{"openapi":"3.1.0","info":{"title":"Confident AI Evals (BYOK) API","version":"0.1.0","description":"Confident AI Evals (BYOK) lets an AI assistant read your own Confident AI Evals data through the Confident AI Evals API with a key you supply (Confident AI project API key). Read-only: 10 tools (list_metrics, get_metric, list_metric_versions, list_metric_collections, get_metric_collection, list_evaluation_rules, list_datasets, list_dataset_versions, list_personas, list_transformers). The BYOK key...\n\nEvery operation is read-only or compute-only. The same tools are available over MCP (streamable HTTP) at https://byok-confident-ai-evals.connectorbay.com/mcp.","contact":{"name":"Unify Dynamics LLC","email":"agent@connectorbay.com","url":"https://connectorbay.com/byok-confident-ai-evals/docs"},"termsOfService":"https://connectorbay.com/byok-confident-ai-evals/terms"},"servers":[{"url":"https://byok-confident-ai-evals.connectorbay.com"}],"externalDocs":{"url":"https://connectorbay.com/byok-confident-ai-evals/docs"},"security":[{"upstreamKey":[]}],"components":{"securitySchemes":{"bearer":{"type":"http","scheme":"bearer","description":"Optional. A Pro API key raises rate limits; no credentials are required."},"upstreamKey":{"type":"apiKey","in":"header","name":"X-Upstream-Key","description":"Your own key for the service this connector wraps (bring your own key). Used for that one call only; never stored, logged or cached."}},"schemas":{"Error":{"type":"object","properties":{"ok":{"const":false},"error":{"type":"object"}}}}},"paths":{"/v1/tools/list_metrics":{"post":{"operationId":"list_metrics","summary":"List the custom evaluation metrics of the project with name, algorithm, scoring criteria or steps, rubric and the test-c","description":"List the custom evaluation metrics of the project with name, algorithm, scoring criteria or steps, rubric and the test-case fields each metric needs. Use for \"Which custom evaluation metrics are defined in my Confident AI project\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{},"title":"list_metricsArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}},"/v1/tools/get_metric":{"post":{"operationId":"get_metric","summary":"Get one custom metric definition: algorithm, criteria or evaluation steps, rubric and required test-case fields. Use for","description":"Get one custom metric definition: algorithm, criteria or evaluation steps, rubric and required test-case fields. Use for \"Show the definition of the Confident AI metric met_123\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"metric_id":{"description":"Metric id, as returned by list_metrics.","maxLength":200,"title":"Metric Id","type":"string"}},"required":["metric_id"],"title":"get_metricArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}},"/v1/tools/list_metric_versions":{"post":{"operationId":"list_metric_versions","summary":"List the definition history of a custom metric, newest first, with version label, sequence number and creation time. Use","description":"List the definition history of a custom metric, newest first, with version label, sequence number and creation time. Use for \"What versions does the Confident AI metric met_123 have\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"metric_id":{"description":"Metric id, as returned by list_metrics.","maxLength":200,"title":"Metric Id","type":"string"},"page":{"anyOf":[{"maximum":100000,"minimum":1,"type":"integer"},{"type":"null"}],"default":1,"description":"Page number, starting at 1.","title":"Page"},"page_size":{"anyOf":[{"maximum":100,"minimum":1,"type":"integer"},{"type":"null"}],"default":25,"description":"How many versions per page (1 to 100).","title":"Page Size"}},"required":["metric_id"],"title":"list_metric_versionsArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}},"/v1/tools/list_metric_collections":{"post":{"operationId":"list_metric_collections","summary":"List the metric collections with sampling rate and, per metric, threshold, strict mode, reason setting and the evaluatio","description":"List the metric collections with sampling rate and, per metric, threshold, strict mode, reason setting and the evaluation model used. Use for \"Which metric collections does my Confident AI project have and what thresholds do they use\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{},"title":"list_metric_collectionsArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}},"/v1/tools/get_metric_collection":{"post":{"operationId":"get_metric_collection","summary":"Get one metric collection with every metric inside it and its settings. Use for \"Show the Confident AI metric collection","description":"Get one metric collection with every metric inside it and its settings. Use for \"Show the Confident AI metric collection col_1\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"collection_id":{"description":"Metric collection id, as returned by list_metric_collections.","maxLength":200,"title":"Collection Id","type":"string"}},"required":["collection_id"],"title":"get_metric_collectionArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}},"/v1/tools/list_evaluation_rules":{"post":{"operationId":"list_evaluation_rules","summary":"List the evaluation rules (name, enabled flag, whether they evaluate traces, spans or threads), newest first. Needs the ","description":"List the evaluation rules (name, enabled flag, whether they evaluate traces, spans or threads), newest first. Needs the Starter plan or above. Use for \"Which automatic evaluation rules are running in my Confident AI project\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"page":{"anyOf":[{"maximum":100000,"minimum":1,"type":"integer"},{"type":"null"}],"default":1,"description":"Page number, starting at 1.","title":"Page"},"page_size":{"anyOf":[{"maximum":100,"minimum":1,"type":"integer"},{"type":"null"}],"default":25,"description":"How many rules per page (1 to 100).","title":"Page Size"},"data_model":{"anyOf":[{"maxLength":20,"type":"string"},{"type":"null"}],"default":null,"description":"Only rules for this kind of item: TRACE, SPAN or THREAD.","title":"Data Model"}},"title":"list_evaluation_rulesArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}},"/v1/tools/list_datasets":{"post":{"operationId":"list_datasets","summary":"List the datasets of the project, newest first, with id, alias and whether they are multi-turn (the goldens inside are n","description":"List the datasets of the project, newest first, with id, alias and whether they are multi-turn (the goldens inside are not returned). Use for \"Which datasets exist in my Confident AI project\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{},"title":"list_datasetsArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}},"/v1/tools/list_dataset_versions":{"post":{"operationId":"list_dataset_versions","summary":"List the versions of one dataset, newest first, with version number and creation time. Needs the Team plan or above. Use","description":"List the versions of one dataset, newest first, with version number and creation time. Needs the Team plan or above. Use for \"What versions does the Confident AI dataset ds_1 have\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"dataset_id":{"description":"Dataset id, as returned by list_datasets.","maxLength":200,"title":"Dataset Id","type":"string"}},"required":["dataset_id"],"title":"list_dataset_versionsArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}},"/v1/tools/list_personas":{"post":{"operationId":"list_personas","summary":"List the simulation personas (id and name only), newest first, used for multi-turn evaluation. Use for \"Which personas c","description":"List the simulation personas (id and name only), newest first, used for multi-turn evaluation. Use for \"Which personas can I use for multi-turn simulations in Confident AI\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"page":{"anyOf":[{"maximum":100000,"minimum":1,"type":"integer"},{"type":"null"}],"default":1,"description":"Page number, starting at 1.","title":"Page"},"page_size":{"anyOf":[{"maximum":100,"minimum":1,"type":"integer"},{"type":"null"}],"default":25,"description":"How many personas per page (1 to 100).","title":"Page Size"}},"title":"list_personasArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}},"/v1/tools/list_transformers":{"post":{"operationId":"list_transformers","summary":"List the payload transformers (name, description, dates) that metric collections and AI connections can use. Use for \"Wh","description":"List the payload transformers (name, description, dates) that metric collections and AI connections can use. Use for \"Which transformers are defined in my Confident AI project\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"page":{"anyOf":[{"maximum":100000,"minimum":1,"type":"integer"},{"type":"null"}],"default":1,"description":"Page number, starting at 1.","title":"Page"},"page_size":{"anyOf":[{"maximum":100,"minimum":1,"type":"integer"},{"type":"null"}],"default":25,"description":"How many transformers per page (1 to 100).","title":"Page Size"}},"title":"list_transformersArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}}}}