From cf61f744ee16a2a19f37a9fbbddc560a4795edfe Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Wed, 1 Apr 2026 09:14:02 +0000 Subject: [PATCH] feat(core): add api_format option to OpenAI/Azure targets Add api_format field ("chat" | "responses") to OpenAI and Azure target configs. Defaults to "chat" (Chat Completions API) which is universally supported by all OpenAI-compatible endpoints. Users can opt in to the Responses API by setting api_format: responses. Closes #896 Co-Authored-By: Claude Opus 4.6 --- .../docs/docs/targets/llm-providers.mdx | 46 +++++++++++++++++++ .../core/src/evaluation/providers/ai-sdk.ts | 8 +--- .../core/src/evaluation/providers/index.ts | 1 + .../core/src/evaluation/providers/targets.ts | 23 ++++++++++ .../validation/targets-validator.ts | 2 + .../test/evaluation/providers/targets.test.ts | 7 +-- 6 files changed, 78 insertions(+), 9 deletions(-) diff --git a/apps/web/src/content/docs/docs/targets/llm-providers.mdx b/apps/web/src/content/docs/docs/targets/llm-providers.mdx index c498d7a89..0bfcd8d11 100644 --- a/apps/web/src/content/docs/docs/targets/llm-providers.mdx +++ b/apps/web/src/content/docs/docs/targets/llm-providers.mdx @@ -7,6 +7,52 @@ sidebar: LLM provider targets call language model APIs directly. These are used both as evaluation targets and as grader targets for scoring. +## OpenAI + +```yaml +targets: + - name: openai-target + provider: openai + api_key: ${{ OPENAI_API_KEY }} + model: gpt-4o +``` + +| Field | Required | Description | +|-------|----------|-------------| +| `api_key` | Yes | OpenAI API key | +| `model` | Yes | Model identifier | +| `base_url` | No | Custom base URL for OpenAI-compatible endpoints | +| `api_format` | No | API format: `chat` (default) or `responses` | + +### `api_format` + +Controls which OpenAI API endpoint is used: + +| Value | Endpoint | When to use | +|-------|----------|-------------| +| `chat` (default) | `/chat/completions` | All OpenAI-compatible endpoints (GitHub Models, local proxies, etc.) | +| `responses` | `/responses` | Only `api.openai.com` — opt in to the Responses API | + +Most users should leave this unset. The default `chat` format is universally supported. Use `responses` only when you need Responses API features on `api.openai.com` directly. + +```yaml +# OpenAI-compatible endpoint (default chat format works) +targets: + - name: github-models + provider: openai + api_format: chat + base_url: https://models.github.ai/inference/v1 + api_key: ${{ GH_MODELS_TOKEN }} + model: ${{ GH_MODELS_MODEL }} + + # Opt in to Responses API for api.openai.com + - name: openai-responses + provider: openai + api_format: responses + api_key: ${{ OPENAI_API_KEY }} + model: gpt-4o +``` + ## Azure OpenAI ```yaml diff --git a/packages/core/src/evaluation/providers/ai-sdk.ts b/packages/core/src/evaluation/providers/ai-sdk.ts index 85278890b..46bf3dd49 100644 --- a/packages/core/src/evaluation/providers/ai-sdk.ts +++ b/packages/core/src/evaluation/providers/ai-sdk.ts @@ -53,12 +53,8 @@ export class OpenAIProvider implements Provider { apiKey: config.apiKey, baseURL: config.baseURL, }); - // Default to Chat Completions API (/chat/completions) which is - // universally supported by all OpenAI-compatible endpoints. - // Only use the Responses API (/responses) for actual OpenAI, which - // is the only provider that supports it. - const isOpenAI = config.baseURL.includes('api.openai.com'); - this.model = isOpenAI ? openai(config.model) : openai.chat(config.model); + this.model = + config.apiFormat === 'responses' ? openai(config.model) : openai.chat(config.model); } async invoke(request: ProviderRequest): Promise { diff --git a/packages/core/src/evaluation/providers/index.ts b/packages/core/src/evaluation/providers/index.ts index 1a215b46b..3f671d6b8 100644 --- a/packages/core/src/evaluation/providers/index.ts +++ b/packages/core/src/evaluation/providers/index.ts @@ -42,6 +42,7 @@ export { extractLastAssistantContent } from './types.js'; export type { AgentVResolvedConfig, AnthropicResolvedConfig, + ApiFormat, AzureResolvedConfig, ClaudeResolvedConfig, CliResolvedConfig, diff --git a/packages/core/src/evaluation/providers/targets.ts b/packages/core/src/evaluation/providers/targets.ts index 57bb41a30..2d6562a87 100644 --- a/packages/core/src/evaluation/providers/targets.ts +++ b/packages/core/src/evaluation/providers/targets.ts @@ -389,6 +389,15 @@ export interface RetryConfig { readonly retryableStatusCodes?: readonly number[]; } +/** + * Selects which OpenAI-compatible API endpoint to use. + * - "chat" (default): POST /chat/completions — universally supported by all OpenAI-compatible providers. + * - "responses": POST /responses — only supported by api.openai.com. + * + * Maps to Vercel AI SDK methods: "chat" → provider.chat(model), "responses" → provider(model). + */ +export type ApiFormat = 'chat' | 'responses'; + /** * Azure OpenAI settings used by the Vercel AI SDK. */ @@ -409,6 +418,7 @@ export interface OpenAIResolvedConfig { readonly baseURL: string; readonly apiKey: string; readonly model: string; + readonly apiFormat?: ApiFormat; readonly temperature?: number; readonly maxOutputTokens?: number; readonly retry?: RetryConfig; @@ -927,6 +937,18 @@ function resolveAzureConfig( }; } +function resolveApiFormat( + target: z.infer, + targetName: string, +): ApiFormat | undefined { + const raw = target.api_format ?? target.apiFormat; + if (raw === undefined) return undefined; + if (raw === 'chat' || raw === 'responses') return raw; + throw new Error( + `Invalid api_format '${raw}' for target '${targetName}'. Must be 'chat' or 'responses'.`, + ); +} + function resolveOpenAIConfig( target: z.infer, env: EnvLookup, @@ -951,6 +973,7 @@ function resolveOpenAIConfig( baseURL, apiKey, model, + apiFormat: resolveApiFormat(target, target.name), temperature: resolveOptionalNumber(temperatureSource, `${target.name} temperature`), maxOutputTokens: resolveOptionalNumber(maxTokensSource, `${target.name} max output tokens`), retry, diff --git a/packages/core/src/evaluation/validation/targets-validator.ts b/packages/core/src/evaluation/validation/targets-validator.ts index 9f9fce9f8..d941900f6 100644 --- a/packages/core/src/evaluation/validation/targets-validator.ts +++ b/packages/core/src/evaluation/validation/targets-validator.ts @@ -60,6 +60,8 @@ const OPENAI_SETTINGS = new Set([ 'model', 'deployment', 'variant', + 'api_format', + 'apiFormat', 'temperature', 'max_output_tokens', 'maxTokens', diff --git a/packages/core/test/evaluation/providers/targets.test.ts b/packages/core/test/evaluation/providers/targets.test.ts index 187b75f18..7b2173ae6 100644 --- a/packages/core/test/evaluation/providers/targets.test.ts +++ b/packages/core/test/evaluation/providers/targets.test.ts @@ -23,9 +23,10 @@ const createAzureMock = mock((options: unknown) => ({ chat: () => ({ provider: 'azure', options }), })); const createOpenAIMock = mock((options: unknown) => { - const defaultFn = () => ({ provider: 'openai', options }); - defaultFn.chat = () => ({ provider: 'openai', options, api: 'chat' }); - return defaultFn; + const fn = () => ({ provider: 'openai', options }); + fn.chat = () => ({ provider: 'openai', options }); + fn.responses = () => ({ provider: 'openai', options }); + return fn; }); const createOpenRouterMock = mock((options: unknown) => () => ({ provider: 'openrouter',