feat(core): add api_format option to OpenAI/Azure targets

Add api_format field ("chat" | "responses") to OpenAI and Azure target
configs. Defaults to "chat" (Chat Completions API) which is universally
supported by all OpenAI-compatible endpoints. Users can opt in to the
Responses API by setting api_format: responses.

Closes #896

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: christso

apps/web/src/content/docs/docs/targets/llm-providers.mdx

@@ -7,6 +7,52 @@ sidebar:
 
 LLM provider targets call language model APIs directly. These are used both as evaluation targets and as grader targets for scoring.
 
+## OpenAI
+
+```yaml
+targets:
+  - name: openai-target
+    provider: openai
+    api_key: ${{ OPENAI_API_KEY }}
+    model: gpt-4o
+```
+
+| Field | Required | Description |
+|-------|----------|-------------|
+| `api_key` | Yes | OpenAI API key |
+| `model` | Yes | Model identifier |
+| `base_url` | No | Custom base URL for OpenAI-compatible endpoints |
+| `api_format` | No | API format: `chat` (default) or `responses` |
+
+### `api_format`
+
+Controls which OpenAI API endpoint is used:
+
+| Value | Endpoint | When to use |
+|-------|----------|-------------|
+| `chat` (default) | `/chat/completions` | All OpenAI-compatible endpoints (GitHub Models, local proxies, etc.) |
+| `responses` | `/responses` | Only `api.openai.com` — opt in to the Responses API |
+
+Most users should leave this unset. The default `chat` format is universally supported. Use `responses` only when you need Responses API features on `api.openai.com` directly.
+
+```yaml
+# OpenAI-compatible endpoint (default chat format works)
+targets:
+  - name: github-models
+    provider: openai
+    api_format: chat
+    base_url: https://models.github.ai/inference/v1
+    api_key: ${{ GH_MODELS_TOKEN }}
+    model: ${{ GH_MODELS_MODEL }}
+
+  # Opt in to Responses API for api.openai.com
+  - name: openai-responses
+    provider: openai
+    api_format: responses
+    api_key: ${{ OPENAI_API_KEY }}
+    model: gpt-4o
+```
+
 ## Azure OpenAI
 
 ```yaml

packages/core/src/evaluation/providers/ai-sdk.ts

@@ -53,12 +53,8 @@ export class OpenAIProvider implements Provider {
       apiKey: config.apiKey,
       baseURL: config.baseURL,
     });
-    // Default to Chat Completions API (/chat/completions) which is
-    // universally supported by all OpenAI-compatible endpoints.
-    // Only use the Responses API (/responses) for actual OpenAI, which
-    // is the only provider that supports it.
-    const isOpenAI = config.baseURL.includes('api.openai.com');
-    this.model = isOpenAI ? openai(config.model) : openai.chat(config.model);
+    this.model =
+      config.apiFormat === 'responses' ? openai(config.model) : openai.chat(config.model);
   }
 
   async invoke(request: ProviderRequest): Promise<ProviderResponse> {

packages/core/src/evaluation/providers/index.ts

@@ -42,6 +42,7 @@ export { extractLastAssistantContent } from './types.js';
 export type {
   AgentVResolvedConfig,
   AnthropicResolvedConfig,
+  ApiFormat,
   AzureResolvedConfig,
   ClaudeResolvedConfig,
   CliResolvedConfig,

packages/core/src/evaluation/providers/targets.ts

@@ -389,6 +389,15 @@ export interface RetryConfig {
   readonly retryableStatusCodes?: readonly number[];
 }
 
+/**
+ * Selects which OpenAI-compatible API endpoint to use.
+ * - "chat" (default): POST /chat/completions — universally supported by all OpenAI-compatible providers.
+ * - "responses": POST /responses — only supported by api.openai.com.
+ *
+ * Maps to Vercel AI SDK methods: "chat" → provider.chat(model), "responses" → provider(model).
+ */
+export type ApiFormat = 'chat' | 'responses';
+
 /**
  * Azure OpenAI settings used by the Vercel AI SDK.
  */
@@ -409,6 +418,7 @@ export interface OpenAIResolvedConfig {
   readonly baseURL: string;
   readonly apiKey: string;
   readonly model: string;
+  readonly apiFormat?: ApiFormat;
   readonly temperature?: number;
   readonly maxOutputTokens?: number;
   readonly retry?: RetryConfig;
@@ -927,6 +937,18 @@ function resolveAzureConfig(
   };
 }
 
+function resolveApiFormat(
+  target: z.infer<typeof BASE_TARGET_SCHEMA>,
+  targetName: string,
+): ApiFormat | undefined {
+  const raw = target.api_format ?? target.apiFormat;
+  if (raw === undefined) return undefined;
+  if (raw === 'chat' || raw === 'responses') return raw;
+  throw new Error(
+    `Invalid api_format '${raw}' for target '${targetName}'. Must be 'chat' or 'responses'.`,
+  );
+}
+
 function resolveOpenAIConfig(
   target: z.infer<typeof BASE_TARGET_SCHEMA>,
   env: EnvLookup,
@@ -951,6 +973,7 @@ function resolveOpenAIConfig(
     baseURL,
     apiKey,
     model,
+    apiFormat: resolveApiFormat(target, target.name),
     temperature: resolveOptionalNumber(temperatureSource, `${target.name} temperature`),
     maxOutputTokens: resolveOptionalNumber(maxTokensSource, `${target.name} max output tokens`),
     retry,

packages/core/src/evaluation/validation/targets-validator.ts

@@ -60,6 +60,8 @@ const OPENAI_SETTINGS = new Set([
   'model',
   'deployment',
   'variant',
+  'api_format',
+  'apiFormat',
   'temperature',
   'max_output_tokens',
   'maxTokens',

packages/core/test/evaluation/providers/targets.test.ts

@@ -23,9 +23,10 @@ const createAzureMock = mock((options: unknown) => ({
   chat: () => ({ provider: 'azure', options }),
 }));
 const createOpenAIMock = mock((options: unknown) => {
-  const defaultFn = () => ({ provider: 'openai', options });
-  defaultFn.chat = () => ({ provider: 'openai', options, api: 'chat' });
-  return defaultFn;
+  const fn = () => ({ provider: 'openai', options });
+  fn.chat = () => ({ provider: 'openai', options });
+  fn.responses = () => ({ provider: 'openai', options });
+  return fn;
 });
 const createOpenRouterMock = mock((options: unknown) => () => ({
   provider: 'openrouter',