From 0a9ab26b68fda6fb5b3b4622dfc0f8597c9ff469 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Wed, 1 Apr 2026 13:32:07 +0000 Subject: [PATCH] fix(core): suppress noisy warning for inline llm-grader prompts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add explicit file:// prefix for prompt file references. Bare strings are always treated as inline text — no file resolution attempted. Closes #901 Co-Authored-By: Claude Opus 4.6 --- .../evaluation/loaders/evaluator-parser.ts | 78 +++++++++++++------ .../loaders/evaluator-parser.test.ts | 58 ++++++++++++++ 2 files changed, 112 insertions(+), 24 deletions(-) diff --git a/packages/core/src/evaluation/loaders/evaluator-parser.ts b/packages/core/src/evaluation/loaders/evaluator-parser.ts index d07e0f87..2cecc40e 100644 --- a/packages/core/src/evaluation/loaders/evaluator-parser.ts +++ b/packages/core/src/evaluation/loaders/evaluator-parser.ts @@ -9,6 +9,17 @@ import { resolveFileReference } from './file-resolver.js'; const ANSI_YELLOW = '\u001b[33m'; const ANSI_RESET = '\u001b[0m'; +/** + * Prefix for explicit file references in prompt strings. + * Consistent with case-file-loader.ts which uses "file://" for test-case file references. + * + * Usage: + * prompt: "file://prompts/grader.md" → explicit file, error if not found + * prompt: "grader.md" → inline text (never resolved as file) + * prompt: "Evaluate the response" → inline text + */ +const PROMPT_FILE_PREFIX = 'file://'; + /** * Normalize evaluator type names from legacy snake_case to internal kebab-case. * Accepts both forms for backward compatibility: @@ -428,14 +439,27 @@ async function parseEvaluatorList( threshold: thresholdValue, }; } else { - // llm-grader aggregator - const aggregatorPrompt = asString(rawAggregator.prompt); + // llm-grader aggregator — same file:// prefix logic as evaluator prompts + const rawAggPrompt = asString(rawAggregator.prompt); + let aggregatorPrompt: string | undefined; let promptPath: string | undefined; - if (aggregatorPrompt) { - const resolved = await resolveFileReference(aggregatorPrompt, searchRoots); - if (resolved.resolvedPath) { - promptPath = path.resolve(resolved.resolvedPath); + if (rawAggPrompt) { + if (rawAggPrompt.startsWith(PROMPT_FILE_PREFIX)) { + // Explicit file reference — error if not found + const fileRef = rawAggPrompt.slice(PROMPT_FILE_PREFIX.length); + aggregatorPrompt = fileRef; + const resolved = await resolveFileReference(fileRef, searchRoots); + if (resolved.resolvedPath) { + promptPath = path.resolve(resolved.resolvedPath); + } else { + throw new Error( + `Composite aggregator in '${evalId}': prompt file not found: ${resolved.displayPath}`, + ); + } + } else { + // Bare string — always treat as inline text, no file resolution + aggregatorPrompt = rawAggPrompt; } } @@ -1144,26 +1168,32 @@ async function parseEvaluatorList( promptScriptConfig = rawPrompt.config as Record; } } else if (typeof rawPrompt === 'string') { - // Text template prompt (existing behavior) - prompt = rawPrompt; - const resolved = await resolveFileReference(prompt, searchRoots); - if (resolved.resolvedPath) { - promptPath = path.resolve(resolved.resolvedPath); - // Validate custom prompt content upfront - throws error if validation fails - try { - await validateCustomPromptContent(promptPath); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - // Add context and re-throw for the caller to handle - throw new Error(`Evaluator '${name}' template (${promptPath}): ${message}`); + // Text template prompt — supports explicit file:// prefix for file references. + // "file://prompts/grader.md" → explicit file reference, error if not found + // "grader.md" → inline text (no file resolution) + // "Evaluate the response" → inline text + + if (rawPrompt.startsWith(PROMPT_FILE_PREFIX)) { + // Explicit file reference — strip prefix and resolve. Error if not found. + const fileRef = rawPrompt.slice(PROMPT_FILE_PREFIX.length); + prompt = fileRef; + const resolved = await resolveFileReference(fileRef, searchRoots); + if (resolved.resolvedPath) { + promptPath = path.resolve(resolved.resolvedPath); + try { + await validateCustomPromptContent(promptPath); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Evaluator '${name}' template (${promptPath}): ${message}`); + } + } else { + throw new Error( + `Evaluator '${name}' in '${evalId}': prompt file not found: ${resolved.displayPath}`, + ); } } else { - logWarning( - `Inline prompt used for evaluator '${name}' in '${evalId}' (file not found: ${resolved.displayPath})`, - resolved.attempted.length > 0 - ? resolved.attempted.map((attempt) => ` Tried: ${attempt}`) - : undefined, - ); + // Bare string — always treat as inline text, no file resolution + prompt = rawPrompt; } } diff --git a/packages/core/test/evaluation/loaders/evaluator-parser.test.ts b/packages/core/test/evaluation/loaders/evaluator-parser.test.ts index 0c56c86a..2f924875 100644 --- a/packages/core/test/evaluation/loaders/evaluator-parser.test.ts +++ b/packages/core/test/evaluation/loaders/evaluator-parser.test.ts @@ -1780,3 +1780,61 @@ describe('parseEvaluators - string shorthand in assertions', () => { expect(evaluators).toBeUndefined(); }); }); + +describe('parseEvaluators - file:// prefix prompt resolution', () => { + let tempDir: string; + + beforeAll(async () => { + tempDir = path.join(os.tmpdir(), `agentv-test-file-prefix-${Date.now()}`); + await mkdir(tempDir, { recursive: true }); + await writeFile(path.join(tempDir, 'grader.md'), 'Evaluate the quality of {{ output }}'); + }); + + afterAll(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + it('file:// prefix resolves existing file', async () => { + const evaluators = await parseEvaluators( + { + assertions: [{ name: 'quality', type: 'llm-grader', prompt: 'file://grader.md' }], + }, + undefined, + [tempDir], + 'test-1', + ); + expect(evaluators).toHaveLength(1); + const config = evaluators?.[0] as LlmGraderEvaluatorConfig; + expect(config.promptPath).toBeTruthy(); + expect(config.promptPath).toContain('grader.md'); + }); + + it('file:// prefix throws when file not found', async () => { + await expect( + parseEvaluators( + { + assertions: [{ name: 'missing', type: 'llm-grader', prompt: 'file://nonexistent.md' }], + }, + undefined, + [tempDir], + 'test-1', + ), + ).rejects.toThrow(/prompt file not found/); + }); + + it('bare path is always treated as inline text even if file exists', async () => { + const evaluators = await parseEvaluators( + { + assertions: [{ name: 'quality', type: 'llm-grader', prompt: 'grader.md' }], + }, + undefined, + [tempDir], + 'test-1', + ); + expect(evaluators).toHaveLength(1); + const config = evaluators?.[0] as LlmGraderEvaluatorConfig; + // Bare string is inline text — no file resolution, no promptPath + expect(config.prompt).toBe('grader.md'); + expect(config.promptPath).toBeUndefined(); + }); +});