diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index d67fae4..9018b11 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -53,6 +53,24 @@ import { import { DNSRebindingProtectionScenario } from './server/dns-rebinding'; +// IncompleteResult scenarios from (SEP-2322) +import { + IncompleteResultBasicElicitationScenario, + IncompleteResultBasicSamplingScenario, + IncompleteResultBasicListRootsScenario, + IncompleteResultRequestStateScenario, + IncompleteResultMultipleInputRequestsScenario, + IncompleteResultMultiRoundScenario, + IncompleteResultMissingInputResponseScenario, + IncompleteResultNonToolRequestScenario +} from './server/incomplete-result'; + +import { + IncompleteResultTaskBasicScenario, + IncompleteResultTaskBadInputResponseScenario, + IncompleteResultTaskInputResponseIncompleteScenario +} from './server/incomplete-result-tasks'; + import { authScenariosList, backcompatScenariosList, @@ -69,7 +87,22 @@ const pendingClientScenariosList: ClientScenario[] = [ // On hold until server-side SSE improvements are made // https://github.com/modelcontextprotocol/typescript-sdk/pull/1129 - new ServerSSEPollingScenario() + new ServerSSEPollingScenario(), + + // IncompleteResult scenarios (SEP-2322) — pending until a conformance test + // server implements IncompleteResult tools. These are draft spec scenarios + // intended to be run via `--spec-version draft` against capable servers. + new IncompleteResultBasicElicitationScenario(), + new IncompleteResultBasicSamplingScenario(), + new IncompleteResultBasicListRootsScenario(), + new IncompleteResultRequestStateScenario(), + new IncompleteResultMultipleInputRequestsScenario(), + new IncompleteResultMultiRoundScenario(), + new IncompleteResultMissingInputResponseScenario(), + new IncompleteResultNonToolRequestScenario(), + new IncompleteResultTaskBasicScenario(), + new IncompleteResultTaskBadInputResponseScenario(), + new IncompleteResultTaskInputResponseIncompleteScenario() ]; // All client scenarios @@ -124,7 +157,22 @@ const allClientScenariosList: ClientScenario[] = [ new PromptsGetWithImageScenario(), // Security scenarios - new DNSRebindingProtectionScenario() + new DNSRebindingProtectionScenario(), + + // IncompleteResult scenarios (SEP-2322) + new IncompleteResultBasicElicitationScenario(), + new IncompleteResultBasicSamplingScenario(), + new IncompleteResultBasicListRootsScenario(), + new IncompleteResultRequestStateScenario(), + new IncompleteResultMultipleInputRequestsScenario(), + new IncompleteResultMultiRoundScenario(), + new IncompleteResultMissingInputResponseScenario(), + new IncompleteResultNonToolRequestScenario(), + + // IncompleteResult Task scenarios (SEP-2322) + new IncompleteResultTaskBasicScenario(), + new IncompleteResultTaskBadInputResponseScenario(), + new IncompleteResultTaskInputResponseIncompleteScenario() ]; // Active client scenarios (excludes pending) diff --git a/src/scenarios/server/client-helper.ts b/src/scenarios/server/client-helper.ts index eebbd9b..5e99374 100644 --- a/src/scenarios/server/client-helper.ts +++ b/src/scenarios/server/client-helper.ts @@ -1,5 +1,13 @@ /** - * Helper utilities for creating MCP clients to test servers + * Helper utilities for creating MCP clients to test servers. + * + * Provides two connection modes: + * 1. SDK-based (connectToServer) — uses the MCP TypeScript SDK for standard + * protocol operations. + * 2. Raw JSON-RPC (RawMcpSession) — uses undici HTTP for draft/experimental + * features that the SDK does not yet support. + * + * Both modes share the same SDK-based initialize handshake and session ID. */ import { Client } from '@modelcontextprotocol/sdk/client/index.js'; @@ -8,14 +16,27 @@ import { LoggingMessageNotificationSchema, ProgressNotificationSchema } from '@modelcontextprotocol/sdk/types.js'; +import { request } from 'undici'; + +// ─── JSON-RPC Types ────────────────────────────────────────────────────────── + +export interface JsonRpcResponse { + jsonrpc: '2.0'; + id: number; + result?: Record; + error?: { code: number; message: string; data?: unknown }; +} + +// ─── SDK-based Connection ──────────────────────────────────────────────────── export interface MCPClientConnection { client: Client; + transport: StreamableHTTPClientTransport; close: () => Promise; } /** - * Create and connect an MCP client to a server + * Create and connect an MCP client to a server using the SDK. */ export async function connectToServer( serverUrl: string @@ -40,15 +61,127 @@ export async function connectToServer( return { client, + transport, close: async () => { await client.close(); } }; } +// ─── Raw JSON-RPC Session ──────────────────────────────────────────────────── + +/** + * A raw MCP session for testing draft/experimental protocol features that the + * SDK does not yet support. Uses the SDK for the standard initialize handshake, + * then sends raw JSON-RPC over HTTP via undici for subsequent requests. + * + * Usage: + * const session = await createRawSession(serverUrl); + * const response = await session.send('tools/call', { name: 'my-tool', arguments: {} }); + */ +export class RawMcpSession { + private nextId = 1; + private serverUrl: string; + private connection: MCPClientConnection | null = null; + + constructor(serverUrl: string) { + this.serverUrl = serverUrl; + } + + /** + * Initialize the MCP session using the SDK's connectToServer(), + * then extract the session ID for subsequent raw requests. + */ + async initialize(): Promise { + this.connection = await connectToServer(this.serverUrl); + } + + /** + * Send a JSON-RPC request via raw HTTP. + * Automatically manages session ID and auto-incrementing JSON-RPC IDs. + * Handles both JSON and SSE response formats. + */ + async send( + method: string, + params?: Record + ): Promise { + const id = this.nextId++; + + const headers: Record = { + 'Content-Type': 'application/json', + Accept: 'application/json, text/event-stream' + }; + + const body = JSON.stringify({ + jsonrpc: '2.0', + id, + method, + params + }); + + const response = await request(this.serverUrl, { + method: 'POST', + headers, + body + }); + + const contentType = response.headers['content-type'] ?? ''; + + // Handle SSE responses — parse the last JSON-RPC message from the stream + // Not doing proper handling of SSE here since none of the MRTR features under test currently require it. + // This can be expanded if necessary for new features. + if (contentType.includes('text/event-stream')) { + const text = await response.body.text(); + return parseSseResponse(text); + } + + // Handle direct JSON responses + return (await response.body.json()) as JsonRpcResponse; + } + + /** + * Close the underlying SDK connection. + */ + async close(): Promise { + if (this.connection) { + await this.connection.close(); + this.connection = null; + } + } +} + +/** + * Create an initialized raw MCP session ready for testing. + */ +export async function createRawSession( + serverUrl: string +): Promise { + const session = new RawMcpSession(serverUrl); + await session.initialize(); + return session; +} + /** - * Helper to collect notifications (logging and progress) + * Parse the last JSON-RPC message from an SSE response body. */ +export function parseSseResponse(sseText: string): JsonRpcResponse { + const lines = sseText.split('\n'); + let lastData: string | null = null; + + for (const line of lines) { + if (line.startsWith('data: ')) { + lastData = line.slice(6); + } + } + + if (!lastData) { + throw new Error('No data found in SSE stream'); + } + + return JSON.parse(lastData) as JsonRpcResponse; +} + +// ─── Notification Collector ────────────────────────────────────────────────── export class NotificationCollector { private loggingNotifications: any[] = []; private progressNotifications: any[] = []; diff --git a/src/scenarios/server/incomplete-result-helpers.ts b/src/scenarios/server/incomplete-result-helpers.ts new file mode 100644 index 0000000..e79189c --- /dev/null +++ b/src/scenarios/server/incomplete-result-helpers.ts @@ -0,0 +1,117 @@ +/** + * Helpers for SEP-2322 conformance tests. + * + * Uses RawMcpSession from client-helper.ts for connection management and + * raw JSON-RPC transport. This file adds IncompleteResult-specific type + * guards and mock response builders. + */ + +import { RawMcpSession, JsonRpcResponse } from './client-helper'; + +export type { RawMcpSession, JsonRpcResponse }; + +// ─── IncompleteResult Types ────────────────────────────────────────────────── + +export interface IncompleteResult { + result_type?: 'incomplete'; + inputRequests?: Record; + requestState?: string; + _meta?: Record; + [key: string]: unknown; +} + +export interface InputRequestObject { + method: string; + params?: Record; +} + +// ─── Type Guards ───────────────────────────────────────────────────────────── + +/** + * Check if a JSON-RPC result is an IncompleteResult. + */ +export function isIncompleteResult( + result: Record | undefined +): result is IncompleteResult { + if (!result) return false; + if (result.result_type === 'incomplete') return true; + // Also detect by presence of IncompleteResult fields + return 'inputRequests' in result || 'requestState' in result; +} + +/** + * Check if a JSON-RPC result is a complete result (not incomplete). + * complete is the default so if result_type is missing we assume it's complete. + */ +export function isCompleteResult( + result: Record | undefined +): boolean { + if (!result) return false; + if (result.result_type === 'complete') return true; + if (!('result_type' in result)) return true; + return !isIncompleteResult(result); +} + +/** + * Extract inputRequests from an IncompleteResult. + */ +export function getInputRequests( + result: IncompleteResult +): Record | undefined { + return result.inputRequests; +} + +// ─── Mock Response Builders ────────────────────────────────────────────────── + +/** + * Build a mock elicitation response (ElicitResult). + */ +export function mockElicitResponse( + content: Record +): Record { + return { + action: 'accept', + content + }; +} + +/** + * Build a mock sampling response (CreateMessageResult). + */ +export function mockSamplingResponse(text: string): Record { + return { + role: 'assistant', + content: { + type: 'text', + text + }, + model: 'test-model', + stopReason: 'endTurn' + }; +} + +/** + * Build a mock list roots response (ListRootsResult). + */ +export function mockListRootsResponse(): Record { + return { + roots: [ + { + uri: 'file:///test/root', + name: 'Test Root' + } + ] + }; +} + +// ─── Spec References ───────────────────────────────────────────────────────── + +/** + * SEP reference for IncompleteResult / MRTR tests. + */ +export const MRTR_SPEC_REFERENCES = [ + { + id: 'SEP-2322', + url: 'https://github.com/modelcontextprotocol/specification/pull/2322' + } +]; diff --git a/src/scenarios/server/incomplete-result-tasks.ts b/src/scenarios/server/incomplete-result-tasks.ts new file mode 100644 index 0000000..c2c9488 --- /dev/null +++ b/src/scenarios/server/incomplete-result-tasks.ts @@ -0,0 +1,639 @@ +/** + * SEP-2322: MRTR Tests for Persistent Workflow aka Tasks + * + * Tests the persistent (task-based) workflow where servers use Tasks to + * manage long-running operations that require additional input via + * tasks/get → input_required → tasks/result → tasks/input_response. + */ + +import { ClientScenario, ConformanceCheck, SpecVersion } from '../../types'; +import { createRawSession } from './client-helper'; +import { + isIncompleteResult, + isCompleteResult, + mockElicitResponse, + MRTR_SPEC_REFERENCES, + RawMcpSession +} from './incomplete-result-helpers'; + +/** + * Poll tasks/get until the task reaches the expected status or times out. + */ +async function pollTaskStatus( + session: RawMcpSession, + taskId: string, + expectedStatus: string, + maxAttempts: number = 20, + intervalMs: number = 250 +): Promise | null> { + for (let i = 0; i < maxAttempts; i++) { + const response = await session.send('tasks/get', { taskId }); + if (response.error) return null; + const result = response.result; + if (!result) return null; + if (result.status === expectedStatus) return result; + // If already completed/failed, stop polling + if ( + result.status === 'completed' || + result.status === 'failed' || + result.status === 'cancelled' + ) { + return result; + } + await new Promise((resolve) => setTimeout(resolve, intervalMs)); + } + return null; +} + +// ─── B1: Basic Persistent Workflow ─────────────────────────────────────────── + +export class IncompleteResultTaskBasicScenario implements ClientScenario { + name = 'incomplete-result-task-basic'; + specVersions: SpecVersion[] = ['draft']; + description = `Test full persistent IncompleteResult workflow via Tasks API (SEP-2322). + +**Server Implementation Requirements:** + +Implement a tool named \`test_incomplete_result_task\` that supports task-augmented execution. + +**Behavior:** +1. When called with \`task\` metadata, return a \`CreateTaskResult\` with \`status: "working"\` +2. After a brief period, set task status to \`"input_required"\` +3. When \`tasks/result\` is called, return an \`IncompleteResult\` with \`inputRequests\`: + +\`\`\`json +{ + "result_type": "incomplete", + "inputRequests": { + "user_input": { + "method": "elicitation/create", + "params": { + "message": "What input should the task use?", + "requestedSchema": { + "type": "object", + "properties": { "input": { "type": "string" } }, + "required": ["input"] + } + } + } + } +} +\`\`\` + +4. When \`tasks/input_response\` is called with \`inputResponses\`, acknowledge and resume +5. Set task status to \`"completed"\` +6. When \`tasks/result\` is called again, return the final result with tool content`; + + async run(serverUrl: string): Promise { + const checks: ConformanceCheck[] = []; + + try { + const session = await createRawSession(serverUrl); + + // Step 1: Call tool with task metadata + const r1 = await session.send('tools/call', { + name: 'test_incomplete_result_task', + arguments: {}, + task: { ttl: 30000 } + }); + + const r1Result = r1.result; + const r1Errors: string[] = []; + let taskId: string | undefined; + + if (r1.error) { + r1Errors.push(`JSON-RPC error: ${r1.error.message}`); + } else if (!r1Result) { + r1Errors.push('No result in response'); + } else { + const task = r1Result.task as + | { taskId?: string; status?: string } + | undefined; + if (!task?.taskId) { + r1Errors.push('Expected CreateTaskResult with task.taskId'); + } else { + taskId = task.taskId; + if (task.status !== 'working') { + r1Errors.push( + `Expected initial task status "working", got "${task.status}"` + ); + } + } + } + + checks.push({ + id: 'incomplete-result-task-created', + name: 'IncompleteResultTaskCreated', + description: 'Server creates task with working status', + status: r1Errors.length === 0 ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: r1Errors.length > 0 ? r1Errors.join('; ') : undefined, + specReferences: MRTR_SPEC_REFERENCES, + details: { result: r1Result, taskId } + }); + + if (!taskId) return checks; + + // Step 2: Poll tasks/get until input_required + const taskState = await pollTaskStatus(session, taskId, 'input_required'); + + const pollErrors: string[] = []; + if (!taskState) { + pollErrors.push( + 'Task did not reach input_required status within timeout' + ); + } else if (taskState.status !== 'input_required') { + pollErrors.push( + `Expected status "input_required", got "${taskState.status}"` + ); + } + + checks.push({ + id: 'incomplete-result-task-input-required', + name: 'IncompleteResultTaskInputRequired', + description: 'Task reaches input_required status', + status: pollErrors.length === 0 ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: pollErrors.length > 0 ? pollErrors.join('; ') : undefined, + specReferences: MRTR_SPEC_REFERENCES, + details: { taskState } + }); + + if (pollErrors.length > 0) return checks; + + // Step 3: Call tasks/result to get inputRequests + const r3 = await session.send('tasks/result', { taskId }); + const r3Result = r3.result; + const r3Errors: string[] = []; + + if (r3.error) { + r3Errors.push(`JSON-RPC error: ${r3.error.message}`); + } else if (!r3Result) { + r3Errors.push('No result from tasks/result'); + } else if (!isIncompleteResult(r3Result)) { + r3Errors.push( + 'Expected IncompleteResult with inputRequests from tasks/result' + ); + } else if (!r3Result.inputRequests) { + r3Errors.push( + 'IncompleteResult from tasks/result missing inputRequests' + ); + } + + checks.push({ + id: 'incomplete-result-task-tasks-result-incomplete', + name: 'IncompleteResultTaskTasksResultIncomplete', + description: 'tasks/result returns IncompleteResult with inputRequests', + status: r3Errors.length === 0 ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: r3Errors.length > 0 ? r3Errors.join('; ') : undefined, + specReferences: MRTR_SPEC_REFERENCES, + details: { result: r3Result } + }); + + if (r3Errors.length > 0 || !isIncompleteResult(r3Result)) return checks; + + // Step 4: Call tasks/input_response with inputResponses + const inputKey = Object.keys(r3Result.inputRequests!)[0]; + const r4 = await session.send('tasks/input_response', { + inputResponses: { + [inputKey]: mockElicitResponse({ input: 'Hello World!' }) + }, + _meta: { + 'io.modelcontextprotocol/related-task': { taskId } + } + }); + + const r4Errors: string[] = []; + if (r4.error) { + r4Errors.push(`JSON-RPC error: ${r4.error.message}`); + } + + checks.push({ + id: 'incomplete-result-task-input-response-sent', + name: 'IncompleteResultTaskInputResponseSent', + description: 'tasks/input_response is acknowledged by server', + status: r4Errors.length === 0 ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: r4Errors.length > 0 ? r4Errors.join('; ') : undefined, + specReferences: MRTR_SPEC_REFERENCES, + details: { result: r4.result } + }); + + // Validate acknowledgment includes task metadata (SHOULD per spec) + if (r4Errors.length === 0) { + const r4Result = r4.result; + const ackErrors: string[] = []; + + if (!r4Result) { + ackErrors.push('No result from tasks/input_response'); + } else { + const meta = r4Result._meta as Record | undefined; + const relatedTask = meta?.['io.modelcontextprotocol/related-task'] as + | { taskId?: string } + | undefined; + if (!relatedTask?.taskId) { + ackErrors.push( + 'Acknowledgment missing _meta.io.modelcontextprotocol/related-task.taskId' + ); + } else if (relatedTask.taskId !== taskId) { + ackErrors.push( + `taskId mismatch: expected "${taskId}", got "${relatedTask.taskId}"` + ); + } + } + + checks.push({ + id: 'incomplete-result-task-ack-structure', + name: 'IncompleteResultTaskAckStructure', + description: + 'tasks/input_response acknowledgment includes task metadata', + status: ackErrors.length === 0 ? 'SUCCESS' : 'WARNING', + timestamp: new Date().toISOString(), + errorMessage: ackErrors.length > 0 ? ackErrors.join('; ') : undefined, + specReferences: MRTR_SPEC_REFERENCES, + details: { result: r4Result } + }); + } + + if (r4Errors.length > 0) return checks; + + // Step 5: Poll until completed + const completedState = await pollTaskStatus(session, taskId, 'completed'); + + const compErrors: string[] = []; + if (!completedState) { + compErrors.push('Task did not reach completed status within timeout'); + } else if (completedState.status !== 'completed') { + compErrors.push( + `Expected status "completed", got "${completedState.status}"` + ); + } + + checks.push({ + id: 'incomplete-result-task-completed', + name: 'IncompleteResultTaskCompleted', + description: 'Task reaches completed status after input_response', + status: compErrors.length === 0 ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: compErrors.length > 0 ? compErrors.join('; ') : undefined, + specReferences: MRTR_SPEC_REFERENCES, + details: { taskState: completedState } + }); + + if (compErrors.length > 0) return checks; + + // Step 6: Get final result + const r6 = await session.send('tasks/result', { taskId }); + const r6Result = r6.result; + const r6Errors: string[] = []; + + if (r6.error) { + r6Errors.push(`JSON-RPC error: ${r6.error.message}`); + } else if (!r6Result) { + r6Errors.push('No result from final tasks/result'); + } else if (!isCompleteResult(r6Result)) { + r6Errors.push('Expected complete result from final tasks/result'); + } else if (!r6Result.content) { + r6Errors.push('Final result missing content'); + } + + checks.push({ + id: 'incomplete-result-task-final-result', + name: 'IncompleteResultTaskFinalResult', + description: 'tasks/result returns complete final result', + status: r6Errors.length === 0 ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: r6Errors.length > 0 ? r6Errors.join('; ') : undefined, + specReferences: MRTR_SPEC_REFERENCES, + details: { result: r6Result } + }); + } catch (error) { + checks.push({ + id: 'incomplete-result-task-created', + name: 'IncompleteResultTaskCreated', + description: 'Server creates task with working status', + status: 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`, + specReferences: MRTR_SPEC_REFERENCES + }); + } + + return checks; + } +} + +// ─── B2: Bad Input Response ────────────────────────────────────────────────── + +export class IncompleteResultTaskBadInputResponseScenario + implements ClientScenario +{ + name = 'incomplete-result-task-bad-input-response'; + specVersions: SpecVersion[] = ['draft']; + description = `Test error handling when tasks/input_response contains wrong data (SEP-2322). + +**Server Implementation Requirements:** + +Use the same tool as B1: \`test_incomplete_result_task\`. + +**Behavior:** When the client sends \`tasks/input_response\` with incorrect keys, the server SHOULD acknowledge the message but keep the task in \`input_required\` status. The next \`tasks/result\` call should return a new \`inputRequests\` re-requesting the needed information.`; + + async run(serverUrl: string): Promise { + const checks: ConformanceCheck[] = []; + + try { + const session = await createRawSession(serverUrl); + + // Create task and wait for input_required + const r1 = await session.send('tools/call', { + name: 'test_incomplete_result_task', + arguments: {}, + task: { ttl: 30000 } + }); + + const task = r1.result?.task as { taskId?: string } | undefined; + if (!task?.taskId) { + checks.push({ + id: 'incomplete-result-task-bad-input-prereq', + name: 'IncompleteResultTaskBadInputPrereq', + description: 'Prerequisite: Task creation', + status: 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: 'Could not create task', + specReferences: MRTR_SPEC_REFERENCES + }); + return checks; + } + + const taskId = task.taskId; + await pollTaskStatus(session, taskId, 'input_required'); + + // Get input requests + const r3 = await session.send('tasks/result', { taskId }); + if ( + r3.error || + !r3.result || + !isIncompleteResult(r3.result) || + !r3.result.inputRequests + ) { + checks.push({ + id: 'incomplete-result-task-bad-input-prereq', + name: 'IncompleteResultTaskBadInputPrereq', + description: 'Prerequisite: Get inputRequests', + status: 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: 'Could not get inputRequests', + specReferences: MRTR_SPEC_REFERENCES + }); + return checks; + } + + // Send wrong inputResponses + const r4 = await session.send('tasks/input_response', { + inputResponses: { + wrong_key: mockElicitResponse({ wrong: 'data' }) + }, + _meta: { + 'io.modelcontextprotocol/related-task': { taskId } + } + }); + + const ackErrors: string[] = []; + if (r4.error) { + // Some servers may error; that's acceptable + ackErrors.push( + 'Server returned error for bad input (acceptable but not preferred)' + ); + } + + // Check task is still input_required + const stateAfter = await session.send('tasks/get', { taskId }); + const stillInputRequired = stateAfter.result?.status === 'input_required'; + + // Try to get new inputRequests + let newInputRequests = false; + if (stillInputRequired) { + const r5 = await session.send('tasks/result', { taskId }); + if ( + r5.result && + isIncompleteResult(r5.result) && + r5.result.inputRequests + ) { + newInputRequests = true; + } + } + + const errors: string[] = []; + if (!stillInputRequired && ackErrors.length === 0) { + errors.push( + 'Task should remain in input_required after bad inputResponses' + ); + } + if (stillInputRequired && !newInputRequests) { + errors.push( + 'tasks/result should return new inputRequests after bad input_response' + ); + } + + checks.push({ + id: 'incomplete-result-task-bad-input-rerequests', + name: 'IncompleteResultTaskBadInputRerequests', + description: + 'Server keeps task in input_required and re-requests after bad inputResponses', + status: + errors.length === 0 + ? ackErrors.length === 0 + ? 'SUCCESS' + : 'WARNING' + : 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: + [...errors, ...ackErrors].length > 0 + ? [...errors, ...ackErrors].join('; ') + : undefined, + specReferences: MRTR_SPEC_REFERENCES, + details: { + stillInputRequired, + newInputRequests, + ackResult: r4.result + } + }); + } catch (error) { + checks.push({ + id: 'incomplete-result-task-bad-input-rerequests', + name: 'IncompleteResultTaskBadInputRerequests', + description: + 'Server keeps task in input_required and re-requests after bad inputResponses', + status: 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`, + specReferences: MRTR_SPEC_REFERENCES + }); + } + + return checks; + } +} + +// ─── B4: tasks/input_response returning IncompleteResult ───────────────────── + +export class IncompleteResultTaskInputResponseIncompleteScenario + implements ClientScenario +{ + name = 'incomplete-result-task-input-response-incomplete'; + specVersions: SpecVersion[] = ['draft']; + description = `Test that tasks/input_response can itself return an IncompleteResult (SEP-2322). + +**Server Implementation Requirements:** + +Implement a tool named \`test_incomplete_result_task_multi_input\` that supports task-augmented execution and requires TWO rounds of input. + +**Behavior:** +1. Create task, transition to \`input_required\` +2. \`tasks/result\` returns IncompleteResult with first \`inputRequests\` +3. \`tasks/input_response\` returns an \`IncompleteResult\` with ADDITIONAL \`inputRequests\` +4. Client sends another \`tasks/input_response\` with the additional responses +5. Task completes + +This tests the schema: \`TaskInputResponseResultResponse.result: Result | IncompleteResult\``; + + async run(serverUrl: string): Promise { + const checks: ConformanceCheck[] = []; + + try { + const session = await createRawSession(serverUrl); + + // Create task + const r1 = await session.send('tools/call', { + name: 'test_incomplete_result_task_multi_input', + arguments: {}, + task: { ttl: 30000 } + }); + + const task = r1.result?.task as { taskId?: string } | undefined; + if (!task?.taskId) { + checks.push({ + id: 'incomplete-result-task-multi-input-prereq', + name: 'IncompleteResultTaskMultiInputPrereq', + description: 'Prerequisite: Task creation', + status: 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: 'Could not create task', + specReferences: MRTR_SPEC_REFERENCES + }); + return checks; + } + + const taskId = task.taskId; + await pollTaskStatus(session, taskId, 'input_required'); + + // Get first inputRequests + const r3 = await session.send('tasks/result', { taskId }); + if ( + r3.error || + !r3.result || + !isIncompleteResult(r3.result) || + !r3.result.inputRequests + ) { + checks.push({ + id: 'incomplete-result-task-multi-input-prereq', + name: 'IncompleteResultTaskMultiInputPrereq', + description: 'Prerequisite: Get first inputRequests', + status: 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: 'Could not get inputRequests from tasks/result', + specReferences: MRTR_SPEC_REFERENCES + }); + return checks; + } + + // Send first input_response — expect IncompleteResult back + const inputKey1 = Object.keys(r3.result.inputRequests!)[0]; + const r4 = await session.send('tasks/input_response', { + inputResponses: { + [inputKey1]: mockElicitResponse({ input: 'step1' }) + }, + _meta: { + 'io.modelcontextprotocol/related-task': { taskId } + } + }); + + const r4Result = r4.result; + const r4Errors: string[] = []; + + if (r4.error) { + r4Errors.push(`JSON-RPC error: ${r4.error.message}`); + } else if (!r4Result) { + r4Errors.push('No result from tasks/input_response'); + } else if (!isIncompleteResult(r4Result)) { + r4Errors.push( + 'Expected IncompleteResult from tasks/input_response (additional input needed)' + ); + } else if (!r4Result.inputRequests) { + r4Errors.push( + 'IncompleteResult from tasks/input_response missing inputRequests' + ); + } + + checks.push({ + id: 'incomplete-result-task-input-response-returns-incomplete', + name: 'IncompleteResultTaskInputResponseReturnsIncomplete', + description: + 'tasks/input_response returns IncompleteResult with additional inputRequests', + status: r4Errors.length === 0 ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: r4Errors.length > 0 ? r4Errors.join('; ') : undefined, + specReferences: MRTR_SPEC_REFERENCES, + details: { result: r4Result } + }); + + // Send second input_response — expect completion + if (r4Errors.length === 0 && isIncompleteResult(r4Result)) { + const inputKey2 = Object.keys(r4Result.inputRequests!)[0]; + const r5 = await session.send('tasks/input_response', { + inputResponses: { + [inputKey2]: mockElicitResponse({ input: 'step2' }) + }, + _meta: { + 'io.modelcontextprotocol/related-task': { taskId } + } + }); + + const r5Ok = !r5.error; + + // Poll for completion + if (r5Ok) { + const finalState = await pollTaskStatus(session, taskId, 'completed'); + + checks.push({ + id: 'incomplete-result-task-multi-input-completed', + name: 'IncompleteResultTaskMultiInputCompleted', + description: 'Task completes after second input_response', + status: finalState?.status === 'completed' ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: + finalState?.status !== 'completed' + ? 'Task did not complete after second input_response' + : undefined, + specReferences: MRTR_SPEC_REFERENCES, + details: { taskState: finalState } + }); + } + } + } catch (error) { + checks.push({ + id: 'incomplete-result-task-input-response-returns-incomplete', + name: 'IncompleteResultTaskInputResponseReturnsIncomplete', + description: + 'tasks/input_response returns IncompleteResult with additional inputRequests', + status: 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`, + specReferences: MRTR_SPEC_REFERENCES + }); + } + + return checks; + } +} diff --git a/src/scenarios/server/incomplete-result.ts b/src/scenarios/server/incomplete-result.ts new file mode 100644 index 0000000..7b4b63c --- /dev/null +++ b/src/scenarios/server/incomplete-result.ts @@ -0,0 +1,1153 @@ +/** + * SEP-2322: IncompleteResult - Ephemeral Workflow Tests + * + * Tests the ephemeral (stateless) workflow where servers respond with + * IncompleteResult containing inputRequests and/or requestState, and + * clients retry with inputResponses and echoed requestState. + */ + +import { ClientScenario, ConformanceCheck, SpecVersion } from '../../types'; +import { createRawSession } from './client-helper'; +import { + isIncompleteResult, + isCompleteResult, + mockElicitResponse, + mockSamplingResponse, + mockListRootsResponse, + MRTR_SPEC_REFERENCES +} from './incomplete-result-helpers'; + +// ─── A1: Basic Elicitation ──────────────────────────────────────────────────── + +export class IncompleteResultBasicElicitationScenario + implements ClientScenario +{ + name = 'incomplete-result-basic-elicitation'; + specVersions: SpecVersion[] = ['draft']; + description = `Test basic ephemeral IncompleteResult flow with a single elicitation input request (SEP-2322). + +**Server Implementation Requirements:** + +Implement a tool named \`test_tool_with_elicitation\` (no arguments required). + +**Behavior (Round 1):** When called without \`inputResponses\`, return an \`IncompleteResult\`: + +\`\`\`json +{ + "result_type": "incomplete", + "inputRequests": { + "user_name": { + "method": "elicitation/create", + "params": { + "message": "What is your name?", + "requestedSchema": { + "type": "object", + "properties": { + "name": { "type": "string" } + }, + "required": ["name"] + } + } + } + } +} +\`\`\` + +**Behavior (Round 2):** When called with \`inputResponses\` containing the key \`"user_name"\`, return a complete result: + +\`\`\`json +{ + "content": [{ "type": "text", "text": "Hello, !" }] +} +\`\`\``; + + async run(serverUrl: string): Promise { + const checks: ConformanceCheck[] = []; + + try { + const session = await createRawSession(serverUrl); + + // Round 1: Initial call — expect IncompleteResult + const r1 = await session.send('tools/call', { + name: 'test_tool_with_elicitation', + arguments: {} + }); + + const r1Result = r1.result; + const r1Errors: string[] = []; + + if (r1.error) { + r1Errors.push(`JSON-RPC error: ${r1.error.message}`); + } else if (!r1Result) { + r1Errors.push('No result in response'); + } else if (!isIncompleteResult(r1Result)) { + r1Errors.push( + 'Expected IncompleteResult but got a complete result. ' + + 'Server should return result_type: "incomplete" with inputRequests.' + ); + } else { + if (!r1Result.inputRequests) { + r1Errors.push('IncompleteResult missing inputRequests'); + } else if (!r1Result.inputRequests['user_name']) { + r1Errors.push('inputRequests missing expected key "user_name"'); + } else { + const req = r1Result.inputRequests['user_name']; + if (req.method !== 'elicitation/create') { + r1Errors.push( + `Expected method "elicitation/create", got "${req.method}"` + ); + } + } + } + + checks.push({ + id: 'incomplete-result-elicitation-incomplete', + name: 'IncompleteResultElicitationIncomplete', + description: + 'Server returns IncompleteResult with elicitation inputRequest', + status: r1Errors.length === 0 ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: r1Errors.length > 0 ? r1Errors.join('; ') : undefined, + specReferences: MRTR_SPEC_REFERENCES, + details: { result: r1Result } + }); + + // Round 2: Retry with inputResponses — expect complete result + if (r1Errors.length === 0 && isIncompleteResult(r1Result)) { + const r2 = await session.send('tools/call', { + name: 'test_tool_with_elicitation', + arguments: {}, + inputResponses: { + user_name: mockElicitResponse({ name: 'Alice' }) + }, + ...(r1Result.requestState !== undefined + ? { requestState: r1Result.requestState } + : {}) + }); + + const r2Result = r2.result; + const r2Errors: string[] = []; + + if (r2.error) { + r2Errors.push(`JSON-RPC error: ${r2.error.message}`); + } else if (!r2Result) { + r2Errors.push('No result in response'); + } else if (!isCompleteResult(r2Result)) { + r2Errors.push( + 'Expected complete result after retry with inputResponses' + ); + } else { + const content = r2Result.content as + | Array<{ type: string; text?: string }> + | undefined; + if (!content || !Array.isArray(content) || content.length === 0) { + r2Errors.push('Complete result missing content array'); + } + } + + checks.push({ + id: 'incomplete-result-elicitation-complete', + name: 'IncompleteResultElicitationComplete', + description: + 'Server returns complete result after retry with inputResponses', + status: r2Errors.length === 0 ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: r2Errors.length > 0 ? r2Errors.join('; ') : undefined, + specReferences: MRTR_SPEC_REFERENCES, + details: { result: r2Result } + }); + } + } catch (error) { + checks.push({ + id: 'incomplete-result-elicitation-incomplete', + name: 'IncompleteResultElicitationIncomplete', + description: + 'Server returns IncompleteResult with elicitation inputRequest', + status: 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`, + specReferences: MRTR_SPEC_REFERENCES + }); + } + + return checks; + } +} + +// ─── A2: Basic Sampling ────────────────────────────────────────────────────── + +export class IncompleteResultBasicSamplingScenario implements ClientScenario { + name = 'incomplete-result-basic-sampling'; + specVersions: SpecVersion[] = ['draft']; + description = `Test basic ephemeral IncompleteResult flow with a single sampling input request (SEP-2322). + +**Server Implementation Requirements:** + +Implement a tool named \`test_incomplete_result_sampling\` (no arguments required). + +**Behavior (Round 1):** When called without \`inputResponses\`, return an \`IncompleteResult\`: + +\`\`\`json +{ + "result_type": "incomplete", + "inputRequests": { + "capital_question": { + "method": "sampling/createMessage", + "params": { + "messages": [{ + "role": "user", + "content": { "type": "text", "text": "What is the capital of France?" } + }], + "maxTokens": 100 + } + } + } +} +\`\`\` + +**Behavior (Round 2):** When called with \`inputResponses\` containing the key \`"capital_question"\`, return a complete result with the sampling response text.`; + + async run(serverUrl: string): Promise { + const checks: ConformanceCheck[] = []; + + try { + const session = await createRawSession(serverUrl); + + // Round 1: Initial call + const r1 = await session.send('tools/call', { + name: 'test_incomplete_result_sampling', + arguments: {} + }); + + const r1Result = r1.result; + const r1Errors: string[] = []; + + if (r1.error) { + r1Errors.push(`JSON-RPC error: ${r1.error.message}`); + } else if (!r1Result) { + r1Errors.push('No result in response'); + } else if (!isIncompleteResult(r1Result)) { + r1Errors.push('Expected IncompleteResult with sampling inputRequest'); + } else { + if (!r1Result.inputRequests) { + r1Errors.push('IncompleteResult missing inputRequests'); + } else { + const key = Object.keys(r1Result.inputRequests)[0]; + if (!key) { + r1Errors.push('inputRequests map is empty'); + } else { + const req = r1Result.inputRequests[key]; + if (req.method !== 'sampling/createMessage') { + r1Errors.push( + `Expected method "sampling/createMessage", got "${req.method}"` + ); + } + } + } + } + + checks.push({ + id: 'incomplete-result-sampling-incomplete', + name: 'IncompleteResultSamplingIncomplete', + description: + 'Server returns IncompleteResult with sampling inputRequest', + status: r1Errors.length === 0 ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: r1Errors.length > 0 ? r1Errors.join('; ') : undefined, + specReferences: MRTR_SPEC_REFERENCES, + details: { result: r1Result } + }); + + // Round 2: Retry with inputResponses + if (r1Errors.length === 0 && isIncompleteResult(r1Result)) { + const inputKey = Object.keys(r1Result.inputRequests!)[0]; + const r2 = await session.send('tools/call', { + name: 'test_incomplete_result_sampling', + arguments: {}, + inputResponses: { + [inputKey]: mockSamplingResponse('The capital of France is Paris.') + }, + ...(r1Result.requestState !== undefined + ? { requestState: r1Result.requestState } + : {}) + }); + + const r2Result = r2.result; + const r2Errors: string[] = []; + + if (r2.error) { + r2Errors.push(`JSON-RPC error: ${r2.error.message}`); + } else if (!r2Result) { + r2Errors.push('No result in response'); + } else if (!isCompleteResult(r2Result)) { + r2Errors.push( + 'Expected complete result after retry with sampling response' + ); + } + + checks.push({ + id: 'incomplete-result-sampling-complete', + name: 'IncompleteResultSamplingComplete', + description: + 'Server returns complete result after retry with sampling response', + status: r2Errors.length === 0 ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: r2Errors.length > 0 ? r2Errors.join('; ') : undefined, + specReferences: MRTR_SPEC_REFERENCES, + details: { result: r2Result } + }); + } + } catch (error) { + checks.push({ + id: 'incomplete-result-sampling-incomplete', + name: 'IncompleteResultSamplingIncomplete', + description: + 'Server returns IncompleteResult with sampling inputRequest', + status: 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`, + specReferences: MRTR_SPEC_REFERENCES + }); + } + + return checks; + } +} + +// ─── A3: Basic ListRoots ───────────────────────────────────────────────────── + +export class IncompleteResultBasicListRootsScenario implements ClientScenario { + name = 'incomplete-result-basic-list-roots'; + specVersions: SpecVersion[] = ['draft']; + description = `Test basic ephemeral IncompleteResult flow with a single roots/list input request (SEP-2322). + +**Server Implementation Requirements:** + +Implement a tool named \`test_incomplete_result_list_roots\` (no arguments required). + +**Behavior (Round 1):** When called without \`inputResponses\`, return an \`IncompleteResult\`: + +\`\`\`json +{ + "result_type": "incomplete", + "inputRequests": { + "client_roots": { + "method": "roots/list", + "params": {} + } + } +} +\`\`\` + +**Behavior (Round 2):** When called with \`inputResponses\` containing the key \`"client_roots"\` (a ListRootsResult with a \`roots\` array), return a complete result that references the provided roots.`; + + async run(serverUrl: string): Promise { + const checks: ConformanceCheck[] = []; + + try { + const session = await createRawSession(serverUrl); + + // Round 1: Initial call + const r1 = await session.send('tools/call', { + name: 'test_incomplete_result_list_roots', + arguments: {} + }); + + const r1Result = r1.result; + const r1Errors: string[] = []; + + if (r1.error) { + r1Errors.push(`JSON-RPC error: ${r1.error.message}`); + } else if (!r1Result) { + r1Errors.push('No result in response'); + } else if (!isIncompleteResult(r1Result)) { + r1Errors.push('Expected IncompleteResult with roots/list inputRequest'); + } else { + if (!r1Result.inputRequests) { + r1Errors.push('IncompleteResult missing inputRequests'); + } else { + const key = Object.keys(r1Result.inputRequests)[0]; + if (!key) { + r1Errors.push('inputRequests map is empty'); + } else { + const req = r1Result.inputRequests[key]; + if (req.method !== 'roots/list') { + r1Errors.push( + `Expected method "roots/list", got "${req.method}"` + ); + } + } + } + } + + checks.push({ + id: 'incomplete-result-list-roots-incomplete', + name: 'IncompleteResultListRootsIncomplete', + description: + 'Server returns IncompleteResult with roots/list inputRequest', + status: r1Errors.length === 0 ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: r1Errors.length > 0 ? r1Errors.join('; ') : undefined, + specReferences: MRTR_SPEC_REFERENCES, + details: { result: r1Result } + }); + + // Round 2: Retry with inputResponses + if (r1Errors.length === 0 && isIncompleteResult(r1Result)) { + const inputKey = Object.keys(r1Result.inputRequests!)[0]; + const r2 = await session.send('tools/call', { + name: 'test_incomplete_result_list_roots', + arguments: {}, + inputResponses: { + [inputKey]: mockListRootsResponse() + }, + ...(r1Result.requestState !== undefined + ? { requestState: r1Result.requestState } + : {}) + }); + + const r2Result = r2.result; + const r2Errors: string[] = []; + + if (r2.error) { + r2Errors.push(`JSON-RPC error: ${r2.error.message}`); + } else if (!r2Result) { + r2Errors.push('No result in response'); + } else if (!isCompleteResult(r2Result)) { + r2Errors.push( + 'Expected complete result after retry with roots response' + ); + } + + checks.push({ + id: 'incomplete-result-list-roots-complete', + name: 'IncompleteResultListRootsComplete', + description: + 'Server returns complete result after retry with roots response', + status: r2Errors.length === 0 ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: r2Errors.length > 0 ? r2Errors.join('; ') : undefined, + specReferences: MRTR_SPEC_REFERENCES, + details: { result: r2Result } + }); + } + } catch (error) { + checks.push({ + id: 'incomplete-result-list-roots-incomplete', + name: 'IncompleteResultListRootsIncomplete', + description: + 'Server returns IncompleteResult with roots/list inputRequest', + status: 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`, + specReferences: MRTR_SPEC_REFERENCES + }); + } + + return checks; + } +} + +// ─── A4: Request State ────────────────────────────────────────────────────── + +export class IncompleteResultRequestStateScenario implements ClientScenario { + name = 'incomplete-result-request-state'; + specVersions: SpecVersion[] = ['draft']; + description = `Test that requestState is correctly round-tripped in ephemeral IncompleteResult flow (SEP-2322). + +**Server Implementation Requirements:** + +Implement a tool named \`test_incomplete_result_request_state\` (no arguments required). + +**Behavior (Round 1):** Return an \`IncompleteResult\` with both \`inputRequests\` and \`requestState\`: + +\`\`\`json +{ + "result_type": "incomplete", + "inputRequests": { + "confirm": { + "method": "elicitation/create", + "params": { + "message": "Please confirm", + "requestedSchema": { + "type": "object", + "properties": { "ok": { "type": "boolean" } }, + "required": ["ok"] + } + } + } + }, + "requestState": "" +} +\`\`\` + +**Behavior (Round 2):** When called with \`inputResponses\` AND the echoed \`requestState\`, validate the state and return a complete result. The text content MUST include the word "state-ok" to confirm the server received and validated the requestState.`; + + async run(serverUrl: string): Promise { + const checks: ConformanceCheck[] = []; + + try { + const session = await createRawSession(serverUrl); + + // Round 1 + const r1 = await session.send('tools/call', { + name: 'test_incomplete_result_request_state', + arguments: {} + }); + + const r1Result = r1.result; + const r1Errors: string[] = []; + + if (r1.error) { + r1Errors.push(`JSON-RPC error: ${r1.error.message}`); + } else if (!r1Result || !isIncompleteResult(r1Result)) { + r1Errors.push('Expected IncompleteResult'); + } else { + if (!r1Result.requestState) { + r1Errors.push('IncompleteResult missing requestState'); + } + if (typeof r1Result.requestState !== 'string') { + r1Errors.push('requestState must be a string'); + } + if (!r1Result.inputRequests) { + r1Errors.push('IncompleteResult missing inputRequests'); + } + } + + checks.push({ + id: 'incomplete-result-request-state-incomplete', + name: 'IncompleteResultRequestStateIncomplete', + description: + 'Server returns IncompleteResult with both inputRequests and requestState', + status: r1Errors.length === 0 ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: r1Errors.length > 0 ? r1Errors.join('; ') : undefined, + specReferences: MRTR_SPEC_REFERENCES, + details: { result: r1Result } + }); + + // Round 2: Retry with inputResponses + requestState + if (r1Errors.length === 0 && isIncompleteResult(r1Result)) { + const inputKey = Object.keys(r1Result.inputRequests!)[0]; + const r2 = await session.send('tools/call', { + name: 'test_incomplete_result_request_state', + arguments: {}, + inputResponses: { + [inputKey]: mockElicitResponse({ ok: true }) + }, + requestState: r1Result.requestState + }); + + const r2Result = r2.result; + const r2Errors: string[] = []; + + if (r2.error) { + r2Errors.push(`JSON-RPC error: ${r2.error.message}`); + } else if (!r2Result) { + r2Errors.push('No result in response'); + } else if (!isCompleteResult(r2Result)) { + r2Errors.push( + 'Expected complete result after retry with requestState' + ); + } else { + // Check that server confirmed it received the state + const content = r2Result.content as + | Array<{ type: string; text?: string }> + | undefined; + const text = content?.find((c) => c.type === 'text')?.text ?? ''; + if (!text.includes('state-ok')) { + r2Errors.push( + 'Server response text should include "state-ok" to confirm requestState was validated' + ); + } + } + + checks.push({ + id: 'incomplete-result-request-state-complete', + name: 'IncompleteResultRequestStateComplete', + description: + 'Server validates echoed requestState and returns complete result', + status: r2Errors.length === 0 ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: r2Errors.length > 0 ? r2Errors.join('; ') : undefined, + specReferences: MRTR_SPEC_REFERENCES, + details: { result: r2Result } + }); + } + } catch (error) { + checks.push({ + id: 'incomplete-result-request-state-incomplete', + name: 'IncompleteResultRequestStateIncomplete', + description: + 'Server returns IncompleteResult with both inputRequests and requestState', + status: 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`, + specReferences: MRTR_SPEC_REFERENCES + }); + } + + return checks; + } +} + +// ─── A5: Multiple Input Requests ───────────────────────────────────────────── + +export class IncompleteResultMultipleInputRequestsScenario + implements ClientScenario +{ + name = 'incomplete-result-multiple-input-requests'; + specVersions: SpecVersion[] = ['draft']; + description = `Test multiple input requests in a single IncompleteResult (SEP-2322). + +**Server Implementation Requirements:** + +Implement a tool named \`test_incomplete_result_multiple_inputs\` (no arguments required). + +**Behavior (Round 1):** Return an \`IncompleteResult\` with multiple \`inputRequests\` — elicitation, sampling, and roots/list — plus \`requestState\`: + +\`\`\`json +{ + "result_type": "incomplete", + "inputRequests": { + "user_name": { + "method": "elicitation/create", + "params": { + "message": "What is your name?", + "requestedSchema": { + "type": "object", + "properties": { "name": { "type": "string" } }, + "required": ["name"] + } + } + }, + "greeting": { + "method": "sampling/createMessage", + "params": { + "messages": [{ "role": "user", "content": { "type": "text", "text": "Generate a greeting" } }], + "maxTokens": 50 + } + }, + "client_roots": { + "method": "roots/list", + "params": {} + } + }, + "requestState": "" +} +\`\`\` + +**Behavior (Round 2):** When called with \`inputResponses\` containing ALL keys and the echoed \`requestState\`, return a complete result.`; + + async run(serverUrl: string): Promise { + const checks: ConformanceCheck[] = []; + + try { + const session = await createRawSession(serverUrl); + + // Round 1 + const r1 = await session.send('tools/call', { + name: 'test_incomplete_result_multiple_inputs', + arguments: {} + }); + + const r1Result = r1.result; + const r1Errors: string[] = []; + + if (r1.error) { + r1Errors.push(`JSON-RPC error: ${r1.error.message}`); + } else if (!r1Result || !isIncompleteResult(r1Result)) { + r1Errors.push('Expected IncompleteResult'); + } else if (!r1Result.inputRequests) { + r1Errors.push('IncompleteResult missing inputRequests'); + } else { + if (!r1Result.requestState) { + r1Errors.push('IncompleteResult missing requestState'); + } + + const keys = Object.keys(r1Result.inputRequests); + if (keys.length < 3) { + r1Errors.push( + `Expected at least 3 inputRequests, got ${keys.length}` + ); + } + + // Check that required method types are present + const methods = new Set( + keys.map((k) => r1Result.inputRequests![k].method) + ); + if (!methods.has('elicitation/create')) { + r1Errors.push('Expected an elicitation/create inputRequest'); + } + if (!methods.has('sampling/createMessage')) { + r1Errors.push('Expected a sampling/createMessage inputRequest'); + } + if (!methods.has('roots/list')) { + r1Errors.push('Expected a roots/list inputRequest'); + } + if (methods.size < 3) { + r1Errors.push( + 'Expected inputRequests with different method types (elicitation + sampling + roots/list)' + ); + } + } + + checks.push({ + id: 'incomplete-result-multiple-inputs-incomplete', + name: 'IncompleteResultMultipleInputsIncomplete', + description: + 'Server returns IncompleteResult with multiple inputRequests of different types', + status: r1Errors.length === 0 ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: r1Errors.length > 0 ? r1Errors.join('; ') : undefined, + specReferences: MRTR_SPEC_REFERENCES, + details: { result: r1Result } + }); + + // Round 2: Respond to all input requests + if (r1Errors.length === 0 && isIncompleteResult(r1Result)) { + const inputResponses: Record = {}; + for (const [key, req] of Object.entries(r1Result.inputRequests!)) { + if (req.method === 'elicitation/create') { + inputResponses[key] = mockElicitResponse({ name: 'Alice' }); + } else if (req.method === 'sampling/createMessage') { + inputResponses[key] = mockSamplingResponse('Hello there!'); + } else if (req.method === 'roots/list') { + inputResponses[key] = mockListRootsResponse(); + } + } + + const r2 = await session.send('tools/call', { + name: 'test_incomplete_result_multiple_inputs', + arguments: {}, + inputResponses, + ...(r1Result.requestState !== undefined + ? { requestState: r1Result.requestState } + : {}) + }); + + const r2Result = r2.result; + const r2Errors: string[] = []; + + if (r2.error) { + r2Errors.push(`JSON-RPC error: ${r2.error.message}`); + } else if (!r2Result) { + r2Errors.push('No result in response'); + } else if (!isCompleteResult(r2Result)) { + r2Errors.push( + 'Expected complete result after providing all inputResponses' + ); + } + + checks.push({ + id: 'incomplete-result-multiple-inputs-complete', + name: 'IncompleteResultMultipleInputsComplete', + description: + 'Server returns complete result after all inputResponses are provided', + status: r2Errors.length === 0 ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: r2Errors.length > 0 ? r2Errors.join('; ') : undefined, + specReferences: MRTR_SPEC_REFERENCES, + details: { result: r2Result } + }); + } + } catch (error) { + checks.push({ + id: 'incomplete-result-multiple-inputs-incomplete', + name: 'IncompleteResultMultipleInputsIncomplete', + description: + 'Server returns IncompleteResult with multiple inputRequests of different types', + status: 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`, + specReferences: MRTR_SPEC_REFERENCES + }); + } + + return checks; + } +} + +// ─── A6: Multi-Round ───────────────────────────────────────────────────────── + +export class IncompleteResultMultiRoundScenario implements ClientScenario { + name = 'incomplete-result-multi-round'; + specVersions: SpecVersion[] = ['draft']; + description = `Test multi-round ephemeral IncompleteResult flow with evolving requestState (SEP-2322). + +**Server Implementation Requirements:** + +Implement a tool named \`test_incomplete_result_multi_round\` (no arguments required). + +**Behavior (Round 1):** Return an \`IncompleteResult\` with an elicitation request and \`requestState\`: + +\`\`\`json +{ + "result_type": "incomplete", + "inputRequests": { + "step1": { + "method": "elicitation/create", + "params": { + "message": "Step 1: What is your name?", + "requestedSchema": { + "type": "object", + "properties": { "name": { "type": "string" } }, + "required": ["name"] + } + } + } + }, + "requestState": "" +} +\`\`\` + +**Behavior (Round 2):** When called with \`inputResponses\` for step1 + requestState, return ANOTHER \`IncompleteResult\` with a new elicitation and updated requestState: + +\`\`\`json +{ + "result_type": "incomplete", + "inputRequests": { + "step2": { + "method": "elicitation/create", + "params": { + "message": "Step 2: What is your favorite color?", + "requestedSchema": { + "type": "object", + "properties": { "color": { "type": "string" } }, + "required": ["color"] + } + } + } + }, + "requestState": "" +} +\`\`\` + +**Behavior (Round 3):** When called with \`inputResponses\` for step2 + updated requestState, return a complete result.`; + + async run(serverUrl: string): Promise { + const checks: ConformanceCheck[] = []; + + try { + const session = await createRawSession(serverUrl); + + // Round 1 + const r1 = await session.send('tools/call', { + name: 'test_incomplete_result_multi_round', + arguments: {} + }); + + const r1Result = r1.result; + let round1Complete = false; + + if ( + !r1.error && + r1Result && + isIncompleteResult(r1Result) && + r1Result.inputRequests && + r1Result.requestState + ) { + round1Complete = true; + } + + checks.push({ + id: 'incomplete-result-multi-round-r1', + name: 'IncompleteResultMultiRoundR1', + description: + 'Round 1: Server returns IncompleteResult with requestState', + status: round1Complete ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: round1Complete + ? undefined + : 'Expected IncompleteResult with inputRequests and requestState', + specReferences: MRTR_SPEC_REFERENCES, + details: { result: r1Result } + }); + + if (!round1Complete || !isIncompleteResult(r1Result)) return checks; + + // Round 2: Retry — expect another IncompleteResult + const r1InputKey = Object.keys(r1Result.inputRequests!)[0]; + const r2 = await session.send('tools/call', { + name: 'test_incomplete_result_multi_round', + arguments: {}, + inputResponses: { + [r1InputKey]: mockElicitResponse({ name: 'Alice' }) + }, + requestState: r1Result.requestState + }); + + const r2Result = r2.result; + let round2Complete = false; + + if ( + !r2.error && + r2Result && + isIncompleteResult(r2Result) && + r2Result.inputRequests && + r2Result.requestState + ) { + // requestState should have changed + if (r2Result.requestState !== r1Result.requestState) { + round2Complete = true; + } + } + + checks.push({ + id: 'incomplete-result-multi-round-r2', + name: 'IncompleteResultMultiRoundR2', + description: + 'Round 2: Server returns another IncompleteResult with updated requestState', + status: round2Complete ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: round2Complete + ? undefined + : 'Expected new IncompleteResult with different requestState', + specReferences: MRTR_SPEC_REFERENCES, + details: { result: r2Result } + }); + + if (!round2Complete || !isIncompleteResult(r2Result)) return checks; + + // Round 3: Final retry — expect complete result + const r2InputKey = Object.keys(r2Result.inputRequests!)[0]; + const r3 = await session.send('tools/call', { + name: 'test_incomplete_result_multi_round', + arguments: {}, + inputResponses: { + [r2InputKey]: mockElicitResponse({ color: 'blue' }) + }, + requestState: r2Result.requestState + }); + + const r3Result = r3.result; + const round3Complete = + !r3.error && r3Result != null && isCompleteResult(r3Result); + + checks.push({ + id: 'incomplete-result-multi-round-r3', + name: 'IncompleteResultMultiRoundR3', + description: 'Round 3: Server returns complete result', + status: round3Complete ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: round3Complete + ? undefined + : 'Expected complete result after final retry', + specReferences: MRTR_SPEC_REFERENCES, + details: { result: r3Result } + }); + } catch (error) { + checks.push({ + id: 'incomplete-result-multi-round-r1', + name: 'IncompleteResultMultiRoundR1', + description: + 'Round 1: Server returns IncompleteResult with requestState', + status: 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`, + specReferences: MRTR_SPEC_REFERENCES + }); + } + + return checks; + } +} + +// ─── A7: Missing Input Response ────────────────────────────────────────────── + +export class IncompleteResultMissingInputResponseScenario + implements ClientScenario +{ + name = 'incomplete-result-missing-input-response'; + specVersions: SpecVersion[] = ['draft']; + description = `Test error handling when client sends wrong/missing inputResponses (SEP-2322). + +**Server Implementation Requirements:** + +Use the same tool as A1: \`test_incomplete_result_elicitation\`. + +**Behavior:** When the client retries with \`inputResponses\` that are missing required keys or contain wrong keys, the server SHOULD respond with a new \`IncompleteResult\` re-requesting the missing information (NOT a JSON-RPC error).`; + + async run(serverUrl: string): Promise { + const checks: ConformanceCheck[] = []; + + try { + const session = await createRawSession(serverUrl); + + // Round 1: Send wrong inputResponses (wrong key) + const r1 = await session.send('tools/call', { + name: 'test_incomplete_result_elicitation', + arguments: {}, + inputResponses: { + wrong_key: mockElicitResponse({ data: 'wrong' }) + } + }); + + const r1Result = r1.result; + const r1Errors: string[] = []; + + if (r1.error) { + // A JSON-RPC error is acceptable but the SEP prefers re-requesting + r1Errors.push( + 'Server returned JSON-RPC error instead of re-requesting via IncompleteResult. ' + + 'SEP-2322 recommends servers re-request missing information.' + ); + } else if (!r1Result) { + r1Errors.push('No result in response'); + } else if (!isIncompleteResult(r1Result)) { + r1Errors.push( + 'Expected IncompleteResult re-requesting missing information, ' + + 'but got a complete result' + ); + } + + checks.push({ + id: 'incomplete-result-missing-response-rerequests', + name: 'IncompleteResultMissingResponseRerequests', + description: + 'Server re-requests missing inputResponses via new IncompleteResult', + status: r1Errors.length === 0 ? 'SUCCESS' : 'WARNING', + timestamp: new Date().toISOString(), + errorMessage: r1Errors.length > 0 ? r1Errors.join('; ') : undefined, + specReferences: MRTR_SPEC_REFERENCES, + details: { result: r1Result } + }); + } catch (error) { + checks.push({ + id: 'incomplete-result-missing-response-rerequests', + name: 'IncompleteResultMissingResponseRerequests', + description: + 'Server re-requests missing inputResponses via new IncompleteResult', + status: 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`, + specReferences: MRTR_SPEC_REFERENCES + }); + } + + return checks; + } +} + +// ─── A9: Non-Tool Request (prompts/get) ────────────────────────────────────── + +export class IncompleteResultNonToolRequestScenario implements ClientScenario { + name = 'incomplete-result-non-tool-request'; + specVersions: SpecVersion[] = ['draft']; + description = `Test IncompleteResult on a non-tool request (prompts/get) to verify IncompleteResult is universal (SEP-2322). + +**Server Implementation Requirements:** + +Implement a prompt named \`test_incomplete_result_prompt\` that requires elicitation input. + +**Behavior (Round 1):** When \`prompts/get\` is called for \`test_incomplete_result_prompt\` without \`inputResponses\`, return an \`IncompleteResult\`: + +\`\`\`json +{ + "result_type": "incomplete", + "inputRequests": { + "user_context": { + "method": "elicitation/create", + "params": { + "message": "What context should the prompt use?", + "requestedSchema": { + "type": "object", + "properties": { "context": { "type": "string" } }, + "required": ["context"] + } + } + } + } +} +\`\`\` + +**Behavior (Round 2):** When called with \`inputResponses\`, return a complete \`GetPromptResult\`.`; + + async run(serverUrl: string): Promise { + const checks: ConformanceCheck[] = []; + + try { + const session = await createRawSession(serverUrl); + + // Round 1 + const r1 = await session.send('prompts/get', { + name: 'test_incomplete_result_prompt' + }); + + const r1Result = r1.result; + const r1Errors: string[] = []; + + if (r1.error) { + r1Errors.push(`JSON-RPC error: ${r1.error.message}`); + } else if (!r1Result || !isIncompleteResult(r1Result)) { + r1Errors.push('Expected IncompleteResult from prompts/get'); + } else if (!r1Result.inputRequests) { + r1Errors.push('IncompleteResult missing inputRequests'); + } + + checks.push({ + id: 'incomplete-result-non-tool-incomplete', + name: 'IncompleteResultNonToolIncomplete', + description: 'prompts/get returns IncompleteResult with inputRequests', + status: r1Errors.length === 0 ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: r1Errors.length > 0 ? r1Errors.join('; ') : undefined, + specReferences: MRTR_SPEC_REFERENCES, + details: { result: r1Result } + }); + + // Round 2: Retry with inputResponses + if (r1Errors.length === 0 && isIncompleteResult(r1Result)) { + const inputKey = Object.keys(r1Result.inputRequests!)[0]; + const r2 = await session.send('prompts/get', { + name: 'test_incomplete_result_prompt', + inputResponses: { + [inputKey]: mockElicitResponse({ context: 'test context' }) + }, + ...(r1Result.requestState !== undefined + ? { requestState: r1Result.requestState } + : {}) + }); + + const r2Result = r2.result; + const r2Errors: string[] = []; + + if (r2.error) { + r2Errors.push(`JSON-RPC error: ${r2.error.message}`); + } else if (!r2Result) { + r2Errors.push('No result in response'); + } else if (!isCompleteResult(r2Result)) { + r2Errors.push('Expected complete GetPromptResult after retry'); + } else if (!r2Result.messages) { + r2Errors.push( + 'Complete result missing messages (expected GetPromptResult)' + ); + } + + checks.push({ + id: 'incomplete-result-non-tool-complete', + name: 'IncompleteResultNonToolComplete', + description: + 'prompts/get returns complete GetPromptResult after retry with inputResponses', + status: r2Errors.length === 0 ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: r2Errors.length > 0 ? r2Errors.join('; ') : undefined, + specReferences: MRTR_SPEC_REFERENCES, + details: { result: r2Result } + }); + } + } catch (error) { + checks.push({ + id: 'incomplete-result-non-tool-incomplete', + name: 'IncompleteResultNonToolIncomplete', + description: 'prompts/get returns IncompleteResult with inputRequests', + status: 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`, + specReferences: MRTR_SPEC_REFERENCES + }); + } + + return checks; + } +}