diff --git a/index.ts b/index.ts index 361e749..2f8c114 100644 --- a/index.ts +++ b/index.ts @@ -133,6 +133,7 @@ interface PluginConfig { }; }; mdMirror?: { enabled?: boolean; dir?: string }; + exposeRetrievalMetadata?: boolean; } type ReflectionThinkLevel = "off" | "minimal" | "low" | "medium" | "high"; @@ -1606,6 +1607,7 @@ const memoryLanceDBProPlugin = { agentId: undefined, // Will be determined at runtime from context workspaceDir: getDefaultWorkspaceDir(), mdMirror, + exposeRetrievalMetadata: config.exposeRetrievalMetadata, }, { enableManagementTools: config.enableManagementTools, @@ -1691,7 +1693,7 @@ const memoryLanceDBProPlugin = { return postProcessAutoRecallResults(retrieved).slice(0, topK); }, formatLine: (row) => - `- [${row.entry.category}:${row.entry.scope}] ${sanitizeForContext(row.entry.text)} (${(row.score * 100).toFixed(0)}%${row.sources?.bm25 ? ", vector+BM25" : ""}${row.sources?.reranked ? "+reranked" : ""})`, + `- [${row.entry.category}:${row.entry.scope}] ${sanitizeForContext(row.entry.text)}`, }); } catch (err) { api.logger.warn(`memory-lancedb-pro: auto-recall failed: ${String(err)}`); @@ -2882,6 +2884,7 @@ export function parsePluginConfig(value: unknown): PluginConfig { : undefined, } : undefined, + exposeRetrievalMetadata: cfg.exposeRetrievalMetadata === true, }; } diff --git a/openclaw.plugin.json b/openclaw.plugin.json index ef842dd..7c769dc 100644 --- a/openclaw.plugin.json +++ b/openclaw.plugin.json @@ -489,6 +489,11 @@ "description": "Fallback directory for Markdown mirror files when agent workspace is unknown" } } + }, + "exposeRetrievalMetadata": { + "type": "boolean", + "default": false, + "description": "Expose retrieval metadata (score, sources) in details.debug field. Useful for debugging retrieval quality. Default: false to keep output clean." } }, "required": [ diff --git a/package.json b/package.json index 57069d9..9909257 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ ] }, "scripts": { - "test": "node test/embedder-error-hints.test.mjs && node test/migrate-legacy-schema.test.mjs && node test/cli-smoke.mjs && node --test test/memory-reflection.test.mjs test/self-improvement.test.mjs" + "test": "node test/embedder-error-hints.test.mjs && node test/migrate-legacy-schema.test.mjs && node test/cli-smoke.mjs && node --test test/memory-recall-metadata.test.mjs test/memory-reflection.test.mjs test/self-improvement.test.mjs" }, "devDependencies": { "commander": "^14.0.0", diff --git a/src/tools.ts b/src/tools.ts index 620845a..e6f9d88 100644 --- a/src/tools.ts +++ b/src/tools.ts @@ -43,6 +43,7 @@ interface ToolContext { agentId?: string; workspaceDir?: string; mdMirror?: MdMirrorWriter | null; + exposeRetrievalMetadata?: boolean; } function resolveAgentId(runtimeAgentId: unknown, fallback?: string): string | undefined { @@ -65,17 +66,50 @@ function clamp01(value: number, fallback = 0.7): number { return Math.min(1, Math.max(0, value)); } -function sanitizeMemoryForSerialization(results: RetrievalResult[]) { - return results.map((r) => ({ - id: r.entry.id, - text: r.entry.text, - category: getDisplayCategoryTag(r.entry), - rawCategory: r.entry.category, - scope: r.entry.scope, - importance: r.entry.importance, - score: r.score, - sources: r.sources, - })); +type SerializedMemory = { + id: string; + text: string; + category: string; + rawCategory: string; + scope: string; + importance: number; +}; + +type DebugMemory = { + id: string; + score: number; + sources: RetrievalResult["sources"]; +}; + +type SerializedMemoryPayload = { + memories: SerializedMemory[]; + debug?: DebugMemory[]; +}; + +function buildSanitizedMemoryPayload( + results: RetrievalResult[], + exposeRetrievalMetadata?: boolean, +): SerializedMemoryPayload { + const payload: SerializedMemoryPayload = { + memories: results.map((r) => ({ + id: r.entry.id, + text: r.entry.text, + category: getDisplayCategoryTag(r.entry), + rawCategory: r.entry.category, + scope: r.entry.scope, + importance: r.entry.importance, + })), + }; + + if (exposeRetrievalMetadata) { + payload.debug = results.map((r) => ({ + id: r.entry.id, + score: r.score, + sources: r.sources, + })); + } + + return payload; } function resolveWorkspaceDir(toolCtx: unknown, fallback?: string): string { @@ -408,16 +442,16 @@ export function registerMemoryRecallTool( const text = results .map((r, i) => { - const sources = []; - if (r.sources.vector) sources.push("vector"); - if (r.sources.bm25) sources.push("BM25"); - if (r.sources.reranked) sources.push("reranked"); - const categoryTag = getDisplayCategoryTag(r.entry); - return `${i + 1}. [${r.entry.id}] [${categoryTag}] ${r.entry.text} (${(r.score * 100).toFixed(0)}%${sources.length > 0 ? `, ${sources.join("+")}` : ""})`; + return `${i + 1}. [${r.entry.id}] [${categoryTag}] ${r.entry.text}`; }) .join("\n"); + const serialized = buildSanitizedMemoryPayload( + results, + context.exposeRetrievalMetadata, + ); + return { content: [ { @@ -427,7 +461,8 @@ export function registerMemoryRecallTool( ], details: { count: results.length, - memories: sanitizeMemoryForSerialization(results), + memories: serialized.memories, + ...(serialized.debug ? { debug: serialized.debug } : {}), query, scopes: scopeFilter, retrievalMode: context.retriever.getConfig().mode, @@ -718,6 +753,11 @@ export function registerMemoryForgetTool( ) .join("\n"); + const serialized = buildSanitizedMemoryPayload( + results, + context.exposeRetrievalMetadata, + ); + return { content: [ { @@ -727,7 +767,8 @@ export function registerMemoryForgetTool( ], details: { action: "candidates", - candidates: sanitizeMemoryForSerialization(results), + candidates: serialized.memories, + ...(serialized.debug ? { debug: serialized.debug } : {}), }, }; } @@ -844,6 +885,11 @@ export function registerMemoryUpdateTool( `- [${r.entry.id.slice(0, 8)}] ${r.entry.text.slice(0, 60)}${r.entry.text.length > 60 ? "..." : ""}`, ) .join("\n"); + const serialized = buildSanitizedMemoryPayload( + results, + context.exposeRetrievalMetadata, + ); + return { content: [ { @@ -853,7 +899,8 @@ export function registerMemoryUpdateTool( ], details: { action: "candidates", - candidates: sanitizeMemoryForSerialization(results), + candidates: serialized.memories, + ...(serialized.debug ? { debug: serialized.debug } : {}), }, }; } diff --git a/test/memory-recall-metadata.test.mjs b/test/memory-recall-metadata.test.mjs new file mode 100644 index 0000000..b0004fd --- /dev/null +++ b/test/memory-recall-metadata.test.mjs @@ -0,0 +1,158 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import jitiFactory from "jiti"; + +const testDir = path.dirname(fileURLToPath(import.meta.url)); +const pluginSdkStubPath = path.resolve(testDir, "helpers", "openclaw-plugin-sdk-stub.mjs"); +const jiti = jitiFactory(import.meta.url, { + interopDefault: true, + alias: { + "openclaw/plugin-sdk": pluginSdkStubPath, + }, +}); + +const { + registerMemoryRecallTool, + registerMemoryForgetTool, + registerMemoryUpdateTool, +} = jiti("../src/tools.ts"); + +function makeResults(count = 1) { + return Array.from({ length: count }, (_, index) => ({ + entry: { + id: `m${index + 1}`, + text: `remember this ${index + 1}`, + category: "fact", + scope: "global", + importance: 0.7, + }, + score: 0.82 - index * 0.01, + sources: { + vector: { score: 0.82 - index * 0.01, rank: index + 1 }, + bm25: { score: 0.88 - index * 0.01, rank: index + 2 }, + }, + })); +} + +function makeApiCapture() { + let capturedCreator = null; + const api = { + registerTool(cb) { + capturedCreator = cb; + }, + logger: { info: () => {}, warn: () => {}, debug: () => {} }, + }; + return { api, getCreator: () => capturedCreator }; +} + +function makeContext({ expose = false, results = makeResults() } = {}) { + return { + retriever: { + async retrieve() { + return results; + }, + getConfig() { + return { mode: "hybrid" }; + }, + }, + store: { + async delete() { + return true; + }, + async update() { + return true; + }, + }, + scopeManager: { + getAccessibleScopes: () => ["global"], + isAccessible: () => true, + getDefaultScope: () => "global", + }, + embedder: { embedPassage: async () => [] }, + agentId: "main", + workspaceDir: "/tmp", + mdMirror: null, + exposeRetrievalMetadata: expose, + }; +} + +function createTool(registerTool, context) { + const { api, getCreator } = makeApiCapture(); + registerTool(api, context); + const creator = getCreator(); + assert.ok(typeof creator === "function"); + return creator({}); +} + +describe("memory metadata exposure", () => { + it("keeps memory_recall text clean when exposeRetrievalMetadata=false", async () => { + const tool = createTool( + registerMemoryRecallTool, + makeContext({ expose: false, results: makeResults(1) }), + ); + + const res = await tool.execute(null, { query: "test" }); + + assert.equal(res.details.count, 1); + assert.ok(Array.isArray(res.details.memories)); + assert.equal(res.details.debug, undefined); + assert.equal(Object.prototype.hasOwnProperty.call(res.details.memories[0], "score"), false); + assert.equal(Object.prototype.hasOwnProperty.call(res.details.memories[0], "sources"), false); + assert.match(res.content[0].text, /remember this 1/); + assert.doesNotMatch(res.content[0].text, /82%|vector|BM25|reranked/); + }); + + it("exposes debug metadata without polluting memory_recall text when enabled", async () => { + const tool = createTool( + registerMemoryRecallTool, + makeContext({ expose: true, results: makeResults(2) }), + ); + + const res = await tool.execute(null, { query: "test" }); + + assert.equal(res.details.count, 2); + assert.ok(Array.isArray(res.details.memories)); + assert.ok(Array.isArray(res.details.debug)); + assert.equal(res.details.debug.length, res.details.memories.length); + assert.equal(typeof res.details.debug[0].score, "number"); + assert.ok(res.details.debug[0].sources.vector); + assert.ok(res.details.debug[0].sources.bm25); + assert.equal(res.details.debug[0].id, res.details.memories[0].id); + assert.equal(res.details.debug[1].id, res.details.memories[1].id); + assert.doesNotMatch(res.content[0].text, /82%|vector|BM25|reranked/); + }); + + it("preserves details.candidates for memory_forget while attaching debug metadata separately", async () => { + const tool = createTool( + registerMemoryForgetTool, + makeContext({ expose: true, results: makeResults(2) }), + ); + + const res = await tool.execute(null, { query: "test" }); + + assert.equal(res.details.action, "candidates"); + assert.ok(Array.isArray(res.details.candidates)); + assert.equal(res.details.memories, undefined); + assert.ok(Array.isArray(res.details.debug)); + assert.equal(Object.prototype.hasOwnProperty.call(res.details.candidates[0], "score"), false); + assert.equal(Object.prototype.hasOwnProperty.call(res.details.candidates[0], "sources"), false); + }); + + it("preserves details.candidates for memory_update while attaching debug metadata separately", async () => { + const tool = createTool( + registerMemoryUpdateTool, + makeContext({ expose: true, results: makeResults(2) }), + ); + + const res = await tool.execute(null, { memoryId: "test", text: "updated text" }); + + assert.equal(res.details.action, "candidates"); + assert.ok(Array.isArray(res.details.candidates)); + assert.equal(res.details.memories, undefined); + assert.ok(Array.isArray(res.details.debug)); + assert.equal(Object.prototype.hasOwnProperty.call(res.details.candidates[0], "score"), false); + assert.equal(Object.prototype.hasOwnProperty.call(res.details.candidates[0], "sources"), false); + }); +});