Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@ interface PluginConfig {
};
};
mdMirror?: { enabled?: boolean; dir?: string };
exposeRetrievalMetadata?: boolean;
}

type ReflectionThinkLevel = "off" | "minimal" | "low" | "medium" | "high";
Expand Down Expand Up @@ -1606,6 +1607,7 @@ const memoryLanceDBProPlugin = {
agentId: undefined, // Will be determined at runtime from context
workspaceDir: getDefaultWorkspaceDir(),
mdMirror,
exposeRetrievalMetadata: config.exposeRetrievalMetadata,
},
{
enableManagementTools: config.enableManagementTools,
Expand Down Expand Up @@ -1691,7 +1693,7 @@ const memoryLanceDBProPlugin = {
return postProcessAutoRecallResults(retrieved).slice(0, topK);
},
formatLine: (row) =>
`- [${row.entry.category}:${row.entry.scope}] ${sanitizeForContext(row.entry.text)} (${(row.score * 100).toFixed(0)}%${row.sources?.bm25 ? ", vector+BM25" : ""}${row.sources?.reranked ? "+reranked" : ""})`,
`- [${row.entry.category}:${row.entry.scope}] ${sanitizeForContext(row.entry.text)}`,
});
} catch (err) {
api.logger.warn(`memory-lancedb-pro: auto-recall failed: ${String(err)}`);
Expand Down Expand Up @@ -2882,6 +2884,7 @@ export function parsePluginConfig(value: unknown): PluginConfig {
: undefined,
}
: undefined,
exposeRetrievalMetadata: cfg.exposeRetrievalMetadata === true,
};
}

Expand Down
5 changes: 5 additions & 0 deletions openclaw.plugin.json
Original file line number Diff line number Diff line change
Expand Up @@ -489,6 +489,11 @@
"description": "Fallback directory for Markdown mirror files when agent workspace is unknown"
}
}
},
"exposeRetrievalMetadata": {
"type": "boolean",
"default": false,
"description": "Expose retrieval metadata (score, sources) in details.debug field. Useful for debugging retrieval quality. Default: false to keep output clean."
}
},
"required": [
Expand Down
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
]
},
"scripts": {
"test": "node test/embedder-error-hints.test.mjs && node test/migrate-legacy-schema.test.mjs && node test/cli-smoke.mjs && node --test test/memory-reflection.test.mjs test/self-improvement.test.mjs"
"test": "node test/embedder-error-hints.test.mjs && node test/migrate-legacy-schema.test.mjs && node test/cli-smoke.mjs && node --test test/memory-recall-metadata.test.mjs test/memory-reflection.test.mjs test/self-improvement.test.mjs"
},
"devDependencies": {
"commander": "^14.0.0",
Expand Down
87 changes: 67 additions & 20 deletions src/tools.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ interface ToolContext {
agentId?: string;
workspaceDir?: string;
mdMirror?: MdMirrorWriter | null;
exposeRetrievalMetadata?: boolean;
}

function resolveAgentId(runtimeAgentId: unknown, fallback?: string): string | undefined {
Expand All @@ -65,17 +66,50 @@ function clamp01(value: number, fallback = 0.7): number {
return Math.min(1, Math.max(0, value));
}

function sanitizeMemoryForSerialization(results: RetrievalResult[]) {
return results.map((r) => ({
id: r.entry.id,
text: r.entry.text,
category: getDisplayCategoryTag(r.entry),
rawCategory: r.entry.category,
scope: r.entry.scope,
importance: r.entry.importance,
score: r.score,
sources: r.sources,
}));
type SerializedMemory = {
id: string;
text: string;
category: string;
rawCategory: string;
scope: string;
importance: number;
};

type DebugMemory = {
id: string;
score: number;
sources: RetrievalResult["sources"];
};

type SerializedMemoryPayload = {
memories: SerializedMemory[];
debug?: DebugMemory[];
};

function buildSanitizedMemoryPayload(
results: RetrievalResult[],
exposeRetrievalMetadata?: boolean,
): SerializedMemoryPayload {
const payload: SerializedMemoryPayload = {
memories: results.map((r) => ({
id: r.entry.id,
text: r.entry.text,
category: getDisplayCategoryTag(r.entry),
rawCategory: r.entry.category,
scope: r.entry.scope,
importance: r.entry.importance,
})),
};

if (exposeRetrievalMetadata) {
payload.debug = results.map((r) => ({
id: r.entry.id,
score: r.score,
sources: r.sources,
}));
}

return payload;
}

function resolveWorkspaceDir(toolCtx: unknown, fallback?: string): string {
Expand Down Expand Up @@ -408,16 +442,16 @@ export function registerMemoryRecallTool(

const text = results
.map((r, i) => {
const sources = [];
if (r.sources.vector) sources.push("vector");
if (r.sources.bm25) sources.push("BM25");
if (r.sources.reranked) sources.push("reranked");

const categoryTag = getDisplayCategoryTag(r.entry);
return `${i + 1}. [${r.entry.id}] [${categoryTag}] ${r.entry.text} (${(r.score * 100).toFixed(0)}%${sources.length > 0 ? `, ${sources.join("+")}` : ""})`;
return `${i + 1}. [${r.entry.id}] [${categoryTag}] ${r.entry.text}`;
})
.join("\n");

const serialized = buildSanitizedMemoryPayload(
results,
context.exposeRetrievalMetadata,
);

return {
content: [
{
Expand All @@ -427,7 +461,8 @@ export function registerMemoryRecallTool(
],
details: {
count: results.length,
memories: sanitizeMemoryForSerialization(results),
memories: serialized.memories,
...(serialized.debug ? { debug: serialized.debug } : {}),
query,
scopes: scopeFilter,
retrievalMode: context.retriever.getConfig().mode,
Expand Down Expand Up @@ -718,6 +753,11 @@ export function registerMemoryForgetTool(
)
.join("\n");

const serialized = buildSanitizedMemoryPayload(
results,
context.exposeRetrievalMetadata,
);

return {
content: [
{
Expand All @@ -727,7 +767,8 @@ export function registerMemoryForgetTool(
],
details: {
action: "candidates",
candidates: sanitizeMemoryForSerialization(results),
candidates: serialized.memories,
...(serialized.debug ? { debug: serialized.debug } : {}),
},
};
}
Expand Down Expand Up @@ -844,6 +885,11 @@ export function registerMemoryUpdateTool(
`- [${r.entry.id.slice(0, 8)}] ${r.entry.text.slice(0, 60)}${r.entry.text.length > 60 ? "..." : ""}`,
)
.join("\n");
const serialized = buildSanitizedMemoryPayload(
results,
context.exposeRetrievalMetadata,
);

return {
content: [
{
Expand All @@ -853,7 +899,8 @@ export function registerMemoryUpdateTool(
],
details: {
action: "candidates",
candidates: sanitizeMemoryForSerialization(results),
candidates: serialized.memories,
...(serialized.debug ? { debug: serialized.debug } : {}),
},
};
}
Expand Down
158 changes: 158 additions & 0 deletions test/memory-recall-metadata.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
import { describe, it } from "node:test";
import assert from "node:assert/strict";
import path from "node:path";
import { fileURLToPath } from "node:url";
import jitiFactory from "jiti";

const testDir = path.dirname(fileURLToPath(import.meta.url));
const pluginSdkStubPath = path.resolve(testDir, "helpers", "openclaw-plugin-sdk-stub.mjs");
const jiti = jitiFactory(import.meta.url, {
interopDefault: true,
alias: {
"openclaw/plugin-sdk": pluginSdkStubPath,
},
});

const {
registerMemoryRecallTool,
registerMemoryForgetTool,
registerMemoryUpdateTool,
} = jiti("../src/tools.ts");

function makeResults(count = 1) {
return Array.from({ length: count }, (_, index) => ({
entry: {
id: `m${index + 1}`,
text: `remember this ${index + 1}`,
category: "fact",
scope: "global",
importance: 0.7,
},
score: 0.82 - index * 0.01,
sources: {
vector: { score: 0.82 - index * 0.01, rank: index + 1 },
bm25: { score: 0.88 - index * 0.01, rank: index + 2 },
},
}));
}

function makeApiCapture() {
let capturedCreator = null;
const api = {
registerTool(cb) {
capturedCreator = cb;
},
logger: { info: () => {}, warn: () => {}, debug: () => {} },
};
return { api, getCreator: () => capturedCreator };
}

function makeContext({ expose = false, results = makeResults() } = {}) {
return {
retriever: {
async retrieve() {
return results;
},
getConfig() {
return { mode: "hybrid" };
},
},
store: {
async delete() {
return true;
},
async update() {
return true;
},
},
scopeManager: {
getAccessibleScopes: () => ["global"],
isAccessible: () => true,
getDefaultScope: () => "global",
},
embedder: { embedPassage: async () => [] },
agentId: "main",
workspaceDir: "/tmp",
mdMirror: null,
exposeRetrievalMetadata: expose,
};
}

function createTool(registerTool, context) {
const { api, getCreator } = makeApiCapture();
registerTool(api, context);
const creator = getCreator();
assert.ok(typeof creator === "function");
return creator({});
}

describe("memory metadata exposure", () => {
it("keeps memory_recall text clean when exposeRetrievalMetadata=false", async () => {
const tool = createTool(
registerMemoryRecallTool,
makeContext({ expose: false, results: makeResults(1) }),
);

const res = await tool.execute(null, { query: "test" });

assert.equal(res.details.count, 1);
assert.ok(Array.isArray(res.details.memories));
assert.equal(res.details.debug, undefined);
assert.equal(Object.prototype.hasOwnProperty.call(res.details.memories[0], "score"), false);
assert.equal(Object.prototype.hasOwnProperty.call(res.details.memories[0], "sources"), false);
assert.match(res.content[0].text, /remember this 1/);
assert.doesNotMatch(res.content[0].text, /82%|vector|BM25|reranked/);
});

it("exposes debug metadata without polluting memory_recall text when enabled", async () => {
const tool = createTool(
registerMemoryRecallTool,
makeContext({ expose: true, results: makeResults(2) }),
);

const res = await tool.execute(null, { query: "test" });

assert.equal(res.details.count, 2);
assert.ok(Array.isArray(res.details.memories));
assert.ok(Array.isArray(res.details.debug));
assert.equal(res.details.debug.length, res.details.memories.length);
assert.equal(typeof res.details.debug[0].score, "number");
assert.ok(res.details.debug[0].sources.vector);
assert.ok(res.details.debug[0].sources.bm25);
assert.equal(res.details.debug[0].id, res.details.memories[0].id);
assert.equal(res.details.debug[1].id, res.details.memories[1].id);
assert.doesNotMatch(res.content[0].text, /82%|vector|BM25|reranked/);
});

it("preserves details.candidates for memory_forget while attaching debug metadata separately", async () => {
const tool = createTool(
registerMemoryForgetTool,
makeContext({ expose: true, results: makeResults(2) }),
);

const res = await tool.execute(null, { query: "test" });

assert.equal(res.details.action, "candidates");
assert.ok(Array.isArray(res.details.candidates));
assert.equal(res.details.memories, undefined);
assert.ok(Array.isArray(res.details.debug));
assert.equal(Object.prototype.hasOwnProperty.call(res.details.candidates[0], "score"), false);
assert.equal(Object.prototype.hasOwnProperty.call(res.details.candidates[0], "sources"), false);
});

it("preserves details.candidates for memory_update while attaching debug metadata separately", async () => {
const tool = createTool(
registerMemoryUpdateTool,
makeContext({ expose: true, results: makeResults(2) }),
);

const res = await tool.execute(null, { memoryId: "test", text: "updated text" });

assert.equal(res.details.action, "candidates");
assert.ok(Array.isArray(res.details.candidates));
assert.equal(res.details.memories, undefined);
assert.ok(Array.isArray(res.details.debug));
assert.equal(Object.prototype.hasOwnProperty.call(res.details.candidates[0], "score"), false);
assert.equal(Object.prototype.hasOwnProperty.call(res.details.candidates[0], "sources"), false);
});
});