From 8ca4351d21e1503a69c99e377d2bcd5547a05423 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Sun, 22 Mar 2026 21:13:44 +0000 Subject: [PATCH] feat: add Voyage AI embedding search with pure-JS cosine similarity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - New src/embedding.ts: Voyage AI client, cosine similarity, vectorSearch - embed() calls POST https://api.voyageai.com/v1/embeddings - cosineSimilarity() pure-JS dot product / magnitude - vectorSearch() brute-force over knowledge BLOBs (<100 entries) - embedKnowledgeEntry() fire-and-forget (errors logged, never thrown) - backfillEmbeddings() batch-embeds entries missing embeddings - checkConfigChange() detects model/dimension changes and clears stale embeddings for re-embedding on next backfill - Schema migration v8: - ADD COLUMN embedding BLOB to knowledge table - CREATE TABLE kv_meta (key-value store for plugin state) - Config: search.embeddings section (enabled, model, dimensions) - Default: disabled, voyage-code-3, 1024 dims - Requires VOYAGE_API_KEY env var - Hook embedding into ltm.create() and ltm.update() - Fire-and-forget after sync DB write - Re-embeds on content change - Add vector search as additional RRF list in recall tool - Same k: key prefix as BM25 knowledge — RRF merges, not duplicates - Entries found by both BM25 and vector get boosted score - Startup backfill when embeddings first enabled - Migration strategy: on startup, compare model+dimensions config fingerprint against stored value — if changed, clear all embeddings and re-embed in background - 18 new tests: cosine similarity, BLOB round-trip, vectorSearch, isAvailable, config schema, config change detection --- src/config.ts | 16 +++ src/db.ts | 14 ++- src/embedding.ts | 280 +++++++++++++++++++++++++++++++++++++++++ src/index.ts | 10 ++ src/ltm.ts | 15 +++ src/reflect.ts | 28 +++++ test/config.test.ts | 34 ++++- test/db.test.ts | 2 +- test/embedding.test.ts | 263 ++++++++++++++++++++++++++++++++++++++ 9 files changed, 659 insertions(+), 3 deletions(-) create mode 100644 src/embedding.ts create mode 100644 test/embedding.test.ts diff --git a/src/config.ts b/src/config.ts index baa801d..28bba33 100644 --- a/src/config.ts +++ b/src/config.ts @@ -66,11 +66,27 @@ export const LoreConfig = z.object({ * When enabled, the configured model generates 2–3 alternative query phrasings * before search, improving recall for ambiguous queries. */ queryExpansion: z.boolean().default(false), + /** Vector embedding search via Voyage AI. Requires VOYAGE_API_KEY env var. */ + embeddings: z + .object({ + /** Enable vector embedding search. Requires VOYAGE_API_KEY env var. Default: false. */ + enabled: z.boolean().default(false), + /** Voyage AI model ID. Default: voyage-code-3. */ + model: z.string().default("voyage-code-3"), + /** Embedding dimensions. Default: 1024. */ + dimensions: z.number().min(256).max(2048).default(1024), + }) + .default({ + enabled: false, + model: "voyage-code-3", + dimensions: 1024, + }), }) .default({ ftsWeights: { title: 6.0, content: 2.0, category: 3.0 }, recallLimit: 10, queryExpansion: false, + embeddings: { enabled: false, model: "voyage-code-3", dimensions: 1024 }, }), crossProject: z.boolean().default(false), agentsFile: z diff --git a/src/db.ts b/src/db.ts index 3970667..5843071 100644 --- a/src/db.ts +++ b/src/db.ts @@ -2,7 +2,7 @@ import { Database } from "bun:sqlite"; import { join, dirname } from "path"; import { mkdirSync } from "fs"; -const SCHEMA_VERSION = 7; +const SCHEMA_VERSION = 8; const MIGRATIONS: string[] = [ ` @@ -208,6 +208,18 @@ const MIGRATIONS: string[] = [ INSERT INTO distillation_fts(rowid, observations) VALUES (new.rowid, new.observations); END; `, + ` + -- Version 8: Embedding BLOB column for vector search (Voyage AI). + -- No backfill — entries get embedded lazily on next create/update + -- or via explicit backfill when embeddings are first enabled. + ALTER TABLE knowledge ADD COLUMN embedding BLOB; + + -- Key-value metadata table for plugin state (e.g. embedding config fingerprint). + CREATE TABLE IF NOT EXISTS kv_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + `, ]; function dataDir() { diff --git a/src/embedding.ts b/src/embedding.ts new file mode 100644 index 0000000..cdf0f03 --- /dev/null +++ b/src/embedding.ts @@ -0,0 +1,280 @@ +/** + * Voyage AI embedding integration for vector search. + * + * Provides embedding generation via Voyage AI's REST API, pure-JS cosine + * similarity, and vector search over the knowledge table. All operations + * are gated behind `search.embeddings.enabled` config + `VOYAGE_API_KEY` + * env var — falls back silently to FTS-only when unavailable. + */ + +import { db } from "./db"; +import { config } from "./config"; +import * as log from "./log"; + +const VOYAGE_API_URL = "https://api.voyageai.com/v1/embeddings"; + +// --------------------------------------------------------------------------- +// Availability +// --------------------------------------------------------------------------- + +function getApiKey(): string | undefined { + return process.env.VOYAGE_API_KEY; +} + +/** Returns true if embedding is configured and the API key is present. */ +export function isAvailable(): boolean { + return config().search.embeddings.enabled && !!getApiKey(); +} + +// --------------------------------------------------------------------------- +// Voyage AI API +// --------------------------------------------------------------------------- + +type VoyageResponse = { + data: Array<{ embedding: number[]; index: number }>; + model: string; + usage: { total_tokens: number }; +}; + +/** + * Call Voyage AI embeddings API. + * + * @param texts Array of texts to embed (max 128 per call) + * @param inputType "document" for storage, "query" for search + * @returns Float32Array per input text + * @throws On API errors or missing API key + */ +export async function embed( + texts: string[], + inputType: "document" | "query", +): Promise { + const apiKey = getApiKey(); + if (!apiKey) throw new Error("VOYAGE_API_KEY not set"); + + const cfg = config().search.embeddings; + + const res = await fetch(VOYAGE_API_URL, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${apiKey}`, + }, + body: JSON.stringify({ + input: texts, + model: cfg.model, + input_type: inputType, + output_dimension: cfg.dimensions, + }), + }); + + if (!res.ok) { + const body = await res.text().catch(() => ""); + throw new Error(`Voyage API ${res.status}: ${body}`); + } + + const json = (await res.json()) as VoyageResponse; + // Sort by index to match input order (API may reorder) + const sorted = [...json.data].sort((a, b) => a.index - b.index); + return sorted.map((d) => new Float32Array(d.embedding)); +} + +// --------------------------------------------------------------------------- +// Cosine similarity (pure JS) +// --------------------------------------------------------------------------- + +/** + * Cosine similarity between two Float32Array vectors. + * Returns -1.0 to 1.0 where 1.0 = identical direction. + * Returns 0 if either vector is zero-length. + */ +export function cosineSimilarity(a: Float32Array, b: Float32Array): number { + const len = Math.min(a.length, b.length); + let dot = 0; + let normA = 0; + let normB = 0; + for (let i = 0; i < len; i++) { + dot += a[i] * b[i]; + normA += a[i] * a[i]; + normB += b[i] * b[i]; + } + const denom = Math.sqrt(normA) * Math.sqrt(normB); + if (denom === 0) return 0; + return dot / denom; +} + +// --------------------------------------------------------------------------- +// BLOB conversion +// --------------------------------------------------------------------------- + +/** Convert Float32Array to Buffer for SQLite BLOB storage. */ +export function toBlob(arr: Float32Array): Buffer { + return Buffer.from(arr.buffer, arr.byteOffset, arr.byteLength); +} + +/** Convert SQLite BLOB (Buffer/Uint8Array) back to Float32Array. */ +export function fromBlob(blob: Buffer | Uint8Array): Float32Array { + const bytes = new Uint8Array(blob); + return new Float32Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 4); +} + +// --------------------------------------------------------------------------- +// Vector search +// --------------------------------------------------------------------------- + +type VectorHit = { id: string; similarity: number }; + +/** + * Search all knowledge entries with embeddings by cosine similarity. + * Returns top-k entries sorted by similarity descending. + * Pure brute-force — fine for <100 entries (microseconds). + */ +export function vectorSearch( + queryEmbedding: Float32Array, + limit = 10, +): VectorHit[] { + const rows = db() + .query("SELECT id, embedding FROM knowledge WHERE embedding IS NOT NULL AND confidence > 0.2") + .all() as Array<{ id: string; embedding: Buffer }>; + + const scored: VectorHit[] = []; + for (const row of rows) { + const vec = fromBlob(row.embedding); + const sim = cosineSimilarity(queryEmbedding, vec); + scored.push({ id: row.id, similarity: sim }); + } + + scored.sort((a, b) => b.similarity - a.similarity); + return scored.slice(0, limit); +} + +// --------------------------------------------------------------------------- +// Fire-and-forget embedding +// --------------------------------------------------------------------------- + +/** + * Embed a knowledge entry and store the result in the DB. + * Fire-and-forget — errors are logged, never thrown. + * The entry remains usable via FTS even if embedding fails. + */ +export function embedKnowledgeEntry( + id: string, + title: string, + content: string, +): void { + const text = `${title}\n${content}`; + embed([text], "document") + .then(([vec]) => { + db() + .query("UPDATE knowledge SET embedding = ? WHERE id = ?") + .run(toBlob(vec), id); + }) + .catch((err) => { + log.info("embedding failed for entry", id, ":", err); + }); +} + +// --------------------------------------------------------------------------- +// Config change detection +// --------------------------------------------------------------------------- + +/** + * Build a config fingerprint from model + dimensions. + * Used to detect when the embedding config changes (model swap, dimension change) + * so we can clear stale embeddings and re-embed. + */ +function configFingerprint(): string { + const cfg = config().search.embeddings; + return `${cfg.model}:${cfg.dimensions}`; +} + +const EMBEDDING_CONFIG_KEY = "lore:embedding_config"; + +/** + * Check if embedding config has changed since the last backfill. + * If so, clear all existing embeddings (they're incompatible) and + * update the stored fingerprint. + * + * Returns true if embeddings were cleared (full re-embed needed). + */ +export function checkConfigChange(): boolean { + // Read stored fingerprint from schema_version metadata (reuse the table) + const stored = db() + .query("SELECT value FROM kv_meta WHERE key = ?") + .get(EMBEDDING_CONFIG_KEY) as { value: string } | null; + + const current = configFingerprint(); + + if (stored && stored.value === current) return false; + + // Config changed (or first run) — clear all embeddings + if (stored) { + const count = db() + .query("SELECT COUNT(*) as n FROM knowledge WHERE embedding IS NOT NULL") + .get() as { n: number }; + if (count.n > 0) { + db().query("UPDATE knowledge SET embedding = NULL").run(); + log.info( + `embedding config changed (${stored.value} → ${current}), cleared ${count.n} stale embeddings`, + ); + } + } + + // Store new fingerprint + db() + .query( + "INSERT INTO kv_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = ?", + ) + .run(EMBEDDING_CONFIG_KEY, current, current); + + return true; +} + +// --------------------------------------------------------------------------- +// Backfill +// --------------------------------------------------------------------------- + +/** + * Embed all knowledge entries that are missing embeddings. + * Called on startup when embeddings are first enabled. + * Also handles config changes: if model/dimensions changed, clears + * stale embeddings first, then re-embeds all entries. + * Returns the number of entries embedded. + */ +export async function backfillEmbeddings(): Promise { + // Detect model/dimension changes and clear stale embeddings + checkConfigChange(); + + const rows = db() + .query("SELECT id, title, content FROM knowledge WHERE embedding IS NULL AND confidence > 0.2") + .all() as Array<{ id: string; title: string; content: string }>; + + if (!rows.length) return 0; + + // Batch embed (Voyage supports up to 128 per call) + const BATCH_SIZE = 128; + let embedded = 0; + + for (let i = 0; i < rows.length; i += BATCH_SIZE) { + const batch = rows.slice(i, i + BATCH_SIZE); + const texts = batch.map((r) => `${r.title}\n${r.content}`); + + try { + const vectors = await embed(texts, "document"); + const update = db().prepare( + "UPDATE knowledge SET embedding = ? WHERE id = ?", + ); + + for (let j = 0; j < batch.length; j++) { + update.run(toBlob(vectors[j]), batch[j].id); + embedded++; + } + } catch (err) { + log.info(`embedding backfill batch ${i}-${i + batch.length} failed:`, err); + } + } + + if (embedded > 0) { + log.info(`embedded ${embedded} knowledge entries`); + } + return embedded; +} diff --git a/src/index.ts b/src/index.ts index 8346c57..c9b1598 100644 --- a/src/index.ts +++ b/src/index.ts @@ -19,6 +19,7 @@ import { import { formatKnowledge, formatDistillations } from "./prompt"; import { createRecallTool } from "./reflect"; import { shouldImport, importFromFile, exportToFile } from "./agents-file"; +import * as embedding from "./embedding"; import * as log from "./log"; /** @@ -678,6 +679,15 @@ End with "I'm ready to continue." so the agent knows to pick up where it left of // appears for a project, the init failed (see catch block below). process.stderr.write(`[lore] active: ${projectPath}\n`); + // Background: backfill embeddings for entries that don't have one yet. + // Fires once when embeddings are first enabled — subsequent entries + // get embedded on create/update via ltm.ts hooks. + if (config().search.embeddings.enabled && embedding.isAvailable()) { + embedding.backfillEmbeddings().catch((err) => { + log.info("embedding backfill failed:", err); + }); + } + return hooks; } catch (e) { // Log the full error before re-throwing so OpenCode's plugin loader diff --git a/src/ltm.ts b/src/ltm.ts index 660dc0f..d6d2b57 100644 --- a/src/ltm.ts +++ b/src/ltm.ts @@ -2,6 +2,7 @@ import { uuidv7 } from "uuidv7"; import { db, ensureProject } from "./db"; import { config } from "./config"; import { ftsQuery, ftsQueryOr, EMPTY_QUERY, extractTopTerms } from "./search"; +import * as embedding from "./embedding"; // ~3 chars per token — validated as best heuristic against real API data. function estimateTokens(text: string): number { @@ -98,6 +99,12 @@ export function create(input: { now, now, ); + + // Fire-and-forget: embed for vector search (errors logged, never thrown) + if (embedding.isAvailable()) { + embedding.embedKnowledgeEntry(id, input.title, input.content); + } + return id; } @@ -121,6 +128,14 @@ export function update( db() .query(`UPDATE knowledge SET ${sets.join(", ")} WHERE id = ?`) .run(...(params as [string, ...string[]])); + + // Re-embed when content changes (fire-and-forget) + if (embedding.isAvailable() && input.content !== undefined) { + const entry = get(id); + if (entry) { + embedding.embedKnowledgeEntry(id, entry.title, input.content); + } + } } export function remove(id: string) { diff --git a/src/reflect.ts b/src/reflect.ts index 284fd43..559478a 100644 --- a/src/reflect.ts +++ b/src/reflect.ts @@ -3,6 +3,7 @@ import type { createOpencodeClient } from "@opencode-ai/sdk"; import * as temporal from "./temporal"; import * as ltm from "./ltm"; import * as log from "./log"; +import * as embedding from "./embedding"; import { db, ensureProject } from "./db"; import { ftsQuery, ftsQueryOr, EMPTY_QUERY, reciprocalRankFusion, expandQuery } from "./search"; import { serialize, inline, h, p, ul, lip, liph, t, root } from "./markdown"; @@ -310,6 +311,33 @@ export function createRecallTool( ); } + // Vector search: embed query and find similar knowledge entries + if (embedding.isAvailable() && knowledgeEnabled && scope !== "session") { + try { + const [queryVec] = await embedding.embed([args.query], "query"); + const vectorHits = embedding.vectorSearch(queryVec, limit); + const vectorTagged: TaggedResult[] = []; + for (const hit of vectorHits) { + const entry = ltm.get(hit.id); + if (entry) { + vectorTagged.push({ + source: "knowledge", + item: { ...entry, rank: -hit.similarity }, + }); + } + } + if (vectorTagged.length) { + // Same `k:` key prefix as BM25 knowledge — RRF merges, not duplicates + allRrfLists.push({ + items: vectorTagged, + key: (r) => `k:${r.item.id}`, + }); + } + } catch (err) { + log.info("recall: vector search failed:", err); + } + } + // Fuse results using Reciprocal Rank Fusion across all query variants const fused = reciprocalRankFusion(allRrfLists); diff --git a/test/config.test.ts b/test/config.test.ts index 4975964..6a82eb5 100644 --- a/test/config.test.ts +++ b/test/config.test.ts @@ -77,13 +77,16 @@ describe("LoreConfig — curator schema", () => { }); describe("LoreConfig — search schema", () => { - test("search defaults: ftsWeights, recallLimit, queryExpansion", () => { + test("search defaults: ftsWeights, recallLimit, queryExpansion, embeddings", () => { const cfg = LoreConfig.parse({}); expect(cfg.search.ftsWeights.title).toBe(6.0); expect(cfg.search.ftsWeights.content).toBe(2.0); expect(cfg.search.ftsWeights.category).toBe(3.0); expect(cfg.search.recallLimit).toBe(10); expect(cfg.search.queryExpansion).toBe(false); + expect(cfg.search.embeddings.enabled).toBe(false); + expect(cfg.search.embeddings.model).toBe("voyage-code-3"); + expect(cfg.search.embeddings.dimensions).toBe(1024); }); test("search.ftsWeights can be customised", () => { @@ -122,6 +125,35 @@ describe("LoreConfig — search schema", () => { expect(cfg.search.ftsWeights.title).toBe(6.0); expect(cfg.search.recallLimit).toBe(20); expect(cfg.search.queryExpansion).toBe(false); + expect(cfg.search.embeddings.enabled).toBe(false); + }); + + test("search.embeddings can be enabled", () => { + const cfg = LoreConfig.parse({ + search: { embeddings: { enabled: true } }, + }); + expect(cfg.search.embeddings.enabled).toBe(true); + expect(cfg.search.embeddings.model).toBe("voyage-code-3"); + expect(cfg.search.embeddings.dimensions).toBe(1024); + }); + + test("search.embeddings model and dimensions can be customised", () => { + const cfg = LoreConfig.parse({ + search: { + embeddings: { enabled: true, model: "voyage-4-lite", dimensions: 512 }, + }, + }); + expect(cfg.search.embeddings.model).toBe("voyage-4-lite"); + expect(cfg.search.embeddings.dimensions).toBe(512); + }); + + test("search.embeddings.dimensions rejects out-of-range values", () => { + expect(() => + LoreConfig.parse({ search: { embeddings: { dimensions: 128 } } }), + ).toThrow(); + expect(() => + LoreConfig.parse({ search: { embeddings: { dimensions: 4096 } } }), + ).toThrow(); }); }); diff --git a/test/db.test.ts b/test/db.test.ts index ad7df99..fe4679e 100644 --- a/test/db.test.ts +++ b/test/db.test.ts @@ -21,7 +21,7 @@ describe("db", () => { const row = db().query("SELECT version FROM schema_version").get() as { version: number; }; - expect(row.version).toBe(7); + expect(row.version).toBe(8); }); test("distillation_fts virtual table exists", () => { diff --git a/test/embedding.test.ts b/test/embedding.test.ts new file mode 100644 index 0000000..79f0ee3 --- /dev/null +++ b/test/embedding.test.ts @@ -0,0 +1,263 @@ +import { describe, test, expect, beforeEach } from "bun:test"; +import { db, ensureProject } from "../src/db"; +import { + cosineSimilarity, + toBlob, + fromBlob, + isAvailable, + vectorSearch, + checkConfigChange, +} from "../src/embedding"; + +describe("cosineSimilarity", () => { + test("identical vectors return 1.0", () => { + const a = new Float32Array([1, 2, 3]); + expect(cosineSimilarity(a, a)).toBeCloseTo(1.0, 5); + }); + + test("opposite vectors return -1.0", () => { + const a = new Float32Array([1, 0, 0]); + const b = new Float32Array([-1, 0, 0]); + expect(cosineSimilarity(a, b)).toBeCloseTo(-1.0, 5); + }); + + test("orthogonal vectors return 0.0", () => { + const a = new Float32Array([1, 0, 0]); + const b = new Float32Array([0, 1, 0]); + expect(cosineSimilarity(a, b)).toBeCloseTo(0.0, 5); + }); + + test("similar vectors return high positive value", () => { + const a = new Float32Array([1, 0.1, 0]); + const b = new Float32Array([0.9, 0.2, 0]); + const sim = cosineSimilarity(a, b); + expect(sim).toBeGreaterThan(0.9); + expect(sim).toBeLessThanOrEqual(1.0); + }); + + test("zero vector returns 0", () => { + const a = new Float32Array([0, 0, 0]); + const b = new Float32Array([1, 2, 3]); + expect(cosineSimilarity(a, b)).toBe(0); + }); + + test("both zero vectors return 0", () => { + const a = new Float32Array([0, 0, 0]); + expect(cosineSimilarity(a, a)).toBe(0); + }); +}); + +describe("BLOB round-trip", () => { + test("Float32Array survives toBlob → fromBlob", () => { + const original = new Float32Array([0.123, -0.456, 0.789, 1.0, -1.0]); + const blob = toBlob(original); + const restored = fromBlob(blob); + expect(restored.length).toBe(original.length); + for (let i = 0; i < original.length; i++) { + expect(restored[i]).toBeCloseTo(original[i], 6); + } + }); + + test("single element array", () => { + const original = new Float32Array([42.5]); + const restored = fromBlob(toBlob(original)); + expect(restored[0]).toBeCloseTo(42.5, 5); + }); + + test("empty array", () => { + const original = new Float32Array([]); + const restored = fromBlob(toBlob(original)); + expect(restored.length).toBe(0); + }); + + test("large array (1024 dims)", () => { + const original = new Float32Array(1024); + for (let i = 0; i < 1024; i++) original[i] = Math.random() * 2 - 1; + const restored = fromBlob(toBlob(original)); + expect(restored.length).toBe(1024); + for (let i = 0; i < 1024; i++) { + expect(restored[i]).toBeCloseTo(original[i], 6); + } + }); +}); + +describe("isAvailable", () => { + test("returns false without VOYAGE_API_KEY", () => { + // In test environment, VOYAGE_API_KEY should not be set + const original = process.env.VOYAGE_API_KEY; + delete process.env.VOYAGE_API_KEY; + expect(isAvailable()).toBe(false); + if (original) process.env.VOYAGE_API_KEY = original; + }); +}); + +describe("vectorSearch", () => { + const PROJECT = "/test/embedding/vectorsearch"; + + beforeEach(() => { + const pid = ensureProject(PROJECT); + db().query("DELETE FROM knowledge WHERE project_id = ?").run(pid); + }); + + test("returns entries sorted by similarity descending", () => { + const pid = ensureProject(PROJECT); + const now = Date.now(); + + // Insert 3 entries with known embeddings + const vecA = new Float32Array([1, 0, 0]); // matches query perfectly + const vecB = new Float32Array([0, 1, 0]); // orthogonal to query + const vecC = new Float32Array([0.9, 0.1, 0]); // similar to query + + db() + .query( + "INSERT INTO knowledge (id, project_id, category, title, content, confidence, created_at, updated_at, embedding) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ) + .run("embed-a", pid, "test", "Entry A", "Perfect match", 1.0, now, now, toBlob(vecA)); + db() + .query( + "INSERT INTO knowledge (id, project_id, category, title, content, confidence, created_at, updated_at, embedding) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ) + .run("embed-b", pid, "test", "Entry B", "Orthogonal", 1.0, now, now, toBlob(vecB)); + db() + .query( + "INSERT INTO knowledge (id, project_id, category, title, content, confidence, created_at, updated_at, embedding) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ) + .run("embed-c", pid, "test", "Entry C", "Similar", 1.0, now, now, toBlob(vecC)); + + const query = new Float32Array([1, 0, 0]); + const results = vectorSearch(query, 10); + + expect(results.length).toBe(3); + // Entry A should be first (exact match, similarity ≈ 1.0) + expect(results[0].id).toBe("embed-a"); + expect(results[0].similarity).toBeCloseTo(1.0, 3); + // Entry C should be second (similar) + expect(results[1].id).toBe("embed-c"); + expect(results[1].similarity).toBeGreaterThan(0.9); + // Entry B should be last (orthogonal, similarity ≈ 0.0) + expect(results[2].id).toBe("embed-b"); + expect(results[2].similarity).toBeCloseTo(0.0, 3); + }); + + test("respects limit parameter", () => { + const pid = ensureProject(PROJECT); + const now = Date.now(); + + for (let i = 0; i < 5; i++) { + const vec = new Float32Array([Math.random(), Math.random(), 0]); + db() + .query( + "INSERT INTO knowledge (id, project_id, category, title, content, confidence, created_at, updated_at, embedding) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ) + .run(`embed-limit-${i}`, pid, "test", `Entry ${i}`, `Content ${i}`, 1.0, now, now, toBlob(vec)); + } + + const query = new Float32Array([1, 0, 0]); + const results = vectorSearch(query, 2); + expect(results.length).toBe(2); + }); + + test("skips entries without embeddings", () => { + const pid = ensureProject(PROJECT); + const now = Date.now(); + + // Entry with embedding + db() + .query( + "INSERT INTO knowledge (id, project_id, category, title, content, confidence, created_at, updated_at, embedding) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ) + .run("embed-yes", pid, "test", "Has Embedding", "Content", 1.0, now, now, toBlob(new Float32Array([1, 0, 0]))); + + // Entry without embedding (NULL) + db() + .query( + "INSERT INTO knowledge (id, project_id, category, title, content, confidence, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + ) + .run("embed-no", pid, "test", "No Embedding", "Content", 1.0, now, now); + + const query = new Float32Array([1, 0, 0]); + const results = vectorSearch(query, 10); + + expect(results.length).toBe(1); + expect(results[0].id).toBe("embed-yes"); + }); + + test("skips low-confidence entries", () => { + const pid = ensureProject(PROJECT); + const now = Date.now(); + + db() + .query( + "INSERT INTO knowledge (id, project_id, category, title, content, confidence, created_at, updated_at, embedding) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ) + .run("embed-high", pid, "test", "High Confidence", "Content", 1.0, now, now, toBlob(new Float32Array([1, 0, 0]))); + + db() + .query( + "INSERT INTO knowledge (id, project_id, category, title, content, confidence, created_at, updated_at, embedding) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ) + .run("embed-low", pid, "test", "Low Confidence", "Content", 0.1, now, now, toBlob(new Float32Array([1, 0, 0]))); + + const query = new Float32Array([1, 0, 0]); + const results = vectorSearch(query, 10); + + expect(results.length).toBe(1); + expect(results[0].id).toBe("embed-high"); + }); +}); + +describe("checkConfigChange", () => { + const PROJECT = "/test/embedding/configchange"; + + beforeEach(() => { + const pid = ensureProject(PROJECT); + db().query("DELETE FROM knowledge WHERE project_id = ?").run(pid); + db().query("DELETE FROM kv_meta WHERE key LIKE 'lore:%'").run(); + }); + + test("first call stores fingerprint and returns true", () => { + const changed = checkConfigChange(); + expect(changed).toBe(true); + + // Fingerprint is now stored + const row = db() + .query("SELECT value FROM kv_meta WHERE key = 'lore:embedding_config'") + .get() as { value: string } | null; + expect(row).not.toBeNull(); + expect(row!.value).toContain("voyage-code-3"); + expect(row!.value).toContain("1024"); + }); + + test("second call with same config returns false", () => { + checkConfigChange(); // first call + const changed = checkConfigChange(); // same config + expect(changed).toBe(false); + }); + + test("clears embeddings when fingerprint changes", () => { + const pid = ensureProject(PROJECT); + const now = Date.now(); + + // Insert entry with embedding + db() + .query( + "INSERT INTO knowledge (id, project_id, category, title, content, confidence, created_at, updated_at, embedding) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ) + .run("cc-1", pid, "test", "Test", "Content", 1.0, now, now, toBlob(new Float32Array([1, 0, 0]))); + + // Store a different fingerprint (simulating a previous config) + db() + .query("INSERT INTO kv_meta (key, value) VALUES (?, ?)") + .run("lore:embedding_config", "old-model:512"); + + // Check config — should detect change and clear embeddings + const changed = checkConfigChange(); + expect(changed).toBe(true); + + // Verify embedding was cleared + const row = db() + .query("SELECT embedding FROM knowledge WHERE id = 'cc-1'") + .get() as { embedding: Buffer | null }; + expect(row.embedding).toBeNull(); + }); +});