feat(search): MiniSearch BM25 ranking with per-heading chunks

Add minisearch to search image, section splitting, mtime index cache.
Fallback to legacy token ranker on empty hits; SEARCH_RANKER=legacy opt-out.
Tests: node:test under docker/search; exclude path from vitest.
Update PRD §5.8, §6, NR-RET-1.

Made-with: Cursor

Author: ilyar

docker/search/Dockerfile

@@ -13,13 +13,19 @@ WORKDIR /workspace
 
 COPY docker/search/entrypoint.sh /usr/local/bin/qwen-entrypoint
 COPY docker/search/qwen-search.sh /usr/local/bin/qwen-search
-COPY docker/search/http-server.mjs /opt/search/http-server.mjs
-COPY docker/search/knowledge-rank.mjs /opt/search/knowledge-rank.mjs
-COPY openapi/knowledge-v1.yaml /opt/search/openapi.yaml
-COPY knowledge /opt/search/knowledge
+
+WORKDIR /opt/search
+COPY docker/search/package.json docker/search/package-lock.json ./
+RUN npm ci --omit=dev
+
+COPY docker/search/http-server.mjs docker/search/knowledge-rank.mjs ./
+COPY openapi/knowledge-v1.yaml ./openapi.yaml
+COPY knowledge ./knowledge
 
 RUN chmod +x /usr/local/bin/qwen-entrypoint /usr/local/bin/qwen-search /opt/search/http-server.mjs
 
+WORKDIR /workspace
+
 VOLUME ["/corpus"]
 
 ENTRYPOINT ["qwen-entrypoint"]

docker/search/knowledge-rank.mjs

@@ -1,9 +1,11 @@
 /**
- * Rank local markdown corpus for public search (mirrors src/local-search.ts logic).
- * Uses Unicode-aware tokenization so Cyrillic queries match the corpus.
+ * Rank local markdown corpus for public search.
+ * Primary: MiniSearch (BM25-style full-text, per-heading chunks).
+ * Fallback: legacy token heuristic (Unicode), same as src/local-search.ts.
  */
 import { readdirSync, readFileSync, statSync } from "node:fs";
 import { extname, join, relative } from "node:path";
+import MiniSearch from "minisearch";
 
 const MAX_RESULTS = 5;
 const MAX_KNOWLEDGE_FILE_BYTES = 2 * 1024 * 1024;
@@ -24,11 +26,18 @@ const STOP_WORDS = new Set([
   "with",
 ]);
 
-function tokenizeQuery(query) {
-  return query
+/** @type {Map<string, { mtime: number, mini: import('minisearch').default, docs: Map<number, { file: string, section: string, content: string }> }>} */
+const indexCache = new Map();
+
+function unicodeTokenize(text) {
+  return text
     .toLowerCase()
     .split(/[^\p{L}\p{N}]+/u)
-    .filter((token) => token.length >= MIN_TOKEN_LENGTH && !STOP_WORDS.has(token));
+    .filter((t) => t.length > 0);
+}
+
+function tokenizeQuery(query) {
+  return unicodeTokenize(query).filter((token) => token.length >= MIN_TOKEN_LENGTH && !STOP_WORDS.has(token));
 }
 
 function countOccurrences(content, token) {
@@ -71,7 +80,33 @@ function extractSnippet(content, matchIndex) {
   return `${rawSnippet.slice(0, 217)}...`;
 }
 
-function rankDocuments(query, documents) {
+function snippetFromContent(content, query) {
+  const terms = tokenizeQuery(query);
+  if (terms.length === 0) {
+    const fb = unicodeTokenize(query).filter((t) => t.length >= 1);
+    for (const t of fb) {
+      const i = content.toLowerCase().indexOf(t);
+      if (i !== -1) {
+        return extractSnippet(content, i);
+      }
+    }
+    return extractSnippet(content, 0);
+  }
+  const lower = content.toLowerCase();
+  let best = -1;
+  for (const t of terms) {
+    const i = lower.indexOf(t);
+    if (i !== -1 && (best === -1 || i < best)) {
+      best = i;
+    }
+  }
+  if (best === -1) {
+    best = 0;
+  }
+  return extractSnippet(content, best);
+}
+
+function rankDocumentsLegacy(query, documents) {
   const tokens = tokenizeQuery(query);
   if (tokens.length === 0) {
     return [];
@@ -134,6 +169,25 @@ function walkKnowledgeTree(dir) {
   return files;
 }
 
+function corpusMaxMtime(rootDir) {
+  let max = 0;
+  try {
+    for (const p of walkKnowledgeTree(rootDir)) {
+      try {
+        const t = statSync(p).mtimeMs;
+        if (t > max) {
+          max = t;
+        }
+      } catch {
+        /* skip */
+      }
+    }
+  } catch {
+    return 0;
+  }
+  return max;
+}
+
 function loadKnowledgeDocuments(dir) {
   let paths;
   try {
@@ -156,16 +210,134 @@ function loadKnowledgeDocuments(dir) {
 }
 
 /**
- * @param {string} query
- * @param {string} rootDir
- * @returns {ReadonlyArray<{ file: string, section: string, snippet: string }>}
+ * Split a markdown file into heading-scoped sections for finer retrieval.
+ * @param {string} relPath
+ * @param {string} content
  */
-export function rankKnowledgeForQuery(query, rootDir) {
+export function splitIntoSections(relPath, content) {
+  const lines = content.split(/\r?\n/);
+  /** @type {{ file: string, section: string, content: string }[]} */
+  const out = [];
+  let sectionTitle = "Overview";
+  /** @type {string[]} */
+  let buf = [];
+  const flush = () => {
+    const text = buf.join("\n").trim();
+    if (text.length > 0) {
+      out.push({ file: relPath, section: sectionTitle, content: text });
+    }
+    buf = [];
+  };
+  for (const line of lines) {
+    const m = /^#{1,6}\s+(.+)$/.exec(line);
+    if (m) {
+      flush();
+      sectionTitle = m[1].trim();
+      continue;
+    }
+    buf.push(line);
+  }
+  flush();
+  return out;
+}
+
+function buildSectionRows(documents) {
+  /** @type {{ id: number, file: string, section: string, content: string }[]} */
+  const rows = [];
+  let id = 0;
+  for (const doc of documents) {
+    for (const sec of splitIntoSections(doc.file, doc.content)) {
+      rows.push({ id: id++, file: sec.file, section: sec.section, content: sec.content });
+    }
+  }
+  return rows;
+}
+
+function getOrBuildMiniIndex(rootDir) {
+  const mtime = corpusMaxMtime(rootDir);
+  const cached = indexCache.get(rootDir);
+  if (mtime > 0 && cached && cached.mtime === mtime) {
+    return cached;
+  }
+
   const documents = loadKnowledgeDocuments(rootDir);
-  const ranked = rankDocuments(query, documents);
+  const rows = buildSectionRows(documents);
+  /** @type {Map<number, { file: string, section: string, content: string }>} */
+  const docs = new Map();
+  for (const r of rows) {
+    docs.set(r.id, { file: r.file, section: r.section, content: r.content });
+  }
+
+  const mini = new MiniSearch({
+    fields: ["content", "section", "file"],
+    storeFields: ["file", "section", "content"],
+    idField: "id",
+    tokenize: (string) => unicodeTokenize(string).filter((t) => t.length >= 1),
+  });
+
+  mini.addAll(rows);
+
+  const entry = { mtime, mini, docs };
+  if (mtime > 0) {
+    indexCache.set(rootDir, entry);
+  }
+  return entry;
+}
+
+function rankMiniSearch(query, rootDir) {
+  const { mini, docs } = getOrBuildMiniIndex(rootDir);
+  if (docs.size === 0) {
+    return [];
+  }
+  const hits = mini.search(query, {
+    prefix: true,
+    fuzzy: 0.12,
+    boost: { section: 2.2, file: 1.65, content: 1 },
+  });
+
+  /** @type {{ file: string, section: string, snippet: string }[]} */
+  const out = [];
+  for (const h of hits.slice(0, MAX_RESULTS)) {
+    const stored = docs.get(h.id);
+    if (!stored) {
+      continue;
+    }
+    out.push({
+      file: stored.file,
+      section: stored.section,
+      snippet: snippetFromContent(stored.content, query),
+    });
+  }
+  return out;
+}
+
+function mapLegacySlice(ranked) {
   return ranked.slice(0, MAX_RESULTS).map((d) => ({
     file: d.file,
     section: d.section,
     snippet: d.snippet,
   }));
 }
+
+/**
+ * @param {string} query
+ * @param {string} rootDir
+ * @returns {ReadonlyArray<{ file: string, section: string, snippet: string }>}
+ */
+export function rankKnowledgeForQuery(query, rootDir) {
+  const mode = (process.env.SEARCH_RANKER || "minisearch").toLowerCase();
+  const documents = loadKnowledgeDocuments(rootDir);
+
+  if (mode === "legacy") {
+    const ranked = rankDocumentsLegacy(query, documents);
+    return mapLegacySlice(ranked);
+  }
+
+  const miniHits = rankMiniSearch(query, rootDir);
+  if (miniHits.length > 0) {
+    return miniHits;
+  }
+
+  const ranked = rankDocumentsLegacy(query, documents);
+  return mapLegacySlice(ranked);
+}

docker/search/package-lock.json

@@ -0,0 +1,9 @@
+{
+  "name": "@spawndock/search-ranker",
+  "private": true,
+  "type": "module",
+  "version": "0.0.0",
+  "dependencies": {
+    "minisearch": "^7.1.0"
+  }
+}

docker/search/package.json

@@ -0,0 +1,51 @@
+import assert from "node:assert/strict";
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { describe, it, beforeEach, afterEach } from "node:test";
+import { rankKnowledgeForQuery, splitIntoSections } from "./knowledge-rank.mjs";
+
+describe("splitIntoSections", () => {
+  it("chunks by headings", () => {
+    const parts = splitIntoSections("g/x.md", "# A\nline\n\n## B\nother");
+    assert.equal(parts.length, 2);
+    assert.equal(parts[0].section, "A");
+    assert.ok(parts[0].content.includes("line"));
+    assert.equal(parts[1].section, "B");
+  });
+});
+
+describe("rankKnowledgeForQuery (MiniSearch)", () => {
+  let root;
+  beforeEach(() => {
+    root = mkdtempSync(join(tmpdir(), "search-rank-"));
+    mkdirSync(join(root, "guides"));
+    writeFileSync(
+      join(root, "guides", "tma.md"),
+      ["# Telegram Mini App", "Use WebApp SDK for TMA buttons.", "", "# Deploy", "Use spawn dock deploy."].join("\n"),
+    );
+  });
+  afterEach(() => {
+    rmSync(root, { recursive: true, force: true });
+  });
+
+  it("returns Telegram section for english query", () => {
+    const r = rankKnowledgeForQuery("telegram WebApp SDK buttons", root);
+    assert.ok(r.length >= 1);
+    assert.ok(r.some((x) => x.file === "guides/tma.md" && x.section === "Telegram Mini App"));
+  });
+
+  it("falls back to legacy when minisearch has no overlap", () => {
+    const r = rankKnowledgeForQuery("zzzznomatch____", root);
+    assert.equal(r.length, 0);
+  });
+
+  it("matches Cyrillic query against Cyrillic heading", () => {
+    writeFileSync(
+      join(root, "guides", "ru.md"),
+      "# Как сделать TMA\nИспользуйте Telegram Mini App SDK.\n",
+    );
+    const r = rankKnowledgeForQuery("как сделать TMA", root);
+    assert.ok(r.some((x) => x.file === "guides/ru.md" && x.section.includes("TMA")));
+  });
+});

docker/search/rank.test.mjs

@@ -270,7 +270,7 @@ rate_limit_tiers:
 | Компонент | Статус | Заметки |
 |-----------|--------|---------|
 | Монтирование корпуса **`/corpus`** (прод) и **`KNOWLEDGE_ROOT`** | **Done** | Образ также содержит снимок **`knowledge/`** в **`/opt/search/knowledge`** для запуска без volume. |
-| Ранжирование фрагментов перед вызовом LLM | **Done** | Эвристика по токенам запроса и `.md` (Unicode-токены для кириллицы); см. `docker/search/knowledge-rank.mjs`, зеркало логики в `src/local-search.ts`. |
+| Ранжирование фрагментов перед вызовом LLM | **Done** | По умолчанию **[MiniSearch](https://github.com/lucaong/minisearch)** (BM25-алгоритм) по секциям Markdown; при пустом hit — fallback на эвристику по токенам. `SEARCH_RANKER=legacy` — только эвристика. См. `docker/search/knowledge-rank.mjs`; зеркало эвристики в `src/local-search.ts`. |
 | Учёт **`locale`** в промпте | **Done** | Явные инструкции `ru` / `en` / авто по языку запроса. |
 | Поле **`sources`** | **Done** | Из JSON ответа модели; если пусто — fallback из ранжированных источников. |
 | Диагностика сбоев Qwen (**stdout/stderr** в **502**) | **Done** | Усечённые потоки в `message` для оператора. |
@@ -288,6 +288,7 @@ rate_limit_tiers:
 | `SEARCH_HTTP_PORT` / `QWEN_HTTP_PORT` | Порт HTTP listener (по умолчанию **8790**). |
 | `SEARCH_HTTP_BIND` / `QWEN_HTTP_BIND` | Bind address (по умолчанию **0.0.0.0**). |
 | `KNOWLEDGE_ROOT` | Корень Markdown-корпуса (**рекомендуется `/corpus`** в проде). |
+| `SEARCH_RANKER` | `minisearch` (по умолчанию) или `legacy` — только эвристика по токенам. |
 | `SEARCH_RATE_LIMIT_TIERS` | JSON override лимитов **free** / **basic** (см. §5.6). |
 | `API_TOKEN` | Общий секрет для **Bearer** и tier **basic** на **`search`**. |
 | `PROD_QWEN_OAUTH_CREDS` / `QWEN_OAUTH_CREDS_B64` | Base64 **oauth_creds** для Qwen CLI в контейнере; после смены секрета — **пересобрать/перезапустить** **`search`**. |
@@ -362,7 +363,7 @@ rate_limit_tiers:
 
 | ID | Требование | Приоритет |
 |----|------------|-----------|
-| **NR-RET-1** | Оценить **FTS5 / BM25** или **сведение рангов (RRF)** для релевантности фрагментов корпуса вместо или вместе с текущей эвристикой; сохранить контракт API. | P2 |
+| **NR-RET-1** | ~~Оценить BM25~~ — **частично done** (MiniSearch + секции). Далее: **FTS5 / RRF / trigram** при необходимости; контракт API без изменений. | P2 |
 | **NR-RET-2** | Опциональный **кэш** ответов по `(query нормализованный, locale, версия корпуса)` при неизменном корпусе — снижение стоимости Qwen и latency. | P3 |
 | **NR-OBS-1** | Метрики (**accepted/429/latency/502**) и точки интеграции с мониторингом хоста. | P2 |
 | **NR-HA-1** | При **>1 реплики** `search` — вынести дневные/минутные счётчики rate limit из in-memory (**Redis** и аналоги); см. §5.6.3. | P2 |
@@ -388,4 +389,4 @@ rate_limit_tiers:
 
 ---
 
-*Document version: 1.5 — 2026-03-25 — §4/§6/§7 актуализированы; §5.8 пайплайн; §12 статус и backlog; §13 context-mode; §9 smoke-тесты; NR для retrieval/observability/HA.*
+*Document version: 1.6 — 2026-03-25 — §5.8 MiniSearch ranker; §6 `SEARCH_RANKER`; NR-RET-1 частично закрыт; vitest exclude `docker/search`.*

docs/PRD-public-knowledge-search-service.md

@@ -8,6 +8,7 @@
     "start": "node dist/index.js",
     "dev": "tsx --env-file=.env src/index.ts",
     "test": "vitest run",
+    "test:search-rank": "cd docker/search && npm ci --omit=dev && node --test rank.test.mjs",
     "smoke:knowledge": "node scripts/smoke-knowledge-search.mjs",
     "bot": "node --env-file=.env dist/bot/polling.js",
     "bot:dev": "tsx --env-file=.env src/bot/polling.ts",

package.json

@@ -2,6 +2,6 @@ import { defineConfig } from "vitest/config";
 
 export default defineConfig({
   test: {
-    exclude: ["dist/**", "node_modules/**"],
+    exclude: ["dist/**", "node_modules/**", "docker/search/**"],
   },
 });