Add vector search index feature and end-to-end logging (#1113)

# Summary

- Implement SQLite-based vector search index with math utilities and
match models, plus factory wiring for embedding providers.
- Centralize constants into src/Core/Constants.cs and update Core/Main
code to consume the new structure.
  - Add `doctor` command and embedding generator factory integration.
- Expand test suite: new vector index unit/integration tests, updated
core/main tests, and **new e2e harness/tests** with per-test C# log
files.
- Update format.sh and add e2e-tests.sh runner; e2e tests default to
Release build and use KM_BIN.

# Details

- New vector search components: IVectorIndex, SqliteVectorIndex,
VectorMath, VectorMatch, plus tests covering persistence, error
handling, and math.
- Constants consolidation removed module-specific constant files and
unified them in Constants.cs (search, embeddings, logging, config/app).
- CLI/Services: added DoctorCommand, EmbeddingGeneratorFactory, enhanced
SearchIndexFactory; CLI builder now emits bootstrap logs and respects
--log-file.
- E2E: added framework helpers (cli, db, logging), five e2e scenarios
with per-test log files; runner script e2e-tests.sh builds if needed and
sets KM_BIN.

Author: dluc

.github/workflows/coverage.yml

@@ -25,6 +25,7 @@ jobs:
         chmod +x ./coverage.sh
         ./coverage.sh 80
       env:
+        OLLAMA_AVAILABLE: "false"
         MIN_COVERAGE: 80
 
     - name: Upload coverage report

.gitignore

@@ -6,6 +6,7 @@ archived/
 node_modules/
 obj/
 bin/
+__pycache__/
 _dev/
 .dev/
 .vs/
@@ -68,4 +69,4 @@ publish/
 *.crt
 *.key
 *.pem
-certs/
+certs/

docs

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+set -e
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)"
+cd "$ROOT"
+
+echo "======================================="
+echo "  Running E2E Tests"
+echo "======================================="
+echo ""
+
+# Choose build configuration (default Release to align with build.sh)
+CONFIGURATION="${CONFIGURATION:-Release}"
+KM_BIN="$ROOT/src/Main/bin/$CONFIGURATION/net10.0/KernelMemory.Main.dll"
+
+# Ensure km binary is built at the selected configuration
+if [ ! -f "$KM_BIN" ]; then
+    echo "km binary not found at $KM_BIN. Building ($CONFIGURATION)..."
+    dotnet build src/Main/Main.csproj -c "$CONFIGURATION"
+fi
+
+if [ ! -f "$KM_BIN" ]; then
+    echo "❌ km binary still not found at $KM_BIN after build. Set KM_BIN to a valid path."
+    exit 1
+fi
+
+export KM_BIN
+
+FAILED=0
+PASSED=0
+
+# Run each test file
+for test_file in tests/e2e/test_*.py; do
+    if [ -f "$test_file" ]; then
+        echo ""
+        echo "Running: $(basename "$test_file")"
+        echo "---------------------------------------"
+
+        if python3 "$test_file"; then
+            PASSED=$((PASSED + 1))
+        else
+            FAILED=$((FAILED + 1))
+        fi
+    fi
+done
+
+echo ""
+echo "======================================="
+echo "  E2E Test Results"
+echo "======================================="
+echo "Passed: $PASSED"
+echo "Failed: $FAILED"
+echo "======================================="
+
+if [ $FAILED -gt 0 ]; then
+    exit 1
+fi
+
+exit 0

e2e-tests.sh

@@ -83,7 +83,8 @@ public static AppConfig CreateDefault()
 
     /// <summary>
     /// Creates a default configuration with a single "personal" node
-    /// using local SQLite storage in the specified base directory
+    /// using local SQLite storage in the specified base directory.
+    /// Includes embeddings cache for efficient vector search operations.
     /// </summary>
     /// <param name="baseDir">Base directory for data storage</param>
     public static AppConfig CreateDefault(string baseDir)
@@ -95,8 +96,10 @@ public static AppConfig CreateDefault(string baseDir)
             Nodes = new Dictionary<string, NodeConfig>
             {
                 ["personal"] = NodeConfig.CreateDefaultPersonalNode(personalNodeDir)
-            }
-            // EmbeddingsCache and LLMCache intentionally omitted - add when features are implemented
+            },
+            EmbeddingsCache = CacheConfig.CreateDefaultSqliteCache(
+                Path.Combine(baseDir, "embeddings-cache.db"))
+            // LLMCache intentionally omitted - add when LLM features are implemented
         };
     }
 }

src/Core/Config/AppConfig.cs

@@ -1,5 +1,6 @@
 // Copyright (c) Microsoft. All rights reserved.
 using System.Text.Json;
+using System.Text.Json.Serialization;
 using System.Text.Json.Serialization.Metadata;
 using KernelMemory.Core.Config.Cache;
 using KernelMemory.Core.Config.ContentIndex;
@@ -28,7 +29,8 @@ public static class ConfigParser
         ReadCommentHandling = JsonCommentHandling.Skip,
         AllowTrailingCommas = true,
         PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
-        TypeInfoResolver = new DefaultJsonTypeInfoResolver()
+        TypeInfoResolver = new DefaultJsonTypeInfoResolver(),
+        Converters = { new JsonStringEnumConverter() }
     };
 
     /// <summary>
@@ -46,13 +48,14 @@ public static class ConfigParser
 
     /// <summary>
     /// Loads configuration from a file, or creates default config if file doesn't exist.
-    /// The config file is always ensured to exist on disk after loading.
+    /// Optionally ensures the config file exists on disk after loading (for write operations).
     /// Performs tilde expansion on paths (~/ → home directory)
     /// </summary>
     /// <param name="filePath">Path to configuration file</param>
+    /// <param name="ensureFileExists">If true, writes config to disk if missing (default: true for backward compatibility)</param>
     /// <returns>Validated AppConfig instance</returns>
     /// <exception cref="ConfigException">Thrown when file exists but parsing or validation fails</exception>
-    public static AppConfig LoadFromFile(string filePath)
+    public static AppConfig LoadFromFile(string filePath, bool ensureFileExists = true)
     {
         AppConfig config;
 
@@ -65,8 +68,11 @@ public static AppConfig LoadFromFile(string filePath)
             // Create default config relative to config file location
             config = AppConfig.CreateDefault(baseDir);
 
-            // Write the config file
-            WriteConfigFile(filePath, config);
+            // Write the config file only if requested
+            if (ensureFileExists)
+            {
+                WriteConfigFile(filePath, config);
+            }
 
             return config;
         }
@@ -82,8 +88,11 @@ public static AppConfig LoadFromFile(string filePath)
             // Expand tilde paths
             ExpandTildePaths(config);
 
-            // Always ensure the config file exists (recreate if deleted between load and save)
-            WriteConfigFileIfMissing(filePath, config);
+            // Optionally ensure the config file exists (recreate if deleted between load and save)
+            if (ensureFileExists)
+            {
+                WriteConfigFileIfMissing(filePath, config);
+            }
 
             return config;
         }

src/Core/Config/ConfigParser.cs

@@ -2,7 +2,6 @@
 using System.Text.Json.Serialization;
 using KernelMemory.Core.Config.Enums;
 using KernelMemory.Core.Config.Validation;
-using KernelMemory.Core.Embeddings;
 
 namespace KernelMemory.Core.Config.Embeddings;
 
@@ -20,7 +19,7 @@ public sealed class HuggingFaceEmbeddingsConfig : EmbeddingsConfig
     /// HuggingFace model name (e.g., "sentence-transformers/all-MiniLM-L6-v2", "BAAI/bge-base-en-v1.5").
     /// </summary>
     [JsonPropertyName("model")]
-    public string Model { get; set; } = EmbeddingConstants.DefaultHuggingFaceModel;
+    public string Model { get; set; } = Constants.EmbeddingDefaults.DefaultHuggingFaceModel;
 
     /// <summary>
     /// HuggingFace API key (token).
@@ -35,7 +34,7 @@ public sealed class HuggingFaceEmbeddingsConfig : EmbeddingsConfig
     /// Can be changed for custom inference endpoints.
     /// </summary>
     [JsonPropertyName("baseUrl")]
-    public string BaseUrl { get; set; } = EmbeddingConstants.DefaultHuggingFaceBaseUrl;
+    public string BaseUrl { get; set; } = Constants.EmbeddingDefaults.DefaultHuggingFaceBaseUrl;
 
     /// <inheritdoc />
     public override void Validate(string path)

src/Core/Config/Embeddings/HuggingFaceEmbeddingsConfig.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft. All rights reserved.
 using System.Text.Json.Serialization;
 using KernelMemory.Core.Config.ContentIndex;
+using KernelMemory.Core.Config.Embeddings;
 using KernelMemory.Core.Config.Enums;
 using KernelMemory.Core.Config.SearchIndex;
 using KernelMemory.Core.Config.Storage;
@@ -106,7 +107,8 @@ public void Validate(string path)
     }
 
     /// <summary>
-    /// Creates a default "personal" node configuration
+    /// Creates a default "personal" node configuration with FTS and vector search.
+    /// Uses Ollama with qwen3-embedding model (1024 dimensions) for local, offline-capable vector search.
     /// </summary>
     /// <param name="nodeDir"></param>
     internal static NodeConfig CreateDefaultPersonalNode(string nodeDir)
@@ -128,7 +130,21 @@ internal static NodeConfig CreateDefaultPersonalNode(string nodeDir)
                     Id = "sqlite-fts",
                     Type = SearchIndexTypes.SqliteFTS,
                     Path = Path.Combine(nodeDir, "fts.db"),
-                    EnableStemming = true
+                    EnableStemming = true,
+                    Required = true
+                },
+                new VectorSearchIndexConfig
+                {
+                    Id = "sqlite-vector",
+                    Type = SearchIndexTypes.SqliteVector,
+                    Path = Path.Combine(nodeDir, "vector.db"),
+                    Dimensions = 1024,
+                    UseSqliteVec = false,
+                    Embeddings = new OllamaEmbeddingsConfig
+                    {
+                        Model = Constants.EmbeddingDefaults.DefaultOllamaModel,
+                        BaseUrl = Constants.EmbeddingDefaults.DefaultOllamaBaseUrl
+                    }
                 }
             }
         };

src/Core/Config/NodeConfig.cs

@@ -1,7 +1,6 @@
 // Copyright (c) Microsoft. All rights reserved.
 using System.Text.Json.Serialization;
 using KernelMemory.Core.Config.Validation;
-using KernelMemory.Core.Search;
 
 namespace KernelMemory.Core.Config;
 
@@ -17,22 +16,22 @@ public sealed class SearchConfig : IValidatable
     /// Default: 0.3 (moderate threshold).
     /// </summary>
     [JsonPropertyName("defaultMinRelevance")]
-    public float DefaultMinRelevance { get; set; } = SearchConstants.DefaultMinRelevance;
+    public float DefaultMinRelevance { get; set; } = Constants.SearchDefaults.DefaultMinRelevance;
 
     /// <summary>
     /// Default maximum number of results to return per search.
     /// Default: 20 results.
     /// </summary>
     [JsonPropertyName("defaultLimit")]
-    public int DefaultLimit { get; set; } = SearchConstants.DefaultLimit;
+    public int DefaultLimit { get; set; } = Constants.SearchDefaults.DefaultLimit;
 
     /// <summary>
     /// Search timeout in seconds per node.
     /// If a node takes longer than this, it times out and is excluded from results.
     /// Default: 30 seconds.
     /// </summary>
     [JsonPropertyName("searchTimeoutSeconds")]
-    public int SearchTimeoutSeconds { get; set; } = SearchConstants.DefaultSearchTimeoutSeconds;
+    public int SearchTimeoutSeconds { get; set; } = Constants.SearchDefaults.DefaultSearchTimeoutSeconds;
 
     /// <summary>
     /// Default maximum results to retrieve from each node (memory safety).
@@ -41,7 +40,7 @@ public sealed class SearchConfig : IValidatable
     /// Default: 1000 results per node.
     /// </summary>
     [JsonPropertyName("maxResultsPerNode")]
-    public int MaxResultsPerNode { get; set; } = SearchConstants.DefaultMaxResultsPerNode;
+    public int MaxResultsPerNode { get; set; } = Constants.SearchDefaults.DefaultMaxResultsPerNode;
 
     /// <summary>
     /// Default nodes to search when no explicit --nodes flag is provided.
@@ -50,7 +49,7 @@ public sealed class SearchConfig : IValidatable
     /// </summary>
     [JsonPropertyName("defaultNodes")]
     [System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1819:Properties should not return arrays")]
-    public string[] DefaultNodes { get; set; } = [SearchConstants.AllNodesWildcard];
+    public string[] DefaultNodes { get; set; } = [Constants.SearchDefaults.AllNodesWildcard];
 
     /// <summary>
     /// Nodes to exclude from search by default.
@@ -67,66 +66,66 @@ public sealed class SearchConfig : IValidatable
     /// Default: 10 levels.
     /// </summary>
     [JsonPropertyName("maxQueryDepth")]
-    public int MaxQueryDepth { get; set; } = SearchConstants.MaxQueryDepth;
+    public int MaxQueryDepth { get; set; } = Constants.SearchDefaults.MaxQueryDepth;
 
     /// <summary>
     /// Maximum number of boolean operators (AND/OR/NOT) in a single query.
     /// Prevents query complexity attacks.
     /// Default: 50 operators.
     /// </summary>
     [JsonPropertyName("maxBooleanOperators")]
-    public int MaxBooleanOperators { get; set; } = SearchConstants.MaxBooleanOperators;
+    public int MaxBooleanOperators { get; set; } = Constants.SearchDefaults.MaxBooleanOperators;
 
     /// <summary>
     /// Maximum length of a field value in query (characters).
     /// Prevents oversized query values.
     /// Default: 1000 characters.
     /// </summary>
     [JsonPropertyName("maxFieldValueLength")]
-    public int MaxFieldValueLength { get; set; } = SearchConstants.MaxFieldValueLength;
+    public int MaxFieldValueLength { get; set; } = Constants.SearchDefaults.MaxFieldValueLength;
 
     /// <summary>
     /// Maximum time allowed for query parsing (milliseconds).
     /// Prevents regex catastrophic backtracking.
     /// Default: 1000ms (1 second).
     /// </summary>
     [JsonPropertyName("queryParseTimeoutMs")]
-    public int QueryParseTimeoutMs { get; set; } = SearchConstants.QueryParseTimeoutMs;
+    public int QueryParseTimeoutMs { get; set; } = Constants.SearchDefaults.QueryParseTimeoutMs;
 
     /// <summary>
     /// Default snippet length in characters when --snippet flag is used.
     /// Default: 200 characters.
     /// </summary>
     [JsonPropertyName("snippetLength")]
-    public int SnippetLength { get; set; } = SearchConstants.DefaultSnippetLength;
+    public int SnippetLength { get; set; } = Constants.SearchDefaults.DefaultSnippetLength;
 
     /// <summary>
     /// Default maximum number of snippets per result when --snippet flag is used.
     /// Default: 1 snippet.
     /// </summary>
     [JsonPropertyName("maxSnippetsPerResult")]
-    public int MaxSnippetsPerResult { get; set; } = SearchConstants.DefaultMaxSnippetsPerResult;
+    public int MaxSnippetsPerResult { get; set; } = Constants.SearchDefaults.DefaultMaxSnippetsPerResult;
 
     /// <summary>
     /// Separator string between multiple snippets.
     /// Default: "..." (ellipsis).
     /// </summary>
     [JsonPropertyName("snippetSeparator")]
-    public string SnippetSeparator { get; set; } = SearchConstants.DefaultSnippetSeparator;
+    public string SnippetSeparator { get; set; } = Constants.SearchDefaults.DefaultSnippetSeparator;
 
     /// <summary>
     /// Prefix marker for highlighting matched terms.
     /// Default: "&lt;mark&gt;" (HTML-style).
     /// </summary>
     [JsonPropertyName("highlightPrefix")]
-    public string HighlightPrefix { get; set; } = SearchConstants.DefaultHighlightPrefix;
+    public string HighlightPrefix { get; set; } = Constants.SearchDefaults.DefaultHighlightPrefix;
 
     /// <summary>
     /// Suffix marker for highlighting matched terms.
     /// Default: "&lt;/mark&gt;" (HTML-style).
     /// </summary>
     [JsonPropertyName("highlightSuffix")]
-    public string HighlightSuffix { get; set; } = SearchConstants.DefaultHighlightSuffix;
+    public string HighlightSuffix { get; set; } = Constants.SearchDefaults.DefaultHighlightSuffix;
 
     /// <summary>
     /// Validates the search configuration.
@@ -135,10 +134,10 @@ public sealed class SearchConfig : IValidatable
     public void Validate(string path)
     {
         // Validate min relevance score
-        if (this.DefaultMinRelevance < SearchConstants.MinRelevanceScore || this.DefaultMinRelevance > SearchConstants.MaxRelevanceScore)
+        if (this.DefaultMinRelevance < Constants.SearchDefaults.MinRelevanceScore || this.DefaultMinRelevance > Constants.SearchDefaults.MaxRelevanceScore)
         {
             throw new ConfigException($"{path}.DefaultMinRelevance",
-                $"Must be between {SearchConstants.MinRelevanceScore} and {SearchConstants.MaxRelevanceScore}");
+                $"Must be between {Constants.SearchDefaults.MinRelevanceScore} and {Constants.SearchDefaults.MaxRelevanceScore}");
         }
 
         // Validate default limit
@@ -167,7 +166,7 @@ public void Validate(string path)
         }
 
         // Validate no contradictory node configuration
-        if (this.DefaultNodes.Length == 1 && this.DefaultNodes[0] == SearchConstants.AllNodesWildcard)
+        if (this.DefaultNodes.Length == 1 && this.DefaultNodes[0] == Constants.SearchDefaults.AllNodesWildcard)
         {
             // Using wildcard - excludeNodes is OK
         }