Merge pull request #46 from Build5Nines/dev

v2.0.2

Author: crpietschmann

CHANGELOG.md

@@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 2.0.2 (2025-04-15)
+
+Add:
+
+- Add `SerializeToBinaryStream` and `DeserializeFromBinaryStream` methods to replace (and mark obselete) `SerializeToJsonStream` and `DeserializeFromJsonStream` methods. They read/write binary zip file data, not json, so they were named incorrectly.
+
+Fixed:
+
+- Fixed `.LoadFromFile` method that was previously inaccessible.
+
 ## 2.0.1 (2025-03-17)
 
 Added:

docs/docs/persistence.md

@@ -0,0 +1,85 @@
+# Data Persistence
+
+The `Build5Nines.SharpVector` library provides easy-to-use methods for saving a memory-based vector database to a file or stream and loading it again later. This is particularly useful for caching indexed content between runs, deploying pre-built vector stores, or shipping databases with your application.
+
+## File Persistence
+
+`Build5Nines.SharpVector` supports persisting the vector database to a file.
+
+### Save to File
+
+To persist your `BasicMemoryVectorDatabase` to disk, use the `SaveToFile` or `SaveToFileAsync` methods:
+
+```csharp
+var vdb = new BasicMemoryVectorDatabase();
+
+var filePath = "vectordata.b59vdb";
+
+// persist vector database to file asynchronously
+await vdb.SaveToFileAsync(filePath);
+
+// -- or --
+
+// persist vector database to file
+vdb.SaveToFile(filePath);
+```
+
+### Load from File
+
+To load a previously saved vector database from disk, use the `LoadFromFile` or `LoadFromFileAsync` methods:
+
+```csharp
+var vdb = new BasicMemoryVectorDatabase();
+
+var filePath = "vectordata.b59vdb";
+
+// load vector database from file
+vdb.LoadFromFile(filePath);
+
+// -- or --
+
+// load vector database from file asynchronously
+await vdb.LoadFromFileAsync(filePath);
+```
+
+## Persist to Stream
+
+The underlying methods used by `SaveToFile` and `LoadFromFile` methods for serializing the vector database to a `Stream` are available to use directly. This provides support for reading/writing to `MemoryStream` (or other streams) if the vector database needs to be persisted to something other than the local file system.
+
+!!! info
+    These `SerializeToBinaryStream` and `DeserializeFromBinaryStream` methods are available in `v2.0.2` and later.
+
+### Write to Stream
+
+To persist your `BasicMemoryVectorDatabase` to a JSON stream, use the `SerializeToBinaryStream` or `SerializeToBinaryStreamAsync` methods:
+
+```csharp
+var vdb = new BasicMemoryVectorDatabase();
+
+var stream = new MemoryStream();
+
+// serialize to JSON stream
+vdb.SerializeToBinaryStream(stream);
+
+// -- or --
+
+// serialize asynchronously to JSON stream
+await vdb.SerializeToBinaryStreamAsync(stream);
+```
+
+### Read from Stream
+
+To load your `BasicMemoryVectorDatabase` from JSON stream, use the `DeserializeFromBinaryStream` and `DeserializeFromBinaryStreamAsync` methods:
+
+```csharp
+// Be sure Stream position is at the start
+stream.Position = 0;
+
+// deserialize from JSON stream
+vdb.DeserializeFromBinaryStream(stream);
+
+// -- or ---
+
+// deserialize asynchronously from JSON stream
+await vdb.DeserializeFromBinaryStreamAsync(stream);
+```

docs/mkdocs.yml

@@ -101,6 +101,7 @@ nav:
   - Get Started: get-started.md
   - Concepts: concepts.md
   - Text Chunking: text-chunking.md
+  - Persistence: persistence.md
   - OpenAI Embeddings: openai-embeddings.md
   - Samples: samples.md
   - Resources: resources.md

src/Build5Nines.SharpVector.OpenAI/Build5Nines.SharpVector.OpenAI.csproj

@@ -9,7 +9,7 @@
     <PackageId>Build5Nines.SharpVector.OpenAI</PackageId>
     <PackageProjectUrl>https://sharpvector.build5nines.com</PackageProjectUrl>
     <RepositoryUrl>https://github.com/Build5Nines/SharpVector</RepositoryUrl>
-    <Version>2.0.0</Version>
+    <Version>2.0.2</Version>
     <Description>Lightweight In-memory Vector Database to embed in any .NET Applications that integrates with OpenAI Embedding model for vector generation.</Description>
     <Copyright>Copyright (c) 2025 Build5Nines LLC</Copyright>
     <PackageReadmeFile>README.md</PackageReadmeFile>
@@ -24,7 +24,7 @@
   </ItemGroup>
 
   <ItemGroup>
-      <PackageReference Include="Build5Nines.SharpVector" Version="[2.0.0,3.0.0)" />
-      <PackageReference Include="OpenAI" Version="2.1.0" />
+      <PackageReference Include="Build5Nines.SharpVector" Version="[2.0.2,3.0.0)" />
+      <PackageReference Include="OpenAI" Version="[2.1.0,3.0.0]" />
   </ItemGroup>
 </Project>

src/Build5Nines.SharpVector.OpenAI/OpenAIMemoryVectorDatabaseBase.cs

@@ -251,13 +251,19 @@ private async Task<IEnumerable<VectorTextResultItem<TMetadata>>> CalculateVector
         return results;
     }
 
-        /// <summary>
+    [Obsolete("Use SerializeToBinaryStreamAsync Instead")]
+    public virtual async Task SerializeToJsonStreamAsync(Stream stream)
+    {
+        await SerializeToBinaryStreamAsync(stream);
+    }
+
+    /// <summary>
     /// Serializes the Vector Database to a JSON stream
     /// </summary>
     /// <param name="stream"></param>
     /// <returns></returns>
     /// <exception cref="InvalidOperationException"></exception>
-    public virtual async Task SerializeToJsonStreamAsync(Stream stream)
+    public virtual async Task SerializeToBinaryStreamAsync(Stream stream)
     {
         var streamVectorStore = new MemoryStream();
         var streamVocabularyStore = new MemoryStream();
@@ -296,16 +302,28 @@ public virtual async Task SerializeToJsonStreamAsync(Stream stream)
         await stream.FlushAsync();
     }
 
+    [Obsolete("Use SerializeToBinaryStream Instead")]
     public virtual void SerializeToJsonStream(Stream stream)
+    {
+        SerializeToBinaryStream(stream);
+    }
+
+    public virtual void SerializeToBinaryStream(Stream stream)
     {
         if (stream == null)
         {
             throw new ArgumentNullException(nameof(stream));
         }
-        SerializeToJsonStreamAsync(stream).Wait();
+        SerializeToBinaryStreamAsync(stream).Wait();
     }
 
+    [Obsolete("Use DeserializeFromBinaryStreamAsync Instead")]
     public virtual async Task DeserializeFromJsonStreamAsync(Stream stream)
+    {
+        await DeserializeFromBinaryStreamAsync(stream);
+    }
+
+    public virtual async Task DeserializeFromBinaryStreamAsync(Stream stream)
     {
         if (stream == null)
         {
@@ -374,13 +392,19 @@ public virtual async Task DeserializeFromJsonStreamAsync(Stream stream)
         }
     }
 
+    [Obsolete("Use DeserializeFromBinaryStream Instead")]
     public virtual void DeserializeFromJsonStream(Stream stream)
+    {
+        DeserializeFromBinaryStream(stream);
+    }
+    
+    public virtual void DeserializeFromBinaryStream(Stream stream)
     {
         if (stream == null)
         {
             throw new ArgumentNullException(nameof(stream));
         }
-        DeserializeFromJsonStreamAsync(stream).Wait();
+        DeserializeFromBinaryStreamAsync(stream).Wait();
     }
 
 }

src/Build5Nines.SharpVector/Build5Nines.SharpVector.csproj

@@ -9,7 +9,7 @@
     <PackageId>Build5Nines.SharpVector</PackageId>
     <PackageProjectUrl>https://sharpvector.build5nines.com</PackageProjectUrl>
     <RepositoryUrl>https://github.com/Build5Nines/SharpVector</RepositoryUrl>
-    <Version>2.0.1</Version>
+    <Version>2.0.2</Version>
     <Description>Lightweight In-memory Vector Database to embed in any .NET Applications</Description>
     <Copyright>Copyright (c) 2025 Build5Nines LLC</Copyright>
     <PackageReadmeFile>README.md</PackageReadmeFile>

src/Build5Nines.SharpVector/IVectorDatabase.cs

@@ -1,3 +1,5 @@
+using System.Reflection.Metadata;
+
 namespace Build5Nines.SharpVector;
 
 public interface IVectorDatabase<TId, TMetadata, TDocument>
@@ -85,33 +87,47 @@ public interface IVectorDatabase<TId, TMetadata, TDocument>
     Task<IVectorTextResult<TDocument, TMetadata>> SearchAsync(TDocument queryText, float? threshold = null, int pageIndex = 0, int? pageCount = null);
 
 
+    [Obsolete("Use SerializeToBinaryStreamAsync Instead")]
+    Task SerializeToJsonStreamAsync(Stream stream);
+
+    [Obsolete("Use SerializeToBinaryStream Instead")]
+    void SerializeToJsonStream(Stream stream);
+
+    [Obsolete("Use DeserializeToBinaryStreamAsync Instead")]
+    Task DeserializeFromJsonStreamAsync(Stream stream);
+
+    [Obsolete("Use DeserializeToBinaryStream Instead")]
+    void DeserializeFromJsonStream(Stream stream);
+
+
+    
     /// <summary>
     /// Serializes the Database to a JSON stream
     /// </summary>
     /// <param name="stream"></param>
     /// <returns></returns>
-    Task SerializeToJsonStreamAsync(Stream stream);
+    Task SerializeToBinaryStreamAsync(Stream stream);
 
     /// <summary>
     /// Serializes the Database to a JSON stream
     /// </summary>
     /// <param name="stream"></param>
     /// <returns></returns>
-    void SerializeToJsonStream(Stream stream);
+    void SerializeToBinaryStream(Stream stream);
 
     /// <summary>
     /// Deserializes the Database from a JSON stream
     /// </summary>
     /// <param name="stream"></param>
     /// <returns></returns>
-    Task DeserializeFromJsonStreamAsync(Stream stream);
+    Task DeserializeFromBinaryStreamAsync(Stream stream);
 
     /// <summary>
     /// Deserializes the Database from a JSON stream
     /// </summary>
     /// <param name="stream"></param>
     /// <returns></returns>
-    void DeserializeFromJsonStream(Stream stream);
+    void DeserializeFromBinaryStream(Stream stream);
 }
 
 public interface IVectorDatabase<TId, TMetadata>

src/Build5Nines.SharpVector/IVectorDatabaseExtensions.cs

@@ -1,3 +1,5 @@
+using System.Text;
+
 namespace Build5Nines.SharpVector;
 
 public static class IVectorDatabaseExtensions
@@ -7,7 +9,7 @@ public static async Task SaveToFileAsync<TId, TMetadata, TDocument>(this IVector
     {
         using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write))
         {
-            await vectorDatabase.SerializeToJsonStreamAsync(stream);
+            await vectorDatabase.SerializeToBinaryStreamAsync(stream);
         }
     }
 
@@ -16,7 +18,7 @@ public static void SaveToFile<TId, TMetadata, TDocument>(this IVectorDatabase<TI
     {
         using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write))
         {
-            vectorDatabase.SerializeToJsonStream(stream);
+            vectorDatabase.SerializeToBinaryStream(stream);
         }
     }
 
@@ -25,16 +27,16 @@ public static async Task LoadFromFileAsync<TId, TMetadata, TDocument>(this IVect
     {
         using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read))
         {
-            await vectorDatabase.DeserializeFromJsonStreamAsync(stream);
+            await vectorDatabase.DeserializeFromBinaryStreamAsync(stream);
         }
     }
 
-    public static void LoadFromFile<TId, TMetadata, TDocument>(IVectorDatabase<TId, TMetadata, TDocument> vectorDatabase, string filePath)
+    public static void LoadFromFile<TId, TMetadata, TDocument>(this IVectorDatabase<TId, TMetadata, TDocument> vectorDatabase, string filePath)
         where TId : notnull
     {
         using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read))
         {
-            vectorDatabase.DeserializeFromJsonStream(stream);
+            vectorDatabase.DeserializeFromBinaryStream(stream);
         }
     }
 }

src/Build5Nines.SharpVector/MemoryVectorDatabase.cs

@@ -33,17 +33,30 @@ public MemoryVectorDatabase()
         )
     { }
 
+
+    [Obsolete("Use DeserializeFromBinaryStreamAsync instead.")]
     public override async Task DeserializeFromJsonStreamAsync(Stream stream)
     {
-        await base.DeserializeFromJsonStreamAsync(stream);
+        await DeserializeFromBinaryStreamAsync(stream);
+    }
+
+    [Obsolete("Use DeserializeFromBinaryStream instead.")]
+    public override void DeserializeFromJsonStream(Stream stream)
+    {
+        DeserializeFromBinaryStream(stream);
+    }
+
+    public override async Task DeserializeFromBinaryStreamAsync(Stream stream)
+    {
+        await base.DeserializeFromBinaryStreamAsync(stream);
 
         // Re-initialize the IdGenerator with the max Id value from the VectorStore
         _idGenerator = new IntIdGenerator(VectorStore.GetIds().Max());
     }
 
-    public override void DeserializeFromJsonStream(Stream stream)
+    public override void DeserializeFromBinaryStream(Stream stream)
     {
-        base.DeserializeFromJsonStream(stream);
+        base.DeserializeFromBinaryStream(stream);
 
         // Re-initialize the IdGenerator with the max Id value from the VectorStore
         _idGenerator = new IntIdGenerator(VectorStore.GetIds().Max());