improve index management in r rollback.c

Author: ProgramComputer

PersonalTesting/setup_test.py

@@ -51,6 +51,7 @@ def get_voyage_embedding(text, cache_key=None):
         "input_type": "document"
     }
 
+    print(f"Calling Voyage API for text: {text[:50]}...")
     response = requests.post(
         "https://api.voyageai.com/v1/embeddings",
         headers=headers,
@@ -81,6 +82,8 @@ def get_openai_embedding(text, cache_key=None):
         "model": "text-embedding-3-small",
         "encoding_format": "float"
     }
+    
+    print(f"Calling OpenAI API for text: {text[:50]}...")
     response = requests.post(
         "https://api.openai.com/v1/embeddings",
         headers=headers,
@@ -94,27 +97,6 @@ def get_openai_embedding(text, cache_key=None):
 
     return embedding
 
-def create_synthetic_embedding(base_embedding, variation=0.05, dims=None):
-    """Create a synthetic embedding based on an existing one with some variation"""
-    if dims is None:
-        dims = base_embedding.shape[0]
-    
-    if base_embedding is not None:
-        # Add random noise to the base embedding
-        noise = np.random.normal(0, variation, base_embedding.shape)
-        synthetic = base_embedding + noise
-        
-        # Normalize the embedding
-        norm = np.linalg.norm(synthetic)
-        if norm > 0:
-            synthetic = synthetic / norm
-    else:
-        # Generate a completely random embedding if no base is provided
-        synthetic = np.random.random(dims).astype(np.float32)
-        synthetic = synthetic / np.linalg.norm(synthetic)
-        
-    return synthetic
-
 def file_exists_and_not_empty(filepath):
     """Check if a file exists and is not empty"""
     return os.path.exists(filepath) and os.path.getsize(filepath) > 0
@@ -199,9 +181,6 @@ def create_test_files():
         "api_design.md": api_doc
     }
 
-    # Keep track of real embeddings for synthetic generation
-    real_embeddings = {}
-
     for filename, content in files.items():
         print(f"\nProcessing {filename}...")
 
@@ -212,40 +191,38 @@ def create_test_files():
             openai_emb = get_openai_embedding(content, cache_key=f"openai_{filename}")
             np.save(openai_path, openai_emb)
             print(f"Created {openai_path}")
-            real_embeddings['openai'] = openai_emb
 
             # Also save as binary
             openai_bin_path = f"test_files/{filename}.openai.bin"
             save_as_bin(openai_emb, openai_bin_path)
         else:
             print(f"Using existing OpenAI embedding: {openai_path}")
-            real_embeddings['openai'] = np.load(openai_path)
 
             # Create bin file if it doesn't exist
             openai_bin_path = f"test_files/{filename}.openai.bin"
             if not file_exists_and_not_empty(openai_bin_path):
-                save_as_bin(real_embeddings['openai'], openai_bin_path)
+                openai_emb = np.load(openai_path)
+                save_as_bin(openai_emb, openai_bin_path)
 
-        # Generate Voyage embedding as a synthetic version of OpenAI to reduce API calls
+        # Generate real Voyage embedding (only if not exists)
         voyage_path = f"test_files/{filename}.voyage.npy"
         if not file_exists_and_not_empty(voyage_path):
-            print("Creating synthetic Voyage embedding based on OpenAI embedding...")
-            voyage_emb = create_synthetic_embedding(real_embeddings['openai'])
+            print("Getting Voyage embedding for full document...")
+            voyage_emb = get_voyage_embedding(content, cache_key=f"voyage_{filename}")
             np.save(voyage_path, voyage_emb)
             print(f"Created {voyage_path}")
-            real_embeddings['voyage'] = voyage_emb
 
             # Also save as binary
             voyage_bin_path = f"test_files/{filename}.voyage.bin"
             save_as_bin(voyage_emb, voyage_bin_path)
         else:
             print(f"Using existing Voyage embedding: {voyage_path}")
-            real_embeddings['voyage'] = np.load(voyage_path)
 
             # Create bin file if it doesn't exist
             voyage_bin_path = f"test_files/{filename}.voyage.bin"
             if not file_exists_and_not_empty(voyage_bin_path):
-                save_as_bin(real_embeddings['voyage'], voyage_bin_path)
+                voyage_emb = np.load(voyage_path)
+                save_as_bin(voyage_emb, voyage_bin_path)
 
 def create_modified_files():
     """Create modified versions of test files to test rollback"""
@@ -277,11 +254,11 @@ def create_modified_files():
                 modified_openai_bin_path = "test_files/technical_doc.txt.modified.openai.bin"
                 save_as_bin(openai_emb, modified_openai_bin_path)
 
-                # Create synthetic Voyage embedding
+                # Create real Voyage embedding
                 modified_voyage_path = "test_files/technical_doc.txt.modified.voyage.npy"
-                voyage_emb = create_synthetic_embedding(openai_emb)
+                voyage_emb = get_voyage_embedding(modified_tech_doc, cache_key="voyage_tech_modified")
                 np.save(modified_voyage_path, voyage_emb)
-                print(f"Created synthetic {modified_voyage_path}")
+                print(f"Created {modified_voyage_path}")
 
                 # Also save as binary
                 modified_voyage_bin_path = "test_files/technical_doc.txt.modified.voyage.bin"
@@ -337,11 +314,11 @@ def create_modified_files():
                     modified_openai_bin_path = "test_files/api_design.md.modified.openai.bin"
                     save_as_bin(openai_emb, modified_openai_bin_path)
 
-                    # Create synthetic Voyage embedding
+                    # Create real Voyage embedding
                     modified_voyage_path = "test_files/api_design.md.modified.voyage.npy"
-                    voyage_emb = create_synthetic_embedding(openai_emb)
+                    voyage_emb = get_voyage_embedding(modified_api_doc, cache_key="voyage_api_modified")
                     np.save(modified_voyage_path, voyage_emb)
-                    print(f"Created synthetic {modified_voyage_path}")
+                    print(f"Created {modified_voyage_path}")
 
                     # Also save as binary
                     modified_voyage_bin_path = "test_files/api_design.md.modified.voyage.bin"
@@ -371,6 +348,7 @@ def print_test_commands():
     print("\n1. Initialize EmbeddingBridge in your project:")
     print("   eb init")
     print("   eb model register openai-3-small --dimensions 1536 --description \"OpenAI text-embedding-3-small model\"")
+    print("   eb model register voyage-2 --dimensions 1024 --description \"Voyage-2 embedding model\"")
 
     print("\n2. Create and set up the test files (this runs your setup script):")
     print("   python PersonalTesting/setup_test.py")
@@ -415,7 +393,7 @@ def print_test_commands():
     print("   # Compare using specific model")
     print("   eb diff --model openai-3-small <short hash> <short hash>")
     print("   # Compare using different models for each file")
-    print("   eb diff --models openai-3-small,voyage <short hash> <short hash>")
+    print("   eb diff --models openai-3-small,voyage-2 <short hash> <short hash>")
 
     print("\n8. Rollback to previous versions:")
     print("   # Rollback by file path (goes to previous version)")

PersonalTesting/test_files/api_design.md

@@ -2,31 +2,31 @@
 
 
 ## Overview
-Dolore dolor neque quisquam etincidunt neque dolore. Ipsum porro dolor dolore quisquam quisquam sit etincidunt. Amet adipisci numquam sed non sed dolore. Porro porro ut porro labore eius non. Amet quaerat quisquam numquam ipsum est voluptatem. Neque ut numquam voluptatem sit tempora ipsum etincidunt.
+Adipisci ut magnam non modi dolor. Sit quiquia tempora magnam dolore sed ipsum. Dolorem dolore porro magnam adipisci. Ipsum voluptatem dolorem velit. Dolor etincidunt aliquam consectetur. Ut neque non adipisci porro quaerat modi.
 
 ## Authentication
-Magnam porro adipisci eius. Aliquam non sit est ipsum est amet voluptatem. Labore ipsum consectetur consectetur dolore neque est. Sit sit sed aliquam modi sed velit. Neque ipsum numquam voluptatem sed etincidunt numquam. Quaerat numquam eius dolor neque velit modi. Quaerat voluptatem ipsum tempora porro amet. Modi quaerat ut etincidunt.
+Adipisci ut amet adipisci porro. Sit dolore aliquam modi ipsum dolorem dolorem. Tempora etincidunt quaerat adipisci magnam dolorem. Consectetur quisquam consectetur velit quisquam non. Quiquia magnam ipsum sit dolore numquam adipisci. Non quaerat dolorem etincidunt etincidunt porro est quisquam. Consectetur aliquam etincidunt est quaerat aliquam ut dolor. Ut labore ut ipsum dolore.
 
-Sit aliquam est voluptatem modi voluptatem voluptatem magnam.
+Consectetur consectetur ipsum est.
 
 ## Endpoints
-- POST /embeddings: Quaerat magnam numquam neque est est.
-- GET /models: Tempora ut magnam modi tempora labore velit.
-- PUT /configurations: Dolorem modi neque dolorem neque velit non porro.
-- DELETE /models/{id}: Consectetur est quiquia amet.
+- POST /embeddings: Quaerat numquam non dolorem eius.
+- GET /models: Aliquam dolor porro eius.
+- PUT /configurations: Quiquia dolore est quiquia quaerat ipsum sit.
+- DELETE /models/{id}: Eius etincidunt labore quaerat est.
 
 ## Error Handling
-Porro etincidunt aliquam quaerat ipsum sit. Neque ipsum dolor eius numquam. Non non quiquia amet amet eius neque. Voluptatem dolore dolore sed neque ut non. Adipisci etincidunt sed est quisquam. Tempora dolore est aliquam non. Adipisci etincidunt non dolor tempora velit dolorem voluptatem. Etincidunt labore numquam etincidunt modi labore porro ipsum.
+Consectetur quiquia ut quisquam neque eius quaerat. Dolor non consectetur modi non. Voluptatem ut est ipsum. Dolore eius sit consectetur aliquam. Tempora neque non voluptatem. Dolor velit numquam neque quaerat.
 
 ## Rate Limiting
-Consectetur dolor numquam sed. Neque neque consectetur aliquam quisquam. Dolore labore numquam ut quisquam. Dolore adipisci adipisci quisquam porro labore velit tempora. Sed ut adipisci modi. Eius dolorem porro quisquam neque voluptatem voluptatem. Velit dolore quisquam velit.
+Amet ipsum amet ipsum dolore etincidunt dolore. Est sed amet sit adipisci consectetur. Consectetur modi amet quisquam ipsum voluptatem ipsum voluptatem. Labore quaerat voluptatem ut. Voluptatem quisquam porro dolore dolor magnam velit amet.
 
 ## New Endpoints
-- PATCH /models/{id}: Quaerat consectetur est adipisci neque dolore.
-- POST /validate: Ut sit aliquam dolorem amet.
+- PATCH /models/{id}: Etincidunt neque quiquia sed adipisci est numquam neque.
+- POST /validate: Neque eius neque dolore sed.
 
 ## Best Practices
-- Magnam dolorem sed dolor.
-- Velit consectetur modi sed.
-- Consectetur sit sit est.
-- Etincidunt quaerat velit quiquia labore.
+- Sit porro etincidunt labore etincidunt numquam dolore.
+- Numquam adipisci quiquia sit non etincidunt.
+- Quiquia quiquia sed aliquam quaerat velit eius dolore.
+- Dolorem eius sit labore est.

README.md

@@ -16,8 +16,8 @@ A command-line tool for managing and versioning embedding vectors. Think of it a
 
 ```bash
 make clean
-make DEBUG=0 all
-make DEBUG=0 install
+make all
+make install
 ```
 
 ## Quick Start