feat: drop memory index

Makefile

@@ -184,4 +184,4 @@ clean-test-services: stop-test-services
 	@docker compose rm -f localai postgres || true
 
 run-e2e:
-	@E2E=true LOCALAI_ENDPOINT=http://localhost:8081 LOCALRECALL_ENDPOINT=http://localhost:8080 go test -v ./test/e2e/...
+	@E2E=true LOCALAI_ENDPOINT=http://localhost:8081 LOCALRECALL_ENDPOINT=http://localhost:8080 go test -v -timeout 30m ./test/e2e/...

pkg/client/client.go

@@ -66,13 +66,20 @@ func (c *Client) ListCollections() ([]string, error) {
 		return nil, errors.New("failed to list collections")
 	}
 
-	var collections []string
-	err = json.NewDecoder(resp.Body).Decode(&collections)
+	var apiResp struct {
+		Success bool   `json:"success"`
+		Message string `json:"message"`
+		Data    struct {
+			Collections []string `json:"collections"`
+			Count       int      `json:"count"`
+		} `json:"data"`
+	}
+	err = json.NewDecoder(resp.Body).Decode(&apiResp)
 	if err != nil {
 		return nil, err
 	}
 
-	return collections, nil
+	return apiResp.Data.Collections, nil
 }
 
 // ListEntries lists all entries in a collection
@@ -132,15 +139,20 @@ func (c *Client) GetEntryContent(collection, entry string) ([]EntryChunk, error)
 
 	var result struct {
 		Data struct {
-			Chunks []EntryChunk `json:"chunks"`
+			Content    string `json:"content"`
+			ChunkCount int    `json:"chunk_count"`
 		} `json:"data"`
 	}
 	err = json.NewDecoder(resp.Body).Decode(&result)
 	if err != nil {
 		return nil, err
 	}
 
-	return result.Data.Chunks, nil
+	if result.Data.Content == "" {
+		return nil, nil
+	}
+
+	return []EntryChunk{{Content: result.Data.Content}}, nil
 }
 
 // GetEntryRawFile returns the original uploaded binary file as a ReadCloser.
@@ -196,13 +208,19 @@ func (c *Client) DeleteEntry(collection, entry string) ([]string, error) {
 		return nil, errors.New("failed to delete collection: " + bodyResult.String())
 	}
 
-	var results []string
-	err = json.NewDecoder(resp.Body).Decode(&results)
+	var apiResp struct {
+		Success bool   `json:"success"`
+		Message string `json:"message"`
+		Data    struct {
+			RemainingEntries []string `json:"remaining_entries"`
+		} `json:"data"`
+	}
+	err = json.NewDecoder(resp.Body).Decode(&apiResp)
 	if err != nil {
 		return nil, err
 	}
 
-	return results, nil
+	return apiResp.Data.RemainingEntries, nil
 }
 
 // Search searches a collection
@@ -229,13 +247,19 @@ func (c *Client) Search(collection, query string, maxResults int) ([]types.Resul
 		return nil, errors.New("failed to search collection")
 	}
 
-	var results []types.Result
-	err = json.NewDecoder(resp.Body).Decode(&results)
+	var apiResp struct {
+		Success bool   `json:"success"`
+		Message string `json:"message"`
+		Data    struct {
+			Results []types.Result `json:"results"`
+		} `json:"data"`
+	}
+	err = json.NewDecoder(resp.Body).Decode(&apiResp)
 	if err != nil {
 		return nil, err
 	}
 
-	return results, nil
+	return apiResp.Data.Results, nil
 }
 
 func (c *Client) Reset(collection string) error {

rag/engine.go

@@ -14,4 +14,5 @@ type Engine interface {
 	Count() int
 	Delete(where map[string]string, whereDocuments map[string]string, ids ...string) error
 	GetByID(id string) (types.Result, error)
+	GetBySource(source string) ([]types.Result, error)
 }

rag/engine/chromem.go

@@ -167,6 +167,32 @@ func (c *ChromemDB) GetByID(id string) (types.Result, error) {
 	return types.Result{ID: res.ID, Metadata: res.Metadata, Content: res.Content}, nil
 }
 
+func (c *ChromemDB) GetBySource(source string) ([]types.Result, error) {
+	ctx := context.Background()
+	count := c.collection.Count()
+	if count == 0 {
+		return nil, nil
+	}
+
+	// Use Query with a where filter to find documents by source metadata.
+	// We use a dummy query and request all documents, relying on the where
+	// filter to narrow results.
+	res, err := c.collection.Query(ctx, ".", count, map[string]string{"source": source}, nil)
+	if err != nil {
+		return nil, fmt.Errorf("error querying by source: %v", err)
+	}
+
+	var results []types.Result
+	for _, r := range res {
+		results = append(results, types.Result{
+			ID:       r.ID,
+			Metadata: r.Metadata,
+			Content:  r.Content,
+		})
+	}
+	return results, nil
+}
+
 func (c *ChromemDB) Search(s string, similarEntries int) ([]types.Result, error) {
 	res, err := c.collection.Query(context.Background(), s, similarEntries, nil, nil)
 	if err != nil {

rag/engine/localai.go

@@ -88,6 +88,10 @@ func (db *LocalAIRAGDB) GetByID(id string) (types.Result, error) {
 	return types.Result{}, fmt.Errorf("not implemented")
 }
 
+func (db *LocalAIRAGDB) GetBySource(source string) ([]types.Result, error) {
+	return nil, fmt.Errorf("not implemented")
+}
+
 func (db *LocalAIRAGDB) Search(s string, similarEntries int) ([]types.Result, error) {
 	resp, err := db.openaiClient.CreateEmbeddings(context.TODO(),
 		openai.EmbeddingRequestStrings{

rag/engine/mock.go

@@ -0,0 +1,156 @@
+package engine
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/mudler/localrecall/rag/types"
+)
+
+// MockEngine is a simple in-memory engine for testing. It requires no
+// external dependencies (no LocalAI, no embeddings).
+type MockEngine struct {
+	mu    sync.Mutex
+	docs  map[string]types.Result
+	index int
+}
+
+func NewMockEngine() *MockEngine {
+	return &MockEngine{
+		docs:  make(map[string]types.Result),
+		index: 1,
+	}
+}
+
+func (m *MockEngine) Store(s string, metadata map[string]string) (Result, error) {
+	results, err := m.StoreDocuments([]string{s}, metadata)
+	if err != nil {
+		return Result{}, err
+	}
+	return results[0], nil
+}
+
+func (m *MockEngine) StoreDocuments(s []string, metadata map[string]string) ([]Result, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	if len(s) == 0 {
+		return nil, fmt.Errorf("empty input")
+	}
+
+	results := make([]Result, len(s))
+	for i, content := range s {
+		id := fmt.Sprintf("%d", m.index)
+		// Copy metadata so each doc has its own map
+		meta := make(map[string]string, len(metadata))
+		for k, v := range metadata {
+			meta[k] = v
+		}
+		m.docs[id] = types.Result{
+			ID:       id,
+			Content:  content,
+			Metadata: meta,
+		}
+		results[i] = Result{ID: id}
+		m.index++
+	}
+	return results, nil
+}
+
+func (m *MockEngine) Search(s string, similarEntries int) ([]types.Result, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	var results []types.Result
+	for _, doc := range m.docs {
+		if strings.Contains(strings.ToLower(doc.Content), strings.ToLower(s)) {
+			results = append(results, doc)
+		}
+	}
+	// If no substring match, return all (useful for generic searches)
+	if len(results) == 0 {
+		for _, doc := range m.docs {
+			results = append(results, doc)
+		}
+	}
+	if len(results) > similarEntries {
+		results = results[:similarEntries]
+	}
+	return results, nil
+}
+
+func (m *MockEngine) Delete(where map[string]string, whereDocuments map[string]string, ids ...string) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	// Delete by IDs
+	if len(ids) > 0 {
+		for _, id := range ids {
+			delete(m.docs, id)
+		}
+		return nil
+	}
+
+	// Delete by metadata where filter
+	if len(where) > 0 {
+		for id, doc := range m.docs {
+			match := true
+			for k, v := range where {
+				if doc.Metadata[k] != v {
+					match = false
+					break
+				}
+			}
+			if match {
+				delete(m.docs, id)
+			}
+		}
+	}
+
+	return nil
+}
+
+func (m *MockEngine) GetByID(id string) (types.Result, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	doc, ok := m.docs[id]
+	if !ok {
+		return types.Result{}, fmt.Errorf("document not found: %s", id)
+	}
+	return doc, nil
+}
+
+func (m *MockEngine) GetBySource(source string) ([]types.Result, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	var results []types.Result
+	for _, doc := range m.docs {
+		if doc.Metadata["source"] == source {
+			results = append(results, doc)
+		}
+	}
+	return results, nil
+}
+
+func (m *MockEngine) Count() int {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	return len(m.docs)
+}
+
+func (m *MockEngine) Reset() error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	m.docs = make(map[string]types.Result)
+	m.index = 1
+	return nil
+}
+
+func (m *MockEngine) GetEmbeddingDimensions() (int, error) {
+	return 384, nil
+}

rag/engine/postgres.go

@@ -586,6 +586,45 @@ func (p *PostgresDB) GetByID(id string) (types.Result, error) {
 	return result, nil
 }
 
+func (p *PostgresDB) GetBySource(source string) ([]types.Result, error) {
+	ctx := context.Background()
+
+	rows, err := p.pool.Query(ctx, fmt.Sprintf(`
+		SELECT id::text, COALESCE(title, '') as title, content, metadata
+		FROM %s WHERE metadata->>'source' = $1
+	`, p.tableName), source)
+	if err != nil {
+		return nil, fmt.Errorf("failed to query by source: %w", err)
+	}
+	defer rows.Close()
+
+	var results []types.Result
+	for rows.Next() {
+		var r types.Result
+		var title string
+		var metadataJSON []byte
+
+		if err := rows.Scan(&r.ID, &title, &r.Content, &metadataJSON); err != nil {
+			return nil, fmt.Errorf("failed to scan row: %w", err)
+		}
+
+		r.Metadata = make(map[string]string)
+		if len(metadataJSON) > 0 {
+			json.Unmarshal(metadataJSON, &r.Metadata)
+		}
+		if title != "" {
+			r.Metadata["title"] = title
+		}
+		results = append(results, r)
+	}
+
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("row iteration error: %w", err)
+	}
+
+	return results, nil
+}
+
 func (p *PostgresDB) Search(s string, similarEntries int) ([]types.Result, error) {
 	ctx := context.Background()