feat: transition to tree-indexed categories (schema 4.0)

- Removed sharding support and replaced it with tree-indexed category handling.
- Updated manifest and operations to accommodate new category structure.
- Introduced tombstones management for tracking deleted items separately.
- Enhanced push and pull operations to work with tree-indexed categories.
- Updated types and interfaces to reflect changes in category management.
- Refactored related helper functions for better clarity and performance.

Author: ErcinDedeoglu

src/sync/engine/helpers.ts

@@ -11,15 +11,11 @@ import type {
   SyncConfig,
   Tombstone,
 } from '../../types/index.js';
-import { isShardedRef } from '../../types/index.js';
-import type { CategoryData, PassphraseOption, ResolvedShard } from '../operations/types.js';
+import type { CategoryData, PassphraseOption } from '../operations/types.js';
 import { isItemCategoryData } from '../operations/types.js';
 import type { PullOptions } from '../operations/pull.js';
 import type { PreparePushResult } from '../operations/push.js';
 import { preparePushData } from '../operations/push.js';
-import type { StorageBackend } from '../../storage/index.js';
-import { fetchCategoryShard } from './manifest.js';
-import { syncLog } from './logger.js';
 
 /**
  * Build checksums map from local item category data for merge-based pull.
@@ -67,18 +63,11 @@ export interface PushOptsBase {
 }
 
 /** Execute push and return files for storage */
-export function executePush(
-  opts: PushOptsBase,
-  remoteManifest?: Manifest,
-  resolvedShards?: Record<SyncCategory, ResolvedShard>
-): PreparePushResult {
+export function executePush(opts: PushOptsBase, remoteManifest?: Manifest): PreparePushResult {
   const pushOpts = { ...opts } as Parameters<typeof preparePushData>[0];
   if (remoteManifest) {
     pushOpts.remoteManifest = remoteManifest;
   }
-  if (resolvedShards) {
-    pushOpts.resolvedShards = resolvedShards;
-  }
   return preparePushData(pushOpts);
 }
 
@@ -120,38 +109,3 @@ export function extractTombstoneIds(
   }
   return result;
 }
-
-/**
- * Fetch resolved shard data for sharded categories.
- * This ensures we merge with existing remote data instead of overwriting.
- */
-export async function fetchResolvedShards(
-  backend: StorageBackend,
-  remote?: Manifest
-): Promise<Record<SyncCategory, ResolvedShard> | undefined> {
-  if (!remote) return undefined;
-
-  const shardedCategories: SyncCategory[] = ['sessions', 'messages'];
-  const resolved: Record<SyncCategory, ResolvedShard> = {} as Record<SyncCategory, ResolvedShard>;
-  let hasAny = false;
-
-  for (const category of shardedCategories) {
-    const info = remote.categories[category];
-    if (info && isShardedRef(info)) {
-      syncLog(`[PUSH] Fetching shard for ${category}: ${info.shardFile}`);
-      const shard = await fetchCategoryShard(backend, info.shardFile);
-      if (shard) {
-        resolved[category] = {
-          items: shard.items,
-          tombstones: shard.tombstones,
-        };
-        hasAny = true;
-        syncLog(
-          `[PUSH] Loaded ${String(Object.keys(shard.items).length)} remote ${category} for merge`
-        );
-      }
-    }
-  }
-
-  return hasAny ? resolved : undefined;
-}

src/sync/engine/manifest.ts

@@ -1,15 +1,13 @@
 /**
  * Manifest Operations
  *
- * Handles fetching and parsing the remote manifest, including sharded manifests.
+ * Handles fetching and parsing the remote manifest.
  */
 
 import type { StorageBackend } from '../../storage/index.js';
 import { RepoApiError } from '../../storage/index.js';
-import type { Manifest, CategoryShard, ItemCategoryInfo } from '../../types/index.js';
-import { isShardedRef, type SyncCategory } from '../../types/index.js';
+import type { Manifest } from '../../types/index.js';
 import { MANIFEST_FILENAME } from './types.js';
-import { syncLog } from './logger.js';
 
 /**
  * Fetch and parse the manifest from storage.
@@ -27,65 +25,3 @@ export async function fetchManifest(backend: StorageBackend): Promise<Manifest |
     throw error;
   }
 }
-
-/**
- * Fetch a category shard from storage.
- */
-export async function fetchCategoryShard(
-  backend: StorageBackend,
-  shardFile: string
-): Promise<CategoryShard | null> {
-  let content: string | null = null;
-  try {
-    content = await backend.getFile(shardFile);
-    if (!content) return null;
-    return JSON.parse(content) as CategoryShard;
-  } catch (error) {
-    if (error instanceof RepoApiError && error.status === 404) {
-      return null;
-    }
-    // Log JSON parse errors for debugging with content preview
-    if (error instanceof SyntaxError) {
-      const preview = content ? content.slice(0, 200) : 'null';
-      syncLog(`[MANIFEST] JSON parse error in ${shardFile}: ${error.message}`);
-      syncLog(`[MANIFEST] Content preview: ${preview}`);
-    }
-    throw error;
-  }
-}
-
-/**
- * Resolve a category's full info, loading shard if needed.
- * Returns ItemCategoryInfo for sharded categories.
- */
-export async function resolveCategoryInfo(
-  manifest: Manifest,
-  category: SyncCategory,
-  backend: StorageBackend
-): Promise<ItemCategoryInfo | null> {
-  const info = manifest.categories[category];
-  if (!info) return null;
-
-  // If it's already an item category, return as-is
-  if (info.type === 'items') {
-    return info;
-  }
-
-  // If it's a sharded reference, load the shard
-  if (isShardedRef(info)) {
-    const shard = await fetchCategoryShard(backend, info.shardFile);
-    if (!shard) return null;
-
-    // Convert shard to ItemCategoryInfo
-    return {
-      type: 'items',
-      items: shard.items,
-      tombstones: shard.tombstones,
-      itemCount: Object.keys(shard.items).length,
-      lastModified: info.lastModified,
-      lastModifiedBy: info.lastModifiedBy,
-    };
-  }
-
-  return null;
-}

src/sync/engine/operations.ts

@@ -24,9 +24,7 @@ import {
   toStorageFiles,
   buildCryptoOptions,
   extractTombstoneIds,
-  fetchResolvedShards,
 } from './helpers.js';
-import { fetchRemoteItemsNotLocal } from './remote-fetch.js';
 
 export interface OperationContext {
   backend: StorageBackend;
@@ -45,41 +43,25 @@ export async function executePushOperation(
   ctx: PushContext,
   data: CategoryData[],
   remote?: Manifest
-): Promise<{ result: SyncResult; newState: LocalSyncState; remoteItems?: CategoryData[] }> {
+): Promise<{ result: SyncResult; newState: LocalSyncState }> {
   const existing = (await ctx.backend.listFiles()).map((f) => f.filename);
 
-  // Pre-fetch shards for sharded categories to enable proper merging
-  const resolvedShards = await fetchResolvedShards(ctx.backend, remote);
-
   const opts = {
     localData: data,
     config: ctx.config,
     localState: ctx.localState,
     passphrase: buildCryptoOptions(ctx.passphrase, ctx.oldPassphrase),
     existingFiles: existing,
   };
-  const { files, manifest, changedCategories } = executePush(opts, remote, resolvedShards);
+  const { files, manifest, changedCategories } = executePush(opts, remote);
   const fileCount = Object.keys(files).length;
   syncLog(`[SYNC] Push: ${String(fileCount)} files, ${String(existing.length)} existing`);
   await ctx.backend.updateFiles(
     toStorageFiles(files, MANIFEST_FILENAME, JSON.stringify(manifest, null, 2))
   );
   const newState = buildLocalState(manifest, data, ctx.getStorageId(), ctx.config.machineId);
 
-  // Fetch remote items we don't have locally (for writing to disk)
-  const remoteItems = resolvedShards
-    ? await fetchRemoteItemsNotLocal(ctx.backend, resolvedShards, data)
-    : undefined;
-
-  const result = buildPushResult({
-    changedCategories,
-    pulledData: remoteItems && remoteItems.length > 0 ? remoteItems : undefined,
-  });
-
-  // Only include remoteItems in return if there are any
-  if (remoteItems && remoteItems.length > 0) {
-    return { result, newState, remoteItems };
-  }
+  const result = buildPushResult({ changedCategories });
   return { result, newState };
 }
 

src/sync/engine/remote-fetch.ts

@@ -45,5 +45,13 @@ export {
   isItemTombstoned,
   getItemsToDelete,
   detectLocalDeletions,
+  // TombstonesFile handling (schema 4.0)
+  parseTombstonesFile,
+  serializeTombstonesFile,
+  getCategoryTombstones,
+  setCategoryTombstones,
+  addTombstone,
+  mergeTombstonesFiles,
+  filterExpiredTombstonesFile,
 } from './tombstone.js';
 export type { FilterTombstonesResult } from './tombstone.js';

src/sync/index.ts

@@ -83,7 +83,7 @@ export function createManifest(
 ): Manifest {
   return {
     version: (localState?.lastSyncedVersion ?? 0) + 1,
-    schemaVersion: '3.0',
+    schemaVersion: '4.0',
     createdAt: localState?.lastSyncedAt ?? now,
     updatedAt: now,
     lastUpdatedBy: machineId,
@@ -103,6 +103,9 @@ export function addSyncHistory(ctx: PushContext, categories: SyncCategory[]): vo
   ctx.manifest.recentSyncs = [entry].slice(0, MAX_SYNC_HISTORY);
 }
 
+/** Files that should never be marked as orphaned */
+const PROTECTED_FILES = new Set(['manifest.json', 'tombstones.json']);
+
 /** Mark orphaned files for deletion. */
 export function markOrphanedFiles(
   files: Record<string, { content: string | null }>,
@@ -111,7 +114,7 @@ export function markOrphanedFiles(
 ): void {
   if (!existingFiles) return;
   for (const filename of existingFiles) {
-    if (filename !== 'manifest.json' && !newFiles.has(filename)) {
+    if (!PROTECTED_FILES.has(filename) && !newFiles.has(filename)) {
       files[filename] = { content: null };
     }
   }