feat(sync): enhance pull and push operations with improved error handling

Author: ErcinDedeoglu

src/storage/repo/graphql-client.ts

@@ -55,6 +55,7 @@ export class GraphQLClient {
 
     // Process in batches to avoid GraphQL query complexity limits
     let apiCalls = 0;
+    const totalBatches = Math.ceil(paths.length / GRAPHQL_BATCH_SIZE);
     for (let i = 0; i < paths.length; i += GRAPHQL_BATCH_SIZE) {
       const batch = paths.slice(i, i + GRAPHQL_BATCH_SIZE);
       const batchResult = await this.fetchFilesViaGraphQL(batch, branch, fetchBlob);
@@ -63,6 +64,11 @@ export class GraphQLClient {
       for (const [path, content] of Object.entries(batchResult)) {
         result[path] = content;
       }
+
+      // Log progress every 10 batches for large fetches
+      if (totalBatches > 10 && apiCalls % 10 === 0) {
+        logProgress(`GraphQL fetch progress: ${String(apiCalls)}/${String(totalBatches)} batches`);
+      }
     }
 
     const duration = Date.now() - startTime;

src/sync/engine/helpers.ts

@@ -38,17 +38,22 @@ export function buildLocalChecksums(
 }
 
 /**
- * Build pull options, conditionally including localChecksums.
+ * Build pull options, conditionally including localChecksums and localRemoteShas.
  */
 export function buildPullOptions(
-  base: Omit<PullOptions, 'localChecksums'>,
-  localData?: CategoryData[]
+  base: Omit<PullOptions, 'localChecksums' | 'localRemoteShas'>,
+  localData?: CategoryData[],
+  localState?: LocalSyncState | null
 ): PullOptions {
   const localChecksums = buildLocalChecksums(localData);
+  const opts: PullOptions = { ...base };
   if (localChecksums) {
-    return { ...base, localChecksums };
+    opts.localChecksums = localChecksums;
+  }
+  if (localState?.remoteShas) {
+    opts.localRemoteShas = localState.remoteShas;
   }
-  return base;
+  return opts;
 }
 
 /** Build push options with conditional remoteManifest */
@@ -58,6 +63,8 @@ export interface PushOptsBase {
   localState: LocalSyncState | null;
   passphrase: PassphraseOption;
   existingFiles: string[];
+  /** Remote file SHAs for incremental push (filename -> git blob SHA) */
+  remoteShas?: Record<string, string>;
 }
 
 /** Execute push and return files for storage */

src/sync/engine/operations.ts

@@ -44,14 +44,22 @@ export async function executePushOperation(
   data: CategoryData[],
   remote?: Manifest
 ): Promise<{ result: SyncResult; newState: LocalSyncState }> {
-  const existing = (await ctx.backend.listFiles()).map((f) => f.filename);
+  const remoteFiles = await ctx.backend.listFiles();
+  const existing = remoteFiles.map((f) => f.filename);
+
+  // Build SHA map for comparison (filename -> git blob SHA)
+  const remoteShas: Record<string, string> = {};
+  for (const f of remoteFiles) {
+    if (f.sha) remoteShas[f.filename] = f.sha;
+  }
 
   const opts = {
     localData: data,
     config: ctx.config,
     localState: ctx.localState,
     passphrase: buildCryptoOptions(ctx.passphrase, ctx.oldPassphrase),
     existingFiles: existing,
+    remoteShas,
   };
   const { files, manifest, changedCategories } = executePush(opts, remote);
   const fileCount = Object.keys(files).length;
@@ -67,7 +75,7 @@ export async function executePushOperation(
 
 /** Execute a pull operation and return updated local state */
 export async function executePullOperation(
-  ctx: OperationContext,
+  ctx: PushContext,
   remote?: Manifest,
   data?: CategoryData[]
 ): Promise<{ result: SyncResult; newState: LocalSyncState | null }> {
@@ -79,12 +87,18 @@ export async function executePullOperation(
       passphrase: buildCryptoOptions(ctx.passphrase, ctx.oldPassphrase),
       backend: ctx.backend,
     },
-    data
+    data,
+    ctx.localState
   );
-  const { pulledData, changedCategories, tombstonedItems } = await pullCategories(opts);
+  const { pulledData, changedCategories, tombstonedItems, remoteShas } = await pullCategories(opts);
   syncLog(`[SYNC] Pull: ${String(changedCategories.length)} categories`);
   const mergedData = mergeDataForState(data, pulledData);
   const newState = buildLocalState(remote, mergedData, ctx.getStorageId(), ctx.config.machineId);
+  // Merge remote SHAs: keep existing ones and add/update from pull
+  newState.remoteShas = {
+    ...(ctx.localState?.remoteShas ?? {}),
+    ...remoteShas,
+  };
   const tombstoneIds = extractTombstoneIds(tombstonedItems);
   return {
     result: buildPullResult({ changedCategories, pulledData, tombstonedItems: tombstoneIds }),

src/sync/operations/pull.ts

@@ -29,6 +29,8 @@ export interface PullOptions {
   backend: StorageBackend;
   /** Map of category → item ID → checksum for existing local items */
   localChecksums?: Record<SyncCategory, Record<string, string>>;
+  /** Map of category → filename → git SHA for incremental pull */
+  localRemoteShas?: Partial<Record<SyncCategory, Record<string, string>>>;
 }
 
 /**
@@ -39,6 +41,8 @@ export interface ExtendedPullResult extends PullResult {
   downloadedItems: Record<SyncCategory, string[]>;
   /** Items that should be deleted locally (tombstoned remotely) */
   tombstonedItems: Record<SyncCategory, Record<string, Tombstone>>;
+  /** Remote SHAs for all synced items (for incremental pull) */
+  remoteShas: Partial<Record<SyncCategory, Record<string, string>>>;
 }
 
 /** Result accumulator for pull operation */
@@ -47,6 +51,7 @@ interface PullAccumulator {
   changedCategories: SyncCategory[];
   downloadedItems: Record<SyncCategory, string[]>;
   tombstonedItems: Record<SyncCategory, Record<string, Tombstone>>;
+  remoteShas: Partial<Record<SyncCategory, Record<string, string>>>;
 }
 
 /** Info about a file to fetch */
@@ -63,6 +68,7 @@ function createPullAccumulator(): PullAccumulator {
     changedCategories: [],
     downloadedItems: {} as Record<SyncCategory, string[]>,
     tombstonedItems: {} as Record<SyncCategory, Record<string, Tombstone>>,
+    remoteShas: {},
   };
 }
 
@@ -71,7 +77,8 @@ function recordItemPull(
   acc: PullAccumulator,
   cat: SyncCategory,
   data: ItemCategoryData,
-  tombstones: Record<string, Tombstone>
+  tombstones: Record<string, Tombstone>,
+  shas: Record<string, string>
 ): void {
   if (Object.keys(data.items).length > 0) {
     acc.pulledData.push(data);
@@ -81,24 +88,36 @@ function recordItemPull(
   if (Object.keys(tombstones).length > 0) {
     acc.tombstonedItems[cat] = tombstones;
   }
+  // Always record SHAs (merge with existing for incremental)
+  acc.remoteShas[cat] = { ...(acc.remoteShas[cat] ?? {}), ...shas };
 }
 
-/** Build list of files to fetch, excluding tombstoned items */
+/** Build list of files to fetch, excluding tombstoned and unchanged items */
 function buildFilesToFetch(
   categoryFiles: StorageFile[],
-  categoryTombstones: Record<string, Tombstone>
+  categoryTombstones: Record<string, Tombstone>,
+  localShas?: Record<string, string>
 ): FetchInfo[] {
   const filesToFetch: FetchInfo[] = [];
+  let skipped = 0;
   for (const file of categoryFiles) {
     const itemId = getItemIdFromFilename(file.filename);
     if (!itemId) continue;
     if (itemId in categoryTombstones) continue;
+    // Skip if local has same SHA (unchanged)
+    if (localShas && file.sha && localShas[itemId] === file.sha) {
+      skipped++;
+      continue;
+    }
     filesToFetch.push({
       itemId,
       filename: file.filename,
       checksum: file.sha ?? '',
     });
   }
+  if (skipped > 0) {
+    syncLog(`[PULL] Skipped ${String(skipped)} unchanged files`);
+  }
   return filesToFetch;
 }
 
@@ -127,8 +146,27 @@ function processDownloadedFiles(
   return { items, checksums };
 }
 
+/** Build SHA map from fetched files */
+function buildShaMap(filesToFetch: FetchInfo[]): Record<string, string> {
+  const shas: Record<string, string> = {};
+  for (const f of filesToFetch) {
+    if (f.checksum) shas[f.itemId] = f.checksum;
+  }
+  return shas;
+}
+
+/** Load tombstones for a category */
+async function loadCategoryTombstones(
+  backend: StorageBackend,
+  cat: SyncCategory
+): Promise<Record<string, Tombstone>> {
+  const content = await backend.getFile(TOMBSTONES_FILENAME);
+  const file = parseTombstonesFile(content);
+  return getCategoryTombstones(file, cat);
+}
+
 export async function pullCategories(options: PullOptions): Promise<ExtendedPullResult> {
-  const { manifest, enabledCategories, backend, localChecksums } = options;
+  const { manifest, enabledCategories, backend, localRemoteShas } = options;
   const acc = createPullAccumulator();
 
   for (const [category, info] of Object.entries(manifest.categories)) {
@@ -138,7 +176,7 @@ export async function pullCategories(options: PullOptions): Promise<ExtendedPull
       continue;
     }
     syncLog(`[PULL] Processing ${cat} (type: ${info.type})`);
-    await pullTreeIndexedCategory(cat, info, localChecksums?.[cat] ?? {}, backend, acc);
+    await pullTreeIndexedCategory(cat, info, localRemoteShas?.[cat], backend, acc);
   }
 
   return acc;
@@ -150,7 +188,7 @@ export async function pullCategories(options: PullOptions): Promise<ExtendedPull
 async function pullTreeIndexedCategory(
   cat: SyncCategory,
   info: TreeIndexedCategoryInfo,
-  _localChecksums: Record<string, string>,
+  localShas: Record<string, string> | undefined,
   backend: StorageBackend,
   acc: PullAccumulator
 ): Promise<void> {
@@ -160,33 +198,28 @@ async function pullTreeIndexedCategory(
   const allFiles = await backend.listFiles();
   const categoryFiles = allFiles.filter((f) => f.filename.startsWith(info.pathPrefix));
   syncLog(`[PULL] ${cat}: found ${String(categoryFiles.length)} files in tree`);
-
   if (categoryFiles.length === 0) return;
 
-  // Load tombstones from separate file
-  const tombstonesContent = await backend.getFile(TOMBSTONES_FILENAME);
-  const tombstonesFile = parseTombstonesFile(tombstonesContent);
-  const categoryTombstones = getCategoryTombstones(tombstonesFile, cat);
+  // Load tombstones and build file list
+  const categoryTombstones = await loadCategoryTombstones(backend, cat);
   syncLog(`[PULL] ${cat}: ${String(Object.keys(categoryTombstones).length)} tombstones`);
-
-  // Build list of files to download (excluding tombstoned items)
-  const filesToFetch = buildFilesToFetch(categoryFiles, categoryTombstones);
+  const filesToFetch = buildFilesToFetch(categoryFiles, categoryTombstones, localShas);
+  syncLog(`[PULL] ${cat}: ${String(filesToFetch.length)} files to fetch`);
 
   if (filesToFetch.length === 0) {
-    if (Object.keys(categoryTombstones).length > 0) {
-      acc.tombstonedItems[cat] = categoryTombstones;
-    }
+    if (Object.keys(categoryTombstones).length > 0) acc.tombstonedItems[cat] = categoryTombstones;
     return;
   }
 
   // Bulk download and process files
+  syncLog(`[PULL] ${cat}: starting bulk download...`);
   const contents = await backend.getFiles(filesToFetch.map((f) => f.filename));
   const { items, checksums } = processDownloadedFiles(filesToFetch, contents);
   syncLog(
     `[PULL] ${cat}: downloaded ${String(Object.keys(items).length)}/${String(filesToFetch.length)} items`
   );
 
-  // Record results
+  // Record results with SHAs for incremental sync
   const data: ItemCategoryData = { category: cat, type: 'items', items, checksums };
-  recordItemPull(acc, cat, data, categoryTombstones);
+  recordItemPull(acc, cat, data, categoryTombstones, buildShaMap(filesToFetch));
 }

src/sync/operations/push.ts

@@ -1,6 +1,6 @@
 /** Push Operation - Pushes local data to remote storage. */
 
-import { packItem, buildItemInfo, diffItems } from '../item-packer.js';
+import { packItem, buildItemInfo, diffItems, getItemFilename } from '../item-packer.js';
 import {
   processTombstonesForPush,
   removeItemsById,
@@ -45,11 +45,17 @@ export function preparePushData(opts: PreparePushOptions): PreparePushResult {
   const newFiles = new Set<string>();
   const changedCategories: SyncCategory[] = [];
 
+  // Get previously synced checksums to detect what actually changed
+  const lastSyncedChecksums = localState?.itemChecksums;
+
   for (const catData of localData) {
     if (!config.sync[catData.category]) continue;
-    const filenames = packCategoryData(catData, remoteManifest, ctx);
+    const prevChecksums = lastSyncedChecksums?.[catData.category];
+    const filenames = packCategoryData(catData, remoteManifest, ctx, prevChecksums);
     for (const f of filenames) newFiles.add(f);
-    changedCategories.push(catData.category);
+    if (filenames.length > 0) {
+      changedCategories.push(catData.category);
+    }
   }
 
   markOrphanedFiles(ctx.files, existingFiles, newFiles);
@@ -61,10 +67,23 @@ export function preparePushData(opts: PreparePushOptions): PreparePushResult {
 function packCategoryData(
   catData: CategoryData,
   _remoteManifest: Manifest | undefined,
-  ctx: PushContext
+  ctx: PushContext,
+  prevChecksums?: Record<string, string>
 ): string[] {
-  // All categories now use tree-indexed sync, no remote item tracking needed
-  return packItemCategoryData(catData, {}, ctx, {});
+  // Use previous checksums to build pseudo ItemInfo for diff comparison
+  const remoteItems: Record<string, ItemInfo> = {};
+  if (prevChecksums) {
+    for (const [itemId, checksum] of Object.entries(prevChecksums)) {
+      remoteItems[itemId] = {
+        filename: getItemFilename(catData.category, itemId),
+        checksum,
+        size: 0,
+        lastModified: '',
+        lastModifiedBy: '',
+      };
+    }
+  }
+  return packItemCategoryData(catData, remoteItems, ctx, {});
 }
 
 /** Result of processing items for push */

src/sync/operations/types.ts

@@ -71,6 +71,8 @@ export interface PreparePushOptions {
   passphrase: PassphraseOption;
   existingFiles?: string[];
   remoteManifest?: Manifest;
+  /** Remote file SHAs for incremental push (filename -> git blob SHA) */
+  remoteShas?: Record<string, string>;
 }
 
 export { type SyncResult, type Manifest, type SyncCategory, type LocalSyncState };

src/types/sync.ts

@@ -18,6 +18,8 @@ export interface LocalSyncState {
   baseVersions: Partial<Record<SyncCategory, string>>; // JSON stringified data
   // Per-item tracking for deletion detection (sessions, messages)
   itemChecksums?: Partial<Record<SyncCategory, Record<string, string>>>;
+  // Remote git blob SHAs for incremental pull (skip unchanged files)
+  remoteShas?: Partial<Record<SyncCategory, Record<string, string>>>;
 }
 
 // ============================================================================