feat(sharding): simplify sharding logic by always using sharded categories and removing inline category writing

Author: ErcinDedeoglu

src/sync/operations/push.ts

@@ -9,7 +9,7 @@ import {
   markOrphanedFiles,
   buildPushContext,
 } from './helpers.js';
-import { shouldShard, writeShardedCategory, writeInlineItemCategory } from './sharding.js';
+import { writeShardedCategory } from './sharding.js';
 import {
   type ItemCategoryInfo,
   type ItemInfo,
@@ -169,14 +169,9 @@ function packItemCategoryData(
     ctx.files[filename] = { content: null };
   }
   const newItems = removeItemsById(processed, tombResult.itemsToRemove);
-  const totalEntries = Object.keys(newItems).length + Object.keys(tombResult.tombstones).length;
 
-  // Use sharding if item count exceeds threshold
-  if (shouldShard(totalEntries)) {
-    writeShardedCategory(category, newItems, tombResult.tombstones, ctx);
-  } else {
-    writeInlineItemCategory(category, newItems, tombResult.tombstones, ctx);
-  }
+  // Always use sharding for item categories
+  writeShardedCategory(category, newItems, tombResult.tombstones, ctx);
 
   return filenames;
 }

src/sync/operations/sharding.ts

@@ -6,36 +6,14 @@
 
 import { calculateChecksum } from '../packer.js';
 import {
-  type ItemCategoryInfo,
   type ItemInfo,
   type CategoryShard,
   type ShardedCategoryRef,
   getShardFilename,
-  shouldShard,
 } from '../../types/index.js';
 import type { Tombstone } from '../../types/manifest.js';
 import type { PushContext, SyncCategory } from './types.js';
 
-export { shouldShard };
-
-/** Write inline item category (not sharded) to manifest */
-export function writeInlineItemCategory(
-  category: SyncCategory,
-  items: Record<string, ItemInfo>,
-  tombstones: Record<string, Tombstone>,
-  ctx: PushContext
-): void {
-  ctx.manifest.categories[category] = {
-    type: 'items',
-    items,
-    tombstones,
-    itemCount: Object.keys(items).length,
-    lastModified: ctx.now,
-    lastModifiedBy: ctx.machineId,
-    vectorClock: { [ctx.machineId]: ctx.newClock[ctx.machineId] ?? 1 },
-  } satisfies ItemCategoryInfo;
-}
-
 /**
  * Write a sharded category (items stored in separate shard file).
  */

src/types/index.ts

@@ -41,8 +41,6 @@ export {
   // Sharded manifest helpers
   isShardedRef,
   getShardFilename,
-  SHARDING_THRESHOLD,
-  shouldShard,
 } from './manifest.js';
 
 // Sync Operations

src/types/manifest.ts

@@ -125,14 +125,6 @@ export function getShardFilename(category: SyncCategory): string {
   return `manifest-${category}.json`;
 }
 
-/** Categories that should use sharding when item count exceeds threshold */
-export const SHARDING_THRESHOLD = 100;
-
-/** Check if a category should be sharded based on item count */
-export function shouldShard(itemCount: number): boolean {
-  return itemCount >= SHARDING_THRESHOLD;
-}
-
 /** Content of a category shard file */
 export interface CategoryShard {
   /** Category this shard belongs to */