Crawl: keep the skills of a failing source, restore the snapshot

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## Unreleased
 
+- **A source that fails no longer drops its skills.** The crawl keeps what that source listed in the previous snapshot. The snapshot of 2026-11-10, built while claude-plugins.dev answered HTTP 402, had lost 35,750 skills; the one of 2026-11-04 is restored.
+
 ## 0.30.0 — 2026-10-23
 
 - **Update checks say when they are incomplete.** A repository GitHub does not answer for (HTTP 403 rate limit, timeout, invalid answer), a truncated tree or a skill folder it no longer has is reported instead of read as up to date: the tree shows a warning, `outdated` and `update` name each on stderr and exit with 1, and `update` still runs the updates it confirmed. `outdated --json` keeps its array on stdout. A skill installed at the repository root is now checked too.

README.md

@@ -348,7 +348,7 @@ The snapshot is built from:
 
 **Rate limits.** The skills.sh search API allows 30 requests per minute. The crawl queries it at that pace, about 2 s per owner. The scheduled workflow covers the top 600 owners; a local crawl covers none by default. With `VERCEL_OIDC_TOKEN`, the crawl uses the [official API](https://skills.sh/docs/api) instead, which returns the full leaderboard.
 
-New snapshots retain collection status per source. If a source was unavailable during collection, the terminal and web interface warn that the catalog is partial. `packs --json` also includes `available`, `total` and `missing` fields. Older snapshots without source diagnostics remain readable; their source health is unknown. No extra source checks run while browsing.
+New snapshots retain collection status per source. If a source was unavailable during collection, the terminal and web interface warn that the catalog is partial; the snapshot keeps the skills that source listed in the previous one (`carried` in the source's status), so that a registry being down does not drop them, with fresh install counts for those another source knows too. `packs --json` also includes `available`, `total` and `missing` fields. Older snapshots without source diagnostics remain readable; their source health is unknown. No extra source checks run while browsing.
 
 **Processing:**
 

scripts/crawl.ts

@@ -39,9 +39,10 @@ const started = Date.now();
 const log = (msg: string) => console.error(`[${((Date.now() - started) / 1000).toFixed(0)}s] ${msg}`);
 
 const token = githubToken();
-const catalog = await buildCatalog({ maxSkillsShOwners, enrich, githubToken: token, log });
-// The snapshot about to be replaced is the previous crawl's: growth is measured against it.
+// The snapshot about to be replaced is the previous crawl's: growth is measured against it,
+// and a source that fails keeps the skills it had there.
 const previous = await loadCatalog(out).catch(() => undefined);
+const catalog = await buildCatalog({ maxSkillsShOwners, enrich, githubToken: token, previous, log });
 catalog.stats.growthSince = markGrowth(catalog.skills, previous);
 log(`growth: ${catalog.skills.filter((s) => s.growth).length} skills gained installs since ${catalog.stats.growthSince ?? 'nothing (no previous snapshot)'}`);
 if (requires) {

src/catalog/build.ts

@@ -13,9 +13,28 @@ export interface BuildOptions {
   /** How many popular skills without description to complete from GitHub (0 = none). */
   enrich?: number;
   githubToken?: string;
+  /** The snapshot being replaced: a source that fails keeps the skills it had there. */
+  previous?: Catalog;
   log?: (msg: string) => void;
 }
 
+/**
+ * What a source listed in the previous snapshot, as raw entries. A skill only that
+ * source knew keeps its counts; one other sources know too takes theirs, which are fresh.
+ */
+export function carriedOver(previous: Catalog, origin: Origin): RawSkill[] {
+  return previous.skills
+    .filter((s) => s.origins.includes(origin))
+    .map((s) => {
+      const alone = s.origins.length === 1;
+      return {
+        origin, name: s.name, source: s.source, path: s.path, description: s.description,
+        ...(alone ? { installs: s.installs, stars: s.stars, labels: s.labels } : {}),
+        ...(origin === 'skills.sh' ? { rank: s.rank } : {}),
+      };
+    });
+}
+
 /**
  * Classifies and sorts merged skills, flags the official ones when the list
  * of official owners is known, then wraps them with stats.
@@ -51,8 +70,9 @@ export function assemble(
 }
 
 /**
- * Crawls every source. A failing source is logged and skipped so that one
- * registry being down does not block the snapshot; all of them failing, or no skill at all, does.
+ * Crawls every source. A failing source is logged and does not block the snapshot:
+ * it keeps the skills it listed in the previous one, so that a registry being down
+ * for a day does not drop half the catalog. All of them failing, or no skill at all, does block.
  */
 export async function buildCatalog(opts: BuildOptions = {}): Promise<Catalog> {
   const log = opts.log ?? (() => {});
@@ -83,6 +103,12 @@ export async function buildCatalog(opts: BuildOptions = {}): Promise<Catalog> {
     }
   });
   if (settled.every((r) => r.status === 'rejected')) throw new Error('every catalog source failed');
+  for (const origin of origins) {
+    if (sources[origin].status !== 'error' || !opts.previous) continue;
+    raw[origin] = carriedOver(opts.previous, origin);
+    sources[origin].carried = raw[origin].length;
+    log(`${origin}: ${raw[origin].length} entries kept from the snapshot of ${opts.previous.generatedAt.slice(0, 10)}`);
+  }
   let merged = dedupe(Object.values(raw).flat());
   if (opts.enrich) {
     const { filled, missing } = await enrichDescriptions(merged, { max: opts.enrich, githubToken: opts.githubToken, log });

src/types.ts

@@ -66,7 +66,8 @@ export interface Catalog {
   skills: Skill[];
   stats: {
     /** Absent in older snapshots, whose source health is unknown. */
-    sources?: Record<Origin, { status: 'ok' | 'error'; count: number; error?: string }>;
+    /** `carried`: entries of a failed source kept from the previous snapshot, as old as that source's last good crawl. */
+    sources?: Record<Origin, { status: 'ok' | 'error'; count: number; error?: string; carried?: number }>;
     raw: Record<Origin, number>;
     merged: number;
     /** Skills flagged `official` (absent from snapshots older than the flag). */

test/catalog-health.test.ts

@@ -28,6 +28,26 @@ describe('catalog source health', () => {
     });
   });
 
+  it('keeps what a failed source listed in the previous snapshot, with fresh counts from the sources that answered', async () => {
+    const previous = await buildCatalog();
+    previous.generatedAt = '2026-11-04T00:00:00.000Z';
+    previous.skills = [
+      { id: 'anthropics/skills@pdf', name: 'pdf', source: 'anthropics/skills', installs: 5, labels: [], origins: ['skills.sh', 'claude-plugins.dev'], themes: [] },
+      { id: 'o/r@only-there', name: 'only-there', source: 'o/r', path: 'skills/only-there', installs: 7, stars: 2, labels: [], origins: ['claude-plugins.dev'], themes: [] },
+      { id: 'o/r@elsewhere', name: 'elsewhere', source: 'o/r', installs: 9, labels: [], origins: ['smithery'], themes: [] },
+    ];
+    vi.mocked(fetchClaudePluginsDev).mockRejectedValue(new Error('HTTP 402'));
+    const catalog = await buildCatalog({ previous });
+    expect(catalog.stats.sources?.['claude-plugins.dev']).toEqual({ status: 'error', count: 0, error: 'HTTP 402', carried: 2 });
+    expect(catalog.skills.map((s) => [s.id, s.installs, s.origins])).toEqual([
+      ['anthropics/skills@pdf', 10, ['skills.sh', 'claude-plugins.dev']],
+      ['o/r@only-there', 7, ['claude-plugins.dev']],
+    ]);
+    expect(catalog.skills[1]).toMatchObject({ path: 'skills/only-there', stars: 2 });
+    // Without a previous snapshot the source is skipped, as before.
+    expect((await buildCatalog()).skills.map((s) => s.id)).toEqual(['anthropics/skills@pdf']);
+  });
+
   it('bounds external diagnostic text and still refuses a completely failed collection', async () => {
     vi.mocked(fetchSmithery).mockRejectedValue(new Error('x'.repeat(1000)));
     expect((await buildCatalog()).stats.sources?.smithery.error).toHaveLength(300);