From 4a9a8db745d81a0c5a726d0746e112b695dd5708 Mon Sep 17 00:00:00 2001 From: Luke Parke <5702154+LukasParke@users.noreply.github.com> Date: Fri, 14 Aug 2026 13:59:32 -0500 Subject: [PATCH 1/3] feat: add package stats adapters and card --- README.md | 4 +- ROADMAP.md | 1 + packages/diffler/README.md | 14 ++ packages/diffler/src/cli.ts | 16 ++ packages/diffler/src/config.ts | 21 +++ packages/diffler/src/index.ts | 7 + packages/diffler/src/packages/index.ts | 81 ++++++++ packages/diffler/src/packages/npm.ts | 177 ++++++++++++++++++ packages/diffler/src/packages/types.ts | 20 ++ packages/diffler/src/stats/index.ts | 6 + packages/diffler/src/stats/output.ts | 20 ++ packages/diffler/src/stats/types.ts | 9 + packages/diffler/tests/config.test.ts | 17 ++ packages/diffler/tests/packages/npm.test.ts | 146 +++++++++++++++ packages/diffler/tests/stats/index.test.ts | 1 + packages/remotion/README.md | 1 + packages/remotion/scripts/render-assets.mjs | 1 + .../remotion/src/cards/PackageImpactCard.tsx | 84 +++++++++ packages/remotion/src/cards/index.ts | 7 + packages/remotion/src/data/adapter.test.ts | 55 ++++++ packages/remotion/src/data/adapter.ts | 43 +++++ packages/remotion/src/data/defaultStats.ts | 14 ++ packages/remotion/src/data/fetch.ts | 35 ++++ packages/schemas/src/v2.ts | 27 +++ packages/schemas/src/zod.ts | 25 +++ 25 files changed, 830 insertions(+), 2 deletions(-) create mode 100644 packages/diffler/src/packages/index.ts create mode 100644 packages/diffler/src/packages/npm.ts create mode 100644 packages/diffler/src/packages/types.ts create mode 100644 packages/diffler/tests/packages/npm.test.ts create mode 100644 packages/remotion/src/cards/PackageImpactCard.tsx diff --git a/README.md b/README.md index 5e49b56..2202128 100644 --- a/README.md +++ b/README.md @@ -50,8 +50,8 @@ pnpm --filter @lukasparke/diffler-remotion test └─────────────────┘ └──────────────────┘ └─────────────────┘ ``` -1. **diffler** collects comprehensive GitHub stats via GraphQL + REST, with caching, rate-limit budgeting, and resumable backfill -2. The collected stats are written as JSON conforming to the v2 schema +1. **diffler** collects comprehensive GitHub stats via GraphQL + REST and package install stats through registry adapters +2. Provider-specific package data is normalized alongside GitHub data in the v2 JSON schema 3. **@lukasparke/diffler-remotion** reads the JSON and renders animated profile cards ## GitHub Actions diff --git a/ROADMAP.md b/ROADMAP.md index d8e55aa..190bbd1 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -19,6 +19,7 @@ rendering. ## Next +- Add a PyPI package-stats adapter using the normalized package metrics seam. - Add a consumer fixture that exercises the released GitHub Action. - Consolidate the three Remotion export commands. - Add image-link validation to the profile repository. diff --git a/packages/diffler/README.md b/packages/diffler/README.md index 4f01091..16640c5 100644 --- a/packages/diffler/README.md +++ b/packages/diffler/README.md @@ -166,6 +166,13 @@ statsAction: includePrivateRepositoryMetrics: true includePrivateRepositoryDetails: false includePrivateCacheDetails: false + # Package names are explicit so only packages you own or maintain are shown. + # Every provider is normalized through the package stats adapter interface. + packageSources: + - provider: "npm" + packages: + - "@octocat/example" + - "another-package" ``` Private aggregate collection requires a token that can read the intended private @@ -175,6 +182,13 @@ repository secret. You can also enable this mode with `STATS_INCLUDE_PRIVATE_REPOSITORY_METRICS=true` or `diffler collect --include-private-metrics`. +Package stats currently support npm download totals for the last day, week, +month, year, and all time, plus the latest version and publish date. Configure +npm from YAML as above, with `STATS_NPM_PACKAGES` as a comma-separated list, or +with `diffler collect --npm-packages "@octocat/example,another-package"`. +The output and cards use provider-neutral package metrics so future registries +such as PyPI can be added as adapters without changing consumers. + --- ## 🤖 GitHub Actions diff --git a/packages/diffler/src/cli.ts b/packages/diffler/src/cli.ts index abe68f8..6cee5ac 100644 --- a/packages/diffler/src/cli.ts +++ b/packages/diffler/src/cli.ts @@ -99,6 +99,10 @@ program "--include-private-metrics", "Include anonymous private repository metrics without repository details" ) + .option( + "--npm-packages ", + "Comma-separated npm package names to include in package stats" + ) .option("--include-private", "Include private repository details") .action(async (options) => { const token = process.env.GITHUB_TOKEN; @@ -126,6 +130,18 @@ program statsConfig.includePrivateRepositoryDetails = true; statsConfig.includePrivateCacheDetails = true; } + if (options.npmPackages) { + statsConfig.packageSources = [ + ...statsConfig.packageSources.filter((source) => source.provider !== "npm"), + { + provider: "npm", + packages: options.npmPackages + .split(",") + .map((packageName: string) => packageName.trim()) + .filter(Boolean), + }, + ]; + } const client = new GitHubClient(config.github); await runStatsCollection(statsConfig, client); diff --git a/packages/diffler/src/config.ts b/packages/diffler/src/config.ts index ac0a636..119a3c1 100644 --- a/packages/diffler/src/config.ts +++ b/packages/diffler/src/config.ts @@ -84,6 +84,14 @@ export const StatsActionConfigSchema = z.object({ includePrivateRepositoryDetails: z.boolean().default(false), includePrivateCacheDetails: z.boolean().default(false), backfillMode: z.enum(["resume", "refresh", "off"]).default("resume"), + packageSources: z + .array( + z.object({ + provider: z.string().min(1), + packages: z.array(z.string().min(1)), + }) + ) + .default([]), }); export type StatsActionConfig = z.infer; @@ -167,6 +175,19 @@ export function buildStatsActionConfig(config: DifflerConfig): StatsActionConfig if (backfillMode === "resume" || backfillMode === "refresh" || backfillMode === "off") { base.backfillMode = backfillMode; } + const npmPackages = env("npm-packages", "") + .split(",") + .map((packageName) => packageName.trim()) + .filter(Boolean); + if (npmPackages.length > 0) { + const configuredSources = base.packageSources.filter( + (source) => source.provider !== "npm" + ); + base.packageSources = [ + ...configuredSources, + { provider: "npm", packages: npmPackages }, + ]; + } return base; } diff --git a/packages/diffler/src/index.ts b/packages/diffler/src/index.ts index 2242674..c762ba8 100644 --- a/packages/diffler/src/index.ts +++ b/packages/diffler/src/index.ts @@ -2,3 +2,10 @@ export { DifflerConfig, GitHubConfig, GitHubProfileConfig } from "./config.js"; export { Engine } from "./core/engine.js"; export { Renderer } from "./core/renderer.js"; export { ContextBuilder } from "./core/context.js"; +export { + collectPackageStats, + NpmPackageStatsAdapter, + type PackageStatsAdapter, + type PackageStatsAdapterResult, + type PackageStatsCollection, +} from "./packages/index.js"; diff --git a/packages/diffler/src/packages/index.ts b/packages/diffler/src/packages/index.ts new file mode 100644 index 0000000..51c1d8a --- /dev/null +++ b/packages/diffler/src/packages/index.ts @@ -0,0 +1,81 @@ +import type { + PackageDownloadCounts, + PackageMetric, + PackageSourceConfig, + PackageStatsAdapter, + PackageStatsCollection, +} from "./types.js"; +import { NpmPackageStatsAdapter } from "./npm.js"; + +const EMPTY_DOWNLOADS: PackageDownloadCounts = { + lastDay: 0, + lastWeek: 0, + lastMonth: 0, + lastYear: 0, + allTime: 0, +}; + +export async function collectPackageStats( + sources: PackageSourceConfig[], + adapters: PackageStatsAdapter[] = [new NpmPackageStatsAdapter()] +): Promise { + const adaptersByProvider = new Map( + adapters.map((adapter) => [adapter.provider, adapter]) + ); + const packages: PackageMetric[] = []; + const warnings: string[] = []; + + for (const source of sources) { + const adapter = adaptersByProvider.get(source.provider); + if (!adapter) { + warnings.push(`No package stats adapter registered for "${source.provider}"`); + continue; + } + + try { + const result = await adapter.collect([...new Set(source.packages)]); + packages.push(...result.packages); + warnings.push(...result.warnings); + } catch (error) { + warnings.push( + `${source.provider} package stats unavailable: ${ + error instanceof Error ? error.message : String(error) + }` + ); + } + } + + return { + packageCount: packages.length, + providers: [...new Set(packages.map((item) => item.provider))], + downloads: sumDownloads(packages), + packages: packages.sort( + (left, right) => right.downloads.lastMonth - left.downloads.lastMonth + ), + complete: warnings.length === 0, + warnings, + }; +} + +function sumDownloads(packages: PackageMetric[]): PackageDownloadCounts { + return packages.reduce( + (total, item) => ({ + lastDay: total.lastDay + item.downloads.lastDay, + lastWeek: total.lastWeek + item.downloads.lastWeek, + lastMonth: total.lastMonth + item.downloads.lastMonth, + lastYear: total.lastYear + item.downloads.lastYear, + allTime: total.allTime + item.downloads.allTime, + }), + { ...EMPTY_DOWNLOADS } + ); +} + +export { NpmPackageStatsAdapter } from "./npm.js"; +export type { + PackageDownloadCounts, + PackageMetric, + PackageSourceConfig, + PackageStatsAdapter, + PackageStatsAdapterResult, + PackageStatsCollection, +} from "./types.js"; diff --git a/packages/diffler/src/packages/npm.ts b/packages/diffler/src/packages/npm.ts new file mode 100644 index 0000000..7d0e9ec --- /dev/null +++ b/packages/diffler/src/packages/npm.ts @@ -0,0 +1,177 @@ +import type { + PackageDownloadCounts, + PackageMetric, + PackageStatsAdapter, + PackageStatsAdapterResult, +} from "./types.js"; + +type FetchResponse = { + ok: boolean; + status: number; + json(): Promise; +}; + +type Fetcher = (url: string) => Promise; + +const DOWNLOAD_PERIODS: Array<[keyof PackageDownloadCounts, string]> = [ + ["lastDay", "last-day"], + ["lastWeek", "last-week"], + ["lastMonth", "last-month"], + ["lastYear", "last-year"], +]; +const NPM_DOWNLOAD_DATA_START = new Date("2015-01-10T00:00:00.000Z"); + +export class NpmPackageStatsAdapter implements PackageStatsAdapter { + readonly provider = "npm"; + + constructor( + private readonly fetcher: Fetcher = fetch, + private readonly now: () => Date = () => new Date() + ) {} + + async collect(packageNames: string[]): Promise { + const results = await Promise.allSettled( + packageNames.map((packageName) => this.collectPackage(packageName)) + ); + const packages: PackageMetric[] = []; + const warnings: string[] = []; + + results.forEach((result, index) => { + if (result.status === "fulfilled") { + packages.push(result.value); + } else { + warnings.push( + `npm package "${packageNames[index]}" unavailable: ${ + result.reason instanceof Error ? result.reason.message : String(result.reason) + }` + ); + } + }); + + return { packages, warnings }; + } + + private async collectPackage(packageName: string): Promise { + const encodedName = encodeURIComponent(packageName); + const metadata = await fetchJson( + this.fetcher, + `https://registry.npmjs.org/${encodedName}` + ); + const metadataRecord = asRecord(metadata); + const createdAt = parseDate(asRecord(metadataRecord.time).created); + const [downloadCounts, allTime] = await Promise.all([ + Promise.all( + DOWNLOAD_PERIODS.map(([, period]) => + fetchJson( + this.fetcher, + `https://api.npmjs.org/downloads/point/${period}/${encodedName}` + ) + ) + ), + this.collectAllTimeDownloads(encodedName, createdAt), + ]); + const latestVersion = asNullableString(asRecord(metadataRecord["dist-tags"]).latest); + const publishedAt = latestVersion + ? asNullableString(asRecord(metadataRecord.time)[latestVersion]) + : null; + const downloads = { + ...Object.fromEntries( + DOWNLOAD_PERIODS.map(([key], index) => [ + key, + asNonNegativeNumber(asRecord(downloadCounts[index]).downloads), + ]) + ), + allTime, + } as PackageDownloadCounts; + + return { + provider: this.provider, + name: packageName, + url: `https://www.npmjs.com/package/${encodedName}`, + latestVersion, + latestPublishedAt: publishedAt, + downloads, + }; + } + + private async collectAllTimeDownloads( + encodedName: string, + createdAt: Date | null + ): Promise { + const yesterday = new Date(this.now()); + yesterday.setUTCHours(0, 0, 0, 0); + yesterday.setUTCDate(yesterday.getUTCDate() - 1); + const firstDay = + createdAt && createdAt > NPM_DOWNLOAD_DATA_START + ? startOfUtcDay(createdAt) + : NPM_DOWNLOAD_DATA_START; + if (firstDay > yesterday) return 0; + + const ranges = buildDownloadRanges(firstDay, yesterday); + const results = await Promise.all( + ranges.map((range) => + fetchJson( + this.fetcher, + `https://api.npmjs.org/downloads/point/${range}/${encodedName}` + ) + ) + ); + return results.reduce( + (total, result) => total + asNonNegativeNumber(asRecord(result).downloads), + 0 + ); + } +} + +function buildDownloadRanges(firstDay: Date, lastDay: Date): string[] { + const ranges: string[] = []; + let start = new Date(firstDay); + + while (start <= lastDay) { + const end = new Date( + Math.min(start.getTime() + 539 * 86_400_000, lastDay.getTime()) + ); + ranges.push(`${formatDate(start)}:${formatDate(end)}`); + start = new Date(end.getTime() + 86_400_000); + } + + return ranges; +} + +function parseDate(value: unknown): Date | null { + if (typeof value !== "string") return null; + const date = new Date(value); + return Number.isNaN(date.getTime()) ? null : date; +} + +function startOfUtcDay(value: Date): Date { + return new Date( + Date.UTC(value.getUTCFullYear(), value.getUTCMonth(), value.getUTCDate()) + ); +} + +function formatDate(value: Date): string { + return value.toISOString().slice(0, 10); +} + +async function fetchJson(fetcher: Fetcher, url: string): Promise { + const response = await fetcher(url); + if (!response.ok) { + throw new Error(`npm request failed with HTTP ${response.status}: ${url}`); + } + return response.json(); +} + +function asRecord(value: unknown): Record { + return value && typeof value === "object" + ? (value as Record) + : {}; +} + +function asNullableString(value: unknown): string | null { + return typeof value === "string" && value.length > 0 ? value : null; +} + +function asNonNegativeNumber(value: unknown): number { + return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : 0; +} diff --git a/packages/diffler/src/packages/types.ts b/packages/diffler/src/packages/types.ts new file mode 100644 index 0000000..eec6c84 --- /dev/null +++ b/packages/diffler/src/packages/types.ts @@ -0,0 +1,20 @@ +import type { + PackageDownloadCounts, + PackageMetric, + PackageMetrics, +} from "@lukasparke/diffler-schemas"; +import type { PackageSourceConfig } from "../stats/types.js"; + +export type PackageStatsAdapter = { + readonly provider: string; + collect(packageNames: string[]): Promise; +}; + +export type PackageStatsAdapterResult = { + packages: PackageMetric[]; + warnings: string[]; +}; + +export type PackageStatsCollection = PackageMetrics; + +export type { PackageDownloadCounts, PackageMetric, PackageSourceConfig }; diff --git a/packages/diffler/src/stats/index.ts b/packages/diffler/src/stats/index.ts index 60873f0..a62b690 100644 --- a/packages/diffler/src/stats/index.ts +++ b/packages/diffler/src/stats/index.ts @@ -30,6 +30,7 @@ import { import { buildOutput } from "./output.js"; import { RequestScheduler } from "./scheduler.js"; import type { GitHubClient } from "../github/client.js"; +import { collectPackageStats } from "../packages/index.js"; export { formatBytes, @@ -52,6 +53,8 @@ export async function runStatsCollection( const volatileCache = readVolatileCache(config.volatileCachePath); const warnings: string[] = []; const errors: string[] = []; + console.log("Collecting configured package registry stats"); + const packageMetricsPromise = collectPackageStats(config.packageSources); console.log("Collecting viewer profile and activity counts"); const { profile, activity } = await collectProfile(client, scheduler); @@ -120,6 +123,8 @@ export async function runStatsCollection( ...contributions.repositories, ] ); + const packageMetrics = await packageMetricsPromise; + warnings.push(...packageMetrics.warnings); const finishedAt = Date.now(); const schedulerState = scheduler.state(); @@ -164,6 +169,7 @@ export async function runStatsCollection( config, collectionStatus, fetchedAt: finishedAt, + packageMetrics, }); writeJsonOutput(config.outputPath, output); diff --git a/packages/diffler/src/stats/output.ts b/packages/diffler/src/stats/output.ts index f59db92..7da3672 100644 --- a/packages/diffler/src/stats/output.ts +++ b/packages/diffler/src/stats/output.ts @@ -3,6 +3,7 @@ import type { CollectionStatus, GitHubStatsOutput, LegacyStats, + PackageMetrics, PresentationData, PrivacyReport, ProfileContributions, @@ -33,6 +34,7 @@ export function buildOutput(params: { config: StatsActionConfig; collectionStatus: CollectionStatus; fetchedAt: number; + packageMetrics?: PackageMetrics; }): GitHubStatsOutput { const includePrivateDetails = params.config.includePrivateRepositoryDetails; const includePrivateMetrics = @@ -278,6 +280,7 @@ export function buildOutput(params: { activity: params.activity, repositories: visibleRepositories, repoMetrics, + packageMetrics: params.packageMetrics ?? emptyPackageMetrics(), presentation, privacy, collectionStatus, @@ -285,6 +288,23 @@ export function buildOutput(params: { }; } +function emptyPackageMetrics(): PackageMetrics { + return { + packageCount: 0, + providers: [], + downloads: { + lastDay: 0, + lastWeek: 0, + lastMonth: 0, + lastYear: 0, + allTime: 0, + }, + packages: [], + complete: true, + warnings: [], + }; +} + function toComputedRepo(repo: RepositoryRecord) { return { ...repo, diff --git a/packages/diffler/src/stats/types.ts b/packages/diffler/src/stats/types.ts index 72190bd..dec320c 100644 --- a/packages/diffler/src/stats/types.ts +++ b/packages/diffler/src/stats/types.ts @@ -22,6 +22,9 @@ export { type RepositoryContributionSummary, type ProfileContributions, type RepoMetrics, + type PackageDownloadCounts, + type PackageMetric, + type PackageMetrics, type PresentationData, type PrivacyReport, type CollectionStatus, @@ -48,6 +51,12 @@ export type StatsActionConfig = { includePrivateRepositoryDetails: boolean; includePrivateCacheDetails: boolean; backfillMode: "resume" | "refresh" | "off"; + packageSources: PackageSourceConfig[]; +}; + +export type PackageSourceConfig = { + provider: string; + packages: string[]; }; export type ContributorStatsSummary = { diff --git a/packages/diffler/tests/config.test.ts b/packages/diffler/tests/config.test.ts index 2c5d4f1..752e12d 100644 --- a/packages/diffler/tests/config.test.ts +++ b/packages/diffler/tests/config.test.ts @@ -31,6 +31,7 @@ describe("config", () => { expect(config.templates.main).toBe("profile.md.j2"); expect(config.cache.enabled).toBe(true); expect(config.statsAction.includePrivateRepositoryMetrics).toBe(false); + expect(config.statsAction.packageSources).toEqual([]); }); }); @@ -165,5 +166,21 @@ describe("config", () => { expect(config.includePrivateRepositoryDetails).toBe(false); expect(config.includePrivateCacheDetails).toBe(false); }); + + it("configures npm package stats from the environment", () => { + process.env.STATS_NPM_PACKAGES = + "@lukasparke/diffler, @lukasparke/diffler-remotion"; + const config = buildStatsActionConfig(loadConfigFromEnv()); + + expect(config.packageSources).toEqual([ + { + provider: "npm", + packages: [ + "@lukasparke/diffler", + "@lukasparke/diffler-remotion", + ], + }, + ]); + }); }); }); diff --git a/packages/diffler/tests/packages/npm.test.ts b/packages/diffler/tests/packages/npm.test.ts new file mode 100644 index 0000000..26fbf96 --- /dev/null +++ b/packages/diffler/tests/packages/npm.test.ts @@ -0,0 +1,146 @@ +import { describe, expect, it } from "vitest"; +import { collectPackageStats, NpmPackageStatsAdapter } from "../../src/packages/index.js"; + +describe("NpmPackageStatsAdapter", () => { + it("normalizes metadata and every supported download window", async () => { + const requests: string[] = []; + const adapter = new NpmPackageStatsAdapter( + async (url) => { + requests.push(url); + if (url.startsWith("https://registry.npmjs.org/")) { + return response({ + "dist-tags": { latest: "1.2.3" }, + time: { + created: "2026-01-01T12:00:00.000Z", + "1.2.3": "2026-08-01T12:00:00.000Z", + }, + }); + } + + const period = url.split("/").at(-2) ?? ""; + return response({ + downloads: period.includes(":") + ? 8_000 + : { + "last-day": 10, + "last-week": 70, + "last-month": 300, + "last-year": 3_650, + }[period], + }); + }, + () => new Date("2026-08-14T12:00:00.000Z") + ); + + const result = await adapter.collect(["@lukasparke/diffler"]); + + expect(result.warnings).toEqual([]); + expect(result.packages).toEqual([ + { + provider: "npm", + name: "@lukasparke/diffler", + url: "https://www.npmjs.com/package/%40lukasparke%2Fdiffler", + latestVersion: "1.2.3", + latestPublishedAt: "2026-08-01T12:00:00.000Z", + downloads: { + lastDay: 10, + lastWeek: 70, + lastMonth: 300, + lastYear: 3_650, + allTime: 8_000, + }, + }, + ]); + expect(requests).toHaveLength(6); + expect(requests.every((url) => url.includes("%40lukasparke%2Fdiffler"))).toBe( + true + ); + }); + + it("keeps successful packages when another package fails", async () => { + const adapter = new NpmPackageStatsAdapter( + async (url) => { + if (url.includes("missing")) { + return response({}, 404); + } + if (url.startsWith("https://registry.npmjs.org/")) { + return response({ + "dist-tags": { latest: "1.0.0" }, + time: { created: "2026-01-01T00:00:00.000Z" }, + }); + } + return response({ downloads: 5 }); + }, + () => new Date("2026-08-14T12:00:00.000Z") + ); + + const result = await adapter.collect(["working", "missing"]); + + expect(result.packages.map((item) => item.name)).toEqual(["working"]); + expect(result.warnings).toHaveLength(1); + expect(result.warnings[0]).toContain('npm package "missing" unavailable'); + }); +}); + +describe("collectPackageStats", () => { + it("deduplicates package names and aggregates normalized providers", async () => { + const requested: string[][] = []; + const result = await collectPackageStats( + [ + { provider: "npm", packages: ["one", "one", "two"] }, + { provider: "pypi", packages: ["future"] }, + ], + [ + { + provider: "npm", + async collect(packageNames) { + requested.push(packageNames); + return { + warnings: [], + packages: packageNames.map((name, index) => ({ + provider: "npm", + name, + url: `https://example.com/${name}`, + latestVersion: "1.0.0", + latestPublishedAt: null, + downloads: { + lastDay: index + 1, + lastWeek: 10, + lastMonth: (index + 1) * 100, + lastYear: 1_000, + allTime: 2_000, + }, + })), + }; + }, + }, + ] + ); + + expect(requested).toEqual([["one", "two"]]); + expect(result.packageCount).toBe(2); + expect(result.providers).toEqual(["npm"]); + expect(result.downloads).toEqual({ + lastDay: 3, + lastWeek: 20, + lastMonth: 300, + lastYear: 2_000, + allTime: 4_000, + }); + expect(result.packages.map((item) => item.name)).toEqual(["two", "one"]); + expect(result.complete).toBe(false); + expect(result.warnings).toEqual([ + 'No package stats adapter registered for "pypi"', + ]); + }); +}); + +function response(body: unknown, status = 200) { + return { + ok: status >= 200 && status < 300, + status, + async json() { + return body; + }, + }; +} diff --git a/packages/diffler/tests/stats/index.test.ts b/packages/diffler/tests/stats/index.test.ts index cb2ad0b..5d6d840 100644 --- a/packages/diffler/tests/stats/index.test.ts +++ b/packages/diffler/tests/stats/index.test.ts @@ -372,6 +372,7 @@ describe("v2 collection helpers", () => { includePrivateRepositoryDetails: false, includePrivateCacheDetails: false, backfillMode: "resume", + packageSources: [], }; const createRepository = ( diff --git a/packages/remotion/README.md b/packages/remotion/README.md index fb1b719..33943e1 100644 --- a/packages/remotion/README.md +++ b/packages/remotion/README.md @@ -19,6 +19,7 @@ Available assets: - `main-stats.webp` - `stats.webp` - `repo-impact.webp` +- `package-impact.webp` - `languages.webp` - `top-languages.webp` - `activity-overview.webp` diff --git a/packages/remotion/scripts/render-assets.mjs b/packages/remotion/scripts/render-assets.mjs index 36a8868..e809e2e 100644 --- a/packages/remotion/scripts/render-assets.mjs +++ b/packages/remotion/scripts/render-assets.mjs @@ -10,6 +10,7 @@ const allCards = [ 'main-stats', 'stats', 'repo-impact', + 'package-impact', 'languages', 'top-languages', 'activity-overview', diff --git a/packages/remotion/src/cards/PackageImpactCard.tsx b/packages/remotion/src/cards/PackageImpactCard.tsx new file mode 100644 index 0000000..056f02f --- /dev/null +++ b/packages/remotion/src/cards/PackageImpactCard.tsx @@ -0,0 +1,84 @@ +import {Box, CalendarDays, Download, PackageOpen} from 'lucide-react'; +import {UserStats} from '../data'; +import {MetricRow, Panel} from '../components/primitives'; +import {defaultTheme} from '../themes/default'; +import {formatCompactNumber} from '../utils'; + +export function PackageImpactCard({userStats}: {userStats: UserStats}) { + const metrics = userStats.packages; + const providerLabel = + metrics.providers.length > 0 + ? metrics.providers.join(' + ') + : 'Package registries'; + + return ( + + {metrics.packageCount === 0 ? ( +
+ Configure package sources to showcase install activity +
+ ) : ( +
+
+ } + label="Last 30 days" + value={formatCompactNumber(metrics.downloads.lastMonth)} + accent={defaultTheme.colors.purple} + /> + } + label="Last year" + value={formatCompactNumber(metrics.downloads.lastYear)} + delay={0.08} + accent={defaultTheme.colors.cyan} + /> + } + label="All-time downloads" + value={formatCompactNumber(metrics.downloads.allTime)} + delay={0.16} + accent={defaultTheme.colors.green} + /> + } + label="Published packages" + value={metrics.packageCount} + delay={0.24} + accent={defaultTheme.colors.yellow} + /> +
+
+

+ Top packages this month +

+ {metrics.packages.slice(0, 3).map((item) => ( +
+
+

{item.name}

+

+ {item.provider} + {item.latestVersion ? ` · v${item.latestVersion}` : ''} +

+
+
+

+ {formatCompactNumber(item.downloads.lastMonth)} +

+

downloads

+
+
+ ))} +
+
+ )} + + ); +} diff --git a/packages/remotion/src/cards/index.ts b/packages/remotion/src/cards/index.ts index 902affe..b63dc9b 100644 --- a/packages/remotion/src/cards/index.ts +++ b/packages/remotion/src/cards/index.ts @@ -6,6 +6,7 @@ import {CommitStreakCard} from './CommitStreakCard'; import {IssueTrackingCard} from './IssueTrackingCard'; import {LanguagesCard} from './LanguagesCard'; import {MainStatsCard} from './MainStatsCard'; +import {PackageImpactCard} from './PackageImpactCard'; import {ReadmeClassicCard, ReadmeSpotlightCard} from './ReadmeCard'; import {RepositoryImpactCard} from './RepositoryImpactCard'; import {StatsCard} from './StatsCard'; @@ -57,6 +58,11 @@ export const cards: CardConfig[] = [ component: RepositoryImpactCard, height: 280, }, + { + id: 'package-impact', + component: PackageImpactCard, + height: 300, + }, { id: 'issue-tracking', component: IssueTrackingCard, @@ -90,6 +96,7 @@ export {CommitStreakCard} from './CommitStreakCard'; export {IssueTrackingCard} from './IssueTrackingCard'; export {LanguagesCard} from './LanguagesCard'; export {MainStatsCard} from './MainStatsCard'; +export {PackageImpactCard} from './PackageImpactCard'; export {ReadmeCard, ReadmeClassicCard, ReadmeSpotlightCard} from './ReadmeCard'; export {RepositoryImpactCard} from './RepositoryImpactCard'; export {StatsCard} from './StatsCard'; diff --git a/packages/remotion/src/data/adapter.test.ts b/packages/remotion/src/data/adapter.test.ts index ee54d86..0495fd8 100644 --- a/packages/remotion/src/data/adapter.test.ts +++ b/packages/remotion/src/data/adapter.test.ts @@ -253,4 +253,59 @@ describe('normalizeGithubStats', () => { expect(stats.repositories.repoViews).toBeNull(); expect(stats.repoViews).toBeNull(); }); + + it('normalizes provider-neutral package metrics', () => { + const stats = normalizeGithubStats( + { + schemaVersion: 2, + generatedAt: '2026-08-14T00:00:00.000Z', + profile: {login: 'octocat'}, + legacy: {contributionStats: {}}, + profileContributions: {contributionCalendar: {weeks: []}}, + presentation: {readmeSummary: {}, timeline: [], cards: [], highlights: []}, + repoMetrics: {contributorStats: {}, traffic: {}, repoStats: {}}, + collectionStatus: {}, + privacy: {}, + activity: {}, + repositories: [], + packageMetrics: { + packageCount: 1, + providers: ['npm'], + downloads: { + lastDay: 4, + lastWeek: 28, + lastMonth: 120, + lastYear: 1440, + allTime: 2000, + }, + packages: [ + { + provider: 'npm', + name: '@example/tool', + url: 'https://npmjs.com/package/example', + latestVersion: '2.0.0', + latestPublishedAt: '2026-08-01T00:00:00.000Z', + downloads: {lastMonth: 120}, + }, + ], + complete: true, + warnings: [], + }, + }, + {allowPrivateRepositoryDetails: false}, + ); + + expect(stats.packages).toMatchObject({ + packageCount: 1, + providers: ['npm'], + downloads: {lastMonth: 120, allTime: 2000}, + complete: true, + }); + expect(stats.packages.packages[0]).toMatchObject({ + provider: 'npm', + name: '@example/tool', + latestVersion: '2.0.0', + downloads: {lastDay: 0, lastMonth: 120}, + }); + }); }); diff --git a/packages/remotion/src/data/adapter.ts b/packages/remotion/src/data/adapter.ts index 5b1dc56..601eb9b 100644 --- a/packages/remotion/src/data/adapter.ts +++ b/packages/remotion/src/data/adapter.ts @@ -32,6 +32,7 @@ function normalizeV2Stats(raw: UnknownRecord): UserStats { const computedStats = asRecord(asRecord(repoMetrics.computedStats)); const contributionStats = asRecord(asRecord(legacy.contributionStats)); const privacy = asRecord(raw.privacy); + const packageMetrics = asRecord(raw.packageMetrics); const collectionStatus = asRecord(raw.collectionStatus); const backfill = asRecord(collectionStatus.backfill); const fetchedAt = asNumber( @@ -278,6 +279,7 @@ function normalizeV2Stats(raw: UnknownRecord): UserStats { forkCount }, topLanguages, + packages: normalizePackageMetrics(packageMetrics), cards: normalizeMetricCards(presentation.cards), highlights: normalizeMetricCards(presentation.highlights), privacy: { @@ -525,6 +527,7 @@ function normalizeLegacyStats(raw: UnknownRecord): UserStats { forkCount: asNumber(raw.forkCount, 0) }, topLanguages, + packages: normalizePackageMetrics({}), cards: [], highlights: [], privacy: { @@ -648,6 +651,46 @@ function normalizeMetricCards(value: unknown): MetricCard[] { .filter((item): item is MetricCard => item !== null); } +function normalizePackageMetrics(value: unknown): UserStats['packages'] { + const metrics = asRecord(value); + const downloads = asRecord(metrics.downloads); + const packages = asArray(metrics.packages) + .map(asRecord) + .map((item) => { + const itemDownloads = asRecord(item.downloads); + return { + provider: asString(item.provider), + name: asString(item.name), + url: asString(item.url), + latestVersion: asNullableString(item.latestVersion), + latestPublishedAt: asNullableString(item.latestPublishedAt), + downloads: { + lastDay: asNumber(itemDownloads.lastDay, 0), + lastWeek: asNumber(itemDownloads.lastWeek, 0), + lastMonth: asNumber(itemDownloads.lastMonth, 0), + lastYear: asNumber(itemDownloads.lastYear, 0), + allTime: asNumber(itemDownloads.allTime, 0) + } + }; + }) + .filter((item) => item.provider && item.name); + + return { + packageCount: asNumber(metrics.packageCount, packages.length), + providers: asStringArray(metrics.providers), + downloads: { + lastDay: asNumber(downloads.lastDay, 0), + lastWeek: asNumber(downloads.lastWeek, 0), + lastMonth: asNumber(downloads.lastMonth, 0), + lastYear: asNumber(downloads.lastYear, 0), + allTime: asNumber(downloads.allTime, 0) + }, + packages, + complete: metrics.complete !== false, + warnings: asStringArray(metrics.warnings) + }; +} + function normalizePeakDay(value: unknown) { const record = asRecord(value); const date = asString(record.date); diff --git a/packages/remotion/src/data/defaultStats.ts b/packages/remotion/src/data/defaultStats.ts index b901ef7..f81df6b 100644 --- a/packages/remotion/src/data/defaultStats.ts +++ b/packages/remotion/src/data/defaultStats.ts @@ -106,6 +106,20 @@ export const defaultStats: UserStats = { percentage: 7.4, }, ], + packages: { + packageCount: 0, + providers: [], + downloads: { + lastDay: 0, + lastWeek: 0, + lastMonth: 0, + lastYear: 0, + allTime: 0, + }, + packages: [], + complete: true, + warnings: [], + }, cards: [], highlights: [], privacy: { diff --git a/packages/remotion/src/data/fetch.ts b/packages/remotion/src/data/fetch.ts index 46ec087..ace7fbf 100644 --- a/packages/remotion/src/data/fetch.ts +++ b/packages/remotion/src/data/fetch.ts @@ -104,6 +104,7 @@ function mergeUserStats(stats: UserStats[]): UserStats { [...first.topLanguages, ...stat.topLanguages], first.code.codeByteTotal, ); + first.packages = mergePackageMetrics(first.packages, stat.packages); first.contributions.timeline = mergeTimeline( first.contributions.timeline, stat.contributions.timeline, @@ -128,6 +129,40 @@ function mergeUserStats(stats: UserStats[]): UserStats { return first; } +function mergePackageMetrics( + current: UserStats['packages'], + next: UserStats['packages'], +): UserStats['packages'] { + const packages = new Map( + [...current.packages, ...next.packages].map((item) => [ + `${item.provider}:${item.name}`, + item, + ]), + ); + const values = [...packages.values()].sort( + (left, right) => right.downloads.lastMonth - left.downloads.lastMonth, + ); + const sum = (period: keyof UserStats['packages']['downloads']) => + values.reduce((total, item) => total + item.downloads[period], 0); + + return { + packageCount: values.length, + providers: [ + ...new Set([...current.providers, ...next.providers]), + ], + downloads: { + lastDay: sum('lastDay'), + lastWeek: sum('lastWeek'), + lastMonth: sum('lastMonth'), + lastYear: sum('lastYear'), + allTime: sum('allTime'), + }, + packages: values, + complete: current.complete && next.complete, + warnings: [...new Set([...current.warnings, ...next.warnings])], + }; +} + function mergeTimeline( current: UserStats['contributions']['timeline'], next: UserStats['contributions']['timeline'], diff --git a/packages/schemas/src/v2.ts b/packages/schemas/src/v2.ts index 8d78066..54f8938 100644 --- a/packages/schemas/src/v2.ts +++ b/packages/schemas/src/v2.ts @@ -254,6 +254,32 @@ export type RepoMetrics = { computedStats: ComputedStats; }; +export type PackageDownloadCounts = { + lastDay: number; + lastWeek: number; + lastMonth: number; + lastYear: number; + allTime: number; +}; + +export type PackageMetric = { + provider: string; + name: string; + url: string; + latestVersion: string | null; + latestPublishedAt: string | null; + downloads: PackageDownloadCounts; +}; + +export type PackageMetrics = { + packageCount: number; + providers: string[]; + downloads: PackageDownloadCounts; + packages: PackageMetric[]; + complete: boolean; + warnings: string[]; +}; + export type PresentationData = { readmeSummary: { name: string; @@ -366,6 +392,7 @@ export type GitHubStatsOutput = LegacyStats & { activity: ActivityStats; repositories: RepositoryRecord[]; repoMetrics: RepoMetrics; + packageMetrics: PackageMetrics; presentation: PresentationData; privacy: PrivacyReport; collectionStatus: CollectionStatus; diff --git a/packages/schemas/src/zod.ts b/packages/schemas/src/zod.ts index bb7a5d0..b770006 100644 --- a/packages/schemas/src/zod.ts +++ b/packages/schemas/src/zod.ts @@ -24,6 +24,23 @@ export const metricCardSchema = z.object({ detail: z.string().optional(), }); +export const packageDownloadCountsSchema = z.object({ + lastDay: z.number(), + lastWeek: z.number(), + lastMonth: z.number(), + lastYear: z.number(), + allTime: z.number(), +}); + +export const packageMetricSchema = z.object({ + provider: z.string(), + name: z.string(), + url: z.string(), + latestVersion: z.string().nullable(), + latestPublishedAt: z.string().nullable(), + downloads: packageDownloadCountsSchema, +}); + export const userStatsSchema = z.object({ schemaVersion: z.number().nullable(), name: z.string(), @@ -108,6 +125,14 @@ export const userStatsSchema = z.object({ forkCount: z.number(), }), topLanguages: z.array(renderLanguageSchema), + packages: z.object({ + packageCount: z.number(), + providers: z.array(z.string()), + downloads: packageDownloadCountsSchema, + packages: z.array(packageMetricSchema), + complete: z.boolean(), + warnings: z.array(z.string()), + }), cards: z.array(metricCardSchema), highlights: z.array(metricCardSchema), privacy: z.object({ From 488d31611831b8263fda18e6a183025a307e602e Mon Sep 17 00:00:00 2001 From: Luke Parke <5702154+LukasParke@users.noreply.github.com> Date: Fri, 14 Aug 2026 14:10:11 -0500 Subject: [PATCH 2/3] fix: bound package stats collection --- packages/diffler/src/cli.ts | 11 ++-- packages/diffler/src/packages/index.ts | 15 ++++-- packages/diffler/src/packages/npm.ts | 56 ++++++++++++++++++--- packages/diffler/tests/packages/npm.test.ts | 56 ++++++++++++++++++--- 4 files changed, 114 insertions(+), 24 deletions(-) diff --git a/packages/diffler/src/cli.ts b/packages/diffler/src/cli.ts index 6cee5ac..d823286 100644 --- a/packages/diffler/src/cli.ts +++ b/packages/diffler/src/cli.ts @@ -130,15 +130,16 @@ program statsConfig.includePrivateRepositoryDetails = true; statsConfig.includePrivateCacheDetails = true; } - if (options.npmPackages) { + const npmPackages = options.npmPackages + ?.split(",") + .map((packageName: string) => packageName.trim()) + .filter(Boolean); + if (npmPackages?.length) { statsConfig.packageSources = [ ...statsConfig.packageSources.filter((source) => source.provider !== "npm"), { provider: "npm", - packages: options.npmPackages - .split(",") - .map((packageName: string) => packageName.trim()) - .filter(Boolean), + packages: npmPackages, }, ]; } diff --git a/packages/diffler/src/packages/index.ts b/packages/diffler/src/packages/index.ts index 51c1d8a..2e1f772 100644 --- a/packages/diffler/src/packages/index.ts +++ b/packages/diffler/src/packages/index.ts @@ -24,21 +24,28 @@ export async function collectPackageStats( ); const packages: PackageMetric[] = []; const warnings: string[] = []; + const packagesByProvider = new Map>(); for (const source of sources) { - const adapter = adaptersByProvider.get(source.provider); + const packageNames = packagesByProvider.get(source.provider) ?? new Set(); + source.packages.forEach((packageName) => packageNames.add(packageName)); + packagesByProvider.set(source.provider, packageNames); + } + + for (const [provider, packageNames] of packagesByProvider) { + const adapter = adaptersByProvider.get(provider); if (!adapter) { - warnings.push(`No package stats adapter registered for "${source.provider}"`); + warnings.push(`No package stats adapter registered for "${provider}"`); continue; } try { - const result = await adapter.collect([...new Set(source.packages)]); + const result = await adapter.collect([...packageNames]); packages.push(...result.packages); warnings.push(...result.warnings); } catch (error) { warnings.push( - `${source.provider} package stats unavailable: ${ + `${provider} package stats unavailable: ${ error instanceof Error ? error.message : String(error) }` ); diff --git a/packages/diffler/src/packages/npm.ts b/packages/diffler/src/packages/npm.ts index 7d0e9ec..26143d8 100644 --- a/packages/diffler/src/packages/npm.ts +++ b/packages/diffler/src/packages/npm.ts @@ -11,7 +11,7 @@ type FetchResponse = { json(): Promise; }; -type Fetcher = (url: string) => Promise; +type Fetcher = (url: string, init?: RequestInit) => Promise; const DOWNLOAD_PERIODS: Array<[keyof PackageDownloadCounts, string]> = [ ["lastDay", "last-day"], @@ -20,6 +20,9 @@ const DOWNLOAD_PERIODS: Array<[keyof PackageDownloadCounts, string]> = [ ["lastYear", "last-year"], ]; const NPM_DOWNLOAD_DATA_START = new Date("2015-01-10T00:00:00.000Z"); +const PACKAGE_CONCURRENCY = 3; +const RANGE_CONCURRENCY = 3; +const REQUEST_TIMEOUT_MS = 15_000; export class NpmPackageStatsAdapter implements PackageStatsAdapter { readonly provider = "npm"; @@ -30,8 +33,10 @@ export class NpmPackageStatsAdapter implements PackageStatsAdapter { ) {} async collect(packageNames: string[]): Promise { - const results = await Promise.allSettled( - packageNames.map((packageName) => this.collectPackage(packageName)) + const results = await mapSettledWithConcurrency( + packageNames, + PACKAGE_CONCURRENCY, + (packageName) => this.collectPackage(packageName) ); const packages: PackageMetric[] = []; const warnings: string[] = []; @@ -108,13 +113,14 @@ export class NpmPackageStatsAdapter implements PackageStatsAdapter { if (firstDay > yesterday) return 0; const ranges = buildDownloadRanges(firstDay, yesterday); - const results = await Promise.all( - ranges.map((range) => + const results = await mapWithConcurrency( + ranges, + RANGE_CONCURRENCY, + (range) => fetchJson( this.fetcher, `https://api.npmjs.org/downloads/point/${range}/${encodedName}` ) - ) ); return results.reduce( (total, result) => total + asNonNegativeNumber(asRecord(result).downloads), @@ -155,13 +161,49 @@ function formatDate(value: Date): string { } async function fetchJson(fetcher: Fetcher, url: string): Promise { - const response = await fetcher(url); + const response = await fetcher(url, { + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); if (!response.ok) { throw new Error(`npm request failed with HTTP ${response.status}: ${url}`); } return response.json(); } +async function mapWithConcurrency( + values: T[], + concurrency: number, + worker: (value: T) => Promise +): Promise { + const results = new Array(values.length); + let nextIndex = 0; + const workers = Array.from( + { length: Math.min(concurrency, values.length) }, + async () => { + while (nextIndex < values.length) { + const index = nextIndex++; + results[index] = await worker(values[index]); + } + } + ); + await Promise.all(workers); + return results; +} + +async function mapSettledWithConcurrency( + values: T[], + concurrency: number, + worker: (value: T) => Promise +): Promise[]> { + return mapWithConcurrency(values, concurrency, async (value) => { + try { + return { status: "fulfilled", value: await worker(value) } as const; + } catch (reason) { + return { status: "rejected", reason } as const; + } + }); +} + function asRecord(value: unknown): Record { return value && typeof value === "object" ? (value as Record) diff --git a/packages/diffler/tests/packages/npm.test.ts b/packages/diffler/tests/packages/npm.test.ts index 26fbf96..22fafb2 100644 --- a/packages/diffler/tests/packages/npm.test.ts +++ b/packages/diffler/tests/packages/npm.test.ts @@ -80,6 +80,41 @@ describe("NpmPackageStatsAdapter", () => { expect(result.warnings).toHaveLength(1); expect(result.warnings[0]).toContain('npm package "missing" unavailable'); }); + + it("splits and sums historical downloads across bounded date ranges", async () => { + const requests: string[] = []; + const adapter = new NpmPackageStatsAdapter( + async (url, init) => { + requests.push(url); + expect(init?.signal).toBeInstanceOf(AbortSignal); + if (url.startsWith("https://registry.npmjs.org/")) { + return response({ + "dist-tags": { latest: "1.0.0" }, + time: { created: "2014-01-01T00:00:00.000Z" }, + }); + } + const period = url.split("/").at(-2) ?? ""; + if (period.startsWith("2015-01-10:")) return response({ downloads: 100 }); + if (period.startsWith("2016-07-03:")) return response({ downloads: 200 }); + if (period.startsWith("2017-12-25:")) return response({ downloads: 300 }); + return response({ downloads: 1 }); + }, + () => new Date("2018-01-01T12:00:00.000Z") + ); + + const result = await adapter.collect(["historical"]); + + expect(requests).toContain( + "https://api.npmjs.org/downloads/point/2015-01-10:2016-07-02/historical" + ); + expect(requests).toContain( + "https://api.npmjs.org/downloads/point/2016-07-03:2017-12-24/historical" + ); + expect(requests).toContain( + "https://api.npmjs.org/downloads/point/2017-12-25:2017-12-31/historical" + ); + expect(result.packages[0].downloads.allTime).toBe(600); + }); }); describe("collectPackageStats", () => { @@ -88,6 +123,7 @@ describe("collectPackageStats", () => { const result = await collectPackageStats( [ { provider: "npm", packages: ["one", "one", "two"] }, + { provider: "npm", packages: ["two", "three"] }, { provider: "pypi", packages: ["future"] }, ], [ @@ -117,17 +153,21 @@ describe("collectPackageStats", () => { ] ); - expect(requested).toEqual([["one", "two"]]); - expect(result.packageCount).toBe(2); + expect(requested).toEqual([["one", "two", "three"]]); + expect(result.packageCount).toBe(3); expect(result.providers).toEqual(["npm"]); expect(result.downloads).toEqual({ - lastDay: 3, - lastWeek: 20, - lastMonth: 300, - lastYear: 2_000, - allTime: 4_000, + lastDay: 6, + lastWeek: 30, + lastMonth: 600, + lastYear: 3_000, + allTime: 6_000, }); - expect(result.packages.map((item) => item.name)).toEqual(["two", "one"]); + expect(result.packages.map((item) => item.name)).toEqual([ + "three", + "two", + "one", + ]); expect(result.complete).toBe(false); expect(result.warnings).toEqual([ 'No package stats adapter registered for "pypi"', From 4b4d6c5881a87790031a125491778d88fe6c1a29 Mon Sep 17 00:00:00 2001 From: Luke Parke <5702154+LukasParke@users.noreply.github.com> Date: Fri, 14 Aug 2026 21:43:02 -0500 Subject: [PATCH 3/3] fix: survive npm rate limiting at realistic package counts Live collection of 46 packages showed the downloads API burst-limiting nearly every request (976 HTTP 429s), and the CDN answers 429s with "Retry-After: 0", so honor that header only as a floor on top of exponential backoff. Also: - retry 429/5xx and network failures with bounded backoff (6 attempts) - pace request starts (200ms) so large lists do not trip the burst limit - treat downloads API 404s as zero downloads for published packages - reduce package/range concurrency to 2 Verified live: 43/46 packages collect cleanly; failures degrade to warnings instead of aborting the run. --- packages/diffler/src/packages/npm.ts | 130 +++++++++++++++++--- packages/diffler/tests/packages/npm.test.ts | 109 ++++++++++++++++ 2 files changed, 223 insertions(+), 16 deletions(-) diff --git a/packages/diffler/src/packages/npm.ts b/packages/diffler/src/packages/npm.ts index 26143d8..fb84b55 100644 --- a/packages/diffler/src/packages/npm.ts +++ b/packages/diffler/src/packages/npm.ts @@ -8,10 +8,12 @@ import type { type FetchResponse = { ok: boolean; status: number; + headers?: { get(name: string): string | null }; json(): Promise; }; type Fetcher = (url: string, init?: RequestInit) => Promise; +type Sleep = (ms: number) => Promise; const DOWNLOAD_PERIODS: Array<[keyof PackageDownloadCounts, string]> = [ ["lastDay", "last-day"], @@ -20,17 +22,29 @@ const DOWNLOAD_PERIODS: Array<[keyof PackageDownloadCounts, string]> = [ ["lastYear", "last-year"], ]; const NPM_DOWNLOAD_DATA_START = new Date("2015-01-10T00:00:00.000Z"); -const PACKAGE_CONCURRENCY = 3; -const RANGE_CONCURRENCY = 3; +const PACKAGE_CONCURRENCY = 2; +const RANGE_CONCURRENCY = 2; const REQUEST_TIMEOUT_MS = 15_000; +// npm's CDN serves short-lived 429 bursts with "Retry-After: 0", so patience +// (several backed-off attempts) matters more than raw concurrency. +const MAX_ATTEMPTS = 6; +const RETRY_BASE_DELAY_MS = 1_000; +const RETRY_MAX_DELAY_MS = 30_000; +// Space out request starts so large package lists do not trip the burst limit. +const REQUEST_PACE_MS = 200; export class NpmPackageStatsAdapter implements PackageStatsAdapter { readonly provider = "npm"; + private readonly pacer: RequestPacer; constructor( private readonly fetcher: Fetcher = fetch, - private readonly now: () => Date = () => new Date() - ) {} + private readonly now: () => Date = () => new Date(), + private readonly sleep: Sleep = defaultSleep, + paceIntervalMs: number = REQUEST_PACE_MS + ) { + this.pacer = new RequestPacer(paceIntervalMs, sleep); + } async collect(packageNames: string[]): Promise { const results = await mapSettledWithConcurrency( @@ -60,15 +74,16 @@ export class NpmPackageStatsAdapter implements PackageStatsAdapter { const encodedName = encodeURIComponent(packageName); const metadata = await fetchJson( this.fetcher, - `https://registry.npmjs.org/${encodedName}` + `https://registry.npmjs.org/${encodedName}`, + this.sleep, + this.pacer ); const metadataRecord = asRecord(metadata); const createdAt = parseDate(asRecord(metadataRecord.time).created); const [downloadCounts, allTime] = await Promise.all([ Promise.all( DOWNLOAD_PERIODS.map(([, period]) => - fetchJson( - this.fetcher, + this.fetchDownloads( `https://api.npmjs.org/downloads/point/${period}/${encodedName}` ) ) @@ -99,6 +114,19 @@ export class NpmPackageStatsAdapter implements PackageStatsAdapter { }; } + // The downloads API 404s for published packages that have no download data + // yet; treat that as zero downloads instead of failing the package. + private async fetchDownloads(url: string): Promise { + try { + return await fetchJson(this.fetcher, url, this.sleep, this.pacer); + } catch (error) { + if (error instanceof NpmRequestError && error.status === 404) { + return { downloads: 0 }; + } + throw error; + } + } + private async collectAllTimeDownloads( encodedName: string, createdAt: Date | null @@ -117,8 +145,7 @@ export class NpmPackageStatsAdapter implements PackageStatsAdapter { ranges, RANGE_CONCURRENCY, (range) => - fetchJson( - this.fetcher, + this.fetchDownloads( `https://api.npmjs.org/downloads/point/${range}/${encodedName}` ) ); @@ -160,14 +187,85 @@ function formatDate(value: Date): string { return value.toISOString().slice(0, 10); } -async function fetchJson(fetcher: Fetcher, url: string): Promise { - const response = await fetcher(url, { - signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), - }); - if (!response.ok) { - throw new Error(`npm request failed with HTTP ${response.status}: ${url}`); +class NpmRequestError extends Error { + constructor( + readonly status: number, + url: string + ) { + super(`npm request failed with HTTP ${status}: ${url}`); + } +} + +// Serializes request starts across the adapter so bursts of parallel packages +// still reach npm at a steady, gentle rate. +class RequestPacer { + private tail: Promise = Promise.resolve(); + + constructor( + private readonly intervalMs: number, + private readonly sleep: Sleep + ) {} + + async wait(): Promise { + if (this.intervalMs <= 0) return; + const next = this.tail.then(() => this.sleep(this.intervalMs)); + this.tail = next; + await next; } - return response.json(); +} + +// npm rate limits bursts of downloads API requests with short-lived 429s, so +// retry with backoff and honor Retry-After when the registry provides it. +async function fetchJson( + fetcher: Fetcher, + url: string, + sleep: Sleep, + pacer: RequestPacer +): Promise { + let lastError: unknown = new Error(`npm request failed: ${url}`); + for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + let retryAfterMs: number | null = null; + try { + await pacer.wait(); + const response = await fetcher(url, { + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + if (response.ok) { + return response.json(); + } + lastError = new NpmRequestError(response.status, url); + if (response.status !== 429 && response.status < 500) { + break; + } + retryAfterMs = parseRetryAfter(response.headers?.get("retry-after")); + } catch (error) { + lastError = error; + } + if (attempt < MAX_ATTEMPTS) { + await sleep(retryDelayMs(attempt, retryAfterMs)); + } + } + throw lastError; +} + +function parseRetryAfter(value: string | null | undefined): number | null { + if (!value) return null; + const seconds = Number(value); + if (!Number.isFinite(seconds) || seconds < 0) return null; + return Math.min(seconds * 1_000, RETRY_MAX_DELAY_MS); +} + +function retryDelayMs(attempt: number, retryAfterMs: number | null): number { + const backoff = RETRY_BASE_DELAY_MS * 2 ** (attempt - 1); + const jitter = Math.floor(Math.random() * RETRY_BASE_DELAY_MS); + const delay = Math.min(backoff + jitter, RETRY_MAX_DELAY_MS); + // Cloudflare fronts npm and answers 429s with "Retry-After: 0", so treat the + // header as a floor rather than a replacement for backoff. + return retryAfterMs !== null ? Math.max(delay, retryAfterMs) : delay; +} + +function defaultSleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); } async function mapWithConcurrency( diff --git a/packages/diffler/tests/packages/npm.test.ts b/packages/diffler/tests/packages/npm.test.ts index 22fafb2..c092c9a 100644 --- a/packages/diffler/tests/packages/npm.test.ts +++ b/packages/diffler/tests/packages/npm.test.ts @@ -81,6 +81,37 @@ describe("NpmPackageStatsAdapter", () => { expect(result.warnings[0]).toContain('npm package "missing" unavailable'); }); + it("treats downloads API 404s as zero downloads for known packages", async () => { + const adapter = new NpmPackageStatsAdapter( + async (url) => { + if (url.startsWith("https://registry.npmjs.org/")) { + return response({ + "dist-tags": { latest: "0.1.0" }, + time: { + created: "2026-08-01T00:00:00.000Z", + "0.1.0": "2026-08-01T00:00:00.000Z", + }, + }); + } + return response({ error: "package not found" }, 404); + }, + () => new Date("2026-08-14T12:00:00.000Z") + ); + + const result = await adapter.collect(["fresh-package"]); + + expect(result.warnings).toEqual([]); + expect(result.packages).toHaveLength(1); + expect(result.packages[0].downloads).toEqual({ + lastDay: 0, + lastWeek: 0, + lastMonth: 0, + lastYear: 0, + allTime: 0, + }); + expect(result.packages[0].latestVersion).toBe("0.1.0"); + }); + it("splits and sums historical downloads across bounded date ranges", async () => { const requests: string[] = []; const adapter = new NpmPackageStatsAdapter( @@ -117,6 +148,84 @@ describe("NpmPackageStatsAdapter", () => { }); }); +describe("NpmPackageStatsAdapter retries", () => { + const noSleep = async () => {}; + + it("retries rate-limited requests until they succeed", async () => { + let callCount = 0; + const delays: number[] = []; + const adapter = new NpmPackageStatsAdapter( + async (url) => { + callCount += 1; + if (callCount <= 2) { + // npm's CDN answers 429s with "Retry-After: 0", which must not + // flatten the backoff delay. + return { + ...response({}, 429), + headers: { get: () => "0" }, + }; + } + if (url.startsWith("https://registry.npmjs.org/")) { + return response({ + "dist-tags": { latest: "1.0.0" }, + time: { created: "2026-01-01T00:00:00.000Z" }, + }); + } + return response({ downloads: 5 }); + }, + () => new Date("2026-08-14T12:00:00.000Z"), + async (ms) => { + delays.push(ms); + }, + 0 + ); + + const result = await adapter.collect(["limited"]); + + expect(result.warnings).toEqual([]); + expect(result.packages).toHaveLength(1); + expect(delays.length).toBeGreaterThanOrEqual(2); + expect(delays.every((delay) => delay >= 1_000)).toBe(true); + }); + + it("bounds attempts and warns when rate limiting persists", async () => { + let callCount = 0; + const adapter = new NpmPackageStatsAdapter( + async () => { + callCount += 1; + return response({}, 429); + }, + () => new Date("2026-08-14T12:00:00.000Z"), + noSleep + ); + + const result = await adapter.collect(["always-limited"]); + + expect(callCount).toBe(6); + expect(result.packages).toEqual([]); + expect(result.warnings).toHaveLength(1); + expect(result.warnings[0]).toContain("HTTP 429"); + }); + + it("does not retry permanent client errors", async () => { + let callCount = 0; + const adapter = new NpmPackageStatsAdapter( + async () => { + callCount += 1; + return response({}, 404); + }, + () => new Date("2026-08-14T12:00:00.000Z"), + noSleep + ); + + const result = await adapter.collect(["missing"]); + + expect(callCount).toBe(1); + expect(result.warnings).toHaveLength(1); + expect(result.warnings[0]).toContain("HTTP 404"); + }); +}); + describe("collectPackageStats", () => { it("deduplicates package names and aggregates normalized providers", async () => { const requested: string[][] = [];