diff --git a/packages/diffler/README.md b/packages/diffler/README.md index 3a1df45..4f01091 100644 --- a/packages/diffler/README.md +++ b/packages/diffler/README.md @@ -159,8 +159,22 @@ helpers: cache: ttl: 3600 + +statsAction: + # Include private repositories in anonymous aggregate numbers while keeping + # names, URLs, topics, IDs, and per-repository records out of public JSON. + includePrivateRepositoryMetrics: true + includePrivateRepositoryDetails: false + includePrivateCacheDetails: false ``` +Private aggregate collection requires a token that can read the intended private +repositories. A repository-scoped `GITHUB_TOKEN` usually cannot access private +repositories outside its own repository; use a fine-grained PAT stored as a +repository secret. You can also enable this mode with +`STATS_INCLUDE_PRIVATE_REPOSITORY_METRICS=true` or +`diffler collect --include-private-metrics`. + --- ## 🤖 GitHub Actions diff --git a/packages/diffler/src/cli.ts b/packages/diffler/src/cli.ts index 4994a98..abe68f8 100644 --- a/packages/diffler/src/cli.ts +++ b/packages/diffler/src/cli.ts @@ -95,6 +95,10 @@ program .option("--cache-path ", "Path for committed stable cache state") .option("--volatile-cache-path ", "Path for volatile API metadata cache") .option("--backfill-mode ", "Backfill mode: resume, refresh, or off") + .option( + "--include-private-metrics", + "Include anonymous private repository metrics without repository details" + ) .option("--include-private", "Include private repository details") .action(async (options) => { const token = process.env.GITHUB_TOKEN; @@ -114,7 +118,11 @@ program statsConfig.backfillMode = options.backfillMode; } } + if (options.includePrivateMetrics) { + statsConfig.includePrivateRepositoryMetrics = true; + } if (options.includePrivate) { + statsConfig.includePrivateRepositoryMetrics = true; statsConfig.includePrivateRepositoryDetails = true; statsConfig.includePrivateCacheDetails = true; } diff --git a/packages/diffler/src/config.ts b/packages/diffler/src/config.ts index e6021e3..ac0a636 100644 --- a/packages/diffler/src/config.ts +++ b/packages/diffler/src/config.ts @@ -80,6 +80,7 @@ export const StatsActionConfigSchema = z.object({ minRestRemaining: z.number().int().default(750), includeTraffic: z.boolean().default(true), includeRestRepoStats: z.boolean().default(true), + includePrivateRepositoryMetrics: z.boolean().default(false), includePrivateRepositoryDetails: z.boolean().default(false), includePrivateCacheDetails: z.boolean().default(false), backfillMode: z.enum(["resume", "refresh", "off"]).default("resume"), @@ -150,6 +151,10 @@ export function buildStatsActionConfig(config: DifflerConfig): StatsActionConfig base.minRestRemaining = envNum("min-rest-remaining", base.minRestRemaining); base.includeTraffic = envBool("include-traffic", base.includeTraffic); base.includeRestRepoStats = envBool("include-rest-repo-stats", base.includeRestRepoStats); + base.includePrivateRepositoryMetrics = envBool( + "include-private-repository-metrics", + base.includePrivateRepositoryMetrics + ); base.includePrivateRepositoryDetails = envBool( "include-private-repository-details", base.includePrivateRepositoryDetails diff --git a/packages/diffler/src/stats/cache.ts b/packages/diffler/src/stats/cache.ts index 4b7d379..bd11531 100644 --- a/packages/diffler/src/stats/cache.ts +++ b/packages/diffler/src/stats/cache.ts @@ -1,4 +1,5 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { createHash } from "node:crypto"; import { dirname } from "node:path"; import { type BackfillFailure, @@ -46,10 +47,11 @@ export function readVolatileCache(path: string): VolatileCache { export function writeStableCache( path: string, cache: StableCache, - includePrivateDetails = false + includePrivateDetails = false, + includePrivateMetrics = false ): void { writeJsonFile(path, { - ...sanitizeStableCache(cache, includePrivateDetails), + ...sanitizeStableCache(cache, includePrivateDetails, includePrivateMetrics), updatedAt: Date.now(), }); } @@ -84,7 +86,8 @@ export function cacheRepository( export function sanitizeStableCache( cache: StableCache, - includePrivateDetails: boolean + includePrivateDetails: boolean, + includePrivateMetrics = false ): StableCache { if (includePrivateDetails) return cache; @@ -94,6 +97,13 @@ export function sanitizeStableCache( ) ) as Record; const publicRepositoryIds = new Set(Object.keys(repositories)); + const metricCacheIds = includePrivateMetrics + ? new Set([ + ...publicRepositoryIds, + ...Object.keys(cache.contributorStats).filter(isPrivateMetricCacheKey), + ...Object.keys(cache.traffic).filter(isPrivateMetricCacheKey), + ]) + : publicRepositoryIds; return { ...cache, @@ -106,9 +116,9 @@ export function sanitizeStableCache( ), contributorStats: filterRecordByPublicRepoId( cache.contributorStats, - publicRepositoryIds + metricCacheIds ), - traffic: filterRecordByPublicRepoId(cache.traffic, publicRepositoryIds), + traffic: filterRecordByPublicRepoId(cache.traffic, metricCacheIds), backfill: { pending: cache.backfill.pending.filter((item) => publicRepositoryIds.has(item.repoId) @@ -119,6 +129,36 @@ export function sanitizeStableCache( }; } +export function repositoryMetricCacheKey( + repository: RepositoryRecord, + includePrivateDetails: boolean +): string { + if (!repository.isPrivate || includePrivateDetails) return repository.id; + return `private:${hashPrivateMetricValue(repository.id)}`; +} + +export function repositoryMetricVersion( + repository: RepositoryRecord, + includePrivateDetails: boolean +): string | null { + if ( + !repository.defaultBranchOid || + !repository.isPrivate || + includePrivateDetails + ) { + return repository.defaultBranchOid; + } + return `private:${hashPrivateMetricValue(repository.defaultBranchOid)}`; +} + +function isPrivateMetricCacheKey(key: string): boolean { + return key.startsWith("private:"); +} + +function hashPrivateMetricValue(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} + export function shouldReuseContributionYear( cached: CachedContributionYear | undefined, year: number, diff --git a/packages/diffler/src/stats/github.ts b/packages/diffler/src/stats/github.ts index a774c76..7dbc899 100644 --- a/packages/diffler/src/stats/github.ts +++ b/packages/diffler/src/stats/github.ts @@ -26,6 +26,8 @@ import { cacheRepository, mergeBackfillQueue, recordBackfillFailure, + repositoryMetricCacheKey, + repositoryMetricVersion, shouldReuseContributionYear, } from "./cache.js"; import { aggregateLanguages, mergeContributionsCollections } from "./aggregate.js"; @@ -408,12 +410,26 @@ export function buildBackfillQueue( const next: BackfillItem[] = []; const forceRefresh = config.backfillMode === "refresh"; for (const repo of repositories) { - if (repo.isPrivate && !config.includePrivateRepositoryDetails) continue; + if ( + repo.isPrivate && + !config.includePrivateRepositoryMetrics && + !config.includePrivateRepositoryDetails + ) { + continue; + } const basePriority = getRepositoryPriority(repo); - const contributorStats = cache.contributorStats[repo.id]; + const metricCacheKey = repositoryMetricCacheKey( + repo, + config.includePrivateRepositoryDetails + ); + const metricVersion = repositoryMetricVersion( + repo, + config.includePrivateRepositoryDetails + ); + const contributorStats = cache.contributorStats[metricCacheKey]; const contributorStatsComplete = - contributorStats?.defaultBranchOid === repo.defaultBranchOid && + contributorStats?.defaultBranchOid === metricVersion && ["fresh", "cached"].includes(contributorStats.status); if ( config.includeRestRepoStats && @@ -430,7 +446,7 @@ export function buildBackfillQueue( }); } - const traffic = cache.traffic[repo.id]; + const traffic = cache.traffic[metricCacheKey]; if ( config.includeTraffic && canReadTraffic(repo) && @@ -486,21 +502,32 @@ export async function processBackfillQueue( } try { + const metricCacheKey = repositoryMetricCacheKey( + repo, + config.includePrivateRepositoryDetails + ); if (item.type === "contributors") { - cache.contributorStats[repo.id] = await fetchContributorStats( + const contributorStats = await fetchContributorStats( client, scheduler, volatileCache, - cache.contributorStats[repo.id], + cache.contributorStats[metricCacheKey], repo, username ); + cache.contributorStats[metricCacheKey] = { + ...contributorStats, + defaultBranchOid: repositoryMetricVersion( + repo, + config.includePrivateRepositoryDetails + ), + }; } else { - cache.traffic[repo.id] = await fetchTraffic( + cache.traffic[metricCacheKey] = await fetchTraffic( client, scheduler, volatileCache, - cache.traffic[repo.id], + cache.traffic[metricCacheKey], repo ); } diff --git a/packages/diffler/src/stats/index.ts b/packages/diffler/src/stats/index.ts index a3bf88d..60873f0 100644 --- a/packages/diffler/src/stats/index.ts +++ b/packages/diffler/src/stats/index.ts @@ -116,6 +116,7 @@ export async function runStatsCollection( repositories = mergeRepositories( [ ...Object.values(stableCache.repositories).map((entry) => entry.repository), + ...repositories, ...contributions.repositories, ] ); @@ -166,7 +167,12 @@ export async function runStatsCollection( }); writeJsonOutput(config.outputPath, output); - writeStableCache(config.cachePath, stableCache, config.includePrivateCacheDetails); + writeStableCache( + config.cachePath, + stableCache, + config.includePrivateCacheDetails, + config.includePrivateRepositoryMetrics + ); writeVolatileCache(config.volatileCachePath, volatileCache); console.log( diff --git a/packages/diffler/src/stats/output.ts b/packages/diffler/src/stats/output.ts index 7d16463..f59db92 100644 --- a/packages/diffler/src/stats/output.ts +++ b/packages/diffler/src/stats/output.ts @@ -22,6 +22,7 @@ import { formatNumber, toTopRepos, } from "./aggregate.js"; +import { repositoryMetricCacheKey } from "./cache.js"; export function buildOutput(params: { profile: UserProfile; @@ -34,10 +35,21 @@ export function buildOutput(params: { fetchedAt: number; }): GitHubStatsOutput { const includePrivateDetails = params.config.includePrivateRepositoryDetails; + const includePrivateMetrics = + includePrivateDetails || params.config.includePrivateRepositoryMetrics; const visibleRepositories = includePrivateDetails ? params.repositories : params.repositories.filter((repo) => !repo.isPrivate); const visibleRepositoryIds = new Set(visibleRepositories.map((repo) => repo.id)); + const metricRepositories = includePrivateMetrics + ? params.repositories + : params.repositories.filter((repo) => !repo.isPrivate); + const metricRepositoryIds = new Set(metricRepositories.map((repo) => repo.id)); + const metricCacheKeys = new Set( + metricRepositories.map((repo) => + repositoryMetricCacheKey(repo, includePrivateDetails) + ) + ); const visibleRepositoryContributions = includePrivateDetails ? params.contributions.repositoryContributions : params.contributions.repositoryContributions.filter((summary) => @@ -45,36 +57,56 @@ export function buildOutput(params: { ); const contributionStats = calculateContributionStats(params.contributions.collection); const { languages: topLanguages, codeByteTotal } = - aggregateRepositoryLanguages(visibleRepositories); - const allComputedRepos = params.repositories.map(toComputedRepo); + aggregateRepositoryLanguages(metricRepositories); const visibleComputedRepos = visibleRepositories.map(toComputedRepo); - const repoStats = calculateRepoStats(allComputedRepos); + const metricComputedRepos = metricRepositories.map(toComputedRepo); + const repoStats = calculateRepoStats(metricComputedRepos); + const visibleComputedStats = calculateComputedStats( + visibleComputedRepos, + aggregateRepositoryLanguages(visibleRepositories).languages, + contributionStats + ); const computedStats = { - ...calculateComputedStats(visibleComputedRepos, topLanguages, contributionStats), + ...calculateComputedStats(metricComputedRepos, topLanguages, contributionStats), ...repoStats, + topTopics: visibleComputedStats.topTopics, + allTopics: visibleComputedStats.allTopics, }; - const visibleContributorStats = Object.entries(params.cache.contributorStats) - .filter(([repoId]) => visibleRepositoryIds.has(repoId)) - .map(([, stats]) => stats); - const visibleTrafficSummaries = Object.entries(params.cache.traffic) - .filter(([repoId]) => visibleRepositoryIds.has(repoId)) - .map(([, traffic]) => traffic); - const ownedVisibleRepos = visibleRepositories.filter((repo) => + const metricContributorStats = metricRepositories + .map( + (repo) => + params.cache.contributorStats[ + repositoryMetricCacheKey(repo, includePrivateDetails) + ] + ) + .filter((stats) => stats !== undefined); + const metricTrafficSummaries = metricRepositories + .map( + (repo) => + params.cache.traffic[repositoryMetricCacheKey(repo, includePrivateDetails)] + ) + .filter((traffic) => traffic !== undefined); + const ownedMetricRepos = metricRepositories.filter((repo) => repo.sources.includes("owned") ); const ownedPublicRepos = params.repositories.filter( (repo) => !repo.isPrivate && repo.sources.includes("owned") ); - const ownedOriginalRepos = ownedPublicRepos.filter((repo) => !repo.isFork); + const ownedPrivateRepos = params.repositories.filter( + (repo) => repo.isPrivate && repo.sources.includes("owned") + ); + const ownedOriginalRepos = ownedMetricRepos.filter((repo) => !repo.isFork); const { languages: profileTopLanguages, codeByteTotal: profileCodeByteTotal, } = aggregateRepositoryLanguages(ownedOriginalRepos); const currentYear = `${new Date(params.fetchedAt).getUTCFullYear()}`; const profileRepoMetrics: NonNullable = { + totalRepos: ownedMetricRepos.length, publicRepos: ownedPublicRepos.length, + privateRepos: includePrivateMetrics ? ownedPrivateRepos.length : 0, originalRepos: ownedOriginalRepos.length, - forkedRepos: ownedPublicRepos.length - ownedOriginalRepos.length, + forkedRepos: ownedMetricRepos.length - ownedOriginalRepos.length, activeOriginalRepos: ownedOriginalRepos.filter((repo) => (repo.pushedAt || repo.updatedAt).startsWith(currentYear) ).length, @@ -86,36 +118,37 @@ export function buildOutput(params: { topLanguages: profileTopLanguages, }; - const linesAdded = visibleContributorStats.reduce( + const linesAdded = metricContributorStats.reduce( (sum, stats) => sum + stats.additions, 0 ); - const linesDeleted = visibleContributorStats.reduce( + const linesDeleted = metricContributorStats.reduce( (sum, stats) => sum + stats.deletions, 0 ); - const commitCount = visibleContributorStats.reduce( + const commitCount = metricContributorStats.reduce( (sum, stats) => sum + stats.commits, 0 ); - const repoViews = visibleTrafficSummaries.reduce( + const repoViews = metricTrafficSummaries.reduce( (sum, traffic) => sum + traffic.count, 0 ); - const repoViewUniques = visibleTrafficSummaries.reduce( + const repoViewUniques = metricTrafficSummaries.reduce( (sum, traffic) => sum + traffic.uniques, 0 ); - const starCount = ownedVisibleRepos.reduce((sum, repo) => sum + repo.stars, 0); - const forkCount = ownedVisibleRepos.reduce((sum, repo) => sum + repo.forks, 0); + const starCount = ownedMetricRepos.reduce((sum, repo) => sum + repo.stars, 0); + const forkCount = ownedMetricRepos.reduce((sum, repo) => sum + repo.forks, 0); const topRepos = toTopRepos(visibleRepositories); const privacy = buildPrivacyReport({ + includePrivateRepositoryMetrics: includePrivateMetrics, includePrivateRepositoryDetails: includePrivateDetails, includePrivateCacheDetails: params.config.includePrivateCacheDetails, repositories: params.repositories, repositoryContributions: params.contributions.repositoryContributions.length, visibleRepositoryContributions: visibleRepositoryContributions.length, - visibleRepositoryIds, + metricCacheKeys, cache: params.cache, }); const collectionStatus = addPrivacyWarnings(params.collectionStatus, privacy); @@ -132,31 +165,31 @@ export function buildOutput(params: { linesAdded, linesDeleted, linesOfCodeChanged: linesAdded + linesDeleted, - reposCompleted: visibleContributorStats.filter((stats) => + reposCompleted: metricContributorStats.filter((stats) => ["fresh", "cached"].includes(stats.status) ).length, reposPending: params.cache.backfill.pending.filter( - (item) => item.type === "contributors" && visibleRepositoryIds.has(item.repoId) + (item) => item.type === "contributors" && metricRepositoryIds.has(item.repoId) ).length, reposFailed: Object.values(params.cache.backfill.failures).filter( (failure) => failure.key.startsWith("contributors:") && - hasVisibleRepositoryId(failure.key, visibleRepositoryIds) + hasVisibleRepositoryId(failure.key, metricRepositoryIds) ).length, }, traffic: { repoViews, repoViewUniques, - reposCompleted: visibleTrafficSummaries.filter((traffic) => + reposCompleted: metricTrafficSummaries.filter((traffic) => ["fresh", "cached"].includes(traffic.status) ).length, reposPending: params.cache.backfill.pending.filter( - (item) => item.type === "traffic" && visibleRepositoryIds.has(item.repoId) + (item) => item.type === "traffic" && metricRepositoryIds.has(item.repoId) ).length, reposFailed: Object.values(params.cache.backfill.failures).filter( (failure) => failure.key.startsWith("traffic:") && - hasVisibleRepositoryId(failure.key, visibleRepositoryIds) + hasVisibleRepositoryId(failure.key, metricRepositoryIds) ).length, }, repoStats, @@ -268,15 +301,17 @@ function toComputedRepo(repo: RepositoryRecord) { } function buildPrivacyReport(params: { + includePrivateRepositoryMetrics: boolean; includePrivateRepositoryDetails: boolean; includePrivateCacheDetails: boolean; repositories: RepositoryRecord[]; repositoryContributions: number; visibleRepositoryContributions: number; - visibleRepositoryIds: Set; + metricCacheKeys: Set; cache: StableCache; }): PrivacyReport { return { + privateRepositoryMetricsIncluded: params.includePrivateRepositoryMetrics, privateRepositoryDetailsIncluded: params.includePrivateRepositoryDetails, privateCacheDetailsIncluded: params.includePrivateCacheDetails, redactedPrivateRepositories: params.includePrivateRepositoryDetails @@ -286,10 +321,10 @@ function buildPrivacyReport(params: { params.repositoryContributions - params.visibleRepositoryContributions, redactedOptionalMetrics: Object.keys(params.cache.contributorStats).filter( - (repoId) => !params.visibleRepositoryIds.has(repoId) + (repoId) => !params.metricCacheKeys.has(repoId) ).length + Object.keys(params.cache.traffic).filter( - (repoId) => !params.visibleRepositoryIds.has(repoId) + (repoId) => !params.metricCacheKeys.has(repoId) ).length, }; } @@ -347,7 +382,7 @@ function buildPresentation(params: { topLanguages: profileMetrics.topLanguages.slice(0, 5), starsReceived: profileMetrics.starsReceived, forksReceived: profileMetrics.forksReceived, - totalRepos: profileMetrics.publicRepos, + totalRepos: profileMetrics.totalRepos ?? profileMetrics.publicRepos, originalRepos: profileMetrics.originalRepos, activeRepos: profileMetrics.activeOriginalRepos, languageCount: profileMetrics.topLanguages.length, @@ -436,7 +471,7 @@ function buildPresentation(params: { { id: "repositories", title: "Repository footprint", - metric: profileMetrics.publicRepos, + metric: profileMetrics.totalRepos ?? profileMetrics.publicRepos, supportingText: `${profileMetrics.originalRepos} original repositories`, }, { diff --git a/packages/diffler/src/stats/types.ts b/packages/diffler/src/stats/types.ts index f33af82..72190bd 100644 --- a/packages/diffler/src/stats/types.ts +++ b/packages/diffler/src/stats/types.ts @@ -44,6 +44,7 @@ export type StatsActionConfig = { minRestRemaining: number; includeTraffic: boolean; includeRestRepoStats: boolean; + includePrivateRepositoryMetrics: boolean; includePrivateRepositoryDetails: boolean; includePrivateCacheDetails: boolean; backfillMode: "resume" | "refresh" | "off"; diff --git a/packages/diffler/tests/config.test.ts b/packages/diffler/tests/config.test.ts index cd8e629..2c5d4f1 100644 --- a/packages/diffler/tests/config.test.ts +++ b/packages/diffler/tests/config.test.ts @@ -5,6 +5,7 @@ import { join } from "node:path"; import { loadConfigFromFile, loadConfigFromEnv, + buildStatsActionConfig, getUsernames, getProfiles, type GitHubConfig, @@ -29,6 +30,7 @@ describe("config", () => { expect(config.github.apiUrl).toBe("https://api.github.com"); expect(config.templates.main).toBe("profile.md.j2"); expect(config.cache.enabled).toBe(true); + expect(config.statsAction.includePrivateRepositoryMetrics).toBe(false); }); }); @@ -155,5 +157,13 @@ describe("config", () => { const config = loadConfigFromEnv(); expect(config.templates.main).toBe("other.md.j2"); }); + + it("enables anonymous private repository metrics", () => { + process.env.STATS_INCLUDE_PRIVATE_REPOSITORY_METRICS = "true"; + const config = buildStatsActionConfig(loadConfigFromEnv()); + expect(config.includePrivateRepositoryMetrics).toBe(true); + expect(config.includePrivateRepositoryDetails).toBe(false); + expect(config.includePrivateCacheDetails).toBe(false); + }); }); }); diff --git a/packages/diffler/tests/stats/index.test.ts b/packages/diffler/tests/stats/index.test.ts index 5b74c5b..cb2ad0b 100644 --- a/packages/diffler/tests/stats/index.test.ts +++ b/packages/diffler/tests/stats/index.test.ts @@ -6,7 +6,14 @@ import { calculateContributionStats, calculateComputedStats, } from "../../src/stats/index.js"; -import { cacheRepository, createEmptyStableCache, shouldReuseContributionYear } from "../../src/stats/cache.js"; +import { + cacheRepository, + createEmptyStableCache, + repositoryMetricCacheKey, + repositoryMetricVersion, + sanitizeStableCache, + shouldReuseContributionYear, +} from "../../src/stats/cache.js"; import { emptyContributionsCollection } from "../../src/stats/aggregate.js"; import { buildBackfillQueue, mergeRepositories } from "../../src/stats/github.js"; import { buildOutput } from "../../src/stats/output.js"; @@ -361,6 +368,7 @@ describe("v2 collection helpers", () => { minRestRemaining: 750, includeTraffic: true, includeRestRepoStats: true, + includePrivateRepositoryMetrics: false, includePrivateRepositoryDetails: false, includePrivateCacheDetails: false, backfillMode: "resume", @@ -492,6 +500,52 @@ describe("v2 collection helpers", () => { expect(buildBackfillQueue([repo], cache, baseConfig)).toHaveLength(0); }); + it("queues anonymous metrics for private repositories without enabling details", () => { + const privateRepo = createRepository({ + id: "R_PRIVATE", + isPrivate: true, + visibility: "PRIVATE", + }); + + expect(buildBackfillQueue([privateRepo], createEmptyStableCache(), baseConfig)).toEqual( + [] + ); + + const queue = buildBackfillQueue([privateRepo], createEmptyStableCache(), { + ...baseConfig, + includePrivateRepositoryMetrics: true, + }); + + expect(queue.map((item) => item.type).sort()).toEqual([ + "contributors", + "traffic", + ]); + + const cache = createEmptyStableCache(); + const metricCacheKey = repositoryMetricCacheKey(privateRepo, false); + cache.contributorStats[metricCacheKey] = { + additions: 10, + deletions: 2, + commits: 3, + fetchedAt: Date.now(), + defaultBranchOid: repositoryMetricVersion(privateRepo, false), + status: "cached", + }; + cache.traffic[metricCacheKey] = { + count: 1, + uniques: 1, + days: [], + fetchedAt: Date.now(), + status: "cached", + }; + expect( + buildBackfillQueue([privateRepo], cache, { + ...baseConfig, + includePrivateRepositoryMetrics: true, + }) + ).toHaveLength(0); + }); + it("reuses immutable cached contribution years but refreshes current years", () => { const cached: CachedContributionYear = { year: "2022", @@ -545,6 +599,33 @@ describe("v2 collection helpers", () => { expect(cache.repositories[privateRepo.id]).toBeUndefined(); }); + it("persists private metrics under anonymous cache keys", () => { + const cache = createEmptyStableCache(); + const privateRepo = createRepository({ + id: "R_PRIVATE", + name: "private-product", + nameWithOwner: "LukasParke/private-product", + isPrivate: true, + }); + const metricCacheKey = repositoryMetricCacheKey(privateRepo, false); + cache.contributorStats[metricCacheKey] = { + additions: 10, + deletions: 2, + commits: 3, + fetchedAt: Date.now(), + defaultBranchOid: repositoryMetricVersion(privateRepo, false), + status: "cached", + }; + + const sanitized = sanitizeStableCache(cache, false, true); + + expect(metricCacheKey).toMatch(/^private:[a-f0-9]{64}$/); + expect(sanitized.contributorStats[metricCacheKey]?.commits).toBe(3); + expect(JSON.stringify(sanitized)).not.toContain(privateRepo.id); + expect(JSON.stringify(sanitized)).not.toContain(privateRepo.nameWithOwner); + expect(JSON.stringify(sanitized)).not.toContain(privateRepo.defaultBranchOid); + }); + it("builds v2 output while preserving legacy top-level aliases", () => { const cache = createEmptyStableCache(); const repo = createRepository(); @@ -760,7 +841,9 @@ describe("v2 collection helpers", () => { }); expect(output.repoMetrics.profile).toEqual({ + totalRepos: 2, publicRepos: 2, + privateRepos: 0, originalRepos: 1, forkedRepos: 1, activeOriginalRepos: 1, @@ -792,7 +875,7 @@ describe("v2 collection helpers", () => { ); }); - it("redacts private repository details from public output while keeping aggregate counts", () => { + it("includes full private aggregates without exposing private repository details", () => { const cache = createEmptyStableCache(); const publicRepo = createRepository(); const privateRepo = createRepository({ @@ -812,6 +895,22 @@ describe("v2 collection helpers", () => { ], codeByteTotal: 5000, }); + const privateMetricCacheKey = repositoryMetricCacheKey(privateRepo, false); + cache.contributorStats[privateMetricCacheKey] = { + additions: 100, + deletions: 25, + commits: 8, + fetchedAt: Date.now(), + defaultBranchOid: repositoryMetricVersion(privateRepo, false), + status: "fresh", + }; + cache.traffic[privateMetricCacheKey] = { + count: 50, + uniques: 12, + days: [], + fetchedAt: Date.now(), + status: "fresh", + }; const collection = emptyContributionsCollection(); const status: CollectionStatus = { startedAt: 1, @@ -839,7 +938,7 @@ describe("v2 collection helpers", () => { errors: [], }; - const output = buildOutput({ + const outputParams = { profile: { name: "Luke Parke", login: "LukasParke", @@ -883,16 +982,61 @@ describe("v2 collection helpers", () => { config: baseConfig, collectionStatus: status, fetchedAt: 1000, + }; + const defaultOutput = buildOutput(outputParams); + const output = buildOutput({ + ...outputParams, + config: { + ...baseConfig, + includePrivateRepositoryMetrics: true, + }, }); + expect(defaultOutput.repoStats).toMatchObject({ + totalRepos: 1, + publicRepos: 1, + privateRepos: 0, + originalRepos: 1, + averageStarsPerRepo: 10, + }); + expect(defaultOutput.topLanguages.map((lang) => lang.languageName)).not.toContain( + "SecretLang" + ); expect(output.repoStats.privateRepos).toBe(1); + expect(output.repoMetrics.profile).toMatchObject({ + totalRepos: 2, + publicRepos: 1, + privateRepos: 1, + originalRepos: 2, + starsReceived: 1009, + codeByteTotal: 6000, + }); + expect(output.repoMetrics.contributorStats).toMatchObject({ + totalCommits: 8, + linesAdded: 100, + linesDeleted: 25, + reposCompleted: 1, + }); + expect(output.repoMetrics.traffic).toMatchObject({ + repoViews: 50, + repoViewUniques: 12, + reposCompleted: 1, + }); + expect(output.topLanguages.map((lang) => lang.languageName)).toContain( + "SecretLang" + ); + expect(output.repoMetrics.topTopics.map((topic) => topic.name)).not.toContain( + "secret-topic" + ); expect(output.repositories.map((repo) => repo.nameWithOwner)).not.toContain( privateRepo.nameWithOwner ); expect(output.profileContributions.repositoryContributions).toHaveLength(0); - expect(output.topLanguages.map((lang) => lang.languageName)).not.toContain( - "SecretLang" - ); + expect(JSON.stringify(output)).not.toContain(privateRepo.nameWithOwner); + expect(JSON.stringify(output)).not.toContain("secret-topic"); + expect(output.privacy.privateRepositoryMetricsIncluded).toBe(true); + expect(output.privacy.privateRepositoryDetailsIncluded).toBe(false); + expect(output.privacy.redactedOptionalMetrics).toBe(0); expect(output.privacy.redactedPrivateRepositories).toBe(1); expect(output.collectionStatus.warnings[0]).toContain("Private repository details redacted"); }); diff --git a/packages/remotion/src/data/adapter.test.ts b/packages/remotion/src/data/adapter.test.ts index b023b9e..ee54d86 100644 --- a/packages/remotion/src/data/adapter.test.ts +++ b/packages/remotion/src/data/adapter.test.ts @@ -152,10 +152,13 @@ describe('normalizeGithubStats', () => { const explicitProfile = normalizeGithubStats( { ...raw, + privacy: {privateRepositoryMetricsIncluded: true}, repoMetrics: { ...raw.repoMetrics, profile: { + totalRepos: 9, publicRepos: 7, + privateRepos: 2, originalRepos: 6, forkedRepos: 1, activeOriginalRepos: 5, @@ -179,13 +182,21 @@ describe('normalizeGithubStats', () => { {allowPrivateRepositoryDetails: false}, ); expect(explicitProfile.summary).toMatchObject({ - totalRepos: 7, + totalRepos: 9, activeRepos: 5, languageCount: 1, starsReceived: 99, forksReceived: 12, profileMetricsComplete: true, }); + expect(explicitProfile.repositories).toMatchObject({ + totalRepos: 9, + publicRepos: 7, + privateRepos: 2, + }); + expect(explicitProfile.privacy.privateRepositoryMetricsIncluded).toBe( + true, + ); expect(explicitProfile.topLanguages[0]?.languageName).toBe('Go'); }); diff --git a/packages/remotion/src/data/adapter.ts b/packages/remotion/src/data/adapter.ts index 4ebe8e5..5b1dc56 100644 --- a/packages/remotion/src/data/adapter.ts +++ b/packages/remotion/src/data/adapter.ts @@ -79,7 +79,10 @@ function normalizeV2Stats(raw: UnknownRecord): UserStats { const starCount = asNumber(profileRepoMetrics.starsReceived, 0); const forkCount = asNumber(profileRepoMetrics.forksReceived, 0); const activeRepos = asNumber(profileRepoMetrics.activeOriginalRepos, 0); - const totalRepos = asNumber(profileRepoMetrics.publicRepos, 0); + const totalRepos = asNumber( + profileRepoMetrics.totalRepos, + asNumber(profileRepoMetrics.publicRepos, 0) + ); const contributionCalendar = normalizeContributionCalendar( asRecord(profileContributions.contributionCalendar) ); @@ -227,13 +230,10 @@ function normalizeV2Stats(raw: UnknownRecord): UserStats { repositories: { totalRepos, publicRepos: asNumber(profileRepoMetrics.publicRepos, totalRepos), - privateRepos: - profileRepoMetrics.publicRepos === undefined - ? asNumber( - repoStats.privateRepos, - asNumber(computedStats.privateRepos, 0) - ) - : 0, + privateRepos: asNumber( + profileRepoMetrics.privateRepos, + Math.max(0, totalRepos - asNumber(profileRepoMetrics.publicRepos, totalRepos)) + ), activeRepos, archivedRepos: asNumber( profileRepoMetrics.archivedOriginalRepos, @@ -281,6 +281,8 @@ function normalizeV2Stats(raw: UnknownRecord): UserStats { cards: normalizeMetricCards(presentation.cards), highlights: normalizeMetricCards(presentation.highlights), privacy: { + privateRepositoryMetricsIncluded: + privacy.privateRepositoryMetricsIncluded === true, privateRepositoryDetailsIncluded: privacy.privateRepositoryDetailsIncluded === true, privateCacheDetailsIncluded: privacy.privateCacheDetailsIncluded === true, @@ -526,6 +528,7 @@ function normalizeLegacyStats(raw: UnknownRecord): UserStats { cards: [], highlights: [], privacy: { + privateRepositoryMetricsIncluded: false, privateRepositoryDetailsIncluded: false, privateCacheDetailsIncluded: false, redactedPrivateRepositories: 0, diff --git a/packages/remotion/src/data/defaultStats.ts b/packages/remotion/src/data/defaultStats.ts index e3b06ab..b901ef7 100644 --- a/packages/remotion/src/data/defaultStats.ts +++ b/packages/remotion/src/data/defaultStats.ts @@ -109,6 +109,7 @@ export const defaultStats: UserStats = { cards: [], highlights: [], privacy: { + privateRepositoryMetricsIncluded: false, privateRepositoryDetailsIncluded: false, privateCacheDetailsIncluded: false, redactedPrivateRepositories: 0, diff --git a/packages/schemas/src/v2.ts b/packages/schemas/src/v2.ts index 06e1d2c..8d78066 100644 --- a/packages/schemas/src/v2.ts +++ b/packages/schemas/src/v2.ts @@ -221,7 +221,9 @@ export type RepoMetrics = { topLanguages: Language[]; topTopics: TopicCount[]; profile?: { + totalRepos?: number; publicRepos: number; + privateRepos?: number; originalRepos: number; forkedRepos: number; activeOriginalRepos: number; @@ -279,6 +281,7 @@ export type PresentationData = { }; export type PrivacyReport = { + privateRepositoryMetricsIncluded: boolean; privateRepositoryDetailsIncluded: boolean; privateCacheDetailsIncluded: boolean; redactedPrivateRepositories: number; diff --git a/packages/schemas/src/zod.ts b/packages/schemas/src/zod.ts index 18c0e18..bb7a5d0 100644 --- a/packages/schemas/src/zod.ts +++ b/packages/schemas/src/zod.ts @@ -111,6 +111,7 @@ export const userStatsSchema = z.object({ cards: z.array(metricCardSchema), highlights: z.array(metricCardSchema), privacy: z.object({ + privateRepositoryMetricsIncluded: z.boolean(), privateRepositoryDetailsIncluded: z.boolean(), privateCacheDetailsIncluded: z.boolean(), redactedPrivateRepositories: z.number(),