diff --git a/.github/scorecard/SCORECARD.md b/.github/scorecard/SCORECARD.md new file mode 100644 index 000000000..584a759a2 --- /dev/null +++ b/.github/scorecard/SCORECARD.md @@ -0,0 +1,80 @@ +# Coder Registry Module Scorecard + +**100 pts** = **Universal criteria (75)** + **one track (25)** + +Score each criterion as **0 / half / full**. + +## Scoring rubric + +### Universal criteria — 75 pts + +| Criterion | Pts | Pass = | +| -------------------------------------------------- | -----: | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Presentation & Onboarding** | **25** | | +| Configuration-mode examples | 12 | If the module has many options, each major mode has a documented example with sensible defaults, for example provider choice, app vs headless | +| Coder-context framing | 8 | Explains what the module adds on top of Coder, names both Coder and the target tool, and shows where Coder fits in the flow | +| Visual preview | 5 | README includes an image, GIF, or video of the module in action | +| **Credential Hygiene** | **20** | | +| Secrets marked sensitive | 16 | Sensitive inputs are marked `sensitive = true`, and README examples avoid inline secrets | +| Non-hardcoded auth path | 4 | If possible, README shows a path that avoids pasting raw keys into templates, for example ServiceAccount, IAM/OAuth/external auth, API key helper, or AI Gateway | +| **Restricted-Network Readiness** _(if applicable)_ | **20** | | +| Mirrorable artifact source | 10 | Download or install URL is configurable so it can point to an internal mirror or artifact store instead of the public internet | +| Bring-your-own binary | 6 | Download or install can be disabled entirely when the tool is already baked into the image | +| Egress transparency | 4 | External endpoints are documented, plus notes for restricted or air-gapped environments | +| **Engineering Quality** | **10** | | +| Input quality | 6 | Inputs have clear descriptions, sensible defaults, and `validation` where appropriate | +| Test coverage | 4 | Clear testing story, `.tftest.hcl` primarily covers business logic, TypeScript tests cover end-to-end behavior | + +### Agent track — 25 pts + +| Criterion | Pts | Pass = | +| --------------------- | --: | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| AI governance | 10 | Documented support for Coder AI Gateway and/or Agent Firewall, including how Coder governs auth, routing, or policy enforcement for the agent | +| Dashboard entry point | 5 | Documented `coder_app` support, built-in or example | +| Session continuity | 5 | Documented support for continuing an existing agent session across reconnects or relaunches, either through native resume/session ID support or a persistent session manager such as boo, tmux, or screen | +| Managed configuration | 5 | Documented support for managed MCP, settings, policies, or workdir config | + +### IDE track — 25 pts + +| Criterion | Pts | Pass = | +| ------------------------------------------ | --: | ---------------------------------------------------------------------------- | +| Dashboard entry point | 7 | `coder_app` support with proper launch behavior | +| Managed configuration | 6 | Documented support for managed IDE settings or config | +| Configurable folder or workdir | 6 | Documented support for opening or starting in a configured folder or workdir | +| Pre-installed extensions _(web IDEs only)_ | 6 | For web IDEs, documented support for pre-installing extensions | + +### Utility track + +Utility modules are scored on Universal criteria only, then normalized: + +`round(raw / 75 * 100)` + +## Notes + +| Topic | Rule | +| ------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Track assignment | Every score should record which track was used | +| Internal building blocks | Modules whose README cautions against direct use (for example "we do not recommend using this module directly") are excluded from scoring entirely | +| If applicable | Criteria or themes marked _if applicable_ are excluded when the concern does not exist by construction, for example a module that downloads nothing. Excluded points are removed from the denominator and the final score is normalized to 100 | +| N/A handling | Outside _if applicable_ criteria, missing support scores zero. Only Utility modules skip the track section | +| Scoring | Full = implemented and documented; half = partial, awkward, or under-documented; zero = absent | + +## Grading + +| Score | Badge | +| -----: | ---------- | +| 90-100 | Exemplary | +| 75-89 | Strong | +| 50-74 | Adequate | +| <50 | Needs work | + +## Output format + +Every scorecard leads with a theme-level summary table, then a collapsed +drilldown with per-criterion tables grouped by theme. + +## Live scorecards + +Every scored module is listed in the pinned +[📊 Module Scorecards](https://github.com/coder/registry/discussions/1011) +discussion, which links each module's dedicated scorecard discussion. diff --git a/.github/scorecard/lib.ts b/.github/scorecard/lib.ts new file mode 100644 index 000000000..5c026284f --- /dev/null +++ b/.github/scorecard/lib.ts @@ -0,0 +1,159 @@ +/** + * Shared constants, types, and helpers for the module scorecard scripts. + */ + +export const REPO_OWNER = "coder"; +export const REPO_NAME = "registry"; +export const REPO_ID = "R_kgDOOVbRAA"; // coder/registry +export const CATEGORY_ID = "DIC_kwDOOVbRAM4DBMLT"; // "Modules" category + +export const MARKER = ""; +export const INDEX_MARKER = ""; +export const INDEX_TITLE = "📊 Module Scorecards"; + +export interface SummaryScores { + track: "Agent" | "IDE" | "Utility"; + presentation: string; + integration: string; + credential: string; + network: string; + engineering: string; + overall: string; + overallNum: number; +} + +export interface Result extends SummaryScores { + module: string; + name: string; + url: string; + scoredAt: string; +} + +export async function graphql( + query: string, + variables: Record = {}, +): Promise { + const res = await fetch("https://api.github.com/graphql", { + method: "POST", + headers: { + Authorization: `Bearer ${process.env.GITHUB_DISCUSSIONS_TOKEN}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ query, variables }), + }); + const body = (await res.json()) as { data?: T; errors?: unknown[] }; + if (!res.ok || body.errors?.length) { + throw new Error( + `GitHub GraphQL error: ${JSON.stringify(body.errors ?? body)}`, + ); + } + return body.data!; +} + +// Parses the theme-level summary table out of scorecard markdown (either a +// raw scorecard or a discussion body containing one). +export function parseSummary(markdown: string): SummaryScores | null { + const lines = markdown.split("\n"); + const headerIdx = lines.findIndex( + (l) => + l.startsWith("|") && l.includes("Overall") && l.includes("Presentation"), + ); + if (headerIdx === -1) return null; + const headers = lines[headerIdx] + .split("|") + .map((c) => c.trim()) + .filter(Boolean); + const cells = lines[headerIdx + 2] + ?.split("|") + .map((c) => c.trim().replace(/\*\*/g, "")) + .filter(Boolean); + if (!cells || cells.length !== headers.length) return null; + const get = (name: string) => { + const i = headers.findIndex((h) => h.toLowerCase().includes(name)); + return i === -1 ? "—" : cells[i]; + }; + const integrationIdx = headers.findIndex((h) => /integration/i.test(h)); + const track: SummaryScores["track"] = + integrationIdx === -1 + ? "Utility" + : /agent/i.test(headers[integrationIdx]) + ? "Agent" + : "IDE"; + const overall = get("overall"); + return { + track, + presentation: get("presentation"), + integration: integrationIdx === -1 ? "—" : cells[integrationIdx], + credential: get("credential"), + network: get("restricted"), + engineering: get("engineering"), + overall, + overallNum: Number(overall.match(/(\d+)\s*\/\s*100/)?.[1] ?? 0), + }; +} + +export interface DiscussionRef { + id: string; + title: string; + url: string; + body: string; +} + +export async function findDiscussionByTitle( + title: string, +): Promise { + const q = `"${title}" in:title repo:${REPO_OWNER}/${REPO_NAME}`; + const data = await graphql<{ search: { nodes: DiscussionRef[] } }>( + ` + query ($q: String!) { + search(query: $q, type: DISCUSSION, first: 10) { + nodes { + ... on Discussion { + id + title + url + body + } + } + } + } + `, + { q }, + ); + return data.search.nodes.find((n) => n.title === title) ?? null; +} + +// Finds a module's scorecard discussion by the registry URL in its body, +// regardless of title. Used to detect display_name renames, where the +// title lookup misses but the discussion still exists. GitHub search +// tokenizes URLs, so search the module slug and filter by the exact URL. +export async function findDiscussionByModuleUrl( + module: string, +): Promise { + const q = `${module} in:body repo:${REPO_OWNER}/${REPO_NAME}`; + const data = await graphql<{ search: { nodes: DiscussionRef[] } }>( + ` + query ($q: String!) { + search(query: $q, type: DISCUSSION, first: 20) { + nodes { + ... on Discussion { + id + title + url + body + } + } + } + } + `, + { q }, + ); + return ( + data.search.nodes.find( + (n) => + n.body.includes(MARKER) && + !n.body.includes(INDEX_MARKER) && + n.body.includes(`registry.coder.com/modules/coder/${module})`), + ) ?? null + ); +} diff --git a/.github/scorecard/score-modules.ts b/.github/scorecard/score-modules.ts new file mode 100644 index 000000000..f9b8b21a1 --- /dev/null +++ b/.github/scorecard/score-modules.ts @@ -0,0 +1,426 @@ +#!/usr/bin/env bun +/** + * Scores Coder Registry modules against SCORECARD.md using Claude, then + * posts (or updates) one GitHub Discussion per module in coder/registry. + * + * Env (required): + * ANTHROPIC_API_KEY Anthropic API key + * GITHUB_DISCUSSIONS_TOKEN GitHub PAT with Discussions read/write + * + * Usage: + * bun run score-modules.ts [--modules a,b,c] [--dry-run] [--limit N] + * + * --dry-run prints scorecards to stdout and skips GitHub entirely. + */ + +import { readdir, readFile } from "node:fs/promises"; +import { existsSync } from "node:fs"; +import path from "node:path"; +import { + CATEGORY_ID, + MARKER, + REPO_ID, + REPO_NAME, + REPO_OWNER, + type SummaryScores, + findDiscussionByModuleUrl, + findDiscussionByTitle, + graphql, + parseSummary, +} from "./lib"; + +const REGISTRY_ROOT = path.resolve(import.meta.dir, "..", ".."); +const MODULES_DIR = path.join(REGISTRY_ROOT, "registry", "coder", "modules"); +const SCORECARD_PATH = path.join(import.meta.dir, "SCORECARD.md"); + +const ANTHROPIC_MODEL = process.env.ANTHROPIC_MODEL ?? "claude-sonnet-4-5"; +const MAX_FILE_BYTES = 30_000; + +async function displayName(moduleName: string): Promise { + const readme = await readFile( + path.join(MODULES_DIR, moduleName, "README.md"), + "utf8", + ); + const match = readme.match(/^display_name:\s*["']?([^"'\n]+)["']?\s*$/m); + return match ? match[1].trim() : moduleName; +} + +// Internal building-block modules caution against direct use and are not +// meant to be consumed by template authors, so they are excluded from +// scoring entirely. +async function isInternalBuildingBlock(moduleName: string): Promise { + const readme = await readFile( + path.join(MODULES_DIR, moduleName, "README.md"), + "utf8", + ); + return /do not recommend using this module directly|not intended to be used directly|internal building block/i.test( + readme, + ); +} + +interface Args { + modules?: string[]; + dryRun: boolean; + limit?: number; + prReport?: string; +} + +function parseArgs(): Args { + const args: Args = { dryRun: false }; + const argv = process.argv.slice(2); + for (let i = 0; i < argv.length; i++) { + switch (argv[i]) { + case "--modules": + args.modules = argv[++i].split(",").map((s) => s.trim()); + break; + case "--dry-run": + args.dryRun = true; + break; + case "--limit": + args.limit = Number(argv[++i]); + break; + case "--pr-report": + args.prReport = argv[++i]; + break; + default: + console.error(`Unknown argument: ${argv[i]}`); + process.exit(1); + } + } + return args; +} + +async function readTruncated(filePath: string): Promise { + const content = await readFile(filePath, "utf8"); + if (content.length <= MAX_FILE_BYTES) return content; + return content.slice(0, MAX_FILE_BYTES) + "\n... [truncated]"; +} + +async function gatherModuleContext(moduleName: string): Promise { + const dir = path.join(MODULES_DIR, moduleName); + const parts: string[] = []; + const files = await readdir(dir, { recursive: true }); + const wanted = files.filter( + (f) => + typeof f === "string" && + (f.endsWith(".md") || + f.endsWith(".tf") || + f.endsWith(".tftest.hcl") || + f.endsWith(".test.ts") || + f.endsWith(".sh") || + f.endsWith(".tftpl")), + ) as string[]; + for (const f of wanted.sort()) { + const full = path.join(dir, f); + parts.push(`=== FILE: ${f} ===\n${await readTruncated(full)}`); + } + return parts.join("\n\n"); +} + +function fixOverall(scorecard: string): string { + // Find the summary table's data row: first row whose cells are bold + // "**a / b**" fractions (or N/A), ending with the overall cell. + const lines = scorecard.split("\n"); + const rowIdx = lines.findIndex( + (l) => /^\|\s*\*\*\d/.test(l.trim()) && l.includes("/ 100"), + ); + if (rowIdx === -1) return scorecard; + const cells = lines[rowIdx] + .split("|") + .map((c) => c.trim()) + .filter((c) => c !== ""); + const themeCells = cells.slice(0, -1); + let raw = 0; + let denom = 0; + for (const cell of themeCells) { + if (/n\/a/i.test(cell)) continue; + const m = cell.match(/(\d+(?:\.\d+)?)\s*\/\s*(\d+)/); + if (!m) return scorecard; // unexpected cell, leave untouched + raw += Number(m[1]); + denom += Number(m[2]); + } + if (denom === 0) return scorecard; + const overall = Math.round((raw / denom) * 100); + cells[cells.length - 1] = `**${overall} / 100**`; + lines[rowIdx] = `| ${cells.join(" | ")} |`; + let out = lines.join("\n"); + out = out.replace( + /#### Overall — \d+(?:\.\d+)? \/ 100/g, + `#### Overall — ${overall} / 100`, + ); + // Rewrite or append the normalization line under the Overall heading. + const normLine = + denom === 100 + ? "" + : `\n\nRaw ${raw} / ${denom} → round(${raw} / ${denom} × 100) = **${overall}**`; + out = out.replace(/\n+Raw \d+(?:\.\d+)?\s*\/\s*\d+[^\n]*/g, ""); + out = out.replace( + new RegExp(`(#### Overall — ${overall} / 100)`), + `$1${normLine}`, + ); + return out; +} + +async function scoreModule( + moduleName: string, + rubric: string, +): Promise { + const context = await gatherModuleContext(moduleName); + const prompt = `You are scoring a Coder Registry module against a scorecard rubric. + + +${rubric} + + + +${context} + + +Score the module strictly against the rubric. Rules: +- First decide the track: Agent, IDE, or Utility. Agent = AI coding agents (CLI agents like Claude Code, Goose, Aider). IDE = editors/IDEs users open (code-server, JetBrains, VS Code). Utility = everything else (auth, regions, git helpers, file browsers, etc). +- Score each criterion 0, half, or full. Full = implemented AND documented. Half = partial, awkward, or under-documented. Zero = absent. +- Apply "if applicable" exclusions per the rubric: when a concern does not exist by construction, mark the criterion N/A, exclude its points from the denominator, and normalize the final score to 100. +- Utility modules skip the track section (denominator 75 before any N/A exclusions), then normalize: round(raw / denominator * 100). +- Notes must be specific and evidence-based (reference actual variables, README sections, or files you saw). Never invent features. + +Output ONLY the scorecard markdown in EXACTLY this structure (this example shows an Agent module; use IDE Integration or omit the track section for Utility): + +| Presentation & Onboarding | Agent Integration | Credential Hygiene | Restricted-Network Readiness | Engineering Quality | Overall | +|---:|---:|---:|---:|---:|---:| +| **X / 25** | **X / 25** | **X / 20** | **X / 20 or N/A** | **X / 10** | **X / 100** | + +
+Drilldown + +#### Presentation & Onboarding — X / 25 + +| Criterion | Max | Score | Notes | +|---|---:|---:|---| +| Configuration-mode examples | 12 | X | ... | +| Coder-context framing | 8 | X | ... | +| Visual preview | 5 | X | ... | + +(...remaining theme tables in rubric order, highest weight first, each with per-criterion rows...) + +#### Overall — X / 100 + +(If normalized, show: Raw X / Y → round(X / Y × 100) = **Z**) + +
+ +Do not add any prose before or after the scorecard.`; + + const res = await fetch("https://api.anthropic.com/v1/messages", { + method: "POST", + headers: { + "x-api-key": process.env.ANTHROPIC_API_KEY!, + "anthropic-version": "2023-06-01", + "content-type": "application/json", + }, + body: JSON.stringify({ + model: ANTHROPIC_MODEL, + max_tokens: 4096, + temperature: 0, + messages: [{ role: "user", content: prompt }], + }), + }); + if (!res.ok) { + throw new Error(`Anthropic API error ${res.status}: ${await res.text()}`); + } + const data = (await res.json()) as { + content: { type: string; text?: string }[]; + }; + const text = data.content + .filter((c) => c.type === "text") + .map((c) => c.text) + .join("\n"); + return fixOverall(text.trim()); +} + +async function upsertDiscussion( + title: string, + body: string, +): Promise<{ id: string; url: string; created: boolean }> { + const existing = await findDiscussionByTitle(title); + if (existing) { + await graphql( + ` + mutation ($id: ID!, $body: String!) { + updateDiscussion(input: { discussionId: $id, body: $body }) { + discussion { + id + } + } + } + `, + { id: existing.id, body }, + ); + return { id: existing.id, url: existing.url, created: false }; + } + const data = await graphql<{ + createDiscussion: { discussion: { id: string; url: string } }; + }>( + ` + mutation ($repoId: ID!, $catId: ID!, $title: String!, $body: String!) { + createDiscussion( + input: { + repositoryId: $repoId + categoryId: $catId + title: $title + body: $body + } + ) { + discussion { + id + url + } + } + } + `, + { repoId: REPO_ID, catId: CATEGORY_ID, title, body }, + ); + return { ...data.createDiscussion.discussion, created: true }; +} + +// Builds one PR-report section comparing a fresh scorecard against the +// module's current discussion (the last scoring from main), so PRs can see +// whether they improve, regress, or hold the score. +function prReportSection( + mod: string, + name: string, + scorecard: string, + summary: SummaryScores | null, + baseline: (SummaryScores & { url: string }) | null, +): string { + const details = `
\nFull scorecard for this PR\n\n${scorecard}\n\n
`; + if (!summary) { + return `### \`coder/${mod}\`\n\nCould not parse the generated scorecard; see full output below.\n\n${details}`; + } + if (!baseline) { + return `### \`coder/${mod}\`: first scorecard, **${summary.overall}**\n\nNo existing scorecard discussion found for ${name}; this is the initial score. A dedicated discussion is created after merge.\n\n${details}`; + } + const delta = summary.overallNum - baseline.overallNum; + const themes: [string, string, string][] = [ + ["Presentation & Onboarding", baseline.presentation, summary.presentation], + ["Integration", baseline.integration, summary.integration], + ["Credential Hygiene", baseline.credential, summary.credential], + ["Restricted-Network", baseline.network, summary.network], + ["Engineering Quality", baseline.engineering, summary.engineering], + ["**Overall**", `**${baseline.overall}**`, `**${summary.overall}**`], + ]; + const table = [ + "| Theme | Before | After |", + "|---|---:|---:|", + ...themes.map(([t, b, a]) => `| ${t} | ${b} | ${a} |`), + ].join("\n"); + let verdict: string; + if (delta < 0) { + verdict = `\u26a0\ufe0f **Score regression**: ${baseline.overallNum} \u2192 ${summary.overallNum} (${delta}). Check the drilldown for which criteria dropped.`; + } else if (delta > 0) { + verdict = `\u2705 **Score improvement**: ${baseline.overallNum} \u2192 ${summary.overallNum} (+${delta}).`; + } else { + verdict = `\u2705 **Score unchanged** at ${summary.overall}. This PR does not affect the module's scorecard; the results are still good.`; + } + return `### [\`coder/${mod}\`](${baseline.url}): ${baseline.overallNum} \u2192 ${summary.overallNum}\n\n${verdict}\n\n${table}\n\n${details}`; +} + +function discussionBody( + moduleName: string, + name: string, + scorecard: string, +): string { + const now = new Date().toISOString().slice(0, 10); + return `A discussion dedicated to the [${name}](https://registry.coder.com/modules/coder/${moduleName}) module. Share your thoughts, questions, and feedback here. + +## Module Scorecard + +${scorecard} + +--- +Scored against [SCORECARD.md](https://github.com/${REPO_OWNER}/${REPO_NAME}/blob/main/.github/scorecard/SCORECARD.md) on ${now} with \`${ANTHROPIC_MODEL}\`. +${MARKER}`; +} + +async function main() { + const args = parseArgs(); + if (!process.env.ANTHROPIC_API_KEY) + throw new Error("ANTHROPIC_API_KEY not set"); + if (!args.dryRun && !process.env.GITHUB_DISCUSSIONS_TOKEN) { + throw new Error("GITHUB_DISCUSSIONS_TOKEN not set"); + } + + const rubric = await readFile(SCORECARD_PATH, "utf8"); + let modules = + args.modules ?? + (await readdir(MODULES_DIR, { withFileTypes: true })) + .filter((d) => d.isDirectory()) + .map((d) => d.name) + .sort(); + if (args.limit) modules = modules.slice(0, args.limit); + + const prSections: string[] = []; + for (const mod of modules) { + if (!existsSync(path.join(MODULES_DIR, mod, "README.md"))) { + console.error(`skip ${mod}: no README.md`); + continue; + } + if (await isInternalBuildingBlock(mod)) { + process.stderr.write(`skip ${mod}: internal building block\n`); + continue; + } + process.stderr.write(`scoring ${mod}... `); + try { + const scorecard = await scoreModule(mod, rubric); + if (args.dryRun) { + console.log(`\n===== coder/${mod} =====\n${scorecard}\n`); + process.stderr.write("done (dry-run)\n"); + continue; + } + const name = await displayName(mod); + if (args.prReport) { + // PR mode: compare against the module's current discussion and + // build a report instead of touching any discussion. + const summary = parseSummary(scorecard); + let existing = await findDiscussionByTitle(`${name} module`); + let renameWarning = ""; + if (!existing) { + // A display_name rename breaks the title lookup while the old + // discussion still exists. Fall back to the registry URL and + // flag it so a reviewer retitles or deletes the old one; the + // post-merge run creates a fresh discussion under the new name. + existing = await findDiscussionByModuleUrl(mod); + if (existing) { + renameWarning = `\n\n> [!WARNING]\n> This module's display name appears to have changed. Its existing scorecard discussion is titled "${existing.title}" (${existing.url}). After merge, a new discussion named "${name} module" will be created; a maintainer should retitle or delete the old one to avoid duplicates.`; + } + } + const parsed = existing ? parseSummary(existing.body) : null; + const baseline = + existing && parsed ? { ...parsed, url: existing.url } : null; + prSections.push( + prReportSection(mod, name, scorecard, summary, baseline) + + renameWarning, + ); + process.stderr.write("compared\n"); + continue; + } + const { url, created } = await upsertDiscussion( + `${name} module`, + discussionBody(mod, name, scorecard), + ); + process.stderr.write(`${created ? "created" : "updated"} ${url}\n`); + } catch (err) { + process.stderr.write(`FAILED: ${err}\n`); + } + } + + if (args.prReport) { + const report = + prSections.length > 0 + ? `## Module Scorecard Check\n\n${prSections.join("\n\n")}\n\n---\nScored against [SCORECARD.md](https://github.com/${REPO_OWNER}/${REPO_NAME}/blob/main/.github/scorecard/SCORECARD.md) with \`${ANTHROPIC_MODEL}\`. Language-model scores are advisory.\n` + : ""; + await Bun.write(args.prReport, report); + process.stderr.write(`wrote PR report to ${args.prReport}\n`); + } +} + +main(); diff --git a/.github/scorecard/scorecard-index.ts b/.github/scorecard/scorecard-index.ts new file mode 100644 index 000000000..768293c02 --- /dev/null +++ b/.github/scorecard/scorecard-index.ts @@ -0,0 +1,218 @@ +#!/usr/bin/env bun +/** + * Builds (or updates) the pinned index discussion listing every module + * scorecard with per-theme scores, grouped by track. Results are read back + * from the module scorecard discussions, so partial scoring runs still + * produce a complete index. + * + * Env (required): + * GITHUB_DISCUSSIONS_TOKEN GitHub PAT with Discussions read/write + * + * Usage: + * bun run scorecard-index.ts [--dry-run] + */ + +import { + CATEGORY_ID, + INDEX_MARKER, + INDEX_TITLE, + MARKER, + REPO_ID, + type Result, + findDiscussionByTitle, + graphql, + parseSummary, +} from "./lib"; + +interface Args { + dryRun: boolean; +} + +function parseArgs(): Args { + const args: Args = { dryRun: false }; + const argv = process.argv.slice(2); + for (let i = 0; i < argv.length; i++) { + switch (argv[i]) { + case "--dry-run": + args.dryRun = true; + break; + default: + console.error(`Unknown argument: ${argv[i]}`); + process.exit(1); + } + } + return args; +} + +async function fetchResults(): Promise> { + const results: Record = {}; + let cursor: string | null = null; + for (;;) { + const data: { + repository: { + discussions: { + pageInfo: { hasNextPage: boolean; endCursor: string }; + nodes: { title: string; url: string; body: string }[]; + }; + }; + } = await graphql( + `query($cursor: String) { + repository(owner: "coder", name: "registry") { + discussions(first: 100, after: $cursor, categoryId: "${CATEGORY_ID}") { + pageInfo { hasNextPage endCursor } + nodes { title url body } + } + } + }`, + { cursor }, + ); + for (const d of data.repository.discussions.nodes) { + if (!d.body.includes(MARKER) || d.body.includes(INDEX_MARKER)) continue; + const summary = parseSummary(d.body); + const module = d.body.match( + /registry\.coder\.com\/modules\/coder\/([a-z0-9-]+)/, + )?.[1]; + if (!summary || !module) { + console.error(`warn: could not parse "${d.title}"`); + continue; + } + results[module] = { + ...summary, + module, + name: d.title.replace(/ module$/, ""), + url: d.url, + scoredAt: new Date().toISOString(), + }; + } + if (!data.repository.discussions.pageInfo.hasNextPage) break; + cursor = data.repository.discussions.pageInfo.endCursor; + } + return results; +} + +function buildBody(results: Record): string { + const rank = (a: Result, b: Result) => b.overallNum - a.overallNum; + + const section = ( + title: string, + blurb: string, + rows: Result[], + withIntegration: boolean, + ): string => { + if (rows.length === 0) return ""; + const integrationHeader = withIntegration ? ` ${title} Integration |` : ""; + const integrationSep = withIntegration ? "---:|" : ""; + const body = rows + .sort(rank) + .map((r) => { + const integration = withIntegration ? ` ${r.integration} |` : ""; + return `| [${r.name}](${r.url}) | ${r.presentation} |${integration} ${r.credential} | ${r.network} | ${r.engineering} | **${r.overall}** |`; + }) + .join("\n"); + return `### ${title} modules + +${blurb} + +| Module | Presentation & Onboarding |${integrationHeader} Credential Hygiene | Restricted-Network | Engineering Quality | Overall | +|---|---:|${integrationSep}---:|---:|---:|---:| +${body} +`; + }; + + const all = Object.values(results); + const sections = [ + section( + "Agent", + "AI coding agents you can run in a workspace.", + all.filter((r) => r.track === "Agent"), + true, + ), + section( + "IDE", + "Editors and IDEs users open from the dashboard.", + all.filter((r) => r.track === "IDE"), + true, + ), + section( + "Utility", + "Auth, regions, git helpers, and other workspace plumbing. Scored on the universal themes only, normalized to 100.", + all.filter((r) => r.track === "Utility"), + false, + ), + ] + .filter(Boolean) + .join("\n"); + + const now = new Date().toISOString().slice(0, 10); + return `Every module in the \`coder\` namespace, scored against [SCORECARD.md](https://github.com/coder/registry/blob/main/.github/scorecard/SCORECARD.md). Each module links to its dedicated discussion, where you can share thoughts, questions, and feedback. + +**Why a scorecard?** It gives us a shared definition of quality (and a goalpost) for modules: what we care about for each one, in one place, for any contributor. It is not a strict gate today, though it may become one, and the criteria will evolve over time. It also helps catch regressions, for example a feature or documented capability being removed, since scores are re-evaluated whenever a module changes. + +**Looking for a way to contribute?** Low scores are contribution opportunities. Pick a module, open its discussion to see exactly which criteria it misses (visual previews, air-gapped install docs, tests, session persistence, and so on), and open a PR against [\`registry/coder/modules/\`](https://github.com/coder/registry/tree/main/registry/coder/modules). + +${sections} +**Notes**: N/A means the theme does not apply by construction and is excluded from the denominator. + +--- +Updated ${now}. Scores refresh when modules change. +${INDEX_MARKER}`; +} + +async function upsertIndex(body: string): Promise { + const existing = await findDiscussionByTitle(INDEX_TITLE); + if (existing) { + await graphql( + ` + mutation ($id: ID!, $body: String!) { + updateDiscussion(input: { discussionId: $id, body: $body }) { + discussion { + id + } + } + } + `, + { id: existing.id, body }, + ); + return existing.url; + } + const created = await graphql<{ + createDiscussion: { discussion: { url: string } }; + }>( + ` + mutation ($repoId: ID!, $catId: ID!, $title: String!, $body: String!) { + createDiscussion( + input: { + repositoryId: $repoId + categoryId: $catId + title: $title + body: $body + } + ) { + discussion { + url + } + } + } + `, + { repoId: REPO_ID, catId: CATEGORY_ID, title: INDEX_TITLE, body }, + ); + return created.createDiscussion.discussion.url; +} + +async function main() { + const args = parseArgs(); + if (!process.env.GITHUB_DISCUSSIONS_TOKEN) { + throw new Error("GITHUB_DISCUSSIONS_TOKEN not set"); + } + const results = await fetchResults(); + console.error(`found ${Object.keys(results).length} module scorecards`); + const body = buildBody(results); + if (args.dryRun) { + console.log(body); + return; + } + const url = await upsertIndex(body); + console.error(`index: ${url}`); +} + +main(); diff --git a/.github/workflows/module-scorecard-check.yaml b/.github/workflows/module-scorecard-check.yaml new file mode 100644 index 000000000..e36f431a6 --- /dev/null +++ b/.github/workflows/module-scorecard-check.yaml @@ -0,0 +1,83 @@ +# Scores modules changed in a pull request against +# .github/scorecard/SCORECARD.md and posts a sticky comment comparing the +# PR's score to the module's current scorecard discussion. Flags +# regressions, celebrates improvements, and confirms unchanged scores. +# +# Never creates or updates discussions; the comparison is read-only against +# GitHub and only writes the PR comment. Skipped for fork PRs because it +# requires secrets. +# +# Required repository secrets: +# SCORECARD_ANTHROPIC_API_KEY Anthropic API key used for scoring +# SCORECARD_DISCUSSIONS_TOKEN GitHub PAT with Discussions read (baseline lookup) + +name: Module Scorecard Check + +on: + pull_request: + paths: + - "registry/coder/modules/**" + +permissions: + contents: read + pull-requests: write + +concurrency: + group: module-scorecard-check-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + check: + name: Compare module scores against main + if: github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: latest + + - name: Determine changed modules + id: changed + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + MODULES=$(git diff --name-only "${BASE_SHA}"...HEAD | grep -oP '^registry/coder/modules/\K[^/]+' | sort -u | paste -sd, -) + echo "modules=${MODULES}" >> "${GITHUB_OUTPUT}" + echo "Changed modules: ${MODULES:-none}" + + - name: Score changed modules + if: steps.changed.outputs.modules != '' + env: + ANTHROPIC_API_KEY: ${{ secrets.SCORECARD_ANTHROPIC_API_KEY }} + GITHUB_DISCUSSIONS_TOKEN: ${{ secrets.SCORECARD_DISCUSSIONS_TOKEN }} + MODULES: ${{ steps.changed.outputs.modules }} + run: bun run .github/scorecard/score-modules.ts --modules "${MODULES}" --pr-report /tmp/scorecard-report.md + + - name: Upsert PR comment + if: steps.changed.outputs.modules != '' + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + # No report means every changed module was skipped (for example, + # internal building blocks); leave no comment in that case. + if [[ ! -s /tmp/scorecard-report.md ]]; then + echo "No scorecard report generated; skipping comment." + exit 0 + fi + MARKER="" + COMMENT_ID=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate --jq ".[] | select(.body | contains(\"${MARKER}\")) | .id" | head -1) + if [[ -n "${COMMENT_ID}" ]]; then + gh api --method PATCH "repos/${GITHUB_REPOSITORY}/issues/comments/${COMMENT_ID}" -F body=@/tmp/scorecard-report.md > /dev/null + echo "Updated comment ${COMMENT_ID}" + else + gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" -F body=@/tmp/scorecard-report.md > /dev/null + echo "Created comment" + fi diff --git a/.github/workflows/module-scorecards.yaml b/.github/workflows/module-scorecards.yaml new file mode 100644 index 000000000..fe737e2dc --- /dev/null +++ b/.github/workflows/module-scorecards.yaml @@ -0,0 +1,97 @@ +# Scores modules in the coder namespace against +# .github/scorecard/SCORECARD.md using Claude, then posts (or updates) one +# GitHub Discussion per module plus the pinned "📊 Module Scorecards" index. +# +# The rubric lives in .github/scorecard/SCORECARD.md. The scoring script +# reads it directly at runtime, so editing that file changes the criteria +# for the next run, and pushing a change to it re-runs full scoring. +# Pushes that touch individual modules re-score only those modules; the +# index is rebuilt either way. +# +# Required repository secrets: +# SCORECARD_ANTHROPIC_API_KEY Anthropic API key used for scoring +# SCORECARD_DISCUSSIONS_TOKEN GitHub PAT with Discussions read/write + +name: Module Scorecards + +on: + schedule: + # Weekly, Mondays 06:00 UTC + - cron: "0 6 * * 1" + push: + branches: + - main + paths: + - ".github/scorecard/SCORECARD.md" + - "registry/coder/modules/**" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: module-scorecards + cancel-in-progress: false + +jobs: + score: + name: Score modules and update discussions + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: latest + + - name: Determine scope + id: scope + env: + EVENT_NAME: ${{ github.event_name }} + BEFORE: ${{ github.event.before }} + AFTER: ${{ github.sha }} + run: | + # Full run for schedule, manual dispatch, rubric changes, or when + # the previous commit is unavailable (force push, new branch). + if [[ "${EVENT_NAME}" != "push" ]]; then + echo "args=" >> "${GITHUB_OUTPUT}" + echo "Full run (${EVENT_NAME})" + exit 0 + fi + if [[ "${BEFORE}" == "0000000000000000000000000000000000000000" ]] || ! git cat-file -e "${BEFORE}" 2>/dev/null; then + echo "args=" >> "${GITHUB_OUTPUT}" + echo "Full run (no previous commit)" + exit 0 + fi + CHANGED=$(git diff --name-only "${BEFORE}" "${AFTER}") + if grep -q "^\.github/scorecard/SCORECARD\.md$" <<< "${CHANGED}"; then + echo "args=" >> "${GITHUB_OUTPUT}" + echo "Full run (rubric changed)" + exit 0 + fi + MODULES=$(grep -oP '^registry/coder/modules/\K[^/]+' <<< "${CHANGED}" | sort -u | paste -sd, -) + if [[ -z "${MODULES}" ]]; then + echo "args=" >> "${GITHUB_OUTPUT}" + echo "Full run (no module paths matched)" + else + echo "args=--modules ${MODULES}" >> "${GITHUB_OUTPUT}" + echo "Scoped run: ${MODULES}" + fi + + - name: Score modules and update module discussions + env: + ANTHROPIC_API_KEY: ${{ secrets.SCORECARD_ANTHROPIC_API_KEY }} + GITHUB_DISCUSSIONS_TOKEN: ${{ secrets.SCORECARD_DISCUSSIONS_TOKEN }} + run: bun run .github/scorecard/score-modules.ts ${{ steps.scope.outputs.args }} + + - name: Update index discussion + env: + GITHUB_DISCUSSIONS_TOKEN: ${{ secrets.SCORECARD_DISCUSSIONS_TOKEN }} + # The index reads scores back from the module discussions, so a + # single-module run still produces a complete index. + run: bun run .github/scorecard/scorecard-index.ts