feat: dist-diff — show reviewers what a PR changes in the built artifacts

.github/workflows/dist-diff-comment.yml

@@ -0,0 +1,85 @@
+name: dist-diff comment
+
+# Posts the dist-diff report as a single sticky PR comment, edited in place on
+# every push. Split from dist-diff.yml (workflow_run) so the report job stays
+# unprivileged and fork PRs get the same comment as internal ones.
+#
+# This is the only privileged part of dist-diff. It treats the artifact as
+# data: the PR number is verified against the run's head SHA before posting,
+# and the report body is only ever posted as comment text.
+
+on:
+  workflow_run:
+    workflows: [dist-diff]
+    types: [completed]
+
+permissions:
+  pull-requests: write
+
+jobs:
+  comment:
+    # No conclusion gate: a failed self-check still uploads a (warning-stamped)
+    # report, and the PR should get its comment either way. Runs where the
+    # report step itself died leave no artifact — the download step's outcome
+    # check below skips the comment quietly in that case.
+    if: github.event.workflow_run.event == 'pull_request'
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Download the report artifact
+        id: download
+        continue-on-error: true
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: dist-diff-report
+          path: dist-diff-report
+          run-id: ${{ github.event.workflow_run.id }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Upsert the sticky comment
+        if: steps.download.outcome == 'success'
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        with:
+          script: |
+            const fs = require('fs');
+            const marker = '<!-- dist-diff -->';
+
+            const prNumber = parseInt(fs.readFileSync('dist-diff-report/pr-number', 'utf8').trim(), 10);
+            if (!Number.isInteger(prNumber)) throw new Error('invalid PR number in artifact');
+
+            // The artifact comes from an unprivileged run of untrusted code:
+            // only post to the PR whose head produced this exact run.
+            const { data: pr } = await github.rest.pulls.get({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: prNumber,
+            });
+            if (pr.head.sha !== context.payload.workflow_run.head_sha) {
+              throw new Error('PR number in artifact does not match the run head SHA');
+            }
+
+            let body = `${marker}\n${fs.readFileSync('dist-diff-report/comment.md', 'utf8')}`;
+            if (body.length > 65000) body = `${body.slice(0, 65000)}\n…truncated — see the workflow summary.`;
+
+            const comments = await github.paginate(github.rest.issues.listComments, {
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: prNumber,
+              per_page: 100,
+            });
+            const existing = comments.find(c => c.body?.startsWith(marker));
+            if (existing) {
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: existing.id,
+                body,
+              });
+            } else {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: prNumber,
+                body,
+              });
+            }

.github/workflows/dist-diff.yml

@@ -0,0 +1,112 @@
+name: dist-diff
+
+# Shows reviewers what a PR changes in the BUILT artifacts (dist/) — the
+# skill menu the wizard reads, the marketplace plugins, the MCP manifest —
+# which a source diff of configs/markdown can't show. Runs unprivileged;
+# the sticky PR comment is posted by dist-diff-comment.yml (workflow_run),
+# so fork PRs get the same report as internal ones.
+#
+# Normalization guarantee: dist-diff reports are only trustworthy if two
+# builds of the same ref produce an empty normalized diff. That self-check
+# runs on every push to main (catches environment drift) and on PRs that
+# touch the build pipeline or dependencies (the only changes that can
+# introduce new nondeterminism) — content PRs can never go red here.
+# A failed self-check still renders and uploads the report (stamped with a
+# warning) before failing the job, so the PR never silently loses its comment.
+
+on:
+  pull_request:
+    branches: [main]
+  push:
+    branches: [main]
+
+permissions:
+  contents: read
+
+jobs:
+  dist-diff:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
+        with:
+          node-version: lts/*
+
+      - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0
+        with:
+          version: latest
+
+      - name: Install dependencies
+        run: pnpm install
+
+      - name: Scope the run
+        id: scope
+        run: |
+          if [ "${{ github.event_name }}" = "push" ]; then
+            echo "self-check=true" >> "$GITHUB_OUTPUT"
+          else
+            MERGE_BASE=$(git merge-base "origin/${{ github.base_ref }}" HEAD)
+            echo "merge-base=$MERGE_BASE" >> "$GITHUB_OUTPUT"
+            if git diff --name-only "$MERGE_BASE"...HEAD | grep -qE '^(scripts/|package\.json$|pnpm-lock\.yaml$|\.github/workflows/dist-diff)'; then
+              echo "self-check=true" >> "$GITHUB_OUTPUT"
+            else
+              echo "self-check=false" >> "$GITHUB_OUTPUT"
+            fi
+          fi
+
+      - name: Build head
+        # Also primes .docs-cache, which every later build in this job shares —
+        # so upstream doc changes between builds cannot leak into the diff.
+        run: pnpm build
+
+      - name: Self-check — two builds of the same ref must normalize identically
+        id: selfcheck
+        if: steps.scope.outputs.self-check == 'true'
+        # Deferred failure: the report steps below still run, then the last
+        # step turns a failed self-check into a red job.
+        continue-on-error: true
+        run: |
+          mv dist dist-head-first
+          pnpm build
+          pnpm --silent diff --exit-code dist-head-first dist
+          rm -rf dist-head-first
+
+      - name: Build merge-base and render the report
+        if: github.event_name == 'pull_request'
+        run: |
+          git worktree add /tmp/base "${{ steps.scope.outputs.merge-base }}"
+          ln -s "$PWD/node_modules" /tmp/base/node_modules
+          mkdir -p .docs-cache
+          ln -s "$PWD/.docs-cache" /tmp/base/.docs-cache
+          (cd /tmp/base && pnpm build)
+
+          RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          WARN=""
+          if [ "${{ steps.selfcheck.outcome }}" = "failure" ]; then
+            WARN="> ⚠️ **Self-check failed on this run** — two builds of the same ref differ after normalization, so this report may contain noise. See the job log."
+          fi
+
+          mkdir -p dist-diff-report
+          { [ -n "$WARN" ] && printf '%s\n\n' "$WARN"; \
+            pnpm --silent diff /tmp/base/dist dist --format comment --summary-url "$RUN_URL"; } > dist-diff-report/comment.md
+          { [ -n "$WARN" ] && printf '%s\n\n' "$WARN"; \
+            pnpm --silent diff /tmp/base/dist dist --format full; } >> "$GITHUB_STEP_SUMMARY"
+          echo "${{ github.event.pull_request.number }}" > dist-diff-report/pr-number
+
+      - name: Upload report for the comment workflow
+        if: github.event_name == 'pull_request'
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: dist-diff-report
+          path: dist-diff-report/
+
+      - name: Fail the job on a failed self-check
+        if: steps.selfcheck.outcome == 'failure'
+        run: |
+          echo "::error::dist-diff self-check failed: the build is nondeterministic. Add a normalization rule in scripts/lib/dist-diff.js for the file(s) named above."
+          exit 1

package.json

@@ -7,6 +7,7 @@
   "scripts": {
     "build": "node scripts/build.js",
     "dev": "node scripts/dev-server.js",
+    "diff": "node scripts/diff-dist.js",
     "visual-dags": "node scripts/visual-dags.js",
     "test:plugins": "vitest run scripts/plugins/tests",
     "test:plugins:watch": "vitest scripts/plugins/tests",
@@ -19,6 +20,8 @@
   "devDependencies": {
     "archiver": "^7.0.1",
     "chokidar": "^4.0.3",
+    "diff": "9.0.0",
+    "fflate": "0.8.3",
     "vitest": "^2.0.0"
   },
   "engines": {