promote: differing same-name copies are an error, jax is considered, and a consumer series class (project-monorepo#74)

.github/workflows/drift-check.yml

@@ -18,6 +18,19 @@ name: drift-check
 #
 # The refresh lands as a pull request rather than a push to main so a human
 # sees the ledger diff: a refresh can change which entries are `interim`.
+# An entry whose refresh fails does not hold the others back: promote still
+# writes every entry it could refresh, the pull request is opened with those
+# and lists each failure with promote's reason, and the run is marked RED.
+# Reasons include a same-name copy edited to differ (settled by
+# sync/canonical.yml or upstream), a sync/canonical.yml line naming a copy
+# that has fallen behind, a canonical copy removed or renamed upstream, an
+# asset that cannot be moved or overwritten, and a canonical series this run
+# excludes. A merely stale copy stays a source and does not fail.
+# A refused entry keeps its old ledger record, so while its canonical copy
+# differs from that record the drift-check asks for a refresh every night:
+# the run stays red until the entry is settled, and a settlement applies the
+# next night. An entry refused when only its series' pin moved (its canonical
+# copy unchanged) is retried only when some canonical copy next moves (#20).
 # PERMISSIONS: uses the default GITHUB_TOKEN. A PR opened with it does not
 # trigger this workflow's own pull_request run; once branch protection is on,
 # the push to `drift/refresh` still works (it is not a protected branch).
@@ -44,10 +57,18 @@ concurrency:
   cancel-in-progress: false
 
 env:
-  # M2 posture (PLAN.md): the real `dp` series is excluded from promotion while
-  # the `dp-test` sandbox stands in for it. Retired at the switch-over
-  # (QuantEcon/project-monorepo#24), when this becomes empty.
-  PROMOTE_EXCLUDE: dp
+  # Series every refresh leaves out (promote --exclude). `dp`: the real dp
+  # series is excluded while the dp-test sandbox stands in for it (M2 posture,
+  # PLAN.md); it leaves this list at QuantEcon/project-monorepo#24. `jax`: it
+  # used to be excluded inside tools/promote and is now excluded here, by the
+  # same per-run flag. With jax considered, short_path fails the refresh
+  # (intro's and jax's are different lectures), while ifp_egm's
+  # sync/canonical.yml line records jax's copy as divergent instead; drop jax
+  # when its renames land (QuantEcon/project-monorepo#76). A refresh fails an
+  # entry whose canonical series is excluded once that series' pin moves, so
+  # jax must leave this list before any jax-only lecture is promoted. An
+  # empty list is fine: the flag is then left off.
+  PROMOTE_EXCLUDE: dp jax
 
 jobs:
   check:
@@ -111,17 +132,48 @@ jobs:
         run: uv run --script tools/sync
 
       - name: Refresh the pool entries whose canonical source moved
-        run: uv run --script tools/promote --refresh --exclude $PROMOTE_EXCLUDE
+        id: refresh
+        # promote writes every entry it can refresh and exits 1 if any entry
+        # failed. The exit status is kept, not acted on here, so the entries
+        # that did refresh still reach the pull request below; the last step
+        # turns the run red.
+        run: |
+          set +e
+          uv run --script tools/promote --refresh ${PROMOTE_EXCLUDE:+--exclude $PROMOTE_EXCLUDE} 2>&1 | tee promote.log
+          rc=${PIPESTATUS[0]}
+          set -e
+          echo "rc=$rc" >> "$GITHUB_OUTPUT"
+          if [ "$rc" = 0 ]; then
+            echo "promote --refresh exit status: 0 (every entry refreshed, or nothing to refresh)"
+          elif grep -q '^summary \[refresh\]' promote.log; then
+            echo "promote --refresh exit status: $rc (some entries were refused -- see the FAILED lines)"
+          else
+            echo "promote --refresh exit status: $rc (it stopped before its summary -- see the next step)"
+          fi
 
       - name: Open or update the refresh pull request
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
           git config user.name  'github-actions[bot]'
           git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
+          # A failing promote prints its summary only after every write, the
+          # ledger included. A non-zero exit without it means the run stopped
+          # before or during its writes (a configuration error, or a crash; not
+          # a refused entry), so what is on disk may be half-written: open nothing.
+          if [ "${{ steps.refresh.outputs.rc }}" != "0" ] && ! grep -q '^summary \[refresh\]' promote.log; then
+            echo "::error::tools/promote --refresh exited ${{ steps.refresh.outputs.rc }} before its summary: a configuration error (see the error line in the refresh step) or a crash part-way. Its writes may be partial, so no pull request is opened."
+            exit 1
+          fi
+          failed=$(grep -E '^\[[^]]+\] FAILED: ' promote.log || true)
           git add lectures sync/ledger.yml
           if git diff --cached --quiet; then
-            echo 'promote --refresh changed nothing -- no pull request.'
+            if [ -n "$failed" ]; then
+              echo 'promote --refresh refreshed nothing; every entry it tried failed:'
+              echo "$failed"
+            else
+              echo 'promote --refresh changed nothing -- no pull request.'
+            fi
             exit 0
           fi
           before=$(git show HEAD:sync/ledger.yml | grep -c 'interim: true' || true)
@@ -131,12 +183,20 @@ jobs:
           git commit -m "promote: refresh $n pool lectures whose canonical source moved"
           git push --force -u origin drift/refresh
           {
-            echo "\`tools/drift-check\` found pool entries whose **canonical copy moved upstream**; \`tools/promote --refresh --exclude $PROMOTE_EXCLUDE\` re-promoted them. Review the ledger diff, not just the lecture text: a refresh re-resolves every source, so an entry whose other copies went stale can change shape."
+            echo "\`tools/drift-check\` found pool entries whose **canonical copy moved upstream**; \`tools/promote --refresh${PROMOTE_EXCLUDE:+ --exclude $PROMOTE_EXCLUDE}\` re-promoted them. Review the ledger diff, not just the lecture text: a refresh re-resolves every source, so an entry whose other copies went stale can change shape."
             echo
             echo "| | before | after |"
             echo "|---|---|---|"
             echo "| \`interim: true\` ledger entries | $before | $after |"
             echo "| lectures re-copied | | $n |"
+            if [ -n "$failed" ]; then
+              echo
+              echo "**Not refreshed.** \`tools/promote\` refused these entries, so their pool copies stay at the previous pin and the run that opened this PR is red. Each line gives promote's reason. A same-name copy edited to differ is settled by naming the winning series in \`sync/canonical.yml\` or by reconciling the copies upstream. A \`sync/canonical.yml\` line that names a lagging copy is settled by pointing it at the recorded canonical series or removing it. A canonical copy removed or renamed upstream needs a decision, which the drift-check's \`canonical-removed\` issue tracks. The line itself explains the other reasons. An entry whose canonical copy moved is refused again every night, so the run stays red until it is settled; an entry refused when only its series' pin moved is retried when a canonical copy next moves."
+              echo
+              echo '```'
+              echo "$failed"
+              echo '```'
+            fi
             echo
             echo "Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}. The branch is force-pushed from \`${{ github.event.repository.default_branch }}\` on every run that still needs a refresh, so this PR always shows the current delta."
             echo
@@ -151,3 +211,9 @@ jobs:
           else
             gh pr create --head drift/refresh --title "promote: refresh $n pool lectures whose canonical source moved" --body-file pr-body.md
           fi
+
+      - name: Keep the run red while any entry could not be refreshed
+        if: steps.refresh.outputs.rc != '0'
+        run: |
+          echo "::error::tools/promote --refresh could not refresh every entry (exit ${{ steps.refresh.outputs.rc }}). The FAILED lines in the refresh step give each entry and promote's reason; any entries it did refresh are in the drift/refresh pull request."
+          exit 1

.github/workflows/tests.yml

@@ -0,0 +1,34 @@
+name: tests
+
+# Run the tools' test suite (tools/tests/). Each test builds a small scratch
+# repository in a temp dir — mirror, manifest, state, empty pool — and runs a
+# copy of tools/promote or tools/drift-check against it, so the suite needs no
+# network, no mirror reconstruction and no access to the real pool.
+#
+# Dependencies are declared inline, as the tools themselves declare theirs
+# (PEP 723), so there is no project file or lockfile to keep in step.
+#
+# Pull requests and manual runs only: a push to a branch triggers nothing.
+
+on:
+  pull_request:
+    paths:
+      - 'tools/**'
+      - '.github/workflows/tests.yml'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  pytest:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+
+      - name: Run the tools' tests
+        run: uv run --no-project --with pytest --with pyyaml pytest tools/tests -v

README.md

@@ -55,15 +55,18 @@ The repository is organised around three layers. Only the first exists in a mean
 ├── products/         # (future) one manifest per published product — empty
 ├── mirror/           # reconstructed, verbatim, GITIGNORED — never committed
 ├── sync/
-│   ├── manifest.yml  # human intent: which repos feed the mirror
+│   ├── manifest.yml  # human intent: which repos feed the mirror, and each series' class
+│   ├── canonical.yml # human intent: the winning series for a same-name collision
 │   └── state.yml     # machine state: pinned SHAs (bot-written)
 ├── tools/
 │   ├── sync          # reconstructs the mirror from pinned SHAs
 │   ├── promote       # promotes lectures + their asset closure into the pool; writes the ledger
-│   └── drift-check   # proves the pool still equals its sources; issues, refresh PR
+│   ├── drift-check   # proves the pool still equals its sources; issues, refresh PR
+│   └── tests/        # pytest suite for promote and drift-check (scratch repos)
 └── .github/workflows/
     ├── sync.yml         # daily job that refreshes the pinned SHAs
-    └── drift-check.yml  # runs after sync: drift issues, refresh PR, red on a pool edit
+    ├── drift-check.yml  # runs after sync: drift issues, refresh PR, red on a pool edit or a refused refresh
+    └── tests.yml        # runs tools/tests on pull requests
 ```
 
 ## Running the sync
@@ -95,7 +98,11 @@ A series' pin (and its timestamp) is only rewritten when the upstream `HEAD` has
 | `tools/drift-check --issues` | Open, update and close `drift` issues to match the findings (CI). |
 | `tools/drift-check --accept-toc dp-test` | Record a series' `_toc.yml` at its current pin as the watched baseline. |
 
-Four outcomes: **refresh** (the canonical copy moved upstream; CI runs `promote --refresh` and opens a pull request), **drift** (a non-canonical copy diverged from its canonical home, a copy was removed, or a watched toc changed; each becomes an issue that closes itself when the finding clears), **violation** (the pool itself was edited; the job goes red), and **info** (a copy is merely stale, or a divergence converged). [`drift-check.yml`](.github/workflows/drift-check.yml) runs it after every `sync`.
+Four outcomes: **refresh** (the canonical copy moved upstream; CI runs `promote --refresh --exclude $PROMOTE_EXCLUDE` and opens a pull request; an entry the refresh refuses is listed with promote's reason, in that pull request when one is opened and always in the red run's refresh step; the usual reason is a same-name copy edited to differ, which `sync/canonical.yml` or upstream settles; while the entry's canonical copy differs from its ledger record the run stays red every night until it is settled, and a settlement applies the next night), **drift** (a non-canonical copy diverged from its canonical home, a copy was removed, or a watched toc changed; each becomes an issue that closes itself when the finding clears), **violation** (the pool itself was edited; the job goes red), and **info** (a copy is merely stale, or a divergence converged). [`drift-check.yml`](.github/workflows/drift-check.yml) runs it after every `sync`.
+
+One kind of copy is never watched. A **consumer** series (`class: consumer` in `sync/manifest.yml`, today `dp`) only republishes other series' lectures, so its copies are recorded as `superseded` and are expected to go stale.
+
+`tools/promote` refuses a lecture name held by two canonical-eligible series with different text, naming the lecture and every series involved. It does not pick a winner by run order. A rename or merge upstream fixes it; until then, [`sync/canonical.yml`](sync/canonical.yml) can name the winning series, and the entry is recorded as `interim`. A copy that is merely stale (a recorded source still at a version the pool already took, while the canonical copy moved on) is not a collision: it stays a source, and the refresh goes through. A `sync/canonical.yml` line that names such a stale copy is refused, though, since following it would take the pool back to the lagging text, so delete a line as soon as promote reports that it settles nothing.
 
 ## Roadmap / coming next
 

sync/canonical.yml

@@ -0,0 +1,55 @@
+# sync/canonical.yml — HUMAN INTENT
+#
+# The canonical map: which series' copy the pool takes when two or more
+# canonical-eligible series hold DIFFERENT files under one lecture name.
+#
+# tools/promote treats a same-named file as one lecture, so a copy that
+# differs from the rest (after asset paths are normalised) is refused with an
+# error naming the lecture and every series involved. Which copy survives must
+# never fall out of run order. An entry here is the one exception: it names
+# the winning series for that slug. The losing copies are recorded under
+# `divergent:` in sync/ledger.yml, the entry is marked `interim: true`, and
+# the drift-check reports their movement as info.
+#
+# An entry is a stop-gap. The fix is a rename or a merge upstream, after
+# which the entry must be deleted promptly: promote warns when an entry
+# settles nothing, and once the copy it names falls behind the canonical
+# copy, the entry is refused rather than followed.
+# A consumer series (`class: consumer` in sync/manifest.yml) is never
+# canonical and cannot appear here; its copies are recorded as superseded.
+#
+# Format: one `<slug>: <series>` line per lecture, series keys as in
+# sync/manifest.yml.
+#
+# Seeded 2026-10-06 (QuantEcon/project-monorepo#74) from the 24 entries that
+# were `interim: true` in sync/ledger.yml: each keeps the canonical copy that
+# promotion had picked by run order before this map existed, so the pool is
+# unchanged. Every one is a dp-test collision, retired with dp-test itself.
+# An entry covers the slug, not a pair of series: `ifp_egm` below also
+# settles jax's different ifp_egm whenever jax takes part (the CI refresh
+# excludes jax), recording jax's copy as divergent instead of failing.
+
+career: intermediate                        # over dp-test
+cons_news: dp-test                          # over advanced
+ifp_advanced: intermediate                  # over dp-test
+ifp_egm: intermediate                       # over dp-test (and jax, see above)
+ifp_egm_transient_shocks: intermediate      # over dp-test
+inventory_q: intermediate                   # over dp-test
+jv: intermediate                            # over dp-test
+lagrangian_lqdp: dp-test                    # over intermediate
+lqcontrol: dp-test                          # over intermediate
+mccall_fitted_vfi: intermediate             # over dp-test
+mccall_model: intermediate                  # over dp-test
+mccall_model_with_sep_markov: dp-test       # over intermediate
+mccall_persist_trans: intermediate          # over dp-test
+mccall_q: intermediate                      # over dp-test
+odu: dp-test                                # over intermediate
+opt_tax_recur: advanced                     # over dp-test
+os: dp-test                                 # over intermediate
+os_egm: dp-test                             # over intermediate
+os_egm_jax: dp-test                         # over intermediate
+os_stochastic: intermediate                 # over dp-test
+os_time_iter: intermediate                  # over dp-test
+perm_income: dp-test                        # over intermediate
+perm_income_cons: dp-test                   # over intermediate
+rs_inventory_q: intermediate                # over dp-test

sync/ledger.yml

@@ -4,9 +4,14 @@
 # provenance for every pool asset. Asset/rewrite paths are relative to the
 # pool root (lectures/); source `path` values are relative to
 # mirror/<series>/lectures/.
-# Every lecture `sources`/`divergent` item records the sha256 `digest` of
-# that series' copy AS OF PROMOTION — the drift-check's baseline for
-# telling fresh upstream movement from already-recorded divergence.
+# Every lecture `sources`/`divergent`/`superseded` item records the sha256
+# `digest` of that series' copy AS OF PROMOTION — the drift-check's
+# baseline for telling fresh upstream movement from already-recorded
+# divergence. `superseded` items are consumer-series copies: provenance
+# only, never watched. A `sources` item whose digest differs from the
+# canonical one is either a STALE copy (still at an earlier promoted
+# version while the canonical copy moved on) or a copy equal to the
+# canonical one once asset paths are normalised.
 # `rewrites` is an ordered list of literal replacements: applying each
 # {from -> to} in turn to the canonical mirror copy reproduces the pool
 # copy byte-for-byte. Each pair is ONE non-rescanning pass with Python

sync/manifest.yml

@@ -10,7 +10,14 @@
 #
 # Series are ordered by curriculum progression: intro -> programming ->
 # intermediate -> advanced -> dp -> jax. (dp-test is a temporary test double,
-# not part of the progression.)
+# not part of the progression.) The order is also tools/promote's dedup
+# priority among identical copies.
+#
+# `class:` is optional. A series without it (or with `class: source`) is
+# canonical-eligible. `class: consumer` marks a series that republishes other
+# series' lectures: it is never canonical, contributes no lecture to the pool,
+# and tools/promote records its copies as superseded, which tools/drift-check
+# never watches.
 
 defaults:
   branch: main
@@ -29,10 +36,15 @@ series:
     repo: QuantEcon/lecture-python-advanced.myst
   dp:
     repo: QuantEcon/lecture-dp
+    # Every dp lecture is a copy of one in another series (often an older
+    # one), so dp consumes the pool rather than feeding it.
+    class: consumer
   # TEST DOUBLE (M2 development only): a disposable public copy of lecture-dp
   # that we deliberately edit to exercise promotion/drift tooling — the live
-  # repos stay read-only. Excluded from canonical-home consideration when the
-  # real dp set is promoted; removed at the M3 switch-over. See its README.
+  # repos stay read-only. Canonical-eligible like any series, so its edited
+  # copies collide with their originals: sync/canonical.yml settles each one
+  # for now, and both go when dp-test is removed at the M3 switch-over
+  # (QuantEcon/project-monorepo#24). See its README.
   dp-test:
     repo: QuantEcon/lecture-dp.monorepo
   jax: