diff --git a/.github/workflows/drift-check.yml b/.github/workflows/drift-check.yml index 911ffe3..ea09a28 100644 --- a/.github/workflows/drift-check.yml +++ b/.github/workflows/drift-check.yml @@ -18,6 +18,19 @@ name: drift-check # # The refresh lands as a pull request rather than a push to main so a human # sees the ledger diff: a refresh can change which entries are `interim`. +# An entry whose refresh fails does not hold the others back: promote still +# writes every entry it could refresh, the pull request is opened with those +# and lists each failure with promote's reason, and the run is marked RED. +# Reasons include a same-name copy edited to differ (settled by +# sync/canonical.yml or upstream), a sync/canonical.yml line naming a copy +# that has fallen behind, a canonical copy removed or renamed upstream, an +# asset that cannot be moved or overwritten, and a canonical series this run +# excludes. A merely stale copy stays a source and does not fail. +# A refused entry keeps its old ledger record, so while its canonical copy +# differs from that record the drift-check asks for a refresh every night: +# the run stays red until the entry is settled, and a settlement applies the +# next night. An entry refused when only its series' pin moved (its canonical +# copy unchanged) is retried only when some canonical copy next moves (#20). # PERMISSIONS: uses the default GITHUB_TOKEN. A PR opened with it does not # trigger this workflow's own pull_request run; once branch protection is on, # the push to `drift/refresh` still works (it is not a protected branch). @@ -44,10 +57,18 @@ concurrency: cancel-in-progress: false env: - # M2 posture (PLAN.md): the real `dp` series is excluded from promotion while - # the `dp-test` sandbox stands in for it. Retired at the switch-over - # (QuantEcon/project-monorepo#24), when this becomes empty. - PROMOTE_EXCLUDE: dp + # Series every refresh leaves out (promote --exclude). `dp`: the real dp + # series is excluded while the dp-test sandbox stands in for it (M2 posture, + # PLAN.md); it leaves this list at QuantEcon/project-monorepo#24. `jax`: it + # used to be excluded inside tools/promote and is now excluded here, by the + # same per-run flag. With jax considered, short_path fails the refresh + # (intro's and jax's are different lectures), while ifp_egm's + # sync/canonical.yml line records jax's copy as divergent instead; drop jax + # when its renames land (QuantEcon/project-monorepo#76). A refresh fails an + # entry whose canonical series is excluded once that series' pin moves, so + # jax must leave this list before any jax-only lecture is promoted. An + # empty list is fine: the flag is then left off. + PROMOTE_EXCLUDE: dp jax jobs: check: @@ -111,7 +132,24 @@ jobs: run: uv run --script tools/sync - name: Refresh the pool entries whose canonical source moved - run: uv run --script tools/promote --refresh --exclude $PROMOTE_EXCLUDE + id: refresh + # promote writes every entry it can refresh and exits 1 if any entry + # failed. The exit status is kept, not acted on here, so the entries + # that did refresh still reach the pull request below; the last step + # turns the run red. + run: | + set +e + uv run --script tools/promote --refresh ${PROMOTE_EXCLUDE:+--exclude $PROMOTE_EXCLUDE} 2>&1 | tee promote.log + rc=${PIPESTATUS[0]} + set -e + echo "rc=$rc" >> "$GITHUB_OUTPUT" + if [ "$rc" = 0 ]; then + echo "promote --refresh exit status: 0 (every entry refreshed, or nothing to refresh)" + elif grep -q '^summary \[refresh\]' promote.log; then + echo "promote --refresh exit status: $rc (some entries were refused -- see the FAILED lines)" + else + echo "promote --refresh exit status: $rc (it stopped before its summary -- see the next step)" + fi - name: Open or update the refresh pull request env: @@ -119,9 +157,23 @@ jobs: run: | git config user.name 'github-actions[bot]' git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + # A failing promote prints its summary only after every write, the + # ledger included. A non-zero exit without it means the run stopped + # before or during its writes (a configuration error, or a crash; not + # a refused entry), so what is on disk may be half-written: open nothing. + if [ "${{ steps.refresh.outputs.rc }}" != "0" ] && ! grep -q '^summary \[refresh\]' promote.log; then + echo "::error::tools/promote --refresh exited ${{ steps.refresh.outputs.rc }} before its summary: a configuration error (see the error line in the refresh step) or a crash part-way. Its writes may be partial, so no pull request is opened." + exit 1 + fi + failed=$(grep -E '^\[[^]]+\] FAILED: ' promote.log || true) git add lectures sync/ledger.yml if git diff --cached --quiet; then - echo 'promote --refresh changed nothing -- no pull request.' + if [ -n "$failed" ]; then + echo 'promote --refresh refreshed nothing; every entry it tried failed:' + echo "$failed" + else + echo 'promote --refresh changed nothing -- no pull request.' + fi exit 0 fi before=$(git show HEAD:sync/ledger.yml | grep -c 'interim: true' || true) @@ -131,12 +183,20 @@ jobs: git commit -m "promote: refresh $n pool lectures whose canonical source moved" git push --force -u origin drift/refresh { - echo "\`tools/drift-check\` found pool entries whose **canonical copy moved upstream**; \`tools/promote --refresh --exclude $PROMOTE_EXCLUDE\` re-promoted them. Review the ledger diff, not just the lecture text: a refresh re-resolves every source, so an entry whose other copies went stale can change shape." + echo "\`tools/drift-check\` found pool entries whose **canonical copy moved upstream**; \`tools/promote --refresh${PROMOTE_EXCLUDE:+ --exclude $PROMOTE_EXCLUDE}\` re-promoted them. Review the ledger diff, not just the lecture text: a refresh re-resolves every source, so an entry whose other copies went stale can change shape." echo echo "| | before | after |" echo "|---|---|---|" echo "| \`interim: true\` ledger entries | $before | $after |" echo "| lectures re-copied | | $n |" + if [ -n "$failed" ]; then + echo + echo "**Not refreshed.** \`tools/promote\` refused these entries, so their pool copies stay at the previous pin and the run that opened this PR is red. Each line gives promote's reason. A same-name copy edited to differ is settled by naming the winning series in \`sync/canonical.yml\` or by reconciling the copies upstream. A \`sync/canonical.yml\` line that names a lagging copy is settled by pointing it at the recorded canonical series or removing it. A canonical copy removed or renamed upstream needs a decision, which the drift-check's \`canonical-removed\` issue tracks. The line itself explains the other reasons. An entry whose canonical copy moved is refused again every night, so the run stays red until it is settled; an entry refused when only its series' pin moved is retried when a canonical copy next moves." + echo + echo '```' + echo "$failed" + echo '```' + fi echo echo "Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}. The branch is force-pushed from \`${{ github.event.repository.default_branch }}\` on every run that still needs a refresh, so this PR always shows the current delta." echo @@ -151,3 +211,9 @@ jobs: else gh pr create --head drift/refresh --title "promote: refresh $n pool lectures whose canonical source moved" --body-file pr-body.md fi + + - name: Keep the run red while any entry could not be refreshed + if: steps.refresh.outputs.rc != '0' + run: | + echo "::error::tools/promote --refresh could not refresh every entry (exit ${{ steps.refresh.outputs.rc }}). The FAILED lines in the refresh step give each entry and promote's reason; any entries it did refresh are in the drift/refresh pull request." + exit 1 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..8c975a2 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,34 @@ +name: tests + +# Run the tools' test suite (tools/tests/). Each test builds a small scratch +# repository in a temp dir — mirror, manifest, state, empty pool — and runs a +# copy of tools/promote or tools/drift-check against it, so the suite needs no +# network, no mirror reconstruction and no access to the real pool. +# +# Dependencies are declared inline, as the tools themselves declare theirs +# (PEP 723), so there is no project file or lockfile to keep in step. +# +# Pull requests and manual runs only: a push to a branch triggers nothing. + +on: + pull_request: + paths: + - 'tools/**' + - '.github/workflows/tests.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + pytest: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v6 + + - name: Run the tools' tests + run: uv run --no-project --with pytest --with pyyaml pytest tools/tests -v diff --git a/README.md b/README.md index cf86ed4..fe4463e 100644 --- a/README.md +++ b/README.md @@ -55,15 +55,18 @@ The repository is organised around three layers. Only the first exists in a mean ├── products/ # (future) one manifest per published product — empty ├── mirror/ # reconstructed, verbatim, GITIGNORED — never committed ├── sync/ -│ ├── manifest.yml # human intent: which repos feed the mirror +│ ├── manifest.yml # human intent: which repos feed the mirror, and each series' class +│ ├── canonical.yml # human intent: the winning series for a same-name collision │ └── state.yml # machine state: pinned SHAs (bot-written) ├── tools/ │ ├── sync # reconstructs the mirror from pinned SHAs │ ├── promote # promotes lectures + their asset closure into the pool; writes the ledger -│ └── drift-check # proves the pool still equals its sources; issues, refresh PR +│ ├── drift-check # proves the pool still equals its sources; issues, refresh PR +│ └── tests/ # pytest suite for promote and drift-check (scratch repos) └── .github/workflows/ ├── sync.yml # daily job that refreshes the pinned SHAs - └── drift-check.yml # runs after sync: drift issues, refresh PR, red on a pool edit + ├── drift-check.yml # runs after sync: drift issues, refresh PR, red on a pool edit or a refused refresh + └── tests.yml # runs tools/tests on pull requests ``` ## Running the sync @@ -95,7 +98,11 @@ A series' pin (and its timestamp) is only rewritten when the upstream `HEAD` has | `tools/drift-check --issues` | Open, update and close `drift` issues to match the findings (CI). | | `tools/drift-check --accept-toc dp-test` | Record a series' `_toc.yml` at its current pin as the watched baseline. | -Four outcomes: **refresh** (the canonical copy moved upstream; CI runs `promote --refresh` and opens a pull request), **drift** (a non-canonical copy diverged from its canonical home, a copy was removed, or a watched toc changed; each becomes an issue that closes itself when the finding clears), **violation** (the pool itself was edited; the job goes red), and **info** (a copy is merely stale, or a divergence converged). [`drift-check.yml`](.github/workflows/drift-check.yml) runs it after every `sync`. +Four outcomes: **refresh** (the canonical copy moved upstream; CI runs `promote --refresh --exclude $PROMOTE_EXCLUDE` and opens a pull request; an entry the refresh refuses is listed with promote's reason, in that pull request when one is opened and always in the red run's refresh step; the usual reason is a same-name copy edited to differ, which `sync/canonical.yml` or upstream settles; while the entry's canonical copy differs from its ledger record the run stays red every night until it is settled, and a settlement applies the next night), **drift** (a non-canonical copy diverged from its canonical home, a copy was removed, or a watched toc changed; each becomes an issue that closes itself when the finding clears), **violation** (the pool itself was edited; the job goes red), and **info** (a copy is merely stale, or a divergence converged). [`drift-check.yml`](.github/workflows/drift-check.yml) runs it after every `sync`. + +One kind of copy is never watched. A **consumer** series (`class: consumer` in `sync/manifest.yml`, today `dp`) only republishes other series' lectures, so its copies are recorded as `superseded` and are expected to go stale. + +`tools/promote` refuses a lecture name held by two canonical-eligible series with different text, naming the lecture and every series involved. It does not pick a winner by run order. A rename or merge upstream fixes it; until then, [`sync/canonical.yml`](sync/canonical.yml) can name the winning series, and the entry is recorded as `interim`. A copy that is merely stale (a recorded source still at a version the pool already took, while the canonical copy moved on) is not a collision: it stays a source, and the refresh goes through. A `sync/canonical.yml` line that names such a stale copy is refused, though, since following it would take the pool back to the lagging text, so delete a line as soon as promote reports that it settles nothing. ## Roadmap / coming next diff --git a/sync/canonical.yml b/sync/canonical.yml new file mode 100644 index 0000000..10174ca --- /dev/null +++ b/sync/canonical.yml @@ -0,0 +1,55 @@ +# sync/canonical.yml — HUMAN INTENT +# +# The canonical map: which series' copy the pool takes when two or more +# canonical-eligible series hold DIFFERENT files under one lecture name. +# +# tools/promote treats a same-named file as one lecture, so a copy that +# differs from the rest (after asset paths are normalised) is refused with an +# error naming the lecture and every series involved. Which copy survives must +# never fall out of run order. An entry here is the one exception: it names +# the winning series for that slug. The losing copies are recorded under +# `divergent:` in sync/ledger.yml, the entry is marked `interim: true`, and +# the drift-check reports their movement as info. +# +# An entry is a stop-gap. The fix is a rename or a merge upstream, after +# which the entry must be deleted promptly: promote warns when an entry +# settles nothing, and once the copy it names falls behind the canonical +# copy, the entry is refused rather than followed. +# A consumer series (`class: consumer` in sync/manifest.yml) is never +# canonical and cannot appear here; its copies are recorded as superseded. +# +# Format: one `: ` line per lecture, series keys as in +# sync/manifest.yml. +# +# Seeded 2026-10-06 (QuantEcon/project-monorepo#74) from the 24 entries that +# were `interim: true` in sync/ledger.yml: each keeps the canonical copy that +# promotion had picked by run order before this map existed, so the pool is +# unchanged. Every one is a dp-test collision, retired with dp-test itself. +# An entry covers the slug, not a pair of series: `ifp_egm` below also +# settles jax's different ifp_egm whenever jax takes part (the CI refresh +# excludes jax), recording jax's copy as divergent instead of failing. + +career: intermediate # over dp-test +cons_news: dp-test # over advanced +ifp_advanced: intermediate # over dp-test +ifp_egm: intermediate # over dp-test (and jax, see above) +ifp_egm_transient_shocks: intermediate # over dp-test +inventory_q: intermediate # over dp-test +jv: intermediate # over dp-test +lagrangian_lqdp: dp-test # over intermediate +lqcontrol: dp-test # over intermediate +mccall_fitted_vfi: intermediate # over dp-test +mccall_model: intermediate # over dp-test +mccall_model_with_sep_markov: dp-test # over intermediate +mccall_persist_trans: intermediate # over dp-test +mccall_q: intermediate # over dp-test +odu: dp-test # over intermediate +opt_tax_recur: advanced # over dp-test +os: dp-test # over intermediate +os_egm: dp-test # over intermediate +os_egm_jax: dp-test # over intermediate +os_stochastic: intermediate # over dp-test +os_time_iter: intermediate # over dp-test +perm_income: dp-test # over intermediate +perm_income_cons: dp-test # over intermediate +rs_inventory_q: intermediate # over dp-test diff --git a/sync/ledger.yml b/sync/ledger.yml index ae46cf6..1f8c299 100644 --- a/sync/ledger.yml +++ b/sync/ledger.yml @@ -4,9 +4,14 @@ # provenance for every pool asset. Asset/rewrite paths are relative to the # pool root (lectures/); source `path` values are relative to # mirror//lectures/. -# Every lecture `sources`/`divergent` item records the sha256 `digest` of -# that series' copy AS OF PROMOTION — the drift-check's baseline for -# telling fresh upstream movement from already-recorded divergence. +# Every lecture `sources`/`divergent`/`superseded` item records the sha256 +# `digest` of that series' copy AS OF PROMOTION — the drift-check's +# baseline for telling fresh upstream movement from already-recorded +# divergence. `superseded` items are consumer-series copies: provenance +# only, never watched. A `sources` item whose digest differs from the +# canonical one is either a STALE copy (still at an earlier promoted +# version while the canonical copy moved on) or a copy equal to the +# canonical one once asset paths are normalised. # `rewrites` is an ordered list of literal replacements: applying each # {from -> to} in turn to the canonical mirror copy reproduces the pool # copy byte-for-byte. Each pair is ONE non-rescanning pass with Python diff --git a/sync/manifest.yml b/sync/manifest.yml index f982604..af18cac 100644 --- a/sync/manifest.yml +++ b/sync/manifest.yml @@ -10,7 +10,14 @@ # # Series are ordered by curriculum progression: intro -> programming -> # intermediate -> advanced -> dp -> jax. (dp-test is a temporary test double, -# not part of the progression.) +# not part of the progression.) The order is also tools/promote's dedup +# priority among identical copies. +# +# `class:` is optional. A series without it (or with `class: source`) is +# canonical-eligible. `class: consumer` marks a series that republishes other +# series' lectures: it is never canonical, contributes no lecture to the pool, +# and tools/promote records its copies as superseded, which tools/drift-check +# never watches. defaults: branch: main @@ -29,10 +36,15 @@ series: repo: QuantEcon/lecture-python-advanced.myst dp: repo: QuantEcon/lecture-dp + # Every dp lecture is a copy of one in another series (often an older + # one), so dp consumes the pool rather than feeding it. + class: consumer # TEST DOUBLE (M2 development only): a disposable public copy of lecture-dp # that we deliberately edit to exercise promotion/drift tooling — the live - # repos stay read-only. Excluded from canonical-home consideration when the - # real dp set is promoted; removed at the M3 switch-over. See its README. + # repos stay read-only. Canonical-eligible like any series, so its edited + # copies collide with their originals: sync/canonical.yml settles each one + # for now, and both go when dp-test is removed at the M3 switch-over + # (QuantEcon/project-monorepo#24). See its README. dp-test: repo: QuantEcon/lecture-dp.monorepo jax: diff --git a/tools/drift-check b/tools/drift-check index eb75877..1157d8f 100755 --- a/tools/drift-check +++ b/tools/drift-check @@ -12,19 +12,33 @@ same lecture. This script compares all of that against the reconstructed ``mirror/`` and sorts what it finds into four outcomes: * ``refresh`` — the canonical copy moved upstream. The pool copy is stale, not - wrong: ``tools/promote --refresh`` re-copies it. CI does this automatically + wrong: ``tools/promote --refresh`` re-copies it. CI does this automatically, + with the exclusions in PROMOTE_EXCLUDE (.github/workflows/drift-check.yml), and opens a pull request. -* ``drift`` — a NON-canonical copy (today: the dp copy of an advanced or - intermediate lecture) no longer matches its canonical home, or a copy was +* ``drift`` — a NON-canonical copy (today: the dp-test copy of an advanced + or intermediate lecture) no longer matches its canonical home, or a copy was removed upstream, or a watched ``_toc.yml`` changed. Each becomes a GitHub issue (``--issues``) that closes itself when the finding clears. * ``violation`` — the POOL itself no longer equals rewrites(canonical), or a pool asset changed: someone edited the pool directly, which the upstream- first rule forbids. Exit code 2; a red run is the alarm. * ``info`` — nothing to act on now, but worth reading: a non-canonical copy - that is merely stale (canonical moved, the copy did not), an ``interim`` + that is merely stale (canonical moved, the copy did not, or the copy is at + the canonical copy's last promoted version), an ``interim`` entry whose divergent copy moved again, or a divergence that converged. +A non-canonical copy is judged by the rule promotion admitted it under: it is +the same lecture as the canonical copy when it is byte-identical, or identical +once each copy's asset references are normalised to their pool paths (series +spell ``/_static/...`` and ``_static/...`` for the same file). That comparison +is ``tools/promote``'s own ``normalised_text``, imported for the purpose, so +the two tools cannot disagree about which copies are one lecture. + +One kind of copy is deliberately NOT watched. A consumer series' copies +(``class: consumer`` in ``sync/manifest.yml``, recorded under ``superseded:`` +in the ledger, or under ``sources``/``divergent`` by a ledger written before +the class was declared) are stale by design and raise nothing. + Usage ----- tools/drift-check # report against the existing mirror/ @@ -83,6 +97,8 @@ TOC_STATE_PATH = ROOT / "sync" / "toc.yml" DRIFT_LABEL = "drift" DIFF_LINE_CAP = 300 # lines of unified diff kept in an issue body +CONSUMER_CLASS = "consumer" + # --------------------------------------------------------------------------- # # Small helpers # @@ -152,6 +168,50 @@ def mirror_lectures(series: str) -> Path: return MIRROR_DIR / series / "lectures" +def consumer_series(manifest: dict) -> set: + """Series declared ``class: consumer``: never canonical, never watched.""" + return {k for k, v in manifest.items() if (v or {}).get("class") == CONSUMER_CLASS} + + +PROMOTE_PATH = Path(__file__).resolve().parent / "promote" +_promote = None + + +def promote_module(): + """``tools/promote`` imported as a module, once, on first use. Only its + ``normalised_text`` is wanted: the form two same-name copies are compared + in at promotion. Importing it (rather than copying the asset parser here) + keeps the drift-check's idea of "one lecture" identical to promote's.""" + global _promote + if _promote is None: + import importlib.util + from importlib.machinery import SourceFileLoader + + loader = SourceFileLoader("qe_promote", str(PROMOTE_PATH)) + spec = importlib.util.spec_from_loader(loader.name, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + _promote = module + return _promote + + +def same_lecture(series: str, canonical: str, path: str, cur: bytes, can_bytes: bytes) -> bool: + """Is the ``series`` copy at ``path`` the same lecture as ``canonical``'s, + by promotion's rule: byte-identical, or identical once each copy's asset + references are normalised to the pool paths promotion would give them. + A copy that is not UTF-8 text is never the same lecture.""" + if cur == can_bytes: + return True + if not path.endswith(".md"): + return False + slug = path[:-3] + promote = promote_module() + try: + return promote.normalised_text(series, slug) == promote.normalised_text(canonical, slug) + except UnicodeDecodeError: + return False + + # --------------------------------------------------------------------------- # # Findings # # --------------------------------------------------------------------------- # @@ -193,8 +253,9 @@ class Finding: # --------------------------------------------------------------------------- # # The checks # # --------------------------------------------------------------------------- # -def series_in_ledger(ledger_lectures, ledger_assets, toc_state): - """Every series whose mirror the checks read.""" +def series_in_ledger(ledger_lectures, ledger_assets, toc_state, consumers): + """Every series whose mirror the checks read. Consumer copies are never + read, so they need no mirror.""" seen = [] def add(s): @@ -203,20 +264,20 @@ def series_in_ledger(ledger_lectures, ledger_assets, toc_state): for entry in ledger_lectures.values(): add(entry.get("canonical")) - for s in entry.get("sources") or []: - add(s.get("series")) - for s in entry.get("divergent") or []: - add(s.get("series")) + for s in (entry.get("sources") or []) + (entry.get("divergent") or []): + if s.get("series") not in consumers: + add(s.get("series")) for meta in ledger_assets.values(): add(meta.get("canonical")) for s in meta.get("sources") or []: - add(s.get("series")) + if s.get("series") not in consumers: + add(s.get("series")) for s in toc_state: add(s) return seen -def check_lecture(key: str, entry: dict, manifest: dict, findings: list): +def check_lecture(key: str, entry: dict, manifest: dict, consumers: set, findings: list): canonical = entry.get("canonical") sources = entry.get("sources") or [] divergent = entry.get("divergent") or [] @@ -297,9 +358,10 @@ def check_lecture(key: str, entry: dict, manifest: dict, findings: list): )) # Non-canonical copies: did any of them stop matching their canonical home? + # A consumer's copy is stale by design and is never watched. for s in sources: series = s.get("series") - if series == canonical: + if series == canonical or series in consumers: continue p = mirror_lectures(series) / s["path"] cur = read_bytes(p) @@ -320,10 +382,21 @@ def check_lecture(key: str, entry: dict, manifest: dict, findings: list): data={"series": series, "path": s["path"]}, )) continue - if cur == can_bytes: + if same_lecture(series, canonical, s["path"], cur, can_bytes): + exact = cur == can_bytes findings.append(Finding( "info", "converged", key, - f"{key}: the {series} copy changed and now matches {canonical} exactly", + f"{key}: the {series} copy changed and now matches {canonical}" + + (" exactly" if exact else " up to asset path spelling"), + data={"series": series, "path": s["path"], "exact": exact}, + )) + continue + if canonical_moved and sha256_bytes(cur) == can_src.get("digest"): + # It caught up with the canonical copy as last promoted, and the + # canonical copy has since moved on: stale, not drift. + findings.append(Finding( + "info", "stale-copy", key, + f"{key}: the {series} copy is at {canonical}'s last promoted version while {canonical} moved — {series} is now stale", data={"series": series, "path": s["path"]}, )) continue @@ -331,9 +404,10 @@ def check_lecture(key: str, entry: dict, manifest: dict, findings: list): "drift", "diverged", key, f"{key}: the {series} copy lectures/{s['path']} no longer matches its canonical home in {canonical}", detail=( - f"At promotion the {series} copy was byte-identical to {canonical}'s. It has since been edited " - f"on its own. Under the upstream-first rule the fix belongs in {canonical} (and flows down); " - f"if the edit is a deliberate variant, record it as one rather than leaving a silent fork." + f"At promotion the {series} copy was the same lecture as {canonical}'s (byte-identical, or " + f"identical up to asset path spelling). It has since been edited on its own. Under the " + f"upstream-first rule the fix belongs in {canonical} (and flows down); if the edit is a " + f"deliberate variant, record it as one rather than leaving a silent fork." ), diff=unified_diff(can_bytes, cur, f"{canonical}/lectures/{can_src['path']}", f"{series}/lectures/{s['path']}"), issue_title=f"drift({series}): lectures/{s['path']} differs from its canonical copy in {canonical}", @@ -343,6 +417,8 @@ def check_lecture(key: str, entry: dict, manifest: dict, findings: list): # Already-divergent copies (interim entries): did they move again, or converge? for d in divergent: series = d.get("series") + if series in consumers: + continue p = mirror_lectures(series) / d["path"] cur = read_bytes(p) if cur is None: @@ -354,11 +430,14 @@ def check_lecture(key: str, entry: dict, manifest: dict, findings: list): continue if sha256_bytes(cur) == d.get("digest"): continue - if cur == can_bytes: + if same_lecture(series, canonical, d["path"], cur, can_bytes): + exact = cur == can_bytes findings.append(Finding( "info", "converged", key, - f"{key}: the divergent {series} copy now matches {canonical} exactly — the interim entry can be re-promoted", - data={"series": series, "path": d["path"], "interim": interim}, + f"{key}: the divergent {series} copy now matches {canonical}" + + (" exactly" if exact else " up to asset path spelling") + + " — the interim entry can be re-promoted", + data={"series": series, "path": d["path"], "interim": interim, "exact": exact}, )) else: findings.append(Finding( @@ -368,7 +447,7 @@ def check_lecture(key: str, entry: dict, manifest: dict, findings: list): )) -def check_asset(pool_rel: str, meta: dict, findings: list): +def check_asset(pool_rel: str, meta: dict, consumers: set, findings: list): canonical = meta.get("canonical") sources = meta.get("sources") or [] recorded = meta.get("digest") @@ -415,7 +494,7 @@ def check_asset(pool_rel: str, meta: dict, findings: list): for s in sources: series = s.get("series") - if series == canonical: + if series == canonical or series in consumers: continue cur = read_bytes(mirror_lectures(series) / s["path"]) if cur is None: @@ -637,7 +716,7 @@ def default_repo(): HEADINGS = { "violation": "VIOLATIONS — the pool no longer equals rewrites(canonical); fix before anything else", "drift": "DRIFT — a copy diverged from its canonical home (each becomes an issue under --issues)", - "refresh": "REFRESH — the canonical copy moved; `tools/promote --refresh` brings the pool up to date", + "refresh": "REFRESH — the canonical copy moved; `tools/promote --refresh` (CI adds --exclude $PROMOTE_EXCLUDE) brings the pool up to date", "info": "INFO — nothing to act on now", } @@ -697,6 +776,7 @@ def main(argv=None): ledger_lectures = {k: v for k, v in ledger.items() if isinstance(v, dict)} state = load_yaml(STATE_PATH) manifest = (load_yaml(MANIFEST_PATH).get("series")) or {} + consumers = consumer_series(manifest) toc_state = load_yaml(TOC_STATE_PATH) try: @@ -709,7 +789,7 @@ def main(argv=None): accept_tocs(args.accept_toc, state, toc_state) return 0 - needed = series_in_ledger(ledger_lectures, ledger_assets, toc_state) + needed = series_in_ledger(ledger_lectures, ledger_assets, toc_state, consumers) if args.reconstruct: reconstruct(needed) missing = [s for s in needed if not mirror_lectures(s).exists()] @@ -721,9 +801,9 @@ def main(argv=None): findings = [] for key in sorted(ledger_lectures): - check_lecture(key, ledger_lectures[key], manifest, findings) + check_lecture(key, ledger_lectures[key], manifest, consumers, findings) for pool_rel in sorted(ledger_assets): - check_asset(pool_rel, ledger_assets[pool_rel], findings) + check_asset(pool_rel, ledger_assets[pool_rel], consumers, findings) check_tocs(toc_state, manifest, findings) findings.sort(key=lambda f: (SEVERITY_ORDER[f.severity], f.subject, f.kind)) diff --git a/tools/promote b/tools/promote index 29c1e7d..efaa99e 100755 --- a/tools/promote +++ b/tools/promote @@ -14,10 +14,31 @@ assets enter the pool, so orphans are excluded by construction. Dedup happens at promotion time: the same-named copy is hashed across every non-excluded series in ``sync/manifest.yml``; the pool takes the copy from the CANONICAL series (the highest-priority series, in manifest order, whose copy is -byte-identical to the requested source) and records every identical series as -a source. Same-named copies that differ in bytes are recorded under -``divergent:`` and the entry is marked ``interim: true`` (canonical pending -human triage). ``jax`` is ALWAYS excluded (variant regime). +identical to the requested source) and records every identical series as a +source. "Identical" means byte-identical, or identical once each copy's asset +references are normalised to the pool paths promotion would give them. + +A same-named copy that DIFFERS is treated as a different lecture under the same +name, and promotion refuses it: the error names the lecture and every series +involved. A copy that is merely STALE is not different: a series recorded as a +source at the entry's last promotion whose copy is still at a version recorded +then (its own, or the canonical copy's), while the canonical copy moved on, +stays a source (the drift-check's ``stale-copy``, which is info). Only a copy +edited to something else is refused, and a canonical-map entry that names a +stale copy, since following it would take the pool back to the lagging text. +Which copy the pool keeps never falls out of run order. The one exception is +the committed canonical map, ``sync/canonical.yml`` (human intent), which +names the winning series per slug; the losing copies are then recorded under +``divergent:`` and the entry is marked ``interim: true`` until a rename or +merge upstream retires the map entry. + +Series classes (``class:`` on a manifest stanza). The default class is +canonical-eligible. A ``consumer`` series (one that republishes other series' +lectures) is never canonical and contributes no lecture to the pool; its +same-named copies, equal or not, are recorded under ``superseded:`` and are +NOT watched by the drift-check. No series is excluded by default: ``jax`` +takes part like any other, so until its renames land its colliding names fail +by the rule above. ``--exclude`` drops series for one run. Asset references are normalised into the pool ``_static`` conventions (``_static//`` for per-lecture assets, ``_static/_shared/`` for assets @@ -45,14 +66,19 @@ Flags --toc SERIES take the slug list from mirror/SERIES/lectures/_toc.yml (product pages — intro, status, troubleshooting, zreferences — are skipped: they become products// - pages in M3, not pool content). + pages in M3, not pool content). For a consumer series + each lecture comes from its canonical home instead. --refresh re-run the promotion pipeline for every ledger entry whose canonical series pin in sync/state.yml differs from its recorded promoted_at pin; unchanged pins are a - strict no-op. + strict no-op. An entry whose canonical series this run + cannot use (excluded, or a consumer) is skipped while + its pin is unchanged and FAILS once the pin has moved. + CI runs it with --exclude $PROMOTE_EXCLUDE (see + .github/workflows/drift-check.yml). --exclude SERIES [SERIES...] series excluded from source/dedup consideration for - this run (jax is always excluded by default). + this run (none is excluded by default). --dry-run print the full plan (sources, closures, rewrites, collisions, skips) without writing anything. -h/--help show help and exit. @@ -93,11 +119,16 @@ SYNC_DIR = REPO_ROOT / "sync" MANIFEST_PATH = SYNC_DIR / "manifest.yml" STATE_PATH = SYNC_DIR / "state.yml" LEDGER_PATH = SYNC_DIR / "ledger.yml" +CANONICAL_MAP_PATH = SYNC_DIR / "canonical.yml" MIRROR_DIR = REPO_ROOT / "mirror" POOL_DIR = REPO_ROOT / "lectures" -# Series that never participate as a source or in dedup (variant regime). -ALWAYS_EXCLUDED = ("jax",) +# Series classes (``class:`` on a manifest stanza). The default is canonical- +# eligible; a consumer republishes other series' lectures, so it is never +# canonical, contributes no lecture, and its copies are recorded as superseded. +SOURCE_CLASS = "source" +CONSUMER_CLASS = "consumer" +SERIES_CLASSES = (SOURCE_CLASS, CONSUMER_CLASS) # Per-series boilerplate: product pages, not pool content (M3 owns them). PRODUCT_PAGES = ("intro", "status", "troubleshooting", "zreferences") @@ -118,9 +149,14 @@ LEDGER_HEADER = ( "# provenance for every pool asset. Asset/rewrite paths are relative to the\n" "# pool root (lectures/); source `path` values are relative to\n" "# mirror//lectures/.\n" - "# Every lecture `sources`/`divergent` item records the sha256 `digest` of\n" - "# that series' copy AS OF PROMOTION — the drift-check's baseline for\n" - "# telling fresh upstream movement from already-recorded divergence.\n" + "# Every lecture `sources`/`divergent`/`superseded` item records the sha256\n" + "# `digest` of that series' copy AS OF PROMOTION — the drift-check's\n" + "# baseline for telling fresh upstream movement from already-recorded\n" + "# divergence. `superseded` items are consumer-series copies: provenance\n" + "# only, never watched. A `sources` item whose digest differs from the\n" + "# canonical one is either a STALE copy (still at an earlier promoted\n" + "# version while the canonical copy moved on) or a copy equal to the\n" + "# canonical one once asset paths are normalised.\n" "# `rewrites` is an ordered list of literal replacements: applying each\n" "# {from -> to} in turn to the canonical mirror copy reproduces the pool\n" "# copy byte-for-byte. Each pair is ONE non-rescanning pass with Python\n" @@ -308,12 +344,41 @@ def slug_of(name: str) -> str: # --------------------------------------------------------------------------- # # Manifest / state / ledger I/O # # --------------------------------------------------------------------------- # -def load_yaml(path: Path): +class UniqueKeyLoader(yaml.SafeLoader): + """A SafeLoader that refuses a mapping with a repeated key, which plain + YAML loading silently resolves to the last value.""" + + +def _construct_unique_mapping(loader, node, deep=False): + loader.flatten_mapping(node) # resolve merge keys first, as safe_load does + seen = set() + for key_node, _ in node.value: + key = loader.construct_object(key_node, deep=deep) + try: + hash(key) + except TypeError: + raise yaml.constructor.ConstructorError( + None, None, "found an unhashable key", key_node.start_mark + ) + if key in seen: + raise yaml.constructor.ConstructorError( + None, None, f"repeated key {key!r}", key_node.start_mark + ) + seen.add(key) + return loader.construct_mapping(node, deep=deep) + + +UniqueKeyLoader.add_constructor( + yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _construct_unique_mapping +) + + +def load_yaml(path: Path, loader=yaml.SafeLoader): """Load a YAML mapping, tolerating an empty/None document (-> {}).""" if not path.exists(): return None try: - data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + data = yaml.load(path.read_text(encoding="utf-8"), Loader=loader) or {} except yaml.YAMLError as exc: raise SystemExit(f"error: could not parse {path}:\n{exc}") if not isinstance(data, dict): @@ -321,8 +386,13 @@ def load_yaml(path: Path): return data -def load_manifest_order(): - """Return the series keys in manifest order (= dedup priority order).""" +def load_manifest(): + """Return ``(order, consumers)``: the series keys in manifest order (= + dedup priority order) and the set of series declared ``class: consumer``. + + A stanza without ``class:`` is canonical-eligible. An unknown class is an + error, never a silent default — it decides what may become canonical. + """ data = load_yaml(MANIFEST_PATH) if data is None: raise SystemExit( @@ -332,7 +402,65 @@ def load_manifest_order(): series_block = data.get("series") or {} if not isinstance(series_block, dict) or not series_block: raise SystemExit(f"error: {MANIFEST_PATH} has no `series:` mapping.") - return list(series_block.keys()) + consumers = set() + for key, cfg in series_block.items(): + cls = (cfg or {}).get("class", SOURCE_CLASS) + if cls not in SERIES_CLASSES: + raise SystemExit( + f"error: series '{key}' in {MANIFEST_PATH} has unknown " + f"class {cls!r} (valid: {', '.join(SERIES_CLASSES)})." + ) + if cls == CONSUMER_CLASS: + consumers.add(key) + return list(series_block.keys()), consumers + + +def load_canonical_map(order, consumers): + """Return the committed canonical map: ``slug -> series``. + + ``sync/canonical.yml`` is HUMAN INTENT — the one way a same-name collision + is settled. It is validated up front: a repeated key, two keys naming one + lecture (``slug`` and ``slug.md``), and an unknown or consumer series are + errors before anything is planned. A slug that no series holds cannot be + checked here (it needs the mirror); ``map_slugs_held_nowhere`` reports + it. A missing or empty file is an empty map. + """ + data = load_yaml(CANONICAL_MAP_PATH, loader=UniqueKeyLoader) + if not data: + return {} + bad, named = [], {} + for slug, series in data.items(): + if not isinstance(slug, str) or not isinstance(series, str): + bad.append(f"{slug!r}: {series!r} (expected `: `)") + continue + if slug_of(slug) in named: + bad.append(f"{slug}: names the same lecture as '{named[slug_of(slug)]}'") + named[slug_of(slug)] = slug + if series not in order: + bad.append(f"{slug}: unknown series '{series}'") + elif series in consumers: + bad.append(f"{slug}: '{series}' is a consumer series and is never canonical") + if bad: + raise SystemExit( + f"error: invalid entries in {CANONICAL_MAP_PATH}:\n " + + "\n ".join(bad) + ) + return {slug_of(slug): series for slug, series in data.items()} + + +def map_slugs_held_nowhere(canonical_map, order, consumers): + """Map slugs whose lecture no reconstructed canonical-eligible mirror + holds: a typo, or a lecture renamed or retired upstream. Names are matched + exactly, so a wrong-case key is caught on a case-insensitive file system + too. Reported, not refused — one dead line must not stop every + promotion.""" + held = set() + for s in order: + root = mirror_lectures(s) + if s in consumers or not root.is_dir(): + continue + held.update(p.relative_to(root).as_posix()[:-3] for p in root.rglob("*.md")) + return sorted(slug for slug in canonical_map if slug not in held) def load_state(): @@ -408,11 +536,13 @@ def dump_ledger(lectures: dict, assets: dict) -> str: return LEDGER_HEADER + body -def ordered_entry(canonical, interim, sources, divergent, promoted_at, assets, rewrites): +def ordered_entry(canonical, interim, sources, divergent, superseded, promoted_at, assets, rewrites): """A ledger lecture entry with a stable key order. ``interim`` is present only while ``divergent`` is non-empty — it marks - the canonical choice as pending human triage. + a canonical choice the canonical map made over differing copies, pending + the rename or merge upstream that retires the map entry. ``superseded`` + is present only when a consumer series holds a copy. """ entry = {"canonical": canonical} if interim: @@ -420,6 +550,8 @@ def ordered_entry(canonical, interim, sources, divergent, promoted_at, assets, r entry["sources"] = sources if divergent: entry["divergent"] = divergent + if superseded: + entry["superseded"] = superseded entry["promoted_at"] = promoted_at entry["assets"] = assets entry["rewrites"] = rewrites @@ -436,8 +568,10 @@ class Plan: self.slug = slug self.requested = requested_series # None => resolve by priority self.canonical = None - self.sources = [] # series with byte-identical copies (manifest order) - self.divergent = [] # series with same-named, byte-different copies + self.sources = [] # series with identical or stale copies (manifest order) + self.divergent = [] # differing copies the canonical map overruled + self.superseded = [] # consumer series holding a copy (manifest order) + self.stale = [] # sources unchanged since promotion while the canonical moved self.digests = {} # series -> sha256 of its copy at promotion time self.refs = [] # (kind, literal, src_rel, digest) — deduped, source order self.warnings = [] @@ -447,12 +581,72 @@ class Plan: self.assets = [] # pool-relative destinations (sorted) -def resolve_sources(plan: Plan, considered): - """Fill in canonical/sources/divergent by hashing across series. +def normalised_text(series: str, slug: str) -> str: + """A copy's text with each local asset reference replaced by the pool + path promotion would give it — the form two copies are COMPARED in. - The requested copy anchors the byte-identity set; the canonical series is - the highest-priority member of that set (manifest order). Series whose - same-named copy differs in bytes land in ``divergent`` for human triage. + Series spell the same reference differently (``/_static/...`` against + ``_static/...``, a ``lecture_specific//`` dir against a new-style + ``_static//`` one); promotion rewrites both to one pool path, so + copies that differ only there are one lecture. Comparison only: the pool + text is always built by ``build_rewrites`` from the canonical copy. A + reference that resolves to no file stays as written. + """ + root = mirror_lectures(series) + text = read_text_exact(root / f"{slug}.md") + pairs = {} + for kind, written in extract_refs(text): + if kind == "url": + literal, src_rel = written, written[written.find("/_static/") + 1 :] + else: + literal = written.split("#", 1)[0] + if not literal or SCHEME_RE.match(literal): + continue + try: + src_rel = to_src_rel(literal) + except PromoteError: + continue + if kind == "link" and src_rel.endswith(".md"): + continue + if (root / src_rel).is_file(): + pairs[literal] = plan_destination(src_rel, [slug]) + for frm, to in sorted(pairs.items(), key=lambda kv: (-len(kv[0]), kv[0])): + text = text.replace(frm, to) + return text + + +def resolve_sources(plan: Plan, considered, consumers, canonical_map, recorded=None, order=()): + """Fill in canonical/sources/divergent/superseded by hashing across series. + + A same-named file is presumed to be ONE lecture. The anchor copy (the + canonical map's choice, else the requested copy, else the highest-priority + one) defines the identity set: every canonical-eligible copy that is + byte-identical to it, or identical after ``normalised_text``. The canonical + series is the highest-priority member of that set (manifest order). + + A copy outside the set is a different lecture under the same name, and an + error naming the lecture and every series involved: which copy survives + must never fall out of run order. Only the committed canonical map settles + it — the map's copy anchors the set, and the differing copies land in + ``divergent`` (the entry is ``interim``). The map picks a COPY, never a + priority: among identical copies the highest-priority series is still + canonical, so deleting an entry that settles nothing changes nothing. + Consumer copies, equal or not, are recorded under ``superseded`` and + never anchor anything. + + ``recorded`` is the slug's ledger entry, if any. A copy is STALE, not a + different lecture, when it is recorded under ``sources``, is not the + recorded canonical copy's lecture as that copy now stands, and is still + at a version the pool already took: its own recorded digest (it has not + moved) or the canonical copy's (it caught up, then the canonical copy + moved on). + Staleness is judged against the recorded canonical copy whatever anchors + the run: a stale requested copy (``--toc`` of a series that lags) gives + way to the canonical copy, and a canonical-map entry naming a stale copy + is refused, since it no longer settles a collision and following it + would keep the lagging text. A stale copy stays a source + (``plan.stale``) — the drift-check's ``stale-copy`` — and only a copy + edited to something else is a collision. """ rel = f"{plan.slug}.md" digests = {} @@ -460,20 +654,169 @@ def resolve_sources(plan: Plan, considered): f = mirror_lectures(series) / rel if f.is_file(): digests[series] = sha256_file(f) - if not digests: + eligible = [s for s in considered if s in digests and s not in consumers] + if not eligible: + held = [s for s in considered if s in digests] + if held: + raise PromoteError( + f"{rel} is held only by consumer series ({', '.join(held)}) — " + "a consumer contributes no lecture to the pool" + ) + if recorded: + raise PromoteError( + f"{rel} is in the pool, but no series this run considers holds " + "it any more (removed or renamed upstream?) — decide whether " + "the lecture was retired, renamed or moved; the drift-check " + "reports it as canonical-removed" + ) raise PromoteError( f"{rel} not found in any non-excluded series " f"(searched: {', '.join(considered)})" ) - requested = plan.requested - if requested is None: - requested = next(s for s in considered if s in digests) - elif requested not in digests: + requested = plan.requested if plan.requested not in consumers else None + if requested is not None and requested not in digests: + if recorded and recorded.get("canonical") == requested: + raise PromoteError( + f"the canonical copy of {rel} is gone from {requested} upstream " + "(removed or renamed) — decide whether the lecture was " + "retired, renamed or moved; the drift-check reports it as " + "canonical-removed" + ) raise PromoteError(f"{rel} not found in mirror/{requested}/lectures/") - anchor = digests[requested] - plan.sources = [s for s in considered if digests.get(s) == anchor] - plan.divergent = [s for s in digests if digests[s] != anchor] - plan.canonical = plan.sources[0] + mapped = canonical_map.get(plan.slug) + if mapped is not None and mapped not in considered: + plan.warnings.append( + f"{CANONICAL_MAP_PATH.name} names '{mapped}' for {plan.slug}, but " + f"'{mapped}' is excluded from this run — the map entry is ignored" + ) + mapped = None + if mapped is not None and mapped not in digests: + raise PromoteError( + f"{CANONICAL_MAP_PATH.name} names '{mapped}' for {plan.slug}, but " + f"{rel} is not in mirror/{mapped}/lectures/" + ) + base = mapped or requested or eligible[0] + + normalised = {} # series -> normalised text, computed only when needed + + def split(anchor): + """``(same, differ)``: the eligible copies that are, and are not, the + anchor's lecture (byte-identical, or identical once normalised).""" + same, differ = [], [] + for series in eligible: + if digests[series] == digests[anchor]: + same.append(series) + continue + for s in (anchor, series): + if s not in normalised: + normalised[s] = normalised_text(s, plan.slug) + (same if normalised[series] == normalised[anchor] else differ).append(series) + return same, differ + + # Staleness is judged against the recorded canonical copy, whatever + # anchors this run, so a refresh and --toc of a lagging series agree. + recorded_canonical, stale_copies = None, [] + if recorded and recorded.get("canonical") in eligible: + recorded_canonical = recorded.get("canonical") + was = {s.get("series"): s.get("digest") for s in recorded.get("sources") or []} + _, differ_c = split(recorded_canonical) + stale_copies = [ + s for s in differ_c + if s in was and digests[s] in (was[s], was.get(recorded_canonical)) + ] + + anchor = base + if base in stale_copies: + if base == mapped: + if digests[recorded_canonical] != was.get(recorded_canonical): + why = ( + f"'{mapped}' holds an earlier promoted version, and the " + f"recorded canonical copy in '{recorded_canonical}' has " + "moved on" + ) + else: + why = ( + f"'{mapped}' no longer matches the recorded canonical copy " + f"in '{recorded_canonical}' (it is an earlier version, or " + "a file it references changed upstream)" + ) + raise PromoteError( + f"{CANONICAL_MAP_PATH.name} names '{mapped}' for {plan.slug}, " + f"but {why}: point the entry at '{recorded_canonical}', remove " + f"it if no other copy differs, or bring {mapped}'s copy up to " + "date upstream" + ) + anchor = recorded_canonical + + same, differ = split(anchor) + stale = [] + if recorded_canonical is not None and recorded_canonical in same: + stale = [s for s in differ if s in stale_copies] + differ = [s for s in differ if s not in stale] + if differ and mapped is None: + detail = "same: " + ", ".join(f"{s}:{digests[s][:12]}" for s in same) + detail += "; different: " + ", ".join(f"{s}:{digests[s][:12]}" for s in differ) + if stale: + detail += "; stale (at an earlier promoted version): " + ", ".join(stale) + raise PromoteError( + f"{rel} differs across series ({detail}) — a same-named file must " + "be one lecture: reconcile or rename it upstream, or name the " + f"canonical series in sync/{CANONICAL_MAP_PATH.name}" + ) + if mapped is not None and not differ: + # ``order`` is every manifest series. A canonical-eligible series this + # run excludes may hold the copy the entry settles, so its copy is + # compared too; without its mirror the run cannot tell. + differing, unchecked = [], [] + for s in order: + if s in considered or s in consumers: + continue + if not mirror_lectures(s).is_dir(): + unchecked.append(s) + continue + f = mirror_lectures(s) / rel + if not f.is_file() or sha256_file(f) == digests[anchor]: + continue + for x in (anchor, s): + if x not in normalised: + normalised[x] = normalised_text(x, plan.slug) + if normalised[s] != normalised[anchor]: + differing.append(s) + head = ( + f"{CANONICAL_MAP_PATH.name} names '{mapped}' for {plan.slug}; " + "every copy this run considers is the same lecture" + ) + if differing: + plan.warnings.append( + f"{head}, but {', '.join(differing)} (excluded from this run) " + + ("holds a different copy" if len(differing) == 1 else "hold different copies") + + ", so keep the entry while it settles that collision" + ) + elif unchecked: + plan.warnings.append( + f"{head}, but {', '.join(unchecked)} (excluded, with no mirror " + f"on disk) could not be checked: run `tools/sync " + f"{' '.join(unchecked)}` and a run that considers " + + ("it" if len(unchecked) == 1 else "them") + + " before removing the entry" + ) + else: + note = ( + f"{CANONICAL_MAP_PATH.name} names '{mapped}' for {plan.slug}, but " + "every copy is the same lecture — the map entry settles nothing " + "and can be removed" + ) + if mapped != same[0]: + note += ( + f"; remove it now, since once {same[0]}'s copy moves on, " + f"an entry naming {mapped}'s lagging copy is refused" + ) + plan.warnings.append(note) + plan.sources = [s for s in eligible if s in same or s in stale] + plan.stale = stale + plan.divergent = differ + plan.superseded = [s for s in considered if s in digests and s in consumers] + plan.canonical = same[0] plan.digests = digests return read_text_exact(mirror_lectures(plan.canonical) / rel) @@ -624,11 +967,12 @@ def plan_destination(src_rel: str, consumers) -> str: return f"_static/{only}/{src_rel}" -def check_admonition_identity(src_rel: str, considered): +def check_admonition_identity(src_rel: str, eligible): """Rule 4 guard: an ``_admonition/`` include merges into ``_shared`` only - if every non-excluded series that carries the file agrees byte-for-byte.""" + if every non-excluded, canonical-eligible series that carries the file + agrees byte-for-byte (a consumer's copy is never a source).""" digests = {} - for series in considered: + for series in eligible: f = mirror_lectures(series) / src_rel if f.is_file(): digests[series] = sha256_file(f) @@ -719,29 +1063,66 @@ def toc_slugs(series: str): return ordered -def refresh_targets(ledger_lectures, state, considered): +def refresh_targets(ledger_lectures, state, considered, consumers, order=None): """Ledger entries whose canonical series pin moved since promotion. Design note: refresh deliberately re-anchors each entry on its RECORDED canonical series (``requested=canonical``), not on a fresh priority resolution — refresh tracks the canonical copy's movement, so the entry's - identity stays stable even when a higher-priority series' copy has since - diverged. Only the canonical pin is watched (per the refresh spec); - ``sources``/``divergent`` provenance is as of each entry's last promotion. + identity stays stable. A copy edited since its last promotion is refused + like any other same-name collision (the canonical map settles it); one + still at a version recorded then, while the canonical copy moved on, is + stale and stays a source (see ``resolve_sources``). Only + the canonical pin is watched (per the refresh spec); ``sources``/ + ``divergent``/``superseded`` provenance is as of each entry's last + promotion. + + Returns ``(targets, skipped, refused)``. An entry this run cannot refresh + (its canonical series is excluded, or is a consumer) is skipped while its + pin has not moved, and REFUSED once it has: a silent skip would leave the + pool copy behind upstream with a clean exit and a green run. """ - targets, skipped = [], [] + targets, skipped, refused = [], [], [] for key in sorted(ledger_lectures): entry = ledger_lectures[key] slug = slug_of(posixpath.basename(key)) canonical = entry.get("canonical") - if canonical not in considered: - skipped.append((slug, f"canonical series '{canonical}' is excluded")) - continue pinned = (entry.get("promoted_at") or {}).get(canonical) current = (state.get(canonical) or {}).get("sha") or "" + if order is not None and canonical not in order: + why = f"canonical series '{canonical}' is not in sync/manifest.yml" + remedy = ( + f"promote {slug} by name to re-home it to a series that holds " + "it, or restore the series' stanza" + ) + skip = f"{why} — {remedy}" + elif canonical in consumers: + why = f"canonical series '{canonical}' is a consumer" + remedy = ( + f"promote {slug} by name to re-home it to a series that holds " + "it, or decide whether to retire it" + ) + skip = f"{why} — {remedy}" + elif canonical not in considered: + why = f"canonical series '{canonical}' is excluded" + remedy = ( + "refresh without excluding it (CI's exclusions are " + "PROMOTE_EXCLUDE in .github/workflows/drift-check.yml)" + ) + skip = why + else: + if pinned != current: + targets.append((slug, canonical)) + continue if pinned != current: - targets.append((slug, canonical)) - return targets, skipped + refused.append(( + slug, canonical, + f"{why}, but its pin moved since promotion " + f"({(pinned or 'none')[:12]} -> {current[:12] or 'none'}): {remedy}", + )) + else: + skipped.append((slug, skip)) + return targets, skipped, refused # --------------------------------------------------------------------------- # @@ -756,7 +1137,10 @@ def build_parser(): "and every rewrite in sync/ledger.yml." ), formatter_class=argparse.RawDescriptionHelpFormatter, - epilog="Series keys are defined in sync/manifest.yml; jax is always excluded.", + epilog=( + "Series keys and classes are defined in sync/manifest.yml; the " + "canonical map is sync/canonical.yml." + ), ) parser.add_argument( "slugs", @@ -779,8 +1163,9 @@ def build_parser(): nargs="+", metavar="SERIES", default=[], - help="series excluded from source/dedup consideration " - "(jax is always excluded).", + help="series excluded from source/dedup consideration for this run " + "(none by default; CI excludes PROMOTE_EXCLUDE in " + ".github/workflows/drift-check.yml).", ) parser.add_argument( "--dry-run", @@ -799,8 +1184,8 @@ def main(argv=None): if sum(modes) != 1: parser.error("choose exactly one of: SLUG..., --toc SERIES, --refresh") - order = load_manifest_order() - excluded = set(ALWAYS_EXCLUDED) | set(args.exclude) + order, consumers = load_manifest() + excluded = set(args.exclude) unknown = [s for s in args.exclude if s not in order] if unknown: parser.error( @@ -809,13 +1194,18 @@ def main(argv=None): + "\nvalid keys: " + ", ".join(order) ) + considered = [s for s in order if s not in excluded] - if not considered: - parser.error("every series is excluded — nothing can be promoted.") + eligible = [s for s in considered if s not in consumers] + if not eligible: + parser.error( + "every canonical-eligible series is excluded — nothing can be promoted." + ) if args.toc and args.toc not in order: parser.error(f"unknown series '{args.toc}' (valid: {', '.join(order)})") if args.toc and args.toc in excluded: parser.error(f"series '{args.toc}' is excluded from consideration.") + canonical_map = load_canonical_map(order, consumers) # Dedup hashes across every considered series, so all their mirrors must # exist — a missing mirror would silently change canonical decisions. @@ -829,6 +1219,12 @@ def main(argv=None): + "` first (dedup must see every non-excluded series).\n" ) return 1 + for slug in map_slugs_held_nowhere(canonical_map, order, consumers): + print( + f"[{slug}] WARN: {CANONICAL_MAP_PATH.name} names {slug}, but no " + f"canonical-eligible series mirror holds {slug}.md — a typo, or a " + "lecture renamed or retired upstream" + ) state = load_state() for s in considered: @@ -846,13 +1242,18 @@ def main(argv=None): # ------------------------------------------------------------------ # skips = [] # (slug, reason) if args.refresh: - targets, refresh_skips = refresh_targets(ledger_lectures, state, considered) + targets, refresh_skips, refresh_refused = refresh_targets( + ledger_lectures, state, considered, consumers, order + ) skips.extend(refresh_skips) - if not targets and not refresh_skips: + if not targets and not refresh_skips and not refresh_refused: print("nothing to refresh — every canonical pin matches its ledger entry.") return 0 elif args.toc: - targets = [(slug, args.toc) for slug in toc_slugs(args.toc)] + # A consumer's toc names lectures it republishes: each comes from its + # canonical home, resolved by priority, never from the consumer. + requested = None if args.toc in consumers else args.toc + targets = [(slug, requested) for slug in toc_slugs(args.toc)] else: targets = [(slug_of(s), None) for s in args.slugs] @@ -878,6 +1279,11 @@ def main(argv=None): plans.append(plan) continue plans.append(Plan(slug, requested)) + if args.refresh: + for slug, canonical, reason in refresh_refused: + plan = Plan(slug, canonical) + plan.error = reason + plans.append(plan) # ------------------------------------------------------------------ # # Phase 1 — resolve sources and parse closures (per-lecture errors # @@ -888,7 +1294,10 @@ def main(argv=None): if plan.error is not None: continue try: - text = resolve_sources(plan, considered) + text = resolve_sources( + plan, considered, consumers, canonical_map, + ledger_lectures.get(f"lectures/{plan.slug}.md"), order, + ) collect_refs(plan, text) texts[plan.slug] = text except PromoteError as exc: @@ -968,7 +1377,7 @@ def main(argv=None): try: for _, _, src_rel, digest in plan.refs: if src_rel.startswith("_admonition/"): - check_admonition_identity(src_rel, considered) + check_admonition_identity(src_rel, eligible) dest = dest_map[(src_rel, digest)] src_file = mirror_lectures(plan.canonical) / src_rel prior = planned.get(dest) @@ -1054,14 +1463,15 @@ def main(argv=None): break # a full pass with no culls: the plan set is stable def asset_provenance(dest): - """Sources for a pool asset: every considered series whose copy at ANY - of the contributing relative paths is byte-identical to the promoted - bytes. Multiple paths per destination arise when different written - forms normalise to the same pool file (same-bytes dedup) — each is - recorded so the drift-check watches every upstream copy.""" + """Sources for a pool asset: every considered, canonical-eligible + series whose copy at ANY of the contributing relative paths is + byte-identical to the promoted bytes. Multiple paths per destination + arise when different written forms normalise to the same pool file + (same-bytes dedup) — each is recorded so the drift-check watches every + upstream copy. A consumer's copy is never a source.""" rec = planned[dest] sources = [] - for series in considered: + for series in eligible: for src_rel in sorted(rec["src_rels"]): f = mirror_lectures(series) / src_rel if f.is_file() and sha256_file(f) == rec["digest"]: @@ -1075,7 +1485,7 @@ def main(argv=None): new_lectures = dict(ledger_lectures) new_assets = dict(ledger_assets) for plan in ok: - pin_series = plan.sources + [s for s in plan.divergent if s not in plan.sources] + pin_series = set(plan.sources) | set(plan.divergent) | set(plan.superseded) promoted_at = { s: (state.get(s) or {}).get("sha") or "" for s in order @@ -1093,6 +1503,11 @@ def main(argv=None): for s in order if s in plan.divergent ], + superseded=[ + {"series": s, "path": f"{plan.slug}.md", "digest": plan.digests[s]} + for s in order + if s in plan.superseded + ], promoted_at=promoted_at, assets=plan.assets, rewrites=[{"from": f, "to": t} for f, t in plan.rewrites], @@ -1124,8 +1539,12 @@ def main(argv=None): if args.dry_run: for plan in ok: line = f"[{plan.slug}] plan: canonical={plan.canonical} sources={plan.sources}" + if plan.stale: + line += f" stale={plan.stale}" if plan.divergent: line += f" divergent={plan.divergent} (interim)" + if plan.superseded: + line += f" superseded={plan.superseded}" print(line) for _, _, src_rel, digest in sorted(set(plan.refs), key=lambda r: (r[2], r[0], r[1])): dest = dest_map[(src_rel, digest)] @@ -1167,7 +1586,8 @@ def main(argv=None): f"[{plan.slug}] {state_word} -> lectures/{plan.slug}.md " f"(canonical {plan.canonical}, {len(plan.assets)} assets, " f"{len(plan.rewrites)} rewrites" - + (", interim: divergent copies pending triage" if plan.divergent else "") + + (f", stale: {', '.join(plan.stale)} at an earlier promoted version" if plan.stale else "") + + (", interim: canonical map over differing copies" if plan.divergent else "") + ")" ) # The ledger is written BEFORE stale assets are unlinked: a crash in diff --git a/tools/tests/conftest.py b/tools/tests/conftest.py new file mode 100644 index 0000000..ce9e160 --- /dev/null +++ b/tools/tests/conftest.py @@ -0,0 +1,135 @@ +"""Scratch repositories for the tools/ tests. + +Each test builds a small repository in a temp dir — ``mirror// +lectures/``, ``sync/manifest.yml``, ``sync/state.yml``, an empty pool — and +copies the real ``tools/promote`` and ``tools/drift-check`` into it. The tools +resolve every path from their own location, so a copy runs against the +scratch repository and never touches the real mirror, pool or ledger. They are +run as subprocesses, exactly as CI runs them. +""" + +from __future__ import annotations + +import hashlib +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + +TOOLS_DIR = Path(__file__).resolve().parent.parent + + +def fake_sha(series: str, n: int = 1) -> str: + """A deterministic 40-hex pin per series (``n`` moves it).""" + return hashlib.sha1(f"{series}:{n}".encode()).hexdigest() + + +def lecture_text(title: str, body: str = "", refs=()) -> str: + """A minimal MyST lecture; ``refs`` become markdown image references.""" + lines = ["---", "jupytext:", " text_representation:", " extension: .md", "---", "", f"# {title}", ""] + if body: + lines += [body, ""] + for ref in refs: + lines += [f"![figure]({ref})", ""] + return "\n".join(lines) + + +class ScratchRepo: + def __init__(self, root: Path, series, consumers=()): + self.root = root + self.order = list(series) + (root / "tools").mkdir(parents=True) + for name in ("promote", "drift-check"): + shutil.copy(TOOLS_DIR / name, root / "tools" / name) + (root / "sync").mkdir() + (root / "lectures").mkdir() + for key in self.order: + (root / "mirror" / key / "lectures").mkdir(parents=True) + self.write_manifest(consumers) + self.write_state({key: fake_sha(key) for key in self.order}) + + # -- inputs ------------------------------------------------------------ # + def write_manifest(self, consumers=()): + lines = ["defaults:", " branch: main", "series:"] + for key in self.order: + lines += [f" {key}:", f" repo: test/{key}"] + if key in consumers: + lines.append(" class: consumer") + (self.root / "sync" / "manifest.yml").write_text("\n".join(lines) + "\n", encoding="utf-8") + + def write_state(self, pins): + state = {key: {"sha": sha, "synced_at": "2026-10-06T00:00:00Z"} for key, sha in pins.items()} + (self.root / "sync" / "state.yml").write_text(yaml.safe_dump(state, sort_keys=False), encoding="utf-8") + + def pin(self, series): + state = yaml.safe_load((self.root / "sync" / "state.yml").read_text(encoding="utf-8")) + return state[series]["sha"] + + def move_pin(self, series, n=2): + state = yaml.safe_load((self.root / "sync" / "state.yml").read_text(encoding="utf-8")) + state[series]["sha"] = fake_sha(series, n) + (self.root / "sync" / "state.yml").write_text(yaml.safe_dump(state, sort_keys=False), encoding="utf-8") + + def write_canonical_map(self, mapping): + text = "# test canonical map\n" + "".join(f"{slug}: {series}\n" for slug, series in mapping.items()) + (self.root / "sync" / "canonical.yml").write_text(text, encoding="utf-8") + + def mirror_file(self, series, rel, data): + path = self.root / "mirror" / series / "lectures" / rel + path.parent.mkdir(parents=True, exist_ok=True) + if isinstance(data, str): + data = data.encode("utf-8") + path.write_bytes(data) + return path + + def lecture(self, series, slug, text): + return self.mirror_file(series, f"{slug}.md", text) + + def toc(self, series, slugs): + chapters = "".join(f" - file: {slug}\n" for slug in slugs) + self.mirror_file(series, "_toc.yml", f"format: jb-book\nroot: intro\nchapters:\n{chapters}") + + # -- running ----------------------------------------------------------- # + def run(self, tool, *args): + return subprocess.run( + [sys.executable, str(self.root / "tools" / tool), *args], + capture_output=True, + text=True, + cwd=str(self.root), + ) + + def promote(self, *args): + return self.run("promote", *args) + + def drift_check(self, *args): + return self.run("drift-check", *args) + + # -- outputs ----------------------------------------------------------- # + @property + def ledger_path(self): + return self.root / "sync" / "ledger.yml" + + def ledger(self): + return yaml.safe_load(self.ledger_path.read_text(encoding="utf-8")) + + def entry(self, slug): + return self.ledger()[f"lectures/{slug}.md"] + + def pool(self, rel): + return self.root / "lectures" / rel + + +@pytest.fixture +def make_repo(tmp_path): + """``make_repo(series, consumers=())`` -> a ScratchRepo; ``series`` is the + manifest order (= dedup priority).""" + count = {"n": 0} + + def factory(series, consumers=()): + count["n"] += 1 + return ScratchRepo(tmp_path / f"repo{count['n']}", series, consumers) + + return factory diff --git a/tools/tests/test_consumer.py b/tools/tests/test_consumer.py new file mode 100644 index 0000000..532bb9a --- /dev/null +++ b/tools/tests/test_consumer.py @@ -0,0 +1,103 @@ +"""The consumer series class: never canonical, superseded, unwatched.""" + +from conftest import lecture_text + + +def test_consumer_copy_is_superseded_and_never_canonical(make_repo): + # dp comes FIRST in manifest order: priority alone would make it canonical. + repo = make_repo(["dp", "intermediate"], consumers=["dp"]) + repo.lecture("dp", "career", lecture_text("Career (an older copy)")) + current = lecture_text("Career") + repo.lecture("intermediate", "career", current) + + res = repo.promote("career") + + assert res.returncode == 0, res.stderr + entry = repo.entry("career") + assert entry["canonical"] == "intermediate" + assert [s["series"] for s in entry["sources"]] == ["intermediate"] + assert [s["series"] for s in entry["superseded"]] == ["dp"] + assert "divergent" not in entry and "interim" not in entry + assert set(entry["promoted_at"]) == {"dp", "intermediate"} + assert repo.pool("career.md").read_text(encoding="utf-8") == current + + +def test_identical_consumer_copy_is_superseded_not_a_source(make_repo): + repo = make_repo(["intermediate", "dp"], consumers=["dp"]) + text = lecture_text("Job Search") + repo.lecture("intermediate", "mccall_model", text) + repo.lecture("dp", "mccall_model", text) + + assert repo.promote("mccall_model").returncode == 0 + entry = repo.entry("mccall_model") + assert [s["series"] for s in entry["sources"]] == ["intermediate"] + assert [s["series"] for s in entry["superseded"]] == ["dp"] + + +def test_a_lecture_held_only_by_a_consumer_is_refused(make_repo): + repo = make_repo(["intermediate", "dp"], consumers=["dp"]) + repo.lecture("dp", "dp_only", lecture_text("Only in dp")) + + res = repo.promote("dp_only") + + assert res.returncode == 1 + assert "held only by consumer series (dp)" in res.stderr + assert not repo.pool("dp_only.md").exists() + + +def test_a_consumer_toc_promotes_each_lecture_from_its_canonical_home(make_repo): + repo = make_repo(["intermediate", "advanced", "dp"], consumers=["dp"]) + repo.lecture("intermediate", "jv", lecture_text("Job Search VI")) + repo.lecture("advanced", "amss", lecture_text("AMSS")) + repo.lecture("dp", "jv", lecture_text("Job Search V (older)")) + repo.lecture("dp", "amss", lecture_text("AMSS")) + repo.toc("dp", ["jv", "amss"]) + + res = repo.promote("--toc", "dp") + + assert res.returncode == 0, res.stderr + assert repo.entry("jv")["canonical"] == "intermediate" + assert repo.entry("amss")["canonical"] == "advanced" + for slug in ("jv", "amss"): + assert [s["series"] for s in repo.entry(slug)["superseded"]] == ["dp"] + + +def test_drift_check_does_not_watch_a_consumers_copies(make_repo): + repo = make_repo(["intermediate", "advanced", "dp"], consumers=["dp"]) + text = lecture_text("Optimal Taxation") + repo.lecture("intermediate", "opt_tax", text) + repo.lecture("advanced", "opt_tax", text) # an identical, watched source + repo.lecture("dp", "opt_tax", lecture_text("Optimal Taxation (older)")) + assert repo.promote("opt_tax").returncode == 0 + assert repo.drift_check().returncode == 0 + + # The consumer's stale copy moves again: nothing to report. + repo.lecture("dp", "opt_tax", lecture_text("Optimal Taxation (older still)")) + res = repo.drift_check() + assert res.returncode == 0, res.stdout + assert "clean:" in res.stdout + + # Control: a canonical-eligible source moving IS drift. + repo.lecture("advanced", "opt_tax", lecture_text("Optimal Taxation, edited in advanced")) + res = repo.drift_check() + assert res.returncode == 1 + assert "the advanced copy lectures/opt_tax.md no longer matches" in res.stdout + + +def test_drift_check_honours_a_class_declared_after_promotion(make_repo): + # A ledger written while dp was an ordinary series lists dp as a source; + # declaring dp a consumer afterwards stops its copy being watched. + repo = make_repo(["intermediate", "dp"]) + text = lecture_text("Career") + repo.lecture("intermediate", "career", text) + repo.lecture("dp", "career", text) + assert repo.promote("career").returncode == 0 + assert [s["series"] for s in repo.entry("career")["sources"]] == ["intermediate", "dp"] + + repo.lecture("dp", "career", lecture_text("Career (dp edit)")) + assert repo.drift_check().returncode == 1 # an ordinary series: drift + + repo.write_manifest(consumers=["dp"]) + res = repo.drift_check() + assert res.returncode == 0, res.stdout + assert "clean:" in res.stdout diff --git a/tools/tests/test_drift_check_normalised.py b/tools/tests/test_drift_check_normalised.py new file mode 100644 index 0000000..11f15b4 --- /dev/null +++ b/tools/tests/test_drift_check_normalised.py @@ -0,0 +1,79 @@ +"""drift-check judges a non-canonical copy by promotion's own rule: the same +lecture when byte-identical, or identical once asset references are +normalised to their pool paths (QuantEcon/lectures#15).""" + +from conftest import lecture_text + +FIG = "_static/lecture_specific/short_path/graph.png" + + +def short_path_repo(make_repo): + """intro (canonical) spells the figure root-relative, intermediate + file-relative; promote admits both as one lecture.""" + repo = make_repo(["intro", "intermediate"]) + for series in ("intro", "intermediate"): + repo.mirror_file(series, FIG, b"png") + repo.lecture("intro", "short_path", lecture_text("Shortest Paths", refs=["/" + FIG])) + repo.lecture("intermediate", "short_path", lecture_text("Shortest Paths", refs=[FIG])) + res = repo.promote("short_path") + assert res.returncode == 0, res.stderr + entry = repo.entry("short_path") + assert [s["series"] for s in entry["sources"]] == ["intro", "intermediate"] + assert entry["sources"][0]["digest"] != entry["sources"][1]["digest"] + assert repo.drift_check().returncode == 0 + return repo + + +def test_a_source_moving_in_lockstep_with_canonical_is_not_drift(make_repo): + repo = short_path_repo(make_repo) + + # The same upstream edit lands in both series; the spelling still differs. + repo.lecture("intro", "short_path", lecture_text("Shortest Paths", "edited", refs=["/" + FIG])) + repo.lecture("intermediate", "short_path", lecture_text("Shortest Paths", "edited", refs=[FIG])) + res = repo.drift_check() + + assert res.returncode == 1, res.stdout # the refresh finding alone + assert "[canonical-moved]" in res.stdout + assert "[diverged]" not in res.stdout + assert "[converged] lectures/short_path.md: the intermediate copy changed and now matches intro up to asset path spelling" in res.stdout + + +def test_a_source_changing_only_its_asset_spelling_is_not_drift(make_repo): + repo = short_path_repo(make_repo) + + # intermediate adopts a third spelling of the same reference. + repo.lecture("intermediate", "short_path", lecture_text("Shortest Paths", refs=["./" + FIG])) + res = repo.drift_check() + + assert res.returncode == 0, res.stdout + assert "[diverged]" not in res.stdout + assert "up to asset path spelling" in res.stdout + + +def test_a_source_edited_on_its_own_is_still_drift(make_repo): + repo = short_path_repo(make_repo) + + repo.lecture("intermediate", "short_path", lecture_text("Shortest Paths", "edited in intermediate", refs=[FIG])) + res = repo.drift_check() + + assert res.returncode == 1 + assert "[diverged] lectures/short_path.md: the intermediate copy lectures/short_path.md no longer matches its canonical home in intro" in res.stdout + + +def test_a_divergent_copy_converging_up_to_spelling_is_reported_as_converged(make_repo): + repo = make_repo(["intro", "intermediate"]) + for series in ("intro", "intermediate"): + repo.mirror_file(series, FIG, b"png") + repo.lecture("intro", "short_path", lecture_text("Shortest Paths", refs=["/" + FIG])) + repo.lecture("intermediate", "short_path", lecture_text("Shortest Paths (intermediate)", refs=[FIG])) + repo.write_canonical_map({"short_path": "intro"}) + assert repo.promote("short_path").returncode == 0 + assert repo.entry("short_path")["interim"] is True + assert repo.drift_check().returncode == 0 + + # intermediate converges on intro's text, keeping its own spelling. + repo.lecture("intermediate", "short_path", lecture_text("Shortest Paths", refs=[FIG])) + res = repo.drift_check() + + assert res.returncode == 0, res.stdout + assert "[converged] lectures/short_path.md: the divergent intermediate copy now matches intro up to asset path spelling — the interim entry can be re-promoted" in res.stdout diff --git a/tools/tests/test_pool_invariants.py b/tools/tests/test_pool_invariants.py new file mode 100644 index 0000000..497cb40 --- /dev/null +++ b/tools/tests/test_pool_invariants.py @@ -0,0 +1,40 @@ +"""Pool invariants the drift-check and promote guard (kept from the switch +tests when --switch moved out of this pull request).""" + +from conftest import lecture_text + + +def test_a_direct_pool_edit_is_a_violation(make_repo): + repo = make_repo(["intro", "intermediate"]) + repo.mirror_file("intermediate", "_static/lecture_specific/mccall/fig.png", b"figure v1") + repo.lecture("intermediate", "mccall", lecture_text("McCall", refs=["_static/lecture_specific/mccall/fig.png"])) + assert repo.promote("mccall").returncode == 0 + assert repo.drift_check().returncode == 0 + + lecture = repo.pool("mccall.md") + original = lecture.read_bytes() + lecture.write_bytes(original + b"\nA fix made in the pool.\n") + res = repo.drift_check() + assert res.returncode == 2, res.stdout + assert "[pool-edited] lectures/mccall.md" in res.stdout + lecture.write_bytes(original) + + repo.pool("_static/mccall/fig.png").write_bytes(b"figure v2, redrawn in the pool") + res = repo.drift_check() + assert res.returncode == 2, res.stdout + assert "[asset-edited] lectures/_static/mccall/fig.png" in res.stdout + + +def test_an_asset_two_lectures_use_is_shared(make_repo): + repo = make_repo(["intro", "intermediate"]) + shared = b"a figure both series use" + for series in ("intro", "intermediate"): + repo.mirror_file(series, "_static/lecture_specific/common/fig.png", shared) + repo.lecture("intro", "lake", lecture_text("Lake", refs=["_static/lecture_specific/common/fig.png"])) + repo.lecture("intermediate", "mccall", lecture_text("McCall", refs=["_static/lecture_specific/common/fig.png"])) + assert repo.promote("lake", "mccall").returncode == 0 + path = "_static/_shared/common/fig.png" + assert list(repo.ledger()["assets"]) == [path] + assert repo.pool(path).read_bytes() == shared + assert path in repo.entry("lake")["assets"] and path in repo.entry("mccall")["assets"] + assert repo.drift_check().returncode == 0 diff --git a/tools/tests/test_promote_collisions.py b/tools/tests/test_promote_collisions.py new file mode 100644 index 0000000..1a785b2 --- /dev/null +++ b/tools/tests/test_promote_collisions.py @@ -0,0 +1,607 @@ +"""Same-name collisions, the canonical map, jax by default, byte stability.""" + +import hashlib + +from conftest import lecture_text + + +def test_identical_copies_keep_the_highest_priority_series_canonical(make_repo): + repo = make_repo(["intro", "intermediate"]) + text = lecture_text("Shared") + repo.lecture("intro", "shared", text) + repo.lecture("intermediate", "shared", text) + + res = repo.promote("shared") + + assert res.returncode == 0, res.stderr + entry = repo.entry("shared") + assert entry["canonical"] == "intro" + assert [s["series"] for s in entry["sources"]] == ["intro", "intermediate"] + assert "divergent" not in entry and "interim" not in entry + assert repo.pool("shared.md").read_text(encoding="utf-8") == text + + +def test_differing_copy_is_an_error_naming_the_lecture_and_both_series(make_repo): + repo = make_repo(["intro", "intermediate"]) + repo.lecture("intro", "lake_model", lecture_text("A Lake Model of Employment")) + repo.lecture("intermediate", "lake_model", lecture_text("A Lake Model of Unemployment")) + + res = repo.promote("lake_model") + + assert res.returncode == 1 + failed = [line for line in res.stderr.splitlines() if line.startswith("[lake_model] FAILED")] + assert len(failed) == 1, res.stderr + assert "lake_model.md differs across series" in failed[0] + assert "intro:" in failed[0] and "intermediate:" in failed[0] + assert "sync/canonical.yml" in failed[0] + # Nothing was overwritten or recorded. + assert not repo.pool("lake_model.md").exists() + assert not repo.ledger_path.exists() + + +def test_run_order_never_decides_a_collision(make_repo): + # Asking for the lower-priority series' toc used to make its copy win + # (recorded as interim); it is now the same error. + repo = make_repo(["intro", "intermediate"]) + repo.lecture("intro", "lln_clt", lecture_text("LLN and CLT (intro)")) + repo.lecture("intermediate", "lln_clt", lecture_text("LLN and CLT (intermediate)")) + repo.toc("intermediate", ["lln_clt"]) + + res = repo.promote("--toc", "intermediate") + + assert res.returncode == 1 + assert "[lln_clt] FAILED: lln_clt.md differs across series" in res.stderr + assert not repo.ledger_path.exists() + + +def test_three_way_collision_names_every_series(make_repo): + repo = make_repo(["intro", "intermediate", "jax"]) + for series in ("intro", "intermediate", "jax"): + repo.lecture(series, "mle", lecture_text(f"Maximum Likelihood ({series})")) + + res = repo.promote("mle") + + assert res.returncode == 1 + line = next(l for l in res.stderr.splitlines() if l.startswith("[mle] FAILED")) + for series in ("intro:", "intermediate:", "jax:"): + assert series in line + # The message groups the copies: which agree with the anchor, which do not. + assert "same: intro:" in line and "different: intermediate:" in line and ", jax:" in line + + +def test_canonical_map_resolves_a_collision(make_repo): + repo = make_repo(["intro", "intermediate"]) + repo.lecture("intro", "lake_model", lecture_text("Lake (intro)")) + winner = lecture_text("Lake (intermediate)") + repo.lecture("intermediate", "lake_model", winner) + repo.write_canonical_map({"lake_model": "intermediate"}) + + res = repo.promote("lake_model") + + assert res.returncode == 0, res.stderr + entry = repo.entry("lake_model") + assert entry["canonical"] == "intermediate" + assert entry["interim"] is True + assert [s["series"] for s in entry["sources"]] == ["intermediate"] + assert [s["series"] for s in entry["divergent"]] == ["intro"] + assert set(entry["promoted_at"]) == {"intro", "intermediate"} + assert repo.pool("lake_model.md").read_text(encoding="utf-8") == winner + + +def test_canonical_map_picks_a_copy_and_priority_picks_among_identical_holders(make_repo): + repo = make_repo(["intro", "intermediate", "jax"]) + repo.lecture("intro", "mle", lecture_text("MLE (intro)")) + shared = lecture_text("MLE (intermediate and jax)") + repo.lecture("intermediate", "mle", shared) + repo.lecture("jax", "mle", shared) + repo.write_canonical_map({"mle": "jax"}) + + res = repo.promote("mle") + + assert res.returncode == 0, res.stderr + entry = repo.entry("mle") + assert entry["canonical"] == "intermediate" + assert [s["series"] for s in entry["sources"]] == ["intermediate", "jax"] + assert [s["series"] for s in entry["divergent"]] == ["intro"] + + +def test_canonical_map_rejects_unknown_and_consumer_series(make_repo): + repo = make_repo(["intermediate", "dp"], consumers=["dp"]) + repo.lecture("intermediate", "career", lecture_text("Career")) + + repo.write_canonical_map({"career": "dp"}) + res = repo.promote("career") + assert res.returncode != 0 + assert "consumer series and is never canonical" in res.stderr + + repo.write_canonical_map({"career": "nonesuch"}) + res = repo.promote("career") + assert res.returncode != 0 + assert "unknown series 'nonesuch'" in res.stderr + + +def test_canonical_map_entry_that_settles_nothing_warns(make_repo): + repo = make_repo(["intro", "intermediate"]) + text = lecture_text("Shared") + repo.lecture("intro", "shared", text) + repo.lecture("intermediate", "shared", text) + repo.write_canonical_map({"shared": "intermediate"}) + + res = repo.promote("shared") + + assert res.returncode == 0, res.stderr + assert "settles nothing" in res.stdout + # The map picks a copy, not a priority: identical copies still go to the + # highest-priority series, so removing the entry changes nothing. + entry = repo.entry("shared") + assert entry["canonical"] == "intro" + assert [s["series"] for s in entry["sources"]] == ["intro", "intermediate"] + assert "interim" not in entry + + +def test_jax_is_considered_by_default(make_repo): + repo = make_repo(["intro", "jax"]) + text = lecture_text("Same everywhere") + repo.lecture("intro", "same", text) + repo.lecture("jax", "same", text) + repo.lecture("intro", "schelling", lecture_text("Schelling (intro)")) + repo.lecture("jax", "schelling", lecture_text("Schelling (jax)")) + + res = repo.promote("same") + assert res.returncode == 0, res.stderr + assert [s["series"] for s in repo.entry("same")["sources"]] == ["intro", "jax"] + + res = repo.promote("schelling") + assert res.returncode == 1 + assert "schelling.md differs across series" in res.stderr and "jax:" in res.stderr + + # --exclude remains the per-run way to leave a series out. + res = repo.promote("schelling", "--exclude", "jax") + assert res.returncode == 0, res.stderr + assert [s["series"] for s in repo.entry("schelling")["sources"]] == ["intro"] + + +def test_copies_differing_only_in_asset_path_spelling_are_one_lecture(make_repo): + repo = make_repo(["intro", "intermediate"]) + figure = b"\x89PNG fake figure bytes" + for series in ("intro", "intermediate"): + repo.mirror_file(series, "_static/lecture_specific/short_path/graph.png", figure) + # Root-relative in one series, file-relative in the other: both normalise + # to the same pool path, so they are the same lecture. + repo.lecture("intro", "short_path", lecture_text("Shortest Paths", refs=["/_static/lecture_specific/short_path/graph.png"])) + repo.lecture("intermediate", "short_path", lecture_text("Shortest Paths", refs=["_static/lecture_specific/short_path/graph.png"])) + + res = repo.promote("short_path") + + assert res.returncode == 0, res.stderr + entry = repo.entry("short_path") + assert entry["canonical"] == "intro" + assert [s["series"] for s in entry["sources"]] == ["intro", "intermediate"] + assert entry["sources"][0]["digest"] != entry["sources"][1]["digest"] + assert "divergent" not in entry + assert entry["assets"] == ["_static/short_path/graph.png"] + assert repo.pool("_static/short_path/graph.png").read_bytes() == figure + + +def test_ledger_write_is_byte_stable(make_repo): + repo = make_repo(["intro", "intermediate", "dp"], consumers=["dp"]) + figure = b"figure bytes" + repo.mirror_file("intermediate", "_static/lecture_specific/mccall/fig.png", figure) + repo.lecture("intermediate", "mccall", lecture_text("McCall", refs=["_static/lecture_specific/mccall/fig.png"])) + repo.lecture("dp", "mccall", lecture_text("McCall, an older copy")) + repo.lecture("intro", "lake_model", lecture_text("Lake (intro)")) + repo.lecture("intermediate", "lake_model", lecture_text("Lake (intermediate)")) + repo.write_canonical_map({"lake_model": "intermediate"}) + + assert repo.promote("mccall", "lake_model").returncode == 0 + first = repo.ledger_path.read_bytes() + + res = repo.promote("mccall", "lake_model") + assert res.returncode == 0, res.stderr + assert "2 unchanged" in res.stdout + assert repo.ledger_path.read_bytes() == first + + res = repo.promote("--refresh") + assert res.returncode == 0 + assert "nothing to refresh" in res.stdout + assert repo.ledger_path.read_bytes() == first + + +def stale_pair(make_repo): + """advanced and dp-test hold ``amss`` identically; advanced then edits it + upstream and dp-test does not move — dp-test's copy is stale.""" + repo = make_repo(["advanced", "dp-test"]) + repo.lecture("advanced", "amss", lecture_text("AMSS")) + repo.lecture("dp-test", "amss", lecture_text("AMSS")) + assert repo.promote("amss").returncode == 0 + recorded = {s["series"]: s["digest"] for s in repo.entry("amss")["sources"]} + repo.lecture("advanced", "amss", lecture_text("AMSS", "A typo fixed upstream.")) + repo.move_pin("advanced") + return repo, recorded + + +def test_a_stale_copy_stays_a_source_and_the_refresh_goes_through(make_repo): + repo, recorded = stale_pair(make_repo) + + res = repo.promote("--refresh") + + assert res.returncode == 0, res.stderr + assert "stale: dp-test at an earlier promoted version" in res.stdout + entry = repo.entry("amss") + assert entry["canonical"] == "advanced" + assert "divergent" not in entry and "interim" not in entry + digests = {s["series"]: s["digest"] for s in entry["sources"]} + assert digests["dp-test"] == recorded["dp-test"] + assert digests["advanced"] != recorded["advanced"] + assert "A typo fixed upstream." in repo.pool("amss.md").read_text(encoding="utf-8") + # The drift-check agrees: a stale copy is not drift. + assert repo.drift_check().returncode == 0 + + +def test_a_copy_edited_since_promotion_still_fails_the_refresh(make_repo): + repo, _ = stale_pair(make_repo) + repo.lecture("dp-test", "amss", lecture_text("AMSS", "Edited on its own.")) + before = repo.ledger_path.read_bytes() + + res = repo.promote("--refresh") + + assert res.returncode == 1 + assert "[amss] FAILED: amss.md differs across series" in res.stderr + assert "different: dp-test:" in res.stderr + assert repo.ledger_path.read_bytes() == before + + +def test_a_copy_that_caught_up_then_fell_behind_again_is_still_stale(make_repo): + repo, _ = stale_pair(make_repo) + assert repo.promote("--refresh").returncode == 0 + caught_up = lecture_text("AMSS", "A typo fixed upstream.") + repo.lecture("dp-test", "amss", caught_up) + repo.move_pin("dp-test") + repo.lecture("advanced", "amss", lecture_text("AMSS", "A second fix upstream.")) + repo.move_pin("advanced", 3) + + dc = repo.drift_check() + + assert "[stale-copy]" in dc.stdout and "[diverged]" not in dc.stdout, dc.stdout + res = repo.promote("--refresh") + assert res.returncode == 0, res.stderr + digests = {s["series"]: s["digest"] for s in repo.entry("amss")["sources"]} + assert digests["dp-test"] == hashlib.sha256(caught_up.encode("utf-8")).hexdigest() + + +def test_a_second_canonical_move_keeps_an_unmoved_copy_stale(make_repo): + # After the first refresh only the copy's OWN recorded digest still + # matches it, so this pins that half of the rule. + repo, recorded = stale_pair(make_repo) + assert repo.promote("--refresh").returncode == 0 + repo.lecture("advanced", "amss", lecture_text("AMSS", "A second fix upstream.")) + repo.move_pin("advanced", 3) + + res = repo.promote("--refresh") + + assert res.returncode == 0, res.stderr + digests = {s["series"]: s["digest"] for s in repo.entry("amss")["sources"]} + assert digests["dp-test"] == recorded["dp-test"] + assert "A second fix upstream." in repo.pool("amss.md").read_text(encoding="utf-8") + + +def test_a_copy_never_recorded_as_a_source_is_not_stale(make_repo): + # A series that holds an old version but was never recorded is a + # collision, even when its copy equals the canonical's recorded version. + repo = make_repo(["advanced", "dp-test"]) + repo.lecture("advanced", "amss", lecture_text("AMSS")) + assert repo.promote("amss").returncode == 0 + repo.lecture("dp-test", "amss", lecture_text("AMSS")) + repo.lecture("advanced", "amss", lecture_text("AMSS", "A fix upstream.")) + repo.move_pin("advanced") + + res = repo.promote("--refresh") + + assert res.returncode == 1 + assert "[amss] FAILED: amss.md differs across series" in res.stderr + + +def test_toc_of_a_lagging_series_takes_the_canonical_copy(make_repo): + repo = make_repo(["advanced", "dp-test"]) + repo.lecture("advanced", "amss", lecture_text("AMSS")) + repo.lecture("dp-test", "amss", lecture_text("AMSS")) + repo.toc("dp-test", ["amss"]) + assert repo.promote("--toc", "dp-test").returncode == 0 + assert repo.entry("amss")["canonical"] == "advanced" + repo.lecture("advanced", "amss", lecture_text("AMSS", "A fix upstream.")) + repo.move_pin("advanced") + + # Before any refresh, and again after one, --toc of the lagging series + # takes the canonical copy and keeps its own copy as a stale source. + for _ in range(2): + res = repo.promote("--toc", "dp-test") + assert res.returncode == 0, res.stderr + assert "A fix upstream." in repo.pool("amss.md").read_text(encoding="utf-8") + entry = repo.entry("amss") + assert entry["canonical"] == "advanced" + assert [s["series"] for s in entry["sources"]] == ["advanced", "dp-test"] + assert "divergent" not in entry and "interim" not in entry + assert repo.promote("--refresh").returncode == 0 + + +def test_a_map_entry_naming_a_stale_copy_is_refused(make_repo): + # The map names the lower-priority holder of two identical copies (it + # settles nothing); when the canonical copy moves on, following the map + # would keep the lagging text, so the entry is refused and named. + repo = make_repo(["advanced", "dp-test"]) + repo.lecture("advanced", "amss", lecture_text("AMSS")) + repo.lecture("dp-test", "amss", lecture_text("AMSS")) + repo.write_canonical_map({"amss": "dp-test"}) + res = repo.promote("amss") + assert res.returncode == 0, res.stderr + assert "settles nothing" in res.stdout + assert repo.entry("amss")["canonical"] == "advanced" + before = repo.ledger_path.read_bytes() + repo.lecture("advanced", "amss", lecture_text("AMSS", "A fix upstream.")) + repo.move_pin("advanced") + + res = repo.promote("--refresh") + + assert res.returncode == 1 + failed = [line for line in res.stderr.splitlines() if line.startswith("[amss] FAILED")] + assert len(failed) == 1, res.stderr + assert "names 'dp-test'" in failed[0] and "earlier promoted version" in failed[0] + assert "point the entry at 'advanced'" in failed[0] and "remove it" in failed[0] + assert repo.ledger_path.read_bytes() == before + + +def test_a_map_entry_over_spelling_only_copies_is_not_refused(make_repo): + # Copies equal after asset-path normalisation, nothing moved: a map line + # naming either holder settles nothing, and a refresh goes through. + repo = make_repo(["intro", "intermediate"]) + for series in ("intro", "intermediate"): + repo.mirror_file(series, "_static/lecture_specific/short_path/graph.png", b"png") + repo.lecture("intro", "short_path", lecture_text("Short paths", refs=["/_static/lecture_specific/short_path/graph.png"])) + repo.lecture("intermediate", "short_path", lecture_text("Short paths", refs=["_static/lecture_specific/short_path/graph.png"])) + repo.write_canonical_map({"short_path": "intermediate"}) + assert repo.promote("short_path").returncode == 0 + repo.move_pin("intro") + + res = repo.promote("--refresh") + + assert res.returncode == 0, res.stderr + assert "settles nothing" in res.stdout + + +def test_canonical_map_rejects_repeated_keys_and_two_keys_for_one_lecture(make_repo): + repo = make_repo(["intro", "intermediate"]) + repo.lecture("intro", "lake_model", lecture_text("Lake (intro)")) + repo.lecture("intermediate", "lake_model", lecture_text("Lake (intermediate)")) + path = repo.root / "sync" / "canonical.yml" + + path.write_text("lake_model: intro\nlake_model: intermediate\n", encoding="utf-8") + res = repo.promote("lake_model") + assert res.returncode == 1 + assert "repeated key 'lake_model'" in res.stderr + + path.write_text("lake_model: intro\nlake_model.md: intermediate\n", encoding="utf-8") + res = repo.promote("lake_model") + assert res.returncode == 1 + assert "lake_model.md: names the same lecture as 'lake_model'" in res.stderr + assert not repo.ledger_path.exists() + + +def test_a_map_slug_no_series_holds_is_reported(make_repo): + repo = make_repo(["intro", "intermediate"]) + repo.lecture("intermediate", "career", lecture_text("Career")) + repo.write_canonical_map({"carrer": "intermediate"}) + + res = repo.promote("career") + + assert res.returncode == 0, res.stderr + assert "[carrer] WARN: canonical.yml names carrer, but no canonical-eligible series mirror holds carrer.md" in res.stdout + + +def test_a_map_entry_settling_an_excluded_series_is_not_called_removable(make_repo): + repo = make_repo(["intermediate", "jax"]) + repo.lecture("intermediate", "ifp_egm", lecture_text("IFP (intermediate)")) + repo.lecture("jax", "ifp_egm", lecture_text("IFP (jax)")) + repo.write_canonical_map({"ifp_egm": "intermediate"}) + assert repo.promote("ifp_egm").returncode == 0 + + res = repo.promote("ifp_egm", "--exclude", "jax") + + assert res.returncode == 0, res.stderr + assert "can be removed" not in res.stdout + assert "jax (excluded from this run) holds a different copy" in res.stdout + + +def test_toc_of_a_series_that_caught_up_then_lagged_takes_the_canonical_copy(make_repo): + # The canonical-digest half of the rule under a non-canonical anchor. + repo, _ = stale_pair(make_repo) + repo.toc("dp-test", ["amss"]) + assert repo.promote("--refresh").returncode == 0 # advanced v2; dp-test stale at v1 + repo.lecture("dp-test", "amss", lecture_text("AMSS", "A typo fixed upstream.")) # caught up to v2 + repo.move_pin("dp-test") + repo.lecture("advanced", "amss", lecture_text("AMSS", "A second fix upstream.")) # v3 + repo.move_pin("advanced", 3) + + res = repo.promote("--toc", "dp-test") + + assert res.returncode == 0, res.stderr + entry = repo.entry("amss") + assert entry["canonical"] == "advanced" + assert [s["series"] for s in entry["sources"]] == ["advanced", "dp-test"] + assert "A second fix upstream." in repo.pool("amss.md").read_text(encoding="utf-8") + + +def test_a_stale_copy_is_divergent_when_the_map_names_a_different_lecture(make_repo): + # The map newly names an edited copy that is not the recorded canonical's + # lecture; a third copy stale against the recorded canonical is then not + # the mapped copy's lecture, so it is divergent, not a source. + repo = make_repo(["intermediate", "advanced", "dp-test"]) + for series in ("intermediate", "advanced", "dp-test"): + repo.lecture(series, "amss", lecture_text("AMSS")) + assert repo.promote("amss").returncode == 0 + repo.lecture("intermediate", "amss", lecture_text("AMSS", "Fixed in intermediate.")) + repo.move_pin("intermediate") + repo.lecture("dp-test", "amss", lecture_text("AMSS", "Rewritten in dp-test.")) + repo.move_pin("dp-test") + repo.write_canonical_map({"amss": "dp-test"}) + + res = repo.promote("--refresh") + + assert res.returncode == 0, res.stderr + entry = repo.entry("amss") + assert entry["canonical"] == "dp-test" + assert [d["series"] for d in entry["divergent"]] == ["intermediate", "advanced"] + + +def test_a_lagging_toc_re_anchors_on_the_recorded_canonical_not_on_priority(make_repo): + # intermediate, higher in priority, comes to hold advanced's current text + # after promotion; --toc of the lagging dp-test re-anchors on the + # recorded canonical (advanced), whose identity set includes intermediate. + repo, _ = stale_pair(make_repo) + repo.order = ["intermediate", "advanced", "dp-test"] + (repo.root / "mirror" / "intermediate" / "lectures").mkdir(parents=True) + repo.write_manifest() + repo.write_state({"intermediate": "1" * 40, "advanced": repo.pin("advanced"), "dp-test": repo.pin("dp-test")}) + repo.lecture("intermediate", "amss", lecture_text("AMSS", "A typo fixed upstream.")) + repo.toc("dp-test", ["amss"]) + + res = repo.promote("--toc", "dp-test") + + assert res.returncode == 0, res.stderr + assert "A typo fixed upstream." in repo.pool("amss.md").read_text(encoding="utf-8") + + +def three_holders_one_lagging(make_repo): + """intermediate (canonical) and advanced move in step; dp-test lags.""" + repo = make_repo(["intermediate", "advanced", "dp-test"]) + for series in ("intermediate", "advanced", "dp-test"): + repo.lecture(series, "amss", lecture_text("AMSS")) + repo.toc("advanced", ["amss"]) + assert repo.promote("amss").returncode == 0 + assert repo.entry("amss")["canonical"] == "intermediate" + for series in ("intermediate", "advanced"): + repo.lecture(series, "amss", lecture_text("AMSS", "Fixed in step.")) + repo.move_pin(series) + return repo + + +def test_toc_of_a_holder_in_step_with_the_canonical_keeps_a_third_copy_stale(make_repo): + repo = three_holders_one_lagging(make_repo) + + res = repo.promote("--toc", "advanced") + + assert res.returncode == 0, res.stderr + entry = repo.entry("amss") + assert entry["canonical"] == "intermediate" + assert [s["series"] for s in entry["sources"]] == ["intermediate", "advanced", "dp-test"] + assert "divergent" not in entry and "interim" not in entry + + +def test_a_map_line_naming_a_holder_in_step_keeps_a_third_copy_stale(make_repo): + repo = three_holders_one_lagging(make_repo) + repo.write_canonical_map({"amss": "advanced"}) + + res = repo.promote("--refresh") + + assert res.returncode == 0, res.stderr + assert "settles nothing" in res.stdout + entry = repo.entry("amss") + assert "divergent" not in entry and "interim" not in entry + + +def test_settles_nothing_is_removable_when_the_excluded_series_lacks_the_lecture(make_repo): + repo = make_repo(["intro", "intermediate", "jax"]) + text = lecture_text("Shared") + repo.lecture("intro", "shared", text) + repo.lecture("intermediate", "shared", text) + repo.write_canonical_map({"shared": "intermediate"}) + + res = repo.promote("shared", "--exclude", "jax") + + assert res.returncode == 0, res.stderr + assert "can be removed" in res.stdout + assert "remove it now, since once intro's copy moves on" in res.stdout + + +def test_settles_nothing_is_removable_when_the_excluded_copy_is_the_same_lecture(make_repo): + repo = make_repo(["intro", "intermediate", "jax"]) + text = lecture_text("Shared") + for series in ("intro", "intermediate", "jax"): + repo.lecture(series, "shared", text) + repo.write_canonical_map({"shared": "intro"}) + + res = repo.promote("shared", "--exclude", "jax") + + assert res.returncode == 0, res.stderr + assert "can be removed" in res.stdout and "remove it now" not in res.stdout + + +def test_settles_nothing_is_removable_when_only_an_excluded_consumer_differs(make_repo): + repo = make_repo(["intro", "intermediate", "dp"], consumers=["dp"]) + text = lecture_text("Shared") + repo.lecture("intro", "shared", text) + repo.lecture("intermediate", "shared", text) + repo.lecture("dp", "shared", lecture_text("Shared, dp's older copy")) + repo.write_canonical_map({"shared": "intermediate"}) + + res = repo.promote("shared", "--exclude", "dp") + + assert res.returncode == 0, res.stderr + assert "can be removed" in res.stdout + + +def test_an_excluded_series_without_a_mirror_is_not_called_settled(make_repo): + import shutil + repo = make_repo(["intermediate", "jax"]) + repo.lecture("intermediate", "ifp_egm", lecture_text("IFP (intermediate)")) + repo.lecture("jax", "ifp_egm", lecture_text("IFP (jax)")) + repo.write_canonical_map({"ifp_egm": "intermediate"}) + assert repo.promote("ifp_egm").returncode == 0 + shutil.rmtree(repo.root / "mirror" / "jax") + + res = repo.promote("ifp_egm", "--exclude", "jax") + + assert res.returncode == 0, res.stderr + assert "can be removed" not in res.stdout + assert "jax (excluded, with no mirror on disk) could not be checked" in res.stdout + + +def test_a_map_slug_held_only_by_an_excluded_series_is_not_reported(make_repo): + repo = make_repo(["intro", "intermediate", "jax"]) + repo.lecture("intro", "career", lecture_text("Career")) + repo.lecture("jax", "jax_nn", lecture_text("NN (jax)")) + repo.write_canonical_map({"jax_nn": "jax"}) + + res = repo.promote("career", "--exclude", "jax") + + assert res.returncode == 0, res.stderr + assert "[jax_nn] WARN" not in res.stdout + + +def test_a_map_slug_only_a_consumer_holds_or_in_the_wrong_case_is_reported(make_repo): + repo = make_repo(["intermediate", "dp"], consumers=["dp"]) + repo.lecture("intermediate", "career", lecture_text("Career")) + repo.lecture("dp", "only_dp", lecture_text("Only in dp")) + repo.write_canonical_map({"only_dp": "intermediate", "Career": "intermediate"}) + + res = repo.promote("career") + + assert res.returncode == 0, res.stderr + assert "[only_dp] WARN: canonical.yml names only_dp" in res.stdout + assert "[Career] WARN: canonical.yml names Career" in res.stdout + + +def test_canonical_map_accepts_merge_keys_and_refuses_unhashable_keys(make_repo): + repo = make_repo(["intro", "intermediate"]) + repo.lecture("intro", "lake_model", lecture_text("Lake (intro)")) + repo.lecture("intermediate", "lake_model", lecture_text("Lake (intermediate)")) + path = repo.root / "sync" / "canonical.yml" + + path.write_text("<<: {lake_model: intermediate}\n", encoding="utf-8") + res = repo.promote("lake_model") + assert res.returncode == 0, res.stderr + assert repo.entry("lake_model")["canonical"] == "intermediate" + + path.write_text("? [lake_model, mle]\n: intermediate\n", encoding="utf-8") + res = repo.promote("lake_model") + assert res.returncode == 1 + assert "found an unhashable key" in res.stderr and "Traceback" not in res.stderr diff --git a/tools/tests/test_refresh.py b/tools/tests/test_refresh.py new file mode 100644 index 0000000..454e021 --- /dev/null +++ b/tools/tests/test_refresh.py @@ -0,0 +1,95 @@ +"""``promote --refresh``: an entry this run cannot refresh is never skipped +silently once its canonical copy has moved.""" + +from conftest import lecture_text + + +def jax_only_entry(make_repo): + """A lecture only jax holds, promoted by a default run (jax considered).""" + repo = make_repo(["intermediate", "jax"]) + repo.lecture("jax", "jax_nn", lecture_text("Neural nets")) + assert repo.promote("jax_nn").returncode == 0 + assert repo.entry("jax_nn")["canonical"] == "jax" + return repo + + +def test_an_excluded_canonical_series_is_skipped_while_its_pin_stands(make_repo): + repo = jax_only_entry(make_repo) + repo.move_pin("intermediate") + + res = repo.promote("--refresh", "--exclude", "jax") + + assert res.returncode == 0, res.stderr + assert "[jax_nn] SKIP: canonical series 'jax' is excluded" in res.stdout + + +def test_an_excluded_canonical_series_whose_pin_moved_fails_the_refresh(make_repo): + repo = jax_only_entry(make_repo) + before = repo.ledger_path.read_bytes() + repo.lecture("jax", "jax_nn", lecture_text("Neural nets", "Fixed upstream.")) + repo.move_pin("jax") + + res = repo.promote("--refresh", "--exclude", "jax") + + assert res.returncode == 1 + failed = [line for line in res.stderr.splitlines() if line.startswith("[jax_nn] FAILED")] + assert len(failed) == 1, res.stderr + assert "canonical series 'jax' is excluded, but its pin moved since promotion" in failed[0] + assert "PROMOTE_EXCLUDE" in failed[0] + assert repo.ledger_path.read_bytes() == before + assert "Fixed upstream." not in repo.pool("jax_nn.md").read_text(encoding="utf-8") + + # With jax considered again, the same entry refreshes. + res = repo.promote("--refresh") + assert res.returncode == 0, res.stderr + assert "Fixed upstream." in repo.pool("jax_nn.md").read_text(encoding="utf-8") + + +def test_a_consumer_canonical_entry_whose_pin_moved_fails_the_refresh(make_repo): + # A ledger written before dp was declared a consumer can name dp canonical. + repo = make_repo(["intermediate", "dp"]) + repo.lecture("dp", "dp_only", lecture_text("Only in dp")) + assert repo.promote("dp_only").returncode == 0 + repo.write_manifest(consumers=["dp"]) + + res = repo.promote("--refresh") + assert res.returncode == 0, res.stderr + assert "[dp_only] SKIP: canonical series 'dp' is a consumer" in res.stdout + + repo.move_pin("dp") + res = repo.promote("--refresh") + assert res.returncode == 1 + assert "[dp_only] FAILED: canonical series 'dp' is a consumer, but its pin moved" in res.stderr + + +def test_a_canonical_series_that_left_the_manifest_is_named_as_such(make_repo): + repo = make_repo(["intro", "extra"]) + repo.lecture("extra", "only_extra", lecture_text("Only in extra")) + assert repo.promote("only_extra").returncode == 0 + # The series leaves the manifest; the next sync drops its pin. + repo.order = ["intro"] + repo.write_manifest() + repo.write_state({"intro": repo.pin("intro")}) + + res = repo.promote("--refresh") + + assert res.returncode == 1 + failed = [line for line in res.stderr.splitlines() if line.startswith("[only_extra] FAILED")] + assert len(failed) == 1, res.stderr + assert "canonical series 'extra' is not in sync/manifest.yml" in failed[0] + assert "-> none)" in failed[0] and "excluded" not in failed[0] + + +def test_a_canonical_copy_gone_upstream_is_named_as_such(make_repo): + repo = make_repo(["advanced", "dp-test"]) + for series in ("advanced", "dp-test"): + repo.lecture(series, "amss", lecture_text("AMSS")) + assert repo.promote("amss").returncode == 0 + (repo.root / "mirror" / "advanced" / "lectures" / "amss.md").unlink() + repo.move_pin("advanced") + + res = repo.promote("--refresh") + + assert res.returncode == 1 + assert "[amss] FAILED: the canonical copy of amss.md is gone from advanced upstream" in res.stderr + assert "canonical-removed" in res.stderr