diff --git a/README.md b/README.md index d025282..2021cbe 100644 --- a/README.md +++ b/README.md @@ -279,7 +279,7 @@ and `smoke-mirror` do nothing until a mirror fills them. | `list` | `rsync -rL --list-only` of `SOURCE`, through `FILTER`, normalised to `.run/upstream.txt` as `path TAB size TAB mtime`, byte-sorted; a listing under `LIST_FLOOR` lines stops the run | | `state` | Fetch `.state/applied.txt.xz` from the bucket; a missing one asks `rebuild` | | `rebuild` | List the bucket and make the state exactly what it holds, at upstream's sizes | -| `diff` | `changed.txt` (upstream has, the state lacks), `deleted.txt` (the state has, upstream lacks), `paths.txt` | +| `diff` | `changed.txt` (upstream has, the state lacks, plus the file each changed signed sha512 names), `deleted.txt` (the state has, upstream lacks), `paths.txt` | | `split` | Refuse a tree over `CEILING_GB` or a file the disk cannot hold; split the delta into `batch-NNNN.txt` of `BATCH_GB`, the decision batch last | | `prepare` | Hook. With `TL_KEY` set and the delta touching `TL`: fetch the tlpdb and check its signature against the pinned key | | `batches` | Work the first `MAX_BATCHES`, each `fetch`, `verify`, `publish`, `checkpoint`; touch `.run/chain` when batches remain | diff --git a/engines/rsync.yml b/engines/rsync.yml index 4ad58cb..3165110 100644 --- a/engines/rsync.yml +++ b/engines/rsync.yml @@ -187,9 +187,16 @@ tasks: - rm -f {{.RUN}}/rebuild-now diff: - desc: RUN/changed.txt (lines upstream has and the state lacks), deleted.txt (paths the state has and upstream lacks), paths.txt + desc: RUN/changed.txt (lines upstream has and the state lacks, plus the file each changed signed sha512 names), deleted.txt (paths the state has and upstream lacks), paths.txt cmds: - LC_ALL=C comm -13 {{.RUN}}/applied.txt {{.RUN}}/upstream.txt > {{.RUN}}/changed.txt + # Upstream can write a signed sha512 again while the file it names stands still: TeX + # Live's nightly build checksums and signs its prebuilt installer every night. The + # named file joins the delta, so verify checks it against the new sha512 in staging. + # The pattern is verify's. + - >- + awk -F'\t' -v TL="{{.TL}}" 'NR == FNR { if ($1 ~ "^" TL "/(tlpkg/texlive\\.tlpdb|[^/]+)\\.sha512$") want[substr($1, 1, length($1) - 7)]; next } + $1 in want' {{.RUN}}/changed.txt {{.RUN}}/upstream.txt | LC_ALL=C sort -u -o {{.RUN}}/changed.txt - {{.RUN}}/changed.txt - cut -f1 {{.RUN}}/upstream.txt > {{.RUN}}/paths.txt - cut -f1 {{.RUN}}/applied.txt | LC_ALL=C comm -23 - {{.RUN}}/paths.txt > {{.RUN}}/deleted.txt @@ -213,7 +220,7 @@ tasks: # a batch by itself and leaves the running batch alone. The decision batch, TL's # tlpkg/ (the tlpdb) and the bucket-root files (timestamp among them), is last, so # nothing a client reads to decide what to fetch can name a key that has not landed. - # A path that extends the previous one (x, x.sha512, x.sha512.asc) never starts a new batch, so verify always finds the file its signature names. + # A path that extends the previous one (x, x.sha512, x.sha512.asc) never starts a new batch, and diff puts x in the delta whenever x.sha512 is, so verify always finds the file its signature names. # ponytail: BATCH_GB is spliced bare because it lands inside $(( )), where a quoted # operand is a syntax error. What keeps it safe is the workflow refusing a vars input # that is not a plain value. Ceiling: another caller handing untrusted text to diff --git a/examples/rsync/Taskfile.yml b/examples/rsync/Taskfile.yml index 656e942..04082c6 100644 --- a/examples/rsync/Taskfile.yml +++ b/examples/rsync/Taskfile.yml @@ -62,6 +62,11 @@ tasks: - {task: diff, vars: {RUN: '{{.F}}/run-empty'}} - {task: merge, vars: {RUN: '{{.F}}/run-empty', STAGING: '{{.F}}/run-empty/staging', B: '{{.F}}/run-empty/changed.txt'}} - cmp {{.F}}/run-empty/applied.new {{.F}}/run-empty/applied.txt + # run-resign: upstream wrote x.exe's sha512 and signature again and left x.exe alone, + # as TeX Live's nightly build does to its prebuilt installer. diff takes x.exe back + # into the delta so verify has the file the sha512 names; y.exe, unmoved, stays out. + - {task: diff, vars: {RUN: '{{.F}}/run-resign', TL: tl}} + - test "$(cut -f1 {{.F}}/run-resign/changed.txt | tr '\n' ' ')" = "tl/x.exe tl/x.exe.sha512 tl/x.exe.sha512.asc " # retry: 23 and 24 are successes, a transport code is retried five times, anything else is final. - task retry CMD='exit 23' RETRY_BASE=0 - task retry CMD='exit 24' RETRY_BASE=0 diff --git a/examples/rsync/fixtures/run-resign/applied.txt b/examples/rsync/fixtures/run-resign/applied.txt new file mode 100644 index 0000000..4497dd7 --- /dev/null +++ b/examples/rsync/fixtures/run-resign/applied.txt @@ -0,0 +1,5 @@ +tl/x.exe 2175321 2026/09/28 21:48:36 +tl/x.exe.sha512 157 2026/09/29 23:54:31 +tl/x.exe.sha512.asc 488 2026/09/29 23:54:31 +tl/y.exe 20726697 2026/09/29 23:54:30 +tl/y.exe.sha512 153 2026/09/29 23:54:31 diff --git a/examples/rsync/fixtures/run-resign/upstream.txt b/examples/rsync/fixtures/run-resign/upstream.txt new file mode 100644 index 0000000..db44cfd --- /dev/null +++ b/examples/rsync/fixtures/run-resign/upstream.txt @@ -0,0 +1,5 @@ +tl/x.exe 2175321 2026/09/28 21:48:36 +tl/x.exe.sha512 157 2026/09/30 23:54:31 +tl/x.exe.sha512.asc 488 2026/09/30 23:54:31 +tl/y.exe 20726697 2026/09/29 23:54:30 +tl/y.exe.sha512 153 2026/09/29 23:54:31 diff --git a/examples/rsync/render.txt b/examples/rsync/render.txt index b8b51f8..6565563 100644 --- a/examples/rsync/render.txt +++ b/examples/rsync/render.txt @@ -51,6 +51,7 @@ task: [push] aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout task: [rebuild] mv /work/examples/rsync/.run/applied.new /work/examples/rsync/.run/applied.txt task: [rebuild] rm -f /work/examples/rsync/.run/rebuild-now task: [diff] LC_ALL=C comm -13 /work/examples/rsync/.run/applied.txt /work/examples/rsync/.run/upstream.txt > /work/examples/rsync/.run/changed.txt +task: [diff] awk -F'\t' -v TL="" 'NR == FNR { if ($1 ~ "^" TL "/(tlpkg/texlive\\.tlpdb|[^/]+)\\.sha512$") want[substr($1, 1, length($1) - 7)]; next } $1 in want' /work/examples/rsync/.run/changed.txt /work/examples/rsync/.run/upstream.txt | LC_ALL=C sort -u -o /work/examples/rsync/.run/changed.txt - /work/examples/rsync/.run/changed.txt task: [diff] cut -f1 /work/examples/rsync/.run/upstream.txt > /work/examples/rsync/.run/paths.txt task: [diff] cut -f1 /work/examples/rsync/.run/applied.txt | LC_ALL=C comm -23 - /work/examples/rsync/.run/paths.txt > /work/examples/rsync/.run/deleted.txt task: [split] awk -F'\t' '{ s += $2 } END { printf "upstream: %d objects, %.2f GB, no ceiling\n", NR, s / 1e9 }' /work/examples/rsync/.run/upstream.txt