From d8b5d43788c587a3c99f1a90b966ea9d5d826231 Mon Sep 17 00:00:00 2001 From: Timothy van der Ham Date: Mon, 31 Aug 2026 07:25:06 +0200 Subject: [PATCH 1/4] Pin CI's qpdf and poppler oracles instead of floating on ubuntu-latest apt's qpdf on the runner is 11.9.0, two majors behind upstream, and --check's output has changed across qpdf majors while #186's acceptance criterion is exactly that output. Install qpdf 12.4.1 from the official release artifact (checksum-verified) instead; pin poppler-utils, fonts-dejavu-core and fonts-texgyre to the exact versions the runner's apt archive serves today, since an unpinned font update silently changes fixture rendering for every downstream oracle. Also deduplicate the veraPDF Docker tag into one job-level env var, bump zxing-cpp to 3.1.1 with pillow pinned alongside it (the barcode oracle suite passes unchanged), bump setup-dotnet/setup-python to their current majors, and add a step that asserts each pinned tool's own version report so drift fails loudly instead of quietly changing what CI validates against. Closes #230 --- .github/workflows/ci.yml | 77 ++++++++++++++++++++++++++++++----- .github/workflows/docs.yml | 2 +- .github/workflows/release.yml | 4 +- CHANGELOG.md | 17 ++++++++ CONTRIBUTING.md | 20 ++++++--- 5 files changed, 101 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3e81af69..512c04b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,10 @@ on: jobs: build: runs-on: ubuntu-latest + env: + # Single source for the veraPDF Docker tag, referenced by both the image pull and the + # shim it backs (#230) — previously duplicated so bumping veraPDF meant editing two lines. + VERAPDF_TAG: v1.30.2 steps: # Full history: the clean-room check scans the branch's commit MESSAGES as well as its files, # and a merged message cannot be corrected afterwards. The default depth-1 checkout gives it @@ -18,7 +22,7 @@ jobs: with: fetch-depth: 0 - - uses: actions/setup-dotnet@v5 + - uses: actions/setup-dotnet@v6 with: dotnet-version: '10.0.x' @@ -45,11 +49,37 @@ jobs: - name: Build run: dotnet build VellumPdf.slnx -c Release --no-restore - - name: Install PDF validators - run: sudo apt-get update && sudo apt-get install -y qpdf poppler-utils fonts-dejavu-core fonts-texgyre + - name: Install poppler and fonts (pinned) + # Versions are whatever ubuntu-latest's noble apt archive currently serves; a font or + # poppler update changes fixture rendering and oracle output with no commit to blame + # (#230), so pin them the same way qpdf is pinned below instead of taking apt's default. + run: | + sudo apt-get update + sudo apt-get install -y \ + poppler-utils=24.02.0-1ubuntu9.9 \ + fonts-dejavu-core=2.37-8 \ + fonts-texgyre=20180621-6 + + - name: Install qpdf (pinned) + # apt's qpdf on ubuntu-latest is 11.9.0, two majors behind upstream; `--check`'s output + # has changed across qpdf majors and #186's acceptance criterion is exactly that output + # (#230). Installed from the official release artifact instead, matched by checksum. + env: + QPDF_VERSION: '12.4.1' + QPDF_SHA256: 'db9122e88ec00c76ac6a14e09ffb92406db1773d47b968911ff6e69f28c09bf9' + run: | + set -euo pipefail + asset="qpdf-${QPDF_VERSION}-bin-linux-x86_64.zip" + curl -fsSL -o /tmp/qpdf.zip \ + "https://github.com/qpdf/qpdf/releases/download/v${QPDF_VERSION}/${asset}" + echo "${QPDF_SHA256} /tmp/qpdf.zip" | sha256sum -c - + sudo mkdir -p "/opt/qpdf-${QPDF_VERSION}" + sudo unzip -q /tmp/qpdf.zip -d "/opt/qpdf-${QPDF_VERSION}" + echo "/opt/qpdf-${QPDF_VERSION}/bin" >> "$GITHUB_PATH" + echo "QPDF_HOME=/opt/qpdf-${QPDF_VERSION}" >> "$GITHUB_ENV" - name: Pull veraPDF Docker image - run: docker pull verapdf/cli:v1.30.2 + run: docker pull verapdf/cli:${{ env.VERAPDF_TAG }} - name: Install veraPDF shim run: | @@ -59,18 +89,45 @@ jobs: # /tmp lets the tests' temp-dir PDF paths resolve identically in-container. sudo tee /usr/local/bin/verapdf > /dev/null << 'SHIM' #!/bin/sh - exec docker run --rm -v /tmp:/tmp -w /tmp verapdf/cli:v1.30.2 "$@" + exec docker run --rm -v /tmp:/tmp -w /tmp verapdf/cli:${{ env.VERAPDF_TAG }} "$@" SHIM sudo chmod +x /usr/local/bin/verapdf - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v7 with: python-version: '3.x' - name: Install barcode decode oracle (zxing-cpp) - # Pinned: the EAN add-on text format differs across zxing-cpp versions. setup-python's - # toolcache interpreter avoids the PEP 668 externally-managed block on the system python. - run: python -m pip install zxing-cpp==3.0.0 pillow + # setup-python's toolcache interpreter avoids the PEP 668 externally-managed block on + # the system python. pillow is pinned alongside it so a Pillow release can't shift + # fixture rendering the same way an unpinned apt font package could (#230). + run: python -m pip install zxing-cpp==3.1.1 pillow==12.3.0 + + - name: Assert pinned oracle versions + # Prints and checks each pinned tool's own version report, so drift between this file + # and what actually got installed fails loudly instead of silently changing what every + # downstream oracle test validates against (#230). + run: | + set -euo pipefail + + qpdf_version=$(qpdf --version) + echo "$qpdf_version" + echo "$qpdf_version" | grep -qF "12.4.1" + + poppler_version=$(pdftotext -v 2>&1) + echo "$poppler_version" + echo "$poppler_version" | grep -qF "24.02.0" + + verapdf_version=$(verapdf --version 2>&1) + echo "$verapdf_version" + echo "$verapdf_version" | grep -qF "1.30.2" + + python -c " + import importlib.metadata as m + v = m.version('zxing-cpp') + print('zxing-cpp', v) + assert v == '3.1.1', v + " - name: Test # REQUIRE_VERAPDF makes the conformance oracle fail (not silently skip) if the verapdf @@ -162,7 +219,7 @@ jobs: steps: - uses: actions/checkout@v7 - - uses: actions/setup-dotnet@v5 + - uses: actions/setup-dotnet@v6 with: dotnet-version: '10.0.x' diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 3e59c914..8c8b8ff0 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -36,7 +36,7 @@ jobs: steps: - uses: actions/checkout@v7 - - uses: actions/setup-dotnet@v5 + - uses: actions/setup-dotnet@v6 with: dotnet-version: '10.0.x' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9e944d47..739154a1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,7 +15,7 @@ jobs: steps: - uses: actions/checkout@v7 - - uses: actions/setup-dotnet@v5 + - uses: actions/setup-dotnet@v6 with: dotnet-version: '10.0.x' @@ -119,7 +119,7 @@ jobs: steps: - uses: actions/checkout@v7 - - uses: actions/setup-dotnet@v5 + - uses: actions/setup-dotnet@v6 with: dotnet-version: '10.0.x' diff --git a/CHANGELOG.md b/CHANGELOG.md index d4ddea80..225e2c29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -188,6 +188,23 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Changed +- **CI's external oracles are now pinned instead of floating on whatever `ubuntu-latest` ships.** + qpdf was the worst offender: apt's `qpdf` on the runner is 11.9.0, two majors behind upstream + 12.4.1, and `--check`'s output has changed across qpdf majors while #186's acceptance criterion + is exactly that output. CI now installs qpdf 12.4.1 from the official release artifact + (checksum-verified) instead of apt; `poppler-utils`, `fonts-dejavu-core` and `fonts-texgyre` stay + on apt but pin to the exact versions `ubuntu-latest`'s noble archive serves today + (24.02.0-1ubuntu9.9, 2.37-8, 20180621-6 respectively), so a font update can no longer silently + shift fixture rendering out from under every downstream oracle. The veraPDF Docker tag, already + pinned, was duplicated across the image pull and the shim that backs it; both now read one + job-level `VERAPDF_TAG`. zxing-cpp moves from 3.0.0 to 3.1.1 (with `pillow` newly pinned to + 12.3.0) — the barcode oracle suite passes unchanged, including the EAN add-on case the 3.0.0 pin + was recorded against, since that test only asserts the main 13 digits and treats the add-on's + presentation as version-dependent. A new version-assert step prints and checks each pinned + tool's own version report after install, so drift between the workflow and what actually landed + fails the build instead of changing what CI validates against with no commit to blame. + `actions/setup-dotnet` and `actions/setup-python` also move to their current majors (v6 and v7) + across all three workflow files. (#230) - The roadmap now describes the scope past 2.5 as two parallel tracks, Kernel and conformance alongside Layout, and adds the milestones covering the ISO/TS extension series, embedded files, graphics, fonts, tagged PDF, PDF/UA-2 and signature verification. The previous table had drifted: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 955d2468..80fe35cd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,8 +9,14 @@ build, test, and submit changes to VellumPdf. - **Docker** — needed to run the veraPDF conformance gate locally. - **qpdf** and **poppler-utils** — needed to run the structural-validator, text-extraction, signature, and barcode-rasterization oracle tests locally - (`qpdf`, `pdftotext`, `pdfsig`, `pdftoppm`). On Debian/Ubuntu: - `sudo apt-get install qpdf poppler-utils fonts-dejavu-core fonts-texgyre`. + (`qpdf`, `pdftotext`, `pdfsig`, `pdftoppm`). CI pins these (#230) rather than taking whatever + the runner image ships, so matching CI locally means matching its versions: qpdf 12.4.1 + (installed in CI from the [official release + artifact](https://github.com/qpdf/qpdf/releases/tag/v12.4.1), since apt's qpdf on + `ubuntu-latest` is 11.9.0 and `--check`'s output has changed across qpdf majors), poppler-utils + 24.02.0-1ubuntu9.9, fonts-dejavu-core 2.37-8, and fonts-texgyre 20180621-6. On Debian/Ubuntu, + install qpdf from the same release artifact and pin the rest with + `sudo apt-get install poppler-utils=24.02.0-1ubuntu9.9 fonts-dejavu-core=2.37-8 fonts-texgyre=20180621-6`. On Windows, PATH order between shells is not reliable — the same bare `pdftotext` can resolve to a completely different program depending on which shell launched the test host, so point `QPDF_HOME` and `POPPLER_HOME` at @@ -18,8 +24,8 @@ build, test, and submit changes to VellumPdf. the directory holding the executable directly, its parent with a `bin` subdirectory under it, or its grandparent with a `Library\bin` subdirectory under it (the shape a Windows poppler build installed via winget uses) — so - `QPDF_HOME` can name either `...\qpdf-12.3.2-msvc64` or - `...\qpdf-12.3.2-msvc64\bin`, and `POPPLER_HOME` can name either the + `QPDF_HOME` can name either `...\qpdf-12.4.1-msvc64` or + `...\qpdf-12.4.1-msvc64\bin`, and `POPPLER_HOME` can name either the poppler install root or its `Library\bin` folder directly. A `*_HOME` that is set but does not resolve through any of these is reported as a misconfiguration (the same skip-locally/fail-on-CI outcome a wrong-tool @@ -36,8 +42,10 @@ build, test, and submit changes to VellumPdf. (`ExternalToolResolutionTests`) checks that each tool resolves to itself on every run. - **Python with zxing-cpp** — the barcode decode oracle: - `python -m pip install zxing-cpp==3.0.0 pillow`. The version is pinned because - the EAN add-on text format differs between zxing-cpp releases. + `python -m pip install zxing-cpp==3.1.1 pillow==12.3.0`. zxing-cpp is pinned because the EAN + add-on text format differs between releases; the barcode oracle test asserts only the main + digits for that reason (`EanBarcode_Ean13WithAddOn_MainDigitsExact_AddOnTolerant`), so the pin + exists to keep CI reproducible rather than to work around a currently-failing assertion. ## Building and testing From 47348f3b04ec5d5619c8d1d1528fed76897a59d7 Mon Sep 17 00:00:00 2001 From: Timothy van der Ham Date: Mon, 31 Aug 2026 09:16:40 +0200 Subject: [PATCH 2/4] Address review: pin the runner, survive apt's revision rotation runs-on moves from the floating ubuntu-latest to ubuntu-24.04, since every apt pin in this job is noble-only and would all break at once the day GitHub retargets the label. The poppler/font =version pins alone were a scheduled outage: noble's -updates/-security pockets keep only the newest revision, so the exact revisions here would 404 the moment Ubuntu ships another one. apt-get now passes -o APT::Snapshot for the date those versions were measured, which rewrites the runner's existing sources to the dated Ubuntu snapshot and keeps the pinned revisions resolvable indefinitely; verified in a container with the live archive already past the pinned revision. setup-python's interpreter is pinned to 3.14 (zxing-cpp 3.1.1 has no wheel past it) instead of floating, and docs-inventory.yml's own setup-python, missed by the previous sweep, moves to v7 to match. The version-assert step now reads every pinned version from job-level env instead of re-hardcoding them, asserts the apt packages' full Ubuntu revision via dpkg-query (pdftotext -v can't see it and fonts can't self-report at all), and compares full lines/values instead of substrings so a future point release can't slip past a careless grep. Restored the zxing-cpp pin's inline rationale in ci.yml, which the previous edit dropped, and updated the CHANGELOG/CONTRIBUTING/CLAUDE.md prose to match the runner label, the snapshot mechanism and the full apt version. tests/VellumPdf.Reader.Tests/Fixtures/Encrypted/README.md's CI/local qpdf version claim was stale in both directions; both are 12.4.1 now. Reran its two qpdf-12.3.2-specific behavioral claims (catalog-in-object-stream under encryption, and /V4 /CFM /V2 ignoring /Length) against 12.4.1 and recorded that both still hold. Refs #230 --- .github/workflows/ci.yml | 124 ++++++++++++------ .github/workflows/docs-inventory.yml | 4 +- CHANGELOG.md | 38 +++--- CONTRIBUTING.md | 17 ++- .../Fixtures/Encrypted/README.md | 15 ++- 5 files changed, 140 insertions(+), 58 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 512c04b3..0a303fd0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,11 +8,27 @@ on: jobs: build: - runs-on: ubuntu-latest + # Pinned to the specific label, not the floating `ubuntu-latest`: every apt pin below is + # noble-only, and GitHub moves `ubuntu-latest` to a new release on its own schedule, which + # would break all of them at once with no commit to blame (#230). + runs-on: ubuntu-24.04 env: - # Single source for the veraPDF Docker tag, referenced by both the image pull and the - # shim it backs (#230) — previously duplicated so bumping veraPDF meant editing two lines. + # Every pinned oracle version in one place, referenced by both the install steps and the + # version-assert step below, so bumping one means editing one line rather than two. VERAPDF_TAG: v1.30.2 + QPDF_VERSION: '12.4.1' + QPDF_SHA256: 'db9122e88ec00c76ac6a14e09ffb92406db1773d47b968911ff6e69f28c09bf9' + POPPLER_VERSION: '24.02.0-1ubuntu9.9' + FONTS_DEJAVU_VERSION: '2.37-8' + FONTS_TEXGYRE_VERSION: '20180621-6' + # noble's apt pockets keep only the newest revision of each package, so a bare `=version` + # pin eventually 404s the moment Ubuntu ships another revision (poppler-utils alone has + # already rolled 9.1 through 9.8 off the archive). Point apt at the Ubuntu snapshot service + # for the date the versions above were measured, so the exact revision keeps resolving + # after the live archive rotates past it — see the "Install poppler and fonts" step. + APT_SNAPSHOT: '20260830T000000Z' + ZXING_VERSION: '3.1.1' + PILLOW_VERSION: '12.3.0' steps: # Full history: the clean-room check scans the branch's commit MESSAGES as well as its files, # and a merged message cannot be corrected afterwards. The default depth-1 checkout gives it @@ -50,31 +66,38 @@ jobs: run: dotnet build VellumPdf.slnx -c Release --no-restore - name: Install poppler and fonts (pinned) - # Versions are whatever ubuntu-latest's noble apt archive currently serves; a font or - # poppler update changes fixture rendering and oracle output with no commit to blame - # (#230), so pin them the same way qpdf is pinned below instead of taking apt's default. + # A font or poppler update changes fixture rendering and oracle output with no commit to + # blame (#230), so pin exact versions rather than taking apt's default. The `=version` + # pins alone are not enough: noble's -updates/-security pockets keep only the newest + # revision, so the moment Ubuntu ships another one, these exact revisions 404 out of the + # live archive and every PR goes red with no code change involved. `-o APT::Snapshot` + # rewrites the runner's existing archive.ubuntu.com sources to the dated Ubuntu snapshot + # instead, which never evicts what it once published — verified against a fresh + # ubuntu:24.04 container with the live archive already past this exact revision, and it + # still resolved. If a real bump is ever needed instead of just chasing a rotated + # revision — a new poppler feature, a font correction — update POPPLER_VERSION / + # FONTS_*_VERSION and APT_SNAPSHOT together (`apt-cache policy` against a fresh + # `ubuntu:24.04` container gives both) and mention it in CONTRIBUTING.md. run: | - sudo apt-get update - sudo apt-get install -y \ - poppler-utils=24.02.0-1ubuntu9.9 \ - fonts-dejavu-core=2.37-8 \ - fonts-texgyre=20180621-6 + set -euo pipefail + sudo apt-get update -o APT::Snapshot="${APT_SNAPSHOT}" + sudo apt-get install -y -o APT::Snapshot="${APT_SNAPSHOT}" \ + "poppler-utils=${POPPLER_VERSION}" \ + "fonts-dejavu-core=${FONTS_DEJAVU_VERSION}" \ + "fonts-texgyre=${FONTS_TEXGYRE_VERSION}" - name: Install qpdf (pinned) - # apt's qpdf on ubuntu-latest is 11.9.0, two majors behind upstream; `--check`'s output + # apt's qpdf on ubuntu-24.04 is 11.9.0, two majors behind upstream; `--check`'s output # has changed across qpdf majors and #186's acceptance criterion is exactly that output # (#230). Installed from the official release artifact instead, matched by checksum. - env: - QPDF_VERSION: '12.4.1' - QPDF_SHA256: 'db9122e88ec00c76ac6a14e09ffb92406db1773d47b968911ff6e69f28c09bf9' run: | set -euo pipefail asset="qpdf-${QPDF_VERSION}-bin-linux-x86_64.zip" - curl -fsSL -o /tmp/qpdf.zip \ + curl -fsSL --retry 3 --retry-delay 2 --retry-all-errors -o /tmp/qpdf.zip \ "https://github.com/qpdf/qpdf/releases/download/v${QPDF_VERSION}/${asset}" echo "${QPDF_SHA256} /tmp/qpdf.zip" | sha256sum -c - sudo mkdir -p "/opt/qpdf-${QPDF_VERSION}" - sudo unzip -q /tmp/qpdf.zip -d "/opt/qpdf-${QPDF_VERSION}" + sudo unzip -o -q /tmp/qpdf.zip -d "/opt/qpdf-${QPDF_VERSION}" echo "/opt/qpdf-${QPDF_VERSION}/bin" >> "$GITHUB_PATH" echo "QPDF_HOME=/opt/qpdf-${QPDF_VERSION}" >> "$GITHUB_ENV" @@ -95,38 +118,65 @@ jobs: - uses: actions/setup-python@v7 with: - python-version: '3.x' + # zxing-cpp 3.1.1 ships prebuilt wheels for cp310–cp314; a floating '3.x' would + # eventually resolve to 3.15 and fall back to a source build with no commit to blame. + python-version: '3.14' - name: Install barcode decode oracle (zxing-cpp) - # setup-python's toolcache interpreter avoids the PEP 668 externally-managed block on - # the system python. pillow is pinned alongside it so a Pillow release can't shift - # fixture rendering the same way an unpinned apt font package could (#230). - run: python -m pip install zxing-cpp==3.1.1 pillow==12.3.0 + # Pinned for CI reproducibility (#230), not because 3.1.1 fails the EAN add-on assertion + # it was originally held against — that test only asserts the main digits and tolerates + # the add-on's presentation varying by version (see CONTRIBUTING.md), and it landed in + # the same commit as this pin, passing under 3.1.1. pillow is pinned alongside it so a + # Pillow release can't shift fixture rendering the same way an unpinned apt font package + # could. + run: python -m pip install "zxing-cpp==${ZXING_VERSION}" "pillow==${PILLOW_VERSION}" - name: Assert pinned oracle versions - # Prints and checks each pinned tool's own version report, so drift between this file - # and what actually got installed fails loudly instead of silently changing what every - # downstream oracle test validates against (#230). + # Prints and checks each pinned tool's own version report against the job-level env vars + # above, so drift between this file and what actually got installed fails loudly instead + # of silently changing what every downstream oracle test validates against (#230). + # Comparisons are exact-line or exact-value, not substring greps: "12.4.1" is a substring + # of "12.4.10", so a careless `grep -qF` would pass against a future revision it was + # never meant to accept. run: | set -euo pipefail - qpdf_version=$(qpdf --version) - echo "$qpdf_version" - echo "$qpdf_version" | grep -qF "12.4.1" + qpdf_line=$(qpdf --version | head -n1) + echo "$qpdf_line" + [ "$qpdf_line" = "qpdf version ${QPDF_VERSION}" ] - poppler_version=$(pdftotext -v 2>&1) - echo "$poppler_version" - echo "$poppler_version" | grep -qF "24.02.0" + # pdftotext -v reports only poppler's upstream version (e.g. "24.02.0"), never the + # Ubuntu package revision after the dash — apt can ship a new revision of the same + # upstream release, and fonts have no version-reporting flag at all. dpkg-query reads + # the installed package record directly, which is the only way to see the full pin. + echo "--- installed package versions ---" + dpkg-query -W -f='${Package} ${Version}\n' poppler-utils fonts-dejavu-core fonts-texgyre + assert_pkg_version() { + local pkg="$1" expected="$2" actual + actual=$(dpkg-query -W -f='${Version}' "$pkg") + if [ "$actual" != "$expected" ]; then + echo "::error::$pkg is $actual, expected $expected" + exit 1 + fi + } + assert_pkg_version poppler-utils "$POPPLER_VERSION" + assert_pkg_version fonts-dejavu-core "$FONTS_DEJAVU_VERSION" + assert_pkg_version fonts-texgyre "$FONTS_TEXGYRE_VERSION" - verapdf_version=$(verapdf --version 2>&1) - echo "$verapdf_version" - echo "$verapdf_version" | grep -qF "1.30.2" + verapdf_line=$(verapdf --version 2>&1 | head -n1) + echo "$verapdf_line" + [ "$verapdf_line" = "veraPDF ${VERAPDF_TAG#v}" ] python -c " import importlib.metadata as m - v = m.version('zxing-cpp') - print('zxing-cpp', v) - assert v == '3.1.1', v + zxing_expected = '${ZXING_VERSION}' + pillow_expected = '${PILLOW_VERSION}' + zxing_actual = m.version('zxing-cpp') + pillow_actual = m.version('pillow') + print('zxing-cpp', zxing_actual) + print('pillow', pillow_actual) + assert zxing_actual == zxing_expected, zxing_actual + assert pillow_actual == pillow_expected, pillow_actual " - name: Test diff --git a/.github/workflows/docs-inventory.yml b/.github/workflows/docs-inventory.yml index 1b5e5a3c..0d3cb6fb 100644 --- a/.github/workflows/docs-inventory.yml +++ b/.github/workflows/docs-inventory.yml @@ -36,9 +36,9 @@ jobs: steps: - uses: actions/checkout@v7 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v7 with: - python-version: '3.13' + python-version: '3.14' - name: Inventory is up to date run: python eng/generate-pdf20-inventory.py --check diff --git a/CHANGELOG.md b/CHANGELOG.md index 225e2c29..061f0387 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -188,23 +188,31 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Changed -- **CI's external oracles are now pinned instead of floating on whatever `ubuntu-latest` ships.** - qpdf was the worst offender: apt's `qpdf` on the runner is 11.9.0, two majors behind upstream - 12.4.1, and `--check`'s output has changed across qpdf majors while #186's acceptance criterion - is exactly that output. CI now installs qpdf 12.4.1 from the official release artifact - (checksum-verified) instead of apt; `poppler-utils`, `fonts-dejavu-core` and `fonts-texgyre` stay - on apt but pin to the exact versions `ubuntu-latest`'s noble archive serves today +- **CI's external oracles are now pinned instead of floating on whatever the runner image ships.** + The build job itself moves off the floating `ubuntu-latest` label onto `ubuntu-24.04`, since + every apt pin below is noble-only and would all break at once the day GitHub retargets the + label. qpdf was the worst offender: apt's `qpdf` on the runner is 11.9.0, two majors behind + upstream 12.4.1, and `--check`'s output has changed across qpdf majors while #186's acceptance + criterion is exactly that output. CI now installs qpdf 12.4.1 from the official release artifact + (checksum-verified) instead of apt; `poppler-utils`, `fonts-dejavu-core` and `fonts-texgyre` + stay on apt, pinned to the exact versions `ubuntu-24.04`'s noble archive serves today (24.02.0-1ubuntu9.9, 2.37-8, 20180621-6 respectively), so a font update can no longer silently - shift fixture rendering out from under every downstream oracle. The veraPDF Docker tag, already - pinned, was duplicated across the image pull and the shim that backs it; both now read one - job-level `VERAPDF_TAG`. zxing-cpp moves from 3.0.0 to 3.1.1 (with `pillow` newly pinned to - 12.3.0) — the barcode oracle suite passes unchanged, including the EAN add-on case the 3.0.0 pin - was recorded against, since that test only asserts the main 13 digits and treats the add-on's - presentation as version-dependent. A new version-assert step prints and checks each pinned - tool's own version report after install, so drift between the workflow and what actually landed - fails the build instead of changing what CI validates against with no commit to blame. + shift fixture rendering out from under every downstream oracle. Since noble's apt pockets keep + only the newest revision of a package, the install step also pins apt to the Ubuntu snapshot + service for the date those versions were measured, so the exact revisions keep resolving after + the live archive rotates past them instead of 404ing with no code change involved. The veraPDF + Docker tag, already pinned, was duplicated across the image pull and the shim that backs it; + both now read one job-level `VERAPDF_TAG`. zxing-cpp moves from 3.0.0 to 3.1.1 (with `pillow` + newly pinned to 12.3.0) — the barcode oracle suite passes unchanged, including the EAN add-on + case the 3.0.0 pin was recorded against, since that test only asserts the main 13 digits and + treats the add-on's presentation as version-dependent; `setup-python`'s interpreter is pinned + to 3.14 rather than floating, since zxing-cpp 3.1.1 ships prebuilt wheels only through 3.14. A + new version-assert step checks each pinned tool's own version report after install — including + the apt packages' full Ubuntu revision via `dpkg-query`, since `pdftotext -v` cannot see that + and fonts cannot self-report a version at all — so drift between the workflow and what actually + landed fails the build instead of changing what CI validates against with no commit to blame. `actions/setup-dotnet` and `actions/setup-python` also move to their current majors (v6 and v7) - across all three workflow files. (#230) + across all four workflow files. (#230) - The roadmap now describes the scope past 2.5 as two parallel tracks, Kernel and conformance alongside Layout, and adds the milestones covering the ISO/TS extension series, embedded files, graphics, fonts, tagged PDF, PDF/UA-2 and signature verification. The previous table had drifted: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 80fe35cd..10d9453b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,10 +13,25 @@ build, test, and submit changes to VellumPdf. the runner image ships, so matching CI locally means matching its versions: qpdf 12.4.1 (installed in CI from the [official release artifact](https://github.com/qpdf/qpdf/releases/tag/v12.4.1), since apt's qpdf on - `ubuntu-latest` is 11.9.0 and `--check`'s output has changed across qpdf majors), poppler-utils + `ubuntu-24.04` is 11.9.0 and `--check`'s output has changed across qpdf majors), poppler-utils 24.02.0-1ubuntu9.9, fonts-dejavu-core 2.37-8, and fonts-texgyre 20180621-6. On Debian/Ubuntu, install qpdf from the same release artifact and pin the rest with `sudo apt-get install poppler-utils=24.02.0-1ubuntu9.9 fonts-dejavu-core=2.37-8 fonts-texgyre=20180621-6`. + + **CI's poppler/font pins have a known maintenance cost.** noble's `-updates`/`-security` + pockets keep only the newest revision of each package, so the exact revisions above eventually + vanish from the live apt archive on their own, with no code change involved. When that happens, + `ci.yml`'s "Install poppler and fonts" step still resolves them, because that step passes + `-o APT::Snapshot=$APT_SNAPSHOT` on both `apt-get update` and `apt-get install`, which rewrites + the runner's existing archive.ubuntu.com sources to the [Ubuntu snapshot + service](https://snapshot.ubuntu.com/) as it stood on that date — a snapshot never evicts what + it once published, even after the live archive moves on. The failure mode to watch for instead + is a genuine version bump: if poppler or a font package needs a newer release on purpose, run + `apt-cache policy poppler-utils fonts-dejavu-core fonts-texgyre` against a fresh + `ubuntu:24.04` container to get the new versions and a matching `APT_SNAPSHOT` date, then + update `POPPLER_VERSION` / `FONTS_DEJAVU_VERSION` / `FONTS_TEXGYRE_VERSION` and + `APT_SNAPSHOT` together in `ci.yml`'s job-level `env:`, and this paragraph to match. + On Windows, PATH order between shells is not reliable — the same bare `pdftotext` can resolve to a completely different program depending on which shell launched the test host, so point `QPDF_HOME` and `POPPLER_HOME` at diff --git a/tests/VellumPdf.Reader.Tests/Fixtures/Encrypted/README.md b/tests/VellumPdf.Reader.Tests/Fixtures/Encrypted/README.md index 6221d72f..bcd6ed69 100644 --- a/tests/VellumPdf.Reader.Tests/Fixtures/Encrypted/README.md +++ b/tests/VellumPdf.Reader.Tests/Fixtures/Encrypted/README.md @@ -1,8 +1,8 @@ # Encrypted reader fixtures -Generated once with qpdf and checked in, rather than generated at test time. CI installs qpdf -from apt on `ubuntu-latest` (11.9.0 at the time of writing, but nothing pins it) while local -development uses 12.3.2; checking the files in makes the corpus +Generated once with qpdf and checked in, rather than generated at test time. CI and local +development both use qpdf 12.4.1 (#230 pins CI to the official release artifact instead of apt, +whose `ubuntu-24.04` version was 11.9.0); checking the files in makes the corpus byte-identical everywhere and keeps qpdf out of the test-execution path, so there is no `OracleGate` skip hole on the core corpus. @@ -195,6 +195,9 @@ what remains: `HandBuiltEncryptedDocuments.BuildCatalogInObjectStream`, shared between `EncryptedExemptionTests` and `EncryptedReconstructionTests` — not a fixture here, and carrying no `Corpus` row, since the guard in `EncryptedFixtureCorpusTests` only fires for files actually embedded from this directory. + Reconfirmed unchanged under qpdf 12.4.1 (#230): RC4-128 and AES-256 with `--object-streams=generate` + both still write the catalog as a plain top-level object, never inside the `/Type /ObjStm` they + emit alongside it. - **Every fixture is qpdf's output.** This is the largest gap in the corpus and the hardest to close: producers differ in exactly the places this code has to decide. A crypt filter `/Length` in @@ -263,6 +266,12 @@ what remains: is available to settle it, which is why the behaviour is left alone and recorded instead. Its failure mode is a rejected password, not wrong bytes. + Reconfirmed unchanged under qpdf 12.4.1 (#230): a genuinely 128-bit-keyed `/V 4` `/CFM /V2` file + with both `/Length` entries hex-patched down to declare 40 bits / 5 bytes still opens under its + original password and decrypts correctly, so qpdf 12.4.1 derives the same 16-byte key regardless + of what the patched entries claim. The full matrix above was not re-run digit-for-digit; this one + case is the mechanism the whole matrix follows from. + - **No file whose `/StmF` and `/StrF` name `/CF` entries of different lengths.** The key-length path reads `/StmF` first and falls back to `/StrF`, and that precedence is observable only on such a document. Failure is a rejected password, not wrong bytes. From e6ebf544f687c204d36f3c4178822df60d82776e Mon Sep 17 00:00:00 2001 From: Timothy van der Ham Date: Mon, 31 Aug 2026 10:21:25 +0200 Subject: [PATCH 3/4] Make the snapshot pin's own success observable, not assumed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit apt-get update exits 0 even when the -o APT::Snapshot fetch itself fails outright, and silently carries on with whatever else is already cached or configured. Reproduced both failure shapes in a container (a TLS trust failure from missing ca-certificates, and a blackholed host) and confirmed apt gives no other signal in either case, so the poppler/fonts install step now greps /var/lib/apt/lists/ for the snapshot's own index files right after the update and fails loudly if they never landed. The mechanism comment, CONTRIBUTING's paragraph, and the qpdf-1ubuntu9.1 verification claim all overstated or misdescribed how -o APT::Snapshot works: it adds the dated snapshot alongside the runner's existing sources rather than rewriting them, ubuntu-24.04 resolves its own sources through mirror+file:/etc/apt/apt-mirrors.txt rather than a literal archive.ubuntu.com host, and the earlier "verified against the live archive already past this revision" claim was false since 9.9 is still current there today. Re-verified instead against poppler-utils 24.02.0-1ubuntu9.1, a revision already absent from the live archive: a snapshot dated to when 9.1 was current resolves and installs it correctly with the live sources still present. The bump-procedure guidance is corrected too: apt-cache policy cannot produce an APT_SNAPSHOT date on its own, so it now documents picking a UTC day at or after the new revision's publication and confirming the stamp resolves before using it. Also: setup-python's 3.14 pin was justified by a wheel-availability cliff at 3.15 that does not exist (zxing-cpp 3.1.1's wheel is cp312-abi3 with no upper Python bound) — reworded to interpreter determinism, the same reason the pip versions beside it are pinned. Restored a pdftotext -v identity check alongside the dpkg-query asserts, since the package record proves the installed version but not which binary the tests actually invoke. Fixed a stale ci.yml line-number citation in OracleGate.cs, a stale ubuntu-latest reference in an unrelated CHANGELOG entry sitting in the same Unreleased section, and noted the --allow-weak-crypto flag the RC4 recipe in Fixtures/Encrypted/README.md needs against qpdf 12.4.1. aot-smoke's own ubuntu-latest is noted as deliberate, since it carries no apt pins. Merged main (PR #379: global.json pins the SDK band, all setup-dotnet steps read global-json-file) — no conflicts. Refs #230 --- .github/workflows/ci.yml | 50 ++++++++++++++----- CHANGELOG.md | 20 +++++--- CONTRIBUTING.md | 29 +++++++---- .../Fixtures/Encrypted/README.md | 23 +++++---- tests/VellumPdf.TestSupport/OracleGate.cs | 10 ++-- 5 files changed, 88 insertions(+), 44 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 52ce1ecc..0e86898a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,16 +73,31 @@ jobs: # pins alone are not enough: noble's -updates/-security pockets keep only the newest # revision, so the moment Ubuntu ships another one, these exact revisions 404 out of the # live archive and every PR goes red with no code change involved. `-o APT::Snapshot` - # rewrites the runner's existing archive.ubuntu.com sources to the dated Ubuntu snapshot - # instead, which never evicts what it once published — verified against a fresh - # ubuntu:24.04 container with the live archive already past this exact revision, and it - # still resolved. If a real bump is ever needed instead of just chasing a rotated - # revision — a new poppler feature, a font correction — update POPPLER_VERSION / - # FONTS_*_VERSION and APT_SNAPSHOT together (`apt-cache policy` against a fresh - # `ubuntu:24.04` container gives both) and mention it in CONTRIBUTING.md. + # adds the dated Ubuntu snapshot alongside whatever apt sources the runner already has + # configured (additive, not a replacement — ubuntu-24.04's own sources resolve through + # `mirror+file:/etc/apt/apt-mirrors.txt`, not a literal archive.ubuntu.com host, so this + # comment does not assume which host actually serves a given fetch). The assertion below + # is what makes the outcome observable instead of assumed: `apt-get update` exits 0 even + # when the snapshot fetch fails outright, so without it a silently-unpinned install would + # pass unnoticed. Verified in a fresh ubuntu:24.04 container against an already-rotated- + # off revision — poppler-utils 24.02.0-1ubuntu9.1, absent from today's live archive — a + # snapshot dated to when 9.1 was current resolved and installed it correctly. If a real + # bump is ever needed instead of just chasing a rotated revision — a new poppler feature, + # a font correction — update POPPLER_VERSION / FONTS_*_VERSION together with a new + # APT_SNAPSHOT: a UTC day at or after the new revision's publication (stamps are midnight + # UTC, so a same-day publication needs the next day's stamp), confirmed with + # `curl -sI https://snapshot.ubuntu.com/ubuntu//dists/noble/InRelease` — a future- + # dated or mistyped stamp silently serves latest instead of erroring. Mention the change + # in CONTRIBUTING.md too. run: | set -euo pipefail sudo apt-get update -o APT::Snapshot="${APT_SNAPSHOT}" + # The step comment above explains why this can't be skipped: a failed snapshot fetch + # does not fail this command on its own. + ls /var/lib/apt/lists/ | grep -q "^snapshot\.ubuntu\.com_ubuntu_${APT_SNAPSHOT}_" || { + echo "::error::apt fell back to the live archive; snapshot ${APT_SNAPSHOT} was not used" + exit 1 + } sudo apt-get install -y -o APT::Snapshot="${APT_SNAPSHOT}" \ "poppler-utils=${POPPLER_VERSION}" \ "fonts-dejavu-core=${FONTS_DEJAVU_VERSION}" \ @@ -120,8 +135,8 @@ jobs: - uses: actions/setup-python@v7 with: - # zxing-cpp 3.1.1 ships prebuilt wheels for cp310–cp314; a floating '3.x' would - # eventually resolve to 3.15 and fall back to a source build with no commit to blame. + # Pinned for the same reason as the pip versions below: a deterministic interpreter + # for the barcode oracle rather than whatever '3.x' resolves to on a given run. python-version: '3.14' - name: Install barcode decode oracle (zxing-cpp) @@ -150,7 +165,14 @@ jobs: # pdftotext -v reports only poppler's upstream version (e.g. "24.02.0"), never the # Ubuntu package revision after the dash — apt can ship a new revision of the same # upstream release, and fonts have no version-reporting flag at all. dpkg-query reads - # the installed package record directly, which is the only way to see the full pin. + # the installed package record directly, which is the only way to see the full pin — + # but it proves the package record, not the binary the tests actually invoke. CLAUDE.md + # documents pdftotext resolving to two different programs on a developer machine + # depending on which shell launched the test host, so keep the identity check too. + poppler_line=$(pdftotext -v 2>&1 | head -n1) + echo "$poppler_line" + [ "$poppler_line" = "pdftotext version ${POPPLER_VERSION%%-*}" ] + echo "--- installed package versions ---" dpkg-query -W -f='${Package} ${Version}\n' poppler-utils fonts-dejavu-core fonts-texgyre assert_pkg_version() { @@ -171,8 +193,9 @@ jobs: python -c " import importlib.metadata as m - zxing_expected = '${ZXING_VERSION}' - pillow_expected = '${PILLOW_VERSION}' + import os + zxing_expected = os.environ['ZXING_VERSION'] + pillow_expected = os.environ['PILLOW_VERSION'] zxing_actual = m.version('zxing-cpp') pillow_actual = m.version('pillow') print('zxing-cpp', zxing_actual) @@ -263,6 +286,9 @@ jobs: run: dotnet pack VellumPdf.slnx -c Release --no-build -o ${{ runner.temp }}/packages aot-smoke: + # Deliberately left on the floating ubuntu-latest label: this job carries no apt version + # pins, so it has nothing for a label move to break the way the build job's oracle installs + # would (#230). strategy: fail-fast: false matrix: diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d493047..0ca3df29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -200,17 +200,23 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). shift fixture rendering out from under every downstream oracle. Since noble's apt pockets keep only the newest revision of a package, the install step also pins apt to the Ubuntu snapshot service for the date those versions were measured, so the exact revisions keep resolving after - the live archive rotates past them instead of 404ing with no code change involved. The veraPDF + the live archive rotates past them instead of 404ing with no code change involved — verified + against an already-rotated-off revision, not merely today's current one. Since apt reports + success even when that snapshot fetch itself silently fails, the step also asserts the + snapshot's own index files actually landed, so a silent fallback to the live archive fails the + build instead of quietly un-pinning the install. The veraPDF Docker tag, already pinned, was duplicated across the image pull and the shim that backs it; both now read one job-level `VERAPDF_TAG`. zxing-cpp moves from 3.0.0 to 3.1.1 (with `pillow` newly pinned to 12.3.0) — the barcode oracle suite passes unchanged, including the EAN add-on case the 3.0.0 pin was recorded against, since that test only asserts the main 13 digits and treats the add-on's presentation as version-dependent; `setup-python`'s interpreter is pinned - to 3.14 rather than floating, since zxing-cpp 3.1.1 ships prebuilt wheels only through 3.14. A - new version-assert step checks each pinned tool's own version report after install — including - the apt packages' full Ubuntu revision via `dpkg-query`, since `pdftotext -v` cannot see that - and fonts cannot self-report a version at all — so drift between the workflow and what actually - landed fails the build instead of changing what CI validates against with no commit to blame. + to 3.14 rather than floating too, for the same reproducibility reason as the pip versions + alongside it. A new version-assert step checks each pinned tool's own version report after + install — the apt packages' full Ubuntu revision via `dpkg-query`, since `pdftotext -v` reports + only poppler's upstream version and fonts cannot self-report a version at all, alongside a + `pdftotext -v` identity check proving which binary the tests actually invoke — so drift between + the workflow and what actually landed fails the build instead of changing what CI validates + against with no commit to blame. `actions/setup-dotnet` and `actions/setup-python` also move to their current majors (v6 and v7) across all four workflow files. (#230) - The roadmap now describes the scope past 2.5 as two parallel tracks, Kernel and conformance @@ -280,7 +286,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). `-tsv` flag; the version banner alone does not tell the two apart, so `pdftoppm` needs the same check via `-png`. veraPDF gets that same `VERAPDF_HOME`-first treatment only on Windows, where it needs the variable to find its `.bat` launcher at all; on every other platform, including CI's - ubuntu-latest runner, `VERAPDF_HOME` is not read here and veraPDF resolves by bare name, same as + ubuntu-24.04 runner, `VERAPDF_HOME` is not read here and veraPDF resolves by bare name, same as before this fix. The barcode decode oracle's `python` leg is unchanged too: it has no `*_HOME` and no identity check of its own, resolving by bare name everywhere, the ambiguity this fix removes for the other four. A hand-check in the "wrong" shell would never catch the swap, and the diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 69c1c7c8..f748c581 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -25,16 +25,25 @@ build, test, and submit changes to VellumPdf. **CI's poppler/font pins have a known maintenance cost.** noble's `-updates`/`-security` pockets keep only the newest revision of each package, so the exact revisions above eventually vanish from the live apt archive on their own, with no code change involved. When that happens, - `ci.yml`'s "Install poppler and fonts" step still resolves them, because that step passes - `-o APT::Snapshot=$APT_SNAPSHOT` on both `apt-get update` and `apt-get install`, which rewrites - the runner's existing archive.ubuntu.com sources to the [Ubuntu snapshot - service](https://snapshot.ubuntu.com/) as it stood on that date — a snapshot never evicts what - it once published, even after the live archive moves on. The failure mode to watch for instead - is a genuine version bump: if poppler or a font package needs a newer release on purpose, run - `apt-cache policy poppler-utils fonts-dejavu-core fonts-texgyre` against a fresh - `ubuntu:24.04` container to get the new versions and a matching `APT_SNAPSHOT` date, then - update `POPPLER_VERSION` / `FONTS_DEJAVU_VERSION` / `FONTS_TEXGYRE_VERSION` and - `APT_SNAPSHOT` together in `ci.yml`'s job-level `env:`, and this paragraph to match. + `ci.yml`'s "Install poppler and fonts" step still resolves them, because that step passes `-o + APT::Snapshot=$APT_SNAPSHOT` on both `apt-get update` and `apt-get install`, which adds the + dated [Ubuntu snapshot service](https://snapshot.ubuntu.com/) alongside whatever sources the + runner already has — additive, not a replacement, and `ubuntu-24.04`'s own sources resolve + through `mirror+file:/etc/apt/apt-mirrors.txt` rather than a literal `archive.ubuntu.com` host, + so do not assume which host actually serves a given fetch. A snapshot never evicts what it once + published, even after the live archive moves on, but `apt-get update` reports success (exit 0) + even when the snapshot fetch itself fails, so the step also greps `/var/lib/apt/lists/` for the + snapshot's own index files right after — that is what makes the snapshot actually having been + used something you can check rather than assume. The failure mode to watch for instead is a + genuine version bump: if poppler or a font package needs a newer release on purpose, run + `apt-cache policy poppler-utils fonts-dejavu-core fonts-texgyre` against a fresh `ubuntu:24.04` + container to get the new versions, pick an `APT_SNAPSHOT` stamp that is a UTC day at or after + the new revision's publication (stamps are midnight UTC, so a same-day publication needs the + next day's stamp), and confirm the stamp actually resolves with `curl -sI + https://snapshot.ubuntu.com/ubuntu//dists/noble/InRelease` before using it — a + future-dated or mistyped stamp does not error, it silently serves whatever is latest. Then + update `POPPLER_VERSION` / `FONTS_DEJAVU_VERSION` / `FONTS_TEXGYRE_VERSION` and `APT_SNAPSHOT` + together in `ci.yml`'s job-level `env:`, and this paragraph to match. On Windows, PATH order between shells is not reliable — the same bare `pdftotext` can resolve to a completely different program depending on which diff --git a/tests/VellumPdf.Reader.Tests/Fixtures/Encrypted/README.md b/tests/VellumPdf.Reader.Tests/Fixtures/Encrypted/README.md index bcd6ed69..0be73fe4 100644 --- a/tests/VellumPdf.Reader.Tests/Fixtures/Encrypted/README.md +++ b/tests/VellumPdf.Reader.Tests/Fixtures/Encrypted/README.md @@ -188,16 +188,19 @@ what remains: - **No committed fixture has its catalog packed inside an object stream.** Measured against qpdf 12.3.2 (#184 PR3): it pulls the catalog out of every object stream it writes the moment the same - run also encrypts — RC4-128 with `--object-streams=generate`, AES-256 the same way, and re-packing - an already-encrypted file all emit a top-level catalog regardless. `VellumPdf.Reader`'s own writer - refuses `UseObjectStreams` together with `Encrypt`, so it cannot produce the shape either. The - layout is pinned instead by a hand-built document, - `HandBuiltEncryptedDocuments.BuildCatalogInObjectStream`, shared between `EncryptedExemptionTests` - and `EncryptedReconstructionTests` — not a fixture here, and carrying no `Corpus` row, since the - guard in `EncryptedFixtureCorpusTests` only fires for files actually embedded from this directory. - Reconfirmed unchanged under qpdf 12.4.1 (#230): RC4-128 and AES-256 with `--object-streams=generate` - both still write the catalog as a plain top-level object, never inside the `/Type /ObjStm` they - emit alongside it. + run also encrypts — RC4-128 with `--allow-weak-crypto --object-streams=generate`, AES-256 the + same way, and re-packing an already-encrypted file all emit a top-level catalog regardless. + `VellumPdf.Reader`'s own writer refuses `UseObjectStreams` together with `Encrypt`, so it cannot + produce the shape either. The layout is pinned instead by a hand-built document, + `HandBuiltEncryptedDocuments.BuildCatalogInObjectStream`, shared between + `EncryptedExemptionTests` and `EncryptedReconstructionTests` — not a fixture here, and carrying + no `Corpus` row, since the guard in `EncryptedFixtureCorpusTests` only fires for files actually + embedded from this directory. + Reconfirmed unchanged under qpdf 12.4.1 (#230): RC4-128 and AES-256 with + `--object-streams=generate` both still write the catalog as a plain top-level object, never + inside the `/Type /ObjStm` they emit alongside it. 12.4.1 also refuses to write RC4 at all + without `--allow-weak-crypto` (exit 2, "refusing to write a file with weak crypto"), unrelated + to this behaviour but needed to reproduce the RC4-128 half of it. - **Every fixture is qpdf's output.** This is the largest gap in the corpus and the hardest to close: producers differ in exactly the places this code has to decide. A crypt filter `/Length` in diff --git a/tests/VellumPdf.TestSupport/OracleGate.cs b/tests/VellumPdf.TestSupport/OracleGate.cs index 2fe7c8f5..4d5d9fe3 100644 --- a/tests/VellumPdf.TestSupport/OracleGate.cs +++ b/tests/VellumPdf.TestSupport/OracleGate.cs @@ -101,11 +101,11 @@ private static bool IsRequired(string dependency) private static bool IsBarcodeDecodeDependency(string dependency) => dependency is "pdftoppm" or "python" or "zxing-cpp"; - // All five switches accept "1" or "true" (case-insensitively). ci.yml sets REQUIRE_VERAPDF and - // REQUIRE_BARCODE_ORACLE to the literal "1" (ci.yml:80-81); CI and GITHUB_ACTIONS are set to - // "true" by the GitHub Actions runner itself, not by this repository's workflow file. A CI - // system that instead exports CI=1 (common outside GitHub Actions) needs the same acceptance, - // and a developer who reaches for "true" on REQUIRE_ORACLES, REQUIRE_VERAPDF or + // All five switches accept "1" or "true" (case-insensitively). The Test step's env block in + // ci.yml sets REQUIRE_VERAPDF and REQUIRE_BARCODE_ORACLE to the literal "1"; CI and + // GITHUB_ACTIONS are set to "true" by the GitHub Actions runner itself, not by this repository's + // workflow file. A CI system that instead exports CI=1 (common outside GitHub Actions) needs the + // same acceptance, and a developer who reaches for "true" on REQUIRE_ORACLES, REQUIRE_VERAPDF or // REQUIRE_BARCODE_ORACLE should get the same behaviour CI and GITHUB_ACTIONS already give — a // "1"-only check on those three would be a silent footgun for exactly that developer. private static bool IsTrueOrOne(string variable) From 85d8c5c7d13c8d56e665f66bd5c56423d592c87d Mon Sep 17 00:00:00 2001 From: Timothy van der Ham Date: Mon, 31 Aug 2026 10:35:57 +0200 Subject: [PATCH 4/4] Stop piping oracle version banners into an early-exiting head The real runner failed the assert step at exit 141 (128+SIGPIPE) right after the dpkg-query lines: ` ... | head -n1` under `set -euo pipefail` lets head exit the moment it has its one line, and the producer (writing a multi-line banner) can still be mid-write when that happens, so the write hits a closed pipe. Local containers never showed it because the timing has to line up, which the runner's own buffering apparently does more often than a clean container does. Fixed by capturing the full output first and taking the first line with a bash parameter expansion instead of a second process in the pipeline: `out=$(cmd); line=${out%%$'\n'*}`. Applied to all three version-banner reads (qpdf, pdftotext, veraPDF). The snapshot-verification guard had the same shape with `grep -q` instead of `head` (`ls ... | grep -q ...`) and got the same treatment: capture the listing, then grep it through a here-string, which reads a fixed string rather than a live pipe and so cannot race a still-writing producer. Re-verified in containers: qpdf/poppler assert logic against a real install (exit 0), the veraPDF line extraction against real multi-line `verapdf --version` output, and the guard in both its failing and passing shapes. Refs #230 --- .github/workflows/ci.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e86898a..acecf674 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,10 +94,11 @@ jobs: sudo apt-get update -o APT::Snapshot="${APT_SNAPSHOT}" # The step comment above explains why this can't be skipped: a failed snapshot fetch # does not fail this command on its own. - ls /var/lib/apt/lists/ | grep -q "^snapshot\.ubuntu\.com_ubuntu_${APT_SNAPSHOT}_" || { + apt_lists=$(ls /var/lib/apt/lists/) + if ! grep -q "^snapshot\.ubuntu\.com_ubuntu_${APT_SNAPSHOT}_" <<< "$apt_lists"; then echo "::error::apt fell back to the live archive; snapshot ${APT_SNAPSHOT} was not used" exit 1 - } + fi sudo apt-get install -y -o APT::Snapshot="${APT_SNAPSHOT}" \ "poppler-utils=${POPPLER_VERSION}" \ "fonts-dejavu-core=${FONTS_DEJAVU_VERSION}" \ @@ -158,7 +159,8 @@ jobs: run: | set -euo pipefail - qpdf_line=$(qpdf --version | head -n1) + qpdf_out=$(qpdf --version) + qpdf_line=${qpdf_out%%$'\n'*} echo "$qpdf_line" [ "$qpdf_line" = "qpdf version ${QPDF_VERSION}" ] @@ -169,7 +171,8 @@ jobs: # but it proves the package record, not the binary the tests actually invoke. CLAUDE.md # documents pdftotext resolving to two different programs on a developer machine # depending on which shell launched the test host, so keep the identity check too. - poppler_line=$(pdftotext -v 2>&1 | head -n1) + poppler_out=$(pdftotext -v 2>&1) + poppler_line=${poppler_out%%$'\n'*} echo "$poppler_line" [ "$poppler_line" = "pdftotext version ${POPPLER_VERSION%%-*}" ] @@ -187,7 +190,8 @@ jobs: assert_pkg_version fonts-dejavu-core "$FONTS_DEJAVU_VERSION" assert_pkg_version fonts-texgyre "$FONTS_TEXGYRE_VERSION" - verapdf_line=$(verapdf --version 2>&1 | head -n1) + verapdf_out=$(verapdf --version 2>&1) + verapdf_line=${verapdf_out%%$'\n'*} echo "$verapdf_line" [ "$verapdf_line" = "veraPDF ${VERAPDF_TAG#v}" ]