From ed35fccd5e16e23ae0005c5446fb9913df63a5d6 Mon Sep 17 00:00:00 2001 From: michael-grunder Date: Thu, 17 Sep 2026 12:49:16 -0700 Subject: [PATCH] Integrate packaging revisions This allows us to add or rebuild a relay package without needing to bump the upstream version of Relay. This is done with `build/revisions/deb.json` and `build/revisions/rpm.json`. They both contain a json object with the upstream relay tab and the package revision. If we need to change the deb package for an existing Relay release we just bump the vevsion id and only that will build. If we want to do both we just bump both. When packaging a new upstream of Relay we will add a new entry in the `deb.json` and `rpm.json` and we should set both to `1`. --- .github/workflows/package.yml | 25 ++- .github/workflows/packaging-tests.yml | 25 +++ .github/workflows/repos.yml | 240 +++++++-------------- .github/workflows/sync.yml | 4 + .gitignore | 3 + README.md | 50 +++++ build/deb-repo.sh | 2 + build/fpm.sh | 20 +- build/helpers.sh | 57 +++-- build/packages.py | 299 ++++++++++++++++++++++++++ build/revisions/deb.json | 3 + build/revisions/rpm.json | 3 + build/rpm-repo.sh | 2 + tests/smoke-packages.sh | 52 +++++ tests/test_packages.py | 229 ++++++++++++++++++++ 15 files changed, 832 insertions(+), 182 deletions(-) create mode 100644 .github/workflows/packaging-tests.yml create mode 100644 build/packages.py create mode 100644 build/revisions/deb.json create mode 100644 build/revisions/rpm.json create mode 100644 tests/smoke-packages.sh create mode 100644 tests/test_packages.py diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index fa54017d..6c3e6303 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -24,7 +24,14 @@ jobs: - name: Checkout code uses: actions/checkout@v7 + - name: Plan unpublished packages + id: plan + env: + TAG: ${{ inputs.tag }} + run: python3 build/packages.py plan "$TAG" + - name: Log in to GHCR + if: steps.plan.outputs.has_packages == 'true' uses: docker/login-action@v4 with: registry: ghcr.io @@ -32,24 +39,36 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Pull the Docker image + if: steps.plan.outputs.has_packages == 'true' run: | docker pull ghcr.io/${{ github.repository_owner }}/fpm:latest docker tag ghcr.io/${{ github.repository_owner }}/fpm:latest fpm - name: Render changelogs + if: steps.plan.outputs.has_packages == 'true' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ inputs.tag }} run: | gh api --paginate 'repos/${{ github.repository_owner }}/relay/releases?per_page=100' \ | jq -s 'add' > /tmp/releases.json - ./build/changelog.sh /tmp/releases.json '${{ github.event.inputs.tag }}' build/changelog + ./build/changelog.sh /tmp/releases.json "$TAG" build/changelog - name: Bundle packages + if: steps.plan.outputs.has_packages == 'true' + env: + TAG: ${{ inputs.tag }} + DEB_REVISION: ${{ steps.plan.outputs.deb_revision }} + RPM_REVISION: ${{ steps.plan.outputs.rpm_revision }} run: | cd build docker run --tty \ - -v ${PWD}:/root/build \ - fpm /bin/bash -c "./fpm.sh ${{ github.event.inputs.tag }}" + -v "${PWD}:/root/build" \ + -e DEB_REVISION -e RPM_REVISION \ + fpm /bin/bash ./fpm.sh "$TAG" + + - name: Record artifact checksums and build revision + run: python3 build/packages.py manifest - name: Upload artifacts uses: actions/upload-artifact@v7 diff --git a/.github/workflows/packaging-tests.yml b/.github/workflows/packaging-tests.yml new file mode 100644 index 00000000..e6f2ccd8 --- /dev/null +++ b/.github/workflows/packaging-tests.yml @@ -0,0 +1,25 @@ +name: Packaging tests + +on: + pull_request: + paths: ['build/**', 'tests/**', '.github/workflows/**'] + push: + branches: [main] + paths: ['build/**', 'tests/**', '.github/workflows/**'] + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Check shell syntax + run: bash -n build/fpm.sh && bash -n build/helpers.sh + - name: Test planning and immutable publication + run: python3 -m unittest discover -s tests -v + - name: Build FPM test image + run: docker build --file build/fpm.Dockerfile --tag fpm-test build + - name: Verify DEB and RPM versions with fixture packages + run: docker run --rm --network none -v "$PWD:/workspace:ro" fpm-test bash /workspace/tests/smoke-packages.sh diff --git a/.github/workflows/repos.yml b/.github/workflows/repos.yml index 5d9f31a7..f56df36f 100644 --- a/.github/workflows/repos.yml +++ b/.github/workflows/repos.yml @@ -5,81 +5,81 @@ on: workflows: [Build packages] types: [completed] -jobs: - - deb-repo: +# Both formats commit to the same branch and publish repository metadata. +concurrency: + group: package-repositories + cancel-in-progress: false - name: Update deb repository +jobs: + publish: + name: Update ${{ matrix.format }} repository runs-on: ubuntu-latest - timeout-minutes: 10 - - if: github.event.workflow_run.conclusion == 'success' + timeout-minutes: 20 + if: >- + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.head_branch == github.event.repository.default_branch && + github.event.workflow_run.head_repository.full_name == github.repository + + strategy: + max-parallel: 1 + fail-fast: false + matrix: + format: [deb, rpm] permissions: contents: write actions: read - steps: + env: + FORMAT: ${{ matrix.format }} - - name: Checkout code + steps: + - name: Checkout current repository uses: actions/checkout@v7 with: + ref: ${{ github.event.repository.default_branch }} lfs: true - # Uses AWS CLI preinstalled on ubuntu-latest runners. - - name: Download artifacts + - name: Download packages and build manifest env: GH_TOKEN: ${{ github.token }} - run: gh run download ${{ github.event.workflow_run.id }} --dir artifacts + BUILD_RUN: ${{ github.event.workflow_run.id }} + run: gh run download "$BUILD_RUN" --name packages --dir artifacts/packages - - name: Get build tag - run: | - TAG_FILE=$(find artifacts -name 'TAG' | head -n 1) - if [ -z "$TAG_FILE" ]; then - echo "No TAG file found in downloaded artifacts" >&2 - exit 1 - fi - TAG=$(tr -d '[:space:]' < "$TAG_FILE") - if [ -z "$TAG" ]; then - echo "TAG file is empty in downloaded artifacts" >&2 - exit 1 - fi - echo "TAG=$TAG" >> $GITHUB_ENV - - - name: Copy packages - run: | - mkdir -p deb/pool/${{ env.TAG }} - find artifacts -name '*.deb' -exec cp {} deb/pool/${{ env.TAG }} \; + - name: Verify and stage packages without overwriting published versions + id: stage + env: + BUILD_COMMIT: ${{ github.event.workflow_run.head_sha }} + run: python3 build/packages.py stage "$FORMAT" --source-commit "$BUILD_COMMIT" - name: Build the Docker image - run: docker build . --tag deb --file build/deb.Dockerfile + if: steps.stage.outputs.has_packages == 'true' + run: docker build . --tag "$FORMAT" --file "build/$FORMAT.Dockerfile" - name: Set up private key - run: echo -n '${{ secrets.PRIVATE_KEY }}' | base64 --decode > key-private.asc - - - name: Set up Git - run: | - git config --local user.name "github-actions[bot]" - git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" + if: steps.stage.outputs.has_packages == 'true' + env: + PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }} + run: printf '%s' "$PRIVATE_KEY" | base64 --decode > key-private.asc - name: Update repository + if: steps.stage.outputs.has_packages == 'true' + env: + GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} run: | docker run --tty \ - -v ${PWD}:/root/deb \ - -e GPG_PASSPHRASE='${{ secrets.GPG_PASSPHRASE }}' \ - deb /bin/bash -c "./build/deb-repo.sh" + -v "${PWD}:/root/$FORMAT" \ + -e GPG_PASSPHRASE \ + "$FORMAT" /bin/bash "./build/$FORMAT-repo.sh" - name: Remove private key + if: always() run: rm -f key-private.asc - - name: Commit changes - run: | - git pull - git add deb/* - git commit -m "Bump deb repo to ${{ env.TAG }}" || echo "No changes to commit" - git push - - - name: Sync repo with R2 + # Upload packages before advertising them in either Git or repository metadata. + # Conditional writes allow identical retries but never replace existing bytes. + - name: Publish immutable package files to R2 + if: steps.stage.outputs.has_packages == 'true' env: R2_ENDPOINT: https://a1220fd38ad4771f7b7b38f5f3c2b00d.r2.cloudflarestorage.com AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} @@ -87,105 +87,26 @@ jobs: AWS_DEFAULT_REGION: auto AWS_DEFAULT_OUTPUT: json run: | - aws s3 cp key.gpg s3://relay-repos/ \ - --no-progress \ - --acl public-read \ - --endpoint-url ${{ env.R2_ENDPOINT }} - aws s3 cp deb/sources.list s3://relay-repos/deb/ \ - --no-progress \ - --acl public-read \ - --endpoint-url ${{ env.R2_ENDPOINT }} - aws s3 sync deb s3://relay-repos/deb \ - --exclude "*" \ - --include "*${{ env.TAG }}*" \ - --delete \ - --no-progress \ - --acl public-read \ - --endpoint-url ${{ env.R2_ENDPOINT }} - aws s3 sync deb/dists s3://relay-repos/deb/dists/ \ - --no-progress \ - --acl public-read \ - --endpoint-url ${{ env.R2_ENDPOINT }} - - rpm-repo: - - name: Update rpm repository - runs-on: ubuntu-latest - needs: deb-repo - timeout-minutes: 10 - - if: github.event.workflow_run.conclusion == 'success' - - permissions: - contents: write - actions: read - - steps: + python3 build/packages.py upload "$FORMAT" \ + --bucket relay-repos --endpoint "$R2_ENDPOINT" - - name: Checkout code - uses: actions/checkout@v7 - with: - lfs: true - - # Uses AWS CLI preinstalled on ubuntu-latest runners. - - name: Download artifacts + - name: Commit changes + if: steps.stage.outputs.has_packages == 'true' env: - GH_TOKEN: ${{ github.token }} - run: gh run download ${{ github.event.workflow_run.id }} --dir artifacts - - - name: Get build tag - run: | - TAG_FILE=$(find artifacts -name 'TAG' | head -n 1) - if [ -z "$TAG_FILE" ]; then - echo "No TAG file found in downloaded artifacts" >&2 - exit 1 - fi - TAG=$(tr -d '[:space:]' < "$TAG_FILE") - if [ -z "$TAG" ]; then - echo "TAG file is empty in downloaded artifacts" >&2 - exit 1 - fi - echo "TAG=$TAG" >> $GITHUB_ENV - - - name: Copy packages - run: | - source build/distros.sh - - for distro in "${el_dists[@]}"; do - mkdir -p "rpm/$distro/${{ env.TAG }}" - find artifacts -name "*-$distro-*.rpm" \ - -exec cp {} "rpm/$distro/${{ env.TAG }}" \; - done - - - name: Build the Docker image - run: docker build . --tag rpm --file build/rpm.Dockerfile - - - name: Set up private key - run: echo -n '${{ secrets.PRIVATE_KEY }}' | base64 --decode > key-private.asc - - - name: Set up Git + TAG: ${{ steps.stage.outputs.tag }} + REVISION: ${{ steps.stage.outputs.revision }} run: | git config --local user.name "github-actions[bot]" git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add "$FORMAT" + if ! git diff --cached --quiet; then + git commit -m "Publish $FORMAT packages for $TAG revision $REVISION" + git pull --rebase + git push + fi - - name: Update repository - run: | - docker run --tty \ - -v ${PWD}:/root/rpm \ - -e GPG_PASSPHRASE='${{ secrets.GPG_PASSPHRASE }}' \ - rpm /bin/bash -c "./build/rpm-repo.sh" - - - name: Remove private key - run: rm -f key-private.asc - - - name: Commit changes - run: | - git pull - git add rpm/* - git commit -m "Bump rpm repos to ${{ env.TAG }}" || echo "No changes to commit" - git push - - - name: Sync repo with R2 + - name: Publish repository metadata to R2 + if: steps.stage.outputs.has_packages == 'true' env: R2_ENDPOINT: https://a1220fd38ad4771f7b7b38f5f3c2b00d.r2.cloudflarestorage.com AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} @@ -194,25 +115,18 @@ jobs: AWS_DEFAULT_OUTPUT: json run: | aws s3 cp key.gpg s3://relay-repos/ \ - --no-progress \ - --acl public-read \ - --endpoint-url ${{ env.R2_ENDPOINT }} - aws s3 cp rpm/el.repo s3://relay-repos/rpm/ \ - --no-progress \ - --acl public-read \ - --endpoint-url ${{ env.R2_ENDPOINT }} - aws s3 sync rpm s3://relay-repos/rpm \ - --exclude "*" \ - --include "*${{ env.TAG }}*" \ - --delete \ - --no-progress \ - --acl public-read \ - --endpoint-url ${{ env.R2_ENDPOINT }} - source build/distros.sh - - for distro in "${el_dists[@]}"; do - aws s3 sync "rpm/$distro/repodata" "s3://relay-repos/rpm/$distro/repodata/" \ - --no-progress \ - --acl public-read \ - --endpoint-url ${{ env.R2_ENDPOINT }} - done + --no-progress --endpoint-url "$R2_ENDPOINT" + if [ "$FORMAT" = deb ]; then + aws s3 cp deb/sources.list s3://relay-repos/deb/ \ + --no-progress --endpoint-url "$R2_ENDPOINT" + aws s3 sync deb/dists s3://relay-repos/deb/dists/ \ + --no-progress --endpoint-url "$R2_ENDPOINT" + else + aws s3 cp rpm/el.repo s3://relay-repos/rpm/ \ + --no-progress --endpoint-url "$R2_ENDPOINT" + source build/distros.sh + for distro in "${el_dists[@]}"; do + aws s3 sync "rpm/$distro/repodata" "s3://relay-repos/rpm/$distro/repodata/" \ + --no-progress --endpoint-url "$R2_ENDPOINT" + done + fi diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index ca62d197..f0b85883 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -5,6 +5,10 @@ on: # schedule: # - cron: '0 0 * * 0' # once a week +concurrency: + group: package-repositories + cancel-in-progress: false + jobs: deb-repo: diff --git a/.gitignore b/.gitignore index cbe33068..209aea84 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ /build/dist /build/src/*/*-php* /build/changelog/ +/build/plan.json +/build/selected-packages.txt +__pycache__/ key-private.asc diff --git a/README.md b/README.md index 4c77144f..38ec59fb 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,56 @@ For detailed installations instruction see [relay.so](https://relay.so/docs/installation). +## Publishing packages + +The **Build packages** action takes an upstream Relay tag, such as `v0.50.0`. +Packaging revisions are recorded separately for each format in +[`build/revisions/deb.json`](build/revisions/deb.json) and +[`build/revisions/rpm.json`](build/revisions/rpm.json), keyed by upstream tag. +Both files must contain the tag; revisions are positive integers. + +- For a new Relay release, add its tag to both files with revision `1`. +- For a packaging fix, increment the affected format's revision for that tag. + A shared change affecting both formats requires both revisions to increase. +- Adding a distribution or PHP target can use the existing revision: only + missing packages are built. Existing packages are skipped even if packaging + code changed, so bump the revision when those packages need the change. +- Commit the revision changes before running the action. Revisions are shared + across distributions, architectures, and PHP variants within each format. + +DEBs use `Version: 0.50.0-1`; RPMs use `Version: 0.50.0` and `Release: 1`. +Filenames also include the revision. Upstream downloads still use `v0.50.0`. +When adopting this scheme, existing unsuffixed DEBs advance to revision `1`. +Existing RPMs already have Release `1`, despite their old filenames, and are +skipped until the RPM revision increases to `2`. + +The action prints a build/skip count for each format. A rerun with no missing +packages is a successful no-op. To preview the plan locally without downloading +or building packages (Python 3.10+ and Bash are required): + +```bash +python3 build/packages.py plan v0.50.0 +``` + +The artifact includes `manifest.json`, recording the upstream tag, revisions, +build commit, filenames, and SHA256 checksums. **Update repositories** consumes +that manifest and updates only formats with built packages. Automatic publishing +is restricted to builds from this repository's default branch; branch builds +produce reviewable artifacts without publishing them. + +Published package files are immutable. Publication rejects conflicting versions +and uses conditional R2 writes to prevent overwrites, while allowing identical +retries. Old revisions and their download URLs are retained. If publication +fails partway through, rerun **Update repositories** for the original build so +it can finish publishing the same artifacts; rebuilding may produce different +bytes. Repository metadata is uploaded after the package files. + +Run the packaging regression tests with: + +```bash +python3 -m unittest discover -s tests -v +``` + ## Using APT (Debian, Ubuntu) ```bash diff --git a/build/deb-repo.sh b/build/deb-repo.sh index 256b7076..adffb13e 100755 --- a/build/deb-repo.sh +++ b/build/deb-repo.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e + gpg --batch --import key-private.asc source build/distros.sh diff --git a/build/fpm.sh b/build/fpm.sh index d2afe748..73d590f5 100755 --- a/build/fpm.sh +++ b/build/fpm.sh @@ -2,10 +2,22 @@ set -e -source /root/build/helpers.sh -source /root/build/distros.sh - -version=$1 +build_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +source "$build_dir/helpers.sh" +source "$build_dir/distros.sh" + +version=${1:?Usage: fpm.sh TAG [--list]} +mode=${2:-build} +if [[ ! "$version" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || + [[ ! "${DEB_REVISION:-}" =~ ^[1-9][0-9]*$ ]] || + [[ ! "${RPM_REVISION:-}" =~ ^[1-9][0-9]*$ ]]; then + echo "A release tag and positive DEB_REVISION/RPM_REVISION are required; run packages.py plan first" >&2 + exit 1 +fi +if [[ "$mode" != build && "$mode" != --list ]]; then + echo "Unknown mode: $mode" >&2 + exit 1 +fi baseurl="https://builds.r2.relay.so/$version/relay-$version" declare -A php_api=( diff --git a/build/helpers.sh b/build/helpers.sh index 86a22e9f..5ffac966 100755 --- a/build/helpers.sh +++ b/build/helpers.sh @@ -9,11 +9,15 @@ DESCRIPTION main() { - rm -rf /tmp/relay* - rm -rf /root/build/dist - mkdir /root/build/dist - - echo -n "$version" > /root/build/dist/TAG + if [[ "$mode" != --list ]]; then + test -f "$build_dir/selected-packages.txt" || { + echo "Missing build plan; run packages.py plan first" >&2 + exit 1 + } + rm -rf /tmp/relay* + rm -rf "$build_dir/dist" + mkdir "$build_dir/dist" + fi for package in "${packages[@]}"; do unset ${!pkg_@} @@ -36,11 +40,27 @@ fpm_build() # we don't have centos builds for v0.1.0 if [[ "$version" == "v0.1.0" && "$type" == "rpm" ]]; then - echo "Skipping RPMs for v0.1.0" + echo "Skipping RPMs for v0.1.0" >&2 return 0 fi - source /root/build/src/$type/config.$config.sh + source "$build_dir/src/$type/config.$config.sh" + + pkg_version=${version#v} + if [[ "$type" == deb ]]; then + pkg_revision=$DEB_REVISION + else + pkg_revision=$RPM_REVISION + fi + pkg_filename="${pkg_name}-${pkg_version}-${pkg_revision}-php${php_version}-${pkg_identifier}-${pkg_arch}.${type}" + + if [[ "$mode" == --list ]]; then + printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$type" "$distro" "$pkg_arch" "$php_version" "$pkg_name" "$pkg_filename" + return 0 + fi + if ! grep -Fxq -- "$pkg_filename" "$build_dir/selected-packages.txt"; then + return 0 + fi echo "Building Relay ($version) .$type package for PHP $php_version on $pkg_arch" @@ -109,9 +129,6 @@ fpm_build() done > $dest_path/usr/share/lintian/overrides/$pkg_name fi - pkg_version=${version#v} - pkg_filename="${pkg_name}-${pkg_version}-php${php_version}-${pkg_identifier}-${pkg_arch}.${type}" - args=( "--input-type dir" "--output-type $type" @@ -123,6 +140,7 @@ fpm_build() "--category 'php'" "--name '$pkg_name'" "--version '$pkg_version'" + "--iteration '$pkg_revision'" "--architecture $pkg_arch" "--package dist/$pkg_filename" @@ -145,10 +163,14 @@ fpm_build() # deb changelog entries embed the package name, rpm ones don't if [[ "$type" == "deb" ]]; then - sed "s/@PKG@/$pkg_name/g" /root/build/changelog/deb.tpl > /tmp/changelog-$pkg_name.deb + sed -e "s/@PKG@/$pkg_name/g" \ + -e "1s/($pkg_version)/($pkg_version-$pkg_revision)/" \ + /root/build/changelog/deb.tpl > /tmp/changelog-$pkg_name.deb args+=("--deb-changelog /tmp/changelog-$pkg_name.deb") else - args+=("--rpm-changelog /root/build/changelog/rpm") + sed "1s/ - $pkg_version-1$/ - $pkg_version-$pkg_revision/" \ + /root/build/changelog/rpm > /tmp/changelog-$pkg_name.rpm + args+=("--rpm-changelog /tmp/changelog-$pkg_name.rpm") fi if [ ! -z "$pkg_provides" ]; then @@ -171,4 +193,15 @@ fpm_build() echo "Building package: $pkg_filename" bash -c "fpm $args $dest_path/=/" + + # Check the actual package version, not just the filename we supplied to FPM. + if [[ "$type" == deb ]]; then + actual_version=$(dpkg-deb -f "dist/$pkg_filename" Version) + else + actual_version=$(rpm -qp --qf '%{VERSION}-%{RELEASE}' "dist/$pkg_filename") + fi + if [[ "$actual_version" != "$pkg_version-$pkg_revision" ]]; then + echo "Unexpected version in $pkg_filename: $actual_version" >&2 + exit 1 + fi } diff --git a/build/packages.py b/build/packages.py new file mode 100644 index 00000000..9c218e2e --- /dev/null +++ b/build/packages.py @@ -0,0 +1,299 @@ +#!/usr/bin/env python3 +"""Plan revisioned packages and publish them without replacing existing bytes.""" + +import argparse +from concurrent.futures import ThreadPoolExecutor +import gzip +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +import xml.etree.ElementTree as ET + + +ROOT = Path(__file__).resolve().parent.parent +FORMATS = ("deb", "rpm") + + +def read_json(path): + return json.loads(path.read_text()) + + +def write_json(path, data): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=2) + "\n") + + +def sha256(path): + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def check_tag(tag): + if not isinstance(tag, str) or not re.fullmatch(r"v[0-9]+\.[0-9]+\.[0-9]+", tag): + raise ValueError("Expected a stable upstream tag such as v0.50.0") + + +def check_revisions(revisions): + if set(revisions) != set(FORMATS) or any( + type(value) is not int or value < 1 for value in revisions.values() + ): + raise ValueError("DEB and RPM revisions must be positive integers") + + +def filename(package, tag, revision): + return ( + f"{package['name']}-{tag[1:]}-{revision}-php{package['php']}-" + f"{package['distro']}-{package['arch']}.{package['format']}" + ) + + +def destination(package, tag): + if package["format"] == "deb": + return Path("deb/pool") / tag / package["filename"] + return Path("rpm") / package["distro"] / tag / package["filename"] + + +def identity(package, version): + # The single-PHP RPMs share a name/version/arch across PHP versions. + return tuple(package[k] for k in ("format", "distro", "name", "arch", "php")) + (version,) + + +def published_packages(root): + """Read indices, including legacy RPM Release 1 files without a suffix.""" + published = {} + + def add(fmt, distro, name, arch, version, location, checksum): + php = re.search(r"-php([0-9]+\.[0-9]+)-", location) + if not php: + raise ValueError(f"Cannot identify PHP version in {location}") + key = (fmt, distro, name, arch, php[1], version) + published.setdefault(key, set()).add(checksum) + + for path in sorted((root / "deb/dists").glob("*/main/binary-*/Packages")): + distro = path.parents[2].name + for stanza in path.read_text().strip().split("\n\n"): + fields = dict(line.split(": ", 1) for line in stanza.splitlines() + if ": " in line and not line.startswith(" ")) + if fields: + add("deb", distro, fields["Package"], fields["Architecture"], + fields["Version"], fields["Filename"], fields["SHA256"]) + + ns = {"r": "http://linux.duke.edu/metadata/repo", + "c": "http://linux.duke.edu/metadata/common"} + for path in sorted((root / "rpm").glob("*/repodata/repomd.xml")): + distro = path.parents[1].name + repomd = ET.parse(path) + location = repomd.find("r:data[@type='primary']/r:location", ns).attrib["href"] + primary = path.parents[1] / location + with gzip.open(primary) as stream: + packages = ET.parse(stream) + for package in packages.findall("c:package", ns): + version = package.find("c:version", ns).attrib + checksum = package.find("c:checksum", ns) + if checksum.attrib["type"] != "sha256": + raise ValueError(f"Expected SHA256 checksums in {primary}") + add("rpm", distro, package.findtext("c:name", namespaces=ns), + package.findtext("c:arch", namespaces=ns), + f"{version['ver']}-{version['rel']}", + package.find("c:location", ns).attrib["href"], checksum.text) + return published + + +def output(name, value): + print(f"{name}={value}") + if os.environ.get("GITHUB_OUTPUT"): + with open(os.environ["GITHUB_OUTPUT"], "a") as stream: + stream.write(f"{name}={value}\n") + + +def plan(root, tag): + check_tag(tag) + revisions = {} + for fmt in FORMATS: + path = root / f"build/revisions/{fmt}.json" + records = read_json(path) + if tag not in records: + raise ValueError(f"Add {tag} to {path.relative_to(root)} before building") + revisions[fmt] = records[tag] + check_revisions(revisions) + published = published_packages(root) + for key in published: + match = re.fullmatch(re.escape(tag[1:]) + r"-([0-9]+)", key[-1]) + if match and int(match[1]) > revisions[key[0]]: + raise ValueError(f"{key[0]} revision {revisions[key[0]]} is older than published {key[-1]}") + + env = dict(os.environ, **{f"{fmt.upper()}_REVISION": str(rev) for fmt, rev in revisions.items()}) + candidates = subprocess.check_output( + ["bash", str(root / "build/fpm.sh"), tag, "--list"], env=env, text=True + ) + selected = [] + skipped = dict.fromkeys(FORMATS, 0) + for line in candidates.splitlines(): + package = dict(zip(("format", "distro", "arch", "php", "name", "filename"), + line.split("\t"), strict=True)) + fmt = package["format"] + expected = filename(package, tag, revisions[fmt]) + if package["filename"] != expected: + raise ValueError(f"Unexpected candidate filename: {package['filename']}") + if identity(package, f"{tag[1:]}-{revisions[fmt]}") in published: + skipped[fmt] += 1 + else: + selected.append(package) + commit = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip() + manifest = {"schema": 1, "tag": tag, "revisions": revisions, + "source_commit": commit, "packages": selected} + write_json(root / "build/plan.json", manifest) + (root / "build/selected-packages.txt").write_text("".join(p["filename"] + "\n" for p in selected)) + for fmt in FORMATS: + count = sum(p["format"] == fmt for p in selected) + print(f"{fmt.upper()} revision {revisions[fmt]}: build {count}, already published {skipped[fmt]}") + output(f"{fmt}_revision", revisions[fmt]) + output("has_packages", str(bool(selected)).lower()) + return manifest + + +def validate_manifest(manifest): + if manifest.get("schema") != 1: + raise ValueError("Unsupported package manifest schema") + check_tag(manifest["tag"]) + check_revisions(manifest["revisions"]) + if not re.fullmatch(r"[0-9a-f]{40}", manifest["source_commit"]): + raise ValueError("Invalid source commit") + seen = set() + for package in manifest["packages"]: + fmt = package["format"] + if fmt not in FORMATS: + raise ValueError(f"Unknown package format: {fmt}") + for field in ("name", "distro", "arch", "php"): + if not re.fullmatch(r"[a-z0-9][a-z0-9._+-]*", package[field]): + raise ValueError(f"Invalid package {field}") + expected = filename(package, manifest["tag"], manifest["revisions"][fmt]) + if package["filename"] != expected or expected in seen: + raise ValueError(f"Invalid or duplicate filename: {package['filename']}") + seen.add(expected) + + +def finish_manifest(root): + manifest = read_json(root / "build/plan.json") + validate_manifest(manifest) + dist = root / "build/dist" + dist.mkdir(exist_ok=True) + expected = {p["filename"] for p in manifest["packages"]} + actual = {p.name for fmt in FORMATS for p in dist.glob(f"*.{fmt}")} + if expected != actual: + raise ValueError(f"Build output differs from plan: missing {expected - actual}, extra {actual - expected}") + for package in manifest["packages"]: + package["sha256"] = sha256(dist / package["filename"]) + write_json(dist / "manifest.json", manifest) + + +def load_artifacts(artifacts): + manifest = read_json(artifacts / "manifest.json") + validate_manifest(manifest) + for package in manifest["packages"]: + if sha256(artifacts / package["filename"]) != package["sha256"]: + raise ValueError(f"Checksum mismatch: {package['filename']}") + return manifest + + +def stage(root, artifacts, fmt, source_commit): + manifest = load_artifacts(artifacts) + if manifest["source_commit"] != source_commit: + raise ValueError("Artifact source commit does not match the build workflow") + published = published_packages(root) + packages = [p for p in manifest["packages"] if p["format"] == fmt] + # Validate the entire batch before copying anything. + for package in packages: + version = f"{manifest['tag'][1:]}-{manifest['revisions'][fmt]}" + checksums = published.get(identity(package, version), set()) + if checksums and checksums != {package["sha256"]}: + raise ValueError(f"Published identity has different bytes: {package['filename']}; bump the revision") + dest = root / destination(package, manifest["tag"]) + if dest.exists() and sha256(dest) != package["sha256"]: + raise ValueError(f"Refusing to overwrite {dest}; bump the revision") + for package in packages: + dest = root / destination(package, manifest["tag"]) + dest.parent.mkdir(parents=True, exist_ok=True) + if not dest.exists(): + shutil.copyfile(artifacts / package["filename"], dest) + output("tag", manifest["tag"]) + output("revision", manifest["revisions"][fmt]) + output("has_packages", str(bool(packages)).lower()) + + +def upload_object(source, key, checksum, bucket, endpoint): + command = ["aws", "s3api", "--endpoint-url", endpoint] + result = subprocess.run(command + [ + "put-object", "--bucket", bucket, "--key", key, "--body", str(source), + "--if-none-match", "*", + ], text=True, capture_output=True) + if result.returncode: + if "PreconditionFailed" not in result.stderr and "ConditionalRequestConflict" not in result.stderr: + raise RuntimeError(result.stderr) + # A retry may encounter an already uploaded object. Compare actual bytes; + # ETags are not necessarily content hashes (e.g. multipart uploads). + with tempfile.TemporaryDirectory() as tmp: + previous = Path(tmp) / "package" + subprocess.run(command + ["get-object", "--bucket", bucket, "--key", key, + str(previous)], check=True, stdout=subprocess.DEVNULL) + if sha256(previous) != checksum: + raise ValueError(f"Published object has different bytes: {key}; bump the revision") + print(f"Verified package object: {key}") + + +def upload(artifacts, fmt, bucket, endpoint): + manifest = load_artifacts(artifacts) + objects = [] + for package in manifest["packages"]: + if package["format"] != fmt: + continue + source = artifacts / package["filename"] + keys = [destination(package, manifest["tag"]).as_posix()] + if fmt == "deb": + # dpkg-scanpackages refers to the distribution symlinks, which use + # an underscore before the architecture. Publish both URLs. + alias = package["filename"].removesuffix(f"-{package['arch']}.deb") + f"_{package['arch']}.deb" + keys.append(f"deb/pools/{package['distro']}/{manifest['tag']}/{alias}") + objects.extend((source, key, package["sha256"], bucket, endpoint) for key in keys) + with ThreadPoolExecutor(max_workers=8) as executor: + list(executor.map(lambda args: upload_object(*args), objects)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + commands.add_parser("plan").add_argument("tag") + commands.add_parser("manifest") + for name in ("stage", "upload"): + sub = commands.add_parser(name) + sub.add_argument("format", choices=FORMATS) + sub.add_argument("--artifacts", type=Path, default=Path("artifacts/packages")) + if name == "stage": + sub.add_argument("--source-commit", required=True) + else: + sub.add_argument("--bucket", required=True) + sub.add_argument("--endpoint", required=True) + args = parser.parse_args() + try: + if args.command == "plan": + plan(ROOT, args.tag) + elif args.command == "manifest": + finish_manifest(ROOT) + elif args.command == "stage": + stage(ROOT, args.artifacts, args.format, args.source_commit) + else: + upload(args.artifacts, args.format, args.bucket, args.endpoint) + except (ValueError, KeyError, OSError, RuntimeError, subprocess.CalledProcessError) as error: + parser.exit(1, f"Error: {error}\n") + + +if __name__ == "__main__": + main() diff --git a/build/revisions/deb.json b/build/revisions/deb.json new file mode 100644 index 00000000..32bf6286 --- /dev/null +++ b/build/revisions/deb.json @@ -0,0 +1,3 @@ +{ + "v0.50.0": 1 +} diff --git a/build/revisions/rpm.json b/build/revisions/rpm.json new file mode 100644 index 00000000..32bf6286 --- /dev/null +++ b/build/revisions/rpm.json @@ -0,0 +1,3 @@ +{ + "v0.50.0": 1 +} diff --git a/build/rpm-repo.sh b/build/rpm-repo.sh index 394aa7d0..89485655 100755 --- a/build/rpm-repo.sh +++ b/build/rpm-repo.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e + export PATH=/opt/gnupg22/bin:$PATH gpg --batch --import key-private.asc diff --git a/tests/smoke-packages.sh b/tests/smoke-packages.sh new file mode 100644 index 00000000..fe87419d --- /dev/null +++ b/tests/smoke-packages.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Run inside the FPM image with the repository mounted read-only at /workspace. +# Uses a fixture shared library; never downloads or publishes Relay binaries. +set -e + +cp -a /workspace/build/. /root/build/ +cd /root/build +mkdir -p dist changelog /tmp/fixture +printf 'int fixture(void) { return 0; }\n' > /tmp/fixture.c +cc -shared -fPIC /tmp/fixture.c -o /tmp/fixture/relay.so +cp /tmp/fixture/relay.so /tmp/fixture/relay-pkg.so +printf '; fixture\n' > /tmp/fixture/relay.ini +printf 'Fixture license\n' > /tmp/fixture/LICENSE +printf '1750000000\n' > changelog/epoch +cat > changelog/deb.tpl <<'EOF' +@PKG@ (0.50.0) unstable; urgency=medium + + * Fixture release. + + -- Relay Team Sun, 15 Jun 2025 15:06:40 +0000 +EOF +cat > changelog/rpm <<'EOF' +* Sun Jun 15 2025 Relay Team - 0.50.0-1 +- Fixture release. +EOF + +export DEB_REVISION=2 RPM_REVISION=3 +bash ./fpm.sh v0.50.0 --list | cut -f6 > selected-packages.txt +source ./helpers.sh +build_dir=/root/build +mode=build +version=v0.50.0 + +for config in base multi ls; do + unset ${!pkg_@} + fpm_build noble deb "$config" amd64 8.4 20240924 https://example.invalid/fixture.tar.gz +done +for config in single.el9 multi.el9 ls.el9; do + unset ${!pkg_@} + fpm_build el9 rpm "$config" x86_64 8.4 20240924 https://example.invalid/fixture.tar.gz +done + +for file in dist/*.deb; do + test "$(dpkg-deb -f "$file" Version)" = 0.50.0-2 +done +for file in dist/*.rpm; do + test "$(rpm -qp --qf '%{VERSION}-%{RELEASE}' "$file")" = 0.50.0-3 + rpm -qp --changelog "$file" | head -1 | grep -F -- '0.50.0-3' +done +dpkg-deb -x dist/php8.4-relay-0.50.0-2-php8.4-noble-amd64.deb /tmp/extracted +gzip -dc /tmp/extracted/usr/share/doc/php8.4-relay/changelog.gz | head -1 | grep -F '(0.50.0-2)' +echo 'Verified three DEBs and three RPMs, including revisioned changelogs.' diff --git a/tests/test_packages.py b/tests/test_packages.py new file mode 100644 index 00000000..a72b76b8 --- /dev/null +++ b/tests/test_packages.py @@ -0,0 +1,229 @@ +import contextlib +import copy +import gzip +import importlib.util +import io +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch +import xml.etree.ElementTree as ET + + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location("packages", ROOT / "build/packages.py") +packages = importlib.util.module_from_spec(spec) +spec.loader.exec_module(packages) +TAG = "v0.50.0" +COMMIT = "a" * 40 + + +class PackageTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + self.revisions = {"deb": 1, "rpm": 1} + self.candidates = [ + {"format": "deb", "distro": distro, "arch": "amd64", "php": "8.4", "name": "php8.4-relay"} + for distro in ("noble", "resolute") + ] + [ + {"format": "rpm", "distro": "el9", "arch": "x86_64", "php": php, "name": "php-relay"} + for php in ("8.3", "8.4") + ] + self.addCleanup(patch.stopall) + patch.dict(os.environ, {"GITHUB_OUTPUT": ""}).start() + + def candidate(self, index, revision=1): + package = dict(self.candidates[index]) + package["filename"] = packages.filename(package, TAG, revision) + return package + + def index(self, candidates, revision=1, checksum="b" * 64, legacy=False): + """Write actual index formats; no package bytes are needed by planning.""" + for package in candidates: + fmt = package["format"] + name = packages.filename(package, TAG, revision) + if legacy: + name = name.replace(f"-{TAG[1:]}-{revision}-", f"-{TAG[1:]}-") + if fmt == "deb": + path = self.root / f"deb/dists/{package['distro']}/main/binary-{package['arch']}/Packages" + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a") as stream: + stream.write( + f"Package: {package['name']}\nArchitecture: {package['arch']}\n" + f"Version: {TAG[1:]}-{revision}\nFilename: pools/{name}\nSHA256: {checksum}\n\n" + ) + else: + directory = self.root / f"rpm/{package['distro']}/repodata" + directory.mkdir(parents=True, exist_ok=True) + primary = directory / "primary.xml.gz" + ns = "http://linux.duke.edu/metadata/common" + tree = ET.fromstring(gzip.decompress(primary.read_bytes())) if primary.exists() else ET.Element(f"{{{ns}}}metadata") + pkg = ET.SubElement(tree, f"{{{ns}}}package") + ET.SubElement(pkg, f"{{{ns}}}name").text = package["name"] + ET.SubElement(pkg, f"{{{ns}}}arch").text = package["arch"] + ET.SubElement(pkg, f"{{{ns}}}version", epoch="0", ver=TAG[1:], rel=str(revision)) + ET.SubElement(pkg, f"{{{ns}}}checksum", type="sha256").text = checksum + ET.SubElement(pkg, f"{{{ns}}}location", href=f"{TAG}/{name}") + primary.write_bytes(gzip.compress(ET.tostring(tree))) + (directory / "repomd.xml").write_text( + '' + '' + ) + + def plan(self, tag=TAG): + for fmt, revision in self.revisions.items(): + packages.write_json(self.root / f"build/revisions/{fmt}.json", {TAG: revision}) + + def command(args, **kwargs): + if args[0] == "git": + return COMMIT + "\n" + return "".join("\t".join([ + *(p[k] for k in ("format", "distro", "arch", "php", "name")), + packages.filename(p, TAG, self.revisions[p["format"]]) + ]) + "\n" for p in self.candidates) + + with patch.object(packages.subprocess, "check_output", side_effect=command): + with contextlib.redirect_stdout(io.StringIO()): + return packages.plan(self.root, tag) + + def artifact(self): + manifest = self.plan() + dist = self.root / "build/dist" + dist.mkdir() + for package in manifest["packages"]: + (dist / package["filename"]).write_bytes(b"package bytes") + packages.finish_manifest(self.root) + return dist, packages.read_json(dist / "manifest.json") + + def test_legacy_rpm_release_one_is_skipped_for_each_php(self): + self.index([self.candidates[2]], legacy=True) + manifest = self.plan() + self.assertEqual([p["php"] for p in manifest["packages"] if p["format"] == "rpm"], ["8.4"]) + + def test_unchanged_revision_is_noop_and_manifest_is_still_produced(self): + self.index(self.candidates) + self.assertEqual(self.plan()["packages"], []) + packages.finish_manifest(self.root) + self.assertEqual(packages.read_json(self.root / "build/dist/manifest.json")["packages"], []) + + def test_only_bumped_format_is_built(self): + self.index(self.candidates) + self.revisions["rpm"] = 2 + manifest = self.plan() + self.assertEqual({p["format"] for p in manifest["packages"]}, {"rpm"}) + self.assertTrue(all("-0.50.0-2-" in p["filename"] for p in manifest["packages"])) + + def test_new_distribution_does_not_rebuild_existing_packages(self): + self.index([self.candidates[i] for i in (0, 2, 3)]) + manifest = self.plan() + self.assertEqual([p["distro"] for p in manifest["packages"]], ["resolute"]) + + def test_revision_cannot_go_backwards(self): + self.index([self.candidates[0]], revision=2) + with self.assertRaisesRegex(ValueError, "older than published"): + self.plan() + + def test_unknown_tag_and_invalid_revisions_fail(self): + with self.assertRaisesRegex(ValueError, "Add v0.51.0"): + self.plan("v0.51.0") + for value in (0, -1, True, "1", 1.5): + self.revisions["deb"] = value + with self.assertRaisesRegex(ValueError, "positive integers"): + self.plan() + + def test_artifact_manifest_checks_every_planned_file(self): + self.plan() + (self.root / "build/dist").mkdir() + with self.assertRaisesRegex(ValueError, "Build output differs"): + packages.finish_manifest(self.root) + + def test_staging_rejects_tampering_and_wrong_commit(self): + dist, manifest = self.artifact() + with self.assertRaisesRegex(ValueError, "source commit"): + packages.stage(self.root, dist, "deb", "c" * 40) + (dist / manifest["packages"][0]["filename"]).write_bytes(b"modified") + with self.assertRaisesRegex(ValueError, "Checksum mismatch"): + packages.stage(self.root, dist, "deb", COMMIT) + + def test_stage_validates_whole_batch_before_writing(self): + dist, manifest = self.artifact() + self.index([self.candidates[1]]) + with self.assertRaisesRegex(ValueError, "Published identity has different bytes"): + packages.stage(self.root, dist, "deb", COMMIT) + self.assertFalse((self.root / packages.destination(manifest["packages"][0], TAG)).exists()) + + def test_stage_is_repeatable_but_rejects_replacement(self): + dist, manifest = self.artifact() + with contextlib.redirect_stdout(io.StringIO()): + packages.stage(self.root, dist, "deb", COMMIT) + packages.stage(self.root, dist, "deb", COMMIT) + dest = self.root / packages.destination(manifest["packages"][0], TAG) + dest.write_bytes(b"older published bytes") + with self.assertRaisesRegex(ValueError, "Refusing to overwrite"): + packages.stage(self.root, dist, "deb", COMMIT) + self.assertEqual(dest.read_bytes(), b"older published bytes") + + def test_manifest_cannot_escape_package_directories(self): + _, manifest = self.artifact() + for field in ("name", "distro", "arch", "php", "filename"): + changed = copy.deepcopy(manifest) + changed["packages"][0][field] = "../../escape" + with self.assertRaises(ValueError): + packages.validate_manifest(changed) + + def test_deb_upload_includes_pool_and_index_alias_only(self): + dist, manifest = self.artifact() + with patch.object(packages, "upload_object") as upload: + packages.upload(dist, "deb", "bucket", "endpoint") + keys = {call.args[1] for call in upload.call_args_list} + self.assertEqual(len(keys), 4) + self.assertIn("deb/pools/resolute/v0.50.0/php8.4-relay-0.50.0-1-php8.4-resolute_amd64.deb", keys) + self.assertTrue(all(key.startswith("deb/") for key in keys)) + + def test_remote_upload_uses_conditional_write_and_checks_retry_bytes(self): + dist, manifest = self.artifact() + package = manifest["packages"][0] + remote_bytes = b"package bytes" + + def aws(args, **kwargs): + if "put-object" in args: + self.assertEqual(args[-2:], ["--if-none-match", "*"]) + return subprocess.CompletedProcess(args, 1, "", "PreconditionFailed") + self.assertIn("get-object", args) + Path(args[-1]).write_bytes(remote_bytes) + return subprocess.CompletedProcess(args, 0) + + with patch.object(packages.subprocess, "run", side_effect=aws): + with contextlib.redirect_stdout(io.StringIO()): + packages.upload_object(dist / package["filename"], "key", package["sha256"], "bucket", "endpoint") + remote_bytes = b"different bytes" + with self.assertRaisesRegex(ValueError, "Published object has different bytes"): + packages.upload_object(dist / package["filename"], "key", package["sha256"], "bucket", "endpoint") + + def test_remote_access_error_is_not_treated_as_missing_object(self): + with patch.object(packages.subprocess, "run", return_value=subprocess.CompletedProcess([], 1, "", "AccessDenied")) as aws: + with self.assertRaisesRegex(RuntimeError, "AccessDenied"): + packages.upload_object(Path("unused"), "key", "checksum", "bucket", "endpoint") + self.assertEqual(aws.call_count, 1) + + def test_real_shell_candidates_include_independent_revisions(self): + result = subprocess.run( + ["bash", str(ROOT / "build/fpm.sh"), TAG, "--list"], + env=dict(os.environ, DEB_REVISION="3", RPM_REVISION="4"), + check=True, capture_output=True, text=True, + ) + rows = [line.split("\t") for line in result.stdout.splitlines()] + self.assertEqual({row[0] for row in rows}, {"deb", "rpm"}) + for row in rows: + self.assertEqual(len(row), 6) + revision = "3" if row[0] == "deb" else "4" + self.assertIn(f"-0.50.0-{revision}-php", row[-1]) + + +if __name__ == "__main__": + unittest.main()