diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index fa54017d..6c3e6303 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -24,7 +24,14 @@ jobs: - name: Checkout code uses: actions/checkout@v7 + - name: Plan unpublished packages + id: plan + env: + TAG: ${{ inputs.tag }} + run: python3 build/packages.py plan "$TAG" + - name: Log in to GHCR + if: steps.plan.outputs.has_packages == 'true' uses: docker/login-action@v4 with: registry: ghcr.io @@ -32,24 +39,36 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Pull the Docker image + if: steps.plan.outputs.has_packages == 'true' run: | docker pull ghcr.io/${{ github.repository_owner }}/fpm:latest docker tag ghcr.io/${{ github.repository_owner }}/fpm:latest fpm - name: Render changelogs + if: steps.plan.outputs.has_packages == 'true' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ inputs.tag }} run: | gh api --paginate 'repos/${{ github.repository_owner }}/relay/releases?per_page=100' \ | jq -s 'add' > /tmp/releases.json - ./build/changelog.sh /tmp/releases.json '${{ github.event.inputs.tag }}' build/changelog + ./build/changelog.sh /tmp/releases.json "$TAG" build/changelog - name: Bundle packages + if: steps.plan.outputs.has_packages == 'true' + env: + TAG: ${{ inputs.tag }} + DEB_REVISION: ${{ steps.plan.outputs.deb_revision }} + RPM_REVISION: ${{ steps.plan.outputs.rpm_revision }} run: | cd build docker run --tty \ - -v ${PWD}:/root/build \ - fpm /bin/bash -c "./fpm.sh ${{ github.event.inputs.tag }}" + -v "${PWD}:/root/build" \ + -e DEB_REVISION -e RPM_REVISION \ + fpm /bin/bash ./fpm.sh "$TAG" + + - name: Record artifact checksums and build revision + run: python3 build/packages.py manifest - name: Upload artifacts uses: actions/upload-artifact@v7 diff --git a/.github/workflows/packaging-tests.yml b/.github/workflows/packaging-tests.yml new file mode 100644 index 00000000..e6f2ccd8 --- /dev/null +++ b/.github/workflows/packaging-tests.yml @@ -0,0 +1,25 @@ +name: Packaging tests + +on: + pull_request: + paths: ['build/**', 'tests/**', '.github/workflows/**'] + push: + branches: [main] + paths: ['build/**', 'tests/**', '.github/workflows/**'] + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Check shell syntax + run: bash -n build/fpm.sh && bash -n build/helpers.sh + - name: Test planning and immutable publication + run: python3 -m unittest discover -s tests -v + - name: Build FPM test image + run: docker build --file build/fpm.Dockerfile --tag fpm-test build + - name: Verify DEB and RPM versions with fixture packages + run: docker run --rm --network none -v "$PWD:/workspace:ro" fpm-test bash /workspace/tests/smoke-packages.sh diff --git a/.github/workflows/repos.yml b/.github/workflows/repos.yml index 5d9f31a7..f56df36f 100644 --- a/.github/workflows/repos.yml +++ b/.github/workflows/repos.yml @@ -5,81 +5,81 @@ on: workflows: [Build packages] types: [completed] -jobs: - - deb-repo: +# Both formats commit to the same branch and publish repository metadata. +concurrency: + group: package-repositories + cancel-in-progress: false - name: Update deb repository +jobs: + publish: + name: Update ${{ matrix.format }} repository runs-on: ubuntu-latest - timeout-minutes: 10 - - if: github.event.workflow_run.conclusion == 'success' + timeout-minutes: 20 + if: >- + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.head_branch == github.event.repository.default_branch && + github.event.workflow_run.head_repository.full_name == github.repository + + strategy: + max-parallel: 1 + fail-fast: false + matrix: + format: [deb, rpm] permissions: contents: write actions: read - steps: + env: + FORMAT: ${{ matrix.format }} - - name: Checkout code + steps: + - name: Checkout current repository uses: actions/checkout@v7 with: + ref: ${{ github.event.repository.default_branch }} lfs: true - # Uses AWS CLI preinstalled on ubuntu-latest runners. - - name: Download artifacts + - name: Download packages and build manifest env: GH_TOKEN: ${{ github.token }} - run: gh run download ${{ github.event.workflow_run.id }} --dir artifacts + BUILD_RUN: ${{ github.event.workflow_run.id }} + run: gh run download "$BUILD_RUN" --name packages --dir artifacts/packages - - name: Get build tag - run: | - TAG_FILE=$(find artifacts -name 'TAG' | head -n 1) - if [ -z "$TAG_FILE" ]; then - echo "No TAG file found in downloaded artifacts" >&2 - exit 1 - fi - TAG=$(tr -d '[:space:]' < "$TAG_FILE") - if [ -z "$TAG" ]; then - echo "TAG file is empty in downloaded artifacts" >&2 - exit 1 - fi - echo "TAG=$TAG" >> $GITHUB_ENV - - - name: Copy packages - run: | - mkdir -p deb/pool/${{ env.TAG }} - find artifacts -name '*.deb' -exec cp {} deb/pool/${{ env.TAG }} \; + - name: Verify and stage packages without overwriting published versions + id: stage + env: + BUILD_COMMIT: ${{ github.event.workflow_run.head_sha }} + run: python3 build/packages.py stage "$FORMAT" --source-commit "$BUILD_COMMIT" - name: Build the Docker image - run: docker build . --tag deb --file build/deb.Dockerfile + if: steps.stage.outputs.has_packages == 'true' + run: docker build . --tag "$FORMAT" --file "build/$FORMAT.Dockerfile" - name: Set up private key - run: echo -n '${{ secrets.PRIVATE_KEY }}' | base64 --decode > key-private.asc - - - name: Set up Git - run: | - git config --local user.name "github-actions[bot]" - git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" + if: steps.stage.outputs.has_packages == 'true' + env: + PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }} + run: printf '%s' "$PRIVATE_KEY" | base64 --decode > key-private.asc - name: Update repository + if: steps.stage.outputs.has_packages == 'true' + env: + GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} run: | docker run --tty \ - -v ${PWD}:/root/deb \ - -e GPG_PASSPHRASE='${{ secrets.GPG_PASSPHRASE }}' \ - deb /bin/bash -c "./build/deb-repo.sh" + -v "${PWD}:/root/$FORMAT" \ + -e GPG_PASSPHRASE \ + "$FORMAT" /bin/bash "./build/$FORMAT-repo.sh" - name: Remove private key + if: always() run: rm -f key-private.asc - - name: Commit changes - run: | - git pull - git add deb/* - git commit -m "Bump deb repo to ${{ env.TAG }}" || echo "No changes to commit" - git push - - - name: Sync repo with R2 + # Upload packages before advertising them in either Git or repository metadata. + # Conditional writes allow identical retries but never replace existing bytes. + - name: Publish immutable package files to R2 + if: steps.stage.outputs.has_packages == 'true' env: R2_ENDPOINT: https://a1220fd38ad4771f7b7b38f5f3c2b00d.r2.cloudflarestorage.com AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} @@ -87,105 +87,26 @@ jobs: AWS_DEFAULT_REGION: auto AWS_DEFAULT_OUTPUT: json run: | - aws s3 cp key.gpg s3://relay-repos/ \ - --no-progress \ - --acl public-read \ - --endpoint-url ${{ env.R2_ENDPOINT }} - aws s3 cp deb/sources.list s3://relay-repos/deb/ \ - --no-progress \ - --acl public-read \ - --endpoint-url ${{ env.R2_ENDPOINT }} - aws s3 sync deb s3://relay-repos/deb \ - --exclude "*" \ - --include "*${{ env.TAG }}*" \ - --delete \ - --no-progress \ - --acl public-read \ - --endpoint-url ${{ env.R2_ENDPOINT }} - aws s3 sync deb/dists s3://relay-repos/deb/dists/ \ - --no-progress \ - --acl public-read \ - --endpoint-url ${{ env.R2_ENDPOINT }} - - rpm-repo: - - name: Update rpm repository - runs-on: ubuntu-latest - needs: deb-repo - timeout-minutes: 10 - - if: github.event.workflow_run.conclusion == 'success' - - permissions: - contents: write - actions: read - - steps: + python3 build/packages.py upload "$FORMAT" \ + --bucket relay-repos --endpoint "$R2_ENDPOINT" - - name: Checkout code - uses: actions/checkout@v7 - with: - lfs: true - - # Uses AWS CLI preinstalled on ubuntu-latest runners. - - name: Download artifacts + - name: Commit changes + if: steps.stage.outputs.has_packages == 'true' env: - GH_TOKEN: ${{ github.token }} - run: gh run download ${{ github.event.workflow_run.id }} --dir artifacts - - - name: Get build tag - run: | - TAG_FILE=$(find artifacts -name 'TAG' | head -n 1) - if [ -z "$TAG_FILE" ]; then - echo "No TAG file found in downloaded artifacts" >&2 - exit 1 - fi - TAG=$(tr -d '[:space:]' < "$TAG_FILE") - if [ -z "$TAG" ]; then - echo "TAG file is empty in downloaded artifacts" >&2 - exit 1 - fi - echo "TAG=$TAG" >> $GITHUB_ENV - - - name: Copy packages - run: | - source build/distros.sh - - for distro in "${el_dists[@]}"; do - mkdir -p "rpm/$distro/${{ env.TAG }}" - find artifacts -name "*-$distro-*.rpm" \ - -exec cp {} "rpm/$distro/${{ env.TAG }}" \; - done - - - name: Build the Docker image - run: docker build . --tag rpm --file build/rpm.Dockerfile - - - name: Set up private key - run: echo -n '${{ secrets.PRIVATE_KEY }}' | base64 --decode > key-private.asc - - - name: Set up Git + TAG: ${{ steps.stage.outputs.tag }} + REVISION: ${{ steps.stage.outputs.revision }} run: | git config --local user.name "github-actions[bot]" git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add "$FORMAT" + if ! git diff --cached --quiet; then + git commit -m "Publish $FORMAT packages for $TAG revision $REVISION" + git pull --rebase + git push + fi - - name: Update repository - run: | - docker run --tty \ - -v ${PWD}:/root/rpm \ - -e GPG_PASSPHRASE='${{ secrets.GPG_PASSPHRASE }}' \ - rpm /bin/bash -c "./build/rpm-repo.sh" - - - name: Remove private key - run: rm -f key-private.asc - - - name: Commit changes - run: | - git pull - git add rpm/* - git commit -m "Bump rpm repos to ${{ env.TAG }}" || echo "No changes to commit" - git push - - - name: Sync repo with R2 + - name: Publish repository metadata to R2 + if: steps.stage.outputs.has_packages == 'true' env: R2_ENDPOINT: https://a1220fd38ad4771f7b7b38f5f3c2b00d.r2.cloudflarestorage.com AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} @@ -194,25 +115,18 @@ jobs: AWS_DEFAULT_OUTPUT: json run: | aws s3 cp key.gpg s3://relay-repos/ \ - --no-progress \ - --acl public-read \ - --endpoint-url ${{ env.R2_ENDPOINT }} - aws s3 cp rpm/el.repo s3://relay-repos/rpm/ \ - --no-progress \ - --acl public-read \ - --endpoint-url ${{ env.R2_ENDPOINT }} - aws s3 sync rpm s3://relay-repos/rpm \ - --exclude "*" \ - --include "*${{ env.TAG }}*" \ - --delete \ - --no-progress \ - --acl public-read \ - --endpoint-url ${{ env.R2_ENDPOINT }} - source build/distros.sh - - for distro in "${el_dists[@]}"; do - aws s3 sync "rpm/$distro/repodata" "s3://relay-repos/rpm/$distro/repodata/" \ - --no-progress \ - --acl public-read \ - --endpoint-url ${{ env.R2_ENDPOINT }} - done + --no-progress --endpoint-url "$R2_ENDPOINT" + if [ "$FORMAT" = deb ]; then + aws s3 cp deb/sources.list s3://relay-repos/deb/ \ + --no-progress --endpoint-url "$R2_ENDPOINT" + aws s3 sync deb/dists s3://relay-repos/deb/dists/ \ + --no-progress --endpoint-url "$R2_ENDPOINT" + else + aws s3 cp rpm/el.repo s3://relay-repos/rpm/ \ + --no-progress --endpoint-url "$R2_ENDPOINT" + source build/distros.sh + for distro in "${el_dists[@]}"; do + aws s3 sync "rpm/$distro/repodata" "s3://relay-repos/rpm/$distro/repodata/" \ + --no-progress --endpoint-url "$R2_ENDPOINT" + done + fi diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index ca62d197..f0b85883 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -5,6 +5,10 @@ on: # schedule: # - cron: '0 0 * * 0' # once a week +concurrency: + group: package-repositories + cancel-in-progress: false + jobs: deb-repo: diff --git a/.gitignore b/.gitignore index cbe33068..209aea84 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ /build/dist /build/src/*/*-php* /build/changelog/ +/build/plan.json +/build/selected-packages.txt +__pycache__/ key-private.asc diff --git a/README.md b/README.md index 4c77144f..38ec59fb 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,56 @@ For detailed installations instruction see [relay.so](https://relay.so/docs/installation). +## Publishing packages + +The **Build packages** action takes an upstream Relay tag, such as `v0.50.0`. +Packaging revisions are recorded separately for each format in +[`build/revisions/deb.json`](build/revisions/deb.json) and +[`build/revisions/rpm.json`](build/revisions/rpm.json), keyed by upstream tag. +Both files must contain the tag; revisions are positive integers. + +- For a new Relay release, add its tag to both files with revision `1`. +- For a packaging fix, increment the affected format's revision for that tag. + A shared change affecting both formats requires both revisions to increase. +- Adding a distribution or PHP target can use the existing revision: only + missing packages are built. Existing packages are skipped even if packaging + code changed, so bump the revision when those packages need the change. +- Commit the revision changes before running the action. Revisions are shared + across distributions, architectures, and PHP variants within each format. + +DEBs use `Version: 0.50.0-1`; RPMs use `Version: 0.50.0` and `Release: 1`. +Filenames also include the revision. Upstream downloads still use `v0.50.0`. +When adopting this scheme, existing unsuffixed DEBs advance to revision `1`. +Existing RPMs already have Release `1`, despite their old filenames, and are +skipped until the RPM revision increases to `2`. + +The action prints a build/skip count for each format. A rerun with no missing +packages is a successful no-op. To preview the plan locally without downloading +or building packages (Python 3.10+ and Bash are required): + +```bash +python3 build/packages.py plan v0.50.0 +``` + +The artifact includes `manifest.json`, recording the upstream tag, revisions, +build commit, filenames, and SHA256 checksums. **Update repositories** consumes +that manifest and updates only formats with built packages. Automatic publishing +is restricted to builds from this repository's default branch; branch builds +produce reviewable artifacts without publishing them. + +Published package files are immutable. Publication rejects conflicting versions +and uses conditional R2 writes to prevent overwrites, while allowing identical +retries. Old revisions and their download URLs are retained. If publication +fails partway through, rerun **Update repositories** for the original build so +it can finish publishing the same artifacts; rebuilding may produce different +bytes. Repository metadata is uploaded after the package files. + +Run the packaging regression tests with: + +```bash +python3 -m unittest discover -s tests -v +``` + ## Using APT (Debian, Ubuntu) ```bash diff --git a/build/deb-repo.sh b/build/deb-repo.sh index 256b7076..adffb13e 100755 --- a/build/deb-repo.sh +++ b/build/deb-repo.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e + gpg --batch --import key-private.asc source build/distros.sh diff --git a/build/fpm.sh b/build/fpm.sh index d2afe748..73d590f5 100755 --- a/build/fpm.sh +++ b/build/fpm.sh @@ -2,10 +2,22 @@ set -e -source /root/build/helpers.sh -source /root/build/distros.sh - -version=$1 +build_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +source "$build_dir/helpers.sh" +source "$build_dir/distros.sh" + +version=${1:?Usage: fpm.sh TAG [--list]} +mode=${2:-build} +if [[ ! "$version" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || + [[ ! "${DEB_REVISION:-}" =~ ^[1-9][0-9]*$ ]] || + [[ ! "${RPM_REVISION:-}" =~ ^[1-9][0-9]*$ ]]; then + echo "A release tag and positive DEB_REVISION/RPM_REVISION are required; run packages.py plan first" >&2 + exit 1 +fi +if [[ "$mode" != build && "$mode" != --list ]]; then + echo "Unknown mode: $mode" >&2 + exit 1 +fi baseurl="https://builds.r2.relay.so/$version/relay-$version" declare -A php_api=( diff --git a/build/helpers.sh b/build/helpers.sh index 86a22e9f..5ffac966 100755 --- a/build/helpers.sh +++ b/build/helpers.sh @@ -9,11 +9,15 @@ DESCRIPTION main() { - rm -rf /tmp/relay* - rm -rf /root/build/dist - mkdir /root/build/dist - - echo -n "$version" > /root/build/dist/TAG + if [[ "$mode" != --list ]]; then + test -f "$build_dir/selected-packages.txt" || { + echo "Missing build plan; run packages.py plan first" >&2 + exit 1 + } + rm -rf /tmp/relay* + rm -rf "$build_dir/dist" + mkdir "$build_dir/dist" + fi for package in "${packages[@]}"; do unset ${!pkg_@} @@ -36,11 +40,27 @@ fpm_build() # we don't have centos builds for v0.1.0 if [[ "$version" == "v0.1.0" && "$type" == "rpm" ]]; then - echo "Skipping RPMs for v0.1.0" + echo "Skipping RPMs for v0.1.0" >&2 return 0 fi - source /root/build/src/$type/config.$config.sh + source "$build_dir/src/$type/config.$config.sh" + + pkg_version=${version#v} + if [[ "$type" == deb ]]; then + pkg_revision=$DEB_REVISION + else + pkg_revision=$RPM_REVISION + fi + pkg_filename="${pkg_name}-${pkg_version}-${pkg_revision}-php${php_version}-${pkg_identifier}-${pkg_arch}.${type}" + + if [[ "$mode" == --list ]]; then + printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$type" "$distro" "$pkg_arch" "$php_version" "$pkg_name" "$pkg_filename" + return 0 + fi + if ! grep -Fxq -- "$pkg_filename" "$build_dir/selected-packages.txt"; then + return 0 + fi echo "Building Relay ($version) .$type package for PHP $php_version on $pkg_arch" @@ -109,9 +129,6 @@ fpm_build() done > $dest_path/usr/share/lintian/overrides/$pkg_name fi - pkg_version=${version#v} - pkg_filename="${pkg_name}-${pkg_version}-php${php_version}-${pkg_identifier}-${pkg_arch}.${type}" - args=( "--input-type dir" "--output-type $type" @@ -123,6 +140,7 @@ fpm_build() "--category 'php'" "--name '$pkg_name'" "--version '$pkg_version'" + "--iteration '$pkg_revision'" "--architecture $pkg_arch" "--package dist/$pkg_filename" @@ -145,10 +163,14 @@ fpm_build() # deb changelog entries embed the package name, rpm ones don't if [[ "$type" == "deb" ]]; then - sed "s/@PKG@/$pkg_name/g" /root/build/changelog/deb.tpl > /tmp/changelog-$pkg_name.deb + sed -e "s/@PKG@/$pkg_name/g" \ + -e "1s/($pkg_version)/($pkg_version-$pkg_revision)/" \ + /root/build/changelog/deb.tpl > /tmp/changelog-$pkg_name.deb args+=("--deb-changelog /tmp/changelog-$pkg_name.deb") else - args+=("--rpm-changelog /root/build/changelog/rpm") + sed "1s/ - $pkg_version-1$/ - $pkg_version-$pkg_revision/" \ + /root/build/changelog/rpm > /tmp/changelog-$pkg_name.rpm + args+=("--rpm-changelog /tmp/changelog-$pkg_name.rpm") fi if [ ! -z "$pkg_provides" ]; then @@ -171,4 +193,15 @@ fpm_build() echo "Building package: $pkg_filename" bash -c "fpm $args $dest_path/=/" + + # Check the actual package version, not just the filename we supplied to FPM. + if [[ "$type" == deb ]]; then + actual_version=$(dpkg-deb -f "dist/$pkg_filename" Version) + else + actual_version=$(rpm -qp --qf '%{VERSION}-%{RELEASE}' "dist/$pkg_filename") + fi + if [[ "$actual_version" != "$pkg_version-$pkg_revision" ]]; then + echo "Unexpected version in $pkg_filename: $actual_version" >&2 + exit 1 + fi } diff --git a/build/packages.py b/build/packages.py new file mode 100644 index 00000000..9c218e2e --- /dev/null +++ b/build/packages.py @@ -0,0 +1,299 @@ +#!/usr/bin/env python3 +"""Plan revisioned packages and publish them without replacing existing bytes.""" + +import argparse +from concurrent.futures import ThreadPoolExecutor +import gzip +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +import xml.etree.ElementTree as ET + + +ROOT = Path(__file__).resolve().parent.parent +FORMATS = ("deb", "rpm") + + +def read_json(path): + return json.loads(path.read_text()) + + +def write_json(path, data): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=2) + "\n") + + +def sha256(path): + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def check_tag(tag): + if not isinstance(tag, str) or not re.fullmatch(r"v[0-9]+\.[0-9]+\.[0-9]+", tag): + raise ValueError("Expected a stable upstream tag such as v0.50.0") + + +def check_revisions(revisions): + if set(revisions) != set(FORMATS) or any( + type(value) is not int or value < 1 for value in revisions.values() + ): + raise ValueError("DEB and RPM revisions must be positive integers") + + +def filename(package, tag, revision): + return ( + f"{package['name']}-{tag[1:]}-{revision}-php{package['php']}-" + f"{package['distro']}-{package['arch']}.{package['format']}" + ) + + +def destination(package, tag): + if package["format"] == "deb": + return Path("deb/pool") / tag / package["filename"] + return Path("rpm") / package["distro"] / tag / package["filename"] + + +def identity(package, version): + # The single-PHP RPMs share a name/version/arch across PHP versions. + return tuple(package[k] for k in ("format", "distro", "name", "arch", "php")) + (version,) + + +def published_packages(root): + """Read indices, including legacy RPM Release 1 files without a suffix.""" + published = {} + + def add(fmt, distro, name, arch, version, location, checksum): + php = re.search(r"-php([0-9]+\.[0-9]+)-", location) + if not php: + raise ValueError(f"Cannot identify PHP version in {location}") + key = (fmt, distro, name, arch, php[1], version) + published.setdefault(key, set()).add(checksum) + + for path in sorted((root / "deb/dists").glob("*/main/binary-*/Packages")): + distro = path.parents[2].name + for stanza in path.read_text().strip().split("\n\n"): + fields = dict(line.split(": ", 1) for line in stanza.splitlines() + if ": " in line and not line.startswith(" ")) + if fields: + add("deb", distro, fields["Package"], fields["Architecture"], + fields["Version"], fields["Filename"], fields["SHA256"]) + + ns = {"r": "http://linux.duke.edu/metadata/repo", + "c": "http://linux.duke.edu/metadata/common"} + for path in sorted((root / "rpm").glob("*/repodata/repomd.xml")): + distro = path.parents[1].name + repomd = ET.parse(path) + location = repomd.find("r:data[@type='primary']/r:location", ns).attrib["href"] + primary = path.parents[1] / location + with gzip.open(primary) as stream: + packages = ET.parse(stream) + for package in packages.findall("c:package", ns): + version = package.find("c:version", ns).attrib + checksum = package.find("c:checksum", ns) + if checksum.attrib["type"] != "sha256": + raise ValueError(f"Expected SHA256 checksums in {primary}") + add("rpm", distro, package.findtext("c:name", namespaces=ns), + package.findtext("c:arch", namespaces=ns), + f"{version['ver']}-{version['rel']}", + package.find("c:location", ns).attrib["href"], checksum.text) + return published + + +def output(name, value): + print(f"{name}={value}") + if os.environ.get("GITHUB_OUTPUT"): + with open(os.environ["GITHUB_OUTPUT"], "a") as stream: + stream.write(f"{name}={value}\n") + + +def plan(root, tag): + check_tag(tag) + revisions = {} + for fmt in FORMATS: + path = root / f"build/revisions/{fmt}.json" + records = read_json(path) + if tag not in records: + raise ValueError(f"Add {tag} to {path.relative_to(root)} before building") + revisions[fmt] = records[tag] + check_revisions(revisions) + published = published_packages(root) + for key in published: + match = re.fullmatch(re.escape(tag[1:]) + r"-([0-9]+)", key[-1]) + if match and int(match[1]) > revisions[key[0]]: + raise ValueError(f"{key[0]} revision {revisions[key[0]]} is older than published {key[-1]}") + + env = dict(os.environ, **{f"{fmt.upper()}_REVISION": str(rev) for fmt, rev in revisions.items()}) + candidates = subprocess.check_output( + ["bash", str(root / "build/fpm.sh"), tag, "--list"], env=env, text=True + ) + selected = [] + skipped = dict.fromkeys(FORMATS, 0) + for line in candidates.splitlines(): + package = dict(zip(("format", "distro", "arch", "php", "name", "filename"), + line.split("\t"), strict=True)) + fmt = package["format"] + expected = filename(package, tag, revisions[fmt]) + if package["filename"] != expected: + raise ValueError(f"Unexpected candidate filename: {package['filename']}") + if identity(package, f"{tag[1:]}-{revisions[fmt]}") in published: + skipped[fmt] += 1 + else: + selected.append(package) + commit = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip() + manifest = {"schema": 1, "tag": tag, "revisions": revisions, + "source_commit": commit, "packages": selected} + write_json(root / "build/plan.json", manifest) + (root / "build/selected-packages.txt").write_text("".join(p["filename"] + "\n" for p in selected)) + for fmt in FORMATS: + count = sum(p["format"] == fmt for p in selected) + print(f"{fmt.upper()} revision {revisions[fmt]}: build {count}, already published {skipped[fmt]}") + output(f"{fmt}_revision", revisions[fmt]) + output("has_packages", str(bool(selected)).lower()) + return manifest + + +def validate_manifest(manifest): + if manifest.get("schema") != 1: + raise ValueError("Unsupported package manifest schema") + check_tag(manifest["tag"]) + check_revisions(manifest["revisions"]) + if not re.fullmatch(r"[0-9a-f]{40}", manifest["source_commit"]): + raise ValueError("Invalid source commit") + seen = set() + for package in manifest["packages"]: + fmt = package["format"] + if fmt not in FORMATS: + raise ValueError(f"Unknown package format: {fmt}") + for field in ("name", "distro", "arch", "php"): + if not re.fullmatch(r"[a-z0-9][a-z0-9._+-]*", package[field]): + raise ValueError(f"Invalid package {field}") + expected = filename(package, manifest["tag"], manifest["revisions"][fmt]) + if package["filename"] != expected or expected in seen: + raise ValueError(f"Invalid or duplicate filename: {package['filename']}") + seen.add(expected) + + +def finish_manifest(root): + manifest = read_json(root / "build/plan.json") + validate_manifest(manifest) + dist = root / "build/dist" + dist.mkdir(exist_ok=True) + expected = {p["filename"] for p in manifest["packages"]} + actual = {p.name for fmt in FORMATS for p in dist.glob(f"*.{fmt}")} + if expected != actual: + raise ValueError(f"Build output differs from plan: missing {expected - actual}, extra {actual - expected}") + for package in manifest["packages"]: + package["sha256"] = sha256(dist / package["filename"]) + write_json(dist / "manifest.json", manifest) + + +def load_artifacts(artifacts): + manifest = read_json(artifacts / "manifest.json") + validate_manifest(manifest) + for package in manifest["packages"]: + if sha256(artifacts / package["filename"]) != package["sha256"]: + raise ValueError(f"Checksum mismatch: {package['filename']}") + return manifest + + +def stage(root, artifacts, fmt, source_commit): + manifest = load_artifacts(artifacts) + if manifest["source_commit"] != source_commit: + raise ValueError("Artifact source commit does not match the build workflow") + published = published_packages(root) + packages = [p for p in manifest["packages"] if p["format"] == fmt] + # Validate the entire batch before copying anything. + for package in packages: + version = f"{manifest['tag'][1:]}-{manifest['revisions'][fmt]}" + checksums = published.get(identity(package, version), set()) + if checksums and checksums != {package["sha256"]}: + raise ValueError(f"Published identity has different bytes: {package['filename']}; bump the revision") + dest = root / destination(package, manifest["tag"]) + if dest.exists() and sha256(dest) != package["sha256"]: + raise ValueError(f"Refusing to overwrite {dest}; bump the revision") + for package in packages: + dest = root / destination(package, manifest["tag"]) + dest.parent.mkdir(parents=True, exist_ok=True) + if not dest.exists(): + shutil.copyfile(artifacts / package["filename"], dest) + output("tag", manifest["tag"]) + output("revision", manifest["revisions"][fmt]) + output("has_packages", str(bool(packages)).lower()) + + +def upload_object(source, key, checksum, bucket, endpoint): + command = ["aws", "s3api", "--endpoint-url", endpoint] + result = subprocess.run(command + [ + "put-object", "--bucket", bucket, "--key", key, "--body", str(source), + "--if-none-match", "*", + ], text=True, capture_output=True) + if result.returncode: + if "PreconditionFailed" not in result.stderr and "ConditionalRequestConflict" not in result.stderr: + raise RuntimeError(result.stderr) + # A retry may encounter an already uploaded object. Compare actual bytes; + # ETags are not necessarily content hashes (e.g. multipart uploads). + with tempfile.TemporaryDirectory() as tmp: + previous = Path(tmp) / "package" + subprocess.run(command + ["get-object", "--bucket", bucket, "--key", key, + str(previous)], check=True, stdout=subprocess.DEVNULL) + if sha256(previous) != checksum: + raise ValueError(f"Published object has different bytes: {key}; bump the revision") + print(f"Verified package object: {key}") + + +def upload(artifacts, fmt, bucket, endpoint): + manifest = load_artifacts(artifacts) + objects = [] + for package in manifest["packages"]: + if package["format"] != fmt: + continue + source = artifacts / package["filename"] + keys = [destination(package, manifest["tag"]).as_posix()] + if fmt == "deb": + # dpkg-scanpackages refers to the distribution symlinks, which use + # an underscore before the architecture. Publish both URLs. + alias = package["filename"].removesuffix(f"-{package['arch']}.deb") + f"_{package['arch']}.deb" + keys.append(f"deb/pools/{package['distro']}/{manifest['tag']}/{alias}") + objects.extend((source, key, package["sha256"], bucket, endpoint) for key in keys) + with ThreadPoolExecutor(max_workers=8) as executor: + list(executor.map(lambda args: upload_object(*args), objects)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + commands.add_parser("plan").add_argument("tag") + commands.add_parser("manifest") + for name in ("stage", "upload"): + sub = commands.add_parser(name) + sub.add_argument("format", choices=FORMATS) + sub.add_argument("--artifacts", type=Path, default=Path("artifacts/packages")) + if name == "stage": + sub.add_argument("--source-commit", required=True) + else: + sub.add_argument("--bucket", required=True) + sub.add_argument("--endpoint", required=True) + args = parser.parse_args() + try: + if args.command == "plan": + plan(ROOT, args.tag) + elif args.command == "manifest": + finish_manifest(ROOT) + elif args.command == "stage": + stage(ROOT, args.artifacts, args.format, args.source_commit) + else: + upload(args.artifacts, args.format, args.bucket, args.endpoint) + except (ValueError, KeyError, OSError, RuntimeError, subprocess.CalledProcessError) as error: + parser.exit(1, f"Error: {error}\n") + + +if __name__ == "__main__": + main() diff --git a/build/revisions/deb.json b/build/revisions/deb.json new file mode 100644 index 00000000..32bf6286 --- /dev/null +++ b/build/revisions/deb.json @@ -0,0 +1,3 @@ +{ + "v0.50.0": 1 +} diff --git a/build/revisions/rpm.json b/build/revisions/rpm.json new file mode 100644 index 00000000..32bf6286 --- /dev/null +++ b/build/revisions/rpm.json @@ -0,0 +1,3 @@ +{ + "v0.50.0": 1 +} diff --git a/build/rpm-repo.sh b/build/rpm-repo.sh index 394aa7d0..89485655 100755 --- a/build/rpm-repo.sh +++ b/build/rpm-repo.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e + export PATH=/opt/gnupg22/bin:$PATH gpg --batch --import key-private.asc diff --git a/tests/smoke-packages.sh b/tests/smoke-packages.sh new file mode 100644 index 00000000..fe87419d --- /dev/null +++ b/tests/smoke-packages.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Run inside the FPM image with the repository mounted read-only at /workspace. +# Uses a fixture shared library; never downloads or publishes Relay binaries. +set -e + +cp -a /workspace/build/. /root/build/ +cd /root/build +mkdir -p dist changelog /tmp/fixture +printf 'int fixture(void) { return 0; }\n' > /tmp/fixture.c +cc -shared -fPIC /tmp/fixture.c -o /tmp/fixture/relay.so +cp /tmp/fixture/relay.so /tmp/fixture/relay-pkg.so +printf '; fixture\n' > /tmp/fixture/relay.ini +printf 'Fixture license\n' > /tmp/fixture/LICENSE +printf '1750000000\n' > changelog/epoch +cat > changelog/deb.tpl <<'EOF' +@PKG@ (0.50.0) unstable; urgency=medium + + * Fixture release. + + -- Relay Team Sun, 15 Jun 2025 15:06:40 +0000 +EOF +cat > changelog/rpm <<'EOF' +* Sun Jun 15 2025 Relay Team - 0.50.0-1 +- Fixture release. +EOF + +export DEB_REVISION=2 RPM_REVISION=3 +bash ./fpm.sh v0.50.0 --list | cut -f6 > selected-packages.txt +source ./helpers.sh +build_dir=/root/build +mode=build +version=v0.50.0 + +for config in base multi ls; do + unset ${!pkg_@} + fpm_build noble deb "$config" amd64 8.4 20240924 https://example.invalid/fixture.tar.gz +done +for config in single.el9 multi.el9 ls.el9; do + unset ${!pkg_@} + fpm_build el9 rpm "$config" x86_64 8.4 20240924 https://example.invalid/fixture.tar.gz +done + +for file in dist/*.deb; do + test "$(dpkg-deb -f "$file" Version)" = 0.50.0-2 +done +for file in dist/*.rpm; do + test "$(rpm -qp --qf '%{VERSION}-%{RELEASE}' "$file")" = 0.50.0-3 + rpm -qp --changelog "$file" | head -1 | grep -F -- '0.50.0-3' +done +dpkg-deb -x dist/php8.4-relay-0.50.0-2-php8.4-noble-amd64.deb /tmp/extracted +gzip -dc /tmp/extracted/usr/share/doc/php8.4-relay/changelog.gz | head -1 | grep -F '(0.50.0-2)' +echo 'Verified three DEBs and three RPMs, including revisioned changelogs.' diff --git a/tests/test_packages.py b/tests/test_packages.py new file mode 100644 index 00000000..a72b76b8 --- /dev/null +++ b/tests/test_packages.py @@ -0,0 +1,229 @@ +import contextlib +import copy +import gzip +import importlib.util +import io +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch +import xml.etree.ElementTree as ET + + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location("packages", ROOT / "build/packages.py") +packages = importlib.util.module_from_spec(spec) +spec.loader.exec_module(packages) +TAG = "v0.50.0" +COMMIT = "a" * 40 + + +class PackageTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + self.revisions = {"deb": 1, "rpm": 1} + self.candidates = [ + {"format": "deb", "distro": distro, "arch": "amd64", "php": "8.4", "name": "php8.4-relay"} + for distro in ("noble", "resolute") + ] + [ + {"format": "rpm", "distro": "el9", "arch": "x86_64", "php": php, "name": "php-relay"} + for php in ("8.3", "8.4") + ] + self.addCleanup(patch.stopall) + patch.dict(os.environ, {"GITHUB_OUTPUT": ""}).start() + + def candidate(self, index, revision=1): + package = dict(self.candidates[index]) + package["filename"] = packages.filename(package, TAG, revision) + return package + + def index(self, candidates, revision=1, checksum="b" * 64, legacy=False): + """Write actual index formats; no package bytes are needed by planning.""" + for package in candidates: + fmt = package["format"] + name = packages.filename(package, TAG, revision) + if legacy: + name = name.replace(f"-{TAG[1:]}-{revision}-", f"-{TAG[1:]}-") + if fmt == "deb": + path = self.root / f"deb/dists/{package['distro']}/main/binary-{package['arch']}/Packages" + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a") as stream: + stream.write( + f"Package: {package['name']}\nArchitecture: {package['arch']}\n" + f"Version: {TAG[1:]}-{revision}\nFilename: pools/{name}\nSHA256: {checksum}\n\n" + ) + else: + directory = self.root / f"rpm/{package['distro']}/repodata" + directory.mkdir(parents=True, exist_ok=True) + primary = directory / "primary.xml.gz" + ns = "http://linux.duke.edu/metadata/common" + tree = ET.fromstring(gzip.decompress(primary.read_bytes())) if primary.exists() else ET.Element(f"{{{ns}}}metadata") + pkg = ET.SubElement(tree, f"{{{ns}}}package") + ET.SubElement(pkg, f"{{{ns}}}name").text = package["name"] + ET.SubElement(pkg, f"{{{ns}}}arch").text = package["arch"] + ET.SubElement(pkg, f"{{{ns}}}version", epoch="0", ver=TAG[1:], rel=str(revision)) + ET.SubElement(pkg, f"{{{ns}}}checksum", type="sha256").text = checksum + ET.SubElement(pkg, f"{{{ns}}}location", href=f"{TAG}/{name}") + primary.write_bytes(gzip.compress(ET.tostring(tree))) + (directory / "repomd.xml").write_text( + '' + '' + ) + + def plan(self, tag=TAG): + for fmt, revision in self.revisions.items(): + packages.write_json(self.root / f"build/revisions/{fmt}.json", {TAG: revision}) + + def command(args, **kwargs): + if args[0] == "git": + return COMMIT + "\n" + return "".join("\t".join([ + *(p[k] for k in ("format", "distro", "arch", "php", "name")), + packages.filename(p, TAG, self.revisions[p["format"]]) + ]) + "\n" for p in self.candidates) + + with patch.object(packages.subprocess, "check_output", side_effect=command): + with contextlib.redirect_stdout(io.StringIO()): + return packages.plan(self.root, tag) + + def artifact(self): + manifest = self.plan() + dist = self.root / "build/dist" + dist.mkdir() + for package in manifest["packages"]: + (dist / package["filename"]).write_bytes(b"package bytes") + packages.finish_manifest(self.root) + return dist, packages.read_json(dist / "manifest.json") + + def test_legacy_rpm_release_one_is_skipped_for_each_php(self): + self.index([self.candidates[2]], legacy=True) + manifest = self.plan() + self.assertEqual([p["php"] for p in manifest["packages"] if p["format"] == "rpm"], ["8.4"]) + + def test_unchanged_revision_is_noop_and_manifest_is_still_produced(self): + self.index(self.candidates) + self.assertEqual(self.plan()["packages"], []) + packages.finish_manifest(self.root) + self.assertEqual(packages.read_json(self.root / "build/dist/manifest.json")["packages"], []) + + def test_only_bumped_format_is_built(self): + self.index(self.candidates) + self.revisions["rpm"] = 2 + manifest = self.plan() + self.assertEqual({p["format"] for p in manifest["packages"]}, {"rpm"}) + self.assertTrue(all("-0.50.0-2-" in p["filename"] for p in manifest["packages"])) + + def test_new_distribution_does_not_rebuild_existing_packages(self): + self.index([self.candidates[i] for i in (0, 2, 3)]) + manifest = self.plan() + self.assertEqual([p["distro"] for p in manifest["packages"]], ["resolute"]) + + def test_revision_cannot_go_backwards(self): + self.index([self.candidates[0]], revision=2) + with self.assertRaisesRegex(ValueError, "older than published"): + self.plan() + + def test_unknown_tag_and_invalid_revisions_fail(self): + with self.assertRaisesRegex(ValueError, "Add v0.51.0"): + self.plan("v0.51.0") + for value in (0, -1, True, "1", 1.5): + self.revisions["deb"] = value + with self.assertRaisesRegex(ValueError, "positive integers"): + self.plan() + + def test_artifact_manifest_checks_every_planned_file(self): + self.plan() + (self.root / "build/dist").mkdir() + with self.assertRaisesRegex(ValueError, "Build output differs"): + packages.finish_manifest(self.root) + + def test_staging_rejects_tampering_and_wrong_commit(self): + dist, manifest = self.artifact() + with self.assertRaisesRegex(ValueError, "source commit"): + packages.stage(self.root, dist, "deb", "c" * 40) + (dist / manifest["packages"][0]["filename"]).write_bytes(b"modified") + with self.assertRaisesRegex(ValueError, "Checksum mismatch"): + packages.stage(self.root, dist, "deb", COMMIT) + + def test_stage_validates_whole_batch_before_writing(self): + dist, manifest = self.artifact() + self.index([self.candidates[1]]) + with self.assertRaisesRegex(ValueError, "Published identity has different bytes"): + packages.stage(self.root, dist, "deb", COMMIT) + self.assertFalse((self.root / packages.destination(manifest["packages"][0], TAG)).exists()) + + def test_stage_is_repeatable_but_rejects_replacement(self): + dist, manifest = self.artifact() + with contextlib.redirect_stdout(io.StringIO()): + packages.stage(self.root, dist, "deb", COMMIT) + packages.stage(self.root, dist, "deb", COMMIT) + dest = self.root / packages.destination(manifest["packages"][0], TAG) + dest.write_bytes(b"older published bytes") + with self.assertRaisesRegex(ValueError, "Refusing to overwrite"): + packages.stage(self.root, dist, "deb", COMMIT) + self.assertEqual(dest.read_bytes(), b"older published bytes") + + def test_manifest_cannot_escape_package_directories(self): + _, manifest = self.artifact() + for field in ("name", "distro", "arch", "php", "filename"): + changed = copy.deepcopy(manifest) + changed["packages"][0][field] = "../../escape" + with self.assertRaises(ValueError): + packages.validate_manifest(changed) + + def test_deb_upload_includes_pool_and_index_alias_only(self): + dist, manifest = self.artifact() + with patch.object(packages, "upload_object") as upload: + packages.upload(dist, "deb", "bucket", "endpoint") + keys = {call.args[1] for call in upload.call_args_list} + self.assertEqual(len(keys), 4) + self.assertIn("deb/pools/resolute/v0.50.0/php8.4-relay-0.50.0-1-php8.4-resolute_amd64.deb", keys) + self.assertTrue(all(key.startswith("deb/") for key in keys)) + + def test_remote_upload_uses_conditional_write_and_checks_retry_bytes(self): + dist, manifest = self.artifact() + package = manifest["packages"][0] + remote_bytes = b"package bytes" + + def aws(args, **kwargs): + if "put-object" in args: + self.assertEqual(args[-2:], ["--if-none-match", "*"]) + return subprocess.CompletedProcess(args, 1, "", "PreconditionFailed") + self.assertIn("get-object", args) + Path(args[-1]).write_bytes(remote_bytes) + return subprocess.CompletedProcess(args, 0) + + with patch.object(packages.subprocess, "run", side_effect=aws): + with contextlib.redirect_stdout(io.StringIO()): + packages.upload_object(dist / package["filename"], "key", package["sha256"], "bucket", "endpoint") + remote_bytes = b"different bytes" + with self.assertRaisesRegex(ValueError, "Published object has different bytes"): + packages.upload_object(dist / package["filename"], "key", package["sha256"], "bucket", "endpoint") + + def test_remote_access_error_is_not_treated_as_missing_object(self): + with patch.object(packages.subprocess, "run", return_value=subprocess.CompletedProcess([], 1, "", "AccessDenied")) as aws: + with self.assertRaisesRegex(RuntimeError, "AccessDenied"): + packages.upload_object(Path("unused"), "key", "checksum", "bucket", "endpoint") + self.assertEqual(aws.call_count, 1) + + def test_real_shell_candidates_include_independent_revisions(self): + result = subprocess.run( + ["bash", str(ROOT / "build/fpm.sh"), TAG, "--list"], + env=dict(os.environ, DEB_REVISION="3", RPM_REVISION="4"), + check=True, capture_output=True, text=True, + ) + rows = [line.split("\t") for line in result.stdout.splitlines()] + self.assertEqual({row[0] for row in rows}, {"deb", "rpm"}) + for row in rows: + self.assertEqual(len(row), 6) + revision = "3" if row[0] == "deb" else "4" + self.assertIn(f"-0.50.0-{revision}-php", row[-1]) + + +if __name__ == "__main__": + unittest.main()