diff --git a/.github/actions/rust/action.yaml b/.github/actions/rust/action.yaml index 808f86cbe32..e1765dfce6b 100644 --- a/.github/actions/rust/action.yaml +++ b/.github/actions/rust/action.yaml @@ -39,7 +39,7 @@ runs: echo "TOOLCHAIN_VERSION=$TOOLCHAIN_VERSION" >> $GITHUB_ENV echo "version=$TOOLCHAIN_VERSION" >> $GITHUB_OUTPUT - - uses: dtolnay/rust-toolchain@master + - uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # v1 name: Install Rust toolchain with: toolchain: ${{ steps.rust_toolchain.outputs.version }} diff --git a/.github/actions/sccache/action.yaml b/.github/actions/sccache/action.yaml index 48d9ae670b9..d541b70f3b1 100644 --- a/.github/actions/sccache/action.yaml +++ b/.github/actions/sccache/action.yaml @@ -77,7 +77,7 @@ runs: - name: Install sccache binary if: steps.check.outputs.available == 'true' && inputs.install == 'true' - uses: mozilla-actions/sccache-action@v0.0.6 + uses: mozilla-actions/sccache-action@9e326ebed976843c9932b3aa0e021c6f50310eb4 # v0.0.6 with: version: "v${{ inputs.version }}" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ed7ca77e7c7..edf0a90a603 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,33 +18,45 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true -# Required for NPM publish +# The OIDC token that publishes to npm is granted to the publish job alone. +# Anywhere else it is a standing credential for republishing every package, +# readable by any action or dependency lifecycle script running in that job. permissions: - id-token: write contents: read +# Debian version knobs, read by scripts/deb_version.js in both the version gate +# and the packaging job. They have to stay in lockstep: a value seen by only one +# of them makes the gate validate a version that is not the one that ships. +# Bump the revision to rebuild an already published version; set the epoch only +# to outrank a version whose upstream part carried a git sha. +env: + DASHMATE_DEB_REVISION: "1" + DASHMATE_DEB_EPOCH: "" + jobs: release-npm: - name: Release NPM packages + name: Build NPM packages runs-on: ubuntu-24.04 timeout-minutes: 60 if: github.event_name != 'workflow_dispatch' + permissions: + contents: read steps: - name: Check out repo uses: actions/checkout@v4 - name: Check package version matches tag - uses: geritol/match-tag-to-package-version@0.2.0 + uses: geritol/match-tag-to-package-version@dd6acafe4382a73f4282687d5ee384b238ea1df7 # 0.2.0 env: TAG_PREFIX: v - - uses: softwareforgood/check-artifact-v4-existence@v0 + - uses: softwareforgood/check-artifact-v4-existence@f988c59be23773bdf226dffa5720d85a13501fba # v0.4.3 id: check-artifact with: name: js-build-${{ github.sha }} - name: Login to DockerHub - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -74,7 +86,7 @@ jobs: uses: ./.github/actions/nodejs - name: Install Cargo binstall - uses: cargo-bins/cargo-binstall@v1.3.1 + uses: cargo-bins/cargo-binstall@82e5fc4781666bf57476d025f1b6e7a5719da4c9 # v1.3.1 if: ${{ steps.check-artifact.outputs.exists != 'true' }} - name: Install wasm-bindgen-cli @@ -85,11 +97,22 @@ jobs: run: cargo binstall wasm-pack if: ${{ steps.check-artifact.outputs.exists != 'true' }} + # Binaryen rewrites the wasm that ships inside the published packages, so + # the download is a build input we have to verify. The checksum is pinned + # here instead of read from the release's .sha256 sidecar, which an + # attacker able to replace the archive could replace as well. - name: Install Binaryen + env: + BINARYEN_VERSION: version_121 + BINARYEN_SHA256: c90e0e295e8f8484ba5b47da92f26e5d1d18db6cd2fcc0c5cc265a5a73609f17 run: | - wget https://github.com/WebAssembly/binaryen/releases/download/version_121/binaryen-version_121-x86_64-linux.tar.gz -P /tmp - tar -xzf /tmp/binaryen-version_121-x86_64-linux.tar.gz -C /tmp - sudo cp -r /tmp/binaryen-version_121/* /usr/local/ + set -euo pipefail + archive="/tmp/binaryen-${BINARYEN_VERSION}-x86_64-linux.tar.gz" + wget -q -O "${archive}" \ + "https://github.com/WebAssembly/binaryen/releases/download/${BINARYEN_VERSION}/binaryen-${BINARYEN_VERSION}-x86_64-linux.tar.gz" + echo "${BINARYEN_SHA256} ${archive}" | sha256sum --check --strict - + tar -xzf "${archive}" -C /tmp + sudo cp -r "/tmp/binaryen-${BINARYEN_VERSION}"/* /usr/local/ if: ${{ steps.check-artifact.outputs.exists != 'true' }} - name: Build packages @@ -98,6 +121,74 @@ jobs: CARGO_BUILD_PROFILE: release if: ${{ steps.check-artifact.outputs.exists != 'true' }} + - name: Ignore only already cached artifacts + run: | + find . -name '.gitignore' -exec rm -f {} + + echo ".yarn" >> .gitignore + echo "target" >> .gitignore + echo "node_modules" >> .gitignore + echo ".nyc_output" >> .gitignore + echo ".idea" >> .gitignore + echo ".ultra.cache.json" >> .gitignore + echo "db/*" >> .gitignore + if: ${{ steps.check-artifact.outputs.exists != 'true' }} + + - name: Get modified files + id: diff + run: | + echo "files<> $GITHUB_OUTPUT + git ls-files --others --exclude-standard >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + if: ${{ steps.check-artifact.outputs.exists != 'true' }} + + - name: Upload the archive of built files + uses: actions/upload-artifact@v4 + with: + name: js-build-${{ github.sha }} + path: ${{ steps.diff.outputs.files }} + retention-days: 1 + if-no-files-found: error + include-hidden-files: true + if: ${{ steps.check-artifact.outputs.exists != 'true' }} + + # Publishing is split from the build so the npm OIDC token is confined to one + # job instead of being present for the whole cargo and wasm build. + # + # The confinement is partial, and knowingly so. Setting up Node runs + # `yarn install`, so dependency lifecycle scripts execute here, and any of them + # can read the runner's OIDC request variables and mint a token that publishes + # as this repository. Removing the install is not currently possible: dashmate + # runs `oclif manifest` from `prepack`, and four other workspaces build from + # their publish hooks, so packing needs the dependency graph. Closing this + # properly means packing in the build job and publishing the resulting + # tarballs, which needs a publish path that can be tested first. + publish-npm: + name: Publish NPM packages + runs-on: ubuntu-24.04 + timeout-minutes: 30 + if: github.event_name != 'workflow_dispatch' + needs: + - release-npm + - check-dashmate-deb-version + permissions: + id-token: write # npm trusted publishing + contents: read + steps: + - name: Check out repo + uses: actions/checkout@v4 + + - name: Download JS build artifacts + uses: actions/download-artifact@v4 + with: + name: js-build-${{ github.sha }} + path: packages + + # Composite action defaults to Node 24+ which ships npm 11.5.1+; + # required for trusted-publishers OIDC at the publish step below. + # See https://docs.npmjs.com/trusted-publishers. + - name: Setup Node.JS + uses: ./.github/actions/nodejs + - name: Set suffix uses: actions/github-script@v6 id: suffix @@ -128,38 +219,59 @@ jobs: echo "NPM suffix: ${{ steps.suffix.outputs.result }}" echo "NPM release tag: ${{ steps.tag.outputs.result }}" + # npm rejects a provenance attestation unless the package declares a public + # `repository` matching, case-sensitively, the repository the build runs + # in, so the flag is applied per workspace rather than to the whole set: a + # manifest that does not qualify publishes exactly as it does today, with a + # warning naming the file to fix, instead of failing the release. + # Attested packages go first: publishing cannot be undone, so the batch + # carrying the newer machinery is the one to fail early. - name: Publish NPM packages - run: yarn workspaces foreach --all --no-private --parallel npm publish --tolerate-republish --access public --tag ${{ steps.tag.outputs.result }} - - - name: Ignore only already cached artifacts - run: | - find . -name '.gitignore' -exec rm -f {} + - echo ".yarn" >> .gitignore - echo "target" >> .gitignore - echo "node_modules" >> .gitignore - echo ".nyc_output" >> .gitignore - echo ".idea" >> .gitignore - echo ".ultra.cache.json" >> .gitignore - echo "db/*" >> .gitignore - if: ${{ steps.check-artifact.outputs.exists != 'true' }} - - - name: Get modified files - id: diff + env: + NPM_RELEASE_TAG: ${{ steps.tag.outputs.result }} run: | - echo "files<> $GITHUB_OUTPUT - git ls-files --others --exclude-standard >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - if: ${{ steps.check-artifact.outputs.exists != 'true' }} - - - name: Upload the archive of built files - uses: actions/upload-artifact@v4 - with: - name: js-build-${{ github.sha }} - path: ${{ steps.diff.outputs.files }} - retention-days: 1 - if-no-files-found: error - include-hidden-files: true - if: ${{ steps.check-artifact.outputs.exists != 'true' }} + set -euo pipefail + if [ -z "${CI:-}" ]; then + echo "Refusing to publish outside CI" + exit 1 + fi + + # Written to a file rather than piped into the loop: a process + # substitution hides its own failure, so an empty or failed listing + # would publish nothing and still report success. + workspaces="${RUNNER_TEMP}/publishable-workspaces" + yarn workspaces list --no-private --json | jq -r '.location' > "${workspaces}" + if [ ! -s "${workspaces}" ]; then + echo "::error::Found no publishable workspaces" + exit 1 + fi + + expected="https://github.com/${GITHUB_REPOSITORY}" + attested=() + plain=() + while IFS= read -r location; do + manifest="${location}/package.json" + name="$(jq -r '.name' "${manifest}")" + url="$(jq -r 'if (.repository | type) == "string" then .repository else .repository.url // "" end' "${manifest}")" + normalized="${url#git+}" + normalized="${normalized%.git}" + if [ "${normalized}" = "${expected}" ]; then + attested+=(--include "${name}") + else + plain+=(--include "${name}") + echo "::warning file=${manifest}::published without provenance: repository must be \"${expected}\", found \"${url:-}\"" + fi + done < "${workspaces}" + + if [ "${#attested[@]}" -gt 0 ]; then + yarn workspaces foreach --all --no-private --parallel "${attested[@]}" \ + npm publish --tolerate-republish --access public --provenance --tag "${NPM_RELEASE_TAG}" + fi + + if [ "${#plain[@]}" -gt 0 ]; then + yarn workspaces foreach --all --no-private --parallel "${plain[@]}" \ + npm publish --tolerate-republish --access public --tag "${NPM_RELEASE_TAG}" + fi release-drive-image: name: Release Drive image @@ -244,13 +356,95 @@ jobs: with: tag: ${{ github.event.release.tag_name }} + check-dashmate-deb-version: + name: Check Dashmate deb version + runs-on: ubuntu-24.04 + timeout-minutes: 10 + if: ${{ !inputs.only_drive }} + permissions: + contents: read + outputs: + validated_version: ${{ steps.check.outputs.validated_version }} + steps: + - name: Check out repo + uses: actions/checkout@v4 + + # apt reads a deb whose version does not sort above the installed one as a + # downgrade and refuses it, leaving operators silently stuck on the older + # release. Catch that before anything is built or published. + - name: Check deb version sorts above the last published release + id: check + env: + GH_TOKEN: ${{ github.token }} + CURRENT_TAG: ${{ inputs.tag || github.event.release.tag_name }} + run: | + set -euo pipefail + compare="${GITHUB_WORKSPACE}/scripts/check_deb_version.sh" + translate="${GITHUB_WORKSPACE}/scripts/deb_version.js" + select_baseline="${GITHUB_WORKSPACE}/scripts/deb_release_baseline.js" + if [ ! -x "${compare}" ] || [ ! -f "${translate}" ] || [ ! -f "${select_baseline}" ]; then + echo "::error::${compare}, ${translate} or ${select_baseline} is missing" + exit 1 + fi + # The comparison runs on Debian versions, never on the semver tags: + # "4.1.0-1" is valid as both and means something different in each. + new_version="$(node "${translate}" "${CURRENT_TAG}")" + # Published so the packaging job can prove the deb it actually builds + # carries the version validated here, rather than both jobs + # independently predicting it from the tag. + echo "validated_version=${new_version}" >> "${GITHUB_OUTPUT}" + + # Every release is fetched, not just the first page: the repository has + # several hundred of them, and a single page silently hides older + # lines. Which one becomes the baseline is decided by the script rather + # than by the API's default ordering, so the choice of predecessor is + # not an undocumented implementation detail. + baseline="$(gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/releases?per_page=100" \ + | node "${select_baseline}" "${CURRENT_TAG}")" + + if [ -z "${baseline}" ]; then + echo "::notice::No published deb to compare ${CURRENT_TAG} against" + exit 0 + fi + + baseline_tag="${baseline%%$'\t'*}" + baseline_asset="${baseline#*$'\t'}" + + # Read the baseline from the deb's own control field: the only version + # apt looks at, the only one a server-side rename of the asset cannot + # alter, and what operators actually installed rather than what the tag + # would produce today. + mkdir -p baseline-deb + gh release download "${baseline_tag}" --repo "${GITHUB_REPOSITORY}" \ + --pattern "${baseline_asset}" --dir baseline-deb --clobber + baseline_version="$(dpkg-deb -f "baseline-deb/${baseline_asset}" Version)" + if [ -z "${baseline_version}" ]; then + echo "::error::Could not read the Version field from ${baseline_asset} in ${baseline_tag}" + exit 1 + fi + + echo "Comparing ${CURRENT_TAG} (${new_version}) against ${baseline_asset} from ${baseline_tag} (${baseline_version})" + status=0 + "${compare}" "${new_version}" "${baseline_version}" || status=$? + if [ "${status}" -eq 1 ]; then + echo "::error::${new_version} does not sort above ${baseline_version}, so apt would refuse this release as a downgrade or report it as already the newest version. Rebuilding an already published version needs DASHMATE_DEB_REVISION; re-releasing a version whose predecessor carried a git sha in its upstream part needs DASHMATE_DEB_EPOCH=1. Both are read by scripts/deb_version.js and must be set for the packaging job as well." + fi + exit "${status}" + + # Holds the Apple signing certificate and the notarization credentials, and + # runs dependency lifecycle scripts while doing so, so it is gated on its own + # environment rather than sharing one with the job that will hold the package + # signing key. On a release the ref is refs/tags/v*, so the protection rule + # has to be a tag rule; a branch-only policy blocks every release. release-dashmate-packages: name: Release Dashmate packages runs-on: ${{ matrix.os }} if: ${{ !inputs.only_drive }} - needs: release-npm + needs: + - publish-npm + - check-dashmate-deb-version + environment: dashmate-release-build permissions: - id-token: write # s3 cache contents: write # update release artifacts strategy: fail-fast: false @@ -281,9 +475,11 @@ jobs: run: | brew install llvm coreutils + # Pinned to a master commit rather than the latest tag: the newest release + # (1.0.12) still declares the removed Node 16 runtime. - name: Set up Docker for macOS if: runner.os == 'macOS' - uses: docker-practice/actions-setup-docker@master + uses: docker-practice/actions-setup-docker@509de4a162d8fd24b2721a9fb3721131a6a4776b # master, 2025-08-24 - name: Install the Apple certificate if: runner.os == 'macOS' @@ -320,6 +516,34 @@ jobs: OSX_KEYCHAIN: ${{ runner.temp }}/app-signing.keychain-db run: "${GITHUB_WORKSPACE}/scripts/pack_dashmate.sh ${{ matrix.package_type }}" + # The gate validates a version derived from the tag; nothing so far proves + # the deb that actually gets built carries it. Without this the gate's + # verdict applies to a prediction rather than to the bytes that ship. + - name: Check the built deb carries the validated version + if: matrix.package_type == 'deb' + env: + VALIDATED_VERSION: ${{ needs.check-dashmate-deb-version.outputs.validated_version }} + run: | + set -euo pipefail + if [ -z "${VALIDATED_VERSION}" ]; then + echo "::error::The version gate did not publish a validated version" + exit 1 + fi + debs="${RUNNER_TEMP}/built-debs" + find packages/dashmate/dist -type f -name '*.deb' > "${debs}" + if [ ! -s "${debs}" ]; then + echo "::error::No deb was produced to check" + exit 1 + fi + while IFS= read -r deb; do + built_version="$(dpkg-deb -f "${deb}" Version)" + if [ "${built_version}" != "${VALIDATED_VERSION}" ]; then + echo "::error::${deb} carries version ${built_version}, but the gate validated ${VALIDATED_VERSION}; the packaging scheme and the gate disagree" + exit 1 + fi + echo "${deb} carries the validated version ${built_version}" + done < "${debs}" + - name: Upload artifacts to action summary uses: actions/upload-artifact@v4 if: github.event_name != 'release' @@ -332,8 +556,200 @@ jobs: run: | find packages/dashmate/dist/ -name '*.pkg' -exec sh -c 'xcrun notarytool submit "{}" --apple-id "${{ secrets.MACOS_APPLE_ID }}" --team-id "${{ secrets.MACOS_TEAM_ID }}" --password "${{ secrets.MACOS_NOTARIZING_PASSWORD }}" --wait;' \; + # Recorded here, after notarization and immediately before upload, so the + # hashes cover the exact bytes that leave this job. The checksums job + # compares them against what the release actually serves, which is what + # makes SHA256SUMS evidence rather than a restatement of whatever is + # attached to the release by the time it runs. + - name: Record built package checksums + if: github.event_name == 'release' + run: | + set -euo pipefail + if command -v sha256sum > /dev/null; then + hash_cmd=(sha256sum) + else + hash_cmd=(shasum -a 256) + fi + cd packages/dashmate/dist + # Hidden files are skipped to match the glob that uploads this + # directory, which does not match them either. Recording a file that + # never gets uploaded would fail the release for a package that was + # never meant to ship. + find . -type f -not -path '*/.*' | sed 's|^\./||' | LC_ALL=C sort > "${RUNNER_TEMP}/built-files" + if [ ! -s "${RUNNER_TEMP}/built-files" ]; then + echo "::error::No built packages found to record" + exit 1 + fi + # Recorded under the name the file will carry on the release rather + # than its path in the build tree: a release has no directories, so + # the upload flattens every file to its basename. A record keyed by + # path could only ever be matched against a published asset by hash, + # which is what lets bytes built for one target be served under + # another target's name. + : > "${RUNNER_TEMP}/built.sha256" + while IFS= read -r file; do + hash="$("${hash_cmd[@]}" "${file}" | cut -d' ' -f1)" + printf '%s %s\n' "${hash}" "${file##*/}" >> "${RUNNER_TEMP}/built.sha256" + done < "${RUNNER_TEMP}/built-files" + cat "${RUNNER_TEMP}/built.sha256" + + - name: Upload built package checksums + uses: actions/upload-artifact@v4 + if: github.event_name == 'release' + with: + name: dashmate-built-checksums-${{ matrix.package_type }} + path: ${{ runner.temp }}/built.sha256 + retention-days: 1 + if-no-files-found: error + - name: Upload artifacts to release - uses: softprops/action-gh-release@v0.1.15 + uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 # v0.1.15 if: github.event_name == 'release' with: files: packages/dashmate/dist/** + + # Publication of the release, kept apart from the jobs that build it: there is + # no checkout and no dependency install, so no repository or package script + # runs beside the signing material this job is going to hold. One first-party + # action still executes here to collect the build checksums. + # + # The environment is where the deployment protection rules live; without them + # this job's secrets would be reachable from any ref someone with write access + # can dispatch the workflow on. It covers this job and the packaging job only + # - the Docker image jobs below receive credentials via `secrets: inherit` and + # are not gated. + release-dashmate-checksums: + name: Release Dashmate checksums + runs-on: ubuntu-24.04 + timeout-minutes: 30 + needs: release-dashmate-packages + if: github.event_name == 'release' + environment: dashmate-release-signing + permissions: + contents: write # update release artifacts + steps: + # Regenerating SHA256SUMS under an existing signature would leave the + # signature attesting a file that no longer exists in that form, which + # reads as valid to anyone who checks it. Removing the stale signature is + # a maintainer decision, so it is deliberately a manual step. + - name: Refuse to regenerate checksums under an existing signature + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ github.event.release.tag_name }} + run: | + set -euo pipefail + signature="$(gh api "repos/${GITHUB_REPOSITORY}/releases/tags/${TAG}" \ + --jq '[.assets[].name | select(. == "SHA256SUMS.asc")] | first // ""')" + if [ -n "${signature}" ]; then + echo "::error::SHA256SUMS.asc is already attached to ${TAG}, and regenerating SHA256SUMS would strand it over a file that no longer exists. Delete it with 'gh release delete-asset ${TAG} SHA256SUMS.asc', re-run, and sign the new file." + exit 1 + fi + + - name: Download built package checksums + uses: actions/download-artifact@v4 + with: + pattern: dashmate-built-checksums-* + path: built-checksums + + # The names and hashes the packaging jobs recorded are the whole + # expectation this job holds: SHA256SUMS covers the packages this workflow + # built, not whatever the release happens to serve. Other jobs attach + # assets of their own, and this job has no build record for those - listing + # them would attest artifacts it knows nothing about, and would depend on + # whether those jobs had finished yet. + - name: Collect the packages built in this run + run: | + set -euo pipefail + cat built-checksums/*/built.sha256 | LC_ALL=C sort -u > "${RUNNER_TEMP}/built.sha256" + if [ ! -s "${RUNNER_TEMP}/built.sha256" ]; then + echo "::error::No built package checksums were recorded" + exit 1 + fi + cut -d' ' -f3- "${RUNNER_TEMP}/built.sha256" | LC_ALL=C sort > "${RUNNER_TEMP}/built-names" + + # A release holds one asset per name, so two built files sharing a name + # means one of them silently replaced the other on the way up and there + # is no longer any way to say which bytes are published under it. + duplicates="$(LC_ALL=C uniq -d "${RUNNER_TEMP}/built-names")" + if [ -n "${duplicates}" ]; then + echo "::error::More than one built file would be published under the same name: $(echo "${duplicates}" | tr '\n' ' ')" + exit 1 + fi + cat "${RUNNER_TEMP}/built.sha256" + + # Fetched by name, one asset at a time, so that a package built here but + # missing from the release is reported as missing rather than quietly + # dropped from the listing. + - name: Download the published packages + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ github.event.release.tag_name }} + run: | + set -euo pipefail + mkdir -p assets + missing=0 + while IFS= read -r name; do + gh release download "${TAG}" --repo "${GITHUB_REPOSITORY}" \ + --pattern "${name}" --dir assets --clobber > /dev/null 2>&1 || true + if [ ! -f "assets/${name}" ]; then + echo "::error::${name} was built in this run but is not published on ${TAG}" + missing=1 + fi + done < "${RUNNER_TEMP}/built-names" + if [ "${missing}" -ne 0 ]; then + echo "::error::The release is missing packages this run built; refusing to publish checksums" + exit 1 + fi + + # A maintainer signs SHA256SUMS out of band and attaches the detached + # signature as SHA256SUMS.asc; apt publication then refuses to publish any + # package whose hash is not in the signed file, so a compromised CI run + # alone cannot ship a package. The listing is generated in byte-wise name + # order so regenerating it reproduces exactly what was signed. + - name: Generate SHA256SUMS + run: | + set -euo pipefail + cd assets + xargs -a "${RUNNER_TEMP}/built-names" -d '\n' sha256sum > SHA256SUMS + cat SHA256SUMS + + # Hashing whatever the release currently serves would attest availability, + # not origin: anyone able to swap an asset between the packaging job and + # this one would simply get the replacement blessed. The comparison is on + # whole (hash, name) pairs and runs in both directions, because each half + # alone leaves a way through: matching hashes only would let bytes built + # for one target be published under another target's name, and matching + # published assets only would let a package be dropped from the release + # entirely without anything noticing. + - name: Verify published assets against the built packages + run: | + set -euo pipefail + LC_ALL=C sort assets/SHA256SUMS > "${RUNNER_TEMP}/published.sha256" + if diff -u "${RUNNER_TEMP}/built.sha256" "${RUNNER_TEMP}/published.sha256" \ + --label built --label published; then + echo "All published packages match the packages built in this run" + exit 0 + fi + echo "::error::Published packages do not match the packages built in this run; refusing to publish checksums" + exit 1 + + # Uploaded with the preinstalled CLI rather than a third-party action: + # code running in this job runs beside the signing material, and an action + # can read the keyring or shim the tools regardless of being pinned. + # The guard at the start of this job cannot cover the whole run: a + # maintainer can attach a signature while the packages are downloaded and + # verified. Re-checking here means the upload never replaces a file a + # signature already covers, leaving that signature over bytes it never saw. + - name: Upload checksums to release + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ github.event.release.tag_name }} + run: | + set -euo pipefail + signature="$(gh api "repos/${GITHUB_REPOSITORY}/releases/tags/${TAG}" \ + --jq '[.assets[].name | select(. == "SHA256SUMS.asc")] | first // ""')" + if [ -n "${signature}" ]; then + echo "::error::SHA256SUMS.asc was attached to ${TAG} while this job was running, so uploading SHA256SUMS now would leave that signature over bytes it did not sign. Delete it with 'gh release delete-asset ${TAG} SHA256SUMS.asc', re-run, and sign the new file." + exit 1 + fi + gh release upload "${TAG}" assets/SHA256SUMS --repo "${GITHUB_REPOSITORY}" --clobber diff --git a/packages/dashmate/docs/installation.md b/packages/dashmate/docs/installation.md index c367b811cb9..497a8a6b19c 100644 --- a/packages/dashmate/docs/installation.md +++ b/packages/dashmate/docs/installation.md @@ -20,22 +20,26 @@ Installing the Linux, MacOS, or Windows packages from the [GitHub releases page] ### Debian package -Download the newest dashmate installation package for your architecture from the [GitHub releases page](https://github.com/dashpay/platform/releases/latest): +Download the newest dashmate installation package for your architecture from the [GitHub releases page](https://github.com/dashpay/platform/releases/latest). +The file name contains the version, so it changes with every release; this downloads the one matching the architecture you are on: ```bash -wget https://github.com/dashpay/platform/releases/download/v1.8.0/dashmate_1.8.0.e4e156c86-1_amd64.deb +DASHMATE_DEB="$(curl -fsSL https://api.github.com/repos/dashpay/platform/releases/latest \ + | grep -o "https://[^\"]*_$(dpkg --print-architecture)\.deb" \ + | head -n 1)" +curl -fLO "$DASHMATE_DEB" ``` -Install dashmate using apt: +Install the package that was just downloaded: ```bash sudo apt update -sudo apt install ./dashmate_1.8.0.e4e156c86-1_amd64.deb +sudo apt install "./$(basename "$DASHMATE_DEB")" ``` > **Note:** At the end of the installation process, apt may display an error due to installing a downloaded package. > You can ignore this error message: -> N: Download is performed unsandboxed as root as file '/home/ubuntu/dashmate_1.8.0.e4e156c86-1_amd64.deb' couldn't be accessed by user '_apt'. - pkgAcquire::Run (13: Permission denied) +> N: Download is performed unsandboxed as root as file '/home/ubuntu/dashmate_4.1.0-1_amd64.deb' couldn't be accessed by user '_apt'. - pkgAcquire::Run (13: Permission denied) ### Node package diff --git a/packages/dashmate/package.json b/packages/dashmate/package.json index 0475b1ccbf7..072c886fa9e 100644 --- a/packages/dashmate/package.json +++ b/packages/dashmate/package.json @@ -16,7 +16,7 @@ }, "repository": { "type": "git", - "url": "git+https://github.com/dashevo/dashmate.git" + "url": "https://github.com/dashpay/platform" }, "type": "module", "bin": "./bin/run.js", diff --git a/packages/dashmate/test/unit/packaging/debReleaseBaseline.spec.js b/packages/dashmate/test/unit/packaging/debReleaseBaseline.spec.js new file mode 100644 index 00000000000..0c7efb43019 --- /dev/null +++ b/packages/dashmate/test/unit/packaging/debReleaseBaseline.spec.js @@ -0,0 +1,168 @@ +import { execFileSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { selectDebBaseline, releaseLine } from '../../../../../scripts/deb_release_baseline.js'; + +const SCRIPT_PATH = fileURLToPath(new URL('../../../../../scripts/deb_release_baseline.js', import.meta.url)); + +/** + * A releases API entry, trimmed to the fields the baseline choice reads. + * + * `createdAt` is never used to order releases and is only set where a test needs the two + * dates to disagree. + */ +function release({ + tag, publishedAt, createdAt = publishedAt, draft = false, assets = [], +}) { + return { + tag_name: tag, + draft, + created_at: createdAt, + published_at: publishedAt, + assets: assets.map((name) => ({ name })), + }; +} + +function deb(version, arch = 'amd64') { + return `dashmate_${version}_${arch}.deb`; +} + +describe('deb_release_baseline.js', () => { + describe('#releaseLine', () => { + it('should group a prerelease with the release it leads up to', () => { + expect(releaseLine('v4.1.0-rc.3')).to.equal('4.1.0'.split('.').slice(0, 2).join('.')); + expect(releaseLine('v4.1.0-rc.3')).to.equal(releaseLine('v4.1.0')); + expect(releaseLine('v4.1.7')).to.equal(releaseLine('v4.1.0')); + }); + + it('should keep separate minor lines apart', () => { + expect(releaseLine('v4.2.0')).to.not.equal(releaseLine('v4.1.0')); + }); + }); + + describe('#selectDebBaseline', () => { + it('should measure a release against the last package offered on its own line', () => { + const releases = [ + release({ tag: 'v4.1.0', publishedAt: '2026-08-01T00:00:00Z', assets: [deb('4.1.0-1')] }), + release({ tag: 'v4.1.1', publishedAt: '2026-08-10T00:00:00Z', assets: [deb('4.1.1-1')] }), + ]; + + expect(selectDebBaseline(releases, 'v4.1.2')).to.deep.equal({ + tag: 'v4.1.1', + asset: deb('4.1.1-1'), + }); + }); + + // An operator on the 4.1 line was never offered 4.2, so measuring a 4.1 hotfix + // against it would demand a version that outranks a release it does not supersede. + it('should not measure a hotfix against a higher line', () => { + const releases = [ + release({ tag: 'v4.1.1', publishedAt: '2026-08-01T00:00:00Z', assets: [deb('4.1.1-1')] }), + release({ tag: 'v4.2.0', publishedAt: '2026-08-20T00:00:00Z', assets: [deb('4.2.0-1')] }), + ]; + + expect(selectDebBaseline(releases, 'v4.1.2').tag).to.equal('v4.1.1'); + }); + + // The first release on a new line still has to outrank whatever apt last installed, + // which is the newest package from the previous line. + it('should fall back to the newest package of any line', () => { + const releases = [ + release({ tag: 'v4.1.0', publishedAt: '2026-08-01T00:00:00Z', assets: [deb('4.1.0-1')] }), + release({ tag: 'v4.1.1', publishedAt: '2026-08-10T00:00:00Z', assets: [deb('4.1.1-1')] }), + ]; + + expect(selectDebBaseline(releases, 'v4.2.0').tag).to.equal('v4.1.1'); + }); + + // The package attached to the current tag is what an earlier run of this same release + // already shipped, and it is exactly what apt measures a rebuild against. Ordering by + // creation instead of publication loses that comparison whenever the release was + // drafted before its own predecessor, and a same-version rebuild then passes the gate + // while apt reports the package as already the newest version. + it('should measure a rerun against the package that release already shipped', () => { + const current = release({ + tag: 'v4.1.1', + createdAt: '2026-08-01T00:00:00Z', + publishedAt: '2026-08-20T00:00:00Z', + assets: [deb('4.1.1-1')], + }); + const predecessor = release({ + tag: 'v4.1.0', + createdAt: '2026-08-05T00:00:00Z', + publishedAt: '2026-08-06T00:00:00Z', + assets: [deb('4.1.0-1')], + }); + + // The two orderings disagree, which is the whole point of the fixture: chosen by + // creation the predecessor wins, chosen by publication the current release does. + expect(current.created_at < predecessor.created_at).to.equal(true); + expect(current.published_at > predecessor.published_at).to.equal(true); + + expect(selectDebBaseline([current, predecessor], 'v4.1.1').asset).to.equal(deb('4.1.1-1')); + }); + + // A draft has never been offered to anyone, so nothing can have installed it. + it('should ignore drafts', () => { + const releases = [ + release({ tag: 'v4.1.0', publishedAt: '2026-08-01T00:00:00Z', assets: [deb('4.1.0-1')] }), + release({ + tag: 'v4.1.9', publishedAt: null, draft: true, assets: [deb('4.1.9-1')], + }), + ]; + + expect(selectDebBaseline(releases, 'v4.1.1').tag).to.equal('v4.1.0'); + }); + + // Prereleases go to the same channel as stable releases, so apt compares against them + // like anything else. + it('should measure against a prerelease', () => { + const releases = [ + release({ tag: 'v4.1.0-rc.3', publishedAt: '2026-08-10T00:00:00Z', assets: [deb('4.1.0.rc.3-1')] }), + release({ tag: 'v4.0.9', publishedAt: '2026-07-01T00:00:00Z', assets: [deb('4.0.9-1')] }), + ]; + + expect(selectDebBaseline(releases, 'v4.1.0').tag).to.equal('v4.1.0-rc.3'); + }); + + it('should skip releases that shipped no package', () => { + const releases = [ + release({ tag: 'v4.1.0', publishedAt: '2026-08-01T00:00:00Z', assets: [deb('4.1.0-1')] }), + release({ tag: 'v4.1.1', publishedAt: '2026-08-10T00:00:00Z', assets: ['dashmate-v4.1.1-x64.pkg'] }), + ]; + + expect(selectDebBaseline(releases, 'v4.1.2').tag).to.equal('v4.1.0'); + }); + + it('should have nothing to compare against when no release shipped a package', () => { + expect(selectDebBaseline([], 'v4.1.0')).to.equal(null); + expect(selectDebBaseline([ + release({ tag: 'v4.1.0', publishedAt: '2026-08-01T00:00:00Z' }), + ], 'v4.1.1')).to.equal(null); + }); + }); + + describe('command line', () => { + function run(currentTag, releases) { + return execFileSync(process.execPath, [SCRIPT_PATH, currentTag], { + encoding: 'utf8', + input: JSON.stringify(releases), + }); + } + + // The workflow pipes `gh api --paginate --slurp`, which emits one array per page. + it('should read a paginated response and print the tag and package name', () => { + const output = run('v4.1.2', [ + [release({ tag: 'v4.1.0', publishedAt: '2026-08-01T00:00:00Z', assets: [deb('4.1.0-1')] })], + [release({ tag: 'v4.1.1', publishedAt: '2026-08-10T00:00:00Z', assets: [deb('4.1.1-1')] })], + ]); + + expect(output).to.equal(`v4.1.1\t${deb('4.1.1-1')}\n`); + }); + + // The workflow reads an empty result as "nothing to compare against" and stops there, + // so anything printed on the happy path would be taken for a package name. + it('should print nothing when no release shipped a package', () => { + expect(run('v4.1.0', [[]])).to.equal(''); + }); + }); +}); diff --git a/packages/dashmate/test/unit/packaging/debVersion.spec.js b/packages/dashmate/test/unit/packaging/debVersion.spec.js new file mode 100644 index 00000000000..34141a4e8c9 --- /dev/null +++ b/packages/dashmate/test/unit/packaging/debVersion.spec.js @@ -0,0 +1,301 @@ +import { execFileSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { debVersionFromSemver, debFileNameVersion } from '../../../../../scripts/deb_version.js'; + +const SCRIPT_PATH = fileURLToPath(new URL('../../../../../scripts/deb_version.js', import.meta.url)); + +/** + * Independent port of dpkg's version comparison (`verrevcmp` in dpkg's version.c), + * used as the oracle here so the expectations below describe what apt actually does + * rather than whatever the mapping under test happens to produce. + */ +function order(char) { + if (char === undefined) { + return 0; + } + + if (char >= '0' && char <= '9') { + return 0; + } + + if ((char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z')) { + return char.charCodeAt(0); + } + + if (char === '~') { + return -1; + } + + return char.charCodeAt(0) + 256; +} + +function isDigit(char) { + return char !== undefined && char >= '0' && char <= '9'; +} + +function verrevcmp(left, right) { + let i = 0; + let j = 0; + + while (i < left.length || j < right.length) { + let firstDiff = 0; + + while ((i < left.length && !isDigit(left[i])) || (j < right.length && !isDigit(right[j]))) { + const leftOrder = order(left[i]); + const rightOrder = order(right[j]); + + if (leftOrder !== rightOrder) { + return leftOrder - rightOrder; + } + + i += 1; + j += 1; + } + + while (left[i] === '0') { + i += 1; + } + + while (right[j] === '0') { + j += 1; + } + + while (isDigit(left[i]) && isDigit(right[j])) { + if (firstDiff === 0) { + firstDiff = left.charCodeAt(i) - right.charCodeAt(j); + } + + i += 1; + j += 1; + } + + if (isDigit(left[i])) { + return 1; + } + + if (isDigit(right[j])) { + return -1; + } + + if (firstDiff !== 0) { + return firstDiff; + } + } + + return 0; +} + +function parseDebVersion(version) { + const [, epoch = '0', rest] = /^(?:(\d+):)?(.*)$/.exec(version); + const revisionAt = rest.lastIndexOf('-'); + + return { + epoch: Number(epoch), + revision: revisionAt === -1 ? '' : rest.slice(revisionAt + 1), + upstream: revisionAt === -1 ? rest : rest.slice(0, revisionAt), + }; +} + +/** + * @returns {number} negative when left sorts below right, 0 when equal, positive above + */ +function compareDebVersions(left, right) { + const a = parseDebVersion(left); + const b = parseDebVersion(right); + + if (a.epoch !== b.epoch) { + return a.epoch - b.epoch; + } + + const upstream = verrevcmp(a.upstream, b.upstream); + + return upstream === 0 ? verrevcmp(a.revision, b.revision) : upstream; +} + +describe('deb_version.js', () => { + describe('version comparison oracle', () => { + it('should order versions the way dpkg does', () => { + expect(compareDebVersions('4.1.0', '4.1.0')).to.equal(0); + expect(compareDebVersions('4.1.0', '4.2.0')).to.be.below(0); + expect(compareDebVersions('4.1.0', '4.1.10')).to.be.below(0); + // `~` sorts below everything, including the end of the string + expect(compareDebVersions('4.1.0~rc.1', '4.1.0')).to.be.below(0); + // letters sort above digits, which is what makes a git sha unusable as a version part + expect(compareDebVersions('4.1.0.a', '4.1.0.9')).to.be.above(0); + // the Debian revision breaks ties on an identical upstream version + expect(compareDebVersions('4.1.0-2', '4.1.0-1')).to.be.above(0); + // an epoch outranks any upstream version + expect(compareDebVersions('1:1.0.0', '99.0.0')).to.be.above(0); + }); + }); + + describe('versions produced by the legacy `.-1` scheme', () => { + // Exactly what was published for the 4.1.0 series. + const published = { + 'v4.1.0': '4.1.0.bfc80249b9-1', + 'v4.1.0-beta.2': '4.1.0.ae554fdd83-1', + 'v4.1.0-rc.1': '4.1.0.08152ea51e-1', + 'v4.1.0-rc.2': '4.1.0.3de436123d-1', + 'v4.1.0-rc.3': '4.1.0.61be67f7bf-1', + }; + + it('should make half of the real 4.1.0 releases look like downgrades to apt', () => { + expect(compareDebVersions(published['v4.1.0-rc.1'], published['v4.1.0-beta.2'])).to.be.below(0); + expect(compareDebVersions(published['v4.1.0-rc.2'], published['v4.1.0-rc.1'])).to.be.below(0); + // and let the other half through, so the ordering is effectively arbitrary + expect(compareDebVersions(published['v4.1.0-rc.3'], published['v4.1.0-rc.2'])).to.be.above(0); + expect(compareDebVersions(published['v4.1.0'], published['v4.1.0-rc.3'])).to.be.above(0); + }); + + it('should never ship a rebuild of an already published version', () => { + const hotfix = '4.1.0.284f02fabb-1'; + + expect(compareDebVersions(hotfix, published['v4.1.0'])).to.be.below(0); + }); + }); + + describe('#debVersionFromSemver', () => { + it('should give a stable release the first Debian revision', () => { + expect(debVersionFromSemver('4.1.0')).to.equal('4.1.0-1'); + expect(debVersionFromSemver('v4.1.0')).to.equal('4.1.0-1'); + }); + + it('should keep a prerelease below its final release', () => { + expect(debVersionFromSemver('4.1.0-rc.3')).to.equal('4.1.0~rc.3-1'); + + expect(compareDebVersions( + debVersionFromSemver('4.1.0-rc.3'), + debVersionFromSemver('4.1.0'), + )).to.be.below(0); + }); + + it('should order every release of the 4.1.0 series upward', () => { + const releases = ['4.1.0-beta.2', '4.1.0-rc.1', '4.1.0-rc.2', '4.1.0-rc.3', '4.1.0', '4.1.1']; + + releases.slice(1).forEach((release, index) => { + const previous = debVersionFromSemver(releases[index]); + const next = debVersionFromSemver(release); + + expect(compareDebVersions(next, previous)).to.be.above( + 0, + `${next} must sort above ${previous}`, + ); + }); + }); + + // Inherent to any correct scheme, semver included, and the reason prereleases cannot + // share a suite with stable: apt offers whatever sorts highest, so a stable node would + // be walked onto a release candidate. + it('should sort a prerelease of the next release above the current stable one', () => { + expect(compareDebVersions( + debVersionFromSemver('4.2.0-rc.1'), + debVersionFromSemver('4.1.0'), + )).to.be.above(0); + }); + + it('should let a rebuild of the same version overtake the published one', () => { + expect(debVersionFromSemver('4.1.0', { revision: '2' })).to.equal('4.1.0-2'); + + expect(compareDebVersions( + debVersionFromSemver('4.1.0', { revision: '2' }), + debVersionFromSemver('4.1.0'), + )).to.be.above(0); + }); + + it('should overtake a legacy git sha version with an epoch', () => { + const legacy = '4.1.0.bfc80249b9-1'; + + // Without an epoch the legacy version wins, because `.bfc80249b9` extends `4.1.0` + expect(compareDebVersions(debVersionFromSemver('4.1.0'), legacy)).to.be.below(0); + + expect(debVersionFromSemver('4.1.0', { epoch: '1' })).to.equal('1:4.1.0-1'); + expect(compareDebVersions(debVersionFromSemver('4.1.0', { epoch: '1' }), legacy)).to.be.above(0); + }); + + it('should treat an empty epoch as no epoch at all', () => { + expect(debVersionFromSemver('4.1.0', { epoch: '' })).to.equal('4.1.0-1'); + }); + + it('should keep build metadata out of the upstream version', () => { + // Semver gives build metadata no weight when ordering, so putting it in the upstream + // version would let `4.1.0+build.5` outrank the plain `4.1.0` release + expect(debVersionFromSemver('4.1.0+build.5')).to.equal('4.1.0-1+build.5'); + + const repackaged = debVersionFromSemver('4.1.0+build.5'); + + // it is still a distinct build, so it has to be installable over the plain one + expect(compareDebVersions(repackaged, debVersionFromSemver('4.1.0'))).to.be.above(0); + // but it must not overtake a revision bump or the next release + expect(compareDebVersions(repackaged, debVersionFromSemver('4.1.0', { revision: '2' }))).to.be.below(0); + expect(compareDebVersions(repackaged, debVersionFromSemver('4.1.1'))).to.be.below(0); + }); + + it('should refuse anything that is not a version it can translate', () => { + expect(() => debVersionFromSemver('4.1')).to.throw('not a version'); + expect(() => debVersionFromSemver('4.1.0-rc.3; rm -rf /')).to.throw('not a version'); + // a `-` inside the prerelease would move where dpkg splits off the Debian revision + expect(() => debVersionFromSemver('4.1.0-rc-3')).to.throw('not a version'); + expect(() => debVersionFromSemver('4.1.0', { revision: '1; id' })).to.throw('revision'); + expect(() => debVersionFromSemver('4.1.0', { epoch: 'x' })).to.throw('epoch'); + }); + + it('should refuse leading zeroes, which dpkg would read as the same version', () => { + expect(compareDebVersions('4.1.0~rc.01-1', '4.1.0~rc.1-1')).to.equal(0); + expect(compareDebVersions('04.1.0-1', '4.1.0-1')).to.equal(0); + + expect(() => debVersionFromSemver('4.1.0-rc.01')).to.throw('not a version'); + expect(() => debVersionFromSemver('04.1.0')).to.throw('not a version'); + expect(() => debVersionFromSemver('4.01.0')).to.throw('not a version'); + }); + }); + + describe('#debFileNameVersion', () => { + it('should leave a version that is already safe to publish alone', () => { + expect(debFileNameVersion('4.1.0-1')).to.equal('4.1.0-1'); + }); + + // GitHub rewrites characters like `~` when a release asset is uploaded, which would + // leave the published file name disagreeing with the index that points at it. + it('should drop the tilde a prerelease is published with', () => { + expect(debFileNameVersion('4.1.0~rc.3-1')).to.equal('4.1.0.rc.3-1'); + expect(debFileNameVersion(debVersionFromSemver('4.1.0-beta.2'))).to.equal('4.1.0.beta.2-1'); + }); + + // Debian leaves the epoch out of file names, and `:` would not survive publishing either + it('should drop the epoch', () => { + expect(debFileNameVersion('1:4.1.0-1')).to.equal('4.1.0-1'); + expect(debFileNameVersion(debVersionFromSemver('4.1.0-rc.3', { epoch: '2' }))).to.equal('4.1.0.rc.3-1'); + }); + + it('should not change the version apt installs', () => { + // the file name is cosmetic; ordering comes from the control field, which keeps `~` + expect(debVersionFromSemver('4.1.0-rc.3')).to.equal('4.1.0~rc.3-1'); + expect(compareDebVersions( + debVersionFromSemver('4.1.0-rc.3'), + debVersionFromSemver('4.1.0'), + )).to.be.below(0); + }); + }); + + describe('command line', () => { + function run(version, env) { + return execFileSync(process.execPath, [SCRIPT_PATH, version], { + encoding: 'utf8', + env: { ...process.env, ...env }, + }).trim(); + } + + it('should take the revision and the epoch from the environment', () => { + expect(run('v4.1.0', { DASHMATE_DEB_REVISION: '2' })).to.equal('4.1.0-2'); + expect(run('v4.1.0', { DASHMATE_DEB_EPOCH: '1' })).to.equal('1:4.1.0-1'); + }); + + // The release workflow declares both variables in one place so that the version gate + // and the packaging job cannot read different values. That exports the epoch as set + // but empty, which has to mean the same thing as not setting it at all. + it('should ignore an epoch that is set to an empty value', () => { + expect(run('v4.1.0', { DASHMATE_DEB_REVISION: '1', DASHMATE_DEB_EPOCH: '' })).to.equal('4.1.0-1'); + expect(run('v4.1.0-rc.3', { DASHMATE_DEB_REVISION: '1', DASHMATE_DEB_EPOCH: '' })).to.equal('4.1.0~rc.3-1'); + }); + }); +}); diff --git a/scripts/check_deb_version.sh b/scripts/check_deb_version.sh new file mode 100755 index 00000000000..c878cc3f9ca --- /dev/null +++ b/scripts/check_deb_version.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +set -e + +cmd_usage="Usage: check_deb_version.sh NEW_VERSION PREVIOUS_VERSION + +Exits successfully only when NEW_VERSION sorts strictly above PREVIOUS_VERSION under +dpkg's version comparison, which is what decides whether apt offers a release as an +upgrade at all. + +Both arguments are Debian package versions ([EPOCH:]UPSTREAM[-REVISION]), not semver +tags. Translate a semver version first: + + scripts/check_deb_version.sh \\ + \"\$(node scripts/deb_version.js 4.1.0-rc.4)\" \\ + \"\$(node scripts/deb_version.js 4.1.0-rc.3)\" + + EXIT CODES: + 0 new version sorts above the previous one + 1 new version is equal to or below the previous one + 2 wrong arguments + 3 dpkg is unavailable, so the comparison could not be made +" + +NEW_VERSION="$1" +PREVIOUS_VERSION="$2" + +if [ -z "$NEW_VERSION" ] || [ -z "$PREVIOUS_VERSION" ] +then + echo "$cmd_usage" >&2 + exit 2 +fi + +# dpkg is the only authority on its own ordering rules, and the rules are subtle enough +# (`~` below the empty string, digits and letters ordered differently) that guessing here +# would defeat the point of the check. Refuse to answer instead of answering wrongly. +if ! command -v dpkg > /dev/null 2>&1 +then + echo "check_deb_version.sh: dpkg not found, cannot compare Debian versions." >&2 + echo "Run this on a Debian based host or inside a container that has dpkg." >&2 + exit 3 +fi + +if dpkg --compare-versions "$NEW_VERSION" gt "$PREVIOUS_VERSION" +then + echo "$NEW_VERSION sorts above $PREVIOUS_VERSION" + exit 0 +fi + +if dpkg --compare-versions "$NEW_VERSION" eq "$PREVIOUS_VERSION" +then + echo "$NEW_VERSION is the version that is already published." >&2 + echo "Set DASHMATE_DEB_REVISION to the next Debian revision to rebuild it." >&2 + exit 1 +fi + +echo "$NEW_VERSION sorts below $PREVIOUS_VERSION, so apt would refuse it as a downgrade." >&2 +echo "Set DASHMATE_DEB_EPOCH to overtake a version published under a different scheme." >&2 +exit 1 diff --git a/scripts/deb_release_baseline.js b/scripts/deb_release_baseline.js new file mode 100644 index 00000000000..2af323bdb27 --- /dev/null +++ b/scripts/deb_release_baseline.js @@ -0,0 +1,100 @@ +/** + * Choose the published release whose Debian package a new release has to sort above. + * + * apt refuses a package whose version does not sort above the one already installed, so + * the release to measure against is the last one an operator was offered on the same + * release line - or, when the line has not shipped a package yet, the last one offered + * at all. The release being built is deliberately a candidate: a package already + * attached to that tag is what an earlier run shipped, and it is exactly what apt + * compares a rebuild against. + * + * Only the choice of release is made here. The version itself is always read from the + * chosen package's own control field, never derived from the tag, because that is the + * only version apt looks at and the only one a rename of the asset cannot alter. + */ + +// Releases are ordered by when they were published rather than when they were created. +// A release drafted early and published late reaches operators after releases created +// after it, and it is the order the packages were offered in that decides what apt has +// already installed. Ordering by creation lets a rerun of such a release measure itself +// against its own predecessor instead of against the package it already shipped, which +// passes a same-version rebuild that apt will then report as already the newest version. +const PUBLISHED_AT = 'published_at'; + +// Only the architecture-independent naming matters here: any published package can be +// read for its version, and the amd64 one is present in every release that shipped debs. +const BASELINE_ASSET_SUFFIX = '_amd64.deb'; + +/** + * The release line a tag belongs to, as `major.minor`. + * + * Used only to group releases, so that a hotfix on an older line is measured against its + * own predecessor instead of a higher line it was never meant to supersede. + * + * @param {string} tag + * @returns {string} + */ +function releaseLine(tag) { + return String(tag).replace(/^v/, '').split('-')[0].split('.').slice(0, 2).join('.'); +} + +/** + * @param {object[]} releases - releases as returned by the GitHub releases API + * @param {string} currentTag - the tag being released + * @returns {{tag: string, asset: string}|null} the release to compare against, if any + */ +function selectDebBaseline(releases, currentTag) { + const candidates = releases + // Drafts have never been offered to anyone and carry no publication date to sort by. + // Prereleases are kept: they go to the same channel, and apt compares against + // whatever the operator installed last regardless of how it was labelled. + .filter((release) => release.draft === false && release[PUBLISHED_AT] != null) + .sort((left, right) => String(right[PUBLISHED_AT]).localeCompare(String(left[PUBLISHED_AT]))) + .map((release) => ({ + tag: release.tag_name, + asset: (release.assets || []) + .map((asset) => asset.name) + .find((name) => String(name).endsWith(BASELINE_ASSET_SUFFIX)), + })) + .filter((candidate) => candidate.asset !== undefined); + + const line = releaseLine(currentTag); + + return candidates.find((candidate) => releaseLine(candidate.tag) === line) + || candidates[0] + || null; +} + +module.exports.selectDebBaseline = selectDebBaseline; +module.exports.releaseLine = releaseLine; + +if (require.main === module) { + const currentTag = process.argv[2]; + + if (!currentTag) { + console.error('Usage: deb_release_baseline.js CURRENT_TAG < releases.json\n\n' + + ' Reads the GitHub releases API response on stdin and prints the tag and the\n' + + ' package file name of the release the current tag has to sort above, separated\n' + + ' by a tab. Prints nothing when no published release has shipped a package.\n'); + + process.exit(1); + } + + const input = require('node:fs').readFileSync(0, 'utf8'); + const parsed = JSON.parse(input); + + if (!Array.isArray(parsed)) { + console.error('Expected the releases API response to be an array'); + + process.exit(1); + } + + // A paginated response arrives as one array per page. + const releases = parsed.flatMap((page) => (Array.isArray(page) ? page : [page])); + + const baseline = selectDebBaseline(releases, currentTag); + + if (baseline !== null) { + process.stdout.write(`${baseline.tag}\t${baseline.asset}\n`); + } +} diff --git a/scripts/deb_version.js b/scripts/deb_version.js new file mode 100644 index 00000000000..4b7c56d0e5e --- /dev/null +++ b/scripts/deb_version.js @@ -0,0 +1,128 @@ +/** + * Translate a semver version into a Debian package version, and into the file name that + * version is published under. + * + * oclif builds the deb version as `.-1`. That drops the + * semver prerelease tag and turns the git sha into an ordering component: dpkg orders + * digits as equal and letters by their character code, so a sha starting with a digit + * sorts below one starting with a letter. Roughly half of the releases published that + * way look like downgrades to apt, and a rebuild of an already published version can + * never be installed at all. + * + * Debian's own idiom is used instead. `~` sorts below everything, including the end of + * the string, so a prerelease stays below its final release, and rebuilds of the same + * upstream version are distinguished by the Debian revision: + * + * 4.1.0 -> 4.1.0-1 + * 4.1.0-rc.3 -> 4.1.0~rc.3-1 + * 4.1.0, second build -> 4.1.0-2 + * + * The git sha is not part of the version; it is carried in the package description. + * + * An epoch is available for the one case the scheme cannot express: an upstream version + * that was already published under the git sha scheme, where `4.1.0-1` sorts below the + * published `4.1.0.bfc80249b9-1` because the sha extends the upstream version. + */ + +// Version identifiers follow semver: no leading zeros, because `4.1.0-rc.01` and +// `4.1.0-rc.1` are two distinct tags that dpkg considers the same version. +const NUMERIC_IDENTIFIER = '(?:0|[1-9]\\d*)'; +// The prerelease is deliberately restricted to alphanumerics and dots. A `-` there would +// end up in the Debian upstream version and move where dpkg splits off the revision. +const PRERELEASE_IDENTIFIER = '(?:0|[1-9]\\d*|\\d*[A-Za-z][0-9A-Za-z]*)'; +const BUILD_IDENTIFIER = '[0-9A-Za-z]+'; + +const VERSION_REGEX = new RegExp(`^v?(${NUMERIC_IDENTIFIER}\\.${NUMERIC_IDENTIFIER}\\.${NUMERIC_IDENTIFIER})` + + `(?:-(${PRERELEASE_IDENTIFIER}(?:\\.${PRERELEASE_IDENTIFIER})*))?` + + `(?:\\+(${BUILD_IDENTIFIER}(?:\\.${BUILD_IDENTIFIER})*))?$`); + +// Debian revisions are alphanumerics plus `+ . ~`; starting with a digit keeps them sortable. +const REVISION_REGEX = /^\d[0-9A-Za-z.+~]*$/; +const EPOCH_REGEX = /^\d+$/; + +/** + * @param {string} version - semver version, with or without a leading `v` + * @param {object} [options] + * @param {string} [options.revision] - Debian revision, bumped for rebuilds of one version + * @param {string} [options.epoch] - Debian epoch, omitted when empty + * @returns {string} + */ +function debVersionFromSemver(version, options = {}) { + const { revision = '1', epoch = '' } = options; + + const parsed = VERSION_REGEX.exec(String(version).trim()); + + if (parsed === null) { + throw new Error(`"${version}" is not a version that can be translated to a Debian version.` + + ' Expected MAJOR.MINOR.PATCH with an optional alphanumeric prerelease, for example' + + ' 4.1.0 or 4.1.0-rc.3'); + } + + if (!REVISION_REGEX.test(String(revision))) { + throw new Error(`"${revision}" is not a valid Debian revision. Expected a number, optionally` + + ' followed by alphanumerics, dots, pluses or tildes'); + } + + if (epoch !== '' && !EPOCH_REGEX.test(String(epoch))) { + throw new Error(`"${epoch}" is not a valid Debian epoch. Expected a number`); + } + + const [, release, prerelease, build] = parsed; + + const upstream = release + (prerelease === undefined ? '' : `~${prerelease}`); + + // Semver gives build metadata no weight when ordering versions, so it must not reach the + // upstream version, where dpkg would sort `4.1.0+build.5` above plain `4.1.0`. It marks a + // repackaging of one upstream release, which is what the Debian revision is for. + const debianRevision = build === undefined ? revision : `${revision}+${build}`; + + return `${epoch === '' ? '' : `${epoch}:`}${upstream}-${debianRevision}`; +} + +/** + * The version as it appears in the package file name. + * + * Debian leaves the epoch out of file names. `~` is dropped as well, because GitHub + * rewrites characters like it when a release asset is uploaded, and the renamed asset + * would no longer match the `Filename:` field of the apt index that points at it. Only + * the name changes; the version apt installs comes from the control file. + * + * @param {string} debVersion - as returned by debVersionFromSemver + * @returns {string} + */ +function debFileNameVersion(debVersion) { + return debVersion.replace(/^\d+:/, '').replace(/~/g, '.'); +} + +module.exports.debVersionFromSemver = debVersionFromSemver; +module.exports.debFileNameVersion = debFileNameVersion; + +if (require.main === module) { + const args = process.argv.slice(2); + const forFileName = args[0] === '--file-name'; + const version = forFileName ? args[1] : args[0]; + + if (!version) { + console.error('Usage: deb_version.js [--file-name] SEMVER_VERSION\n\n' + + ' Prints the Debian package version for a semver version, or the version as it\n' + + ' appears in the package file name.\n\n' + + ' DASHMATE_DEB_REVISION Debian revision, default 1. Bump it to rebuild a version\n' + + ' that was already published.\n' + + ' DASHMATE_DEB_EPOCH Debian epoch, unset by default.\n'); + + process.exit(1); + } + + try { + const debVersion = debVersionFromSemver(version, { + revision: process.env.DASHMATE_DEB_REVISION || '1', + epoch: process.env.DASHMATE_DEB_EPOCH || '', + }); + + console.log(forFileName ? debFileNameVersion(debVersion) : debVersion); + } catch (e) { + console.error(e.message); + + process.exit(1); + } +} diff --git a/scripts/pack_dashmate.sh b/scripts/pack_dashmate.sh index 063239b7b1d..9387b5aafcf 100755 --- a/scripts/pack_dashmate.sh +++ b/scripts/pack_dashmate.sh @@ -39,6 +39,136 @@ FULL_PATH=$(realpath "$0") DIR_PATH=$(dirname "$FULL_PATH") ROOT_PATH=$(dirname "$DIR_PATH") +# oclif hardcodes the deb version as `.-1`. That drops the +# semver prerelease tag and makes the git sha an ordering component: dpkg orders digits +# as equal and letters by character code, so a sha starting with a digit sorts below one +# starting with a letter. Apt then reads about half of the releases as downgrades, and a +# rebuild of an already published version is never offered at all. +# +# Rebuild the packages with the Debian idiom instead (4.1.0-1, 4.1.0~rc.3-1, rebuilds +# bumping the Debian revision through DASHMATE_DEB_REVISION) and carry the git sha in the +# package description, where it has no ordering weight. The packages go back through +# dpkg-deb rather than being patched in place so they keep whatever archive format and +# compression the system dpkg produces. Nothing else in the control file has to be kept in +# step with the payload, because oclif's template declares neither md5sums nor +# Installed-Size. +# +# The file name is not the version: it carries no epoch, and no `~`, so that the name +# survives being uploaded as a release asset and still matches the index that points at it. +rewrite_deb_versions() { + DIST_PATH="$1" + STAGING_PATH="tmp/rewritten" + + if ! ls "$DIST_PATH"/*.deb > /dev/null 2>&1 + then + echo "No deb packages to rewrite in $DIST_PATH" + exit 1 + fi + + SEMVER_VERSION=$(node -p "require('./package.json').version") + DEB_VERSION=$(node "$DIR_PATH/deb_version.js" "$SEMVER_VERSION") + DEB_FILE_VERSION=$(node "$DIR_PATH/deb_version.js" --file-name "$SEMVER_VERSION") + + rm -rf "$STAGING_PATH" + mkdir -p "$STAGING_PATH" + + # Every package is rebuilt into a staging directory and only swapped in once they have + # all succeeded, so a failure part way through cannot leave the indexes describing a + # mix of old and new file names. + for DEB_PATH in "$DIST_PATH"/*.deb + do + PACKAGE=$(dpkg-deb --field "$DEB_PATH" Package) + ARCH=$(dpkg-deb --field "$DEB_PATH" Architecture) + OCLIF_VERSION=$(dpkg-deb --field "$DEB_PATH" Version) + + GIT_SHA=${OCLIF_VERSION##*.} + GIT_SHA=${GIT_SHA%%-*} + + if ! echo "$GIT_SHA" | grep -Eq '^[0-9a-f]{7,40}$' + then + echo "Expected a git sha in the version $OCLIF_VERSION built by oclif" + exit 1 + fi + + WORKSPACE_PATH="tmp/repack/$ARCH" + rm -rf "$WORKSPACE_PATH" + # dpkg-deb creates the last path component only. + mkdir -p "$(dirname "$WORKSPACE_PATH")" + dpkg-deb --raw-extract "$DEB_PATH" "$WORKSPACE_PATH" + + # The sha is appended to the end of the description, after any lines continuing it. + # It is the only record of which commit the package was built from now that it is out + # of the version, so a control file without a description to attach it to is an error + # rather than something to skip quietly. + if ! awk -v version="$DEB_VERSION" -v sha="$GIT_SHA" ' + /^Version: / { print "Version: " version; next } + /^Description: / { found = 1; in_description = 1; print; next } + in_description && /^[ \t]/ { print; next } + in_description { print " Built from git commit " sha "."; in_description = 0 } + { print } + END { + if (in_description) print " Built from git commit " sha "." + if (!found) exit 1 + } + ' "$WORKSPACE_PATH/DEBIAN/control" > "$WORKSPACE_PATH/DEBIAN/control.rewritten" + then + echo "No Description field in the control file of $DEB_PATH to record the git sha in" + exit 1 + fi + + mv "$WORKSPACE_PATH/DEBIAN/control.rewritten" "$WORKSPACE_PATH/DEBIAN/control" + + # oclif builds the payload as root, so restore that after extracting and rebuilding + # it as the current user. + dpkg-deb --root-owner-group --build "$WORKSPACE_PATH" "$STAGING_PATH/${PACKAGE}_${DEB_FILE_VERSION}_${ARCH}.deb" + + rm -rf "$WORKSPACE_PATH" + done + + rm -f "$DIST_PATH"/*.deb + mv "$STAGING_PATH"/*.deb "$DIST_PATH" + rm -rf "$STAGING_PATH" + + # Apt takes the version and the file name from these indexes, so they have to be built + # again around the renamed packages. + ORIGIN=$(sed -n 's/^Origin: //p' "$DIST_PATH/Release") + SUITE=$(sed -n 's/^Suite: //p' "$DIST_PATH/Release") + + if [ -z "$ORIGIN" ] || [ -z "$SUITE" ] + then + echo "Could not read Origin and Suite from $DIST_PATH/Release" + exit 1 + fi + + # Kept beside the build rather than in a temporary directory so it goes away with the + # rest of the build even if the script stops early, and out of the indexed directory so + # apt-ftparchive does not list it. + FTPARCHIVE_CONF="$PWD/tmp/apt-ftparchive.conf" + + printf 'APT::FTPArchive::Release {\n Origin "%s";\n Suite "%s";\n};\n' "$ORIGIN" "$SUITE" > "$FTPARCHIVE_CONF" + + ( + cd "$DIST_PATH" || exit 1 + rm -f Packages Packages.gz Packages.bz2 Packages.xz Release InRelease Release.gpg + apt-ftparchive packages . > Packages + gzip -c Packages > Packages.gz + bzip2 -c Packages > Packages.bz2 + xz -c Packages > Packages.xz + apt-ftparchive -c "$FTPARCHIVE_CONF" release . > Release + + # The signatures oclif made cover the metadata from before the rewrite. + if [ -n "${DASHMATE_DEB_KEY:-}" ] + then + gpg --digest-algo SHA512 --clearsign -u "$DASHMATE_DEB_KEY" -o InRelease Release + gpg --digest-algo SHA512 -abs -u "$DASHMATE_DEB_KEY" -o Release.gpg Release + fi + ) + + rm -f "$FTPARCHIVE_CONF" + + echo "Rewrote deb packages as $DEB_VERSION, published as ${DEB_FILE_VERSION}" +} + cd $ROOT_PATH/packages/dashmate || exit 1 yarn pack --install-if-needed tar zxvf package.tgz -C . @@ -49,10 +179,14 @@ echo "nodeLinker: node-modules" > .yarnrc.yml yarn install --no-immutable yarn oclif manifest yarn oclif pack $COMMAND $FLAGS + +if [ "$COMMAND" = "deb" ] +then + rewrite_deb_versions "dist/deb" +fi + cd .. || exit 1 rm package.tgz -rm -rf package/dist/Release -rm -rf package/dist/Packages cp -R package/dist "$ROOT_PATH/packages/dashmate" # fix for deb package build