From eeed3d5b8ceb08cc20c83c3d0654808656efbf5a Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Tue, 3 Nov 2026 16:47:26 -0600 Subject: [PATCH 1/9] Add pre-publication installed-native acceptance and checksum-bound publish Security's #170 gate order requires 4.11.2 to pass same-installed-worker lifecycle acceptance against published native Server 3.0.5 before any tag or crates.io publication. The harness could only test a crate already on crates.io. Harness (scripts/acceptance/installed-native/run.py), --package-from mode: - Requires a clean checkout whose HEAD is --package-commit. Copies that commit into a self-contained repository and checks it again. - Runs `cargo package --locked [--list]` in a bounded --rm container with the source mounted read-only, and requires Cargo 1.86.0, the release workflow's. - Binds the .crate SHA-256, the exact member set (equal to `--list`), the version (source and packaged manifests), and .cargo_vcs_info.json (the clean commit). Every non-generated member must byte-equal the commit's blob. Optionally pins --expect-package-sha256. - Installs only those bytes as a crates.io directory-source replacement whose package checksum is the .crate SHA-256, then builds offline (--network none). The fixture lock must carry that checksum, and the installed files must equal the archive members. - Runs the same lifecycle/deadline cells and keeps every bound, ownership label and cleanup rule. Only a fully passing run writes approved-package.json. Release (scripts/ci/publish-rust-sdk.sh, release.yml, release_package.py): - Reads the reviewed record from main at release/approved-packages/.json. - A new version is never published without it. The reproduced package must equal its SHA-256, file set, version, commit and Cargo release, or the workflow refuses before cargo publish. - After publication, the crates.io checksum, the downloaded archive and the origin tag must all read back as the approved artifact. - CI now checks that `cargo package` is byte-reproducible. Refs #170 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 13 + .github/workflows/release.yml | 3 + scripts/acceptance/installed-native/README.md | 104 +++- scripts/acceptance/installed-native/run.py | 417 ++++++++++++++- .../acceptance/installed-native/test_run.py | 473 +++++++++++++++++- scripts/ci/README.md | 8 +- scripts/ci/publish-rust-sdk.sh | 91 ++++ scripts/ci/release_package.py | 138 ++++- scripts/ci/test-publish-rust-sdk.py | 189 ++++++- scripts/ci/test-release-package.py | 83 +++ 10 files changed, 1483 insertions(+), 36 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05844b3d..1d58509f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -354,6 +354,19 @@ jobs: archive="target/package/durable-workflow-$(cargo metadata --no-deps --format-version 1 | python3 -c 'import json,sys; print(json.load(sys.stdin)["packages"][0]["version"])').crate" python3 scripts/ci/release_package.py --archive "$archive" python3 scripts/ci/verify-fresh-consumer.py package + - name: Check that the package is byte-reproducible + # Pre-publication approval binds a .crate SHA-256 that the release + # workflow must reproduce, so packaging twice must give identical bytes. + run: | + version="$(cargo metadata --no-deps --format-version 1 | python3 -c 'import json,sys; print(json.load(sys.stdin)["packages"][0]["version"])')" + first="$(sha256sum "target/package/durable-workflow-${version}.crate" | awk '{print $1}')" + CARGO_TARGET_DIR="${RUNNER_TEMP}/repackage" cargo package --locked --no-verify + second="$(sha256sum "${RUNNER_TEMP}/repackage/package/durable-workflow-${version}.crate" | awk '{print $1}')" + printf 'commit %s cargo %s crate %s\n' "$(git rev-parse HEAD)" "$(cargo -V)" "$first" + if [[ "$first" != "$second" ]]; then + printf 'cargo package is not reproducible: %s != %s\n' "$first" "$second" >&2 + exit 1 + fi - name: Validate release tooling run: | bash -n scripts/ci/crates-io-publish-lib.sh scripts/ci/install-release-tooling.sh scripts/ci/publish-rust-sdk.sh scripts/ci/test-crates-io-publish.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c5752d6c..68c50778 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -119,6 +119,9 @@ jobs: CARGO_REGISTRY_TOKEN: ${{ steps.crates-auth.outputs.token }} RELEASE_RUN_ID: ${{ github.run_id }} RELEASE_RUN_ATTEMPT: ${{ github.run_attempt }} + # Reviewed record of the pre-publication package that passed installed-native + # acceptance, committed to main (the maintained tooling), never the tagged source. + RUST_SDK_APPROVED_PACKAGE_PATH: ../release-tools/release/approved-packages/${{ env.RELEASE_TAG }}.json run: ../release-tools/scripts/ci/publish-rust-sdk.sh - name: Create the source GitHub Release diff --git a/scripts/acceptance/installed-native/README.md b/scripts/acceptance/installed-native/README.md index a26cf881..c71e8ef0 100644 --- a/scripts/acceptance/installed-native/README.md +++ b/scripts/acceptance/installed-native/README.md @@ -1,7 +1,8 @@ # Installed-package acceptance against native Server -This harness checks one exact published `durable-workflow` crate against one -exact native Server image. It runs in disposable local containers on an +This harness checks one exact published `durable-workflow` crate, or (in +pre-publication mode, below) the exact `cargo package` bytes of one clean +release commit, against one exact native Server image. It runs in disposable local containers on an internal Docker network and never contacts Cloud. It does the following: @@ -60,6 +61,105 @@ python3 scripts/acceptance/installed-native/run.py \ For a crate published before the map existed (≤ 4.10.2), pass `--baselines absent --baselines-control Cargo.toml`. +### Pre-publication mode + +A new version must pass this acceptance **before** it is tagged or published, +so crates.io never holds an unqualified immutable version. `--package-from` +replaces `--crate-version`: the harness packages one exact clean commit and +installs only those packaged bytes. + +```sh +python3 scripts/acceptance/installed-native/run.py \ + --package-from \ + --package-commit <40-hex release commit> \ + --expect-version 4.11.2 \ + --server-index sha256:3cb6e5bf8ec766ac2abd933369b40a77af62794b6d7b921ce21d3fb263e88b72 \ + --platform linux/arm64 \ + --expect-server-version 3.0.5 \ + --expect-release-tag 3.0.5 --expect-revision bc24589db02161a21febbb30211309537fd1e24e \ + --baselines required --container-prefix general \ + --work-dir +``` + +Use the commit that will be tagged. Cargo records the commit in +`.cargo_vcs_info.json`, so a different commit (for example a later merge +commit with the same tree) produces a different package and will not match. + +1. **Exact source.** The checkout's HEAD must equal `--package-commit` and + its tree must be fully clean (no changed or untracked paths). The harness + copies that commit into `/package-source`, a self-contained + repository, and checks it the same way. +2. **Package.** A bounded `--rm` container (same limits as the fixture build) + runs `cargo package --locked --no-verify --list` and + `cargo package --locked --no-verify`, with the source mounted read-only. The + container must report Cargo `--package-cargo-version` (default `1.86.0`, + the release workflow's), because a different Cargo writes different bytes. + `--no-verify` skips only Cargo's own test build. The archive is already + written by then, and the offline fixture build compiles it. +3. **Bind.** The harness refuses the package unless all of these hold: + - exactly `durable-workflow-.crate` was produced, and the source and + packaged manifests both declare `--expect-version`; + - every member is a regular file under `durable-workflow-/`, with + no links, traversal or duplicates; + - the member set equals the `--list` output exactly; + - `.cargo_vcs_info.json` names the clean commit, with `path_in_vcs` empty; + - every member except the generated `Cargo.toml` and `.cargo_vcs_info.json` + is byte-identical to the commit's blob (`Cargo.toml.orig` against + `Cargo.toml`). + + `--expect-package-sha256` also pins the archive SHA-256, for example when + re-running an earlier result. +4. **Install only those bytes.** One container (default network) patches the + dependency to the unpacked package, only to resolve and `cargo vendor` + third-party crates. The harness then adds the verified members as a + directory-source entry whose package checksum is the `.crate` SHA-256 and + replaces crates.io with that directory. A second container, with + `--network none`, regenerates the lockfile and builds offline. Cargo checks + every vendored file against the checksums. The fixture lock must name + `durable-workflow ` from the crates.io source with checksum equal + to the `.crate` SHA-256. The installed files must equal the archive + members exactly. There is no path dependency on checkout bytes. +5. **Same cells.** The lifecycle and deadline cells, Server identity and + same-worker assertions are unchanged. After the cells, the saved `.crate` + must still hash to the bound SHA-256. + +`summary.json` records `package` (commit, version, SHA-256, size, sorted file +list and its digest, VCS record, Cargo version, Rust image identity) and +`crate` (lock checksum and source, installed file count, baselines). Only a run +whose checks and cleanup both pass writes `/approved-package.json`: + +```json +{ + "schema": "durable-workflow.rust-sdk.approved-package/v1", + "package": "durable-workflow", "version": "4.11.2", "commit": "<40 hex>", + "crate_sha256": "<64 hex>", "cargo_version": "1.86.0", + "files": ["..."], "file_list_sha256": "<64 hex>", + "acceptance": {"summary_sha256": "<64 hex>", "server_index": "...", "...": "..."} +} +``` + +A reviewer commits that record unchanged to `main` as +`release/approved-packages/.json`. `release/` is outside the crate's +`include` list, so it never changes package bytes. The Release workflow reads +the record from its maintained tooling checkout (`main`), never from the +tagged source. Then: + +- **Before publishing,** it reproduces `cargo package --locked` from the tag and + refuses (`approved_package_mismatch`) unless the archive's SHA-256, file set, + version, VCS commit and Cargo release all equal the record. A version that is + not yet on crates.io is never published without a record + (`approved_package_required`). +- **After publishing,** the crates.io checksum and the downloaded archive must + equal the approved SHA-256. The downloaded archive must match the record + again. The tag must read back from `origin` as the release commit. Any + difference fails the release (`published_checksum_differs_from_approved_package`, + `published_package_differs_from_approved_package`, + `source_tag_readback_mismatch`), and nothing may claim the version is qualified. + +Cargo cannot upload a prebuilt `.crate`, so `cargo publish` packages again from +the same clean tag with the same Cargo. The readback above is what proves that +the published bytes are the accepted ones. + The run writes `/summary.json` with every identity, receipt and history result, plus per-container logs under `cells/*/logs`. The exit code is 0 only when every check passes and cleanup is complete (see below). diff --git a/scripts/acceptance/installed-native/run.py b/scripts/acceptance/installed-native/run.py index c8e0def9..3e7b49e0 100755 --- a/scripts/acceptance/installed-native/run.py +++ b/scripts/acceptance/installed-native/run.py @@ -2,8 +2,9 @@ """Installed-package lifecycle acceptance of the Rust SDK against a native Server image. Everything runs in disposable local containers on an internal Docker network, -against one exact Server image index digest and one exact published crate -version. Nothing here contacts Cloud. See README.md for usage. +against one exact Server image index digest and either one exact published +crate version or (pre-publication mode) the exact `cargo package` bytes of one +clean source commit. Nothing here contacts Cloud. See README.md for usage. """ import argparse import datetime @@ -41,6 +42,23 @@ BUILD_WALL_SECONDS = 3300 BUILD_KILL_AFTER_SECONDS = 60 BUILD_CLIENT_TIMEOUT = 3600 +# Pre-publication mode. The release workflow packages with Rust 1.86.0, and a +# different Cargo writes different archive bytes, so packaging here must use the +# same Cargo release. +PACKAGE_NAME = 'durable-workflow' +PACKAGE_CARGO_VERSION = '1.86.0' +CRATES_IO_SOURCE = 'registry+https://github.com/rust-lang/crates.io-index' +# Files Cargo writes into the archive instead of copying them from the commit. +GENERATED_PACKAGE_FILES = ('.cargo_vcs_info.json', 'Cargo.toml') +REQUIRED_PACKAGE_FILES = ('.cargo_vcs_info.json', 'Cargo.toml', 'Cargo.toml.orig', 'Cargo.lock') +APPROVAL_SCHEMA = 'durable-workflow.rust-sdk.approved-package/v1' +COMMIT_PATTERN = re.compile(r'[0-9a-f]{40}') +SHA256_PATTERN = re.compile(r'[0-9a-f]{64}') +VERSION_PATTERN = re.compile(r'[0-9]+\.[0-9]+\.[0-9]+([-+][0-9A-Za-z.-]+)?') + + +class PackageError(RuntimeError): + """The locally packaged crate is not exactly the requested clean commit.""" def container_prefix(value): @@ -112,9 +130,39 @@ def remove_owned_network(name, run): return network_id -def parse_args(): +def full_commit(value): + if not COMMIT_PATTERN.fullmatch(value): + raise argparse.ArgumentTypeError('%r must be a full lowercase 40-hex commit' % value) + return value + + +def exact_sha256(value): + if not SHA256_PATTERN.fullmatch(value): + raise argparse.ArgumentTypeError('%r must be a lowercase 64-hex SHA-256' % value) + return value + + +def exact_version(value): + if not VERSION_PATTERN.fullmatch(value): + raise argparse.ArgumentTypeError('%r must be an exact SemVer version' % value) + return value + + +def parse_args(argv=None): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--crate-version', required=True, help='exact published durable-workflow version, e.g. 4.10.3') + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument('--crate-version', help='exact published durable-workflow version, e.g. 4.10.3') + source.add_argument('--package-from', type=pathlib.Path, + help='pre-publication mode: a clean git checkout whose HEAD is --package-commit; ' + 'the harness packages that commit and installs only the packaged bytes') + parser.add_argument('--package-commit', type=full_commit, + help='pre-publication mode: the exact release commit to package (40 hex)') + parser.add_argument('--expect-version', type=exact_version, + help='pre-publication mode: required Cargo.toml version of the packaged crate') + parser.add_argument('--expect-package-sha256', type=exact_sha256, + help='pre-publication mode: require the packaged .crate to have this SHA-256') + parser.add_argument('--package-cargo-version', default=PACKAGE_CARGO_VERSION, + help='pre-publication mode: required `cargo -V` release in --rust-image (the release workflow\'s)') parser.add_argument('--server-index', required=True, help='exact image index digest, sha256:...') parser.add_argument('--server-image', default='ghcr.io/durable-workflow/server') parser.add_argument('--runtime', choices=['native', 'php'], default='native', @@ -140,7 +188,24 @@ def parse_args(): parser.add_argument('--container-prefix', type=container_prefix, default='sdkrust', help='owner prefix for container, network and task names, e.g. general or security ' '(^%s$)' % PREFIX_PATTERN.pattern) - return parser.parse_args() + args = parser.parse_args(argv) + if args.package_from: + # An approval candidate is only meaningful against one exact Server release. + required = (('--package-commit', args.package_commit), ('--expect-version', args.expect_version), + ('--expect-server-version', args.expect_server_version), + ('--expect-revision', args.expect_revision), ('--expect-release-tag', args.expect_release_tag)) + absent = [option for option, value in required if not value] + if absent: + parser.error('--package-from requires %s' % ', '.join(absent)) + for option, value in (('--fixture-from', args.fixture_from), ('--source-tag', args.source_tag)): + if value: + parser.error('%s cannot be combined with --package-from' % option) + else: + for option, value in (('--package-commit', args.package_commit), ('--expect-version', args.expect_version), + ('--expect-package-sha256', args.expect_package_sha256)): + if value: + parser.error('%s requires --package-from' % option) + return args def call(command, timeout=120, check=True, **kwargs): @@ -189,36 +254,311 @@ def server_identity(args): 'image': by_manifest, 'image_by_index': by_index, 'labels': labels} -def build_fixture(args, fixture, run): - shutil.copytree(HERE / 'fixture' / 'src', fixture / 'src') - template = (HERE / 'fixture' / 'Cargo.toml.in').read_text() - (fixture / 'Cargo.toml').write_text(template.replace('@CRATE_VERSION@', args.crate_version)) - call(['docker', 'pull', args.rust_image], timeout=1800) - # Bounded like every other harness container: --init, CPU/RAM/PID caps and an - # in-container wall bound. It keeps the default network only for the - # crates.io fetch and mounts only the fixture directory. - name = run.base + '-build' +FIXTURE_ENV = ['CARGO_HOME=/work/.cargo', 'CARGO_RESOLVER_INCOMPATIBLE_RUST_VERSIONS=fallback'] + + +def bounded_cargo(args, run, role, volumes, env, script, workdir='/work', network=None): + """Run one foreground Cargo step in a bounded, labelled, self-removing container. + + Bounded like every other harness container: --init, CPU/RAM/PID caps, no + capabilities and an in-container wall bound. `network` None keeps the + default network (registry fetches); 'none' runs fully offline. + """ + name = '%s-%s' % (run.base, role) command = (['docker', 'run', '--rm', '--init', '--name', name] + run.labels(name) + ['--cpus', str(max(1, args.jobs)), '--memory', '3g', '--pids-limit', '1024', '--security-opt', 'no-new-privileges', '--cap-drop', 'ALL', - '--user', '%d:%d' % (os.getuid(), os.getgid()), - '-v', '%s:/work' % fixture, '-w', '/work', '-e', 'CARGO_HOME=/work/.cargo', - '-e', 'CARGO_RESOLVER_INCOMPATIBLE_RUST_VERSIONS=fallback', args.rust_image, - 'timeout', '--kill-after=%ds' % BUILD_KILL_AFTER_SECONDS, '%ds' % BUILD_WALL_SECONDS, 'sh', '-c', - 'rustc -V && cargo -V && cargo generate-lockfile && cargo build --locked -j %d' % args.jobs]) + '--user', '%d:%d' % (os.getuid(), os.getgid())]) + if network: + command += ['--network', network] + for volume in volumes: + command += ['-v', volume] + command += ['-w', workdir] + for value in env: + command += ['-e', value] + command += [args.rust_image, 'timeout', '--kill-after=%ds' % BUILD_KILL_AFTER_SECONDS, + '%ds' % BUILD_WALL_SECONDS, 'sh', '-c', script] try: - build = call(command, timeout=BUILD_CLIENT_TIMEOUT) + return call(command, timeout=BUILD_CLIENT_TIMEOUT) finally: # Covers a client-side timeout or error while this process is alive. If - # the client dies, the in-container timeout ends the build and --rm fires. - # A cleanup failure is recorded and never replaces the build's own error. - run.best_effort('build-cleanup', name, remove_owned_container, name, run) + # the client dies, the in-container timeout ends the step and --rm fires. + # A cleanup failure is recorded and never replaces the step's own error. + run.best_effort('%s-cleanup' % role, name, remove_owned_container, name, run) + + +def fixture_manifest(version): + return (HERE / 'fixture' / 'Cargo.toml.in').read_text().replace('@CRATE_VERSION@', version) + + +def build_fixture(args, fixture, run): + shutil.copytree(HERE / 'fixture' / 'src', fixture / 'src') + (fixture / 'Cargo.toml').write_text(fixture_manifest(args.crate_version)) + call(['docker', 'pull', args.rust_image], timeout=1800) + # The default network is kept only for the crates.io fetch; only the fixture + # directory is mounted. + build = bounded_cargo(args, run, 'build', ['%s:/work' % fixture], FIXTURE_ENV, + 'rustc -V && cargo -V && cargo generate-lockfile && cargo build --locked -j %d' % args.jobs) (fixture / 'build.stdout').write_text(build.stdout) (fixture / 'build.stderr').write_text(build.stderr) return {'rust_image': image_identity(args.rust_image), 'toolchain': build.stdout.strip().splitlines()[:2], 'binary_sha256': sha256(fixture / 'target' / 'debug' / BINARY)} +# --------------------------------------------------------------------------- +# Pre-publication mode: package one clean commit, bind its bytes, and install +# only those bytes into the fixture as a checksummed crates.io replacement. +# --------------------------------------------------------------------------- + +def git(path, *arguments, **kwargs): + return call(['git', '-C', str(path)] + list(arguments), **kwargs) + + +def require_clean_checkout(path, commit): + """The checkout's HEAD must be exactly `commit` and its tree fully clean.""" + head = git(path, 'rev-parse', '--verify', 'HEAD^{commit}', check=False) + if head.returncode or head.stdout.strip() != commit: + raise PackageError('checkout %s HEAD is %r, not requested commit %s' % (path, head.stdout.strip(), commit)) + status = git(path, 'status', '--porcelain=v1', '--untracked-files=all', '--ignore-submodules=none', check=False) + if status.returncode or status.stdout.strip(): + raise PackageError('checkout %s is not clean (%d changed or untracked paths)' + % (path, len(status.stdout.splitlines()))) + + +def clone_exact_commit(checkout, commit, destination): + """Copy exactly `commit` into a fresh, self-contained repository. + + A linked worktree's .git points outside itself, so the packaging container + could not read it, and Cargo would silently omit .cargo_vcs_info.json. + """ + require_clean_checkout(checkout, commit) + git(destination.parent, 'init', '--quiet', str(destination)) + git(destination, 'fetch', '--quiet', '--no-tags', '--no-recurse-submodules', str(checkout), + '+HEAD:refs/heads/acceptance-package', timeout=600) + git(destination, 'checkout', '--quiet', '--detach', commit) + require_clean_checkout(destination, commit) + + +def file_list_sha256(files): + return hashlib.sha256(''.join(name + '\n' for name in sorted(files)).encode()).hexdigest() + + +def verify_package_archive(crate_bytes, version, commit, listed, git_blob): + """Check a .crate against its source commit and `cargo package --list`. + + Returns {relative path: bytes}. Every member must be a regular file under + `-/`; the member set must equal the listing exactly; the VCS + record must name the clean commit; the generated manifest must carry the + version; every other file must be byte-identical to the commit's blob. + """ + prefix = '%s-%s/' % (PACKAGE_NAME, version) + members = {} + try: + with tarfile.open(fileobj=io.BytesIO(crate_bytes), mode='r:gz') as archive: + for member in archive.getmembers(): + relative = member.name.removeprefix(prefix) + parts = relative.split('/') + if (not member.name.startswith(prefix) or not member.isfile() or not relative + or any(part in ('', '.', '..') for part in parts)): + raise PackageError('crate member %r is not a regular file under %s' % (member.name, prefix)) + if relative in members: + raise PackageError('crate member %r appears more than once' % relative) + members[relative] = archive.extractfile(member).read() + except (tarfile.TarError, OSError, EOFError) as error: + raise PackageError('could not read the packaged crate: %s' % type(error).__name__) from error + listed = list(listed) + if len(set(listed)) != len(listed): + raise PackageError('cargo package --list repeats a path') + missing, extra = sorted(set(listed) - set(members)), sorted(set(members) - set(listed)) + if missing or extra: + raise PackageError('crate file set differs from cargo package --list: missing %s, extra %s' + % (missing[:10], extra[:10])) + absent = [name for name in REQUIRED_PACKAGE_FILES if name not in members] + if absent: + raise PackageError('crate lacks required files %s' % absent) + try: + vcs = json.loads(members['.cargo_vcs_info.json']) + manifest = tomllib.loads(members['Cargo.toml'].decode()) + except (ValueError, UnicodeError) as error: + raise PackageError('crate VCS record or manifest is unreadable: %s' % type(error).__name__) from error + git_record = vcs.get('git') if isinstance(vcs, dict) else None + if (not isinstance(git_record, dict) or git_record.get('sha1') != commit + or git_record.get('dirty', False) is not False or vcs.get('path_in_vcs') != ''): + raise PackageError('crate VCS record %r does not name clean commit %s' % (vcs, commit)) + package = manifest.get('package', {}) + if package.get('name') != PACKAGE_NAME or package.get('version') != version: + raise PackageError('packaged manifest is %s %s, expected %s %s' + % (package.get('name'), package.get('version'), PACKAGE_NAME, version)) + drift = [] + for relative, content in members.items(): + if relative in GENERATED_PACKAGE_FILES: + continue + if git_blob('Cargo.toml' if relative == 'Cargo.toml.orig' else relative) != content: + drift.append(relative) + if drift: + raise PackageError('packaged files differ from commit %s: %s' % (commit, sorted(drift)[:10])) + return members + + +def write_members(members, destination): + """Write verified archive members (never tar extraction) under `destination`.""" + destination.mkdir(parents=True) + for relative, content in sorted(members.items()): + path = destination / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + + +def package_release(args, work_dir, run): + """`cargo package` exactly args.package_commit and bind the resulting bytes.""" + commit, version = args.package_commit, args.expect_version + checkout = args.package_from.resolve() + source, out = work_dir / 'package-source', work_dir / 'package-out' + clone_exact_commit(checkout, commit, source) + source_manifest = tomllib.loads(git(source, 'show', '%s:Cargo.toml' % commit).stdout)['package'] + if source_manifest.get('name') != PACKAGE_NAME or source_manifest.get('version') != version: + raise PackageError('commit %s declares %s %s, expected %s %s' + % (commit, source_manifest.get('name'), source_manifest.get('version'), PACKAGE_NAME, version)) + out.mkdir() + call(['docker', 'pull', args.rust_image], timeout=1800) + # The source is mounted read-only; Cargo writes only under /out. The default + # network is kept for the registry index Cargo needs to package --locked. + # --no-verify skips Cargo's own test build only: archive bytes are written + # first and are identical, and the offline fixture build below compiles them. + packaged = bounded_cargo(args, run, 'package', ['%s:/source:ro' % source, '%s:/out' % out], + ['CARGO_HOME=/out/.cargo', 'CARGO_TARGET_DIR=/out/target'], + 'cargo -V && cargo package --locked --no-verify --list > /out/package.list ' + '&& cargo package --locked --no-verify', workdir='/source') + (out / 'package.stdout').write_text(packaged.stdout) + (out / 'package.stderr').write_text(packaged.stderr) + cargo_version = (packaged.stdout.strip().splitlines() or [''])[0] + if not cargo_version.startswith('cargo %s ' % args.package_cargo_version): + raise PackageError('packaging Cargo is %r, expected cargo %s' % (cargo_version, args.package_cargo_version)) + # The worktree must still be exactly the commit after Cargo ran. + require_clean_checkout(source, commit) + crates = sorted((out / 'target' / 'package').glob('*.crate')) + expected = out / 'target' / 'package' / ('%s-%s.crate' % (PACKAGE_NAME, version)) + if crates != [expected]: + raise PackageError('expected exactly %s, found %s' % (expected.name, [c.name for c in crates])) + crate_bytes = expected.read_bytes() + crate_sha256 = hashlib.sha256(crate_bytes).hexdigest() + if args.expect_package_sha256 and crate_sha256 != args.expect_package_sha256: + raise PackageError('packaged crate SHA-256 %s differs from expected %s' % (crate_sha256, args.expect_package_sha256)) + listed = [line for line in (out / 'package.list').read_text().splitlines() if line] + + def blob(path): + result = git(source, 'cat-file', 'blob', '%s:%s' % (commit, path), check=False, text=False) + return result.stdout if result.returncode == 0 else None + members = verify_package_archive(crate_bytes, version, commit, listed, blob) + unpacked = work_dir / 'package' / ('%s-%s' % (PACKAGE_NAME, version)) + write_members(members, unpacked) + (work_dir / 'package' / expected.name).write_bytes(crate_bytes) + metadata = tomllib.loads(members['Cargo.toml'].decode())['package'].get('metadata', {}).get('durable-workflow', {}) + return {'mode': 'pre-publication', 'package': PACKAGE_NAME, 'version': version, 'commit': commit, + 'source_checkout': str(checkout), 'crate_file': expected.name, 'crate_sha256': crate_sha256, + 'crate_size': len(crate_bytes), 'files': sorted(members), 'file_count': len(members), + 'file_list_sha256': file_list_sha256(members), + 'vcs_info': json.loads(members['.cargo_vcs_info.json']), 'cargo_version': cargo_version, + 'rust_image': image_identity(args.rust_image), 'product_train': metadata.get('product-train'), + 'verified_source_files': len(members) - len(GENERATED_PACKAGE_FILES), + 'unpacked': str(unpacked)}, members + + +def cargo_checksum(members, crate_sha256): + return json.dumps({'files': {name: hashlib.sha256(content).hexdigest() for name, content in sorted(members.items())}, + 'package': crate_sha256}, sort_keys=True) + + +def build_packaged_fixture(args, fixture, package, members, run): + """Build the fixture against only the packaged bytes, offline. + + 1. Resolve: with the default network, `[patch]` the dependency to the + unpacked package only to resolve and `cargo vendor` third-party crates. + Cargo never vendors that path package. + 2. Install: add the verified members as a directory-source entry whose + package checksum is the .crate SHA-256, replace crates.io with the vendor + directory, drop the patch and the lockfile, and build with --network none. + Cargo then locks durable-workflow as a crates.io package whose checksum is + the packaged SHA-256 and verifies every vendored file against it. + """ + version = package['version'] + shutil.copytree(HERE / 'fixture' / 'src', fixture / 'src') + final_manifest = fixture_manifest(version) + (fixture / 'Cargo.toml').write_text(final_manifest + '\n[patch.crates-io]\n%s = { path = "/package" }\n' % PACKAGE_NAME) + resolve = bounded_cargo(args, run, 'resolve', ['%s:/work' % fixture, '%s:/package:ro' % package['unpacked']], + FIXTURE_ENV, 'cargo -V && cargo generate-lockfile && cargo vendor --locked --versioned-dirs vendor') + (fixture / 'resolve.stdout').write_text(resolve.stdout) + (fixture / 'resolve.stderr').write_text(resolve.stderr) + vendor = fixture / 'vendor' + foreign = sorted(p.name for p in vendor.glob(PACKAGE_NAME + '-*')) + if foreign: + raise PackageError('third-party vendoring produced %s; only the packaged bytes may be installed' % foreign) + entry = vendor / ('%s-%s' % (PACKAGE_NAME, version)) + write_members(members, entry) + (entry / '.cargo-checksum.json').write_text(cargo_checksum(members, package['crate_sha256'])) + (fixture / 'Cargo.toml').write_text(final_manifest) + (fixture / 'Cargo.lock').unlink() + (fixture / '.cargo').mkdir(exist_ok=True) + (fixture / '.cargo' / 'config.toml').write_text( + '[source.crates-io]\nreplace-with = "acceptance-package"\n\n' + '[source.acceptance-package]\ndirectory = "vendor"\n') + build = bounded_cargo(args, run, 'build', ['%s:/work' % fixture], FIXTURE_ENV, + 'rustc -V && cargo -V && cargo generate-lockfile --offline ' + '&& cargo build --locked --offline -j %d' % args.jobs, network='none') + (fixture / 'build.stdout').write_text(build.stdout) + (fixture / 'build.stderr').write_text(build.stderr) + return {'rust_image': image_identity(args.rust_image), 'toolchain': build.stdout.strip().splitlines()[:2], + 'network': 'none', 'binary_sha256': sha256(fixture / 'target' / 'debug' / BINARY)} + + +def packaged_provenance(args, fixture, package, members): + """The built fixture locked and installed exactly the packaged bytes.""" + version, crate_sha256 = package['version'], package['crate_sha256'] + lock = tomllib.loads((fixture / 'Cargo.lock').read_text()) + locked = [p for p in lock['package'] if p['name'] == PACKAGE_NAME] + if len(locked) != 1 or locked[0].get('version') != version: + raise PackageError('fixture did not lock exactly %s %s: %r' % (PACKAGE_NAME, version, locked)) + if locked[0].get('source') != CRATES_IO_SOURCE or locked[0].get('checksum') != crate_sha256: + raise PackageError('fixture lock %r is not the packaged crate %s' % (locked[0], crate_sha256)) + entries = sorted(p.name for p in (fixture / 'vendor').glob(PACKAGE_NAME + '-*')) + entry = fixture / 'vendor' / ('%s-%s' % (PACKAGE_NAME, version)) + if entries != [entry.name]: + raise PackageError('vendored %s entries are %s' % (PACKAGE_NAME, entries)) + paths = list(entry.rglob('*')) + if any(p.is_symlink() or not (p.is_file() or p.is_dir()) for p in paths): + raise PackageError('installed package contains a link or special file') + installed = {str(p.relative_to(entry)): p.read_bytes() for p in paths if p.is_file()} + checksum = json.loads(installed.pop('.cargo-checksum.json', b'null')) + if installed != members or not isinstance(checksum, dict) or checksum.get('package') != crate_sha256: + raise PackageError('installed package bytes differ from the packaged crate') + metadata = tomllib.loads(members['Cargo.toml'].decode())['package'].get('metadata', {}).get('durable-workflow', {}) + return {'mode': 'pre-publication', 'version': version, 'sha256': crate_sha256, 'lock_checksum': locked[0]['checksum'], + 'lock_source': locked[0]['source'], 'registry_published': False, 'source_commit': package['commit'], + 'installed_files': len(installed), 'verified_source_files': package['verified_source_files'], + 'product_train': metadata.get('product-train'), + 'baselines': check_baselines(metadata, args.baselines, args.expected_baselines)} + + +def approval_candidate(summary, summary_bytes): + """The approved-package record a reviewer commits after a passing run.""" + package = summary['package'] + return {'schema': APPROVAL_SCHEMA, 'package': package['package'], 'version': package['version'], + 'commit': package['commit'], 'crate_sha256': package['crate_sha256'], + 'cargo_version': package['cargo_version'].split()[1], 'files': package['files'], + 'file_list_sha256': package['file_list_sha256'], + 'acceptance': {'summary_sha256': hashlib.sha256(summary_bytes).hexdigest(), + 'harness_commit': summary.get('harness_commit'), + 'server_index': summary['server']['index'], + 'executed_manifest': summary['server']['executed_manifest'], + 'platform': summary['server']['platform'], + 'expected_server_version': summary.get('expected_server', {}).get('version'), + 'expected_revision': summary.get('expected_server', {}).get('revision'), + 'expected_release_tag': summary.get('expected_server', {}).get('release_tag'), + 'cluster_versions': sorted({c.get('cluster_info', {}).get('version') or '' + for c in summary['cells'].values()}), + 'cells': sorted(summary['cells']), 'finished_at': summary['finished_at']}} + + def check_baselines(metadata, mode, expected): present = 'supported-server-release-baselines' in metadata baselines = metadata.get('supported-server-release-baselines') @@ -480,7 +820,11 @@ def main(): args.work_dir.mkdir(parents=True) run = Run(args.container_prefix) summary = {'started_at': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'outcome': 'fail', + 'mode': 'pre-publication' if args.package_from else 'published', + 'expected_server': {'version': args.expect_server_version, 'revision': args.expect_revision, + 'release_tag': args.expect_release_tag}, 'harness_commit': call(['git', '-C', str(REPO), 'rev-parse', 'HEAD'], check=False).stdout.strip()} + approval = None try: if args.baselines_control: control = tomllib.loads(args.baselines_control.read_text()) @@ -488,14 +832,20 @@ def main(): control['package'].get('metadata', {}).get('durable-workflow', {}), 'required', args.expected_baselines), source=str(args.baselines_control)) summary['server'] = server_identity(args) - if args.fixture_from: + if args.package_from: + fixture = args.work_dir / 'fixture' + summary['package'], members = package_release(args, args.work_dir, run) + summary['build'] = build_packaged_fixture(args, fixture, summary['package'], members, run) + summary['crate'] = packaged_provenance(args, fixture, summary['package'], members) + elif args.fixture_from: fixture = args.fixture_from.resolve() summary['build'] = {'reused_fixture': str(fixture), 'toolchain': (fixture / 'build.stdout').read_text().strip().splitlines()[:2], 'binary_sha256': sha256(fixture / 'target' / 'debug' / BINARY)} else: fixture = args.work_dir / 'fixture' summary['build'] = build_fixture(args, fixture, run) - summary['crate'] = crate_provenance(args, fixture) + if not args.package_from: + summary['crate'] = crate_provenance(args, fixture) # Pull first: inspect alone fails on a clean host that never fetched the helper image. call(['docker', 'pull', '--quiet', args.python_image], timeout=600) summary['python_image'] = image_identity(args.python_image) @@ -503,7 +853,13 @@ def main(): for name in args.cells.split(','): cells[name] = Cell(args, name, args.work_dir, summary['server'], fixture, run).run() summary['cells'] = cells - passed = (summary['crate']['baselines']['pass'] + package_intact = True + if args.package_from: + # The approved bytes are the ones that were packaged, not a later copy. + after = sha256(args.work_dir / 'package' / summary['package']['crate_file']) + summary['package']['crate_sha256_after_cells'] = after + package_intact = after == summary['package']['crate_sha256'] + passed = (package_intact and summary['crate']['baselines']['pass'] and summary.get('baselines_control', {'pass': True})['pass'] and all(c['outcome'] == 'pass' for c in summary['cells'].values())) summary['outcome'] = 'pass' if passed else 'fail' @@ -529,8 +885,15 @@ def main(): 'local_leftovers': run.local_leftovers, 'leftover_containers': leftovers, 'leftover_networks': leftover_networks} summary['finished_at'] = datetime.datetime.now(datetime.timezone.utc).isoformat() - (args.work_dir / 'summary.json').write_text(json.dumps(summary, ensure_ascii=False, indent=2) + '\n') + summary_bytes = (json.dumps(summary, ensure_ascii=False, indent=2) + '\n').encode() + (args.work_dir / 'summary.json').write_bytes(summary_bytes) + if args.package_from and summary['outcome'] == 'pass': + # Only a fully passing pre-publication run (checks and cleanup) yields + # an approval candidate; a reviewer commits it for the release workflow. + approval = args.work_dir / 'approved-package.json' + approval.write_text(json.dumps(approval_candidate(summary, summary_bytes), indent=2) + '\n') print(json.dumps({'outcome': summary['outcome'], 'error': summary.get('error'), + 'approval_candidate': str(approval) if approval else None, 'summary': str(args.work_dir / 'summary.json'), 'leftover_containers': leftovers, 'leftover_networks': leftover_networks, 'local_leftovers': run.local_leftovers, 'cleanup_errors': run.cleanup_errors})) diff --git a/scripts/acceptance/installed-native/test_run.py b/scripts/acceptance/installed-native/test_run.py index 51a994cc..94cc4d4c 100644 --- a/scripts/acceptance/installed-native/test_run.py +++ b/scripts/acceptance/installed-native/test_run.py @@ -4,6 +4,8 @@ """ import argparse import contextlib +import gzip +import hashlib import io import json import os @@ -12,6 +14,7 @@ import shutil import subprocess import sys +import tarfile import tempfile import unittest from unittest import mock @@ -47,7 +50,8 @@ def __init__(self, containers=None, networks=None, faults=None, created=None, ra self.containers = dict(containers or {}) self.networks = dict(networks or {}) self.faults = dict(faults or {}) - self.created = created # side effect of a successful build, e.g. writing the binary + # Side effect of a foreground run, e.g. writing the binary; a returned string is its stdout. + self.created = created self.raise_on = dict(raise_on or {}) def __call__(self, command, timeout=120, check=True, **kwargs): @@ -73,9 +77,10 @@ def __call__(self, command, timeout=120, check=True, **kwargs): result = self._create(self.containers, name, command, timeout, check) if '--rm' in command and '-d' not in command: self.containers.pop(name) # a foreground --rm container exits and is removed - if self.created: - self.created(command) - return done(command, stdout=BUILD_STDOUT if '-d' not in command else 'cid\n') if result is None else result + stdout = self.created(command) if self.created else None + if stdout is None: + stdout = BUILD_STDOUT if '-d' not in command else 'cid\n' + return done(command, stdout=stdout) if result is None else result if command[:3] == ['docker', 'network', 'create']: result = self._create(self.networks, command[-1], command, timeout, check) return done(command) if result is None else result @@ -768,5 +773,465 @@ def cell_run(h): [{'step': 'network-cleanup', 'object': 'n', 'error': 'TimeoutExpired'}]) +# --------------------------------------------------------------------------- +# Pre-publication mode: exact packaging, byte binding and offline installation. +# --------------------------------------------------------------------------- + +VERSION = '4.11.2' +CARGO_V = 'cargo 1.86.0 (adf9b6ad1 2025-02-28)' +SOURCE_MANIFEST = ('[package]\nname = "durable-workflow"\nversion = "%s"\nedition = "2021"\n\n' + '[package.metadata.durable-workflow]\nproduct-train = "%s"\n' + 'supported-server-versions = "2.0.0"\n' + 'supported-server-release-baselines = { "2" = "2.0.0", "3" = "3.0.2" }\n') % (VERSION, VERSION) +SOURCE_FILES = {'Cargo.toml': SOURCE_MANIFEST.encode(), 'Cargo.lock': b'version = 4\n', + 'CHANGELOG.md': b'# Changelog\n\n## 4.11.2\n\n- Fix.\n', 'src/lib.rs': b'pub fn settle() {}\n', + 'schema/contract.json': b'{"v": 1}\n'} +REAL_CALL = run.call + + +def git_env(): + return dict(os.environ, GIT_AUTHOR_NAME='t', GIT_AUTHOR_EMAIL='t@example.invalid', GIT_COMMITTER_NAME='t', + GIT_COMMITTER_EMAIL='t@example.invalid', GIT_CONFIG_NOSYSTEM='1', GIT_CONFIG_GLOBAL=os.devnull) + + +def make_repo(path, files=None): + path.mkdir(parents=True) + for name, content in (files or SOURCE_FILES).items(): + (path / name).parent.mkdir(parents=True, exist_ok=True) + (path / name).write_bytes(content) + for command in (['init', '--quiet'], ['add', '-A'], ['-c', 'commit.gpgsign=false', 'commit', '--quiet', '-m', 'release']): + subprocess.run(['git', '-C', str(path)] + command, check=True, env=git_env(), capture_output=True) + return subprocess.run(['git', '-C', str(path), 'rev-parse', 'HEAD'], check=True, capture_output=True, + text=True).stdout.strip() + + +def packaged_members(commit, files=None): + files = dict(files or SOURCE_FILES) + members = {name: content for name, content in files.items() if name != 'Cargo.toml'} + members['Cargo.toml.orig'] = files['Cargo.toml'] + members['Cargo.toml'] = b'# normalized by cargo\n' + files['Cargo.toml'] + members['.cargo_vcs_info.json'] = json.dumps({'git': {'sha1': commit}, 'path_in_vcs': ''}).encode() + return members + + +def make_crate(members, version=VERSION, raw=()): + """A gzip tar like Cargo's; `raw` adds (TarInfo, bytes-or-None) members verbatim.""" + buffer = io.BytesIO() + with gzip.GzipFile(fileobj=buffer, mode='wb', mtime=0) as compressed, \ + tarfile.open(fileobj=compressed, mode='w') as archive: + for name, content in members.items(): + info = tarfile.TarInfo('durable-workflow-%s/%s' % (version, name)) + info.size = len(content) + archive.addfile(info, io.BytesIO(content)) + for info, content in raw: + archive.addfile(info, io.BytesIO(content) if content is not None else None) + return buffer.getvalue() + + +def hybrid(docker): + """git runs for real (temporary repositories); Docker is the stateful fake.""" + def dispatch(command, timeout=120, check=True, **kwargs): + if command[0] == 'git': + return REAL_CALL(command, timeout=timeout, check=check, **kwargs) + return docker(command, timeout, check, **kwargs) + return dispatch + + +def volume(command, destination): + """Host side of the `-v host:destination[:ro]` mount.""" + for index, value in enumerate(command): + if command[index - 1] == '-v' and value.split(':')[1] == destination: + return pathlib.Path(value.split(':')[0]) + raise AssertionError('no %s mount in %r' % (destination, command)) + + +class FakeCargo: + """What the three Cargo containers write, with knobs for each counterfactual.""" + + def __init__(self, commit, members=None, crate_name=None, cargo=CARGO_V, listing=None, lock=None, + vendor_foreign=False, tamper_install=None, crate=None): + self.commit, self.cargo = commit, cargo + self.members = members if members is not None else packaged_members(commit) + self.crate = crate + self.crate_name = crate_name or 'durable-workflow-%s.crate' % VERSION + self.listing = listing + self.lock, self.vendor_foreign, self.tamper_install = lock, vendor_foreign, tamper_install + + def crate_bytes(self): + return self.crate if self.crate is not None else make_crate(self.members) + + def __call__(self, command): + name = command[command.index('--name') + 1] + if name.endswith('-package'): + out = volume(command, '/out') + (out / 'target' / 'package').mkdir(parents=True) + (out / 'target' / 'package' / self.crate_name).write_bytes(self.crate_bytes()) + listing = self.listing if self.listing is not None else sorted(self.members) + (out / 'package.list').write_text(''.join(line + '\n' for line in listing)) + return self.cargo + '\n' + work = volume(command, '/work') + if name.endswith('-resolve'): + (work / 'Cargo.lock').write_text('version = 4\n') + (work / 'vendor' / 'serde_json-1.0.0').mkdir(parents=True) + if self.vendor_foreign: + (work / 'vendor' / ('durable-workflow-%s' % VERSION)).mkdir() + return self.cargo + '\n' + # The offline build: lock what the directory source declares. + entry = work / 'vendor' / ('durable-workflow-%s' % VERSION) + declared = json.loads((entry / '.cargo-checksum.json').read_text())['package'] + lock = self.lock or {'name': 'durable-workflow', 'version': VERSION, 'source': run.CRATES_IO_SOURCE, + 'checksum': declared} + (work / 'Cargo.lock').write_text('version = 4\n\n[[package]]\n' + ''.join( + '%s = "%s"\n' % item for item in lock.items())) + if self.tamper_install: + self.tamper_install(entry) + write_binary(work)(command) + return 'rustc 1.86.0\n' + self.cargo + '\n' + + +def package_args(checkout, commit, **overrides): + values = dict(package_from=checkout, package_commit=commit, expect_version=VERSION, expect_package_sha256=None, + package_cargo_version='1.86.0', rust_image=RUST, jobs=2, baselines='required', + expected_baselines={'2': '2.0.0', '3': '3.0.2'}) + values.update(overrides) + return argparse.Namespace(**values) + + +class PrePublication(HarnessRun): + def setUp(self): + super().setUp() + self.checkout = self.tmp / 'release' + self.commit = make_repo(self.checkout) + self.work = self.tmp / 'work' + self.work.mkdir() + + def package(self, cargo=None, **overrides): + self.docker = FakeDocker(created=cargo or FakeCargo(self.commit)) + with mock.patch.object(run, 'call', hybrid(self.docker)): + return run.package_release(package_args(self.checkout, self.commit, **overrides), self.work, self.run) + + def install(self, cargo=None): + cargo = cargo or FakeCargo(self.commit) + identity, members = self.package(cargo) + fixture = self.work / 'fixture' + with mock.patch.object(run, 'call', hybrid(self.docker)): + build = run.build_packaged_fixture(package_args(self.checkout, self.commit), fixture, identity, members, self.run) + crate = run.packaged_provenance(package_args(self.checkout, self.commit), fixture, identity, members) + return identity, build, crate + + def refused(self, pattern, cargo=None, **overrides): + with self.assertRaisesRegex(run.PackageError, pattern): + self.package(cargo, **overrides) + + def test_exact_clean_commit_is_packaged_bound_and_installed_offline(self): + identity, build, crate = self.install() + crate_bytes = make_crate(packaged_members(self.commit)) + digest = hashlib.sha256(crate_bytes).hexdigest() + self.assertEqual((identity['commit'], identity['version'], identity['crate_sha256']), (self.commit, VERSION, digest)) + self.assertEqual(identity['files'], sorted(packaged_members(self.commit))) + self.assertEqual(identity['file_list_sha256'], run.file_list_sha256(packaged_members(self.commit))) + self.assertEqual(identity['vcs_info'], {'git': {'sha1': self.commit}, 'path_in_vcs': ''}) + self.assertEqual(identity['cargo_version'], CARGO_V) + self.assertEqual((self.work / 'package' / identity['crate_file']).read_bytes(), crate_bytes) + self.assertEqual((crate['sha256'], crate['lock_checksum'], crate['lock_source']), + (digest, digest, run.CRATES_IO_SOURCE)) + self.assertIs(crate['registry_published'], False) + self.assertTrue(crate['baselines']['pass']) + self.assertEqual(build['network'], 'none') + json.dumps(identity) # evidence is serializable and carries no raw bytes + self.assertEqual(self.run.cleanup_errors, []) + + def test_package_container_is_bounded_read_only_and_labelled(self): + self.package() + [(command, timeout)] = self.docker.runs() + name = self.run.base + '-package' + image = command.index(RUST) + self.assertEqual(command[:image], [ + 'docker', 'run', '--rm', '--init', '--name', name, + '--label', 'task=' + name, '--label', 'dev.durable-workflow.acceptance.run=' + TOKEN, + '--label', 'dev.durable-workflow.acceptance.owner=general', + '--cpus', '2', '--memory', '3g', '--pids-limit', '1024', + '--security-opt', 'no-new-privileges', '--cap-drop', 'ALL', '--user', '%d:%d' % (os.getuid(), os.getgid()), + '-v', '%s:/source:ro' % (self.work / 'package-source'), '-v', '%s:/out' % (self.work / 'package-out'), + '-w', '/source', '-e', 'CARGO_HOME=/out/.cargo', '-e', 'CARGO_TARGET_DIR=/out/target']) + self.assertEqual(command[image + 1:image + 6], ['timeout', '--kill-after=60s', '3300s', 'sh', '-c']) + self.assertIn('cargo package --locked --no-verify --list', command[-1]) + self.assertEqual(timeout, run.BUILD_CLIENT_TIMEOUT) + + def test_resolve_and_offline_build_containers(self): + self.install() + [package, resolve, build] = [c for c, _ in self.docker.runs()] + self.assertNotIn('--network', resolve) + self.assertIn('%s:/package:ro' % (self.work / 'package' / ('durable-workflow-' + VERSION)), resolve) + self.assertIn('cargo vendor --locked --versioned-dirs vendor', resolve[-1]) + self.assertEqual(build[build.index('--network') + 1], 'none') + self.assertEqual([build[i + 1] for i, a in enumerate(build) if a in ('-v', '--mount')], + ['%s:/work' % (self.work / 'fixture')]) + self.assertIn('cargo build --locked --offline -j 2', build[-1]) + fixture = self.work / 'fixture' + self.assertNotIn('[patch', (fixture / 'Cargo.toml').read_text()) + self.assertIn('replace-with = "acceptance-package"', (fixture / '.cargo' / 'config.toml').read_text()) + for container in (package, resolve, build): + self.assertIn('--rm', container) + self.assertIn('--init', container) + + # Counterfactuals: each refuses before any lifecycle cell can run. + + def test_dirty_tree_is_refused_before_packaging(self): + for change in ('modify', 'untracked'): + with self.subTest(change=change): + target = self.checkout / ('src/lib.rs' if change == 'modify' else 'notes.txt') + original = target.read_bytes() if target.exists() else None + target.write_bytes(b'drift\n') + try: + self.refused('is not clean') + self.assertEqual(self.docker.runs(), []) + finally: + target.write_bytes(original) if original is not None else target.unlink() + shutil.rmtree(self.work / 'package-source', ignore_errors=True) + + def test_head_commit_mismatch_is_refused(self): + with self.assertRaisesRegex(run.PackageError, 'not requested commit'): + self.docker = FakeDocker(created=FakeCargo(self.commit)) + with mock.patch.object(run, 'call', hybrid(self.docker)): + run.package_release(package_args(self.checkout, 'f' * 40), self.work, self.run) + self.assertEqual(self.docker.runs(), []) + + def test_source_version_mismatch_is_refused_before_packaging(self): + self.refused('declares durable-workflow 4.11.2, expected durable-workflow 4.11.3', expect_version='4.11.3') + self.assertEqual(self.docker.runs(), []) + + def test_packaged_manifest_version_mismatch_is_refused(self): + members = packaged_members(self.commit) + members['Cargo.toml'] = members['Cargo.toml'].replace(b'version = "4.11.2"', b'version = "4.11.1"') + self.refused('packaged manifest is durable-workflow 4.11.1', FakeCargo(self.commit, members=members)) + + def test_crate_file_name_or_count_mismatch_is_refused(self): + self.refused('expected exactly durable-workflow-4.11.2.crate', + FakeCargo(self.commit, crate_name='durable-workflow-4.11.1.crate')) + + def test_vcs_info_mismatch_is_refused(self): + for record in ({'git': {'sha1': 'e' * 40}, 'path_in_vcs': ''}, + {'git': {'sha1': None}, 'path_in_vcs': ''}, + {'git': {'dirty': True}, 'path_in_vcs': ''}, + {'path_in_vcs': ''}, + {'git': {'sha1': 'COMMIT', 'dirty': True}, 'path_in_vcs': ''}, + {'git': {'sha1': 'COMMIT'}, 'path_in_vcs': 'sdk'}): + with self.subTest(record=record): + members = packaged_members(self.commit) + members['.cargo_vcs_info.json'] = json.dumps(record).replace('COMMIT', self.commit).encode() + self.refused('does not name clean commit', FakeCargo(self.commit, members=members)) + shutil.rmtree(self.work) + self.work.mkdir() + + def test_one_byte_package_drift_from_the_commit_is_refused(self): + for name in ('src/lib.rs', 'schema/contract.json', 'Cargo.lock', 'Cargo.toml.orig', 'CHANGELOG.md'): + with self.subTest(name=name): + members = packaged_members(self.commit) + members[name] = members[name][:-1] + bytes([members[name][-1] ^ 1]) + self.refused('differ from commit', FakeCargo(self.commit, members=members)) + shutil.rmtree(self.work) + self.work.mkdir() + + def test_extra_or_missing_file_against_cargo_listing_is_refused(self): + members = packaged_members(self.commit) + extra = dict(members, **{'src/extra.rs': b'// not listed\n'}) + missing = {k: v for k, v in members.items() if k != 'schema/contract.json'} + cases = (('extra', FakeCargo(self.commit, members=extra, listing=sorted(members))), + ('missing', FakeCargo(self.commit, members=missing, listing=sorted(members))), + ('listed-twice', FakeCargo(self.commit, listing=sorted(members) + ['src/lib.rs']))) + for label, cargo in cases: + with self.subTest(label=label): + self.refused('differs from cargo package --list|repeats a path', cargo) + shutil.rmtree(self.work) + self.work.mkdir() + + def test_extra_file_absent_from_the_commit_is_refused_even_when_listed(self): + members = dict(packaged_members(self.commit), **{'src/injected.rs': b'// not in git\n'}) + self.refused('differ from commit', FakeCargo(self.commit, members=members)) + + def test_missing_required_generated_file_is_refused(self): + members = {k: v for k, v in packaged_members(self.commit).items() if k != 'Cargo.lock'} + self.refused('lacks required files', FakeCargo(self.commit, members=members)) + + def test_unsafe_or_duplicate_members_are_refused(self): + members = packaged_members(self.commit) + link = tarfile.TarInfo('durable-workflow-%s/src/link.rs' % VERSION) + link.type, link.linkname = tarfile.SYMTYPE, '/etc/passwd' + outside = tarfile.TarInfo('durable-workflow-%s/../escape.rs' % VERSION) + outside.size = 1 + foreign = tarfile.TarInfo('other-1.0.0/src/lib.rs') + foreign.size = 1 + duplicate = tarfile.TarInfo('durable-workflow-%s/src/lib.rs' % VERSION) + duplicate.size = len(members['src/lib.rs']) + for label, raw in (('symlink', [(link, None)]), ('traversal', [(outside, b'x')]), + ('foreign-root', [(foreign, b'x')]), ('duplicate', [(duplicate, members['src/lib.rs'])])): + with self.subTest(label=label): + self.refused('not a regular file|more than once', + FakeCargo(self.commit, crate=make_crate(members, raw=raw))) + shutil.rmtree(self.work) + self.work.mkdir() + + def test_corrupt_archive_is_refused(self): + self.refused('could not read the packaged crate', FakeCargo(self.commit, crate=b'not a gzip tar')) + + def test_expected_package_sha256_mismatch_is_refused(self): + self.refused('differs from expected', expect_package_sha256='0' * 64) + shutil.rmtree(self.work) + self.work.mkdir() + identity, _ = self.package(expect_package_sha256=hashlib.sha256( + make_crate(packaged_members(self.commit))).hexdigest()) + self.assertEqual(identity['crate_sha256'], hashlib.sha256(make_crate(packaged_members(self.commit))).hexdigest()) + + def test_wrong_packaging_cargo_is_refused(self): + self.refused('packaging Cargo is', FakeCargo(self.commit, cargo='cargo 1.99.0 (5f94df478 2026-08-27)')) + + def test_package_container_failure_is_cleaned_by_ownership(self): + name = self.run.base + '-package' + self.docker = FakeDocker(faults={name: 'created-then-timeout'}) + with mock.patch.object(run, 'call', hybrid(self.docker)), self.assertRaises(subprocess.TimeoutExpired): + run.package_release(package_args(self.checkout, self.commit), self.work, self.run) + self.assertEqual(self.docker.removals(), [['docker', 'rm', '-f', 'id-' + name]]) + + def test_foreign_vendored_package_is_refused(self): + with self.assertRaisesRegex(run.PackageError, 'only the packaged bytes'): + self.install(FakeCargo(self.commit, vendor_foreign=True)) + + def test_lock_that_is_not_the_packaged_crate_is_refused(self): + for lock in ({'name': 'durable-workflow', 'version': VERSION, 'source': run.CRATES_IO_SOURCE, 'checksum': '0' * 64}, + {'name': 'durable-workflow', 'version': VERSION, 'source': 'path+file:///package'}, + {'name': 'durable-workflow', 'version': '4.11.1', 'source': run.CRATES_IO_SOURCE, 'checksum': '0' * 64}): + with self.subTest(lock=lock): + with self.assertRaises(run.PackageError): + self.install(FakeCargo(self.commit, lock=lock)) + shutil.rmtree(self.work) + self.work.mkdir() + + def test_installed_bytes_that_drift_from_the_package_are_refused(self): + def drift(entry): + (entry / 'src' / 'lib.rs').write_bytes(b'pub fn other() {}\n') + + def extra(entry): + (entry / 'src' / 'extra.rs').write_bytes(b'\n') + + def link(entry): + (entry / 'src' / 'lib.rs').unlink() + os.symlink(self.checkout / 'src' / 'lib.rs', entry / 'src' / 'lib.rs') + for tamper in (drift, extra, link): + with self.subTest(tamper=tamper.__name__): + with self.assertRaisesRegex(run.PackageError, 'installed package'): + self.install(FakeCargo(self.commit, tamper_install=tamper)) + shutil.rmtree(self.work) + self.work.mkdir() + + def test_directory_source_checksum_is_the_crate_sha256(self): + identity, _, _ = self.install() + entry = self.work / 'fixture' / 'vendor' / ('durable-workflow-' + VERSION) + checksum = json.loads((entry / '.cargo-checksum.json').read_text()) + self.assertEqual(checksum['package'], identity['crate_sha256']) + self.assertEqual(sorted(checksum['files']), identity['files']) + + +class PrePublicationArguments(unittest.TestCase): + BASE = ['--server-index', 'sha256:x', '--work-dir', '/tmp/x'] + PACKAGE = ['--package-from', '/tmp/src', '--package-commit', 'a' * 40, '--expect-version', VERSION, + '--expect-server-version', '3.0.5', '--expect-revision', 'b' * 40, '--expect-release-tag', '3.0.5'] + + def parse(self, extra): + with contextlib.redirect_stderr(io.StringIO()): + return run.parse_args(self.BASE + extra) + + def test_package_mode_parses(self): + args = self.parse(self.PACKAGE) + self.assertEqual((args.package_commit, args.expect_version, args.crate_version), ('a' * 40, VERSION, None)) + + def test_invalid_combinations_are_refused(self): + def without(option): + index = self.PACKAGE.index(option) + return self.PACKAGE[:index] + self.PACKAGE[index + 2:] + for extra in ([], self.PACKAGE + ['--crate-version', '4.11.1'], + without('--package-commit'), without('--expect-version'), without('--expect-server-version'), + without('--expect-revision'), without('--expect-release-tag'), + self.PACKAGE + ['--fixture-from', '/tmp/f'], self.PACKAGE + ['--source-tag', '4.11.2'], + ['--crate-version', '4.11.1', '--package-commit', 'a' * 40], + ['--crate-version', '4.11.1', '--expect-package-sha256', 'a' * 64], + self.PACKAGE[:3] + ['abc'] + self.PACKAGE[4:], + self.PACKAGE + ['--expect-package-sha256', 'A' * 64], + without('--expect-version') + ['--expect-version', 'v4.11.2']): + with self.subTest(extra=extra), self.assertRaises(SystemExit): + self.parse(extra) + + +class PrePublicationOutcome(unittest.TestCase): + """The approval candidate exists only for a fully passing run, and binds the package.""" + + def main(self, cell_run=None, tamper_after_cells=False): + tmp = temporary_directory(self) + work = tmp / 'work' + identity = {'mode': 'pre-publication', 'package': 'durable-workflow', 'version': VERSION, 'commit': 'a' * 40, + 'crate_file': 'durable-workflow-%s.crate' % VERSION, 'crate_sha256': hashlib.sha256(b'crate').hexdigest(), + 'files': ['Cargo.toml'], 'file_list_sha256': run.file_list_sha256(['Cargo.toml']), 'cargo_version': CARGO_V} + + def package_release(args_, work_dir, harness): + (work_dir / 'package').mkdir() + (work_dir / 'package' / identity['crate_file']).write_bytes(b'crate') + return dict(identity), {} + + def cell(args_, name, root, server, fixture_, harness): + instance = mock.Mock() + + def run_cell(): + if tamper_after_cells: + (work / 'package' / identity['crate_file']).write_bytes(b'other') + (cell_run or (lambda h: None))(harness) + return {'outcome': 'pass', 'cluster_info': {'version': '3.0.5'}} + instance.run.side_effect = run_cell + return instance + argv = ['run.py'] + PrePublicationArguments.PACKAGE + [ + '--server-index', 'sha256:x', '--cells', 'lifecycle', '--work-dir', str(work), '--container-prefix', 'general'] + out = io.StringIO() + server = {'index': 'img@sha256:x', 'executed_manifest': 'img@sha256:y', 'platform': 'linux/arm64'} + with mock.patch.object(sys, 'argv', argv), mock.patch.object(run, 'call', FakeDocker()), \ + mock.patch.object(run, 'server_identity', return_value=server), \ + mock.patch.object(run, 'package_release', package_release), \ + mock.patch.object(run, 'build_packaged_fixture', return_value={'network': 'none'}), \ + mock.patch.object(run, 'packaged_provenance', return_value={'baselines': {'pass': True}}), \ + mock.patch.object(run, 'crate_provenance') as published, mock.patch.object(run, 'Cell', cell), \ + contextlib.redirect_stdout(out), self.assertRaises(SystemExit) as exited: + run.main() + published.assert_not_called() + self.identity = identity + return exited.exception.code, json.loads(out.getvalue()), work + + def test_passing_run_writes_a_bound_approval_candidate(self): + code, receipt, work = self.main() + self.assertEqual(code, 0) + approval = json.loads((work / 'approved-package.json').read_text()) + self.assertEqual(receipt['approval_candidate'], str(work / 'approved-package.json')) + self.assertEqual(approval['schema'], run.APPROVAL_SCHEMA) + for key in ('package', 'version', 'commit', 'crate_sha256', 'files', 'file_list_sha256'): + self.assertEqual(approval[key], self.identity[key]) + self.assertEqual(approval['cargo_version'], '1.86.0') + self.assertEqual(approval['acceptance']['summary_sha256'], + hashlib.sha256((work / 'summary.json').read_bytes()).hexdigest()) + self.assertEqual((approval['acceptance']['expected_server_version'], approval['acceptance']['expected_release_tag'], + approval['acceptance']['platform'], approval['acceptance']['cluster_versions']), + ('3.0.5', '3.0.5', 'linux/arm64', ['3.0.5'])) + summary = json.loads((work / 'summary.json').read_text()) + self.assertEqual((summary['mode'], summary['package']['crate_sha256_after_cells']), + ('pre-publication', self.identity['crate_sha256'])) + + def test_cleanup_failure_writes_no_approval(self): + error = {'step': 'container-remove', 'object': 'x', 'error': 'TimeoutExpired'} + code, receipt, work = self.main(cell_run=lambda h: h.cleanup_errors.append(error)) + self.assertEqual((code, receipt['approval_candidate']), (1, None)) + self.assertFalse((work / 'approved-package.json').exists()) + + def test_package_changed_during_cells_writes_no_approval(self): + code, receipt, work = self.main(tamper_after_cells=True) + self.assertEqual((code, receipt['outcome'], receipt['approval_candidate']), (1, 'fail', None)) + self.assertFalse((work / 'approved-package.json').exists()) + + if __name__ == '__main__': unittest.main() diff --git a/scripts/ci/README.md b/scripts/ci/README.md index 9f55d1fc..ee5e3b7c 100644 --- a/scripts/ci/README.md +++ b/scripts/ci/README.md @@ -8,7 +8,13 @@ These scripts support four public repository checks: - API documentation, analytics, navigation, and base-URL checks. The release workflow uses the crates.io publishing scripts after an immutable -tag already exists. Ordinary pull requests run the product tests and package +tag already exists. A version that is not yet on crates.io is published only +when `cargo package` of the tag reproduces the reviewed pre-publication record +`release/approved-packages/.json` exactly. The record must match the +SHA-256, file set, version, VCS commit and Cargo release. After publication, the +crates.io checksum, the downloaded archive and the tag read back from `origin` +must still match it. See `scripts/acceptance/installed-native/README.md` +(pre-publication mode) for how a passing acceptance run produces the record. Ordinary pull requests run the product tests and package checks directly from `.github/workflows/ci.yml`. GitHub release notes come from the exact versioned CHANGELOG section selected by diff --git a/scripts/ci/publish-rust-sdk.sh b/scripts/ci/publish-rust-sdk.sh index d17b9088..b09d33cc 100755 --- a/scripts/ci/publish-rust-sdk.sh +++ b/scripts/ci/publish-rust-sdk.sh @@ -14,6 +14,10 @@ release_run_id="${RELEASE_RUN_ID:-}" release_run_attempt="${RELEASE_RUN_ATTEMPT:-}" release_tooling_commit="$(git -C "$script_dir" rev-parse HEAD)" lockfile_source="source_checkout" +# A reviewed record of the pre-publication package that passed installed-native +# acceptance (scripts/acceptance/installed-native, --package-from). New versions +# publish only when the reproduced package is exactly that artifact. +approved_package_path="${RUST_SDK_APPROVED_PACKAGE_PATH:-}" for command in cargo curl git jq python3 rustc sha256sum; do if ! command -v "$command" >/dev/null 2>&1; then @@ -185,6 +189,11 @@ release_entry_checksum="" published_changelog_checksum="" published_release_entry_checksum="" fresh_consumer_verified="false" +approval_status="absent" +approved_checksum="" +approved_record_sha256="" +approved_matches_registry="false" +source_tag_readback_commit="" write_evidence() { local outcome="$1" @@ -265,6 +274,12 @@ write_evidence() { --arg published_changelog_checksum "$published_changelog_checksum" \ --arg published_release_entry_checksum "$published_release_entry_checksum" \ --arg fresh_consumer_verified "$fresh_consumer_verified" \ + --arg approval_status "$approval_status" \ + --arg approved_package_path "$approved_package_path" \ + --arg approved_checksum "$approved_checksum" \ + --arg approved_record_sha256 "$approved_record_sha256" \ + --arg approved_matches_registry "$approved_matches_registry" \ + --arg source_tag_readback_commit "$source_tag_readback_commit" \ --arg outcome "$outcome" \ --arg reason "$reason" \ --arg registry_verified "$registry_verified" \ @@ -353,6 +368,17 @@ write_evidence() { commit: $release_tooling_commit, lockfile_source: $lockfile_source }, + approved_package: { + status: $approval_status, + record_path: $approved_package_path, + record_sha256: $approved_record_sha256, + crate_sha256: $approved_checksum, + registry_matches_approved_package: ($approved_matches_registry == "true") + }, + source_tag_readback: { + tag: $release_tag, + commit: $source_tag_readback_commit + }, fresh_consumer: { rust_version: $rust_version, exact_dependency: ($package + " = \"=" + $version + "\""), @@ -424,6 +450,26 @@ archive_vcs_dirty="$(jq -r '.archive.vcs_dirty' "$local_identity")" changelog_checksum="$(jq -er '.changelog.sha256' "$local_identity")" release_entry_checksum="$(jq -er '.changelog.entry_sha256' "$local_identity")" +if [[ -n "$approved_package_path" && -e "$approved_package_path" ]]; then + packaging_cargo="$(cargo -V | awk '{print $2}')" + local_approval="$tmp_dir/local-approval.json" + if ! python3 "$script_dir/release_package.py" \ + --manifest "$manifest_path" \ + --changelog "$changelog_path" \ + --archive "$local_archive" \ + --expected-vcs-commit "$release_commit" \ + --approved-package "$approved_package_path" \ + --cargo-version "$packaging_cargo" > "$local_approval"; then + approval_status="reproduction_mismatch" + write_evidence "failed" "approved_package_mismatch" + printf 'reproduced package is not the approved pre-publication package; refusing to publish\n' >&2 + exit 1 + fi + approved_checksum="$(jq -er '.approval.crate_sha256' "$local_approval")" + approved_record_sha256="$(jq -er '.approval.record_sha256' "$local_approval")" + approval_status="reproduced" +fi + write_evidence "pending" "checking_public_registry" if ! http_status="$(crates_io_version_lookup "$version_api" "$version_response" "$user_agent")"; then @@ -432,6 +478,13 @@ if ! http_status="$(crates_io_version_lookup "$version_api" "$version_response" exit 1 fi +if [[ "$http_status" == "404" && "$approval_status" != "reproduced" ]]; then + write_evidence "failed" "approved_package_required" + printf 'publishing %s %s requires a reviewed approved-package record (RUST_SDK_APPROVED_PACKAGE_PATH)\n' \ + "$package_name" "$package_version" >&2 + exit 1 +fi + if ! publish_exact_crate_if_absent "$http_status" "$manifest_path"; then write_evidence "failed" "$publish_error" case "$publish_error" in @@ -505,6 +558,12 @@ if [[ "$download_checksum" != "$published_checksum" ]]; then printf 'downloaded crate checksum does not match the crates.io registry checksum\n' >&2 exit 1 fi +if [[ "$approval_status" == "reproduced" && \ + ( "$published_checksum" != "$approved_checksum" || "$download_checksum" != "$approved_checksum" ) ]]; then + write_evidence "failed" "published_checksum_differs_from_approved_package" + printf 'crates.io checksum %s differs from the approved package %s\n' "$published_checksum" "$approved_checksum" >&2 + exit 1 +fi if [[ "$local_checksum" != "$published_checksum" ]]; then write_evidence "failed" "published_source_archive_mismatch" printf 'published crate archive differs from the exact release checkout package\n' >&2 @@ -534,6 +593,38 @@ if [[ "$published_archive_vcs_commit" != "$archive_vcs_commit" || \ exit 1 fi +if [[ "$approval_status" == "reproduced" ]]; then + if ! python3 "$script_dir/release_package.py" \ + --manifest "$manifest_path" \ + --changelog "$changelog_path" \ + --archive "$published_archive" \ + --expected-vcs-commit "$release_commit" \ + --approved-package "$approved_package_path" > "$tmp_dir/published-approval.json"; then + write_evidence "failed" "published_package_differs_from_approved_package" + printf 'downloaded crate is not the approved pre-publication package\n' >&2 + exit 1 + fi + approved_matches_registry="true" +fi + +# Read the source tag back from the repository: it must still name the release +# (and approved) commit after publication. +if ! remote_tag_refs="$(git ls-remote origin "refs/tags/${release_tag}" "refs/tags/${release_tag}^{}")"; then + write_evidence "failed" "source_tag_readback_failed" + printf 'could not read source tag %s back from origin\n' "$release_tag" >&2 + exit 1 +fi +source_tag_readback_commit="$(awk -v ref="refs/tags/${release_tag}^{}" '$2 == ref {print $1}' <<<"$remote_tag_refs")" +if [[ -z "$source_tag_readback_commit" ]]; then + source_tag_readback_commit="$(awk -v ref="refs/tags/${release_tag}" '$2 == ref {print $1}' <<<"$remote_tag_refs")" +fi +if [[ "$source_tag_readback_commit" != "$release_commit" ]]; then + write_evidence "failed" "source_tag_readback_mismatch" + printf 'source tag %s reads back as %s, not release commit %s\n' \ + "$release_tag" "${source_tag_readback_commit:-}" "$release_commit" >&2 + exit 1 +fi + if ! python3 "$script_dir/verify-fresh-consumer.py" \ registry --manifest "$manifest_path"; then write_evidence "failed" "published_fresh_consumer_msrv_build_failed" "true" diff --git a/scripts/ci/release_package.py b/scripts/ci/release_package.py index 9b0be459..04fb716d 100644 --- a/scripts/ci/release_package.py +++ b/scripts/ci/release_package.py @@ -369,11 +369,124 @@ def restore_missing_lockfile( return identity +APPROVAL_SCHEMA = "durable-workflow.rust-sdk.approved-package/v1" +SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") +CARGO_VERSION_PATTERN = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+$") +APPROVAL_FIELDS = { + "schema", "package", "version", "commit", "crate_sha256", "cargo_version", + "files", "file_list_sha256", "acceptance", +} + + +def file_list_sha256(files: list[str]) -> str: + return sha256("".join(f"{name}\n" for name in sorted(files)).encode()) + + +def load_approval(path: Path) -> tuple[dict[str, Any], str]: + """Read one reviewed approved-package record and check its shape.""" + try: + raw = path.read_bytes() + approval = json.loads(raw) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise ReleasePackageError(f"could not read approved package record: {error}") from error + if not isinstance(approval, dict) or set(approval) != APPROVAL_FIELDS: + raise ReleasePackageError("approved package record has unexpected fields") + files = approval["files"] + acceptance = approval["acceptance"] + if ( + approval["schema"] != APPROVAL_SCHEMA + or not isinstance(approval["package"], str) + or not isinstance(approval["version"], str) + or not isinstance(approval["commit"], str) + or not COMMIT_PATTERN.fullmatch(approval["commit"]) + or not isinstance(approval["crate_sha256"], str) + or not SHA256_PATTERN.fullmatch(approval["crate_sha256"]) + or not isinstance(approval["cargo_version"], str) + or not CARGO_VERSION_PATTERN.fullmatch(approval["cargo_version"]) + or not isinstance(files, list) + or not files + or not all(isinstance(name, str) and name for name in files) + or files != sorted(set(files)) + or approval["file_list_sha256"] != file_list_sha256(files) + or not isinstance(acceptance, dict) + or not isinstance(acceptance.get("summary_sha256"), str) + or not SHA256_PATTERN.fullmatch(acceptance["summary_sha256"]) + or not isinstance(acceptance.get("server_index"), str) + or not acceptance["server_index"] + ): + raise ReleasePackageError("approved package record is malformed") + return approval, sha256(raw) + + +def _archive_files(archive_path: Path, root: str) -> list[str]: + prefix = f"{root}/" + try: + with tarfile.open(archive_path, "r:gz") as archive: + members = archive.getmembers() + except (OSError, tarfile.TarError, EOFError) as error: + raise ReleasePackageError(f"could not inspect crate archive: {error}") from error + names = [] + for member in members: + if not member.isfile() or not member.name.startswith(prefix): + raise ReleasePackageError(f"crate member {member.name!r} is not a regular package file") + names.append(member.name[len(prefix):]) + if len(names) != len(set(names)): + raise ReleasePackageError("crate archive repeats a member") + return sorted(names) + + +def verify_approved_archive( + archive_path: Path, + manifest: Path, + changelog: Path, + expected_vcs_commit: str, + approval_path: Path, + cargo_version: str | None = None, +) -> dict[str, Any]: + """Require an archive to be exactly the approved pre-publication package.""" + identity = verify_archive(archive_path, manifest, changelog, expected_vcs_commit) + approval, approval_sha256 = load_approval(approval_path) + if approval["package"] != identity["package"]: + raise ReleasePackageError("approved package name differs from the release manifest") + if approval["version"] != identity["package_version"]: + raise ReleasePackageError("approved package version differs from the release manifest") + if approval["commit"] != expected_vcs_commit or approval["commit"] != identity["archive"]["vcs_commit"]: + raise ReleasePackageError("approved package commit differs from the release commit") + if cargo_version is not None and approval["cargo_version"] != cargo_version: + raise ReleasePackageError( + f"release Cargo {cargo_version} differs from approved Cargo {approval['cargo_version']}" + ) + files = _archive_files(archive_path, f"{identity['package']}-{identity['package_version']}") + if files != approval["files"]: + missing = sorted(set(approval["files"]) - set(files)) + extra = sorted(set(files) - set(approval["files"])) + raise ReleasePackageError( + f"archive file set differs from approved package: missing {missing[:10]}, extra {extra[:10]}" + ) + if identity["archive"]["sha256"] != approval["crate_sha256"]: + raise ReleasePackageError( + "archive SHA-256 differs from the approved pre-publication package" + ) + identity["approval"] = { + "record_sha256": approval_sha256, + "crate_sha256": approval["crate_sha256"], + "commit": approval["commit"], + "version": approval["version"], + "cargo_version": approval["cargo_version"], + "file_count": len(approval["files"]), + "file_list_sha256": approval["file_list_sha256"], + "acceptance_summary_sha256": approval["acceptance"]["summary_sha256"], + "server_index": approval["acceptance"]["server_index"], + } + return identity + + def _validate_notes_arguments( arguments: argparse.Namespace, parser: argparse.ArgumentParser, ) -> None: if any((arguments.archive, arguments.expected_vcs_commit, - arguments.expected_archive_sha256, arguments.restore_missing_lockfile)): + arguments.expected_archive_sha256, arguments.restore_missing_lockfile, + arguments.approved_package, arguments.cargo_version)): parser.error("--release-notes cannot be combined with archive verification") if (arguments.manifest != Path(SOURCE_MANIFEST) or arguments.changelog != Path(SOURCE_CHANGELOG)): @@ -388,6 +501,13 @@ def main() -> int: parser.add_argument("--expected-vcs-commit") parser.add_argument("--expected-archive-sha256") parser.add_argument("--restore-missing-lockfile", action="store_true") + parser.add_argument( + "--approved-package", type=Path, + help="require --archive to be exactly this reviewed approved-package record", + ) + parser.add_argument( + "--cargo-version", help="with --approved-package: the packaging Cargo release (X.Y.Z)", + ) parser.add_argument( "--release-notes", metavar="VERSION", help="print the exact current-version Markdown after manifest validation", @@ -395,6 +515,13 @@ def main() -> int: arguments = parser.parse_args() if arguments.expected_archive_sha256 is not None and not arguments.restore_missing_lockfile: parser.error("--expected-archive-sha256 requires --restore-missing-lockfile") + if arguments.cargo_version is not None and arguments.approved_package is None: + parser.error("--cargo-version requires --approved-package") + if arguments.approved_package is not None and ( + arguments.archive is None or arguments.expected_vcs_commit is None + or arguments.restore_missing_lockfile or arguments.release_notes is not None + ): + parser.error("--approved-package requires --archive and --expected-vcs-commit only") try: if arguments.release_notes is not None: @@ -416,6 +543,15 @@ def main() -> int: "version": 1, **verify_source(arguments.manifest, arguments.changelog), } + elif arguments.approved_package is not None: + evidence = verify_approved_archive( + arguments.archive, + arguments.manifest, + arguments.changelog, + arguments.expected_vcs_commit, + arguments.approved_package, + arguments.cargo_version, + ) else: evidence = verify_archive( arguments.archive, diff --git a/scripts/ci/test-publish-rust-sdk.py b/scripts/ci/test-publish-rust-sdk.py index 6f8be6b3..3555eac6 100644 --- a/scripts/ci/test-publish-rust-sdk.py +++ b/scripts/ci/test-publish-rust-sdk.py @@ -7,10 +7,12 @@ import json import os from pathlib import Path +import hashlib import ssl import shutil import subprocess import sys +import tarfile import tempfile import textwrap import threading @@ -65,6 +67,9 @@ def _write_mock_commands(self) -> None: import tomllib command = sys.argv[1] + if command == "-V": + print(f'cargo {os.environ.get("MOCK_CARGO_VERSION", "1.86.0")} (adf9b6ad1 2025-02-28)') + raise SystemExit(0) manifest = Path(sys.argv[sys.argv.index("--manifest-path") + 1]) package = tomllib.loads(manifest.read_text(encoding="utf-8"))["package"] target = Path(os.environ["CARGO_TARGET_DIR"]) @@ -106,6 +111,10 @@ def _write_mock_commands(self) -> None: member = tarfile.TarInfo(name) member.size = len(content) crate.addfile(member, io.BytesIO(content)) + elif command == "publish": + if "--locked" not in sys.argv: + raise SystemExit("publication must use --locked") + Path(os.environ["MOCK_PUBLISH_MARKER"]).write_text("published\n") elif command == "build": if os.environ.get("MOCK_CONSUMER_BUILD_OUTCOME") == "fail": raise SystemExit("mock fresh consumer build failed") @@ -153,6 +162,14 @@ def _write_mock_commands(self) -> None: pass elif command in {"rev-parse", "rev-list"}: print(os.environ["MOCK_RELEASE_COMMIT"]) + elif command == "ls-remote": + if sys.argv[2] != "origin": + raise SystemExit("tag readback must use origin") + tag = sys.argv[3] + peeled = os.environ.get("MOCK_REMOTE_TAG_COMMIT", os.environ["MOCK_RELEASE_COMMIT"]) + print("1111111111111111111111111111111111111111\t" + tag) + if peeled: + print(peeled + "\t" + tag + "^{}") else: raise SystemExit(f"unexpected git command: {command}") """, @@ -187,7 +204,8 @@ def _write_mock_commands(self) -> None: else: raise SystemExit(f"unexpected curl URL: {{url}}") if "--write-out" in args: - print(os.environ.get("MOCK_LOOKUP_STATUS", "200"), end="") + published = Path(os.environ["MOCK_PUBLISH_MARKER"]).exists() + print("200" if published else os.environ.get("MOCK_LOOKUP_STATUS", "200"), end="") ''', ) def _publish( @@ -195,6 +213,8 @@ def _publish( manifest: Path = MANIFEST, environment: dict[str, str] | None = None, cwd: Path = ROOT, + approval: dict[str, object] | None = None, + registry_changelog: bytes | None = None, ) -> subprocess.CompletedProcess[str]: env = os.environ.copy() env.update( @@ -208,6 +228,8 @@ def _publish( "RELEASE_TAG": PACKAGE_VERSION, "RUST_SDK_MANIFEST_PATH": str(manifest), "RUST_SDK_RELEASE_EVIDENCE_PATH": str(self.evidence), + "RUST_SDK_APPROVED_PACKAGE_PATH": str(self.temp / "approved" / f"{PACKAGE_VERSION}.json"), + "MOCK_PUBLISH_MARKER": str(self.temp / "published-marker"), } ) env.update(environment or {}) @@ -221,6 +243,24 @@ def _publish( [str(self.bin_dir / "cargo"), "package", "--manifest-path", str(MANIFEST)], env=fixture_env, check=True, ) + registry_archive = Path(env["MOCK_REGISTRY_ARCHIVE"]) + if approval is not None: + record = self._approval(registry_archive) + record.update(approval) + path = Path(env["RUST_SDK_APPROVED_PACKAGE_PATH"]) + path.parent.mkdir(exist_ok=True) + path.write_text(json.dumps(record), encoding="utf-8") + if registry_changelog is not None: + # crates.io serves different (self-consistent) bytes than were approved. + changed = self.temp / "changed-registry" + changelog = self.temp / "registry-CHANGELOG.md" + changelog.write_bytes(registry_changelog) + subprocess.run( + [str(self.bin_dir / "cargo"), "package", "--manifest-path", str(MANIFEST)], + env={**fixture_env, "CARGO_TARGET_DIR": str(changed), + "RUST_SDK_CHANGELOG_PATH": str(changelog)}, check=True, + ) + env["MOCK_REGISTRY_ARCHIVE"] = str(changed / "package" / registry_archive.name) return subprocess.run( ["bash", str(PUBLISH)], cwd=cwd, @@ -231,6 +271,153 @@ def _publish( check=False, ) + @staticmethod + def _approval(archive: Path) -> dict[str, object]: + root = f"durable-workflow-{PACKAGE_VERSION}/" + with tarfile.open(archive) as crate: + files = sorted(member.name.removeprefix(root) for member in crate.getmembers()) + return { + "schema": "durable-workflow.rust-sdk.approved-package/v1", + "package": "durable-workflow", + "version": PACKAGE_VERSION, + "commit": RELEASE_COMMIT, + "crate_sha256": hashlib.sha256(archive.read_bytes()).hexdigest(), + "cargo_version": "1.86.0", + "files": files, + "file_list_sha256": hashlib.sha256("".join(f"{name}\n" for name in files).encode()).hexdigest(), + "acceptance": {"summary_sha256": "c" * 64, "server_index": "ghcr.io/durable-workflow/server@sha256:" + "d" * 64}, + } + + def _evidence(self) -> dict[str, object]: + return json.loads(self.evidence.read_text(encoding="utf-8")) + + def _published(self) -> bool: + return (self.temp / "published-marker").exists() + + # Pre-publication approval binding (sdk-rust#170 Security ruling). + + def test_new_version_publishes_only_the_approved_reproduced_package(self) -> None: + result = self._publish(environment={"MOCK_LOOKUP_STATUS": "404", "CARGO_REGISTRY_TOKEN": "test-token"}, + approval={}) + self.assertEqual(0, result.returncode, result.stderr) + self.assertTrue(self._published()) + evidence = self._evidence() + self.assertEqual("published", evidence["outcome"]) + self.assertEqual("reproduced", evidence["approved_package"]["status"]) + self.assertTrue(evidence["approved_package"]["registry_matches_approved_package"]) + self.assertEqual(evidence["registry_checksum_sha256"], evidence["approved_package"]["crate_sha256"]) + self.assertEqual(evidence["local_package_checksum_sha256"], evidence["approved_package"]["crate_sha256"]) + self.assertEqual(RELEASE_COMMIT, evidence["source_tag_readback"]["commit"]) + self.assertNotIn("test-token", self.evidence.read_text(encoding="utf-8")) + + def test_new_version_without_an_approval_record_is_never_published(self) -> None: + result = self._publish(environment={"MOCK_LOOKUP_STATUS": "404", "CARGO_REGISTRY_TOKEN": "test-token"}) + self.assertNotEqual(0, result.returncode) + self.assertFalse(self._published()) + self.assertEqual("approved_package_required", self._evidence()["reason"]) + result = self._publish(environment={"MOCK_LOOKUP_STATUS": "404", "CARGO_REGISTRY_TOKEN": "test-token", + "RUST_SDK_APPROVED_PACKAGE_PATH": ""}) + self.assertNotEqual(0, result.returncode) + self.assertFalse(self._published()) + + def test_publish_time_repackage_mismatch_refuses_to_publish(self) -> None: + cases = { + "one-byte checksum drift": {"crate_sha256": "0" * 64}, + "wrong commit": {"commit": "f" * 40}, + "wrong version": {"version": "4.11.9"}, + "wrong cargo": {"cargo_version": "1.99.0"}, + "malformed": {"crate_sha256": "not-a-digest"}, + "unexpected field": {"approved_by": "nobody"}, + } + for label, change in cases.items(): + with self.subTest(label=label): + result = self._publish( + environment={"MOCK_LOOKUP_STATUS": "404", "CARGO_REGISTRY_TOKEN": "test-token"}, + approval=change, + ) + self.assertNotEqual(0, result.returncode) + self.assertFalse(self._published()) + evidence = self._evidence() + self.assertEqual("approved_package_mismatch", evidence["reason"]) + self.assertEqual("reproduction_mismatch", evidence["approved_package"]["status"]) + self.assertIn("refusing to publish", result.stderr) + + def test_extra_or_missing_approved_file_refuses_to_publish(self) -> None: + for label, mutate in ( + ("extra", lambda files: sorted(files + ["src/extra.rs"])), + ("missing", lambda files: files[1:]), + ): + with self.subTest(label=label): + record = self._approval_files_override(mutate) + result = self._publish( + environment={"MOCK_LOOKUP_STATUS": "404", "CARGO_REGISTRY_TOKEN": "test-token"}, + approval=record, + ) + self.assertNotEqual(0, result.returncode) + self.assertFalse(self._published()) + self.assertEqual("approved_package_mismatch", self._evidence()["reason"]) + self.assertIn("file set differs", result.stderr) + + @staticmethod + def _approval_files_override(mutate) -> dict[str, object]: + # The mock package holds exactly these members. + files = mutate([".cargo_vcs_info.json", "CHANGELOG.md", "Cargo.lock"]) + return {"files": files, + "file_list_sha256": hashlib.sha256("".join(f"{n}\n" for n in files).encode()).hexdigest()} + + def test_post_publish_registry_readback_mismatch_fails(self) -> None: + result = self._publish( + environment={"MOCK_LOOKUP_STATUS": "404", "CARGO_REGISTRY_TOKEN": "test-token"}, + approval={}, + registry_changelog=(ROOT / "CHANGELOG.md").read_bytes() + b"\n", + ) + self.assertNotEqual(0, result.returncode) + self.assertTrue(self._published()) + evidence = self._evidence() + self.assertEqual("failed", evidence["outcome"]) + self.assertEqual("published_checksum_differs_from_approved_package", evidence["reason"]) + self.assertFalse(evidence["approved_package"]["registry_matches_approved_package"]) + + def test_registry_checksum_claim_differing_from_approval_fails(self) -> None: + result = self._publish( + environment={"MOCK_LOOKUP_STATUS": "404", "CARGO_REGISTRY_TOKEN": "test-token", + "MOCK_REGISTRY_CHECKSUM": "0" * 64}, + approval={}, + ) + self.assertNotEqual(0, result.returncode) + self.assertNotEqual("published", self._evidence()["outcome"]) + self.assertFalse(self._evidence()["approved_package"]["registry_matches_approved_package"]) + + def test_already_published_version_must_still_equal_its_approval(self) -> None: + result = self._publish(approval={}, registry_changelog=(ROOT / "CHANGELOG.md").read_bytes() + b"\n") + self.assertNotEqual(0, result.returncode) + self.assertEqual("published_checksum_differs_from_approved_package", self._evidence()["reason"]) + + def test_post_publish_source_tag_readback_mismatch_fails(self) -> None: + for label, commit in (("moved", "e" * 40), ("missing", "")): + with self.subTest(label=label): + result = self._publish( + environment={"MOCK_LOOKUP_STATUS": "404", "CARGO_REGISTRY_TOKEN": "test-token", + "MOCK_REMOTE_TAG_COMMIT": commit}, + approval={}, + ) + self.assertNotEqual(0, result.returncode) + evidence = self._evidence() + self.assertEqual("source_tag_readback_mismatch", evidence["reason"]) + self.assertEqual("failed", evidence["outcome"]) + (self.temp / "published-marker").unlink(missing_ok=True) + + def test_release_workflow_supplies_the_reviewed_approval_record(self) -> None: + workflow = RELEASE_WORKFLOW.read_text(encoding="utf-8") + self.assertIn( + "RUST_SDK_APPROVED_PACKAGE_PATH: ../release-tools/release/approved-packages/${{ env.RELEASE_TAG }}.json", + workflow, + ) + # The record comes from maintained tooling (main), never the tagged source. + self.assertIn("ref: ${{ github.workflow_sha }}", workflow) + self.assertLess(workflow.index("Resolve exact release identity"), workflow.index("RUST_SDK_APPROVED_PACKAGE_PATH")) + self.assertNotIn("/release/", MANIFEST.read_text(encoding="utf-8").split("include = ", 1)[1].split("\n", 1)[0]) + def _manifest_with(self, old: str, new: str) -> Path: manifest = self.temp / "Cargo.toml" source = MANIFEST.read_text(encoding="utf-8") diff --git a/scripts/ci/test-release-package.py b/scripts/ci/test-release-package.py index 73ccbea8..9fcc3904 100644 --- a/scripts/ci/test-release-package.py +++ b/scripts/ci/test-release-package.py @@ -14,11 +14,14 @@ import unittest from release_package import ( + APPROVAL_SCHEMA, ReleasePackageError, + file_list_sha256, load_package, release_entry, render_release_notes, restore_missing_lockfile, + verify_approved_archive, verify_archive, verify_source, ) @@ -448,5 +451,85 @@ def test_archive_rejects_changed_notes_or_vcs_identity(self) -> None: ) + def approval(self, archive: Path, **changes: object) -> Path: + root = f"{self.package['name']}-{self.package['version']}/" + with tarfile.open(archive) as crate: + files = sorted(member.name.removeprefix(root) for member in crate.getmembers()) + record = { + "schema": APPROVAL_SCHEMA, + "package": self.package["name"], + "version": self.package["version"], + "commit": COMMIT, + "crate_sha256": hashlib.sha256(archive.read_bytes()).hexdigest(), + "cargo_version": "1.86.0", + "files": files, + "file_list_sha256": file_list_sha256(files), + "acceptance": {"summary_sha256": "c" * 64, "server_index": "ghcr.io/durable-workflow/server@sha256:" + "d" * 64}, + } + record.update(changes) + path = self.root / "approved.json" + path.write_text(json.dumps(record), encoding="utf-8") + return path + + def test_approved_archive_binds_checksum_files_commit_version_and_cargo(self) -> None: + archive = self.archive() + identity = verify_approved_archive( + archive, MANIFEST, CHANGELOG, COMMIT, self.approval(archive), "1.86.0" + ) + self.assertEqual(hashlib.sha256(archive.read_bytes()).hexdigest(), identity["approval"]["crate_sha256"]) + self.assertEqual(COMMIT, identity["approval"]["commit"]) + self.assertEqual(3, identity["approval"]["file_count"]) + + def test_approved_archive_counterfactuals_are_refused(self) -> None: + archive = self.archive() + files = sorted([".cargo_vcs_info.json", "CHANGELOG.md", "Cargo.lock"]) + extra = files + ["src/extra.rs"] + cases = { + "one-byte checksum drift": ({"crate_sha256": "0" * 64}, "SHA-256 differs"), + "wrong commit": ({"commit": "f" * 40}, "commit differs"), + "wrong version": ({"version": "9.9.9"}, "version differs"), + "wrong package": ({"package": "other"}, "name differs"), + "extra file": ({"files": sorted(extra), "file_list_sha256": file_list_sha256(extra)}, "file set differs"), + "missing file": ({"files": files[1:], "file_list_sha256": file_list_sha256(files[1:])}, "file set differs"), + "unsorted files": ({"files": list(reversed(files))}, "malformed"), + "list digest drift": ({"file_list_sha256": "0" * 64}, "malformed"), + "wrong schema": ({"schema": "other"}, "malformed"), + "no acceptance": ({"acceptance": {}}, "malformed"), + "extra field": ({"note": "x"}, "unexpected fields"), + } + for label, (change, message) in cases.items(): + with self.subTest(label=label), self.assertRaisesRegex(ReleasePackageError, message): + verify_approved_archive(archive, MANIFEST, CHANGELOG, COMMIT, self.approval(archive, **change)) + with self.assertRaisesRegex(ReleasePackageError, "differs from approved Cargo"): + verify_approved_archive(archive, MANIFEST, CHANGELOG, COMMIT, self.approval(archive), "1.99.0") + # A one-byte drift in the archive itself, with the approval recorded before it. + approval = self.approval(archive) + drifted = self.archive(changelog=CHANGELOG.read_bytes(), name="drifted.crate", lockfile=(ROOT / "Cargo.lock").read_bytes() + b"#") + with self.assertRaisesRegex(ReleasePackageError, "SHA-256 differs"): + verify_approved_archive(drifted, MANIFEST, CHANGELOG, COMMIT, approval) + with self.assertRaises(ReleasePackageError): + verify_approved_archive(archive, MANIFEST, CHANGELOG, "e" * 40, approval) + + def test_approved_package_cli_requires_archive_and_commit(self) -> None: + archive = self.archive() + approval = self.approval(archive) + script = Path(__file__).with_name("release_package.py") + for arguments in ( + ["--approved-package", str(approval)], + ["--approved-package", str(approval), "--archive", str(archive)], + ["--archive", str(archive), "--expected-vcs-commit", COMMIT, "--cargo-version", "1.86.0"], + ): + with self.subTest(arguments=arguments): + result = subprocess.run([sys.executable, str(script), *arguments], capture_output=True, text=True) + self.assertEqual(2, result.returncode, result.stderr) + result = subprocess.run( + [sys.executable, str(script), "--archive", str(archive), "--expected-vcs-commit", COMMIT, + "--approved-package", str(approval), "--cargo-version", "1.86.0"], + capture_output=True, text=True, + ) + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual(COMMIT, json.loads(result.stdout)["approval"]["commit"]) + + if __name__ == "__main__": unittest.main() From 501d031186802e033e5af48d312cccb00ea6151b Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Tue, 3 Nov 2026 16:54:58 -0600 Subject: [PATCH 2/9] Address static-analysis findings in the pre-publication change Split the approval argument checks out of release_package.main to reduce its cognitive complexity. Match the Cargo version with an ASCII-only \d class. Hoist test setup out of the exception-asserting blocks so each block makes exactly one call that can raise. Co-Authored-By: Claude Opus 5.5 --- .../acceptance/installed-native/test_run.py | 35 ++++++++++--------- scripts/ci/release_package.py | 22 +++++++----- scripts/ci/test-release-package.py | 6 ++-- 3 files changed, 37 insertions(+), 26 deletions(-) diff --git a/scripts/acceptance/installed-native/test_run.py b/scripts/acceptance/installed-native/test_run.py index 94cc4d4c..a0b0a076 100644 --- a/scripts/acceptance/installed-native/test_run.py +++ b/scripts/acceptance/installed-native/test_run.py @@ -991,10 +991,11 @@ def test_dirty_tree_is_refused_before_packaging(self): shutil.rmtree(self.work / 'package-source', ignore_errors=True) def test_head_commit_mismatch_is_refused(self): - with self.assertRaisesRegex(run.PackageError, 'not requested commit'): - self.docker = FakeDocker(created=FakeCargo(self.commit)) - with mock.patch.object(run, 'call', hybrid(self.docker)): - run.package_release(package_args(self.checkout, 'f' * 40), self.work, self.run) + self.docker = FakeDocker(created=FakeCargo(self.commit)) + wrong = package_args(self.checkout, 'f' * 40) + with mock.patch.object(run, 'call', hybrid(self.docker)), \ + self.assertRaisesRegex(run.PackageError, 'not requested commit'): + run.package_release(wrong, self.work, self.run) self.assertEqual(self.docker.runs(), []) def test_source_version_mismatch_is_refused_before_packaging(self): @@ -1089,23 +1090,25 @@ def test_wrong_packaging_cargo_is_refused(self): def test_package_container_failure_is_cleaned_by_ownership(self): name = self.run.base + '-package' self.docker = FakeDocker(faults={name: 'created-then-timeout'}) + arguments = package_args(self.checkout, self.commit) with mock.patch.object(run, 'call', hybrid(self.docker)), self.assertRaises(subprocess.TimeoutExpired): - run.package_release(package_args(self.checkout, self.commit), self.work, self.run) + run.package_release(arguments, self.work, self.run) self.assertEqual(self.docker.removals(), [['docker', 'rm', '-f', 'id-' + name]]) def test_foreign_vendored_package_is_refused(self): + cargo = FakeCargo(self.commit, vendor_foreign=True) with self.assertRaisesRegex(run.PackageError, 'only the packaged bytes'): - self.install(FakeCargo(self.commit, vendor_foreign=True)) + self.install(cargo) def test_lock_that_is_not_the_packaged_crate_is_refused(self): for lock in ({'name': 'durable-workflow', 'version': VERSION, 'source': run.CRATES_IO_SOURCE, 'checksum': '0' * 64}, {'name': 'durable-workflow', 'version': VERSION, 'source': 'path+file:///package'}, {'name': 'durable-workflow', 'version': '4.11.1', 'source': run.CRATES_IO_SOURCE, 'checksum': '0' * 64}): - with self.subTest(lock=lock): - with self.assertRaises(run.PackageError): - self.install(FakeCargo(self.commit, lock=lock)) - shutil.rmtree(self.work) - self.work.mkdir() + cargo = FakeCargo(self.commit, lock=lock) + with self.subTest(lock=lock), self.assertRaises(run.PackageError): + self.install(cargo) + shutil.rmtree(self.work) + self.work.mkdir() def test_installed_bytes_that_drift_from_the_package_are_refused(self): def drift(entry): @@ -1118,11 +1121,11 @@ def link(entry): (entry / 'src' / 'lib.rs').unlink() os.symlink(self.checkout / 'src' / 'lib.rs', entry / 'src' / 'lib.rs') for tamper in (drift, extra, link): - with self.subTest(tamper=tamper.__name__): - with self.assertRaisesRegex(run.PackageError, 'installed package'): - self.install(FakeCargo(self.commit, tamper_install=tamper)) - shutil.rmtree(self.work) - self.work.mkdir() + cargo = FakeCargo(self.commit, tamper_install=tamper) + with self.subTest(tamper=tamper.__name__), self.assertRaisesRegex(run.PackageError, 'installed package'): + self.install(cargo) + shutil.rmtree(self.work) + self.work.mkdir() def test_directory_source_checksum_is_the_crate_sha256(self): identity, _, _ = self.install() diff --git a/scripts/ci/release_package.py b/scripts/ci/release_package.py index 04fb716d..30b1c6d4 100644 --- a/scripts/ci/release_package.py +++ b/scripts/ci/release_package.py @@ -371,7 +371,7 @@ def restore_missing_lockfile( APPROVAL_SCHEMA = "durable-workflow.rust-sdk.approved-package/v1" SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") -CARGO_VERSION_PATTERN = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+$") +CARGO_VERSION_PATTERN = re.compile(r"^\d+\.\d+\.\d+$", re.ASCII) APPROVAL_FIELDS = { "schema", "package", "version", "commit", "crate_sha256", "cargo_version", "files", "file_list_sha256", "acceptance", @@ -493,6 +493,18 @@ def _validate_notes_arguments( parser.error("--release-notes uses the current checkout files; path overrides are refused") +def _validate_approval_arguments( + arguments: argparse.Namespace, parser: argparse.ArgumentParser, +) -> None: + if arguments.cargo_version is not None and arguments.approved_package is None: + parser.error("--cargo-version requires --approved-package") + if arguments.approved_package is not None and ( + arguments.archive is None or arguments.expected_vcs_commit is None + or arguments.restore_missing_lockfile or arguments.release_notes is not None + ): + parser.error("--approved-package requires --archive and --expected-vcs-commit only") + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--manifest", type=Path, default=Path(SOURCE_MANIFEST)) @@ -515,13 +527,7 @@ def main() -> int: arguments = parser.parse_args() if arguments.expected_archive_sha256 is not None and not arguments.restore_missing_lockfile: parser.error("--expected-archive-sha256 requires --restore-missing-lockfile") - if arguments.cargo_version is not None and arguments.approved_package is None: - parser.error("--cargo-version requires --approved-package") - if arguments.approved_package is not None and ( - arguments.archive is None or arguments.expected_vcs_commit is None - or arguments.restore_missing_lockfile or arguments.release_notes is not None - ): - parser.error("--approved-package requires --archive and --expected-vcs-commit only") + _validate_approval_arguments(arguments, parser) try: if arguments.release_notes is not None: diff --git a/scripts/ci/test-release-package.py b/scripts/ci/test-release-package.py index 9fcc3904..87492de6 100644 --- a/scripts/ci/test-release-package.py +++ b/scripts/ci/test-release-package.py @@ -498,10 +498,12 @@ def test_approved_archive_counterfactuals_are_refused(self) -> None: "extra field": ({"note": "x"}, "unexpected fields"), } for label, (change, message) in cases.items(): + approval = self.approval(archive, **change) with self.subTest(label=label), self.assertRaisesRegex(ReleasePackageError, message): - verify_approved_archive(archive, MANIFEST, CHANGELOG, COMMIT, self.approval(archive, **change)) + verify_approved_archive(archive, MANIFEST, CHANGELOG, COMMIT, approval) + approval = self.approval(archive) with self.assertRaisesRegex(ReleasePackageError, "differs from approved Cargo"): - verify_approved_archive(archive, MANIFEST, CHANGELOG, COMMIT, self.approval(archive), "1.99.0") + verify_approved_archive(archive, MANIFEST, CHANGELOG, COMMIT, approval, "1.99.0") # A one-byte drift in the archive itself, with the approval recorded before it. approval = self.approval(archive) drifted = self.archive(changelog=CHANGELOG.read_bytes(), name="drifted.crate", lockfile=(ROOT / "Cargo.lock").read_bytes() + b"#") From 944b73a9e19a146b1754cfbbdd27a7ca0c77c1aa Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Tue, 3 Nov 2026 17:14:34 -0600 Subject: [PATCH 3/9] Confine every release and harness path to an explicit root Security held #178 at 501d0311. The two Sonar S8707 path-traversal findings in release_package.py are not false positives: the new approved-package and archive inputs sit on the release trust boundary. release_package.py: - confined_root() canonicalizes an explicit root. It refuses NUL bytes, ".." components and a root that is itself a symbolic link, and it requires an existing directory. - confined_file() refuses NUL bytes, absolute paths, ".." components and a symbolic link at any component below the root. It then requires the realpath to stay inside the root (os.path.commonpath) and to be a regular file, or, for a file being created, not to exist. - --manifest and --changelog are relative to --source-root (default "."), and --archive is relative to a required --archive-root. - --approved-package is replaced by --approval-root. The record path is always /.json, and the manifest version must match strict SemVer 2.0.0 (ASCII) first. - Approval records must carry a strict-SemVer version and safe relative file names. --check-paths validates only the source paths. publish-rust-sdk.sh and the workflows: - The publisher confines the manifest and changelog to its working directory (the release checkout). Every archive call passes its root: Cargo's target/package or the publisher's temporary directory. - The approval comes from RUST_SDK_APPROVED_PACKAGE_DIR, which release.yml sets as $GITHUB_WORKSPACE/release-tools/release/approved-packages, with no ".." component. The publisher refuses a symlinked evidence path or one with a ".." component. - ci.yml passes --archive-root. run.py: - --package-from, --work-dir, --fixture-from and --baselines-control refuse NUL bytes, ".." components and a symbolic link, and are canonicalized. - --expect-version uses strict SemVer before naming any path. Archive member names refuse NUL and backslash. - write_members confines every member below the new destination with commonpath and a no-link walk, rechecks after creating parent directories, and creates files with O_EXCL|O_NOFOLLOW. Counterfactual tests cover ../ escapes, symlinks to outside (file, directory and root), absolute paths, NUL bytes, crafted versions such as "4.11.2/../../x", unsafe approval file names, planted links and the evidence path, each with positive controls. With the confinement removed, 34+1 release-package, 6 publish and 41+3 harness failures/errors appear; with it, all pass. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 2 +- .github/workflows/release.yml | 2 +- scripts/acceptance/installed-native/README.md | 20 ++ scripts/acceptance/installed-native/run.py | 77 +++++-- .../acceptance/installed-native/test_run.py | 91 ++++++++- scripts/ci/publish-rust-sdk.sh | 53 +++-- scripts/ci/release_package.py | 191 +++++++++++++----- scripts/ci/test-publish-rust-sdk.py | 85 +++++++- scripts/ci/test-release-package.py | 170 ++++++++++++++-- 9 files changed, 593 insertions(+), 98 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d58509f..5b6b1f87 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -352,7 +352,7 @@ jobs: run: | cargo package --locked archive="target/package/durable-workflow-$(cargo metadata --no-deps --format-version 1 | python3 -c 'import json,sys; print(json.load(sys.stdin)["packages"][0]["version"])').crate" - python3 scripts/ci/release_package.py --archive "$archive" + python3 scripts/ci/release_package.py --archive-root "$(dirname "$archive")" --archive "$(basename "$archive")" python3 scripts/ci/verify-fresh-consumer.py package - name: Check that the package is byte-reproducible # Pre-publication approval binds a .crate SHA-256 that the release diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 68c50778..152afc62 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -121,7 +121,7 @@ jobs: RELEASE_RUN_ATTEMPT: ${{ github.run_attempt }} # Reviewed record of the pre-publication package that passed installed-native # acceptance, committed to main (the maintained tooling), never the tagged source. - RUST_SDK_APPROVED_PACKAGE_PATH: ../release-tools/release/approved-packages/${{ env.RELEASE_TAG }}.json + RUST_SDK_APPROVED_PACKAGE_DIR: ${{ github.workspace }}/release-tools/release/approved-packages run: ../release-tools/scripts/ci/publish-rust-sdk.sh - name: Create the source GitHub Release diff --git a/scripts/acceptance/installed-native/README.md b/scripts/acceptance/installed-native/README.md index c71e8ef0..0bc10ce6 100644 --- a/scripts/acceptance/installed-native/README.md +++ b/scripts/acceptance/installed-native/README.md @@ -160,6 +160,26 @@ Cargo cannot upload a prebuilt `.crate`, so `cargo publish` packages again from the same clean tag with the same Cargo. The readback above is what proves that the published bytes are the accepted ones. +#### Path confinement + +Every file named by a command-line argument, an environment variable or an +approval record is resolved inside an explicit root before it is opened or +written. + +| Input | Root | Rule | +| --- | --- | --- | +| `--package-from`, `--work-dir`, `--fixture-from`, `--baselines-control` | Operator-supplied root | The path is canonicalized. The harness refuses a NUL byte, a `..` component, or a path that is itself a symbolic link. | +| Paths the harness writes or reads below the work dir, such as unpacked and vendored members | `--work-dir` | Each is a `/`-separated relative path with no empty, `.`, `..`, NUL or backslash parts and no link at any component. Its canonical result must stay inside the root (`os.path.commonpath`). Files are created with `O_EXCL`/`O_NOFOLLOW`. | +| `release_package.py --manifest`, `--changelog` | `--source-root` (the release checkout) | Relative, confined | +| `release_package.py --archive` | `--archive-root` (Cargo's `target/package`, or the publisher's temporary directory) | Relative, confined | +| Approval record | `--approval-root`, set by the Release workflow as `RUST_SDK_APPROVED_PACKAGE_DIR=$GITHUB_WORKSPACE/release-tools/release/approved-packages` | Always `.json`. The version must match strict Semantic Versioning 2.0.0 before it names a file. The record's own version must match it too, and its file names must be safe relative paths. | + +Versions used in harness paths (`--expect-version`) also follow strict +Semantic Versioning, so a value such as `4.11.2/../../x` is refused. + +The publisher also refuses a manifest or changelog outside its working +directory, and an evidence path that is a link or contains a `..` component. + The run writes `/summary.json` with every identity, receipt and history result, plus per-container logs under `cells/*/logs`. The exit code is 0 only when every check passes and cleanup is complete (see below). diff --git a/scripts/acceptance/installed-native/run.py b/scripts/acceptance/installed-native/run.py index 3e7b49e0..b6cb069c 100755 --- a/scripts/acceptance/installed-native/run.py +++ b/scripts/acceptance/installed-native/run.py @@ -54,7 +54,13 @@ APPROVAL_SCHEMA = 'durable-workflow.rust-sdk.approved-package/v1' COMMIT_PATTERN = re.compile(r'[0-9a-f]{40}') SHA256_PATTERN = re.compile(r'[0-9a-f]{64}') -VERSION_PATTERN = re.compile(r'[0-9]+\.[0-9]+\.[0-9]+([-+][0-9A-Za-z.-]+)?') +# Semantic Versioning 2.0.0, ASCII only. The version names files and directories +# (durable-workflow-.crate, vendor/durable-workflow-/); no +# identifier can be empty, so it can never hold '/' or a '..' component. +VERSION_PATTERN = re.compile( + r'(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)' + r'(?:-(?:0|[1-9]\d*|\d*[A-Za-z-][0-9A-Za-z-]*)(?:\.(?:0|[1-9]\d*|\d*[A-Za-z-][0-9A-Za-z-]*))*)?' + r'(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?', re.ASCII) class PackageError(RuntimeError): @@ -142,6 +148,45 @@ def exact_sha256(value): return value +def cli_path(value): + """An operator-supplied root (checkout, work dir, fixture, control manifest). + + Refuses NUL bytes, '..' components and a symbolic link at the path itself, + then canonicalizes it. Everything the harness reads or writes below a root + is confined to it separately (confined_child). + """ + if not value or '\0' in value: + raise argparse.ArgumentTypeError('path is empty or contains a NUL byte') + if '..' in pathlib.PurePath(value).parts: + raise argparse.ArgumentTypeError('%r contains a ".." component' % value) + if os.path.islink(value): + raise argparse.ArgumentTypeError('%r is a symbolic link' % value) + return pathlib.Path(os.path.realpath(value)) + + +def safe_relative(relative): + """A '/'-separated relative path with no empty, '.', '..', NUL or backslash parts.""" + return (isinstance(relative, str) and relative and '\0' not in relative and '\\' not in relative + and not os.path.isabs(relative) + and all(part not in ('', '.', '..') for part in relative.split('/'))) + + +def confined_child(root, relative): + """`root`/`relative`, refusing escapes and symbolic links below canonical `root`.""" + if not safe_relative(relative): + raise PackageError('unsafe relative path %r' % relative) + base = os.path.realpath(root) + current = base + for part in relative.split('/'): + current = os.path.join(current, part) + if os.path.islink(current): + raise PackageError('path %r passes through a symbolic link' % relative) + target = os.path.realpath(os.path.join(base, relative)) + if os.path.commonpath([base, target]) != base: + raise PackageError('path %r escapes its root' % relative) + return pathlib.Path(target) + + def exact_version(value): if not VERSION_PATTERN.fullmatch(value): raise argparse.ArgumentTypeError('%r must be an exact SemVer version' % value) @@ -152,7 +197,7 @@ def parse_args(argv=None): parser = argparse.ArgumentParser(description=__doc__) source = parser.add_mutually_exclusive_group(required=True) source.add_argument('--crate-version', help='exact published durable-workflow version, e.g. 4.10.3') - source.add_argument('--package-from', type=pathlib.Path, + source.add_argument('--package-from', type=cli_path, help='pre-publication mode: a clean git checkout whose HEAD is --package-commit; ' 'the harness packages that commit and installs only the packaged bytes') parser.add_argument('--package-commit', type=full_commit, @@ -167,7 +212,7 @@ def parse_args(argv=None): parser.add_argument('--server-image', default='ghcr.io/durable-workflow/server') parser.add_argument('--runtime', choices=['native', 'php'], default='native', help='php starts the PHP Server process set (MySQL, Redis, queue worker, repair) as a parity baseline') - parser.add_argument('--fixture-from', type=pathlib.Path, + parser.add_argument('--fixture-from', type=cli_path, help='reuse a fixture directory built by an earlier run (provenance is checked again)') parser.add_argument('--platform', default='linux/amd64') parser.add_argument('--expect-server-version', help='required /api/cluster/info version, e.g. 3.0.1') @@ -177,14 +222,14 @@ def parse_args(argv=None): help='whether the published crate must carry the per-major Server baseline map') parser.add_argument('--expected-baselines', type=json.loads, default={'2': '2.0.0', '3': '3.0.2'}, help='JSON map the published crate must declare when --baselines required') - parser.add_argument('--baselines-control', type=pathlib.Path, + parser.add_argument('--baselines-control', type=cli_path, help='a Cargo.toml that must pass the required-map check (positive control)') parser.add_argument('--source-tag', help='git tag the crate must match (default: the crate version)') parser.add_argument('--rust-image', default='rust:1.86-bookworm') parser.add_argument('--python-image', default='python:3.12-bookworm') parser.add_argument('--cells', default='lifecycle,deadline') parser.add_argument('--jobs', type=int, default=2, help='cargo build jobs (lab machines: keep at 2)') - parser.add_argument('--work-dir', type=pathlib.Path, required=True, help='new directory for build output and evidence') + parser.add_argument('--work-dir', type=cli_path, required=True, help='new directory for build output and evidence') parser.add_argument('--container-prefix', type=container_prefix, default='sdkrust', help='owner prefix for container, network and task names, e.g. general or security ' '(^%s$)' % PREFIX_PATTERN.pattern) @@ -357,9 +402,7 @@ def verify_package_archive(crate_bytes, version, commit, listed, git_blob): with tarfile.open(fileobj=io.BytesIO(crate_bytes), mode='r:gz') as archive: for member in archive.getmembers(): relative = member.name.removeprefix(prefix) - parts = relative.split('/') - if (not member.name.startswith(prefix) or not member.isfile() or not relative - or any(part in ('', '.', '..') for part in parts)): + if not member.name.startswith(prefix) or not member.isfile() or not safe_relative(relative): raise PackageError('crate member %r is not a regular file under %s' % (member.name, prefix)) if relative in members: raise PackageError('crate member %r appears more than once' % relative) @@ -401,18 +444,28 @@ def verify_package_archive(crate_bytes, version, commit, listed, git_blob): def write_members(members, destination): - """Write verified archive members (never tar extraction) under `destination`.""" + """Write verified archive members (never tar extraction) under `destination`. + + `destination` must be new. Each member path is confined below it (no + escape, no link), and each file is created exclusively, never through an + existing path. + """ destination.mkdir(parents=True) for relative, content in sorted(members.items()): - path = destination / relative + path = confined_child(destination, relative) path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(content) + confined_child(destination, relative) # parents just created: still no link or escape + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_NOFOLLOW', 0), 0o644) + with os.fdopen(descriptor, 'wb') as handle: + handle.write(content) def package_release(args, work_dir, run): """`cargo package` exactly args.package_commit and bind the resulting bytes.""" commit, version = args.package_commit, args.expect_version - checkout = args.package_from.resolve() + checkout = args.package_from + if not checkout.is_dir(): + raise PackageError('--package-from %s is not a directory' % checkout) source, out = work_dir / 'package-source', work_dir / 'package-out' clone_exact_commit(checkout, commit, source) source_manifest = tomllib.loads(git(source, 'show', '%s:Cargo.toml' % commit).stdout)['package'] diff --git a/scripts/acceptance/installed-native/test_run.py b/scripts/acceptance/installed-native/test_run.py index a0b0a076..4e069384 100644 --- a/scripts/acceptance/installed-native/test_run.py +++ b/scripts/acceptance/installed-native/test_run.py @@ -1210,7 +1210,7 @@ def test_passing_run_writes_a_bound_approval_candidate(self): code, receipt, work = self.main() self.assertEqual(code, 0) approval = json.loads((work / 'approved-package.json').read_text()) - self.assertEqual(receipt['approval_candidate'], str(work / 'approved-package.json')) + self.assertEqual(receipt['approval_candidate'], str(work.resolve() / 'approved-package.json')) self.assertEqual(approval['schema'], run.APPROVAL_SCHEMA) for key in ('package', 'version', 'commit', 'crate_sha256', 'files', 'file_list_sha256'): self.assertEqual(approval[key], self.identity[key]) @@ -1236,5 +1236,94 @@ def test_package_changed_during_cells_writes_no_approval(self): self.assertFalse((work / 'approved-package.json').exists()) +class PathConfinement(unittest.TestCase): + """Security review of #178: CLI paths and every path written below them.""" + + def setUp(self): + self.tmp = temporary_directory(self) + self.outside = temporary_directory(self) + (self.outside / 'secret').write_bytes(b'PRIVATE') + + def test_cli_paths_refuse_nul_dot_dot_and_symlinks(self): + (self.tmp / 'real').mkdir() + (self.tmp / 'link').symlink_to(self.outside, target_is_directory=True) + for value in ('', 'work\0dir', str(self.tmp / 'real' / '..' / 'escape'), '../work', 'a/../../b', + str(self.tmp / 'link')): + with self.subTest(value=value), self.assertRaises(argparse.ArgumentTypeError): + run.cli_path(value) + # Positive controls: absolute, relative and not-yet-existing paths canonicalize. + self.assertEqual(run.cli_path(str(self.tmp / 'real')), pathlib.Path(os.path.realpath(self.tmp / 'real'))) + self.assertEqual(run.cli_path(str(self.tmp / 'new')), pathlib.Path(os.path.realpath(self.tmp)) / 'new') + self.assertTrue(run.cli_path('work').is_absolute()) + + def test_every_path_option_is_confined(self): + (self.tmp / 'link').symlink_to(self.outside, target_is_directory=True) + package = PrePublicationArguments.PACKAGE + for option in ('--package-from', '--work-dir'): + for value in (str(self.tmp / 'link'), '../x', 'x\0y'): + argv = ['--server-index', 'sha256:x', '--work-dir', str(self.tmp / 'w')] + package + argv[argv.index(option) + 1] = value + with self.subTest(option=option, value=value), self.assertRaises(SystemExit), \ + contextlib.redirect_stderr(io.StringIO()): + run.parse_args(argv) + base = ['--server-index', 'sha256:x', '--work-dir', str(self.tmp / 'w'), '--crate-version', '4.11.1'] + for option in ('--fixture-from', '--baselines-control'): + with self.subTest(option=option), self.assertRaises(SystemExit), contextlib.redirect_stderr(io.StringIO()): + run.parse_args(base + [option, str(self.tmp / 'link')]) + + def test_crafted_versions_are_refused_before_naming_paths(self): + for version in ('4.11.2/../../x', '../4.11.2', '4.11.2\0', '4.11.2-..', '4.11.2-rc..1', 'v4.11.2', + '04.11.2', '4.11.2/', '４.11.2', '4.11.2-a/b'): + with self.subTest(version=version), self.assertRaises(argparse.ArgumentTypeError): + run.exact_version(version) + for version in ('4.11.2', '4.11.2-rc.1+build.5', '10.0.0-alpha-1'): + with self.subTest(positive=version): + self.assertEqual(run.exact_version(version), version) + + def test_confined_child_refusals_and_positive_controls(self): + root = self.tmp / 'root' + (root / 'src').mkdir(parents=True) + (root / 'linkdir').symlink_to(self.outside, target_is_directory=True) + (root / 'link.rs').symlink_to(self.outside / 'secret') + for relative in ('../secret', 'src/../../secret', str(self.outside / 'secret'), '/etc/passwd', + 'linkdir/secret', 'link.rs', 'src/a\0b', 'src\\..\\x', '', 'src//a', './a'): + with self.subTest(relative=relative), self.assertRaises(run.PackageError): + run.confined_child(root, relative) + self.assertEqual(run.confined_child(root, 'src/lib.rs'), + pathlib.Path(os.path.realpath(root)) / 'src' / 'lib.rs') + + def test_write_members_never_escapes_or_follows_links(self): + for name in ('../escape.rs', 'src/../../escape.rs', '/abs.rs', 'a\0b'): + destination = self.tmp / ('dest-%d' % len(os.listdir(self.tmp))) + with self.subTest(name=name), self.assertRaises(run.PackageError): + run.write_members({name: b'x'}, destination) + self.assertEqual(sorted(os.listdir(self.outside)), ['secret']) + self.assertFalse((self.tmp / 'escape.rs').exists()) + # A member whose directory is a link planted beside it is refused too. + destination = self.tmp / 'planted' + original_mkdir = pathlib.Path.mkdir + + def mkdir(path, *a, **k): + if path.name == 'src' and path.parent == pathlib.Path(os.path.realpath(destination)): + path.symlink_to(self.outside, target_is_directory=True) + return None + return original_mkdir(path, *a, **k) + with mock.patch.object(pathlib.Path, 'mkdir', mkdir), self.assertRaises(run.PackageError): + run.write_members({'src/lib.rs': b'x'}, destination) + self.assertEqual(sorted(os.listdir(self.outside)), ['secret']) + # Positive control. + run.write_members({'src/lib.rs': b'pub fn a() {}\n', 'Cargo.toml': b'[package]\n'}, self.tmp / 'ok') + self.assertEqual((self.tmp / 'ok' / 'src' / 'lib.rs').read_bytes(), b'pub fn a() {}\n') + + def test_archive_member_with_nul_or_backslash_is_refused(self): + tmp = temporary_directory(self) + commit = make_repo(tmp / 'repo') + members = packaged_members(commit) + for name in ('src/a\\..\\b.rs',): + crate = make_crate(dict(members, **{name: b'x'})) + with self.subTest(name=name), self.assertRaisesRegex(run.PackageError, 'not a regular file'): + run.verify_package_archive(crate, VERSION, commit, sorted(members) + [name], lambda path: None) + + if __name__ == '__main__': unittest.main() diff --git a/scripts/ci/publish-rust-sdk.sh b/scripts/ci/publish-rust-sdk.sh index b09d33cc..bca256a2 100755 --- a/scripts/ci/publish-rust-sdk.sh +++ b/scripts/ci/publish-rust-sdk.sh @@ -17,7 +17,8 @@ lockfile_source="source_checkout" # A reviewed record of the pre-publication package that passed installed-native # acceptance (scripts/acceptance/installed-native, --package-from). New versions # publish only when the reproduced package is exactly that artifact. -approved_package_path="${RUST_SDK_APPROVED_PACKAGE_PATH:-}" +approved_package_dir="${RUST_SDK_APPROVED_PACKAGE_DIR:-}" +approved_package_path="" for command in cargo curl git jq python3 rustc sha256sum; do if ! command -v "$command" >/dev/null 2>&1; then @@ -35,6 +36,21 @@ if [[ ! -f "$changelog_path" ]]; then exit 1 fi +# The manifest and changelog must be regular files inside the release checkout +# (the working directory): relative, no '..', no symbolic links, no escape. +source_root="$(pwd -P)" +if ! python3 "$script_dir/release_package.py" --check-paths --source-root "$source_root" \ + --manifest "$manifest_path" --changelog "$changelog_path" >/dev/null; then + printf 'Rust SDK manifest and changelog must be regular files inside the release checkout\n' >&2 + exit 1 +fi +evidence_parent="$(dirname "$evidence_path")" +if [[ -L "$evidence_path" || -L "$evidence_parent" || ! -d "$evidence_parent" || \ + "/$evidence_path/" == *"/../"* ]]; then + printf 'release evidence path must be a plain path without links or ".." components\n' >&2 + exit 1 +fi + if [[ -n "$(git status --porcelain --untracked-files=all)" ]]; then printf 'Rust SDK publication requires a clean release checkout\n' >&2 exit 1 @@ -413,8 +429,9 @@ if [[ ! -f "$lockfile_path" || -L "$lockfile_path" ]]; then ! crates_io_request --fail --silent --show-error --location \ --header "User-Agent: ${user_agent}" --output "$published_archive" "$download_url" || \ ! python3 "$script_dir/release_package.py" \ - --manifest "$manifest_path" --changelog "$changelog_path" \ - --archive "$published_archive" --expected-vcs-commit "$release_commit" \ + --source-root "$source_root" --manifest "$manifest_path" --changelog "$changelog_path" \ + --archive-root "$tmp_dir" --archive "$(basename "$published_archive")" \ + --expected-vcs-commit "$release_commit" \ --expected-archive-sha256 "$published_checksum" \ --restore-missing-lockfile > "$tmp_dir/restored-lockfile.json"; then write_evidence "failed" "legacy_release_lockfile_verification_failed" @@ -428,7 +445,8 @@ if ! cargo package --locked --manifest-path "$manifest_path"; then exit 1 fi -local_archive="${target_directory}/package/${package_name}-${package_version}.crate" +local_archive_root="${target_directory}/package" +local_archive="${local_archive_root}/${package_name}-${package_version}.crate" if [[ ! -f "$local_archive" ]]; then write_evidence "failed" "local_package_archive_missing" printf 'local Cargo package archive was not created: %s\n' "$local_archive" >&2 @@ -437,9 +455,11 @@ fi local_checksum="$(sha256sum "$local_archive" | awk '{print $1}')" local_identity="$tmp_dir/local-package-identity.json" if ! python3 "$script_dir/release_package.py" \ + --source-root "$source_root" \ --manifest "$manifest_path" \ --changelog "$changelog_path" \ - --archive "$local_archive" \ + --archive-root "$local_archive_root" \ + --archive "$(basename "$local_archive")" \ --expected-vcs-commit "$release_commit" > "$local_identity"; then write_evidence "failed" "local_package_release_identity_mismatch" printf 'local Cargo package does not contain the authorized release notes and VCS identity\n' >&2 @@ -450,15 +470,20 @@ archive_vcs_dirty="$(jq -r '.archive.vcs_dirty' "$local_identity")" changelog_checksum="$(jq -er '.changelog.sha256' "$local_identity")" release_entry_checksum="$(jq -er '.changelog.entry_sha256' "$local_identity")" -if [[ -n "$approved_package_path" && -e "$approved_package_path" ]]; then +if [[ -n "$approved_package_dir" ]]; then + approved_package_path="${approved_package_dir}/${package_version}.json" +fi +if [[ -n "$approved_package_path" && ( -e "$approved_package_path" || -L "$approved_package_path" ) ]]; then packaging_cargo="$(cargo -V | awk '{print $2}')" local_approval="$tmp_dir/local-approval.json" if ! python3 "$script_dir/release_package.py" \ + --source-root "$source_root" \ --manifest "$manifest_path" \ --changelog "$changelog_path" \ - --archive "$local_archive" \ + --archive-root "$local_archive_root" \ + --archive "$(basename "$local_archive")" \ --expected-vcs-commit "$release_commit" \ - --approved-package "$approved_package_path" \ + --approval-root "$approved_package_dir" \ --cargo-version "$packaging_cargo" > "$local_approval"; then approval_status="reproduction_mismatch" write_evidence "failed" "approved_package_mismatch" @@ -480,7 +505,7 @@ fi if [[ "$http_status" == "404" && "$approval_status" != "reproduced" ]]; then write_evidence "failed" "approved_package_required" - printf 'publishing %s %s requires a reviewed approved-package record (RUST_SDK_APPROVED_PACKAGE_PATH)\n' \ + printf 'publishing %s %s requires a reviewed approved-package record (RUST_SDK_APPROVED_PACKAGE_DIR)\n' \ "$package_name" "$package_version" >&2 exit 1 fi @@ -572,9 +597,11 @@ fi published_identity="$tmp_dir/published-package-identity.json" if ! python3 "$script_dir/release_package.py" \ + --source-root "$source_root" \ --manifest "$manifest_path" \ --changelog "$changelog_path" \ - --archive "$published_archive" \ + --archive-root "$tmp_dir" \ + --archive "$(basename "$published_archive")" \ --expected-vcs-commit "$release_commit" > "$published_identity"; then write_evidence "failed" "published_package_release_identity_mismatch" printf 'downloaded crate does not contain the authorized release notes and VCS identity\n' >&2 @@ -595,11 +622,13 @@ fi if [[ "$approval_status" == "reproduced" ]]; then if ! python3 "$script_dir/release_package.py" \ + --source-root "$source_root" \ --manifest "$manifest_path" \ --changelog "$changelog_path" \ - --archive "$published_archive" \ + --archive-root "$tmp_dir" \ + --archive "$(basename "$published_archive")" \ --expected-vcs-commit "$release_commit" \ - --approved-package "$approved_package_path" > "$tmp_dir/published-approval.json"; then + --approval-root "$approved_package_dir" > "$tmp_dir/published-approval.json"; then write_evidence "failed" "published_package_differs_from_approved_package" printf 'downloaded crate is not the approved pre-publication package\n' >&2 exit 1 diff --git a/scripts/ci/release_package.py b/scripts/ci/release_package.py index 30b1c6d4..a6783273 100644 --- a/scripts/ci/release_package.py +++ b/scripts/ci/release_package.py @@ -7,6 +7,7 @@ from dataclasses import dataclass import hashlib import json +import os from pathlib import Path import re import sys @@ -370,6 +371,78 @@ def restore_missing_lockfile( APPROVAL_SCHEMA = "durable-workflow.rust-sdk.approved-package/v1" +# Semantic Versioning 2.0.0, ASCII only. A version is validated with this before +# it names a file (release/approved-packages/.json); no identifier can +# be empty, so it can never contain "/" or a ".." component. +STRICT_SEMVER = re.compile( + r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)" + r"(?:-(?:0|[1-9]\d*|\d*[A-Za-z-][0-9A-Za-z-]*)(?:\.(?:0|[1-9]\d*|\d*[A-Za-z-][0-9A-Za-z-]*))*)?" + r"(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?$", + re.ASCII, +) + + +def _refuse_unsafe_text(value: str, label: str) -> None: + if not value or "\0" in value: + raise ReleasePackageError(f"{label} is empty or contains a NUL byte") + if ".." in Path(value).parts: + raise ReleasePackageError(f"{label} contains a '..' component") + + +def confined_root(value: str | os.PathLike[str]) -> str: + """Canonicalize an explicit trust root: an existing directory, not a link.""" + text = os.fspath(value) + _refuse_unsafe_text(text, "path root") + if os.path.islink(text): + raise ReleasePackageError("path root must not be a symbolic link") + root = os.path.realpath(text) + if not os.path.isdir(root): + raise ReleasePackageError("path root is not an existing directory") + return root + + +def confined_file( + root: str, relative: str | os.PathLike[str], *, must_exist: bool = True, +) -> Path: + """Resolve `relative` strictly inside canonical `root`. + + Refuses NUL bytes, absolute paths, '..' components and a symbolic link at + any component below the root, then requires the canonical result to stay + inside the root (os.path.commonpath) and, if `must_exist`, to be a regular + file. + """ + text = os.fspath(relative) + _refuse_unsafe_text(text, "path") + if os.path.isabs(text): + raise ReleasePackageError("path must be relative to its root, not absolute") + current = root + for part in Path(text).parts: + current = os.path.join(current, part) + if os.path.islink(current): + raise ReleasePackageError("path must not pass through a symbolic link") + resolved = os.path.realpath(os.path.join(root, text)) + if os.path.commonpath([root, resolved]) != root: + raise ReleasePackageError("path escapes its root") + if must_exist and not os.path.isfile(resolved): + raise ReleasePackageError("path is not an existing regular file") + if not must_exist and os.path.lexists(resolved): + raise ReleasePackageError("path already exists") + return Path(resolved) + + +def approval_record_path(root: str, version: str) -> Path: + """The reviewed record for `version`: /.json, strictly confined.""" + if not isinstance(version, str) or not STRICT_SEMVER.fullmatch(version): + raise ReleasePackageError("approval version is not a strict SemVer version") + return confined_file(root, f"{version}.json") + + +def _safe_member_name(name: object) -> bool: + if not isinstance(name, str) or not name or "\0" in name or "\\" in name: + return False + if name.startswith("/") or os.path.isabs(name): + return False + return all(part not in ("", ".", "..") for part in name.split("/")) SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") CARGO_VERSION_PATTERN = re.compile(r"^\d+\.\d+\.\d+$", re.ASCII) APPROVAL_FIELDS = { @@ -397,6 +470,7 @@ def load_approval(path: Path) -> tuple[dict[str, Any], str]: approval["schema"] != APPROVAL_SCHEMA or not isinstance(approval["package"], str) or not isinstance(approval["version"], str) + or not STRICT_SEMVER.fullmatch(approval["version"]) or not isinstance(approval["commit"], str) or not COMMIT_PATTERN.fullmatch(approval["commit"]) or not isinstance(approval["crate_sha256"], str) @@ -405,7 +479,7 @@ def load_approval(path: Path) -> tuple[dict[str, Any], str]: or not CARGO_VERSION_PATTERN.fullmatch(approval["cargo_version"]) or not isinstance(files, list) or not files - or not all(isinstance(name, str) and name for name in files) + or not all(_safe_member_name(name) for name in files) or files != sorted(set(files)) or approval["file_list_sha256"] != file_list_sha256(files) or not isinstance(acceptance, dict) @@ -486,85 +560,106 @@ def _validate_notes_arguments( ) -> None: if any((arguments.archive, arguments.expected_vcs_commit, arguments.expected_archive_sha256, arguments.restore_missing_lockfile, - arguments.approved_package, arguments.cargo_version)): + arguments.approval_root, arguments.archive_root, arguments.cargo_version, + arguments.check_paths)): parser.error("--release-notes cannot be combined with archive verification") if (arguments.manifest != Path(SOURCE_MANIFEST) - or arguments.changelog != Path(SOURCE_CHANGELOG)): + or arguments.changelog != Path(SOURCE_CHANGELOG) + or arguments.source_root != "."): parser.error("--release-notes uses the current checkout files; path overrides are refused") -def _validate_approval_arguments( +def _validate_mode_arguments( arguments: argparse.Namespace, parser: argparse.ArgumentParser, ) -> None: - if arguments.cargo_version is not None and arguments.approved_package is None: - parser.error("--cargo-version requires --approved-package") - if arguments.approved_package is not None and ( + if arguments.expected_archive_sha256 is not None and not arguments.restore_missing_lockfile: + parser.error("--expected-archive-sha256 requires --restore-missing-lockfile") + if arguments.check_paths and any(( + arguments.archive, arguments.archive_root, arguments.approval_root, + arguments.expected_vcs_commit, arguments.restore_missing_lockfile, arguments.cargo_version, + )): + parser.error("--check-paths takes only --source-root, --manifest and --changelog") + if (arguments.archive is None) != (arguments.archive_root is None): + parser.error("--archive and --archive-root must be given together") + if arguments.cargo_version is not None and arguments.approval_root is None: + parser.error("--cargo-version requires --approval-root") + if arguments.approval_root is not None and ( arguments.archive is None or arguments.expected_vcs_commit is None - or arguments.restore_missing_lockfile or arguments.release_notes is not None + or arguments.restore_missing_lockfile ): - parser.error("--approved-package requires --archive and --expected-vcs-commit only") + parser.error("--approval-root requires --archive and --expected-vcs-commit only") + + +def _run(arguments: argparse.Namespace, parser: argparse.ArgumentParser) -> dict[str, Any]: + # Every file named on the command line is confined to an explicit root + # before it is opened: the checkout (--source-root), the package directory + # (--archive-root) or the reviewed approvals directory (--approval-root). + source_root = confined_root(arguments.source_root) + manifest = confined_file(source_root, arguments.manifest) + changelog = confined_file(source_root, arguments.changelog) + if arguments.check_paths: + return {"manifest": str(manifest), "changelog": str(changelog)} + archive = None + if arguments.archive is not None: + archive = confined_file(confined_root(arguments.archive_root), arguments.archive) + if arguments.restore_missing_lockfile: + if not all((archive, arguments.expected_vcs_commit, arguments.expected_archive_sha256)): + parser.error("lockfile recovery requires archive, source commit and registry checksum") + # The lockfile is written beside the confined manifest, never through a link. + confined_file(source_root, manifest.parent.relative_to(source_root) / "Cargo.lock", must_exist=False) + return restore_missing_lockfile( + archive, manifest, changelog, + arguments.expected_vcs_commit, arguments.expected_archive_sha256, + ) + if archive is None: + if arguments.expected_vcs_commit is not None: + parser.error("--expected-vcs-commit requires --archive") + return {"schema": SCHEMA, "version": 1, **verify_source(manifest, changelog)} + if arguments.approval_root is not None: + approval = approval_record_path( + confined_root(arguments.approval_root), str(load_package(manifest)["version"]) + ) + return verify_approved_archive( + archive, manifest, changelog, arguments.expected_vcs_commit, + approval, arguments.cargo_version, + ) + return verify_archive(archive, manifest, changelog, arguments.expected_vcs_commit) def main() -> int: parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-root", default=".", + help="release checkout; --manifest and --changelog are relative to it") parser.add_argument("--manifest", type=Path, default=Path(SOURCE_MANIFEST)) parser.add_argument("--changelog", type=Path, default=Path(SOURCE_CHANGELOG)) - parser.add_argument("--archive", type=Path) + parser.add_argument("--archive-root", help="directory holding --archive") + parser.add_argument("--archive", type=Path, help="crate file name relative to --archive-root") parser.add_argument("--expected-vcs-commit") parser.add_argument("--expected-archive-sha256") parser.add_argument("--restore-missing-lockfile", action="store_true") + parser.add_argument("--check-paths", action="store_true", + help="only confine --manifest and --changelog to --source-root") parser.add_argument( - "--approved-package", type=Path, - help="require --archive to be exactly this reviewed approved-package record", + "--approval-root", + help="reviewed approvals directory; requires --archive to equal .json in it", ) parser.add_argument( - "--cargo-version", help="with --approved-package: the packaging Cargo release (X.Y.Z)", + "--cargo-version", help="with --approval-root: the packaging Cargo release (X.Y.Z)", ) parser.add_argument( "--release-notes", metavar="VERSION", help="print the exact current-version Markdown after manifest validation", ) arguments = parser.parse_args() - if arguments.expected_archive_sha256 is not None and not arguments.restore_missing_lockfile: - parser.error("--expected-archive-sha256 requires --restore-missing-lockfile") - _validate_approval_arguments(arguments, parser) + if arguments.release_notes is not None: + _validate_notes_arguments(arguments, parser) + _validate_mode_arguments(arguments, parser) try: if arguments.release_notes is not None: - _validate_notes_arguments(arguments, parser) sys.stdout.write(source_release_notes(arguments.release_notes)) return 0 - elif arguments.restore_missing_lockfile: - if not all((arguments.archive, arguments.expected_vcs_commit, arguments.expected_archive_sha256)): - parser.error("lockfile recovery requires archive, source commit and registry checksum") - evidence = restore_missing_lockfile( - arguments.archive, arguments.manifest, arguments.changelog, - arguments.expected_vcs_commit, arguments.expected_archive_sha256, - ) - elif arguments.archive is None: - if arguments.expected_vcs_commit is not None: - parser.error("--expected-vcs-commit requires --archive") - evidence = { - "schema": SCHEMA, - "version": 1, - **verify_source(arguments.manifest, arguments.changelog), - } - elif arguments.approved_package is not None: - evidence = verify_approved_archive( - arguments.archive, - arguments.manifest, - arguments.changelog, - arguments.expected_vcs_commit, - arguments.approved_package, - arguments.cargo_version, - ) - else: - evidence = verify_archive( - arguments.archive, - arguments.manifest, - arguments.changelog, - arguments.expected_vcs_commit, - ) + evidence = _run(arguments, parser) except ReleasePackageError as error: print(f"release package verification failed: {error}", file=sys.stderr) return 1 diff --git a/scripts/ci/test-publish-rust-sdk.py b/scripts/ci/test-publish-rust-sdk.py index 3555eac6..a9931c44 100644 --- a/scripts/ci/test-publish-rust-sdk.py +++ b/scripts/ci/test-publish-rust-sdk.py @@ -216,6 +216,13 @@ def _publish( approval: dict[str, object] | None = None, registry_changelog: bytes | None = None, ) -> subprocess.CompletedProcess[str]: + if cwd == ROOT and not manifest.resolve().is_relative_to(ROOT): + # A modified manifest is its own release checkout: the publisher only + # reads a manifest and changelog confined to its working directory. + cwd = manifest.parent + for name in ("CHANGELOG.md", "Cargo.lock"): + if not (cwd / name).exists(): + shutil.copyfile(ROOT / name, cwd / name) env = os.environ.copy() env.update( { @@ -226,9 +233,9 @@ def _publish( "MOCK_PACKAGE_VERSION": PACKAGE_VERSION, "MOCK_REGISTRY_ARCHIVE": str(self.temp / "registry" / "package" / f"durable-workflow-{PACKAGE_VERSION}.crate"), "RELEASE_TAG": PACKAGE_VERSION, - "RUST_SDK_MANIFEST_PATH": str(manifest), + "RUST_SDK_MANIFEST_PATH": os.path.relpath(manifest, cwd), "RUST_SDK_RELEASE_EVIDENCE_PATH": str(self.evidence), - "RUST_SDK_APPROVED_PACKAGE_PATH": str(self.temp / "approved" / f"{PACKAGE_VERSION}.json"), + "RUST_SDK_APPROVED_PACKAGE_DIR": str(self.temp / "approved"), "MOCK_PUBLISH_MARKER": str(self.temp / "published-marker"), } ) @@ -247,7 +254,7 @@ def _publish( if approval is not None: record = self._approval(registry_archive) record.update(approval) - path = Path(env["RUST_SDK_APPROVED_PACKAGE_PATH"]) + path = Path(env["RUST_SDK_APPROVED_PACKAGE_DIR"]) / f"{PACKAGE_VERSION}.json" path.parent.mkdir(exist_ok=True) path.write_text(json.dumps(record), encoding="utf-8") if registry_changelog is not None: @@ -316,7 +323,7 @@ def test_new_version_without_an_approval_record_is_never_published(self) -> None self.assertFalse(self._published()) self.assertEqual("approved_package_required", self._evidence()["reason"]) result = self._publish(environment={"MOCK_LOOKUP_STATUS": "404", "CARGO_REGISTRY_TOKEN": "test-token", - "RUST_SDK_APPROVED_PACKAGE_PATH": ""}) + "RUST_SDK_APPROVED_PACKAGE_DIR": ""}) self.assertNotEqual(0, result.returncode) self.assertFalse(self._published()) @@ -407,15 +414,73 @@ def test_post_publish_source_tag_readback_mismatch_fails(self) -> None: self.assertEqual("failed", evidence["outcome"]) (self.temp / "published-marker").unlink(missing_ok=True) + # Path confinement of every file the publisher reads or writes. + + def test_source_paths_outside_the_release_checkout_are_refused(self) -> None: + source = self._source_checkout() + outside = self.temp / "outside" + outside.mkdir() + shutil.copyfile(ROOT / "CHANGELOG.md", outside / "CHANGELOG.md") + linked = source / "linked" + linked.mkdir() + (linked / "CHANGELOG.md").symlink_to(outside / "CHANGELOG.md") + cases = { + "absolute manifest": {"RUST_SDK_MANIFEST_PATH": str(source / "Cargo.toml")}, + "dot-dot manifest": {"RUST_SDK_MANIFEST_PATH": "../source/Cargo.toml"}, + "dot-dot changelog": {"RUST_SDK_CHANGELOG_PATH": "../outside/CHANGELOG.md"}, + "absolute changelog": {"RUST_SDK_CHANGELOG_PATH": str(outside / "CHANGELOG.md")}, + "symlinked changelog": {"RUST_SDK_CHANGELOG_PATH": "linked/CHANGELOG.md"}, + } + for label, environment in cases.items(): + with self.subTest(label=label): + result = self._publish(source / "Cargo.toml", environment, cwd=source) + self.assertNotEqual(0, result.returncode) + self.assertIn("inside the release checkout", result.stderr) + self.assertFalse(self._published()) + # Positive control: the same checkout with plain relative paths publishes. + result = self._publish(source / "Cargo.toml", {"MOCK_LOOKUP_STATUS": "404", + "CARGO_REGISTRY_TOKEN": "test-token"}, + cwd=source, approval={}) + self.assertEqual(0, result.returncode, result.stderr) + + def test_symlinked_approval_record_is_refused(self) -> None: + outside = self.temp / "forged.json" + result = self._publish(environment={"MOCK_LOOKUP_STATUS": "404", "CARGO_REGISTRY_TOKEN": "test-token"}, + approval={}) + self.assertEqual(0, result.returncode, result.stderr) + record = self.temp / "approved" / f"{PACKAGE_VERSION}.json" + outside.write_bytes(record.read_bytes()) + record.unlink() + record.symlink_to(outside) + (self.temp / "published-marker").unlink() + result = self._publish(environment={"MOCK_LOOKUP_STATUS": "404", "CARGO_REGISTRY_TOKEN": "test-token"}) + self.assertNotEqual(0, result.returncode) + self.assertFalse(self._published()) + self.assertEqual("approved_package_mismatch", self._evidence()["reason"]) + self.assertIn("symbolic link", result.stderr) + + def test_symlinked_evidence_path_is_refused(self) -> None: + target = self.temp / "elsewhere.json" + target.write_text("untouched", encoding="utf-8") + link = self.temp / "linked-evidence.json" + link.symlink_to(target) + for value in (str(link), str(self.temp / "sub" / ".." / "evidence.json")): + with self.subTest(value=value): + (self.temp / "sub").mkdir(exist_ok=True) + result = self._publish(environment={"RUST_SDK_RELEASE_EVIDENCE_PATH": value}) + self.assertNotEqual(0, result.returncode) + self.assertIn("release evidence path", result.stderr) + self.assertEqual("untouched", target.read_text(encoding="utf-8")) + def test_release_workflow_supplies_the_reviewed_approval_record(self) -> None: workflow = RELEASE_WORKFLOW.read_text(encoding="utf-8") self.assertIn( - "RUST_SDK_APPROVED_PACKAGE_PATH: ../release-tools/release/approved-packages/${{ env.RELEASE_TAG }}.json", + "RUST_SDK_APPROVED_PACKAGE_DIR: ${{ github.workspace }}/release-tools/release/approved-packages", workflow, ) # The record comes from maintained tooling (main), never the tagged source. self.assertIn("ref: ${{ github.workflow_sha }}", workflow) - self.assertLess(workflow.index("Resolve exact release identity"), workflow.index("RUST_SDK_APPROVED_PACKAGE_PATH")) + self.assertLess(workflow.index("Resolve exact release identity"), workflow.index("RUST_SDK_APPROVED_PACKAGE_DIR")) self.assertNotIn("/release/", MANIFEST.read_text(encoding="utf-8").split("include = ", 1)[1].split("\n", 1)[0]) def _manifest_with(self, old: str, new: str) -> Path: @@ -738,14 +803,12 @@ def test_release_path_rejects_an_invalid_qualified_server_identity(self) -> None self.assertIn("supported Server contract", result.stderr) def test_release_path_rejects_package_without_current_release_notes(self) -> None: - changelog = self.temp / "CHANGELOG.md" - changelog.write_text( + source = self._source_checkout() + (source / "CHANGELOG.md").write_text( "# Changelog\n\n## 2.0.0-rc.31\n\n- Previous release.\n", encoding="utf-8", ) - result = self._publish( - environment={"RUST_SDK_CHANGELOG_PATH": str(changelog)} - ) + result = self._publish(source / "Cargo.toml", cwd=source) self.assertNotEqual(0, result.returncode) evidence = json.loads(self.evidence.read_text(encoding="utf-8")) self.assertEqual("local_package_release_identity_mismatch", evidence["reason"]) diff --git a/scripts/ci/test-release-package.py b/scripts/ci/test-release-package.py index 87492de6..a59c3983 100644 --- a/scripts/ci/test-release-package.py +++ b/scripts/ci/test-release-package.py @@ -6,7 +6,9 @@ import hashlib import io import json +import os from pathlib import Path +import shutil import subprocess import sys import tarfile @@ -16,7 +18,11 @@ from release_package import ( APPROVAL_SCHEMA, ReleasePackageError, + approval_record_path, + confined_file, + confined_root, file_list_sha256, + load_approval, load_package, release_entry, render_release_notes, @@ -512,26 +518,166 @@ def test_approved_archive_counterfactuals_are_refused(self) -> None: with self.assertRaises(ReleasePackageError): verify_approved_archive(archive, MANIFEST, CHANGELOG, "e" * 40, approval) + def approvals(self, archive: Path, **changes: object) -> Path: + """An approvals directory holding .json for `archive`.""" + directory = self.root / "approvals" + directory.mkdir(exist_ok=True) + record = self.approval(archive, **changes) + record.replace(directory / f"{self.package['version']}.json") + return directory + + def cli(self, *arguments: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(ROOT / "scripts/ci/release_package.py"), *arguments], + cwd=ROOT, capture_output=True, text=True, + ) + def test_approved_package_cli_requires_archive_and_commit(self) -> None: archive = self.archive() - approval = self.approval(archive) - script = Path(__file__).with_name("release_package.py") + approvals = self.approvals(archive) + base = ["--archive-root", str(self.root), "--archive", archive.name] for arguments in ( - ["--approved-package", str(approval)], - ["--approved-package", str(approval), "--archive", str(archive)], - ["--archive", str(archive), "--expected-vcs-commit", COMMIT, "--cargo-version", "1.86.0"], + ["--approval-root", str(approvals)], + ["--approval-root", str(approvals), *base], + [*base, "--expected-vcs-commit", COMMIT, "--cargo-version", "1.86.0"], + ["--archive", archive.name, "--expected-vcs-commit", COMMIT], + ["--check-paths", *base], ): with self.subTest(arguments=arguments): - result = subprocess.run([sys.executable, str(script), *arguments], capture_output=True, text=True) - self.assertEqual(2, result.returncode, result.stderr) - result = subprocess.run( - [sys.executable, str(script), "--archive", str(archive), "--expected-vcs-commit", COMMIT, - "--approved-package", str(approval), "--cargo-version", "1.86.0"], - capture_output=True, text=True, - ) + self.assertEqual(2, self.cli(*arguments).returncode) + result = self.cli(*base, "--expected-vcs-commit", COMMIT, + "--approval-root", str(approvals), "--cargo-version", "1.86.0") self.assertEqual(0, result.returncode, result.stderr) self.assertEqual(COMMIT, json.loads(result.stdout)["approval"]["commit"]) + # Path confinement (Security review of #178): every CLI path is confined to + # an explicit root before it is opened. + + def outside_file(self) -> Path: + outside = Path(tempfile.mkdtemp(prefix="release-package-outside-")) + self.addCleanup(shutil.rmtree, outside, ignore_errors=True) + secret = outside / "secret.crate" + secret.write_bytes(b"PRIVATE-TEST-CONTENT") + return secret + + def test_confined_file_refusals_and_positive_controls(self) -> None: + root = confined_root(self.root) + (self.root / "nested").mkdir() + (self.root / "nested" / "ok.crate").write_bytes(b"x") + (self.root / "ok.crate").write_bytes(b"x") + secret = self.outside_file() + (self.root / "link.crate").symlink_to(secret) + (self.root / "linkdir").symlink_to(secret.parent, target_is_directory=True) + refusals = { + "dot-dot escape": "../" + secret.parent.name + "/secret.crate", + "inner dot-dot": "nested/../ok.crate", + "symlink to outside": "link.crate", + "symlinked directory": "linkdir/secret.crate", + "absolute": str(secret), + "absolute inside root": str(self.root / "ok.crate"), + "NUL byte": "ok.crate\0.json", + "empty": "", + "directory": "nested", + "missing": "absent.crate", + } + for label, value in refusals.items(): + with self.subTest(label=label), self.assertRaises(ReleasePackageError): + confined_file(root, value) + for value in ("ok.crate", "nested/ok.crate", "./ok.crate"): + with self.subTest(positive=value): + self.assertEqual(Path(root, value).resolve(), confined_file(root, value)) + with self.assertRaisesRegex(ReleasePackageError, "already exists"): + confined_file(root, "ok.crate", must_exist=False) + self.assertEqual(Path(root, "new.lock"), confined_file(root, "new.lock", must_exist=False)) + + def test_confined_root_refusals(self) -> None: + (self.root / "real").mkdir() + (self.root / "rootlink").symlink_to(self.root / "real", target_is_directory=True) + (self.root / "file").write_bytes(b"x") + for label, value in (("symlink", str(self.root / "rootlink")), ("dot-dot", str(self.root / "real" / "..")), + ("NUL", str(self.root) + "\0"), ("file", str(self.root / "file")), + ("missing", str(self.root / "absent")), ("empty", "")): + with self.subTest(label=label), self.assertRaises(ReleasePackageError): + confined_root(value) + self.assertEqual(os.path.realpath(self.root / "real"), confined_root(self.root / "real")) + + def test_approval_path_requires_strict_semver_before_naming_a_file(self) -> None: + root = confined_root(self.root) + (self.root / "4.11.2.json").write_text("{}") + (self.root / "4.11.2-rc.1+build.5.json").write_text("{}") + secret = self.outside_file() + (secret.parent / "x.json").write_text("{}") + for version in ("4.11.2/../../x", "../4.11.2", "4.11.2\0", "v4.11.2", "4.11", "04.11.2", + "4.11.2-..", "4.11.2-rc..1", "4.11.2/", "/4.11.2", "４.11.2", "", + os.path.relpath(secret.parent / "x", self.root)): + with self.subTest(version=version), self.assertRaisesRegex(ReleasePackageError, "strict SemVer"): + approval_record_path(root, version) + for version in ("4.11.2", "4.11.2-rc.1+build.5"): + with self.subTest(positive=version): + self.assertEqual(Path(root, f"{version}.json"), approval_record_path(root, version)) + (self.root / "4.11.2.json").unlink() + (self.root / "4.11.2.json").symlink_to(secret) + with self.assertRaisesRegex(ReleasePackageError, "symbolic link"): + approval_record_path(root, "4.11.2") + + def test_approval_record_rejects_unsafe_file_names_and_versions(self) -> None: + archive = self.archive() + for label, change in { + "dot-dot file": {"files": ["../escape"]}, + "absolute file": {"files": ["/etc/passwd"]}, + "NUL file": {"files": ["a\0b"]}, + "backslash file": {"files": ["a\\..\\b"]}, + "crafted version": {"version": "4.11.2/../../x"}, + }.items(): + files = change.get("files") + if files is not None: + change = {**change, "file_list_sha256": file_list_sha256(files)} + approval = self.approval(archive, **change) + with self.subTest(label=label), self.assertRaisesRegex(ReleasePackageError, "malformed"): + load_approval(approval) + + def test_cli_refuses_paths_outside_their_roots(self) -> None: + archive = self.archive() + approvals = self.approvals(archive) + secret = self.outside_file() + (self.root / "escape.crate").symlink_to(secret) + verify = ["--expected-vcs-commit", COMMIT] + # A NUL byte cannot cross argv (exec refuses it); confined_file's own + # NUL refusal is covered in test_confined_file_refusals_and_positive_controls. + cases = { + "archive dot-dot": ["--archive-root", str(self.root), "--archive", "../" + secret.parent.name + "/secret.crate", *verify], + "archive absolute": ["--archive-root", str(self.root), "--archive", str(secret), *verify], + "archive symlink": ["--archive-root", str(self.root), "--archive", "escape.crate", *verify], + "manifest absolute": ["--manifest", str(MANIFEST), "--archive-root", str(self.root), "--archive", archive.name, *verify], + "manifest dot-dot": ["--manifest", "../" + ROOT.name + "/Cargo.toml", "--archive-root", str(self.root), "--archive", archive.name, *verify], + "changelog absolute": ["--check-paths", "--changelog", str(CHANGELOG)], + "root symlink": ["--archive-root", str(self.root / "rootlink"), "--archive", archive.name, *verify], + } + (self.root / "rootlink").symlink_to(secret.parent, target_is_directory=True) + for label, arguments in cases.items(): + with self.subTest(label=label): + result = self.cli(*arguments) + self.assertNotEqual(0, result.returncode) + self.assertEqual("", result.stdout) + self.assertNotIn("PRIVATE-TEST-CONTENT", result.stderr) + record = approvals / f"{self.package['version']}.json" + moved = secret.parent / "approved.json" + record.replace(moved) + record.symlink_to(moved) + result = self.cli("--archive-root", str(self.root), "--archive", archive.name, *verify, + "--approval-root", str(approvals)) + self.assertEqual(1, result.returncode) + self.assertIn("symbolic link", result.stderr) + # Positive controls: the same inputs, confined, are accepted. + record.unlink() + moved.replace(record) + for arguments in (["--check-paths"], + ["--archive-root", str(self.root), "--archive", archive.name, *verify], + ["--archive-root", str(self.root), "--archive", archive.name, *verify, + "--approval-root", str(approvals)]): + with self.subTest(positive=arguments): + result = self.cli(*arguments) + self.assertEqual(0, result.returncode, result.stderr) if __name__ == "__main__": unittest.main() From 94af7844cdbe118b9614d71e4f874708946a0b03 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Tue, 3 Nov 2026 17:27:07 -0600 Subject: [PATCH 4/9] Require a complete passing receipt and the tracked approval record Security's source review of 501d0311 held two publication-boundary gaps. Fabricated or truncated acceptance: - validate_acceptance() requires the exact receipt approval_candidate() emits, with no missing or extra keys and strict types. That covers: - a 64-hex summary SHA-256 and 40-hex harness and Server revision commits; - canonical @sha256:<64 hex> index and executed-manifest digests from one repository; - a supported platform, and a SemVer expected Server version equal to the release tag and to every cell's cluster version; - cells exactly deadline+lifecycle, a valid offset timestamp, and outcome "pass". - The harness emits "outcome". Pre-publication mode now requires a digest --server-index, a supported --platform, both cells, a 40-hex --expect-revision, and a tag equal to the Server version, so a passing run always produces an acceptable receipt. A test feeds the harness candidate through the publication parser to catch schema drift. Approval read confinement: - read_approval_record() derives release/approved-packages/.json only after strict SemVer, and confines it to the maintained-tooling checkout (RUST_SDK_APPROVED_PACKAGE_REPOSITORY = $GITHUB_WORKSPACE/ release-tools) with no link at any component. - It reads with lstat + O_NOFOLLOW + fstat. The bytes must equal a regular 100644 blob tracked at exactly that path at HEAD. Every .crate member is now checked independently at publication as well as acceptance: regular files only (no symlink, hardlink or directory), under the exact -/ prefix, with canonical relative POSIX names (no absolute form, empty/./.. component, backslash or NUL) and no duplicates. New counterfactuals: - a receipt deletion, wrong type and mismatch for every field; - external, tracked and directory symlinks; an untracked record; a record tracked at the wrong path; worktree/blob drift; - symlink, hardlink, directory, traversal, backslash, empty/dot component, wrong-prefix and duplicate members; - publish-level fabricated receipts and tracked-path cases. Removing each guard fails them: 71 failures and 1 error, plus 4 publish failures, for the receipt; 6 failures and 2 errors, plus 5 publish failures, for the tracked read; 6 failures for member canonicalization. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release.yml | 2 +- scripts/acceptance/installed-native/README.md | 32 +- scripts/acceptance/installed-native/run.py | 17 +- .../acceptance/installed-native/test_run.py | 31 +- scripts/ci/README.md | 4 +- scripts/ci/publish-rust-sdk.sh | 25 +- scripts/ci/release_package.py | 201 ++++++++-- scripts/ci/test-publish-rust-sdk.py | 148 ++++++- scripts/ci/test-release-package.py | 364 ++++++++++++++---- 9 files changed, 684 insertions(+), 140 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 152afc62..2b977e63 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -121,7 +121,7 @@ jobs: RELEASE_RUN_ATTEMPT: ${{ github.run_attempt }} # Reviewed record of the pre-publication package that passed installed-native # acceptance, committed to main (the maintained tooling), never the tagged source. - RUST_SDK_APPROVED_PACKAGE_DIR: ${{ github.workspace }}/release-tools/release/approved-packages + RUST_SDK_APPROVED_PACKAGE_REPOSITORY: ${{ github.workspace }}/release-tools run: ../release-tools/scripts/ci/publish-rust-sdk.sh - name: Create the source GitHub Release diff --git a/scripts/acceptance/installed-native/README.md b/scripts/acceptance/installed-native/README.md index 0bc10ce6..914c5783 100644 --- a/scripts/acceptance/installed-native/README.md +++ b/scripts/acceptance/installed-native/README.md @@ -134,10 +134,32 @@ whose checks and cleanup both pass writes `/approved-package.json`: "package": "durable-workflow", "version": "4.11.2", "commit": "<40 hex>", "crate_sha256": "<64 hex>", "cargo_version": "1.86.0", "files": ["..."], "file_list_sha256": "<64 hex>", - "acceptance": {"summary_sha256": "<64 hex>", "server_index": "...", "...": "..."} + "acceptance": { + "summary_sha256": "<64 hex>", "harness_commit": "<40 hex>", + "server_index": "ghcr.io/durable-workflow/server@sha256:<64 hex>", + "executed_manifest": "ghcr.io/durable-workflow/server@sha256:<64 hex>", + "platform": "linux/arm64", "expected_server_version": "3.0.5", + "expected_revision": "<40 hex>", "expected_release_tag": "3.0.5", + "cluster_versions": ["3.0.5"], "cells": ["deadline", "lifecycle"], + "finished_at": "", "outcome": "pass" + } } ``` +Publication accepts only this exact shape: no missing or extra keys at +either level, and strict types. It also checks how the receipt's fields relate: +- the index and executed manifest are canonical digests in one repository; +- the tag equals the expected Server version, and so does every cell's cluster + version; +- the cells are exactly lifecycle and deadline; +- the platform is `linux/amd64` or `linux/arm64`; +- the completion time is a valid timestamp with an offset; +- the outcome is `pass`. + +To match, pre-publication mode requires `--server-index sha256:<64 hex>`, a +supported `--platform`, both cells, a 40-hex `--expect-revision`, and +`--expect-release-tag` equal to `--expect-server-version`. + A reviewer commits that record unchanged to `main` as `release/approved-packages/.json`. `release/` is outside the crate's `include` list, so it never changes package bytes. The Release workflow reads @@ -172,7 +194,7 @@ written. | Paths the harness writes or reads below the work dir, such as unpacked and vendored members | `--work-dir` | Each is a `/`-separated relative path with no empty, `.`, `..`, NUL or backslash parts and no link at any component. Its canonical result must stay inside the root (`os.path.commonpath`). Files are created with `O_EXCL`/`O_NOFOLLOW`. | | `release_package.py --manifest`, `--changelog` | `--source-root` (the release checkout) | Relative, confined | | `release_package.py --archive` | `--archive-root` (Cargo's `target/package`, or the publisher's temporary directory) | Relative, confined | -| Approval record | `--approval-root`, set by the Release workflow as `RUST_SDK_APPROVED_PACKAGE_DIR=$GITHUB_WORKSPACE/release-tools/release/approved-packages` | Always `.json`. The version must match strict Semantic Versioning 2.0.0 before it names a file. The record's own version must match it too, and its file names must be safe relative paths. | +| Approval record | `--approval-repository`, set by the Release workflow as `RUST_SDK_APPROVED_PACKAGE_REPOSITORY=$GITHUB_WORKSPACE/release-tools` (the maintained-tooling checkout of `main`) | Always `release/approved-packages/.json`. The version must match strict Semantic Versioning 2.0.0 before it names the path. The record is read with `lstat` and `O_NOFOLLOW` and must be a regular file with no link at any component. It must also be byte-identical to a regular (`100644`) blob tracked at exactly that path in the checked-out commit. Its own version must match, and its file names must be canonical relative paths. | Versions used in harness paths (`--expect-version`) also follow strict Semantic Versioning, so a value such as `4.11.2/../../x` is refused. @@ -180,6 +202,12 @@ Semantic Versioning, so a value such as `4.11.2/../../x` is refused. The publisher also refuses a manifest or changelog outside its working directory, and an evidence path that is a link or contains a `..` component. +Every `.crate` member is checked independently, in acceptance and again at +publication. Each must be a regular file, never a symlink, hardlink or +directory. Its name must sit under the exact `-/` prefix, as a +canonical relative POSIX path with no absolute form, empty, `.` or `..` +component, backslash or NUL. No name may appear twice. + The run writes `/summary.json` with every identity, receipt and history result, plus per-container logs under `cells/*/logs`. The exit code is 0 only when every check passes and cleanup is complete (see below). diff --git a/scripts/acceptance/installed-native/run.py b/scripts/acceptance/installed-native/run.py index b6cb069c..5cb0f313 100755 --- a/scripts/acceptance/installed-native/run.py +++ b/scripts/acceptance/installed-native/run.py @@ -52,6 +52,8 @@ GENERATED_PACKAGE_FILES = ('.cargo_vcs_info.json', 'Cargo.toml') REQUIRED_PACKAGE_FILES = ('.cargo_vcs_info.json', 'Cargo.toml', 'Cargo.toml.orig', 'Cargo.lock') APPROVAL_SCHEMA = 'durable-workflow.rust-sdk.approved-package/v1' +APPROVAL_CELLS = ['deadline', 'lifecycle'] +APPROVAL_PLATFORMS = ('linux/amd64', 'linux/arm64') COMMIT_PATTERN = re.compile(r'[0-9a-f]{40}') SHA256_PATTERN = re.compile(r'[0-9a-f]{64}') # Semantic Versioning 2.0.0, ASCII only. The version names files and directories @@ -242,6 +244,18 @@ def parse_args(argv=None): absent = [option for option, value in required if not value] if absent: parser.error('--package-from requires %s' % ', '.join(absent)) + # The approval receipt the release workflow accepts is exactly this shape. + if not re.fullmatch(r'sha256:[0-9a-f]{64}', args.server_index): + parser.error('--package-from requires --server-index sha256:<64 hex>') + if args.platform not in APPROVAL_PLATFORMS: + parser.error('--package-from requires --platform in %s' % sorted(APPROVAL_PLATFORMS)) + if sorted(args.cells.split(',')) != APPROVAL_CELLS: + parser.error('--package-from requires --cells lifecycle,deadline') + if not COMMIT_PATTERN.fullmatch(args.expect_revision): + parser.error('--package-from requires a 40-hex --expect-revision') + if not VERSION_PATTERN.fullmatch(args.expect_server_version) or \ + args.expect_release_tag != args.expect_server_version: + parser.error('--package-from requires --expect-release-tag equal to a SemVer --expect-server-version') for option, value in (('--fixture-from', args.fixture_from), ('--source-tag', args.source_tag)): if value: parser.error('%s cannot be combined with --package-from' % option) @@ -609,7 +623,8 @@ def approval_candidate(summary, summary_bytes): 'expected_release_tag': summary.get('expected_server', {}).get('release_tag'), 'cluster_versions': sorted({c.get('cluster_info', {}).get('version') or '' for c in summary['cells'].values()}), - 'cells': sorted(summary['cells']), 'finished_at': summary['finished_at']}} + 'cells': sorted(summary['cells']), 'finished_at': summary['finished_at'], + 'outcome': summary['outcome']}} def check_baselines(metadata, mode, expected): diff --git a/scripts/acceptance/installed-native/test_run.py b/scripts/acceptance/installed-native/test_run.py index 4e069384..3340dd1f 100644 --- a/scripts/acceptance/installed-native/test_run.py +++ b/scripts/acceptance/installed-native/test_run.py @@ -1135,8 +1135,11 @@ def test_directory_source_checksum_is_the_crate_sha256(self): self.assertEqual(sorted(checksum['files']), identity['files']) +INDEX = 'sha256:' + '3' * 64 + + class PrePublicationArguments(unittest.TestCase): - BASE = ['--server-index', 'sha256:x', '--work-dir', '/tmp/x'] + BASE = ['--server-index', INDEX, '--work-dir', '/tmp/x'] PACKAGE = ['--package-from', '/tmp/src', '--package-commit', 'a' * 40, '--expect-version', VERSION, '--expect-server-version', '3.0.5', '--expect-revision', 'b' * 40, '--expect-release-tag', '3.0.5'] @@ -1148,6 +1151,11 @@ def test_package_mode_parses(self): args = self.parse(self.PACKAGE) self.assertEqual((args.package_commit, args.expect_version, args.crate_version), ('a' * 40, VERSION, None)) + def test_server_index_must_be_a_digest_in_package_mode(self): + with self.assertRaises(SystemExit), contextlib.redirect_stderr(io.StringIO()): + run.parse_args(['--server-index', 'sha256:x', '--work-dir', '/tmp/x'] + self.PACKAGE) + self.assertEqual(self.parse(self.PACKAGE + ['--cells', 'deadline,lifecycle']).cells, 'deadline,lifecycle') + def test_invalid_combinations_are_refused(self): def without(option): index = self.PACKAGE.index(option) @@ -1160,7 +1168,13 @@ def without(option): ['--crate-version', '4.11.1', '--expect-package-sha256', 'a' * 64], self.PACKAGE[:3] + ['abc'] + self.PACKAGE[4:], self.PACKAGE + ['--expect-package-sha256', 'A' * 64], - without('--expect-version') + ['--expect-version', 'v4.11.2']): + without('--expect-version') + ['--expect-version', 'v4.11.2'], + # The approval receipt shape the release workflow accepts. + self.PACKAGE + ['--cells', 'lifecycle'], self.PACKAGE + ['--cells', 'lifecycle,deadline,extra'], + self.PACKAGE + ['--platform', 'linux/riscv64'], + without('--expect-release-tag') + ['--expect-release-tag', 'native-3.0.5'], + without('--expect-revision') + ['--expect-revision', 'main'], + without('--expect-server-version') + ['--expect-server-version', 'v3.0.5']): with self.subTest(extra=extra), self.assertRaises(SystemExit): self.parse(extra) @@ -1191,7 +1205,7 @@ def run_cell(): instance.run.side_effect = run_cell return instance argv = ['run.py'] + PrePublicationArguments.PACKAGE + [ - '--server-index', 'sha256:x', '--cells', 'lifecycle', '--work-dir', str(work), '--container-prefix', 'general'] + '--server-index', INDEX, '--cells', 'lifecycle,deadline', '--work-dir', str(work), '--container-prefix', 'general'] out = io.StringIO() server = {'index': 'img@sha256:x', 'executed_manifest': 'img@sha256:y', 'platform': 'linux/arm64'} with mock.patch.object(sys, 'argv', argv), mock.patch.object(run, 'call', FakeDocker()), \ @@ -1215,6 +1229,12 @@ def test_passing_run_writes_a_bound_approval_candidate(self): for key in ('package', 'version', 'commit', 'crate_sha256', 'files', 'file_list_sha256'): self.assertEqual(approval[key], self.identity[key]) self.assertEqual(approval['cargo_version'], '1.86.0') + self.assertEqual(sorted(approval['acceptance']), sorted([ + 'summary_sha256', 'harness_commit', 'server_index', 'executed_manifest', 'platform', + 'expected_server_version', 'expected_revision', 'expected_release_tag', 'cluster_versions', + 'cells', 'finished_at', 'outcome'])) + self.assertEqual((approval['acceptance']['outcome'], approval['acceptance']['cells']), + ('pass', ['deadline', 'lifecycle'])) self.assertEqual(approval['acceptance']['summary_sha256'], hashlib.sha256((work / 'summary.json').read_bytes()).hexdigest()) self.assertEqual((approval['acceptance']['expected_server_version'], approval['acceptance']['expected_release_tag'], @@ -1319,6 +1339,11 @@ def test_archive_member_with_nul_or_backslash_is_refused(self): tmp = temporary_directory(self) commit = make_repo(tmp / 'repo') members = packaged_members(commit) + hardlink = tarfile.TarInfo('durable-workflow-%s/src/hard.rs' % VERSION) + hardlink.type, hardlink.linkname = tarfile.LNKTYPE, 'durable-workflow-%s/Cargo.lock' % VERSION + crate = make_crate(members, raw=[(hardlink, None)]) + with self.assertRaisesRegex(run.PackageError, 'not a regular file'): + run.verify_package_archive(crate, VERSION, commit, sorted(members) + ['src/hard.rs'], lambda path: None) for name in ('src/a\\..\\b.rs',): crate = make_crate(dict(members, **{name: b'x'})) with self.subTest(name=name), self.assertRaisesRegex(run.PackageError, 'not a regular file'): diff --git a/scripts/ci/README.md b/scripts/ci/README.md index ee5e3b7c..5fe5b3e4 100644 --- a/scripts/ci/README.md +++ b/scripts/ci/README.md @@ -10,7 +10,9 @@ These scripts support four public repository checks: The release workflow uses the crates.io publishing scripts after an immutable tag already exists. A version that is not yet on crates.io is published only when `cargo package` of the tag reproduces the reviewed pre-publication record -`release/approved-packages/.json` exactly. The record must match the +`release/approved-packages/.json` exactly. That record must be the +regular file tracked at that path in the maintained-tooling checkout, and its +acceptance receipt must be complete and passing. The record must match the SHA-256, file set, version, VCS commit and Cargo release. After publication, the crates.io checksum, the downloaded archive and the tag read back from `origin` must still match it. See `scripts/acceptance/installed-native/README.md` diff --git a/scripts/ci/publish-rust-sdk.sh b/scripts/ci/publish-rust-sdk.sh index bca256a2..45b50c4c 100755 --- a/scripts/ci/publish-rust-sdk.sh +++ b/scripts/ci/publish-rust-sdk.sh @@ -17,7 +17,9 @@ lockfile_source="source_checkout" # A reviewed record of the pre-publication package that passed installed-native # acceptance (scripts/acceptance/installed-native, --package-from). New versions # publish only when the reproduced package is exactly that artifact. -approved_package_dir="${RUST_SDK_APPROVED_PACKAGE_DIR:-}" +# The maintained-tooling checkout (main) that tracks the reviewed records at +# release/approved-packages/.json. +approved_package_repository="${RUST_SDK_APPROVED_PACKAGE_REPOSITORY:-}" approved_package_path="" for command in cargo curl git jq python3 rustc sha256sum; do @@ -470,10 +472,19 @@ archive_vcs_dirty="$(jq -r '.archive.vcs_dirty' "$local_identity")" changelog_checksum="$(jq -er '.changelog.sha256' "$local_identity")" release_entry_checksum="$(jq -er '.changelog.entry_sha256' "$local_identity")" -if [[ -n "$approved_package_dir" ]]; then - approved_package_path="${approved_package_dir}/${package_version}.json" +# package_version is already an exact SemVer here; release_package.py validates +# it strictly again before naming the record, and reads only a regular, +# link-free file that equals the blob tracked at that exact path. +approved_package_path="release/approved-packages/${package_version}.json" +approval_present="false" +if [[ -n "$approved_package_repository" ]]; then + if [[ -e "$approved_package_repository/$approved_package_path" || \ + -L "$approved_package_repository/$approved_package_path" ]] || \ + [[ -n "$(git -C "$approved_package_repository" ls-tree --full-tree HEAD -- "$approved_package_path" 2>/dev/null || true)" ]]; then + approval_present="true" + fi fi -if [[ -n "$approved_package_path" && ( -e "$approved_package_path" || -L "$approved_package_path" ) ]]; then +if [[ "$approval_present" == "true" ]]; then packaging_cargo="$(cargo -V | awk '{print $2}')" local_approval="$tmp_dir/local-approval.json" if ! python3 "$script_dir/release_package.py" \ @@ -483,7 +494,7 @@ if [[ -n "$approved_package_path" && ( -e "$approved_package_path" || -L "$appro --archive-root "$local_archive_root" \ --archive "$(basename "$local_archive")" \ --expected-vcs-commit "$release_commit" \ - --approval-root "$approved_package_dir" \ + --approval-repository "$approved_package_repository" \ --cargo-version "$packaging_cargo" > "$local_approval"; then approval_status="reproduction_mismatch" write_evidence "failed" "approved_package_mismatch" @@ -505,7 +516,7 @@ fi if [[ "$http_status" == "404" && "$approval_status" != "reproduced" ]]; then write_evidence "failed" "approved_package_required" - printf 'publishing %s %s requires a reviewed approved-package record (RUST_SDK_APPROVED_PACKAGE_DIR)\n' \ + printf 'publishing %s %s requires a reviewed approved-package record (RUST_SDK_APPROVED_PACKAGE_REPOSITORY)\n' \ "$package_name" "$package_version" >&2 exit 1 fi @@ -628,7 +639,7 @@ if [[ "$approval_status" == "reproduced" ]]; then --archive-root "$tmp_dir" \ --archive "$(basename "$published_archive")" \ --expected-vcs-commit "$release_commit" \ - --approval-root "$approved_package_dir" > "$tmp_dir/published-approval.json"; then + --approval-repository "$approved_package_repository" > "$tmp_dir/published-approval.json"; then write_evidence "failed" "published_package_differs_from_approved_package" printf 'downloaded crate is not the approved pre-publication package\n' >&2 exit 1 diff --git a/scripts/ci/release_package.py b/scripts/ci/release_package.py index a6783273..40083781 100644 --- a/scripts/ci/release_package.py +++ b/scripts/ci/release_package.py @@ -5,11 +5,14 @@ import argparse from dataclasses import dataclass +from datetime import datetime import hashlib import json import os from pathlib import Path import re +import stat +import subprocess import sys import tarfile import tomllib @@ -279,6 +282,7 @@ def verify_archive( ): raise ReleasePackageError("expected VCS commit must be a full lowercase SHA-1") + _archive_files(archive_path, f"{source['package']}-{source['package_version']}") try: archive_bytes = archive_path.read_bytes() with tarfile.open(archive_path, "r:*") as archive: @@ -430,11 +434,11 @@ def confined_file( return Path(resolved) -def approval_record_path(root: str, version: str) -> Path: - """The reviewed record for `version`: /.json, strictly confined.""" +def approval_record_relative(version: str) -> str: + """release/approved-packages/.json, only for a strict SemVer version.""" if not isinstance(version, str) or not STRICT_SEMVER.fullmatch(version): raise ReleasePackageError("approval version is not a strict SemVer version") - return confined_file(root, f"{version}.json") + return f"{APPROVAL_DIRECTORY}/{version}.json" def _safe_member_name(name: object) -> bool: @@ -455,43 +459,150 @@ def file_list_sha256(files: list[str]) -> str: return sha256("".join(f"{name}\n" for name in sorted(files)).encode()) -def load_approval(path: Path) -> tuple[dict[str, Any], str]: - """Read one reviewed approved-package record and check its shape.""" +ACCEPTANCE_FIELDS = { + "summary_sha256", "harness_commit", "server_index", "executed_manifest", "platform", + "expected_server_version", "expected_revision", "expected_release_tag", + "cluster_versions", "cells", "finished_at", "outcome", +} +ACCEPTANCE_CELLS = ["deadline", "lifecycle"] +ACCEPTANCE_PLATFORMS = {"linux/amd64", "linux/arm64"} +# /@sha256:, lowercase, as `docker manifest` names it. +IMAGE_DIGEST_PATTERN = re.compile( + r"^((?:[a-z0-9]+(?:[.-][a-z0-9]+)*(?::[0-9]+)?/)?" + r"[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*(?:/[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*)*)" + r"@sha256:[0-9a-f]{64}$", + re.ASCII, +) +APPROVAL_DIRECTORY = "release/approved-packages" + + +def _is_text(value: object, pattern: re.Pattern[str] | None = None) -> bool: + return isinstance(value, str) and (pattern is None or pattern.fullmatch(value) is not None) + + +def _valid_timestamp(value: object) -> bool: + if not isinstance(value, str): + return False + try: + parsed = datetime.fromisoformat(value) + except ValueError: + return False + return parsed.tzinfo is not None and parsed.utcoffset() is not None + + +def validate_acceptance(acceptance: object) -> None: + """The receipt of one passing exact-Server lifecycle and deadline run. + + The inner schema is exact (no missing or extra keys, strict types) and its + relationships are bound: one image repository for index and executed + manifest, tag equal to the expected Server version, every cell reporting + that version, both cells, a supported platform and a valid completion time. + """ + if not isinstance(acceptance, dict) or set(acceptance) != ACCEPTANCE_FIELDS: + raise ReleasePackageError("approval acceptance receipt has missing or unexpected fields") + index = IMAGE_DIGEST_PATTERN.fullmatch(acceptance["server_index"]) if _is_text(acceptance["server_index"]) else None + executed = (IMAGE_DIGEST_PATTERN.fullmatch(acceptance["executed_manifest"]) + if _is_text(acceptance["executed_manifest"]) else None) + version = acceptance["expected_server_version"] + checks = { + "summary_sha256": _is_text(acceptance["summary_sha256"], SHA256_PATTERN), + "harness_commit": _is_text(acceptance["harness_commit"], COMMIT_PATTERN), + "expected_revision": _is_text(acceptance["expected_revision"], COMMIT_PATTERN), + "server_index": index is not None, + "executed_manifest": executed is not None and index is not None and executed.group(1) == index.group(1), + "platform": _is_text(acceptance["platform"]) and acceptance["platform"] in ACCEPTANCE_PLATFORMS, + "expected_server_version": _is_text(version, STRICT_SEMVER), + "expected_release_tag": _is_text(acceptance["expected_release_tag"]) and acceptance["expected_release_tag"] == version, + "cluster_versions": isinstance(acceptance["cluster_versions"], list) and acceptance["cluster_versions"] == [version], + "cells": isinstance(acceptance["cells"], list) and acceptance["cells"] == ACCEPTANCE_CELLS, + "finished_at": _valid_timestamp(acceptance["finished_at"]), + "outcome": _is_text(acceptance["outcome"]) and acceptance["outcome"] == "pass", + } + failed = sorted(field for field, ok in checks.items() if not ok) + if failed: + raise ReleasePackageError(f"approval acceptance receipt is invalid: {', '.join(failed)}") + + +def parse_approval(raw: bytes) -> tuple[dict[str, Any], str]: + """Parse one reviewed approved-package record and check its exact shape.""" try: - raw = path.read_bytes() approval = json.loads(raw) - except (OSError, UnicodeError, json.JSONDecodeError) as error: + except (UnicodeError, json.JSONDecodeError) as error: raise ReleasePackageError(f"could not read approved package record: {error}") from error if not isinstance(approval, dict) or set(approval) != APPROVAL_FIELDS: raise ReleasePackageError("approved package record has unexpected fields") files = approval["files"] - acceptance = approval["acceptance"] if ( approval["schema"] != APPROVAL_SCHEMA - or not isinstance(approval["package"], str) - or not isinstance(approval["version"], str) - or not STRICT_SEMVER.fullmatch(approval["version"]) - or not isinstance(approval["commit"], str) - or not COMMIT_PATTERN.fullmatch(approval["commit"]) - or not isinstance(approval["crate_sha256"], str) - or not SHA256_PATTERN.fullmatch(approval["crate_sha256"]) - or not isinstance(approval["cargo_version"], str) - or not CARGO_VERSION_PATTERN.fullmatch(approval["cargo_version"]) + or not _is_text(approval["package"]) + or not _is_text(approval["version"], STRICT_SEMVER) + or not _is_text(approval["commit"], COMMIT_PATTERN) + or not _is_text(approval["crate_sha256"], SHA256_PATTERN) + or not _is_text(approval["cargo_version"], CARGO_VERSION_PATTERN) or not isinstance(files, list) or not files or not all(_safe_member_name(name) for name in files) or files != sorted(set(files)) or approval["file_list_sha256"] != file_list_sha256(files) - or not isinstance(acceptance, dict) - or not isinstance(acceptance.get("summary_sha256"), str) - or not SHA256_PATTERN.fullmatch(acceptance["summary_sha256"]) - or not isinstance(acceptance.get("server_index"), str) - or not acceptance["server_index"] ): raise ReleasePackageError("approved package record is malformed") + validate_acceptance(approval["acceptance"]) return approval, sha256(raw) +def _read_regular_no_follow(path: Path) -> bytes: + try: + if not stat.S_ISREG(os.lstat(path).st_mode): + raise ReleasePackageError("approved package record is not a regular file") + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + except OSError as error: + raise ReleasePackageError(f"could not open approved package record: {error.strerror}") from error + with os.fdopen(descriptor, "rb") as handle: + if not stat.S_ISREG(os.fstat(handle.fileno()).st_mode): + raise ReleasePackageError("approved package record is not a regular file") + return handle.read() + + +def load_approval(path: Path) -> tuple[dict[str, Any], str]: + """Read (without following links) and parse one approved-package record.""" + return parse_approval(_read_regular_no_follow(path)) + + +def _git(repository: str, *arguments: str) -> bytes: + result = subprocess.run( + ["git", "-C", repository, *arguments], capture_output=True, check=False, timeout=60, + ) + if result.returncode: + raise ReleasePackageError(f"git {arguments[0]} failed in the approvals repository") + return result.stdout + + +def read_approval_record(repository: str, version: str) -> tuple[bytes, str]: + """The reviewed record for `version` from the maintained-tooling checkout. + + Its path is always release/approved-packages/.json, built only + after `version` passes strict SemVer. The worktree file must be a regular + file reached without any symbolic link (lstat + O_NOFOLLOW), confined to + the repository, and byte-identical to a regular (mode 100644) blob tracked + at exactly that path in the checked-out commit. + """ + relative = approval_record_relative(version) + path = confined_file(repository, relative) + content = _read_regular_no_follow(path) + listing = _git(repository, "ls-tree", "-z", "--full-tree", "HEAD", "--", relative) + entries = [entry for entry in listing.split(b"\0") if entry] + if len(entries) != 1: + raise ReleasePackageError("approved package record is not tracked at its exact path") + meta, _, tracked_path = entries[0].partition(b"\t") + fields = meta.split() + if (tracked_path != relative.encode() or len(fields) != 3 + or fields[0] != b"100644" or fields[1] != b"blob"): + raise ReleasePackageError("approved package record is not a tracked regular file at its exact path") + if _git(repository, "cat-file", "blob", fields[2].decode("ascii")) != content: + raise ReleasePackageError("approved package record differs from its tracked content") + return content, relative + + def _archive_files(archive_path: Path, root: str) -> list[str]: prefix = f"{root}/" try: @@ -501,9 +612,13 @@ def _archive_files(archive_path: Path, root: str) -> list[str]: raise ReleasePackageError(f"could not inspect crate archive: {error}") from error names = [] for member in members: - if not member.isfile() or not member.name.startswith(prefix): + # Each member independently: a regular file (no symlink or hardlink) + # under the exact -/ prefix, with a canonical relative + # POSIX path (no absolute form, empty, '.', '..', backslash or NUL). + relative = member.name[len(prefix):] if member.name.startswith(prefix) else None + if not member.isfile() or relative is None or not _safe_member_name(relative): raise ReleasePackageError(f"crate member {member.name!r} is not a regular package file") - names.append(member.name[len(prefix):]) + names.append(relative) if len(names) != len(set(names)): raise ReleasePackageError("crate archive repeats a member") return sorted(names) @@ -514,12 +629,12 @@ def verify_approved_archive( manifest: Path, changelog: Path, expected_vcs_commit: str, - approval_path: Path, + approval_record: bytes, cargo_version: str | None = None, ) -> dict[str, Any]: """Require an archive to be exactly the approved pre-publication package.""" identity = verify_archive(archive_path, manifest, changelog, expected_vcs_commit) - approval, approval_sha256 = load_approval(approval_path) + approval, approval_sha256 = parse_approval(approval_record) if approval["package"] != identity["package"]: raise ReleasePackageError("approved package name differs from the release manifest") if approval["version"] != identity["package_version"]: @@ -551,6 +666,7 @@ def verify_approved_archive( "file_list_sha256": approval["file_list_sha256"], "acceptance_summary_sha256": approval["acceptance"]["summary_sha256"], "server_index": approval["acceptance"]["server_index"], + "server_version": approval["acceptance"]["expected_server_version"], } return identity @@ -560,7 +676,7 @@ def _validate_notes_arguments( ) -> None: if any((arguments.archive, arguments.expected_vcs_commit, arguments.expected_archive_sha256, arguments.restore_missing_lockfile, - arguments.approval_root, arguments.archive_root, arguments.cargo_version, + arguments.approval_repository, arguments.archive_root, arguments.cargo_version, arguments.check_paths)): parser.error("--release-notes cannot be combined with archive verification") if (arguments.manifest != Path(SOURCE_MANIFEST) @@ -575,25 +691,25 @@ def _validate_mode_arguments( if arguments.expected_archive_sha256 is not None and not arguments.restore_missing_lockfile: parser.error("--expected-archive-sha256 requires --restore-missing-lockfile") if arguments.check_paths and any(( - arguments.archive, arguments.archive_root, arguments.approval_root, + arguments.archive, arguments.archive_root, arguments.approval_repository, arguments.expected_vcs_commit, arguments.restore_missing_lockfile, arguments.cargo_version, )): parser.error("--check-paths takes only --source-root, --manifest and --changelog") if (arguments.archive is None) != (arguments.archive_root is None): parser.error("--archive and --archive-root must be given together") - if arguments.cargo_version is not None and arguments.approval_root is None: - parser.error("--cargo-version requires --approval-root") - if arguments.approval_root is not None and ( + if arguments.cargo_version is not None and arguments.approval_repository is None: + parser.error("--cargo-version requires --approval-repository") + if arguments.approval_repository is not None and ( arguments.archive is None or arguments.expected_vcs_commit is None or arguments.restore_missing_lockfile ): - parser.error("--approval-root requires --archive and --expected-vcs-commit only") + parser.error("--approval-repository requires --archive and --expected-vcs-commit only") def _run(arguments: argparse.Namespace, parser: argparse.ArgumentParser) -> dict[str, Any]: # Every file named on the command line is confined to an explicit root # before it is opened: the checkout (--source-root), the package directory - # (--archive-root) or the reviewed approvals directory (--approval-root). + # (--archive-root) or the reviewed approvals directory (--approval-repository). source_root = confined_root(arguments.source_root) manifest = confined_file(source_root, arguments.manifest) changelog = confined_file(source_root, arguments.changelog) @@ -615,14 +731,16 @@ def _run(arguments: argparse.Namespace, parser: argparse.ArgumentParser) -> dict if arguments.expected_vcs_commit is not None: parser.error("--expected-vcs-commit requires --archive") return {"schema": SCHEMA, "version": 1, **verify_source(manifest, changelog)} - if arguments.approval_root is not None: - approval = approval_record_path( - confined_root(arguments.approval_root), str(load_package(manifest)["version"]) + if arguments.approval_repository is not None: + record, relative = read_approval_record( + confined_root(arguments.approval_repository), str(load_package(manifest)["version"]) ) - return verify_approved_archive( + identity = verify_approved_archive( archive, manifest, changelog, arguments.expected_vcs_commit, - approval, arguments.cargo_version, + record, arguments.cargo_version, ) + identity["approval"]["record_path"] = relative + return identity return verify_archive(archive, manifest, changelog, arguments.expected_vcs_commit) @@ -640,11 +758,12 @@ def main() -> int: parser.add_argument("--check-paths", action="store_true", help="only confine --manifest and --changelog to --source-root") parser.add_argument( - "--approval-root", - help="reviewed approvals directory; requires --archive to equal .json in it", + "--approval-repository", + help="maintained-tooling checkout; requires --archive to equal its tracked " + "release/approved-packages/.json", ) parser.add_argument( - "--cargo-version", help="with --approval-root: the packaging Cargo release (X.Y.Z)", + "--cargo-version", help="with --approval-repository: the packaging Cargo release (X.Y.Z)", ) parser.add_argument( "--release-notes", metavar="VERSION", diff --git a/scripts/ci/test-publish-rust-sdk.py b/scripts/ci/test-publish-rust-sdk.py index a9931c44..d44c529f 100644 --- a/scripts/ci/test-publish-rust-sdk.py +++ b/scripts/ci/test-publish-rust-sdk.py @@ -35,6 +35,23 @@ RELEASE_COMMIT = "0123456789abcdef0123456789abcdef01234567" TOOLING_COMMIT = "fedcba9876543210fedcba9876543210fedcba98" CHECKSUM = "a" * 64 +REAL_GIT = shutil.which("git") +APPROVAL_RELATIVE = f"release/approved-packages/{PACKAGE_VERSION}.json" +# The exact receipt approval_candidate() emits for a passing 3.0.5 run. +ACCEPTANCE = { + "summary_sha256": "c" * 64, + "harness_commit": "e" * 40, + "server_index": "ghcr.io/durable-workflow/server@sha256:" + "d" * 64, + "executed_manifest": "ghcr.io/durable-workflow/server@sha256:" + "f" * 64, + "platform": "linux/arm64", + "expected_server_version": "3.0.5", + "expected_revision": "b" * 40, + "expected_release_tag": "3.0.5", + "cluster_versions": ["3.0.5"], + "cells": ["deadline", "lifecycle"], + "finished_at": "2026-11-03T23:00:00+00:00", + "outcome": "pass", +} class PublishRustSdkContractTest(unittest.TestCase): @@ -45,6 +62,9 @@ def setUp(self) -> None: self.bin_dir = self.temp / "bin" self.bin_dir.mkdir() self.evidence = self.temp / "evidence.json" + self.tooling = self.temp / "tooling" + self.tooling.mkdir() + self._tooling_git("init", "--quiet") self._write_mock_commands() def _write_executable(self, name: str, source: str) -> None: @@ -154,7 +174,11 @@ def _write_mock_commands(self) -> None: import sys command = sys.argv[1] - if command == "-C": + approvals = os.environ.get("MOCK_APPROVAL_REPOSITORY") + if command == "-C" and approvals and os.path.realpath(sys.argv[2]) == os.path.realpath(approvals): + # The approvals checkout is a real repository. + os.execv(os.environ["MOCK_REAL_GIT"], ["git", *sys.argv[1:]]) + elif command == "-C": print(os.environ["MOCK_TOOLING_COMMIT"]) elif command == "check-ignore": raise SystemExit(0 if os.environ.get("MOCK_IGNORED_LOCKFILE") else 1) @@ -235,7 +259,9 @@ def _publish( "RELEASE_TAG": PACKAGE_VERSION, "RUST_SDK_MANIFEST_PATH": os.path.relpath(manifest, cwd), "RUST_SDK_RELEASE_EVIDENCE_PATH": str(self.evidence), - "RUST_SDK_APPROVED_PACKAGE_DIR": str(self.temp / "approved"), + "RUST_SDK_APPROVED_PACKAGE_REPOSITORY": str(self.tooling), + "MOCK_APPROVAL_REPOSITORY": str(self.tooling), + "MOCK_REAL_GIT": REAL_GIT, "MOCK_PUBLISH_MARKER": str(self.temp / "published-marker"), } ) @@ -254,9 +280,7 @@ def _publish( if approval is not None: record = self._approval(registry_archive) record.update(approval) - path = Path(env["RUST_SDK_APPROVED_PACKAGE_DIR"]) / f"{PACKAGE_VERSION}.json" - path.parent.mkdir(exist_ok=True) - path.write_text(json.dumps(record), encoding="utf-8") + self._commit_approval(json.dumps(record).encode()) if registry_changelog is not None: # crates.io serves different (self-consistent) bytes than were approved. changed = self.temp / "changed-registry" @@ -278,6 +302,24 @@ def _publish( check=False, ) + def _tooling_git(self, *arguments: str) -> None: + subprocess.run( + [REAL_GIT, "-C", str(self.tooling), "-c", "user.name=t", "-c", "user.email=t@example.invalid", + "-c", "commit.gpgsign=false", *arguments], + check=True, capture_output=True, + env={**os.environ, "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_NOSYSTEM": "1"}, + ) + + def _commit_approval(self, content: bytes, relative: str = APPROVAL_RELATIVE) -> Path: + path = self.tooling / relative + path.parent.mkdir(parents=True, exist_ok=True) + if path.is_symlink(): + path.unlink() + path.write_bytes(content) + self._tooling_git("add", "--", relative) + self._tooling_git("commit", "--quiet", "--allow-empty", "-m", "approve") + return path + @staticmethod def _approval(archive: Path) -> dict[str, object]: root = f"durable-workflow-{PACKAGE_VERSION}/" @@ -292,7 +334,7 @@ def _approval(archive: Path) -> dict[str, object]: "cargo_version": "1.86.0", "files": files, "file_list_sha256": hashlib.sha256("".join(f"{name}\n" for name in files).encode()).hexdigest(), - "acceptance": {"summary_sha256": "c" * 64, "server_index": "ghcr.io/durable-workflow/server@sha256:" + "d" * 64}, + "acceptance": dict(ACCEPTANCE), } def _evidence(self) -> dict[str, object]: @@ -323,7 +365,7 @@ def test_new_version_without_an_approval_record_is_never_published(self) -> None self.assertFalse(self._published()) self.assertEqual("approved_package_required", self._evidence()["reason"]) result = self._publish(environment={"MOCK_LOOKUP_STATUS": "404", "CARGO_REGISTRY_TOKEN": "test-token", - "RUST_SDK_APPROVED_PACKAGE_DIR": ""}) + "RUST_SDK_APPROVED_PACKAGE_REPOSITORY": ""}) self.assertNotEqual(0, result.returncode) self.assertFalse(self._published()) @@ -443,21 +485,91 @@ def test_source_paths_outside_the_release_checkout_are_refused(self) -> None: cwd=source, approval={}) self.assertEqual(0, result.returncode, result.stderr) - def test_symlinked_approval_record_is_refused(self) -> None: - outside = self.temp / "forged.json" + def _approved_record(self) -> bytes: + """Publish once with a valid record, then reset the registry state.""" result = self._publish(environment={"MOCK_LOOKUP_STATUS": "404", "CARGO_REGISTRY_TOKEN": "test-token"}, approval={}) self.assertEqual(0, result.returncode, result.stderr) - record = self.temp / "approved" / f"{PACKAGE_VERSION}.json" - outside.write_bytes(record.read_bytes()) - record.unlink() - record.symlink_to(outside) (self.temp / "published-marker").unlink() - result = self._publish(environment={"MOCK_LOOKUP_STATUS": "404", "CARGO_REGISTRY_TOKEN": "test-token"}) + return (self.tooling / APPROVAL_RELATIVE).read_bytes() + + def _publish_new(self) -> subprocess.CompletedProcess[str]: + return self._publish(environment={"MOCK_LOOKUP_STATUS": "404", "CARGO_REGISTRY_TOKEN": "test-token"}) + + def test_approval_record_must_be_the_tracked_regular_file(self) -> None: + record = self._approved_record() + path = self.tooling / APPROVAL_RELATIVE + outside = self.temp / "forged.json" + outside.write_bytes(record) + + def external_symlink() -> None: + path.unlink() + path.symlink_to(outside) + + def tracked_symlink() -> None: + path.unlink() + path.symlink_to(outside) + self._tooling_git("add", "--", APPROVAL_RELATIVE) + self._tooling_git("commit", "--quiet", "-m", "symlink") + + def symlinked_directory() -> None: + directory = self.tooling / "release" / "approved-packages" + shutil.move(directory, self.temp / "elsewhere") + directory.symlink_to(self.temp / "elsewhere", target_is_directory=True) + + def untracked_copy() -> None: + self._tooling_git("rm", "--quiet", "--cached", "--", APPROVAL_RELATIVE) + self._tooling_git("commit", "--quiet", "-m", "untrack") + + def edited_worktree() -> None: + path.write_bytes(record.replace(b'"outcome": "pass"', b'"outcome": "pass" ')) + + for label, mutate, message in ( + ("external symlink", external_symlink, "symbolic link"), + ("tracked symlink", tracked_symlink, "symbolic link"), + ("symlinked directory", symlinked_directory, "symbolic link"), + ("untracked record", untracked_copy, "not tracked at its exact path"), + ("worktree differs from tracked blob", edited_worktree, "differs from its tracked content"), + ): + with self.subTest(label=label): + mutate() + result = self._publish_new() + self.assertNotEqual(0, result.returncode) + self.assertFalse(self._published()) + self.assertEqual("approved_package_mismatch", self._evidence()["reason"]) + self.assertIn(message, result.stderr) + # Restore the tracked regular record for the next case. + if (self.tooling / "release" / "approved-packages").is_symlink(): + (self.tooling / "release" / "approved-packages").unlink() + shutil.move(self.temp / "elsewhere", self.tooling / "release" / "approved-packages") + self._commit_approval(record) + + def test_record_tracked_at_a_wrong_path_is_not_an_approval(self) -> None: + record = self._approved_record() + self._tooling_git("rm", "--quiet", "--", APPROVAL_RELATIVE) + for relative in ("release/approved-packages/4.11.0.json", f"release/{PACKAGE_VERSION}.json", + f"release/approved-packages/nested/{PACKAGE_VERSION}.json", + f"approved-packages/{PACKAGE_VERSION}.json"): + self._commit_approval(record, relative) + result = self._publish_new() self.assertNotEqual(0, result.returncode) self.assertFalse(self._published()) - self.assertEqual("approved_package_mismatch", self._evidence()["reason"]) - self.assertIn("symbolic link", result.stderr) + self.assertEqual("approved_package_required", self._evidence()["reason"]) + + def test_truncated_or_fabricated_acceptance_receipt_refuses_to_publish(self) -> None: + cases = {"empty receipt": {}, "receipt without outcome": {k: v for k, v in ACCEPTANCE.items() if k != "outcome"}, + "failed receipt": {**ACCEPTANCE, "outcome": "fail"}, + "other Server": {**ACCEPTANCE, "cluster_versions": ["3.0.4"]}} + for label, acceptance in cases.items(): + with self.subTest(label=label): + result = self._publish( + environment={"MOCK_LOOKUP_STATUS": "404", "CARGO_REGISTRY_TOKEN": "test-token"}, + approval={"acceptance": acceptance}, + ) + self.assertNotEqual(0, result.returncode) + self.assertFalse(self._published()) + self.assertEqual("approved_package_mismatch", self._evidence()["reason"]) + self.assertIn("acceptance receipt", result.stderr) def test_symlinked_evidence_path_is_refused(self) -> None: target = self.temp / "elsewhere.json" @@ -475,12 +587,12 @@ def test_symlinked_evidence_path_is_refused(self) -> None: def test_release_workflow_supplies_the_reviewed_approval_record(self) -> None: workflow = RELEASE_WORKFLOW.read_text(encoding="utf-8") self.assertIn( - "RUST_SDK_APPROVED_PACKAGE_DIR: ${{ github.workspace }}/release-tools/release/approved-packages", + "RUST_SDK_APPROVED_PACKAGE_REPOSITORY: ${{ github.workspace }}/release-tools", workflow, ) # The record comes from maintained tooling (main), never the tagged source. self.assertIn("ref: ${{ github.workflow_sha }}", workflow) - self.assertLess(workflow.index("Resolve exact release identity"), workflow.index("RUST_SDK_APPROVED_PACKAGE_DIR")) + self.assertLess(workflow.index("Resolve exact release identity"), workflow.index("RUST_SDK_APPROVED_PACKAGE_REPOSITORY")) self.assertNotIn("/release/", MANIFEST.read_text(encoding="utf-8").split("include = ", 1)[1].split("\n", 1)[0]) def _manifest_with(self, old: str, new: str) -> Path: diff --git a/scripts/ci/test-release-package.py b/scripts/ci/test-release-package.py index a59c3983..d7cb782a 100644 --- a/scripts/ci/test-release-package.py +++ b/scripts/ci/test-release-package.py @@ -18,11 +18,14 @@ from release_package import ( APPROVAL_SCHEMA, ReleasePackageError, - approval_record_path, + approval_record_relative, confined_file, confined_root, file_list_sha256, load_approval, + parse_approval, + read_approval_record, + validate_acceptance, load_package, release_entry, render_release_notes, @@ -38,6 +41,21 @@ CHANGELOG = ROOT / "CHANGELOG.md" REQUIREMENTS = ROOT / "scripts/ci/release-tooling-requirements.txt" COMMIT = "0123456789abcdef0123456789abcdef01234567" +# The exact receipt approval_candidate() emits for a passing 3.0.5 run. +ACCEPTANCE = { + "summary_sha256": "c" * 64, + "harness_commit": "e" * 40, + "server_index": "ghcr.io/durable-workflow/server@sha256:" + "d" * 64, + "executed_manifest": "ghcr.io/durable-workflow/server@sha256:" + "f" * 64, + "platform": "linux/arm64", + "expected_server_version": "3.0.5", + "expected_revision": "b" * 40, + "expected_release_tag": "3.0.5", + "cluster_versions": ["3.0.5"], + "cells": ["deadline", "lifecycle"], + "finished_at": "2026-11-03T23:00:00+00:00", + "outcome": "pass", +} class ReleasePackageTest(unittest.TestCase): @@ -457,7 +475,7 @@ def test_archive_rejects_changed_notes_or_vcs_identity(self) -> None: ) - def approval(self, archive: Path, **changes: object) -> Path: + def approval_bytes(self, archive: Path, **changes: object) -> bytes: root = f"{self.package['name']}-{self.package['version']}/" with tarfile.open(archive) as crate: files = sorted(member.name.removeprefix(root) for member in crate.getmembers()) @@ -470,21 +488,25 @@ def approval(self, archive: Path, **changes: object) -> Path: "cargo_version": "1.86.0", "files": files, "file_list_sha256": file_list_sha256(files), - "acceptance": {"summary_sha256": "c" * 64, "server_index": "ghcr.io/durable-workflow/server@sha256:" + "d" * 64}, + "acceptance": dict(ACCEPTANCE), } record.update(changes) + return json.dumps(record).encode() + + def approval(self, archive: Path, **changes: object) -> Path: path = self.root / "approved.json" - path.write_text(json.dumps(record), encoding="utf-8") + path.write_bytes(self.approval_bytes(archive, **changes)) return path def test_approved_archive_binds_checksum_files_commit_version_and_cargo(self) -> None: archive = self.archive() identity = verify_approved_archive( - archive, MANIFEST, CHANGELOG, COMMIT, self.approval(archive), "1.86.0" + archive, MANIFEST, CHANGELOG, COMMIT, self.approval_bytes(archive), "1.86.0" ) self.assertEqual(hashlib.sha256(archive.read_bytes()).hexdigest(), identity["approval"]["crate_sha256"]) self.assertEqual(COMMIT, identity["approval"]["commit"]) self.assertEqual(3, identity["approval"]["file_count"]) + self.assertEqual("3.0.5", identity["approval"]["server_version"]) def test_approved_archive_counterfactuals_are_refused(self) -> None: archive = self.archive() @@ -500,31 +522,274 @@ def test_approved_archive_counterfactuals_are_refused(self) -> None: "unsorted files": ({"files": list(reversed(files))}, "malformed"), "list digest drift": ({"file_list_sha256": "0" * 64}, "malformed"), "wrong schema": ({"schema": "other"}, "malformed"), - "no acceptance": ({"acceptance": {}}, "malformed"), + "empty acceptance": ({"acceptance": {}}, "acceptance receipt"), "extra field": ({"note": "x"}, "unexpected fields"), } for label, (change, message) in cases.items(): - approval = self.approval(archive, **change) + record = self.approval_bytes(archive, **change) with self.subTest(label=label), self.assertRaisesRegex(ReleasePackageError, message): - verify_approved_archive(archive, MANIFEST, CHANGELOG, COMMIT, approval) - approval = self.approval(archive) + verify_approved_archive(archive, MANIFEST, CHANGELOG, COMMIT, record) + record = self.approval_bytes(archive) with self.assertRaisesRegex(ReleasePackageError, "differs from approved Cargo"): - verify_approved_archive(archive, MANIFEST, CHANGELOG, COMMIT, approval, "1.99.0") + verify_approved_archive(archive, MANIFEST, CHANGELOG, COMMIT, record, "1.99.0") # A one-byte drift in the archive itself, with the approval recorded before it. - approval = self.approval(archive) drifted = self.archive(changelog=CHANGELOG.read_bytes(), name="drifted.crate", lockfile=(ROOT / "Cargo.lock").read_bytes() + b"#") with self.assertRaisesRegex(ReleasePackageError, "SHA-256 differs"): - verify_approved_archive(drifted, MANIFEST, CHANGELOG, COMMIT, approval) + verify_approved_archive(drifted, MANIFEST, CHANGELOG, COMMIT, record) with self.assertRaises(ReleasePackageError): - verify_approved_archive(archive, MANIFEST, CHANGELOG, "e" * 40, approval) + verify_approved_archive(archive, MANIFEST, CHANGELOG, "e" * 40, record) + + # Security review of #178 (501d0311), hold 1: a truncated or fabricated + # acceptance receipt must never authorize publication. + + def test_acceptance_receipt_positive_control(self) -> None: + validate_acceptance(dict(ACCEPTANCE)) + validate_acceptance({**ACCEPTANCE, "platform": "linux/amd64", "finished_at": "2026-11-03T23:00:00Z", + "server_index": "docker.io/durableworkflow/server@sha256:" + "1" * 64, + "executed_manifest": "docker.io/durableworkflow/server@sha256:" + "2" * 64}) + + def test_harness_candidate_is_exactly_the_receipt_publication_accepts(self) -> None: + # Guard against schema drift between installed-native/run.py and this verifier. + sys.path.insert(0, str(ROOT / "scripts/acceptance/installed-native")) + self.addCleanup(sys.path.remove, str(ROOT / "scripts/acceptance/installed-native")) + import run as harness + files = sorted(["Cargo.toml", "Cargo.toml.orig", "src/lib.rs"]) + summary = { + "harness_commit": "e" * 40, "outcome": "pass", "finished_at": "2026-11-03T23:00:00.000001+00:00", + "expected_server": {"version": "3.0.5", "revision": "b" * 40, "release_tag": "3.0.5"}, + "server": {"index": "ghcr.io/durable-workflow/server@sha256:" + "d" * 64, + "executed_manifest": "ghcr.io/durable-workflow/server@sha256:" + "f" * 64, + "platform": "linux/arm64"}, + "cells": {"lifecycle": {"cluster_info": {"version": "3.0.5"}}, + "deadline": {"cluster_info": {"version": "3.0.5"}}}, + "package": {"package": "durable-workflow", "version": "4.11.2", "commit": COMMIT, + "crate_sha256": "a" * 64, "cargo_version": "cargo 1.86.0 (adf9b6ad1 2025-02-28)", + "files": files, "file_list_sha256": file_list_sha256(files)}, + } + candidate = harness.approval_candidate(summary, b"{}") + approval, _ = parse_approval(json.dumps(candidate).encode()) + self.assertEqual("3.0.5", approval["acceptance"]["expected_server_version"]) + failed = {**summary, "outcome": "fail"} + with self.assertRaisesRegex(ReleasePackageError, "outcome"): + parse_approval(json.dumps(harness.approval_candidate(failed, b"{}")).encode()) + + def test_acceptance_receipt_deletions_are_refused(self) -> None: + for field in sorted(ACCEPTANCE): + receipt = {key: value for key, value in ACCEPTANCE.items() if key != field} + with self.subTest(deleted=field), self.assertRaisesRegex(ReleasePackageError, "missing or unexpected"): + validate_acceptance(receipt) + for label, receipt in (("extra key", {**ACCEPTANCE, "note": "x"}), ("not an object", ["pass"]), + ("null", None)): + with self.subTest(label=label), self.assertRaisesRegex(ReleasePackageError, "missing or unexpected"): + validate_acceptance(receipt) + + def test_acceptance_receipt_wrong_types_are_refused(self) -> None: + wrong = { + "summary_sha256": 0, "harness_commit": None, "server_index": ["x"], "executed_manifest": {}, + "platform": 1, "expected_server_version": 3.05, "expected_revision": True, + "expected_release_tag": 305, "cluster_versions": "3.0.5", "cells": "deadline,lifecycle", + "finished_at": 1793745600, "outcome": True, + } + self.assertEqual(set(wrong), set(ACCEPTANCE)) + for field, value in wrong.items(): + with self.subTest(field=field), self.assertRaisesRegex(ReleasePackageError, "is invalid"): + validate_acceptance({**ACCEPTANCE, field: value}) + + def test_acceptance_receipt_mismatches_are_refused(self) -> None: + digest = "@sha256:" + "d" * 64 + mismatches = { + "summary_sha256": ["C" * 64, "c" * 63, ""], + "harness_commit": ["e" * 39, "E" * 40, "main"], + "expected_revision": ["b" * 41, "3.0.5"], + "server_index": ["ghcr.io/durable-workflow/server:3.0.5", "ghcr.io/durable-workflow/server@sha256:" + "d" * 63, + "ghcr.io/Durable/server" + digest, "img@sha256:x", "sha256:" + "d" * 64], + "executed_manifest": ["ghcr.io/other/server@sha256:" + "f" * 64, "ghcr.io/durable-workflow/server:3.0.5"], + "platform": ["linux/riscv64", "windows/amd64", "linux/arm64/v8", ""], + "expected_server_version": ["3.0", "v3.0.5", "3.0.5/../x"], + "expected_release_tag": ["native-3.0.5", "3.0.4", ""], + "cluster_versions": [[], ["3.0.4"], ["3.0.5", "3.0.5"], ["", "3.0.5"], [None]], + "cells": [["lifecycle"], ["deadline"], ["lifecycle", "deadline"], ["deadline", "lifecycle", "extra"], []], + "finished_at": ["", "yesterday", "2026-11-03T23:00:00", "2026-13-03T23:00:00+00:00"], + "outcome": ["fail", "PASS", ""], + } + self.assertEqual(set(mismatches), set(ACCEPTANCE)) + for field, values in mismatches.items(): + for value in values: + with self.subTest(field=field, value=value), self.assertRaisesRegex(ReleasePackageError, field): + validate_acceptance({**ACCEPTANCE, field: value}) + # Relationship: tag, cluster versions and expected version must agree. + moved = {**ACCEPTANCE, "expected_server_version": "3.0.6"} + with self.assertRaisesRegex(ReleasePackageError, "cluster_versions, expected_release_tag"): + validate_acceptance(moved) + + # Hold 2: the approval is the tracked regular file at its exact path in + # the maintained-tooling checkout. + + def git(self, repository: Path, *arguments: str) -> None: + subprocess.run( + ["git", "-C", str(repository), "-c", "user.name=t", "-c", "user.email=t@example.invalid", + "-c", "commit.gpgsign=false", *arguments], + check=True, capture_output=True, + env={**os.environ, "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_NOSYSTEM": "1"}, + ) - def approvals(self, archive: Path, **changes: object) -> Path: - """An approvals directory holding .json for `archive`.""" - directory = self.root / "approvals" - directory.mkdir(exist_ok=True) - record = self.approval(archive, **changes) - record.replace(directory / f"{self.package['version']}.json") - return directory + def tooling(self, record: bytes, relative: str | None = None) -> Path: + """A maintained-tooling repository tracking `record` (default: the exact path).""" + repository = self.root / "tooling" + if not repository.exists(): + repository.mkdir() + self.git(repository, "init", "--quiet") + relative = relative or approval_record_relative(self.package["version"]) + path = repository / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(record) + self.git(repository, "add", "--", relative) + self.git(repository, "commit", "--quiet", "--allow-empty", "-m", "approve") + return repository + + def test_tracked_approval_record_positive_control(self) -> None: + record = self.approval_bytes(self.archive()) + repository = self.tooling(record) + content, relative = read_approval_record(confined_root(repository), self.package["version"]) + self.assertEqual((record, f"release/approved-packages/{self.package['version']}.json"), (content, relative)) + + def test_approval_record_symlinks_and_wrong_paths_are_refused(self) -> None: + record = self.approval_bytes(self.archive()) + version = self.package["version"] + relative = approval_record_relative(version) + secret = self.outside_file() + forged = secret.parent / "forged.json" + forged.write_bytes(record) + cases = {} + + def external_symlink(repository: Path) -> None: + (repository / relative).unlink() + (repository / relative).symlink_to(forged) + cases["external symlink"] = (external_symlink, "symbolic link") + + def tracked_symlink(repository: Path) -> None: + external_symlink(repository) + self.git(repository, "add", "--", relative) + self.git(repository, "commit", "--quiet", "-m", "link") + cases["tracked symlink"] = (tracked_symlink, "symbolic link") + + def symlinked_directory(repository: Path) -> None: + directory = repository / "release" / "approved-packages" + shutil.move(directory, secret.parent / "approved-packages") + directory.symlink_to(secret.parent / "approved-packages", target_is_directory=True) + cases["symlinked directory"] = (symlinked_directory, "symbolic link") + + def untracked(repository: Path) -> None: + self.git(repository, "rm", "--quiet", "--cached", "--", relative) + self.git(repository, "commit", "--quiet", "-m", "untrack") + cases["untracked record"] = (untracked, "not tracked at its exact path") + + def wrong_tracked_path(repository: Path) -> None: + self.git(repository, "rm", "--quiet", "--", relative) + self.git(repository, "commit", "--quiet", "-m", "move") + self.tooling(record, f"release/approved-packages/nested/{version}.json") + self.tooling(record, f"release/{version}.json") + cases["wrong tracked path"] = (wrong_tracked_path, "not an existing regular file") + + def edited(repository: Path) -> None: + (repository / relative).write_bytes(record + b" ") + cases["worktree differs from tracked blob"] = (edited, "differs from its tracked content") + + def directory_in_place(repository: Path) -> None: + (repository / relative).unlink() + (repository / relative).mkdir() + cases["directory at the record path"] = (directory_in_place, "not an existing regular file") + + for label, (mutate, message) in cases.items(): + with self.subTest(label=label): + shutil.rmtree(self.root / "tooling", ignore_errors=True) + shutil.rmtree(secret.parent / "approved-packages", ignore_errors=True) + repository = self.tooling(record) + mutate(repository) + with self.assertRaisesRegex(ReleasePackageError, message): + read_approval_record(confined_root(repository), version) + + def test_approval_version_is_strict_semver_before_naming_a_path(self) -> None: + secret = self.outside_file() + for version in ("4.11.2/../../x", "../4.11.2", "4.11.2\0", "v4.11.2", "4.11", "04.11.2", + "4.11.2-..", "4.11.2-rc..1", "4.11.2/", "/4.11.2", "４.11.2", "", + os.path.relpath(secret.parent / "x", self.root)): + with self.subTest(version=version), self.assertRaisesRegex(ReleasePackageError, "strict SemVer"): + approval_record_relative(version) + for version in ("4.11.2", "4.11.2-rc.1+build.5"): + with self.subTest(positive=version): + self.assertEqual(f"release/approved-packages/{version}.json", approval_record_relative(version)) + + def test_approval_record_rejects_unsafe_file_names_and_versions(self) -> None: + archive = self.archive() + for label, change in { + "dot-dot file": {"files": ["../escape"]}, + "absolute file": {"files": ["/etc/passwd"]}, + "NUL file": {"files": ["a\0b"]}, + "backslash file": {"files": ["a\\..\\b"]}, + "empty component": {"files": ["src//lib.rs"]}, + "dot component": {"files": ["./Cargo.toml"]}, + "crafted version": {"version": "4.11.2/../../x"}, + }.items(): + files = change.get("files") + if files is not None: + change = {**change, "file_list_sha256": file_list_sha256(files)} + approval = self.approval(archive, **change) + with self.subTest(label=label), self.assertRaisesRegex(ReleasePackageError, "malformed"): + load_approval(approval) + # The plain reader never follows a link either. + link = self.root / "linked.json" + link.symlink_to(self.approval(archive)) + with self.assertRaisesRegex(ReleasePackageError, "not a regular file"): + load_approval(link) + + # Defense in depth: every .crate member is checked independently. + + def raw_archive(self, extra: list[tuple[tarfile.TarInfo, bytes | None]], name: str = "raw.crate") -> Path: + base = self.archive(name="base-" + name) + path = self.root / name + with tarfile.open(base) as source, tarfile.open(path, "w:gz") as archive: + for member in source.getmembers(): + archive.addfile(member, source.extractfile(member)) + for info, content in extra: + archive.addfile(info, io.BytesIO(content) if content is not None else None) + return path + + def test_crate_members_must_be_canonical_regular_files(self) -> None: + root = f"{self.package['name']}-{self.package['version']}/" + + def regular(name: str) -> tuple[tarfile.TarInfo, bytes]: + info = tarfile.TarInfo(name) + info.size = 1 + return info, b"x" + + symlink = tarfile.TarInfo(root + "src/link.rs") + symlink.type, symlink.linkname = tarfile.SYMTYPE, "/etc/passwd" + hardlink = tarfile.TarInfo(root + "src/hard.rs") + hardlink.type, hardlink.linkname = tarfile.LNKTYPE, root + "Cargo.lock" + directory = tarfile.TarInfo(root + "src") + directory.type = tarfile.DIRTYPE + cases = { + "symlink": [(symlink, None)], + "hardlink": [(hardlink, None)], + "directory": [(directory, None)], + "dot-dot": [regular(root + "../escape.rs")], + "inner dot-dot": [regular(root + "src/../../escape.rs")], + "absolute": [regular("/" + root + "x.rs")], + "backslash": [regular(root + "src\\..\\x.rs")], + "empty component": [regular(root + "src//x.rs")], + "dot component": [regular(root + "./x.rs")], + "wrong prefix": [regular("other-1.0.0/x.rs")], + "prefix only": [regular(root)], + "duplicate": [regular(root + "Cargo.lock")], + } + for label, extra in cases.items(): + archive = self.raw_archive(extra, name=label.replace(" ", "-") + ".crate") + with self.subTest(label=label), self.assertRaises(ReleasePackageError): + verify_archive(archive, MANIFEST, CHANGELOG, expected_vcs_commit=COMMIT) + # Positive control: an extra canonical regular member is accepted. + archive = self.raw_archive([regular(root + "src/extra.rs")], name="canonical.crate") + verify_archive(archive, MANIFEST, CHANGELOG, expected_vcs_commit=COMMIT) def cli(self, *arguments: str) -> subprocess.CompletedProcess[str]: return subprocess.run( @@ -534,11 +799,11 @@ def cli(self, *arguments: str) -> subprocess.CompletedProcess[str]: def test_approved_package_cli_requires_archive_and_commit(self) -> None: archive = self.archive() - approvals = self.approvals(archive) + repository = self.tooling(self.approval_bytes(archive)) base = ["--archive-root", str(self.root), "--archive", archive.name] for arguments in ( - ["--approval-root", str(approvals)], - ["--approval-root", str(approvals), *base], + ["--approval-repository", str(repository)], + ["--approval-repository", str(repository), *base], [*base, "--expected-vcs-commit", COMMIT, "--cargo-version", "1.86.0"], ["--archive", archive.name, "--expected-vcs-commit", COMMIT], ["--check-paths", *base], @@ -546,12 +811,14 @@ def test_approved_package_cli_requires_archive_and_commit(self) -> None: with self.subTest(arguments=arguments): self.assertEqual(2, self.cli(*arguments).returncode) result = self.cli(*base, "--expected-vcs-commit", COMMIT, - "--approval-root", str(approvals), "--cargo-version", "1.86.0") + "--approval-repository", str(repository), "--cargo-version", "1.86.0") self.assertEqual(0, result.returncode, result.stderr) - self.assertEqual(COMMIT, json.loads(result.stdout)["approval"]["commit"]) + identity = json.loads(result.stdout) + self.assertEqual(COMMIT, identity["approval"]["commit"]) + self.assertEqual(approval_record_relative(self.package["version"]), identity["approval"]["record_path"]) - # Path confinement (Security review of #178): every CLI path is confined to - # an explicit root before it is opened. + # Path confinement: every CLI path is confined to an explicit root before + # it is opened. def outside_file(self) -> Path: outside = Path(tempfile.mkdtemp(prefix="release-package-outside-")) @@ -601,44 +868,9 @@ def test_confined_root_refusals(self) -> None: confined_root(value) self.assertEqual(os.path.realpath(self.root / "real"), confined_root(self.root / "real")) - def test_approval_path_requires_strict_semver_before_naming_a_file(self) -> None: - root = confined_root(self.root) - (self.root / "4.11.2.json").write_text("{}") - (self.root / "4.11.2-rc.1+build.5.json").write_text("{}") - secret = self.outside_file() - (secret.parent / "x.json").write_text("{}") - for version in ("4.11.2/../../x", "../4.11.2", "4.11.2\0", "v4.11.2", "4.11", "04.11.2", - "4.11.2-..", "4.11.2-rc..1", "4.11.2/", "/4.11.2", "４.11.2", "", - os.path.relpath(secret.parent / "x", self.root)): - with self.subTest(version=version), self.assertRaisesRegex(ReleasePackageError, "strict SemVer"): - approval_record_path(root, version) - for version in ("4.11.2", "4.11.2-rc.1+build.5"): - with self.subTest(positive=version): - self.assertEqual(Path(root, f"{version}.json"), approval_record_path(root, version)) - (self.root / "4.11.2.json").unlink() - (self.root / "4.11.2.json").symlink_to(secret) - with self.assertRaisesRegex(ReleasePackageError, "symbolic link"): - approval_record_path(root, "4.11.2") - - def test_approval_record_rejects_unsafe_file_names_and_versions(self) -> None: - archive = self.archive() - for label, change in { - "dot-dot file": {"files": ["../escape"]}, - "absolute file": {"files": ["/etc/passwd"]}, - "NUL file": {"files": ["a\0b"]}, - "backslash file": {"files": ["a\\..\\b"]}, - "crafted version": {"version": "4.11.2/../../x"}, - }.items(): - files = change.get("files") - if files is not None: - change = {**change, "file_list_sha256": file_list_sha256(files)} - approval = self.approval(archive, **change) - with self.subTest(label=label), self.assertRaisesRegex(ReleasePackageError, "malformed"): - load_approval(approval) - def test_cli_refuses_paths_outside_their_roots(self) -> None: archive = self.archive() - approvals = self.approvals(archive) + repository = self.tooling(self.approval_bytes(archive)) secret = self.outside_file() (self.root / "escape.crate").symlink_to(secret) verify = ["--expected-vcs-commit", COMMIT] @@ -660,12 +892,12 @@ def test_cli_refuses_paths_outside_their_roots(self) -> None: self.assertNotEqual(0, result.returncode) self.assertEqual("", result.stdout) self.assertNotIn("PRIVATE-TEST-CONTENT", result.stderr) - record = approvals / f"{self.package['version']}.json" + record = repository / approval_record_relative(self.package["version"]) moved = secret.parent / "approved.json" record.replace(moved) record.symlink_to(moved) result = self.cli("--archive-root", str(self.root), "--archive", archive.name, *verify, - "--approval-root", str(approvals)) + "--approval-repository", str(repository)) self.assertEqual(1, result.returncode) self.assertIn("symbolic link", result.stderr) # Positive controls: the same inputs, confined, are accepted. @@ -674,7 +906,7 @@ def test_cli_refuses_paths_outside_their_roots(self) -> None: for arguments in (["--check-paths"], ["--archive-root", str(self.root), "--archive", archive.name, *verify], ["--archive-root", str(self.root), "--archive", archive.name, *verify, - "--approval-root", str(approvals)]): + "--approval-repository", str(repository)]): with self.subTest(positive=arguments): result = self.cli(*arguments) self.assertEqual(0, result.returncode, result.stderr) From f9b28f4f6610abf922bad0f1e6ef43b6f4e336ae Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Tue, 3 Nov 2026 17:33:38 -0600 Subject: [PATCH 5/9] Keep caller values out of git argv for the approval read Sonar flagged the approval repository path reaching `git -C` (S8705). The confined repository is now the subprocess working directory, not an argument. Every git argument is a constant, the SemVer-derived record path after "--", or an object id validated as 40/64 hex. Also merges the nested presence check in publish-rust-sdk.sh and hoists test setup out of two exception blocks. Co-Authored-By: Claude Opus 5.5 --- scripts/ci/publish-rust-sdk.sh | 12 +++++++----- scripts/ci/release_package.py | 16 ++++++++++++++-- scripts/ci/test-publish-rust-sdk.py | 3 +++ scripts/ci/test-release-package.py | 7 ++++--- 4 files changed, 28 insertions(+), 10 deletions(-) diff --git a/scripts/ci/publish-rust-sdk.sh b/scripts/ci/publish-rust-sdk.sh index 45b50c4c..12debdda 100755 --- a/scripts/ci/publish-rust-sdk.sh +++ b/scripts/ci/publish-rust-sdk.sh @@ -477,12 +477,14 @@ release_entry_checksum="$(jq -er '.changelog.entry_sha256' "$local_identity")" # link-free file that equals the blob tracked at that exact path. approved_package_path="release/approved-packages/${package_version}.json" approval_present="false" +approval_tracked="" if [[ -n "$approved_package_repository" ]]; then - if [[ -e "$approved_package_repository/$approved_package_path" || \ - -L "$approved_package_repository/$approved_package_path" ]] || \ - [[ -n "$(git -C "$approved_package_repository" ls-tree --full-tree HEAD -- "$approved_package_path" 2>/dev/null || true)" ]]; then - approval_present="true" - fi + approval_tracked="$(git -C "$approved_package_repository" ls-tree --full-tree HEAD -- "$approved_package_path" 2>/dev/null || true)" +fi +if [[ -n "$approved_package_repository" && \ + ( -e "$approved_package_repository/$approved_package_path" || \ + -L "$approved_package_repository/$approved_package_path" || -n "$approval_tracked" ) ]]; then + approval_present="true" fi if [[ "$approval_present" == "true" ]]; then packaging_cargo="$(cargo -V | awk '{print $2}')" diff --git a/scripts/ci/release_package.py b/scripts/ci/release_package.py index 40083781..01307de7 100644 --- a/scripts/ci/release_package.py +++ b/scripts/ci/release_package.py @@ -568,9 +568,18 @@ def load_approval(path: Path) -> tuple[dict[str, Any], str]: return parse_approval(_read_regular_no_follow(path)) +GIT_OBJECT_PATTERN = re.compile(r"^(?:[0-9a-f]{40}|[0-9a-f]{64})$") + + def _git(repository: str, *arguments: str) -> bytes: + """Run a fixed git query in the confined repository. + + The repository is the working directory, never an argument, and every + argument is a constant, the SemVer-derived record path after "--", or a + validated object id, so no caller value can become a git option. + """ result = subprocess.run( - ["git", "-C", repository, *arguments], capture_output=True, check=False, timeout=60, + ["git", *arguments], cwd=repository, capture_output=True, check=False, timeout=60, ) if result.returncode: raise ReleasePackageError(f"git {arguments[0]} failed in the approvals repository") @@ -598,7 +607,10 @@ def read_approval_record(repository: str, version: str) -> tuple[bytes, str]: if (tracked_path != relative.encode() or len(fields) != 3 or fields[0] != b"100644" or fields[1] != b"blob"): raise ReleasePackageError("approved package record is not a tracked regular file at its exact path") - if _git(repository, "cat-file", "blob", fields[2].decode("ascii")) != content: + object_id = fields[2].decode("ascii") + if not GIT_OBJECT_PATTERN.fullmatch(object_id): + raise ReleasePackageError("approved package record has an invalid tracked object id") + if _git(repository, "cat-file", "blob", object_id) != content: raise ReleasePackageError("approved package record differs from its tracked content") return content, relative diff --git a/scripts/ci/test-publish-rust-sdk.py b/scripts/ci/test-publish-rust-sdk.py index d44c529f..70332b47 100644 --- a/scripts/ci/test-publish-rust-sdk.py +++ b/scripts/ci/test-publish-rust-sdk.py @@ -175,6 +175,9 @@ def _write_mock_commands(self) -> None: command = sys.argv[1] approvals = os.environ.get("MOCK_APPROVAL_REPOSITORY") + in_approvals = approvals and os.path.realpath(os.getcwd()) == os.path.realpath(approvals) + if command in {"ls-tree", "cat-file"} and in_approvals: + os.execv(os.environ["MOCK_REAL_GIT"], ["git", *sys.argv[1:]]) if command == "-C" and approvals and os.path.realpath(sys.argv[2]) == os.path.realpath(approvals): # The approvals checkout is a real repository. os.execv(os.environ["MOCK_REAL_GIT"], ["git", *sys.argv[1:]]) diff --git a/scripts/ci/test-release-package.py b/scripts/ci/test-release-package.py index d7cb782a..8b6a05f5 100644 --- a/scripts/ci/test-release-package.py +++ b/scripts/ci/test-release-package.py @@ -569,9 +569,9 @@ def test_harness_candidate_is_exactly_the_receipt_publication_accepts(self) -> N candidate = harness.approval_candidate(summary, b"{}") approval, _ = parse_approval(json.dumps(candidate).encode()) self.assertEqual("3.0.5", approval["acceptance"]["expected_server_version"]) - failed = {**summary, "outcome": "fail"} + failed = json.dumps(harness.approval_candidate({**summary, "outcome": "fail"}, b"{}")).encode() with self.assertRaisesRegex(ReleasePackageError, "outcome"): - parse_approval(json.dumps(harness.approval_candidate(failed, b"{}")).encode()) + parse_approval(failed) def test_acceptance_receipt_deletions_are_refused(self) -> None: for field in sorted(ACCEPTANCE): @@ -706,8 +706,9 @@ def directory_in_place(repository: Path) -> None: shutil.rmtree(secret.parent / "approved-packages", ignore_errors=True) repository = self.tooling(record) mutate(repository) + root = confined_root(repository) with self.assertRaisesRegex(ReleasePackageError, message): - read_approval_record(confined_root(repository), version) + read_approval_record(root, version) def test_approval_version_is_strict_semver_before_naming_a_path(self) -> None: secret = self.outside_file() From f7ef1d58956fc289b0994650129256c006bfba9d Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Tue, 3 Nov 2026 18:18:24 -0600 Subject: [PATCH 6/9] Isolate the crates.io credential by job and bind the committed acceptance summary Interim review of f9b28f4f held F1-F4. The cross-repo audit added that step-scoping the token is not enough: Runner.Worker holds every secret a job references in memory, hosted runners grant passwordless sudo, and any step of an id-token job can mint OIDC tokens. F1, release.yml as four jobs: - gate (contents: read, no credential) runs publish-rust-sdk.sh gate. It reproduces the tag's package with Cargo's full verify build, so every build script and proc-macro runs without the credential. It requires the reviewed approval and an absent version, then uploads the exact .crate unarchived through the central digest-bound artifact handoff. - publish is the only job with id-token, the crates-io environment and the token. Its steps: - the central handoff validator; - `publish-rust-sdk.sh preflight` (still without the credential): `cargo package --no-verify`, which neither builds nor runs build scripts, must equal the handed-off file, the artifact digest, the gate output and the approval, and the version must still be absent; - minting the token; - `publish-rust-sdk.sh publish`, which runs only `cargo publish --locked --no-verify`. No build or test runs in this job. Cargo is pinned with RUSTUP_TOOLCHAIN, and the tagged source may not carry .cargo/config* or rust-toolchain*. - verify (no credential) runs publish-rust-sdk.sh verify: registry, approval and tag readback, then the fresh-consumer build. - release creates the GitHub Release from the validated notes. The publisher refuses the credential in every other phase and requires it for publish. release.yml passes the central action policy, including the privileged artifact handoff contract, and actionlint. F2: pre-publication mode refuses a harness checkout with modified, staged or untracked files before any work, because the approval names the harness commit. Evidence no longer records absolute local paths. F3: the approval PR commits the run's exact summary.json as release/approved-packages/.summary.json. It is read with the same no-follow, tracked-blob rules. Its SHA-256 must equal summary_sha256. Its package (commit, crate SHA-256 before and after the cells, file list and digest, VCS record, Cargo version), crate lock checksum, outcome, checks_outcome, cleanup, harness commit, Server tuple, finish time and cells must all equal the record and receipt. F4: a record tracked at HEAD but deleted from the worktree still binds, so it can never fall back to legacy verification. The new test fails if the `-n "$approval_tracked"` branch is removed. Removing each guard fails its tests: - F4, the tracked-record branch: 2 failures; - F1, the credential refusal: 6 failures; - F1, the upload without --no-verify: 9 failures; - F1, the preflight handoff equality: 7 failures; - F3, the summary verification: release-package and publish failures; - F2, the clean-harness check: 2 failures and 1 error. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release.yml | 407 ++++++++++++++++-- scripts/acceptance/installed-native/README.md | 82 +++- scripts/acceptance/installed-native/run.py | 24 +- .../acceptance/installed-native/test_run.py | 48 +++ scripts/ci/README.md | 7 +- scripts/ci/publish-rust-sdk.sh | 162 ++++++- scripts/ci/release_package.py | 114 ++++- scripts/ci/test-publish-rust-sdk.py | 387 ++++++++++++++--- scripts/ci/test-release-package.py | 177 +++++++- 9 files changed, 1254 insertions(+), 154 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2b977e63..78fb01d8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,21 +21,37 @@ concurrency: group: release-${{ inputs.release_tag }} cancel-in-progress: false +# The crates.io credential never shares a runner with crate code. Hosted +# runners keep every secret a job references in memory, grant passwordless +# sudo, and let any step of an id-token job mint OIDC tokens, so isolation is +# per job, not per step: +# gate - no credential: reproduce the tag's package with Cargo's full +# verify build, require the reviewed approval and an absent +# version, and hand the exact .crate on as a digest-bound artifact. +# publish - the only job with id-token and the credential. It runs no build, +# test or other crate code: it validates the handed-off .crate, +# reproduces it with --no-verify, requires it to equal the gate +# output and the approval, then runs cargo publish --no-verify. +# verify - no credential: registry, approval and tag readback, then the +# fresh-consumer build. +# release - GitHub release notes and release, after verification. jobs: - publish: - name: Publish exact crates.io package + gate: + name: Gate the approved package (no publication credential) if: >- github.repository == 'durable-workflow/sdk-rust' && github.ref == 'refs/heads/main' runs-on: ubuntu-latest - environment: crates-io - timeout-minutes: 20 + timeout-minutes: 30 permissions: - contents: write - id-token: write - defaults: - run: - working-directory: release-source + contents: read + outputs: + publish_required: ${{ steps.gate.outputs.publish_required }} + crate_sha256: ${{ steps.gate.outputs.crate_sha256 }} + gated_artifact_id: ${{ steps.gated-upload.outputs.artifact-id }} + gated_artifact_digest: ${{ steps.gated-upload.outputs.artifact-digest }} + source_run_id: ${{ github.run_id }} + source_run_attempt: ${{ github.run_attempt }} steps: - name: Check out maintained release tooling uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 @@ -53,6 +69,7 @@ jobs: persist-credentials: false - name: Resolve exact release identity + working-directory: release-source env: REQUESTED_RELEASE_TAG: ${{ inputs.release_tag }} REQUESTED_RELEASE_COMMIT: ${{ inputs.release_commit }} @@ -95,16 +112,180 @@ jobs: clients: pip,python,cargo,curl,git - name: Install hash-locked release tooling + working-directory: release-source run: ../release-tools/scripts/ci/install-release-tooling.sh "${RUNNER_TEMP}/release-tooling" - - name: Prepare exact source release notes + - name: Install Rust 1.86 + uses: dtolnay/rust-toolchain@4d407b29186a635f0cc27475ef0bc0ae605a8866 # 1.86.0 + + - name: Reproduce, build and gate the approved package + id: gate + working-directory: release-source + env: + RUSTUP_TOOLCHAIN: 1.86.0 + RELEASE_RUN_ID: ${{ github.run_id }} + RELEASE_RUN_ATTEMPT: ${{ github.run_attempt }} + RUST_SDK_APPROVED_PACKAGE_REPOSITORY: ${{ github.workspace }}/release-tools + RUST_SDK_GATED_PACKAGE_FILE: ${{ github.workspace }}/gated-package.crate + run: ../release-tools/scripts/ci/publish-rust-sdk.sh gate + + - name: Hand the gated package to the publish job + id: gated-upload + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + archive: 'false' + if-no-files-found: error + path: gated-package.crate + + - name: Upload gate evidence + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: rust-sdk-release-evidence-gate-${{ inputs.release_tag }} + path: release-source/rust-sdk-release-evidence.json + if-no-files-found: error + + publish: + name: Upload the gated package (only job with the publication credential) + needs: gate + if: >- + github.repository == 'durable-workflow/sdk-rust' && + github.ref == 'refs/heads/main' && + needs.gate.outputs.publish_required == 'true' + runs-on: ubuntu-latest + environment: crates-io + timeout-minutes: 20 + permissions: + contents: read + id-token: write + steps: + - name: Check out maintained release tooling + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + fetch-depth: '0' + persist-credentials: 'false' + ref: ${{ github.sha }} + + - name: Download the gated package + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + artifact-ids: ${{ needs.gate.outputs.gated_artifact_id }} + digest-mismatch: error + github-token: ${{ github.token }} + path: isolated-gated-package + repository: ${{ github.repository }} + run-id: ${{ needs.gate.outputs.source_run_id }} + + - name: Validate the gated package handoff + env: + ARTIFACT_DIRECTORY: isolated-gated-package + EXPECTED_ARTIFACT_DIGEST: ${{ needs.gate.outputs.gated_artifact_digest }} + EXPECTED_ARTIFACT_ID: ${{ needs.gate.outputs.gated_artifact_id }} + EXPECTED_SOURCE_RUN_ATTEMPT: ${{ needs.gate.outputs.source_run_attempt }} + EXPECTED_SOURCE_RUN_ID: ${{ needs.gate.outputs.source_run_id }} run: | - python3 ../release-tools/scripts/ci/release_package.py \ - --release-notes "$RELEASE_TAG" > "${RUNNER_TEMP}/rust-sdk-release-notes.md" + set -euo pipefail + if [[ ! "$EXPECTED_ARTIFACT_DIGEST" =~ ^[0-9a-f]{64}$ ]]; then + printf 'producer artifact digest is not an exact SHA-256 digest\n' >&2 + exit 1 + fi + for identity in "$EXPECTED_ARTIFACT_ID" "$EXPECTED_SOURCE_RUN_ID" "$EXPECTED_SOURCE_RUN_ATTEMPT"; do + if [[ ! "$identity" =~ ^[1-9][0-9]*$ ]]; then + printf 'producer artifact identity is invalid\n' >&2 + exit 1 + fi + done + if [[ ! "$ARTIFACT_DIRECTORY" =~ ^isolated-[a-z0-9][a-z0-9._-]*$ ]]; then + printf 'artifact validation directory is unsafe\n' >&2 + exit 1 + fi + mapfile -d '' entries < <( + /usr/bin/find "$ARTIFACT_DIRECTORY" -mindepth 1 -maxdepth 1 -print0 + ) + if [ "${#entries[@]}" -ne 1 ] || [ ! -f "${entries[0]}" ] || [ -L "${entries[0]}" ]; then + printf 'artifact handoff must contain exactly one regular file\n' >&2 + exit 1 + fi + observed_digest="$(/usr/bin/sha256sum "${entries[0]}")" + observed_digest="${observed_digest%% *}" + if [ "$observed_digest" != "$EXPECTED_ARTIFACT_DIGEST" ]; then + printf 'artifact digest mismatch: expected %s, got %s\n' \ + "$EXPECTED_ARTIFACT_DIGEST" "$observed_digest" >&2 + exit 1 + fi + printf 'validated artifact %s from run %s attempt %s\n' \ + "$EXPECTED_ARTIFACT_ID" "$EXPECTED_SOURCE_RUN_ID" "$EXPECTED_SOURCE_RUN_ATTEMPT" + + - name: Check out immutable package source + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + fetch-depth: 0 + ref: ${{ format('refs/tags/{0}', inputs.release_tag) }} + path: release-source + persist-credentials: false + + - name: Resolve exact release identity + working-directory: release-source + env: + REQUESTED_RELEASE_TAG: ${{ inputs.release_tag }} + REQUESTED_RELEASE_COMMIT: ${{ inputs.release_commit }} + run: | + set -euo pipefail + if [[ ! "$REQUESTED_RELEASE_TAG" =~ ^[0-9]+\.[0-9]+\.[0-9]+([-+][0-9A-Za-z.-]+)?$ ]]; then + printf 'release tag must be an exact SDK SemVer: %s\n' "$REQUESTED_RELEASE_TAG" >&2 + exit 1 + fi + if [[ ! "$REQUESTED_RELEASE_COMMIT" =~ ^[0-9a-f]{40}$ ]]; then + printf 'release commit must be a full lowercase Git commit: %s\n' \ + "$REQUESTED_RELEASE_COMMIT" >&2 + exit 1 + fi + head_commit="$(git rev-parse HEAD)" + tag_commit="$(git rev-list -n 1 "$REQUESTED_RELEASE_TAG")" + if [[ "$tag_commit" != "$head_commit" ]]; then + printf 'release tag %s resolves to %s, not checked-out commit %s\n' \ + "$REQUESTED_RELEASE_TAG" "$tag_commit" "$head_commit" >&2 + exit 1 + fi + if [[ "$tag_commit" != "$REQUESTED_RELEASE_COMMIT" ]]; then + printf 'release tag %s resolves to %s, not requested commit %s at checkout %s\n' \ + "$REQUESTED_RELEASE_TAG" "$tag_commit" "$REQUESTED_RELEASE_COMMIT" "$head_commit" >&2 + exit 1 + fi + { + printf 'RELEASE_TAG=%s\n' "$REQUESTED_RELEASE_TAG" + printf 'RELEASE_COMMIT=%s\n' "$head_commit" + } >> "$GITHUB_ENV" + + - name: Set up Python for release tooling + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: '3.12' + + - name: Bootstrap patched pip trust + uses: durable-workflow/.github/.github/actions/bootstrap-patched-pip@75587fc33ff40c34f32ef777293dad56bb6f48c7 # clean-trust + with: + clients: pip,python,cargo,curl,git + + - name: Install hash-locked release tooling + working-directory: release-source + run: ../scripts/ci/install-release-tooling.sh "${RUNNER_TEMP}/release-tooling" - name: Install Rust 1.86 uses: dtolnay/rust-toolchain@4d407b29186a635f0cc27475ef0bc0ae605a8866 # 1.86.0 + - name: Preflight the gated package without the publication credential + working-directory: release-source + env: + RUSTUP_TOOLCHAIN: 1.86.0 + RELEASE_RUN_ID: ${{ github.run_id }} + RELEASE_RUN_ATTEMPT: ${{ github.run_attempt }} + RUST_SDK_APPROVED_PACKAGE_REPOSITORY: ${{ github.workspace }} + RUST_SDK_GATED_PACKAGE_FILE: ${{ github.workspace }}/isolated-gated-package/gated-package.crate + RUST_SDK_GATED_SHA256: ${{ needs.gate.outputs.gated_artifact_digest }} + RUST_SDK_GATED_CRATE_SHA256: ${{ needs.gate.outputs.crate_sha256 }} + run: ../scripts/ci/publish-rust-sdk.sh preflight + - name: Obtain a short-lived crates.io publication token id: crates-auth uses: rust-lang/crates-io-auth-action@c6f97d42243bad5fab37ca0427f495c86d5b1a18 # v1.0.5 @@ -114,17 +295,203 @@ jobs: NODE_USE_SYSTEM_CA: '0' NODE_TLS_REJECT_UNAUTHORIZED: '1' - - name: Publish and verify exact crate + - name: Publish the gated package + working-directory: release-source env: + RUSTUP_TOOLCHAIN: 1.86.0 + RELEASE_RUN_ID: ${{ github.run_id }} + RELEASE_RUN_ATTEMPT: ${{ github.run_attempt }} CARGO_REGISTRY_TOKEN: ${{ steps.crates-auth.outputs.token }} + run: ../scripts/ci/publish-rust-sdk.sh publish + + - name: Upload publish evidence + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: rust-sdk-release-evidence-publish-${{ inputs.release_tag }} + path: release-source/rust-sdk-release-evidence.json + if-no-files-found: error + + verify: + name: Verify the published package (no publication credential) + needs: [gate, publish] + if: >- + always() && + github.repository == 'durable-workflow/sdk-rust' && + github.ref == 'refs/heads/main' && + needs.gate.result == 'success' && + (needs.publish.result == 'success' || needs.publish.result == 'skipped') + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + steps: + - name: Check out maintained release tooling + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + ref: ${{ github.workflow_sha }} + path: release-tools + persist-credentials: false + + - name: Check out immutable package source + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + fetch-depth: 0 + ref: ${{ format('refs/tags/{0}', inputs.release_tag) }} + path: release-source + persist-credentials: false + + - name: Resolve exact release identity + working-directory: release-source + env: + REQUESTED_RELEASE_TAG: ${{ inputs.release_tag }} + REQUESTED_RELEASE_COMMIT: ${{ inputs.release_commit }} + run: | + set -euo pipefail + if [[ ! "$REQUESTED_RELEASE_TAG" =~ ^[0-9]+\.[0-9]+\.[0-9]+([-+][0-9A-Za-z.-]+)?$ ]]; then + printf 'release tag must be an exact SDK SemVer: %s\n' "$REQUESTED_RELEASE_TAG" >&2 + exit 1 + fi + if [[ ! "$REQUESTED_RELEASE_COMMIT" =~ ^[0-9a-f]{40}$ ]]; then + printf 'release commit must be a full lowercase Git commit: %s\n' \ + "$REQUESTED_RELEASE_COMMIT" >&2 + exit 1 + fi + head_commit="$(git rev-parse HEAD)" + tag_commit="$(git rev-list -n 1 "$REQUESTED_RELEASE_TAG")" + if [[ "$tag_commit" != "$head_commit" ]]; then + printf 'release tag %s resolves to %s, not checked-out commit %s\n' \ + "$REQUESTED_RELEASE_TAG" "$tag_commit" "$head_commit" >&2 + exit 1 + fi + if [[ "$tag_commit" != "$REQUESTED_RELEASE_COMMIT" ]]; then + printf 'release tag %s resolves to %s, not requested commit %s at checkout %s\n' \ + "$REQUESTED_RELEASE_TAG" "$tag_commit" "$REQUESTED_RELEASE_COMMIT" "$head_commit" >&2 + exit 1 + fi + { + printf 'RELEASE_TAG=%s\n' "$REQUESTED_RELEASE_TAG" + printf 'RELEASE_COMMIT=%s\n' "$head_commit" + } >> "$GITHUB_ENV" + + - name: Set up Python for release tooling + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: '3.12' + + - name: Bootstrap patched pip trust + uses: durable-workflow/.github/.github/actions/bootstrap-patched-pip@75587fc33ff40c34f32ef777293dad56bb6f48c7 # clean-trust + with: + clients: pip,python,cargo,curl,git + + - name: Install hash-locked release tooling + working-directory: release-source + run: ../release-tools/scripts/ci/install-release-tooling.sh "${RUNNER_TEMP}/release-tooling" + + - name: Install Rust 1.86 + uses: dtolnay/rust-toolchain@4d407b29186a635f0cc27475ef0bc0ae605a8866 # 1.86.0 + + - name: Read back and verify the exact published crate + working-directory: release-source + env: + RUSTUP_TOOLCHAIN: 1.86.0 RELEASE_RUN_ID: ${{ github.run_id }} RELEASE_RUN_ATTEMPT: ${{ github.run_attempt }} - # Reviewed record of the pre-publication package that passed installed-native - # acceptance, committed to main (the maintained tooling), never the tagged source. RUST_SDK_APPROVED_PACKAGE_REPOSITORY: ${{ github.workspace }}/release-tools - run: ../release-tools/scripts/ci/publish-rust-sdk.sh + RUST_SDK_PUBLISHED_IN_RUN: ${{ needs.publish.result == 'success' }} + run: ../release-tools/scripts/ci/publish-rust-sdk.sh verify + + - name: Upload verify evidence + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: rust-sdk-release-evidence-verify-${{ inputs.release_tag }} + path: release-source/rust-sdk-release-evidence.json + if-no-files-found: error + + release: + name: Create the source GitHub Release + needs: verify + if: >- + always() && + github.repository == 'durable-workflow/sdk-rust' && + github.ref == 'refs/heads/main' && + needs.verify.result == 'success' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + steps: + - name: Check out maintained release tooling + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + ref: ${{ github.workflow_sha }} + path: release-tools + persist-credentials: false + + - name: Check out immutable package source + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + fetch-depth: 0 + ref: ${{ format('refs/tags/{0}', inputs.release_tag) }} + path: release-source + persist-credentials: false + + - name: Resolve exact release identity + working-directory: release-source + env: + REQUESTED_RELEASE_TAG: ${{ inputs.release_tag }} + REQUESTED_RELEASE_COMMIT: ${{ inputs.release_commit }} + run: | + set -euo pipefail + if [[ ! "$REQUESTED_RELEASE_TAG" =~ ^[0-9]+\.[0-9]+\.[0-9]+([-+][0-9A-Za-z.-]+)?$ ]]; then + printf 'release tag must be an exact SDK SemVer: %s\n' "$REQUESTED_RELEASE_TAG" >&2 + exit 1 + fi + if [[ ! "$REQUESTED_RELEASE_COMMIT" =~ ^[0-9a-f]{40}$ ]]; then + printf 'release commit must be a full lowercase Git commit: %s\n' \ + "$REQUESTED_RELEASE_COMMIT" >&2 + exit 1 + fi + head_commit="$(git rev-parse HEAD)" + tag_commit="$(git rev-list -n 1 "$REQUESTED_RELEASE_TAG")" + if [[ "$tag_commit" != "$head_commit" ]]; then + printf 'release tag %s resolves to %s, not checked-out commit %s\n' \ + "$REQUESTED_RELEASE_TAG" "$tag_commit" "$head_commit" >&2 + exit 1 + fi + if [[ "$tag_commit" != "$REQUESTED_RELEASE_COMMIT" ]]; then + printf 'release tag %s resolves to %s, not requested commit %s at checkout %s\n' \ + "$REQUESTED_RELEASE_TAG" "$tag_commit" "$REQUESTED_RELEASE_COMMIT" "$head_commit" >&2 + exit 1 + fi + { + printf 'RELEASE_TAG=%s\n' "$REQUESTED_RELEASE_TAG" + printf 'RELEASE_COMMIT=%s\n' "$head_commit" + } >> "$GITHUB_ENV" + + - name: Set up Python for release tooling + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: '3.12' + + - name: Bootstrap patched pip trust + uses: durable-workflow/.github/.github/actions/bootstrap-patched-pip@75587fc33ff40c34f32ef777293dad56bb6f48c7 # clean-trust + with: + clients: pip,python,cargo,curl,git + + - name: Install hash-locked release tooling + working-directory: release-source + run: ../release-tools/scripts/ci/install-release-tooling.sh "${RUNNER_TEMP}/release-tooling" + + - name: Prepare exact source release notes + working-directory: release-source + run: | + python3 ../release-tools/scripts/ci/release_package.py \ + --release-notes "$RELEASE_TAG" > "${RUNNER_TEMP}/rust-sdk-release-notes.md" - name: Create the source GitHub Release + working-directory: release-source env: GH_TOKEN: ${{ github.token }} run: | @@ -135,11 +502,3 @@ jobs: fi gh release create "$RELEASE_TAG" "${arguments[@]}" fi - - - name: Upload release evidence - if: ${{ always() }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: rust-sdk-release-evidence-${{ env.RELEASE_TAG }} - path: release-source/rust-sdk-release-evidence.json - if-no-files-found: error diff --git a/scripts/acceptance/installed-native/README.md b/scripts/acceptance/installed-native/README.md index 914c5783..f2fc165f 100644 --- a/scripts/acceptance/installed-native/README.md +++ b/scripts/acceptance/installed-native/README.md @@ -81,6 +81,10 @@ python3 scripts/acceptance/installed-native/run.py \ --work-dir ``` +Run the harness from a clean checkout. Pre-publication mode refuses a +harness tree with changed, staged or untracked files, because the approval +records the harness commit. + Use the commit that will be tagged. Cargo records the commit in `.cargo_vcs_info.json`, so a different commit (for example a later merge commit with the same tree) produces a different package and will not match. @@ -161,26 +165,64 @@ supported `--platform`, both cells, a 40-hex `--expect-revision`, and `--expect-release-tag` equal to `--expect-server-version`. A reviewer commits that record unchanged to `main` as -`release/approved-packages/.json`. `release/` is outside the crate's -`include` list, so it never changes package bytes. The Release workflow reads -the record from its maintained tooling checkout (`main`), never from the -tagged source. Then: - -- **Before publishing,** it reproduces `cargo package --locked` from the tag and - refuses (`approved_package_mismatch`) unless the archive's SHA-256, file set, - version, VCS commit and Cargo release all equal the record. A version that is - not yet on crates.io is never published without a record - (`approved_package_required`). -- **After publishing,** the crates.io checksum and the downloaded archive must - equal the approved SHA-256. The downloaded archive must match the record - again. The tag must read back from `origin` as the release commit. Any - difference fails the release (`published_checksum_differs_from_approved_package`, - `published_package_differs_from_approved_package`, - `source_tag_readback_mismatch`), and nothing may claim the version is qualified. - -Cargo cannot upload a prebuilt `.crate`, so `cargo publish` packages again from -the same clean tag with the same Cargo. The readback above is what proves that -the published bytes are the accepted ones. +`release/approved-packages/.json`. The same change commits the run's +exact `summary.json` bytes as `release/approved-packages/.summary.json`. +`release/` is outside the crate's `include` list, so neither file changes +package bytes. Both files are read from the maintained tooling checkout +(`main`), never from the tagged source, with the tracked-blob and no-follow +rules below. Publication requires all of the following: +- the summary's SHA-256 equals `acceptance.summary_sha256`; +- the summary is a passing pre-publication run, with checks, outcome and + cleanup all passing and no error; +- its harness commit, Server tuple, finish time and cells match the receipt; +- its package commit, `.crate` SHA-256 (before and after the cells), file list + and digest, VCS record and Cargo version, and its locked crate checksum, all + equal the record. + +The Release workflow (`release.yml`) runs as four jobs. Hosted runners keep +every secret a job references in memory, grant passwordless sudo, and let any +step of an `id-token` job mint OIDC tokens. The credential is therefore +isolated by job, not by step: + +1. **`gate`** has no credential (`contents: read`). + - It reproduces `cargo package --locked` from the tag with Cargo's full + verify build. Every build script and proc-macro runs here, without the + credential. + - It refuses (`approved_package_mismatch`) unless the archive's SHA-256, + file set, version, VCS commit, Cargo release and committed summary all + equal the record. + - It never publishes a missing version without a record + (`approved_package_required`). + - It uploads the `.crate` unarchived as a digest-bound artifact. +2. **`publish`** is the only job with `id-token: write`, the `crates-io` + environment and the minted `CARGO_REGISTRY_TOKEN`. + - It runs no build, test or other crate code. + - The central handoff validator checks the artifact's ID, run and SHA-256. + - Before minting, it reproduces the package with `--no-verify`, which + neither builds nor runs build scripts. It requires that package to equal + the handed-off file, the artifact digest, the gate output and the approval + (`gated_package_mismatch`), and requires the version to still be absent. + - Only then is the token minted, and only `cargo publish --locked + --no-verify` sees it. + - The tagged source is package data only. Cargo is pinned with + `RUSTUP_TOOLCHAIN=1.86.0`, and the publisher refuses an in-repo + `.cargo/config*` or `rust-toolchain*`. +3. **`verify`** has no credential. The crates.io checksum and the downloaded + archive must equal the approved SHA-256, and the downloaded archive must + match the record again. The tag must read back from `origin` as the release + commit. Then the fresh-consumer build runs. Any difference fails the release + (`published_checksum_differs_from_approved_package`, + `published_package_differs_from_approved_package`, + `source_tag_readback_mismatch`), and nothing may claim the version is + qualified. +4. **`release`** creates the GitHub Release from the validated changelog + entry. + +Cargo cannot upload a prebuilt `.crate`, so `cargo publish --no-verify` +packages again from the same clean tag with the same Cargo. The preflight +equality before upload and the readback afterwards are what prove that the +published bytes are the accepted ones. CI's package job also checks every +change: the `--no-verify` repackage must equal the verify-built package. #### Path confinement diff --git a/scripts/acceptance/installed-native/run.py b/scripts/acceptance/installed-native/run.py index 5cb0f313..d33c901a 100755 --- a/scripts/acceptance/installed-native/run.py +++ b/scripts/acceptance/installed-native/run.py @@ -522,13 +522,13 @@ def blob(path): (work_dir / 'package' / expected.name).write_bytes(crate_bytes) metadata = tomllib.loads(members['Cargo.toml'].decode())['package'].get('metadata', {}).get('durable-workflow', {}) return {'mode': 'pre-publication', 'package': PACKAGE_NAME, 'version': version, 'commit': commit, - 'source_checkout': str(checkout), 'crate_file': expected.name, 'crate_sha256': crate_sha256, + 'crate_file': expected.name, 'crate_sha256': crate_sha256, 'crate_size': len(crate_bytes), 'files': sorted(members), 'file_count': len(members), 'file_list_sha256': file_list_sha256(members), 'vcs_info': json.loads(members['.cargo_vcs_info.json']), 'cargo_version': cargo_version, 'rust_image': image_identity(args.rust_image), 'product_train': metadata.get('product-train'), 'verified_source_files': len(members) - len(GENERATED_PACKAGE_FILES), - 'unpacked': str(unpacked)}, members + 'unpacked': str(unpacked.relative_to(work_dir))}, members def cargo_checksum(members, crate_sha256): @@ -552,7 +552,7 @@ def build_packaged_fixture(args, fixture, package, members, run): shutil.copytree(HERE / 'fixture' / 'src', fixture / 'src') final_manifest = fixture_manifest(version) (fixture / 'Cargo.toml').write_text(final_manifest + '\n[patch.crates-io]\n%s = { path = "/package" }\n' % PACKAGE_NAME) - resolve = bounded_cargo(args, run, 'resolve', ['%s:/work' % fixture, '%s:/package:ro' % package['unpacked']], + resolve = bounded_cargo(args, run, 'resolve', ['%s:/work' % fixture, '%s:/package:ro' % (fixture.parent / package['unpacked'])], FIXTURE_ENV, 'cargo -V && cargo generate-lockfile && cargo vendor --locked --versioned-dirs vendor') (fixture / 'resolve.stdout').write_text(resolve.stdout) (fixture / 'resolve.stderr').write_text(resolve.stderr) @@ -881,8 +881,24 @@ def save_logs(self, logs, name, container_id): (logs / (name.removeprefix(self.task + '-') + '.stderr')).write_text(out.stderr) +def harness_identity(args): + """The harness commit, and in pre-publication mode proof that it is clean. + + An approval names the harness commit that produced it, so a locally edited + harness must not be able to produce an approval that names a clean commit. + """ + head = call(['git', '-C', str(REPO), 'rev-parse', '--verify', 'HEAD^{commit}'], check=False).stdout.strip() + if args.package_from: + try: + require_clean_checkout(REPO, head) + except PackageError as error: + raise SystemExit('pre-publication mode requires a clean harness checkout: %s' % error) from error + return head + + def main(): args = parse_args() + harness_commit = harness_identity(args) if args.work_dir.exists(): raise SystemExit('work dir %s already exists; use a new one' % args.work_dir) args.work_dir.mkdir(parents=True) @@ -891,7 +907,7 @@ def main(): 'mode': 'pre-publication' if args.package_from else 'published', 'expected_server': {'version': args.expect_server_version, 'revision': args.expect_revision, 'release_tag': args.expect_release_tag}, - 'harness_commit': call(['git', '-C', str(REPO), 'rev-parse', 'HEAD'], check=False).stdout.strip()} + 'harness_commit': harness_commit} approval = None try: if args.baselines_control: diff --git a/scripts/acceptance/installed-native/test_run.py b/scripts/acceptance/installed-native/test_run.py index 3340dd1f..80307ad6 100644 --- a/scripts/acceptance/installed-native/test_run.py +++ b/scripts/acceptance/installed-native/test_run.py @@ -1350,5 +1350,53 @@ def test_archive_member_with_nul_or_backslash_is_refused(self): run.verify_package_archive(crate, VERSION, commit, sorted(members) + [name], lambda path: None) +class HarnessCleanliness(unittest.TestCase): + """F2: an approval names the harness commit, so package mode needs a clean harness.""" + + def setUp(self): + self.tmp = temporary_directory(self) + self.harness = self.tmp / 'harness' + self.commit = make_repo(self.harness) + self.argv = ['run.py'] + PrePublicationArguments.PACKAGE + [ + '--server-index', INDEX, '--cells', 'lifecycle,deadline', '--work-dir', str(self.tmp / 'work')] + + def main(self, docker): + with mock.patch.object(run, 'REPO', self.harness), mock.patch.object(run, 'call', hybrid(docker)), \ + mock.patch.object(sys, 'argv', self.argv), contextlib.redirect_stdout(io.StringIO()), \ + self.assertRaises(SystemExit) as exited: + run.main() + return exited.exception + + def test_dirty_harness_checkout_is_refused_before_any_work(self): + for change in ('modified', 'untracked', 'staged'): + with self.subTest(change=change): + target = self.harness / ('src/lib.rs' if change != 'untracked' else 'scratch.py') + original = target.read_bytes() if target.exists() else None + target.write_bytes(b'# local edit\n') + if change == 'staged': + subprocess.run(['git', '-C', str(self.harness), 'add', '-A'], check=True, env=git_env()) + docker = FakeDocker() + error = self.main(docker) + self.assertIn('clean harness checkout', str(error)) + self.assertEqual(docker.commands, []) + self.assertFalse((self.tmp / 'work').exists()) + if original is None: + target.unlink() + else: + target.write_bytes(original) + subprocess.run(['git', '-C', str(self.harness), 'reset', '--quiet', '--hard'], check=True, env=git_env()) + + def test_clean_harness_commit_is_recorded(self): + args = run.parse_args(self.argv[1:]) + with mock.patch.object(run, 'REPO', self.harness), mock.patch.object(run, 'call', hybrid(FakeDocker())): + self.assertEqual(run.harness_identity(args), self.commit) + + def test_published_mode_still_only_records_head(self): + (self.harness / 'scratch.py').write_bytes(b'x') + args = run.parse_args(['--crate-version', '4.11.1', '--server-index', 'sha256:x', '--work-dir', str(self.tmp / 'w')]) + with mock.patch.object(run, 'REPO', self.harness), mock.patch.object(run, 'call', hybrid(FakeDocker())): + self.assertEqual(run.harness_identity(args), self.commit) + + if __name__ == '__main__': unittest.main() diff --git a/scripts/ci/README.md b/scripts/ci/README.md index 5fe5b3e4..df90583e 100644 --- a/scripts/ci/README.md +++ b/scripts/ci/README.md @@ -11,8 +11,11 @@ The release workflow uses the crates.io publishing scripts after an immutable tag already exists. A version that is not yet on crates.io is published only when `cargo package` of the tag reproduces the reviewed pre-publication record `release/approved-packages/.json` exactly. That record must be the -regular file tracked at that path in the maintained-tooling checkout, and its -acceptance receipt must be complete and passing. The record must match the +regular file tracked at that path in the maintained-tooling checkout. Its +acceptance receipt must be complete and passing, and must equal the run's +committed `.summary.json`. The release runs as four jobs (`gate`, +`publish`, `verify` and `release`). Only `publish` has `id-token` and the +crates.io credential, and it runs no crate build or test. The record must match the SHA-256, file set, version, VCS commit and Cargo release. After publication, the crates.io checksum, the downloaded archive and the tag read back from `origin` must still match it. See `scripts/acceptance/installed-native/README.md` diff --git a/scripts/ci/publish-rust-sdk.sh b/scripts/ci/publish-rust-sdk.sh index 12debdda..883c387a 100755 --- a/scripts/ci/publish-rust-sdk.sh +++ b/scripts/ci/publish-rust-sdk.sh @@ -2,6 +2,36 @@ set -euo pipefail script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# The release runs as separate jobs so the publication credential never shares +# a runner with crate code: +# gate (job 1, no credential): reproduce the tag's package with Cargo's +# full verify build, require the approved record, require that the +# version is absent, and hand the exact .crate to job 2. +# preflight (job 2, before minting): reproduce the package with --no-verify, +# require it to equal job 1's .crate, its output digest and the +# approval, and require the version to still be absent. +# publish (job 2, the only phase with the credential): cargo publish +# --locked --no-verify. No build, test or other crate code runs. +# verify (job 3, no credential): registry and tag readback against the +# approval, then the fresh-consumer build. +phase="${1:-}" +case "$phase" in + gate | preflight | publish | verify) ;; + *) + printf 'usage: %s gate|preflight|publish|verify\n' "$0" >&2 + exit 2 + ;; +esac +if [[ "$phase" == "publish" ]]; then + if [[ -z "${CARGO_REGISTRY_TOKEN:-}" ]]; then + printf 'the publish phase requires CARGO_REGISTRY_TOKEN\n' >&2 + exit 1 + fi +elif [[ -n "${CARGO_REGISTRY_TOKEN:-}${CARGO_REGISTRIES_CRATES_IO_TOKEN:-}" ]]; then + printf 'the %s phase must not see a crates.io publication credential\n' "$phase" >&2 + exit 1 +fi # shellcheck source=crates-io-publish-lib.sh source "$script_dir/crates-io-publish-lib.sh" @@ -57,6 +87,14 @@ if [[ -n "$(git status --porcelain --untracked-files=all)" ]]; then printf 'Rust SDK publication requires a clean release checkout\n' >&2 exit 1 fi +# The tagged source is package data, never configuration for the tools that +# package or upload it: no Cargo configuration or toolchain override. +for override in .cargo/config .cargo/config.toml rust-toolchain rust-toolchain.toml; do + if [[ -e "$override" || -L "$override" ]]; then + printf 'release source must not contain %s\n' "$override" >&2 + exit 1 + fi +done metadata="$(cargo metadata --manifest-path "$manifest_path" --no-deps --format-version 1)" package_name="$(jq -er '.packages[0].name' <<<"$metadata")" @@ -410,6 +448,21 @@ write_evidence() { }' > "$evidence_path" } +if [[ "$phase" == "publish" ]]; then + # Token-only step. --no-verify packages and uploads without building, so no + # build script, proc-macro or test of the crate or its dependencies runs + # here. The preflight phase in this job already required this clean tag's + # --no-verify package to equal job 1's gated .crate and the approval; the + # verify job reads the published bytes back against the approval. + if ! cargo publish --locked --no-verify --manifest-path "$manifest_path" --registry crates-io; then + write_evidence "failed" "cargo_publish_failed" + exit 1 + fi + write_evidence "published" "exact_public_version_published" + printf 'Rust SDK %s %s was uploaded to crates.io.\n' "$package_name" "$package_version" + exit 0 +fi + # Older immutable tags ignored Cargo.lock. Recover only that missing input from # their verified registry archive; complete local/archive equality still follows. lockfile_path="$(dirname "$manifest_path")/Cargo.lock" @@ -442,7 +495,13 @@ if [[ ! -f "$lockfile_path" || -L "$lockfile_path" ]]; then lockfile_source="verified_registry_archive" fi -if ! cargo package --locked --manifest-path "$manifest_path"; then +# Only the gate phase (job 1, no credential) runs Cargo's verify build of the +# package; the other phases reproduce the same bytes without building. +package_arguments=(package --locked --manifest-path "$manifest_path") +if [[ "$phase" != "gate" ]]; then + package_arguments+=(--no-verify) +fi +if ! cargo "${package_arguments[@]}"; then write_evidence "failed" "local_package_archive_build_failed" exit 1 fi @@ -516,24 +575,101 @@ if ! http_status="$(crates_io_version_lookup "$version_api" "$version_response" exit 1 fi -if [[ "$http_status" == "404" && "$approval_status" != "reproduced" ]]; then +if [[ "$http_status" != "200" && "$http_status" != "404" ]]; then + write_evidence "failed" "registry_version_lookup_http_${http_status}" + printf 'crates.io version lookup failed with HTTP %s\n' "$http_status" >&2 + exit 1 +fi +if [[ "$http_status" == "404" && "$phase" != "verify" && "$approval_status" != "reproduced" ]]; then write_evidence "failed" "approved_package_required" printf 'publishing %s %s requires a reviewed approved-package record (RUST_SDK_APPROVED_PACKAGE_REPOSITORY)\n' \ "$package_name" "$package_version" >&2 exit 1 fi -if ! publish_exact_crate_if_absent "$http_status" "$manifest_path"; then - write_evidence "failed" "$publish_error" - case "$publish_error" in - registry_token_missing) - printf 'CARGO_REGISTRY_TOKEN is required to publish %s %s\n' "$package_name" "$package_version" >&2 - ;; - registry_version_lookup_http_*) - printf 'crates.io version lookup failed with HTTP %s\n' "$http_status" >&2 - ;; - esac - exit 1 +write_output() { + if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + printf '%s=%s\n' "$1" "$2" >> "$GITHUB_OUTPUT" + fi +} + +if [[ "$phase" == "gate" ]]; then + # Hand the exact reproduced bytes to the publish job (uploaded unarchived, + # digest-bound by the central artifact handoff contract). Job 2 trusts + # neither this file nor these outputs alone: it requires both to equal its + # own reproduction and the reviewed approval. + gated_file="${RUST_SDK_GATED_PACKAGE_FILE:-}" + if [[ -z "$gated_file" || -e "$gated_file" || -L "$gated_file" ]]; then + write_evidence "failed" "gated_package_path_invalid" + printf 'RUST_SDK_GATED_PACKAGE_FILE must name a new file\n' >&2 + exit 1 + fi + cp "$local_archive" "$gated_file" + gated_file_sha256="$(sha256sum "$gated_file" | awk '{print $1}')" + if [[ "$gated_file_sha256" != "$local_checksum" ]]; then + write_evidence "failed" "gated_package_mismatch" + exit 1 + fi + write_output crate_sha256 "$gated_file_sha256" + if [[ "$http_status" == "200" ]]; then + write_output publish_required false + write_evidence "gated" "exact_public_version_already_exists" + exit 0 + fi + if [[ "$gated_file_sha256" != "$approved_checksum" ]]; then + write_evidence "failed" "gated_package_mismatch" + exit 1 + fi + write_output publish_required true + write_evidence "gated" "approved_package_reproduced_and_built" + printf 'Rust SDK %s %s is approved, reproduced, built and absent from crates.io.\n' "$package_name" "$package_version" + exit 0 +fi + +if [[ "$phase" == "preflight" ]]; then + if [[ "$http_status" != "404" ]]; then + write_evidence "failed" "registry_version_already_exists" + printf '%s %s already exists on crates.io; nothing to publish\n' "$package_name" "$package_version" >&2 + exit 1 + fi + gated_file="${RUST_SDK_GATED_PACKAGE_FILE:-}" + gated_digest="${RUST_SDK_GATED_SHA256:-}" + gated_output="${RUST_SDK_GATED_CRATE_SHA256:-}" + gated_entries="" + if [[ -n "$gated_file" ]]; then + gated_entries="$(find "$(dirname "$gated_file")" -mindepth 1 -maxdepth 1 -print 2>/dev/null || true)" + fi + if [[ ! "$gated_digest" =~ ^[0-9a-f]{64}$ || ! "$gated_output" =~ ^[0-9a-f]{64}$ || \ + -z "$gated_file" || "$gated_entries" != "$gated_file" || -L "$gated_file" || ! -f "$gated_file" ]]; then + write_evidence "failed" "gated_package_mismatch" + printf 'the gated package handoff must be exactly one regular file with exact digests\n' >&2 + exit 1 + fi + gated_file_sha256="$(sha256sum "$gated_file" | awk '{print $1}')" + if [[ "$gated_file_sha256" != "$gated_digest" || "$gated_file_sha256" != "$gated_output" || \ + "$gated_file_sha256" != "$local_checksum" || "$gated_file_sha256" != "$approved_checksum" ]]; then + write_evidence "failed" "gated_package_mismatch" + printf 'gated package %s, artifact digest %s, gate output %s, reproduction %s and approval %s must be equal\n' \ + "$gated_file_sha256" "$gated_digest" "$gated_output" "$local_checksum" "$approved_checksum" >&2 + exit 1 + fi + write_evidence "preflight_passed" "approved_package_reproduced_without_build" + printf 'Rust SDK %s %s matches the gated package and the approval.\n' "$package_name" "$package_version" + exit 0 +fi + +# verify phase +if [[ "${RUST_SDK_PUBLISHED_IN_RUN:-false}" == "true" ]]; then + publish_outcome="published" + publish_reason="exact_public_version_published" +else + if [[ "$http_status" != "200" ]]; then + write_evidence "failed" "registry_version_absent" + printf '%s %s is not on crates.io\n' "$package_name" "$package_version" >&2 + exit 1 + fi + publish_outcome="already_published" + publish_reason="exact_public_version_already_exists" fi if [[ "$publish_outcome" == "published" ]]; then diff --git a/scripts/ci/release_package.py b/scripts/ci/release_package.py index 01307de7..33e1b322 100644 --- a/scripts/ci/release_package.py +++ b/scripts/ci/release_package.py @@ -441,6 +441,11 @@ def approval_record_relative(version: str) -> str: return f"{APPROVAL_DIRECTORY}/{version}.json" +def approval_summary_relative(version: str) -> str: + """release/approved-packages/.summary.json, the run's exact summary.json.""" + return approval_record_relative(version).removesuffix(".json") + ".summary.json" + + def _safe_member_name(name: object) -> bool: if not isinstance(name, str) or not name or "\0" in name or "\\" in name: return False @@ -586,33 +591,108 @@ def _git(repository: str, *arguments: str) -> bytes: return result.stdout -def read_approval_record(repository: str, version: str) -> tuple[bytes, str]: - """The reviewed record for `version` from the maintained-tooling checkout. +def read_tracked_record(repository: str, relative: str) -> bytes: + """One reviewed file from the maintained-tooling checkout. - Its path is always release/approved-packages/.json, built only - after `version` passes strict SemVer. The worktree file must be a regular - file reached without any symbolic link (lstat + O_NOFOLLOW), confined to - the repository, and byte-identical to a regular (mode 100644) blob tracked - at exactly that path in the checked-out commit. + The worktree file must be a regular file reached without any symbolic link + (lstat + O_NOFOLLOW + fstat), confined to the repository, and + byte-identical to a regular (mode 100644) blob tracked at exactly + `relative` in the checked-out commit. """ - relative = approval_record_relative(version) path = confined_file(repository, relative) content = _read_regular_no_follow(path) listing = _git(repository, "ls-tree", "-z", "--full-tree", "HEAD", "--", relative) entries = [entry for entry in listing.split(b"\0") if entry] if len(entries) != 1: - raise ReleasePackageError("approved package record is not tracked at its exact path") + raise ReleasePackageError(f"approved package file {relative} is not tracked at its exact path") meta, _, tracked_path = entries[0].partition(b"\t") fields = meta.split() if (tracked_path != relative.encode() or len(fields) != 3 or fields[0] != b"100644" or fields[1] != b"blob"): - raise ReleasePackageError("approved package record is not a tracked regular file at its exact path") + raise ReleasePackageError(f"approved package file {relative} is not a tracked regular file at its exact path") object_id = fields[2].decode("ascii") if not GIT_OBJECT_PATTERN.fullmatch(object_id): - raise ReleasePackageError("approved package record has an invalid tracked object id") + raise ReleasePackageError(f"approved package file {relative} has an invalid tracked object id") if _git(repository, "cat-file", "blob", object_id) != content: - raise ReleasePackageError("approved package record differs from its tracked content") - return content, relative + raise ReleasePackageError(f"approved package file {relative} differs from its tracked content") + return content + + +def read_approval_record(repository: str, version: str) -> tuple[bytes, bytes, str]: + """The reviewed record and its acceptance summary for `version`. + + Both live under release/approved-packages/, named only after `version` + passes strict SemVer: .json and .summary.json (the + harness's exact summary.json bytes). Each is read with read_tracked_record. + """ + relative = approval_record_relative(version) + record = read_tracked_record(repository, relative) + summary = read_tracked_record(repository, approval_summary_relative(version)) + return record, summary, relative + + +def _summary_mismatches(approval: dict[str, Any], summary: Any) -> list[str]: + acceptance = approval["acceptance"] + if not isinstance(summary, dict): + return ["summary"] + package = summary.get("package") if isinstance(summary.get("package"), dict) else {} + crate = summary.get("crate") if isinstance(summary.get("crate"), dict) else {} + server = summary.get("server") if isinstance(summary.get("server"), dict) else {} + cells = summary.get("cells") if isinstance(summary.get("cells"), dict) else {} + cargo = package.get("cargo_version") + version = acceptance["expected_server_version"] + checks = { + "mode": summary.get("mode") == "pre-publication", + "outcome": summary.get("outcome") == "pass" and summary.get("checks_outcome") == "pass", + "error": "error" not in summary, + "cleanup": summary.get("cleanup") == { + "pass": True, "errors": [], "local_leftovers": [], + "leftover_containers": [], "leftover_networks": [], + }, + "harness_commit": summary.get("harness_commit") == acceptance["harness_commit"], + "finished_at": summary.get("finished_at") == acceptance["finished_at"], + "expected_server": summary.get("expected_server") == { + "version": version, "revision": acceptance["expected_revision"], + "release_tag": acceptance["expected_release_tag"], + }, + "server": (server.get("index") == acceptance["server_index"] + and server.get("executed_manifest") == acceptance["executed_manifest"] + and server.get("platform") == acceptance["platform"]), + "package.identity": (package.get("package") == approval["package"] + and package.get("version") == approval["version"] + and package.get("commit") == approval["commit"] + and package.get("vcs_info") == {"git": {"sha1": approval["commit"]}, "path_in_vcs": ""}), + "package.crate_sha256": (package.get("crate_sha256") == approval["crate_sha256"] + and package.get("crate_sha256_after_cells") == approval["crate_sha256"]), + "package.files": (package.get("files") == approval["files"] + and package.get("file_list_sha256") == approval["file_list_sha256"]), + "package.cargo_version": (isinstance(cargo, str) and len(cargo.split()) >= 2 + and cargo.split()[0] == "cargo" and cargo.split()[1] == approval["cargo_version"]), + "crate": (crate.get("sha256") == approval["crate_sha256"] + and crate.get("lock_checksum") == approval["crate_sha256"] + and crate.get("source_commit") == approval["commit"] + and crate.get("registry_published") is False + and isinstance(crate.get("baselines"), dict) and crate["baselines"].get("pass") is True), + "cells": (sorted(cells) == acceptance["cells"] + and all(isinstance(cell, dict) and cell.get("outcome") == "pass" + and isinstance(cell.get("cluster_info"), dict) + and cell["cluster_info"].get("version") == version + for cell in cells.values())), + } + return sorted(field for field, ok in checks.items() if not ok) + + +def verify_acceptance_summary(approval: dict[str, Any], summary_record: bytes) -> None: + """The committed summary.json is the run the approval names, and it passed.""" + if sha256(summary_record) != approval["acceptance"]["summary_sha256"]: + raise ReleasePackageError("acceptance summary SHA-256 differs from the approval's summary_sha256") + try: + summary = json.loads(summary_record) + except (UnicodeError, json.JSONDecodeError) as error: + raise ReleasePackageError("acceptance summary is not valid JSON") from error + mismatches = _summary_mismatches(approval, summary) + if mismatches: + raise ReleasePackageError(f"acceptance summary does not match the approval: {', '.join(mismatches)}") def _archive_files(archive_path: Path, root: str) -> list[str]: @@ -642,6 +722,7 @@ def verify_approved_archive( changelog: Path, expected_vcs_commit: str, approval_record: bytes, + summary_record: bytes, cargo_version: str | None = None, ) -> dict[str, Any]: """Require an archive to be exactly the approved pre-publication package.""" @@ -668,6 +749,8 @@ def verify_approved_archive( raise ReleasePackageError( "archive SHA-256 differs from the approved pre-publication package" ) + # The record matches the archive; now its committed acceptance run must too. + verify_acceptance_summary(approval, summary_record) identity["approval"] = { "record_sha256": approval_sha256, "crate_sha256": approval["crate_sha256"], @@ -679,6 +762,7 @@ def verify_approved_archive( "acceptance_summary_sha256": approval["acceptance"]["summary_sha256"], "server_index": approval["acceptance"]["server_index"], "server_version": approval["acceptance"]["expected_server_version"], + "summary_verified": True, } return identity @@ -744,12 +828,12 @@ def _run(arguments: argparse.Namespace, parser: argparse.ArgumentParser) -> dict parser.error("--expected-vcs-commit requires --archive") return {"schema": SCHEMA, "version": 1, **verify_source(manifest, changelog)} if arguments.approval_repository is not None: - record, relative = read_approval_record( + record, summary, relative = read_approval_record( confined_root(arguments.approval_repository), str(load_package(manifest)["version"]) ) identity = verify_approved_archive( archive, manifest, changelog, arguments.expected_vcs_commit, - record, arguments.cargo_version, + record, summary, arguments.cargo_version, ) identity["approval"]["record_path"] = relative return identity diff --git a/scripts/ci/test-publish-rust-sdk.py b/scripts/ci/test-publish-rust-sdk.py index 70332b47..0066dd35 100644 --- a/scripts/ci/test-publish-rust-sdk.py +++ b/scripts/ci/test-publish-rust-sdk.py @@ -6,6 +6,7 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import json import os +import re from pathlib import Path import hashlib import ssl @@ -37,6 +38,7 @@ CHECKSUM = "a" * 64 REAL_GIT = shutil.which("git") APPROVAL_RELATIVE = f"release/approved-packages/{PACKAGE_VERSION}.json" +SUMMARY_RELATIVE = f"release/approved-packages/{PACKAGE_VERSION}.summary.json" # The exact receipt approval_candidate() emits for a passing 3.0.5 run. ACCEPTANCE = { "summary_sha256": "c" * 64, @@ -62,6 +64,7 @@ def setUp(self) -> None: self.bin_dir = self.temp / "bin" self.bin_dir.mkdir() self.evidence = self.temp / "evidence.json" + self.cargo_log = self.temp / "cargo-calls.jsonl" self.tooling = self.temp / "tooling" self.tooling.mkdir() self._tooling_git("init", "--quiet") @@ -87,6 +90,13 @@ def _write_mock_commands(self) -> None: import tomllib command = sys.argv[1] + if os.environ.get("MOCK_CARGO_LOG"): + with open(os.environ["MOCK_CARGO_LOG"], "a", encoding="utf-8") as log: + log.write(json.dumps({ + "argv": sys.argv[1:], + "phase": os.environ.get("MOCK_PHASE", ""), + "credential": bool(os.environ.get("CARGO_REGISTRY_TOKEN")), + }) + "\n") if command == "-V": print(f'cargo {os.environ.get("MOCK_CARGO_VERSION", "1.86.0")} (adf9b6ad1 2025-02-28)') raise SystemExit(0) @@ -132,8 +142,8 @@ def _write_mock_commands(self) -> None: member.size = len(content) crate.addfile(member, io.BytesIO(content)) elif command == "publish": - if "--locked" not in sys.argv: - raise SystemExit("publication must use --locked") + if "--locked" not in sys.argv or "--no-verify" not in sys.argv: + raise SystemExit("publication must use --locked --no-verify") Path(os.environ["MOCK_PUBLISH_MARKER"]).write_text("published\n") elif command == "build": if os.environ.get("MOCK_CONSUMER_BUILD_OUTCOME") == "fail": @@ -242,6 +252,8 @@ def _publish( cwd: Path = ROOT, approval: dict[str, object] | None = None, registry_changelog: bytes | None = None, + between_jobs=None, + preflight_environment: dict[str, str] | None = None, ) -> subprocess.CompletedProcess[str]: if cwd == ROOT and not manifest.resolve().is_relative_to(ROOT): # A modified manifest is its own release checkout: the publisher only @@ -266,11 +278,13 @@ def _publish( "MOCK_APPROVAL_REPOSITORY": str(self.tooling), "MOCK_REAL_GIT": REAL_GIT, "MOCK_PUBLISH_MARKER": str(self.temp / "published-marker"), + "MOCK_CARGO_LOG": str(self.cargo_log), } ) env.update(environment or {}) + # The simulated registry's own packaging is not part of the release run. fixture_env = { - **env, + **{key: value for key, value in env.items() if key not in {"CARGO_REGISTRY_TOKEN", "MOCK_CARGO_LOG"}}, "CARGO_TARGET_DIR": str(self.temp / "registry"), "RUST_SDK_CHANGELOG_PATH": str(ROOT / "CHANGELOG.md"), "MOCK_FIXTURE_CREATION": "1", @@ -281,9 +295,9 @@ def _publish( ) registry_archive = Path(env["MOCK_REGISTRY_ARCHIVE"]) if approval is not None: - record = self._approval(registry_archive) + record, summary = self._approval(registry_archive) record.update(approval) - self._commit_approval(json.dumps(record).encode()) + self._commit_approval(json.dumps(record).encode(), summary=summary) if registry_changelog is not None: # crates.io serves different (self-consistent) bytes than were approved. changed = self.temp / "changed-registry" @@ -295,15 +309,69 @@ def _publish( "RUST_SDK_CHANGELOG_PATH": str(changelog)}, check=True, ) env["MOCK_REGISTRY_ARCHIVE"] = str(changed / "package" / registry_archive.name) - return subprocess.run( - ["bash", str(PUBLISH)], - cwd=cwd, - env=env, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=False, + return self._run_release(env, cwd, between_jobs, preflight_environment or {}) + + def _phase(self, phase: str, env: dict[str, str], cwd: Path) -> subprocess.CompletedProcess[str]: + if getattr(self, "_fresh_lockfile", False): + (cwd / "Cargo.lock").unlink(missing_ok=True) + result = subprocess.run( + ["bash", str(PUBLISH), phase], cwd=cwd, env={**env, "MOCK_PHASE": phase}, + text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, ) + self.phases.append(phase) + return result + + def _run_release(self, env, cwd, between_jobs, preflight_environment) -> subprocess.CompletedProcess[str]: + """Run the four release jobs as release.yml does, stopping at the first failure. + + Only the publish phase receives CARGO_REGISTRY_TOKEN; every other phase + runs without it, as their jobs have no credential. + """ + self.phases = [] + token = env.pop("CARGO_REGISTRY_TOKEN", None) + # Each job starts from a fresh checkout: a lockfile a legacy phase + # restored is not visible to the next job. + self._fresh_lockfile = not (cwd / "Cargo.lock").exists() + jobs = self.temp / "jobs" + shutil.rmtree(jobs, ignore_errors=True) + jobs.mkdir() + outputs = jobs / "gate-output" + gated = jobs / "gated-package.crate" + result = self._phase("gate", {**env, "GITHUB_OUTPUT": str(outputs), "RUST_SDK_GATED_PACKAGE_FILE": str(gated)}, cwd) + if result.returncode: + return result + gate = dict(line.split("=", 1) for line in outputs.read_text().splitlines()) + published_in_run = "false" + if gate.get("publish_required") == "true": + # actions/upload-artifact + download-artifact + the central validator: + # job 2 receives the same single file in an isolated directory. + isolated = jobs / "isolated-gated-package" + isolated.mkdir() + shutil.copyfile(gated, isolated / gated.name) + if between_jobs is not None: + between_jobs(isolated / gated.name) + handoff = { + "RUST_SDK_GATED_PACKAGE_FILE": str(isolated / gated.name), + "RUST_SDK_GATED_SHA256": hashlib.sha256(gated.read_bytes()).hexdigest(), + "RUST_SDK_GATED_CRATE_SHA256": gate.get("crate_sha256", ""), + **preflight_environment, + } + result = self._phase("preflight", {**env, **handoff}, cwd) + if result.returncode: + return result + publish_env = dict(env) + if token is not None: + publish_env["CARGO_REGISTRY_TOKEN"] = token + result = self._phase("publish", publish_env, cwd) + if result.returncode: + return result + published_in_run = "true" + return self._phase("verify", {**env, "RUST_SDK_PUBLISHED_IN_RUN": published_in_run}, cwd) + + def _cargo_calls(self) -> list[dict[str, object]]: + if not self.cargo_log.exists(): + return [] + return [json.loads(line) for line in self.cargo_log.read_text(encoding="utf-8").splitlines()] def _tooling_git(self, *arguments: str) -> None: subprocess.run( @@ -313,21 +381,45 @@ def _tooling_git(self, *arguments: str) -> None: env={**os.environ, "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_NOSYSTEM": "1"}, ) - def _commit_approval(self, content: bytes, relative: str = APPROVAL_RELATIVE) -> Path: - path = self.tooling / relative - path.parent.mkdir(parents=True, exist_ok=True) - if path.is_symlink(): - path.unlink() - path.write_bytes(content) - self._tooling_git("add", "--", relative) + def _commit_approval(self, content: bytes, relative: str = APPROVAL_RELATIVE, + summary: bytes | None = None) -> Path: + files = {relative: content} + if summary is not None: + files[SUMMARY_RELATIVE] = summary + for name, data in files.items(): + path = self.tooling / name + path.parent.mkdir(parents=True, exist_ok=True) + if path.is_symlink(): + path.unlink() + path.write_bytes(data) + self._tooling_git("add", "--", name) self._tooling_git("commit", "--quiet", "--allow-empty", "-m", "approve") - return path + return self.tooling / relative @staticmethod - def _approval(archive: Path) -> dict[str, object]: + def _approval(archive: Path) -> tuple[dict[str, object], bytes]: + """A record and the exact harness summary.json it binds.""" root = f"durable-workflow-{PACKAGE_VERSION}/" with tarfile.open(archive) as crate: files = sorted(member.name.removeprefix(root) for member in crate.getmembers()) + digest = hashlib.sha256(archive.read_bytes()).hexdigest() + file_list = hashlib.sha256("".join(f"{name}\n" for name in files).encode()).hexdigest() + summary = (json.dumps({ + "outcome": "pass", "checks_outcome": "pass", "mode": "pre-publication", + "expected_server": {"version": "3.0.5", "revision": ACCEPTANCE["expected_revision"], "release_tag": "3.0.5"}, + "harness_commit": ACCEPTANCE["harness_commit"], + "server": {"index": ACCEPTANCE["server_index"], "executed_manifest": ACCEPTANCE["executed_manifest"], + "platform": ACCEPTANCE["platform"]}, + "package": {"package": "durable-workflow", "version": PACKAGE_VERSION, "commit": RELEASE_COMMIT, + "crate_sha256": digest, "crate_sha256_after_cells": digest, "files": files, + "file_list_sha256": file_list, "vcs_info": {"git": {"sha1": RELEASE_COMMIT}, "path_in_vcs": ""}, + "cargo_version": "cargo 1.86.0 (adf9b6ad1 2025-02-28)"}, + "crate": {"sha256": digest, "lock_checksum": digest, "source_commit": RELEASE_COMMIT, + "registry_published": False, "baselines": {"pass": True}}, + "cells": {name: {"outcome": "pass", "cluster_info": {"version": "3.0.5"}} for name in ("lifecycle", "deadline")}, + "cleanup": {"pass": True, "errors": [], "local_leftovers": [], "leftover_containers": [], "leftover_networks": []}, + "finished_at": ACCEPTANCE["finished_at"], + }, indent=2) + "\n").encode() return { "schema": "durable-workflow.rust-sdk.approved-package/v1", "package": "durable-workflow", @@ -336,9 +428,9 @@ def _approval(archive: Path) -> dict[str, object]: "crate_sha256": hashlib.sha256(archive.read_bytes()).hexdigest(), "cargo_version": "1.86.0", "files": files, - "file_list_sha256": hashlib.sha256("".join(f"{name}\n" for name in files).encode()).hexdigest(), - "acceptance": dict(ACCEPTANCE), - } + "file_list_sha256": file_list, + "acceptance": {**ACCEPTANCE, "summary_sha256": hashlib.sha256(summary).hexdigest()}, + }, summary def _evidence(self) -> dict[str, object]: return json.loads(self.evidence.read_text(encoding="utf-8")) @@ -587,6 +679,126 @@ def test_symlinked_evidence_path_is_refused(self) -> None: self.assertIn("release evidence path", result.stderr) self.assertEqual("untouched", target.read_text(encoding="utf-8")) + # F1: the publication credential never shares a job with crate code. + + def test_credential_reaches_only_the_no_verify_upload(self) -> None: + result = self._publish(environment={"MOCK_LOOKUP_STATUS": "404", "CARGO_REGISTRY_TOKEN": "test-token"}, + approval={}) + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual(["gate", "preflight", "publish", "verify"], self.phases) + calls = self._cargo_calls() + with_credential = [call for call in calls if call["credential"]] + self.assertEqual({"publish"}, {call["phase"] for call in with_credential}) + self.assertEqual({"metadata", "publish"}, {call["argv"][0] for call in with_credential}) + [upload] = [call for call in calls if call["argv"][0] == "publish"] + self.assertTrue({"--locked", "--no-verify"} <= set(upload["argv"])) + packages = {call["phase"]: call["argv"] for call in calls if call["argv"][0] == "package"} + self.assertNotIn("--no-verify", packages["gate"]) # the verify build runs credential-free + self.assertIn("--no-verify", packages["preflight"]) + self.assertIn("--no-verify", packages["verify"]) + for call in calls: + if call["argv"][0] in {"build", "test", "run", "check"}: + self.assertFalse(call["credential"], call) + self.assertNotEqual("publish", call["phase"]) + self.assertEqual(set(), {call["argv"][0] for call in calls if call["phase"] == "publish"} - {"metadata", "publish"}) + + def test_phases_refuse_a_misplaced_credential(self) -> None: + def run(phase: str, **environment: str) -> subprocess.CompletedProcess[str]: + env = {key: value for key, value in os.environ.items() + if key not in {"CARGO_REGISTRY_TOKEN", "CARGO_REGISTRIES_CRATES_IO_TOKEN"}} + return subprocess.run(["bash", str(PUBLISH), *([phase] if phase else [])], cwd=ROOT, + env={**env, **environment}, text=True, capture_output=True, check=False) + for phase in ("gate", "preflight", "verify"): + for variable in ("CARGO_REGISTRY_TOKEN", "CARGO_REGISTRIES_CRATES_IO_TOKEN"): + with self.subTest(phase=phase, variable=variable): + result = run(phase, **{variable: "test-token"}) + self.assertEqual(1, result.returncode) + self.assertIn("must not see a crates.io publication credential", result.stderr) + self.assertNotIn("test-token", result.stderr + result.stdout) + result = run("publish") + self.assertEqual(1, result.returncode) + self.assertIn("requires CARGO_REGISTRY_TOKEN", result.stderr) + for phase in ("", "all", "--help"): + with self.subTest(phase=phase): + self.assertEqual(2, run(phase).returncode) + + def test_gated_handoff_must_equal_the_reproduction_and_approval(self) -> None: + def flip_byte(path: Path) -> None: + data = bytearray(path.read_bytes()) + data[-1] ^= 1 + path.write_bytes(bytes(data)) + + def extra_file(path: Path) -> None: + (path.parent / "extra.crate").write_bytes(b"x") + + def symlink(path: Path) -> None: + copy = path.parent.parent / "elsewhere.crate" + path.replace(copy) + path.symlink_to(copy) + + def missing(path: Path) -> None: + path.unlink() + cases = { + "one-byte artifact drift": (flip_byte, {}), + "extra file in the handoff": (extra_file, {}), + "symlinked artifact": (symlink, {}), + "missing artifact": (missing, {}), + "artifact digest output differs": (None, {"RUST_SDK_GATED_SHA256": "0" * 64}), + "gate output differs": (None, {"RUST_SDK_GATED_CRATE_SHA256": "0" * 64}), + "malformed digest": (None, {"RUST_SDK_GATED_SHA256": "sha256:" + "0" * 64}), + } + for label, (tamper, override) in cases.items(): + with self.subTest(label=label): + self.cargo_log.unlink(missing_ok=True) + result = self._publish( + environment={"MOCK_LOOKUP_STATUS": "404", "CARGO_REGISTRY_TOKEN": "test-token"}, + approval={}, between_jobs=tamper, preflight_environment=override, + ) + self.assertNotEqual(0, result.returncode) + self.assertEqual(["gate", "preflight"], self.phases) + self.assertEqual("gated_package_mismatch", self._evidence()["reason"]) + self.assertFalse(self._published()) + self.assertNotIn("publish", [call["argv"][0] for call in self._cargo_calls()]) + + def test_version_published_between_jobs_is_not_uploaded_again(self) -> None: + result = self._publish( + environment={"MOCK_LOOKUP_STATUS": "404", "CARGO_REGISTRY_TOKEN": "test-token"}, approval={}, + between_jobs=lambda _path: (self.temp / "published-marker").write_text("elsewhere\n"), + ) + self.assertNotEqual(0, result.returncode) + self.assertEqual("registry_version_already_exists", self._evidence()["reason"]) + self.assertNotIn("publish", [call["argv"][0] for call in self._cargo_calls()]) + + def test_tagged_source_cannot_configure_cargo_or_the_toolchain(self) -> None: + for override in (".cargo/config.toml", ".cargo/config", "rust-toolchain.toml", "rust-toolchain"): + with self.subTest(override=override): + source = self.temp / "source" + shutil.rmtree(source, ignore_errors=True) + source = self._source_checkout() + (source / override).parent.mkdir(parents=True, exist_ok=True) + (source / override).write_text("[toolchain]\npath = \"/tmp/x\"\n", encoding="utf-8") + result = self._publish(source / "Cargo.toml", {"MOCK_LOOKUP_STATUS": "404", + "CARGO_REGISTRY_TOKEN": "test-token"}, + cwd=source, approval={}) + self.assertNotEqual(0, result.returncode) + self.assertIn("must not contain " + override, result.stderr) + self.assertFalse(self._published()) + + # F4: a record tracked at HEAD but absent from the worktree is still an + # approval, so it can never silently fall back to legacy verification. + + def test_tracked_record_missing_from_the_worktree_still_binds(self) -> None: + self._approved_record() + (self.tooling / APPROVAL_RELATIVE).unlink() + for status in ("200", "404"): + with self.subTest(registry=status): + result = self._publish(environment={"MOCK_LOOKUP_STATUS": status, "CARGO_REGISTRY_TOKEN": "test-token"}) + self.assertNotEqual(0, result.returncode) + self.assertEqual("approved_package_mismatch", self._evidence()["reason"]) + self.assertEqual("reproduction_mismatch", self._evidence()["approved_package"]["status"]) + self.assertIn("not an existing regular file", result.stderr) + self.assertFalse(self._published()) + def test_release_workflow_supplies_the_reviewed_approval_record(self) -> None: workflow = RELEASE_WORKFLOW.read_text(encoding="utf-8") self.assertIn( @@ -929,45 +1141,102 @@ def test_release_path_rejects_package_without_current_release_notes(self) -> Non self.assertEqual("local_package_release_identity_mismatch", evidence["reason"]) self.assertIn("current-version entry", result.stderr) + @staticmethod + def _workflow_jobs() -> dict[str, str]: + """release.yml job name -> that job's source text.""" + workflow = RELEASE_WORKFLOW.read_text(encoding="utf-8") + body = workflow.split("\njobs:\n", maxsplit=1)[1] + parts = re.split(r"^ ([a-z][a-z0-9-]*):\n", body, flags=re.MULTILINE) + return {parts[index]: parts[index + 1] for index in range(1, len(parts), 2)} + + @staticmethod + def _steps(job: str) -> list[str]: + return [" - " + step for step in job.split("\n - ")[1:]] + def test_release_workflow_only_publishes_planned_dispatches(self) -> None: workflow = RELEASE_WORKFLOW.read_text(encoding="utf-8") triggers = workflow.split("\njobs:\n", maxsplit=1)[0] self.assertIn("workflow_dispatch:", triggers) self.assertNotIn("push:", triggers) - self.assertIn("scripts/ci/publish-rust-sdk.sh", workflow) - self.assertIn( - 'scripts/ci/install-release-tooling.sh "${RUNNER_TEMP}/release-tooling"', - workflow, - ) - self.assertLess( - workflow.index("Install hash-locked release tooling"), - workflow.index("CARGO_REGISTRY_TOKEN"), - ) - self.assertNotIn("secrets.CARGO_REGISTRY_TOKEN", workflow) - self.assertIn("environment: crates-io", workflow) - self.assertEqual(1, workflow.count("id-token: write")) - self.assertIn( - "uses: rust-lang/crates-io-auth-action@c6f97d42243bad5fab37ca0427f495c86d5b1a18", - workflow, - ) - self.assertIn("CARGO_REGISTRY_TOKEN: ${{ steps.crates-auth.outputs.token }}", workflow) - self.assertLess(workflow.index("id: crates-auth"), workflow.index("CARGO_REGISTRY_TOKEN")) - self.assertLess(workflow.index("Bootstrap patched pip trust"), workflow.index("Install hash-locked release tooling")) - self.assertIn("ref: ${{ github.workflow_sha }}", workflow) - self.assertIn("working-directory: release-source", workflow) - self.assertIn("run: ../release-tools/scripts/ci/publish-rust-sdk.sh", workflow) - self.assertIn("path: release-source/rust-sdk-release-evidence.json", workflow) - self.assertEqual(2, workflow.count("persist-credentials: false")) - - def test_github_notes_use_validated_source_before_publication(self) -> None: - workflow = RELEASE_WORKFLOW.read_text(encoding="utf-8") - self.assertIn('--release-notes "$RELEASE_TAG"', workflow) - self.assertIn('--notes-file "${RUNNER_TEMP}/rust-sdk-release-notes.md"', workflow) - self.assertNotIn('--generate-notes', workflow) - self.assertLess(workflow.index("Install hash-locked release tooling"), - workflow.index("Prepare exact source release notes")) - self.assertLess(workflow.index("Prepare exact source release notes"), - workflow.index("id: crates-auth")) + self.assertNotIn("secrets.", workflow) + self.assertEqual(["gate", "publish", "verify", "release"], list(self._workflow_jobs())) + for name, job in self._workflow_jobs().items(): + with self.subTest(job=name): + self.assertIn("github.ref == 'refs/heads/main'", job) + self.assertIn('scripts/ci/install-release-tooling.sh "${RUNNER_TEMP}/release-tooling"', job) + self.assertLess(job.index("Bootstrap patched pip trust"), job.index("Install hash-locked release tooling")) + self.assertIn("persist-credentials: false", job.replace("'false'", "false")) + + def test_publication_credential_exists_only_in_the_publish_job(self) -> None: + jobs = self._workflow_jobs() + for marker in ("CARGO_REGISTRY_TOKEN", "id-token", "crates-io-auth-action", "crates-auth", + "environment: crates-io"): + holders = [name for name, job in jobs.items() if marker in job] + with self.subTest(marker=marker): + self.assertEqual(["publish"], holders) + publish = jobs["publish"] + self.assertEqual(1, publish.count("id-token: write")) + self.assertEqual(1, publish.count("CARGO_REGISTRY_TOKEN")) + # The credential is minted after the credential-free preflight, and only + # the single upload step receives it. + steps = self._steps(publish) + [token_step] = [step for step in steps if "CARGO_REGISTRY_TOKEN" in step] + self.assertIn("CARGO_REGISTRY_TOKEN: ${{ steps.crates-auth.outputs.token }}", token_step) + self.assertTrue(token_step.rstrip().endswith("run: ../scripts/ci/publish-rust-sdk.sh publish")) + names = [step.split("\n", 1)[0] for step in steps] + self.assertLess(names.index(" - name: Preflight the gated package without the publication credential"), + names.index(" - name: Obtain a short-lived crates.io publication token")) + self.assertLess(names.index(" - name: Obtain a short-lived crates.io publication token"), + names.index(" - name: Publish the gated package")) + for name in ("gate", "verify", "release"): + with self.subTest(job=name): + self.assertNotIn("id-token", jobs[name]) + self.assertNotIn("environment:", jobs[name]) + + def test_publish_job_runs_no_crate_build_or_test(self) -> None: + publish = self._workflow_jobs()["publish"] + for forbidden in ("cargo build", "cargo test", "cargo run", "cargo check", "verify-fresh-consumer", + "publish-rust-sdk.sh gate", "publish-rust-sdk.sh verify", "cargo package", + "cargo publish"): + with self.subTest(forbidden=forbidden): + self.assertNotIn(forbidden, publish) + phases = re.findall(r"publish-rust-sdk\.sh (\w+)", publish) + self.assertEqual(["preflight", "publish"], phases) + # The tagged source is data for --no-verify packaging: Cargo cannot be + # redirected by an in-repo toolchain file. + for step in self._steps(publish): + if "publish-rust-sdk.sh" in step: + self.assertIn("RUSTUP_TOOLCHAIN: 1.86.0", step) + # The handed-off .crate is digest-bound before any other use. + steps = self._steps(publish) + self.assertIn("uses: actions/download-artifact@", steps[1]) + self.assertIn("artifact-ids: ${{ needs.gate.outputs.gated_artifact_id }}", steps[1]) + self.assertIn("digest-mismatch: error", steps[1]) + self.assertIn("EXPECTED_ARTIFACT_DIGEST: ${{ needs.gate.outputs.gated_artifact_digest }}", steps[2]) + self.assertIn("RUST_SDK_GATED_SHA256: ${{ needs.gate.outputs.gated_artifact_digest }}", publish) + self.assertIn("RUST_SDK_GATED_CRATE_SHA256: ${{ needs.gate.outputs.crate_sha256 }}", publish) + + def test_gate_and_verify_jobs_build_without_the_credential(self) -> None: + jobs = self._workflow_jobs() + self.assertIn("run: ../release-tools/scripts/ci/publish-rust-sdk.sh gate", jobs["gate"]) + self.assertIn("path: gated-package.crate", jobs["gate"]) + self.assertIn("run: ../release-tools/scripts/ci/publish-rust-sdk.sh verify", jobs["verify"]) + self.assertIn("RUST_SDK_PUBLISHED_IN_RUN: ${{ needs.publish.result == 'success' }}", jobs["verify"]) + self.assertIn("needs: [gate, publish]", jobs["verify"]) + self.assertIn("needs.publish.result == 'success' || needs.publish.result == 'skipped'", jobs["verify"]) + self.assertIn("needs: verify", jobs["release"]) + self.assertIn("contents: write", jobs["release"]) + for name in ("gate", "publish", "verify"): + self.assertNotIn("contents: write", jobs[name]) + + def test_github_notes_use_validated_source_after_verification(self) -> None: + release = self._workflow_jobs()["release"] + self.assertIn('--release-notes "$RELEASE_TAG"', release) + self.assertIn('--notes-file "${RUNNER_TEMP}/rust-sdk-release-notes.md"', release) + self.assertNotIn('--generate-notes', release) + self.assertLess(release.index("- name: Prepare exact source release notes"), + release.index("- name: Create the source GitHub Release")) + self.assertIsNone(re.search(r"\bcargo (build|test|run|check|package|publish)\b", release)) class ReleaseToolingInstallerTrustTest(unittest.TestCase): diff --git a/scripts/ci/test-release-package.py b/scripts/ci/test-release-package.py index 8b6a05f5..1d64cc86 100644 --- a/scripts/ci/test-release-package.py +++ b/scripts/ci/test-release-package.py @@ -19,6 +19,7 @@ APPROVAL_SCHEMA, ReleasePackageError, approval_record_relative, + approval_summary_relative, confined_file, confined_root, file_list_sha256, @@ -26,6 +27,7 @@ parse_approval, read_approval_record, validate_acceptance, + verify_acceptance_summary, load_package, release_entry, render_release_notes, @@ -475,10 +477,43 @@ def test_archive_rejects_changed_notes_or_vcs_identity(self) -> None: ) - def approval_bytes(self, archive: Path, **changes: object) -> bytes: + def archive_files(self, archive: Path) -> list[str]: root = f"{self.package['name']}-{self.package['version']}/" with tarfile.open(archive) as crate: - files = sorted(member.name.removeprefix(root) for member in crate.getmembers()) + return sorted(member.name.removeprefix(root) for member in crate.getmembers()) + + def summary_dict(self, archive: Path) -> dict[str, object]: + """The harness summary.json of a passing run that produced `archive`.""" + digest = hashlib.sha256(archive.read_bytes()).hexdigest() + files = self.archive_files(archive) + return { + "started_at": "2026-11-03T22:00:00+00:00", "outcome": "pass", "mode": "pre-publication", + "expected_server": {"version": "3.0.5", "revision": ACCEPTANCE["expected_revision"], "release_tag": "3.0.5"}, + "harness_commit": ACCEPTANCE["harness_commit"], + "server": {"index": ACCEPTANCE["server_index"], "executed_manifest": ACCEPTANCE["executed_manifest"], + "platform": ACCEPTANCE["platform"]}, + "package": {"mode": "pre-publication", "package": self.package["name"], "version": self.package["version"], + "commit": COMMIT, "crate_sha256": digest, "crate_sha256_after_cells": digest, + "files": files, "file_list_sha256": file_list_sha256(files), + "vcs_info": {"git": {"sha1": COMMIT}, "path_in_vcs": ""}, + "cargo_version": "cargo 1.86.0 (adf9b6ad1 2025-02-28)"}, + "crate": {"mode": "pre-publication", "sha256": digest, "lock_checksum": digest, "source_commit": COMMIT, + "registry_published": False, "baselines": {"pass": True}}, + "cells": {name: {"cell": name, "outcome": "pass", "cluster_info": {"version": "3.0.5"}} + for name in ("lifecycle", "deadline")}, + "checks_outcome": "pass", + "cleanup": {"pass": True, "errors": [], "local_leftovers": [], "leftover_containers": [], + "leftover_networks": []}, + "finished_at": ACCEPTANCE["finished_at"], + } + + def summary_bytes(self, archive: Path, **changes: object) -> bytes: + return (json.dumps({**self.summary_dict(archive), **changes}, ensure_ascii=False, indent=2) + "\n").encode() + + def approval_bytes(self, archive: Path, summary: bytes | None = None, **changes: object) -> bytes: + """A record bound to `summary` (default: the passing summary), kept as self.summary.""" + self.summary = summary if summary is not None else self.summary_bytes(archive) + files = self.archive_files(archive) record = { "schema": APPROVAL_SCHEMA, "package": self.package["name"], @@ -488,7 +523,7 @@ def approval_bytes(self, archive: Path, **changes: object) -> bytes: "cargo_version": "1.86.0", "files": files, "file_list_sha256": file_list_sha256(files), - "acceptance": dict(ACCEPTANCE), + "acceptance": {**ACCEPTANCE, "summary_sha256": hashlib.sha256(self.summary).hexdigest()}, } record.update(changes) return json.dumps(record).encode() @@ -501,7 +536,7 @@ def approval(self, archive: Path, **changes: object) -> Path: def test_approved_archive_binds_checksum_files_commit_version_and_cargo(self) -> None: archive = self.archive() identity = verify_approved_archive( - archive, MANIFEST, CHANGELOG, COMMIT, self.approval_bytes(archive), "1.86.0" + archive, MANIFEST, CHANGELOG, COMMIT, self.approval_bytes(archive), self.summary, "1.86.0" ) self.assertEqual(hashlib.sha256(archive.read_bytes()).hexdigest(), identity["approval"]["crate_sha256"]) self.assertEqual(COMMIT, identity["approval"]["commit"]) @@ -528,16 +563,16 @@ def test_approved_archive_counterfactuals_are_refused(self) -> None: for label, (change, message) in cases.items(): record = self.approval_bytes(archive, **change) with self.subTest(label=label), self.assertRaisesRegex(ReleasePackageError, message): - verify_approved_archive(archive, MANIFEST, CHANGELOG, COMMIT, record) + verify_approved_archive(archive, MANIFEST, CHANGELOG, COMMIT, record, self.summary) record = self.approval_bytes(archive) with self.assertRaisesRegex(ReleasePackageError, "differs from approved Cargo"): - verify_approved_archive(archive, MANIFEST, CHANGELOG, COMMIT, record, "1.99.0") + verify_approved_archive(archive, MANIFEST, CHANGELOG, COMMIT, record, self.summary, "1.99.0") # A one-byte drift in the archive itself, with the approval recorded before it. drifted = self.archive(changelog=CHANGELOG.read_bytes(), name="drifted.crate", lockfile=(ROOT / "Cargo.lock").read_bytes() + b"#") with self.assertRaisesRegex(ReleasePackageError, "SHA-256 differs"): - verify_approved_archive(drifted, MANIFEST, CHANGELOG, COMMIT, record) + verify_approved_archive(drifted, MANIFEST, CHANGELOG, COMMIT, record, self.summary) with self.assertRaises(ReleasePackageError): - verify_approved_archive(archive, MANIFEST, CHANGELOG, "e" * 40, record) + verify_approved_archive(archive, MANIFEST, CHANGELOG, "e" * 40, record, self.summary) # Security review of #178 (501d0311), hold 1: a truncated or fabricated # acceptance receipt must never authorize publication. @@ -633,25 +668,30 @@ def git(self, repository: Path, *arguments: str) -> None: env={**os.environ, "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_NOSYSTEM": "1"}, ) - def tooling(self, record: bytes, relative: str | None = None) -> Path: - """A maintained-tooling repository tracking `record` (default: the exact path).""" + def tooling(self, record: bytes, relative: str | None = None, summary: bytes | None = None) -> Path: + """A maintained-tooling repository tracking `record` (default: the exact + path, together with its summary at .summary.json).""" repository = self.root / "tooling" if not repository.exists(): repository.mkdir() self.git(repository, "init", "--quiet") - relative = relative or approval_record_relative(self.package["version"]) - path = repository / relative - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(record) - self.git(repository, "add", "--", relative) + files = {relative or approval_record_relative(self.package["version"]): record} + if relative is None: + files[approval_summary_relative(self.package["version"])] = summary if summary is not None else self.summary + for name, content in files.items(): + path = repository / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + self.git(repository, "add", "--", name) self.git(repository, "commit", "--quiet", "--allow-empty", "-m", "approve") return repository def test_tracked_approval_record_positive_control(self) -> None: record = self.approval_bytes(self.archive()) repository = self.tooling(record) - content, relative = read_approval_record(confined_root(repository), self.package["version"]) - self.assertEqual((record, f"release/approved-packages/{self.package['version']}.json"), (content, relative)) + content, summary, relative = read_approval_record(confined_root(repository), self.package["version"]) + self.assertEqual((record, self.summary, f"release/approved-packages/{self.package['version']}.json"), + (content, summary, relative)) def test_approval_record_symlinks_and_wrong_paths_are_refused(self) -> None: record = self.approval_bytes(self.archive()) @@ -744,6 +784,109 @@ def test_approval_record_rejects_unsafe_file_names_and_versions(self) -> None: with self.assertRaisesRegex(ReleasePackageError, "not a regular file"): load_approval(link) + # F3: the committed summary.json is the run the approval names. + + def test_committed_summary_positive_control(self) -> None: + archive = self.archive() + approval, _ = parse_approval(self.approval_bytes(archive)) + verify_acceptance_summary(approval, self.summary) + self.assertEqual(f"release/approved-packages/{self.package['version']}.summary.json", + approval_summary_relative(self.package["version"])) + + def test_summary_hash_must_equal_the_approval(self) -> None: + archive = self.archive() + approval, _ = parse_approval(self.approval_bytes(archive)) + for label, summary in (("one byte appended", self.summary + b" "), + ("reformatted", json.dumps(json.loads(self.summary)).encode()), + ("empty", b"")): + with self.subTest(label=label), self.assertRaisesRegex(ReleasePackageError, "summary SHA-256"): + verify_acceptance_summary(approval, summary) + + def test_summary_content_must_match_the_record(self) -> None: + archive = self.archive() + base = self.summary_dict(archive) + package, crate, cells = base["package"], base["crate"], base["cells"] + cases = { + "failed run": ({"outcome": "fail"}, "outcome"), + "failed checks": ({"checks_outcome": "fail"}, "outcome"), + "run error": ({"error": "RuntimeError: x"}, "error"), + "cleanup error": ({"cleanup": {**base["cleanup"], "pass": False, + "errors": [{"step": "x", "object": "y", "error": "Z"}]}}, "cleanup"), + "leftover container": ({"cleanup": {**base["cleanup"], "leftover_containers": ["abc"]}}, "cleanup"), + "published mode": ({"mode": "published"}, "mode"), + "other harness": ({"harness_commit": "0" * 40}, "harness_commit"), + "other finish": ({"finished_at": "2026-11-04T00:00:00+00:00"}, "finished_at"), + "other Server": ({"expected_server": {**base["expected_server"], "revision": "0" * 40}}, "expected_server"), + "other index": ({"server": {**base["server"], "index": "ghcr.io/durable-workflow/server@sha256:" + "0" * 64}}, "server"), + "package commit": ({"package": {**package, "commit": "f" * 40}}, "package.identity"), + "package sha256": ({"package": {**package, "crate_sha256": "0" * 64}}, "package.crate_sha256"), + "sha256 after cells": ({"package": {**package, "crate_sha256_after_cells": "0" * 64}}, "package.crate_sha256"), + "file-list digest": ({"package": {**package, "file_list_sha256": "0" * 64}}, "package.files"), + "file set": ({"package": {**package, "files": package["files"][1:]}}, "package.files"), + "Cargo version": ({"package": {**package, "cargo_version": "cargo 1.99.0 (5f94df478 2026-08-27)"}}, "package.cargo_version"), + "lock checksum": ({"crate": {**crate, "lock_checksum": "0" * 64}}, "crate"), + "baselines failed": ({"crate": {**crate, "baselines": {"pass": False}}}, "crate"), + "failed cell": ({"cells": {**cells, "deadline": {**cells["deadline"], "outcome": "fail"}}}, "cells"), + "missing cell": ({"cells": {"lifecycle": cells["lifecycle"]}}, "cells"), + "cell on other Server": ({"cells": {**cells, "lifecycle": {**cells["lifecycle"], "cluster_info": {"version": "3.0.4"}}}}, "cells"), + } + for label, (change, field) in cases.items(): + summary = self.summary_bytes(archive, **change) + approval, _ = parse_approval(self.approval_bytes(archive, summary=summary)) + with self.subTest(label=label), self.assertRaisesRegex(ReleasePackageError, field): + verify_acceptance_summary(approval, summary) + # The same refusal through the archive verifier the publisher calls. + record = self.approval_bytes(archive, summary=self.summary_bytes(archive, outcome="fail")) + with self.assertRaisesRegex(ReleasePackageError, "outcome"): + verify_approved_archive(archive, MANIFEST, CHANGELOG, COMMIT, record, self.summary) + not_object = b"[]" + approval, _ = parse_approval(self.approval_bytes(archive, summary=not_object)) + with self.assertRaisesRegex(ReleasePackageError, "summary"): + verify_acceptance_summary(approval, not_object) + + def test_committed_summary_is_read_with_the_tracked_rules(self) -> None: + archive = self.archive() + record = self.approval_bytes(archive) + version = self.package["version"] + summary_path = approval_summary_relative(version) + secret = self.outside_file() + forged = secret.parent / "forged.summary.json" + forged.write_bytes(self.summary) + + def symlink(repository: Path) -> None: + (repository / summary_path).unlink() + (repository / summary_path).symlink_to(forged) + + def untracked(repository: Path) -> None: + self.git(repository, "rm", "--quiet", "--cached", "--", summary_path) + self.git(repository, "commit", "--quiet", "-m", "untrack") + + def missing(repository: Path) -> None: + self.git(repository, "rm", "--quiet", "--", summary_path) + self.git(repository, "commit", "--quiet", "-m", "remove") + + def edited(repository: Path) -> None: + (repository / summary_path).write_bytes(self.summary + b" ") + for label, mutate, message in (("symlink", symlink, "symbolic link"), + ("untracked", untracked, "not tracked"), + ("missing", missing, "not an existing regular file"), + ("worktree edit", edited, "differs from its tracked content")): + with self.subTest(label=label): + shutil.rmtree(self.root / "tooling", ignore_errors=True) + repository = self.tooling(record) + mutate(repository) + root = confined_root(repository) + with self.assertRaisesRegex(ReleasePackageError, message): + read_approval_record(root, version) + # Through the CLI, a committed summary that is not the bound one refuses. + shutil.rmtree(self.root / "tooling", ignore_errors=True) + other = self.summary_bytes(archive, finished_at="2026-11-04T00:00:00+00:00") + repository = self.tooling(record, summary=other) + result = self.cli("--archive-root", str(self.root), "--archive", archive.name, + "--expected-vcs-commit", COMMIT, "--approval-repository", str(repository)) + self.assertEqual(1, result.returncode) + self.assertIn("summary SHA-256", result.stderr) + # Defense in depth: every .crate member is checked independently. def raw_archive(self, extra: list[tuple[tarfile.TarInfo, bytes | None]], name: str = "raw.crate") -> Path: From 675236e27ee17a84b872f818814759ad8838f1a8 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:12:11 -0600 Subject: [PATCH 7/9] Keep tagged-tree code out of release tool installation (F5) Interim re-review of f7ef1d58 held F5: the publish (id-token) and release (contents: write) jobs ran install-release-tooling.sh with the tagged tree as the working directory, and the installer ran `python3 -m venv` without -I. With -m, Python puts the working directory first on sys.path, so a tag-root venv/ package or a stdlib-shadowing module (shutil.py) ran before preflight and could mint the crates.io token or poison GITHUB_PATH/ENV. - install-release-tooling.sh: `python3 -I -m venv --without-pip`. - release.yml: every install step runs the maintained copy from a trusted working directory (scripts/ci/... from the main checkout in publish, release-tools/scripts/ci/... elsewhere). Identity resolution reads the tag with `git -C release-source`; the GitHub Release step uses GH_REPO. Only publish-rust-sdk.sh and the release-notes release_package.py call still run with release-source as the working directory. - Every interpreter that runs with the tagged tree as its working directory is isolated: the seven python3 calls in publish-rust-sdk.sh and the release-notes step use -I (none import sibling modules). Tests (test-publish-rust-sdk.py): - The installer runs from a directory holding a planted venv/__init__.py or shutil.py with the real interpreter (only the networked pip install is stubbed); the planted code never runs and the venv is created. - Static allowlist: in every job, a release-source working directory runs only the listed entrypoint (publish: preflight/publish; release: the isolated release-notes call), the installer has no working-directory and is the maintained copy, and no defaults block exists. - Every python3 call in publish-rust-sdk.sh and the installer uses -I. - S5906: assertLessEqual for the cargo publish flag subset. Counterfactual: with f7ef1d58's installer, workflow and publish script the new tests fail (both planted cases mark the run); old line 20 alone, either old install working-directory alone, a non-isolated release-notes or publish-script call, an identity or gh step back in release-source, and a defaults working-directory each fail a test. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release.yml | 40 ++++----- scripts/ci/install-release-tooling.sh | 5 +- scripts/ci/publish-rust-sdk.sh | 16 ++-- scripts/ci/test-publish-rust-sdk.py | 119 +++++++++++++++++++++++++- 4 files changed, 147 insertions(+), 33 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 78fb01d8..bc061e2a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -35,6 +35,10 @@ concurrency: # verify - no credential: registry, approval and tag readback, then the # fresh-consumer build. # release - GitHub release notes and release, after verification. +# The tagged tree (release-source) is data. A step runs with it as the working +# directory only to invoke the maintained publish-rust-sdk.sh or +# release_package.py, and every interpreter there runs isolated (-I). Tool +# installation and other steps run from a maintained checkout or the workspace. jobs: gate: name: Gate the approved package (no publication credential) @@ -69,7 +73,6 @@ jobs: persist-credentials: false - name: Resolve exact release identity - working-directory: release-source env: REQUESTED_RELEASE_TAG: ${{ inputs.release_tag }} REQUESTED_RELEASE_COMMIT: ${{ inputs.release_commit }} @@ -84,8 +87,8 @@ jobs: "$REQUESTED_RELEASE_COMMIT" >&2 exit 1 fi - head_commit="$(git rev-parse HEAD)" - tag_commit="$(git rev-list -n 1 "$REQUESTED_RELEASE_TAG")" + head_commit="$(git -C release-source rev-parse HEAD)" + tag_commit="$(git -C release-source rev-list -n 1 "$REQUESTED_RELEASE_TAG")" if [[ "$tag_commit" != "$head_commit" ]]; then printf 'release tag %s resolves to %s, not checked-out commit %s\n' \ "$REQUESTED_RELEASE_TAG" "$tag_commit" "$head_commit" >&2 @@ -112,8 +115,7 @@ jobs: clients: pip,python,cargo,curl,git - name: Install hash-locked release tooling - working-directory: release-source - run: ../release-tools/scripts/ci/install-release-tooling.sh "${RUNNER_TEMP}/release-tooling" + run: release-tools/scripts/ci/install-release-tooling.sh "${RUNNER_TEMP}/release-tooling" - name: Install Rust 1.86 uses: dtolnay/rust-toolchain@4d407b29186a635f0cc27475ef0bc0ae605a8866 # 1.86.0 @@ -225,7 +227,6 @@ jobs: persist-credentials: false - name: Resolve exact release identity - working-directory: release-source env: REQUESTED_RELEASE_TAG: ${{ inputs.release_tag }} REQUESTED_RELEASE_COMMIT: ${{ inputs.release_commit }} @@ -240,8 +241,8 @@ jobs: "$REQUESTED_RELEASE_COMMIT" >&2 exit 1 fi - head_commit="$(git rev-parse HEAD)" - tag_commit="$(git rev-list -n 1 "$REQUESTED_RELEASE_TAG")" + head_commit="$(git -C release-source rev-parse HEAD)" + tag_commit="$(git -C release-source rev-list -n 1 "$REQUESTED_RELEASE_TAG")" if [[ "$tag_commit" != "$head_commit" ]]; then printf 'release tag %s resolves to %s, not checked-out commit %s\n' \ "$REQUESTED_RELEASE_TAG" "$tag_commit" "$head_commit" >&2 @@ -268,8 +269,7 @@ jobs: clients: pip,python,cargo,curl,git - name: Install hash-locked release tooling - working-directory: release-source - run: ../scripts/ci/install-release-tooling.sh "${RUNNER_TEMP}/release-tooling" + run: scripts/ci/install-release-tooling.sh "${RUNNER_TEMP}/release-tooling" - name: Install Rust 1.86 uses: dtolnay/rust-toolchain@4d407b29186a635f0cc27475ef0bc0ae605a8866 # 1.86.0 @@ -342,7 +342,6 @@ jobs: persist-credentials: false - name: Resolve exact release identity - working-directory: release-source env: REQUESTED_RELEASE_TAG: ${{ inputs.release_tag }} REQUESTED_RELEASE_COMMIT: ${{ inputs.release_commit }} @@ -357,8 +356,8 @@ jobs: "$REQUESTED_RELEASE_COMMIT" >&2 exit 1 fi - head_commit="$(git rev-parse HEAD)" - tag_commit="$(git rev-list -n 1 "$REQUESTED_RELEASE_TAG")" + head_commit="$(git -C release-source rev-parse HEAD)" + tag_commit="$(git -C release-source rev-list -n 1 "$REQUESTED_RELEASE_TAG")" if [[ "$tag_commit" != "$head_commit" ]]; then printf 'release tag %s resolves to %s, not checked-out commit %s\n' \ "$REQUESTED_RELEASE_TAG" "$tag_commit" "$head_commit" >&2 @@ -385,8 +384,7 @@ jobs: clients: pip,python,cargo,curl,git - name: Install hash-locked release tooling - working-directory: release-source - run: ../release-tools/scripts/ci/install-release-tooling.sh "${RUNNER_TEMP}/release-tooling" + run: release-tools/scripts/ci/install-release-tooling.sh "${RUNNER_TEMP}/release-tooling" - name: Install Rust 1.86 uses: dtolnay/rust-toolchain@4d407b29186a635f0cc27475ef0bc0ae605a8866 # 1.86.0 @@ -438,7 +436,6 @@ jobs: persist-credentials: false - name: Resolve exact release identity - working-directory: release-source env: REQUESTED_RELEASE_TAG: ${{ inputs.release_tag }} REQUESTED_RELEASE_COMMIT: ${{ inputs.release_commit }} @@ -453,8 +450,8 @@ jobs: "$REQUESTED_RELEASE_COMMIT" >&2 exit 1 fi - head_commit="$(git rev-parse HEAD)" - tag_commit="$(git rev-list -n 1 "$REQUESTED_RELEASE_TAG")" + head_commit="$(git -C release-source rev-parse HEAD)" + tag_commit="$(git -C release-source rev-list -n 1 "$REQUESTED_RELEASE_TAG")" if [[ "$tag_commit" != "$head_commit" ]]; then printf 'release tag %s resolves to %s, not checked-out commit %s\n' \ "$REQUESTED_RELEASE_TAG" "$tag_commit" "$head_commit" >&2 @@ -481,18 +478,17 @@ jobs: clients: pip,python,cargo,curl,git - name: Install hash-locked release tooling - working-directory: release-source - run: ../release-tools/scripts/ci/install-release-tooling.sh "${RUNNER_TEMP}/release-tooling" + run: release-tools/scripts/ci/install-release-tooling.sh "${RUNNER_TEMP}/release-tooling" - name: Prepare exact source release notes working-directory: release-source run: | - python3 ../release-tools/scripts/ci/release_package.py \ + python3 -I ../release-tools/scripts/ci/release_package.py \ --release-notes "$RELEASE_TAG" > "${RUNNER_TEMP}/rust-sdk-release-notes.md" - name: Create the source GitHub Release - working-directory: release-source env: + GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ github.token }} run: | if ! gh release view "$RELEASE_TAG" >/dev/null 2>&1; then diff --git a/scripts/ci/install-release-tooling.sh b/scripts/ci/install-release-tooling.sh index 9bb1ce46..d83c1d6d 100755 --- a/scripts/ci/install-release-tooling.sh +++ b/scripts/ci/install-release-tooling.sh @@ -17,7 +17,10 @@ fi script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" release_tooling_venv="$1" -python3 -m venv --without-pip "$release_tooling_venv" +# Isolated mode (-I): with -m, a plain interpreter puts the working directory +# first on sys.path, so a venv/ package or stdlib-shadowing module there would +# run before the release tooling exists. +python3 -I -m venv --without-pip "$release_tooling_venv" # Keep dependency requests on the bootstrapped client, outside ensurepip's bundle. python3 -I -m pip --python "$release_tooling_venv/bin/python" install --disable-pip-version-check \ --require-hashes --only-binary=:all: \ diff --git a/scripts/ci/publish-rust-sdk.sh b/scripts/ci/publish-rust-sdk.sh index 883c387a..684c20b4 100755 --- a/scripts/ci/publish-rust-sdk.sh +++ b/scripts/ci/publish-rust-sdk.sh @@ -70,8 +70,10 @@ fi # The manifest and changelog must be regular files inside the release checkout # (the working directory): relative, no '..', no symbolic links, no escape. +# The working directory is the tagged tree, so every interpreter here runs +# isolated (-I): neither that tree nor PYTHON* variables can add import paths. source_root="$(pwd -P)" -if ! python3 "$script_dir/release_package.py" --check-paths --source-root "$source_root" \ +if ! python3 -I "$script_dir/release_package.py" --check-paths --source-root "$source_root" \ --manifest "$manifest_path" --changelog "$changelog_path" >/dev/null; then printf 'Rust SDK manifest and changelog must be regular files inside the release checkout\n' >&2 exit 1 @@ -483,7 +485,7 @@ if [[ ! -f "$lockfile_path" || -L "$lockfile_path" ]]; then "$version_response")" || \ ! crates_io_request --fail --silent --show-error --location \ --header "User-Agent: ${user_agent}" --output "$published_archive" "$download_url" || \ - ! python3 "$script_dir/release_package.py" \ + ! python3 -I "$script_dir/release_package.py" \ --source-root "$source_root" --manifest "$manifest_path" --changelog "$changelog_path" \ --archive-root "$tmp_dir" --archive "$(basename "$published_archive")" \ --expected-vcs-commit "$release_commit" \ @@ -515,7 +517,7 @@ if [[ ! -f "$local_archive" ]]; then fi local_checksum="$(sha256sum "$local_archive" | awk '{print $1}')" local_identity="$tmp_dir/local-package-identity.json" -if ! python3 "$script_dir/release_package.py" \ +if ! python3 -I "$script_dir/release_package.py" \ --source-root "$source_root" \ --manifest "$manifest_path" \ --changelog "$changelog_path" \ @@ -548,7 +550,7 @@ fi if [[ "$approval_present" == "true" ]]; then packaging_cargo="$(cargo -V | awk '{print $2}')" local_approval="$tmp_dir/local-approval.json" - if ! python3 "$script_dir/release_package.py" \ + if ! python3 -I "$script_dir/release_package.py" \ --source-root "$source_root" \ --manifest "$manifest_path" \ --changelog "$changelog_path" \ @@ -745,7 +747,7 @@ if [[ "$local_checksum" != "$published_checksum" ]]; then fi published_identity="$tmp_dir/published-package-identity.json" -if ! python3 "$script_dir/release_package.py" \ +if ! python3 -I "$script_dir/release_package.py" \ --source-root "$source_root" \ --manifest "$manifest_path" \ --changelog "$changelog_path" \ @@ -770,7 +772,7 @@ if [[ "$published_archive_vcs_commit" != "$archive_vcs_commit" || \ fi if [[ "$approval_status" == "reproduced" ]]; then - if ! python3 "$script_dir/release_package.py" \ + if ! python3 -I "$script_dir/release_package.py" \ --source-root "$source_root" \ --manifest "$manifest_path" \ --changelog "$changelog_path" \ @@ -803,7 +805,7 @@ if [[ "$source_tag_readback_commit" != "$release_commit" ]]; then exit 1 fi -if ! python3 "$script_dir/verify-fresh-consumer.py" \ +if ! python3 -I "$script_dir/verify-fresh-consumer.py" \ registry --manifest "$manifest_path"; then write_evidence "failed" "published_fresh_consumer_msrv_build_failed" "true" exit 1 diff --git a/scripts/ci/test-publish-rust-sdk.py b/scripts/ci/test-publish-rust-sdk.py index 0066dd35..d9b1fe98 100644 --- a/scripts/ci/test-publish-rust-sdk.py +++ b/scripts/ci/test-publish-rust-sdk.py @@ -691,7 +691,7 @@ def test_credential_reaches_only_the_no_verify_upload(self) -> None: self.assertEqual({"publish"}, {call["phase"] for call in with_credential}) self.assertEqual({"metadata", "publish"}, {call["argv"][0] for call in with_credential}) [upload] = [call for call in calls if call["argv"][0] == "publish"] - self.assertTrue({"--locked", "--no-verify"} <= set(upload["argv"])) + self.assertLessEqual({"--locked", "--no-verify"}, set(upload["argv"])) packages = {call["phase"]: call["argv"] for call in calls if call["argv"][0] == "package"} self.assertNotIn("--no-verify", packages["gate"]) # the verify build runs credential-free self.assertIn("--no-verify", packages["preflight"]) @@ -1238,6 +1238,76 @@ def test_github_notes_use_validated_source_after_verification(self) -> None: release.index("- name: Create the source GitHub Release")) self.assertIsNone(re.search(r"\bcargo (build|test|run|check|package|publish)\b", release)) + @staticmethod + def _step_run(step: str) -> str: + """A step's run command, with block lines and continuations joined.""" + match = re.search(r"^ run: (.*)$", step, flags=re.MULTILINE) + if match is None: + return "" + if match.group(1).strip() != "|": + return match.group(1).strip() + lines = [] + for line in step[match.end():].splitlines()[1:]: + if line.strip() and not line.startswith(" "): + break + lines.append(line.strip()) + return re.sub(r"\s+", " ", " ".join(lines).replace("\\ ", " ")).strip() + + # F5: the tagged tree is data. Only the maintained entrypoints run with it + # as the working directory; the installer and other steps never do. + TAGGED_TREE_ENTRYPOINTS = { + "gate": {"../release-tools/scripts/ci/publish-rust-sdk.sh gate"}, + "publish": {"../scripts/ci/publish-rust-sdk.sh preflight", "../scripts/ci/publish-rust-sdk.sh publish"}, + "verify": {"../release-tools/scripts/ci/publish-rust-sdk.sh verify"}, + "release": {'python3 -I ../release-tools/scripts/ci/release_package.py --release-notes "$RELEASE_TAG" ' + '> "${RUNNER_TEMP}/rust-sdk-release-notes.md"'}, + } + MAINTAINED_INSTALLER = { + "gate": "release-tools/scripts/ci/install-release-tooling.sh", + "publish": "scripts/ci/install-release-tooling.sh", + "verify": "release-tools/scripts/ci/install-release-tooling.sh", + "release": "release-tools/scripts/ci/install-release-tooling.sh", + } + + def test_tagged_tree_working_directory_runs_only_maintained_entrypoints(self) -> None: + workflow = RELEASE_WORKFLOW.read_text(encoding="utf-8") + # A defaults block would set a working directory no step declares. + self.assertNotIn("defaults:", workflow) + jobs = self._workflow_jobs() + self.assertEqual(set(self.TAGGED_TREE_ENTRYPOINTS), set(jobs)) + for name, job in jobs.items(): + observed = set() + for step in self._steps(job): + directories = re.findall(r"^ working-directory: (.*)$", step, flags=re.MULTILINE) + if not directories: + continue + with self.subTest(job=name, step=step.split("\n", 1)[0]): + self.assertEqual(["release-source"], directories) + self.assertNotIn("\n uses: ", step) + command = self._step_run(step) + self.assertIn(command, self.TAGGED_TREE_ENTRYPOINTS[name]) + observed.add(command) + with self.subTest(job=name): + # Every allowlisted entrypoint is still present: the list matches the workflow. + self.assertEqual(self.TAGGED_TREE_ENTRYPOINTS[name], observed) + [installer] = [step for step in self._steps(job) if "install-release-tooling.sh" in step] + self.assertNotIn("working-directory:", installer) + self.assertEqual(f'{self.MAINTAINED_INSTALLER[name]} "${{RUNNER_TEMP}}/release-tooling"', + self._step_run(installer)) + + def test_tagged_tree_python_runs_isolated(self) -> None: + publish = PUBLISH.read_text(encoding="utf-8") + invocations = re.findall(r"\bpython3\b[^\n]*", publish) + interpreters = [line for line in invocations if not line.startswith("python3 rustc")] + self.assertGreater(len(interpreters), 5) + for line in interpreters: + with self.subTest(line=line): + self.assertTrue(line.startswith("python3 -I "), line) + installer = RELEASE_TOOLING_INSTALLER.read_text(encoding="utf-8") + for line in re.findall(r"^python3\b[^\n]*", installer, flags=re.MULTILINE): + with self.subTest(line=line): + self.assertTrue(line.startswith("python3 -I -m "), line) + class ReleaseToolingInstallerTrustTest(unittest.TestCase): def test_isolated_tools_use_the_bootstrapped_pip_client(self) -> None: @@ -1255,7 +1325,7 @@ def test_isolated_tools_use_the_bootstrapped_pip_client(self) -> None: arguments = sys.argv[1:] with pathlib.Path(os.environ["INSTALLER_CALLS"]).open("a") as handle: handle.write(json.dumps(arguments) + "\\n") - if arguments[:2] == ["-m", "venv"]: + if arguments[:3] == ["-I", "-m", "venv"]: binary_directory = pathlib.Path(arguments[-1]) / "bin" binary_directory.mkdir(parents=True) bundled = binary_directory / "python" @@ -1273,12 +1343,55 @@ def test_isolated_tools_use_the_bootstrapped_pip_client(self) -> None: ) self.assertEqual(0, result.returncode, result.stderr) calls = [json.loads(line) for line in calls_file.read_text().splitlines()] - self.assertEqual(["-m", "venv", "--without-pip", str(venv)], calls[0]) + self.assertEqual(["-I", "-m", "venv", "--without-pip", str(venv)], calls[0]) self.assertEqual(["-I", "-m", "pip", "--python", str(venv / "bin/python")], calls[1][:5]) self.assertIn("--require-hashes", calls[1]) self.assertIn("--only-binary=:all:", calls[1]) self.assertEqual(str(venv / "bin") + "\n", github_path.read_text()) + def test_working_directory_cannot_shadow_the_venv_module(self) -> None: + """A venv/ package or stdlib-shadowing module in the caller's working + directory (the tagged tree, before F5) never runs in the installer.""" + planted = { + "venv-package": {"venv/__init__.py"}, + "stdlib-shadow": {"shutil.py"}, + } + for case, files in planted.items(): + with self.subTest(case=case), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + marker = root / "planted-code-ran" + hostile = root / "tagged-tree" + for relative in files: + path = hostile / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(f"open({str(marker)!r}, 'w').write('ran')\n", encoding="utf-8") + command_directory = root / "bin" + command_directory.mkdir() + # The real interpreter creates the venv exactly as invoked; only + # the networked, hash-locked dependency install is stubbed. + wrapper = command_directory / "python3" + wrapper.write_text(textwrap.dedent("""\ + #!/bin/sh + case " $* " in + *" -m pip "*) exit 0 ;; + esac + exec "$INSTALLER_REAL_PYTHON" "$@" + """), encoding="utf-8") + wrapper.chmod(0o755) + venv = root / "tools" + github_path = root / "github-path" + result = subprocess.run( + ["bash", str(RELEASE_TOOLING_INSTALLER), str(venv)], cwd=hostile, + env={**os.environ, "PATH": f"{command_directory}:{os.environ['PATH']}", + "INSTALLER_REAL_PYTHON": sys.executable, "GITHUB_PATH": str(github_path)}, + capture_output=True, text=True, check=False, timeout=60, + ) + self.assertFalse(marker.exists(), "working-directory code ran in the installer") + self.assertEqual(0, result.returncode, result.stderr) + self.assertTrue((venv / "pyvenv.cfg").is_file(), result.stderr) + self.assertTrue(os.access(venv / "bin" / "python", os.X_OK)) + self.assertEqual(str(venv / "bin") + "\n", github_path.read_text()) + class RegistryHttpsTest(unittest.TestCase): @classmethod From 4bcbca51db415ed8fcc19879d680b46490e2a3eb Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Wed, 4 Nov 2026 00:27:07 -0600 Subject: [PATCH 8/9] Read crate archives without following links; inspect them from memory SonarCloud S8707 flagged tarfile.open(archive_path) in _archive_files: a command-line path reaching a file open. The path is already confined by confined_file, but each archive read now goes through the same no-follow regular-file read as approval records (lstat, O_NOFOLLOW open, fstat) and tarfile inspects the bytes from memory. verify_archive now hashes and inspects the same single read. Nothing is extracted. Adds a regression test that a symlinked crate archive is refused. Co-Authored-By: Claude Opus 5.5 --- scripts/ci/release_package.py | 23 +++++++++++++++-------- scripts/ci/test-release-package.py | 13 +++++++++++++ 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/scripts/ci/release_package.py b/scripts/ci/release_package.py index 33e1b322..56ecbfd7 100644 --- a/scripts/ci/release_package.py +++ b/scripts/ci/release_package.py @@ -7,6 +7,7 @@ from dataclasses import dataclass from datetime import datetime import hashlib +import io import json import os from pathlib import Path @@ -283,9 +284,11 @@ def verify_archive( raise ReleasePackageError("expected VCS commit must be a full lowercase SHA-1") _archive_files(archive_path, f"{source['package']}-{source['package_version']}") + # One no-follow read: the checksum and the inspected members come from the + # same bytes. + archive_bytes = _read_regular_no_follow(archive_path, "crate archive") try: - archive_bytes = archive_path.read_bytes() - with tarfile.open(archive_path, "r:*") as archive: + with tarfile.open(fileobj=io.BytesIO(archive_bytes), mode="r:*") as archive: root = f"{source['package']}-{source['package_version']}" packaged_changelog = _member_bytes( archive, _regular_member(archive, f"{root}/CHANGELOG.md") @@ -354,8 +357,9 @@ def restore_missing_lockfile( or identity["archive"]["sha256"] != expected_archive_sha256 ): raise ReleasePackageError("registry archive checksum differs from expected SHA-256") + archive_bytes = _read_regular_no_follow(archive_path, "crate archive") try: - with tarfile.open(archive_path, "r:*") as archive: + with tarfile.open(fileobj=io.BytesIO(archive_bytes), mode="r:*") as archive: member = _regular_member( archive, f"{identity['package']}-{identity['package_version']}/Cargo.lock" ) @@ -555,16 +559,16 @@ def parse_approval(raw: bytes) -> tuple[dict[str, Any], str]: return approval, sha256(raw) -def _read_regular_no_follow(path: Path) -> bytes: +def _read_regular_no_follow(path: Path, what: str = "approved package record") -> bytes: try: if not stat.S_ISREG(os.lstat(path).st_mode): - raise ReleasePackageError("approved package record is not a regular file") + raise ReleasePackageError(f"{what} is not a regular file") descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) except OSError as error: - raise ReleasePackageError(f"could not open approved package record: {error.strerror}") from error + raise ReleasePackageError(f"could not open {what}: {error.strerror}") from error with os.fdopen(descriptor, "rb") as handle: if not stat.S_ISREG(os.fstat(handle.fileno()).st_mode): - raise ReleasePackageError("approved package record is not a regular file") + raise ReleasePackageError(f"{what} is not a regular file") return handle.read() @@ -697,8 +701,11 @@ def verify_acceptance_summary(approval: dict[str, Any], summary_record: bytes) - def _archive_files(archive_path: Path, root: str) -> list[str]: prefix = f"{root}/" + # Read the confined crate as a regular file without following links, then + # list its members from memory; nothing is ever extracted. + data = _read_regular_no_follow(archive_path, "crate archive") try: - with tarfile.open(archive_path, "r:gz") as archive: + with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as archive: members = archive.getmembers() except (OSError, tarfile.TarError, EOFError) as error: raise ReleasePackageError(f"could not inspect crate archive: {error}") from error diff --git a/scripts/ci/test-release-package.py b/scripts/ci/test-release-package.py index 1d64cc86..0637b934 100644 --- a/scripts/ci/test-release-package.py +++ b/scripts/ci/test-release-package.py @@ -460,6 +460,19 @@ def test_archive_binds_packaged_notes_to_clean_release_commit(self) -> None: identity["archive"]["release_entry_sha256"], ) + def test_archive_reached_through_a_symlink_is_refused(self) -> None: + archive = self.archive() + link = archive.with_name("linked.crate") + link.symlink_to(archive.name) + self.assertEqual( + COMMIT, + verify_archive(archive, MANIFEST, CHANGELOG, expected_vcs_commit=COMMIT)[ + "archive" + ]["vcs_commit"], + ) + with self.assertRaisesRegex(ReleasePackageError, "crate archive is not a regular file"): + verify_archive(link, MANIFEST, CHANGELOG, expected_vcs_commit=COMMIT) + def test_archive_rejects_changed_notes_or_vcs_identity(self) -> None: cases = ( self.archive( From 8325ac7869282fa40dd97f85abe806be5c5de3eb Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Wed, 4 Nov 2026 00:32:29 -0600 Subject: [PATCH 9/9] Justify the confined no-follow read for SonarCloud S8707 S8707 traces the CLI path into _read_regular_no_follow's lstat and open. Every caller confines it under an explicit root first (--archive-root or the approvals repository via confined_root/confined_file), and these are two different roots, so mark the two reads NOSONAR with the reason next to them. Name the repeated "crate archive" label CRATE_ARCHIVE (S1192). Co-Authored-By: Claude Opus 5.5 --- scripts/ci/release_package.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/scripts/ci/release_package.py b/scripts/ci/release_package.py index 56ecbfd7..5a466fde 100644 --- a/scripts/ci/release_package.py +++ b/scripts/ci/release_package.py @@ -286,7 +286,7 @@ def verify_archive( _archive_files(archive_path, f"{source['package']}-{source['package_version']}") # One no-follow read: the checksum and the inspected members come from the # same bytes. - archive_bytes = _read_regular_no_follow(archive_path, "crate archive") + archive_bytes = _read_regular_no_follow(archive_path, CRATE_ARCHIVE) try: with tarfile.open(fileobj=io.BytesIO(archive_bytes), mode="r:*") as archive: root = f"{source['package']}-{source['package_version']}" @@ -357,7 +357,7 @@ def restore_missing_lockfile( or identity["archive"]["sha256"] != expected_archive_sha256 ): raise ReleasePackageError("registry archive checksum differs from expected SHA-256") - archive_bytes = _read_regular_no_follow(archive_path, "crate archive") + archive_bytes = _read_regular_no_follow(archive_path, CRATE_ARCHIVE) try: with tarfile.open(fileobj=io.BytesIO(archive_bytes), mode="r:*") as archive: member = _regular_member( @@ -559,11 +559,23 @@ def parse_approval(raw: bytes) -> tuple[dict[str, Any], str]: return approval, sha256(raw) +CRATE_ARCHIVE = "crate archive" + + def _read_regular_no_follow(path: Path, what: str = "approved package record") -> bytes: + """Read one regular file without following a link at its last component. + + Callers pass only paths already confined under their explicit trust root + (confined_file under --archive-root or the approvals repository, which also + refuses links below the root), so this is the final read of a confined path. + """ + # S8707 (both NOSONAR lines): the path is a CLI argument, but every caller + # confines it under its explicit root first (confined_root/confined_file); + # lstat and O_NOFOLLOW refuse a final link and fstat requires a regular file. try: - if not stat.S_ISREG(os.lstat(path).st_mode): + if not stat.S_ISREG(os.lstat(path).st_mode): # NOSONAR raise ReleasePackageError(f"{what} is not a regular file") - descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) # NOSONAR except OSError as error: raise ReleasePackageError(f"could not open {what}: {error.strerror}") from error with os.fdopen(descriptor, "rb") as handle: @@ -703,7 +715,7 @@ def _archive_files(archive_path: Path, root: str) -> list[str]: prefix = f"{root}/" # Read the confined crate as a regular file without following links, then # list its members from memory; nothing is ever extracted. - data = _read_regular_no_follow(archive_path, "crate archive") + data = _read_regular_no_follow(archive_path, CRATE_ARCHIVE) try: with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as archive: members = archive.getmembers()