diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ddb78feb5..da2706724 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -162,6 +162,7 @@ jobs: upgrade: "FALSE" packages: | deps::. + any::RSQLite - name: Inventory Windows development prerequisites run: | scripts/preflight.cmd --json @@ -396,7 +397,9 @@ jobs: printf 'Dir::Cache::archives "%s";\nAPT::Install-Recommends "false";\n' \ "$RUNNER_TEMP/apt-archives" | sudo tee /etc/apt/apt.conf.d/99-ci-archives sudo apt-get update - sudo apt-get install -y --no-upgrade build-essential pkg-config libcurl4-openssl-dev libcap-dev + sudo apt-get install -y --no-upgrade \ + build-essential pkg-config libcurl4-openssl-dev libcap-dev \ + libharfbuzz-dev libfribidi-dev libjpeg-dev libpng-dev libtiff-dev libwebp-dev sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - name: Configure R cache paths diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9c8e0e41c..ea7e826a1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,6 +1,6 @@ name: Release -# Build native macOS, Linux, and Windows x64 wheels and publish them to PyPI. +# Build native archives and wheels on x86-64 and ARM64. # # Releases are tag-driven. A manual workflow run builds artifacts for # inspection but does not publish. @@ -59,6 +59,10 @@ jobs: with: python-version: "3.14" + # Keep the guard inside each build job so a targeted rerun also checks PyPI. + - name: Refuse to rebuild a version already on PyPI + run: python scripts/release.py check-unpublished + - name: Read private sandbox source pin id: sandbox-pin run: scripts/stage-sandbox-runner --github-output "$GITHUB_OUTPUT" @@ -69,6 +73,7 @@ jobs: repository: ${{ steps.sandbox-pin.outputs.repository }} ref: ${{ steps.sandbox-pin.outputs.commit }} path: .sandbox-runner-source + persist-credentials: false # Chrome is unused here; its repository can block apt with inconsistent metadata. - name: Disable unused Chrome apt repository @@ -102,24 +107,27 @@ jobs: cache-local-path: ${{ runner.temp }}/mcp-console-cache cache-suffix: release-v1-${{ steps.cache-epoch.outputs.value }} - - name: Install Linux build and sandbox dependencies + - name: Build Linux wheel and companions on glibc 2.28 if: runner.os == 'Linux' + env: + TARGET: ${{ matrix.target }} run: | - sudo apt-get update - sudo apt-get install -y build-essential pkgconf libcurl4-openssl-dev libcap-dev libssl-dev binutils + docker run --rm -v "$PWD:/work" -w /work \ + "quay.io/pypa/manylinux_2_28_${TARGET%%-*}" bash scripts/linux-wheel/build.sh "$TARGET" - name: Stage private sandbox runner + if: runner.os == 'macOS' run: >- scripts/stage-sandbox-runner .sandbox-runner-source --target "${{ matrix.target }}" - - name: Build wheel + - name: Build macOS wheel + if: runner.os == 'macOS' uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 with: rust-toolchain: stable target: ${{ matrix.target }} - # Use native runners while retaining PyPI-compatible Linux wheel tags. container: off args: --release --locked --compatibility pypi --out dist @@ -149,7 +157,9 @@ jobs: MAKEFLAGS: -j1 R_MAKEVARS_USER: ${{ runner.temp }}/preparation.Makevars run: | + sudo apt-get update sudo apt-get install -y --no-install-recommends \ + build-essential pkgconf libcurl4-openssl-dev libssl-dev binutils \ r-base-dev libxml2-dev libcairo2-dev libfontconfig1-dev \ libharfbuzz-dev libfribidi-dev libjpeg-dev libpng-dev libtiff-dev \ libuv1-dev libwebp-dev @@ -159,10 +169,11 @@ jobs: export PATH="$RUNNER_TEMP/cmake/bin:$PATH" R_HOME="$(R RHOME)" export R_HOME + printf 'R_HOME=%s\n' "$R_HOME" >> "$GITHUB_ENV" uv tool run --isolated --python 3.14 --from r-lib-ir ir run \ --rscript "$R_HOME/bin/Rscript" \ --with tidyverse --with dplyr --with dbplyr --with reticulate --with DBI \ - --with duckdb --with arrow --with nanoarrow --with yyjsonr --with ggplot2 \ + --with duckdb --with nanoarrow --with yyjsonr --with ggplot2 \ --with jsonlite --with pillar --with tibble --with utf8 \ --isolated --vanilla -e 'sessionInfo()' uv cache clean cmake @@ -188,7 +199,14 @@ jobs: target/${{ matrix.target }}/release/mcp-console \ --target "${{ matrix.target }}" \ --without-r - python3 tests/sandbox_installation.py "$UV_TOOL_BIN_DIR/mcp-console" + python3 tests/sandbox_installation.py --wheel dist/*.whl "$UV_TOOL_BIN_DIR/mcp-console" + + - name: Build and test the standalone archive + env: + TARGET: ${{ matrix.target }} + run: | + python3 scripts/release.py build-archive dist/*.whl --target "$TARGET" --out-dir archives + python3 tests/standalone.py "archives/mcp-console-$TARGET.tar.gz" --target "$TARGET" - name: Report Linux preparation failure if: failure() && runner.os == 'Linux' @@ -219,6 +237,13 @@ jobs: path: linux-compatibility if-no-files-found: warn + - name: Upload standalone archive + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: archive-${{ matrix.target }} + path: archives/* + if-no-files-found: error + - name: Upload wheel uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: @@ -227,11 +252,29 @@ jobs: if-no-files-found: error windows-wheel: - name: pypi-x86_64-pc-windows-msvc - runs-on: windows-latest + name: pypi-${{ matrix.target }} + runs-on: ${{ matrix.os }} + defaults: + run: + shell: pwsh + strategy: + fail-fast: false + matrix: + include: + - os: windows-latest + target: x86_64-pc-windows-msvc + wheel-tag: win_amd64 + python-arch: x64 + r-option: "" + - os: windows-11-arm + target: aarch64-pc-windows-msvc + wheel-tag: win_arm64 + python-arch: arm64 + r-option: --preinstalled-r timeout-minutes: 60 env: MCP_CONSOLE_SANDBOX_SOURCE: ${{ github.workspace }}/.sandbox-runner-source + MCP_CONSOLE_TEST_REQUIRE_MXC: ${{ matrix.os == 'windows-11-arm' && '1' || '0' }} UV_NO_CONFIG: "1" LANGUAGE: en steps: @@ -241,8 +284,13 @@ jobs: - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: + architecture: ${{ matrix.python-arch }} python-version: "3.14" + # Keep the guard inside each build job so a targeted rerun also checks PyPI. + - name: Refuse to rebuild a version already on PyPI + run: python scripts/release.py check-unpublished + - name: Set up uv uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: @@ -271,6 +319,11 @@ jobs: path: .sandbox-runner-source persist-credentials: false + - name: Select native MSVC build environment + uses: ./.sandbox-runner-source/.github/actions/setup-msvc-env + with: + target: ${{ matrix.target }} + # Use the PEP 517 backend: it relocates the native command and adds the # environment-bound Windows launcher that direct Maturin builds omit. - name: Build Windows wheel @@ -278,7 +331,9 @@ jobs: - name: Install wheel through its public tool command run: | - $wheel = (Get-Item dist/*-win_amd64.whl).FullName + $wheel = (Get-Item dist/*-${{ matrix.wheel-tag }}.whl).FullName + python tests/windows_distribution.py $wheel + if ($LASTEXITCODE) { exit $LASTEXITCODE } uv tool install --python (Get-Command python).Source $wheel if ($LASTEXITCODE) { exit $LASTEXITCODE } @@ -288,9 +343,11 @@ jobs: use-public-rspm: false - name: Configure binary R repositories + if: matrix.python-arch == 'x64' run: Rscript --vanilla scripts/configure-ci-r.R - name: Install native smoke-test dependencies + if: matrix.python-arch == 'x64' uses: r-lib/actions/setup-r-dependencies@f9a764fea8d5c63df6ef9a5c7795bf7deb5d7e05 # v2 with: working-directory: r @@ -301,23 +358,42 @@ jobs: any::reticulate any::jsonlite - - name: Prepare Python and select the R runtime + - name: Install Python bridge dependency + if: matrix.python-arch == 'x64' + run: uv pip install --python (Get-Command python).Source numpy + + - name: Install prebuilt R bridge for native Windows ARM64 + if: matrix.python-arch == 'arm64' + run: Rscript --vanilla scripts/install-ci-windows-arm-r-bridge.R + + - name: Select native R run: | - $ciPython = (Get-Command python).Source - uv pip install --python $ciPython numpy - if ($LASTEXITCODE) { exit $LASTEXITCODE } $rHome = & (Get-Command Rscript.exe).Source --vanilla -e 'cat(R.home())' if ($LASTEXITCODE) { exit $LASTEXITCODE } "R_HOME=$rHome" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + - name: Build and test the standalone Windows archive + run: | + $wheel = (Get-Item dist/*-${{ matrix.wheel-tag }}.whl).FullName + python scripts/release.py build-archive $wheel --target "${{ matrix.target }}" --out-dir archives + if ($LASTEXITCODE) { exit $LASTEXITCODE } + python tests/windows_install.py --archive (Get-Item archives/*.zip).FullName ${{ matrix.r-option }} + - name: Accept installed Windows wheel - run: python tests/windows_install.py --installed-binary "$env:UV_TOOL_BIN_DIR/mcp-console.exe" + run: python tests/windows_install.py --installed-binary "$env:UV_TOOL_BIN_DIR/mcp-console.exe" ${{ matrix.r-option }} + + - name: Upload standalone archive + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: archive-${{ matrix.target }} + path: archives/* + if-no-files-found: error - name: Upload wheel uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: pypi-x86_64-pc-windows-msvc - path: dist/*-win_amd64.whl + name: pypi-${{ matrix.target }} + path: dist/*-${{ matrix.wheel-tag }}.whl if-no-files-found: error linux-floor: @@ -352,11 +428,17 @@ jobs: name: pypi-${{ matrix.target }} path: dist - - name: Validate Linux wheel in clean floor runtimes + - name: Download standalone archive + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: archive-${{ matrix.target }} + path: archives + + - name: Validate Linux archive and wheel in clean floor runtimes shell: bash run: | mkdir -p linux-compatibility - python3 scripts/check-linux-wheel dist/*.whl \ + python3 scripts/check-linux-wheel dist/*.whl --archive archives/*.tar.gz \ --target "${{ matrix.target }}" \ --evidence linux-compatibility \ 2>&1 | tee linux-compatibility/runtime.log @@ -369,9 +451,33 @@ jobs: path: linux-compatibility if-no-files-found: warn + collect: + name: Verify six wheels and checksum six archives + needs: [wheels, windows-wheel] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: pypi-* + path: dist + merge-multiple: true + - run: python3 scripts/release.py verify-wheel-set dist + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: archive-* + path: archives + merge-multiple: true + - run: python3 scripts/release.py archive-checksums archives + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: github-release + path: archives + if-no-files-found: error + publish-pypi: name: publish PyPI - needs: [wheels, windows-wheel, linux-floor] + needs: [collect, linux-floor] # Manual dispatch is build-only. Only tag-triggered runs may publish. if: ${{ github.event_name == 'push' && github.ref_type == 'tag' }} @@ -422,6 +528,27 @@ jobs: --check-url https://pypi.org/simple \ dist/*.whl + publish-github: + name: Publish GitHub release assets + needs: publish-pypi + if: ${{ github.event_name == 'push' && github.ref_type == 'tag' }} + runs-on: ubuntu-latest + permissions: + contents: write + actions: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: github-release + path: archives + - name: Verify and attach the tested archives + env: + GH_TOKEN: ${{ github.token }} + run: python3 scripts/release.py publish-archives archives + verify-pypi: name: public-pypi-${{ matrix.target }} needs: publish-pypi @@ -457,7 +584,7 @@ jobs: if: runner.os == 'Linux' run: | sudo apt-get update - sudo apt-get install -y --no-install-recommends libcap2 libgcc-s1 libstdc++6 binutils + sudo apt-get install -y --no-install-recommends libgcc-s1 libstdc++6 - name: Download published artifacts for hash comparison uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 @@ -560,11 +687,29 @@ jobs: PY verify-pypi-windows: - name: public-pypi-x86_64-pc-windows-msvc + name: public-pypi-${{ matrix.target }} needs: publish-pypi - runs-on: windows-latest + runs-on: ${{ matrix.os }} + defaults: + run: + shell: pwsh + strategy: + fail-fast: false + matrix: + include: + - os: windows-latest + target: x86_64-pc-windows-msvc + wheel-tag: win_amd64 + python-arch: x64 + r-option: "" + - os: windows-11-arm + target: aarch64-pc-windows-msvc + wheel-tag: win_arm64 + python-arch: arm64 + r-option: --preinstalled-r timeout-minutes: 30 env: + MCP_CONSOLE_TEST_REQUIRE_MXC: ${{ matrix.os == 'windows-11-arm' && '1' || '0' }} UV_NO_CONFIG: "1" LANGUAGE: en steps: @@ -574,6 +719,7 @@ jobs: - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: + architecture: ${{ matrix.python-arch }} python-version: "3.14" - name: Set up uv @@ -593,9 +739,11 @@ jobs: use-public-rspm: false - name: Configure binary R repositories + if: matrix.python-arch == 'x64' run: Rscript --vanilla scripts/configure-ci-r.R - name: Install native smoke-test dependencies + if: matrix.python-arch == 'x64' uses: r-lib/actions/setup-r-dependencies@f9a764fea8d5c63df6ef9a5c7795bf7deb5d7e05 # v2 with: working-directory: r @@ -606,6 +754,10 @@ jobs: any::reticulate any::jsonlite + - name: Install prebuilt R bridge for native Windows ARM64 + if: matrix.python-arch == 'arm64' + run: Rscript --vanilla scripts/install-ci-windows-arm-r-bridge.R + - name: Download published artifacts for hash comparison uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: @@ -625,16 +777,18 @@ jobs: uv tool install --python (Get-Command python).Source --no-cache --no-sources --default-index https://pypi.org/simple "mcp-console==$version" if ($LASTEXITCODE) { exit $LASTEXITCODE } - - name: Prepare Python and select the R runtime + - name: Install Python bridge dependency + if: matrix.python-arch == 'x64' + run: uv pip install --python (Get-Command python).Source numpy + + - name: Select native R run: | - uv pip install --python (Get-Command python).Source numpy - if ($LASTEXITCODE) { exit $LASTEXITCODE } $rHome = & (Get-Command Rscript.exe).Source --vanilla -e 'cat(R.home())' if ($LASTEXITCODE) { exit $LASTEXITCODE } "R_HOME=$rHome" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - name: Accept pinned public-index installation - run: python tests/windows_install.py --installed-binary "$env:UV_TOOL_BIN_DIR/mcp-console.exe" + run: python tests/windows_install.py --installed-binary "$env:UV_TOOL_BIN_DIR/mcp-console.exe" ${{ matrix.r-option }} - name: Verify a fresh unqualified public-index installation run: | @@ -647,4 +801,4 @@ jobs: if ($actual -ne "mcp-console $version") { throw "Unexpected latest uvx version: $actual" } uv tool install --python (Get-Command python).Source --no-cache --no-sources --default-index https://pypi.org/simple mcp-console if ($LASTEXITCODE) { exit $LASTEXITCODE } - python tests/windows_install.py --installed-binary "$env:UV_TOOL_BIN_DIR/mcp-console.exe" + python tests/windows_install.py --installed-binary "$env:UV_TOOL_BIN_DIR/mcp-console.exe" ${{ matrix.r-option }} diff --git a/README.md b/README.md index be3eb6329..61c013102 100644 --- a/README.md +++ b/README.md @@ -10,11 +10,12 @@ Console prepares environments and packages automatically; no Console configurati It works with MCP-compatible clients such as Codex and Claude Code and does not depend on a particular model provider. **Development preview:** interfaces may change. -macOS, Linux, and Windows x64 are supported. +The development version supports macOS, Linux, and Windows on x86-64 and ARM64. ## Quickstart -Install [uv](https://docs.astral.sh/uv/getting-started/installation/), then register Console with your client: +For native Windows ARM64, use the [development installation](docs/CLIENTS.md#development-version) until the first `win_arm64` wheel is published. +On [platforms with a published wheel](docs/CLIENTS.md#supported-platforms), install [uv](https://docs.astral.sh/uv/getting-started/installation/) and run the command for your client. ```sh uvx mcp-console configure-client codex diff --git a/RELEASE.md b/RELEASE.md index 70377a1e0..6ea524acd 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,9 +1,16 @@ # Releasing MCP Console -The release workflow publishes five binary-only PyPI wheels: Apple Silicon and Intel macOS, ARM64 and x86-64 Linux, and Windows x64. -Linux release validation targets glibc 2.35+ with native Ubuntu 22.04 builds. +The release workflow builds six platform wheels for PyPI and six native archives for GitHub Releases. +For available downloads, see the [installation guide](docs/CLIENTS.md#supported-platforms). + +| OS | x86-64 target | ARM64 target | Wheel platform | +| ------------------- | -------------------------- | --------------------------- | ------------------------------------------------- | +| Linux (glibc 2.28+) | `x86_64-unknown-linux-gnu` | `aarch64-unknown-linux-gnu` | `manylinux_2_28_x86_64`, `manylinux_2_28_aarch64` | +| macOS | `x86_64-apple-darwin` | `aarch64-apple-darwin` | `macosx_*_x86_64`, `macosx_*_arm64` | +| Windows | `x86_64-pc-windows-msvc` | `aarch64-pc-windows-msvc` | `win_amd64`, `win_arm64` | + +Linux builds use the native manylinux_2_28 images for the public executable, sandbox runner, and bubblewrap. Windows MSVC builds statically link the runtime. -No source distribution or GitHub release archive is published by this workflow. `Cargo.toml` owns the version; keep its root package entry in `Cargo.lock` synchronized. Between releases, use the most recently released version plus `+dev`. @@ -11,6 +18,50 @@ Development executables built from Git also report the first 12 characters of `H The current workflows and [`scripts/release.py`](scripts/release.py) are authoritative for mechanics. This page owns the release procedure and required evidence. +## Consuming a standalone release + +Applications such as Posit Assistant can pin a tag, download `mcp-console-.tar.gz` (Unix) or `mcp-console-.zip` (Windows) and verify its SHA-256 against `SHA256SUMS` from the same release. +Keep the bundle and any `--config-file` policy in an application-owned directory outside every path the sandboxed program can modify, including any writable workspace. +This prevents the program from replacing the launcher or policy used on the next run. +Extract the whole archive, preserving Unix executable permissions. +For Linux x86-64, download the archive and checksum file into the application-owned directory and run these commands there. + +```sh +archive=mcp-console-x86_64-unknown-linux-gnu.tar.gz +awk -v archive="$archive" '$2 == archive' SHA256SUMS | sha256sum --check --strict && + mkdir console && + tar -xzf "$archive" -C console && + console/bin/mcp-console --version && + console/bin/mcp-console sandbox --no-config -- /usr/bin/true +``` + +An application can invoke `bin/mcp-console sandbox --no-config -- PROGRAM ARG...` to use the built-in sandbox policy, or pass `--config-file` with the absolute path to its own policy in place of `--no-config`. +On Windows, use `bin\mcp-console.exe` with the same arguments. +Keep `libexec/` and `share/` alongside `bin/` so Console can find and verify its companion. +The [sandbox launcher](docs/SANDBOX_COMMAND.md) requires no MCP server, `ir`, Python, pip, or uv. +The application supplies the target program and its dependencies. + +### Embedding Console in a notarized macOS app + +Build Console from source for a notarized macOS app. +The prebuilt ARM64 launcher and sandbox helper have ad hoc signatures, and the x86-64 executables are unsigned. +Apple requires a Developer ID signature on each executable, including helpers inside a signed app. +See [Apple’s notarization requirements](https://developer.apple.com/documentation/security/notarizing-macos-software-before-distribution). + +For both the sandbox helper and the launcher, use `codesign --force --options runtime --timestamp --sign "$IDENTITY" "$executable"` with your Developer ID Application identity. +In `scripts/stage-sandbox-runner`, sign the staged `libexec/mcp-console-sandbox` after stripping it and before recording its SHA-256 in `target/sandbox-runner-build.json`. +Then build `mcp-console` to embed that digest. +Preserve the signed helper byte for byte because Console checks its digest at launch. + +The `sandbox` subcommand launches the requested program, which loads its own libraries. +With `serve`, `bin/mcp-console` also runs the worker that loads libR, libpython, and native R and Python extensions. +By default, library validation permits the worker to load code signed by Apple or by the same team as the worker. +If the worker must load third-party code outside that signing team, set `com.apple.security.cs.disable-library-validation` to `true` in the worker's entitlements plist. +See [Apple’s library validation guidance](https://developer.apple.com/documentation/bundleresources/entitlements/com.apple.security.cs.disable-library-validation). + +Sign `bin/mcp-console` with the same runtime and timestamp options, adding `--entitlements PATH_TO_PLIST` if the worker needs that exception. +Then sign the enclosing app and submit it for notarization. + ## Private sandbox executable Source builds need Python 3.11+, Git, rustup, and native build tools. @@ -43,7 +94,10 @@ Changes to SDKs, compilers, linkers, or system libraries may require cleaning bo Unix installations contain the public executable, the private runner under `libexec`, and licenses under `share/licenses/mcp-console`. Linux additionally bundles bubblewrap and its notices/source metadata. -Windows contains its native Console executable and the single `mcp-console-sandbox.exe` companion under `libexec`, with an environment-bound Python launcher exposing the public command. +The Linux archive keeps bundled libraries at the same paths relative to the installation prefix as the wheel, so the native executables can load them after extraction. +Windows wheels keep the native Console and companion under `libexec` and expose an environment-bound Python launcher. +The standalone archive copies the same native Console executable to `bin/mcp-console.exe`. +Every archive carries Console’s MIT license alongside the companion licenses. Move the complete native bundle when relocating it; a symlink to the native executable is supported. Do not split Cargo's shared target/build layout. @@ -51,7 +105,9 @@ Native launch verifies the selected companion and does not download or extract o Staging/build checks must retain artifact digests, component identity, pin consistency, and required license files. Linux staging rejects inherited `CODEX_BWRAP_SOURCE_DIR` and `CODEX_SKIP_BWRAP_BUILD`, including empty values. -Dynamically linked libcap is a host dependency; static linkage requires a nonempty applicable `MCP_CONSOLE_LIBCAP_NOTICE`. +Release builds link the EL8 static libcap archive into bubblewrap and include its license in the bundle. +Source builds that link libcap dynamically require it on the host. +Static builds supply its license through `MCP_CONSOLE_LIBCAP_NOTICE`. Supplying text is not legal certification of its contents. When advancing the pin, review the companion's protocol, implementation, executable tests, and toolchain together. @@ -66,16 +122,29 @@ python3 scripts/release.py inspect-wheel WHEEL --target TARGET --report abi.json Audit every ELF member, including private helpers and extra native libraries, against all advertised wheel tags. Preserve architecture, loader, dependency/path, symbol-version, and metadata checks; unknown policies fail rather than inheriting the build host's assumptions. -`--release` additionally enforces the release floor. +`--release` also checks the glibc 2.28 floor and requires bubblewrap to link libcap statically. Exact loader algorithms and symbol allowlists belong in the release tooling, not a second prose specification. On macOS/Linux use `smoke-wheel WHEEL --installed-only --sandbox-pin PIN_FILE` in a clean runtime without the source checkout/Cargo outputs. +Run `python3 tests/standalone.py ARCHIVE --target TARGET` on the matching Unix runner to test the relocated archive. +For macOS, this checks that both the launcher and its helper contain the requested Mach-O architecture before launching either executable. Windows builds use the PEP 517 backend through `uv build --wheel` so the installed command retains its native bundle. Run `python tests/windows_distribution.py WHEEL` to audit normal and delayed PE imports in every bundled native executable or library. Redistributable VC runtime dependencies are rejected; ordinary Windows system and UCRT libraries are allowed. -Install that wheel and run `python tests/windows_install.py --installed-binary PUBLIC_COMMAND` on a clean host. +On Windows x86-64, install the wheel and run `python tests/windows_install.py --installed-binary PUBLIC_COMMAND` on a clean host. Installation acceptance repeats the import audit for installed wheels and source builds, including the environment-bound public launcher. -Acceptance probes MXC and runs native execution cases when supported, without provisioning accounts or selecting another backend. +On Windows 11 ARM64, install the wheel and the `reticulate` and `RSQLite` packages for native R. +Run both acceptance checks in PowerShell. + +```powershell +$env:MCP_CONSOLE_TEST_REQUIRE_MXC = '1' +python tests/windows_install.py --preinstalled-r --installed-binary (Get-Command mcp-console.exe).Source +python tests/windows_install.py --preinstalled-r --archive archives/mcp-console-aarch64-pc-windows-msvc.zip +``` + +This matches the release job's MXC requirement and exercises the preinstalled R packages without `ir`. +The hosted x86-64 Windows Server runner probes for MXC and skips native execution cases when it is unavailable. +Run those cases on an MXC-capable x86-64 Windows 11 host before release. This checks version/help, empty-PATH bundled sandbox execution, filesystem/exit-status enforcement, Python-only MCP, and the R/Python bridge without rebuilding from source. Retain empty-PATH bundled-helper acceptance; a working system bwrap does not validate the relocated bundle. An ABI report alone is not runtime or sandbox acceptance. @@ -85,16 +154,21 @@ An ABI report alone is not runtime or sandbox acceptance. Run on a matching native Docker host with the evidence directory shared with the daemon: ```sh -scripts/check-linux-wheel dist/*.whl \ +scripts/check-linux-wheel dist/*.whl --archive archives/*.tar.gz \ --target x86_64-unknown-linux-gnu --evidence linux-compatibility ``` Use `aarch64-unknown-linux-gnu` for ARM64. +Before starting Docker, the driver compares the installed paths, Unix permissions, and SHA-256 digests of the archive's executables, libraries, and licenses with the wheel. +It also checks that each tar member has UID and GID 0 and empty owner and group names. Emulation is rejected; `--docker-context` can select an explicit daemon. -The floor fixtures exercise installed wheels without R and with R, with runtime prerequisites separated from build tools. +The Python-only AlmaLinux 8 runtime audits the wheel's ABI and companion pin before either artifact runs. +A separate AlmaLinux 8 minimal image launches the relocated archive with its bundled sandbox helpers and no Python or uv. +The wheel runs in compiler-free images with Python alone and with R. The R runtime includes R's recommended packages, which the isolated resolver expects the R installation to provide. -They refresh R preparation before producing compiler-free runtime images, rather than treating old Docker layers as fresh package resolution. +The R preparation stage resolves the default packages on each floor run using the dated Posit Package Manager snapshot in `scripts/linux-wheel/Dockerfile`. +That snapshot supplies a version of `textshaping` compatible with EL8's HarfBuzz on both architectures. Language-package build flags do not change distributed wheel binaries. Native probes require an approved disposable namespace-capable container. @@ -117,7 +191,10 @@ Do not cache accounts, credentials, ACLs, or provisioning state. MXC capability probing and full Windows acceptance still run on a clean host after restoring build/staging caches; unavailable native execution cases are skipped as documented in [Windows validation](docs/WINDOWS_DEVELOPMENT.md#validation). Exact cache-key and timestamp-restoration mechanics belong in the workflow and staging scripts. -CI uses CRAN's native Windows/macOS binaries and PPM's portable Linux binaries on both architectures, including the Jammy floor images. +CI uses CRAN binaries for macOS and Windows x64 and PPM's portable Linux binaries for both Linux architectures. +Windows ARM64 tests both startup orders with native R and Python, preinstalled `reticulate`, and `r.resolution` set to `disabled`. +The R-first test uses `RSQLite` to open the connection required by its configured startup block. +The managed R resolver, `r-lib-ir`, currently lacks a Windows ARM64 wheel. Configure both R's repository data file and site profile so profile-free preparation and ordinary R sessions select the same repository. Packages without compatible binaries still build from source. @@ -143,7 +220,8 @@ gh workflow run release.yml --ref release/X.Y.Z ``` Manual dispatch must skip publication. -Require all five wheel builds/installed smoke tests and both Linux floor-runtime jobs to pass. +Require the six native archive and installed-wheel jobs, both Linux floor-runtime jobs, and the aggregate artifact check to pass. +The `github-release` workflow artifact contains the archives and their checksum manifest for inspection. Exercise mixed-language and R-free installed wheels; `smoke-wheel --without-r` isolates R discovery. Establish readiness before ordinary cell assertions rather than treating asynchronous tool discovery as runtime readiness. @@ -169,12 +247,14 @@ git push origin "refs/tags/v$release_version" ``` Use `-F MESSAGE_FILE` for a noninteractive annotation. -The tag workflow must verify version, main ancestry, exact-commit push CI, and the five install-tested artifacts before Trusted Publishing. +The tag workflow must verify version, main ancestry, exact-commit push CI, and the install-tested artifacts before Trusted Publishing. +After the PyPI upload, a separate job verifies that `SHA256SUMS` lists all six archives exactly once with their measured digests. +It then uploads the archives and manifest to a draft GitHub release, compares the remote asset digests, and publishes the release. ## Verify the publication -Check PyPI's version JSON: all five wheels must be present, unyanked, and match the tag workflow's artifact SHA-256 digests. -The tag workflow checks these hashes and fresh pinned/unqualified public-index installations on all five release platforms, including installed sandbox execution and Python-only MCP evaluation. +Check PyPI's version JSON: all six wheels must be present, unyanked, and match the tag workflow's artifact SHA-256 digests. +The tag workflow checks these hashes and fresh pinned/unqualified public-index installations on all six release platforms, including installed sandbox execution and Python-only MCP evaluation. It waits up to two minutes for both the version JSON and package index to expose all expected files, including retries for temporary request failures; a visible hash mismatch, yanked file, or HTTP permission error fails immediately. Verify fresh public-index installation on every supported release platform: @@ -194,13 +274,14 @@ uv tool install --no-cache --no-sources --default-index https://pypi.org/simple Through an MCP client, establish readiness for that exact version and verify Python `6 * 7` returns `42`; check R when installed. A server waiting for protocol input is normal. On Windows use PowerShell environment variables and `mcp-console.exe`; replace `/usr/bin/true` with the absolute `System32/cmd.exe` command and `/d /c "exit 0"`. -Provision native sandbox setup before testing it; the post-publication Windows job also verifies the R/Python bridge. +Use an MXC-capable Windows host to verify native sandbox execution and the R/Python bridge. Repeat unqualified `uvx mcp-console --version` and `uv tool install mcp-console` with fresh directories and the same isolation/public-index options; confirm they select the new version. Leave older releases unchanged. ## Return to development +After verifying the first public six-platform release, update the [installation guide](docs/CLIENTS.md#supported-platforms) and [Linux compatibility guide](docs/LINUX_COMPATIBILITY.md) to point to its wheels and archives. After publication checks pass, set the version in `Cargo.toml` and its root package entry in `Cargo.lock` to `X.Y.Z+dev`. Run `scripts/format` and `scripts/check`, then commit the two version files as `Mark post-release builds as development versions` through the usual review process. Keep the release tag on the verified release commit. @@ -213,5 +294,10 @@ Moving a pushed tag requires explicit user authorization, confirmation that its After **any** wheel is published, preserve the tag and all uploaded files. **Never yank or remove a PyPI file.** For partial publication, rerun the failed job from the same workflow run while its original artifacts remain. +The pre-build version check rejects versions already visible on PyPI. +If the GitHub upload stopped midway, rerun the failed job with the original `github-release` artifact. +It checks existing assets by digest and uploads the missing files to the draft. +If GitHub retained an empty asset in the `starter` state after an interrupted upload, remove that placeholder from the draft before retrying. +A digest mismatch on an uploaded asset requires investigation. Do not rebuild that version or replace an uploaded filename. Defective published code requires a new version. diff --git a/docs/CLIENTS.md b/docs/CLIENTS.md index 9d3defabb..39a1b6cc9 100644 --- a/docs/CLIENTS.md +++ b/docs/CLIENTS.md @@ -3,12 +3,17 @@ Install Console's prerequisites, then connect your coding agent or another MCP client. [Get started](getting-started.qmd) walks through a first analysis once you are connected. -Install [uv](https://docs.astral.sh/uv/getting-started/installation/) so `uvx` is available to your client. +For the `uvx` launcher, install [uv](https://docs.astral.sh/uv/getting-started/installation/). It can run Console without a separate Console installation. ## Supported platforms -Prebuilt packages are available for macOS and Linux on ARM64 and x86-64, and for Windows x64. +Console 0.0.7 provides wheels for macOS and Linux on x86-64 and ARM64, and for Windows on x86-64. +Its Linux wheels are tagged `manylinux_2_34`. +See the [published files on PyPI](https://pypi.org/project/mcp-console/0.0.7/#files). +The next release adds a Windows ARM64 wheel and lowers the Linux glibc requirement to 2.28 on both architectures. +To use these targets before the wheels are published, [build from a source revision that includes this support](BUILD_FROM_SOURCE.md#install-console). + Sandboxed Windows sessions require Windows 11 with [Microsoft Execution Containers (MXC)](https://support.microsoft.com/en-us/servicing/os/windows-11/2026/08/kb5120998-windows-11-24h2-25h2-update#microsoft-execution-containers). See [Windows sandbox requirements](SANDBOX_CONFIGURATION.md#windows-mxc-requirement) for host setup and troubleshooting, or [Windows development](WINDOWS_DEVELOPMENT.md) for native builds and validation. @@ -20,6 +25,21 @@ To let your agent use R, install it separately, for example with [rig](https://g Python and SQL work without R. The first session may download Python and packages. +On Windows ARM64, if your development build uses native ARM64 Python and discovers R, the default R package setup needs a separate `ir` executable on `PATH`. +PyPI has no Windows ARM64 wheel for `r-lib-ir`. +To use R without `ir`, preinstall the R packages you need, including `reticulate` for sessions that use both R and Python. +Before connecting your client, disable resolution in your project's `.agents/console/config.yaml` and clear any package list inherited from the global configuration. + +```yaml +r: + resolution: disabled + packages: [] +``` + +On Windows ARM64 with reticulate 1.47.0, R-first discovery cannot distinguish native ARM64 Python from x64 Python running under emulation. +If you have both, set `python` in the same configuration file to the absolute path of the native interpreter, for example `python: 'C:/Python314-arm64/python.exe'`. +If an R profile or default package initializes Python, [defer that call until after native R startup](RUNTIME_CONFIGURATION.md#native-r-startup). + To verify the selected runtimes and prepare caches before opening a client, follow [Check your installation](CLI.md#check-your-installation). @@ -80,6 +100,38 @@ The `--no-uvx` option registers the installed executable by its absolute path; t To update, rerun the `uv tool install` command and restart your client. [Build from source](BUILD_FROM_SOURCE.md) covers shell setup, other clients, and installing a specific branch, tag, or commit. +## Standalone executable + +The first standalone archives for all six targets are planned for the next release. +Until then, use the [development installation](#development-version). +Once published, download the archive for the host's OS and architecture and verify it against `SHA256SUMS` from the same [GitHub release](https://github.com/posit-dev/mcp-console/releases). +Extract the whole archive, preserving permissions. +Keep `bin`, `libexec`, and `share` together when moving the installation. + +The checksum verifies the archive's contents. +The macOS binaries ship without Apple notarization. +If files extracted from a browser download retain their quarantine flag, Gatekeeper can block the launcher or its sandbox helper even when you run Console from Terminal. +For command-line use on macOS, [install the PyPI wheel with `uv tool install mcp-console`](SANDBOX_COMMAND.md#install). +For a notarized macOS app, [rebuild Console with a Developer ID–signed sandbox helper](../RELEASE.md#embedding-console-in-a-notarized-macos-app). + +To run a program with Console's built-in `read_only` policy, invoke `bin/mcp-console sandbox --no-config -- PROGRAM ARG...`. +On Windows, use `bin\mcp-console.exe` with the same arguments. +For an application-owned policy, replace `--no-config` with `--config-file` followed by the absolute path to the application's YAML file. +Keep both the bundle and that policy file outside every directory the sandboxed program can modify, including any writable workspace. +Omit both options when the application intends to honor global and project settings, which can widen filesystem and network permissions. +See [Run commands in a sandbox](SANDBOX_COMMAND.md) and [Configuration is part of the trust decision](SANDBOX.md#configuration-is-part-of-the-trust-decision). + +The launcher requires no MCP server, `ir`, Python, pip, or uv. +The target program uses its own runtime and dependencies. + +To embed the MCP server, set the client command to the executable's absolute path and the arguments to `["serve", "--no-config"]` for built-in settings. +For an application-owned policy, use `["serve", "--config-file", "/absolute/path/to/policy.yaml"]`. +Keep the bundle and configuration outside directories writable by evaluated code or dependency preparation. +Use bare `serve` only when the application trusts the global and project settings for both evaluation and dependency preparation. +If the application supplies R and its packages, set `r.resolution` to `disabled` and `r.packages` to `[]` in its configuration file. +Console then uses those libraries without invoking the R package resolver (`ir`). +Python can also use an [existing environment](RUNTIME_CONFIGURATION.md#use-an-existing-environment). + ## R and Python clients {#connect-from-application-code} diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md index f94d26d06..17b68aba2 100644 --- a/docs/ENVIRONMENT.md +++ b/docs/ENVIRONMENT.md @@ -181,7 +181,7 @@ Managed R preparation also supplies `IR_NO_LOCAL_SOURCES=1` and a package-subpro | `XDG_CONFIG_HOME` | Participates in Linux Matplotlib config lookup, not Console YAML discovery. | | `LD_LIBRARY_PATH` | Adjusted for selected R/Python libraries on Linux. | | `LD_PRELOAD`, `DYLD_INSERT_LIBRARIES` | Removed when launching the native sandbox runner; this is not a blanket guarantee for direct execution. | -| `PROCESSOR_ARCHITECTURE` | On Windows x64, absent/empty values default to `AMD64` before runtime loading; supplied nonempty values remain. | +| `PROCESSOR_ARCHITECTURE` | On Windows, Console sets an absent or empty value to its native architecture (`AMD64` or `ARM64`) before loading runtimes. | Private plotting/cache initialization still occurs with `cache: host` when `sandbox.tempdir: managed` is selected. Other tempdir modes and `--no-sandbox` retain normal library cache selection after the configuration/cache assignments above. diff --git a/docs/LINUX_COMPATIBILITY.md b/docs/LINUX_COMPATIBILITY.md index 35004cb91..de41d91ce 100644 --- a/docs/LINUX_COMPATIBILITY.md +++ b/docs/LINUX_COMPATIBILITY.md @@ -9,25 +9,30 @@ For installation and client setup, see [Get started](getting-started.qmd). ## Runtime libraries -Console supports Linux hosts with glibc 2.35 or newer. -The following runtime libraries may already be installed on your system; the installation commands below are only needed when they are missing. -They are system dependencies, separate from the Python packages installed by `uv`. +Console 0.0.7 has `manylinux_2_34` wheels for x86-64 and ARM64. +See the [published files on PyPI](https://pypi.org/project/mcp-console/0.0.7/#files). +Their bundled bubblewrap requires the system libcap library. +The development release workflow builds wheels and standalone archives for glibc 2.28 or newer, including RHEL 8, and links libcap statically into bubblewrap. +Console and its companion use the system GNU runtime libraries. +Native language packages can have additional dependencies. + +| RHEL package | Ubuntu/Debian package | Shared library | Use | +| ------------ | --------------------- | ---------------- | ------------------------------------------------------------------- | +| `libgcc` | `libgcc-s1` | `libgcc_s.so.1` | GNU compiler runtime | +| `libstdc++` | `libstdc++6` | `libstdc++.so.6` | C++ runtime for native code | +| `libcap` | `libcap2` | `libcap.so.2` | Bubblewrap in the 0.0.7 wheels and dynamically linked source builds | -| Ubuntu/Debian package | Shared library | Needed for | -| --------------------- | ---------------- | ---------------------------------------------------------------- | -| `libcap2` | `libcap.so.2` | Linux process capabilities in the bundled Bubblewrap sandbox. | -| `libgcc-s1` | `libgcc_s.so.1` | GNU runtime support for Console and its sandbox executables. | -| `libstdc++6` | `libstdc++.so.6` | C++ runtime support for native analysis packages such as DuckDB. | +A missing library can appear as `error while loading shared libraries` before Console starts, or as an import error when a native Python extension loads. +Use the missing `.so` filename in the error to identify the package in the table above. -On Ubuntu or Debian, install missing runtime libraries with: +## R package preparation on EL8 -```sh -sudo apt-get update -sudo apt-get install -y libcap2 libgcc-s1 libstdc++6 -``` +With R 4.5 on RHEL 8 or a compatible distribution, preparing the default `tidyverse` package can fail because its dependency `textshaping` 1.1.0 needs a newer HarfBuzz than EL8 supplies. +For managed R sessions, set the `CRAN` URL in the selected R installation's `etc/repositories` file to `https://packagemanager.posit.co/cran/2026-11-18/bin/linux/manylinux_2_28-ARCH/4.5`, replacing `ARCH` with `x86_64` or `aarch64`. +This snapshot supplies a compatible `textshaping` 1.0.5 binary on both architectures. -A missing library can appear as `error while loading shared libraries` before Console starts, or as an import error when a native Python extension loads. -Use the missing `.so` filename in the error to identify the package in the table above. +If you already have the R packages you need, including `reticulate` for mixed R/Python sessions, set `r: {resolution: disabled, packages: []}` in your project's `.agents/console/config.yaml` to use those libraries. +The standalone `mcp-console sandbox` command launches its target without R or R package preparation. ## Host and container setup diff --git a/docs/RUNTIME_CONFIGURATION.md b/docs/RUNTIME_CONFIGURATION.md index 56054631f..c5ce76a31 100644 --- a/docs/RUNTIME_CONFIGURATION.md +++ b/docs/RUNTIME_CONFIGURATION.md @@ -156,6 +156,11 @@ A profile diagnostic that R survives does not make the session unusable. If R fails initialization or is interrupted, fix the startup file externally and explicitly restart; ordinary calls do not retry initialization. Edits to `.Rprofile` take effect on runtime restart because R reads the file again, even though YAML configuration remains captured for the server connection. +On Windows ARM64, reticulate 1.47.0 reports R's architecture as `Unknown` during native R startup and rejects even an explicitly selected ARM64 Python interpreter. +Move calls that initialize Python from `.Rprofile` or `.First()` into your first R cell. +If a default package initializes Python when loaded, remove it from R's default packages and load it in that cell. +A [Console R startup block](#session-startup-source) that selects a SQL connection can also run these calls. + ## Select a SQL connection at startup {#session-startup-source} diff --git a/docs/SANDBOX_COMMAND.md b/docs/SANDBOX_COMMAND.md index a08b5e846..fdfc0bc89 100644 --- a/docs/SANDBOX_COMMAND.md +++ b/docs/SANDBOX_COMMAND.md @@ -7,13 +7,13 @@ Read [Sandbox and trust](SANDBOX.md) for the permission model, or [Sandbox permi ## Install -Install Console with [uv](https://docs.astral.sh/uv/getting-started/installation/): +On [platforms with a published wheel](CLIENTS.md#supported-platforms), install Console with [uv](https://docs.astral.sh/uv/getting-started/installation/). ```sh uv tool install mcp-console ``` -The sandbox supports macOS, Linux, and Windows x64. +For native Windows ARM64, use the [development installation](CLIENTS.md#development-version) until the first `win_arm64` wheel is published. See [platform requirements](SANDBOX_CONFIGURATION.md#native-capabilities) for host setup, including MXC on Windows. ## Run a program diff --git a/docs/SANDBOX_CONFIGURATION.md b/docs/SANDBOX_CONFIGURATION.md index 769e6e811..648ddded8 100644 --- a/docs/SANDBOX_CONFIGURATION.md +++ b/docs/SANDBOX_CONFIGURATION.md @@ -112,6 +112,9 @@ Platform behavior matters. macOS and Linux can grant individual files. macOS can grant creation at absent paths; Linux skips absent write roots until a later launch. Linux cannot reliably reopen a narrower read beneath a broader denied mount. +On Windows, a read-only exception inside a writable directory remains inaccessible for the duration of the launch if the path was absent at setup. +This requires MXC support for denying access to a path. +The next launch checks whether the path now exists and can be granted read-only access. Create intended writable directories first. Console does not grant extra paths or switch backends to make an unsupported policy work. @@ -373,11 +376,11 @@ See [Cache selection](PACKAGE_CONFIGURATION.md#cache-locations) for clearing inh ## Platform support {#native-capabilities} -| Platform | Important limits | -| -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| macOS | Seatbelt enforcement and managed proxy supported. Console rejects `tcp_udp`. | -| Linux | Bubblewrap enforcement; managed bridge is TCP-only. Nonempty Unix-socket path lists and `tcp_udp` are rejected. Socket allow-all is supported, but is not path-specific. | -| Windows | Windows x64 requires MXC for workers and preparation. TCP proxies resolve local binding to true under `auto`; explicit false is rejected. UDP and Unix sockets are rejected. | +| Platform | Important limits | +| -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| macOS | Seatbelt enforcement and managed proxy supported. Console rejects `tcp_udp`. | +| Linux | Bubblewrap enforcement; managed bridge is TCP-only. Nonempty Unix-socket path lists and `tcp_udp` are rejected. Socket allow-all is supported, but is not path-specific. | +| Windows | Windows requires MXC for workers and preparation. TCP proxies resolve local binding to true under `auto`; explicit false is rejected. UDP and Unix sockets are rejected. | A common schema does not imply equal native capabilities. Console rejects unsupported explicit requests rather than silently ignoring them. diff --git a/docs/WINDOWS_DEVELOPMENT.md b/docs/WINDOWS_DEVELOPMENT.md index 4563a98cb..4245a82eb 100644 --- a/docs/WINDOWS_DEVELOPMENT.md +++ b/docs/WINDOWS_DEVELOPMENT.md @@ -20,6 +20,7 @@ Console and its `cc`-built native dependencies use the static MSVC runtime selec Use Python 3.14 to match CI. Full coverage needs R/Python, PATH uv/ir, repository access, prepared packages, and an MXC-capable host. +The R-first bridge test runs with package resolution disabled and needs `reticulate` and `RSQLite` in the selected R installation's library paths. Acceptance probes MXC through a real public sandbox launch. Only the native unavailable-runtime diagnostic permits skipping execution cases; installation, policy, and workload failures remain failures. Hosts without MXC still test configuration, bundle integrity, explicit host execution, and fail-closed startup. diff --git a/docs/index.qmd b/docs/index.qmd index ee09bca7c..ee81be18f 100644 --- a/docs/index.qmd +++ b/docs/index.qmd @@ -21,7 +21,7 @@ You can also use Console from [R](R.md) or [Python](PYTHON.md). [Get started](getting-started.qmd){.btn .btn-primary} [Build on Console](BUILD.md){.btn .btn-outline-primary} [Reference](REFERENCE.md){.btn .btn-outline-primary} ::: -Console provides a unified API across macOS, Linux, and Windows x64. +Console provides a unified API across macOS, Linux, and Windows on x86-64 and ARM64. It is currently a **development preview**. ## Work iteratively diff --git a/sandbox-runner.json b/sandbox-runner.json index 4198e9fce..735dd9b33 100644 --- a/sandbox-runner.json +++ b/sandbox-runner.json @@ -1,5 +1,5 @@ { "repository": "t-kalinowski/cobox", "release": "rust-v0.163.0", - "commit": "74218bd69ceb35e855027a7e3123656200bb3c89" + "commit": "699fc9f8e987039a09979e01af4ccfa01984c25e" } diff --git a/scripts/check-linux-wheel b/scripts/check-linux-wheel index a71833bdb..7582cd6db 100755 --- a/scripts/check-linux-wheel +++ b/scripts/check-linux-wheel @@ -1,21 +1,59 @@ #!/usr/bin/env python3 -"""Test a built Linux wheel in clean native Ubuntu 22.04 runtime images.""" +"""Test release archives and wheels in clean native AlmaLinux 8 runtime images.""" from __future__ import annotations import argparse +import hashlib import json from pathlib import Path import shutil import subprocess +import tarfile import tempfile +import zipfile + +from release import standalone_members, wheel_identity, wheel_permissions ROOT = Path(__file__).resolve().parent.parent +def verify_archive(wheel: Path, archive: Path, target: str) -> None: + version, _ = wheel_identity(wheel) + with zipfile.ZipFile(wheel) as dist, tarfile.open(archive, "r:gz") as bundle: + expected = standalone_members(dist, version, target) + for member in bundle: + # The builder leaves directories implicit. An explicit directory + # could restrict access to files whose own permissions match. + source = expected.pop(member.name, None) + if source is None: + raise SystemExit(f"unexpected archive member: {member.name}") + if not member.isfile(): + raise SystemExit(f"archive {member.name} must be a regular file") + # A privileged extractor can restore a named owner in preference + # to its numeric ID. Match the builder's ownership in both forms. + if (member.uid, member.gid, member.uname, member.gname) != (0, 0, "", ""): + raise SystemExit( + f"archive {member.name} must use UID and GID 0 with empty owner names" + ) + entry = dist.getinfo(source) + mode = wheel_permissions(entry) + if member.mode != mode: + raise SystemExit( + f"archive {member.name} has mode {member.mode:04o}, expected {mode:04o}" + ) + with dist.open(entry) as original, bundle.extractfile(member) as archived: + wheel_hash = hashlib.file_digest(original, "sha256").digest() + if hashlib.file_digest(archived, "sha256").digest() != wheel_hash: + raise SystemExit(f"archive {member.name} differs from the wheel") + if expected: + raise SystemExit(f"missing archive member: {', '.join(sorted(expected))}") + + def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("wheel", type=Path) + parser.add_argument("--archive", type=Path, required=True) parser.add_argument( "--target", required=True, @@ -25,23 +63,14 @@ def main() -> None: parser.add_argument("--docker-context") args = parser.parse_args() wheel = args.wheel.resolve() - assert wheel.is_file() + archive = args.archive.resolve() + assert wheel.is_file() and archive.is_file() evidence = args.evidence.resolve() - evidence.mkdir(parents=True, exist_ok=True) + evidence.parent.mkdir(parents=True, exist_ok=True) docker = [ "docker", *(["--context", args.docker_context] if args.docker_context else []), ] - architecture = args.target.split("-", 1)[0] - daemon_architecture = subprocess.check_output( - [*docker, "info", "--format", "{{.Architecture}}"], text=True - ).strip() - assert daemon_architecture == architecture, ( - "floor validation requires a matching native Docker host" - ) - for image in ("ubuntu:22.04", "ghcr.io/astral-sh/uv:0.13.1"): - subprocess.run([*docker, "pull", image], check=True) - # Keep bind inputs beside evidence, on the filesystem shared with the daemon. with tempfile.TemporaryDirectory( prefix="mcp-console-linux-floor-", dir=evidence.parent @@ -50,6 +79,8 @@ def main() -> None: # Only the artifact and minimal public smoke/installation harness enter. for source, destination in ( (wheel, inputs / wheel.name), + (archive, inputs / archive.name), + (ROOT / "scripts/linux-wheel/standalone.sh", inputs / "standalone.sh"), (ROOT / "scripts/linux-wheel/Dockerfile", inputs / "Dockerfile"), (ROOT / "scripts/configure-ci-r.R", inputs / "configure-ci-r.R"), (ROOT / "scripts/release.py", inputs / "release.py"), @@ -63,9 +94,22 @@ def main() -> None: ): destination.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(source, destination) - for mode in ("no-r", "with-r"): + + verify_archive(inputs / wheel.name, inputs / archive.name, args.target) + architecture = args.target.split("-", 1)[0] + daemon_architecture = subprocess.check_output( + [*docker, "info", "--format", "{{.Architecture}}"], text=True + ).strip() + assert daemon_architecture == architecture, ( + "floor validation requires a matching native Docker host" + ) + for image in ("almalinux:8-minimal", "ghcr.io/astral-sh/uv:0.13.1"): + subprocess.run([*docker, "pull", image], check=True) + # The no-R image runs inspect-wheel --release against the checkout's pin + # before launching anything from the wheel or its matching archive. + for mode in ("no-r", "standalone", "with-r"): output = evidence / mode - output.mkdir(exist_ok=True) + output.mkdir(parents=True, exist_ok=True) image = f"mcp-console-linux-floor:{inputs.name}-{mode}" subprocess.run( [ @@ -89,7 +133,7 @@ def main() -> None: *docker, "image", "inspect", - "ubuntu:22.04", + "almalinux:8-minimal", "ghcr.io/astral-sh/uv:0.13.1", image, ], @@ -111,13 +155,18 @@ def main() -> None: "--mount", f"type=bind,src={output},dst=/evidence", image, - "python3", - "/test/linux_wheel_runtime.py", - f"/test/{wheel.name}", - "--target", - args.target, - *(["--with-r"] if mode == "with-r" else []), ] + if mode == "standalone": + command += ["sh", "/test/standalone.sh", f"/test/{archive.name}"] + else: + command += [ + "python3", + "/test/linux_wheel_runtime.py", + f"/test/{wheel.name}", + "--target", + args.target, + *(["--with-r"] if mode == "with-r" else []), + ] (output / "command.json").write_text(json.dumps(command), encoding="utf-8") subprocess.run(command, check=True) diff --git a/scripts/configure-ci-r.R b/scripts/configure-ci-r.R index 994204937..c7454a449 100644 --- a/scripts/configure-ci-r.R +++ b/scripts/configure-ci-r.R @@ -13,7 +13,8 @@ if (system == "Linux") { # Jammy ARM64 has no distro-specific binaries. Portable binaries retain the # same glibc floor on both architectures and need no HTTP user-agent rewrite. cran <- sprintf( - "https://packagemanager.posit.co/cran/latest/bin/linux/manylinux_2_28-%s/%s", + "https://packagemanager.posit.co/cran/%s/bin/linux/manylinux_2_28-%s/%s", + Sys.getenv("CI_PPM_SNAPSHOT", unset = "latest"), R.version$arch, version ) diff --git a/scripts/install-ci-windows-arm-r-bridge.R b/scripts/install-ci-windows-arm-r-bridge.R new file mode 100644 index 000000000..e61732588 --- /dev/null +++ b/scripts/install-ci-windows-arm-r-bridge.R @@ -0,0 +1,11 @@ +# R-universe publishes native Windows ARM64 binaries for the CRAN packages. +stopifnot(.Platform$OS.type == "windows", R.version$arch == "aarch64") +install.packages( + c("reticulate", "RSQLite"), + repos = "https://cran.r-universe.dev", + type = "win.binary.aarch64" +) +stopifnot( + requireNamespace("reticulate", quietly = TRUE), + requireNamespace("RSQLite", quietly = TRUE) +) diff --git a/scripts/linux-wheel/Dockerfile b/scripts/linux-wheel/Dockerfile index 598f0cf27..08391d14d 100644 --- a/scripts/linux-wheel/Dockerfile +++ b/scripts/linux-wheel/Dockerfile @@ -1,45 +1,37 @@ -# Native wheel validation on the supported userspace; no checkout enters here. +# Final native artifacts on an EL8 userspace; no checkout enters this context. FROM ghcr.io/astral-sh/uv:0.13.1 AS uv -FROM ubuntu:22.04 AS base -ENV DEBIAN_FRONTEND=noninteractive UV_NO_CONFIG=1 \ +FROM almalinux:8-minimal AS standalone +RUN microdnf -y --setopt=install_weak_deps=0 install tar gzip && microdnf clean all +WORKDIR /empty-workspace + +FROM standalone AS no-r +ENV UV_NO_CONFIG=1 \ UV_PYTHON_INSTALL_DIR=/opt/console-cache/mcp-console/dependencies/uv/python COPY --from=uv /uv /usr/local/bin/uv -RUN apt-get update && apt-get install -y --no-install-recommends \ - ca-certificates libcap2 libgcc-s1 libstdc++6 binutils \ - && rm -rf /var/lib/apt/lists/* +RUN microdnf -y --setopt=install_weak_deps=0 install binutils && microdnf clean all RUN uv python install 3.14 \ && ln -s "$(uv python find --managed-python 3.14)" /usr/local/bin/python3 ENV UV_PYTHON_DOWNLOADS=never UV_TOOL_DIR=/tmp/tools UV_TOOL_BIN_DIR=/tmp/bin UV_CACHE_DIR=/tmp/uv-cache ENV XDG_CACHE_HOME=/opt/console-cache -WORKDIR /empty-workspace - -FROM base AS no-r - -# R and its package preparation are independent of Console's native libraries. -# Jammy's stock R 4.1 cannot install the current default Arrow/DuckDB packages. -FROM base AS r-repository -RUN apt-get update && apt-get install -y --no-install-recommends curl gnupg \ - && curl -fsSL https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc -o /etc/apt/trusted.gpg.d/cran.asc \ - && test "$(gpg --show-keys --with-colons /etc/apt/trusted.gpg.d/cran.asc | awk -F: '$1 == "fpr" {print $10; exit}')" = E298A3A825C0D65DFD57CBB651716619E084DAB9 \ - && printf '%s\n' 'deb [signed-by=/etc/apt/trusted.gpg.d/cran.asc] https://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/' > /etc/apt/sources.list.d/cran.list -FROM base AS r-runtime -COPY --from=r-repository /etc/apt/trusted.gpg.d/cran.asc /etc/apt/trusted.gpg.d/cran.asc -COPY --from=r-repository /etc/apt/sources.list.d/cran.list /etc/apt/sources.list.d/cran.list -# IR expects R's recommended packages, including reticulate's Matrix dependency. -RUN apt-get update && apt-get install -y --no-install-recommends \ - r-base-core r-recommended libcurl4 libssl3 libxml2 libcairo2 libfontconfig1 \ - libharfbuzz0b libfribidi0 libjpeg-turbo8 libpng16-16 libtiff5 libuv1 libwebp7 libwebpmux3 \ - && rm -rf /var/lib/apt/lists/* +# EPEL's R-core includes R's recommended packages on both architectures. +FROM no-r AS r-runtime +# Both the preparation layer and the final smoke test use this R installation. +ENV R_HOME=/usr/lib64/R +RUN microdnf -y --setopt=install_weak_deps=0 install epel-release \ + && microdnf -y --enablerepo=powertools --setopt=install_weak_deps=0 install \ + R-core libcurl openssl-libs libxml2 cairo fontconfig harfbuzz fribidi \ + libjpeg-turbo libpng libtiff libuv libwebp && microdnf clean all COPY configure-ci-r.R /tmp/configure-ci-r.R -RUN Rscript --vanilla /tmp/configure-ci-r.R && rm /tmp/configure-ci-r.R +# This snapshot supplies textshaping 1.0.5 binaries for EL8's HarfBuzz 1.7.5. +RUN CI_PPM_SNAPSHOT=2026-11-18 Rscript --vanilla /tmp/configure-ci-r.R && rm /tmp/configure-ci-r.R FROM r-runtime AS r-preparation -RUN apt-get update && apt-get install -y --no-install-recommends \ - r-base-dev build-essential pkgconf curl libcurl4-openssl-dev libssl-dev \ - libxml2-dev libcairo2-dev libfontconfig1-dev libharfbuzz-dev libfribidi-dev \ - libjpeg-dev libpng-dev libtiff-dev libuv1-dev libwebp-dev \ - && rm -rf /var/lib/apt/lists/* +RUN microdnf -y --enablerepo=powertools --setopt=install_weak_deps=0 install \ + R-core-devel gcc gcc-c++ gcc-gfortran make pkgconf-pkg-config curl \ + libcurl-devel openssl-devel libxml2-devel cairo-devel fontconfig-devel \ + harfbuzz-devel fribidi-devel libjpeg-turbo-devel libpng-devel libtiff-devel \ + libuv-devel libwebp-devel && microdnf clean all # Match the documented Console cache layout, without opting into host caches. ENV IR_CACHE_DIR=/opt/console-cache/mcp-console/dependencies/ir \ IR_LIBRARY_ROOT=/opt/console-cache/mcp-console/dependencies/ir/libraries \ @@ -49,7 +41,6 @@ ENV IR_CACHE_DIR=/opt/console-cache/mcp-console/dependencies/ir \ R_PKG_CACHE_DIR=/opt/console-cache/mcp-console/dependencies \ UV_CACHE_DIR=/opt/console-cache/mcp-console/dependencies/uv/cache ENV LIBARROW_BINARY=true MAKEFLAGS=-j1 -ENV R_HOME=/usr/lib/R # Bound DuckDB's source-build memory in the language preparation fixture. # These flags affect test dependencies, never the distributed wheel binaries. RUN printf '%s\n' 'CXX17FLAGS=-O0 -g0' > /opt/preparation.Makevars diff --git a/scripts/linux-wheel/build.sh b/scripts/linux-wheel/build.sh new file mode 100644 index 000000000..527e6cc91 --- /dev/null +++ b/scripts/linux-wheel/build.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Run inside the official manylinux_2_28 image for this native target. +set -euo pipefail +target=${1:?expected a native GNU Rust target} +test "$target" = "$(uname -m)-unknown-linux-gnu" +test "$(getconf GNU_LIBC_VERSION)" = 'glibc 2.28' +export PATH="/opt/python/cp314-cp314/bin:$HOME/.cargo/bin:$PATH" + +# The release checkout and its pinned companion are bind-mounted from CI. +git config --global --add safe.directory "$PWD" +git config --global --add safe.directory "$PWD/.sandbox-runner-source" +curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs -o /tmp/rustup-init.sh +sh /tmp/rustup-init.sh -y --profile minimal --no-modify-path +python -m pip install 'maturin>=1.15,<2' + +# libcap is outside the manylinux host ABI. Use AlmaLinux's signed static +# archive. Removing the development symlink makes the pinned build's -lcap +# resolve to libcap.a; libcap.so.2 remains available to system programs. +dnf -y --repofrompath='alma-devel,https://repo.almalinux.org/almalinux/8/devel/$basearch/os/' \ + --setopt=alma-devel.gpgcheck=1 \ + --setopt=alma-devel.gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-AlmaLinux \ + install libcap-devel libcap-static openssl-devel +libdir=$(pkg-config --variable=libdir libcap) +test -f "$libdir/libcap.a" +unlink "$libdir/libcap.so" +export MCP_CONSOLE_LIBCAP_NOTICE=/usr/share/licenses/libcap/License + +scripts/stage-sandbox-runner .sandbox-runner-source --target "$target" +scripts/with-checkout maturin build --release --locked --target "$target" \ + --compatibility manylinux_2_28 --out dist diff --git a/scripts/linux-wheel/standalone.sh b/scripts/linux-wheel/standalone.sh new file mode 100644 index 000000000..499faab28 --- /dev/null +++ b/scripts/linux-wheel/standalone.sh @@ -0,0 +1,19 @@ +#!/bin/sh +# Exercise the archive exactly as a native client would in an EL8 userspace. +set -eu +test "$(getconf GNU_LIBC_VERSION)" = 'glibc 2.28' +test -z "$(rpm -qa '*python*')" +for name in uv pip bwrap cc make cargo; do + if command -v "$name"; then exit 1; fi +done +root=$(mktemp -d) +mkdir "$root/extract" +tar -xzf "$1" -C "$root/extract" +mv "$root/extract" "$root/relocated bundle" +console="$root/relocated bundle/bin/mcp-console" +mkdir "$root/workspace" +cd "$root/workspace" +export PATH="$root/workspace" +"$console" --version +"$console" --help +"$console" sandbox -- /usr/bin/true diff --git a/scripts/release.py b/scripts/release.py index e7b7343ec..aa8188805 100644 --- a/scripts/release.py +++ b/scripts/release.py @@ -1,6 +1,7 @@ from __future__ import annotations import argparse +import fnmatch import hashlib import itertools import json @@ -12,27 +13,30 @@ import shutil import subprocess import sys +import tarfile import tempfile import time import zipfile from collections import deque from email.parser import BytesParser from http.client import IncompleteRead -from pathlib import Path +from pathlib import Path, PureWindowsPath from typing import Any from urllib.error import HTTPError, URLError from urllib.request import Request, urlopen STABLE_TAG = re.compile(r"v[0-9]+\.[0-9]+\.[0-9]+") PACKAGE_VERSION = re.compile(r'^version\s*=\s*"([^"]+)"\s*$') -TARGET_ARCHITECTURES = { - "aarch64-apple-darwin": "arm64", - "x86_64-apple-darwin": "x86_64", - "aarch64-unknown-linux-gnu": "aarch64", - "x86_64-unknown-linux-gnu": "x86_64", +# Platform tags of the wheels used for both distributions. +RELEASE_PLATFORMS = { + "aarch64-apple-darwin": "macosx_*_arm64", + "x86_64-apple-darwin": "macosx_*_x86_64", + "aarch64-unknown-linux-gnu": "manylinux_2_28_aarch64", + "x86_64-unknown-linux-gnu": "manylinux_2_28_x86_64", + "aarch64-pc-windows-msvc": "win_arm64", + "x86_64-pc-windows-msvc": "win_amd64", } -LINUX_RELEASE_GLIBC = (2, 35) -LINUX_RELEASE_CPP = {"GLIBCXX": (3, 4, 30), "CXXABI": (1, 3, 13)} +LINUX_RELEASE_GLIBC = (2, 28) # Numeric GLIBCXX_3.4.N / CXXABI_1.3.N policy ceilings, not host library versions. # https://github.com/pypa/auditwheel/blob/7cec8ec5b1436336bc03e560b785cc63d9c4190f/src/auditwheel/policy/manylinux-policy.json MANYLINUX_CPP = { @@ -102,7 +106,6 @@ "librt.so.1", "libgcc_s.so.1", "libstdc++.so.6", - "libcap.so.2", "ld-linux-x86-64.so.2", "ld-linux-aarch64.so.1", } @@ -433,6 +436,20 @@ def wheel_identity(wheel: Path) -> tuple[str, set[str]]: return version, tags +def wheel_permissions(entry: zipfile.ZipInfo) -> int: + """Unix read, write, and execute bits carried into standalone archives.""" + return entry.external_attr >> 16 & 0o777 + + +def wheel_prefix_path(member: str, data_prefix: str) -> str | None: + """Installed Unix path relative to the prefix for a wheel data or script file.""" + if member.startswith(data_prefix + "data/"): + return member.removeprefix(data_prefix + "data/") + if member.startswith(data_prefix + "scripts/"): + return "bin/" + member.removeprefix(data_prefix + "scripts/") + return None + + def inspect_linux_abi( wheel: Path, version: str, tags: set[str], *, target: str | None, release: bool ) -> dict[str, Any]: @@ -457,14 +474,13 @@ def inspect_linux_abi( architecture = architectures.pop() if target: require( - target.endswith("-linux-gnu") - and TARGET_ARCHITECTURES[target] == architecture, + target.endswith("-linux-gnu") and target.partition("-")[0] == architecture, f"wheel does not match {target}: {wheel.name}", ) if release: require( len(floors) == len(tags) and max(floors) <= LINUX_RELEASE_GLIBC, - "wheel tag exceeds the glibc 2.35 release floor", + "wheel must advertise a manylinux tag within the glibc 2.28 release floor", ) ceilings: dict[str, list[tuple[int, ...]]] = { "GLIBC": floors, @@ -490,9 +506,6 @@ def inspect_linux_abi( for number in numbers } gcc_allowed = gcc_policy if gcc_allowed is None else gcc_allowed & gcc_policy - if release: - for family, ceiling in LINUX_RELEASE_CPP.items(): - ceilings[family].append(ceiling) limits = {family: min(values, default=None) for family, values in ceilings.items()} named_allowed = { family: { @@ -502,16 +515,14 @@ def inspect_linux_abi( } for family, versions in MANYLINUX_NAMED.items() } + # Source builds may use host libcap. Release bundles link it statically. + system_libraries = ( + LINUX_SYSTEM_LIBRARIES if release else LINUX_SYSTEM_LIBRARIES | {"libcap.so.2"} + ) machine, loader = LINUX_MACHINES[architecture] evidence = {} # Model a Unix installation prefix; Python's version does not change its depth. site_packages = "lib/pythonX.Y/site-packages" - schemes = { - "scripts": "bin", - "data": ".", - "purelib": site_packages, - "platlib": site_packages, - } data_prefix = f"mcp_console-{version}.data/" installed_elves: dict[str, str] = {} rpaths: dict[str, tuple[str, ...]] = {} @@ -521,14 +532,17 @@ def inspect_linux_abi( with archive.open(member) as stream: if stream.read(4) != b"\x7fELF": continue - if member.startswith(data_prefix): - scheme, relative = member.removeprefix(data_prefix).split("/", 1) - require(scheme in schemes, f"{member}: unsupported ELF install scheme") - installed = posixpath.normpath( - posixpath.join(schemes[scheme], relative) - ) - else: - installed = posixpath.join(site_packages, member) + installed = wheel_prefix_path(member, data_prefix) + if installed is None: + relative = member + if member.startswith(data_prefix): + scheme, relative = member.removeprefix(data_prefix).split("/", 1) + require( + scheme in {"purelib", "platlib"}, + f"{member}: unsupported ELF install scheme", + ) + installed = posixpath.join(site_packages, relative) + installed = posixpath.normpath(installed) installed_elves[installed] = member elf = Path(directory) / "artifact" elf.write_bytes(archive.read(member)) @@ -638,6 +652,7 @@ def field(name: str) -> str | None: checked: set[str] = set() def check_dependencies(entry: str) -> None: + standalone = release and wheel_prefix_path(entry, data_prefix) is not None loaded_names: set[str] = set() loaded_members = {entry} pending: deque[tuple[str, tuple[str, ...]]] = deque([(entry, ())]) @@ -659,7 +674,7 @@ def check_dependencies(entry: str) -> None: f"{member}: unsupported DT_NEEDED filename {needed}; " "dependencies must be library names without /", ) - if needed in LINUX_SYSTEM_LIBRARIES or needed in loaded_names: + if needed in system_libraries or needed in loaded_names: continue dependency = next( ( @@ -678,6 +693,11 @@ def check_dependencies(entry: str) -> None: dependency is not None, f"{member}: undeclared dependency {needed}", ) + if standalone: + require( + wheel_prefix_path(dependency, data_prefix) is not None, + f"{member}: standalone dependency {needed} uses a Python install scheme", + ) loaded_names.add(needed) if dependency not in loaded_members: loaded_members.add(dependency) @@ -699,6 +719,14 @@ def check_dependencies(entry: str) -> None: return evidence +def sandbox_notices(linux: bool) -> tuple[str, ...]: + return ("LICENSE", "NOTICE") + ( + ("bubblewrap-COPYING", "bubblewrap-NOTICE", "bubblewrap-SOURCE.json") + if linux + else () + ) + + def inspect_wheel_commands( wheel: Path, *, @@ -720,18 +748,10 @@ def inspect_wheel_commands( f"wheel must contain exactly one private sandbox runner {name} under libexec", ) require( - archive.getinfo(runner).external_attr >> 16 & 0o111 != 0, + wheel_permissions(archive.getinfo(runner)) & 0o111 != 0, f"private sandbox runner {name} is not executable", ) - for name in ( - "LICENSE", - "NOTICE", - *( - ["bubblewrap-COPYING", "bubblewrap-NOTICE", "bubblewrap-SOURCE.json"] - if linux - else [] - ), - ): + for name in sandbox_notices(linux): require( f"{data}/share/licenses/mcp-console/{name}" in members, f"private sandbox runner is missing {name}", @@ -774,6 +794,11 @@ def inspect_wheel_commands( if any(name.startswith("libcap.so.") for name in needed) else "static" ) + if release: + require( + linkage == "static", + "release Bubblewrap must link libcap statically", + ) require( provenance.get("elf_needed") == needed, "Bubblewrap provenance has inconsistent ELF dependencies", @@ -853,7 +878,11 @@ def smoke_wheel(args: argparse.Namespace) -> None: ) if args.target is not None: - architecture = TARGET_ARCHITECTURES[args.target] + architecture = ( + "arm64" + if args.target == "aarch64-apple-darwin" + else args.target.partition("-")[0] + ) require( wheel.name.endswith(f"_{architecture}.whl") and linux == ("linux" in args.target), @@ -985,6 +1014,83 @@ def smoke_wheel(args: argparse.Namespace) -> None: ) +def publish_archives(args: argparse.Namespace) -> None: + validate_publish(args) + directory = Path(args.directory) + digests, expected_manifest = archive_manifest(directory) + manifest = (directory / "SHA256SUMS").read_bytes() + require( + sorted(manifest.split(b"\n")) == sorted(expected_manifest.split(b"\n")), + "SHA256SUMS does not match the release archives", + ) + digests["SHA256SUMS"] = hashlib.sha256(manifest).hexdigest() + expected = {name: f"sha256:{digest}" for name, digest in digests.items()} + tag = os.environ["GITHUB_REF_NAME"] + tags = command_output( + [ + "gh", + "api", + "--paginate", + "repos/{owner}/{repo}/releases?per_page=100", + "--jq", + ".[].tag_name", + ] + ).splitlines() + if tag not in tags: + run_command( + [ + "gh", + "release", + "create", + tag, + "--verify-tag", + "--draft", + "--generate-notes", + ] + ) + + def inspect() -> tuple[bool, set[str]]: + remote = json.loads( + command_output(["gh", "release", "view", tag, "--json", "isDraft,assets"]) + ) + uploaded = {asset["name"]: asset["digest"] for asset in remote["assets"]} + for name, digest in uploaded.items(): + require(name in expected, f"unexpected GitHub asset: {name}") + require(digest == expected[name], f"GitHub asset digest differs: {name}") + return remote["isDraft"], expected.keys() - uploaded.keys() + + draft, missing = inspect() + if missing: + require(draft, "published GitHub release is missing assets") + run_command( + [ + "gh", + "release", + "upload", + tag, + *[str(directory / name) for name in sorted(missing)], + ] + ) + draft, missing = inspect() + require(not missing, f"GitHub release is missing assets: {sorted(missing)}") + if draft: + run_command(["gh", "release", "edit", tag, "--draft=false"]) + + +def check_unpublished(args: argparse.Namespace) -> None: + version = package_version() + url = f"{args.pypi_url.rstrip('/')}/pypi/mcp-console/{version}/json" + try: + with urlopen(url, timeout=30): + raise ReleaseError( + f"{version} is already on PyPI; reuse the original build artifacts" + ) + except HTTPError as error: + error.close() + if error.code != 404: + raise + + def validate_publish(_: argparse.Namespace) -> None: event_name = os.environ["GITHUB_EVENT_NAME"] ref_type = os.environ["GITHUB_REF_TYPE"] @@ -1047,34 +1153,155 @@ def validate_publish(_: argparse.Namespace) -> None: ) +def release_wheels(directory: Path) -> list[Path]: + wheels = sorted(directory.glob("*.whl")) + require( + len(wheels) == len(RELEASE_PLATFORMS), + f"expected exactly six wheels, found {len(wheels)}", + ) + version = package_version() + for target, platform in RELEASE_PLATFORMS.items(): + pattern = f"mcp_console-{version}-py3-none-{platform}.whl" + require( + sum(wheel.match(pattern) for wheel in wheels) == 1, + f"expected exactly one {target} wheel", + ) + return wheels + + def verify_wheel_set(args: argparse.Namespace) -> None: directory = Path(args.directory) - wheels = sorted(directory.glob("*.whl")) - arm64 = list(directory.glob("mcp_console-*-macosx_*_arm64.whl")) - x86_64 = list(directory.glob("mcp_console-*-macosx_*_x86_64.whl")) - linux_arm64 = list(directory.glob("mcp_console-*-manylinux_*_aarch64.whl")) - linux_x86_64 = list(directory.glob("mcp_console-*-manylinux_*_x86_64.whl")) - windows_x86_64 = list(directory.glob("mcp_console-*-win_amd64.whl")) - universal = list(directory.glob("*-none-any.whl")) - sdists = list(directory.glob("*.tar.gz")) - - require(len(wheels) == 5, f"expected exactly five wheels, found {len(wheels)}") - require(len(arm64) == 1, "expected exactly one Apple Silicon wheel") - require(len(x86_64) == 1, "expected exactly one Intel macOS wheel") - require(len(linux_arm64) == 1, "expected exactly one ARM64 Linux wheel") - require(len(linux_x86_64) == 1, "expected exactly one x86-64 Linux wheel") - require(len(windows_x86_64) == 1, "expected exactly one x86-64 Windows wheel") - require(not universal, "a platform-independent wheel must not be published") - require(not sdists, "a source distribution must not be published") - - print("Publishing:") + wheels = release_wheels(directory) + require( + set(directory.iterdir()) == set(wheels), "only release wheels may be published" + ) for wheel in wheels: - print(f" {wheel}") + print(wheel) + + +def archive_name(target: str) -> str: + suffix = "zip" if "windows" in target else "tar.gz" + return f"mcp-console-{target}.{suffix}" + + +def standalone_members( + source: zipfile.ZipFile, version: str, target: str +) -> dict[str, str]: + """Map native installation paths to their source members in the wheel.""" + windows = "windows" in target + linux = "linux" in target + data = f"mcp_console-{version}.data" + licenses = "share/licenses/mcp-console" + suffix = ".exe" if windows else "" + public = f"bin/mcp-console{suffix}" + # The Windows wheel entry point is a Python launcher. The backend keeps + # the native Console alongside the runner so both stay in the environment. + native = ( + f"{data}/data/libexec/mcp-console.exe" + if windows + else f"{data}/scripts/mcp-console" + ) + + members = { + public: native, + f"{licenses}/mcp-console-LICENSE": f"mcp_console-{version}.dist-info/licenses/LICENSE", + } + # Wheel data installs directly under the prefix. Preserve all of it, + # including resources and licenses beside libraries. The .gitignore + # is the staging marker excluded from wheels by pyproject.toml. + data_prefix = f"{data}/" + for info in source.infolist(): + if info.is_dir() or info.filename == native: + continue + relative = wheel_prefix_path(info.filename, data_prefix) + if relative is None or relative == ".gitignore": + continue + if info.filename.startswith(data_prefix + "scripts/"): + if not linux: + continue + with source.open(info) as stream: + if stream.read(4) != b"\x7fELF": + continue + require( + all(part not in ("", ".", "..") for part in relative.split("/")) + and "\\" not in relative + and not (windows and PureWindowsPath(relative).drive) + and relative not in members, + f"invalid native bundle member: {info.filename}", + ) + members[relative] = info.filename + required = { + f"libexec/mcp-console-sandbox{suffix}", + *(f"{licenses}/{name}" for name in sandbox_notices(linux)), + } + if linux: + required.add("libexec/bwrap") + require( + required <= members.keys(), + f"missing native bundle files: {sorted(required - members.keys())}", + ) + return members + + +def build_archive(args: argparse.Namespace) -> None: + wheel = Path(args.wheel) + version, tags = wheel_identity(wheel) + require( + all( + fnmatch.fnmatchcase(tag, f"py3-none-{RELEASE_PLATFORMS[args.target]}") + for tag in tags + ), + f"wheel does not match {args.target}: {wheel.name}", + ) + output = Path(args.out_dir) / archive_name(args.target) + output.parent.mkdir(parents=True, exist_ok=True) + with ( + zipfile.ZipFile(wheel) as source, + tempfile.TemporaryDirectory(dir=output.parent) as temp, + ): + members = standalone_members(source, version, args.target) + # Build in a temporary path; a failed wheel read leaves no partial archive. + pending = Path(temp) / output.name + if "windows" in args.target: + with zipfile.ZipFile(pending, "w", zipfile.ZIP_DEFLATED) as archive: + for dest, src in sorted(members.items()): + archive.writestr(dest, source.read(src)) + else: + with tarfile.open(pending, "w:gz") as tar: + for dest, src in sorted(members.items()): + entry = source.getinfo(src) + item = tarfile.TarInfo(dest) + item.mode = wheel_permissions(entry) + item.size = entry.file_size + with source.open(entry) as content: + tar.addfile(item, content) + pending.replace(output) + print(output) + + +def archive_manifest(directory: Path) -> tuple[dict[str, str], bytes]: + expected = {directory / archive_name(target) for target in RELEASE_PLATFORMS} + found = set(directory.iterdir()) - {directory / "SHA256SUMS"} + require( + found == expected, + f"archive set differs: {sorted(path.name for path in expected ^ found)}", + ) + digests = {} + for path in sorted(expected): + with path.open("rb") as stream: + digests[path.name] = hashlib.file_digest(stream, "sha256").hexdigest() + manifest = "".join(f"{digest} {name}\n" for name, digest in digests.items()) + return digests, manifest.encode("ascii") + + +def archive_checksums(args: argparse.Namespace) -> None: + directory = Path(args.directory) + _, manifest = archive_manifest(directory) + (directory / "SHA256SUMS").write_bytes(manifest) def wait_pypi(args: argparse.Namespace) -> None: - wheels = sorted(Path(args.directory).glob("*.whl")) - require(len(wheels) == 5, f"expected exactly five wheels, found {len(wheels)}") + wheels = release_wheels(Path(args.directory)) for value in (args.timeout_seconds, args.poll_interval_seconds): require( math.isfinite(value) and value > 0, @@ -1155,13 +1382,23 @@ def parser() -> argparse.ArgumentParser: argument_parser = argparse.ArgumentParser() commands = argument_parser.add_subparsers(required=True) + unpublished = commands.add_parser("check-unpublished") + unpublished.add_argument("--pypi-url", default="https://pypi.org") + unpublished.set_defaults(function=check_unpublished) + + publish = commands.add_parser("publish-archives") + publish.add_argument("directory") + publish.set_defaults(function=publish_archives) + smoke = commands.add_parser("smoke-wheel") smoke.add_argument("wheel") smoke.add_argument("cargo_bin", nargs="?") smoke.add_argument("--installed-only", action="store_true") smoke.add_argument("--sandbox-pin", default="sandbox-runner.json") smoke.add_argument("--release", action="store_true") - smoke.add_argument("--target", choices=sorted(TARGET_ARCHITECTURES)) + smoke.add_argument( + "--target", choices=sorted(t for t in RELEASE_PLATFORMS if "windows" not in t) + ) smoke.add_argument("--without-r", action="store_true") smoke.add_argument("--startup-timeout-seconds", type=float, default=1200.0) smoke.add_argument("--response-timeout-seconds", type=float, default=30.0) @@ -1169,7 +1406,9 @@ def parser() -> argparse.ArgumentParser: inspect = commands.add_parser("inspect-wheel") inspect.add_argument("wheel") - inspect.add_argument("--target", choices=sorted(TARGET_ARCHITECTURES)) + inspect.add_argument( + "--target", choices=sorted(t for t in RELEASE_PLATFORMS if "linux" in t) + ) inspect.add_argument("--release", action="store_true") inspect.add_argument("--sandbox-pin", default="sandbox-runner.json") inspect.add_argument("--report") @@ -1178,6 +1417,16 @@ def parser() -> argparse.ArgumentParser: validate = commands.add_parser("validate-publish") validate.set_defaults(function=validate_publish) + archive = commands.add_parser("build-archive") + archive.add_argument("wheel") + archive.add_argument("--target", choices=sorted(RELEASE_PLATFORMS), required=True) + archive.add_argument("--out-dir", required=True) + archive.set_defaults(function=build_archive) + + sums = commands.add_parser("archive-checksums") + sums.add_argument("directory") + sums.set_defaults(function=archive_checksums) + verify = commands.add_parser("verify-wheel-set") verify.add_argument("directory") verify.set_defaults(function=verify_wheel_set) diff --git a/scripts/stage-sandbox-runner b/scripts/stage-sandbox-runner index 74a0e2c45..57119b9f6 100755 --- a/scripts/stage-sandbox-runner +++ b/scripts/stage-sandbox-runner @@ -20,6 +20,7 @@ TARGETS = ( "x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + "aarch64-pc-windows-msvc", ) diff --git a/src/local_runtime/r_selection.rs b/src/local_runtime/r_selection.rs index 56f7eff14..e443749d0 100644 --- a/src/local_runtime/r_selection.rs +++ b/src/local_runtime/r_selection.rs @@ -166,10 +166,11 @@ impl RInstallation { } else { home.join("bin/x64/Rscript.exe") }; + #[cfg(windows)] + let library = harp::sys::library::find_r_shared_library_folder(&home).join("R.dll"); + #[cfg(unix)] let library = home.join(if cfg!(target_os = "macos") { "lib/libR.dylib" - } else if cfg!(windows) { - "bin/x64/R.dll" } else { "lib/libR.so" }); diff --git a/src/main.rs b/src/main.rs index 27d35c101..a3dc8a48f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -53,13 +53,17 @@ mod worker_protocol; mod worker_relay; fn main() -> ExitCode { - #[cfg(all(windows, target_arch = "x86_64"))] + #[cfg(windows)] if std::env::var_os("PROCESSOR_ARCHITECTURE").is_none_or(|value| value.is_empty()) { // Filtered MCP environments can omit this Windows runtime variable. // R DuckDB's native teardown can crash without it, including in the // preparation child. Each Console process restores its own architecture. + #[cfg(target_arch = "x86_64")] + let architecture = "AMD64"; + #[cfg(target_arch = "aarch64")] + let architecture = "ARM64"; // SAFETY: entry is single-threaded, before native runtime loading. - unsafe { std::env::set_var("PROCESSOR_ARCHITECTURE", "AMD64") }; + unsafe { std::env::set_var("PROCESSOR_ARCHITECTURE", architecture) }; } let cli = cli::Cli::parse(); let mut overrides = cli.overrides; diff --git a/src/python/bridge.R b/src/python/bridge.R index e3a4bced3..742070d65 100644 --- a/src/python/bridge.R +++ b/src/python/bridge.R @@ -78,6 +78,7 @@ base::local( version_string = metadata$version, version = as.package_version(metadata$version_number), architecture = metadata$architecture, + platform = selection$platform, anaconda = grepl( "anaconda|continuum", metadata$version, @@ -467,17 +468,14 @@ base::local( !is.na(managed) )) } - # R-first attachment precedes shared site and workspace setup. Publish - # metadata after those paths are active, as for Python-first attachment. + # A startup package may have already published reticulate's config. + # Refresh it from the running interpreter after site and workspace setup. live <- conversion_config(jsonlite::fromJSON(.Call( "mcp_console_running_python" ))) globals <- get(".globals", envir = asNamespace("reticulate")) - globals$py_config[c("pythonpath", "numpy", "ephemeral")] <- live[c( - "pythonpath", - "numpy", - "ephemeral" - )] + fields <- c("platform", "pythonpath", "numpy", "ephemeral") + globals$py_config[fields] <- live[fields] invisible(TRUE) } diff --git a/src/python/initialize.R b/src/python/initialize.R index c6dea0c57..ee9837501 100644 --- a/src/python/initialize.R +++ b/src/python/initialize.R @@ -75,6 +75,28 @@ base::local( # do not rediscover an executable or rerun environment activation. config <- globals$py_config sys <- reticulate::import("sys", convert = TRUE) + platform <- config[["platform"]] + if (is.null(platform)) { + # Older reticulate did not record the platform. Ask the same Python + # executable with workspace imports and site startup disabled. + probe <- shQuote("import sysconfig; print(sysconfig.get_platform())") + platform <- system2( + sys$executable, + c("-I", "-S", "-c", probe), + stdout = TRUE + ) + if ( + !is.null(attr(platform, "status")) || + length(platform) != 1L || + !nzchar(platform) + ) { + stop( + "Could not determine the platform of Python at ", + sys$executable, + call. = FALSE + ) + } + } inspected <<- jsonlite::toJSON( list( embedding = list( @@ -82,6 +104,7 @@ base::local( libpython = config$libpython, python_home = config$pythonhome ), + platform = platform, prefix = sys$prefix, exec_prefix = sys$exec_prefix, base_prefix = sys$base_prefix, @@ -152,11 +175,11 @@ base::local( "Installation of Python not found, Python bindings not loaded." ) } else if (reticulate:::is_incompatible_arch(config)) { - fmt <- "Your current architecture is %s; however, this version of Python was compiled for %s." message <- sprintf( - fmt, - reticulate:::current_python_arch(), - config$architecture + "Python at '%s' (%s) is incompatible with this build of R (%s).", + config$python, + paste(c(config$platform, config$architecture), collapse = ", "), + R.version$arch ) python_not_found(message) } @@ -171,7 +194,9 @@ base::local( stop(error) } ) - embedding <- jsonlite::fromJSON(inspected)$embedding + identity <- jsonlite::fromJSON(inspected) + embedding <- identity$embedding + config$platform <- identity$platform config$python <- embedding$python config$executable <- embedding$python config$libpython <- embedding$libpython @@ -243,16 +268,21 @@ base::local( # Console owns CPython's environment. Reticulate attaches conversion # and callbacks without activating or changing the running interpreter. on.exit(finish_python_initialization(), add = TRUE) - reticulate:::py_initialize( - config$python, - config$libpython, - config$pythonhome, - "", - config$version$major, - config$version$minor, - interactive(), - numpy_load_error + arguments <- list( + python = config$python, + libpython = config$libpython, + pythonhome = config$pythonhome, + virtualenv_activate = "", + python_minor_version = config$version$minor, + interactive = interactive(), + numpy_load_error = numpy_load_error ) + py_initialize <- reticulate:::py_initialize + # Reticulate dropped this argument when it removed Python 2 support. + if ("python_major_version" %in% names(formals(py_initialize))) { + arguments$python_major_version <- config$version$major + } + do.call(py_initialize, arguments) # Reticulate owns conversion, cross-language calls, and event integration. reg.finalizer( @@ -305,6 +335,19 @@ base::local( install_python_initializer <- function(...) { namespace <<- asNamespace("reticulate") globals <<- get(".globals", envir = namespace) + if ( + .Platform$OS.type == "windows" && + exists("current_python_arch", envir = namespace, inherits = FALSE) && + namespace$current_python_arch() == "Unknown" + ) { + # Reticulate 1.47 infers pointer width from R's subarchitecture suffix, + # which is empty on native Windows ARM64. Mixed x64/ARM64 installations + # must select a native Python explicitly with this reticulate version. + # Newer reticulate checks the build platform and has no such helper. + replace_binding("current_python_arch", function() { + paste0(.Machine$sizeof.pointer * 8L, "bit") + }) + } replace_binding("initialize_python", initialize_python) original_ensure_initialized <- get( "ensure_python_initialized", diff --git a/src/python/inspection.py b/src/python/inspection.py index ce96f6d4f..ffc8eaac7 100644 --- a/src/python/inspection.py +++ b/src/python/inspection.py @@ -123,6 +123,7 @@ def describe() -> dict[str, object]: pass return { + "platform": sysconfig.get_platform(), "metadata": { "base_executable": sys._base_executable, "pythonpath": os.pathsep.join(sys.path), diff --git a/src/python/inspection.rs b/src/python/inspection.rs index b1505c0b7..1e4406b69 100644 --- a/src/python/inspection.rs +++ b/src/python/inspection.rs @@ -31,6 +31,8 @@ pub(crate) fn explicit_executable(value: &std::ffi::OsStr) -> Result Iterator[tuple[McpClient, FifoCheckpoint, FifoCheckpoint, Path]]: +) -> Iterator[tuple[McpClient, FifoCheckpoint, Path]]: # fmt: python server = code(""" import os @@ -107,7 +108,19 @@ def before_resolver_spawn( ) try: client.initialize_and_list_tools() - yield client, started, release, root + started.wait(f"resolver fork {ordinal}", client=client) + # The resolver can reach fork before the server registers the + # operation. Observe both before submitting the cell or interrupt. + deadline = client.response_deadline() + while True: + client.send(session="default", timeout_ms=100) + status = last_tool_text(client) + if status == "\n[phase: dependency preparation]\n[worker starting]": + break + assert status == "\n[phase: startup]\n[worker starting]", status + assert time.monotonic() < deadline, "resolver handle was not registered" + client.transcript.pop() + yield client, release, root finally: # Release the resolver fork before transport teardown, even when an # assertion fails while it is paused before child creation. @@ -120,16 +133,8 @@ def before_resolver_spawn( def test_interrupts_first_cell_before_resolver_registration( binary: Path, execution: Execution ) -> Transcript: - with before_resolver_spawn(binary, execution, 1) as ( - client, - started, - release, - root, - ): - started.wait("first resolver has not been spawned", client=client) + with before_resolver_spawn(binary, execution, 1) as (client, release, root): assert not (root / "resolver.jsonl").exists() - # The preparation handle is registered, but its materializer is held - # before process creation. Observe that owner before and after interrupt. client.send(r="startup_cell_ran <- TRUE", timeout_ms=0) assert phase_progress(last_tool_text(client)) == "dependency preparation" assert without_elapsed(last_tool_text(client)) == RUNNING @@ -152,13 +157,7 @@ def test_interrupts_first_cell_before_resolver_registration( def test_interrupts_first_cell_between_resolver_phases( binary: Path, execution: Execution ) -> Transcript: - with before_resolver_spawn(binary, execution, 2) as ( - client, - started, - release, - root, - ): - started.wait("default preparation has not been spawned", client=client) + with before_resolver_spawn(binary, execution, 2) as (client, release, root): invocations = [ json.loads(line) for line in (root / "resolver.jsonl").read_text().splitlines() @@ -189,13 +188,7 @@ def test_interrupts_first_cell_admitted_during_stdin_startup( binary: Path, execution: Execution, ) -> Transcript: - with before_resolver_spawn(binary, execution, 1) as ( - client, - started, - release, - root, - ): - started.wait("stdin startup has not spawned its first resolver", client=client) + with before_resolver_spawn(binary, execution, 1) as (client, release, root): assert not (root / "resolver.jsonl").exists() client.send(stdin="old input\n", timeout_ms=0) assert last_tool_text(client) == ( diff --git a/tests/boundaries/client_server/python/test_peer_runtime.py b/tests/boundaries/client_server/python/test_peer_runtime.py index 834026c7b..cefd53200 100644 --- a/tests/boundaries/client_server/python/test_peer_runtime.py +++ b/tests/boundaries/client_server/python/test_peer_runtime.py @@ -6,6 +6,7 @@ import shutil import subprocess import sys +import sysconfig import tempfile from contextlib import ExitStack from functools import cache @@ -265,6 +266,7 @@ def test_late_attachment_preserves_environment_metadata( records = [] with tempfile.TemporaryDirectory() as directory: root = Path(directory).resolve() + (root / "sysconfig.py").write_text("project_value = 42\n") modules = root / "modules" modules.mkdir() (modules / "sitecustomize.py").write_text(DEFER_R_STARTUP) @@ -293,14 +295,26 @@ def test_late_attachment_preserves_environment_metadata( RETICULATE_PYTHON=str(executable), RETICULATE_PYTHONPATH=str(modules), MCP_CONSOLE_TEST_PYTHON=str(executable), + MCP_CONSOLE_TEST_PYTHON_PLATFORM=sysconfig.get_platform(), MCP_CONSOLE_TEST_ENVIRONMENT_KIND=kind, MCP_CONSOLE_TEST_VIRTUALENV="" if kind == "base" else str(prefix), ) with McpClient(binary, execution.serve(), environment, root) as client: client.initialize_and_list_tools() defer_r_bootstrap(client) + # A startup hook can cache the standard library's sysconfig. + # Load the workspace module before asking R to attach. client.expect( - python="peer_object = object(); peer_identity = id(peer_object)", + # fmt: python + python=code(""" + import sys + + if "sysconfig" in sys.modules: + del sys.modules["sysconfig"] + import sysconfig as project_sysconfig + + assert project_sysconfig.project_value == 42 + """), ) # fmt: r client.expect( @@ -310,6 +324,7 @@ def test_late_attachment_preserves_environment_metadata( sys <- reticulate::import("sys") stopifnot( identical(config$python, Sys.getenv("MCP_CONSOLE_TEST_PYTHON")), + identical(config$platform, Sys.getenv("MCP_CONSOLE_TEST_PYTHON_PLATFORM")), identical(normalizePath(config$prefix), normalizePath(sys$prefix)), identical(normalizePath(config$exec_prefix), normalizePath(sys$exec_prefix)), identical(normalizePath(config$base_prefix), normalizePath(sys$base_prefix)), @@ -333,7 +348,13 @@ def test_late_attachment_preserves_environment_metadata( ) client.expect( "same interpreter\n", - python="assert id(peer_object) == peer_identity; print('same interpreter')", + # fmt: python + python=code(""" + import sysconfig + + assert sysconfig is project_sysconfig + print("same interpreter") + """), ) records.extend([{"environment": kind}, *client.finish()[3:]]) return records @@ -1356,6 +1377,7 @@ def test_conversion_metadata_matches_configured_import_paths( sys <- reticulate::import("sys") numpy <- reticulate::import("numpy") stopifnot( + identical(config$platform, reticulate::import("sysconfig")$get_platform()), identical(config$pythonpath, paste(sys$path, collapse = .Platform$path.sep)), identical( normalizePath(config$numpy$path), @@ -1454,11 +1476,20 @@ def test_attaches_to_reticulate_initialized_by_r_startup( MCP_CONSOLE_LANGUAGES="r", R_LIBS=os.pathsep.join(filter(None, (str(library), environment.get("R_LIBS")))), R_DEFAULT_PACKAGES="datasets,utils,grDevices,graphics,stats,methods,mcpconsoleearlypython", - RETICULATE_PYTHON=sys.executable, + MCP_CONSOLE_TEST_PYTHON_PLATFORM=sysconfig.get_platform(), + RETICULATE_CHECK_REQUIRED_PACKAGES="false", ) with tempfile.TemporaryDirectory() as directory: root = Path(directory) environment["TMPDIR"] = str(root) + # NumPy can preload the standard library's sysconfig before adoption. + selected = root / "early Python" + subprocess.run( + [sys.executable, "-m", "venv", "--without-pip", str(selected)], + check=True, + ) + environment["RETICULATE_PYTHON"] = str(virtualenv_python(selected)) + (root / "sysconfig.py").write_text("") worker = root / "worker" worker.write_text( "#!/bin/sh\nexec " + shlex.join([str(binary), "worker"]) + "\n" @@ -1477,7 +1508,13 @@ def test_attaches_to_reticulate_initialized_by_r_startup( reticulate::py_run_string( "assert id(early_object) == early_identity; assert sys.executable == early_executable; assert (sys.prefix, sys.exec_prefix, sys.base_prefix, sys.base_exec_prefix) == early_prefixes" ) - stopifnot(identical(reticulate::py_eval("{'answer': 42}"), list(answer = 42L))) + stopifnot( + identical(reticulate::py_eval("{'answer': 42}"), list(answer = 42L)), + identical( + reticulate::py_config()$platform, + Sys.getenv("MCP_CONSOLE_TEST_PYTHON_PLATFORM") + ) + ) cat("attached to startup Python\\n") """), ) diff --git a/tests/boundaries/client_server/python/test_without_r.py b/tests/boundaries/client_server/python/test_without_r.py index 1ebb15d08..bc4052b43 100644 --- a/tests/boundaries/client_server/python/test_without_r.py +++ b/tests/boundaries/client_server/python/test_without_r.py @@ -135,6 +135,7 @@ def preparation_environment(root: Path, *, with_r: bool = False) -> dict[str, st description = { "executable": str(root / "invalid-python"), "libpython": str(root / "missing-libpython"), + "platform": "fixture-platform", "metadata": { "base_executable": str(root / "invalid-python"), "pythonpath": "", diff --git a/tests/install.py b/tests/install.py index 1e13a1a82..8aae38ee2 100644 --- a/tests/install.py +++ b/tests/install.py @@ -525,21 +525,41 @@ def stage_stale_wheel_data() -> None: launch_checks = ( "SandboxInstallationTests.test_runs_original_binary", "SandboxInstallationTests.test_resolves_private_runner_through_public_symlink", - "SandboxInstallationTests.test_relocates_the_bundle_without_a_writable_home_or_path_lookup", + "SandboxInstallationTests.test_relocates_without_a_writable_console_home_or_path_lookup", ) + # Damaging the fixture must leave the original linked runner executable. + runner = bundle / "libexec/mcp-console-sandbox" + original_runner = runner.rename(runner.with_name("original-runner")) + original_mode = original_runner.stat().st_mode + runner.symlink_to(original_runner) # Every installation must launch, follow its public symlink, - # and relocate. Bundle damage and allocation checks exercise - # the same copied layout, so run them once against the wheel. - for installed, checks in ( - (binary, launch_checks), - (directory / "uv-bin" / "mcp-console", launch_checks), - (directory / "wheel-bin" / "mcp-console", ()), + # and relocate. Run the full damage suite once against the wheel. + for installed, bundle_args, checks in ( + ( + binary, + ["--standalone"], + ( + *launch_checks, + "SandboxInstallationTests.test_rejects_a_non_executable_private_runner", + ), + ), + ( + directory / "uv-bin" / "mcp-console", + ["--wheel", wheels[0]], + launch_checks, + ), + ( + directory / "wheel-bin" / "mcp-console", + ["--wheel", wheels[0]], + (), + ), ): with self.subTest(installer=installed): result = subprocess.run( [ sys.executable, str(ROOT / "tests" / "sandbox_installation.py"), + *bundle_args, str(installed), *checks, ], @@ -551,6 +571,7 @@ def stage_stale_wheel_data() -> None: self.assertEqual( result.returncode, 0, result.stdout + result.stderr ) + self.assertEqual(original_runner.stat().st_mode, original_mode) finally: hidden.rename(target) diff --git a/tests/linux_wheel_runtime.py b/tests/linux_wheel_runtime.py index 93fa40061..704c89e6f 100644 --- a/tests/linux_wheel_runtime.py +++ b/tests/linux_wheel_runtime.py @@ -24,7 +24,7 @@ def main() -> None: glibc = subprocess.check_output(["getconf", "GNU_LIBC_VERSION"], text=True).strip() print(glibc, Path("/etc/os-release").read_text(), flush=True) - assert glibc == "glibc 2.35" + assert glibc == "glibc 2.28" assert platform.machine() == args.target.split("-", 1)[0] assert all( shutil.which(tool) is None @@ -34,11 +34,14 @@ def main() -> None: assert not Path("Cargo.toml").exists() assert not Path("target").exists() subprocess.run(["uname", "-a"], check=True) - packages = subprocess.check_output(["dpkg-query", "-W"], text=True) - print(packages, flush=True) - assert not any( - line.split()[0].split(":")[0].endswith("-dev") for line in packages.splitlines() + packages = subprocess.check_output( + ["rpm", "-qa", "--queryformat", "%{NAME}\t%{VERSION}-%{RELEASE}.%{ARCH}\n"], + text=True, ) + print(packages, end="", flush=True) + for line in packages.splitlines(): + name = line.split("\t", 1)[0] + assert not name.endswith("-devel"), f"development package installed: {name}" subprocess.run(["uv", "--version"], check=True) subprocess.run([sys.executable, "--version"], check=True) if args.with_r: @@ -101,7 +104,13 @@ def main() -> None: path in result.stdout for path in ("/work/", "/build/", "/usr/local/cargo/") ) subprocess.run( - [sys.executable, str(fixture / "sandbox_installation.py"), str(installed)], + [ + sys.executable, + str(fixture / "sandbox_installation.py"), + "--wheel", + str(args.wheel), + str(installed), + ], check=True, ) print( diff --git a/tests/release.py b/tests/release.py index 1f60cfd0a..9dc08bd2e 100755 --- a/tests/release.py +++ b/tests/release.py @@ -34,6 +34,16 @@ STAGE_SCRIPT = ROOT / "scripts" / "stage-sandbox-runner" +RELEASE_FIXTURES = { + "aarch64-apple-darwin": "macosx_11_0_arm64", + "x86_64-apple-darwin": "macosx_10_12_x86_64", + "aarch64-unknown-linux-gnu": "manylinux_2_28_aarch64", + "x86_64-unknown-linux-gnu": "manylinux_2_28_x86_64", + "aarch64-pc-windows-msvc": "win_arm64", + "x86_64-pc-windows-msvc": "win_amd64", +} + + def write_executable(path: Path, source: str) -> None: path.write_text(code(source), encoding="utf-8") path.chmod(0o755) @@ -57,7 +67,9 @@ def bubblewrap_notice( ) -def helper_metadata(pin: dict[str, object], helper: bytes) -> dict[str, object]: +def helper_metadata( + pin: dict[str, object], helper: bytes, *, dynamic: bool = True +) -> dict[str, object]: return { "source_repository": pin["repository"], "source_revision": pin["commit"], @@ -65,8 +77,8 @@ def helper_metadata(pin: dict[str, object], helper: bytes) -> dict[str, object]: "build_script": "codex-rs/bwrap/build.rs", "wrapper_directory": "codex-rs/bwrap", "sha256": hashlib.sha256(helper).hexdigest(), - "elf_needed": ["libc.so.6", "libcap.so.2"], - "libcap_linkage": "dynamic", + "elf_needed": ["libc.so.6", *(["libcap.so.2"] if dynamic else [])], + "libcap_linkage": "dynamic" if dynamic else "static", } @@ -75,7 +87,7 @@ def elf_fixture(**changes: object) -> bytes: "machine": "Advanced Micro Devices X86-64", "interpreter": "/lib64/ld-linux-x86-64.so.2", "needed": ["libc.so.6"], - "versions": ["GLIBC_2.34"], + "versions": ["GLIBC_2.28"], "rpath": None, "runpath": None, } @@ -267,6 +279,118 @@ def test_validate_publish_rejects_successful_pull_request_ci(self) -> None: self.assertNotEqual(result.returncode, 0) self.assertIn("lacks successful CI from a push to main", result.stderr) + def test_archive_publication_resumes_from_a_matching_draft(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + env = self.validation_environment(root) + env["FAKE_GH_RESPONSE"] = json.dumps( + { + "workflow_runs": [ + { + "head_sha": "release-commit", + "event": "push", + "head_branch": "main", + "conclusion": "success", + } + ] + } + ) + assets = root / "archives" + assets.mkdir() + names = [ + f"mcp-console-{arch}-{suffix}" + for arch in ("x86_64", "aarch64") + for suffix in ( + "pc-windows-msvc.zip", + "apple-darwin.tar.gz", + "unknown-linux-gnu.tar.gz", + ) + ] + one = names[0] + for name in names: + (assets / name).write_bytes(name.encode()) + initial = { + "isDraft": True, + "assets": [ + { + "name": one, + "digest": "sha256:" + hashlib.sha256(one.encode()).hexdigest(), + } + ], + } + state = root / "release.json" + state.write_text(json.dumps(initial)) + write_python_executable( + root / "commands/gh", + """ + import hashlib, json, os, sys + from pathlib import Path + state = Path("release.json") + data = json.loads(state.read_text()) + command = sys.argv[1:] + if command[:1] == ["api"] and "--paginate" in command: + print("v0.0.2") + elif command[:1] == ["api"]: + print(os.environ["FAKE_GH_RESPONSE"]) + elif command[:2] == ["release", "view"]: + print(json.dumps(data)) + elif command[:2] == ["release", "upload"]: + assert data["isDraft"] + for arg in command[3:]: + path = Path(arg) + assert path.name not in {a["name"] for a in data["assets"]} + data["assets"].append({"name": path.name, "digest": + "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest()}) + elif command == ["release", "edit", "v0.0.2", "--draft=false"]: + data["isDraft"] = False + else: + raise ValueError(command) + state.write_text(json.dumps(data)) + """, + ) + manifest = assets / "SHA256SUMS" + checksums = self.run_script("archive-checksums", "archives", cwd=root) + self.assertEqual(checksums.returncode, 0, checksums.stderr) + lines = manifest.read_bytes().splitlines(keepends=True) + for label, invalid in ( + ("truncated", lines[:1]), + ("duplicate", lines + lines[:1]), + ("wrong digest", [b"0" * 64 + lines[0][64:], *lines[1:]]), + ("wrong filename", [lines[0][:66] + b"other.zip\n", *lines[1:]]), + ): + with self.subTest(label): + manifest.write_bytes(b"".join(invalid)) + rejected = self.run_script( + "publish-archives", "archives", cwd=root, env=env + ) + self.assertNotEqual(rejected.returncode, 0) + self.assertIn( + "SHA256SUMS does not match the release archives", + rejected.stderr, + ) + self.assertEqual(json.loads(state.read_text()), initial) + (assets / one).write_text("different") + checksums = self.run_script("archive-checksums", "archives", cwd=root) + self.assertEqual(checksums.returncode, 0, checksums.stderr) + mismatch = self.run_script( + "publish-archives", "archives", cwd=root, env=env + ) + self.assertIn("GitHub asset digest differs", mismatch.stderr) + self.assertEqual(json.loads(state.read_text()), initial) + (assets / one).write_bytes(one.encode()) + manifest.write_bytes(b"".join(reversed(lines))) + for _ in range(2): + result = self.run_script( + "publish-archives", "archives", cwd=root, env=env + ) + self.assertEqual(result.returncode, 0, result.stderr) + final = json.loads(state.read_text()) + self.assertFalse(final["isDraft"]) + self.assertEqual( + {item["name"] for item in final["assets"]}, + {path.name for path in assets.iterdir()}, + ) + def test_validate_publish_rejects_manual_dispatch(self) -> None: with tempfile.TemporaryDirectory() as temporary_directory: directory = Path(temporary_directory) @@ -318,22 +442,30 @@ def test_linux_floor_checks_libraries_through_executable_entry_points(self) -> N original_mkdir = Path.mkdir def read_text(path: Path, *args, **kwargs) -> str: if path == Path("/etc/os-release"): - return "ID=ubuntu\\nVERSION_ID=22.04\\n" + return "ID=almalinux\\nVERSION_ID=8\\n" return original_read_text(path, *args, **kwargs) def mkdir(path: Path, *args, **kwargs) -> None: if path != Path("/evidence"): original_mkdir(path, *args, **kwargs) def check_output(command: list[str], **kwargs) -> str: if command == ["getconf", "GNU_LIBC_VERSION"]: - return "glibc 2.35\\n" + return "glibc 2.28\\n" if command[0] == "readelf": assert command[1:-1] == ["-l", "-W"] fields = json.loads(Path(command[-1]).read_bytes()[4:]) if fields["interpreter"]: return "[Requesting program interpreter: " + fields["interpreter"] + "]" return "" - assert command == ["dpkg-query", "-W"] - return "libc6 2.35\\n" + assert command[:3] == ["rpm", "-qa", "--queryformat"] + row = command[3] + for tag, value in { + "NAME": os.environ["FLOOR_RPM_NAME"], + "VERSION": "8.5.0", + "RELEASE": "28.el8", + "ARCH": "x86_64", + }.items(): + row = row.replace("%{" + tag + "}", value) + return row def run(command: list[str], **kwargs) -> subprocess.CompletedProcess[str]: output = "" if command[0] == "ldd": @@ -360,12 +492,13 @@ def run(command: list[str], **kwargs) -> subprocess.CompletedProcess[str]: runpy.run_path(sys.argv[0], run_name="__main__") """, ) - for output, accepted in ( - ("libc.so.6 => /lib/libc.so.6", True), - ("libmissing.so => not found", False), - ("libb.so => /build/libb.so", False), + for package, output, accepted in ( + ("libstdc++", "libc.so.6 => /lib/libc.so.6", True), + ("libstdc++-devel", "libc.so.6 => /lib/libc.so.6", False), + ("libstdc++", "libmissing.so => not found", False), + ("libstdc++", "libb.so => /build/libb.so", False), ): - with self.subTest(output=output): + with self.subTest(package=package, output=output): calls = directory / "loader-calls" calls.unlink(missing_ok=True) result = subprocess.run( @@ -378,6 +511,7 @@ def run(command: list[str], **kwargs) -> subprocess.CompletedProcess[str]: env=os.environ | { "UV_TOOL_BIN_DIR": str(prefix / "bin"), + "FLOOR_RPM_NAME": package, "LOADER_CALLS": str(calls), "EXTRA_LOADER_OUTPUT": output, }, @@ -385,6 +519,9 @@ def run(command: list[str], **kwargs) -> subprocess.CompletedProcess[str]: text=True, check=False, ) + self.assertIn( + f"{package}\t8.5.0-28.el8.x86_64", result.stdout.splitlines() + ) if accepted: self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual( @@ -398,8 +535,159 @@ def run(command: list[str], **kwargs) -> subprocess.CompletedProcess[str]: ) else: self.assertNotEqual(result.returncode, 0, result.stdout) + if package.endswith("-devel"): + self.assertIn( + f"development package installed: {package}", + result.stderr, + ) + self.assertFalse(calls.exists()) + + def linux_floor_bundle( + self, directory: Path, architecture: str + ) -> tuple[Path, Path]: + target = f"{architecture}-unknown-linux-gnu" + wheel = directory / f"mcp_console-0.0.2-py3-none-{RELEASE_FIXTURES[target]}.whl" + self.write_wheel(wheel) + with zipfile.ZipFile(wheel, "a") as archive: + archive.writestr("mcp_console-0.0.2.dist-info/licenses/LICENSE", "license") + archive.writestr( + "mcp_console-0.0.2.data/data/libfixture.so", + elf_fixture(interpreter=None), + ) + result = self.run_script( + "build-archive", + str(wheel), + "--target", + target, + "--out-dir", + ".", + cwd=directory, + ) + self.assertEqual(result.returncode, 0, result.stderr) + return wheel, directory / f"mcp-console-{target}.tar.gz" + + def assert_linux_floor_rejects( + self, wheel: Path, archive: Path, error: str + ) -> None: + evidence = wheel.parent / "evidence" + result = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts/check-linux-wheel"), + str(wheel), + "--archive", + str(archive), + "--target", + "x86_64-unknown-linux-gnu", + "--evidence", + str(evidence), + ], + env=os.environ | {"PATH": str(wheel.parent)}, + capture_output=True, + text=True, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn(error, result.stderr) + self.assertFalse(evidence.exists()) + + def test_linux_floor_requires_matching_bundle_files(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + wheel, archive = self.linux_floor_bundle(directory, "x86_64") + with tarfile.open(archive) as bundle: + files = { + member.name: (member.mode, bundle.extractfile(member).read()) + for member in bundle.getmembers() + } + library = "libfixture.so" + public = "bin/mcp-console" + public_mode, public_bytes = files[public] + library_mode, library_bytes = files[library] + changed = elf_fixture(build="other") + for error, entries in ( + ( + f"archive {public} differs", + (files | {public: (public_mode, changed)}).items(), + ), + ( + f"archive {library} differs", + (files | {library: (library_mode, changed)}).items(), + ), + ( + f"archive {public} has mode 0700, expected 0755", + (files | {public: (0o700, public_bytes)}).items(), + ), + ( + f"archive {library} has mode 0777, expected {library_mode:04o}", + (files | {library: (0o777, library_bytes)}).items(), + ), + ( + f"missing archive member: {library}", + [(name, data) for name, data in files.items() if name != library], + ), + ( + "unexpected archive member: extra.so", + (files | {"extra.so": (0o644, changed)}).items(), + ), + ( + "unexpected archive member: bin", + (files | {"bin/": (0o700, b"")}).items(), + ), + ( + f"unexpected archive member: {library}", + [*files.items(), (library, files[library])], + ), + ( + f"archive {library} must be a regular file", + (files | {library: (library_mode, None)}).items(), + ), + ): + with self.subTest(error=error): + with tarfile.open(archive, "w:gz") as bundle: + for name, (mode, contents) in entries: + member = tarfile.TarInfo(name) + member.mode = mode + if name.endswith("/"): + member.type = tarfile.DIRTYPE + bundle.addfile(member) + elif contents is None: + member.type = tarfile.SYMTYPE + member.linkname = public + bundle.addfile(member) + else: + member.size = len(contents) + bundle.addfile(member, io.BytesIO(contents)) + self.assert_linux_floor_rejects(wheel, archive, error) - def test_linux_floor_refreshes_expired_r_preparation(self) -> None: + def test_linux_floor_rejects_altered_archive_ownership(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + wheel, archive = self.linux_floor_bundle(directory, "x86_64") + altered = directory / "altered.tar.gz" + for field, value in ( + ("uid", 1000), + ("gid", 1000), + ("uname", "nobody"), + ("gname", "nogroup"), + ): + with self.subTest(field=field): + with ( + tarfile.open(archive) as source, + tarfile.open(altered, "w:gz") as dest, + ): + for member in source: + if member.name == "bin/mcp-console": + setattr(member, field, value) + dest.addfile(member, source.extractfile(member)) + self.assert_linux_floor_rejects( + wheel, + altered, + "archive bin/mcp-console must use UID and GID 0 with empty owner names", + ) + + def test_linux_floor_audits_before_standalone_and_refreshes_r_preparation( + self, + ) -> None: with tempfile.TemporaryDirectory() as temporary: directory = Path(temporary) commands = directory / "commands" @@ -441,6 +729,11 @@ def test_linux_floor_refreshes_expired_r_preparation(self) -> None: elif arguments[:2] == ["image", "inspect"]: print("[]") elif arguments[0] == "run": + with Path(os.environ["FLOOR_RUNS"]).open("a") as log: + log.write(json.dumps(arguments) + "\\n") + if arguments[-2] == "/test/standalone.sh": + target = os.environ["FLOOR_ARCHITECTURE"] + "-unknown-linux-gnu" + assert arguments[-1] == f"/test/mcp-console-{target}.tar.gz" if "--with-r" in arguments: state = json.loads(state_path.read_text()) if now - state["prepared"] >= 86400: @@ -453,9 +746,8 @@ def test_linux_floor_refreshes_expired_r_preparation(self) -> None: raise SystemExit(f"unexpected Docker command: {arguments}") """, ) - wheel = directory / "wheel.whl" - wheel.touch() for architecture in ("x86_64", "aarch64"): + wheel, archive = self.linux_floor_bundle(directory, architecture) environment = os.environ | { "PATH": f"{commands}{os.pathsep}{os.environ['PATH']}", "FLOOR_ARCHITECTURE": architecture, @@ -465,22 +757,38 @@ def test_linux_floor_refreshes_expired_r_preparation(self) -> None: # only the cached latest-package resolution has expired. for now in (0, 86401): with self.subTest(architecture=architecture, now=now): + runs = directory / f"{architecture}-{now}-runs.jsonl" result = subprocess.run( [ sys.executable, str(ROOT / "scripts/check-linux-wheel"), str(wheel), + "--archive", + str(archive), "--target", f"{architecture}-unknown-linux-gnu", "--evidence", str(directory / architecture), ], - env=environment | {"FLOOR_NOW": str(now)}, + env=environment + | {"FLOOR_NOW": str(now), "FLOOR_RUNS": str(runs)}, capture_output=True, text=True, check=False, ) self.assertEqual(result.returncode, 0, result.stderr) + launches = [ + json.loads(line) for line in runs.read_text().splitlines() + ] + # The wheel runtime audits the release ABI and source pin + # before its first launch. Standalone goes straight to exec. + self.assertIn("/test/linux_wheel_runtime.py", launches[0]) + self.assertTrue( + any( + "/test/standalone.sh" in launch + for launch in launches[1:] + ) + ) def smoke_environment(self, directory: Path) -> tuple[dict[str, str], Path, Path]: (directory / "Cargo.toml").write_text( @@ -709,18 +1017,29 @@ def write_wheel( *, omit: str | None = None, executable: bool = True, - glibc: str = "2.34", + glibc: str = "2.28", + dynamic_libcap: bool = False, + bwrap_runpath: str | None = None, ) -> None: version = wheel.name.split("-")[1] with zipfile.ZipFile(wheel, "w") as archive: linux = "linux" in wheel.name elf_fields = {"versions": [f"GLIBC_{glibc}"]} + helper_libraries = [ + "libc.so.6", + *(["libcap.so.2"] if dynamic_libcap else []), + ] if linux and "aarch64" in wheel.name: elf_fields.update( machine="AArch64", interpreter="/lib/ld-linux-aarch64.so.1" ) + bwrap = elf_fixture( + **elf_fields, needed=helper_libraries, runpath=bwrap_runpath + ) + console = zipfile.ZipInfo(f"mcp_console-{version}.data/scripts/mcp-console") + console.external_attr = (stat.S_IFREG | 0o755) << 16 archive.writestr( - f"mcp_console-{version}.data/scripts/mcp-console", + console, elf_fixture(**elf_fields) if linux else b"fixture\n", ) archive.writestr( @@ -739,6 +1058,7 @@ def write_wheel( "bubblewrap-COPYING", "bubblewrap-NOTICE", "bubblewrap-SOURCE.json", + *([] if dynamic_libcap else ["libcap-NOTICE"]), ) for name in names: if name == omit: @@ -755,12 +1075,7 @@ def write_wheel( info.external_attr = (stat.S_IFREG | mode) << 16 contents: str | bytes = "fixture\n" if linux and name in ("mcp-console-sandbox", "bwrap"): - contents = elf_fixture( - **elf_fields, - needed=["libc.so.6", "libcap.so.2"] - if name == "bwrap" - else ["libc.so.6"], - ) + contents = bwrap if name == "bwrap" else elf_fixture(**elf_fields) if name == "bubblewrap-NOTICE": contents = bubblewrap_notice( json.loads((ROOT / "sandbox-runner.json").read_text()) @@ -768,12 +1083,7 @@ def write_wheel( if name == "bubblewrap-SOURCE.json": pin = json.loads((ROOT / "sandbox-runner.json").read_text()) contents = json.dumps( - helper_metadata( - pin, - elf_fixture( - **elf_fields, needed=["libc.so.6", "libcap.so.2"] - ), - ) + helper_metadata(pin, bwrap, dynamic=dynamic_libcap) ) archive.writestr(info, contents) @@ -781,7 +1091,7 @@ def test_inspect_wheel_reports_every_shipped_elf_and_matching_tags(self) -> None with tempfile.TemporaryDirectory() as temporary: directory = Path(temporary) environment, _, _ = self.smoke_environment(directory) - wheel = directory / "mcp_console-0.0.2-py3-none-manylinux_2_35_x86_64.whl" + wheel = directory / "mcp_console-0.0.2-py3-none-manylinux_2_28_x86_64.whl" self.write_wheel(wheel) with zipfile.ZipFile(wheel, "a") as archive: archive.writestr("mcp_console/extra.so", elf_fixture(interpreter=None)) @@ -799,19 +1109,38 @@ def test_inspect_wheel_reports_every_shipped_elf_and_matching_tags(self) -> None ) self.assertEqual(result.returncode, 0, result.stderr) evidence = json.loads(report.read_text()) - self.assertEqual(evidence["tags"], ["py3-none-manylinux_2_35_x86_64"]) + self.assertEqual(evidence["tags"], ["py3-none-manylinux_2_28_x86_64"]) self.assertEqual(len(evidence["elf"]), 4) - self.assertEqual(evidence["elf"]["mcp_console/extra.so"]["glibc"], ["2.34"]) - self.assertIn( + self.assertEqual(evidence["elf"]["mcp_console/extra.so"]["glibc"], ["2.28"]) + self.assertNotIn( "libcap.so.2", evidence["elf"]["mcp_console-0.0.2.data/data/libexec/bwrap"]["needed"], ) + def test_release_rejects_bwrap_linked_to_bundled_libcap(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + environment, _, _ = self.smoke_environment(directory) + wheel = directory / "mcp_console-0.0.2-py3-none-manylinux_2_28_x86_64.whl" + self.write_wheel(wheel, dynamic_libcap=True, bwrap_runpath="$ORIGIN") + with zipfile.ZipFile(wheel, "a") as archive: + archive.writestr( + "mcp_console-0.0.2.data/data/libexec/libcap.so.2", + elf_fixture(interpreter=None), + ) + result = self.run_script( + "inspect-wheel", str(wheel), "--release", cwd=directory, env=environment + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn( + "release Bubblewrap must link libcap statically", result.stderr + ) + def test_inspect_wheel_resolves_bundled_libraries_after_installation(self) -> None: with tempfile.TemporaryDirectory() as temporary: directory = Path(temporary) environment, _, _ = self.smoke_environment(directory) - wheel = directory / "mcp_console-0.0.2-py3-none-manylinux_2_35_x86_64.whl" + wheel = directory / "mcp_console-0.0.2-py3-none-manylinux_2_28_x86_64.whl" for member, runpath, library_directory in ( ( "mcp_console-0.0.2.data/scripts/mcp-console", @@ -871,11 +1200,64 @@ def test_inspect_wheel_resolves_bundled_libraries_after_installation(self) -> No evidence = json.loads(report.read_text())["elf"] self.assertEqual(evidence[member]["search"], [library_directory]) + def test_archive_preserves_resolved_native_dependencies(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + environment, _, _ = self.smoke_environment(directory) + target = "x86_64-unknown-linux-gnu" + wheel = directory / "mcp_console-0.0.2-py3-none-manylinux_2_28_x86_64.whl" + self.write_wheel(wheel) + data = "mcp_console-0.0.2.data" + runner = elf_fixture(needed=["libfixture.so"], runpath="$ORIGIN/..") + library = elf_fixture( + interpreter=None, needed=["libchild.so"], runpath="$ORIGIN/bin" + ) + child = elf_fixture(interpreter=None) + rewrite_wheel(wheel, {f"{data}/data/libexec/mcp-console-sandbox": runner}) + with zipfile.ZipFile(wheel, "a") as archive: + archive.writestr(f"{data}/data/libfixture.so", library) + archive.writestr(f"{data}/data/libfixture-NOTICE", b"fixture license") + archive.writestr(f"{data}/scripts/libchild.so", child) + archive.writestr("mcp_console-0.0.2.dist-info/licenses/LICENSE", "MIT") + for args in ( + ("inspect-wheel", str(wheel), "--release"), + ("build-archive", str(wheel), "--target", target, "--out-dir", "."), + ): + result = self.run_script(*args, cwd=directory, env=environment) + self.assertEqual(result.returncode, 0, result.stderr) + with tarfile.open(directory / f"mcp-console-{target}.tar.gz") as bundle: + self.assertEqual(bundle.extractfile("libfixture.so").read(), library) + self.assertEqual( + bundle.extractfile("libfixture-NOTICE").read(), b"fixture license" + ) + self.assertEqual(bundle.extractfile("bin/libchild.so").read(), child) + + def test_release_native_commands_cannot_depend_on_site_packages(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + environment, _, _ = self.smoke_environment(directory) + wheel = directory / "mcp_console-0.0.2-py3-none-manylinux_2_28_x86_64.whl" + self.write_wheel(wheel) + runner = "mcp_console-0.0.2.data/data/libexec/mcp-console-sandbox" + executable = elf_fixture( + needed=["libfixture.so"], + runpath="$ORIGIN/../lib/pythonX.Y/site-packages", + ) + rewrite_wheel(wheel, {runner: executable}) + with zipfile.ZipFile(wheel, "a") as archive: + archive.writestr("libfixture.so", elf_fixture(interpreter=None)) + result = self.run_script( + "inspect-wheel", str(wheel), "--release", cwd=directory, env=environment + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("libfixture.so", result.stderr) + self.assertIn("Python install scheme", result.stderr) + def test_inspect_wheel_rejects_relative_dependency_filenames(self) -> None: with tempfile.TemporaryDirectory() as temporary: directory = Path(temporary) environment, _, _ = self.smoke_environment(directory) - wheel = directory / "mcp_console-0.0.2-py3-none-manylinux_2_35_x86_64.whl" + wheel = directory / "mcp_console-0.0.2-py3-none-manylinux_2_28_x86_64.whl" executable = "mcp_console-0.0.2.data/scripts/mcp-console" for kind in ("rpath", "runpath"): for needed in ( @@ -921,7 +1303,7 @@ def test_inspect_wheel_inherits_rpath_only_along_dependency_chains(self) -> None with tempfile.TemporaryDirectory() as temporary: directory = Path(temporary) environment, _, _ = self.smoke_environment(directory) - wheel = directory / "mcp_console-0.0.2-py3-none-manylinux_2_35_x86_64.whl" + wheel = directory / "mcp_console-0.0.2-py3-none-manylinux_2_28_x86_64.whl" executable = "mcp_console-0.0.2.data/scripts/mcp-console" libexec = "mcp_console-0.0.2.data/data/libexec" for name, root_paths, child_paths, accepted in ( @@ -1036,7 +1418,7 @@ def test_inspect_wheel_reuses_loaded_dependencies_per_entry_point(self) -> None: with tempfile.TemporaryDirectory() as temporary: directory = Path(temporary) environment, _, _ = self.smoke_environment(directory) - wheel = directory / "mcp_console-0.0.2-py3-none-manylinux_2_35_x86_64.whl" + wheel = directory / "mcp_console-0.0.2-py3-none-manylinux_2_28_x86_64.whl" executable = "mcp_console-0.0.2.data/scripts/mcp-console" libexec = "mcp_console-0.0.2.data/data/libexec" for name, needed, libraries, separate_executable, accepted in ( @@ -1128,7 +1510,7 @@ def test_inspect_wheel_rejects_loader_paths_outside_installed_prefix(self) -> No with tempfile.TemporaryDirectory() as temporary: directory = Path(temporary) environment, _, _ = self.smoke_environment(directory) - wheel = directory / "mcp_console-0.0.2-py3-none-manylinux_2_35_x86_64.whl" + wheel = directory / "mcp_console-0.0.2-py3-none-manylinux_2_28_x86_64.whl" for member, runpath in ( ("mcp_console-0.0.2.data/scripts/mcp-console", "$ORIGIN/../../outside"), ( @@ -1152,18 +1534,19 @@ def test_inspect_wheel_rejects_loader_paths_outside_installed_prefix(self) -> No def test_inspect_wheel_rejects_incompatible_private_and_extra_elf(self) -> None: defects = ( - ({"versions": ["GLIBC_2.36"]}, "GLIBC_2.36"), - ({"versions": ["GLIBCXX_3.4.31"]}, "GLIBCXX_3.4.31"), - ({"versions": ["CXXABI_1.3.14"]}, "CXXABI_1.3.14"), + ({"versions": ["GLIBC_2.29"]}, "GLIBC_2.29"), + ({"versions": ["GLIBCXX_3.4.25"]}, "GLIBCXX_3.4.25"), + ({"versions": ["CXXABI_1.3.12"]}, "CXXABI_1.3.12"), ({"machine": "AArch64"}, "machine"), ({"interpreter": "/opt/builder/ld-linux.so"}, "interpreter"), ({"runpath": "/opt/builder/lib"}, "RUNPATH"), ({"needed": ["libssl.so.3"]}, "undeclared dependency"), + ({"needed": ["libcap.so.2"]}, "undeclared dependency"), ) with tempfile.TemporaryDirectory() as temporary: directory = Path(temporary) environment, _, _ = self.smoke_environment(directory) - wheel = directory / "mcp_console-0.0.2-py3-none-manylinux_2_35_x86_64.whl" + wheel = directory / "mcp_console-0.0.2-py3-none-manylinux_2_28_x86_64.whl" for member in ( "mcp_console-0.0.2.data/data/libexec/mcp-console-sandbox", "mcp_console/extra.so", @@ -1187,6 +1570,27 @@ def test_inspect_wheel_rejects_incompatible_private_and_extra_elf(self) -> None: self.assertIn(diagnostic, result.stderr) self.assertIn(member, result.stderr) + def test_release_rejects_ubuntu_22_04_wheels(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + environment, _, _ = self.smoke_environment(directory) + for arch in ("x86_64", "aarch64"): + with self.subTest(arch=arch): + wheel = ( + directory + / f"mcp_console-0.0.2-py3-none-manylinux_2_35_{arch}.whl" + ) + self.write_wheel(wheel) + result = self.run_script( + "inspect-wheel", + str(wheel), + "--release", + cwd=directory, + env=environment, + ) + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertIn("glibc 2.28 release floor", result.stderr) + def test_inspect_wheel_rejects_false_or_inconsistent_tags(self) -> None: with tempfile.TemporaryDirectory() as temporary: directory = Path(temporary) @@ -1194,17 +1598,17 @@ def test_inspect_wheel_rejects_false_or_inconsistent_tags(self) -> None: for platform, metadata_tag, diagnostic in ( ("manylinux_2_28_x86_64", None, "GLIBC_2.34"), ("manylinux_2_39_x86_64", None, "release floor"), - ("manylinux_2_33_x86_64", None, "unsupported manylinux C++ policy"), + ("manylinux_2_25_x86_64", None, "unsupported manylinux C++ policy"), ( "manylinux_2_35_x86_64", "py3-none-manylinux_2_35_aarch64", "WHEEL tags", ), - ("manylinux_2_35_x86_64.manylinux_2_28_x86_64", None, "GLIBC_2.34"), + ("manylinux_2_28_x86_64.manylinux_2_24_x86_64", None, "GLIBC_2.34"), ): with self.subTest(platform=platform, metadata=metadata_tag): wheel = directory / f"mcp_console-0.0.2-py3-none-{platform}.whl" - self.write_wheel(wheel) + self.write_wheel(wheel, glibc="2.34") if metadata_tag: rewrite_wheel( wheel, @@ -1233,6 +1637,7 @@ def test_inspect_wheel_checks_cpp_symbols_against_every_advertised_policy( 22 if architecture == "x86_64" else 24, 10 if architecture == "x86_64" else 11, ), + ("manylinux_2_28", 24, 11), ("manylinux_2_31", 28, 12), ("manylinux_2_34", 29, 13), ("manylinux_2_35", 30, 13), @@ -1281,7 +1686,14 @@ def test_inspect_wheel_checks_cpp_symbols_against_every_advertised_policy( cwd=directory, env=environment, ) - if release and policy == "manylinux_2_39": + if release and policy.startswith( + ( + "manylinux_2_31", + "manylinux_2_34", + "manylinux_2_35", + "manylinux_2_39", + ) + ): self.assertNotEqual( result.returncode, 0, result.stdout ) @@ -1303,6 +1715,7 @@ def test_wheel_commands_check_named_glibc_against_every_advertised_policy( for architecture in ("x86_64", "aarch64"): arm = architecture == "aarch64" cases = [ + ("manylinux_2_28", "ABI_DT_RELR", False), ("manylinux_2_35", "ABI_DT_RELR", False), ("manylinux_2_36", "ABI_DT_RELR", True), ("manylinux_2_39", "ABI_DT_RELR", True), @@ -1354,7 +1767,7 @@ def test_wheel_commands_check_named_glibc_against_every_advertised_policy( cwd=directory, env=environment, ) - if release and policy != "manylinux_2_35": + if release and policy != "manylinux_2_28": self.assertNotEqual(result.returncode, 0, result.stdout) self.assertIn("release floor", result.stderr) elif not permitted: @@ -1391,12 +1804,12 @@ def test_inspect_wheel_checks_named_cxxabi_against_every_advertised_policy( ("manylinux2014", {"TM_1"}), ("manylinux_2_24", named), ("manylinux_2_34", named), - ("manylinux_2_35", named), - ("manylinux_2_35.manylinux2014", {"TM_1"}), + ("manylinux_2_28", named), + ("manylinux_2_28.manylinux2014", {"TM_1"}), ] if not arm: policies[:0] = [("manylinux2010", set())] - policies.append(("manylinux_2_35.manylinux2010", set())) + policies.append(("manylinux_2_28.manylinux2010", set())) with tempfile.TemporaryDirectory() as temporary: directory = Path(temporary) environment, _, _ = self.smoke_environment(directory) @@ -1406,7 +1819,7 @@ def test_inspect_wheel_checks_named_cxxabi_against_every_advertised_policy( ) wheel = directory / f"mcp_console-0.0.2-py3-none-{platform}.whl" names = ["TM_1", "FLOAT128"] - if policy == "manylinux_2_35": + if policy == "manylinux_2_28": names.append("UNKNOWN") for number in names: self.write_wheel(wheel, glibc="2.17" if arm else "2.5") @@ -1442,7 +1855,10 @@ def test_inspect_wheel_checks_named_cxxabi_against_every_advertised_policy( cwd=directory, env=environment, ) - if number not in permitted: + if release and policy == "manylinux_2_34": + self.assertNotEqual(result.returncode, 0) + self.assertIn("release floor", result.stderr) + elif number not in permitted: self.assertNotEqual( result.returncode, 0, result.stdout ) @@ -1515,7 +1931,14 @@ def test_inspect_wheel_checks_gcc_symbols_against_every_advertised_policy( }, ) for release in (False, True): - if release and policy == "manylinux_2_39": + if release and policy.startswith( + ( + "manylinux_2_31", + "manylinux_2_34", + "manylinux_2_35", + "manylinux_2_39", + ) + ): continue with self.subTest( platform=platform, number=number, release=release @@ -2019,7 +2442,7 @@ def test_smoke_linux_wheel_requires_sandbox_and_bundled_helper(self) -> None: linux_wheel = wheel.with_name( "mcp_console-0.0.2-py3-none-manylinux_2_34_x86_64.whl" ) - self.write_wheel(linux_wheel) + self.write_wheel(linux_wheel, dynamic_libcap=True) wheel.unlink() record = directory / "arguments.jsonl" environment["FAKE_MCP_ARGUMENTS"] = str(record) @@ -2048,7 +2471,7 @@ def test_smoke_linux_wheel_requires_sandbox_and_bundled_helper(self) -> None: "bubblewrap-SOURCE.json", ): with self.subTest(missing=missing): - self.write_wheel(linux_wheel, omit=missing) + self.write_wheel(linux_wheel, omit=missing, dynamic_libcap=True) result = self.run_script(*command, cwd=directory, env=environment) self.assertNotEqual(result.returncode, 0) self.assertIn("private sandbox runner", result.stderr) @@ -2064,7 +2487,7 @@ def test_smoke_linux_wheel_requires_sandbox_and_bundled_helper(self) -> None: ("libcap_linkage", "static"), ): with self.subTest(field=field): - self.write_wheel(linux_wheel) + self.write_wheel(linux_wheel, dynamic_libcap=True) with zipfile.ZipFile(linux_wheel) as archive: entries = [ (info, archive.read(info)) for info in archive.infolist() @@ -2085,7 +2508,7 @@ def test_smoke_linux_wheel_requires_sandbox_and_bundled_helper(self) -> None: ("bubblewrap-NOTICE", b""), ("bubblewrap-COPYING", b""), ): - self.write_wheel(linux_wheel) + self.write_wheel(linux_wheel, dynamic_libcap=True) with zipfile.ZipFile(linux_wheel) as archive: entries = [ (info, archive.read(info)) for info in archive.infolist() @@ -2100,7 +2523,7 @@ def test_smoke_linux_wheel_requires_sandbox_and_bundled_helper(self) -> None: self.assertIn(notice_name, result.stderr) for missing_notice in (True, False): - self.write_wheel(linux_wheel) + self.write_wheel(linux_wheel, dynamic_libcap=True) with zipfile.ZipFile(linux_wheel) as archive: entries = [ (info, archive.read(info)) for info in archive.infolist() @@ -2671,6 +3094,95 @@ def test_cargo_rejects_changed_staged_artifacts(self) -> None: class PortableReleaseTests(ReleaseFixture): + @unittest.skipUnless(sys.platform == "darwin", "requires macOS Mach-O tools") + def test_macos_archive_requires_the_target_slice_in_each_executable(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source = root / "main.c" + source.write_text("int main(void) { return 0; }\n") + for arch in ("arm64", "x86_64"): + triple = f"{arch}-apple-macos11" + subprocess.run( + ["cc", "-target", triple, source, "-o", root / arch], check=True + ) + launcher = "bin/mcp-console" + helper = "libexec/mcp-console-sandbox" + archive = root / "mixed.tar.gz" + with tarfile.open(archive, "w:gz") as bundle: + bundle.add(root / "arm64", arcname=launcher) + bundle.add(root / "x86_64", arcname=helper) + command = [ + sys.executable, + ROOT / "tests/standalone.py", + archive, + "--target", + ] + for target, wrong_member, missing in ( + ("aarch64-apple-darwin", helper, "arm64"), + ("x86_64-apple-darwin", launcher, "x86_64"), + ): + with self.subTest(target=target, member=wrong_member): + result = subprocess.run( + [*command, target], capture_output=True, text=True + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn( + f"{wrong_member} lacks the {missing} slice", result.stderr + ) + + @unittest.skipUnless(os.name == "nt", "native Windows installation checks") + def test_windows_install_finds_artifacts_relative_to_the_caller(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + # Both inputs open as ZIPs and reach their respective artifact audits. + fixtures = [ + ( + "--archive", + "bundle.zip", + "no native distribution artifacts inspected", + ), + ( + "--installed-binary", + "launcher.exe", + "missing installed launcher interpreter", + ), + ] + for option, name, diagnostic in fixtures: + with self.subTest(option=option): + with zipfile.ZipFile(directory / name, "w") as archive: + archive.writestr("__main__.py", "pass\n") + result = subprocess.run( + [ + sys.executable, + ROOT / "tests/windows_install.py", + option, + name, + ], + cwd=directory, + capture_output=True, + text=True, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn(diagnostic, result.stderr) + + def test_windows_source_install_rejects_preinstalled_r_before_building( + self, + ) -> None: + result = subprocess.run( + [ + sys.executable, + str(ROOT / "tests/windows_install.py"), + "--preinstalled-r", + ], + env=os.environ | {"PATH": ""}, + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 2, result.stderr) + self.assertIn( + "--preinstalled-r requires --archive or --installed-binary", result.stderr + ) + @contextmanager def publication_server( self, responses: dict[str, list[tuple[int | str, dict[str, object]]]] @@ -2716,13 +3228,7 @@ def publication_files(self, root: Path) -> list[dict[str, object]]: directory = root / "dist" directory.mkdir() files = [] - for tag in ( - "macosx_11_0_arm64", - "macosx_10_12_x86_64", - "manylinux_2_35_aarch64", - "manylinux_2_35_x86_64", - "win_amd64", - ): + for tag in RELEASE_FIXTURES.values(): filename = f"mcp_console-0.0.2-py3-none-{tag}.whl" payload = filename.encode() (directory / filename).write_bytes(payload) @@ -2737,6 +3243,30 @@ def publication_files(self, root: Path) -> list[dict[str, object]]: ) return files + def test_build_requires_unpublished_version(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + files = self.publication_files(root) + for status in (404, 200, 503): + with ( + self.subTest(status=status), + self.publication_server( + { + "/pypi/mcp-console/0.0.2/json": [ + (status, {"urls": files[:1]}) + ] + } + ) as (url, _), + ): + result = self.run_script( + "check-unpublished", "--pypi-url", url, cwd=root + ) + self.assertEqual( + result.returncode == 0, status == 404, result.stderr + ) + if status == 200: + self.assertIn("already on PyPI", result.stderr) + def test_wait_pypi_waits_for_version_json_and_simple_index(self) -> None: with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) @@ -2754,27 +3284,32 @@ def test_wait_pypi_waits_for_version_json_and_simple_index(self) -> None: ], } ) as (url, observed): - result = self.run_script( - "wait-pypi", - "dist", - "--pypi-url", - url, - "--timeout-seconds", - "5", - "--poll-interval-seconds", - "0.01", - "--report", - "publication.json", - cwd=root, - ) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn(("/simple/mcp-console/", {"files": files}), observed) - report = json.loads((root / "publication.json").read_text()) + for _ in range(2): + observed.clear() + result = self.run_script( + "wait-pypi", + "dist", + "--pypi-url", + url, + "--timeout-seconds", + "5", + "--poll-interval-seconds", + "0.01", + "--report", + "dist/publication.json", + cwd=root, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn(("/simple/mcp-console/", {"files": files}), observed) + report = json.loads((root / "dist/publication.json").read_text()) self.assertEqual(report["version"], "0.0.2") self.assertEqual( report["wheels"], {item["filename"]: item["digests"]["sha256"] for item in files}, ) + result = self.run_script("verify-wheel-set", "dist", cwd=root) + self.assertNotEqual(result.returncode, 0) + self.assertIn("only release wheels may be published", result.stderr) def test_wait_pypi_retries_temporary_endpoint_failures(self) -> None: with tempfile.TemporaryDirectory() as temporary: @@ -2808,7 +3343,7 @@ def test_wait_pypi_retries_temporary_endpoint_failures(self) -> None: ) self.assertEqual(result.returncode, 0, result.stderr) self.assertIn(("/simple/mcp-console/", {"files": files}), observed) - self.assertEqual(len(json.loads(result.stdout)["wheels"]), 5) + self.assertEqual(len(json.loads(result.stdout)["wheels"]), 6) def test_wait_pypi_rejects_forbidden_endpoint(self) -> None: with tempfile.TemporaryDirectory() as temporary: @@ -3113,41 +3648,144 @@ def git(*arguments: str, **options): + hashlib.sha256(archive.read_bytes()).hexdigest(), ) - def test_verify_wheel_set_requires_macos_linux_and_windows_architectures( - self, - ) -> None: - with tempfile.TemporaryDirectory() as temporary_directory: - directory = Path(temporary_directory) - arm64 = directory / "mcp_console-0.0.2-py3-none-macosx_11_0_arm64.whl" - x86_64 = directory / "mcp_console-0.0.2-py3-none-macosx_11_0_x86_64.whl" - arm64.touch() - x86_64.touch() - for architecture in ("aarch64", "x86_64"): - ( - directory - / f"mcp_console-0.0.2-py3-none-manylinux_2_39_{architecture}.whl" - ).touch() - - result = self.run_script("verify-wheel-set", str(directory), cwd=ROOT) + def test_verify_wheel_set_requires_all_six_platforms(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + self.publication_files(root) + directory = root / "dist" + result = self.run_script("verify-wheel-set", "dist", cwd=root) + self.assertEqual(result.returncode, 0, result.stderr) + arm = directory / "mcp_console-0.0.2-py3-none-win_arm64.whl" + arm.unlink() + result = self.run_script("verify-wheel-set", "dist", cwd=root) + self.assertNotEqual(result.returncode, 0) + self.assertIn("six wheels", result.stderr) + arm.with_name("mcp_console-0.0.2-py3-none-any.whl").touch() + result = self.run_script("verify-wheel-set", "dist", cwd=root) self.assertNotEqual(result.returncode, 0) - self.assertIn("expected exactly five wheels", result.stderr) + self.assertIn("aarch64-pc-windows-msvc", result.stderr) - windows = directory / "mcp_console-0.0.2-py3-none-win_amd64.whl" - windows.touch() - result = self.run_script("verify-wheel-set", str(directory), cwd=ROOT) - self.assertEqual(result.returncode, 0, result.stderr) + def test_release_archives_copy_native_wheel_files_and_licenses(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + sums = {} + for target, tag in RELEASE_FIXTURES.items(): + with self.subTest(target=target): + windows = "windows" in target + suffix = ".exe" if windows else "" + binary = f"bin/mcp-console{suffix}" + native = ( + "data/libexec/mcp-console.exe" + if windows + else "scripts/mcp-console" + ) + licenses = "share/licenses/mcp-console" + inputs = { + binary: f"mcp_console-0.0.2.data/{native}", + f"{licenses}/mcp-console-LICENSE": "mcp_console-0.0.2.dist-info/licenses/LICENSE", + } + private = [ + f"libexec/mcp-console-sandbox{suffix}", + f"{licenses}/LICENSE", + f"{licenses}/NOTICE", + ] + if "linux" in target: + private += [ + "libexec/bwrap", + *[ + f"{licenses}/{name}" + for name in ( + "bubblewrap-COPYING", + "bubblewrap-NOTICE", + "bubblewrap-SOURCE.json", + "libcap-NOTICE", + ) + ], + ] + inputs.update( + { + name: f"mcp_console-0.0.2.data/data/{name}" + for name in private + } + ) + wheel = root / f"mcp_console-0.0.2-py3-none-{tag}.whl" + with zipfile.ZipFile(wheel, "w") as whl: + for path, src in inputs.items(): + info = zipfile.ZipInfo(src) + info.external_attr = ( + stat.S_IFREG + | ( + 0o755 + if path.startswith(("bin/", "libexec/")) + else 0o644 + ) + ) << 16 + whl.writestr(info, src.encode()) + whl.writestr( + "mcp_console-0.0.2.data/data/.gitignore", b"staging" + ) + whl.writestr("mcp_console/_launcher.py", b"launcher") + whl.writestr( + "mcp_console-0.0.2.dist-info/WHEEL", + f"Root-Is-Purelib: false\nTag: py3-none-{tag}\n", + ) + whl.writestr( + "mcp_console-0.0.2.dist-info/METADATA", + "Name: mcp-console\nVersion: 0.0.2\n", + ) - windows.rename( - windows.with_name("mcp_console-0.0.2-py3-none-win_arm64.whl") - ) - result = self.run_script("verify-wheel-set", str(directory), cwd=ROOT) - self.assertNotEqual(result.returncode, 0) - self.assertIn("expected exactly one x86-64 Windows wheel", result.stderr) + result = self.run_script( + "build-archive", + str(wheel), + "--target", + target, + "--out-dir", + "archives", + cwd=root, + ) + self.assertEqual(result.returncode, 0, result.stderr) + archive = ( + root + / "archives" + / f"mcp-console-{target}.{'zip' if windows else 'tar.gz'}" + ) + if windows: + with zipfile.ZipFile(archive) as zipped: + actual = { + item.filename: zipped.read(item) + for item in zipped.infolist() + if not item.is_dir() + } + else: + with tarfile.open(archive) as tar: + actual = { + item.name: tar.extractfile(item).read() + for item in tar + if item.isfile() + } + for path in inputs: + self.assertEqual( + bool(tar.getmember(path).mode & 0o111), + path.startswith(("bin/", "libexec/")), + ) + self.assertEqual( + actual, {path: src.encode() for path, src in inputs.items()} + ) + sums[archive.name] = hashlib.sha256( + archive.read_bytes() + ).hexdigest() - x86_64.unlink() - result = self.run_script("verify-wheel-set", str(directory), cwd=ROOT) + result = self.run_script("archive-checksums", "archives", cwd=root) + self.assertEqual(result.returncode, 0, result.stderr) + manifest = root / "archives" / "SHA256SUMS" + self.assertEqual( + manifest.read_text().splitlines(), + [f"{sums[name]} {name}" for name in sorted(sums)], + ) + archive.unlink() + result = self.run_script("archive-checksums", "archives", cwd=root) self.assertNotEqual(result.returncode, 0) - self.assertIn("expected exactly five wheels", result.stderr) + self.assertIn(archive.name, result.stderr) @unittest.skipUnless(os.name == "posix", "Unix Rscript executable fixture") diff --git a/tests/sandbox_installation.py b/tests/sandbox_installation.py index 39f88022d..24a9cabd8 100644 --- a/tests/sandbox_installation.py +++ b/tests/sandbox_installation.py @@ -10,28 +10,34 @@ import sys import tempfile import unittest +import zipfile from pathlib import Path from support.native import build_interposer +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) +from release import wheel_identity, wheel_prefix_path + @unittest.skipUnless(sys.platform in ("darwin", "linux"), "requires macOS or Linux") class SandboxInstallationTests(unittest.TestCase): binary_source: Path + bundle_sources: dict[str, Path] def setUp(self) -> None: temporary = tempfile.TemporaryDirectory(prefix="mcp-console-installation-") self.addCleanup(temporary.cleanup) self.root = Path(temporary.name) prefix = self.root / "installation" - (prefix / "bin").mkdir(parents=True) + for relative, source in self.bundle_sources.items(): + destination = prefix / relative + destination.parent.mkdir(parents=True, exist_ok=True) + if source.is_dir(): + shutil.copytree(source, destination) + else: + shutil.copy2(source, destination) self.binary = prefix / "bin" / "mcp-console" - shutil.copy2(self.binary_source, self.binary) - source_prefix = self.binary_source.parent.parent - for relative in ("libexec", "share/licenses/mcp-console"): - shutil.copytree(source_prefix / relative, prefix / relative) - self.home = self.root / "home" - self.home.mkdir() + self.console_home = self.root / "console-home" self.path = self.root / "path" self.path.mkdir() decoy = self.path / "mcp-console-sandbox" @@ -39,7 +45,10 @@ def setUp(self) -> None: "#!/bin/sh\nprintf 'PATH runner was used\\n'\n", encoding="utf-8" ) decoy.chmod(0o755) - self.environment = os.environ | {"PATH": str(self.path), "HOME": str(self.home)} + self.environment = os.environ | { + "PATH": str(self.path), + "MCP_CONSOLE_HOME": str(self.console_home), + } @property def runner(self) -> Path: @@ -127,12 +136,14 @@ def test_setup_is_one_shot_with_idle_stdin(self) -> None: for stream in (process.stdin, process.stdout, process.stderr): stream.close() - def test_relocates_the_bundle_without_a_writable_home_or_path_lookup(self) -> None: + def test_relocates_without_a_writable_console_home_or_path_lookup(self) -> None: relocated = self.root / "relocated" self.binary.parent.parent.rename(relocated) self.binary = relocated / "bin/mcp-console" - self.home.rmdir() - self.home.touch() + # A regular file makes writes beneath either home fail even as root. + self.console_home.touch() + self.environment.update(HOME=str(self.console_home), PATH="") + self.environment.pop("XDG_CACHE_HOME", None) result = self.run_sandbox() self.assertEqual( (result.returncode, result.stdout, result.stderr), (0, b"", b"") @@ -158,21 +169,26 @@ def test_rejects_a_non_executable_private_runner(self) -> None: def test_rejects_invalid_bundle_files(self) -> None: prefix = self.binary.parent.parent - for relative in ( + licenses = "share/licenses/mcp-console" + artifacts = [ "libexec/mcp-console-sandbox", - "share/licenses/mcp-console/LICENSE", - "share/licenses/mcp-console/NOTICE", - *( - ( + f"{licenses}/LICENSE", + f"{licenses}/NOTICE", + ] + if sys.platform == "linux": + provenance = f"{licenses}/bubblewrap-SOURCE.json" + artifacts.extend( + [ "libexec/bwrap", - "share/licenses/mcp-console/bubblewrap-COPYING", - "share/licenses/mcp-console/bubblewrap-NOTICE", - "share/licenses/mcp-console/bubblewrap-SOURCE.json", - ) - if sys.platform == "linux" - else () - ), - ): + f"{licenses}/bubblewrap-COPYING", + f"{licenses}/bubblewrap-NOTICE", + provenance, + ] + ) + linkage = json.loads((prefix / provenance).read_text())["libcap_linkage"] + if linkage == "static": + artifacts.append(f"{licenses}/libcap-NOTICE") + for relative in artifacts: artifact = prefix / relative original = artifact.read_bytes() for defect in ("missing", "modified", "fifo"): @@ -193,8 +209,13 @@ def test_rejects_invalid_bundle_files(self) -> None: else b"private sandbox runner", result.stderr, ) - if relative == "libexec/bwrap" and defect == "modified": - self.assertIn(b"digest mismatch", result.stderr) + if defect == "modified": + self.assertIn( + b"digest mismatch" + if relative == "libexec/bwrap" + else b"private artifact does not match this installation", + result.stderr, + ) if defect == "fifo" and relative != "libexec/bwrap": self.assertIn( b"private artifact is not a readable file or executable", @@ -208,11 +229,9 @@ def test_rejects_invalid_bundle_files(self) -> None: def test_unused_bundled_helper_does_not_block_a_suitable_host_helper(self) -> None: # Native discovery excludes helpers inside cwd. Supply a trusted helper # outside that tree independently of the relocated bundle we damage. - host = tempfile.TemporaryDirectory(prefix="mcp-console-host-helper-") - self.addCleanup(host.cleanup) - helper = Path(host.name) / "bwrap" - shutil.copy2(self.binary_source.parent.parent / "libexec/bwrap", helper) - (self.path / "bwrap").symlink_to(helper) + (self.path / "bwrap").symlink_to( + self.binary_source.parent.parent / "libexec/bwrap" + ) bundled = self.binary.parent.parent / "libexec/bwrap" for defect in ("missing", "modified", "fifo"): with self.subTest(defect=defect): @@ -241,8 +260,37 @@ def test_first_and_repeated_launches_use_bounded_allocations(self) -> None: if __name__ == "__main__": parser = argparse.ArgumentParser() + source = parser.add_mutually_exclusive_group() + source.add_argument( + "--standalone", action="store_true", help="copy the entire extracted prefix" + ) + source.add_argument( + "--wheel", type=Path, help="copy the wheel's installed native bundle" + ) parser.add_argument("binary", type=Path) parser.add_argument("tests", nargs="*", metavar="TEST") args = parser.parse_args() - SandboxInstallationTests.binary_source = args.binary.resolve() + binary = args.binary.resolve() + prefix = binary.parent.parent + SandboxInstallationTests.binary_source = binary + if args.standalone: + sources = {".": prefix} + elif args.wheel: + version, _ = wheel_identity(args.wheel) + data = f"mcp_console-{version}.data/" + with zipfile.ZipFile(args.wheel) as wheel: + sources = { + relative: prefix / relative + for member in wheel.infolist() + if not member.is_dir() + and (relative := wheel_prefix_path(member.filename, data)) is not None + } + else: + # A Cargo target directory also contains build products outside the bundle. + sources = { + "bin/mcp-console": binary, + "libexec": prefix / "libexec", + "share/licenses/mcp-console": prefix / "share/licenses/mcp-console", + } + SandboxInstallationTests.bundle_sources = sources unittest.main(argv=[sys.argv[0], *args.tests]) diff --git a/tests/snapshots/cli/relay/test_commands/forwards_worker_commands_and_keeps_controls_local.yaml b/tests/snapshots/cli/relay/test_commands/forwards_worker_commands_and_keeps_controls_local.yaml index d86505c06..6d2bf5177 100644 --- a/tests/snapshots/cli/relay/test_commands/forwards_worker_commands_and_keeps_controls_local.yaml +++ b/tests/snapshots/cli/relay/test_commands/forwards_worker_commands_and_keeps_controls_local.yaml @@ -43,6 +43,7 @@ forwarded: python: /native/bin/python libpython: /native/lib/libpython python_home: /native/home + platform: fixture-platform prefix: /native/prefix exec_prefix: /native/exec-prefix base_prefix: /native/base-prefix diff --git a/tests/snapshots/client_server/lifecycle/test_startup_interrupt/interrupts_first_cell_admitted_during_stdin_startup.yaml b/tests/snapshots/client_server/lifecycle/test_startup_interrupt/interrupts_first_cell_admitted_during_stdin_startup.yaml index a9660c460..6fd352c8a 100644 --- a/tests/snapshots/client_server/lifecycle/test_startup_interrupt/interrupts_first_cell_admitted_during_stdin_startup.yaml +++ b/tests/snapshots/client_server/lifecycle/test_startup_interrupt/interrupts_first_cell_admitted_during_stdin_startup.yaml @@ -1,6 +1,18 @@ --- !same-as MCP initialization for this execution mode --- +send: + session: default + timeout_ms: 100 +result: + content: + - type: text + text: |- + + [phase: dependency preparation] + [worker starting] + isError: false +--- send: stdin: | old input diff --git a/tests/snapshots/client_server/lifecycle/test_startup_interrupt/interrupts_first_cell_before_resolver_registration.yaml b/tests/snapshots/client_server/lifecycle/test_startup_interrupt/interrupts_first_cell_before_resolver_registration.yaml index b35e9dcaa..16d619094 100644 --- a/tests/snapshots/client_server/lifecycle/test_startup_interrupt/interrupts_first_cell_before_resolver_registration.yaml +++ b/tests/snapshots/client_server/lifecycle/test_startup_interrupt/interrupts_first_cell_before_resolver_registration.yaml @@ -1,6 +1,18 @@ --- !same-as MCP initialization for this execution mode --- +send: + session: default + timeout_ms: 100 +result: + content: + - type: text + text: |- + + [phase: dependency preparation] + [worker starting] + isError: false +--- send: r: startup_cell_ran <- TRUE timeout_ms: 0 diff --git a/tests/snapshots/client_server/lifecycle/test_startup_interrupt/interrupts_first_cell_between_resolver_phases.yaml b/tests/snapshots/client_server/lifecycle/test_startup_interrupt/interrupts_first_cell_between_resolver_phases.yaml index b35e9dcaa..16d619094 100644 --- a/tests/snapshots/client_server/lifecycle/test_startup_interrupt/interrupts_first_cell_between_resolver_phases.yaml +++ b/tests/snapshots/client_server/lifecycle/test_startup_interrupt/interrupts_first_cell_between_resolver_phases.yaml @@ -1,6 +1,18 @@ --- !same-as MCP initialization for this execution mode --- +send: + session: default + timeout_ms: 100 +result: + content: + - type: text + text: |- + + [phase: dependency preparation] + [worker starting] + isError: false +--- send: r: startup_cell_ran <- TRUE timeout_ms: 0 diff --git a/tests/snapshots/client_server/python/test_environment/rejects_python_older_than_3_10.win32.yaml b/tests/snapshots/client_server/python/test_environment/rejects_python_older_than_3_10.win32.yaml index 5a3edaf3b..d689c86c2 100644 --- a/tests/snapshots/client_server/python/test_environment/rejects_python_older_than_3_10.win32.yaml +++ b/tests/snapshots/client_server/python/test_environment/rejects_python_older_than_3_10.win32.yaml @@ -7,7 +7,7 @@ send: result: content: - type: text - text: "selected Python inspection failed (exit code: 1): Traceback (most recent call last):\r\n File \"\", line 145, in \r\n File \"\", line 19, in describe\r\nRuntimeError: MCP Console requires Python 3.10 or later\r\n" + text: "selected Python inspection failed (exit code: 1): Traceback (most recent call last):\r\n File \"\", line 146, in \r\n File \"\", line 19, in describe\r\nRuntimeError: MCP Console requires Python 3.10 or later\r\n" isError: true --- send: @@ -15,7 +15,7 @@ send: result: content: - type: text - text: "selected Python inspection failed (exit code: 1): Traceback (most recent call last):\r\n File \"\", line 145, in \r\n File \"\", line 19, in describe\r\nRuntimeError: MCP Console requires Python 3.10 or later\r\n" + text: "selected Python inspection failed (exit code: 1): Traceback (most recent call last):\r\n File \"\", line 146, in \r\n File \"\", line 19, in describe\r\nRuntimeError: MCP Console requires Python 3.10 or later\r\n" isError: true --- -stderr: "selected Python inspection failed (exit code: 1): Traceback (most recent call last):\r\n File \"\", line 145, in \r\n File \"\", line 19, in describe\r\nRuntimeError: MCP Console requires Python 3.10 or later\r\n\n" +stderr: "selected Python inspection failed (exit code: 1): Traceback (most recent call last):\r\n File \"\", line 146, in \r\n File \"\", line 19, in describe\r\nRuntimeError: MCP Console requires Python 3.10 or later\r\n\n" diff --git a/tests/snapshots/client_server/python/test_environment/rejects_python_older_than_3_10.yaml b/tests/snapshots/client_server/python/test_environment/rejects_python_older_than_3_10.yaml index d640038d5..bbd7b96bc 100644 --- a/tests/snapshots/client_server/python/test_environment/rejects_python_older_than_3_10.yaml +++ b/tests/snapshots/client_server/python/test_environment/rejects_python_older_than_3_10.yaml @@ -9,7 +9,7 @@ result: - type: text text: |- Error: selected Python inspection failed (exit status: 1): Traceback (most recent call last): - File "", line 145, in + File "", line 146, in File "", line 19, in describe RuntimeError: MCP Console requires Python 3.10 or later @@ -24,7 +24,7 @@ result: - type: text text: |+ Error: selected Python inspection failed (exit status: 1): Traceback (most recent call last): - File "", line 145, in + File "", line 146, in File "", line 19, in describe RuntimeError: MCP Console requires Python 3.10 or later diff --git a/tests/snapshots/client_server/python/test_peer_runtime/late_attachment_preserves_environment_metadata.yaml b/tests/snapshots/client_server/python/test_peer_runtime/late_attachment_preserves_environment_metadata.yaml index b946e747e..a67294853 100644 --- a/tests/snapshots/client_server/python/test_peer_runtime/late_attachment_preserves_environment_metadata.yaml +++ b/tests/snapshots/client_server/python/test_peer_runtime/late_attachment_preserves_environment_metadata.yaml @@ -2,7 +2,14 @@ environment: base --- send: - python: peer_object = object(); peer_identity = id(peer_object) + python: | + import sys + + if "sysconfig" in sys.modules: + del sys.modules["sysconfig"] + import sysconfig as project_sysconfig + + assert project_sysconfig.project_value == 42 result: content: - type: text @@ -15,6 +22,7 @@ send: sys <- reticulate::import("sys") stopifnot( identical(config$python, Sys.getenv("MCP_CONSOLE_TEST_PYTHON")), + identical(config$platform, Sys.getenv("MCP_CONSOLE_TEST_PYTHON_PLATFORM")), identical(normalizePath(config$prefix), normalizePath(sys$prefix)), identical(normalizePath(config$exec_prefix), normalizePath(sys$exec_prefix)), identical(normalizePath(config$base_prefix), normalizePath(sys$base_prefix)), @@ -42,7 +50,11 @@ result: isError: false --- send: - python: assert id(peer_object) == peer_identity; print('same interpreter') + python: | + import sysconfig + + assert sysconfig is project_sysconfig + print("same interpreter") result: content: - type: text @@ -53,7 +65,14 @@ result: environment: virtualenv --- send: - python: peer_object = object(); peer_identity = id(peer_object) + python: | + import sys + + if "sysconfig" in sys.modules: + del sys.modules["sysconfig"] + import sysconfig as project_sysconfig + + assert project_sysconfig.project_value == 42 result: content: - type: text @@ -66,6 +85,7 @@ send: sys <- reticulate::import("sys") stopifnot( identical(config$python, Sys.getenv("MCP_CONSOLE_TEST_PYTHON")), + identical(config$platform, Sys.getenv("MCP_CONSOLE_TEST_PYTHON_PLATFORM")), identical(normalizePath(config$prefix), normalizePath(sys$prefix)), identical(normalizePath(config$exec_prefix), normalizePath(sys$exec_prefix)), identical(normalizePath(config$base_prefix), normalizePath(sys$base_prefix)), @@ -93,7 +113,11 @@ result: isError: false --- send: - python: assert id(peer_object) == peer_identity; print('same interpreter') + python: | + import sysconfig + + assert sysconfig is project_sysconfig + print("same interpreter") result: content: - type: text @@ -104,7 +128,14 @@ result: environment: conda-marker --- send: - python: peer_object = object(); peer_identity = id(peer_object) + python: | + import sys + + if "sysconfig" in sys.modules: + del sys.modules["sysconfig"] + import sysconfig as project_sysconfig + + assert project_sysconfig.project_value == 42 result: content: - type: text @@ -117,6 +148,7 @@ send: sys <- reticulate::import("sys") stopifnot( identical(config$python, Sys.getenv("MCP_CONSOLE_TEST_PYTHON")), + identical(config$platform, Sys.getenv("MCP_CONSOLE_TEST_PYTHON_PLATFORM")), identical(normalizePath(config$prefix), normalizePath(sys$prefix)), identical(normalizePath(config$exec_prefix), normalizePath(sys$exec_prefix)), identical(normalizePath(config$base_prefix), normalizePath(sys$base_prefix)), @@ -144,7 +176,11 @@ result: isError: false --- send: - python: assert id(peer_object) == peer_identity; print('same interpreter') + python: | + import sysconfig + + assert sysconfig is project_sysconfig + print("same interpreter") result: content: - type: text diff --git a/tests/staging.py b/tests/staging.py index a8e463b38..20a5cb719 100644 --- a/tests/staging.py +++ b/tests/staging.py @@ -74,7 +74,9 @@ def setUp(self) -> None: stream.write(b"1") with open(os.environ["BUILD_RELEASE"], "rb", buffering=0) as stream: assert stream.read(1) == b"1" - binary = output / "aarch64-apple-darwin/release/mcp-console-sandbox" + target = sys.argv[sys.argv.index("--target") + 1] + name = "mcp-console-sandbox.exe" if "windows" in target else "mcp-console-sandbox" + binary = output / target / "release" / name binary.parent.mkdir(parents=True, exist_ok=True) binary.write_text(os.environ.get("RUSTFLAGS", "initial")) """, @@ -140,6 +142,22 @@ def describe(self, root: Path, **environment: str) -> dict: ) return json.loads(result.stdout) + def test_windows_arm64_companion_is_staged_for_the_requested_target(self) -> None: + root = self.roots[0] + target = "aarch64-pc-windows-msvc" + result = subprocess.run( + self.command(root, "--target", target), + env=self.environment, + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.describe(root)["staged_target"], target) + self.assertEqual( + (root / "wheel-data/data/libexec/mcp-console-sandbox.exe").read_text(), + "initial", + ) + def test_separate_checkouts_reuse_the_pinned_source_and_cargo_output(self) -> None: first, second = self.roots descriptions = [self.describe(root) for root in self.roots] diff --git a/tests/standalone.py b/tests/standalone.py new file mode 100644 index 000000000..47e46a7e7 --- /dev/null +++ b/tests/standalone.py @@ -0,0 +1,97 @@ +"""Accept a standalone Unix release archive after relocation.""" + +from __future__ import annotations + +import argparse +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile +import tomllib + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +from release import RELEASE_PLATFORMS, smoke_mcp + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("archive", type=Path) + parser.add_argument( + "--target", + required=True, + choices=[target for target in RELEASE_PLATFORMS if "windows" not in target], + ) + args = parser.parse_args() + version = tomllib.loads((ROOT / "Cargo.toml").read_text())["package"]["version"] + expected = re.escape(f"mcp-console {version}") + if version.endswith("+dev"): + expected += r"(?:\.g[0-9a-f]{12})?" + + with tempfile.TemporaryDirectory(prefix="mcp-console-archive-") as temporary: + root = Path(temporary) + unpacked = root / "unpacked" + with tarfile.open(args.archive) as archive: + archive.extractall(unpacked, filter="data") + prefix = root / "moved installation" + unpacked.rename(prefix) + if args.target.endswith("-apple-darwin"): + cpu = args.target.removesuffix("-apple-darwin") + arch = "arm64" if cpu == "aarch64" else cpu + for member in ("bin/mcp-console", "libexec/mcp-console-sandbox"): + result = subprocess.run(["lipo", prefix / member, "-verify_arch", arch]) + if result.returncode: + raise SystemExit( + f"{member} lacks the {arch} slice for {args.target}" + ) + binary = prefix / "bin/mcp-console" + empty_path = root / "empty" + empty_path.mkdir() + environment = os.environ | { + "PATH": str(empty_path), + "MCP_CONSOLE_HOME": str(root / "console-home"), + } + actual = subprocess.check_output( + [binary, "--version"], cwd=root, env=environment, text=True + ).strip() + assert re.fullmatch(expected, actual), actual + help_text = subprocess.check_output( + [binary, "--help"], cwd=root, env=environment, text=True + ) + assert "sandbox" in help_text + subprocess.run( + [binary, "sandbox", "--", "/usr/bin/true"], + cwd=root, + env=environment, + check=True, + ) + # Language preparation uses the host runtimes after native launch. + smoke_mcp( + binary, + actual.removeprefix("mcp-console "), + os.environ | {"MCP_CONSOLE_HOME": str(root / "console-home")}, + root, + 1200, + 30, + r_available=bool(os.environ.get("R_HOME") or shutil.which("R")), + ) + # The installation suite modifies each helper and notice in a disposable + # copy to exercise the digests embedded in this archive's public binary. + subprocess.run( + [ + sys.executable, + ROOT / "tests/sandbox_installation.py", + "--standalone", + binary, + "SandboxInstallationTests.test_rejects_invalid_bundle_files", + ], + check=True, + ) + + +if __name__ == "__main__": + main() diff --git a/tests/support/relay_commands.py b/tests/support/relay_commands.py index 4e285530a..f81c3ec13 100644 --- a/tests/support/relay_commands.py +++ b/tests/support/relay_commands.py @@ -34,6 +34,7 @@ "libpython": "/native/lib/libpython", "python_home": "/native/home", }, + "platform": "fixture-platform", "prefix": "/native/prefix", "exec_prefix": "/native/exec-prefix", "base_prefix": "/native/base-prefix", diff --git a/tests/support/windows_faults.py b/tests/support/windows_faults.py index 8125f7de8..bcc2475cc 100644 --- a/tests/support/windows_faults.py +++ b/tests/support/windows_faults.py @@ -2,7 +2,8 @@ Only the verified companion's private process memory is changed. DLLs on disk and the parent process remain untouched. -Windows x64 is the supported native acceptance target. +The injected failure fixture requires x64 Python and an x64 companion. +Hosts without MXC exercise the actual startup failure on both architectures. """ import ctypes @@ -11,8 +12,12 @@ from pathlib import Path import struct import subprocess +import sysconfig from threading import Thread import time +import unittest + +from windows_distribution import PE_MACHINE, pe_offset_and_machine def mcp_startup_failure(process: subprocess.Popen) -> tuple[bytes, bytes]: @@ -72,7 +77,13 @@ def request(identifier: int, method: str, params: dict) -> dict: def run_with_unavailable_mxc( command, *, inject_unavailable, cwd, env, input=b"", interaction=None, timeout=40 ): - assert ctypes.sizeof(ctypes.c_void_p) == 8, "native Windows acceptance requires x64" + assert ctypes.sizeof(ctypes.c_void_p) == 8, ( + "native Windows acceptance requires a 64-bit interpreter" + ) + if inject_unavailable and sysconfig.get_platform() != "win-amd64": + raise unittest.SkipTest( + "simulated unavailable MXC on a capable host requires the x64 fixture" + ) kernel = ctypes.WinDLL("kernel32", use_last_error=True) pointer = ctypes.c_void_p size = ctypes.c_size_t @@ -261,11 +272,21 @@ def event_step(*, injecting): if event.kind == 3: # CREATE_PROCESS_DEBUG_EVENT info = event.data.create try: - processes[event.pid] = ( - info.process, - Path(path_for(info.file)).name.lower(), - {}, - ) + image = Path(path_for(info.file)) + name = image.name.lower() + processes[event.pid] = (info.process, name, {}) + if ( + injecting + and inject_unavailable + and name == "mcp-console-sandbox.exe" + ): + # The launcher can select a companion of a different architecture. + with image.open("rb") as executable: + _, machine = pe_offset_and_machine(executable) + if machine != PE_MACHINE["win_amd64"]: + raise unittest.SkipTest( + "simulated unavailable MXC requires an x64 sandbox companion" + ) finally: close(info.file) elif event.kind == 6: # LOAD_DLL_DEBUG_EVENT diff --git a/tests/windows.py b/tests/windows.py index a9efaae30..e64bb67e3 100644 --- a/tests/windows.py +++ b/tests/windows.py @@ -17,6 +17,7 @@ import shutil import subprocess import sys +import sysconfig import tempfile import time from threading import Thread @@ -29,6 +30,7 @@ from support.installation import native_console from support.normalization import code from support.r import r_duckdb_snapshot_environment, r_test_environment +from support.windows_mxc import requires_mxc from windows_distribution import NATIVE_SUFFIXES, inspect_artifacts from windows_cargo import WindowsCargo # noqa: F401 -- include build acceptance @@ -49,6 +51,24 @@ NATIVE_BINARY = native_console(BINARY) +def preinstalled_r_environment() -> dict[str, str]: + # Keep the selected R installation when hiding interpreter and package launchers. + r_home = ( + os.environ.get("R_HOME") + or subprocess.check_output( + [shutil.which("R") or "R", "RHOME"], text=True + ).strip() + ) + environment = dict( + os.environ, + R_HOME=r_home, + PATH=str(Path(os.environ["SystemRoot"]) / "System32"), + ) + environment.pop("RETICULATE_PYTHON", None) + environment.pop("RETICULATE_UV", None) + return environment + + class Session: def __init__( self, @@ -584,7 +604,8 @@ def test_distribution_audit_checks_normal_and_delayed_imports(self): timeout=60, ) self.assertEqual(build.returncode, 0, build.stderr) - wheel = root / f"{kind}.whl" + platform = sysconfig.get_platform().replace("-", "_") + wheel = root / f"{kind}-1-py3-none-{platform}.whl" with zipfile.ZipFile(wheel, "w") as archive: archive.write( binary, f"fixture.data/data/libexec/{binary.name}" @@ -598,6 +619,22 @@ def test_distribution_audit_checks_normal_and_delayed_imports(self): if kind == "static": self.assertEqual(audit.returncode, 0, audit.stderr) self.assertIn("kernel32.dll", audit.stdout.lower()) + other = "win_arm64" if platform == "win_amd64" else "win_amd64" + mislabeled = wheel.rename( + root / f"static-1-py3-none-{other}.whl" + ) + audit = subprocess.run( + [ + sys.executable, + ROOT / "tests/windows_distribution.py", + mislabeled, + ], + capture_output=True, + text=True, + timeout=30, + ) + self.assertNotEqual(audit.returncode, 0) + self.assertIn(f"expected {other}", audit.stderr) else: self.assertNotEqual(audit.returncode, 0) self.assertIn(f"{kind} import VCRUNTIME140.dll", audit.stderr) @@ -1145,20 +1182,14 @@ def test_explicit_r_selections_use_windows_installation(self): session.close() def test_r_without_python(self): - # Capture R before removing the interpreter launchers from PATH. - r_home = ( - os.environ.get("R_HOME") - or subprocess.check_output( - [shutil.which("R") or "R", "RHOME"], text=True - ).strip() - ) - environment = dict( - os.environ, - R_HOME=r_home, - PATH=str(Path(os.environ["SystemRoot"]) / "System32"), + environment = preinstalled_r_environment() + session = Session( + environment, + bare_r=True, + overrides=[ + "r=" + json.dumps(str(Path(environment["R_HOME"]) / "bin/R.exe")) + ], ) - environment.pop("RETICULATE_PYTHON", None) - session = Session(environment, bare_r=True) self.addCleanup(session.close) session.initialize() self.assertIn("42", json.dumps(session.send(r="answer <- 42L; answer"))) @@ -1320,6 +1351,7 @@ def test_python_without_r(self): RETICULATE_PYTHON=sys.executable, ) environment.pop("R_HOME", None) + environment.pop("PROCESSOR_ARCHITECTURE", None) session = Session(environment) self.addCleanup(session.close) session.initialize() @@ -1327,9 +1359,13 @@ def test_python_without_r(self): result = session.send( python=dedent(""" import ctypes + import os from pathlib import Path import sys + import sysconfig + architecture = {"win-amd64": "AMD64", "win-arm64": "ARM64"} + assert os.environ["PROCESSOR_ARCHITECTURE"] == architecture[sysconfig.get_platform()] assert ctypes.windll.kernel32.GetModuleHandleW("R.dll") == 0 # Console must find Python's supplied runtime without relying # on a separately installed system redistributable. A Python @@ -1383,6 +1419,59 @@ def test_python_without_r(self): ) self.assertIn("fresh False", json.dumps(result)) + @requires_mxc(BINARY) + def test_sandboxed_preinstalled_r_python_bridge(self): + session = Session( + preinstalled_r_environment(), + python=sys.executable, + overrides=["r.resolution=disabled"], + sandbox=True, + ) + self.addCleanup(session.close) + session.initialize() + session.expect( + "41", + # fmt: r + r=code(""" + platform <- reticulate::import("sysconfig")$get_platform() + stopifnot( + Sys.getenv("MCP_CONSOLE_SANDBOX") == "1", + identical(reticulate::py_config()$platform, platform) + ) + seed <- 41L + print(seed) + """), + ) + session.expect("42", python="answer = r.seed + 1; print(answer)") + session.expect( + "round trip complete", + r="stopifnot(py$answer == 42L); cat('round trip complete')", + ) + + def test_preinstalled_r_startup_python_bridge(self): + # fmt: r + source = code(""" + stopifnot(!reticulate::py_available(initialize = FALSE)) + .console$sql_connection(DBI::dbConnect(RSQLite::SQLite(), ":memory:")) + stopifnot(identical(reticulate::py$`__name__`, "__main__")) + seed <- reticulate::py_eval("40 + 1") + platform <- reticulate::import("sysconfig")$get_platform() + stopifnot(identical(reticulate::py_config()$platform, platform)) + """) + startup = json.dumps({"language": "r", "code": source}) + session = Session( + preinstalled_r_environment(), + python=sys.executable, + overrides=["r.resolution=disabled", f"startup={startup}"], + ) + self.addCleanup(session.close) + session.initialize() + session.expect("42", python="answer = r.seed + 1; print(answer)") + session.expect( + "round trip complete", + r="stopifnot(py$answer == 42L); cat('round trip complete')", + ) + def test_r_initializes_before_python(self): session = self.session(defer_bootstrap=True) # fmt: r @@ -1857,7 +1946,10 @@ def test_startup_without_processor_architecture(self): python=code(""" import os - assert os.environ["PROCESSOR_ARCHITECTURE"] == "AMD64" + import sysconfig + + expected = {"win-amd64": "AMD64", "win-arm64": "ARM64"} + assert os.environ["PROCESSOR_ARCHITECTURE"] == expected[sysconfig.get_platform()] print("PYTHON_SMOKE", 2 + 2) """), ) diff --git a/tests/windows_distribution.py b/tests/windows_distribution.py index 19b654d7e..3afcb014d 100644 --- a/tests/windows_distribution.py +++ b/tests/windows_distribution.py @@ -5,11 +5,14 @@ """ import argparse +from io import BytesIO import json from pathlib import Path import re import struct import subprocess +import sysconfig +from typing import BinaryIO import zipfile @@ -18,15 +21,27 @@ re.IGNORECASE, ) NATIVE_SUFFIXES = {".exe", ".dll", ".pyd"} +PE_MACHINE = {"win_amd64": 0x8664, "win_arm64": 0xAA64} -def pe_imports(content): +def pe_offset_and_machine(image: BinaryIO) -> tuple[int, int]: + assert image.read(2) == b"MZ", "not a PE image" + image.seek(0x3C) + (offset,) = struct.unpack(" Path: + # Resolve user input before the acceptance checks switch to the checkout. + return Path(value).resolve() + + +def accept(binary: Path, native: Path, *, preinstalled_r: bool) -> None: + version = tomllib.loads((ROOT / "Cargo.toml").read_text())["package"]["version"] + environment = os.environ | {"PATH": ""} + actual_version = subprocess.check_output( + [binary, "--version"], text=True, env=environment + ).strip() + version_pattern = re.escape(f"mcp-console {version}") + if version.endswith("+dev"): + version_pattern += r"(?:\.g[0-9a-f]{12})?" + assert re.fullmatch(version_pattern, actual_version), actual_version + run(binary, "--help", environment=environment) + assert not (binary.parent / "mcp-console-sandbox.exe").exists() + from support.windows_mxc import mxc_requirement + + if mxc_requirement(binary).available: + with workspace() as directory: + run( + binary, + "sandbox", + "--no-config", + "--", + Path(os.environ["SystemRoot"]) / "System32/cmd.exe", + "/d", + "/c", + "exit 0", + environment=os.environ | {"PATH": str(directory)}, + ) + cases = [ + "WindowsMxc.test_proxy_launch", + "WindowsMxc.test_startup_failures_withhold_code_at_every_public_entry_point", + "WindowsMxc.test_complete_policy_cannot_disable_mxc", + "WindowsSandbox.test_modified_companion_is_rejected_before_execution", + "WindowsSandbox.test_first_python_send_from_short_profile_directory", + "WindowsSandbox.test_python_state_and_restart_with_private_temporary_storage", + "WindowsSandbox.test_cwd_alias_does_not_require_parent_enumeration", + "WindowsSandbox.test_stdio_write_policy_and_exit_status", + "WindowsConsole.test_python_without_r", + "WindowsConsole.test_r_without_python", + ] + if preinstalled_r: + cases += [ + "WindowsConsole.test_sandboxed_preinstalled_r_python_bridge", + "WindowsConsole.test_preinstalled_r_startup_python_bridge", + ] + else: + cases += [ + "WindowsConsole.test_r_initializes_before_python", + "WindowsConsole.test_python_initializes_before_r", + "WindowsConsole.test_python_sleep_interrupt", + "WindowsConsole.test_sql_with_r", + "WindowsSandbox.test_network_enabled_sql", + ] + run( + sys.executable, + ROOT / "tests/windows.py", + "-v", + *cases, + environment=os.environ + | { + "MCP_CONSOLE_TEST_BINARY": str(binary), + "MCP_CONSOLE_TEST_NATIVE_BINARY": str(native), + }, + ) + + if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( + artifacts = parser.add_mutually_exclusive_group() + artifacts.add_argument( "--installed-binary", - type=Path, + type=artifact_path, help="accept an already installed release wheel without building from source", ) + artifacts.add_argument( + "--archive", type=artifact_path, help="accept a relocated native release ZIP" + ) + parser.add_argument( + "--preinstalled-r", + action="store_true", + help="use preinstalled R packages when accepting an archive or installed wheel", + ) options = parser.parse_args() + if ( + options.preinstalled_r + and options.archive is None + and options.installed_binary is None + ): + parser.error("--preinstalled-r requires --archive or --installed-binary") if sys.platform != "win32": raise SystemExit("native Windows installation checks require Windows") os.chdir(ROOT) if options.installed_binary is not None: - binary = options.installed_binary.resolve() + binary = options.installed_binary native, _ = inspect_installation(binary) - version = tomllib.loads((ROOT / "Cargo.toml").read_text())["package"]["version"] - actual_version = subprocess.check_output( - [binary, "--version"], text=True - ).strip() - version_pattern = re.escape(f"mcp-console {version}") - if version.endswith("+dev"): - version_pattern += r"(?:\.g[0-9a-f]{12})?" - assert re.fullmatch(version_pattern, actual_version), actual_version - run(binary, "--help") - assert not (binary.parent / "mcp-console-sandbox.exe").exists() - from support.windows_mxc import mxc_requirement - - if mxc_requirement(binary).available: - with workspace() as directory: - run( - binary, - "sandbox", - "--no-config", - "--", - Path(os.environ["SystemRoot"]) / "System32/cmd.exe", - "/d", - "/c", - "exit 0", - environment=os.environ | {"PATH": str(directory)}, - ) - run( - sys.executable, - ROOT / "tests/windows.py", - "-v", - "WindowsMxc.test_proxy_launch", - "WindowsMxc.test_startup_failures_withhold_code_at_every_public_entry_point", - "WindowsMxc.test_complete_policy_cannot_disable_mxc", - "WindowsSandbox.test_first_python_send_from_short_profile_directory", - "WindowsSandbox.test_python_state_and_restart_with_private_temporary_storage", - "WindowsSandbox.test_cwd_alias_does_not_require_parent_enumeration", - "WindowsSandbox.test_stdio_write_policy_and_exit_status", - "WindowsConsole.test_python_without_r", - "WindowsConsole.test_r_initializes_before_python", - "WindowsConsole.test_python_initializes_before_r", - "WindowsConsole.test_python_sleep_interrupt", - "WindowsConsole.test_sql_with_r", - "WindowsSandbox.test_network_enabled_sql", - environment=os.environ - | { - "MCP_CONSOLE_TEST_BINARY": str(binary), - "MCP_CONSOLE_TEST_NATIVE_BINARY": str(native), - }, - ) + accept(binary, native, preinstalled_r=options.preinstalled_r) + raise SystemExit(0) + if options.archive is not None: + with workspace() as directory: + extracted = directory / "extracted" + with zipfile.ZipFile(options.archive) as archive: + archive.extractall(extracted) + prefix = extracted.rename(directory / "relocated Console") + binary = prefix / "bin/mcp-console.exe" + inspect_artifacts( + (path, path.read_bytes()) + for path in prefix.rglob("*") + if path.suffix.lower() in NATIVE_SUFFIXES + ) + accept(binary, binary, preinstalled_r=options.preinstalled_r) raise SystemExit(0) # Exercise installed commands from a workspace with ordinary inherited ACLs. with checkout_owner(ROOT), workspace() as directory: diff --git a/tests/windows_mxc.py b/tests/windows_mxc.py index 49e3c6899..6467563d8 100644 --- a/tests/windows_mxc.py +++ b/tests/windows_mxc.py @@ -2,10 +2,13 @@ import json import os +import shutil import subprocess import sys +import sysconfig import unittest from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path from threading import Thread from support.normalization import code @@ -19,6 +22,32 @@ @unittest.skipUnless(os.name == "nt", "native Windows MXC") class WindowsMxc(unittest.TestCase): + @unittest.skipUnless(sysconfig.get_platform() == "win-amd64", "requires x64 Python") + def test_x64_fault_fixture_skips_a_non_x64_companion(self): + from support.windows_faults import run_with_unavailable_mxc + + # A 32-bit stand-in exercises the mismatch on ordinary x64 Windows CI. + windows = Path(os.environ["SystemRoot"]) + cmd32 = windows / "SysWOW64/cmd.exe" + if not cmd32.is_file(): + self.skipTest("requires the 32-bit Windows command processor") + with workspace() as root: + shutil.copy2(cmd32, root / "mcp-console-sandbox.exe") + with self.assertRaisesRegex( + unittest.SkipTest, "requires an x64 sandbox companion" + ): + run_with_unavailable_mxc( + [ + windows / "System32/cmd.exe", + "/d", + "/c", + r".\mcp-console-sandbox.exe /d /c exit 0", + ], + inject_unavailable=True, + cwd=root, + env=os.environ, + ) + def test_startup_failures_withhold_code_at_every_public_entry_point(self): from support.windows_faults import run_with_unavailable_mxc diff --git a/tests/windows_sandbox.py b/tests/windows_sandbox.py index c777cdeab..ff3efccc7 100644 --- a/tests/windows_sandbox.py +++ b/tests/windows_sandbox.py @@ -170,6 +170,22 @@ def worker_session(environment=None, *, overrides=(), **options): ) +def python_sandbox_session(temporary_root: Path): + # Exercise the installed Python worker even on hosts with discoverable R. + environment = dict( + os.environ, + PATH=str(Path(os.environ["SystemRoot"]) / "System32"), + RETICULATE_PYTHON=sys.executable, + ) + environment.pop("R_HOME", None) + return worker_session( + environment, + sandbox=True, + temporary_root=temporary_root, + overrides=['sandbox.network="enabled"'], + ) + + @unittest.skipUnless(os.name == "nt", "native Windows sandbox") class WindowsSandbox(unittest.TestCase): @requires_mxc(BINARY) @@ -659,11 +675,7 @@ def test_first_python_send_from_short_profile_directory(self): if short_name.value.casefold() == str(profile).casefold(): self.skipTest("user profile has no 8.3 alias") temporary = Path(short_name.value) / "AppData/Local/Temp" - session = worker_session( - sandbox=True, - temporary_root=temporary, - overrides=["r=null", 'sandbox.network="enabled"'], - ) + session = python_sandbox_session(temporary) try: session.initialize() result = session.send(python="print(6 * 7)") @@ -687,13 +699,8 @@ def test_first_python_send_from_short_profile_directory(self): @requires_mxc(BINARY) def test_python_state_and_restart_with_private_temporary_storage(self): - with workspace(): - session = worker_session( - sandbox=True, - overrides=[ - 'sandbox.network="enabled"', - ], - ) + with workspace() as root: + session = python_sandbox_session(root) try: session.initialize() result = session.send(