diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6adb7ce..296d444 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,45 @@ permissions: contents: read jobs: + public-contracts: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + outputs: + candidate: ${{ steps.revisions.outputs.candidate }} + checker: ${{ steps.revisions.outputs.checker }} + specification: ${{ steps.revisions.outputs.specification }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + path: reference_server + persist-credentials: false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: selvage-protocol/specification + ref: 12795c817566897d0af4c13877d229f4555581f8 + path: contract-checker + persist-credentials: false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: selvage-protocol/specification + ref: main + path: specification + persist-credentials: false + - name: record the contract revisions + id: revisions + run: | + candidate=$(git -C reference_server rev-parse HEAD) + checker=$(git -C contract-checker rev-parse HEAD) + specification=$(git -C specification rev-parse HEAD) + echo "candidate=$candidate" | tee -a "$GITHUB_OUTPUT" + echo "checker=$checker" | tee -a "$GITHUB_OUTPUT" + echo "specification=$specification" | tee -a "$GITHUB_OUTPUT" + - name: public workspace contracts against the candidate + run: >- + python3 contract-checker/scripts/check-workspace.py --workspace . + --consumer reference_server --revision reference_server=HEAD --revision specification=HEAD + checks: runs-on: ubuntu-24.04 timeout-minutes: 45 diff --git a/README.md b/README.md index a7ccf96..6d6ce93 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,8 @@ with its default. `--version` prints `selvaged/`. - [The server's shape](docs/the-server.md): rooms in memory, the grace period and `/meta`. - [The first room](docs/rooms.md): who mints a room, and which clients host or join. - [Deploying](docs/deploying.md): the addresses a client dials, the bind and TLS. +- [Operating the demo](docs/operating-the-demo.md): provisioning, the CI privilege boundary, verification, and recovery. +- [Publishing](docs/publishing.md): version tags, release gates, and partial-release recovery. - [Serving the page](docs/serving-the-page.md): `--serve-page` and the served headers. - [Licence](docs/licence.md): what each path is under, and what the image carries. diff --git a/crates/harness/tests/decisions.rs b/crates/harness/tests/decisions.rs index f1b0117..105d6a0 100644 --- a/crates/harness/tests/decisions.rs +++ b/crates/harness/tests/decisions.rs @@ -339,7 +339,7 @@ fn forward(stdout: ChildStdout, sender: &Sender) { } impl Drop for Subject { - /// A test that fails must not leave a child behind: `AGENTS.md` §5 bounds every spawn. + /// A failed test must release its child so it cannot keep the harness alive. fn drop(&mut self) { let _ = self.child.kill(); let _ = self.child.wait(); diff --git a/crates/harness/tests/limits.rs b/crates/harness/tests/limits.rs index 59e847b..5a8159e 100644 --- a/crates/harness/tests/limits.rs +++ b/crates/harness/tests/limits.rs @@ -8,9 +8,8 @@ //! entry disagree. //! //! The file is vendored inside the Cargo workspace and handed in as `SELVAGE_LIMITS`, because -//! the Nix sandbox receives only the workspace and a test that reads outside it fails there -//! (`ai_notes/AGENTS.md` §2). A missing file or a missing entry is a failure, not a skip: a pin -//! that reaches nothing reports a clean tree (§4). +//! the Nix sandbox receives only the workspace. A missing file or entry must fail rather +//! than silently skip the comparison. use std::collections::BTreeMap; use std::env; @@ -56,8 +55,8 @@ fn path() -> PathBuf { /// The published bounds by name. /// /// A file that is missing, unreadable, empty, malformed or names a bound twice is an error -/// rather than a smaller run: a pin that reads nothing would otherwise report a clean tree -/// (`ai_notes/AGENTS.md` §4). The caller turns it into the test's failure. +/// rather than a smaller run: a pin that reads nothing would otherwise report a clean tree. +/// The caller turns it into the test's failure. fn published() -> Result { let path = path(); let text = fs::read_to_string(&path).map_err(|error| { diff --git a/docs/operating-the-demo.md b/docs/operating-the-demo.md new file mode 100644 index 0000000..50e3986 --- /dev/null +++ b/docs/operating-the-demo.md @@ -0,0 +1,166 @@ +# Operating the public demo + +The demo at runs the tracked +[`deploy/`](../deploy) shape. For general self-hosting and TLS requirements, see +[Deploying](deploying.md); for page-serving privacy requirements, see +[Serving the page](serving-the-page.md). This guide describes the operational +contract, not a live host inventory. Read image versions, digests, credentials, +firewall rules, and installed files from their authoritative systems. + +## Installation and privilege boundary + +The Compose project is `selvage-prod`, rooted at `/srv/selvage`. It contains the +TLS front, `selvaged`, and the separate page image. Only the front publishes a +port. The proxy configuration is baked into its locally built image; a proxy +change requires a rebuild, not just a restart. + +Initial provisioning is an administrator's operation, not a CI deploy: + +1. Provision Docker Compose, the host's firewall, TLS origin certificate, and + Tailscale SSH access. Restrict origin ingress to the intended edge. Refresh + the firewall's Cloudflare ranges together with + [`cloudflare-ips.conf`](../deploy/proxy/conf.d/cloudflare-ips.conf). +2. Install `deploy/compose.yaml` and `deploy/proxy/` from a reviewed commit into + `/srv/selvage`. Copy `deploy/.env.example` to `.env` once, edit it for this + instance, and make `.env` mode `0600`; never overwrite it during an upgrade. +3. Install the origin certificate in `tls/origin.pem` (`root:root`, `0644`) and + key in `tls/origin.key` (`root:101`, `0640`), inside `tls/` (`root:root`, + `0750`). Certificates and keys are instance material, not repository files. +4. Install `deploy/selvage-update.service` and `deploy/selvage-update.timer` in + `/etc/systemd/system/`, then run `systemctl daemon-reload`. Bring the shape + up from `/srv/selvage` with `docker compose up -d --build proxy`; enable the + timer only after checking the configured images and origin responses. +5. Create the unprivileged `deployci` account. Install `deploy/deploy.py` as + `/usr/local/sbin/selvage-deploy` and `deploy/install_shape.py` as + `/usr/local/sbin/selvage-install-shape`, both `root:root`, `0755`. Validate + the corresponding `deployci.sudoers` and `installci.sudoers` with `visudo -c` + before installing them in `/etc/sudoers.d/` as `root:root`, `0440`. + +The two sudoers commands must retain the trailing `""`: it permits **no +arguments**. Omitting that argument constraint permits arbitrary arguments. +Read the result with `sudo -l -U deployci`. Do not grant CI Docker-group +membership, blanket sudo, or permission to replace these scripts or sudoers +files. Changing either privileged program, the systemd units, `.env` settings +other than image references, or TLS material remains administrator work. + +The image deployer accepts only tags of the two allowed GHCR images on stdin. +The shape installer accepts only `compose.yaml` and `proxy/` from a tar stream; +it rejects traversal, absolute paths, links, and special files. Shape access is +still powerful: a reviewed Compose file controls mounts, ports, and capabilities. +The installer does not delete obsolete proxy files; removed files need an +administrator's cleanup before rebuilding the front. + +Instance-specific `compose.override.yaml` and notice files are not installed by +CI. A notice mounted into the front must survive shape upgrades. In particular, +nginx `sub_filter` directives at a more specific location replace the inherited +list rather than appending to it; `nginx -t` alone cannot prove a notice renders. + +## CI access + +[deploy-prod.yml](../.github/workflows/deploy-prod.yml) is dispatched at `main`. +It has **no environment approval gate**. Tailscale workload identity federation +matches the GitHub OIDC subject exactly: + +```text +repo:selvage-protocol@328844633/reference_server@1368869493:ref:refs/heads/main +``` + +The numeric IDs are part of the organization's immutable subject form. Adding +an `environment:` to this job changes the subject and breaks the exchange. +Repository secrets `TS_OAUTH_CLIENT_ID` and `TS_AUDIENCE` name the credential +pair; their values belong in the credential console and GitHub, not this guide. +The ephemeral runner receives `tag:ci`. Tailnet policy must restrict it to SSH on +`tag:selvaged-prod`, with the SSH rule permitting only `deployci`. Tailscale SSH +cannot constrain the command: the host's two argument-free sudo rules are the +command boundary. Tailnet Lock requires a separate authentication design if +enabled; do not silently replace federation with a broad reusable key. + +To rotate the pair, create a replacement identity with the same constrained +subject, update both repository secrets, verify a deploy, then remove the old +identity. An exchange refusal needs the actual claims and credential inspected: +a missing identity and a rejected subject are different failures. Never loosen +the subject or host privileges to make a retry pass. + +## Deploy and verify + +From any checkout, explicitly name this repository: + +```sh +gh workflow run deploy-prod.yml -R selvage-protocol/reference_server --ref main \ + -f server_version= +gh workflow run deploy-prod.yml -R selvage-protocol/reference_server --ref main \ + -f server_version= -f install_shape=true +``` + +An empty image input leaves that line unchanged; `latest` unpins it. The server +release changes its own `SELVAGED_IMAGE` line. The page repository normally +changes `SELVAGE_WEB_IMAGE` independently. The deploy accepts tags, not digests: +a version tag is a deliberate pin but is still mutable by a registry writer. + +With `install_shape=true`, CI streams the tracked pair from the run's `main` +ref, installs it under the update lock, and reads installed file hashes back. +Matching files are a no-op; a Compose-only change runs `up -d`; a proxy change +rebuilds the front. To install the same reviewed shape by hand from this repo: + +```sh +git archive --format=tar :deploy compose.yaml proxy \ + | ssh deployci@selvage-protocol-prod \ + '/usr/bin/sudo /usr/local/sbin/selvage-install-shape' +``` + +The image deploy holds `/srv/selvage/.update.lock`, stages `.env.deploy`, and +writes `.env` only after convergence, keeping `.env.prev`. A failed deploy can +already have changed running containers even though `.env` is unchanged; inspect +both, then reconcile with the intended references. A healthy origin is asserted +through the front over SSH: `/meta` must report the requested server version +when one was specified, and `/` must answer successfully. Read current state: + +```sh +ssh selvage@selvage-protocol-prod 'cd /srv/selvage && docker compose ps && docker compose images' +ssh selvage@selvage-protocol-prod \ + "curl -sk -H 'Host: selvage-demo.dontblameme.dev' https://127.0.0.1/meta" +``` + +The public edge may challenge a datacenter client. Verification reports a +challenge or unreadable public response; it does not bypass bot controls or +claim that the public path passed. A readable public version different from the +requested version fails the deploy. `latest` has no expected version to compare. +A page answering 200 is **not proof of its build identity**. Neither these reads +nor local proxy tests verify the separately hosted site's deployed response +headers or their limits. Read those from the deployed response before claiming +them. Automated checks do not establish human acceptance. + +## Timer, recovery, and downtime + +The update timer pulls configured tags every five minutes under the same lock. +A line left at `latest` follows registry updates without another approval. +Inspect with `systemctl list-timers selvage-update.timer` and +`journalctl -u selvage-update.service -n 50`. + +For manual operations, run Compose as the instance operator in `/srv/selvage`, +without a `-f` override, so it reads the same `.env` and optional override as the +timer. **Stop the timer before stopping services** or its next tick undoes the +stop. To keep it stopped across boots, disable it as well. + +```sh +sudo systemctl stop selvage-update.timer +cd /srv/selvage +docker compose down +# Restore the reviewed shape and intended image references before restarting. +docker compose pull --ignore-buildable +docker compose up -d --build proxy +sudo systemctl start selvage-update.timer +``` + +Rollback is an explicit pin to a known previous image tag followed by pull and +up. After a successful deploy, `.env.prev` is the previous environment; inspect +it before copying it back. Restore older proxy files from a reviewed commit and +rebuild for a front rollback. These are recovery procedures, not a claim that a +live rollback has been exercised. + +Replacing `selvaged` ends every room: all state is in memory, with no drain or +restart reclaim. A front replacement drops WebSockets and requires clients to +reconnect; it does not itself restore lost server state. Coordinate disruptive +changes with users rather than treating a green deploy as a seamless upgrade. + +The repository's [publishing guide](publishing.md) covers release/image recovery. diff --git a/docs/publishing.md b/docs/publishing.md new file mode 100644 index 0000000..2ce5344 --- /dev/null +++ b/docs/publishing.md @@ -0,0 +1,83 @@ +# Publishing the server + +This repository versions the server independently of the clients. `Cargo.toml`'s +workspace version owns the release number; `scripts/bump-version.sh` updates it, +and `scripts/release-tags.sh` asserts image/tag agreement. A server release does +not change the protocol's wire version. + +## Dispatch and gates + +[release.yml](../.github/workflows/release.yml) is dispatch-only at `main`: + +```sh +gh workflow run release.yml -R selvage-protocol/reference_server --ref main \ + -f bump=patch -f dry_run=true +gh workflow run release.yml -R selvage-protocol/reference_server --ref main \ + -f bump=patch +``` + +Choose `patch`, `minor`, or `major` deliberately. The read-only plan requires an +exact remote tag for the current manifest version and no tag for the target. +The write job rechecks these preconditions before bumping; a retried write job +cannot rely on the plan's earlier remote read. Dry run prints the plan and stops +before modifying the tree, running the build gates, or publishing anything. + +Merge with green [CI](../.github/workflows/ci.yml), including the public +workspace-contract job comparing vendored vectors and bounds with the +specification. The release runs its repository-local gate on the bumped tree +before an atomic push of its commit and lightweight tag. It never force-pushes +or moves a release tag. The workflow commits as the owner; it does not open a +bump PR whose checks would never start under its own `GITHUB_TOKEN`. + +[image.yml](../.github/workflows/image.yml) builds and publishes the server-only +image for `linux/amd64` and `linux/arm64`. Its tag-gated publish uses +`GITHUB_TOKEN` with `packages: write`; no separate owner token or approval +environment is configured in that workflow. Tags are `-`, +``, and, for non-prereleases, `latest`. The release explicitly +dispatches the image workflow because a tag push using `GITHUB_TOKEN` does not +start it automatically. + +## Deployment and recovery + +After the image becomes available, release dispatches `deploy-prod.yml` at +`main` for its own server version. It sets `install_shape=true` when `deploy/` +changed, waits for the resulting deploy run, and fails unless that run succeeds. +A published tag or image can therefore remain after a red release. A green run +means the deploy's assertions passed, not that human acceptance or a live +rollback was performed. The host privilege boundary, credentials by name, +verification limits, and rollback procedure are in +[Operating the demo](operating-the-demo.md). + +Recover the failed stage rather than dispatching a new bump blindly: + +- **Tag exists, image missing:** inspect the original image run and rerun the + workflow at the existing tag: + + ```sh + gh workflow run image.yml -R selvage-protocol/reference_server --ref 'v' + ``` + + The image workflow reasserts tag/manifest agreement. Do not move the tag to + include a packaging fix; fix forward with a new version instead. +- **Image exists, demo deploy failed:** inspect the deploy's origin/read-back + failure, then retry the bounded deploy at `main` with the intended version + and shape input. Another release dispatch would compute a new version. +- **Image exists, GitHub Release missing:** the image workflow creates the + GitHub Release in a separate job; the image and even the deployed demo can + exist while that job is red. Inspect its failure, then rerun `image.yml` at + the existing `v` tag as above to create the missing release. Do not + dispatch another version bump merely to recover the release record. +- **Atomic push refused:** reconcile with current `main` and tags before + planning again. Do not force the old candidate over concurrent work. + +Verify the artifact and deployed version, not just a tag's existence: + +```sh +gh release view 'v' -R selvage-protocol/reference_server --json tagName,assets +docker run --rm ghcr.io/selvage-protocol/selvaged: --version +``` + +Inspect the published multi-platform manifest and its per-platform layers when +checking architecture claims. A two-platform manifest alone does not prove the +binaries inside differ. The image workflow's smoke and layer checks cover that +artifact contract; origin `/meta` checks cover the running server version.