Implement protected private CI release and state recovery

.github/workflows/ci.yml

@@ -33,7 +33,7 @@ jobs:
       - name: Set up Node.js
         uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
-          node-version: 20.x
+          node-version: '22.23.1'
           cache: pnpm
           registry-url: https://npm.pkg.github.com
           scope: "@cuny-ai-lab"

.github/workflows/deploy.yml

@@ -1,56 +1,52 @@
-name: Deploy to Staging
-
-# Disabled: server is on Tailscale (100.x.x.x), not reachable from GitHub Actions.
-# Use `./deploy-staging.sh` from a machine on the Tailscale network instead.
+name: Private fleet release
 on:
   workflow_dispatch:
-
+    inputs:
+      release_sha:
+        description: Exact current-main SHA with successful CI
+        required: true
+        type: string
 permissions:
   contents: read
-
+  actions: read
+  deployments: read
+  packages: read
+concurrency:
+  group: slide-maker-production
+  cancel-in-progress: false
 jobs:
-  check:
-    name: Test and build
-    runs-on: ubuntu-latest
-    timeout-minutes: 20
+  release:
+    if: github.ref == 'refs/heads/main'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 35
+    environment: production
+    env:
+      RELEASE_SHA: ${{ inputs.release_sha }}
+      GITHUB_TOKEN: ${{ github.token }}
     steps:
-      - name: Check out source
-        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
+          # Evaluate the guard from trusted main, never an untrusted input ref.
+          ref: main
           persist-credentials: false
-
-      - name: Set up pnpm
-        uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
-          version: 9.15.0
-
-      - name: Set up Node.js
-        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+          node-version: '22.23.1'
+      - name: Verify main, successful checks and protected environment
+        run: node scripts/release/verify-source.mjs
+      - name: Connect only to the authorized private deployment host
+        uses: tailscale/github-action@6cae46e2d796f265265cfcf628b72a32b4d7cade # v3
         with:
-          node-version: 20.x
-          cache: pnpm
-
-      - name: Install dependencies
-        run: pnpm install --frozen-lockfile
-
-      - name: Build workspaces
-        run: pnpm build
-
-      - name: Run tests
-        run: pnpm test
-
-  deploy:
-    needs: check
-    runs-on: ubuntu-latest
-    steps:
-      - name: Deploy via SSH
-        uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2 # v1.2.5
+          oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}
+          oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}
+          tags: tag:slide-maker-deploy
+          version: '1.102.4'
+          sha256sum: '50748df1045e60b5b695f19f4c56b0da36c019948b440fb456b6584a50f0d8b9'
+          targets: 100.111.252.53
+          retry: '1'
+      - name: Upload exact source, rehearse restoration and verify private release
         env:
-          DEPLOY_PASSWORD: ${{ secrets.DEPLOY_PASSWORD }}
-        with:
-          host: 100.111.252.53
-          username: smorello.adm@gc.cuny.edu
-          password: ${{ secrets.DEPLOY_PASSWORD }}
-          envs: DEPLOY_PASSWORD
-          script: |
-            printf '%s\n' "$DEPLOY_PASSWORD" | sudo -S /data/slide-maker/deploy.sh
+          SLIDE_SSH_USER: ${{ secrets.SLIDE_SSH_USER }}
+          SLIDE_SSH_KEY: ${{ secrets.SLIDE_SSH_KEY }}
+          SLIDE_SSH_KNOWN_HOSTS: ${{ secrets.SLIDE_SSH_KNOWN_HOSTS }}
+        run: node scripts/release/private-ssh.mjs

.github/workflows/release-checks.yml

@@ -0,0 +1,23 @@
+name: Release safeguards
+on:
+  pull_request:
+  push:
+    branches: [main]
+permissions:
+  contents: read
+jobs:
+  release-safeguards:
+    name: Release safeguards
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: '22.23.1'
+      - name: Install disposable process-test runtime
+        run: npm install --prefix "$RUNNER_TEMP/release-runtime" --no-audit --no-fund pm2@7.0.4 tsx@4.21.0
+      - name: Verify source guards, release failure behavior and real process recovery
+        run: CAIL_RELEASE_PM2="$RUNNER_TEMP/release-runtime/node_modules/pm2/bin/pm2" node --test scripts/release/*.test.mjs

CLAUDE.md

@@ -6,7 +6,7 @@ Check `TODO.md` for the current task list when planning work, but default to the
 
 ## Institutional integration authority
 
-The current receiver contract and release proposal are in [docs/fleet-integration.md](docs/fleet-integration.md). They supersede the historical direct-provider, local-token-cap, password-production-auth and manual-deployment descriptions below. Model calls now use Gateway; production identity maps verified canonical subjects to unchanged local IDs. Do not deploy or activate a mount from historical staging instructions.
+The current receiver contract is in [docs/fleet-integration.md](docs/fleet-integration.md), and the protected CI release implementation and host prerequisites are in [docs/private-ci-release.md](docs/private-ci-release.md). They supersede the historical direct-provider, local-token-cap, password-production-auth and manual-deployment descriptions below. Model calls now use Gateway; production identity maps verified canonical subjects to unchanged local IDs. Do not deploy or activate a mount from historical staging instructions.
 
 ## What This Is
 

docs/fleet-integration.md

@@ -41,7 +41,11 @@ The script refuses invalid subjects, unknown users, collisions and replacing an
 
 Set a random operator-held `READINESS_TOKEN` and the exact 40-character deployed `RELEASE_SHA`. `GET /internal/ready` requires `Authorization: Bearer <READINESS_TOKEN>`; unauthenticated probes get 404. It awaits identity configuration, checks the migrated user column, requires Gateway configuration and returns the exact release. It does not prove a successful Gateway model call or a browser sign-in. Do not publish this route through the institutional application mount.
 
-The existing SSH/manual staging workflow is historical and is not an approved institutional release mechanism. A subsequent reviewed release PR must establish the actual CI runner's supported private reachability; this proposal supplies no invented transport. The release design must:
+The protected manual CI workflow and host scripts are implemented in the
+[private release runbook](private-ci-release.md). They replace the historical
+password/sudo workflow with pinned Tailscale access, verified SSH keys and an
+immutable source archive. Host credentials, the protected environment and live
+reachability remain operator prerequisites. The implementation follows this order:
 
 1. Run the authoritative checks on main with private package read access, no deploy credentials in PR jobs, and a reviewed dependency lockfile.
 2. Serialize releases, reject stale queued SHAs, deploy the exact checked SHA, and obtain a consistent database/upload backup before the additive migration.
@@ -51,6 +55,13 @@ The existing SSH/manual staging workflow is historical and is not an approved in
 
 Source checks and disposable synthetic-provider tests are deterministic gates. Production traffic, paid model calls, deployment, mount activation and operator mappings are outside this branch's execution authority.
 
+The release additionally rechecks current main after its snapshot/restore
+rehearsal, verifies the existing database and upload paths against the running
+API, and restores prior process definitions on failure without replacing live
+data. Release safeguard tests exercise stale/failed sources, unsafe storage,
+partial stops, failed snapshots/rehearsals and failed readiness/recovery. CI
+transport does not establish browser ingress or live sign-in/model acceptance.
+
 ## Verification commands and scope
 
 `bun run check` runs Svelte type checking, unit/SQLite/Hono regressions, the existing shell checks, and API/web builds. The receiver integration test is explicitly skipped without `CAIL_GATEWAY_SOURCE`; ordinary CI does not require ambient sibling repositories or private cross-repository credentials.

docs/operations/2026-09-23-private-host-audit.md

@@ -0,0 +1,72 @@
+# September 23 private host audit
+
+Administrative SSH to the existing Tailscale host succeeded. The live checkout
+remains at `9efd29c62f07a9e431116ee89c5b4513c4499c8f`, with its existing local
+files preserved. Neither the fleet receiver nor the protected CI release was
+deployed during this audit.
+
+## Existing API repair
+
+PM2 reported the API online, but port 3004 had no listener. The TypeScript
+watcher remained alive after its child failed to load the native SQLite module:
+the module required Node ABI 127, while the host's Node 24.19.0 used ABI 137.
+
+A private Node 22.23.1 runtime was installed at
+`/data/slide-maker-storage/operations/runtime/node-v22.23.1-linux-x64`. Its
+official Linux x64 archive was verified with SHA-256
+`9749e988f437343b7fa832c69ded82a312e41a03116d766797ac14f6f9eee578`.
+An in-memory SQLite check passed before any process was restarted.
+
+Only `slide-maker-api` was restarted with this runtime first in its PATH. Its
+saved PM2 definition received the same PATH change; the other saved process
+definitions were preserved. No shared Node, pnpm or PM2 installation changed.
+The earlier process definition, saved PM2 state and SQLite backup remain private
+under `/data/slide-maker-storage/operations/runtime-repair-20260923T144922Z`.
+
+Readback:
+
+- Loopback `/api/health`: 200; anonymous `/api/auth/me` and `/api/decks`: 401.
+- Database integrity: `quick_check=ok`; no foreign-key violations.
+- Existing records preserved: 9 users, 26 decks, 52 uploaded-file records,
+  281 slides and 740 content blocks.
+- Other PM2 process IDs and statuses were unchanged.
+- The legacy public URL returned 403 from the host's network, so this receipt
+  does not establish public or campus browser access.
+- Saved startup configuration was checked, but no server reboot was performed.
+
+## Production-copy rehearsal
+
+The snapshot utility and migration module came from source
+`7e5275a118aeb13909e83f4c2f84bc304283d5a5`. The installed Identity 5.2.6, jose
+6.2.3 and zod 4.4.3 runtime files were copied into the private rehearsal directory;
+the running checkout and its dependencies were unchanged.
+
+The API was stopped, absence of database/WAL/SHM holders was verified, and the
+utility took a SQLite backup plus all uploads. The API recovered in 4.33 seconds
+from the stop request. The snapshot was restored into a new directory and
+verified against its byte/hash manifest: 78 files total, including all 77 upload
+files (48,799,114 bytes). The 52 database upload records and 77 stored files are
+different inventories; the rehearsal retained the entire storage directory.
+
+The actual additive identity migration ran twice against the restored copy.
+All original column values across 16 tables were preserved. The new nullable
+column and unique index were present, all nine accounts remained unlinked, and
+SQLite integrity and foreign-key checks passed. The original snapshot still
+passed its manifest check after the migration rehearsal.
+
+Private artifacts and the machine-readable receipt are under
+`/data/slide-maker-storage/operations/rehearsal-20260923-7e5275a`.
+No production identity column, account association or role was changed.
+
+## Remaining release requirements
+
+The host's shared defaults are Node 24.19.0, pnpm 10.33.0 and PM2 6.0.13. The
+protected release still requires its pinned runtime, private directories and
+configuration, Tailscale OAuth access, a dedicated SSH key and verified host
+key material in the main-only GitHub environment. The administrative password
+used for this inspection was not installed in CI.
+
+Nginx still proxies the legacy Slide Maker routes directly to the existing
+API/web pair. Doorway-to-Node ingress, mapped-user CUNY login, current Gateway
+model work, exported bytes and public/campus access need their own deployed
+acceptance. A successful state rehearsal does not establish those outcomes.

docs/private-ci-release.md

@@ -0,0 +1,104 @@
+# Private CI release
+
+The manual `Private fleet release` workflow replaces the historical password/
+sudo staging workflow. It is source implementation, not a record of deployment.
+The existing host is `100.111.252.53` on Tailscale. The workflow uses an ephemeral
+Tailscale node and SSH with a verified host key; it does not provision either
+access mechanism or expose a new public listener.
+
+## Operator prerequisites
+
+- Merge the receiver and release PRs, and wait for **Test and build** and
+  **Release safeguards** on the exact `main` push. The source check and release
+  use Node 22.23.1; the existing pnpm lock and version 9.15.0 remain unchanged.
+- Create the GitHub `production` environment with exactly one deployment branch
+  rule: branch `main`. Supply its `TS_OAUTH_CLIENT_ID`, `TS_OAUTH_SECRET`,
+  `SLIDE_SSH_USER`, `SLIDE_SSH_KEY` and `SLIDE_SSH_KNOWN_HOSTS` secrets. Obtain the
+  SSH fingerprint through the host's trusted administrative channel. Do not
+  populate known hosts from an unverified connection or disable host checking.
+- Restrict the OAuth client/tag to `tag:slide-maker-deploy`, with network access
+  only to this host's SSH port. Tailscale action, binary version and binary digest
+  are pinned. A network connection does not grant an SSH account permission.
+- The existing service owner must have key-based SSH access, Node 22.23.1,
+  pnpm 9.15.0, the tested PM2 CLI 7.0.4, `flock`, `lsof`, `ps`, `tar` and `sha256sum`. Use the same
+  service account and PM2 home as the existing API/web processes. No sudo,
+  password injection or automatic user creation is performed.
+- Pre-create `/data/slide-maker-releases` and its `incoming` subdirectory,
+  `/data/slide-maker-backups`, `/data/slide-maker-rehearsals` and
+  `/data/slide-maker-config`, owned by that account with mode 0700. These must
+  be separate from the current database and uploads. Allow room for the source,
+  dependencies, a full snapshot and a disposable restored copy.
+- In `/data/slide-maker-config/production.env` (owner only, mode 0600), preserve
+  existing application configuration and provide the canonical fleet values
+  from [fleet-integration.md](fleet-integration.md), `SESSION_SECRET`, a private
+  `READINESS_TOKEN`, and **verified existing** absolute `DATABASE_URL=file:...`
+  and `SLIDE_UPLOADS_PATH=...`. The script checks that the current API actually
+  holds that database and that its existing uploads resolve to that directory.
+  It refuses guessed, missing or deployment-local data paths. Keep package/CI
+  credentials and Node/PM2 override settings out of this application file.
+- Stop any independent database or upload writers for the release window.
+  The workflow stops the API and checks for remaining database handles; uploads
+  are verified unchanged while copying. Account linking is a separate,
+  independently verified operator operation; the release never matches email
+  addresses or grants local roles.
+
+The job token needs read access to the private CAIL packages. It crosses only
+the verified SSH stdin for installation and source checks, remains out of
+application environment and PM2 files, and is never written to disk. A temporary
+npm configuration contains only an environment placeholder and is removed even
+when installation fails. Command output that could include credentials stays
+out of CI logs. Snapshot and PM2 recovery files are private host data.
+
+## Release and failure behavior
+
+Dispatch from `main` with its full current SHA. The workflow loads its guard
+from trusted main, requires the latest successful main-push CI and release-check
+runs, and rejects stale SHAs or missing environment protection. It uses both
+GitHub concurrency and a host file lock. A release is a `git archive` of the
+verified SHA, transferred once, digest-checked, and extracted into a unique
+SHA/run/attempt directory. It never changes the old checkout or storage paths.
+
+After a frozen install and production build, the host validates both identity
+verifiers. It then records existing PM2 definitions, stops the API, verifies
+exclusive database access, creates a consistent SQLite/uploads snapshot, and
+restores and migrates a disposable copy. A further main/CI check precedes the
+live additive identity migration. Any earlier failure cannot migrate live data.
+The two application processes are recreated to prevent stale listener arguments.
+Both stay on `127.0.0.1` (API 3004, web 4173); the API runs TypeScript through
+the pinned installed runtime without a file watcher.
+
+Acceptance checks require exactly one online process for each component, the
+new source directory and SHA, one loopback listener on each port, unauthenticated
+readiness rejection, authenticated readiness at the exact SHA, and the mounted
+web response. Only then does PM2 save its state and the `current` symlink advance.
+
+A failure after stopping writers attempts to restore the captured old process
+definitions and save that process state. It does **not** restore the live database
+or uploads. The nullable identity migration is backward compatible with the
+previous code. Failed recovery is an explicit error. No upload, migration or
+activation is retried automatically; inspect the exact run before dispatching
+again. Failed release directories and state snapshots remain for investigation.
+
+## Remaining acceptance
+
+The [September 23 host audit](operations/2026-09-23-private-host-audit.md)
+verified administrative SSH access, the existing database/uploads paths, and a
+backup, restoration and additive identity migration against a private copy of
+actual production state. It also repaired an existing native SQLite/Node ABI
+mismatch by pinning only the legacy API to Node 22.23.1. The live application
+source and identity schema were not migrated.
+
+The protected host release has not been run. Its Tailscale OAuth connection,
+dedicated SSH key, private release configuration/directories and release runtime
+still need provisioning. The shared host defaults remain Node 24.19.0, pnpm
+10.33.0 and PM2 6.0.13. Do not globally upgrade or replace these to satisfy the
+release pins: the account also runs other applications. The API's private Node
+22 runtime does not by itself satisfy the CI host's PATH/toolchain contract.
+Source checks and the successful production-copy rehearsal do not prove the
+remaining network, credential or live traffic requirements.
+
+The CI connection does not create the private Doorway-to-Node browser ingress.
+Keep that mount disabled until its transport is established and the receiver is
+verified. Then separately exercise mapped-user CUNY login, unchanged local roles,
+old decks/files/sharing, a real Gateway model action, cancellation and exported
+file bytes through the deployed route, including public and campus access.

scripts/release/activate.mjs

@@ -0,0 +1,24 @@
+// Every callback is awaited. A failed backup or rehearsal cannot reach the live
+// migration. Rollback restores process definitions only, never application data.
+export async function activateRelease(steps) {
+  await steps.verifySource();
+  let stopped = false;
+  try {
+    stopped = true; // A partially failed stop still requires recovery.
+    await steps.stopWriters();
+    await steps.verifyWritersStopped();
+    await steps.backup();
+    await steps.rehearseRestore();
+    await steps.verifySource(); // Main may have advanced during the rehearsal.
+    await steps.migrate();
+    await steps.start();
+    await steps.verify();
+    await steps.save();
+  } catch (error) {
+    if (stopped) {
+      try { await steps.rollbackCode(); }
+      catch { throw new Error('Release failed and process recovery failed. Keep the saved state and inspect the host; no data was restored.'); }
+    }
+    throw error;
+  }
+}