From 7e5275a118aeb13909e83f4c2f84bc304283d5a5 Mon Sep 17 00:00:00 2001
From: zmuhls <zmuhlbauer1@gmail.com>
Date: Wed, 23 Sep 2026 02:06:15 -0400
Subject: [PATCH 1/2] Add protected private CI release with state rehearsal and
 process recovery

---
 .github/workflows/ci.yml                      |   2 +-
 .github/workflows/deploy.yml                  |  86 +++++-----
 .github/workflows/release-checks.yml          |  23 +++
 CLAUDE.md                                     |   2 +-
 docs/fleet-integration.md                     |  13 +-
 docs/private-ci-release.md                    |  94 +++++++++++
 scripts/release/activate.mjs                  |  24 +++
 scripts/release/activate.test.mjs             |  42 +++++
 scripts/release/contract.mjs                  |  47 ++++++
 scripts/release/contract.test.mjs             |  36 ++++
 scripts/release/host.mjs                      | 156 ++++++++++++++++++
 scripts/release/private-ssh.mjs               |  49 ++++++
 .../release/processes.integration.test.mjs    |  70 ++++++++
 scripts/release/processes.mjs                 |  14 ++
 scripts/release/verify-source.mjs             |  66 ++++++++
 scripts/release/verify-source.test.mjs        |  42 +++++
 16 files changed, 718 insertions(+), 48 deletions(-)
 create mode 100644 .github/workflows/release-checks.yml
 create mode 100644 docs/private-ci-release.md
 create mode 100644 scripts/release/activate.mjs
 create mode 100644 scripts/release/activate.test.mjs
 create mode 100644 scripts/release/contract.mjs
 create mode 100644 scripts/release/contract.test.mjs
 create mode 100644 scripts/release/host.mjs
 create mode 100644 scripts/release/private-ssh.mjs
 create mode 100644 scripts/release/processes.integration.test.mjs
 create mode 100644 scripts/release/processes.mjs
 create mode 100644 scripts/release/verify-source.mjs
 create mode 100644 scripts/release/verify-source.test.mjs

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index bdc8f92..3c64dec 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -33,7 +33,7 @@ jobs:
       - name: Set up Node.js
         uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
-          node-version: 20.x
+          node-version: '22.23.1'
           cache: pnpm
           registry-url: https://npm.pkg.github.com
           scope: "@cuny-ai-lab"
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
index be77be9..0d4bea0 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -1,56 +1,52 @@
-name: Deploy to Staging
-
-# Disabled: server is on Tailscale (100.x.x.x), not reachable from GitHub Actions.
-# Use `./deploy-staging.sh` from a machine on the Tailscale network instead.
+name: Private fleet release
 on:
   workflow_dispatch:
-
+    inputs:
+      release_sha:
+        description: Exact current-main SHA with successful CI
+        required: true
+        type: string
 permissions:
   contents: read
-
+  actions: read
+  deployments: read
+  packages: read
+concurrency:
+  group: slide-maker-production
+  cancel-in-progress: false
 jobs:
-  check:
-    name: Test and build
-    runs-on: ubuntu-latest
-    timeout-minutes: 20
+  release:
+    if: github.ref == 'refs/heads/main'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 35
+    environment: production
+    env:
+      RELEASE_SHA: ${{ inputs.release_sha }}
+      GITHUB_TOKEN: ${{ github.token }}
     steps:
-      - name: Check out source
-        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
+          # Evaluate the guard from trusted main, never an untrusted input ref.
+          ref: main
           persist-credentials: false
-
-      - name: Set up pnpm
-        uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
-          version: 9.15.0
-
-      - name: Set up Node.js
-        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+          node-version: '22.23.1'
+      - name: Verify main, successful checks and protected environment
+        run: node scripts/release/verify-source.mjs
+      - name: Connect only to the authorized private deployment host
+        uses: tailscale/github-action@6cae46e2d796f265265cfcf628b72a32b4d7cade # v3
         with:
-          node-version: 20.x
-          cache: pnpm
-
-      - name: Install dependencies
-        run: pnpm install --frozen-lockfile
-
-      - name: Build workspaces
-        run: pnpm build
-
-      - name: Run tests
-        run: pnpm test
-
-  deploy:
-    needs: check
-    runs-on: ubuntu-latest
-    steps:
-      - name: Deploy via SSH
-        uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2 # v1.2.5
+          oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}
+          oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}
+          tags: tag:slide-maker-deploy
+          version: '1.102.4'
+          sha256sum: '50748df1045e60b5b695f19f4c56b0da36c019948b440fb456b6584a50f0d8b9'
+          targets: 100.111.252.53
+          retry: '1'
+      - name: Upload exact source, rehearse restoration and verify private release
         env:
-          DEPLOY_PASSWORD: ${{ secrets.DEPLOY_PASSWORD }}
-        with:
-          host: 100.111.252.53
-          username: smorello.adm@gc.cuny.edu
-          password: ${{ secrets.DEPLOY_PASSWORD }}
-          envs: DEPLOY_PASSWORD
-          script: |
-            printf '%s\n' "$DEPLOY_PASSWORD" | sudo -S /data/slide-maker/deploy.sh
+          SLIDE_SSH_USER: ${{ secrets.SLIDE_SSH_USER }}
+          SLIDE_SSH_KEY: ${{ secrets.SLIDE_SSH_KEY }}
+          SLIDE_SSH_KNOWN_HOSTS: ${{ secrets.SLIDE_SSH_KNOWN_HOSTS }}
+        run: node scripts/release/private-ssh.mjs
diff --git a/.github/workflows/release-checks.yml b/.github/workflows/release-checks.yml
new file mode 100644
index 0000000..a5e7253
--- /dev/null
+++ b/.github/workflows/release-checks.yml
@@ -0,0 +1,23 @@
+name: Release safeguards
+on:
+  pull_request:
+  push:
+    branches: [main]
+permissions:
+  contents: read
+jobs:
+  release-safeguards:
+    name: Release safeguards
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: '22.23.1'
+      - name: Install disposable process-test runtime
+        run: npm install --prefix "$RUNNER_TEMP/release-runtime" --no-audit --no-fund pm2@7.0.4 tsx@4.21.0
+      - name: Verify source guards, release failure behavior and real process recovery
+        run: CAIL_RELEASE_PM2="$RUNNER_TEMP/release-runtime/node_modules/pm2/bin/pm2" node --test scripts/release/*.test.mjs
diff --git a/CLAUDE.md b/CLAUDE.md
index 5b3076c..6140315 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -6,7 +6,7 @@ Check `TODO.md` for the current task list when planning work, but default to the
 
 ## Institutional integration authority
 
-The current receiver contract and release proposal are in [docs/fleet-integration.md](docs/fleet-integration.md). They supersede the historical direct-provider, local-token-cap, password-production-auth and manual-deployment descriptions below. Model calls now use Gateway; production identity maps verified canonical subjects to unchanged local IDs. Do not deploy or activate a mount from historical staging instructions.
+The current receiver contract is in [docs/fleet-integration.md](docs/fleet-integration.md), and the protected CI release implementation and host prerequisites are in [docs/private-ci-release.md](docs/private-ci-release.md). They supersede the historical direct-provider, local-token-cap, password-production-auth and manual-deployment descriptions below. Model calls now use Gateway; production identity maps verified canonical subjects to unchanged local IDs. Do not deploy or activate a mount from historical staging instructions.
 
 ## What This Is
 
diff --git a/docs/fleet-integration.md b/docs/fleet-integration.md
index 8a191f6..a9b0028 100644
--- a/docs/fleet-integration.md
+++ b/docs/fleet-integration.md
@@ -41,7 +41,11 @@ The script refuses invalid subjects, unknown users, collisions and replacing an
 
 Set a random operator-held `READINESS_TOKEN` and the exact 40-character deployed `RELEASE_SHA`. `GET /internal/ready` requires `Authorization: Bearer <READINESS_TOKEN>`; unauthenticated probes get 404. It awaits identity configuration, checks the migrated user column, requires Gateway configuration and returns the exact release. It does not prove a successful Gateway model call or a browser sign-in. Do not publish this route through the institutional application mount.
 
-The existing SSH/manual staging workflow is historical and is not an approved institutional release mechanism. A subsequent reviewed release PR must establish the actual CI runner's supported private reachability; this proposal supplies no invented transport. The release design must:
+The protected manual CI workflow and host scripts are implemented in the
+[private release runbook](private-ci-release.md). They replace the historical
+password/sudo workflow with pinned Tailscale access, verified SSH keys and an
+immutable source archive. Host credentials, the protected environment and live
+reachability remain operator prerequisites. The implementation follows this order:
 
 1. Run the authoritative checks on main with private package read access, no deploy credentials in PR jobs, and a reviewed dependency lockfile.
 2. Serialize releases, reject stale queued SHAs, deploy the exact checked SHA, and obtain a consistent database/upload backup before the additive migration.
@@ -51,6 +55,13 @@ The existing SSH/manual staging workflow is historical and is not an approved in
 
 Source checks and disposable synthetic-provider tests are deterministic gates. Production traffic, paid model calls, deployment, mount activation and operator mappings are outside this branch's execution authority.
 
+The release additionally rechecks current main after its snapshot/restore
+rehearsal, verifies the existing database and upload paths against the running
+API, and restores prior process definitions on failure without replacing live
+data. Release safeguard tests exercise stale/failed sources, unsafe storage,
+partial stops, failed snapshots/rehearsals and failed readiness/recovery. CI
+transport does not establish browser ingress or live sign-in/model acceptance.
+
 ## Verification commands and scope
 
 `bun run check` runs Svelte type checking, unit/SQLite/Hono regressions, the existing shell checks, and API/web builds. The receiver integration test is explicitly skipped without `CAIL_GATEWAY_SOURCE`; ordinary CI does not require ambient sibling repositories or private cross-repository credentials.
diff --git a/docs/private-ci-release.md b/docs/private-ci-release.md
new file mode 100644
index 0000000..540e70e
--- /dev/null
+++ b/docs/private-ci-release.md
@@ -0,0 +1,94 @@
+# Private CI release
+
+The manual `Private fleet release` workflow replaces the historical password/
+sudo staging workflow. It is source implementation, not a record of deployment.
+The existing host is `100.111.252.53` on Tailscale. The workflow uses an ephemeral
+Tailscale node and SSH with a verified host key; it does not provision either
+access mechanism or expose a new public listener.
+
+## Operator prerequisites
+
+- Merge the receiver and release PRs, and wait for **Test and build** and
+  **Release safeguards** on the exact `main` push. The source check and release
+  use Node 22.23.1; the existing pnpm lock and version 9.15.0 remain unchanged.
+- Create the GitHub `production` environment with exactly one deployment branch
+  rule: branch `main`. Supply its `TS_OAUTH_CLIENT_ID`, `TS_OAUTH_SECRET`,
+  `SLIDE_SSH_USER`, `SLIDE_SSH_KEY` and `SLIDE_SSH_KNOWN_HOSTS` secrets. Obtain the
+  SSH fingerprint through the host's trusted administrative channel. Do not
+  populate known hosts from an unverified connection or disable host checking.
+- Restrict the OAuth client/tag to `tag:slide-maker-deploy`, with network access
+  only to this host's SSH port. Tailscale action, binary version and binary digest
+  are pinned. A network connection does not grant an SSH account permission.
+- The existing service owner must have key-based SSH access, Node 22.23.1,
+  pnpm 9.15.0, the tested PM2 CLI 7.0.4, `flock`, `lsof`, `ps`, `tar` and `sha256sum`. Use the same
+  service account and PM2 home as the existing API/web processes. No sudo,
+  password injection or automatic user creation is performed.
+- Pre-create `/data/slide-maker-releases` and its `incoming` subdirectory,
+  `/data/slide-maker-backups`, `/data/slide-maker-rehearsals` and
+  `/data/slide-maker-config`, owned by that account with mode 0700. These must
+  be separate from the current database and uploads. Allow room for the source,
+  dependencies, a full snapshot and a disposable restored copy.
+- In `/data/slide-maker-config/production.env` (owner only, mode 0600), preserve
+  existing application configuration and provide the canonical fleet values
+  from [fleet-integration.md](fleet-integration.md), `SESSION_SECRET`, a private
+  `READINESS_TOKEN`, and **verified existing** absolute `DATABASE_URL=file:...`
+  and `SLIDE_UPLOADS_PATH=...`. The script checks that the current API actually
+  holds that database and that its existing uploads resolve to that directory.
+  It refuses guessed, missing or deployment-local data paths. Keep package/CI
+  credentials and Node/PM2 override settings out of this application file.
+- Stop any independent database or upload writers for the release window.
+  The workflow stops the API and checks for remaining database handles; uploads
+  are verified unchanged while copying. Account linking is a separate,
+  independently verified operator operation; the release never matches email
+  addresses or grants local roles.
+
+The job token needs read access to the private CAIL packages. It crosses only
+the verified SSH stdin for installation and source checks, remains out of
+application environment and PM2 files, and is never written to disk. A temporary
+npm configuration contains only an environment placeholder and is removed even
+when installation fails. Command output that could include credentials stays
+out of CI logs. Snapshot and PM2 recovery files are private host data.
+
+## Release and failure behavior
+
+Dispatch from `main` with its full current SHA. The workflow loads its guard
+from trusted main, requires the latest successful main-push CI and release-check
+runs, and rejects stale SHAs or missing environment protection. It uses both
+GitHub concurrency and a host file lock. A release is a `git archive` of the
+verified SHA, transferred once, digest-checked, and extracted into a unique
+SHA/run/attempt directory. It never changes the old checkout or storage paths.
+
+After a frozen install and production build, the host validates both identity
+verifiers. It then records existing PM2 definitions, stops the API, verifies
+exclusive database access, creates a consistent SQLite/uploads snapshot, and
+restores and migrates a disposable copy. A further main/CI check precedes the
+live additive identity migration. Any earlier failure cannot migrate live data.
+The two application processes are recreated to prevent stale listener arguments.
+Both stay on `127.0.0.1` (API 3004, web 4173); the API runs TypeScript through
+the pinned installed runtime without a file watcher.
+
+Acceptance checks require exactly one online process for each component, the
+new source directory and SHA, one loopback listener on each port, unauthenticated
+readiness rejection, authenticated readiness at the exact SHA, and the mounted
+web response. Only then does PM2 save its state and the `current` symlink advance.
+
+A failure after stopping writers attempts to restore the captured old process
+definitions and save that process state. It does **not** restore the live database
+or uploads. The nullable identity migration is backward compatible with the
+previous code. Failed recovery is an explicit error. No upload, migration or
+activation is retried automatically; inspect the exact run before dispatching
+again. Failed release directories and state snapshots remain for investigation.
+
+## Remaining acceptance
+
+Source safeguards and disposable tests do not prove this host's credentials,
+network ACLs, storage paths, installed runtime or live traffic. The host release
+has not been run. The existing SSH account rejected the available local key.
+An operator must establish the prerequisites above and rehearse against actual
+production state before calling the release accepted.
+
+The CI connection does not create the private Doorway-to-Node browser ingress.
+Keep that mount disabled until its transport is established and the receiver is
+verified. Then separately exercise mapped-user CUNY login, unchanged local roles,
+old decks/files/sharing, a real Gateway model action, cancellation and exported
+file bytes through the deployed route, including public and campus access.
diff --git a/scripts/release/activate.mjs b/scripts/release/activate.mjs
new file mode 100644
index 0000000..7f8c280
--- /dev/null
+++ b/scripts/release/activate.mjs
@@ -0,0 +1,24 @@
+// Every callback is awaited. A failed backup or rehearsal cannot reach the live
+// migration. Rollback restores process definitions only, never application data.
+export async function activateRelease(steps) {
+  await steps.verifySource();
+  let stopped = false;
+  try {
+    stopped = true; // A partially failed stop still requires recovery.
+    await steps.stopWriters();
+    await steps.verifyWritersStopped();
+    await steps.backup();
+    await steps.rehearseRestore();
+    await steps.verifySource(); // Main may have advanced during the rehearsal.
+    await steps.migrate();
+    await steps.start();
+    await steps.verify();
+    await steps.save();
+  } catch (error) {
+    if (stopped) {
+      try { await steps.rollbackCode(); }
+      catch { throw new Error('Release failed and process recovery failed. Keep the saved state and inspect the host; no data was restored.'); }
+    }
+    throw error;
+  }
+}
diff --git a/scripts/release/activate.test.mjs b/scripts/release/activate.test.mjs
new file mode 100644
index 0000000..43398d2
--- /dev/null
+++ b/scripts/release/activate.test.mjs
@@ -0,0 +1,42 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { activateRelease } from './activate.mjs';
+
+function fixture(failure, rollbackFails = false) {
+  const events = [];
+  let verifies = 0;
+  const steps = Object.fromEntries(['verifySource', 'stopWriters', 'verifyWritersStopped', 'backup', 'rehearseRestore', 'migrate', 'start', 'verify', 'save', 'rollbackCode'].map(name => [name, async () => {
+    events.push(name);
+    if (name === 'verifySource') verifies++;
+    if (failure === name || (failure === 'staleAfterBackup' && name === 'verifySource' && verifies === 2) || (name === 'rollbackCode' && rollbackFails)) throw new Error(`Failed ${name}`);
+  }]));
+  return { steps, events };
+}
+test('release verifies source, stops writers, rehearses restoration, and verifies new code before saving', async () => {
+  const f = fixture(); await activateRelease(f.steps);
+  assert.deepEqual(f.events, ['verifySource', 'stopWriters', 'verifyWritersStopped', 'backup', 'rehearseRestore', 'verifySource', 'migrate', 'start', 'verify', 'save']);
+});
+test('initial source refusal cannot stop the application', async () => {
+  const f = fixture('verifySource'); await assert.rejects(activateRelease(f.steps));
+  assert.deepEqual(f.events, ['verifySource']);
+});
+test('failed stop, exclusive-access check, backup, rehearsal or superseded SHA cannot migrate live data', async () => {
+  for (const failure of ['stopWriters', 'verifyWritersStopped', 'backup', 'rehearseRestore', 'staleAfterBackup']) {
+    const f = fixture(failure); await assert.rejects(activateRelease(f.steps));
+    assert.equal(f.events.at(-1), 'rollbackCode');
+    for (const name of ['migrate', 'start', 'verify', 'save']) assert.ok(!f.events.includes(name), `${failure} reached ${name}`);
+  }
+});
+test('migration, start, readiness and persistence failures recover code without restoring live data', async () => {
+  for (const failure of ['migrate', 'start', 'verify', 'save']) {
+    const f = fixture(failure); await assert.rejects(activateRelease(f.steps));
+    assert.equal(f.events.at(-1), 'rollbackCode');
+    assert.equal(f.events.filter(name => name === 'rehearseRestore').length, 1);
+    if (failure !== 'save') assert.ok(!f.events.includes('save'));
+  }
+});
+test('a failed code recovery is explicit and cannot report release success', async () => {
+  const f = fixture('verify', true);
+  await assert.rejects(activateRelease(f.steps), /process recovery failed/);
+  assert.ok(!f.events.includes('save'));
+});
diff --git a/scripts/release/contract.mjs b/scripts/release/contract.mjs
new file mode 100644
index 0000000..1167c0c
--- /dev/null
+++ b/scripts/release/contract.mjs
@@ -0,0 +1,47 @@
+import { isAbsolute, resolve, sep } from 'node:path';
+
+export const paths = Object.freeze({
+  releases: '/data/slide-maker-releases',
+  backups: '/data/slide-maker-backups',
+  rehearsals: '/data/slide-maker-rehearsals',
+  environment: '/data/slide-maker-config/production.env',
+});
+export function releaseIdentity(sha, run, attempt) {
+  if (!/^[a-f0-9]{40}$/.test(sha || '') || !/^[1-9][0-9]*$/.test(run || '') || !/^[1-9][0-9]*$/.test(attempt || '')) {
+    throw new Error('An immutable SHA and GitHub run identity are required.');
+  }
+  return `${sha}-${run}-${attempt}`;
+}
+export function isWithin(root, path) { return path === root || path.startsWith(`${root}${sep}`); }
+export function validateConfiguration(env) {
+  for (const key of ['SESSION_SECRET', 'READINESS_TOKEN', 'CAIL_IDENTITY_JWKS']) {
+    if (typeof env[key] !== 'string' || !env[key].trim()) throw new Error(`Production configuration is missing ${key}.`);
+  }
+  if (env.CAIL_IDENTITY_ISSUER !== 'https://tools.ailab.gc.cuny.edu/cail-sso' ||
+      env.CAIL_GATEWAY_URL !== 'https://tools.ailab.gc.cuny.edu' ||
+      env.PUBLIC_URL !== 'https://tools.ailab.gc.cuny.edu/slide-maker') throw new Error('Production identity, Gateway or public URL is not canonical.');
+  for (const key of ['NODE_OPTIONS', 'NODE_PATH', 'NODE_AUTH_TOKEN', 'GITHUB_TOKEN', 'NPM_CONFIG_USERCONFIG', 'PM2_HOME']) {
+    if (env[key]) throw new Error(`Remove deployment-only or runtime-override setting ${key} from the application environment.`);
+  }
+  const database = env.DATABASE_URL?.startsWith('file:') ? env.DATABASE_URL.slice(5) : '';
+  const uploads = env.SLIDE_UPLOADS_PATH;
+  for (const value of [database, uploads]) {
+    if (!value || !isAbsolute(value) || resolve(value) !== value || value === '/' || /[\r\n\0]/.test(value)) throw new Error('Existing database and uploads must have explicit absolute paths.');
+    if ([paths.releases, paths.backups, paths.rehearsals].some(root => isWithin(root, value))) throw new Error('Live state must remain outside release, backup and rehearsal directories.');
+  }
+  if (isWithin(uploads, database)) throw new Error('Database and uploads must be separate.');
+  return { database, uploads };
+}
+export function processDefinitions(list) {
+  return ['slide-maker-api', 'slide-maker-web'].map(name => {
+    const matches = list.filter(item => item.name === name);
+    if (matches.length !== 1 || matches[0].pm2_env?.status !== 'online' || matches[0].pm2_env?.exec_mode !== 'fork_mode') {
+      throw new Error('Exactly one online fork process is required for each existing application component.');
+    }
+    const current = matches[0].pm2_env;
+    if (!isAbsolute(current.pm_exec_path || '') || !isAbsolute(current.pm_cwd || '')) throw new Error('Existing process paths are invalid.');
+    return { name, script: current.pm_exec_path, cwd: current.pm_cwd,
+      interpreter: current.exec_interpreter, args: current.args || [], node_args: current.node_args || [],
+      env: current.env || {}, exec_mode: 'fork', instances: 1, autorestart: true };
+  });
+}
diff --git a/scripts/release/contract.test.mjs b/scripts/release/contract.test.mjs
new file mode 100644
index 0000000..ac53d3e
--- /dev/null
+++ b/scripts/release/contract.test.mjs
@@ -0,0 +1,36 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { processDefinitions, releaseIdentity, validateConfiguration } from './contract.mjs';
+
+const environment = {
+  SESSION_SECRET: 'fixture', READINESS_TOKEN: 'fixture', CAIL_IDENTITY_JWKS: 'public-fixture',
+  CAIL_IDENTITY_ISSUER: 'https://tools.ailab.gc.cuny.edu/cail-sso',
+  CAIL_GATEWAY_URL: 'https://tools.ailab.gc.cuny.edu', PUBLIC_URL: 'https://tools.ailab.gc.cuny.edu/slide-maker',
+  DATABASE_URL: 'file:/data/existing/database.sqlite', SLIDE_UPLOADS_PATH: '/data/existing/uploads',
+};
+test('release identity cannot contain a ref, path or shell expression', () => {
+  assert.equal(releaseIdentity('a'.repeat(40), '12', '1'), `${'a'.repeat(40)}-12-1`);
+  for (const values of [['main', '12', '1'], ['a'.repeat(40), '../12', '1'], ['a'.repeat(40), '12', '$(whoami)']]) assert.throws(() => releaseIdentity(...values));
+});
+test('state paths must be explicit, separate and outside deployment storage', () => {
+  assert.deepEqual(validateConfiguration(environment), { database: '/data/existing/database.sqlite', uploads: '/data/existing/uploads' });
+  for (const patch of [
+    { DATABASE_URL: 'file:./data/app.db' }, { SLIDE_UPLOADS_PATH: '' }, { SLIDE_UPLOADS_PATH: '/' },
+    { DATABASE_URL: 'file:/data/slide-maker-releases/old/db' }, { SLIDE_UPLOADS_PATH: '/data/slide-maker-backups/new' },
+    { DATABASE_URL: 'file:/data/existing/uploads/database.sqlite' }, { SLIDE_UPLOADS_PATH: '/data/existing/../uploads' },
+  ]) assert.throws(() => validateConfiguration({ ...environment, ...patch }));
+});
+test('application configuration cannot silently replace fleet endpoints or retain CI credentials', () => {
+  for (const patch of [
+    { CAIL_GATEWAY_URL: 'https://tools.ailab.gc.cuny.edu/v1' }, { CAIL_IDENTITY_ISSUER: 'https://wrong.test' },
+    { PUBLIC_URL: 'http://localhost:4173' }, { READINESS_TOKEN: '' }, { GITHUB_TOKEN: 'fixture' }, { NODE_AUTH_TOKEN: 'fixture' }, { NODE_OPTIONS: '--inspect=0.0.0.0' },
+  ]) assert.throws(() => validateConfiguration({ ...environment, ...patch }));
+});
+test('rollback captures exactly one online process per component and preserves existing arguments and environment', () => {
+  const list = ['slide-maker-api', 'slide-maker-web'].map(name => ({ name, pm2_env: { status: 'online', exec_mode: 'fork_mode', pm_exec_path: '/usr/bin/node', pm_cwd: '/data/old', args: ['server.js'], env: { DATABASE_URL: environment.DATABASE_URL } } }));
+  assert.equal(processDefinitions(list)[0].env.DATABASE_URL, environment.DATABASE_URL);
+  assert.deepEqual(processDefinitions(list)[0].args, ['server.js']);
+  assert.throws(() => processDefinitions([list[0]]));
+  assert.throws(() => processDefinitions([...list, list[0]]));
+  assert.throws(() => processDefinitions([{ ...list[0], pm2_env: { ...list[0].pm2_env, status: 'stopped' } }, list[1]]));
+});
diff --git a/scripts/release/host.mjs b/scripts/release/host.mjs
new file mode 100644
index 0000000..fb6f0ee
--- /dev/null
+++ b/scripts/release/host.mjs
@@ -0,0 +1,156 @@
+import { execFileSync } from 'node:child_process';
+import { existsSync, lstatSync, mkdirSync, readFileSync, realpathSync, renameSync, rmSync, statSync, symlinkSync, writeFileSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { parseEnv } from 'node:util';
+import { pathToFileURL } from 'node:url';
+import { activateRelease } from './activate.mjs';
+import { applicationDefinitions, replaceProcesses } from './processes.mjs';
+import { paths, isWithin, processDefinitions, releaseIdentity, validateConfiguration } from './contract.mjs';
+import { config, githubApi, verifyReleaseSource } from './verify-source.mjs';
+
+function run(command, args, options = {}) {
+  try { return execFileSync(command, args, { encoding: 'utf8', timeout: 120000, maxBuffer: 8 * 1024 * 1024, stdio: ['ignore', 'pipe', 'pipe'], ...options }); }
+  catch { throw new Error(`Host release command failed: ${command}. Its output is withheld because it may contain private configuration.`); }
+}
+function json(text) { try { return JSON.parse(text); } catch { throw new Error('Invalid release command response.'); } }
+function privatePath(path, directory = true) {
+  const info = lstatSync(path);
+  if ((directory ? !info.isDirectory() : !info.isFile()) || info.uid !== process.getuid() || (info.mode & 0o077)) {
+    throw new Error('Release directories and configuration must be owned by this user and accessible only to this user.');
+  }
+}
+async function pause() { await new Promise(resolve => setTimeout(resolve, 1000)); }
+export async function main() {
+  process.umask(0o077);
+  if (process.platform !== 'linux' || process.version !== 'v22.23.1') throw new Error('Release host requires Linux and Node 22.23.1.');
+  const [sha, runId, attempt] = process.argv.slice(2);
+  const id = releaseIdentity(sha, runId, attempt);
+  const root = join(paths.releases, id);
+  if (realpathSync(new URL('../..', import.meta.url)) !== root) throw new Error('Release source is outside its immutable directory.');
+  for (const path of [paths.releases, paths.backups, paths.rehearsals, root]) privatePath(path);
+  privatePath(paths.environment, false);
+  const environment = parseEnv(readFileSync(paths.environment, 'utf8'));
+  const storage = validateConfiguration(environment);
+  for (const key of ['database', 'uploads']) {
+    storage[key] = realpathSync(storage[key]);
+    if ([paths.releases, paths.backups, paths.rehearsals].some(path => isWithin(path, storage[key]))) throw new Error('Live state resolves inside deployment storage.');
+  }
+  if (!statSync(storage.database).isFile() || !statSync(storage.uploads).isDirectory() || isWithin(storage.uploads, storage.database)) throw new Error('Existing live state is invalid.');
+  const sourceToken = readFileSync(0, 'utf8'); // One ephemeral token, sent over verified SSH stdin only.
+  if (sourceToken.length > 16384 || !/^[A-Za-z0-9_]+$/.test(sourceToken)) throw new Error('GitHub read token is missing or malformed.');
+  const verifySource = () => verifyReleaseSource({ ...config, sha, environment: 'production', api: githubApi(sourceToken) });
+  const baseEnv = { PATH: process.env.PATH, HOME: process.env.HOME, LANG: 'C.UTF-8' };
+  const appEnv = { ...baseEnv, ...environment, DATABASE_URL: `file:${storage.database}`, NODE_ENV: 'production', API_HOST: '127.0.0.1', API_PORT: '3004', RELEASE_SHA: sha };
+  const command = (exe, args, options = {}) => run(exe, args, { cwd: root, env: baseEnv, ...options });
+  if (command('pnpm', ['--version']).trim() !== '9.15.0') throw new Error('Release host requires pnpm 9.15.0.');
+  if (command('pm2', ['--version']).trim() !== '7.0.4') throw new Error('Release host requires the tested PM2 CLI 7.0.4.');
+  command('lsof', ['-v']);
+  await verifySource();
+  // Install output stays private. The temporary credential is never written or
+  // included in PM2 environment; the only config file contains a placeholder.
+  if (readFileSync(join(root, '.npmrc'), 'utf8').trim() !== '@cuny-ai-lab:registry=https://npm.pkg.github.com') throw new Error('Unexpected tracked npm configuration.');
+  const npmrc = join(root, '.release-npmrc');
+  writeFileSync(npmrc, '@cuny-ai-lab:registry=https://npm.pkg.github.com\n//npm.pkg.github.com/:_authToken=${NODE_AUTH_TOKEN}\n', { mode: 0o600, flag: 'wx' });
+  try { command('pnpm', ['install', '--frozen-lockfile'], { timeout: 600000, env: { ...baseEnv, NODE_AUTH_TOKEN: sourceToken, NPM_CONFIG_USERCONFIG: npmrc } }); }
+  finally { rmSync(npmrc, { force: true }); }
+  command('pnpm', ['build'], { timeout: 600000, env: { ...baseEnv, NODE_ENV: 'production' } });
+  const apiDir = join(root, 'apps/api');
+  const node = (args, extraEnv = {}) => command(process.execPath, ['--import', 'tsx', ...args], { cwd: apiDir, env: { ...appEnv, ...extraEnv } });
+  node(['--input-type=module', '-e', "import {loadIdentityConfigs} from './src/auth/identity.ts'; await loadIdentityConfigs();"]);
+  if (existsSync(join(apiDir, 'uploads'))) throw new Error('Release source contains an unexpected uploads directory.');
+  symlinkSync(storage.uploads, join(apiDir, 'uploads'));
+
+  const current = join(paths.releases, 'current');
+  let previousTarget;
+  try {
+    if (!lstatSync(current).isSymbolicLink()) throw new Error('Current release marker must be a symlink.');
+    previousTarget = realpathSync(current);
+  } catch (error) { if (error.code !== 'ENOENT') throw error; }
+  const list = () => json(command('pm2', ['jlist'])); // Never print PM2 environments.
+  const oldList = list();
+  const oldApps = processDefinitions(oldList);
+  // Confirm that the configured state is actually used by the running API,
+  // including the legacy pnpm -> tsx child process chain. Never migrate a
+  // guessed path that happens to contain another valid SQLite database.
+  const apiPid = oldList.find(item => item.name === 'slide-maker-api').pid;
+  const owners = command('lsof', ['-t', storage.database]).trim().split(/\s+/).map(Number);
+  for (let owner of owners) {
+    let matched = false;
+    for (let depth = 0; depth < 32 && owner > 1; depth++) {
+      if (owner === apiPid) { matched = true; break; }
+      owner = Number(command('ps', ['-o', 'ppid=', '-p', String(owner)]).trim());
+    }
+    if (!matched) throw new Error('Configured database is not exclusively held by the existing API process tree.');
+  }
+  if (!owners.length || owners.some(pid => !Number.isSafeInteger(pid) || pid < 2)) throw new Error('Could not establish the existing API database.');
+  const oldApiCwd = oldApps[0].cwd;
+  const oldUploads = existsSync(join(oldApiCwd, 'apps/api/uploads')) ? join(oldApiCwd, 'apps/api/uploads') : join(oldApiCwd, 'uploads');
+  if (realpathSync(oldUploads) !== storage.uploads) throw new Error('Configured uploads do not match the existing API storage.');
+  const backupRoot = join(paths.backups, id);
+  mkdirSync(backupRoot, { mode: 0o700 });
+  const rollback = join(backupRoot, 'previous-processes.json');
+  writeFileSync(rollback, JSON.stringify({ apps: oldApps }), { mode: 0o600, flag: 'wx' });
+  const next = join(backupRoot, 'new-processes.json');
+  const nextApps = applicationDefinitions(root, appEnv, baseEnv);
+  writeFileSync(next, JSON.stringify({ apps: nextApps }), { mode: 0o600, flag: 'wx' });
+  async function waitFor(check) {
+    for (let i = 0; i < 30; i++) { try { await check(); return; } catch { await pause(); } }
+    throw new Error('Release process or private readiness verification timed out.');
+  }
+  function mark(target) {
+    const temporary = join(paths.releases, `.current-${id}`);
+    symlinkSync(target, temporary);
+    renameSync(temporary, current);
+  }
+  await activateRelease({
+    verifySource,
+    stopWriters: () => command('pm2', ['stop', 'slide-maker-api']),
+    verifyWritersStopped: () => {
+      const databaseFiles = [storage.database, `${storage.database}-wal`, `${storage.database}-shm`].filter(existsSync);
+      try { execFileSync('lsof', ['-t', ...databaseFiles], { env: baseEnv, stdio: ['ignore', 'pipe', 'pipe'], timeout: 10000 }); }
+      catch (error) { if (error.status === 1 && !error.stdout?.length && !error.stderr?.length) return; }
+      throw new Error('A database process is still open or exclusive access could not be verified.');
+    },
+    backup: () => node(['scripts/state-snapshot.ts', 'backup', storage.database, storage.uploads, join(backupRoot, 'snapshot'), '--writers-stopped']),
+    rehearseRestore: () => {
+      const destination = join(paths.rehearsals, id);
+      node(['scripts/state-snapshot.ts', 'restore-new', join(backupRoot, 'snapshot'), destination]);
+      node(['scripts/migrate-identity.ts'], { DATABASE_URL: `file:${join(destination, 'database.sqlite')}` });
+    },
+    migrate: () => node(['scripts/migrate-identity.ts']),
+    start: () => replaceProcesses(command, list, next),
+    verify: () => waitFor(async () => {
+      const definitions = processDefinitions(list());
+      for (const [index, expected] of nextApps.entries()) {
+        if (definitions[index].cwd !== expected.cwd || definitions[index].env.RELEASE_SHA !== sha) throw new Error('Unexpected active source.');
+      }
+      for (const port of [3004, 4173]) {
+        const listeners = command('lsof', ['-nP', `-iTCP:${port}`, '-sTCP:LISTEN', '-Fn']).split('\n').filter(line => line.startsWith('n'));
+        if (listeners.length !== 1 || listeners[0] !== `n127.0.0.1:${port}`) throw new Error('Expected one loopback-only listener per component.');
+      }
+      const request = (url, headers) => fetch(url, { headers, redirect: 'error', signal: AbortSignal.timeout(3000) });
+      const denied = await request('http://127.0.0.1:3004/internal/ready');
+      await denied.body?.cancel();
+      if (denied.status !== 404) throw new Error('Readiness is not private.');
+      const response = await request('http://127.0.0.1:3004/internal/ready', { Authorization: `Bearer ${environment.READINESS_TOKEN}` });
+      const ready = await response.json();
+      if (!response.ok || ready.status !== 'ready' || ready.release !== sha || ready.service !== 'slide-maker') throw new Error('Exact release is not ready.');
+      const web = await request('http://127.0.0.1:4173/slide-maker/');
+      await web.body?.cancel();
+      if (!web.ok) throw new Error('Frontend is unavailable.');
+    }),
+    save: () => { command('pm2', ['save']); mark(root); },
+    rollbackCode: async () => {
+      replaceProcesses(command, list, rollback);
+      await waitFor(() => {
+        const restored = processDefinitions(list());
+        if (restored.some((app, index) => app.cwd !== oldApps[index].cwd || app.script !== oldApps[index].script)) throw new Error('Prior processes did not recover.');
+      });
+      command('pm2', ['save']);
+      if (previousTarget) mark(previousTarget);
+      console.log('Recovered prior application processes. Live database and uploads were not restored.');
+    },
+  });
+  console.log(`Verified release ${sha}, one loopback process per component, a state snapshot and restore rehearsal, and private readiness. Live signed-in acceptance remains separate.`);
+}
+if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) main().catch(error => { console.error(error.message); process.exitCode = 1; });
diff --git a/scripts/release/private-ssh.mjs b/scripts/release/private-ssh.mjs
new file mode 100644
index 0000000..5e0b330
--- /dev/null
+++ b/scripts/release/private-ssh.mjs
@@ -0,0 +1,49 @@
+import { execFileSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { paths, releaseIdentity } from './contract.mjs';
+import { config, verifyCurrentRelease } from './verify-source.mjs';
+
+const host = '100.111.252.53';
+function run(command, args, options = {}) {
+  try { return execFileSync(command, args, { encoding: 'utf8', timeout: 60000, maxBuffer: 1024 * 1024, stdio: ['ignore', 'pipe', 'pipe'], ...options }); }
+  catch { throw new Error(`Private release command failed: ${command}. Inspect the host before retrying; no automatic upload or activation retry was made.`); }
+}
+// Values passed to the remote shell are generated IDs, digests or fixed paths.
+function quote(value) { return `'${value.replaceAll("'", "'\\''")}'`; }
+async function main() {
+  const sha = await verifyCurrentRelease(config);
+  const id = releaseIdentity(sha, process.env.GITHUB_RUN_ID, process.env.GITHUB_RUN_ATTEMPT);
+  const username = process.env.SLIDE_SSH_USER;
+  if (!/^[a-zA-Z0-9._@-]+$/.test(username || '') || !process.env.SLIDE_SSH_KEY || !process.env.SLIDE_SSH_KNOWN_HOSTS) throw new Error('Private SSH user, key and verified known-host entries are required.');
+  const temporary = mkdtempSync(join(process.env.RUNNER_TEMP || tmpdir(), 'slide-release-'));
+  try {
+    const key = join(temporary, 'key');
+    const hosts = join(temporary, 'known_hosts');
+    writeFileSync(key, `${process.env.SLIDE_SSH_KEY.trim()}\n`, { mode: 0o600 });
+    writeFileSync(hosts, `${process.env.SLIDE_SSH_KNOWN_HOSTS.trim()}\n`, { mode: 0o600 });
+    run('ssh-keygen', ['-F', host, '-f', hosts]);
+    const options = ['-o', `User=${username}`, '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=yes', '-o', 'IdentitiesOnly=yes', '-o', 'ConnectTimeout=15', '-o', 'ServerAliveInterval=15', '-o', 'ServerAliveCountMax=3', '-o', `UserKnownHostsFile=${hosts}`, '-i', key];
+    const archive = join(temporary, 'source.tar');
+    run('git', ['archive', '--format=tar', '--output', archive, sha]);
+    const digest = createHash('sha256').update(readFileSync(archive)).digest('hex');
+    const incoming = join(paths.releases, 'incoming', `${id}.tar`);
+    const root = join(paths.releases, id);
+    // Pre-provisioned, private directories; never overwrite an incoming release.
+    run('ssh', [...options, host, `test ! -e ${quote(incoming)} && test ! -e ${quote(root)}`]);
+    run('scp', [...options, archive, `${host}:${incoming}`], { timeout: 120000 });
+    await verifyCurrentRelease(config);
+    const command = [
+      'set -eu', 'umask 077',
+      `exec 9>${quote(join(paths.releases, 'release.lock'))}`, 'flock -n 9',
+      `test "$(sha256sum ${quote(incoming)} | cut -d ' ' -f 1)" = ${quote(digest)}`,
+      `mkdir ${quote(root)}`, `tar -xf ${quote(incoming)} -C ${quote(root)}`,
+      `node ${quote(join(root, 'scripts/release/host.mjs'))} ${quote(sha)} ${quote(process.env.GITHUB_RUN_ID)} ${quote(process.env.GITHUB_RUN_ATTEMPT)}`,
+    ].join('\n');
+    const result = run('ssh', [...options, host, command], { input: process.env.GITHUB_TOKEN, timeout: 25 * 60 * 1000 });
+    console.log(result.trim());
+  } finally { rmSync(temporary, { recursive: true, force: true }); }
+}
+main().catch(error => { console.error(error.message); process.exitCode = 1; });
diff --git a/scripts/release/processes.integration.test.mjs b/scripts/release/processes.integration.test.mjs
new file mode 100644
index 0000000..e4fc049
--- /dev/null
+++ b/scripts/release/processes.integration.test.mjs
@@ -0,0 +1,70 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { execFileSync } from 'node:child_process';
+import { mkdtempSync, mkdirSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { applicationDefinitions, replaceProcesses } from './processes.mjs';
+import { processDefinitions } from './contract.mjs';
+
+const pm2 = process.env.CAIL_RELEASE_PM2;
+test('real isolated PM2 starts the deployed runtime profile and restores captured processes and environment', { skip: !pm2, timeout: 60000 }, async () => {
+  const temporary = realpathSync(mkdtempSync(join(tmpdir(), 'slide-pm2-test-')));
+  const baseEnv = { PATH: process.env.PATH, HOME: process.env.HOME, PM2_HOME: join(temporary, 'pm2'), LANG: 'C.UTF-8' };
+  const command = (_, args) => execFileSync(process.execPath, [pm2, ...args], { env: baseEnv, encoding: 'utf8', timeout: 15000, stdio: ['ignore', 'pipe', 'pipe'] });
+  const list = () => JSON.parse(command('pm2', ['jlist']));
+  const typescriptModules = dirname(dirname(dirname(pm2)));
+  async function ready(root, sha) {
+    let failure;
+    for (let attempt = 0; attempt < 40; attempt++) {
+      try {
+        const definitions = processDefinitions(list());
+        assert.equal(definitions[0].cwd, join(root, 'apps/api'));
+        for (const [index, name] of ['api', 'web'].entries()) {
+          const url = readFileSync(join(root, `${name}.url`), 'utf8');
+          const response = await fetch(url, { signal: AbortSignal.timeout(1000) });
+          const body = await response.json();
+          assert.equal(body.release, sha);
+          assert.equal(body.cwd, definitions[index].cwd);
+          assert.equal(body.ciToken, undefined);
+          assert.equal(body.applicationSecret, index === 0 ? 'preserved-fixture-secret' : undefined);
+        }
+        return;
+      } catch (error) { failure = error; await new Promise(resolve => setTimeout(resolve, 150)); }
+    }
+    const diagnostic = list().map(item => ({ name: item.name, status: item.pm2_env?.status, cwd: item.pm2_env?.pm_cwd, error: readFileSync(item.pm2_env.pm_err_log_path, 'utf8').slice(-1500) }));
+    assert.fail(`Disposable PM2 release did not become ready: ${failure?.message}; ${JSON.stringify(diagnostic)}`);
+  }
+  function stage(label, sha) {
+    const root = join(temporary, label);
+    for (const path of ['apps/api/src', 'apps/web/node_modules/vite/bin']) mkdirSync(join(root, path), { recursive: true });
+    writeFileSync(join(root, 'package.json'), '{"type":"module"}');
+    writeFileSync(join(root, 'apps/web/node_modules/vite/package.json'), '{"type":"module"}');
+    symlinkSync(typescriptModules, join(root, 'apps/api/node_modules'));
+    const server = name => `import {createServer} from 'node:http'; import {writeFileSync} from 'node:fs';\nconst server=createServer((q,r)=>r.end(JSON.stringify({release:process.env.RELEASE_SHA,cwd:process.cwd(),applicationSecret:process.env.SESSION_SECRET,ciToken:process.env.GITHUB_TOKEN})));\nserver.listen(0,'127.0.0.1',()=>writeFileSync(${JSON.stringify(join(root, `${name}.url`))},'http://127.0.0.1:'+server.address().port));`;
+    writeFileSync(join(root, 'apps/api/src/index.ts'), server('api'));
+    writeFileSync(join(root, 'apps/web/node_modules/vite/bin/vite.js'), server('web'));
+    const apps = applicationDefinitions(root, { ...baseEnv, RELEASE_SHA: sha, SESSION_SECRET: 'preserved-fixture-secret' }, baseEnv);
+    const file = join(root, 'ecosystem.json');
+    writeFileSync(file, JSON.stringify({ apps }));
+    return { root, file };
+  }
+  try {
+    const old = stage('old', 'a'.repeat(40));
+    command('pm2', ['start', old.file]);
+    await ready(old.root, 'a'.repeat(40));
+    const original = processDefinitions(list());
+    const rollback = join(temporary, 'rollback.json');
+    writeFileSync(rollback, JSON.stringify({ apps: original }));
+    const next = stage('new', 'b'.repeat(40));
+    replaceProcesses(command, list, next.file);
+    await ready(next.root, 'b'.repeat(40));
+    command('pm2', ['save']);
+    replaceProcesses(command, list, rollback);
+    await ready(old.root, 'a'.repeat(40));
+    command('pm2', ['save']);
+    assert.equal(list().length, 2);
+  } finally {
+    try { command('pm2', ['kill']); } finally { rmSync(temporary, { recursive: true, force: true }); }
+  }
+});
diff --git a/scripts/release/processes.mjs b/scripts/release/processes.mjs
new file mode 100644
index 0000000..b5e1151
--- /dev/null
+++ b/scripts/release/processes.mjs
@@ -0,0 +1,14 @@
+import { join } from 'node:path';
+
+export function applicationDefinitions(root, appEnv, baseEnv, node = process.execPath) {
+  return [
+    { name: 'slide-maker-api', script: node, args: ['--import', 'tsx', 'src/index.ts'], cwd: join(root, 'apps/api'), interpreter: 'none', env: appEnv, exec_mode: 'fork', instances: 1 },
+    { name: 'slide-maker-web', script: node, args: [join(root, 'apps/web/node_modules/vite/bin/vite.js'), 'preview', '--host', '127.0.0.1', '--port', '4173', '--strictPort'], cwd: join(root, 'apps/web'), interpreter: 'none', env: { ...baseEnv, NODE_ENV: 'production', RELEASE_SHA: appEnv.RELEASE_SHA }, exec_mode: 'fork', instances: 1 },
+  ];
+}
+export function replaceProcesses(command, list, file) {
+  for (const name of ['slide-maker-api', 'slide-maker-web']) {
+    if (list().some(item => item.name === name)) command('pm2', ['delete', name]);
+  }
+  command('pm2', ['start', file]);
+}
diff --git a/scripts/release/verify-source.mjs b/scripts/release/verify-source.mjs
new file mode 100644
index 0000000..af8e69d
--- /dev/null
+++ b/scripts/release/verify-source.mjs
@@ -0,0 +1,66 @@
+import { execFileSync } from 'node:child_process';
+import { pathToFileURL } from 'node:url';
+
+export async function verifyReleaseSource({ repository, sha, checks, api, environment }) {
+  if (!/^[a-f0-9]{40}$/.test(sha || '')) throw new Error('Release requires a full lowercase commit SHA.');
+  const ref = await api(`repos/${repository}/git/ref/heads/main`);
+  if (ref.object?.sha !== sha) throw new Error('Release SHA is no longer current main.');
+  for (const [workflow, names] of Object.entries(checks)) {
+    const result = await api(`repos/${repository}/actions/workflows/${workflow}/runs?head_sha=${sha}&event=push&branch=main&per_page=20`);
+    const run = result.workflow_runs?.filter(item => item.head_sha === sha && item.event === 'push' && item.head_branch === 'main')
+      .sort((a, b) => b.id - a.id)[0];
+    if (!run || run.status !== 'completed' || run.conclusion !== 'success') throw new Error(`Current-main ${workflow} has not passed.`);
+    const jobs = await api(`repos/${repository}/actions/runs/${run.id}/jobs?per_page=100`);
+    for (const name of names) {
+      const matches = jobs.jobs?.filter(job => job.name === name) || [];
+      if (matches.length !== 1 || matches[0].conclusion !== 'success') throw new Error(`Required release check has not passed: ${name}`);
+    }
+  }
+  if (environment) {
+    const config = await api(`repos/${repository}/environments/${environment}`);
+    if (config.deployment_branch_policy?.custom_branch_policies !== true) throw new Error('Release environment must allow only main.');
+    const rules = await api(`repos/${repository}/environments/${environment}/deployment-branch-policies`);
+    if (rules.branch_policies?.length !== 1 || rules.branch_policies[0].name !== 'main' || rules.branch_policies[0].type !== 'branch') {
+      throw new Error('Release environment must allow only the main branch.');
+    }
+  }
+}
+
+export function githubApi(token, transport = fetch) {
+  if (!token) throw new Error('GitHub read token is required.');
+  return async path => {
+    const response = await transport(new URL(path, 'https://api.github.com/'), {
+      headers: { Authorization: `Bearer ${token}`, Accept: 'application/vnd.github+json', 'X-GitHub-Api-Version': '2022-11-28' },
+      redirect: 'error', signal: AbortSignal.timeout(15000),
+    });
+    if (!response.ok) throw new Error(`GitHub release verification failed (${response.status}).`);
+    const text = await response.text();
+    if (text.length > 2 * 1024 * 1024) throw new Error('GitHub release verification response is too large.');
+    return JSON.parse(text);
+  };
+}
+
+export async function verifyCurrentRelease(config) {
+  if (process.env.GITHUB_EVENT_NAME !== 'workflow_dispatch' || process.env.GITHUB_REF !== 'refs/heads/main' ||
+      process.env.GITHUB_REPOSITORY !== config.repository) throw new Error('Release must be dispatched from canonical main.');
+  const sha = process.env.RELEASE_SHA;
+  const head = execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim();
+  if (head !== sha || execFileSync('git', ['status', '--porcelain'], { encoding: 'utf8' }).trim()) throw new Error('Release checkout must be clean at the requested SHA.');
+  await verifyReleaseSource({ ...config, sha, api: githubApi(process.env.GITHUB_TOKEN), environment: 'production' });
+  return sha;
+}
+
+export const config = {
+  "repository": "CUNY-AI-Lab/slide-maker",
+  "checks": {
+    "ci.yml": [
+      "Test and build"
+    ],
+    "release-checks.yml": [
+      "Release safeguards"
+    ]
+  }
+};
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+  verifyCurrentRelease(config).then(sha => console.log(`Release source verified: ${sha}`)).catch(error => { console.error(error.message); process.exitCode = 1; });
+}
diff --git a/scripts/release/verify-source.test.mjs b/scripts/release/verify-source.test.mjs
new file mode 100644
index 0000000..149eb41
--- /dev/null
+++ b/scripts/release/verify-source.test.mjs
@@ -0,0 +1,42 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { verifyReleaseSource } from './verify-source.mjs';
+const sha = 'a'.repeat(40);
+const goodRun = { id: 10, head_sha: sha, head_branch: 'main', event: 'push', status: 'completed', conclusion: 'success' };
+const base = { repository: 'CUNY-AI-Lab/example', sha, checks: { 'ci.yml': ['Source gate'] }, environment: 'production' };
+function fixture(overrides = {}) {
+  const visited = [];
+  return { visited, api: async path => {
+    visited.push(path);
+    if (path.endsWith('/git/ref/heads/main')) return { object: { sha: overrides.head || sha } };
+    if (path.includes('/workflows/')) return { workflow_runs: overrides.runs || [goodRun] };
+    if (path.includes('/jobs?')) return { jobs: overrides.jobs || [{ name: 'Source gate', conclusion: 'success' }] };
+    if (path.endsWith('/deployment-branch-policies')) return { branch_policies: overrides.branches || [{ name: 'main', type: 'branch' }] };
+    if (path.endsWith('/environments/production')) return { deployment_branch_policy: { custom_branch_policies: overrides.protected !== false } };
+    throw new Error('Unexpected API request');
+  } };
+}
+test('release requires exact current main, successful main CI and a protected main-only environment', async () => {
+  const f = fixture(); await verifyReleaseSource({ ...base, api: f.api }); assert.equal(f.visited.length, 5);
+});
+test('stale or malformed SHA stops before deployment prerequisites are accessed', async () => {
+  for (const options of [{ ...base, sha: 'main' }, { ...base, sha: 'b'.repeat(40) }]) {
+    const f = fixture(); await assert.rejects(verifyReleaseSource({ ...options, api: f.api })); assert.ok(f.visited.length <= 1);
+  }
+});
+test('PR success, wrong source, pending checks and a newer failed run cannot authorize release', async () => {
+  for (const runs of [
+    [{ ...goodRun, event: 'pull_request' }], [{ ...goodRun, head_sha: 'b'.repeat(40) }],
+    [{ ...goodRun, status: 'in_progress', conclusion: null }], [goodRun, { ...goodRun, id: 11, conclusion: 'failure' }],
+  ]) await assert.rejects(verifyReleaseSource({ ...base, api: fixture({ runs }).api }));
+});
+test('missing, skipped, failed or ambiguous source jobs cannot authorize release', async () => {
+  for (const jobs of [[], [{ name: 'Source gate', conclusion: 'skipped' }], [{ name: 'Source gate', conclusion: 'failure' }], [{ name: 'Source gate', conclusion: 'success' }, { name: 'Source gate', conclusion: 'success' }]]) {
+    await assert.rejects(verifyReleaseSource({ ...base, api: fixture({ jobs }).api }));
+  }
+});
+test('unprotected, wildcard or additional deployment branches are rejected', async () => {
+  for (const options of [{ protected: false }, { branches: [{ name: '*', type: 'branch' }] }, { branches: [{ name: 'main', type: 'tag' }] }, { branches: [{ name: 'main', type: 'branch' }, { name: 'dev', type: 'branch' }] }]) {
+    await assert.rejects(verifyReleaseSource({ ...base, api: fixture(options).api }));
+  }
+});

From 51852bdc27dc9d4ab3acc156d2d8fb3fc8c46804 Mon Sep 17 00:00:00 2001
From: zmuhls <zmuhlbauer1@gmail.com>
Date: Wed, 23 Sep 2026 11:02:25 -0400
Subject: [PATCH 2/2] docs: record private host recovery and production state
 rehearsal

---
 .../2026-09-23-private-host-audit.md          | 72 +++++++++++++++++++
 docs/private-ci-release.md                    | 20 ++++--
 2 files changed, 87 insertions(+), 5 deletions(-)
 create mode 100644 docs/operations/2026-09-23-private-host-audit.md

diff --git a/docs/operations/2026-09-23-private-host-audit.md b/docs/operations/2026-09-23-private-host-audit.md
new file mode 100644
index 0000000..594b039
--- /dev/null
+++ b/docs/operations/2026-09-23-private-host-audit.md
@@ -0,0 +1,72 @@
+# September 23 private host audit
+
+Administrative SSH to the existing Tailscale host succeeded. The live checkout
+remains at `9efd29c62f07a9e431116ee89c5b4513c4499c8f`, with its existing local
+files preserved. Neither the fleet receiver nor the protected CI release was
+deployed during this audit.
+
+## Existing API repair
+
+PM2 reported the API online, but port 3004 had no listener. The TypeScript
+watcher remained alive after its child failed to load the native SQLite module:
+the module required Node ABI 127, while the host's Node 24.19.0 used ABI 137.
+
+A private Node 22.23.1 runtime was installed at
+`/data/slide-maker-storage/operations/runtime/node-v22.23.1-linux-x64`. Its
+official Linux x64 archive was verified with SHA-256
+`9749e988f437343b7fa832c69ded82a312e41a03116d766797ac14f6f9eee578`.
+An in-memory SQLite check passed before any process was restarted.
+
+Only `slide-maker-api` was restarted with this runtime first in its PATH. Its
+saved PM2 definition received the same PATH change; the other saved process
+definitions were preserved. No shared Node, pnpm or PM2 installation changed.
+The earlier process definition, saved PM2 state and SQLite backup remain private
+under `/data/slide-maker-storage/operations/runtime-repair-20260923T144922Z`.
+
+Readback:
+
+- Loopback `/api/health`: 200; anonymous `/api/auth/me` and `/api/decks`: 401.
+- Database integrity: `quick_check=ok`; no foreign-key violations.
+- Existing records preserved: 9 users, 26 decks, 52 uploaded-file records,
+  281 slides and 740 content blocks.
+- Other PM2 process IDs and statuses were unchanged.
+- The legacy public URL returned 403 from the host's network, so this receipt
+  does not establish public or campus browser access.
+- Saved startup configuration was checked, but no server reboot was performed.
+
+## Production-copy rehearsal
+
+The snapshot utility and migration module came from source
+`7e5275a118aeb13909e83f4c2f84bc304283d5a5`. The installed Identity 5.2.6, jose
+6.2.3 and zod 4.4.3 runtime files were copied into the private rehearsal directory;
+the running checkout and its dependencies were unchanged.
+
+The API was stopped, absence of database/WAL/SHM holders was verified, and the
+utility took a SQLite backup plus all uploads. The API recovered in 4.33 seconds
+from the stop request. The snapshot was restored into a new directory and
+verified against its byte/hash manifest: 78 files total, including all 77 upload
+files (48,799,114 bytes). The 52 database upload records and 77 stored files are
+different inventories; the rehearsal retained the entire storage directory.
+
+The actual additive identity migration ran twice against the restored copy.
+All original column values across 16 tables were preserved. The new nullable
+column and unique index were present, all nine accounts remained unlinked, and
+SQLite integrity and foreign-key checks passed. The original snapshot still
+passed its manifest check after the migration rehearsal.
+
+Private artifacts and the machine-readable receipt are under
+`/data/slide-maker-storage/operations/rehearsal-20260923-7e5275a`.
+No production identity column, account association or role was changed.
+
+## Remaining release requirements
+
+The host's shared defaults are Node 24.19.0, pnpm 10.33.0 and PM2 6.0.13. The
+protected release still requires its pinned runtime, private directories and
+configuration, Tailscale OAuth access, a dedicated SSH key and verified host
+key material in the main-only GitHub environment. The administrative password
+used for this inspection was not installed in CI.
+
+Nginx still proxies the legacy Slide Maker routes directly to the existing
+API/web pair. Doorway-to-Node ingress, mapped-user CUNY login, current Gateway
+model work, exported bytes and public/campus access need their own deployed
+acceptance. A successful state rehearsal does not establish those outcomes.
diff --git a/docs/private-ci-release.md b/docs/private-ci-release.md
index 540e70e..a75ee9c 100644
--- a/docs/private-ci-release.md
+++ b/docs/private-ci-release.md
@@ -81,11 +81,21 @@ again. Failed release directories and state snapshots remain for investigation.
 
 ## Remaining acceptance
 
-Source safeguards and disposable tests do not prove this host's credentials,
-network ACLs, storage paths, installed runtime or live traffic. The host release
-has not been run. The existing SSH account rejected the available local key.
-An operator must establish the prerequisites above and rehearse against actual
-production state before calling the release accepted.
+The [September 23 host audit](operations/2026-09-23-private-host-audit.md)
+verified administrative SSH access, the existing database/uploads paths, and a
+backup, restoration and additive identity migration against a private copy of
+actual production state. It also repaired an existing native SQLite/Node ABI
+mismatch by pinning only the legacy API to Node 22.23.1. The live application
+source and identity schema were not migrated.
+
+The protected host release has not been run. Its Tailscale OAuth connection,
+dedicated SSH key, private release configuration/directories and release runtime
+still need provisioning. The shared host defaults remain Node 24.19.0, pnpm
+10.33.0 and PM2 6.0.13. Do not globally upgrade or replace these to satisfy the
+release pins: the account also runs other applications. The API's private Node
+22 runtime does not by itself satisfy the CI host's PATH/toolchain contract.
+Source checks and the successful production-copy rehearsal do not prove the
+remaining network, credential or live traffic requirements.
 
 The CI connection does not create the private Doorway-to-Node browser ingress.
 Keep that mount disabled until its transport is established and the receiver is