Release v0.12.19: curation, Comms and ingestion

.github/workflows/ci.yml

@@ -42,7 +42,7 @@ jobs:
         run: uv run --no-sync python -c "import switchbay.daemon"
 
   frontend:
-    name: Frontend typecheck + build
+    name: Frontend Node tests + typecheck + builds
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -54,5 +54,188 @@ jobs:
           install: false
       - name: Install
         run: pnpm --dir frontend install --frozen-lockfile
+      - name: HTML extraction Node tests (6)
+        working-directory: frontend
+        run: |
+          set -euo pipefail
+          node --version
+          node --experimental-strip-types --test --test-reporter tap \
+            src/lib/htmlExtraction.test.ts > "${RUNNER_TEMP}/html-extract.tap"
+          cat "${RUNNER_TEMP}/html-extract.tap"
+          python3 - "${RUNNER_TEMP}/html-extract.tap" <<'PY'
+          from pathlib import Path
+          import sys
+          text = Path(sys.argv[1]).read_text(encoding="utf-8", errors="replace")
+          lines = [ln.strip() for ln in text.splitlines()]
+          oks = [ln for ln in lines if ln.startswith("ok ")]
+          notoks = [ln for ln in lines if ln.startswith("not ok ")]
+          meta = {}
+          for ln in lines:
+              for key in ("pass", "fail", "skipped", "todo", "cancelled"):
+                  prefix = f"# {key} "
+                  if ln.startswith(prefix):
+                      meta[key] = int(ln.split()[-1])
+          plan = [ln for ln in lines if ln.startswith("1..")]
+          if notoks or meta.get("fail", 0) != 0 or meta.get("skipped", 0) != 0 \
+                  or meta.get("todo", 0) != 0 or meta.get("cancelled", 0) != 0:
+              raise SystemExit(
+                  f"html extraction tests must all run and pass with no skips: "
+                  f"{meta} notoks={notoks}"
+              )
+          pass_n = meta.get("pass", len(oks))
+          if pass_n != 6 or len(oks) != 6 or "1..6" not in plan:
+              raise SystemExit(
+                  f"expected 6 passing html extraction tests, got pass={pass_n} "
+                  f"oks={len(oks)} plan={plan} meta={meta}"
+              )
+          print("htmlExtraction: 6/6 (no skips)")
+          PY
+      - name: Web-policy race Node test
+        working-directory: frontend
+        run: |
+          set -euo pipefail
+          node --experimental-strip-types tests/webPolicy.race.test.ts \
+            | tee "${RUNNER_TEMP}/web-policy-race.txt"
+          grep -F "webPolicy.race.test.ts ok" "${RUNNER_TEMP}/web-policy-race.txt"
+          if grep -Ei 'skip|todo' "${RUNNER_TEMP}/web-policy-race.txt"; then
+            echo "web-policy race test must not skip"
+            exit 1
+          fi
+          echo "webPolicy.race: 1/1 (no skips)"
       - name: Typecheck + build
         run: pnpm --dir frontend run build
+      - name: Webview graph build
+        run: pnpm --dir frontend run build:webview
+
+  browser-e2e:
+    name: Browser e2e (mocked backend)
+    runs-on: ubuntu-latest
+    # Isolated Chromium + mock HTTP/WS. Never starts switchbay.daemon,
+    # never binds or curls the live :8765 PWA, never sets E2E_BASE_URL
+    # (that would collide the two specs and point at vite :5173).
+    env:
+      E2E_MOCK: "1"
+      CI: "true"
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pnpm/setup@v2
+        with:
+          version: 11
+          runtime: node@22
+          cache: true
+          install: false
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+      - name: Set up Python 3.13 + locked dev deps
+        run: |
+          uv python install 3.13
+          uv sync --locked --group dev
+      - name: Install frontend
+        run: pnpm --dir frontend install --frozen-lockfile
+      - name: Build frontend dist
+        run: pnpm --dir frontend run build
+      - name: Require mock e2e fixtures
+        run: |
+          set -euo pipefail
+          test -f frontend/tests/e2e/mock_backend.py
+          test -f frontend/tests/e2e/zen-web-policy.spec.ts
+          test -f frontend/tests/e2e/comms-desks.spec.ts
+          grep -F 'e2e_mock' frontend/tests/e2e/zen-web-policy.spec.ts
+          grep -F 'e2e_mock' frontend/tests/e2e/comms-desks.spec.ts
+          grep -F 'e2e_mock' frontend/tests/e2e/mock_backend.py
+          grep -F '/tmp/switchbay-e2e-mock-ws' frontend/tests/e2e/zen-web-policy.spec.ts
+          grep -F '/tmp/switchbay-e2e-mock-ws' frontend/tests/e2e/comms-desks.spec.ts
+          grep -F '/tmp/switchbay-e2e-mock-ws' frontend/tests/e2e/mock_backend.py
+          if grep -nE 'test\.(skip|fixme)\(' \
+              frontend/tests/e2e/zen-web-policy.spec.ts \
+              frontend/tests/e2e/comms-desks.spec.ts; then
+            echo "e2e specs must not skip"
+            exit 1
+          fi
+      - name: Install Playwright Chromium
+        run: pnpm --dir frontend exec playwright install --with-deps chromium
+      - name: Playwright zen-web-policy (mock port 41765)
+        working-directory: frontend
+        env:
+          E2E_MOCK: "1"
+          E2E_MOCK_PORT: "41765"
+          CI: "true"
+        run: |
+          set -euo pipefail
+          unset E2E_BASE_URL || true
+          if [ -n "${E2E_BASE_URL:-}" ]; then
+            echo "E2E_BASE_URL must stay unset so specs use their own ports"
+            exit 1
+          fi
+          pnpm exec playwright test tests/e2e/zen-web-policy.spec.ts \
+            --reporter=list --forbid-only --workers=1 \
+            | tee "${RUNNER_TEMP}/pw-zen-web-policy.txt"
+          python3 - "${RUNNER_TEMP}/pw-zen-web-policy.txt" 4 zen-web-policy <<'PY'
+          from pathlib import Path
+          import re, sys
+          plain = re.sub(r"\x1b\[[0-9;]*[A-Za-z]", "", Path(sys.argv[1]).read_text(errors="replace"))
+          want, label = int(sys.argv[2]), sys.argv[3]
+          if "zen-web-policy.spec.ts" not in plain:
+              raise SystemExit(f"{label}: spec file did not run")
+          if re.search(r"\b[1-9]\d*\s+skipped\b", plain, re.I):
+              raise SystemExit(f"{label}: skips are not allowed")
+          if re.search(r"\b[1-9]\d*\s+flaky\b", plain, re.I):
+              raise SystemExit(f"{label}: flaky results are not allowed")
+          failed = re.search(r"\b(\d+)\s+failed\b", plain, re.I)
+          if failed and int(failed.group(1)) != 0:
+              raise SystemExit(f"{label}: failed tests")
+          passed = re.findall(r"(?m)^\s*(\d+)\s+passed\b", plain)
+          if not passed:
+              raise SystemExit(f"{label}: no 'N passed' summary")
+          got = int(passed[-1])
+          if got != want:
+              raise SystemExit(f"{label}: expected {want} passed, got {got}")
+          print(f"{label}: {got}/{want} passed, no skips")
+          PY
+      - name: Playwright comms-desks (mock port 41766)
+        working-directory: frontend
+        env:
+          E2E_MOCK: "1"
+          E2E_COMMS_PORT: "41766"
+          CI: "true"
+        run: |
+          set -euo pipefail
+          unset E2E_BASE_URL || true
+          if [ -n "${E2E_BASE_URL:-}" ]; then
+            echo "E2E_BASE_URL must stay unset so specs use their own ports"
+            exit 1
+          fi
+          pnpm exec playwright test tests/e2e/comms-desks.spec.ts \
+            --reporter=list --forbid-only --workers=1 \
+            | tee "${RUNNER_TEMP}/pw-comms-desks.txt"
+          python3 - "${RUNNER_TEMP}/pw-comms-desks.txt" 4 comms-desks <<'PY'
+          from pathlib import Path
+          import re, sys
+          plain = re.sub(r"\x1b\[[0-9;]*[A-Za-z]", "", Path(sys.argv[1]).read_text(errors="replace"))
+          want, label = int(sys.argv[2]), sys.argv[3]
+          if "comms-desks.spec.ts" not in plain:
+              raise SystemExit(f"{label}: spec file did not run")
+          if re.search(r"\b[1-9]\d*\s+skipped\b", plain, re.I):
+              raise SystemExit(f"{label}: skips are not allowed")
+          if re.search(r"\b[1-9]\d*\s+flaky\b", plain, re.I):
+              raise SystemExit(f"{label}: flaky results are not allowed")
+          failed = re.search(r"\b(\d+)\s+failed\b", plain, re.I)
+          if failed and int(failed.group(1)) != 0:
+              raise SystemExit(f"{label}: failed tests")
+          passed = re.findall(r"(?m)^\s*(\d+)\s+passed\b", plain)
+          if not passed:
+              raise SystemExit(f"{label}: no 'N passed' summary")
+          got = int(passed[-1])
+          if got != want:
+              raise SystemExit(f"{label}: expected {want} passed, got {got}")
+          print(f"{label}: {got}/{want} passed, no skips")
+          PY
+      - name: Upload Playwright failure artifacts
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: playwright-e2e-mock
+          path: |
+            frontend/test-results/
+          if-no-files-found: warn
+          retention-days: 7

CHANGELOG.md

@@ -3,6 +3,23 @@
 Human-curated release notes. Earlier 0.9.x notes also live on the
 [GitHub releases](https://github.com/benjsmith/switchbay/releases) page.
 
+## 2026-09-20 — v0.12.19
+
+No wiki/vault format migration. Review Web, Comms, and watch-folder behavior changes below. After updating, run `make refresh BUILD=1` and hard-reload the PWA. No new VSIX.
+
+- **Curate:** repaired settlement, resume, expiry, and continuous package waves. Idle desks wait for new work. Content receipts preserve useful partial results; quota-related research prose no longer creates false provider failures.
+- **Desk limits:** chief-counted concurrent live-worker cap in Settings; minimum 4, default/hard maximum 8. Admin `orchestration.max_live_workers` can tighten the ceiling. Excess workers wait; completed workers leave the live DAG while findings remain available.
+- **Comms:** metadata-first review queue with explicit per-workspace approval and revocation. Relevance suggestions never auto-approve. Secret/Top Secret, and enterprise unknown/missing classifications, are refused before body retrieval. Tenant classification GUIDs are configurable. Teams/Slack remain discovery/review only; message bodies are blocked.
+- **Web/Research:** one default-off Web policy across Rail, Settings, and Zen, with per-call approvals that are never persisted. Codex native search is disabled. Auto can hire Research; workspace model allowlists remain enforced. Installed global curiosity-engine skills can be used read-only when the workspace copy is absent.
+- **Chat:** full-height docked Zen Chat with a bottom composer; floating Chat retains two columns and visible Web control. Fixed narrow-rail control overlap.
+- **Runtime/export:** launchd discovers user-managed Node/pnpm runtimes without sourcing shell profiles. Slideshow PDF export resolves executables and Playwright before spawning.
+- **Document ingestion:** PPTX can use bundled `python-pptx` when the workspace lacks it, while preserving workspace PDF/XLSX extractors. Re-import historical PPTX files whose extracts contain the old unavailable placeholder.
+- **Watch folders:** new arrivals produce deterministic vault extracts for later Curate. macOS iCloud placeholders request per-file downloads and retry while unavailable; no permanent pin or whole-tree download. Retry state, source authorization, workspace binding, and existing vault copies are protected. Other cloud providers are unsupported.
+- **CI:** added frontend Node tests, webview build, and isolated Playwright checks for Web policy, Comms, desk controls, and browser geometry.
+- Includes v0.12.17 Editor HTML extraction preview and v0.12.18 Close on vault-source tabs, previously on `main` but not yet in a published release.
+
+Cloud-state fixtures and the native download API were tested; no live iCloud placeholder was available for end-to-end verification. Comms fixture tests do not certify a real enterprise tenant. See [release notes](docs/releases/v0.12.19.md), [skill-shell parity baseline](docs/handoff/2026-09-20-skill-shell-parity.md), and [enterprise configuration](docs/enterprise.md).
+
 ## 2026-09-13 — v0.12.18
 
 **Migration:** none. **Breaking:** none. After pull, run

docs/concepts-and-data-flow.md

@@ -220,8 +220,25 @@ flowchart LR
 
 - **Capture** writes curiosity-engine's own staging shapes; the CE
   sweeps *are* the async curation half. Ingesting a file (Browser `+`)
-  dispatches a background agent that classifies it into CE types and
-  records `extracted_from` provenance.
+  or a **watch folder** (Settings → Watch folders) stages a copy into
+  `vault/` and runs curiosity-engine `local_ingest.py` for supported
+  formats (text, HTML, PDF, CSV, XLSX, **PPTX**). Watch ingest writes
+  vault extracts only — it does **not** create wiki pages; Curate does
+  that later. Watch folders only pick up files that arrive *after* you
+  add the folder (existing contents are baselined). On macOS, iCloud
+  Drive placeholders in a folder you already authorized are downloaded
+  on demand — that file only, overall wait bounded at 8s per attempt
+  including helper calls — before ingest; other cloud providers are
+  not auto-hydrated. A file is not marked seen until staging +
+  extraction succeed; placeholders, timeouts, and corrupt PPTX stay
+  pending/retryable rather than counting as ingested. Each ingest file
+  uses the interpreter that already has that extractor: workspace CE
+  venv for pypdf/openpyxl when present, Switch Bay host for
+  `python-pptx` when the workspace lacks it. Scan/graph stay on the
+  workspace venv (kuzu). Old vault extracts that say
+  `PPTX extraction unavailable` need an explicit re-ingest; Switch
+  Bay does not rewrite them. The extract's `source_path` /
+  `extracted_from` record the authorized original watch path.
 - **Curation** (curiosity-engine, a bundled first-party skill) links and
   promotes captured material into the wiki graph. A wiki write schedules
   a background graph rebuild → `data.json`.
@@ -268,9 +285,10 @@ roles stay computational kinds (investigate / verify / synthesize /
 execute), not job titles. **Standing desks** (Curate, Work, Code, Deck,
 Auto) reuse a chief and org across waves: working while a run is live,
 quiet after Stop or a finished wave, dismissed only when you drop the
-row. `/curate`, `/work`, and `/code` always seat; authoring an HTML
-slideshow reuses one Deck desk; a wiki question (`what do we know
-about X`) does not seat. Recurring Auto prompts live in the
+row. `/curate`, `/work`, and `/code` always seat — including a duration
+brief such as `/curate for 10 mins`. Authoring an HTML slideshow
+reuses one Deck desk; a wiki question (`what do we know about X`)
+does not seat. Recurring Auto prompts live in the
 **Schedules** tab (per workspace; the daemon fires due items even when
 that vault is not focused). A desk may keep `.orchestrator/APPROACH.md`
 as the overnight problem-solving sequence.
@@ -371,6 +389,28 @@ group chat.
 | **A2A** | Agent/thread interoperability (`message/send`). Not the orchestration algorithm. |
 | **Model ladder** | Available model/provider capability and cost hierarchy. The orchestrator may use it; it must not bypass it. |
 
+### Live seats (Settings)
+
+Each standing desk has a **live-seat** cap — queue / backpressure, not a lifetime stop. When every seat is taken, extra workers wait; they are not dropped, and the desk does not shut down. The **chief of staff is counted**. Floor **4** (chief + verifier + synthesizer + specialist), default **8**, current hard max **8**. Settings → Auto orchestration (`desk_max_live_workers`) cannot go below 4 or above the hard max. Admin policy may only **tighten** the ceiling:
+
+```json
+{
+  "orchestration": { "max_live_workers": 5 }
+}
+```
+
+A baked enterprise cap is never raised or erased by a missing, zero, or malformed overlay. Nested Curate workers and Comms wiki curation share the same per-workspace Curate desk gate.
+
+---
+
+## Data flow — Comms streams
+
+Email and chat accounts are **curation sources**, not Switch Bay threads. Discovery is **metadata only** (headers, labels, channel names). No body content is fetched or written to the wiki until you **explicitly approve** a thread for a **specific workspace**. **Revoke** stops future retrieval for that source. Auto-relevance may suggest a workspace; it never approves.
+
+**Secret / Top Secret** (and unknown or missing classification under enterprise) is refused **before any body fetch**. Tenant secret label GUIDs can be listed in admin policy (`comms.tenant_label_ids`) and match when they appear inside `MSIP_Labels`. Gmail system labels such as `INBOX` / `UNREAD` are not classifications.
+
+**Teams and Slack message bodies are not retrieved**, even after Approve. Their adapters currently lack trustworthy pre-body classification; listing and review remain, and the UI does not imply content will flow. Wiki ingest of approved Gmail / Outlook / IMAP mail needs a **keyed, allowlisted, file-capable CLI** (Claude Code, Grok Build, Codex, Muse Code) — not an HTTP-only provider.
+
 ---
 
 ## Data flow 4 — rich answers become artifacts (not chat walls)