diff --git a/CHANGELOG.md b/CHANGELOG.md index c137f6d..a722a9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,101 @@ +## 2026-09-20 — Obsidian vault import (folder + zip) + +**Migration:** none. **Breaking:** none. + +### Added + +- **`scripts/obsidian_import.py`:** import an Obsidian vault (folder path or zip) + into a CE workspace as `wiki/`. Preserves `[[wikilinks]]` verbatim. Sandboxes + sources under `$HOME` / `$CE_WORKSPACE_HOME` (same as workspace registry); + zip-slip safe extract. Optional `--target` bootstraps a minimal workspace; + `--register` records it in the CE workspace registry. +- **`POST /api/import/obsidian`:** JSON `{path}` or multipart zip; honours + `CE_PUBLIC_BASE` embed prefix. +- **CLI:** `python3 obsidian_import.py ` (conceptual `ce import-obsidian`). +- **Tests:** `tests/test_obsidian_import.py`. +- **Fixture (umbrella):** `corpora/obsidian-dummy-vault/` for Omarchy E2E prep. + +### Notes + +- **Mapping:** Obsidian vault root → CE `wiki/` (documented; matches existing + “open wiki/ as Obsidian vault” guidance in `docs/viewers.md`). +- Omarchy GUI pick-folder remains shell-side later. + +## 2026-09-19 — Phase 2b++++++ rubber-band split targeting UI + +**Migration:** none. **Breaking:** none. + +### Added + +- **Classic graph rubber-band:** Ctrl/⌘-drag marquee select; Shift-click multi-select; + click toggle membership; Alt-click / right-click flips move↔copy (entities/concepts + default to copy). `Graph.splitEnter` / `splitExit` sync with the split panel. +- **Split panel:** listens `ce:split-selection`, `ce:split-proposal`, and + `sy:split-proposal`; still POSTs `/api/split` via `ceApi` / `CE_PUBLIC_BASE` + (embed v2 under `/embed/ce`). +- **Pure helpers:** `packages/knowledge-atlas` `rubberBand.ts` + `rubberBand.test.ts`. +- Atlas: `splitEnter`/`splitExit` seed engine multi-select (canvas marquee deferred). + +### Notes + +- Still Switchbay-only / deferred: Atlas canvas rubber-band, link-heal agents, + tab open after split, workspace-root figures / `.workbench` sketches, OS Trash prune. + See `skills/curiosity-engine/docs/MIGRATION-INTAKE.md`. + +## 2026-09-19 — Phase 2b+++++ split registry + CM export hook + +**Migration:** none. **Breaking:** none. + +### Added + +- **Workspace registry** (`workspace_registry.py`): CE-owned persistence under + `$CE_WORKSPACE_REGISTRY` or `$XDG_CONFIG_HOME/curiosity-engine/workspaces.json`, + sandboxed to `$HOME` / `$CE_WORKSPACE_HOME`. Records split provenance. +- **APIs:** `GET/POST/DELETE /api/workspaces`; successful `POST /api/split` auto-registers + the target (shells still own tab-open chrome). +- **CM export hook** (`cm_export.py` + `POST /api/cm-export`): invokes curiosity-merge + `subgraph_export.py` for a selected page set (license modes via `--include-vault`). + Supports `dry_run` (plan argv, no spawn). Discovers CM via `$CE_CM_ROOT` / + `$CURIOSITY_MERGE_ROOT` / sibling checkout / skill install paths. Target sandboxed + outside the source workspace and inside the registry home root. +- **Tests:** `tests/test_workspace_registry_cm_export.py` (registry persistence + + export dry-run + split auto-register under embed prefix). + +### Notes + +- Native `wiki_partition` remains the default split path (no CM required). +- Still Switchbay-only: rubber-band / `sy:split-proposal` UI, link-heal agents, + tab open after split, workspace-root figures / `.workbench` sketches, OS Trash prune. + See `skills/curiosity-engine/docs/MIGRATION-INTAKE.md`. + +## 2026-09-19 — Phase 2b++++ pack action dispatch + +**Migration:** none. **Breaking:** none. + +### Added + +- Pack APIs (Switchbay `pack.json` shape): `GET /api/packs`, `GET /api/packs//actions`, + `POST /api/packs//action/` (vault/wiki sandbox queue → `{run_id}`), + `POST /api/packs/toggle`, `POST /api/packs/install` (local path), `DELETE /api/packs?name=`. +- Filebrowser context menu runs pack `file_routes` via the dispatch API. +- Tests: `tests/test_filebrowser_packs.py` (escape / disabled / embed prefix). + +### Notes + +- Agent/LLM skill execution, git pack install, pip extras, reveal-in-OS, and drop-ingest + remain Switchbay-only (see `MIGRATION-INTAKE.md`). + +## 2026-09-19 — Phase 2b++ filebrowser FS mutate + pack routes stub + +**Migration:** none. **Breaking:** none. + +### Added + +- Sandboxed FS mutate APIs under `vault/` + `wiki/`: `POST /api/fs/{create,mkdir,rename,move,delete,duplicate}`, `GET /api/fs/stat`. +- Pack `file_routes` discovery: `GET /api/file-routes` (read-only; install/dispatch stays shell-side). +- Wiki-view Files context menu: New file/folder, Rename, Duplicate, Delete. +- Tests: `tests/test_filebrowser_fs.py` (sandbox escape + embed-prefix API). + ## 2026-09-16 — Wiki viewer mobile modal + sidebar **Migration:** none. **Breaking:** none. diff --git a/RELEASE_CHECKLIST.md b/RELEASE_CHECKLIST.md index f5ee1e3..17fba04 100644 --- a/RELEASE_CHECKLIST.md +++ b/RELEASE_CHECKLIST.md @@ -13,7 +13,7 @@ Run through this before tagging a new release. Items marked **(security)** must | `d3.min.js` | 7.9.0 | `f2094bbf6141b359722c4fe454eb6c4b0f0e42cc10cc7af921fc158fceb86539` | `https://cdn.jsdelivr.net/npm/d3@7.9.0/dist/d3.min.js` | | `fuse.min.js` | 7.0.0 | `e3621b53cb77b4ec306dec41ed95511e6dd80d17fae5a04f3e346d214b9f8f92` | `https://cdn.jsdelivr.net/npm/fuse.js@7.0.0/dist/fuse.min.js` | | `jszip.min.js` | 3.10.1 | `acc7e41455a80765b5fd9c7ee1b8078a6d160bbbca455aeae854de65c947d59e` | `https://cdn.jsdelivr.net/npm/jszip@3.10.1/dist/jszip.min.js` | -| `knowledge-atlas.js` | 0.2.0 (in-repo) | `26ffe14abe0c3476702486c2796bb166c8c8e1275e2adb98ae3066b8609df7cd` | built in-repo: `cd packages/knowledge-atlas && pnpm run build`, then copy `dist/knowledge-atlas.iife.js` here (no CDN — first-party code; refresh the hash on every rebuild) | +| `knowledge-atlas.js` | 0.2.0 (in-repo) | `b2ba92eb162a1a71bb71a3fcfeda52450997d8e218a6f34eef4c71c627589e18` | built in-repo: `cd packages/knowledge-atlas && pnpm run build`, then copy `dist/knowledge-atlas.iife.js` here (no CDN — first-party code; refresh the hash on every rebuild) | To refresh: diff --git a/packages/knowledge-atlas/src/animation/historyFromGraph.ts b/packages/knowledge-atlas/src/animation/historyFromGraph.ts new file mode 100644 index 0000000..6814cbd --- /dev/null +++ b/packages/knowledge-atlas/src/animation/historyFromGraph.ts @@ -0,0 +1,185 @@ +/** + * Build / scrub helpers for curation-replay HistoryDoc. + * + * Synthesizes a deterministic timeline from CE `data.json` graph + * membership when git-first-seen history is unavailable. Hosts + * (wiki-view replay UI) own SVG rendering; this module stays pure. + */ + +import type { HistoryDoc, ReplayEvent } from "./replayTimeline.ts"; +import { compareSyntheticNodes } from "./replayCamera.ts"; + +export type GraphNodeIn = { + id: string; + title?: string; + type?: string; + degree?: number; + path?: string; + /** ISO date from frontmatter `created` when available. */ + created?: string; +}; + +export type GraphEdgeIn = { + source: string | { id: string }; + target: string | { id: string }; +}; + +export type BuildHistoryOptions = { + /** Wall-clock span for event `t` values (seconds). */ + duration?: number; + /** Cap total events (nodes kept; edges down-sampled). */ + maxEvents?: number; + source?: string; +}; + +const DEFAULT_DURATION = 12; +const DEFAULT_MAX_EVENTS = 2500; + +function edgeId(ref: string | { id: string } | null | undefined): string { + if (ref == null) return ""; + if (typeof ref === "string") return ref; + return typeof ref.id === "string" ? ref.id : ""; +} + +/** Stable sort: time asc, nodes before edges at equal t. */ +export function sortReplayEvents(events: ReplayEvent[]): ReplayEvent[] { + return events.slice().sort((a, b) => { + if (a.t !== b.t) return a.t - b.t; + const ao = a.op === "node" ? 0 : 1; + const bo = b.op === "node" ? 0 : 1; + return ao - bo; + }); +} + +/** + * Synthetic chronology: sources first, then by descending degree, + * then id. Edges fire at the later endpoint's slot so both ends exist. + */ +export function buildHistoryFromGraph( + nodesIn: readonly GraphNodeIn[] | null | undefined, + edgesIn: readonly GraphEdgeIn[] | null | undefined, + opts: BuildHistoryOptions = {}, +): HistoryDoc { + const duration = opts.duration && opts.duration > 0 ? opts.duration : DEFAULT_DURATION; + const maxEvents = opts.maxEvents && opts.maxEvents > 0 ? opts.maxEvents : DEFAULT_MAX_EVENTS; + const source = opts.source ?? "ce-data-json"; + + const nodes = (nodesIn ?? []) + .map((n) => ({ + id: String(n.id || "").trim(), + title: String(n.title || n.id || "").trim() || String(n.id || ""), + type: String(n.type || "unclassified"), + degree: typeof n.degree === "number" ? n.degree : 0, + created: typeof n.created === "string" ? n.created.trim() : "", + })) + .filter((n) => n.id); + + if (!nodes.length) { + return { duration, events: [], source: "empty", node_count: 0, degree: {} }; + } + + // Prefer frontmatter `created` when present; else type-tier → degree → id. + nodes.sort(compareSyntheticNodes); + + const byId = new Map(nodes.map((n) => [n.id, n])); + const indexOf = new Map(nodes.map((n, i) => [n.id, i])); + const nCount = nodes.length; + const tOf = (i: number) => (nCount <= 1 ? 0 : (i / (nCount - 1)) * duration); + + const events: ReplayEvent[] = nodes.map((n, i) => ({ + t: Math.round(tOf(i) * 1000) / 1000, + op: "node" as const, + id: n.id, + title: n.title, + type: n.type, + })); + + const seenPairs = new Set(); + const edgeEvents: ReplayEvent[] = []; + for (const e of edgesIn ?? []) { + const s = edgeId(e.source); + const t = edgeId(e.target); + if (!s || !t || s === t || !byId.has(s) || !byId.has(t)) continue; + const pair = s < t ? `${s}|${t}` : `${t}|${s}`; + if (seenPairs.has(pair)) continue; + seenPairs.add(pair); + const later = Math.max(indexOf.get(s)!, indexOf.get(t)!); + edgeEvents.push({ + t: Math.round(tOf(later) * 1000) / 1000, + op: "edge", + source: s, + target: t, + }); + } + + let merged = sortReplayEvents([...events, ...edgeEvents]); + if (merged.length > maxEvents) { + const nodeEv = merged.filter((e) => e.op === "node"); + let edgeEv = merged.filter((e) => e.op === "edge"); + const keep = Math.max(0, maxEvents - nodeEv.length); + if (keep < edgeEv.length) { + const step = edgeEv.length / Math.max(1, keep); + edgeEv = Array.from({ length: keep }, (_, i) => edgeEv[Math.floor(i * step)]!); + } + merged = sortReplayEvents([...nodeEv, ...edgeEv]); + } + + const degree: Record = {}; + for (const n of nodes) degree[n.id] = 0; + for (const e of merged) { + if (e.op !== "edge") continue; + degree[e.source] = (degree[e.source] ?? 0) + 1; + degree[e.target] = (degree[e.target] ?? 0) + 1; + } + + return { + duration, + events: merged, + source, + generated_at: Date.now() / 1000, + node_count: nodes.length, + degree, + }; +} + +export type ReplaySnapshot = { + /** Exclusive end index into sorted events (0 = empty). */ + index: number; + nodes: Array<{ id: string; title: string; type: string }>; + edges: Array<{ source: string; target: string }>; +}; + +/** Apply events `[0, index)` into a structural snapshot for scrub/step. */ +export function snapshotAt( + history: HistoryDoc | null | undefined, + index: number, +): ReplaySnapshot { + const events = sortReplayEvents(history?.events ?? []); + const n = events.length; + const clamped = Math.max(0, Math.min(n, Math.floor(index))); + const nodes: ReplaySnapshot["nodes"] = []; + const edges: ReplaySnapshot["edges"] = []; + const seen = new Set(); + for (let i = 0; i < clamped; i++) { + const ev = events[i]!; + if (ev.op === "node") { + if (seen.has(ev.id)) continue; + seen.add(ev.id); + nodes.push({ id: ev.id, title: ev.title, type: ev.type }); + } else if (seen.has(ev.source) && seen.has(ev.target)) { + edges.push({ source: ev.source, target: ev.target }); + } + } + return { index: clamped, nodes, edges }; +} + +/** Largest index such that all events with `t <= wall` are included. */ +export function indexAtTime( + history: HistoryDoc | null | undefined, + wall: number, +): number { + const events = sortReplayEvents(history?.events ?? []); + let i = 0; + while (i < events.length && events[i]!.t <= wall) i += 1; + return i; +} diff --git a/packages/knowledge-atlas/src/animation/replayCamera.ts b/packages/knowledge-atlas/src/animation/replayCamera.ts new file mode 100644 index 0000000..fb5851c --- /dev/null +++ b/packages/knowledge-atlas/src/animation/replayCamera.ts @@ -0,0 +1,194 @@ +/** + * Pure camera / fit helpers for curation-replay SVG hosts. + * + * Mirrors Switchbay curationReplayAnim easeAutoFit math without d3 — + * wiki-view replay.js and tests share the same contract. + */ + +export type FitBounds = { + minX: number; + minY: number; + maxX: number; + maxY: number; +}; + +export type ZoomTransform = { x: number; y: number; k: number }; + +export type FitViewOpts = { + viewW: number; + viewH: number; + /** Minimum cloud size so a tiny cluster does not over-zoom. */ + minCloud?: number; + /** Cap scale so sparse graphs do not fill the viewport too aggressively. */ + maxScale?: number; + padRatio?: number; + minPad?: number; +}; + +const DEFAULT_MIN_CLOUD = 220; +const DEFAULT_MAX_SCALE = 1.5; +const DEFAULT_PAD_RATIO = 0.12; +const DEFAULT_MIN_PAD = 40; + +const TYPE_TIER: Record = { + source: 0, + sources: 0, + project: 1, + entity: 2, + concept: 3, + evidence: 4, + fact: 4, + figure: 5, + table: 5, + note: 6, + todo: 6, + "todo-list": 6, + analysis: 7, + unclassified: 8, +}; + +export function typeTier(type: string | undefined | null): number { + const t = (type || "unclassified").toLowerCase(); + return TYPE_TIER[t] ?? 8; +} + +/** Axis-aligned bounds -> zoomIdentity-style translate+scale that centres the cloud. */ +export function computeFitTransform( + bounds: FitBounds, + opts: FitViewOpts, +): ZoomTransform { + const viewW = opts.viewW; + const viewH = opts.viewH; + const minCloud = opts.minCloud ?? DEFAULT_MIN_CLOUD; + const maxScale = opts.maxScale ?? DEFAULT_MAX_SCALE; + const padRatio = opts.padRatio ?? DEFAULT_PAD_RATIO; + const minPad = opts.minPad ?? DEFAULT_MIN_PAD; + + const { minX, minY, maxX, maxY } = bounds; + const spanX = Math.max(0, maxX - minX); + const spanY = Math.max(0, maxY - minY); + const pad = Math.max(minPad, Math.min(spanX, spanY) * padRatio); + const cloudW = Math.max(minCloud, spanX + pad * 2); + const cloudH = Math.max(minCloud, spanY + pad * 2); + const scale = Math.min(viewW / cloudW, viewH / cloudH, maxScale); + const cx = (minX + maxX) / 2; + const cy = (minY + maxY) / 2; + return { + x: viewW / 2 - cx * scale, + y: viewH / 2 - cy * scale, + k: scale, + }; +} + +/** Bounds from node positions; empty → null. */ +export function boundsFromPositions( + positions: ReadonlyArray<{ x?: number | null; y?: number | null }>, + fallbackCx = 0, + fallbackCy = 0, +): FitBounds | null { + if (!positions.length) return null; + let minX = Infinity; + let minY = Infinity; + let maxX = -Infinity; + let maxY = -Infinity; + for (const n of positions) { + const x = n.x ?? fallbackCx; + const y = n.y ?? fallbackCy; + if (x < minX) minX = x; + if (y < minY) minY = y; + if (x > maxX) maxX = x; + if (y > maxY) maxY = y; + } + if (!Number.isFinite(minX)) return null; + return { minX, minY, maxX, maxY }; +} + +/** Exponential blend toward a target zoom (Switchbay k≈0.08 per tick). */ +export function blendZoom( + current: ZoomTransform, + target: ZoomTransform, + k = 0.08, +): ZoomTransform { + const t = Math.max(0, Math.min(1, k)); + return { + x: current.x + (target.x - current.x) * t, + y: current.y + (target.y - current.y) * t, + k: current.k + (target.k - current.k) * t, + }; +} + +/** + * Comparator for synthetic node chronology. + * Prefer ISO `created` (asc) when present on either side; else type tier → + * degree desc → id. Deterministic. + */ +export function compareSyntheticNodes( + a: { id: string; type?: string; degree?: number; created?: string | null }, + b: { id: string; type?: string; degree?: number; created?: string | null }, +): number { + const ac = (a.created || "").trim(); + const bc = (b.created || "").trim(); + if (ac || bc) { + if (ac && !bc) return -1; + if (!ac && bc) return 1; + const at = Date.parse(ac) || 0; + const bt = Date.parse(bc) || 0; + if (at !== bt) return at - bt; + } + const ta = typeTier(a.type); + const tb = typeTier(b.type); + if (ta !== tb) return ta - tb; + const da = -(a.degree || 0); + const db = -(b.degree || 0); + if (da !== db) return da - db; + return a.id < b.id ? -1 : a.id > b.id ? 1 : 0; +} + +/** Atlas mount camera (centre-origin): screen = world * scale + {x,y}. */ +export type AtlasCamera = { x: number; y: number; scale: number }; + +/** + * Fit an Atlas camera so `bounds` (layout / world space) centres in the + * viewport. Same padding / maxScale contract as computeFitTransform, but + * returns centre-origin pan (camera.x = -cx * scale). + */ +export function computeAtlasCameraFit( + bounds: FitBounds, + opts: FitViewOpts, +): AtlasCamera { + const viewW = opts.viewW; + const viewH = opts.viewH; + const minCloud = opts.minCloud ?? DEFAULT_MIN_CLOUD; + const maxScale = opts.maxScale ?? DEFAULT_MAX_SCALE; + const padRatio = opts.padRatio ?? DEFAULT_PAD_RATIO; + const minPad = opts.minPad ?? DEFAULT_MIN_PAD; + + const { minX, minY, maxX, maxY } = bounds; + const spanX = Math.max(0, maxX - minX); + const spanY = Math.max(0, maxY - minY); + const pad = Math.max(minPad, Math.min(spanX, spanY) * padRatio); + const cloudW = Math.max(minCloud, spanX + pad * 2); + const cloudH = Math.max(minCloud, spanY + pad * 2); + const scale = Math.min(viewW / cloudW, viewH / cloudH, maxScale); + const cx = (minX + maxX) / 2; + const cy = (minY + maxY) / 2; + return { + x: -cx * scale, + y: -cy * scale, + scale, + }; +} + +/** Exponential blend for Atlas cameras (same k as blendZoom). */ +export function blendAtlasCamera( + current: AtlasCamera, + target: AtlasCamera, + k = 0.08, +): AtlasCamera { + const t = Math.max(0, Math.min(1, k)); + return { + x: current.x + (target.x - current.x) * t, + y: current.y + (target.y - current.y) * t, + scale: current.scale + (target.scale - current.scale) * t, + }; +} diff --git a/packages/knowledge-atlas/src/animation/replayTimeline.ts b/packages/knowledge-atlas/src/animation/replayTimeline.ts new file mode 100644 index 0000000..4f6a8a7 --- /dev/null +++ b/packages/knowledge-atlas/src/animation/replayTimeline.ts @@ -0,0 +1,85 @@ +/** + * Curation-replay timeline hook (Phase 2b spike). + * + * Pure scheduler extracted from Switchbay `curationReplayAnim.ts` event + * model — no SVG/d3. Hosts (wiki-view / Graph tab) own rendering; CE owns + * the timing contract so animation can later drive atlas/canvas. + */ + +export type ReplayEvent = + | { t: number; op: "node"; id: string; title: string; type: string } + | { t: number; op: "edge"; source: string; target: string }; + +export type HistoryDoc = { + duration: number; + events: ReplayEvent[]; + source: string; + generated_at?: number; + node_count?: number; + degree?: Record; +}; + +export type ReplayHandlers = { + onEvent: (ev: ReplayEvent, index: number) => void; + onDone?: () => void; + /** Wall-clock multiplier; 1 = real time vs HistoryDoc.t units. */ + rate?: number; + /** Inject clock for tests (defaults to performance/Date). */ + now?: () => number; + /** Inject timer APIs for tests. */ + schedule?: (fn: () => void, ms: number) => number; + cancel?: (id: number) => void; +}; + +/** + * Play events in ascending `t` order. Returns a cancel function. + * Empty/missing history completes on the next microtask-equivalent tick. + */ +export function playReplayTimeline( + history: HistoryDoc | null | undefined, + handlers: ReplayHandlers, +): () => void { + const events = (history?.events ?? []).slice().sort((a, b) => a.t - b.t); + const rate = handlers.rate && handlers.rate > 0 ? handlers.rate : 1; + const now = handlers.now ?? (() => (typeof performance !== "undefined" ? performance.now() : Date.now())); + const schedule = + handlers.schedule ?? + ((fn, ms) => setTimeout(fn, ms) as unknown as number); + const cancel = + handlers.cancel ?? + ((id) => { + clearTimeout(id as unknown as ReturnType); + }); + + let cancelled = false; + let timer: number | null = null; + let index = 0; + const t0 = now(); + + const tick = () => { + if (cancelled) return; + if (index >= events.length) { + handlers.onDone?.(); + return; + } + const wall = (now() - t0) * rate; + while (index < events.length && events[index]!.t <= wall) { + handlers.onEvent(events[index]!, index); + index += 1; + } + if (index >= events.length) { + handlers.onDone?.(); + return; + } + const nextT = events[index]!.t; + const delay = Math.max(0, (nextT - wall) / rate); + timer = schedule(tick, delay); + }; + + timer = schedule(tick, 0); + + return () => { + cancelled = true; + if (timer != null) cancel(timer); + }; +} diff --git a/packages/knowledge-atlas/src/core/engine.ts b/packages/knowledge-atlas/src/core/engine.ts index dc92255..416e4b1 100644 --- a/packages/knowledge-atlas/src/core/engine.ts +++ b/packages/knowledge-atlas/src/core/engine.ts @@ -28,6 +28,7 @@ import type { DiscoveryClass, ExplanationRequest, LayoutKind, + LayoutPoint, LayoutResult, SceneBudget, SceneData, @@ -64,6 +65,26 @@ export type EngineSnapshot = { stats: SceneStats | null; }; + +function coerceInitialPositions( + raw: + | Map + | Record + | undefined, +): Map | null { + if (!raw) return null; + if (raw instanceof Map) { + return raw.size ? raw : null; + } + const m = new Map(); + for (const id of Object.keys(raw)) { + const p = raw[id]; + if (!p || typeof p.x !== "number" || typeof p.y !== "number") continue; + m.set(id, { x: p.x, y: p.y, r: typeof p.r === "number" ? p.r : 4 }); + } + return m.size ? m : null; +} + export class AtlasEngine implements AtlasController { readonly hitTester = new HitTester(); @@ -110,6 +131,10 @@ export class AtlasEngine implements AtlasController { this.budget = { ...DEFAULT_BUDGET, ...config.budget }; this.physics = { ...DEFAULT_PHYSICS, ...config.physics }; this.layoutKind = config.layout ?? "focus"; + const seeded = coerceInitialPositions(config.initialPositions); + if (seeded) { + this.layoutResult = { positions: seeded, displacement: 0 }; + } } // ── events ──────────────────────────────────────────────────────── diff --git a/packages/knowledge-atlas/src/core/layout/force.ts b/packages/knowledge-atlas/src/core/layout/force.ts index e4e3d7f..c114a0b 100644 --- a/packages/knowledge-atlas/src/core/layout/force.ts +++ b/packages/knowledge-atlas/src/core/layout/force.ts @@ -13,7 +13,7 @@ import { forceManyBody, forceSimulation, } from "d3-force"; -import { aggregateRadius, meanDisplacement, nodeRadius, type LayoutAdapter, type LayoutContext } from "./types.ts"; +import { aggregateRadius, collideForceRadius, meanDisplacement, nodeRadius, type LayoutAdapter, type LayoutContext } from "./types.ts"; import { DEFAULT_PHYSICS, type LayoutPoint, type LayoutResult, type SceneData } from "../types.ts"; function prewarmTicks(count: number): number { @@ -53,7 +53,12 @@ export const forceLayout: LayoutAdapter = { ) .force("charge", forceManyBody().strength(physics.charge).distanceMax(2500)) .force("center", forceCenter(0, 0).strength(0.04)) - .force("collide", forceCollide((d) => (d as unknown as SimNode).r + physics.collide)) + .force( + "collide", + forceCollide((d) => collideForceRadius((d as unknown as SimNode).r, physics.collide)) + .strength(1) + .iterations(simNodes.length > 2000 ? 2 : 1), + ) .alpha(1) .alphaDecay(0.05) .stop(); diff --git a/packages/knowledge-atlas/src/core/layout/hybrid.ts b/packages/knowledge-atlas/src/core/layout/hybrid.ts index e99a46b..6787129 100644 --- a/packages/knowledge-atlas/src/core/layout/hybrid.ts +++ b/packages/knowledge-atlas/src/core/layout/hybrid.ts @@ -33,6 +33,8 @@ import { import { coreRadius, coreRadiusAt, rimRadiusAt, type Viewport } from "../geometry.ts"; import { aggregateRadius, + COLLIDE_RADIUS_SCALE_LARGE, + collideForceRadius, meanDisplacement, nodeRadius, type LayoutAdapter, @@ -80,6 +82,42 @@ export function populatedShellBands(scene: SceneData): number { type SimNode = { id: string; r: number; x?: number; y?: number }; type AnchoredNode = SimNode & { x0: number; y0: number; survivor: boolean }; +/** Collide force: hub-scaled radius, extra iterations on large N. */ +function makeCollide(pad: number, nodeCount: number) { + // N>10k overview: more collide iterations + stronger hub scale so hubs + // separate under the reduced many-body tick budget. + const iters = nodeCount > 10_000 ? 3 : nodeCount > 2000 ? 2 : 1; + const scale = nodeCount > 10_000 ? COLLIDE_RADIUS_SCALE_LARGE : undefined; + return forceCollide((d) => + collideForceRadius((d as unknown as SimNode).r, pad, scale), + ) + .strength(1) + .iterations(iters); +} + +/** Soft collide-only settle: separates overlapping hubs while gently + * pinning to the coarse force solution so the atlas look stays intact. */ +function relaxCollideHubs(nodes: SimNode[], pad: number, ticks: number): void { + if (nodes.length === 0 || ticks <= 0) return; + // Weaker soft-pin (0.10) lets hubs travel farther while collide + // resolves; still anchored enough to preserve atlas clustering. + const soft = forceSimulation(nodes as never[]) + .force( + "collide", + forceCollide((d) => + collideForceRadius((d as unknown as SimNode).r, pad, COLLIDE_RADIUS_SCALE_LARGE), + ) + .strength(1) + .iterations(4), + ) + .force("x", forceX((d: unknown) => (d as SimNode).x ?? 0).strength(0.1)) + .force("y", forceY((d: unknown) => (d as SimNode).y ?? 0).strength(0.1)) + .alpha(0.55) + .alphaDecay(0.05) + .stop(); + for (let i = 0; i < ticks; i++) soft.tick(); +} + export const hybridLayout: LayoutAdapter = { id: "hybrid", layout(scene: SceneData, ctx: LayoutContext): LayoutResult { @@ -234,7 +272,7 @@ function fullGraphLayout(scene: SceneData, ctx: LayoutContext): LayoutResult { ) .force("charge", forceManyBody().strength(physics.charge).distanceMax(2500)) .force("center", forceCenter(0, 0).strength(0.04)) - .force("collide", forceCollide((d) => (d as unknown as SimNode).r + physics.collide)) + .force("collide", makeCollide(physics.collide, nodes.length)) .alpha(1) .alphaDecay(0.05) .stop(); @@ -242,8 +280,15 @@ function fullGraphLayout(scene: SceneData, ctx: LayoutContext): LayoutResult { // scale, progressively reduce solver work; labels/edges are already // absent there and the stable global clustering matters more than a // fully converged individual-node packing. - const ticks = nodes.length > 10_000 ? 24 : nodes.length > 2_000 ? 60 : nodes.length > 800 ? 140 : 350; + const ticks = nodes.length > 10_000 ? 32 : nodes.length > 2_000 ? 60 : nodes.length > 800 ? 140 : 350; for (let i = 0; i < ticks; i++) sim.tick(); + // Hub de-overlap at overview scale (tick budget ~28): soft-pinned + // collide-only settle separates stacked hubs without a second + // many-body solve. Skip below 10k — main-sim collide iterations=2 + // already covers that band within the large-density timing budget. + if (nodes.length > 10_000) { + relaxCollideHubs(nodes, physics.collide, 36); + } for (const sn of nodes) { positions.set(sn.id, { x: sn.x ?? 0, y: sn.y ?? 0, r: sn.r }); } @@ -283,10 +328,13 @@ function fullCoreSolve( ) .force("charge", forceManyBody().strength(charge).distanceMax(Rcore * 2)) .force("center", forceCenter(0, 0).strength(0.05)) - .force("collide", forceCollide((d) => (d as unknown as SimNode).r + physics.collide)) + .force("collide", makeCollide(physics.collide, n)) .stop(); - const ticks = n > 10_000 ? 24 : n > 2_000 ? 60 : n > 800 ? 140 : 350; + const ticks = n > 10_000 ? 32 : n > 2_000 ? 60 : n > 800 ? 140 : 350; for (let i = 0; i < ticks; i++) sim.tick(); + if (n > 10_000) { + relaxCollideHubs(coreNodes, physics.collide, 36); + } let cx = 0; let cy = 0; @@ -380,7 +428,12 @@ function settleCore( function relaxAnchored(nodes: AnchoredNode[], ticks: number, collide = 6): void { const relax = forceSimulation(nodes as never[]) - .force("collide", forceCollide((d) => (d as unknown as AnchoredNode).r + collide).strength(1)) + .force( + "collide", + forceCollide((d) => collideForceRadius((d as unknown as AnchoredNode).r, collide)) + .strength(1) + .iterations(2), + ) .force("x", forceX((d: unknown) => (d as AnchoredNode).x0).strength((d: unknown) => ((d as AnchoredNode).survivor ? 0.55 : 0.08))) .force("y", forceY((d: unknown) => (d as AnchoredNode).y0).strength((d: unknown) => ((d as AnchoredNode).survivor ? 0.55 : 0.08))) .stop(); diff --git a/packages/knowledge-atlas/src/core/layout/types.ts b/packages/knowledge-atlas/src/core/layout/types.ts index c01b51d..1ebd344 100644 --- a/packages/knowledge-atlas/src/core/layout/types.ts +++ b/packages/knowledge-atlas/src/core/layout/types.ts @@ -29,6 +29,22 @@ export function aggregateRadius(count: number): number { return Math.min(34, 10 + Math.sqrt(count) * 2.4); } +/** Hub-scaled collide radius: visual r * scale + flat pad. + * Large hubs need more than r+pad so they separate under the reduced + * large-N tick budget without a full many-body re-solve. */ +export const COLLIDE_RADIUS_SCALE = 1.4; +/** Stronger hub clearance for overview corpora (N>10k). Kept off the + * default path so grade-1 survivor settles stay within stability bounds. */ +export const COLLIDE_RADIUS_SCALE_LARGE = 1.65; + +export function collideForceRadius( + visualR: number, + pad: number, + scale: number = COLLIDE_RADIUS_SCALE, +): number { + return visualR * scale + pad; +} + export function meanDisplacement( positions: Map, previous?: LayoutResult, diff --git a/packages/knowledge-atlas/src/core/types.ts b/packages/knowledge-atlas/src/core/types.ts index 4e4aaf7..c6df0a2 100644 --- a/packages/knowledge-atlas/src/core/types.ts +++ b/packages/knowledge-atlas/src/core/types.ts @@ -380,6 +380,14 @@ export type AtlasConfig = { /** Default true. */ keyboard?: boolean; typeMeta?: Record; + /** + * Seed layout positions (e.g. from IndexedDB atlas cache). When every + * scene node id is present, hybrid layout skips the force solve — + * cold Graph paint stays instant after a hard reload. + */ + initialPositions?: + | Map + | Record; }; export type AtlasPhysics = { @@ -398,7 +406,7 @@ export const DEFAULT_PHYSICS: AtlasPhysics = { export const LARGE_WIKI_PHYSICS: AtlasPhysics = { charge: -1000, link: 220, - collide: 28, + collide: 48, }; export type AtlasThemeToken = diff --git a/packages/knowledge-atlas/src/iife.ts b/packages/knowledge-atlas/src/iife.ts index de09116..f981d02 100644 --- a/packages/knowledge-atlas/src/iife.ts +++ b/packages/knowledge-atlas/src/iife.ts @@ -13,6 +13,21 @@ import { coreRadius, isFullGraphScene, populatedShellBands } from "./core/layout import { viewScaleToFit } from "./core/scene/shells.ts"; import { hybridLayout } from "./core/layout/hybrid.ts"; import { clampCameraScale, projectCamera, responsiveNodeScale, wheelZoomFactor } from "./interaction/camera.ts"; +import { + boundsFromPositions, + computeAtlasCameraFit, + computeFitTransform, + type AtlasCamera, + type FitBounds, +} from "./animation/replayCamera.ts"; +import { + clientRubberToScene, + clientToScenePoint, + defaultPartitionPolicyForType, + idsInRubberBand, + isRubberClick, + normalizeRubberRect, +} from "./partition/rubberBand.ts"; import { boundaryHoverDelay, projectedBoundaryDepth } from "./interaction/hover.ts"; import { AggregateTooltip, LONG_PRESS_MS } from "./interaction/tooltip.ts"; import { @@ -43,12 +58,23 @@ export type MountOptions = { export type MountHandle = { engine: AtlasEngine; + /** Drop hover intent / visible hover (pointer leave). */ + clearHover: () => void; + /** Clear hover + selection + focus decoration (pointer leave). */ + clearHighlights: () => void; /** Update the label policy live (wired to the host's label picker). */ setLabels: (mode: "auto" | "on" | "off", types?: readonly string[] | null) => void; /** Update edge stroke policy live (wired to the host's edges control). */ setEdges: (mode: "auto" | "on" | "off") => void; /** Update the classic force controls and re-solve the central graph. */ setPhysics: (physics: Partial) => void; + /** Live camera (centre-origin) for host rubber-band / replay fit. */ + getCamera: () => AtlasCamera; + setCamera: (next: Partial) => void; + /** Fit camera to layout-space bounds (curation-replay / host autofit). */ + fitToBounds: (bounds: FitBounds, opts?: { maxScale?: number }) => void; + /** Fit camera to current layout positions (whole scene). */ + fitToContent: (opts?: { maxScale?: number }) => void; destroy: () => void; }; @@ -591,10 +617,23 @@ export function mount(container: HTMLElement, opts: MountOptions): MountHandle { } engine.zoomTo(engine.getState().semanticScale + (ev.deltaY < 0 ? 0.2 : -0.2)); }; + const onPointerLeave = () => { + // Leaving the canvas must drop hover AND selection/focus — otherwise + // accent rings / hover halos stick after the pointer exits the graph. + clearHoverIntent(true); + try { + engine.select([], "replace"); + engine.clearFocus(); + } catch { + /* ignore */ + } + draw(1); + }; canvas.addEventListener("pointerdown", onDown); canvas.addEventListener("pointermove", onMove); canvas.addEventListener("pointerup", onUp); canvas.addEventListener("pointercancel", onCancel); + canvas.addEventListener("pointerleave", onPointerLeave); canvas.addEventListener("dblclick", onDbl); canvas.addEventListener("wheel", onWheel, { passive: false }); @@ -621,6 +660,17 @@ export function mount(container: HTMLElement, opts: MountOptions): MountHandle { return { engine, + clearHover: () => clearHoverIntent(true), + clearHighlights: () => { + clearHoverIntent(true); + try { + engine.select([], "replace"); + engine.clearFocus(); + } catch { + /* ignore */ + } + draw(1); + }, setLabels: (mode, types) => { labelState.mode = mode; if (types !== undefined) labelState.types = types ? new Set(types) : null; @@ -643,6 +693,50 @@ export function mount(container: HTMLElement, opts: MountOptions): MountHandle { } engine.setPhysics(physics); }, + getCamera: () => ({ x: camera.x, y: camera.y, scale: camera.scale }), + setCamera: (next) => { + if (typeof next.x === "number") camera.x = next.x; + if (typeof next.y === "number") camera.y = next.y; + if (typeof next.scale === "number") { + userZoomed = true; + camera.scale = clampCameraScale(next.scale); + scheduleDensity(); + } + draw(1); + }, + fitToBounds: (bounds, fitOpts) => { + const fitted = computeAtlasCameraFit(bounds, { + viewW: viewport.width, + viewH: viewport.height, + maxScale: fitOpts?.maxScale, + }); + userZoomed = true; + camera.x = fitted.x; + camera.y = fitted.y; + camera.scale = clampCameraScale(fitted.scale); + scheduleDensity(); + draw(1); + }, + fitToContent: (fitOpts) => { + const snap = engine.snapshot(); + const layout = snap.layout; + if (!layout?.positions?.size) return; + const pts: { x: number; y: number }[] = []; + for (const p of layout.positions.values()) pts.push({ x: p.x, y: p.y }); + const b = boundsFromPositions(pts); + if (!b) return; + const fitted = computeAtlasCameraFit(b, { + viewW: viewport.width, + viewH: viewport.height, + maxScale: fitOpts?.maxScale ?? 1.5, + }); + userZoomed = true; + camera.x = fitted.x; + camera.y = fitted.y; + camera.scale = clampCameraScale(fitted.scale); + scheduleDensity(); + draw(1); + }, destroy: () => { destroyed = true; cancelAnimationFrame(raf); @@ -663,3 +757,16 @@ export function mount(container: HTMLElement, opts: MountOptions): MountHandle { }, }; } + +/** Host-facing pure helpers (also on window.KnowledgeAtlas after IIFE build). */ +export { + clientToScenePoint, + clientRubberToScene, + normalizeRubberRect, + isRubberClick, + idsInRubberBand, + defaultPartitionPolicyForType, + computeAtlasCameraFit, + computeFitTransform, + boundsFromPositions, +}; diff --git a/packages/knowledge-atlas/src/index.ts b/packages/knowledge-atlas/src/index.ts index d41ea21..ab40804 100644 --- a/packages/knowledge-atlas/src/index.ts +++ b/packages/knowledge-atlas/src/index.ts @@ -55,3 +55,60 @@ export { export { LocalSceneSource } from "./datasources/local.ts"; export { ScaledDataSource, SCALED_TOTAL_LEAVES } from "./datasources/scaled.ts"; export { RemoteDataSource } from "./datasources/remote.ts"; + +export { + playReplayTimeline, + type ReplayEvent, + type HistoryDoc, + type ReplayHandlers, +} from "./animation/replayTimeline.ts"; + +export { + buildHistoryFromGraph, + sortReplayEvents, + snapshotAt, + indexAtTime, + type GraphNodeIn, + type GraphEdgeIn, + type BuildHistoryOptions, + type ReplaySnapshot, +} from "./animation/historyFromGraph.ts"; + +export { + computeFitTransform, + boundsFromPositions, + blendZoom, + compareSyntheticNodes, + typeTier, + computeAtlasCameraFit, + blendAtlasCamera, + type FitBounds, + type ZoomTransform, + type FitViewOpts, + type AtlasCamera, +} from "./animation/replayCamera.ts"; + +export { + buildPartitionSelection, + setPartitionPolicy, + addToPartitionSelection, + removeFromPartitionSelection, + partitionPayload, + type PartitionPolicy, + type PartitionEntry, + type PartitionSeed, +} from "./partition/partitionSelection.ts"; + +export { + normalizeRubberRect, + isRubberClick, + idsInRubberBand, + defaultPartitionPolicyForType, + clientToScenePoint, + clientRubberToScene, + type RubberPoint, + type RubberRect, + type NormalizedRect, + type CanvasClientRect, +} from "./partition/rubberBand.ts"; + diff --git a/packages/knowledge-atlas/src/partition/partitionSelection.ts b/packages/knowledge-atlas/src/partition/partitionSelection.ts new file mode 100644 index 0000000..d4fe31b --- /dev/null +++ b/packages/knowledge-atlas/src/partition/partitionSelection.ts @@ -0,0 +1,94 @@ +/** + * Workspace-partition selection helpers (Phase 2b+ split spike). + * + * Atlas already exposes multi-select (`select` / `selection-changed`). + * This module owns the move|copy policy map Switchbay's GraphTab used, + * so CE can drive POST /api/split without shell-specific state. + */ + +export type PartitionPolicy = "move" | "copy"; + +export type PartitionEntry = { id: string; policy: PartitionPolicy }; + +export type PartitionSeed = + | string + | { id: string; policy?: PartitionPolicy }; + +/** Normalize seeds into a de-duplicated selection (first policy wins). */ +export function buildPartitionSelection( + seeds: PartitionSeed[], + defaultPolicy: PartitionPolicy = "move", +): PartitionEntry[] { + const out: PartitionEntry[] = []; + const seen = new Set(); + for (const seed of seeds) { + const id = typeof seed === "string" ? seed.trim() : String(seed.id || "").trim(); + if (!id || seen.has(id)) continue; + seen.add(id); + const policy = + typeof seed === "string" + ? defaultPolicy + : seed.policy === "copy" || seed.policy === "move" + ? seed.policy + : defaultPolicy; + out.push({ id, policy }); + } + return out; +} + +/** Toggle or set policy for one id. */ +export function setPartitionPolicy( + selection: PartitionEntry[], + id: string, + policy: PartitionPolicy, +): PartitionEntry[] { + const clean = id.trim(); + if (!clean) return selection.slice(); + let found = false; + const next = selection.map((e) => { + if (e.id !== clean) return e; + found = true; + return { id: e.id, policy }; + }); + if (!found) next.push({ id: clean, policy }); + return next; +} + +/** Add ids (default policy) without dropping existing policies. */ +export function addToPartitionSelection( + selection: PartitionEntry[], + ids: string[], + defaultPolicy: PartitionPolicy = "move", +): PartitionEntry[] { + let next = selection.slice(); + for (const raw of ids) { + const id = raw.trim(); + if (!id) continue; + if (next.some((e) => e.id === id)) continue; + next.push({ id, policy: defaultPolicy }); + } + return next; +} + +/** Remove one id. */ +export function removeFromPartitionSelection( + selection: PartitionEntry[], + id: string, +): PartitionEntry[] { + const clean = id.trim(); + return selection.filter((e) => e.id !== clean); +} + +/** Split selection into move/copy ref lists for POST /api/split. */ +export function partitionPayload(selection: PartitionEntry[]): { + move: string[]; + copy: string[]; +} { + const move: string[] = []; + const copy: string[] = []; + for (const e of selection) { + if (e.policy === "copy") copy.push(e.id); + else move.push(e.id); + } + return { move, copy }; +} diff --git a/packages/knowledge-atlas/src/partition/rubberBand.ts b/packages/knowledge-atlas/src/partition/rubberBand.ts new file mode 100644 index 0000000..d183a4e --- /dev/null +++ b/packages/knowledge-atlas/src/partition/rubberBand.ts @@ -0,0 +1,117 @@ +/** + * Rubber-band (marquee) selection helpers for workspace split targeting. + * + * Pure geometry — hosts map node positions into the same coordinate + * space as the drag rect (screen px for Classic SVG; scene px for Atlas + * hitTester.boxQuery). Used by wiki-view Graph.splitEnter and tests. + */ + +export type RubberPoint = { + id: string; + x: number; + y: number; + /** Page type — drives default move|copy policy when seeding. */ + type?: string; +}; + +export type RubberRect = { + x0: number; + y0: number; + x1: number; + y1: number; +}; + +export type NormalizedRect = { + minX: number; + maxX: number; + minY: number; + maxY: number; + width: number; + height: number; +}; + +/** Axis-aligned normalize (drag may go any direction). */ +export function normalizeRubberRect(rect: RubberRect): NormalizedRect { + const minX = Math.min(rect.x0, rect.x1); + const maxX = Math.max(rect.x0, rect.x1); + const minY = Math.min(rect.y0, rect.y1); + const maxY = Math.max(rect.y0, rect.y1); + return { + minX, + maxX, + minY, + maxY, + width: maxX - minX, + height: maxY - minY, + }; +} + +/** True when the gesture is a click, not a band (Switchbay uses 4px). */ +export function isRubberClick(rect: RubberRect, minPx = 4): boolean { + const n = normalizeRubberRect(rect); + return n.width < minPx && n.height < minPx; +} + +/** Ids whose centers lie inside the rubber-band (inclusive edges). */ +export function idsInRubberBand( + points: readonly RubberPoint[], + rect: RubberRect, +): string[] { + if (isRubberClick(rect)) return []; + const { minX, maxX, minY, maxY } = normalizeRubberRect(rect); + const out: string[] = []; + const seen = new Set(); + for (const p of points) { + const id = (p.id || "").trim(); + if (!id || seen.has(id)) continue; + if (p.x >= minX && p.x <= maxX && p.y >= minY && p.y <= maxY) { + seen.add(id); + out.push(id); + } + } + return out; +} + +/** + * Charter / Switchbay default: entities & concepts copy to both sides; + * everything else moves. + */ +export function defaultPartitionPolicyForType( + type?: string | null, +): "move" | "copy" { + const t = (type || "").toLowerCase(); + return t === "entity" || t === "concept" ? "copy" : "move"; +} + +/** Canvas client rect (DOM getBoundingClientRect shape). */ +export type CanvasClientRect = { + left: number; + top: number; + width: number; + height: number; +}; + +/** + * Map a client (viewport) point into Atlas scene space — origin at the + * canvas centre, matching hitTester / iife `toScene`. + */ +export function clientToScenePoint( + clientX: number, + clientY: number, + canvasRect: CanvasClientRect, +): { x: number; y: number } { + return { + x: clientX - canvasRect.left - canvasRect.width / 2, + y: clientY - canvasRect.top - canvasRect.height / 2, + }; +} + +/** Map a client-space rubber rect into scene space for `boxQuery`. */ +export function clientRubberToScene( + rect: RubberRect, + canvasRect: CanvasClientRect, +): RubberRect { + const a = clientToScenePoint(rect.x0, rect.y0, canvasRect); + const b = clientToScenePoint(rect.x1, rect.y1, canvasRect); + return { x0: a.x, y0: a.y, x1: b.x, y1: b.y }; +} diff --git a/packages/knowledge-atlas/src/react/KnowledgeAtlas.tsx b/packages/knowledge-atlas/src/react/KnowledgeAtlas.tsx index a9d2c53..a5baad7 100644 --- a/packages/knowledge-atlas/src/react/KnowledgeAtlas.tsx +++ b/packages/knowledge-atlas/src/react/KnowledgeAtlas.tsx @@ -625,10 +625,21 @@ export const KnowledgeAtlas = forwardRef( } } }; + const onPointerLeave = () => { + clearHoverIntent(true); + try { + engine.select([], "replace"); + engine.clearFocus(); + } catch { + /* ignore */ + } + draw(1); + }; canvas.addEventListener("pointerdown", onPointerDown); canvas.addEventListener("pointermove", onPointerMove); canvas.addEventListener("pointerup", onPointerUp); canvas.addEventListener("pointercancel", onPointerCancel); + canvas.addEventListener("pointerleave", onPointerLeave); canvas.addEventListener("dblclick", onDblClick); canvas.addEventListener("wheel", onWheel, { passive: false }); canvas.addEventListener("keydown", onKeyDown); @@ -664,6 +675,7 @@ export const KnowledgeAtlas = forwardRef( canvas.removeEventListener("pointermove", onPointerMove); canvas.removeEventListener("pointerup", onPointerUp); canvas.removeEventListener("pointercancel", onPointerCancel); + canvas.removeEventListener("pointerleave", onPointerLeave); canvas.removeEventListener("dblclick", onDblClick); canvas.removeEventListener("wheel", onWheel); canvas.removeEventListener("keydown", onKeyDown); diff --git a/packages/knowledge-atlas/tests/historyFromGraph.test.ts b/packages/knowledge-atlas/tests/historyFromGraph.test.ts new file mode 100644 index 0000000..42aa9da --- /dev/null +++ b/packages/knowledge-atlas/tests/historyFromGraph.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "vitest"; +import { + buildHistoryFromGraph, + indexAtTime, + snapshotAt, + sortReplayEvents, +} from "../src/animation/historyFromGraph.ts"; +import type { ReplayEvent } from "../src/animation/replayTimeline.ts"; + +describe("buildHistoryFromGraph", () => { + it("emits nodes then edges with both endpoints present", () => { + const hist = buildHistoryFromGraph( + [ + { id: "b", title: "B", type: "concept", degree: 1 }, + { id: "a", title: "A", type: "source", degree: 0 }, + ], + [{ source: "a", target: "b" }], + { duration: 10, source: "test" }, + ); + expect(hist.source).toBe("test"); + expect(hist.events.filter((e) => e.op === "node").map((e) => e.op === "node" && e.id)).toEqual([ + "a", + "b", + ]); + const edge = hist.events.find((e) => e.op === "edge"); + expect(edge).toEqual({ t: expect.any(Number), op: "edge", source: "a", target: "b" }); + expect(hist.degree).toEqual({ a: 1, b: 1 }); + }); + + it("returns empty for no nodes", () => { + const hist = buildHistoryFromGraph([], []); + expect(hist.events).toEqual([]); + expect(hist.source).toBe("empty"); + }); +}); + +describe("snapshotAt / indexAtTime", () => { + const events: ReplayEvent[] = [ + { t: 0, op: "node", id: "a", title: "A", type: "entity" }, + { t: 5, op: "node", id: "b", title: "B", type: "entity" }, + { t: 5, op: "edge", source: "a", target: "b" }, + ]; + const hist = { duration: 10, events: sortReplayEvents(events), source: "t" }; + + it("scrubs by index", () => { + expect(snapshotAt(hist, 0).nodes).toEqual([]); + expect(snapshotAt(hist, 1).nodes.map((n) => n.id)).toEqual(["a"]); + expect(snapshotAt(hist, 3).edges).toEqual([{ source: "a", target: "b" }]); + }); + + it("maps wall time to index", () => { + expect(indexAtTime(hist, 0)).toBe(1); + expect(indexAtTime(hist, 4.9)).toBe(1); + expect(indexAtTime(hist, 5)).toBe(3); + }); +}); diff --git a/packages/knowledge-atlas/tests/partitionSelection.test.ts b/packages/knowledge-atlas/tests/partitionSelection.test.ts new file mode 100644 index 0000000..65aec99 --- /dev/null +++ b/packages/knowledge-atlas/tests/partitionSelection.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; +import { + addToPartitionSelection, + buildPartitionSelection, + partitionPayload, + removeFromPartitionSelection, + setPartitionPolicy, +} from "../src/partition/partitionSelection.ts"; + +describe("partitionSelection", () => { + it("builds from mixed seeds with first-wins dedupe", () => { + const sel = buildPartitionSelection([ + "a", + { id: "b", policy: "copy" }, + { id: "a", policy: "copy" }, + " ", + ]); + expect(sel).toEqual([ + { id: "a", policy: "move" }, + { id: "b", policy: "copy" }, + ]); + }); + + it("toggles policy and builds API payload", () => { + let sel = buildPartitionSelection(["x", "y"]); + sel = setPartitionPolicy(sel, "y", "copy"); + sel = addToPartitionSelection(sel, ["z"]); + sel = removeFromPartitionSelection(sel, "x"); + expect(partitionPayload(sel)).toEqual({ + move: ["z"], + copy: ["y"], + }); + }); +}); diff --git a/packages/knowledge-atlas/tests/replayCamera.test.ts b/packages/knowledge-atlas/tests/replayCamera.test.ts new file mode 100644 index 0000000..13c5792 --- /dev/null +++ b/packages/knowledge-atlas/tests/replayCamera.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, it } from "vitest"; +import { + blendAtlasCamera, + blendZoom, + boundsFromPositions, + compareSyntheticNodes, + computeAtlasCameraFit, + computeFitTransform, + typeTier, +} from "../src/animation/replayCamera.ts"; +import { buildHistoryFromGraph } from "../src/animation/historyFromGraph.ts"; + +describe("computeFitTransform / blendZoom", () => { + it("centres a bounds cloud in the view", () => { + const t = computeFitTransform( + { minX: 0, minY: 0, maxX: 200, maxY: 100 }, + { viewW: 1000, viewH: 600 }, + ); + expect(t.k).toBeGreaterThan(0); + // Centre of cloud (100, 50) maps near view centre after transform. + const sx = 100 * t.k + t.x; + const sy = 50 * t.k + t.y; + expect(sx).toBeCloseTo(500, 0); + expect(sy).toBeCloseTo(300, 0); + }); + + it("blends toward a target", () => { + const a = { x: 0, y: 0, k: 1 }; + const b = { x: 100, y: 50, k: 2 }; + const mid = blendZoom(a, b, 0.5); + expect(mid).toEqual({ x: 50, y: 25, k: 1.5 }); + }); + + it("boundsFromPositions returns null for empty", () => { + expect(boundsFromPositions([])).toBeNull(); + expect(boundsFromPositions([{ x: 1, y: 2 }, { x: 3, y: 4 }])).toEqual({ + minX: 1, + minY: 2, + maxX: 3, + maxY: 4, + }); + }); +}); + +describe("compareSyntheticNodes / created ordering", () => { + it("orders by created when present", () => { + const nodes = [ + { id: "late", type: "source", degree: 99, created: "2024-06-01" }, + { id: "early", type: "concept", degree: 0, created: "2023-01-01" }, + ]; + nodes.sort(compareSyntheticNodes); + expect(nodes.map((n) => n.id)).toEqual(["early", "late"]); + }); + + it("falls back to type tier then degree", () => { + expect(typeTier("source")).toBe(0); + expect(typeTier("concept")).toBe(3); + const nodes = [ + { id: "c", type: "concept", degree: 1 }, + { id: "s", type: "source", degree: 0 }, + ]; + nodes.sort(compareSyntheticNodes); + expect(nodes.map((n) => n.id)).toEqual(["s", "c"]); + }); + + it("buildHistoryFromGraph respects created", () => { + const hist = buildHistoryFromGraph( + [ + { id: "b", title: "B", type: "source", degree: 10, created: "2025-01-01" }, + { id: "a", title: "A", type: "note", degree: 0, created: "2020-01-01" }, + ], + [], + { duration: 10, source: "test" }, + ); + const ids = hist.events.filter((e) => e.op === "node").map((e) => (e.op === "node" ? e.id : "")); + expect(ids).toEqual(["a", "b"]); + }); +}); + +describe("computeAtlasCameraFit / blendAtlasCamera", () => { + it("centres cloud at scene origin", () => { + const cam = computeAtlasCameraFit( + { minX: 0, minY: 0, maxX: 200, maxY: 100 }, + { viewW: 1000, viewH: 600 }, + ); + expect(cam.scale).toBeGreaterThan(0); + // World centre (100, 50) → screen 0 after camera. + expect(100 * cam.scale + cam.x).toBeCloseTo(0, 5); + expect(50 * cam.scale + cam.y).toBeCloseTo(0, 5); + }); + + it("blends atlas cameras", () => { + const mid = blendAtlasCamera( + { x: 0, y: 0, scale: 1 }, + { x: 100, y: 50, scale: 2 }, + 0.5, + ); + expect(mid).toEqual({ x: 50, y: 25, scale: 1.5 }); + }); +}); diff --git a/packages/knowledge-atlas/tests/replayTimeline.test.ts b/packages/knowledge-atlas/tests/replayTimeline.test.ts new file mode 100644 index 0000000..f7f4c16 --- /dev/null +++ b/packages/knowledge-atlas/tests/replayTimeline.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it } from "vitest"; +import { playReplayTimeline, type ReplayEvent } from "../src/animation/replayTimeline.ts"; + +describe("playReplayTimeline", () => { + it("emits events in time order and calls onDone", () => { + const events: ReplayEvent[] = [ + { t: 0, op: "node", id: "a", title: "A", type: "entity" }, + { t: 10, op: "node", id: "b", title: "B", type: "concept" }, + { t: 15, op: "edge", source: "a", target: "b" }, + ]; + let wall = 0; + const seen: string[] = []; + const timers: Array<{ id: number; ms: number; fn: () => void }> = []; + let nextId = 1; + + const cancel = playReplayTimeline( + { duration: 20, events, source: "test" }, + { + now: () => wall, + schedule: (fn, ms) => { + const id = nextId++; + timers.push({ id, ms, fn }); + return id; + }, + cancel: (id) => { + const i = timers.findIndex((t) => t.id === id); + if (i >= 0) timers.splice(i, 1); + }, + onEvent: (ev) => { + seen.push(ev.op === "node" ? `n:${ev.id}` : `e:${ev.source}->${ev.target}`); + }, + onDone: () => seen.push("done"), + }, + ); + + // Drain: run scheduled callbacks, advancing wall to cover delays. + while (timers.length) { + const t = timers.shift()!; + wall += t.ms; + t.fn(); + } + expect(seen).toEqual(["n:a", "n:b", "e:a->b", "done"]); + cancel(); + }); + + it("cancel stops further events", () => { + let wall = 0; + const seen: string[] = []; + const timers: Array<{ id: number; ms: number; fn: () => void }> = []; + let nextId = 1; + const stop = playReplayTimeline( + { + duration: 100, + source: "test", + events: [ + { t: 0, op: "node", id: "a", title: "A", type: "entity" }, + { t: 50, op: "node", id: "b", title: "B", type: "entity" }, + ], + }, + { + now: () => wall, + schedule: (fn, ms) => { + const id = nextId++; + timers.push({ id, ms, fn }); + return id; + }, + cancel: (id) => { + const i = timers.findIndex((t) => t.id === id); + if (i >= 0) timers.splice(i, 1); + }, + onEvent: (ev) => { + if (ev.op === "node") seen.push(ev.id); + }, + }, + ); + // First tick at t=0 + const first = timers.shift()!; + wall += first.ms; + first.fn(); + expect(seen).toEqual(["a"]); + stop(); + // Remaining timers cleared by cancel path — drain whatever is left should not add b after stop if cancel removed them + while (timers.length) { + const t = timers.shift()!; + wall += t.ms; + t.fn(); + } + expect(seen).toEqual(["a"]); + }); +}); diff --git a/packages/knowledge-atlas/tests/rubberBand.test.ts b/packages/knowledge-atlas/tests/rubberBand.test.ts new file mode 100644 index 0000000..3ff9ab6 --- /dev/null +++ b/packages/knowledge-atlas/tests/rubberBand.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from "vitest"; +import { + clientRubberToScene, + clientToScenePoint, + defaultPartitionPolicyForType, + idsInRubberBand, + isRubberClick, + normalizeRubberRect, +} from "../src/partition/rubberBand.ts"; + +describe("rubberBand", () => { + it("normalizes inverted drag corners", () => { + expect(normalizeRubberRect({ x0: 10, y0: 20, x1: 4, y1: 8 })).toEqual({ + minX: 4, + maxX: 10, + minY: 8, + maxY: 20, + width: 6, + height: 12, + }); + }); + + it("treats tiny rects as clicks", () => { + expect(isRubberClick({ x0: 0, y0: 0, x1: 3, y1: 3 })).toBe(true); + expect(isRubberClick({ x0: 0, y0: 0, x1: 5, y1: 5 })).toBe(false); + }); + + it("selects node centers inside the band", () => { + const points = [ + { id: "a", x: 10, y: 10 }, + { id: "b", x: 50, y: 50 }, + { id: "c", x: 100, y: 10 }, + { id: " ", x: 20, y: 20 }, + ]; + expect(idsInRubberBand(points, { x0: 0, y0: 0, x1: 60, y1: 60 })).toEqual([ + "a", + "b", + ]); + expect(idsInRubberBand(points, { x0: 0, y0: 0, x1: 2, y1: 2 })).toEqual([]); + }); + + it("defaults entity/concept to copy", () => { + expect(defaultPartitionPolicyForType("entity")).toBe("copy"); + expect(defaultPartitionPolicyForType("concept")).toBe("copy"); + expect(defaultPartitionPolicyForType("note")).toBe("move"); + expect(defaultPartitionPolicyForType(undefined)).toBe("move"); + }); +}); + +describe("clientToScenePoint / clientRubberToScene", () => { + const rect = { left: 100, top: 50, width: 400, height: 300 }; + + it("maps client coords to centre-origin scene space", () => { + expect(clientToScenePoint(300, 200, rect)).toEqual({ x: 0, y: 0 }); + expect(clientToScenePoint(100, 50, rect)).toEqual({ x: -200, y: -150 }); + }); + + it("maps a client rubber rect into scene space", () => { + const scene = clientRubberToScene( + { x0: 100, y0: 50, x1: 300, y1: 200 }, + rect, + ); + expect(scene).toEqual({ x0: -200, y0: -150, x1: 0, y1: 0 }); + }); +}); diff --git a/skills/curiosity-engine/docs/ADR-001-proxy-embed-and-hosted.md b/skills/curiosity-engine/docs/ADR-001-proxy-embed-and-hosted.md new file mode 100644 index 0000000..836d09a --- /dev/null +++ b/skills/curiosity-engine/docs/ADR-001-proxy-embed-and-hosted.md @@ -0,0 +1,41 @@ +# ADR-001 — Proxy embed prefix + hosted-shell contract (Phase 2a) + +**Status:** Accepted (2026-09-18, Europe/Zurich) +**Charter:** umbrella `/workspace/skill-shell-rationalization/docs/CHARTER.md` +**Related:** locked decisions #1 (no iframes), #2 (migrate Switchbay features into CE before thinning), #6 (hosted mode — okstratr settings; CE mirrors the hook). + +## Context + +Switchbay and okbay must **same-origin reverse-proxy** CE at `127.0.0.1:8766` under `/embed/ce/` (not iframes). Absolute frontend fetches (`/api/page`, `/api/vault/…`) break under a path prefix unless the server and static bundle share a public base. Shells also need a `host=switchbay|okbay` signal so CE can later defer workspace/harness chrome to the shell. + +## Decision + +1. **Configurable public base** via env `CE_PUBLIC_BASE` (normalized, no trailing slash; typical value `/embed/ce`). `viewer_server.py` strips that prefix from request paths before routing/static, and injects: + + - `window.CE_PUBLIC_BASE` + - `window.CE_HOSTED` (`""` | `"switchbay"` | `"okbay"`) + - `window.ceApi(path)` → `CE_PUBLIC_BASE + path` + + Wiki-view `edit.js` / `vault.js` call `ceApi` (with a local fallback) so API URLs stay under the embed prefix when proxied. + +2. **Hosted-shell detection** (header wins over query): + + - Header: `X-CE-Host: switchbay|okbay` + - Query: `?host=switchbay|okbay` + - Probe: `GET /api/hosted` → stub policy JSON (`hosted_settings_policy`) + +3. **Loopback bind unchanged:** always `127.0.0.1`. Bare `viewer.sh` default port remains **8090**. Embed/host daemons commonly use **8766** (charter). Setting `CE_PUBLIC_BASE` does not change bind host or imply a port change. + +4. **Settings split stub:** when hosted, `shell_owns_workspace_settings: true` but `ce_viewer_knobs_enabled: true` until a parity-gated follow-up. Full HTML settings ownership split is large and deferred (2b+); this ADR only locks the wire contract. + +## Consequences + +- Proxies may forward either stripped paths (`/api/…`) or prefixed paths (`/embed/ce/api/…`); CE handles the latter when `CE_PUBLIC_BASE` is set. JS always needs the injected base for browser same-origin fetches. +- Migration of Switchbay filebrowser / graph animation / split into CE is tracked in [`MIGRATION-INTAKE.md`](MIGRATION-INTAKE.md); do not delete Switchbay surfaces until the umbrella parity checklist is green. +- okbay’s current `:8766` Atlas host and Switchbay’s tab chrome remain dual-stack until Phase 4/5 thinning. + +## Alternatives considered + +- **`` only** — fragile with absolute `/api` and mixed relative assets; rejected. +- **Iframe embed** — forbidden by charter #1. +- **Full settings disable in 2a** — too large a UX split; stub + document instead. diff --git a/skills/curiosity-engine/docs/MIGRATION-INTAKE.md b/skills/curiosity-engine/docs/MIGRATION-INTAKE.md new file mode 100644 index 0000000..ee912b7 --- /dev/null +++ b/skills/curiosity-engine/docs/MIGRATION-INTAKE.md @@ -0,0 +1,233 @@ +# Migration intake map — Switchbay → CE (Phase 2a / 2b / 2b++) + +Charter north star: **CE** owns graph, atlas, wiki, search, source viewer, **filebrowser**, **graph animation + split**. Switchbay keeps PWA shell (workspaces, watcher, tabs chrome, rail UI) and reverse-proxies CE under `/embed/ce/`. + +**Do not delete** Switchbay implementations until `/workspace/skill-shell-rationalization/docs/PARITY-CHECKLIST.md` (umbrella) is checked and a feature flag gates the cutover. + +## Target homes in CE + +| Capability | CE destination | Notes | +|------------|----------------|-------| +| Atlas / graph engine | `packages/knowledge-atlas` | Already the shared engine; Switchbay Graph tab adapts via examples/switchbay | +| Classic wiki viewer + APIs | `skills/curiosity-engine/template/wiki-view` + `scripts/viewer_server.py` | Phase 2a: `CE_PUBLIC_BASE` / hosted hook | +| Filebrowser | wiki-view sidebar **Files** mode + `GET /api/tree` + `/api/fs/*` | Phase 2b browse; 2b++ FS mutate + file-routes stub | +| Graph opening animation | knowledge-atlas timeline + wiki-view SVG replay | Phase 2b hook; 2b+++ minimal SVG play/pause/scrub/step | +| Workspace split | CE API + Classic rubber-band UI + atlas multi-select; shell may keep tab-open chrome | Phase 2b++++++ (partition API + registry + rubber-band) | + +## Switchbay source map (intake) + +Paths relative to the Switchbay repo root (branch `feat/skill-shell-rationalization`). + +### Filebrowser + +| Artifact | Role | +|----------|------| +| `frontend/src/sidebar/FileBrowser.tsx` (~1.3k LOC) | Tree, sort, search/regex, context menu, pack file-routes | +| `frontend/src/sidebar/Sidebar.tsx` | Host chrome wrapping browser + selection | +| `frontend/src/sidebar/SourceBrowser.tsx` | Source-oriented browse | +| `frontend/src/sidebar/WikiPane.tsx` | Wiki pane adjacent to browser | +| `frontend/src/sidebar/ingestDrop.ts` | Drop-to-ingest bridge | + +**CE intake notes:** Prefer CE vault/wiki path semantics (`vault/`, `wiki/`) over Switchbay workspace-relative pack routes. Right-click / highlight / search parity is on the umbrella checklist. Pack-specific `file_routes` may stay shell-side or become CE extension hooks later. + +#### Phase 2b landed (CE) + +| Artifact | Role | +|----------|------| +| `scripts/filebrowser_tree.py` | Walk `vault/` + `wiki/` only; prune SKIP_DIRS / dot dirs | +| `scripts/filebrowser_match.py` | Substring / `/re/flags` / `*.ext` matchers (Switchbay parity) | +| `GET /api/tree` in `viewer_server.py` | `{ ok, roots, files, count }` — honors `CE_PUBLIC_BASE` strip | +| `template/wiki-view/static/filebrowser.js` | Pages\|Files toggle, tree, filter, highlight, context menu | +| Tests | `tests/test_filebrowser.py` | + +**Parity vs Switchbay (filebrowser):** + +| Behavior | Status | +|----------|--------| +| Browse vault + wiki tree | ✅ CE (`/api/tree`) | +| Search substring / regex / `*.ext` | ✅ | +| Highlight from graph search | ✅ (`FileBrowser.setSearchHits`) | +| Right-click Open / Copy / Reveal | ✅ (minimal) | +| Sort A↔Z | ✅ | +| Open wiki page → modal | ✅ | +| Open `*.extracted.md` via vault API | ✅ (`VaultSources.open`) | +| Pack `file_routes` / ext handlers | ≈ CE discovery + dispatch/install/enable (`GET /api/file-routes`, `/api/packs/*`); agent/LLM skill exec still shell | +| FS mutate (create/rename/move/delete/dup) | ✅ CE (`/api/fs/*`, vault/wiki sandbox) | +| Reveal-in-OS / open-external | ❌ deferred (shell / OS helpers) | +| Drop-to-ingest | ❌ deferred | +| SourceBrowser / WikiPane dual pane | ❌ deferred | +| Ext filter chip UI | ❌ deferred | + +#### Phase 2b++ landed (CE) — FS mutate + pack routes stub + +| Artifact | Role | +|----------|------| +| `scripts/filebrowser_fs.py` | Sandboxed create / mkdir / rename / move / delete / duplicate / stat under `vault/` + `wiki/` only | +| `scripts/filebrowser_packs.py` | Read-only `pack.json` `file_routes` from `.workbench/packs/`, `packs/`, or `CE_PACKS_DIR` | +| `POST /api/fs/{create,mkdir,rename,move,delete,duplicate}` + `GET /api/fs/stat` | Honors `CE_PUBLIC_BASE`; wiki writes trigger viewer rebuild | +| `GET /api/file-routes` | `{ ok, routes, count }` — enabled packs only | +| `template/wiki-view/static/filebrowser.js` | Context menu: New file/folder, Rename, Duplicate, Delete | +| Tests | `tests/test_filebrowser_fs.py` (sandbox escape + API) | + +**API notes for Switchbay embed:** call CE under `/embed/ce/api/fs/*` with the same JSON bodies as Switchbay Step E where overlapping (`delete` `{path}`, `duplicate` `{path}`). CE-only: `create` `{path, kind?, content?}`, `mkdir` `{path}`, `rename`/`move` `{path, to}`. Trash → OS trash when available, else `.workbench/trash/`. + +#### Phase 2b++++ landed (CE) — pack action dispatch + +| Artifact | Role | +|----------|------| +| `scripts/filebrowser_packs.py` | list / enable / local install / uninstall + sandboxed `dispatch_action` (same `pack.json` Manifest) | +| `GET /api/packs`, `GET /api/packs//actions` | Pack registry + per-pack file_routes | +| `POST /api/packs//action/` | `{path}` under vault/wiki → `{run_id, pack, action}` queue (`.workbench/pack-runs/`) | +| `POST /api/packs/toggle`, `POST /api/packs/install`, `DELETE /api/packs?name=` | Enable state + local-path install into `.workbench/packs/` (no git) | +| `template/wiki-view/static/filebrowser.js` | Context menu pack actions → dispatch API | +| Tests | `tests/test_filebrowser_packs.py` (sandbox escape + embed-prefix API) | + +**Still Switchbay-only (filebrowser):** reveal-in-OS / open-external, drop-to-ingest, git pack install, pip `requires_extra`, agent/LLM skill execution for queued runs, SourceBrowser/WikiPane dual pane, ext filter chips. + +Deep-link: `?filebrowser=1` opens Files mode. + +### Graph animation (curation replay) + +| Artifact | Role | +|----------|------| +| `frontend/src/widgets/graph/CurationReplay.tsx` | Graph-tab opening animation UI | +| `frontend/src/widgets/graph/curationReplayAnim.ts` | Shared rAF animation (~389 LOC) | +| `src/switchbay/curation_history.py` | History data behind replay | + +**CE intake notes:** Animation should consume CE graph/atlas data (or OKF/CE `data.json`), not Switchbay-only tab state. Ship behind a viewer flag; keep Switchbay button until parity. + +#### Phase 2b landed (CE) + +| Artifact | Role | +|----------|------| +| `packages/knowledge-atlas/src/animation/replayTimeline.ts` | Pure `playReplayTimeline` + `ReplayEvent` / `HistoryDoc` types | +| `packages/knowledge-atlas/tests/replayTimeline.test.ts` | Scheduler unit tests | + +#### Phase 2b+++ landed (CE) + +| Artifact | Role | +|----------|------| +| `packages/knowledge-atlas/src/animation/historyFromGraph.ts` | Pure `buildHistoryFromGraph` / `snapshotAt` / `indexAtTime` | +| `packages/knowledge-atlas/tests/historyFromGraph.test.ts` | Helper unit tests | +| `scripts/curation_history.py` | HistoryDoc from `data.json` (+ optional `.workbench/curation-history.json` cache) | +| `GET /api/curation/history` | Switchbay-compatible HistoryDoc JSON; honors `CE_PUBLIC_BASE` | +| `template/wiki-view/static/replay.js` | Vanilla SVG overlay: play/pause/scrub/step (`?replay=1`) | +| Tests | `tests/test_curation_history.py` | + +**Parity vs Switchbay (CurationReplay) — Phase 2b+++ polish:** + +| Behavior | Status | +|----------|--------| +| HistoryDoc + timed play | ✅ (`playReplayTimeline` contract + vanilla port) | +| Play / pause / scrub / step chrome | ✅ CE (Switchbay is autoplay-only + ↻) | +| SVG force grow animation | ✅ enter fade + radius inflate + settle kick | +| `CE_PUBLIC_BASE` / `ceApi` | ✅ | +| `GET /api/curation/history` | ≈ synthetic from `data.json` (`created`/type-tier order); prefers `.workbench/curation-history.json` when present | +| Git-log chronology rebuild | ❌ deferred (use Switchbay/workbench cache if present) | +| Opening autoplay | ✅ `?replay=1` / `?replay=auto` short intro autoplay; button opens paused | +| Atlas canvas binding | ✅ `CurationReplay.bindViewer(CEViewer)` + `fitToContent` after fade; pure `computeAtlasCameraFit` | +| Fade crossfade onto live graph | ✅ settle → overlay fade-out after autoplay intro | +| Top-label degree ranking + zoom autofit | ✅ top-30 labels; easeAutoFit on ticks & major scrub jumps | + +Pure helpers: `packages/knowledge-atlas/src/animation/replayCamera.ts` (`computeFitTransform`, `compareSyntheticNodes`, …). + + +### Split (subgraph → new workspace) + +| Artifact | Role | +|----------|------| +| `frontend/src/widgets/graph/GraphTab.tsx` | Split mode UI, rubber-band / proposal listener (`sy:split-proposal`) | +| `frontend/src/widgets/graph/atlas.ts` | Atlas stubs `splitEnter` / `splitExit` | +| `src/switchbay/splitting.py` | `split_workspace` implementation | +| `src/switchbay/daemon.py` | `POST /api/workspaces/split`, status, `POST /api/split/proposal` | + +**CE intake notes:** Engine multi-select already exists in knowledge-atlas (`select` / `selection-changed`). Move **page move/copy + wiki partition** logic into CE; Switchbay may retain workspace registry / tab open after split. Feature-flag dual-stack until checklist green. + +#### Phase 2b+ landed (CE) + +| Artifact | Role | +|----------|------| +| `scripts/wiki_partition.py` | Page resolve, copy wiki + vault citations + figures, manifests, MOVE → `wiki/.deleted//` | +| `POST /api/split` + `GET /api/split` in `viewer_server.py` | Sync partition; `target` or `CE_SPLIT_HOME/`; honors `CE_PUBLIC_BASE` | +| `template/wiki-view/static/split.js` | Minimal panel (`?split=1` / split control); move\|copy policies | +| `packages/knowledge-atlas` `partitionSelection` | Pure move\|copy selection helpers for atlas multi-select | +| `scripts/workspace_registry.py` | CE-owned workspace registry (`CE_WORKSPACE_REGISTRY` / XDG); sandbox under `$HOME` or `CE_WORKSPACE_HOME` | +| `scripts/cm_export.py` | Optional curiosity-merge `subgraph_export` subprocess hook (dry-run + sandbox) | +| `GET/POST/DELETE /api/workspaces` | Registry list / register / unregister | +| `POST /api/cm-export` | CM export for selected pages (`dry_run` supported) | +| `template/wiki-view/static/graph.js` + `split.js` | Classic rubber-band (Ctrl/⌘-drag), Shift-click multi-select, click toggle, Alt/right-click policy flip; `ce:split-proposal` / `sy:split-proposal` | +| `packages/knowledge-atlas` `rubberBand` | Pure rect/hit helpers + `defaultPartitionPolicyForType` | +| Tests | `tests/test_wiki_partition.py` + `tests/test_workspace_registry_cm_export.py` + `partitionSelection.test.ts` + `rubberBand.test.ts` | + +#### Phase 2b++++++ landed (CE) — rubber-band split targeting + +| Artifact | Role | +|----------|------| +| `template/wiki-view/static/graph.js` | `splitEnter` / `splitExit`; Ctrl/⌘-drag rubber-band; Shift-click add; click toggle; Alt/right-click flip move↔copy | +| `template/wiki-view/static/split.js` | Syncs panel ↔ graph selection; `POST /api/split` via `ceApi`; listens `ce:split-proposal` + `sy:split-proposal` | +| `packages/knowledge-atlas/src/partition/rubberBand.ts` | Pure `idsInRubberBand` / policy defaults (unit-tested) | +| Atlas facade | `splitEnter`/`splitExit` + Ctrl/⌘ canvas rubber-band; `fitToContent` for replay bind | + +Smoke: open wiki-view with `CE_PUBLIC_BASE=/embed/ce`, `?split=1`, Ctrl-drag over nodes, name workspace, Split. + +#### Phase 2b+++++++ landed (CE) — Atlas canvas bind + rubber-band + +| Artifact | Role | +|----------|------| +| `packages/knowledge-atlas` `replayCamera` | `computeAtlasCameraFit` / `blendAtlasCamera` (centre-origin) | +| `packages/knowledge-atlas` `rubberBand` | `clientToScenePoint` / `clientRubberToScene` for Atlas boxQuery | +| `MountHandle` / IIFE | `getCamera` / `setCamera` / `fitToBounds` / `fitToContent` | +| `template/wiki-view/static/atlas.js` | Ctrl/⌘-drag rubber-band → `hitTester.boxQuery` → split selection | +| `template/wiki-view/static/replay.js` | `bindViewer(CEViewer)`; `fitAtlasHost` after overlay fade | +| Tests | extended `rubberBand.test.ts` + `replayCamera.test.ts` | + +**Parity vs Switchbay (split):** + +| Behavior | Status | +|----------|--------| +| Export selected pages to new workspace | ✅ CE (`wiki_partition`) | +| MOVE vs COPY policy | ✅ | +| Transitive vault citations | ✅ | +| Wiki-relative figure embeds | ✅ | +| Dual-side `.curator/splits/` manifests | ✅ | +| Recoverable source prune (MOVE) | ✅ (`wiki/.deleted/`) | +| `CE_PUBLIC_BASE` embed paths | ✅ | +| Atlas multi-select policy helpers | ✅ | +| Workspace registry persistence | ✅ CE (`workspace_registry` + `GET/POST/DELETE /api/workspaces`; auto-register on split) | +| Tab open / chrome after split | ❌ shell (Switchbay) | +| Rubber-band / `sy:split-proposal` UI | ✅ CE Classic + **Atlas canvas** Ctrl/⌘-drag (`atlas.js` + `hitTester.boxQuery`); `ce:`/`sy:split-proposal` | +| CM `subgraph_export` hook (license modes) | ✅ CE (`cm_export` + `POST /api/cm-export`, dry-run supported; optional CM install) | +| Workspace-root figures / `.workbench` sketches | ❌ deferred | +| Async curator link-heal agents | ❌ deferred (shell) | +| OS Trash prune | ❌ deferred (CE uses `.deleted/`) | + + +#### Phase 2b+++++++ landed (CE) — Obsidian vault import + +| Artifact | Role | +|----------|------| +| `scripts/obsidian_import.py` | Folder/zip import; `$HOME`/`$CE_WORKSPACE_HOME` sandbox; zip-slip safe; maps **Obsidian vault root → `wiki/`** (wikilinks preserved verbatim); `.obsidian/` kept, `.trash` skipped | +| `POST /api/import/obsidian` | JSON `{path, target?, register?, force?}` or multipart zip; honours `CE_PUBLIC_BASE` | +| CLI | `python3 obsidian_import.py ` (conceptual `ce import-obsidian`) | +| Fixture | Umbrella `corpora/obsidian-dummy-vault/` (Omarchy E2E prep; not blocked on CE Benchmarker) | +| Tests | `tests/test_obsidian_import.py` (sandbox + zip-slip + wikilink presence + embed-prefix API) | + +**Mapping choice:** Obsidian vault root lands in CE `wiki/` (same surface Atlas / classic wiki / “open folder as vault” already expect). No Work/symlink dance. + +**Still deferred:** Omarchy/Switchbay GUI folder-picker (shell-side); richer Benchmarker corpus pack. + + +## Embed contract (for Switchbay Phase 4a) + +- Upstream: `http://127.0.0.1:8766` (loopback only) +- Public prefix: `/embed/ce` → set `CE_PUBLIC_BASE=/embed/ce` on the CE process +- Forward `X-CE-Host: switchbay` (okbay: `okbay`) or `?host=` +- No iframes — in-app panels load first-party proxied routes + +## Sequencing + +1. **2a (landed):** proxy prefix + hosted stub + this map + ADR +2. **2b (landed):** filebrowser shell API + minimal UI; animation timeline hook +3. **2b+ (this spike):** wiki partition / split API + minimal UI + atlas selection helpers +4. **2b++ / 2b+++ / 2b++++ / 2b+++++ / 2b++++++ :** FS mutate + packs + SVG replay + registry + CM export + Classic **and Atlas** rubber-band + Atlas replay camera bind landed; heal agents / git-history rebuild / reveal-in-OS / drop-ingest / tab-open chrome still deferred +5. **4b:** Switchbay tabs thin to `/embed/ce/` once parity checklist passes diff --git a/skills/curiosity-engine/docs/architecture.md b/skills/curiosity-engine/docs/architecture.md index c2d961a..0e0d001 100644 --- a/skills/curiosity-engine/docs/architecture.md +++ b/skills/curiosity-engine/docs/architecture.md @@ -95,6 +95,31 @@ Known failure modes, in rough order of when you'll hit them: - **~500-page wiki ceiling.** Beyond that, plan latency grows. The incremental score cache helps; the tiered-vault design (bounded wiki + unbounded indexed vault) unlocks more; cluster-scoped CURATE (see below) keeps individual waves coherent past the threshold. - **Single user.** No protocol for multiple humans editing the same wiki live. Two asynchronous export paths cover different audiences: **OKF** (`okf_export.py build wiki --output-dir `) projects the wiki to a vendor-neutral [Open Knowledge Format](https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md) bundle for cross-tool / cross-org exchange (read-only; CE structure rides in `x_ce_*` keys — see [`okf-interop.md`](okf-interop.md)); **curiosity-merge** subgraph-export/merge shares whole or sub-wikis with other CE workspaces. Neither is concurrent multi-user editing. +## Shell embed + hosted mode (Phase 2a / 2b) + +Umbrella charter: Switchbay/okbay **same-origin reverse-proxy** CE (typically +`127.0.0.1:8766`) under `/embed/ce/` — **no iframes**. CE readiness: + +- Env `CE_PUBLIC_BASE` (e.g. `/embed/ce`) — see `scripts/public_base.py` and + `viewer_server.py` +- Hosted hook: `X-CE-Host` / `?host=switchbay|okbay` and `GET /api/hosted` +- ADR: [`ADR-001-proxy-embed-and-hosted.md`](ADR-001-proxy-embed-and-hosted.md) +- **Phase 2b:** `GET /api/tree` filebrowser (vault/ + wiki/) + wiki-view Files mode; + knowledge-atlas `playReplayTimeline` animation hook +- **Phase 2b+:** wiki partition / `POST /api/split` + minimal split UI +- **Phase 2b++:** `/api/fs/*` mutate (vault/wiki sandbox) + `GET /api/file-routes` +- **Phase 2b+++:** SVG curation replay UI (`GET /api/curation/history`, play/pause/scrub) +- **Phase 2b++++:** pack list / enable / local install + sandboxed action dispatch + (`/api/packs/*`); agent/LLM skill execution still Switchbay +- **Phase 2b+++++:** workspace registry persistence (`/api/workspaces`) + optional + curiosity-merge `subgraph_export` hook (`POST /api/cm-export`, dry-run) +- **Phase 2b++++++ / +++++++:** Classic + Atlas canvas rubber-band split targeting; + Atlas replay camera bind (`fitToContent`); heal / tab-open chrome still Switchbay + Details: [`MIGRATION-INTAKE.md`](MIGRATION-INTAKE.md) + +Bare `viewer.sh` loopback default remains port **8090**; bind host is always +`127.0.0.1`. + ## Cluster scoping at scale Past `cluster_scope_threshold` non-source pages (default 500), `epoch_summary.py` emits a `wave_scope` field — the worst-scoring page plus every page within two wikilink hops, in either direction. Phase 1 of the CURATE loop honours the scope for **repair mode**: editorial and frontier target selection are restricted to pages inside the scope. Create and wire modes stay global (new pages aren't in the graph yet; inbound-link starvation legitimately crosses clusters). diff --git a/skills/curiosity-engine/docs/viewers.md b/skills/curiosity-engine/docs/viewers.md index b1c59b6..6aad35f 100644 --- a/skills/curiosity-engine/docs/viewers.md +++ b/skills/curiosity-engine/docs/viewers.md @@ -59,8 +59,37 @@ Each workspace's bundle goes into rebuilds it after every inline edit, so refresh and the change is visible. + +## Embed / reverse-proxy (Switchbay, okbay) + +For same-origin shell embeds, run the viewer server with +`CE_PUBLIC_BASE=/embed/ce` (and typically port **8766**). The server strips +that prefix and injects `window.CE_PUBLIC_BASE` / `ceApi()` into HTML. +Optional hosted signal: `X-CE-Host: switchbay|okbay` or `?host=`. See +[`ADR-001-proxy-embed-and-hosted.md`](ADR-001-proxy-embed-and-hosted.md). +Bare local `viewer.sh` is unchanged (default port 8090, `127.0.0.1`). + +**Filebrowser (Phase 2b).** Sidebar **Pages | Files** toggle. Files mode +calls `GET /api/tree` (vault/ + wiki/ only), supports substring / `/re/` / +`*.ext` filter, graph-search highlight, and a minimal context menu. +Honors `ceApi()` / `CE_PUBLIC_BASE`. Open `?filebrowser=1` to land in Files +mode. FS mutate via `/api/fs/*` and pack APIs (`GET /api/file-routes`, `/api/packs/*` list/dispatch/enable/local-install) are CE-owned; agent/LLM skill *execution*, reveal-in-OS, and drop-ingest remain Switchbay-side until later parity. + +**Curation replay (Phase 2b+++ polish).** Graph-controls **replay ↻** opens an +SVG overlay driven by `GET /api/curation/history` (HistoryDoc; synthetic from +`data.json` with `created`/type-tier ordering, or `.workbench/curation-history.json` +when a shell pre-warmed it). Play / pause / scrub / step; `?replay=1` opens +and autoplays a short intro (settle + fade). Enter fade, radius inflate, +easeAutoFit after major jumps. Atlas canvas camera bind landed (`bindViewer` + `fitToContent`); git-log rebuild remain +deferred — see [`MIGRATION-INTAKE.md`](MIGRATION-INTAKE.md). + ## Obsidian (alternative — same underlying markdown) +To **import** an existing Obsidian vault into CE (no Work/symlink dance), +use `POST /api/import/obsidian` or `python3 scripts/obsidian_import.py `. +Mapping: Obsidian vault root → `wiki/` (wikilinks preserved). +See `MIGRATION-INTAKE.md` Phase 2b+++++++. + `wiki/` is plain markdown with `[[wikilinks]]`. Open Obsidian → **Open folder as vault** → pick `/wiki`. Backlinks and Obsidian's own graph view light up immediately, no plugins. diff --git a/skills/curiosity-engine/scripts/cm_export.py b/skills/curiosity-engine/scripts/cm_export.py new file mode 100644 index 0000000..66b495e --- /dev/null +++ b/skills/curiosity-engine/scripts/cm_export.py @@ -0,0 +1,285 @@ +"""cm_export.py — invoke curiosity-merge ``subgraph_export`` for a page set. + +CE's native ``wiki_partition`` remains the default split path (no CM +dependency). This module is the Switchbay-parity *export* hook: when a +shell (or the CE viewer) wants CM license modes / preflight / manifest +shape, it calls ``POST /api/cm-export``. + +Discovery order for the CM skill root: + 1. ``$CE_CM_ROOT`` / ``$CURIOSITY_MERGE_ROOT`` / ``$SWITCHBAY_CM_ROOT`` + 2. Sibling checkout ``/../curiosity-merge`` (dev) + 3. ``~/.claude/skills/curiosity-merge`` + 4. ``~/.agents/skills/curiosity-merge`` (npx skills layout) + +Sandbox: + * ``--to`` must resolve **outside** the source workspace + * ``--to`` must resolve under ``$HOME`` (or ``$CE_WORKSPACE_HOME``) + * page refs validated via ``wiki_partition.validate_refs`` + +``dry_run=True`` validates + returns the planned argv **without** spawning +CM (and without requiring CM to be installed — discovery still runs; a +missing CM skill yields a clear error so callers can surface install +guidance). +""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from typing import Any + +import wiki_partition +import workspace_registry + + +class CmExportError(Exception): + """User-visible CM export failure.""" + + +_EXPORT_TIMEOUT = 900 + +_INCLUDE_VAULT = frozenset({"none", "owned", "all"}) + + +def discover_cm_root() -> Path: + candidates: list[Path] = [] + for key in ("CE_CM_ROOT", "CURIOSITY_MERGE_ROOT", "SWITCHBAY_CM_ROOT"): + raw = (os.environ.get(key) or "").strip() + if raw: + candidates.append(Path(raw).expanduser()) + # Dev sibling: .../repos/curiosity-engine → .../repos/curiosity-merge + here = Path(__file__).resolve() + # skills/curiosity-engine/scripts → repo root is parents[3] + try: + repo_root = here.parents[3] + candidates.append(repo_root.parent / "curiosity-merge") + except IndexError: + pass + candidates.append(Path.home() / ".claude" / "skills" / "curiosity-merge") + candidates.append(Path.home() / ".agents" / "skills" / "curiosity-merge") + for c in candidates: + script = c / "scripts" / "subgraph_export.py" + if script.is_file(): + return c.resolve() + raise CmExportError( + "curiosity-merge skill not found — set CE_CM_ROOT to the skill " + "root (must contain scripts/subgraph_export.py), or install via " + "`npx skills add -g -y benjsmith/curiosity-merge`" + ) + + +def subgraph_export_script(cm_root: Path | None = None) -> Path: + root = cm_root or discover_cm_root() + script = root / "scripts" / "subgraph_export.py" + if not script.is_file(): + raise CmExportError(f"subgraph_export.py missing under {root}") + return script + + +def _ce_scripts_dir() -> Path: + return Path(__file__).resolve().parent + + +def _safe_export_target(workspace: Path, target: Path) -> Path: + """Refuse exports inside the source workspace or outside the sandbox.""" + ws = workspace.resolve() + dest = target.expanduser().resolve() + if ".." in Path(str(target)).parts: + raise CmExportError(f"refusing path with .. segments: {target}") + try: + dest.relative_to(ws) + raise CmExportError(f"refusing to write export inside workspace: {dest}") + except ValueError: + pass + if dest == ws: + raise CmExportError("target cannot be the source workspace") + if not workspace_registry.is_within_sandbox(dest): + raise CmExportError( + f"export target must live inside {workspace_registry._home_root()}; " + f"refusing {dest}" + ) + return dest + + +def plan_export( + workspace: Path, + target: Path, + pages: list[str], + *, + include_vault: str = "all", + include_non_native: bool = True, + no_preflight: bool = True, + force: bool = True, + label: str | None = None, + cm_root: Path | None = None, +) -> dict[str, Any]: + """Validate inputs and build the CM argv. Does not spawn. + + Returns a plan dict with ``argv``, ``cwd``, ``env`` extras, ``pages``, + ``target``, ``cm_root``, ``script``. + """ + workspace = workspace.resolve() + if not (workspace / "wiki").is_dir(): + raise CmExportError(f"workspace has no wiki/: {workspace}") + refs = list(dict.fromkeys(str(r).strip() for r in pages if str(r).strip())) + if not refs: + raise CmExportError("nothing selected") + missing = wiki_partition.validate_refs(workspace, refs) + if missing: + raise CmExportError("not wiki pages: " + ", ".join(missing[:8])) + vault_mode = (include_vault or "all").strip().lower() + if vault_mode not in _INCLUDE_VAULT: + raise CmExportError( + f"include_vault must be one of {sorted(_INCLUDE_VAULT)}; got {vault_mode!r}" + ) + dest = _safe_export_target(workspace, target) + root = cm_root or discover_cm_root() + script = subgraph_export_script(root) + # pages-file path is filled in at execute time (temp file); plan uses a + # placeholder token so dry-run JSON stays stable. + argv = [ + sys.executable, + str(script), + "--pages-file", + "{pages_file}", + "--to", + str(dest), + "--workspace", + str(workspace), + "--include-vault", + vault_mode, + ] + if include_non_native: + argv.append("--include-non-native") + if no_preflight: + argv.append("--no-preflight") + if force: + argv.append("--force") + if label: + argv.extend(["--label", str(label)]) + return { + "ok": True, + "dry_run": True, + "pages": refs, + "target": str(dest), + "cm_root": str(root), + "script": str(script), + "include_vault": vault_mode, + "include_non_native": include_non_native, + "argv": argv, + "cwd": str(workspace), + "ce_scripts": str(_ce_scripts_dir()), + } + + +def run_export( + workspace: Path, + target: Path, + pages: list[str], + *, + include_vault: str = "all", + include_non_native: bool = True, + no_preflight: bool = True, + force: bool = True, + label: str | None = None, + cm_root: Path | None = None, + dry_run: bool = False, + register: bool = True, + timeout: int = _EXPORT_TIMEOUT, +) -> dict[str, Any]: + """Plan (and optionally execute) a CM subgraph export. + + When ``dry_run`` is True, returns the plan without spawning. When + ``register`` is True and the export succeeds, the target is added to + the CE workspace registry. + """ + plan = plan_export( + workspace, + target, + pages, + include_vault=include_vault, + include_non_native=include_non_native, + no_preflight=no_preflight, + force=force, + label=label, + cm_root=cm_root, + ) + if dry_run: + return plan + + dest = Path(plan["target"]) + if dest.exists() and any(dest.iterdir()) and not force: + raise CmExportError(f"target already exists: {dest}") + + pages_file: str | None = None + try: + with tempfile.NamedTemporaryFile( + "w", suffix=".json", delete=False, encoding="utf-8" + ) as f: + json.dump(plan["pages"], f) + pages_file = f.name + argv = [ + (pages_file if tok == "{pages_file}" else tok) for tok in plan["argv"] + ] + env = { + k: v + for k, v in os.environ.items() + if k not in {"VIRTUAL_ENV", "UV_PROJECT_ENVIRONMENT", "PYTHONPATH"} + } + env["CURIOSITY_ENGINE_SCRIPTS_DIR"] = plan["ce_scripts"] + try: + proc = subprocess.run( + argv, + cwd=plan["cwd"], + env=env, + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + except subprocess.TimeoutExpired as e: + raise CmExportError(f"CM subgraph_export timed out after {timeout}s") from e + out = (proc.stdout or "") + (proc.stderr or "") + if proc.returncode != 0: + tail = "\n".join(out.strip().splitlines()[-12:]) + raise CmExportError( + f"subgraph export failed (rc={proc.returncode}):\n{tail}" + ) + result = { + "ok": True, + "dry_run": False, + "pages": plan["pages"], + "target": plan["target"], + "cm_root": plan["cm_root"], + "include_vault": plan["include_vault"], + "exported": len(plan["pages"]), + "log_tail": "\n".join(out.strip().splitlines()[-20:]), + "stamp": time.strftime("%Y%m%d-%H%M%S"), + } + if register: + try: + workspace_registry.register( + dest, + set_active=False, + split={ + "name": dest.name, + "source": str(workspace.resolve()), + "target": str(dest), + "stamp": result["stamp"], + }, + ) + result["registered"] = True + except workspace_registry.RegistryError as e: + result["registered"] = False + result["register_error"] = str(e) + return result + finally: + if pages_file: + try: + os.unlink(pages_file) + except OSError: + pass diff --git a/skills/curiosity-engine/scripts/curation_history.py b/skills/curiosity-engine/scripts/curation_history.py new file mode 100644 index 0000000..05191b5 --- /dev/null +++ b/skills/curiosity-engine/scripts/curation_history.py @@ -0,0 +1,240 @@ +"""Curation-history timeline for the wiki-view replay UI (Phase 2b+++). + +Produces a HistoryDoc compatible with knowledge-atlas ``playReplayTimeline`` +and Switchbay's ``/api/curation/history`` shape. + +CE spike source of truth: the viewer ``data.json`` graph (nodes + edges). +Git first-seen chronology (Switchbay ``curation_history.py``) remains a +parity gap — optional cache hit if ``.workbench/curation-history.json`` +already exists from a shell pre-warm. +""" +from __future__ import annotations + +import json +import time +from pathlib import Path +from typing import Any + +SCHEMA_VERSION = 1 +DEFAULT_DURATION_S = 12.0 +MAX_EVENTS = 2500 +CACHE_REL = Path(".workbench") / "curation-history.json" + + + +_TYPE_TIER = { + "source": 0, + "sources": 0, + "project": 1, + "entity": 2, + "concept": 3, + "evidence": 4, + "fact": 4, + "figure": 5, + "table": 5, + "note": 6, + "todo": 6, + "todo-list": 6, + "analysis": 7, +} + + +def _type_tier(t: str) -> int: + return _TYPE_TIER.get((t or "unclassified").lower(), 8) + + +def _node_sort_key(n: dict[str, Any]) -> tuple: + """Prefer ISO created asc; else type tier → degree desc → id.""" + created = str(n.get("created") or "").strip() + if created: + return (0, created, -(n.get("degree") or 0), n["id"]) + return (1, _type_tier(str(n.get("type") or "")), -(n.get("degree") or 0), n["id"]) + +def _edge_id(ref: Any) -> str: + if isinstance(ref, str): + return ref + if isinstance(ref, dict): + v = ref.get("id") + return str(v) if isinstance(v, str) else "" + return "" + + +def build_from_data( + data: dict[str, Any] | None, + *, + duration_s: float = DEFAULT_DURATION_S, + max_events: int = MAX_EVENTS, + source: str = "ce-data-json", +) -> dict[str, Any]: + """Synthesize HistoryDoc from CE data.json (deterministic).""" + nodes_raw = (data or {}).get("nodes") or [] + edges_raw = (data or {}).get("edges") or [] + nodes: list[dict[str, Any]] = [] + for n in nodes_raw: + if not isinstance(n, dict): + continue + nid = str(n.get("id") or "").strip() + if not nid: + continue + created = n.get("created") + if not created and isinstance(n.get("properties"), dict): + created = n["properties"].get("created") + nodes.append( + { + "id": nid, + "title": str(n.get("title") or nid), + "type": str(n.get("type") or "unclassified"), + "degree": int(n.get("degree") or 0), + "created": str(created).strip() if created else "", + } + ) + if not nodes: + return { + "duration": duration_s, + "events": [], + "source": "empty", + "node_count": 0, + "degree": {}, + "version": SCHEMA_VERSION, + } + + + pages = (data or {}).get("pages") or {} + if isinstance(pages, dict): + for n in nodes: + if n.get("created"): + continue + page = pages.get(n["id"]) + if not isinstance(page, dict): + continue + props = page.get("properties") + if not isinstance(props, dict): + props = {} + created = props.get("created") or page.get("created") + if created: + n["created"] = str(created).strip() + nodes.sort(key=_node_sort_key) + by_id = {n["id"]: n for n in nodes} + index_of = {n["id"]: i for i, n in enumerate(nodes)} + n_count = len(nodes) + + def t_of(i: int) -> float: + if n_count <= 1: + return 0.0 + return round((i / (n_count - 1)) * duration_s, 3) + + events: list[dict[str, Any]] = [] + for i, n in enumerate(nodes): + events.append( + { + "t": t_of(i), + "op": "node", + "id": n["id"], + "title": n["title"], + "type": n["type"], + } + ) + + seen_pairs: set[tuple[str, str]] = set() + edge_events: list[dict[str, Any]] = [] + for e in edges_raw: + if not isinstance(e, dict): + continue + s = _edge_id(e.get("source")) + t = _edge_id(e.get("target")) + if not s or not t or s == t or s not in by_id or t not in by_id: + continue + pair = (s, t) if s < t else (t, s) + if pair in seen_pairs: + continue + seen_pairs.add(pair) + later = max(index_of[s], index_of[t]) + edge_events.append( + { + "t": t_of(later), + "op": "edge", + "source": s, + "target": t, + } + ) + + events.extend(edge_events) + events.sort(key=lambda e: (e["t"], 0 if e["op"] == "node" else 1)) + + if len(events) > max_events: + node_events = [e for e in events if e["op"] == "node"] + edge_ev = [e for e in events if e["op"] == "edge"] + keep_n = max(0, max_events - len(node_events)) + if keep_n < len(edge_ev): + step = len(edge_ev) / max(1, keep_n) + edge_ev = [edge_ev[int(i * step)] for i in range(keep_n)] + events = sorted( + node_events + edge_ev, + key=lambda e: (e["t"], 0 if e["op"] == "node" else 1), + ) + + degree: dict[str, int] = {n["id"]: 0 for n in nodes} + for e in events: + if e["op"] != "edge": + continue + degree[e["source"]] = degree.get(e["source"], 0) + 1 + degree[e["target"]] = degree.get(e["target"], 0) + 1 + + return { + "duration": duration_s, + "events": events, + "source": source, + "generated_at": time.time(), + "node_count": len(nodes), + "degree": degree, + "version": SCHEMA_VERSION, + } + + +def _read_json(path: Path) -> dict[str, Any] | None: + try: + raw = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + return raw if isinstance(raw, dict) else None + + +def _cache_usable(doc: dict[str, Any]) -> bool: + events = doc.get("events") + return isinstance(events, list) and "duration" in doc + + +def read_or_build( + workspace: Path, + bundle_dir: Path | None = None, + *, + duration_s: float = DEFAULT_DURATION_S, +) -> dict[str, Any]: + """Prefer shell cache; else build from bundle data.json.""" + ws = Path(workspace) + cache = ws / CACHE_REL + if cache.is_file(): + cached = _read_json(cache) + if cached and _cache_usable(cached): + out = dict(cached) + out.setdefault("source", "workbench-cache") + out["version"] = SCHEMA_VERSION + return out + + data: dict[str, Any] | None = None + if bundle_dir is not None: + data = _read_json(Path(bundle_dir) / "data.json") + if data is None: + # Fallback: wiki_render cache under ~/.cache is caller-provided via bundle. + data = {} + return build_from_data(data, duration_s=duration_s) + + +def history_payload( + workspace: Path, + bundle_dir: Path | None = None, +) -> dict[str, Any]: + """API envelope: HistoryDoc fields at top level (Switchbay-compatible).""" + doc = read_or_build(workspace, bundle_dir) + # Keep Switchbay clients happy: they expect the doc itself, not {ok, …}. + return doc diff --git a/skills/curiosity-engine/scripts/filebrowser_fs.py b/skills/curiosity-engine/scripts/filebrowser_fs.py new file mode 100644 index 0000000..2f3225f --- /dev/null +++ b/skills/curiosity-engine/scripts/filebrowser_fs.py @@ -0,0 +1,285 @@ +"""CE filebrowser FS mutations — vault/ + wiki/ sandbox (Phase 2b++). + +Switchbay Step E ships delete / duplicate / reveal / stat over the whole +workspace. CE intake prefers vault/wiki path semantics (architecture.md +three-object model) and adds create / rename / move so the embed surface +can mutate without shell round-trips. + +All relative paths are re-resolved under the workspace and refused if they +escape it, leave vault|wiki, or land in a protected / hidden component. +Trash lands in workspace `.workbench/trash/` (Switchbay fallback shape); +OS trash helpers are optional best-effort. +""" +from __future__ import annotations + +import os +import re +import shutil +import subprocess +import sys +import time +from pathlib import Path + +from filebrowser_tree import SKIP_DIRS, TREE_ROOTS + +ALLOWED_ROOTS = frozenset(TREE_ROOTS) +PROTECTED_TOP = frozenset({"node_modules", "venv", "__pycache__", "dist", "build"}) +_SAFE_SEGMENT = re.compile(r"^[A-Za-z0-9._+-]+$") + + +class FileOpError(Exception): + """User-visible FS failure (bad path, missing file, escape, etc.).""" + + +def _norm_rel(rel: str) -> str: + raw = (rel or "").strip().replace("\\", "/") + while raw.startswith("./"): + raw = raw[2:] + return raw.strip("/") + + +def resolve(workspace: Path, rel: str, *, must_exist: bool = False) -> Path: + """Resolve a vault/ or wiki/ relative path; refuse escapes.""" + raw = _norm_rel(rel) + if not raw or raw.startswith("/") or "\x00" in raw: + raise FileOpError("invalid path") + parts = Path(raw).parts + if ".." in parts: + raise FileOpError("path may not contain ..") + if not parts or parts[0] not in ALLOWED_ROOTS: + raise FileOpError("path must start with vault/ or wiki/") + for part in parts: + if part.startswith("."): + raise FileOpError(f"refusing to touch hidden path component {part}") + if part in SKIP_DIRS or part in PROTECTED_TOP: + raise FileOpError(f"refusing to touch {part}/") + ws = Path(workspace).resolve() + try: + target = (ws / raw).resolve() + except (OSError, ValueError) as e: + raise FileOpError(f"resolve failed: {e}") from e + try: + relative = target.relative_to(ws) + except ValueError as e: + raise FileOpError("path escapes workspace") from e + if not relative.parts or relative.parts[0] not in ALLOWED_ROOTS: + raise FileOpError("path escapes vault/wiki sandbox") + # Belt-and-brace: every component of the resolved relative path. + for part in relative.parts: + if part.startswith(".") or part in SKIP_DIRS or part in PROTECTED_TOP: + raise FileOpError(f"refusing to touch {part}/") + if must_exist and not target.exists(): + raise FileOpError("not found") + return target + + +def _rel_of(workspace: Path, target: Path) -> str: + return target.resolve().relative_to(Path(workspace).resolve()).as_posix() + + +def stat(workspace: Path, rel: str) -> dict: + target = resolve(workspace, rel, must_exist=True) + s = target.stat() + return { + "path": _norm_rel(rel), + "size": s.st_size, + "mtime": s.st_mtime, + "kind": "dir" if target.is_dir() else "file", + } + + +def _unique_sibling(parent: Path, stem: str, suffix: str) -> Path: + candidate = parent / f"{stem}{suffix}" + n = 2 + while candidate.exists(): + candidate = parent / f"{stem} {n}{suffix}" + n += 1 + return candidate + + +def create( + workspace: Path, + rel: str, + *, + kind: str = "file", + content: str = "", +) -> str: + """Create a file (parents must exist) or empty directory under vault|wiki.""" + kind = (kind or "file").strip().lower() + if kind not in ("file", "dir", "directory"): + raise FileOpError("kind must be file or dir") + raw = _norm_rel(rel) + if not raw: + raise FileOpError("invalid path") + parts = Path(raw).parts + if len(parts) < 2: + raise FileOpError("path must be under vault/ or wiki/ (not the root itself)") + leaf = parts[-1] + if not leaf or leaf.startswith(".") or not _SAFE_SEGMENT.match(leaf): + raise FileOpError("invalid name") + # Validate full destination path (may not exist yet). + resolve(workspace, raw) + parent_rel = Path(*parts[:-1]).as_posix() + parent = resolve(workspace, parent_rel, must_exist=True) + if not parent.is_dir(): + raise FileOpError("parent must be a directory") + target = parent / leaf + if target.exists(): + raise FileOpError("already exists") + if kind in ("dir", "directory"): + target.mkdir(parents=False, exist_ok=False) + else: + if content and not content.endswith("\n"): + content += "\n" + target.write_text(content, encoding="utf-8") + return _rel_of(workspace, target) + + +def mkdir(workspace: Path, rel: str) -> str: + return create(workspace, rel, kind="dir") + + +def rename(workspace: Path, src: str, dst: str) -> str: + """Rename or relocate within vault|wiki. ``dst`` is a full relative path.""" + source = resolve(workspace, src, must_exist=True) + dest_raw = _norm_rel(dst) + if not dest_raw: + raise FileOpError("invalid destination") + # Destination must not exist; parent must exist. + dest_parent_rel = Path(dest_raw).parent.as_posix() + if dest_parent_rel in ("", "."): + raise FileOpError("destination must be under vault/ or wiki/") + parent = resolve(workspace, dest_parent_rel, must_exist=True) + if not parent.is_dir(): + raise FileOpError("destination parent must be a directory") + leaf = Path(dest_raw).name + if not leaf or leaf.startswith(".") or not _SAFE_SEGMENT.match(leaf): + raise FileOpError("invalid destination name") + dest = parent / leaf + # Validate resolved dest stays in sandbox (even if not yet existing). + resolve(workspace, dest_raw) + if dest.exists(): + raise FileOpError("destination already exists") + if source.resolve() == dest.resolve(): + return _rel_of(workspace, source) + # Refuse moving a directory into itself. + if source.is_dir(): + try: + dest.resolve().relative_to(source.resolve()) + raise FileOpError("cannot move a directory into itself") + except ValueError: + pass + shutil.move(str(source), str(dest)) + return _rel_of(workspace, dest) + + +def move(workspace: Path, src: str, dst: str) -> str: + """Move ``src`` to ``dst``. + + If ``dst`` names an existing directory, place the source basename inside it. + Otherwise treat ``dst`` as the full destination path (same as rename). + """ + source = resolve(workspace, src, must_exist=True) + dest_raw = _norm_rel(dst) + if not dest_raw: + raise FileOpError("invalid destination") + # Probe whether dst is an existing dir without requiring existence for files. + try: + dest_probe = resolve(workspace, dest_raw, must_exist=False) + except FileOpError: + raise + if dest_probe.is_dir(): + dest_raw = f"{dest_raw.rstrip('/')}/{source.name}" + return rename(workspace, src, dest_raw) + + +def delete(workspace: Path, rel: str) -> str: + """Delete-to-trash. Files always; empty directories only. + + Prefer OS trash when available; fall back to ``.workbench/trash/``. + Returns a human-readable destination for UI toasts. + """ + target = resolve(workspace, rel, must_exist=True) + if target.is_dir(): + try: + next(target.iterdir()) + raise FileOpError("delete: directory not empty") + except StopIteration: + pass + cmd: list[str] | None = None + if sys.platform == "darwin" and os.path.exists("/usr/bin/trash"): + cmd = ["/usr/bin/trash", str(target)] + elif sys.platform.startswith("linux") and shutil.which("gio"): + cmd = ["gio", "trash", str(target)] + if cmd is not None: + try: + res = subprocess.run( + cmd, capture_output=True, timeout=10, check=False + ) + if res.returncode == 0: + return "the system Trash" + except (OSError, subprocess.TimeoutExpired): + pass + ws = Path(workspace).resolve() + bin_dir = ws / ".workbench" / "trash" + bin_dir.mkdir(parents=True, exist_ok=True) + stamp = time.strftime("%Y%m%d-%H%M%S") + dest = bin_dir / f"{stamp}-{target.name}" + n = 2 + while dest.exists(): + dest = bin_dir / f"{stamp}-{n}-{target.name}" + n += 1 + shutil.move(str(target), str(dest)) + return f".workbench/trash/{dest.name}" + + +def duplicate(workspace: Path, rel: str) -> str: + """Copy a file to a sibling with a unique `` copy`` suffix.""" + src = resolve(workspace, rel, must_exist=True) + if not src.is_file(): + raise FileOpError("source must be a file") + candidate = _unique_sibling(src.parent, f"{src.stem} copy", src.suffix) + # Validate candidate stays sandboxed before writing. + resolve(workspace, _rel_of(workspace, candidate)) + shutil.copy2(src, candidate) + return _rel_of(workspace, candidate) + + +def reveal(workspace: Path, rel: str) -> dict: + """Open the path in the OS file manager (Finder / Explorer / xdg). + + Mirrors Switchbay ``fileops.reveal``: resolve under the vault/wiki + sandbox, then platform-open the parent directory (Linux) or select + the file (macOS / Windows). Returns a small status dict for the API. + """ + target = resolve(workspace, rel, must_exist=True) + if sys.platform == "darwin": + argv = ["open", "-R", str(target)] + elif sys.platform == "win32": + argv = ["explorer", "/select,", str(target)] + else: + # Linux: open the containing directory (xdg-open has no -R). + argv = ["xdg-open", str(target.parent if target.is_file() else target)] + try: + subprocess.run(argv, capture_output=True, timeout=15, check=False) + except (OSError, subprocess.TimeoutExpired) as e: + raise FileOpError(f"reveal failed: {e}") from e + return {"path": _norm_rel(rel), "revealed": True} + + +def open_external(workspace: Path, rel: str) -> dict: + """Hand the file off to the OS default app (Preview / xdg-open / start).""" + target = resolve(workspace, rel, must_exist=True) + if not target.is_file(): + raise FileOpError("open_external: path must be a file") + if sys.platform == "darwin": + argv = ["open", str(target)] + elif sys.platform == "win32": + argv = ["cmd", "/c", "start", "", str(target)] + else: + argv = ["xdg-open", str(target)] + try: + subprocess.run(argv, capture_output=True, timeout=15, check=False) + except (OSError, subprocess.TimeoutExpired) as e: + raise FileOpError(f"open_external failed: {e}") from e + return {"path": _norm_rel(rel), "opened": True} diff --git a/skills/curiosity-engine/scripts/filebrowser_ingest.py b/skills/curiosity-engine/scripts/filebrowser_ingest.py new file mode 100644 index 0000000..bbb22c0 --- /dev/null +++ b/skills/curiosity-engine/scripts/filebrowser_ingest.py @@ -0,0 +1,165 @@ +"""CE drop-ingest — stage uploads into vault/ + queue ingest runs. + +Mirrors Switchbay ``/api/ingest/from-upload`` semantics at the API layer +so the filebrowser (and Switchbay drag UI via ``/embed/ce/``) can drop +files into the vault without inventing a parallel pipeline. + +Staging lands under ``vault/raw/`` (CE drop-folder convention; also what +``local_ingest.py`` drains). Agent/LLM classification stays shell-side — +CE writes ``.workbench/ingest-runs/.json`` with status ``queued`` +(Switchbay pack-run drain shape) and returns ``{run_id, vault_path, …}``. + +Allowlist: ``DEFAULT_EXTS`` from local_ingest (same set the CLI accepts). +Refuse hidden names, path components, null bytes, and payloads over 50 MB. +""" +from __future__ import annotations + +import hashlib +import json +import re +import uuid +from pathlib import Path +from typing import Any + +# Keep in sync with local_ingest.DEFAULT_EXTS / DEFAULT_MAX_RAW_BYTES. +ALLOWED_EXTS = frozenset({ + ".md", ".txt", ".rst", ".html", ".htm", ".json", ".jsonl", ".yaml", + ".yml", ".org", ".pdf", ".csv", ".xlsx", ".pptx", +}) +MAX_BYTES = 50 * 1024 * 1024 +_RUNS_DIR = "ingest-runs" +_SAFE_NAME = re.compile(r"^[A-Za-z0-9._+-]+$") + + +class IngestError(Exception): + """User-visible ingest failure (bad name, ext, size, escape).""" + + def __init__(self, message: str, *, status: int = 400): + super().__init__(message) + self.status = status + + +def safe_filename(name: str) -> str: + """Basename-only sanitise; refuse hidden / empty / null.""" + raw = (name or "").strip().replace("\\", "/") + if "\x00" in raw: + raise IngestError("invalid filename") + base = Path(raw).name + base = re.sub(r"[^A-Za-z0-9._+-]+", "_", base) + if not base or base.startswith(".") or not _SAFE_NAME.match(base): + raise IngestError("invalid filename") + if ".." in base: + raise IngestError("invalid filename") + return base + + +def assert_allowed_ext(filename: str) -> str: + """Return lowercased suffix including leading ``.``; raise if not allowlisted.""" + suffix = Path(filename).suffix.lower() + if not suffix or suffix not in ALLOWED_EXTS: + raise IngestError( + f"extension not allowlisted: {suffix or '(none)'} " + f"(allowed: {', '.join(sorted(ALLOWED_EXTS))})", + status=400, + ) + return suffix + + +def _unique_raw_path(raw_dir: Path, filename: str) -> Path: + candidate = raw_dir / filename + if not candidate.exists(): + return candidate + stem = Path(filename).stem + suffix = Path(filename).suffix + n = 2 + while True: + cand = raw_dir / f"{stem}-{n}{suffix}" + if not cand.exists(): + return cand + n += 1 + + +def stage_bytes(workspace: Path, filename: str, data: bytes) -> dict[str, Any]: + """Validate + write ``data`` under ``vault/raw/``; queue ingest run. + + Returns Switchbay-shaped ``{ok, run_id, vault_path, filename, size}``. + """ + if data is None: + raise IngestError("empty body") + if len(data) == 0: + raise IngestError("empty file") + if len(data) > MAX_BYTES: + raise IngestError("file too large (>50 MB)", status=413) + + safe = safe_filename(filename) + assert_allowed_ext(safe) + + ws = Path(workspace).resolve() + raw_dir = ws / "vault" / "raw" + raw_dir.mkdir(parents=True, exist_ok=True) + # Belt-and-brace: dest must stay under vault/raw. + dest = _unique_raw_path(raw_dir, safe) + try: + dest_res = dest.resolve() + dest_res.relative_to((ws / "vault" / "raw").resolve()) + except ValueError as e: + raise IngestError("path escapes vault/raw") from e + if any(p.startswith(".") for p in dest.relative_to(ws).parts): + raise IngestError("refusing hidden path") + + dest.write_bytes(data) + rel = dest.relative_to(ws).as_posix() + digest = hashlib.sha1(data).hexdigest()[:12] + + run_id = f"run-{uuid.uuid4().hex[:8]}" + run_rec: dict[str, Any] = { + "run_id": run_id, + "status": "queued", + "kind": "ingest-upload", + "vault_path": rel, + "filename": dest.name, + "size": len(data), + "sha1_12": digest, + "mode": "staged", + "note": ( + "CE staged the upload under vault/raw/; " + "agent/LLM ingest classification remains shell-side " + "(or run local_ingest.py on the drop folder)." + ), + } + runs_dir = ws / ".workbench" / _RUNS_DIR + runs_dir.mkdir(parents=True, exist_ok=True) + (runs_dir / f"{run_id}.json").write_text( + json.dumps(run_rec, indent=2) + "\n", encoding="utf-8" + ) + + return { + "ok": True, + "run_id": run_id, + "vault_path": rel, + "filename": dest.name, + "size": len(data), + "status": "queued", + } + + +def stage_path(workspace: Path, src: str | Path) -> dict[str, Any]: + """Stage an already-on-disk file (absolute path) into vault/raw/. + + Used by ``POST /api/ingest/from-path``. Source must be a real file; + contents are copied (not moved) so callers keep the original. + """ + src_path = Path(src).expanduser() + if not src_path.is_absolute(): + raise IngestError("source must be an absolute path") + try: + src_path = src_path.resolve() + except (OSError, ValueError) as e: + raise IngestError(f"resolve failed: {e}") from e + if not src_path.is_file(): + raise IngestError("source must be a file", status=400) + try: + data = src_path.read_bytes() + except OSError as e: + raise IngestError(f"read failed: {e}") from e + return stage_bytes(workspace, src_path.name, data) diff --git a/skills/curiosity-engine/scripts/filebrowser_match.py b/skills/curiosity-engine/scripts/filebrowser_match.py new file mode 100644 index 0000000..8be9906 --- /dev/null +++ b/skills/curiosity-engine/scripts/filebrowser_match.py @@ -0,0 +1,61 @@ +"""Filebrowser path matchers — parity with Switchbay FileBrowser buildMatcher. + +Empty query → match all. Supports: + - /pattern/flags (invalid regex → None, so UI can show feedback) + - *.ext filetype-only (case-insensitive) + - plain substring (case-insensitive against the full relative path) +""" +from __future__ import annotations + +import re +from collections.abc import Callable + +Matcher = Callable[[str], bool] + + +def file_ext(path: str) -> str: + """Match Switchbay FileBrowser.fileExt (last dot after last slash).""" + slash = path.rfind("/") + dot = path.rfind(".") + if dot <= slash: + return "" + return path[dot + 1 :].lower() + + +def build_matcher(query: str) -> Matcher | None: + """Return a matcher, or None when the regex form is invalid.""" + q = (query or "").strip() + if not q: + return lambda _p: True + + m = re.fullmatch(r"/(.+)/([gimsuy]*)", q) + if m: + pattern, flags_s = m.group(1), m.group(2) or "i" + flag = 0 + if "i" in flags_s: + flag |= re.IGNORECASE + if "m" in flags_s: + flag |= re.MULTILINE + if "s" in flags_s: + flag |= re.DOTALL + try: + rx = re.compile(pattern, flag) + except re.error: + return None + return lambda p, _rx=rx: _rx.search(p) is not None + + glob = re.fullmatch(r"\*\.([A-Za-z0-9_]+)", q) + if glob: + ext = glob.group(1).lower() + return lambda p, _e=ext: file_ext(p) == _e + + lower = q.lower() + return lambda p, _l=lower: _l in p.lower() + + +def filter_paths(paths: list[str], query: str) -> tuple[list[str] | None, bool]: + """Filter paths. Returns (matches_or_None_on_bad_regex, ok).""" + matcher = build_matcher(query) + if matcher is None: + return None, False + return [p for p in paths if matcher(p)], True diff --git a/skills/curiosity-engine/scripts/filebrowser_packs.py b/skills/curiosity-engine/scripts/filebrowser_packs.py new file mode 100644 index 0000000..9df1474 --- /dev/null +++ b/skills/curiosity-engine/scripts/filebrowser_packs.py @@ -0,0 +1,545 @@ +"""CE pack discovery + sandboxed action dispatch (Phase 2b++++). + +Switchbay owns the full packstore (git install, pip extras, agent skill +dispatch via skillkit). CE mirrors the **same** ``pack.json`` Manifest +shape — ``file_routes``, enable state, local-path install — so the +filebrowser can list and dispatch named actions under the vault/wiki +sandbox without inventing a parallel pack format. + +Search order (first wins per pack name): + 1. ``/.workbench/packs//pack.json`` (Switchbay workspace) + 2. ``/packs//pack.json`` (optional CE-local) + 3. ``$CE_PACKS_DIR//pack.json`` when set + +Enable state: ``/.workbench/packs-state.json`` (Switchbay- +compatible: name → bool; missing key = enabled). + +Dispatch (``POST /api/packs//action/`` with ``{path}``): + · path must resolve under vault/|wiki/ (filebrowser_fs sandbox) + · pack must be discovered + enabled + · action must appear in that pack's ``file_routes`` + · skill ``{pack}-{action}`` preferred at ``/skills//SKILL.md`` + (Switchbay skillkit layout); falls back to ``skills//`` + · queues a run under ``.workbench/pack-runs/.json`` and returns + ``{run_id, pack, action}`` (Switchbay-compatible). Agent/LLM execution + stays shell-side — CE accepts + sandboxes the request. + +Install: local-path copy into ``.workbench/packs/``, or git clone +``--depth 1`` from an http(s)/git@ URL (same sandbox as path install). +""" +from __future__ import annotations + +import json +import os +import re +import shutil +import subprocess +import uuid +from pathlib import Path +from typing import Any + +import filebrowser_fs + +_NAME_RE = re.compile(r"^[a-z][a-z0-9_-]{0,63}$") +_ACTION_RE = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9._-]{0,63}$") +PACK_FILE = "pack.json" +_STATE_FILE = "packs-state.json" +_RUNS_DIR = "pack-runs" + + +class PackError(Exception): + """User-visible pack failure (not found, disabled, bad path, etc.).""" + + def __init__(self, message: str, *, status: int = 400): + super().__init__(message) + self.status = status + + +def _parse_file_routes(raw: dict[str, Any], *, pack: str, scope: str) -> list[dict[str, Any]]: + routes_raw = raw.get("file_routes") or [] + out: list[dict[str, Any]] = [] + if not isinstance(routes_raw, list): + return out + for r in routes_raw: + if not isinstance(r, dict): + continue + ext = str(r.get("ext") or "").strip().lower() + action = str(r.get("action") or "").strip() + if not ext or not action: + continue + if not ext.startswith("."): + ext = "." + ext + entry: dict[str, Any] = { + "ext": ext, + "action": action, + "pack": pack, + "scope": scope, + } + for k in ( + "label", + "description", + "endpoint", + "tab_kind", + "selection_kind", + "requires_binary", + ): + v = r.get(k) + if isinstance(v, str) and v: + entry[k] = v + if r.get("primary") is True: + entry["primary"] = True + out.append(entry) + return out + + +def _iter_pack_dirs(workspace: Path) -> list[tuple[Path, str]]: + """Return (pack_dir, scope) candidates; workspace overrides CE_PACKS_DIR.""" + ws = Path(workspace).resolve() + pairs: list[tuple[Path, str]] = [ + (ws / ".workbench" / "packs", "workspace"), + (ws / "packs", "workspace-ce"), + ] + extra = (os.environ.get("CE_PACKS_DIR") or "").strip() + if extra: + pairs.append((Path(extra).expanduser().resolve(), "ce-packs")) + return pairs + + +def _workspace_state_path(workspace: Path) -> Path: + return Path(workspace).resolve() / ".workbench" / _STATE_FILE + + +def _load_state(workspace: Path) -> dict[str, Any]: + p = _workspace_state_path(workspace) + if not p.is_file(): + return {} + try: + raw = json.loads(p.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return {} + return raw if isinstance(raw, dict) else {} + + +def _save_state(workspace: Path, state: dict[str, Any]) -> None: + p = _workspace_state_path(workspace) + p.parent.mkdir(parents=True, exist_ok=True) + tmp = p.with_suffix(".tmp") + tmp.write_text(json.dumps(state, indent=2, sort_keys=True) + "\n", encoding="utf-8") + tmp.replace(p) + + +def _is_enabled(workspace: Path, name: str) -> bool: + state = _load_state(workspace) + if name in state: + return bool(state[name]) + return True + + +def _read_manifest(pack_dir: Path) -> dict[str, Any] | None: + pf = pack_dir / PACK_FILE + if not pf.is_file(): + return None + try: + raw = json.loads(pf.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + if not isinstance(raw, dict): + return None + return raw + + +def _pack_record(pack_dir: Path, scope: str, workspace: Path) -> dict[str, Any] | None: + raw = _read_manifest(pack_dir) + if raw is None: + return None + name = str(raw.get("name") or pack_dir.name).strip() or pack_dir.name + if not _NAME_RE.match(name): + return None + routes = _parse_file_routes(raw, pack=name, scope=scope) + skills_raw = raw.get("skills") or [] + skills = [str(s) for s in skills_raw if isinstance(s, str)] + return { + "name": name, + "version": str(raw.get("version") or "").strip() or "0.0.0", + "description": str(raw.get("description") or ""), + "skills": skills, + "file_routes": routes, + "actions": sorted({r["action"] for r in routes}), + "scope": scope, + "path": str(pack_dir.resolve()), + "enabled": _is_enabled(workspace, name), + "requires_extra": [ + str(e) for e in (raw.get("requires_extra") or []) if isinstance(e, str) + ], + } + + +def list_packs(workspace: Path) -> list[dict[str, Any]]: + """Every discovered pack (workspace first; first name wins).""" + seen: set[str] = set() + out: list[dict[str, Any]] = [] + for root, scope in _iter_pack_dirs(workspace): + if not root.is_dir(): + continue + try: + children = sorted(root.iterdir(), key=lambda p: p.name) + except OSError: + continue + for child in children: + if not child.is_dir(): + continue + name = child.name + if not _NAME_RE.match(name) or name in seen: + continue + rec = _pack_record(child, scope, workspace) + if rec is None: + continue + seen.add(name) + out.append(rec) + return out + + +def get_pack(workspace: Path, name: str) -> dict[str, Any] | None: + if not _NAME_RE.match(name): + return None + for root, scope in _iter_pack_dirs(workspace): + d = root / name + if d.is_dir(): + rec = _pack_record(d, scope, workspace) + if rec is not None: + return rec + return None + + +def packs_payload(workspace: Path) -> dict[str, Any]: + packs = list_packs(workspace) + return {"ok": True, "packs": packs, "count": len(packs)} + + +def actions_for_pack(workspace: Path, name: str) -> dict[str, Any]: + rec = get_pack(workspace, name) + if rec is None: + raise PackError(f"pack not found: {name}", status=404) + routes = list(rec.get("file_routes") or []) + return { + "ok": True, + "pack": rec["name"], + "enabled": bool(rec.get("enabled", True)), + "actions": list(rec.get("actions") or []), + "routes": routes, + "count": len(routes), + } + + +def file_routes_for(workspace: Path) -> list[dict[str, Any]]: + """Flatten file_routes from enabled packs only (Switchbay parity).""" + routes: list[dict[str, Any]] = [] + for pack in list_packs(workspace): + if not pack.get("enabled", True): + continue + routes.extend(pack.get("file_routes") or []) + return routes + + +def file_routes_payload(workspace: Path) -> dict[str, Any]: + routes = file_routes_for(workspace) + return {"ok": True, "routes": routes, "count": len(routes)} + + +def set_enabled(workspace: Path, name: str, enabled: bool) -> dict[str, Any]: + if not _NAME_RE.match(name): + raise PackError(f"invalid pack name: {name!r}") + rec = get_pack(workspace, name) + if rec is None: + raise PackError(f"pack not found: {name}", status=404) + state = _load_state(workspace) + state[name] = bool(enabled) + _save_state(workspace, state) + updated = get_pack(workspace, name) + return {"ok": True, "pack": updated} + + + +_GIT_URL_PREFIXES = ("http://", "https://", "git@", "ssh://", "git+") + + +def looks_like_git_url(source: str) -> bool: + """True for Switchbay-shaped pack install URL inputs.""" + s = (source or "").strip() + if not s: + return False + if s.startswith(_GIT_URL_PREFIXES) or s.endswith(".git"): + return True + return False + + +def _slug_from_url(url: str) -> str: + s = url.rstrip("/") + if s.endswith(".git"): + s = s[: -len(".git")] + s = s.rsplit("/", 1)[-1] + s = s.rsplit(":", 1)[-1] + s = re.sub(r"[^a-z0-9._-]+", "-", s.lower()).strip("-") + return s or "pack" + + +def _validate_git_url(url: str) -> str: + """Allow only remote git transports; refuse file:// and local paths.""" + u = (url or "").strip() + if not u: + raise PackError("url is required") + lower = u.lower() + if lower.startswith("file:"): + raise PackError("file:// URLs are not allowed for pack install") + if u.startswith("/") or u.startswith("\\") or (len(u) > 1 and u[1] == ":"): + raise PackError("local paths must use install_from_path, not git") + if not looks_like_git_url(u): + raise PackError(f"not a git URL: {u!r}") + # Strip git+ prefix used by some lockfiles. + if u.startswith("git+"): + u = u[4:] + return u + + +def install_from_git(workspace: Path, url: str) -> dict[str, Any]: + """Clone ``url`` (``git clone --depth 1``) into ``.workbench/packs//``. + + Directory name is derived from the URL slug; if pack.json ``name`` differs, + the directory is renamed to match (Switchbay parity). Refuses file:// and + nested installs into the destination tree. Does **not** fetch remote skills. + """ + u = _validate_git_url(url) + ws = Path(workspace).resolve() + target_name = _slug_from_url(u) + if not _NAME_RE.match(target_name): + raise PackError(f"can't derive a safe pack name from {url!r}") + dest_root = ws / ".workbench" / "packs" + dest_root.mkdir(parents=True, exist_ok=True) + dest_root_res = dest_root.resolve() + target = dest_root / target_name + if target.exists(): + raise PackError(f"pack {target_name!r} already installed in workspace") + # Ensure target stays under dest_root. + if not str(target.resolve()).startswith(str(dest_root_res) + os.sep): + raise PackError("refusing to install outside packs dir") + env = { + k: v + for k, v in os.environ.items() + if k not in {"VIRTUAL_ENV", "UV_PROJECT_ENVIRONMENT", "PYTHONPATH"} + } + try: + proc = subprocess.run( + ["git", "clone", "--depth", "1", u, str(target)], + capture_output=True, + timeout=120, + check=False, + env=env, + ) + except FileNotFoundError as e: + raise PackError("git is not available on PATH") from e + except subprocess.TimeoutExpired as e: + if target.exists(): + shutil.rmtree(target, ignore_errors=True) + raise PackError("git clone timed out") from e + if proc.returncode != 0: + if target.exists(): + shutil.rmtree(target, ignore_errors=True) + err = (proc.stderr or b"").decode("utf-8", "replace").strip()[:400] + raise PackError(f"git clone failed (rc={proc.returncode}): {err}") + raw = _read_manifest(target) + if raw is None: + shutil.rmtree(target, ignore_errors=True) + raise PackError(f"cloned repo has no valid {PACK_FILE}") + name = str(raw.get("name") or "").strip() + if not _NAME_RE.match(name): + shutil.rmtree(target, ignore_errors=True) + raise PackError(f"manifest name invalid: {name!r}") + if name != target_name: + renamed = dest_root / name + if renamed.exists(): + shutil.rmtree(target, ignore_errors=True) + raise PackError(f"manifest name {name!r} collides with existing pack") + if not str(renamed.resolve()).startswith(str(dest_root_res) + os.sep): + shutil.rmtree(target, ignore_errors=True) + raise PackError("refusing to install outside packs dir") + target.rename(renamed) + target = renamed + rec = get_pack(ws, name) + if rec is None: + raise PackError("install succeeded but pack not discoverable") + return {"ok": True, "pack": rec, "source": "git"} + + +def install_from_path(workspace: Path, src: str | Path) -> dict[str, Any]: + """Copy a local pack directory into ``.workbench/packs//``. + + Source must be an absolute path to a directory with a valid pack.json. + Refuses to install from inside the destination tree (escape / nest). + """ + ws = Path(workspace).resolve() + src_path = Path(src).expanduser() + if not src_path.is_absolute(): + raise PackError("install source must be an absolute path") + try: + src_path = src_path.resolve() + except (OSError, ValueError) as e: + raise PackError(f"resolve failed: {e}") from e + if not src_path.is_dir(): + raise PackError(f"not a directory: {src_path}") + raw = _read_manifest(src_path) + if raw is None: + raise PackError(f"{src_path} has no valid {PACK_FILE}") + name = str(raw.get("name") or "").strip() + if not _NAME_RE.match(name): + raise PackError(f"manifest name invalid: {name!r}") + dest_root = ws / ".workbench" / "packs" + dest_root.mkdir(parents=True, exist_ok=True) + dest_root_res = dest_root.resolve() + # Refuse installing from inside dest (or the workspace packs tree). + try: + src_path.relative_to(dest_root_res) + raise PackError("refusing to install from inside the packs destination") + except ValueError: + pass + target = dest_root / name + if target.exists(): + raise PackError(f"pack {name!r} already installed in workspace") + # Ensure target stays under dest_root after resolve. + if dest_root_res not in target.resolve().parents and target.resolve() != dest_root_res / name: + # name is validated by _NAME_RE so this is belt-and-brace + if not str(target.resolve()).startswith(str(dest_root_res) + os.sep): + raise PackError("refusing to install outside packs dir") + shutil.copytree(src_path, target) + rec = get_pack(ws, name) + if rec is None: + raise PackError("install succeeded but pack not discoverable") + return {"ok": True, "pack": rec} + + +def uninstall_pack(workspace: Path, name: str) -> dict[str, Any]: + if not _NAME_RE.match(name): + raise PackError(f"invalid pack name: {name!r}") + ws = Path(workspace).resolve() + dest_root = (ws / ".workbench" / "packs").resolve() + target = dest_root / name + if not target.exists(): + raise PackError(f"pack not found in workspace scope: {name}", status=404) + if dest_root not in target.resolve().parents: + raise PackError("refusing to delete outside the pack scope dir") + shutil.rmtree(target) + return {"ok": True, "uninstalled": name} + + +def _find_skill(pack_dir: Path, pack: str, action: str) -> dict[str, Any] | None: + """Locate ``{pack}-{action}`` skill (Switchbay layout), then ``{action}``.""" + skills_root = pack_dir / "skills" + if not skills_root.is_dir(): + return None + candidates = [f"{pack}-{action}", action] + for skill_name in candidates: + skill_dir = skills_root / skill_name + skill_md = skill_dir / "SKILL.md" + if skill_md.is_file(): + try: + body = skill_md.read_text(encoding="utf-8") + except OSError: + body = "" + return { + "name": skill_name, + "path": str(skill_dir.resolve()), + "body_chars": len(body), + } + return None + + +def _route_for_action(rec: dict[str, Any], action: str) -> dict[str, Any] | None: + for r in rec.get("file_routes") or []: + if isinstance(r, dict) and r.get("action") == action: + return r + return None + + +def dispatch_action( + workspace: Path, + pack: str, + action: str, + path: str, +) -> dict[str, Any]: + """Sandbox-validate and queue a pack file-route action. + + Returns Switchbay-shaped ``{run_id, pack, action}`` plus CE fields + (``status``, ``skill``, ``path``). Does **not** spawn an LLM agent. + """ + if not _NAME_RE.match(pack): + raise PackError(f"invalid pack name: {pack!r}") + if not _ACTION_RE.match(action): + raise PackError(f"invalid action name: {action!r}") + rel = (path or "").strip() + if not rel: + raise PackError("path required") + + rec = get_pack(workspace, pack) + if rec is None: + raise PackError(f"pack not found: {pack}", status=404) + if not rec.get("enabled", True): + raise PackError( + f"pack '{pack}' is not active — enable it via POST /api/packs/toggle", + status=409, + ) + + route = _route_for_action(rec, action) + if route is None: + raise PackError( + f"action {action!r} not declared in pack {pack!r} file_routes", + status=404, + ) + + # Sandbox: vault/wiki only; must be an existing file. + try: + target = filebrowser_fs.resolve(workspace, rel, must_exist=True) + except filebrowser_fs.FileOpError as e: + raise PackError(str(e), status=400) from e + if not target.is_file(): + raise PackError("path must be a file", status=400) + + pack_dir = Path(str(rec.get("path") or "")) + skill = _find_skill(pack_dir, pack, action) if pack_dir.is_dir() else None + + run_id = f"run-{uuid.uuid4().hex[:8]}" + norm_path = rel.replace("\\", "/").lstrip("./").strip("/") + while norm_path.startswith("./"): + norm_path = norm_path[2:] + run_rec: dict[str, Any] = { + "run_id": run_id, + "pack": pack, + "action": action, + "path": norm_path, + "status": "queued", + "kind": f"pack:{pack}:{action}", + "skill": skill, + "route": { + k: route[k] + for k in ("ext", "action", "label", "endpoint", "tab_kind") + if k in route + }, + "mode": "skill_queue" if skill else "accepted", + "note": ( + "CE queued the pack action under workspace sandbox; " + "agent/LLM skill execution remains Switchbay-side." + ), + } + + runs_dir = Path(workspace).resolve() / ".workbench" / _RUNS_DIR + runs_dir.mkdir(parents=True, exist_ok=True) + run_path = runs_dir / f"{run_id}.json" + run_path.write_text(json.dumps(run_rec, indent=2) + "\n", encoding="utf-8") + + return { + "run_id": run_id, + "pack": pack, + "action": action, + "status": run_rec["status"], + "path": run_rec["path"], + "skill": skill["name"] if skill else None, + "mode": run_rec["mode"], + } diff --git a/skills/curiosity-engine/scripts/filebrowser_tree.py b/skills/curiosity-engine/scripts/filebrowser_tree.py new file mode 100644 index 0000000..4dfd8c5 --- /dev/null +++ b/skills/curiosity-engine/scripts/filebrowser_tree.py @@ -0,0 +1,77 @@ +"""CE filebrowser tree walk — vault/ + wiki/ only (Phase 2b). + +Switchbay's /api/tree walks the whole workspace. CE intake prefers vault +and wiki path semantics so the embed surface matches the three-object +model (architecture.md). Hidden/dot dirs and a small SKIP set are pruned +during os.walk (never descend). +""" +from __future__ import annotations + +import os +from pathlib import Path + +# Parallel Switchbay daemon.SKIP_DIRS / fileops._SKIP_DIRS (subset relevant +# under vault/wiki; kept for safety if a nested tool tree appears). +SKIP_DIRS = frozenset( + { + ".git", + ".workbench", + "node_modules", + ".venv", + "venv", + "__pycache__", + "dist", + "build", + ".vite", + ".cache", + ".idea", + ".vscode", + ".pytest_cache", + ".curator", + } +) + +# Roots relative to the workspace. Order is stable for UI grouping. +TREE_ROOTS = ("vault", "wiki") + + +def walk_ce_tree(workspace: Path) -> list[str]: + """Return sorted relative paths under vault/ and wiki/. + + Paths use forward slashes and always start with ``vault/`` or + ``wiki/``. Missing roots are skipped (empty list for that root). + """ + ws = Path(workspace).resolve() + out: list[str] = [] + for root_name in TREE_ROOTS: + root = ws / root_name + if not root.is_dir(): + continue + for dirpath, dirnames, filenames in os.walk(root): + dirnames[:] = [ + d + for d in dirnames + if not d.startswith(".") and d not in SKIP_DIRS + ] + for fn in filenames: + if fn.startswith("."): + continue + full = Path(dirpath) / fn + try: + rel = full.resolve().relative_to(ws).as_posix() + except ValueError: + continue + out.append(rel) + out.sort() + return out + + +def tree_payload(workspace: Path) -> dict: + """JSON body for GET /api/tree.""" + files = walk_ce_tree(workspace) + return { + "ok": True, + "roots": list(TREE_ROOTS), + "files": files, + "count": len(files), + } diff --git a/skills/curiosity-engine/scripts/obsidian_import.py b/skills/curiosity-engine/scripts/obsidian_import.py new file mode 100644 index 0000000..bdeed09 --- /dev/null +++ b/skills/curiosity-engine/scripts/obsidian_import.py @@ -0,0 +1,457 @@ +#!/usr/bin/env python3 +"""Import an Obsidian vault into a CE workspace as ``wiki/``. + +**Mapping (documented choice):** Obsidian vault root → CE ``wiki/``. + +CE already treats ``wiki/`` as Obsidian-shaped markdown with ``[[wikilinks]]`` +(see ``docs/viewers.md`` — open folder as vault → ``/wiki``). +Import copies notes/assets into ``wiki/`` so Atlas / classic wiki see them +without a Work/symlink dance. Wikilink text is preserved verbatim. + +``.obsidian/`` is copied under ``wiki/.obsidian/`` for round-trip. Other +junk/dot dirs (``.trash``, ``.git``, ``node_modules``, …) are skipped. + +**Source sandbox:** absolute folder/zip paths must resolve under ``$HOME`` +or ``$CE_WORKSPACE_HOME`` (same contract as ``workspace_registry``). Zip +members are zip-slip checked before extract. + +**Targets:** + * into current workspace (default): merge into ``/wiki/`` + * ``--target DIR``: bootstrap a minimal CE workspace at DIR, then import + +CLI (conceptual ``ce import-obsidian ``):: + + python3 obsidian_import.py [--workspace DIR] [--target DIR] + [--register] [--force] + +API: ``POST /api/import/obsidian`` — JSON ``{path}`` or multipart zip +(honours ``CE_PUBLIC_BASE`` via viewer path rewrite). +""" +from __future__ import annotations + +import argparse +import json +import os +import re +import shutil +import sys +import tempfile +import time +import zipfile +from pathlib import Path +from typing import Any, Iterable + +import workspace_registry + +WIKILINK_RE = re.compile(r"\[\[([^\]|]+)(?:\|[^\]]+)?\]\]") + +SKIP_DIR_NAMES = frozenset({ + ".trash", + ".git", + ".svn", + ".hg", + "node_modules", + "__pycache__", + "venv", + ".venv", +}) +KEEP_DOT_DIRS = frozenset({".obsidian"}) + +MAX_ZIP_BYTES = 200 * 1024 * 1024 +MAX_ZIP_MEMBERS = 50_000 +SCHEMA = "ce-obsidian-import/1" + + +class ObsidianImportError(Exception): + """User-visible import failure.""" + + def __init__(self, message: str, *, status: int = 400): + super().__init__(message) + self.status = status + + +def _norm(p: Path) -> Path: + return Path(p).expanduser().resolve() + + +def resolve_source_path(path: str | Path) -> Path: + """Absolute source under home/workspace sandbox; refuse escapes.""" + raw = str(path or "").strip() + if not raw: + raise ObsidianImportError("path required") + if "\x00" in raw: + raise ObsidianImportError("invalid path") + p = Path(raw).expanduser() + if not p.is_absolute(): + raise ObsidianImportError("source must be an absolute path") + try: + resolved = p.resolve() + except (OSError, ValueError) as e: + raise ObsidianImportError(f"resolve failed: {e}") from e + if not workspace_registry.is_within_sandbox(resolved): + raise ObsidianImportError( + f"path escapes workspace sandbox ({workspace_registry._home_root()})", + status=403, + ) + return resolved + + +def looks_like_obsidian_vault(root: Path) -> bool: + if not root.is_dir(): + return False + if (root / ".obsidian").is_dir(): + return True + for p in root.rglob("*.md"): + parts = p.relative_to(root).parts + if any(part.startswith(".") and part not in KEEP_DOT_DIRS for part in parts[:-1]): + continue + return True + return False + + +def count_wikilinks(text: str) -> int: + return len(WIKILINK_RE.findall(text or "")) + + +def _should_skip_dir(name: str) -> bool: + if name in KEEP_DOT_DIRS: + return False + if name in SKIP_DIR_NAMES: + return True + if name.startswith("."): + return True + return False + + +def iter_vault_files(vault_root: Path) -> Iterable[Path]: + root = vault_root.resolve() + for dirpath, dirnames, filenames in os.walk(root): + dirnames[:] = [d for d in dirnames if not _should_skip_dir(d)] + for name in filenames: + if name in (".DS_Store", "Thumbs.db"): + continue + yield Path(dirpath) / name + + +def ensure_workspace_skeleton(dest: Path, *, force: bool = False) -> Path: + """Create minimal CE workspace markers at dest (wiki/, vault/, .curator/).""" + dest = _norm(dest) + if not workspace_registry.is_within_sandbox(dest): + raise ObsidianImportError( + f"target escapes workspace sandbox ({workspace_registry._home_root()})", + status=403, + ) + if dest.exists() and any(dest.iterdir()) and not force: + if not ( + (dest / "wiki").is_dir() + or (dest / ".curator" / "config.json").is_file() + ): + raise ObsidianImportError( + f"target exists and is not an empty/CE workspace: {dest}" + ) + dest.mkdir(parents=True, exist_ok=True) + (dest / "wiki").mkdir(exist_ok=True) + (dest / "vault").mkdir(exist_ok=True) + (dest / "vault" / "raw").mkdir(exist_ok=True) + curator = dest / ".curator" + curator.mkdir(exist_ok=True) + cfg = curator / "config.json" + if not cfg.is_file(): + cfg.write_text( + json.dumps( + { + "wiki_viewer_mode": "obsidian", + "imported_from": "obsidian", + "schema": SCHEMA, + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + return dest + + +def _safe_extract_zip(zf: zipfile.ZipFile, dest: Path) -> Path: + dest = dest.resolve() + dest.mkdir(parents=True, exist_ok=True) + members = zf.infolist() + if len(members) > MAX_ZIP_MEMBERS: + raise ObsidianImportError("zip has too many members") + total = 0 + for info in members: + total += info.file_size + if total > MAX_ZIP_BYTES: + raise ObsidianImportError( + "zip too large (uncompressed >200 MB)", status=413 + ) + name = info.filename.replace("\\", "/") + if not name or name.endswith("/"): + continue + if name.startswith("/") or name.startswith("../") or "/../" in f"/{name}/": + raise ObsidianImportError(f"zip slip refused: {info.filename}") + target = (dest / name).resolve() + try: + target.relative_to(dest) + except ValueError as e: + raise ObsidianImportError(f"zip slip refused: {info.filename}") from e + zf.extractall(dest) + children = [p for p in dest.iterdir() if p.name not in ("__MACOSX",)] + if len(children) == 1 and children[0].is_dir(): + return children[0] + return dest + + +def extract_zip_bytes(data: bytes, work_dir: Path) -> Path: + if not data: + raise ObsidianImportError("empty zip") + if len(data) > MAX_ZIP_BYTES: + raise ObsidianImportError("zip too large", status=413) + work_dir = work_dir.resolve() + work_dir.mkdir(parents=True, exist_ok=True) + zip_path = work_dir / "upload.zip" + zip_path.write_bytes(data) + extract_dir = work_dir / "extracted" + extract_dir.mkdir(exist_ok=True) + try: + with zipfile.ZipFile(zip_path, "r") as zf: + if zf.testzip() is not None: + raise ObsidianImportError("corrupt zip") + return _safe_extract_zip(zf, extract_dir) + except zipfile.BadZipFile as e: + raise ObsidianImportError(f"not a zip: {e}") from e + + +def extract_zip_path(zip_path: Path, work_dir: Path) -> Path: + zip_path = resolve_source_path(zip_path) + if not zip_path.is_file(): + raise ObsidianImportError("zip path must be a file") + try: + data = zip_path.read_bytes() + except OSError as e: + raise ObsidianImportError(f"read failed: {e}") from e + return extract_zip_bytes(data, work_dir) + + +def copy_vault_into_wiki( + vault_root: Path, + wiki_dir: Path, + *, + force: bool = False, +) -> dict[str, Any]: + vault_root = vault_root.resolve() + wiki_dir = wiki_dir.resolve() + if not looks_like_obsidian_vault(vault_root): + raise ObsidianImportError( + f"not an Obsidian vault (no .obsidian/ and no .md): {vault_root}" + ) + wiki_dir.mkdir(parents=True, exist_ok=True) + + copied: list[str] = [] + skipped: list[str] = [] + wikilink_total = 0 + md_files = 0 + + for src in iter_vault_files(vault_root): + rel = src.relative_to(vault_root) + dest = (wiki_dir / rel).resolve() + try: + dest.relative_to(wiki_dir) + except ValueError as e: + raise ObsidianImportError(f"refusing path escape: {rel}") from e + if dest.exists() and not force: + skipped.append(rel.as_posix()) + continue + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dest) + copied.append(rel.as_posix()) + if src.suffix.lower() == ".md": + md_files += 1 + try: + text = src.read_text(encoding="utf-8", errors="replace") + except OSError: + text = "" + wikilink_total += count_wikilinks(text) + + return { + "copied": copied, + "skipped": skipped, + "files_copied": len(copied), + "files_skipped": len(skipped), + "md_files": md_files, + "wikilink_occurrences": wikilink_total, + } + + +def import_vault( + source: str | Path | None = None, + *, + workspace: str | Path | None = None, + target: str | Path | None = None, + zip_bytes: bytes | None = None, + zip_filename: str | None = None, + register: bool = False, + force: bool = False, +) -> dict[str, Any]: + """Import Obsidian vault from folder path or zip into a CE workspace.""" + if source is not None and zip_bytes is not None: + raise ObsidianImportError("provide path or zip, not both") + if source is None and zip_bytes is None: + raise ObsidianImportError("path or zip required") + + if target is not None: + dest_ws = ensure_workspace_skeleton(Path(target), force=force) + mode = "new" + else: + if workspace is None: + raise ObsidianImportError("workspace required when target is unset") + dest_ws = _norm(workspace) + if not workspace_registry.is_within_sandbox(dest_ws): + raise ObsidianImportError( + f"workspace escapes sandbox ({workspace_registry._home_root()})", + status=403, + ) + ensure_workspace_skeleton(dest_ws, force=True) + mode = "into_current" + + wiki_dir = dest_ws / "wiki" + staging: Path | None = None + vault_root: Path + source_label: str + + try: + if zip_bytes is not None: + wb = dest_ws / ".workbench" / "obsidian-import" + wb.mkdir(parents=True, exist_ok=True) + staging = Path(tempfile.mkdtemp(prefix="zip-", dir=str(wb))) + vault_root = extract_zip_bytes(zip_bytes, staging) + source_label = zip_filename or "upload.zip" + else: + src_path = resolve_source_path(source) # type: ignore[arg-type] + if src_path.is_file() and src_path.suffix.lower() == ".zip": + wb = dest_ws / ".workbench" / "obsidian-import" + wb.mkdir(parents=True, exist_ok=True) + staging = Path(tempfile.mkdtemp(prefix="zip-", dir=str(wb))) + vault_root = extract_zip_path(src_path, staging) + source_label = str(src_path) + elif src_path.is_dir(): + vault_root = src_path + source_label = str(src_path) + else: + raise ObsidianImportError( + "source must be a directory or .zip file" + ) + + if vault_root.resolve() == wiki_dir.resolve(): + raise ObsidianImportError("source vault is the destination wiki/") + + stats = copy_vault_into_wiki(vault_root, wiki_dir, force=force) + finally: + if staging is not None and staging.exists(): + shutil.rmtree(staging, ignore_errors=True) + + registered = False + reg_error = None + if register: + try: + workspace_registry.register(dest_ws, set_active=False) + registered = True + except TypeError: + # Older/newer kw spelling + try: + workspace_registry.register(dest_ws, set_active=False) + registered = True + except Exception as e: + registered = False + reg_error = str(e) + except Exception as e: + registered = False + reg_error = str(e) + + receipt_dir = dest_ws / ".curator" / "imports" + receipt_dir.mkdir(parents=True, exist_ok=True) + receipt = { + "schema": SCHEMA, + "ok": True, + "mode": mode, + "mapping": "obsidian-vault-root → wiki/", + "source": source_label, + "workspace": str(dest_ws), + "wiki_root": str(wiki_dir), + "wikilinks_preserved": True, + **stats, + "registered": registered, + } + if reg_error: + receipt["register_error"] = reg_error + stamp = time.strftime("%Y%m%d-%H%M%S") + (receipt_dir / f"{stamp}-obsidian.json").write_text( + json.dumps(receipt, indent=2) + "\n", encoding="utf-8" + ) + return receipt + + +def main(argv: list[str] | None = None) -> int: + argv = list(sys.argv[1:] if argv is None else argv) + if argv and argv[0] in ("import-obsidian", "obsidian", "import"): + argv = argv[1:] + + ap = argparse.ArgumentParser( + prog="ce import-obsidian", + description=( + "Import an Obsidian vault (folder or zip) into a CE workspace as wiki/. " + "Preserves [[wikilinks]]. Mapping: Obsidian vault root → wiki/." + ), + ) + ap.add_argument("path", help="Absolute path to Obsidian vault folder or .zip") + ap.add_argument( + "--workspace", + default=None, + help="Existing CE workspace (default: cwd). Ignored when --target is set.", + ) + ap.add_argument( + "--target", + default=None, + help="Create/use a new CE workspace at this absolute path and import there.", + ) + ap.add_argument( + "--register", + action="store_true", + help="Register the destination workspace in CE workspace registry.", + ) + ap.add_argument( + "--force", + action="store_true", + help="Overwrite existing files under wiki/ on name collision.", + ) + ap.add_argument("--json", action="store_true", help="Print receipt JSON only.") + args = ap.parse_args(argv) + + ws = args.workspace or (None if args.target else os.getcwd()) + try: + result = import_vault( + args.path, + workspace=ws, + target=args.target, + register=args.register, + force=args.force, + ) + except ObsidianImportError as e: + print(f"error: {e}", file=sys.stderr) + return 2 + if args.json: + print(json.dumps(result, indent=2)) + else: + print( + f"Imported {result['files_copied']} files " + f"({result['md_files']} md, {result['wikilink_occurrences']} wikilinks) " + f"→ {result['wiki_root']}" + ) + print(f"mapping: {result['mapping']}") + if result.get("files_skipped"): + print(f"skipped existing: {result['files_skipped']}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/curiosity-engine/scripts/public_base.py b/skills/curiosity-engine/scripts/public_base.py new file mode 100644 index 0000000..51954f4 --- /dev/null +++ b/skills/curiosity-engine/scripts/public_base.py @@ -0,0 +1,147 @@ +"""Public URL base path + hosted-shell detection (Phase 2a). + +Switchbay/okbay same-origin reverse-proxy CE under a configurable prefix +(default empty; typical embed: ``/embed/ce``). See umbrella CHARTER locked +decisions #1 (no iframes) and the CE hosted-mode contract in +``docs/ADR-001-proxy-embed-and-hosted.md``. + +Loopback serve stays ``127.0.0.1`` (bare viewer default port 8090; embed +hosts commonly use **8766**). Setting ``CE_PUBLIC_BASE`` does not change +the bind address. +""" + +from __future__ import annotations + +import os +from typing import Mapping +from urllib.parse import parse_qs + +HOSTED_SHELLS = frozenset({"switchbay", "okbay"}) +HOST_HEADER = "X-CE-Host" +# Recommended public base when proxied (no trailing slash). +DEFAULT_EMBED_BASE = "/embed/ce" + + +def public_base() -> str: + """Normalized public base path (no trailing slash), or \"\" when unset. + + Env: ``CE_PUBLIC_BASE`` — e.g. ``/embed/ce``. + """ + raw = (os.environ.get("CE_PUBLIC_BASE") or "").strip() + if not raw or raw == "/": + return "" + if not raw.startswith("/"): + raw = "/" + raw + return raw.rstrip("/") + + +def strip_public_base(path: str) -> str: + """Strip configured public base from an absolute request path.""" + base = public_base() + if not base: + return path or "/" + p = path or "/" + if p == base: + return "/" + if p.startswith(base + "/"): + rest = p[len(base) :] + return rest if rest else "/" + return p + + +def normalize_hosted_shell(raw: str | None) -> str | None: + """Return ``switchbay``|``okbay`` or None.""" + if not raw: + return None + h = str(raw).strip().lower() + if h in HOSTED_SHELLS: + return h + return None + + +def hosted_shell_from_request( + headers: Mapping[str, str] | None, + query: Mapping[str, list[str]] | None = None, +) -> str | None: + """Resolve hosted shell from ``X-CE-Host`` or ``?host=``. + + Header wins when both are present and valid. + """ + if headers: + for key, val in headers.items(): + if key.lower() == HOST_HEADER.lower(): + found = normalize_hosted_shell(val) + if found: + return found + break + if query: + vals = query.get("host") or [] + if vals: + return normalize_hosted_shell(vals[0]) + return None + + +def hosted_shell_from_query_string(qs: str) -> str | None: + if not qs: + return None + return hosted_shell_from_request(None, parse_qs(qs)) + + +def viewer_bootstrap_js(*, hosted: str | None = None, api_base: str | None = None) -> str: + """Inline script assigning window.CE_* for atlas/wiki/static under a proxy prefix.""" + base = public_base() if api_base is None else (api_base or "") + base = base.rstrip("/") + host = normalize_hosted_shell(hosted) or "" + return ( + "window.CE_PUBLIC_BASE=" + + _js_str(base) + + ";window.CE_HOSTED=" + + _js_str(host) + + ";window.ceApi=function(p){var b=window.CE_PUBLIC_BASE||\"\";" + "if(!p)p=\"/\";if(p.charAt(0)!==\"/\")p=\"/\"+p;return b+p;};" + ) + + +def _js_str(s: str) -> str: + return '"' + s.replace("\\", "\\\\").replace('"', '\\"') + '"' + + +def inject_viewer_bootstrap(html: bytes | str, *, hosted: str | None = None) -> bytes: + """Inject bootstrap script before ```` (or start of ````).""" + text = html.decode("utf-8") if isinstance(html, (bytes, bytearray)) else str(html) + snippet = "\n" + lower = text.lower() + idx = lower.find("") + if idx >= 0: + text = text[:idx] + snippet + text[idx:] + else: + bidx = lower.find("= 0: + gt = text.find(">", bidx) + if gt >= 0: + text = text[: gt + 1] + "\n" + snippet + text[gt + 1 :] + else: + text = snippet + text + else: + text = snippet + text + return text.encode("utf-8") + + +def hosted_settings_policy(hosted: str | None) -> dict: + """Stub contract for shell-hosted CE (Phase 2a). + + Full settings UI split lands later. Today we only advertise intent: + when ``host=switchbay|okbay``, shells own workspace/harness chrome; + CE may later hide duplicate settings surfaces. Physics / viewer knobs + remain available until a parity-gated follow-up. + """ + shell = normalize_hosted_shell(hosted) + return { + "hosted": shell, + "shell_owns_workspace_settings": shell is not None, + "ce_viewer_knobs_enabled": True, # stub: keep atlas/physics until 2b+ + "note": ( + "Hosted mode detected via X-CE-Host or ?host=. " + "Settings chrome split is deferred; see ADR-001." + ), + } diff --git a/skills/curiosity-engine/scripts/viewer_server.py b/skills/curiosity-engine/scripts/viewer_server.py index 0c3a9ad..95d6360 100644 --- a/skills/curiosity-engine/scripts/viewer_server.py +++ b/skills/curiosity-engine/scripts/viewer_server.py @@ -11,8 +11,45 @@ GET / static file (anything in the bundle) GET /api/page?path= raw markdown of a wiki page GET /api/vault/ vault/*.extracted.md basenames only + GET /api/hosted hosted-mode stub (host=switchbay|okbay) + GET /api/tree vault/ + wiki/ relative paths (filebrowser) + GET /api/file-routes pack Manifest.file_routes (enabled packs) + GET /api/packs list discovered packs + actions + GET /api/packs//actions file_routes / actions for one pack + POST /api/packs//action/ dispatch named action {path} (sandbox queue) + POST /api/packs/toggle {name, enabled} workspace enable state + POST /api/packs/install {path|source|url} local or git → .workbench/packs/ + DELETE /api/packs?name= uninstall workspace-scope pack + DELETE /api/workspaces?path= unregister workspace path + GET /api/fs/stat?path= sandbox stat under vault/|wiki/ + POST /api/fs/reveal {path} reveal-in-OS (Finder/xdg/explorer) + POST /api/fs/open-external {path} open with OS default app + POST /api/ingest/from-upload multipart file → vault/raw/ + queue run + POST /api/ingest/from-path {path} absolute file → vault/raw/ + queue + POST /api/import/obsidian {path} vault/zip OR multipart zip -> wiki/ + GET /api/split last partition status (workspace split spike) + GET /api/workspaces CE workspace registry (paths + split provenance) + GET /api/curation/history HistoryDoc for graph replay UI POST /api/page JSON {path, content} → overwrite file POST /api/upload-vault multipart form → save to vault/raw/ + POST /api/split partition wiki pages into a new workspace + POST /api/workspaces register path {path, set_active?, split?} + POST /api/cm-export CM subgraph_export hook {pages, target, dry_run?} + POST /api/fs/create {path, kind?, content?} create file/dir + POST /api/fs/mkdir {path} create empty directory + POST /api/fs/rename {path, to} rename within sandbox + POST /api/fs/move {path, to} move within sandbox + POST /api/fs/delete {path} trash (OS or .workbench/trash) + POST /api/fs/duplicate {path} sibling copy (Switchbay parity) + +Proxy / embed (Phase 2a) +──────────────────────── + Optional ``CE_PUBLIC_BASE`` (e.g. ``/embed/ce``) strips that prefix from + request paths and injects ``window.CE_PUBLIC_BASE`` / ``ceApi()`` into + HTML so absolute ``/api/*`` fetches stay same-origin under a reverse + proxy. Hosted shell: header ``X-CE-Host`` or ``?host=switchbay|okbay``. + Always binds ``127.0.0.1`` (loopback). Bare ``viewer.sh`` default port + remains 8090; embed hosts commonly use **8766**. Writes are constrained: * /api/page only accepts paths that start with `notes/` or `todos/` @@ -20,6 +57,17 @@ stays inside `wiki/`. * /api/upload-vault sanitises the filename (strips directories, replaces non-alnum chars with `_`) before writing to vault/raw/. + * /api/fs/* mutates only under `vault/` + `wiki/` (see filebrowser_fs.py); + escapes, hidden components, and SKIP_DIRS are refused. Delete goes to + OS trash when available, else `.workbench/trash/`. + * /api/packs/*/action/* requires the target file under vault/|wiki/; + install copies or `git clone --depth 1` into `.workbench/packs/`. + Agent/LLM skill execution stays Switchbay-side — CE queues the run. + * /api/fs/reveal|open-external only resolve vault/|wiki/ paths. + * /api/ingest/from-upload|from-path allowlist DEFAULT_EXTS, stage under + vault/raw/, queue `.workbench/ingest-runs/` (shell drains / local_ingest). + * /api/import/obsidian sandboxes source under $HOME/$CE_WORKSPACE_HOME; + zip-slip safe extract; maps Obsidian vault root -> wiki/ (wikilinks kept). After any successful write the server invokes `wiki_render.py build --output-dir ` so the @@ -39,6 +87,7 @@ from __future__ import annotations import http.server +import os import json import re import socketserver @@ -49,6 +98,17 @@ from email.policy import default as default_email_policy from pathlib import Path +import public_base +import filebrowser_tree +import filebrowser_fs +import filebrowser_packs +import filebrowser_ingest +import wiki_partition +import workspace_registry +import cm_export +import curation_history +import obsidian_import + SCRIPT_DIR = Path(__file__).resolve().parent BUNDLE_DIR: Path | None = None @@ -56,6 +116,7 @@ WIKI_DIR: Path | None = None VAULT_DIR: Path | None = None VAULT_RAW_DIR: Path | None = None +SPLIT_LAST: dict | None = None def _safe_wiki_path(rel: str) -> Path: @@ -108,23 +169,134 @@ def log_message(self, fmt, *args): def __init__(self, *args, **kwargs): super().__init__(*args, directory=str(BUNDLE_DIR), **kwargs) + def _rewrite_path(self) -> urllib.parse.ParseResult: + """Strip CE_PUBLIC_BASE so routing + static map to the bundle root.""" + url = urllib.parse.urlparse(self.path) + stripped = public_base.strip_public_base(url.path) + if stripped != url.path: + self.path = urllib.parse.urlunparse( + (url.scheme, url.netloc, stripped, url.params, url.query, url.fragment) + ) + url = urllib.parse.urlparse(self.path) + return url + + def _hosted(self, url: urllib.parse.ParseResult) -> str | None: + return public_base.hosted_shell_from_request( + self.headers, urllib.parse.parse_qs(url.query) + ) + # ── routing ──────────────────────────────────────────────────── def do_GET(self): - url = urllib.parse.urlparse(self.path) + url = self._rewrite_path() if url.path == "/api/page": return self._handle_get_page(urllib.parse.parse_qs(url.query)) if url.path.startswith("/api/vault/"): return self._handle_get_vault(url.path[len("/api/vault/"):]) + if url.path == "/api/hosted": + return self._json(200, public_base.hosted_settings_policy(self._hosted(url))) + if url.path == "/api/tree": + return self._handle_get_tree() + if url.path == "/api/file-routes": + return self._handle_get_file_routes() + if url.path == "/api/packs": + return self._handle_get_packs() + m_actions = self._match_pack_actions(url.path) + if m_actions is not None: + return self._handle_get_pack_actions(m_actions) + if url.path == "/api/fs/stat": + return self._handle_fs_stat(urllib.parse.parse_qs(url.query)) + if url.path == "/api/split": + return self._handle_get_split() + if url.path == "/api/workspaces": + return self._handle_get_workspaces() + if url.path == "/api/curation/history": + return self._handle_get_curation_history() + # Inject embed bootstrap into HTML so /api fetches honor CE_PUBLIC_BASE. + if self._looks_like_html(url.path): + return self._serve_html_with_bootstrap(url) return super().do_GET() def do_POST(self): - url = urllib.parse.urlparse(self.path) + url = self._rewrite_path() if url.path == "/api/page": return self._handle_post_page() if url.path == "/api/upload-vault": return self._handle_upload() + if url.path == "/api/split": + return self._handle_post_split() + if url.path == "/api/workspaces": + return self._handle_post_workspaces() + if url.path == "/api/cm-export": + return self._handle_post_cm_export() + if url.path == "/api/fs/create": + return self._handle_fs_create() + if url.path == "/api/fs/mkdir": + return self._handle_fs_mkdir() + if url.path == "/api/fs/rename": + return self._handle_fs_rename() + if url.path == "/api/fs/move": + return self._handle_fs_move() + if url.path == "/api/fs/delete": + return self._handle_fs_delete() + if url.path == "/api/fs/duplicate": + return self._handle_fs_duplicate() + if url.path == "/api/fs/reveal": + return self._handle_fs_reveal() + if url.path == "/api/fs/open-external": + return self._handle_fs_open_external() + if url.path == "/api/ingest/from-upload": + return self._handle_ingest_from_upload() + if url.path == "/api/ingest/from-path": + return self._handle_ingest_from_path() + if url.path == "/api/import/obsidian": + return self._handle_import_obsidian() + if url.path == "/api/packs/toggle": + return self._handle_packs_toggle() + if url.path == "/api/packs/install": + return self._handle_packs_install() + m_action = self._match_pack_action(url.path) + if m_action is not None: + return self._handle_pack_action(m_action[0], m_action[1]) return self._json(404, {"error": "not found"}) + def do_DELETE(self): + url = self._rewrite_path() + if url.path == "/api/workspaces": + return self._handle_delete_workspaces(urllib.parse.parse_qs(url.query)) + if url.path == "/api/packs": + return self._handle_packs_uninstall(urllib.parse.parse_qs(url.query)) + return self._json(404, {"error": "not found"}) + + def _looks_like_html(self, path: str) -> bool: + p = path.rstrip("/") or "/" + if p == "/" or p.endswith(".html") or p.endswith(".htm"): + return True + return False + + def _serve_html_with_bootstrap(self, url: urllib.parse.ParseResult) -> None: + # Resolve like SimpleHTTPRequestHandler: "" / "/" → index.html + rel = url.path.lstrip("/") + if not rel or rel.endswith("/"): + candidate = BUNDLE_DIR / rel / "index.html" + else: + candidate = BUNDLE_DIR / rel + if candidate.is_dir(): + candidate = candidate / "index.html" + try: + candidate = candidate.resolve() + candidate.relative_to(BUNDLE_DIR.resolve()) + except Exception: + return self.send_error(404, "File not found") + if not candidate.is_file(): + return self.send_error(404, "File not found") + raw = candidate.read_bytes() + body = public_base.inject_viewer_bootstrap(raw, hosted=self._hosted(url)) + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + # ── helpers ──────────────────────────────────────────────────── def _json(self, status: int, payload: dict) -> None: body = json.dumps(payload).encode("utf-8") @@ -170,6 +342,588 @@ def _rebuild(self) -> None: sys.stderr.write(f"viewer-server: rebuild failed: {e}\n") # ── handlers ─────────────────────────────────────────────────── + def _handle_get_tree(self) -> None: + """List vault/ + wiki/ files for the CE filebrowser (Phase 2b).""" + if WORKSPACE_DIR is None: + return self._json(500, {"error": "workspace unset"}) + return self._json(200, filebrowser_tree.tree_payload(WORKSPACE_DIR)) + + def _handle_get_file_routes(self) -> None: + """Pack file_routes from enabled packs (Phase 2b++++).""" + if WORKSPACE_DIR is None: + return self._json(500, {"error": "workspace unset"}) + return self._json(200, filebrowser_packs.file_routes_payload(WORKSPACE_DIR)) + + def _match_pack_actions(self, path: str) -> str | None: + """GET /api/packs//actions → pack name.""" + prefix = "/api/packs/" + suffix = "/actions" + if not path.startswith(prefix) or not path.endswith(suffix): + return None + mid = path[len(prefix):-len(suffix)] + if not mid or "/" in mid: + return None + return mid + + def _match_pack_action(self, path: str) -> tuple[str, str] | None: + """POST /api/packs//action/ → (pack, action).""" + prefix = "/api/packs/" + if not path.startswith(prefix): + return None + rest = path[len(prefix):] + parts = rest.split("/") + if len(parts) != 3 or parts[1] != "action": + return None + return parts[0], parts[2] + + def _handle_get_packs(self) -> None: + if WORKSPACE_DIR is None: + return self._json(500, {"error": "workspace unset"}) + return self._json(200, filebrowser_packs.packs_payload(WORKSPACE_DIR)) + + def _handle_get_pack_actions(self, name: str) -> None: + if WORKSPACE_DIR is None: + return self._json(500, {"error": "workspace unset"}) + try: + return self._json(200, filebrowser_packs.actions_for_pack(WORKSPACE_DIR, name)) + except filebrowser_packs.PackError as e: + return self._json(e.status, {"error": str(e)}) + + def _handle_pack_action(self, pack: str, action: str) -> None: + if WORKSPACE_DIR is None: + return self._json(500, {"error": "workspace unset"}) + try: + body = self._read_json_body() + except ValueError as e: + return self._json(400, {"error": str(e)}) + rel = str(body.get("path") or "").strip() + try: + result = filebrowser_packs.dispatch_action( + WORKSPACE_DIR, pack, action, rel + ) + except filebrowser_packs.PackError as e: + return self._json(e.status, {"error": str(e)}) + except Exception as e: + return self._json(500, {"error": str(e)}) + return self._json(200, result) + + def _handle_packs_toggle(self) -> None: + if WORKSPACE_DIR is None: + return self._json(500, {"error": "workspace unset"}) + try: + body = self._read_json_body() + except ValueError as e: + return self._json(400, {"error": str(e)}) + name = str(body.get("name") or "").strip() + if "enabled" not in body: + return self._json(400, {"error": "enabled required"}) + enabled = bool(body.get("enabled")) + try: + return self._json( + 200, filebrowser_packs.set_enabled(WORKSPACE_DIR, name, enabled) + ) + except filebrowser_packs.PackError as e: + return self._json(e.status, {"error": str(e)}) + + def _handle_packs_install(self) -> None: + if WORKSPACE_DIR is None: + return self._json(500, {"error": "workspace unset"}) + try: + body = self._read_json_body() + except ValueError as e: + return self._json(400, {"error": str(e)}) + src = str( + body.get("source") or body.get("url") or body.get("path") or "" + ).strip() + if not src: + return self._json(400, {"error": "path or source/url required"}) + try: + if filebrowser_packs.looks_like_git_url(src): + result = filebrowser_packs.install_from_git(WORKSPACE_DIR, src) + else: + result = filebrowser_packs.install_from_path(WORKSPACE_DIR, src) + return self._json(200, result) + except filebrowser_packs.PackError as e: + return self._json(e.status, {"error": str(e)}) + + def _handle_packs_uninstall(self, qs: dict) -> None: + if WORKSPACE_DIR is None: + return self._json(500, {"error": "workspace unset"}) + name = (qs.get("name") or [""])[0].strip() + if not name: + return self._json(400, {"error": "name required"}) + try: + return self._json( + 200, filebrowser_packs.uninstall_pack(WORKSPACE_DIR, name) + ) + except filebrowser_packs.PackError as e: + return self._json(e.status, {"error": str(e)}) + + def _fs_workspace(self): + if WORKSPACE_DIR is None: + self._json(500, {"error": "workspace unset"}) + return None + return WORKSPACE_DIR + + def _handle_fs_stat(self, qs: dict) -> None: + ws = self._fs_workspace() + if ws is None: + return + rel = (qs.get("path") or [""])[0] + try: + return self._json(200, filebrowser_fs.stat(ws, rel)) + except filebrowser_fs.FileOpError as e: + return self._json(400, {"error": str(e)}) + + def _fs_mutate_result(self, op: str, fn) -> None: + """Run a filebrowser_fs mutation; rebuild viewer if wiki/ changed.""" + ws = self._fs_workspace() + if ws is None: + return + try: + body = self._read_json_body() + except ValueError as e: + return self._json(400, {"error": str(e)}) + try: + result = fn(ws, body) + except filebrowser_fs.FileOpError as e: + return self._json(400, {"error": str(e)}) + except Exception as e: + return self._json(500, {"error": str(e)}) + # Rebuild when wiki content may have changed. + touched = [] + if isinstance(result, dict): + for key in ("path", "trashed_from"): + v = result.get(key) + if isinstance(v, str): + touched.append(v) + src = body.get("path") + if isinstance(src, str): + touched.append(src) + if any(t.replace("\\", "/").startswith("wiki/") for t in touched): + self._rebuild() + payload = {"ok": True, **result} if isinstance(result, dict) else {"ok": True} + return self._json(200, payload) + + def _handle_fs_create(self) -> None: + def _op(ws, body): + rel = str(body.get("path") or "") + kind = str(body.get("kind") or "file") + content = body.get("content", "") + if content is None: + content = "" + if not isinstance(content, str): + raise filebrowser_fs.FileOpError("content must be a string") + path = filebrowser_fs.create(ws, rel, kind=kind, content=content) + return {"path": path, "op": "create"} + return self._fs_mutate_result("create", _op) + + def _handle_fs_mkdir(self) -> None: + def _op(ws, body): + path = filebrowser_fs.mkdir(ws, str(body.get("path") or "")) + return {"path": path, "op": "mkdir"} + return self._fs_mutate_result("mkdir", _op) + + def _handle_fs_rename(self) -> None: + def _op(ws, body): + path = filebrowser_fs.rename( + ws, str(body.get("path") or ""), str(body.get("to") or "") + ) + return {"path": path, "op": "rename"} + return self._fs_mutate_result("rename", _op) + + def _handle_fs_move(self) -> None: + def _op(ws, body): + path = filebrowser_fs.move( + ws, str(body.get("path") or ""), str(body.get("to") or "") + ) + return {"path": path, "op": "move"} + return self._fs_mutate_result("move", _op) + + def _handle_fs_delete(self) -> None: + def _op(ws, body): + rel = str(body.get("path") or "") + trashed_to = filebrowser_fs.delete(ws, rel) + return {"trashed_to": trashed_to, "trashed_from": rel, "op": "delete"} + return self._fs_mutate_result("delete", _op) + + def _handle_fs_duplicate(self) -> None: + def _op(ws, body): + path = filebrowser_fs.duplicate(ws, str(body.get("path") or "")) + return {"path": path, "op": "duplicate"} + return self._fs_mutate_result("duplicate", _op) + + + def _handle_get_curation_history(self) -> None: + """HistoryDoc for wiki-view replay (CE_PUBLIC_BASE-aware via rewrite).""" + if WORKSPACE_DIR is None: + return self._json( + 500, + { + "error": "workspace not configured", + "events": [], + "duration": 0, + "source": "error", + }, + ) + try: + doc = curation_history.history_payload(WORKSPACE_DIR, BUNDLE_DIR) + except Exception as e: + return self._json( + 500, + {"error": str(e), "events": [], "duration": 0, "source": "error"}, + ) + return self._json(200, doc) + + def _handle_get_split(self) -> None: + """Last workspace-partition status (Phase 2b+ split spike).""" + return self._json(200, {"last": SPLIT_LAST}) + + def _handle_post_split(self) -> None: + """Partition selected wiki pages into a new workspace outside source. + + Body: {target|name, move: [ref], copy: [ref], name?}. + If ``target`` is omitted, ``CE_SPLIT_HOME/`` is used when set. + Honors CE_PUBLIC_BASE via path rewrite. Sync for the spike; shells + may wrap async + workspace registry themselves. + """ + global SPLIT_LAST + if WORKSPACE_DIR is None: + return self._json(500, {"error": "workspace unset"}) + if isinstance(SPLIT_LAST, dict) and SPLIT_LAST.get("state") == "running": + return self._json(409, {"error": "a split is already running"}) + try: + body = self._read_json_body() + except ValueError as e: + return self._json(400, {"error": str(e)}) + move = [str(r).strip() for r in (body.get("move") or []) if str(r).strip()] + copy = [str(r).strip() for r in (body.get("copy") or []) if str(r).strip()] + if not move and not copy: + return self._json(400, {"error": "nothing selected"}) + name = wiki_partition.sanitize_name(str(body.get("name") or "")) + target_raw = str(body.get("target") or "").strip() + if not target_raw: + home = (os.environ.get("CE_SPLIT_HOME") or "").strip() + if not home or not name: + return self._json( + 400, + {"error": "target required (or set CE_SPLIT_HOME + name)"}, + ) + target_raw = str(Path(home).expanduser() / name) + elif not name: + name = wiki_partition.sanitize_name(Path(target_raw).name) + if not name: + return self._json(400, {"error": "name required"}) + target = Path(target_raw) + rec: dict = { + "state": "running", + "step": "starting", + "target": str(target), + "name": name, + "error": None, + } + SPLIT_LAST = rec + + def _progress(step: str) -> None: + rec["step"] = step + + try: + stats = wiki_partition.partition_workspace( + WORKSPACE_DIR, + target, + move, + copy, + name=name, + progress=_progress, + ) + rec.update(state="done", step="done", **stats) + # Persist split target in CE workspace registry (shells may also register). + try: + workspace_registry.register( + Path(stats["target"]), + set_active=False, + split={ + "name": stats.get("name") or name, + "source": str(WORKSPACE_DIR), + "target": stats["target"], + "stamp": stats.get("stamp") or "", + }, + ) + rec["registered"] = True + stats = {**stats, "registered": True} + except workspace_registry.RegistryError as reg_err: + rec["registered"] = False + rec["register_error"] = str(reg_err) + stats = { + **stats, + "registered": False, + "register_error": str(reg_err), + } + # Rebuild source viewer if pages were moved. + if stats.get("moved"): + self._rebuild() + return self._json(200, {"ok": True, "started": False, **stats}) + except wiki_partition.PartitionError as e: + rec.update(state="error", error=str(e), step="error") + return self._json(400, {"error": str(e)}) + except Exception as e: + rec.update(state="error", error=str(e), step="error") + return self._json(500, {"error": str(e)}) + + + def _handle_get_workspaces(self) -> None: + """CE workspace registry snapshot (Phase 2b+++++ registry slice).""" + return self._json(200, workspace_registry.payload()) + + def _handle_post_workspaces(self) -> None: + """Register a workspace path. Body: {path, set_active?, split?}.""" + try: + body = self._read_json_body() + except ValueError as e: + return self._json(400, {"error": str(e)}) + raw = str(body.get("path") or body.get("target") or "").strip() + if not raw: + return self._json(400, {"error": "path required"}) + set_active = bool(body.get("set_active") or body.get("active")) + split = body.get("split") if isinstance(body.get("split"), dict) else None + try: + data = workspace_registry.register( + Path(raw), set_active=set_active, split=split + ) + except workspace_registry.RegistryError as e: + return self._json(400, {"error": str(e)}) + return self._json(200, {"ok": True, **workspace_registry.payload(), "registered": data["paths"]}) + + def _handle_delete_workspaces(self, qs: dict) -> None: + raw = (qs.get("path") or [""])[0].strip() + if not raw: + return self._json(400, {"error": "path required"}) + try: + workspace_registry.unregister(Path(raw)) + except Exception as e: + return self._json(400, {"error": str(e)}) + return self._json(200, workspace_registry.payload()) + + def _handle_post_cm_export(self) -> None: + """Invoke curiosity-merge subgraph_export for selected pages. + + Body: { + pages|move|copy: [ref], target|name, include_vault?, dry_run?, + include_non_native?, label?, register? + }. + When ``target`` is omitted, ``CE_SPLIT_HOME/`` is used. + ``dry_run: true`` validates + returns planned argv without spawning CM. + """ + if WORKSPACE_DIR is None: + return self._json(500, {"error": "workspace unset"}) + try: + body = self._read_json_body() + except ValueError as e: + return self._json(400, {"error": str(e)}) + pages = [str(r).strip() for r in (body.get("pages") or []) if str(r).strip()] + move = [str(r).strip() for r in (body.get("move") or []) if str(r).strip()] + copy = [str(r).strip() for r in (body.get("copy") or []) if str(r).strip()] + pages = list(dict.fromkeys([*pages, *move, *copy])) + if not pages: + return self._json(400, {"error": "nothing selected"}) + name = wiki_partition.sanitize_name(str(body.get("name") or "")) + target_raw = str(body.get("target") or "").strip() + if not target_raw: + home = (os.environ.get("CE_SPLIT_HOME") or "").strip() + if not home or not name: + return self._json( + 400, + {"error": "target required (or set CE_SPLIT_HOME + name)"}, + ) + target_raw = str(Path(home).expanduser() / name) + dry_run = bool(body.get("dry_run")) + include_vault = str(body.get("include_vault") or "all") + include_non_native = body.get("include_non_native") + if include_non_native is None: + include_non_native = True + register = body.get("register") + if register is None: + register = not dry_run + label = body.get("label") + label_s = str(label).strip() if label else None + try: + result = cm_export.run_export( + WORKSPACE_DIR, + Path(target_raw), + pages, + include_vault=include_vault, + include_non_native=bool(include_non_native), + dry_run=dry_run, + register=bool(register), + label=label_s, + ) + except cm_export.CmExportError as e: + return self._json(400, {"error": str(e)}) + except Exception as e: + return self._json(500, {"error": str(e)}) + return self._json(200, result) + + def _handle_fs_reveal(self) -> None: + ws = self._fs_workspace() + if ws is None: + return + try: + body = self._read_json_body() + except ValueError as e: + return self._json(400, {"error": str(e)}) + rel = str(body.get("path") or "").strip() + if not rel: + return self._json(400, {"error": "path required"}) + try: + result = filebrowser_fs.reveal(ws, rel) + except filebrowser_fs.FileOpError as e: + return self._json(400, {"error": str(e)}) + return self._json(200, {"ok": True, **result}) + + def _handle_fs_open_external(self) -> None: + ws = self._fs_workspace() + if ws is None: + return + try: + body = self._read_json_body() + except ValueError as e: + return self._json(400, {"error": str(e)}) + rel = str(body.get("path") or "").strip() + if not rel: + return self._json(400, {"error": "path required"}) + try: + result = filebrowser_fs.open_external(ws, rel) + except filebrowser_fs.FileOpError as e: + return self._json(400, {"error": str(e)}) + return self._json(200, {"ok": True, **result}) + + def _handle_ingest_from_upload(self) -> None: + """Multipart ``file`` → vault/raw/ + ingest-run queue (Switchbay shape).""" + if WORKSPACE_DIR is None: + return self._json(500, {"error": "workspace unset"}) + ctype = self.headers.get("Content-Type", "") + if "multipart/form-data" not in ctype: + return self._json(400, {"error": "multipart/form-data required"}) + length = int(self.headers.get("Content-Length", "0") or "0") + if length <= 0: + return self._json(400, {"error": "empty body"}) + if length > filebrowser_ingest.MAX_BYTES + 64 * 1024: + return self._json(413, {"error": "file too large (>50 MB)"}) + body = self.rfile.read(length) + head = (f"Content-Type: {ctype}\r\n\r\n").encode("ascii") + msg = BytesParser(policy=default_email_policy).parsebytes(head + body) + if not msg.is_multipart(): + return self._json(400, {"error": "not multipart"}) + filename = None + data = None + for part in msg.iter_parts(): + cd = part.get("Content-Disposition", "") + if "filename=" not in cd: + continue + filename = part.get_filename() or "" + data = part.get_payload(decode=True) + if filename and data is not None: + break + if not filename or data is None: + return self._json(400, {"error": "no `file` field"}) + try: + result = filebrowser_ingest.stage_bytes(WORKSPACE_DIR, filename, data) + except filebrowser_ingest.IngestError as e: + return self._json(e.status, {"error": str(e)}) + return self._json(200, result) + + def _handle_ingest_from_path(self) -> None: + if WORKSPACE_DIR is None: + return self._json(500, {"error": "workspace unset"}) + try: + body = self._read_json_body() + except ValueError as e: + return self._json(400, {"error": str(e)}) + src = str(body.get("path") or "").strip() + if not src: + return self._json(400, {"error": "path required"}) + try: + result = filebrowser_ingest.stage_path(WORKSPACE_DIR, src) + except filebrowser_ingest.IngestError as e: + return self._json(e.status, {"error": str(e)}) + return self._json(200, result) + + + def _handle_import_obsidian(self) -> None: + """Import Obsidian vault folder or zip into workspace wiki/. + + JSON ``{path, target?, register?, force?}`` for an on-disk vault/zip, + or multipart form with a ``file`` zip field (optional form fields + ``target`` / ``register`` / ``force``). Honours CE_PUBLIC_BASE via + path rewrite. Mapping: Obsidian vault root -> wiki/. + """ + if WORKSPACE_DIR is None: + return self._json(500, {"error": "workspace unset"}) + ctype = self.headers.get("Content-Type", "") + try: + if "multipart/form-data" in ctype: + length = int(self.headers.get("Content-Length", "0") or "0") + if length <= 0: + return self._json(400, {"error": "empty body"}) + if length > obsidian_import.MAX_ZIP_BYTES + 64 * 1024: + return self._json(413, {"error": "zip too large"}) + body = self.rfile.read(length) + head = (f"Content-Type: {ctype}\r\n\r\n").encode("ascii") + msg = BytesParser(policy=default_email_policy).parsebytes(head + body) + if not msg.is_multipart(): + return self._json(400, {"error": "not multipart"}) + filename = None + data = None + target = None + register = False + force = False + for part in msg.iter_parts(): + cd = part.get("Content-Disposition", "") + name = part.get_param("name", header="Content-Disposition") + if name in ("target", "path_target"): + raw = part.get_payload(decode=True) or b"" + target = raw.decode("utf-8", errors="replace").strip() or None + elif name == "register": + raw = (part.get_payload(decode=True) or b"").decode().strip().lower() + register = raw in ("1", "true", "yes", "on") + elif name == "force": + raw = (part.get_payload(decode=True) or b"").decode().strip().lower() + force = raw in ("1", "true", "yes", "on") + elif "filename=" in cd: + filename = part.get_filename() or "vault.zip" + data = part.get_payload(decode=True) + if data is None: + return self._json(400, {"error": "no zip `file` field"}) + result = obsidian_import.import_vault( + workspace=WORKSPACE_DIR, + target=target, + zip_bytes=data, + zip_filename=filename, + register=register, + force=force, + ) + else: + body = self._read_json_body() + src = str(body.get("path") or "").strip() + if not src: + return self._json(400, {"error": "path required (or multipart zip)"}) + target = body.get("target") + target_s = str(target).strip() if target else None + register = bool(body.get("register") or False) + force = bool(body.get("force") or False) + result = obsidian_import.import_vault( + src, + workspace=WORKSPACE_DIR, + target=target_s, + register=register, + force=force, + ) + except ValueError as e: + return self._json(400, {"error": str(e)}) + except obsidian_import.ObsidianImportError as e: + return self._json(e.status, {"error": str(e)}) + return self._json(200, result) + def _handle_get_page(self, qs: dict) -> None: rel = (qs.get("path") or [""])[0] try: @@ -289,9 +1043,15 @@ def main() -> None: print(f"wiki dir missing: {WIKI_DIR}", file=sys.stderr) sys.exit(1) + base = public_base.public_base() sys.stderr.write( f"viewer-server: serving {BUNDLE_DIR} on http://127.0.0.1:{port}\n" ) + if base: + sys.stderr.write( + f"viewer-server: CE_PUBLIC_BASE={base} " + f"(public URLs under http://127.0.0.1:{port}{base}/)\n" + ) sys.stderr.write( f"viewer-server: edits go to {WIKI_DIR}, uploads to {VAULT_RAW_DIR}\n" ) diff --git a/skills/curiosity-engine/scripts/wiki_partition.py b/skills/curiosity-engine/scripts/wiki_partition.py new file mode 100644 index 0000000..7cc850d --- /dev/null +++ b/skills/curiosity-engine/scripts/wiki_partition.py @@ -0,0 +1,330 @@ +"""wiki_partition.py — CE-owned workspace partition (split) spike. + +Charter: CE owns page move/copy + wiki partition. Switchbay may still own +workspace registry / tab chrome after a split. + +This module is intentionally self-contained (no curiosity-merge dependency): + 1. Resolve page refs → wiki/*.md + 2. Copy selected pages (+ transitive vault: citations + wiki-relative figures) + into a NEW target workspace outside the source tree + 3. Write `.curator/splits/` manifests on BOTH sides + 4. MOVE policy: relocate source pages under wiki/.deleted// (recoverable) + +Shells call POST /api/split on the CE viewer; they register the new workspace +themselves. Feature-flag dual-stack until umbrella parity checklist is green. +""" +from __future__ import annotations + +import json +import re +import shutil +import time +from pathlib import Path +from typing import Callable + +from naming import CITATION_RE + +Progress = Callable[[str], None] + +_IMG_EMBED_RE = re.compile(r"!\[\[([^\]]+)\]\]") +_MD_IMAGE_RE = re.compile(r"!\[[^\]]*\]\(([^)]+)\)") +_FIGURE_SUFFIXES = {".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp"} +_SAFE_NAME_RE = re.compile(r"[^A-Za-z0-9_.-]+") + + +class PartitionError(Exception): + """User-visible partition failure.""" + + +def sanitize_name(name: str) -> str: + cleaned = _SAFE_NAME_RE.sub("-", (name or "").strip()).strip("-.") + return cleaned + + +def _wiki_pages(wiki_dir: Path) -> list[Path]: + if not wiki_dir.is_dir(): + return [] + return [ + p + for p in wiki_dir.rglob("*.md") + if not any(part.startswith(".") for part in p.relative_to(wiki_dir).parts) + and "_suspect" not in p.parts + ] + + +def match_page_ref(ref: str, all_pages: list[Path], wiki_dir: Path) -> Path | None: + """Resolve bare stem or wiki-relative path (optional .md) case-insensitively.""" + target = ref.strip().lower().replace(" ", "-") + if target.endswith(".md"): + target = target[: -len(".md")] + if not target or ".." in Path(target).parts: + return None + for p in all_pages: + rel = str(p.relative_to(wiki_dir)).lower().replace(".md", "") + if p.stem.lower() == target or rel == target: + return p + return None + + +def validate_refs(workspace: Path, refs: list[str]) -> list[str]: + """Return refs that do not resolve to a wiki page.""" + wiki = workspace / "wiki" + pages = _wiki_pages(wiki) + return [r for r in refs if match_page_ref(r, pages, wiki) is None] + + +def resolve_pages(workspace: Path, refs: list[str]) -> list[Path]: + wiki = workspace / "wiki" + pages = _wiki_pages(wiki) + out: list[Path] = [] + seen: set[Path] = set() + for ref in refs: + hit = match_page_ref(ref, pages, wiki) + if hit is None: + raise PartitionError(f"not a wiki page: {ref}") + if hit not in seen: + out.append(hit) + seen.add(hit) + return out + + +def _safe_target(workspace: Path, target: Path) -> Path: + ws = workspace.resolve() + dest = target.expanduser().resolve() + if dest.exists(): + raise PartitionError(f"target already exists: {dest}") + try: + dest.relative_to(ws) + raise PartitionError(f"refusing to write partition inside workspace: {dest}") + except ValueError: + pass + if dest == ws: + raise PartitionError("target cannot be the source workspace") + return dest + + +def _collect_vault(pages: list[Path], vault_dir: Path) -> list[Path]: + if not vault_dir.is_dir(): + return [] + cited: set[str] = set() + for p in pages: + try: + text = p.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + for m in CITATION_RE.finditer(text): + cited.add(m.group(1).strip()) + out: list[Path] = [] + seen: set[Path] = set() + root = vault_dir.resolve() + for rel in sorted(cited): + if not rel or ".." in Path(rel).parts or Path(rel).is_absolute(): + continue + candidate = (vault_dir / rel).resolve() + try: + candidate.relative_to(root) + except ValueError: + continue + if candidate.is_file() and candidate not in seen: + out.append(candidate) + seen.add(candidate) + return out + + +def _normalise_figure_path(path: str) -> str: + path = path.strip().split("|", 1)[0].strip() + if path.startswith("figures/_assets/"): + return path + if path.startswith("_assets/"): + return "figures/" + path + if "/" not in path: + return "figures/_assets/" + path + return path + + +def _collect_figures(pages: list[Path], wiki_dir: Path) -> list[Path]: + refs: set[str] = set() + for p in pages: + try: + text = p.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + for m in _IMG_EMBED_RE.finditer(text): + refs.add(_normalise_figure_path(m.group(1))) + for m in _MD_IMAGE_RE.finditer(text): + refs.add(_normalise_figure_path(m.group(1))) + out: list[Path] = [] + seen: set[Path] = set() + root = wiki_dir.resolve() + for rel in sorted(refs): + if not rel or ".." in Path(rel).parts or Path(rel).is_absolute(): + continue + suffix = Path(rel).suffix.lower() + if suffix and suffix not in _FIGURE_SUFFIXES: + continue + candidate = (wiki_dir / rel).resolve() + try: + candidate.relative_to(root) + except ValueError: + continue + if candidate.is_file() and candidate not in seen: + out.append(candidate) + seen.add(candidate) + return out + + +def _copy_tree_files(files: list[Path], src_root: Path, dest_root: Path) -> list[str]: + rels: list[str] = [] + for f in files: + rel = f.relative_to(src_root) + dst = dest_root / rel + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(f, dst) + rels.append(str(rel).replace("\\", "/")) + return sorted(rels) + + +def _page_ref(page: Path, wiki_dir: Path) -> str: + return str(page.relative_to(wiki_dir)).replace("\\", "/")[: -len(".md")] + + +def _write_manifests( + workspace: Path, + target: Path, + name: str, + move: list[str], + copy: list[str], + stamp: str, +) -> None: + body = { + "name": name, + "created": stamp, + "source": str(workspace), + "target": str(target), + "moved": move, + "copied": copy, + "schema": "ce-wiki-partition/1", + } + for side, extra in ((workspace, {"role": "source"}), (target, {"role": "target"})): + d = side / ".curator" / "splits" + d.mkdir(parents=True, exist_ok=True) + (d / f"{stamp}-{name}.json").write_text( + json.dumps({**body, **extra}, indent=2) + "\n", + encoding="utf-8", + ) + + +def _prune_moved( + workspace: Path, move_pages: list[Path], stamp: str +) -> tuple[list[str], list[str]]: + """Relocate moved pages under wiki/.deleted// (recoverable).""" + wiki = workspace / "wiki" + pruned: list[str] = [] + errors: list[str] = [] + trash_root = wiki / ".deleted" / stamp + for page in move_pages: + ref = _page_ref(page, wiki) + try: + rel = page.relative_to(wiki) + dst = trash_root / rel + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(page), str(dst)) + pruned.append(ref) + except OSError as e: + errors.append(f"{ref}: {e}") + return pruned, errors + + +def partition_workspace( + workspace: Path, + target: Path, + move: list[str], + copy: list[str], + *, + name: str | None = None, + progress: Progress | None = None, +) -> dict: + """Full pipeline. On failure before prune, removes the half-built target. + + Returns stats dict. Source is untouched until export + manifests succeed; + MOVE prune runs last via recoverable .deleted relocate. + """ + def _progress(step: str) -> None: + if progress: + progress(step) + + refs = list(dict.fromkeys([*(move or []), *(copy or [])])) + if not refs: + raise PartitionError("nothing selected") + workspace = workspace.resolve() + if not (workspace / "wiki").is_dir(): + raise PartitionError(f"workspace has no wiki/: {workspace}") + dest = _safe_target(workspace, target) + label = sanitize_name(name or dest.name) + if not label: + raise PartitionError("name required") + + move_u = list(dict.fromkeys(move or [])) + copy_u = list(dict.fromkeys(copy or [])) + # A ref listed in both → MOVE wins (export once, then prune). + copy_u = [r for r in copy_u if r not in set(move_u)] + + try: + _progress(f"resolving {len(refs)} pages") + all_pages = resolve_pages(workspace, refs) + move_pages = resolve_pages(workspace, move_u) if move_u else [] + wiki = workspace / "wiki" + vault = workspace / "vault" + + _progress("creating target layout") + dest.mkdir(parents=True, exist_ok=False) + (dest / "wiki").mkdir() + (dest / "vault").mkdir() + (dest / ".curator").mkdir() + + _progress("copying wiki pages") + pages_rel = _copy_tree_files(all_pages, wiki, dest / "wiki") + + _progress("copying vault citations") + vault_files = _collect_vault(all_pages, vault) + vault_rel = ( + _copy_tree_files(vault_files, vault, dest / "vault") if vault_files else [] + ) + + _progress("copying figure embeds") + figures = _collect_figures(all_pages, wiki) + figures_rel = ( + _copy_tree_files(figures, wiki, dest / "wiki") if figures else [] + ) + + stamp = time.strftime("%Y%m%d-%H%M%S") + move_refs = [_page_ref(p, wiki) for p in move_pages] + copy_refs = [ + _page_ref(p, wiki) + for p in all_pages + if _page_ref(p, wiki) not in set(move_refs) + ] + _progress("writing split manifests") + _write_manifests(workspace, dest, label, move_refs, copy_refs, stamp) + + except Exception: + if dest.exists(): + shutil.rmtree(dest, ignore_errors=True) + raise + + _progress(f"pruning {len(move_pages)} moved pages") + pruned, prune_errors = _prune_moved(workspace, move_pages, stamp) + + return { + "ok": True, + "name": label, + "target": str(dest), + "exported": len(pages_rel), + "pages": pages_rel, + "moved": len(pruned), + "copied": len(copy_refs), + "vault": len(vault_rel), + "figures": len(figures_rel), + "prune_errors": prune_errors, + "stamp": stamp, + } diff --git a/skills/curiosity-engine/scripts/workspace_registry.py b/skills/curiosity-engine/scripts/workspace_registry.py new file mode 100644 index 0000000..f0f880c --- /dev/null +++ b/skills/curiosity-engine/scripts/workspace_registry.py @@ -0,0 +1,217 @@ +"""workspace_registry.py — CE-owned workspace registry persistence. + +Charter: shells (Switchbay/okbay) still own tab chrome and may keep their +own XDG registries. CE persists a sibling registry so split targets are +recorded even when no shell is seated, and so shells can sync from +``GET /api/workspaces``. + +Storage (first hit wins for reads; writes go to the resolved path): + 1. ``$CE_WORKSPACE_REGISTRY`` — explicit file path + 2. ``$XDG_CONFIG_HOME/curiosity-engine/workspaces.json`` + 3. ``~/.config/curiosity-engine/workspaces.json`` + +Shape (``ce-workspace-registry/1``):: + + { + "schema": "ce-workspace-registry/1", + "paths": ["/abs/a", "/abs/b"], + "active": "/abs/a" | null, + "splits": [ + {"name": "child", "source": "/abs/a", "target": "/abs/b", + "stamp": "...", "registered_at": "..."} + ] + } + +Sandbox: paths must resolve under ``$HOME`` (or ``$CE_WORKSPACE_HOME`` when +set). Entries outside the sandbox are dropped on load. +""" +from __future__ import annotations + +import json +import os +import tempfile +import time +from pathlib import Path +from typing import Any + + +SCHEMA = "ce-workspace-registry/1" + + +class RegistryError(Exception): + """User-visible registry failure.""" + + +def _home_root() -> Path: + override = (os.environ.get("CE_WORKSPACE_HOME") or "").strip() + if override: + return Path(override).expanduser().resolve() + return Path.home().resolve() + + +def is_within_sandbox(path: Path) -> bool: + try: + target = Path(path).expanduser().resolve() + target.relative_to(_home_root()) + return True + except (ValueError, OSError): + return False + + +def registry_path() -> Path: + override = (os.environ.get("CE_WORKSPACE_REGISTRY") or "").strip() + if override: + return Path(override).expanduser().resolve() + xdg = (os.environ.get("XDG_CONFIG_HOME") or "").strip() + base = Path(xdg).expanduser() if xdg else Path.home() / ".config" + return (base / "curiosity-engine" / "workspaces.json").resolve() + + +def _empty() -> dict[str, Any]: + return {"schema": SCHEMA, "paths": [], "active": None, "splits": []} + + +def load() -> dict[str, Any]: + p = registry_path() + if not p.is_file(): + return _empty() + try: + data = json.loads(p.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return _empty() + if not isinstance(data, dict): + return _empty() + raw_paths = data.get("paths") or [] + if not isinstance(raw_paths, list): + raw_paths = [] + safe_paths: list[str] = [] + for entry in raw_paths: + s = str(entry) + if is_within_sandbox(Path(s)): + safe_paths.append(str(Path(s).expanduser().resolve())) + # de-dupe preserving order + seen: set[str] = set() + paths: list[str] = [] + for s in safe_paths: + if s not in seen: + seen.add(s) + paths.append(s) + active = data.get("active") + active_s = str(active) if active else None + if active_s and not is_within_sandbox(Path(active_s)): + active_s = paths[0] if paths else None + elif active_s: + active_s = str(Path(active_s).expanduser().resolve()) + splits_raw = data.get("splits") or [] + splits: list[dict[str, Any]] = [] + if isinstance(splits_raw, list): + for item in splits_raw: + if not isinstance(item, dict): + continue + target = str(item.get("target") or "") + if not target or not is_within_sandbox(Path(target)): + continue + splits.append( + { + "name": str(item.get("name") or Path(target).name), + "source": str(item.get("source") or ""), + "target": str(Path(target).expanduser().resolve()), + "stamp": str(item.get("stamp") or ""), + "registered_at": str(item.get("registered_at") or ""), + } + ) + return { + "schema": SCHEMA, + "paths": paths, + "active": active_s, + "splits": splits, + } + + +def _atomic_write(path: Path, data: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + raw = json.dumps(data, indent=2) + "\n" + fd, tmp = tempfile.mkstemp( + dir=str(path.parent), prefix=".workspaces-", suffix=".tmp" + ) + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + f.write(raw) + f.flush() + os.fsync(f.fileno()) + os.replace(tmp, path) + except Exception: + try: + os.unlink(tmp) + except OSError: + pass + raise + + +def save(data: dict[str, Any]) -> None: + body = { + "schema": SCHEMA, + "paths": list(data.get("paths") or []), + "active": data.get("active"), + "splits": list(data.get("splits") or []), + } + _atomic_write(registry_path(), body) + + +def register( + path: Path, + *, + set_active: bool = False, + split: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Add ``path`` to the registry. Optionally record a split provenance row.""" + if not is_within_sandbox(path): + raise RegistryError( + f"workspaces must live inside {_home_root()}; refusing {path}" + ) + resolved = Path(path).expanduser().resolve() + data = load() + s = str(resolved) + if s not in data["paths"]: + data["paths"].append(s) + if set_active: + data["active"] = s + if split: + row = { + "name": str(split.get("name") or resolved.name), + "source": str(split.get("source") or ""), + "target": s, + "stamp": str(split.get("stamp") or ""), + "registered_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + } + # Replace prior row for same target. + data["splits"] = [ + r for r in data["splits"] if str(r.get("target")) != s + ] + data["splits"].append(row) + save(data) + return data + + +def unregister(path: Path) -> dict[str, Any]: + resolved = str(Path(path).expanduser().resolve()) + data = load() + data["paths"] = [p for p in data["paths"] if p != resolved] + data["splits"] = [r for r in data["splits"] if str(r.get("target")) != resolved] + if data.get("active") == resolved: + data["active"] = data["paths"][0] if data["paths"] else None + save(data) + return data + + +def payload() -> dict[str, Any]: + """JSON body for GET /api/workspaces.""" + data = load() + return { + "ok": True, + "schema": data["schema"], + "path": str(registry_path()), + "paths": data["paths"], + "active": data["active"], + "splits": data["splits"], + } diff --git a/skills/curiosity-engine/template/wiki-view/index.html b/skills/curiosity-engine/template/wiki-view/index.html index d0b4fa7..3038e64 100644 --- a/skills/curiosity-engine/template/wiki-view/index.html +++ b/skills/curiosity-engine/template/wiki-view/index.html @@ -26,6 +26,10 @@ {{WORKSPACE}} +
+ + +
@@ -44,6 +48,16 @@
+ +