From 40651d4a9c158dfe830ed4973228a0698525c352 Mon Sep 17 00:00:00 2001 From: HelloThisWorld Date: Sun, 19 Jul 2026 01:31:11 +0800 Subject: [PATCH] OpenMind v2 Phase 1: tool-first runtime foundation Turn OpenMind from a UI-centric application into a headless engineering knowledge runtime that the CLI, the MCP server, the FastAPI app and the tests all drive through one shared bootstrap. This is an architectural foundation; it implements none of the later v2 enterprise knowledge features. Runtime and services Add openmind/runtime.py as the composition root: an idempotent, ordered bootstrap (ensure dirs -> open database -> migrate to head -> build services) that every adapter shares. Previously three call sites initialized the process differently, and jobs.start_worker() ran only under FastAPI, so a job enqueued from any other process stayed queued forever. Worker startup is opt-in per surface, so doctor, status and export never spawn a background thread. Add openmind/services/ (workspace, ingest, job, export, health, container), openmind/domain/ (typed errors and boundary types) and openmind/ports/ (three narrow protocols with real test seams: the workspace repository, the job reader/engine, and a clock that makes bounded waits testable in milliseconds). Services never import FastAPI and return plain dicts or dataclasses. Database migrations Replace the untracked CREATE TABLE IF NOT EXISTS block with a checksummed, transactional migration runner and a schema_migrations ledger. A legacy database is baselined, never recreated. Editing an already-applied migration fails loudly. DDL is transactional via explicit BEGIN/COMMIT, since sqlite3 would otherwise run it in autocommit and survive a rollback. CLI Add openmind/cli.py: doctor, init, add, ingest, status, export, mcp serve and serve. Under --json exactly one object goes to stdout with diagnostics on stderr, and exit codes 0-5 are stable and documented. serve stays loopback-only unless --allow-non-loopback is passed explicitly. Adapters Route the project, path, template, ingest, job, lifecycle and health operations in main.py through the services. Add mcp_server.create_mcp_server(runtime) and remove the import-time db.init_db() side effect; the nine MCP tools keep their names and contracts. Compatibility Verified by probing the refactored routes on main and on this branch through a git worktree and diffing the transcripts: across 29 probes there are zero status-code differences and zero keys removed. The 14 differences are purely additive - /api/health gains version and schema_version, and error bodies gain an "error" key alongside the unchanged "detail". One deliberate fix: DELETE /projects/{id}/paths returned None from a handler annotated -> Dict[str, Any] for an unknown project, which FastAPI's response validation turned into a 500 ResponseValidationError. It now returns 404 like every sibling route. Version Add openmind/version.py (1.1.0-dev), reported by --version, doctor, the FastAPI app and the artifact generator. Deliberately not 2.0.0: none of the v2 enterprise features exist yet. The artifact schema stays frozen at 1.1.0. Tests and CI Add verify_migrations, verify_runtime, verify_services, verify_cli and verify_adapters (374 new checks), plus scripts/run_acceptance.py - an isolated, offline runner in which a skipped core script counts as a failure and every tests/verify_*.py must be registered, so a new test cannot silently go unrun. Expand CI into a dependency-free artifact-contract job, an Ubuntu full gate, a three-OS smoke matrix, and an optional pinned-ref skill-verification job. All 24 acceptance scripts pass; no test invokes a paid or public model API. Docs Add docs/v2/phase-1-core-foundation.md, docs/cli.md and docs/database-migrations.md, and give the README a tool-first quick start. --- .github/workflows/ci.yml | 272 ++++++++ README.md | 143 +++- docs/cli.md | 276 ++++++++ docs/database-migrations.md | 207 ++++++ docs/v2/phase-1-core-foundation.md | 477 ++++++++++++++ openmind/__init__.py | 4 +- openmind/cli.py | 613 ++++++++++++++++++ openmind/db.py | 116 +--- openmind/domain/__init__.py | 17 + openmind/domain/errors.py | 135 ++++ openmind/domain/types.py | 149 +++++ openmind/main.py | 174 ++--- openmind/mcp_server.py | 104 ++- openmind/migrations/__init__.py | 21 + openmind/migrations/runner.py | 335 ++++++++++ openmind/migrations/versions/__init__.py | 10 + .../migrations/versions/v0001_baseline.py | 85 +++ .../versions/v0002_paths_sidecar.py | 40 ++ openmind/ports/__init__.py | 22 + openmind/ports/job_repository.py | 72 ++ openmind/ports/runtime_ports.py | 61 ++ openmind/ports/workspace_repository.py | 42 ++ openmind/runtime.py | 166 +++++ openmind/services/__init__.py | 50 ++ openmind/services/export_service.py | 67 ++ openmind/services/health_service.py | 299 +++++++++ openmind/services/ingest_service.py | 108 +++ openmind/services/job_service.py | 152 +++++ openmind/services/service_container.py | 70 ++ openmind/services/workspace_service.py | 211 ++++++ openmind/version.py | 23 + package.json | 4 +- scripts/run_acceptance.py | 294 +++++++++ tests/verify_adapters.py | 341 ++++++++++ tests/verify_cli.py | 354 ++++++++++ tests/verify_migrations.py | 255 ++++++++ tests/verify_runtime.py | 142 ++++ tests/verify_services.py | 375 +++++++++++ 38 files changed, 6084 insertions(+), 202 deletions(-) create mode 100644 docs/cli.md create mode 100644 docs/database-migrations.md create mode 100644 docs/v2/phase-1-core-foundation.md create mode 100644 openmind/cli.py create mode 100644 openmind/domain/__init__.py create mode 100644 openmind/domain/errors.py create mode 100644 openmind/domain/types.py create mode 100644 openmind/migrations/__init__.py create mode 100644 openmind/migrations/runner.py create mode 100644 openmind/migrations/versions/__init__.py create mode 100644 openmind/migrations/versions/v0001_baseline.py create mode 100644 openmind/migrations/versions/v0002_paths_sidecar.py create mode 100644 openmind/ports/__init__.py create mode 100644 openmind/ports/job_repository.py create mode 100644 openmind/ports/runtime_ports.py create mode 100644 openmind/ports/workspace_repository.py create mode 100644 openmind/runtime.py create mode 100644 openmind/services/__init__.py create mode 100644 openmind/services/export_service.py create mode 100644 openmind/services/health_service.py create mode 100644 openmind/services/ingest_service.py create mode 100644 openmind/services/job_service.py create mode 100644 openmind/services/service_container.py create mode 100644 openmind/services/workspace_service.py create mode 100644 openmind/version.py create mode 100644 scripts/run_acceptance.py create mode 100644 tests/verify_adapters.py create mode 100644 tests/verify_cli.py create mode 100644 tests/verify_migrations.py create mode 100644 tests/verify_runtime.py create mode 100644 tests/verify_services.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c35665a..3f6018d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,7 +6,24 @@ on: pull_request: workflow_dispatch: +# Deterministic and offline for every job: the hashing embedder instead of a +# downloaded model, and no enrichment or source-link egress. No job in this +# workflow may call a paid API or a public LLM endpoint. +env: + OPENMIND_EMBED_OFFLINE: "1" + OPENMIND_EMBED_DEVICE: cpu + OPENMIND_INGEST_FREE_GPU: "0" + OPENMIND_ENRICH_EGRESS: "0" + OPENMIND_SOURCELINK_EGRESS: "0" + PYTHONUTF8: "1" + PYTHONIOENCODING: utf-8 + jobs: + # --------------------------------------------------------------------------- + # The external integration contract, proven WITHOUT installing anything. + # Keeping this job dependency-free IS the guarantee: an external consumer + # (e.g. open-mind-mcp-server) can run the exporter with nothing but Python. + # --------------------------------------------------------------------------- artifact-contract: name: Artifact export contract (stdlib-only) runs-on: ubuntu-latest @@ -31,3 +48,258 @@ jobs: # that external consumers (e.g. open-mind-mcp-server) depend on. - name: Verify artifact export contract run: python tests/verify_artifacts.py + + # The CLI must reach that same contract with no dependencies installed. + - name: CLI export reproduces the contract (stdlib-only) + run: | + python -m openmind.cli export \ + --repo ./fixtures/sample-repo \ + --output ./.ci-openmind \ + --json > export.json + python - <<'PY' + import json, pathlib + summary = json.load(open("export.json")) + assert summary["ok"] is True, summary + assert summary["schemaVersion"] == "1.1.0", summary["schemaVersion"] + manifest = json.load(open(".ci-openmind/manifest.json")) + assert manifest["schemaVersion"] == "1.1.0", manifest + assert manifest["generator"]["name"] == "open-mind", manifest + for name in ("glossary.json", "architecture.json", "flows.json", + "source-index.json", "metadata.json"): + assert pathlib.Path(".ci-openmind", name).is_file(), name + print("artifact contract OK:", manifest["schemaVersion"]) + PY + + # --------------------------------------------------------------------------- + # The full gate. Everything that must be green before main moves. + # --------------------------------------------------------------------------- + full-gate: + name: Full gate (Ubuntu) + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Byte-compile package + run: python -m compileall -q openmind + + # Cheap, high-signal checks first, so an obvious breakage fails in + # seconds instead of after the whole suite. + - name: Migration tests + run: python scripts/run_acceptance.py --only verify_migrations + + - name: CLI tests + run: python scripts/run_acceptance.py --only verify_cli + + - name: MCP smoke test + run: | + python - <<'PY' + import asyncio, os, tempfile + os.environ["OPENMIND_DATA_DIR"] = tempfile.mkdtemp() + os.environ["OPENMIND_MACHINE_DIR"] = tempfile.mkdtemp() + from openmind import mcp_server + from openmind.runtime import get_runtime + + expected = {"search", "route", "dispatch", "get_glossary", + "find_similar_cases", "save_case", "get_doc", + "propose_fix", "apply_fix"} + server = mcp_server.create_mcp_server(get_runtime()) + names = {t.name for t in asyncio.run(server.list_tools())} + assert names == expected, f"MCP tool drift: {names ^ expected}" + assert server.name == "open-mind", server.name + print("MCP smoke OK:", len(names), "tools") + PY + + - name: Skill bridge smoke test + run: | + python - <<'PY' + import json, subprocess, sys + proc = subprocess.Popen( + [sys.executable, "-m", "openmind.skill_bridge", + "--root", "./fixtures/sample-repo"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True, bufsize=1) + try: + ready = json.loads(proc.stdout.readline()) + assert ready.get("ready") is True, ready + assert ready.get("files", 0) > 0, ready + proc.stdin.write(json.dumps( + {"id": 1, "op": "route", "arg": "what does ISR mean"}) + "\n") + proc.stdin.flush() + reply = json.loads(proc.stdout.readline()) + assert reply.get("ok") is True, reply + assert reply["result"]["capability"], reply + print("skill bridge smoke OK:", ready["files"], "files") + finally: + proc.stdin.close() + proc.wait(timeout=30) + PY + + # The whole supported suite. Every tests/verify_*.py must be registered in + # the runner's manifest, so a new test cannot go silently unrun, and a + # skipped core script is counted as a failure rather than a pass. + - name: Acceptance suite (core tier) + run: python scripts/run_acceptance.py --json > acceptance.json + + - name: Upload acceptance report + if: always() + uses: actions/upload-artifact@v4 + with: + name: acceptance-report + path: acceptance.json + if-no-files-found: ignore + + # --------------------------------------------------------------------------- + # Cross-platform smoke. Deliberately light: no embedding-heavy ingest on three + # operating systems, just proof that the package imports and the tool-first + # entry points work everywhere. + # --------------------------------------------------------------------------- + smoke: + name: Smoke (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Package imports + run: python -c "import openmind; print('openmind', openmind.__version__)" + + - name: CLI --version + run: python -m openmind.cli --version + + - name: CLI doctor --json + shell: bash + run: | + python -m openmind.cli doctor --json > doctor.json + python - <<'PY' + import json + report = json.load(open("doctor.json")) + names = {c["name"] for c in report["checks"]} + for required in ("data_dir", "database", "migrations", "project_dirs", + "vectorstore", "embeddings", "mcp", "runtime_version"): + assert required in names, f"missing check: {required}" + # A missing optional local model must never fail doctor. + assert report["ok"] is True, report + print("doctor OK:", report["version"], report["status"]) + PY + + - name: Artifact export contract + run: python tests/verify_artifacts.py + + - name: Skill bridge startup + shell: bash + run: | + python - <<'PY' + import json, subprocess, sys + proc = subprocess.Popen( + [sys.executable, "-m", "openmind.skill_bridge", + "--root", "./fixtures/sample-repo"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True, bufsize=1) + try: + ready = json.loads(proc.stdout.readline()) + assert ready.get("ready") is True, ready + print("skill bridge startup OK:", ready["files"], "files") + finally: + proc.stdin.close() + proc.wait(timeout=30) + PY + + # --------------------------------------------------------------------------- + # Optional: run the independent Agent Skill Verification Template against the + # REAL OpenMind implementation via the skill bridge. + # + # This job never downloads an unpinned repository. It runs only when the + # template is provided deliberately, through the AGENT_SKILL_VERIFICATION_REF + # repository variable in explicit 'owner/repo@ref' form. Without it the job + # reports "not configured" and does NOT claim the verification ran. + # + # A future phase will use Agent Skill Forge to generate the v2 workflow + # Skills; this job is the seam for running them, not those Skills. + # --------------------------------------------------------------------------- + skill-verification: + name: Agent Skill Verification (optional) + runs-on: ubuntu-latest + if: github.event_name == 'workflow_dispatch' + timeout-minutes: 30 + steps: + - name: Checkout OpenMind + uses: actions/checkout@v4 + with: + path: open-mind + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: open-mind/requirements.txt + + - name: Install dependencies + working-directory: open-mind + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Check out the verification template (pinned ref only) + id: template + env: + TEMPLATE_REF: ${{ vars.AGENT_SKILL_VERIFICATION_REF }} + run: | + if [ -z "${TEMPLATE_REF}" ]; then + echo "available=false" >> "$GITHUB_OUTPUT" + echo "AGENT_SKILL_VERIFICATION_REF is not set." + echo "Set it to 'owner/repo@ref' to enable this job." + exit 0 + fi + repo="${TEMPLATE_REF%@*}" + ref="${TEMPLATE_REF##*@}" + if [ "${repo}" = "${ref}" ]; then + echo "::error::AGENT_SKILL_VERIFICATION_REF must be 'owner/repo@ref' (a pinned ref); got '${TEMPLATE_REF}'." + exit 1 + fi + git clone --filter=blob:none --no-checkout "https://github.com/${repo}.git" template + git -C template fetch --depth 1 origin "${ref}" + git -C template checkout FETCH_HEAD + echo "available=true" >> "$GITHUB_OUTPUT" + + - name: Report that verification is not configured + if: steps.template.outputs.available != 'true' + run: | + echo "Agent Skill Verification did NOT run (template not configured)." + echo "This step is a no-op, NOT a pass for the skill suite." + + - name: Run the verification suite against the real implementation + if: steps.template.outputs.available == 'true' + working-directory: template + env: + OPENMIND_ROOT: ${{ github.workspace }}/open-mind + run: | + npm ci + npm run eval:openmind diff --git a/README.md b/README.md index 234a179..3b53b7d 100644 --- a/README.md +++ b/README.md @@ -260,19 +260,60 @@ an AI agent can inspect, cite, and act on. **Prerequisites:** Python 3.12+ on Windows, macOS, or Linux. -```powershell +```bash git clone https://github.com/HelloThisWorld/open-mind.git cd open-mind pip install -r requirements.txt +``` + +Open Mind is a **headless runtime with several front ends**. The CLI, the MCP +server and the FastAPI app all drive the same code — the web UI is one adapter +over the runtime, not the place behaviour is defined, and it is entirely +optional. + +### Tool-first (no UI) + +```bash +# check the runtime: data dir, database, schema version, backends +python -m openmind.cli doctor -# Web UI + REST API on http://127.0.0.1:8077 -./run.ps1 +# create a workspace over a local repository +python -m openmind.cli init --name demo --path ./fixtures/sample-repo + +# learn it (incremental; unchanged files are skipped by content hash) +python -m openmind.cli ingest --workspace --wait + +# what does it know? +python -m openmind.cli status --workspace + +# expose the knowledge layer to an editor or agent over MCP +python -m openmind.cli mcp serve + +# ...or start the optional web UI +python -m openmind.cli serve ``` -Cross-platform launch: +Every command takes `--json` for one machine-readable object on stdout, with +diagnostics on stderr and [stable exit codes](docs/cli.md#exit-codes), so the +runtime scripts cleanly: ```bash -python -m uvicorn openmind.main:app --host 127.0.0.1 --port 8077 +WS=$(python -m openmind.cli init --name demo --path ./src --json | jq -r .workspace_id) +python -m openmind.cli ingest --workspace "$WS" --wait --json | jq '.progress' +python -m openmind.cli status --workspace "$WS" --json | jq '.counts' +``` + +Full contract: **[docs/cli.md](docs/cli.md)**. + +### Web UI + +```powershell +./run.ps1 # Windows +``` + +```bash +python -m openmind.cli serve # cross-platform +python -m uvicorn openmind.main:app --host 127.0.0.1 --port 8077 # equivalent ``` Then open `http://127.0.0.1:8077`, create a project, select a local repository, @@ -280,14 +321,37 @@ and start learning. The deterministic glossary, graph, and exact-token search features do not require an LLM. Ask/Q&A uses an optional local OpenAI-compatible `llama-server`. +The server binds to loopback by default and refuses a non-loopback bind unless +you pass `--allow-non-loopback` explicitly. + +### Things worth knowing + +* **The UI is optional.** Nothing in the CLI or MCP path needs it running. +* **One runtime, one bootstrap.** CLI, MCP and FastAPI share the same + configuration, database connection, migrations and services, so a workspace + created in one is immediately visible in the others. +* **`--workspace` is the project id.** The stored entity is still a *project* + and the REST API still says `/projects`; "workspace" is internal vocabulary + that a later phase will build on. Nothing about the stored shape changed. +* **The artifact schema is stable.** `.openmind` export stays at schema **1.x**; + external consumers keep working. Export is standalone — it needs no database, + vector store or model, and runs with nothing but the standard library. +* **Schema changes are migrations.** The database carries a version and upgrades + itself; an existing project database is baselined, never recreated. See + [docs/database-migrations.md](docs/database-migrations.md). +* Enterprise Asset and Knowledge Graph models are **future v2 work** and are not + implemented — see [Roadmap](#roadmap). + --- ## Use It From An Agent (MCP) -Open Mind ships an MCP stdio server: +Open Mind ships an MCP stdio server. Either command starts the same server — +they share one implementation, not two copies of the tools: ```bash python -m openmind.mcp_server +python -m openmind.cli mcp serve ``` Example client registration: @@ -489,6 +553,14 @@ under the template's `reports/` directory. ```text openmind/ + cli.py the command-line front end (argparse; JSON + exit codes) + runtime.py composition root: one idempotent bootstrap for every adapter + version.py the single runtime version constant + domain/ typed application errors + the types crossing service calls + ports/ the three narrow boundaries services depend on + services/ use-case orchestration shared by CLI, MCP and FastAPI + (workspace, ingest, job, export, health + the container) + migrations/ versioned, checksummed SQLite schema migrations walker.py selection-aware walk, .gitignore handling, hashing detect.py manifest/language detection and stack cues langspec.py declarative language registry @@ -513,7 +585,7 @@ openmind/ artifacts.py .openmind artifact export (the stable integration contract) mcp_server.py MCP stdio server skill_bridge.py JSON-lines bridge exposing the skills to the eval harness - main.py FastAPI REST/SSE API and single-page UI + main.py FastAPI REST/SSE API and single-page UI (one adapter) netguard.py audited outbound HTTP guard for app-controlled egress machine.py machine-local source roots and GitHub source links jobs.py resumable background jobs for ingest, Ask, enrichment @@ -584,7 +656,23 @@ Two layers of checks back the claims in this README. results and reproduction steps. **Focused acceptance scripts** — small checks covering the verification -contracts implemented in this build: +contracts implemented in this build. One command runs the supported suite: + +```bash +python scripts/run_acceptance.py # the CI gate (core tier) +python scripts/run_acceptance.py --all # every tier, including local-only +python scripts/run_acceptance.py --list # what is in the suite, and why +python scripts/run_acceptance.py --only verify_migrations +``` + +The runner gives each script its own isolated `OPENMIND_DATA_DIR` and +`OPENMIND_MACHINE_DIR`, forces offline embeddings and disables all egress, and +returns non-zero on any failure. Two properties are deliberate: **a skipped +core script is counted as a failure**, never a pass, and **every +`tests/verify_*.py` on disk must be registered in the runner's manifest** — so a +new test cannot silently go unrun while the suite still reports green. + +The individual scripts, each mapping to a concrete reliability claim: ```bash python tests/verify_artifacts.py # .openmind artifact contract: schema, evidence, determinism @@ -599,15 +687,22 @@ python tests/verify_facets.py # template facets: roles, captures, projec python tests/verify_guide.py # learning guide: cited pages, notes preservation, honest reset python tests/verify_grounding.py # glossary-first Ask routing, honest miss/empty sources python tests/verify.py # 12 cross-cutting design invariants end to end +python tests/verify_migrations.py # schema ledger: empty, legacy baseline, checksum, rollback +python tests/verify_runtime.py # runtime bootstrap idempotency, worker opt-in, one version +python tests/verify_services.py # application services and their typed errors +python tests/verify_cli.py # CLI contract: JSON output, exit codes, every command +python tests/verify_adapters.py # REST route, MCP tool and skill-bridge compatibility ``` -These are intentionally small acceptance scripts rather than a hidden benchmark -suite; each one maps to a concrete reliability claim in this README. The -remaining `tests/verify_*.py` files are regression suites (Ask flows, model -server, async delete, specific fix batches). Run each script with an isolated -`OPENMIND_DATA_DIR` (plus `OPENMIND_EMBED_OFFLINE=1` for deterministic offline -embeddings); under that setup the whole `tests/` directory passes on a fresh -clone against the neutral fixture repos in `fixtures/`. +The remaining `tests/verify_*.py` files are regression suites (Ask flows, model +server, async delete, specific fix batches). One script, +`verify_delete_responsive.py`, is excluded from the CI gate and marked +`local` in the manifest with its reason: it asserts sub-2-second API latency +while a delete drains, which a shared CI runner's scheduling jitter makes flaky. +Run it locally before releasing a change to the delete path. + +Under the runner's setup the whole `tests/` directory passes on a fresh clone +against the neutral fixture repos in `fixtures/`. --- @@ -615,6 +710,24 @@ clone against the neutral fixture repos in `fixtures/`. The following are not claimed as complete in the current build: +**v2 enterprise knowledge layer** — none of this is implemented. Phase 1 +(shipped) is the tool-first runtime foundation described in +[docs/v2/phase-1-core-foundation.md](docs/v2/phase-1-core-foundation.md); it +deliberately creates extension points for the items below rather than building +them: + +- enterprise Asset, Revision, Claim and Relation models; +- the engineering Knowledge Graph; +- PDF, DOCX and XLSX parsing; COBOL and JCL support; +- cloud model providers (OpenAI, Anthropic, Bedrock, Azure, Vertex) — the + runtime remains local-first with no cloud credentials; +- requirement-to-code traceability, conflict detection and branch overlays; +- webhook integration and a Bundle 2.0 artifact schema (export stays at + schema 1.x); +- a typed worker pool or job DAG replacing the current single-worker engine. + +**Other work not yet done:** + - installable Claude Skill packaging for the tracked capability contracts; - deeper change impact analysis beyond name-based caller/callee neighborhoods; - IDE extension integration; diff --git a/docs/cli.md b/docs/cli.md new file mode 100644 index 0000000..33430c5 --- /dev/null +++ b/docs/cli.md @@ -0,0 +1,276 @@ +# OpenMind CLI + +```bash +python -m openmind.cli --help +``` + +> **Why `python -m` and not a bare `openmind`?** This repository has no Python +> packaging metadata (no `pyproject.toml` or `setup.py`) — it is run from a +> checkout, and `requirements.txt` is the dependency contract. A console entry +> point needs an installable distribution, so adding one would mean introducing +> packaging that nothing else here uses. `python -m openmind.cli` works from a +> clean clone with no install step. If the project later gains packaging +> metadata, exposing `openmind` as a console script is a two-line addition. + +The CLI, the MCP server and the FastAPI app are three front ends over **one +runtime**. A workspace created here is the workspace the UI shows; a job +enqueued here is executed by the same worker. The web UI is optional. + +> **Vocabulary.** `--workspace` takes the id the rest of OpenMind calls a +> *project id* (`p_...`). The stored entity is still a project and the REST API +> still says `/projects`. "Workspace" is internal naming that a later v2 phase +> will build on; nothing about the stored shape has changed. + +--- + +## Global flags + +| Flag | Meaning | +| --- | --- | +| `--help` | usage for the CLI or a subcommand | +| `--version` | print the runtime version and exit | +| `--json` | print one machine-readable JSON object on stdout | +| `--quiet`, `-q` | suppress human progress output | +| `--verbose`, `-v` | print extra diagnostics on stderr | + +All five work on **either side** of the subcommand — `openmind --json doctor` +and `openmind doctor --json` are equivalent. + +### `--json` guarantees + +* Exactly **one** JSON object is printed, to **stdout**, once, at the end. +* Every human message, warning and progress line goes to **stderr**. +* No ANSI escape codes are emitted, in either mode. +* A failure still prints a JSON object, so the output is parseable even when + the exit code is non-zero: + +```json +{ + "ok": false, + "error": { + "code": "workspace_not_found", + "message": "workspace not found: 'p_nope'", + "details": { "workspace_id": "p_nope" } + } +} +``` + +This means `openmind status --workspace "$WS" --json | jq .counts` is safe to +script against. + +--- + +## Exit codes + +| Code | Meaning | Typical cause | +| --- | --- | --- | +| `0` | success | | +| `1` | the operation ran, the domain result is a failure | `doctor` found an error-severity problem; the workspace or job does not exist | +| `2` | invalid arguments or configuration | a missing required flag, a path that is not a directory, `--template` with `--no-template`, a non-loopback `serve` bind | +| `3` | a required runtime dependency or backend is unavailable | the `mcp` package is not installed; `uvicorn` is missing | +| `4` | job or execution failure | an ingest reached `failed`, `paused` or `interrupted` instead of `done` | +| `5` | timeout or cancellation | `--wait` exceeded `--timeout`; Ctrl+C | + +argparse's own usage errors also exit `2`. + +A note on `4`: `paused` and `interrupted` are *not* terminal states, but the +worker will not advance them without an explicit resume. `--wait` returns them +with exit `4` and the real status rather than blocking until the timeout — a +stalled job is reported honestly, not waited on forever. + +--- + +## Commands + +### `doctor` + +```bash +python -m openmind.cli doctor +python -m openmind.cli doctor --json +``` + +Non-destructive diagnostics: data directory, database, migration version, +project-directory permissions, vector-store backend, embedding backend, MCP +dependency, model configuration, local-model readiness, network policy, and the +runtime version. + +**Severity policy.** Only an `error` fails `doctor` (exit `1`). A `warn` means +degraded-but-usable and exits `0`. This matters most for the local model: +OpenMind ingests, extracts, exports artifacts and answers glossary/structure +queries with **no model at all**, so a missing `llama-server` is a warning, never +a failure. A model-dependent operation checks readiness separately. + +### `init` + +```bash +python -m openmind.cli init --name demo --path ./fixtures/sample-repo +python -m openmind.cli init --name demo --path ./src --ingest --wait +``` + +| Flag | Meaning | +| --- | --- | +| `--name` | workspace display name (required) | +| `--path` | source directory to register | +| `--exclude` | path to exclude (repeatable) | +| `--ingest` | ingest immediately — **off by default** | +| `--wait` | with `--ingest`, wait for the job to finish | +| `--timeout` | bound for `--wait`, in seconds (default 3600) | + +Creating a workspace never ingests unless `--ingest` is given. Prints the +workspace id on stdout (or the full record with `--json`). + +Relative paths are resolved to absolute before being stored, so a later ingest +does not depend on the worker's working directory. The absolute path is written +to the **machine-local sidecar** (`~/.openmind`), never to the portable +database or an exported artifact. + +### `add` + +```bash +python -m openmind.cli add --workspace p_abc123 --path ./services/api \ + --exclude target --exclude build +``` + +Registers a new source path or updates an existing one's exclude set. Repeating +`add` for the same path updates it in place rather than duplicating it. + +### `ingest` + +```bash +python -m openmind.cli ingest --workspace p_abc123 +python -m openmind.cli ingest --workspace p_abc123 --wait --timeout 1800 +python -m openmind.cli ingest --workspace p_abc123 --path ./services/api +``` + +| Flag | Meaning | +| --- | --- | +| `--workspace` | workspace id (required) | +| `--path` | restrict the ingest to this subtree | +| `--wait` | poll until the job reaches a terminal state | +| `--timeout` | bound for `--wait`, in seconds (default 3600) | + +Without `--wait` it returns the persisted job id immediately; the job runs in +the background and survives the CLI process. With `--wait` it starts the job +worker, then polls the **persisted** job row — state is never cached, because +the database is the single source of truth that pause, resume and restart +recovery all depend on. + +Ingestion is incremental: unchanged files are skipped by content hash. + +A timeout does **not** cancel the job. It keeps running and can be polled with +`status` or waited on again. + +### `status` + +```bash +python -m openmind.cli status --workspace p_abc123 +python -m openmind.cli status --workspace p_abc123 --jobs 20 --json +``` + +Reports workspace metadata, registered paths, state, template selection, +indexed-file / code-chunk / glossary-term / solved-case counts, active and +recent jobs, the schema version and the runtime version. Does **not** require a +running FastAPI server. + +A count of `unknown` (`null` in JSON) means that store could not be read — it is +deliberately distinct from `0`, which means "read successfully, and empty". + +### `export` + +```bash +python -m openmind.cli export --repo ./fixtures/sample-repo --output ./.openmind +python -m openmind.cli export --repo ./repo --output ./out --template spring-boot +python -m openmind.cli export --repo ./repo --output ./out --no-template --json +``` + +Writes the deterministic `.openmind` artifact directory — **schema 1.1.0**, +unchanged and frozen. Equivalent to `python -m openmind.artifacts`, which also +still works. + +Export is offline and standalone: it does not open the database, the vector +store, the model server or the web app, so it runs from a clean checkout with +nothing installed beyond the standard library. + +`--generated-at` overrides the manifest timestamp for byte-reproducible builds. + +### `mcp serve` + +```bash +python -m openmind.cli mcp serve +``` + +Runs the MCP stdio server — the *same* implementation as +`python -m openmind.mcp_server`, not a second copy of the tools. The nine tools +(`search`, `route`, `dispatch`, `get_glossary`, `find_similar_cases`, +`save_case`, `get_doc`, `propose_fix`, `apply_fix`) are unchanged. + +stdout is the MCP transport, so all CLI chatter goes to stderr on this command. + +### `serve` + +```bash +python -m openmind.cli serve +python -m openmind.cli serve --host 127.0.0.1 --port 8077 +``` + +Runs the FastAPI application. + +**Loopback by default.** OpenMind serves project content — source snippets, +paths, prompts. `serve` refuses a non-loopback bind with exit `2` unless +`--allow-non-loopback` is passed explicitly. There is no silent `0.0.0.0`. + +--- + +## Scripting examples + +Create, ingest and read back a workspace: + +```bash +WS=$(python -m openmind.cli init --name demo --path ./src --json | jq -r .workspace_id) +python -m openmind.cli ingest --workspace "$WS" --wait --json | jq '.progress' +python -m openmind.cli status --workspace "$WS" --json | jq '.counts' +``` + +Fail a build when diagnostics report a real problem: + +```bash +if ! python -m openmind.cli doctor --json > doctor.json; then + jq -r '.checks[] | select(.status=="error") | "\(.name): \(.detail)"' doctor.json + exit 1 +fi +``` + +Distinguish a timeout from a failure: + +```bash +python -m openmind.cli ingest --workspace "$WS" --wait --timeout 300 --json +case $? in + 0) echo "ingested" ;; + 4) echo "job did not complete — check 'status' for why" ;; + 5) echo "still running; poll later" ;; +esac +``` + +--- + +## Environment variables + +| Variable | Effect | +| --- | --- | +| `OPENMIND_DATA_DIR` | where the database, vector store and per-project data live (default `./data`) | +| `OPENMIND_MACHINE_DIR` | the machine-local sidecar holding absolute source roots (default `~/.openmind`) | +| `OPENMIND_EMBED_OFFLINE` | `1` forces the deterministic hashing embedder; no model download | +| `OPENMIND_EMBED_DEVICE` | `cpu`, `auto`, ... | +| `OPENMIND_ENRICH_EGRESS` | `0` disables Wikipedia glossary enrichment | +| `OPENMIND_SOURCELINK_EGRESS` | `0` disables on-demand GitHub source fetching | +| `OPENMIND_INGEST_FREE_GPU` | `0` keeps a resident model loaded during bulk embedding | + +The acceptance runner sets the offline/no-egress set for every test. + +--- + +## See also + +* [`docs/database-migrations.md`](database-migrations.md) — the schema ledger +* [`docs/v2/phase-1-core-foundation.md`](v2/phase-1-core-foundation.md) — why the + runtime is shaped this way diff --git a/docs/database-migrations.md b/docs/database-migrations.md new file mode 100644 index 0000000..e596054 --- /dev/null +++ b/docs/database-migrations.md @@ -0,0 +1,207 @@ +# Database migrations + +OpenMind's SQLite schema evolves through a small, strict migration runner: +[`openmind/migrations/`](../openmind/migrations/). Migrations run automatically +on every `db.init_db()` — which means on every runtime bootstrap, from the CLI, +the MCP server, the FastAPI app and the tests alike. + +Check the current version at any time: + +```bash +python -m openmind.cli doctor --json | jq '.checks[] | select(.name=="migrations")' +``` + +--- + +## Why not Alembic + +This is one SQLite file with a handful of tables, already serialized behind a +single WAL connection and one process-level lock. Alembic would add a +dependency, a config file, an `env.py` and a revision graph to solve problems +OpenMind does not have (multiple backends, branching revisions, autogenerate). + +What OpenMind *does* need is the part Alembic is usually not configured to +enforce: a recorded checksum per migration, so editing an already-applied +migration is caught loudly instead of silently diverging one developer's +database from another's. That is about 150 lines. + +If a future phase brings multiple backends or a real revision graph, this is the +seam to replace. + +--- + +## The ledger + +```sql +CREATE TABLE schema_migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + checksum TEXT NOT NULL, -- sha256 of the migration payload + applied_at TEXT NOT NULL +); +``` + +## Guarantees + +1. Works on a completely empty database. +2. Works on a legacy database that has the current tables but no ledger. +3. Never destroys user data. +4. Applies in ascending numeric order — for every caller, including tests that + inject an unordered list. +5. Each migration runs in **one transaction**; a failure rolls it back whole and + writes no ledger row for it. +6. Records a SHA-256 checksum of the migration payload. +7. A changed checksum on an applied migration raises + `MigrationChecksumMismatch` naming the version and both checksums. +8. Runs under the same lock that guards the shared connection, so the job worker + and request threads cannot race it. +9. Idempotent — a second run applies nothing. +10. The version is reported through `doctor` and `GET /api/health`. + +### Transactions and DDL + +`sqlite3` only opens an implicit transaction for DML, not DDL, so a +`CREATE TABLE` would otherwise run in autocommit and survive a rollback. The +runner switches the connection to explicit-transaction mode +(`isolation_level = None`) for its own duration and issues `BEGIN` / `COMMIT` / +`ROLLBACK` itself, restoring the previous mode afterwards. It never uses +`executescript`, which would `COMMIT` before running. + +--- + +## Current migrations + +| Version | Name | What it does | +| --- | --- | --- | +| 1 | `baseline` | the schema as it stood before migrations existed: `projects`, `jobs`, `model_config`, `file_index`, `kv`, `ask_history` + its index | +| 2 | `paths_sidecar` | moves legacy in-DB project paths into the machine-local sidecar and blanks `projects.paths_json` | + +### Upgrading an existing database + +Nothing to do — open OpenMind and it migrates itself. Concretely: + +```text +empty database -> v0001 creates every table -> ledger records 1, 2 +legacy database -> v0001 statements are all no-ops -> ledger records 1, 2 + (data untouched) +current database -> nothing to apply -> no writes +``` + +A legacy database is **baselined, not recreated**. `v0001` is written entirely +with `CREATE TABLE IF NOT EXISTS`, so against a database that already has those +tables every statement is a no-op and the runner simply records version 1. +Project, job, file-index, cases and Ask data is never touched. +`runner.detect_legacy()` reports this explicitly, and `doctor` surfaces it as +`baselined_legacy`. + +`v0002` carries the path-sidecar logic that previously ran unconditionally on +every startup. It is still idempotent, but recording it means the full scan of +every project row happens once instead of on every process start. + +--- + +## Writing a migration + +Create `openmind/migrations/versions/v_.py`. Declare **exactly one** +of `STATEMENTS` or `upgrade(conn)`. + +SQL — the common case: + +```python +"""Add a per-workspace ingest cursor.""" +from __future__ import annotations + +STATEMENTS = ( + "ALTER TABLE projects ADD COLUMN ingest_cursor TEXT NOT NULL DEFAULT ''", + "CREATE INDEX IF NOT EXISTS idx_projects_state ON projects(state)", +) +``` + +Python — when the change needs logic: + +```python +"""Backfill the ingest cursor from the newest file-index row.""" +from __future__ import annotations + +import sqlite3 + + +def upgrade(conn: sqlite3.Connection) -> None: + for row in conn.execute("SELECT id FROM projects").fetchall(): + newest = conn.execute( + "SELECT MAX(updated_at) FROM file_index WHERE project_id=?", + (row[0],)).fetchone()[0] + conn.execute("UPDATE projects SET ingest_cursor=? WHERE id=?", + (newest or "", row[0])) +``` + +Rules: + +* **One statement per entry** in `STATEMENTS`. The runner executes them + individually; it does not split a SQL blob, so no fragile `;` parsing. +* **Do not** `commit()`, `BEGIN` or `ROLLBACK` inside a migration — the runner + owns the transaction. +* Keep imports of application modules *inside* `upgrade()`. Discovery imports + every migration module, and a migration should not drag the application layer + in just by being enumerated. +* Prefer additive changes. SQLite cannot drop or retype a column in place; a + destructive change needs a create-copy-swap and an explicit review. + +Then add a case to `tests/verify_migrations.py`. + +--- + +## Checksums: applied migrations are immutable + +The checksum covers the migration payload — the joined `STATEMENTS`, or the +source of `upgrade()`. Editing a migration that has already been applied +anywhere raises: + +```text +migration 0002_paths_sidecar has already been applied, but its content changed +on disk (stored checksum a1b2c3d4e5f6..., computed 9f8e7d6c5b4a...). Applied +migrations are immutable: revert the edit, or add a NEW migration expressing +the change. +``` + +This includes comment and whitespace edits, because the checksum is over the +source text. That is deliberate: the ledger's promise is "version N is exactly +this change", and a build cannot tell a cosmetic edit from a semantic one. + +**To fix a bad migration that has already shipped:** write a new migration that +corrects it. Do not edit the old one. + +**During local development**, before a migration has left your machine, it is +fine to edit it and start from a clean database — delete `data/openmind.db` (or +point `OPENMIND_DATA_DIR` at a temp directory) and let it re-apply. + +--- + +## A database newer than the code + +If the ledger contains a version this build does not know about — someone ran a +newer OpenMind against the same data directory — the runner does **not** fail +and does **not** try to undo anything. It reports the unknown versions in +`unknown_applied`, and `doctor` shows a warning: + +```text +[ warn ] migrations: schema version 5, but this build does not know + migration(s) [4, 5] — the database was written by a newer OpenMind +``` + +Refusing to start would lock a user out of their own data over a situation that +is usually harmless (additive migrations). Reporting it loudly is the honest +middle ground. + +--- + +## Tests + +`tests/verify_migrations.py` covers: empty database, repeat no-op, legacy +baseline with data preserved, checksum mismatch, transactional rollback, +ordering, a newer-than-code database, and the shape of the real migration set. + +```bash +python tests/verify_migrations.py +python scripts/run_acceptance.py --only verify_migrations +``` diff --git a/docs/v2/phase-1-core-foundation.md b/docs/v2/phase-1-core-foundation.md new file mode 100644 index 0000000..7d2d368 --- /dev/null +++ b/docs/v2/phase-1-core-foundation.md @@ -0,0 +1,477 @@ +# OpenMind v2 — Phase 1: Core Extraction and Tool-First Foundation + +Status: implemented on branch `feat/v2-phase1-core-foundation`. +Runtime version introduced by this phase: **`1.1.0-dev`**. + +This phase is an **architectural foundation**, not a feature release. It moves +OpenMind from a UI-centric local application to a headless engineering knowledge +runtime that CLI, MCP, FastAPI and tests all drive through one shared bootstrap. +It deliberately implements **none** of the later v2 enterprise knowledge +features (see [Deferred work](#8-deferred-v2-work)). + +--- + +## 1. Baseline: what already worked + +Phase 1 started from a clean working tree at `f76aac7`, with the whole existing +acceptance suite green. Recorded before any change was made: + +| Check | Result | +| --- | --- | +| `python -m compileall -q openmind` | pass | +| `tests/verify_artifacts.py` | 31 passed, 0 failed | +| `tests/verify.py` | 17/17 | +| `tests/verify_structure.py` | 14 passed | +| `tests/verify_glossary.py` | 20 passed | +| `tests/verify_router.py` | 23 passed | +| `tests/verify_diagrams.py` | 7 passed | +| `tests/verify_templates.py` | 21/21 | +| `tests/verify_facets.py` | 22/22 | +| `tests/verify_guide.py` | 19/19 | +| `tests/verify_resources.py` | 15 passed | +| `tests/verify_fixes.py` | 7/7 | +| `tests/verify_fixes2.py` | 7/7 | +| `tests/verify_grounding.py` | 8 passed | +| `tests/verify_ask.py` | 15/15 | +| `tests/verify_ask2.py` | 29/29 | +| `tests/verify_async_delete.py` | 6/6 | +| `tests/verify_delete_responsive.py` | 8/8 | +| `tests/verify_modelserver.py` | 10/10 | +| `tests/verify_source_link.py` | 25 passed, 0 failed | + +This is the regression bar. Phase 1 refactors **behind** these tests; it does not +relax them. + +--- + +## 2. Current coupling (the problem) + +### 2.1 Three divergent startup paths + +`db.init_db()` is reachable from three places that do materially different work: + +| Entry point | What it initializes | +| --- | --- | +| `main.py:_lifespan` | `config.ensure_dirs()`, `db.init_db()`, `vectorstore.sweep_orphan_segment_dirs()`, `jobs.start_worker()`, vector-store warm-up thread | +| `mcp_server.py` (module import, line 23) | `db.init_db()` only | +| `db._c()` (lazy, any caller) | `db.init_db()` only | + +Consequences: + +* **A job enqueued outside FastAPI never runs.** `jobs.start_worker()` is called + only from the FastAPI lifespan. Any other process can write a `queued` row and + then wait forever. This is the single hardest blocker for a useful CLI. +* The orphan-segment sweep must run *before* any Chroma client exists in the + process (raw sqlite read vs. the Rust bindings' GIL/lock interaction). Only the + FastAPI path honours that ordering; nothing encodes it as a rule. +* Import-time side effects in `mcp_server.py` mean importing the module for a + test opens the live database. + +### 2.2 `main.py` is the only orchestrator + +Orchestration — not just HTTP concerns — lives inside route bodies: + +* `POST /projects` — create row, create dirs, register path, run template + auto-detection, persist `meta.template_auto`. +* `POST /projects/{id}/paths` — normalize, validate, sidecar write, state bump. +* `POST /ingest` — resolve scope, enqueue, project-state transition. +* `POST /projects/{id}/terminate`, `DELETE /projects/{id}` — multi-step teardown. +* `GET /api/health` — assembles diagnostics inline. + +None of this is reachable without importing FastAPI and constructing a request. + +### 2.3 Schema evolution is untracked + +`db.init_db()` runs one `executescript` of `CREATE TABLE IF NOT EXISTS` +statements plus one ad-hoc, idempotent fixer (`_migrate_paths_to_sidecar`) that +re-scans every project row on **every** startup. There is: + +* no recorded schema version — nothing can report or gate on it; +* no way to express a non-additive change (a column rename, a backfill) safely; +* no protection against an older build opening a newer database. + +### 2.4 Version is scattered and stale + +`openmind/__init__.py` (`0.1.0`), `FastAPI(version="0.1.0")`, `package.json` +(`0.1.0`), and `artifacts._generator_version()` each carry the version +independently. There is no single constant to report from `doctor` or `--version`. + +### 2.5 No first-class CLI + +The only module entry points are `python -m openmind.artifacts`, +`python -m openmind.skill_bridge` and `python -m openmind.mcp_server`. Creating a +project, ingesting it, or inspecting its state requires the web UI. + +--- + +## 3. Target boundaries + +```text +CLI MCP FastAPI Tests + \ | | / + \ | | / + +---------+--------------+------------+ + | + OpenMindRuntime (composition root; idempotent) + | + Application services (use-case orchestration) + workspace / ingest / job / export / health + | + Existing deterministic implementation + db jobs machine templates artifacts glossary structure rag ... + | + SQLite / Vector store / Filesystem +``` + +Rules that keep this honest: + +1. **Services never import FastAPI.** They return plain dicts / dataclasses. +2. **Adapters map, they do not orchestrate.** A route validates input, calls one + service method, and shapes the response. +3. **Strangler, not rewrite.** Services wrap the existing modules. No algorithm + in `jobs.py`, `rag.py`, `glossary.py`, `structure.py` or `artifacts.py` is + rewritten in this phase. +4. **Ports only where a second implementation or a test seam really exists.** + Phase 1 introduces exactly two: the workspace repository and the job + repository, both of which the migration/services tests substitute. No + speculative interfaces. + +### 3.1 Module layout added by this phase + +```text +openmind/ +├── version.py single runtime version constant +├── domain/ +│ ├── errors.py typed application errors -> exit codes / HTTP status +│ └── types.py dataclasses crossing the service boundary +├── ports/ +│ ├── workspace_repository.py protocol implemented by db (+ test fakes) +│ ├── job_repository.py protocol implemented by db/jobs +│ └── runtime_ports.py clock / worker-control seams +├── services/ +│ ├── service_container.py lazily constructed, cached service instances +│ ├── workspace_service.py +│ ├── ingest_service.py +│ ├── job_service.py +│ ├── export_service.py +│ └── health_service.py +├── migrations/ +│ ├── runner.py ordered, checksummed, transactional runner +│ └── versions/ +│ ├── v0001_baseline.py +│ └── v0002_paths_sidecar.py +├── runtime.py OpenMindRuntime + get_runtime() +└── cli.py argparse front end +``` + +`workspace` is an **internal** vocabulary choice only. The stored entity is still +a project, the REST API still says `/projects`, and `WorkspaceService` methods +return records whose id field is the existing `p_*` project id. Renaming the +public API is explicitly out of scope for this phase. + +--- + +## 4. Compatibility constraints + +These are hard constraints. Each has a regression test added in this phase. + +### 4.1 REST + +Every existing route keeps its path, method, query-parameter aliases (notably +`?scope=` binding to `scope_id`), status codes and response body shape. In +particular `/projects` is **not** renamed to `/workspaces`. + +`GET /api/health` is additive-only: every pre-existing key is unchanged, and it +gains `version` and `schema_version`. The full `doctor` report is available at +`GET /api/health?diagnostics=1` — kept opt-in because it probes the model +endpoint with a timeout, stats the disk and writes a temp file, which is right +for an on-demand diagnostic and wrong for a liveness endpoint. + +One deliberate behaviour fix, covered by `tests/verify_adapters.py`: +`DELETE /projects/{id}/paths` was **broken** for an unknown project. It called +straight through to the database, returned `None` from a handler annotated +`-> Dict[str, Any]`, and FastAPI's response validation turned that into a +**500 `ResponseValidationError`** — verified by running the route against `main`: + +```text +fastapi.exceptions.ResponseValidationError: 1 validation error: + {'type': 'dict_type', 'loc': ('response',), 'msg': 'Input should be a valid + dictionary', 'input': None} +``` + +It now returns 404 like every sibling route. This is the "unless a current route +is demonstrably broken" case. + +### Verified empirically, not by inspection + +The refactored routes were probed on `main` and on this branch through a git +worktree, and the two transcripts diffed (status code + response shape, with +volatile values normalized). Across 29 probes: + +* **zero status-code differences**; +* **zero keys removed** from any response; +* 14 purely additive differences — `/api/health` gains `version` and + `schema_version`, and 13 error bodies gain an `error` key **alongside** the + unchanged `detail` key, so a client reading `detail` sees no change. + +### 4.2 MCP + +`python -m openmind.mcp_server` keeps working, and the nine tools keep their +names, argument names and response contracts: + +```text +search route dispatch get_glossary find_similar_cases +save_case get_doc propose_fix apply_fix +``` + +The refactor adds `create_mcp_server(runtime)` so the server can be constructed +in a test without module-import side effects. The module-level `mcp` object and +`main()` remain, so the documented command is unchanged. + +### 4.3 Artifacts + +`python -m openmind.artifacts --repo ... --output ...` and the `.openmind` +schema **1.1.0** are frozen. `ExportService` wraps `generate_artifacts()` +without touching it, and the CLI `export` command delegates to the same +function. Bundle 2.0 is not introduced. + +Export stays **standalone**: no database, no vector store, no model server, no +web app, and no heavy dependency. That required making +`openmind/services/__init__.py` resolve its re-exports lazily — importing +`ExportService` through an eagerly-importing package `__init__` pulled in +`WorkspaceService`, and through it `openmind.vectorstore`, and through that +numpy and chromadb. The dependency-free `artifact-contract` CI job runs the CLI +exporter with nothing installed, and `tests/verify_cli.py` asserts it by +simulating the absent dependencies rather than trusting the layering. + +One deliberate, non-breaking change: `manifest.generator.version` now reports +`1.1.0-dev` instead of `0.1.0`, because it reads `openmind.__version__`. +`schemaVersion` is untouched, the verifier asserts only the generator *name*, +and determinism is preserved (the value is constant within and across runs). + +### 4.4 Skill bridge + +`python -m openmind.skill_bridge --root ` keeps its JSON-lines protocol and +keeps computing everything in memory from the corpus. It is **not** routed +through the runtime — it must not open the app database. The external +Agent Skill Verification Template continues to exercise the real implementation. + +### 4.5 Existing databases + +A database created by any previous build must open, migrate and keep all +project, job, file-index, cases and Ask data. See §5. + +--- + +## 5. Database migration strategy + +### 5.1 Table + +```sql +CREATE TABLE schema_migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + checksum TEXT NOT NULL, + applied_at TEXT NOT NULL +); +``` + +### 5.2 Runner guarantees + +1. Works on an empty database. +2. Works on a legacy database that has the current tables but no + `schema_migrations` — see baselining below. +3. Never destroys user data. Migrations are additive in this phase. +4. Applies in ascending numeric order. +5. Each migration runs in one transaction; a failure rolls that migration back + and leaves `schema_migrations` unchanged for it. +6. Records a `sha256` checksum of the migration's SQL/source. +7. A changed checksum on an already-applied migration raises + `MigrationChecksumMismatch` with the version, name, stored and computed + checksums. It does not silently re-apply or ignore. +8. Runs under `db._lock` with the same single WAL connection the app uses, so + the worker thread and request threads cannot race it. +9. Idempotent: a second run applies nothing and returns the same version. +10. Current version is exposed through `HealthService` → `doctor` and + `GET /api/health`. + +### 5.3 Baselining a legacy database + +`v0001_baseline` is the *current* schema expressed as `CREATE TABLE IF NOT +EXISTS`. That makes it safe to run against a legacy database: the statements are +no-ops, and the runner then records version 1 as applied. Concretely: + +```text +empty db -> v0001 creates every table -> record 1, 2 +legacy db -> v0001 statements are all no-ops -> record 1, 2 (data intact) +current db -> nothing to do -> no writes +``` + +`v0002_paths_sidecar` carries the existing `_migrate_paths_to_sidecar` logic. It +stays idempotent and remains callable by `db.init_db()` for defence in depth, +but it is now *recorded*, so the every-startup rescan of all project rows stops +being unconditional. + +Alembic is **not** adopted: this is one SQLite file with five tables and a +single-writer lock already in place. A ~150-line runner with real checksum +enforcement is less operational weight than a migration framework, an extra +dependency, and an `alembic.ini`. + +--- + +## 6. CLI contract + +Invocation: `python -m openmind.cli` (primary). + +### 6.1 Global flags + +| Flag | Meaning | +| --- | --- | +| `--help` | usage | +| `--version` | runtime version, then exit 0 | +| `--json` | one JSON object on stdout, nothing else on stdout | +| `--quiet` | suppress human progress output | +| `--verbose` | debug diagnostics on stderr | + +`--json` guarantees: exactly one object printed to **stdout**; no ANSI codes; +every diagnostic on **stderr**; a structured `{"ok": false, "error": {...}}` body +on failure, so the output is still parseable when the exit code is non-zero. + +### 6.2 Exit codes + +| Code | Meaning | +| --- | --- | +| 0 | success | +| 1 | operation completed, domain result is a failure (e.g. `doctor` found a problem) | +| 2 | invalid arguments or configuration | +| 3 | missing runtime dependency or unavailable backend | +| 4 | job or execution failure | +| 5 | timeout or cancellation | + +### 6.3 Commands + +```text +doctor non-destructive diagnostics +init --name --path --exclude --ingest --wait +add --workspace --path --exclude +ingest --workspace --path --wait --timeout +status --workspace +export --repo --output [--name --template|--no-template --generated-at] +mcp serve same implementation as python -m openmind.mcp_server +serve --host 127.0.0.1 --port 8077 +``` + +`serve` keeps the loopback default and refuses a non-loopback bind unless +`--allow-non-loopback` is passed explicitly, so no silent `0.0.0.0`. + +`ingest --wait` requires a running job worker. The runtime starts it on demand +(§7), then polls the **persisted** job row — it never caches job state, because +reconnectability depends on the database being the single source of truth. + +--- + +## 7. Runtime bootstrap + +`OpenMindRuntime.bootstrap()` is idempotent and ordered: + +1. `config.ensure_dirs()` +2. open the database connection +3. run migrations to head +4. construct the service container + +Job-worker startup is **separate and opt-in** (`runtime.ensure_worker()`), +because `doctor`, `status` and `export` must not spawn a worker thread, and +`mcp serve` must not either. FastAPI's lifespan and `ingest --wait` call it. + +The orphan-segment sweep keeps its "before any Chroma client" ordering and stays +in the FastAPI lifespan; it is a server-startup concern, not a bootstrap concern, +and running it from short-lived CLI processes would be both wasteful and unsafe +while a server is live. + +Waiting on jobs treats `done` and `failed` as terminal. `paused` and +`interrupted` are **not** terminal but are also not progressing, so the waiter +returns them with an explicit status rather than blocking until timeout — +preserving pause/resume semantics without hanging a CLI. + +--- + +## 8. Testing strategy + +New tests, all runnable offline and in an isolated `OPENMIND_DATA_DIR`: + +| Test | Covers | +| --- | --- | +| `tests/verify_migrations.py` | empty db, legacy baseline, repeat no-op, checksum mismatch, transaction rollback | +| `tests/verify_runtime.py` | bootstrap idempotency, container identity, worker opt-in | +| `tests/verify_services.py` | workspace create / path register / template selection / ingest enqueue / status | +| `tests/verify_cli.py` | JSON contract, exit codes, doctor, init, add, ingest, status, export | +| `tests/verify_adapters.py` | MCP construction, FastAPI route compatibility, skill-bridge smoke | + +The existing acceptance scripts run unchanged under a new runner: + +```bash +python scripts/run_acceptance.py +``` + +which isolates `OPENMIND_DATA_DIR` and `OPENMIND_MACHINE_DIR` per script, forces +`OPENMIND_EMBED_OFFLINE=1`, `OPENMIND_ENRICH_EGRESS=0`, +`OPENMIND_SOURCELINK_EGRESS=0`, aggregates results, and returns non-zero on any +failure. + +Two integrity properties: + +* **A skip is never a pass.** A script that cannot run is reported as `skipped` + with a reason, and a skipped CORE script fails the run. +* **A test cannot silently go unrun.** Every `tests/verify_*.py` on disk must + appear in the runner's manifest; an unregistered file fails the run with exit + 2 rather than being quietly excluded while the suite reports green. + +Tier assignment was made by checking what each script actually does, not by +assumption: + +* `verify_modelserver.py` — CORE. It spawns a *Python stub* HTTP server, not a + real `llama-server`, so it needs no model binary. +* `verify_source_link.py` — CORE. It exercises URL parsing and the netguard + egress *policy*; it makes no live network call. +* `verify_delete_responsive.py` — **LOCAL**, the one exclusion. It asserts + sub-2-second API latency while a delete drains; a shared CI runner's + scheduling jitter makes that flaky, and a flaky gate is worse than an honest + exclusion. The manifest records that reason, and `--list` prints it. + +--- + +## 9. Deferred v2 work + +Explicitly **not** built in this phase, and no placeholder code pretends +otherwise: + +PDF / DOCX / XLSX parsing · COBOL / JCL support · enterprise Asset, Revision, +Claim, Relation tables · engineering Knowledge Graph · cloud model providers +(OpenAI, Anthropic, Bedrock, Azure, Vertex) · requirement-to-code traceability · +conflict detection · branch overlays · webhooks · new Titan Mind integration · +new graph UI · new chat interface · autonomous code modification · TypeScript +rewrite · monorepo conversion · Neo4j or another graph database · Bundle 2.0 +artifact schema · typed worker pool / job DAG. + +Extension points that exist for them: the migration runner (new versioned +tables), `ports/` (a second repository implementation), `ServiceContainer` (new +services), and the CLI subcommand table. + +--- + +## 10. Known risks + +* **`main.py` remains large.** It holds ~50 routes; Phase 1 moves orchestration + out of the project/path/ingest/job/terminate/health routes but deliberately + leaves Ask, glossary-enrichment, graph and source-navigation routes calling + their modules directly. Those are thin already, and moving them for symmetry + would be churn without a test seam. +* **Job-engine internals are still reachable.** `tests/verify.py` and + `tests/verify_delete_responsive.py` touch `jobs._deleting`, `jobs._shutdown` + and `jobs._recover_on_restart`. `JobService` wraps the public surface only, so + those tests keep working, but the private surface is not yet sealed. +* **Enqueue dedupe is still read-then-write** without a lock in `jobs.py`. + Phase 1 does not change the job engine, so the race is inherited, not fixed. +* **`jobs.py` has a dead `"cancelled"` job-status branch** (`_aborted`, line + ~887); `"cancelled"` is only ever an *exchange* status. `JobService` does not + model a `cancelled` job status, matching reality rather than the dead branch. diff --git a/openmind/__init__.py b/openmind/__init__.py index d0d132a..021ece6 100644 --- a/openmind/__init__.py +++ b/openmind/__init__.py @@ -1,3 +1,5 @@ """Open Mind — a local, grounded code RAG + QA agent with a deterministic glossary, code graphs, and solved-case recall.""" -__version__ = "0.1.0" +from .version import RUNTIME_VERSION + +__version__ = RUNTIME_VERSION diff --git a/openmind/cli.py b/openmind/cli.py new file mode 100644 index 0000000..7f86817 --- /dev/null +++ b/openmind/cli.py @@ -0,0 +1,613 @@ +"""The OpenMind command-line interface. + + python -m openmind.cli --help + +One runtime, four front ends: this CLI, the MCP server, the FastAPI app and the +tests all go through :func:`openmind.runtime.get_runtime`, so a workspace +created here is the same workspace the UI shows, and a job enqueued here is +executed by the same worker. + +MACHINE-READABLE MODE +--------------------- +``--json`` prints exactly ONE JSON object on stdout and nothing else. Every +human message, warning and progress line goes to stderr. No ANSI escapes are +ever emitted, in either mode. A failure still prints a JSON object — +``{"ok": false, "error": {"code": ..., "message": ...}}`` — so the output stays +parseable when the exit code is non-zero. + +EXIT CODES +---------- + 0 success + 1 the operation completed but the domain result is a failure + (doctor found a problem; the workspace or job does not exist) + 2 invalid arguments or configuration + 3 a required runtime dependency or backend is unavailable + 4 job or execution failure + 5 timeout or cancellation + +argparse's own usage errors also exit 2, which matches. + +WHY argparse +------------ +The contract above is a handful of subcommands, five global flags and six exit +codes. The standard library covers all of it, and a dependency added only to +save a few lines of parser wiring would be a dependency to audit, pin and +install on every platform in CI. +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +from typing import Any, Callable, Dict, List, Optional, Tuple + +from .domain.errors import (DependencyUnavailable, InvalidRequest, + OpenMindError, OperationTimeout) +from .version import RUNTIME_VERSION + +EXIT_OK = 0 +EXIT_DOMAIN_FAILURE = 1 +EXIT_INVALID_USAGE = 2 +EXIT_DEPENDENCY = 3 +EXIT_JOB_FAILURE = 4 +EXIT_TIMEOUT = 5 + +PROG = "openmind" + + +# --------------------------------------------------------------------------- +# Output +# --------------------------------------------------------------------------- +class Output: + """Routes human text to stderr and the machine-readable result to stdout. + + In JSON mode stdout carries exactly one object, emitted once at the end. + That is what lets a caller do ``openmind status --json | jq`` without + filtering progress noise, and why every ``say``/``warn`` goes to stderr + even in human mode — the two streams keep the same meaning in both modes. + """ + + def __init__(self, as_json: bool = False, quiet: bool = False, + verbose: bool = False) -> None: + self.as_json = as_json + self.quiet = quiet + self.verbose = verbose + + def say(self, message: str) -> None: + """Human progress. Suppressed by --quiet, never on stdout.""" + if not self.quiet and not self.as_json: + print(message, file=sys.stderr, flush=True) + + def detail(self, message: str) -> None: + """Extra diagnostics, only with --verbose.""" + if self.verbose: + print(message, file=sys.stderr, flush=True) + + def warn(self, message: str) -> None: + """A warning. Survives --quiet (it is not progress chatter), but is + still stderr so it never contaminates JSON output.""" + if not self.as_json or self.verbose: + print(f"warning: {message}", file=sys.stderr, flush=True) + + def emit(self, payload: Dict[str, Any], + human: Optional[Callable[[Dict[str, Any]], None]] = None) -> None: + """Print the final result.""" + if self.as_json: + print(json.dumps(payload, indent=2, default=str), flush=True) + elif human is not None and not self.quiet: + human(payload) + + def emit_error(self, error: Dict[str, Any]) -> None: + if self.as_json: + print(json.dumps({"ok": False, "error": error}, indent=2, default=str), + flush=True) + else: + print(f"error: {error.get('message', 'unknown error')}", + file=sys.stderr, flush=True) + details = error.get("details") or {} + available = details.get("available") + if available: + print(" available: " + ", ".join(str(a) for a in available), + file=sys.stderr, flush=True) + + +def _ok(payload: Dict[str, Any]) -> Dict[str, Any]: + out = {"ok": True} + out.update(payload) + return out + + +def _abs_path(path: str) -> str: + """Resolve a user-supplied path to an absolute, forward-slash form. + + The CLI resolves relative paths against the current directory because a + stored ``./fixtures/sample-repo`` would otherwise be re-resolved against + whatever directory the job worker happens to run in. This absolute path is + stored in the MACHINE-LOCAL sidecar only — it never reaches the portable + database or an exported artifact. + """ + from . import walker + return walker.norm(os.path.abspath(os.path.expanduser(path))) + + +# --------------------------------------------------------------------------- +# Commands +# --------------------------------------------------------------------------- +def cmd_doctor(args: argparse.Namespace, out: Output) -> Tuple[int, Dict[str, Any]]: + """Non-destructive runtime diagnostics.""" + from .runtime import get_runtime + + runtime = get_runtime() + report = runtime.health.report() + payload = _ok(report.as_dict()) + payload["runtime"] = runtime.info() + + def human(_p: Dict[str, Any]) -> None: + print(f"OpenMind {report.version}") + print(f" data dir: {runtime.info()['data_dir']}") + print(f" schema version: {runtime.info()['schema_version']}") + print() + for check in report.checks: + mark = {"ok": " ok ", "warn": " warn ", "error": "ERROR "}.get( + check.status, " ? ") + print(f"[{mark}] {check.name}: {check.detail}") + print() + failures, warnings = report.failures(), report.warnings() + if failures: + print(f"{len(failures)} problem(s) found.") + elif warnings: + print(f"No problems. {len(warnings)} warning(s) — " + f"optional features are degraded or unconfigured.") + else: + print("All checks passed.") + + out.emit(payload, human) + # Warnings never fail doctor: a missing optional local model must not make + # the whole command fail for someone who never asked for a model. + return (EXIT_DOMAIN_FAILURE if not report.ok else EXIT_OK), payload + + +def cmd_init(args: argparse.Namespace, out: Output) -> Tuple[int, Dict[str, Any]]: + """Create a workspace, optionally register a path and ingest it.""" + from .runtime import get_runtime + + runtime = get_runtime() + path = _abs_path(args.path) if args.path else None + if path and not os.path.isdir(path): + raise InvalidRequest(f"source path is not a directory: {path}", + details={"path": path}) + + workspace = runtime.workspaces.create(args.name, path=path, + exclude=args.exclude or []) + out.say(f"Created workspace {workspace['id']} ({workspace['name']}).") + payload = _ok({"workspace_id": workspace["id"], "workspace": workspace}) + + if not args.ingest: + if path: + out.say(f"Registered source path: {path}") + out.say("Not ingesting (pass --ingest to learn this workspace now).") + out.emit(payload, lambda p: print(p["workspace_id"])) + return EXIT_OK, payload + + code, ingest_payload = _run_ingest(runtime, out, workspace["id"], None, + args.wait, args.timeout) + # Merge the ingest outcome INCLUDING its ok flag: a workspace that was + # created but whose ingest failed or timed out must not report ok=true. + payload.update(ingest_payload) + payload["workspace_id"] = workspace["id"] + payload["workspace"] = workspace + out.emit(payload, lambda p: print(p["workspace_id"])) + return code, payload + + +def cmd_add(args: argparse.Namespace, out: Output) -> Tuple[int, Dict[str, Any]]: + """Register or update a source path on an existing workspace.""" + from .runtime import get_runtime + + runtime = get_runtime() + path = _abs_path(args.path) + if not os.path.isdir(path): + raise InvalidRequest(f"source path is not a directory: {path}", + details={"path": path}) + + workspace = runtime.workspaces.add_path(args.workspace, path, + args.exclude or []) + out.say(f"Registered {path} on workspace {args.workspace}.") + payload = _ok({"workspace_id": workspace["id"], "workspace": workspace, + "paths": workspace.get("paths", [])}) + + def human(p: Dict[str, Any]) -> None: + for spec in p["paths"]: + excludes = spec.get("exclude") or [] + suffix = f" (excluding {len(excludes)})" if excludes else "" + print(f"{spec['path']}{suffix}") + + out.emit(payload, human) + return EXIT_OK, payload + + +def cmd_ingest(args: argparse.Namespace, out: Output) -> Tuple[int, Dict[str, Any]]: + """Enqueue an ingest, optionally waiting for it.""" + from .runtime import get_runtime + + runtime = get_runtime() + code, payload = _run_ingest(runtime, out, args.workspace, args.path, + args.wait, args.timeout) + # The job id is present on every path, including the timeout one — a + # timed-out job is still running and still pollable by id. + out.emit(payload, lambda p: print(p.get("job_id", ""))) + return code, payload + + +def _run_ingest(runtime: Any, out: Output, workspace_id: str, + path: Optional[str], wait: bool, + timeout: float) -> Tuple[int, Dict[str, Any]]: + """Shared by ``init --ingest`` and ``ingest``. + + Returns (exit_code, payload) and deliberately does NOT emit: the caller owns + the single final ``out.emit`` call. Emitting here as well would put two JSON + objects on stdout and break the one-object contract. + """ + if wait: + out.say("Starting the job worker...") + + try: + result = runtime.ingest.start(workspace_id, path=path, wait=wait, + timeout=timeout) + except OperationTimeout as exc: + # The job is NOT cancelled — say so, and hand back the id to poll. + out.warn(str(exc)) + payload = {"ok": False, "error": exc.as_dict()} + payload.update(exc.details) + return EXIT_TIMEOUT, payload + + payload = _ok({"workspace_id": workspace_id, "job_id": result["job_id"], + "job": result["job"], "waited": result["waited"]}) + + if not result["waited"]: + out.say(f"Queued ingest job {result['job_id']}.") + out.say("It runs in the background; poll it with " + f"'{PROG} status --workspace {workspace_id}'.") + return EXIT_OK, payload + + status = result.get("status", "") + payload["status"] = status + payload["completed"] = result.get("completed", False) + payload["waited_seconds"] = result.get("waited_seconds") + progress = (result.get("job") or {}).get("progress") or {} + payload["progress"] = progress + + if result.get("completed"): + out.say(f"Ingest finished in {result.get('waited_seconds')}s: " + f"{progress.get('files_indexed', 0)} file(s) indexed, " + f"{progress.get('chunks', 0)} chunk(s).") + return EXIT_OK, payload + + # failed / paused / interrupted — all "did not complete", reported honestly. + error = (result.get("job") or {}).get("error") or "" + payload["ok"] = False + payload["error"] = { + "code": "job_failed", + "message": f"ingest job {result['job_id']} did not complete " + f"(status: {status})" + (f": {error}" if error else ""), + "details": {"job_id": result["job_id"], "status": status}, + } + out.warn(payload["error"]["message"]) + return EXIT_JOB_FAILURE, payload + + +def cmd_status(args: argparse.Namespace, out: Output) -> Tuple[int, Dict[str, Any]]: + """Report workspace metadata, counts, jobs and the schema version.""" + from .runtime import get_runtime + + runtime = get_runtime() + status = runtime.workspaces.status(args.workspace) + jobs = runtime.jobs.list([args.workspace]) + active = [j for j in jobs if j.get("status") in ("queued", "running")] + + payload = _ok({ + "workspace_id": args.workspace, + "workspace": status["workspace"], + "state": status["state"], + "paths": status["paths"], + "template": status["template"], + "counts": status["counts"], + "source_root": runtime.workspaces.source_root(args.workspace), + "active_jobs": active, + "recent_jobs": jobs[:args.jobs], + "schema_version": runtime.info()["schema_version"], + "version": runtime.version, + }) + + def human(p: Dict[str, Any]) -> None: + ws = p["workspace"] + print(f"{ws['name']} ({ws['id']})") + print(f" state: {p['state']}") + print(f" created: {ws.get('created_at', '')}") + template = p["template"].get("effective") + source = p["template"].get("effective_source") + print(f" template: " + f"{template or 'none'}{f' ({source})' if template else ''}") + print(f" schema version: {p['schema_version']}") + print(" source paths:") + if not p["paths"]: + print(" (none registered)") + for spec in p["paths"]: + excludes = spec.get("exclude") or [] + suffix = f" [{len(excludes)} exclusion(s)]" if excludes else "" + print(f" {spec['path']}{suffix}") + print(" counts:") + for label, key in (("indexed files", "indexed_files"), + ("code chunks", "code_chunks"), + ("glossary terms", "glossary_terms"), + ("solved cases", "solved_cases")): + value = p["counts"].get(key) + # None means "could not read that store", NOT zero. + print(f" {label:16} {'unknown' if value is None else value}") + if p["active_jobs"]: + print(" active jobs:") + for job in p["active_jobs"]: + print(f" {job['job_id']} {job['type']:8} {job['status']:8} " + f"{job.get('step', '')}") + else: + print(" active jobs: none") + if p["recent_jobs"]: + print(" recent jobs:") + for job in p["recent_jobs"]: + print(f" {job['job_id']} {job['type']:8} {job['status']:8} " + f"{job.get('created_at', '')}") + + out.emit(payload, human) + return EXIT_OK, payload + + +def cmd_export(args: argparse.Namespace, out: Output) -> Tuple[int, Dict[str, Any]]: + """Generate the deterministic ``.openmind`` artifact directory. + + Deliberately does NOT bootstrap the runtime: export is offline and + standalone, and must not open the database or the vector store. + """ + from .services.export_service import ExportService + + summary = ExportService().export( + args.repo, args.output, name=args.name, template=args.template, + no_template=args.no_template, generated_at=args.generated_at) + payload = _ok(dict(summary)) + + def human(p: Dict[str, Any]) -> None: + print(f"Open Mind artifacts written to {p['output']}") + print(f" schema version: {p['schemaVersion']}") + print(f" repository: {p['repository']['name']} " + f"(commit: {p['repository']['commit'] or 'n/a'})") + print(f" template profile: {p['template'] or 'none'}") + print(f" files indexed: {p['filesIndexed']}") + print(f" symbols indexed: {p['symbolsIndexed']}") + print(f" glossary entries: {p['glossaryEntries']}") + print(f" architecture components: {p['architectureComponents']}") + print(f" flows: {p['flows']}") + print(f" artifact files: {', '.join(p['files'])}") + + out.emit(payload, human) + return EXIT_OK, payload + + +def cmd_mcp_serve(args: argparse.Namespace, out: Output) -> Tuple[int, Dict[str, Any]]: + """Run the MCP stdio server — the same implementation as + ``python -m openmind.mcp_server``, not a second copy of the tools.""" + from .runtime import get_runtime + + try: + from .mcp_server import create_mcp_server + except SystemExit as exc: + # mcp_server raises SystemExit when the 'mcp' package is absent. + raise DependencyUnavailable(str(exc) or "the 'mcp' package is required", + dependency="mcp") from exc + + runtime = get_runtime() + # stderr only: stdout IS the MCP transport and must carry protocol frames. + out.say("Starting the OpenMind MCP server on stdio...") + server = create_mcp_server(runtime) + server.run() + return EXIT_OK, _ok({"served": "mcp"}) + + +def cmd_serve(args: argparse.Namespace, out: Output) -> Tuple[int, Dict[str, Any]]: + """Run the FastAPI application.""" + from . import config + + host = args.host + if host.strip().lower() not in config.LOOPBACK_HOSTS and not args.allow_non_loopback: + # Never silently bind to a public interface: every prompt, path and + # snippet the API serves would be reachable from the network. + raise InvalidRequest( + f"refusing to bind to non-loopback host {host!r}. OpenMind serves " + f"project content and is loopback-only by default; pass " + f"--allow-non-loopback if you really intend to expose it.", + details={"host": host}) + + try: + import uvicorn + except ImportError as exc: + raise DependencyUnavailable( + "uvicorn is required to serve the web application " + "(pip install 'uvicorn[standard]')", dependency="uvicorn") from exc + + out.say(f"Starting Open Mind on http://{host}:{args.port} (Ctrl+C to stop)") + # timeout_graceful_shutdown: an open SSE job stream never completes on its + # own, so without a bound uvicorn waits forever on Ctrl+C. + uvicorn.run("openmind.main:app", host=host, port=args.port, + timeout_graceful_shutdown=5, log_level=( + "debug" if args.verbose else "warning" if args.quiet else "info")) + return EXIT_OK, _ok({"served": "http", "host": host, "port": args.port}) + + +# --------------------------------------------------------------------------- +# Parser +# --------------------------------------------------------------------------- +def _global_flags() -> argparse.ArgumentParser: + """Flags accepted both before and after the subcommand, so + ``openmind --json doctor`` and ``openmind doctor --json`` both work. + + ``default=SUPPRESS`` is load-bearing. A shared parent parser is applied to + BOTH the top-level parser and every subparser, and both write into the same + namespace — so with an ordinary ``default=False`` the subparser would reset + a flag that was given before the subcommand, and ``openmind --quiet doctor`` + would silently not be quiet. With SUPPRESS an unspecified flag sets nothing, + the earlier value survives, and :func:`build_parser` supplies the base + defaults once. + """ + parent = argparse.ArgumentParser(add_help=False) + parent.add_argument("--json", action="store_true", dest="as_json", + default=argparse.SUPPRESS, + help="print one machine-readable JSON object on stdout") + parent.add_argument("--quiet", "-q", action="store_true", + default=argparse.SUPPRESS, + help="suppress human progress output") + parent.add_argument("--verbose", "-v", action="store_true", + default=argparse.SUPPRESS, + help="print extra diagnostics on stderr") + return parent + + +def build_parser() -> argparse.ArgumentParser: + common = _global_flags() + parser = argparse.ArgumentParser( + prog=PROG, parents=[common], + description="OpenMind — a local, grounded engineering knowledge runtime. " + "The web UI is optional: this CLI, the MCP server and the " + "FastAPI app all drive the same runtime.", + epilog="Exit codes: 0 success, 1 domain failure, 2 invalid usage, " + "3 missing dependency, 4 job failure, 5 timeout.") + parser.add_argument("--version", action="version", + version=f"{PROG} {RUNTIME_VERSION}") + # NOTE: deliberately no parser.set_defaults() for the global flags. + # set_defaults() rewrites action.default in place, and `parents=` SHARES + # Action objects with every subparser — so setting a default here would + # undo the SUPPRESS in _global_flags() and re-break `openmind --quiet + # doctor`. main() supplies the defaults with getattr instead. + sub = parser.add_subparsers(dest="command", metavar="") + + doctor = sub.add_parser("doctor", parents=[common], + help="run non-destructive runtime diagnostics") + doctor.set_defaults(func=cmd_doctor) + + init = sub.add_parser("init", parents=[common], + help="create a workspace and optionally ingest it") + init.add_argument("--name", required=True, help="workspace display name") + init.add_argument("--path", help="source directory to register") + init.add_argument("--exclude", action="append", metavar="PATTERN", + help="path to exclude (repeatable)") + init.add_argument("--ingest", action="store_true", + help="ingest immediately after creating (off by default)") + init.add_argument("--wait", action="store_true", + help="with --ingest, wait for the job to finish") + init.add_argument("--timeout", type=float, default=3600.0, metavar="SECONDS", + help="bound for --wait (default: 3600)") + init.set_defaults(func=cmd_init) + + add = sub.add_parser("add", parents=[common], + help="register or update a source path") + add.add_argument("--workspace", required=True, help="workspace id") + add.add_argument("--path", required=True, help="source directory") + add.add_argument("--exclude", action="append", metavar="PATTERN", + help="path to exclude (repeatable)") + add.set_defaults(func=cmd_add) + + ingest = sub.add_parser("ingest", parents=[common], + help="enqueue an ingest job") + ingest.add_argument("--workspace", required=True, help="workspace id") + ingest.add_argument("--path", help="restrict the ingest to this subtree") + ingest.add_argument("--wait", action="store_true", + help="wait for the job to reach a terminal state") + ingest.add_argument("--timeout", type=float, default=3600.0, metavar="SECONDS", + help="bound for --wait (default: 3600)") + ingest.set_defaults(func=cmd_ingest) + + status = sub.add_parser("status", parents=[common], + help="report workspace state, counts and jobs") + status.add_argument("--workspace", required=True, help="workspace id") + status.add_argument("--jobs", type=int, default=5, metavar="N", + help="how many recent jobs to list (default: 5)") + status.set_defaults(func=cmd_status) + + export = sub.add_parser("export", parents=[common], + help="write the deterministic .openmind artifacts") + export.add_argument("--repo", required=True, help="repository root to analyze") + export.add_argument("--output", required=True, help="directory to write into") + export.add_argument("--name", help="repository display name") + export.add_argument("--generated-at", dest="generated_at", metavar="ISO8601", + help="override generatedAt (reproducible builds)") + tgroup = export.add_mutually_exclusive_group() + tgroup.add_argument("--template", metavar="NAME", + help="apply a named template profile " + "(default: deterministic auto-selection)") + tgroup.add_argument("--no-template", dest="no_template", action="store_true", + help="skip template profiles entirely") + export.set_defaults(func=cmd_export) + + mcp = sub.add_parser("mcp", parents=[common], + help="Model Context Protocol server") + mcp_sub = mcp.add_subparsers(dest="mcp_command", metavar="") + mcp_serve = mcp_sub.add_parser("serve", parents=[common], + help="run the MCP stdio server") + mcp_serve.set_defaults(func=cmd_mcp_serve) + mcp.set_defaults(func=None, _parser=mcp) + + serve = sub.add_parser("serve", parents=[common], + help="run the FastAPI web application") + serve.add_argument("--host", default="127.0.0.1", + help="bind host (default: 127.0.0.1, loopback only)") + serve.add_argument("--port", type=int, default=8077, help="bind port") + serve.add_argument("--allow-non-loopback", dest="allow_non_loopback", + action="store_true", + help="permit binding to a non-loopback interface " + "(exposes project content to the network)") + serve.set_defaults(func=cmd_serve) + + return parser + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- +def main(argv: Optional[List[str]] = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + out = Output(as_json=getattr(args, "as_json", False), + quiet=getattr(args, "quiet", False), + verbose=getattr(args, "verbose", False)) + + if not getattr(args, "command", None): + parser.print_help(sys.stderr) + return EXIT_INVALID_USAGE + if getattr(args, "func", None) is None: + # A group like `mcp` with no subcommand. + getattr(args, "_parser", parser).print_help(sys.stderr) + return EXIT_INVALID_USAGE + + try: + code, _payload = args.func(args, out) + return code + except OpenMindError as exc: + out.emit_error(exc.as_dict()) + return exc.exit_code + except KeyboardInterrupt: + out.emit_error({"code": "cancelled", "message": "cancelled by user"}) + return EXIT_TIMEOUT + except BrokenPipeError: + # `openmind status --json | head` — not an error worth a traceback. + return EXIT_OK + except Exception as exc: + # Unexpected: report it as structured data too, and keep the traceback + # available behind --verbose rather than swallowing it. + out.emit_error({"code": "internal_error", + "message": f"{type(exc).__name__}: {exc}"}) + if out.verbose: + import traceback + traceback.print_exc() + return EXIT_DOMAIN_FAILURE + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/openmind/db.py b/openmind/db.py index f7279de..5faafde 100644 --- a/openmind/db.py +++ b/openmind/db.py @@ -14,7 +14,7 @@ import uuid from typing import Any, Dict, List, Optional -from . import config, machine +from . import config, machine, migrations _conn: Optional[sqlite3.Connection] = None _lock = threading.RLock() @@ -38,100 +38,38 @@ def _connect() -> sqlite3.Connection: return conn +_migration_result: Optional["migrations.MigrationResult"] = None + + def init_db() -> None: - global _conn + """Open the shared connection and bring the schema up to date. + + The schema itself lives in :mod:`openmind.migrations.versions` — this is + now a migration run, not a pile of ``CREATE TABLE IF NOT EXISTS``. It stays + idempotent and safe to call from anywhere (the runtime bootstrap, the + FastAPI lifespan, a lazy ``_c()``), and it never destroys data: a legacy + database that predates the ledger is baselined, not recreated. + """ + global _conn, _migration_result with _lock: if _conn is None: _conn = _connect() - c = _conn - c.executescript( - """ - CREATE TABLE IF NOT EXISTS projects ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL, - state TEXT NOT NULL DEFAULT 'init', - paths_json TEXT NOT NULL DEFAULT '[]', - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL, - meta_json TEXT NOT NULL DEFAULT '{}' - ); - - CREATE TABLE IF NOT EXISTS jobs ( - job_id TEXT PRIMARY KEY, - project_id TEXT NOT NULL, - type TEXT NOT NULL, - path TEXT, - status TEXT NOT NULL, - step TEXT DEFAULT '', - progress_json TEXT NOT NULL DEFAULT '{}', - log_tail_json TEXT NOT NULL DEFAULT '[]', - control_json TEXT NOT NULL DEFAULT '{}', - error TEXT DEFAULT '', - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL, - started_at TEXT, - finished_at TEXT - ); - - CREATE TABLE IF NOT EXISTS model_config ( - id INTEGER PRIMARY KEY CHECK (id = 1), - config_json TEXT NOT NULL, - updated_at TEXT NOT NULL - ); - - CREATE TABLE IF NOT EXISTS file_index ( - project_id TEXT NOT NULL, - file_path TEXT NOT NULL, - file_hash TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'indexed', - chunk_ids_json TEXT NOT NULL DEFAULT '[]', - service TEXT DEFAULT '', - topics_json TEXT NOT NULL DEFAULT '[]', - updated_at TEXT NOT NULL, - PRIMARY KEY (project_id, file_path) - ); - - CREATE TABLE IF NOT EXISTS kv ( - key TEXT PRIMARY KEY, - value TEXT - ); - - -- Ask conversation history (UI memory): persistent, per-scope, bounded. - -- Distinct from solved cases (curated knowledge in the cases store). - CREATE TABLE IF NOT EXISTS ask_history ( - exchange_id TEXT PRIMARY KEY, - scope_id TEXT NOT NULL, - project_id TEXT, - job_id TEXT, - status TEXT NOT NULL DEFAULT 'queued', - payload_json TEXT NOT NULL DEFAULT '{}', - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL - ); - CREATE INDEX IF NOT EXISTS idx_ask_scope ON ask_history(scope_id); - """ - ) - c.commit() - _migrate_paths_to_sidecar(c) + # The runner takes the same lock; RLock makes the re-entry safe. + _migration_result = migrations.migrate(_conn, _lock) -def _migrate_paths_to_sidecar(c: sqlite3.Connection) -> None: - """One-time, idempotent: move any legacy in-DB project paths (the absolute - ingest root used to live in ``projects.paths_json``) into the machine-local - sidecar, then blank the column. Keeps the portable DB free of the absolute - machine path. After the first run paths_json is '[]' so this is a no-op.""" - rows = c.execute("SELECT id, paths_json FROM projects").fetchall() - for r in rows: - try: - legacy = json.loads(r["paths_json"] or "[]") - except Exception: - legacy = [] - if not legacy: - continue - if not machine.get_paths(r["id"]): - machine.set_paths(r["id"], legacy) - c.execute("UPDATE projects SET paths_json='[]' WHERE id=?", (r["id"],)) - c.commit() +def migration_status() -> Dict[str, Any]: + """What the last :func:`init_db` migration run did, plus the live schema + version read back from the database. Reported by ``doctor`` and + ``GET /api/health``.""" + with _lock: + version = migrations.current_version(_conn) if _conn is not None else 0 + base = _migration_result.as_dict() if _migration_result else { + "version": version, "applied": [], "already_applied": [], + "baselined_legacy": False, "unknown_applied": [], + } + base["version"] = version + return base def _c() -> sqlite3.Connection: diff --git a/openmind/domain/__init__.py b/openmind/domain/__init__.py new file mode 100644 index 0000000..e16a4bf --- /dev/null +++ b/openmind/domain/__init__.py @@ -0,0 +1,17 @@ +"""Domain layer: application errors and the types that cross service +boundaries. Imports nothing from FastAPI, the CLI, or the persistence layer. +""" +from .errors import (DependencyUnavailable, InvalidRequest, JobFailed, + JobNotFound, NotFound, OpenMindError, OperationTimeout, + WorkspaceNotFound) +from .types import (ACTIVE_JOB_STATUSES, SETTLED_JOB_STATUSES, + TERMINAL_JOB_STATUSES, HealthCheck, HealthReport, + JobWaitResult, STATUS_ERROR, STATUS_OK, STATUS_WARN) + +__all__ = [ + "DependencyUnavailable", "InvalidRequest", "JobFailed", "JobNotFound", + "NotFound", "OpenMindError", "OperationTimeout", "WorkspaceNotFound", + "ACTIVE_JOB_STATUSES", "SETTLED_JOB_STATUSES", "TERMINAL_JOB_STATUSES", + "HealthCheck", "HealthReport", "JobWaitResult", + "STATUS_ERROR", "STATUS_OK", "STATUS_WARN", +] diff --git a/openmind/domain/errors.py b/openmind/domain/errors.py new file mode 100644 index 0000000..0fcda90 --- /dev/null +++ b/openmind/domain/errors.py @@ -0,0 +1,135 @@ +"""Typed application errors. + +Services raise these instead of ``HTTPException`` (which would drag FastAPI into +the application layer) or bare ``ValueError`` (which an adapter cannot map +without string-matching). Each adapter translates them on its own terms: + + error HTTP CLI exit + --------------------- ---- -------- + WorkspaceNotFound 404 1 + JobNotFound 404 1 + InvalidRequest 400 2 + DependencyUnavailable 503 3 + JobFailed (n/a) 4 + OperationTimeout (n/a) 5 + +Every error carries a machine-readable ``code`` and an optional ``details`` +dict, so ``--json`` output stays parseable and callers never have to parse +prose. +""" +from __future__ import annotations + +from typing import Any, Dict, Optional + + +class OpenMindError(Exception): + """Base class for every application-layer error.""" + + code = "openmind_error" + exit_code = 1 + http_status = 500 + + def __init__(self, message: str, *, details: Optional[Dict[str, Any]] = None) -> None: + super().__init__(message) + self.message = message + self.details: Dict[str, Any] = dict(details or {}) + + def as_dict(self) -> Dict[str, Any]: + payload: Dict[str, Any] = {"code": self.code, "message": self.message} + if self.details: + payload["details"] = dict(self.details) + return payload + + +class NotFound(OpenMindError): + """A requested entity does not exist. Honest 'not found', never an empty + stand-in that reads like a real record.""" + + code = "not_found" + exit_code = 1 + http_status = 404 + + +class WorkspaceNotFound(NotFound): + code = "workspace_not_found" + + def __init__(self, workspace_id: str) -> None: + super().__init__(f"workspace not found: {workspace_id!r}", + details={"workspace_id": workspace_id}) + self.workspace_id = workspace_id + + +class JobNotFound(NotFound): + code = "job_not_found" + + def __init__(self, job_id: str) -> None: + super().__init__(f"job not found: {job_id!r}", details={"job_id": job_id}) + self.job_id = job_id + + +class InvalidRequest(OpenMindError): + """Caller-supplied arguments or configuration are unusable.""" + + code = "invalid_request" + exit_code = 2 + http_status = 400 + + +class DependencyUnavailable(OpenMindError): + """A required runtime dependency or backend is missing (the ``mcp`` package, + a vector-store backend, ``uvicorn``).""" + + code = "dependency_unavailable" + exit_code = 3 + http_status = 503 + + def __init__(self, message: str, *, dependency: str = "", + details: Optional[Dict[str, Any]] = None) -> None: + merged = dict(details or {}) + if dependency: + merged.setdefault("dependency", dependency) + super().__init__(message, details=merged) + self.dependency = dependency + + +class JobFailed(OpenMindError): + """A job reached a terminal state that is not success, or stopped without + completing (``paused``/``interrupted`` are not progressing).""" + + code = "job_failed" + exit_code = 4 + http_status = 500 + + def __init__(self, message: str, *, job_id: str = "", status: str = "", + details: Optional[Dict[str, Any]] = None) -> None: + merged = dict(details or {}) + if job_id: + merged.setdefault("job_id", job_id) + if status: + merged.setdefault("status", status) + super().__init__(message, details=merged) + self.job_id = job_id + self.status = status + + +class OperationTimeout(OpenMindError): + """A bounded wait expired. The underlying work is NOT cancelled — the job + keeps running and can still be polled.""" + + code = "timeout" + exit_code = 5 + http_status = 504 + + def __init__(self, message: str, *, timeout: Optional[float] = None, + details: Optional[Dict[str, Any]] = None) -> None: + merged = dict(details or {}) + if timeout is not None: + merged.setdefault("timeout_seconds", timeout) + super().__init__(message, details=merged) + self.timeout = timeout + + +__all__ = [ + "OpenMindError", "NotFound", "WorkspaceNotFound", "JobNotFound", + "InvalidRequest", "DependencyUnavailable", "JobFailed", "OperationTimeout", +] diff --git a/openmind/domain/types.py b/openmind/domain/types.py new file mode 100644 index 0000000..0e9ac28 --- /dev/null +++ b/openmind/domain/types.py @@ -0,0 +1,149 @@ +"""Types that cross the service boundary. + +Services return plain dictionaries for records that already have a stable, +externally-visible shape (project rows, job rows) — those shapes are part of the +REST contract, and wrapping them in dataclasses would mean maintaining two +definitions of the same thing and converting on every call. + +Dataclasses are used where this phase introduces a NEW shape that has no +existing contract to honour: health checks, the health report, and the +terminal-wait outcome. + +VOCABULARY +---------- +"Workspace" is internal vocabulary only. The stored entity is still a project, +the REST API still says ``/projects``, and ``workspace_id`` is the existing +``p_*`` project id. A later v2 phase may introduce a real Asset/Workspace model; +this phase does not. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Dict, List + +# Job statuses, exactly as jobs.py writes them. +JOB_STATUS_QUEUED = "queued" +JOB_STATUS_RUNNING = "running" +JOB_STATUS_PAUSED = "paused" +JOB_STATUS_INTERRUPTED = "interrupted" +JOB_STATUS_DONE = "done" +JOB_STATUS_FAILED = "failed" + +#: Statuses a job never leaves on its own. +TERMINAL_JOB_STATUSES = frozenset({JOB_STATUS_DONE, JOB_STATUS_FAILED}) + +#: Not terminal, but not progressing either: the worker will not advance these +#: without an explicit resume. A bounded wait must return rather than block. +SETTLED_JOB_STATUSES = frozenset({JOB_STATUS_PAUSED, JOB_STATUS_INTERRUPTED}) + +#: Statuses that mean the job is still moving. +ACTIVE_JOB_STATUSES = frozenset({JOB_STATUS_QUEUED, JOB_STATUS_RUNNING}) + +#: Health severities, ordered worst-first. +STATUS_ERROR = "error" +STATUS_WARN = "warn" +STATUS_OK = "ok" +_SEVERITY = {STATUS_ERROR: 0, STATUS_WARN: 1, STATUS_OK: 2} + + +@dataclass +class HealthCheck: + """One diagnostic. ``status`` is ok / warn / error. + + ``warn`` is for a degraded-but-usable condition — a missing optional local + model, an embedding fallback. It must NOT fail ``doctor``: only ``error`` + does, so an absent optional model never breaks diagnostics for someone who + never asked for a model-dependent operation. + """ + + name: str + status: str + detail: str = "" + data: Dict[str, Any] = field(default_factory=dict) + + @property + def ok(self) -> bool: + return self.status != STATUS_ERROR + + def as_dict(self) -> Dict[str, Any]: + out: Dict[str, Any] = {"name": self.name, "status": self.status} + if self.detail: + out["detail"] = self.detail + if self.data: + out["data"] = dict(self.data) + return out + + +@dataclass +class HealthReport: + """The aggregate ``doctor`` result.""" + + version: str + checks: List[HealthCheck] = field(default_factory=list) + + @property + def status(self) -> str: + """Worst severity across all checks.""" + if not self.checks: + return STATUS_OK + return min((c.status for c in self.checks), + key=lambda s: _SEVERITY.get(s, 0)) + + @property + def ok(self) -> bool: + """True unless some check is a hard error.""" + return all(c.ok for c in self.checks) + + def failures(self) -> List[HealthCheck]: + return [c for c in self.checks if c.status == STATUS_ERROR] + + def warnings(self) -> List[HealthCheck]: + return [c for c in self.checks if c.status == STATUS_WARN] + + def as_dict(self) -> Dict[str, Any]: + return { + "version": self.version, + "status": self.status, + "ok": self.ok, + "checks": [c.as_dict() for c in self.checks], + } + + +@dataclass +class JobWaitResult: + """The outcome of a bounded wait on a job. + + ``completed`` means the job reached ``done``. A ``failed``, ``paused`` or + ``interrupted`` job returns with ``completed=False`` and the real status — + the waiter never pretends a stopped job finished, and never blocks forever + on one that will not progress without a resume. + """ + + job: Dict[str, Any] + status: str + completed: bool + waited_seconds: float + timed_out: bool = False + + @property + def job_id(self) -> str: + return str(self.job.get("job_id", "")) + + def as_dict(self) -> Dict[str, Any]: + return { + "job_id": self.job_id, + "status": self.status, + "completed": self.completed, + "timed_out": self.timed_out, + "waited_seconds": round(self.waited_seconds, 3), + "job": self.job, + } + + +__all__ = [ + "JOB_STATUS_QUEUED", "JOB_STATUS_RUNNING", "JOB_STATUS_PAUSED", + "JOB_STATUS_INTERRUPTED", "JOB_STATUS_DONE", "JOB_STATUS_FAILED", + "TERMINAL_JOB_STATUSES", "SETTLED_JOB_STATUSES", "ACTIVE_JOB_STATUSES", + "STATUS_OK", "STATUS_WARN", "STATUS_ERROR", + "HealthCheck", "HealthReport", "JobWaitResult", +] diff --git a/openmind/main.py b/openmind/main.py index f19a5b7..4042cd1 100644 --- a/openmind/main.py +++ b/openmind/main.py @@ -2,6 +2,18 @@ LOCAL-ONLY: the server binds to 127.0.0.1 by default; the only outbound HTTP is the local LLM call (via netguard). See /netlog for the outbound audit trail. + +ONE ADAPTER AMONG SEVERAL +------------------------- +This module is no longer where application behaviour is defined. Workspace, +path, ingest, job, lifecycle and health use cases live in +:mod:`openmind.services` and are shared with the CLI and the MCP server; the +routes below validate HTTP input, call one service, and shape the response. + +Routes that are already a thin call into a deterministic query module (glossary, +structure, graphs, source navigation, Ask) deliberately keep calling it +directly: routing a one-line query through a service method would add +indirection without adding a seam. """ from __future__ import annotations @@ -21,7 +33,10 @@ docs as docsmod, embeddings, fs_tree, glossary, jobs, llm_client, machine, mapio, models, netguard, rag, router, scope, structure, sysload, templates, vectorstore, walker) +from .domain.errors import OpenMindError from .model_server import server as model_server +from .runtime import get_runtime +from .version import RUNTIME_VERSION def _sweep_orphan_project_dirs() -> List[str]: @@ -74,33 +89,55 @@ def _warm_vectorstore() -> None: async def _lifespan(app: FastAPI): # lifespan (not deprecated on_event) so the background job worker reliably # starts under every server/runtime — otherwise jobs would sit queued. - config.ensure_dirs() - db.init_db() + # bootstrap() is the SAME path the CLI and the MCP server take: ensure dirs, + # open the database, migrate to head, build the services. + runtime = get_runtime() # sweep leaked HNSW segment dirs FIRST — it reads chroma.sqlite3 with a raw # (read-only) connection, which is only safe while NO chroma client exists # in this process (start_worker may spawn delete-cleanup threads that open - # one; racing the rust bindings here deadlocks GIL <-> sqlite lock). + # one; racing the rust bindings here deadlocks GIL <-> sqlite lock). This + # stays here rather than in bootstrap(): it is a long-lived-server concern, + # and running it from a short-lived CLI process beside a live server would + # be unsafe. swept = vectorstore.sweep_orphan_segment_dirs() if swept: print(f"[startup] removed {len(swept)} leaked vector segment dir(s).", flush=True) - jobs.start_worker() + # The worker is opt-in per surface; the web app always needs one. + runtime.ensure_worker() threading.Thread(target=_warm_vectorstore, name="vs-warmup", daemon=True).start() yield # shutdown: stop background delete-cleanup at its next batch so Ctrl+C # exits promptly; 'deleting' tombstones resume on the next start. - jobs.begin_shutdown() + runtime.shutdown() # docs_url disabled: the spec reserves GET /docs and /docs/{page} for the # generated knowledge-base docs (not Swagger UI). OpenAPI JSON stays available. -app = FastAPI(title="Open Mind", version="0.1.0", +app = FastAPI(title="Open Mind", version=RUNTIME_VERSION, docs_url=None, redoc_url=None, lifespan=_lifespan) +@app.exception_handler(OpenMindError) +async def _application_error(_request: Request, exc: OpenMindError) -> JSONResponse: + """Map a service-layer error onto its HTTP status. + + Services raise typed errors rather than HTTPException so the CLI and MCP can + map the same failure their own way. ``detail`` keeps the shape FastAPI's own + HTTPException produces, so existing clients see no difference; ``error`` + carries the machine-readable code alongside it. + """ + return JSONResponse(status_code=exc.http_status, + content={"detail": exc.message, "error": exc.as_dict()}) + + # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- +#: The shared application services. Same container the CLI and MCP server use. +_svc = get_runtime + + def _scope_pids(scope_id: Optional[str]) -> List[str]: pids = scope.resolve(scope_id) if not pids: @@ -117,8 +154,19 @@ def index() -> FileResponse: @app.get("/api/health") -def api_health() -> Dict[str, Any]: - return { +def api_health(diagnostics: bool = False) -> Dict[str, Any]: + """Liveness + backend summary. + + ADDITIVE ONLY: every key this returned before is unchanged. ``version`` and + ``schema_version`` are new and cheap. + + The full ``doctor`` report is behind ``?diagnostics=1`` rather than always + included: it probes the model endpoint with a timeout, stats the disk, and + writes a temp file to prove the data dir is writable. That is right for an + on-demand diagnostic and wrong for a liveness endpoint a monitor may poll. + """ + runtime = _svc() + payload = { "ok": True, "embeddings_backend": embeddings.backend_name(), "vectorstore_backend": vectorstore.backend_name(), @@ -126,7 +174,12 @@ def api_health() -> Dict[str, Any]: "llm_local": llm_client.is_local_endpoint(), "outbound_calls_logged": len(netguard.get_log(1000)), "server": model_server.status()["status"], + "version": RUNTIME_VERSION, + "schema_version": runtime.info()["schema_version"], } + if diagnostics: + payload["diagnostics"] = runtime.health.summary() + return payload @app.get("/netlog") @@ -147,48 +200,42 @@ def system_load() -> Dict[str, Any]: # --------------------------------------------------------------------------- @app.get("/projects") def list_projects(state: Optional[str] = None) -> Dict[str, Any]: - return {"projects": db.list_projects(state)} + return {"projects": _svc().workspaces.list(state)} @app.post("/projects") def create_project(req: models.CreateProjectReq) -> Dict[str, Any]: - p = db.create_project(req.name) - if req.path: - db.upsert_project_path(p["id"], req.path, req.exclude) - return db.get_project(p["id"]) + return _svc().workspaces.create(req.name, path=req.path, exclude=req.exclude) @app.get("/projects/{project_id}") def get_project(project_id: str) -> Dict[str, Any]: - p = db.get_project(project_id) - if not p: - raise HTTPException(404, "project not found") - p["code_chunks"] = vectorstore.get_code_store(project_id).count() - p["cases_count"] = vectorstore.get_cases_store(project_id).count() - p["files_indexed"] = len(db.get_file_index(project_id)) - return p + return _svc().workspaces.describe(project_id) @app.post("/projects/{project_id}/paths") def add_path(project_id: str, req: models.AddPathReq) -> Dict[str, Any]: - if not db.get_project(project_id): - raise HTTPException(404, "project not found") - db.upsert_project_path(project_id, req.path, req.exclude) - return db.get_project(project_id) + return _svc().workspaces.add_path(project_id, req.path, req.exclude) @app.post("/projects/{project_id}/selection") def save_selection(project_id: str, req: models.SaveSelectionReq) -> Dict[str, Any]: - if not db.get_project(project_id): - raise HTTPException(404, "project not found") - db.upsert_project_path(project_id, req.path, req.exclude) - return db.get_project(project_id) + """Persist the folder selection (path + exclude-set). Same operation as + /paths; kept as a separate route because the UI's selection panel posts + here.""" + return _svc().workspaces.add_path(project_id, req.path, req.exclude) @app.delete("/projects/{project_id}/paths") def remove_path(project_id: str, path: str) -> Dict[str, Any]: - db.remove_project_path(project_id, path) - return db.get_project(project_id) + """Remove one source path. + + BEHAVIOUR FIX: this used to call straight through to the database and return + ``None`` for an unknown project. Since the handler is annotated + ``-> Dict[str, Any]``, FastAPI's response validation turned that into a + 500 ResponseValidationError. It now 404s like every sibling route. + """ + return _svc().workspaces.remove_path(project_id, path) @app.get("/templates") @@ -204,10 +251,7 @@ def list_templates() -> Dict[str, Any]: def get_project_template(project_id: str) -> Dict[str, Any]: """This project's template selection: the user override, the auto-selection recorded at learn time, and which one is effective.""" - p = db.get_project(project_id) - if not p: - raise HTTPException(404, "project not found") - return templates.selection_info(p) + return _svc().workspaces.template_selection(project_id) @app.post("/projects/{project_id}/template") @@ -215,20 +259,7 @@ def set_project_template(project_id: str, req: models.SetTemplateReq) -> Dict[st """Set (or clear, with name=null) the project's template override. The name must resolve to a valid template at write time — a typo fails HERE, not silently at the next learn.""" - p = db.get_project(project_id) - if not p: - raise HTTPException(404, "project not found") - name = (req.name or "").strip().lower() - meta = dict(p.get("meta") or {}) - if name: - if not templates.get_template(name): - raise HTTPException(400, f"unknown or invalid template: {name!r} " - "(GET /templates lists what is available)") - meta["template"] = name - else: - meta.pop("template", None) - db.update_project_meta(project_id, meta) - return templates.selection_info(db.get_project(project_id)) + return _svc().workspaces.set_template(project_id, req.name) @app.post("/projects/{project_id}/source-link") @@ -261,9 +292,7 @@ def set_source_link(project_id: str, req: models.SourceLinkReq) -> Dict[str, Any @app.post("/projects/{project_id}/terminate") def terminate(project_id: str, req: models.TerminateReq) -> Dict[str, Any]: - if not db.get_project(project_id): - raise HTTPException(404, "project not found") - return jobs.terminate_project(project_id, clear_cases=req.clear_cases) + return _svc().workspaces.terminate(project_id, clear_cases=req.clear_cases) @app.delete("/projects/{project_id}") @@ -271,9 +300,7 @@ def delete_project(project_id: str) -> Dict[str, Any]: """Permanently remove the project. Returns IMMEDIATELY ({deleting}); the project vanishes from the listing at once and its storage is reclaimed in the background so the UI never freezes on a large project.""" - if not db.get_project(project_id): - raise HTTPException(404, "project not found") - return jobs.request_delete(project_id) + return _svc().workspaces.request_delete(project_id) @app.get("/scope/{scope_id}") @@ -379,17 +406,20 @@ def server_status() -> Dict[str, Any]: # --------------------------------------------------------------------------- @app.post("/ingest") def ingest(req: models.IngestReq) -> Dict[str, Any]: - if not db.get_project(req.project_id): - raise HTTPException(404, "project not found") - job = jobs.enqueue_ingest(req.project_id, req.path) + # Enqueue-only, exactly as before: the HTTP client polls /jobs or the SSE + # stream. The service's path-registration guard is deliberately NOT applied + # here — it would turn a currently-accepted request into a 400. + runtime = _svc() + runtime.workspaces.get(req.project_id) # 404 for an unknown project + job = runtime.jobs.enqueue_ingest(req.project_id, req.path) return {"job_id": job["job_id"], "job": job} @app.post("/gendocs") def gendocs(req: models.GendocsReq) -> Dict[str, Any]: - if not db.get_project(req.project_id): - raise HTTPException(404, "project not found") - job = jobs.enqueue_gendocs(req.project_id) + runtime = _svc() + runtime.workspaces.get(req.project_id) + job = runtime.jobs.enqueue_gendocs(req.project_id) return {"job_id": job["job_id"], "job": job} @@ -400,41 +430,31 @@ def gendocs(req: models.GendocsReq) -> Dict[str, Any]: def list_jobs(scope_id: Optional[str] = Query(None, alias="scope")) -> Dict[str, Any]: pids = None if scope_id: + # '__none__' is a sentinel that matches no project, so an unresolvable + # scope lists nothing instead of silently listing EVERY project's jobs. pids = scope.resolve(scope_id) or ["__none__"] - return {"jobs": db.list_jobs(project_ids=pids)} + return {"jobs": _svc().jobs.list(pids)} @app.get("/jobs/{job_id}") def get_job(job_id: str) -> Dict[str, Any]: - j = db.get_job(job_id) - if not j: - raise HTTPException(404, "job not found") - return j + return _svc().jobs.get(job_id) @app.post("/jobs/{job_id}/pause") def pause_job(job_id: str) -> Dict[str, Any]: - try: - return jobs.pause(job_id) - except KeyError: - raise HTTPException(404, "job not found") + return _svc().jobs.pause(job_id) @app.post("/jobs/{job_id}/resume") def resume_job(job_id: str) -> Dict[str, Any]: - try: - return jobs.resume(job_id) - except KeyError: - raise HTTPException(404, "job not found") + return _svc().jobs.resume(job_id) @app.post("/jobs/{job_id}/cancel") def cancel_job(job_id: str) -> Dict[str, Any]: """Cancel a queued/running job (used by the Ask queue; Part 2).""" - try: - return jobs.cancel_job(job_id) - except KeyError: - raise HTTPException(404, "job not found") + return _svc().jobs.cancel(job_id) @app.get("/jobs/{job_id}/stream") diff --git a/openmind/mcp_server.py b/openmind/mcp_server.py index ff49a8b..4cffe6a 100644 --- a/openmind/mcp_server.py +++ b/openmind/mcp_server.py @@ -2,26 +2,43 @@ Exposes the read/query surface a client (your editor, agent, or the CLI) needs — all in-process, all local: - search, get_glossary, find_similar_cases, save_case, get_doc, propose_fix, - apply_fix. + search, route, dispatch, get_glossary, find_similar_cases, save_case, get_doc, + propose_fix, apply_fix. Run: python -m openmind.mcp_server -(register this command as an MCP stdio server in your client). + openmind mcp serve (identical; the CLI calls straight into here) +(register either command as an MCP stdio server in your client). + +CONSTRUCTION +------------ +The tools are plain module-level functions collected in :data:`TOOLS`, and +:func:`create_mcp_server` registers them on a fresh ``FastMCP``. That means a +test can build a server and inspect its tool set without the import of this +module having already opened a database — the import-time ``db.init_db()`` this +module used to run is now the runtime's job, done once per process by the shared +bootstrap. + +The module-level ``mcp`` object is still available and is created on first +attribute access, so anything that referenced it keeps working. + +The tools themselves are deliberately NOT routed through the application +services: they are pure, deterministic queries over the glossary, structure, +cases and RAG modules. Wrapping a one-line query module call in a service method +would add indirection without adding a seam. """ from __future__ import annotations import os -from typing import Any, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional -from . import cases, codemod, db, docs as docsmod, glossary, machine, mapio, rag, router, scope +from . import cases, codemod, docs as docsmod, glossary, machine, mapio, rag, router, scope try: from mcp.server.fastmcp import FastMCP except Exception as exc: # pragma: no cover raise SystemExit("The 'mcp' package is required: pip install mcp\n" + str(exc)) -db.init_db() -mcp = FastMCP("open-mind") +SERVER_NAME = "open-mind" def _pids(scope_id: str) -> List[str]: @@ -44,9 +61,12 @@ def _anchor(file_path: str, scope_id: Optional[str]) -> str: return file_path -@mcp.tool() +# --------------------------------------------------------------------------- +# Tools — argument and response contracts are STABLE; external clients depend +# on them. Changing a name or a returned key is a breaking change. +# --------------------------------------------------------------------------- def search(scope: str, query: str, k: int = 12, case_sensitive: bool = True, - subword: bool = False) -> Dict[str, Any]: + subword: bool = False) -> Dict[str, Any]: """Hybrid code search (cases-first, then RAG). scope = project id. A bare identifier/literal query is matched as an EXACT token (token-boundary; @@ -55,7 +75,8 @@ def search(scope: str, query: str, k: int = 12, case_sensitive: bool = True, also match camelCase/snake_case components; case_sensitive=False to ignore case.""" pids = _pids(scope) case_hits = cases.search_cases(pids, query, k=5) - result = rag.retrieve(pids, query, k=k, case_sensitive=case_sensitive, subword=subword) + result = rag.retrieve(pids, query, k=k, case_sensitive=case_sensitive, + subword=subword) return { "case_hits": case_hits, "case_shortcircuit": any(c.get("similarity", 0) >= 0.65 for c in case_hits), @@ -65,7 +86,6 @@ def search(scope: str, query: str, k: int = 12, case_sensitive: bool = True, } -@mcp.tool() def route(query: str) -> Dict[str, Any]: """Agent-style capability routing with deterministic graceful degradation. @@ -76,14 +96,12 @@ def route(query: str) -> Dict[str, Any]: return router.route(query) -@mcp.tool() def dispatch(scope: str, query: str) -> Dict[str, Any]: """Route the query to one capability and INVOKE it; returns the result plus the routing trace. Each capability is deterministic/grounded; the router only chooses.""" return router.dispatch(_pids(scope), query) -@mcp.tool() def get_glossary(scope: str, term: Optional[str] = None) -> Dict[str, Any]: """Deterministic term/acronym resolution from the persisted glossary map. @@ -95,13 +113,11 @@ def get_glossary(scope: str, term: Optional[str] = None) -> Dict[str, Any]: return glossary.get_glossary(mapio.merged_glossary(_pids(scope)), term) -@mcp.tool() def find_similar_cases(problem: str, scope: str, k: int = 5) -> Dict[str, Any]: """Search the solved-cases store for similar problems (with staleness flags).""" return {"cases": cases.search_cases(_pids(scope), problem, k=k)} -@mcp.tool() def save_case(scope: str, problem_text: str, resolution_summary: str, involved_services: Optional[List[str]] = None, involved_topics: Optional[List[str]] = None, @@ -118,7 +134,6 @@ def save_case(scope: str, problem_text: str, resolution_summary: str, }) -@mcp.tool() def get_doc(page: str, scope: str) -> Dict[str, Any]: """Fetch a generated documentation page (markdown) with its sync status.""" doc = docsmod.get_doc(_pids(scope), page) @@ -127,7 +142,6 @@ def get_doc(page: str, scope: str) -> Dict[str, Any]: return doc -@mcp.tool() def propose_fix(file_path: str, find: str, replace: str, scope: Optional[str] = None) -> Dict[str, Any]: """Preview a SMALL literal find/replace edit as a unified diff. Writes @@ -138,7 +152,6 @@ def propose_fix(file_path: str, find: str, replace: str, return codemod.propose(_anchor(file_path, scope), find, replace) -@mcp.tool() def apply_fix(file_path: str, find: str, replace: str, test_cmd: str, cwd: Optional[str] = None, scope: Optional[str] = None) -> Dict[str, Any]: """Apply a literal find/replace ONLY if it keeps the test suite green. @@ -152,8 +165,61 @@ def apply_fix(file_path: str, find: str, replace: str, test_cmd: str, return codemod.apply_fix(_anchor(file_path, scope), find, replace, test_cmd, cwd=cwd) +#: The published tool set, in registration order. The names are the MCP tool +#: names clients call. +TOOLS: List[Callable[..., Any]] = [ + search, route, dispatch, get_glossary, find_similar_cases, save_case, + get_doc, propose_fix, apply_fix, +] + +TOOL_NAMES = tuple(fn.__name__ for fn in TOOLS) + + +# --------------------------------------------------------------------------- +# Construction +# --------------------------------------------------------------------------- +def create_mcp_server(runtime: Optional[Any] = None) -> FastMCP: + """Build a ``FastMCP`` server exposing :data:`TOOLS`. + + *runtime* is an :class:`~openmind.runtime.OpenMindRuntime`; when omitted the + process-wide one is created and bootstrapped. Bootstrapping here (rather than + at import time) is what makes the module importable in a test without opening + a database, while still guaranteeing that a served process has run its + migrations. + """ + if runtime is None: + from .runtime import get_runtime + get_runtime() + else: + runtime.bootstrap() + + server = FastMCP(SERVER_NAME) + for fn in TOOLS: + server.tool()(fn) + return server + + +_mcp: Optional[FastMCP] = None + + +def __getattr__(name: str) -> Any: + """Lazily provide the module-level ``mcp`` server. + + PEP 562 module ``__getattr__``: ``mcp_server.mcp`` still resolves for anyone + who referenced it, but merely importing this module no longer builds a + server or touches the database. + """ + if name == "mcp": + global _mcp + if _mcp is None: + _mcp = create_mcp_server() + return _mcp + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + def main() -> None: - mcp.run() + from .runtime import get_runtime + create_mcp_server(get_runtime()).run() if __name__ == "__main__": diff --git a/openmind/migrations/__init__.py b/openmind/migrations/__init__.py new file mode 100644 index 0000000..f5e63a6 --- /dev/null +++ b/openmind/migrations/__init__.py @@ -0,0 +1,21 @@ +"""SQLite schema migrations for the Open Mind database. + +See :mod:`openmind.migrations.runner` for the guarantees, and +``docs/database-migrations.md`` for the operational guide. +""" +from .runner import (Migration, MigrationApplyError, MigrationChecksumMismatch, + MigrationError, MigrationResult, applied_migrations, + current_version, detect_legacy, discover, migrate) + +__all__ = [ + "Migration", + "MigrationApplyError", + "MigrationChecksumMismatch", + "MigrationError", + "MigrationResult", + "applied_migrations", + "current_version", + "detect_legacy", + "discover", + "migrate", +] diff --git a/openmind/migrations/runner.py b/openmind/migrations/runner.py new file mode 100644 index 0000000..1300d35 --- /dev/null +++ b/openmind/migrations/runner.py @@ -0,0 +1,335 @@ +"""A small, strict SQLite migration runner. + +WHY NOT ALEMBIC +--------------- +This is one SQLite file with a handful of tables, already serialized behind a +single WAL connection and one process-level lock. Alembic would add a +dependency, a config file, an env.py and a revision graph to solve a problem we +do not have (multiple backends, branching revisions, autogenerate). What we DO +need is the part Alembic is usually not configured to enforce: a recorded +checksum per migration, so an edit to an already-applied migration is caught +loudly instead of silently diverging one developer's database from another's. +That is ~150 lines. + +GUARANTEES +---------- +1. Works on a completely empty database. +2. Works on a legacy database that already has the current tables but no + ``schema_migrations`` table (see :func:`detect_legacy` — such a database is + *baselined*, never recreated). +3. Never destroys user data. Every migration in this phase is additive. +4. Applies migrations in ascending numeric order. +5. Each migration runs inside ONE transaction: on failure it is rolled back + whole and no ``schema_migrations`` row is written for it. +6. Records a SHA-256 checksum of the migration's payload. +7. A changed checksum on an already-applied migration raises + :class:`MigrationChecksumMismatch` naming the version, the stored checksum + and the computed one. Migrations are immutable once applied. +8. The caller passes the lock guarding the shared connection, so the job + worker and request threads cannot race a migration. +9. Idempotent: a second run applies nothing and reports the same version. +10. The resulting version is reported through the health service, ``doctor`` + and ``GET /api/health``. + +TRANSACTIONS +------------ +``sqlite3`` in Python only opens an implicit transaction for DML, not DDL, so a +``CREATE TABLE`` would otherwise run in autocommit and survive a rollback. The +runner therefore switches the connection to explicit-transaction mode +(``isolation_level = None``) for its own duration and issues BEGIN / COMMIT / +ROLLBACK itself, restoring the previous mode afterwards. It never uses +``executescript``, which would COMMIT before running. +""" +from __future__ import annotations + +import hashlib +import importlib +import inspect +import pkgutil +import re +import sqlite3 +import threading +import time +from contextlib import contextmanager +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple + +_MODULE_RE = re.compile(r"^v(\d+)_([a-z0-9_]+)$") + +# Tables that identify a pre-migration ("legacy") Open Mind database. +LEGACY_MARKER_TABLES = ("projects", "jobs", "file_index") + + +class MigrationError(RuntimeError): + """Base class for every migration failure.""" + + +class MigrationChecksumMismatch(MigrationError): + """An already-applied migration's payload changed on disk.""" + + def __init__(self, version: int, name: str, stored: str, computed: str) -> None: + super().__init__( + f"migration {version:04d}_{name} has already been applied, but its " + f"content changed on disk (stored checksum {stored[:12]}..., " + f"computed {computed[:12]}...). Applied migrations are immutable: " + f"revert the edit, or add a NEW migration expressing the change." + ) + self.version = version + self.name = name + self.stored = stored + self.computed = computed + + +class MigrationApplyError(MigrationError): + """A migration raised while being applied; it was rolled back.""" + + def __init__(self, version: int, name: str, cause: BaseException) -> None: + super().__init__( + f"migration {version:04d}_{name} failed and was rolled back: {cause}" + ) + self.version = version + self.name = name + self.cause = cause + + +@dataclass(frozen=True) +class Migration: + """One versioned, checksummed schema change. + + A migration module declares either ``STATEMENTS`` (a sequence of SQL + statements, executed in order) or an ``upgrade(conn)`` function. The + checksum covers whichever of the two carries the change, so editing an + applied migration is always detected. + """ + + version: int + name: str + payload: str + apply: Callable[[sqlite3.Connection], None] + + @property + def checksum(self) -> str: + return hashlib.sha256(self.payload.encode("utf-8")).hexdigest() + + @property + def label(self) -> str: + return f"{self.version:04d}_{self.name}" + + +@dataclass +class MigrationResult: + """What a :func:`migrate` call did — the machine-readable answer that + ``doctor`` and ``GET /api/health`` report.""" + + version: int = 0 + applied: List[str] = field(default_factory=list) + already_applied: List[str] = field(default_factory=list) + baselined_legacy: bool = False + unknown_applied: List[int] = field(default_factory=list) + + def as_dict(self) -> Dict[str, Any]: + return { + "version": self.version, + "applied": list(self.applied), + "already_applied": list(self.already_applied), + "baselined_legacy": self.baselined_legacy, + "unknown_applied": list(self.unknown_applied), + } + + +# --------------------------------------------------------------------------- +# Discovery +# --------------------------------------------------------------------------- +def _build(module: Any, version: int, name: str) -> Migration: + statements: Optional[Sequence[str]] = getattr(module, "STATEMENTS", None) + upgrade: Optional[Callable[[sqlite3.Connection], None]] = getattr( + module, "upgrade", None) + + if statements is not None and upgrade is not None: + raise MigrationError( + f"migration {version:04d}_{name} declares both STATEMENTS and " + f"upgrade(); it must declare exactly one") + + if statements is not None: + sql = tuple(str(s).strip() for s in statements if str(s).strip()) + if not sql: + raise MigrationError( + f"migration {version:04d}_{name} has an empty STATEMENTS") + + def _apply(conn: sqlite3.Connection, _sql: Tuple[str, ...] = sql) -> None: + for statement in _sql: + conn.execute(statement) + + return Migration(version, name, "\n".join(sql), _apply) + + if upgrade is not None: + try: + payload = inspect.getsource(upgrade) + except (OSError, TypeError) as exc: # pragma: no cover - source always available + raise MigrationError( + f"cannot read the source of {version:04d}_{name}.upgrade() to " + f"checksum it: {exc}") from exc + return Migration(version, name, payload, upgrade) + + raise MigrationError( + f"migration {version:04d}_{name} declares neither STATEMENTS nor upgrade()") + + +def discover() -> List[Migration]: + """Every migration under ``openmind.migrations.versions``, ordered by + version. Raises on a duplicate version — an ambiguous order is a bug, not + something to resolve arbitrarily.""" + from . import versions as versions_pkg + + found: Dict[int, Migration] = {} + for info in sorted(pkgutil.iter_modules(versions_pkg.__path__), + key=lambda i: i.name): + match = _MODULE_RE.match(info.name) + if not match: + continue + version, name = int(match.group(1)), match.group(2) + module = importlib.import_module(f"{versions_pkg.__name__}.{info.name}") + if version in found: + raise MigrationError( + f"duplicate migration version {version}: " + f"{found[version].label} and {version:04d}_{name}") + found[version] = _build(module, version, name) + return [found[v] for v in sorted(found)] + + +# --------------------------------------------------------------------------- +# Bookkeeping +# --------------------------------------------------------------------------- +def _table_exists(conn: sqlite3.Connection, table: str) -> bool: + row = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,) + ).fetchone() + return row is not None + + +def detect_legacy(conn: sqlite3.Connection) -> bool: + """True when this database predates migrations: it carries the known + Open Mind tables but has no ``schema_migrations``. + + Such a database is *baselined* — v0001 is written to the ledger without + recreating anything, because v0001 is expressed with ``CREATE TABLE IF NOT + EXISTS`` and is a no-op against it. Existing project, job, file-index, + cases and Ask data is never touched. + """ + if _table_exists(conn, "schema_migrations"): + return False + return any(_table_exists(conn, t) for t in LEGACY_MARKER_TABLES) + + +def ensure_ledger(conn: sqlite3.Connection) -> None: + """Create ``schema_migrations`` if absent. Idempotent.""" + conn.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + checksum TEXT NOT NULL, + applied_at TEXT NOT NULL + ) + """ + ) + + +def applied_migrations(conn: sqlite3.Connection) -> Dict[int, Dict[str, str]]: + """version -> {name, checksum, applied_at} for everything already applied.""" + if not _table_exists(conn, "schema_migrations"): + return {} + rows = conn.execute( + "SELECT version, name, checksum, applied_at FROM schema_migrations" + ).fetchall() + out: Dict[int, Dict[str, str]] = {} + for row in rows: + version = int(row[0]) + out[version] = {"name": row[1], "checksum": row[2], "applied_at": row[3]} + return out + + +def current_version(conn: sqlite3.Connection) -> int: + """The highest applied migration version, or 0 on an unmigrated database.""" + applied = applied_migrations(conn) + return max(applied) if applied else 0 + + +@contextmanager +def _explicit_transactions(conn: sqlite3.Connection): + """Run with manual BEGIN/COMMIT/ROLLBACK so DDL is transactional too.""" + previous = conn.isolation_level + conn.isolation_level = None + try: + yield + finally: + conn.isolation_level = previous + + +# --------------------------------------------------------------------------- +# The runner +# --------------------------------------------------------------------------- +def migrate(conn: sqlite3.Connection, + lock: Optional[threading.RLock] = None, + migrations: Optional[Sequence[Migration]] = None) -> MigrationResult: + """Bring *conn* up to the newest migration and report what happened. + + *lock* is the lock guarding the shared connection (``db._lock``); pass it so + no other thread can use the connection mid-migration. *migrations* is an + injection point for tests — production callers omit it and get + :func:`discover`. + """ + # Sorted here, not only in discover(): guarantee 4 is "deterministic numeric + # order" for EVERY caller, including a test that injects an unordered list. + pending = sorted(migrations if migrations is not None else discover(), + key=lambda m: m.version) + if lock is None: + return _migrate_locked(conn, pending) + with lock: + return _migrate_locked(conn, pending) + + +def _migrate_locked(conn: sqlite3.Connection, + migrations: Sequence[Migration]) -> MigrationResult: + result = MigrationResult() + result.baselined_legacy = detect_legacy(conn) + + with _explicit_transactions(conn): + ensure_ledger(conn) + applied = applied_migrations(conn) + + known = {m.version for m in migrations} + result.unknown_applied = sorted(v for v in applied if v not in known) + + for migration in migrations: + record = applied.get(migration.version) + if record is not None: + # Immutability check BEFORE anything else: a changed payload + # means the database and the code disagree about what version + # N even is, so continuing would silently diverge schemas. + if record["checksum"] != migration.checksum: + raise MigrationChecksumMismatch( + migration.version, migration.name, + record["checksum"], migration.checksum) + result.already_applied.append(migration.label) + continue + + conn.execute("BEGIN") + try: + migration.apply(conn) + conn.execute( + "INSERT INTO schema_migrations (version,name,checksum,applied_at)" + " VALUES (?,?,?,?)", + (migration.version, migration.name, migration.checksum, + time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime())), + ) + except Exception as exc: + conn.execute("ROLLBACK") + raise MigrationApplyError( + migration.version, migration.name, exc) from exc + conn.execute("COMMIT") + result.applied.append(migration.label) + + result.version = current_version(conn) + + return result diff --git a/openmind/migrations/versions/__init__.py b/openmind/migrations/versions/__init__.py new file mode 100644 index 0000000..917b0d8 --- /dev/null +++ b/openmind/migrations/versions/__init__.py @@ -0,0 +1,10 @@ +"""Versioned migration modules, discovered by filename: ``v_.py``. + +Each module declares EXACTLY ONE of: + +* ``STATEMENTS`` — a sequence of SQL statements executed in order; +* ``upgrade(conn)`` — a function, for changes that need Python. + +Whichever it declares is checksummed. Once a migration has been applied to any +database, its content is frozen: express a change as a NEW migration. +""" diff --git a/openmind/migrations/versions/v0001_baseline.py b/openmind/migrations/versions/v0001_baseline.py new file mode 100644 index 0000000..5ccf7d9 --- /dev/null +++ b/openmind/migrations/versions/v0001_baseline.py @@ -0,0 +1,85 @@ +"""Baseline: the schema as it stood before migrations existed. + +Every statement is ``IF NOT EXISTS``, which is what makes baselining a legacy +database safe: against a database that already has these tables the whole +migration is a no-op, and the runner simply records version 1. No table is +recreated and no row is touched, so existing project, job, file-index, cases +and Ask data survives untouched. + +FROZEN. This migration has been applied to real databases; its content is +checksummed and must not change. Express any schema change as a new migration. +""" +from __future__ import annotations + +STATEMENTS = ( + """ + CREATE TABLE IF NOT EXISTS projects ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + state TEXT NOT NULL DEFAULT 'init', + paths_json TEXT NOT NULL DEFAULT '[]', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + meta_json TEXT NOT NULL DEFAULT '{}' + ) + """, + """ + CREATE TABLE IF NOT EXISTS jobs ( + job_id TEXT PRIMARY KEY, + project_id TEXT NOT NULL, + type TEXT NOT NULL, + path TEXT, + status TEXT NOT NULL, + step TEXT DEFAULT '', + progress_json TEXT NOT NULL DEFAULT '{}', + log_tail_json TEXT NOT NULL DEFAULT '[]', + control_json TEXT NOT NULL DEFAULT '{}', + error TEXT DEFAULT '', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + started_at TEXT, + finished_at TEXT + ) + """, + """ + CREATE TABLE IF NOT EXISTS model_config ( + id INTEGER PRIMARY KEY CHECK (id = 1), + config_json TEXT NOT NULL, + updated_at TEXT NOT NULL + ) + """, + """ + CREATE TABLE IF NOT EXISTS file_index ( + project_id TEXT NOT NULL, + file_path TEXT NOT NULL, + file_hash TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'indexed', + chunk_ids_json TEXT NOT NULL DEFAULT '[]', + service TEXT DEFAULT '', + topics_json TEXT NOT NULL DEFAULT '[]', + updated_at TEXT NOT NULL, + PRIMARY KEY (project_id, file_path) + ) + """, + """ + CREATE TABLE IF NOT EXISTS kv ( + key TEXT PRIMARY KEY, + value TEXT + ) + """, + # Ask conversation history (UI memory): persistent, per-scope, bounded. + # Distinct from solved cases (curated knowledge in the cases store). + """ + CREATE TABLE IF NOT EXISTS ask_history ( + exchange_id TEXT PRIMARY KEY, + scope_id TEXT NOT NULL, + project_id TEXT, + job_id TEXT, + status TEXT NOT NULL DEFAULT 'queued', + payload_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ) + """, + "CREATE INDEX IF NOT EXISTS idx_ask_scope ON ask_history(scope_id)", +) diff --git a/openmind/migrations/versions/v0002_paths_sidecar.py b/openmind/migrations/versions/v0002_paths_sidecar.py new file mode 100644 index 0000000..96e4a4d --- /dev/null +++ b/openmind/migrations/versions/v0002_paths_sidecar.py @@ -0,0 +1,40 @@ +"""Move legacy in-DB project paths into the machine-local sidecar. + +The absolute ingest root used to live in ``projects.paths_json``. That put a +machine-specific absolute path inside the portable database, which breaks the +zero-origin-traces constraint: a copied ``data/`` folder carried the original +checkout's location. Paths now live in the machine-local sidecar +(:mod:`openmind.machine`), outside the data directory. + +This logic ran unconditionally on every startup before migrations existed. It +is idempotent either way, but recording it here means the full scan of every +project row happens once instead of on every process start. + +FROZEN. Applied to real databases; checksummed. Do not edit — add a new +migration instead. +""" +from __future__ import annotations + +import json +import sqlite3 + + +def upgrade(conn: sqlite3.Connection) -> None: + # Imported inside the function, not at module scope: discovery imports every + # migration module, and a migration should not drag the application layer in + # just by being enumerated. + from ... import machine + + rows = conn.execute("SELECT id, paths_json FROM projects").fetchall() + for row in rows: + project_id, raw = row[0], row[1] + try: + legacy = json.loads(raw or "[]") + except (TypeError, ValueError): + legacy = [] + if not legacy: + continue + # Never clobber a sidecar entry the user already has on this machine. + if not machine.get_paths(project_id): + machine.set_paths(project_id, legacy) + conn.execute("UPDATE projects SET paths_json='[]' WHERE id=?", (project_id,)) diff --git a/openmind/ports/__init__.py b/openmind/ports/__init__.py new file mode 100644 index 0000000..d2446d2 --- /dev/null +++ b/openmind/ports/__init__.py @@ -0,0 +1,22 @@ +"""Ports: the narrow boundaries the application services depend on. + +Only three exist, and each is justified by a real test seam that is exercised +by ``tests/verify_services.py``: + +* :class:`~openmind.ports.workspace_repository.WorkspaceRepository` — satisfied + by :mod:`openmind.db`; +* :class:`~openmind.ports.job_repository.JobReader` / :class:`JobEngine` — + satisfied by :mod:`openmind.db` and :mod:`openmind.jobs`; +* :class:`~openmind.ports.runtime_ports.Clock` — real vs. fake, so bounded + waits are tested in milliseconds. + +Deliberately absent: ports over configuration, the filesystem, the vector store +and the deterministic extractors. Those have one implementation and an existing +env-var test seam; a Protocol over them would be indirection without coverage. +""" +from .job_repository import JobEngine, JobReader +from .runtime_ports import Clock, FakeClock, SystemClock +from .workspace_repository import WorkspaceRepository + +__all__ = ["JobEngine", "JobReader", "Clock", "FakeClock", "SystemClock", + "WorkspaceRepository"] diff --git a/openmind/ports/job_repository.py b/openmind/ports/job_repository.py new file mode 100644 index 0000000..9eebb48 --- /dev/null +++ b/openmind/ports/job_repository.py @@ -0,0 +1,72 @@ +"""The job surfaces :class:`~openmind.services.job_service.JobService` and +:class:`~openmind.services.ingest_service.IngestService` depend on. + +Two ports, because the existing code really is split in two and pretending +otherwise would hide it: + +* :class:`JobReader` — job *state*, owned by :mod:`openmind.db`. There is no + public ``get_job``/``list_jobs`` in :mod:`openmind.jobs`; callers read the + database directly. +* :class:`JobEngine` — job *lifecycle*, owned by :mod:`openmind.jobs`. + +Both are :class:`typing.Protocol` definitions that the existing modules satisfy +structurally. The test seam is the point: a fake :class:`JobReader` drives the +bounded-wait state machine through queued -> running -> done, or straight to +``paused``, in milliseconds and without a worker thread. + +The job engine itself is NOT replaced in this phase. These ports are the +boundary a later phase would implement with a typed worker pool or a job DAG. +""" +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Protocol, runtime_checkable + + +@runtime_checkable +class JobReader(Protocol): + """Persisted job state. Satisfied by :mod:`openmind.db`. + + Implementations must read through to storage on every call. Job state is + written from the worker thread, request threads and delete-cleanup threads; + a caching reader would break the reconnect/resume story that depends on the + database being the single source of truth. + """ + + def get_job(self, job_id: str) -> Optional[Dict[str, Any]]: ... + + def list_jobs(self, project_ids: Optional[List[str]] = None, + status: Optional[str] = None) -> List[Dict[str, Any]]: ... + + def active_jobs(self) -> List[Dict[str, Any]]: ... + + +@runtime_checkable +class JobEngine(Protocol): + """Job lifecycle control. Satisfied by :mod:`openmind.jobs`. + + ``pause``, ``resume`` and ``cancel_job`` raise :class:`KeyError` for an + unknown job id; :class:`~openmind.services.job_service.JobService` + translates that into + :class:`~openmind.domain.errors.JobNotFound`. + """ + + def start_worker(self) -> None: ... + + def enqueue_ingest(self, project_id: str, + path: Optional[str] = None) -> Dict[str, Any]: ... + + def enqueue_gendocs(self, project_id: str) -> Dict[str, Any]: ... + + def pause(self, job_id: str) -> Dict[str, Any]: ... + + def resume(self, job_id: str) -> Dict[str, Any]: ... + + def cancel_job(self, job_id: str) -> Dict[str, Any]: ... + + def terminate_project(self, project_id: str, + clear_cases: bool = False) -> Dict[str, Any]: ... + + def request_delete(self, project_id: str) -> Dict[str, Any]: ... + + +__all__ = ["JobReader", "JobEngine"] diff --git a/openmind/ports/runtime_ports.py b/openmind/ports/runtime_ports.py new file mode 100644 index 0000000..2dface3 --- /dev/null +++ b/openmind/ports/runtime_ports.py @@ -0,0 +1,61 @@ +"""Runtime seams that make bounded waiting testable. + +:class:`Clock` exists because :class:`~openmind.services.job_service.JobService` +polls persisted job state on a timeout. Testing "a 30-second wait times out" +against the real clock would mean a 30-second test; against a fake clock it is +instant and deterministic. That is a genuine second implementation, not an +abstraction for its own sake. + +Nothing else in the runtime is abstracted here. Configuration, the filesystem +and the database are used directly — they already have a test seam +(``OPENMIND_DATA_DIR`` / ``OPENMIND_MACHINE_DIR``) that the whole existing +suite relies on, and wrapping them would add indirection without adding +coverage. +""" +from __future__ import annotations + +import time +from typing import List, Protocol, runtime_checkable + + +@runtime_checkable +class Clock(Protocol): + """Monotonic time + sleep, so bounded waits can be driven deterministically.""" + + def monotonic(self) -> float: ... + + def sleep(self, seconds: float) -> None: ... + + +class SystemClock: + """The real clock. Uses ``time.monotonic`` so a system clock adjustment + mid-wait cannot make a timeout fire early or hang.""" + + def monotonic(self) -> float: + return time.monotonic() + + def sleep(self, seconds: float) -> None: + if seconds > 0: + time.sleep(seconds) + + +class FakeClock: + """A clock that advances only when slept on. + + Turns "poll until timeout" into a deterministic, instant test. Records every + sleep so a test can assert the poll interval as well as the outcome. + """ + + def __init__(self, start: float = 0.0) -> None: + self._now = float(start) + self.sleeps: List[float] = [] + + def monotonic(self) -> float: + return self._now + + def sleep(self, seconds: float) -> None: + self.sleeps.append(float(seconds)) + self._now += float(seconds) + + +__all__ = ["Clock", "SystemClock", "FakeClock"] diff --git a/openmind/ports/workspace_repository.py b/openmind/ports/workspace_repository.py new file mode 100644 index 0000000..c5370e0 --- /dev/null +++ b/openmind/ports/workspace_repository.py @@ -0,0 +1,42 @@ +"""The persistence surface :class:`~openmind.services.workspace_service.WorkspaceService` +depends on. + +This is a :class:`typing.Protocol`, not a base class: :mod:`openmind.db` +already satisfies it structurally, so nothing has to be registered, subclassed +or adapted. The port exists for one concrete reason — service tests inject a +fake repository to exercise ordering, error and not-found paths without a real +SQLite file or a real vector store. + +It is deliberately narrow. It covers only what the workspace use cases need; +the rest of :mod:`openmind.db` (Ask history, kv, model config) is reached +directly by the modules that own those concerns. +""" +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Protocol, runtime_checkable + + +@runtime_checkable +class WorkspaceRepository(Protocol): + """Structural contract satisfied by :mod:`openmind.db`.""" + + def create_project(self, name: str, + project_id: Optional[str] = None) -> Dict[str, Any]: ... + + def get_project(self, project_id: str) -> Optional[Dict[str, Any]]: ... + + def list_projects(self, state: Optional[str] = None) -> List[Dict[str, Any]]: ... + + def update_project_meta(self, project_id: str, meta: Dict[str, Any]) -> None: ... + + def get_project_paths(self, project_id: str) -> List[Dict[str, Any]]: ... + + def upsert_project_path(self, project_id: str, path: str, + exclude: List[str]) -> None: ... + + def remove_project_path(self, project_id: str, path: str) -> None: ... + + def get_file_index(self, project_id: str) -> Dict[str, Dict[str, Any]]: ... + + +__all__ = ["WorkspaceRepository"] diff --git a/openmind/runtime.py b/openmind/runtime.py new file mode 100644 index 0000000..c607c71 --- /dev/null +++ b/openmind/runtime.py @@ -0,0 +1,166 @@ +"""The composition root. + +One bootstrap path, shared by the CLI, the MCP server, the FastAPI app and the +tests. Before this existed, three call sites initialized the process +differently: the FastAPI lifespan did directories + database + orphan sweep + +job worker, ``mcp_server`` did ``db.init_db()`` at import time, and a lazy +``db._c()`` did the same for anyone else. The most damaging consequence was that +``jobs.start_worker()`` ran ONLY under FastAPI, so a job enqueued from any other +process stayed ``queued`` forever. + +BOOTSTRAP IS IDEMPOTENT AND ORDERED +----------------------------------- +1. ensure directories exist +2. open the database connection +3. migrate the schema to head +4. build the service container + +Calling :meth:`bootstrap` again is a no-op that returns the same container. + +THE WORKER IS OPT-IN +-------------------- +Starting the job worker is deliberately NOT part of bootstrap. +``doctor``, ``status`` and ``export`` must not spawn a background thread, and +``mcp serve`` exposes a read/query surface that does not enqueue jobs. Surfaces +that genuinely need work executed — the FastAPI app, and ``ingest --wait`` — +call :meth:`ensure_worker` explicitly. + +WHAT STAYS IN THE FASTAPI LIFESPAN +---------------------------------- +The leaked-vector-segment sweep. It reads ``chroma.sqlite3`` with a raw sqlite +connection, which is only safe while no Chroma client exists in the process, and +it is a long-lived-server concern. Running it from short-lived CLI processes +would be both wasteful and unsafe against a live server. +""" +from __future__ import annotations + +import threading +from typing import Any, Dict, Optional + +from . import config, db +from . import jobs as jobs_engine # aliased: the class exposes a `jobs` property +from .services.service_container import ServiceContainer +from .version import RUNTIME_VERSION + + +class OpenMindRuntime: + """A bootstrapped OpenMind process: configuration, database, services.""" + + def __init__(self) -> None: + self._lock = threading.RLock() + self._container: Optional[ServiceContainer] = None + self._worker_started = False + + # -- lifecycle ---------------------------------------------------------- + def bootstrap(self) -> ServiceContainer: + """Initialize the process and return the service container. + + Idempotent and thread-safe: concurrent callers get the same container, + and the migration run happens once. + """ + with self._lock: + if self._container is None: + config.ensure_dirs() + db.init_db() # opens the connection + migrates to head + self._container = ServiceContainer(ensure_worker=self.ensure_worker) + return self._container + + def ensure_worker(self) -> None: + """Start the background job worker if it is not already running. + + ``jobs.start_worker()`` is itself idempotent; the extra flag here means + a caller can ask repeatedly without re-entering the engine. + """ + with self._lock: + if self._worker_started: + return + self.bootstrap() + jobs_engine.start_worker() + self._worker_started = True + + def shutdown(self) -> None: + """Ask background delete-cleanup to stop at its next batch. + + Does not join the worker thread — it is a daemon and exits with the + process. ``deleting`` tombstones resume on the next start. + """ + jobs_engine.begin_shutdown() + + # -- accessors ---------------------------------------------------------- + @property + def services(self) -> ServiceContainer: + return self.bootstrap() + + @property + def workspaces(self): + return self.services.workspaces + + @property + def jobs(self): + return self.services.jobs + + @property + def ingest(self): + return self.services.ingest + + @property + def export(self): + return self.services.export + + @property + def health(self): + return self.services.health + + @property + def version(self) -> str: + return RUNTIME_VERSION + + @property + def worker_running(self) -> bool: + return self._worker_started + + def info(self) -> Dict[str, Any]: + """Identifying facts about this runtime, for ``doctor`` and health.""" + return { + "version": RUNTIME_VERSION, + "data_dir": str(config.DATA_DIR), + "database": str(config.DB_PATH), + "schema_version": db.migration_status().get("version", 0), + "worker_running": self._worker_started, + } + + +# --------------------------------------------------------------------------- +# Process-wide default +# --------------------------------------------------------------------------- +_default: Optional[OpenMindRuntime] = None +_default_lock = threading.Lock() + + +def get_runtime() -> OpenMindRuntime: + """The process-wide runtime, created and bootstrapped on first use. + + Every adapter goes through here, so there is exactly one database + connection, one migration run and one service container per process. + """ + global _default + with _default_lock: + if _default is None: + _default = OpenMindRuntime() + runtime = _default + # Bound to a local INSIDE the lock, then bootstrapped outside it: bootstrap + # runs migrations and must not hold the module lock, but re-reading the + # global here would race a concurrent reset_runtime() and hit None. + runtime.bootstrap() + return runtime + + +def reset_runtime() -> None: + """Drop the process-wide runtime. For tests that need a fresh bootstrap + (e.g. after repointing ``OPENMIND_DATA_DIR``); not used in production.""" + global _default + with _default_lock: + _default = None + + +__all__ = ["OpenMindRuntime", "get_runtime", "reset_runtime"] diff --git a/openmind/services/__init__.py b/openmind/services/__init__.py new file mode 100644 index 0000000..11878af --- /dev/null +++ b/openmind/services/__init__.py @@ -0,0 +1,50 @@ +"""Application services: use-case orchestration, shared by every adapter. + +Nothing in this package imports FastAPI, argparse or the MCP SDK. Services take +plain arguments, return plain dictionaries or dataclasses from +:mod:`openmind.domain.types`, and raise :mod:`openmind.domain.errors`. Each +adapter maps those to its own transport. + +Access them through :class:`~openmind.runtime.OpenMindRuntime`, which wires the +container to a bootstrapped database. + +LAZY BY DESIGN +-------------- +The convenience re-exports below resolve on first attribute access (PEP 562) +instead of being imported eagerly. Importing ONE service must not drag in all of +them: :class:`~openmind.services.export_service.ExportService` is standalone and +offline — artifact export needs neither the vector store nor the database — but +:class:`~openmind.services.workspace_service.WorkspaceService` imports +:mod:`openmind.vectorstore`, which needs numpy and chromadb. + +With eager imports here, ``openmind export`` would have required the full +dependency set on a machine that only wanted the deterministic exporter, and the +dependency-free artifact-contract CI job would fail. ``ServiceContainer`` still +imports every service directly, which is correct: a bootstrapped runtime does +need all of them. +""" +from typing import Any + +__all__ = ["ExportService", "HealthService", "IngestService", "JobService", + "ServiceContainer", "WorkspaceService"] + +_EXPORTS = { + "ExportService": ".export_service", + "HealthService": ".health_service", + "IngestService": ".ingest_service", + "JobService": ".job_service", + "ServiceContainer": ".service_container", + "WorkspaceService": ".workspace_service", +} + + +def __getattr__(name: str) -> Any: + module_name = _EXPORTS.get(name) + if module_name is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + from importlib import import_module + return getattr(import_module(module_name, __name__), name) + + +def __dir__() -> list: + return sorted(__all__) diff --git a/openmind/services/export_service.py b/openmind/services/export_service.py new file mode 100644 index 0000000..aaba9ac --- /dev/null +++ b/openmind/services/export_service.py @@ -0,0 +1,67 @@ +"""Artifact export use case. + +A deliberately thin wrapper over :func:`openmind.artifacts.generate_artifacts`. +The ``.openmind`` directory is a FROZEN integration contract (schema 1.1.0) that +external consumers depend on, so this service adds no fields, reorders nothing, +and changes no defaults. Its whole job is to make export reachable from the CLI +and from tests the same way it is reachable from +``python -m openmind.artifacts``. + +Export is standalone by design: no web app, no model server, no vector store, no +database. This service therefore does NOT require a bootstrapped runtime, and +the CLI's ``export`` command runs without initializing one. +""" +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, Optional + +from .. import artifacts +from ..domain.errors import InvalidRequest + + +class ExportService: + """Generate the deterministic ``.openmind`` artifact directory.""" + + #: The artifact schema version this build writes. Frozen for this phase. + schema_version = artifacts.SCHEMA_VERSION + + def export(self, repo: str, output: str, name: Optional[str] = None, + template: Optional[str] = None, no_template: bool = False, + generated_at: Optional[str] = None) -> Dict[str, Any]: + """Analyze *repo* and write artifacts into *output*. + + *generated_at* overrides the manifest timestamp, which is what makes a + reproducible build byte-identical across runs. + """ + if not (repo or "").strip(): + raise InvalidRequest("repo path must not be empty", + details={"field": "repo"}) + if not (output or "").strip(): + raise InvalidRequest("output path must not be empty", + details={"field": "output"}) + if template and no_template: + raise InvalidRequest( + "--template and --no-template are mutually exclusive", + details={"template": template}) + + repo_path = Path(repo) + if not repo_path.is_dir(): + raise InvalidRequest(f"repository not found: {repo}", + details={"repo": str(repo)}) + + try: + summary = artifacts.generate_artifacts( + repo, output, name=name, generated_at=generated_at, + template=template, no_template=no_template) + except (FileNotFoundError, ValueError) as exc: + # The exporter's own argument/IO failures are caller errors, not + # crashes. Anything else propagates untouched. + raise InvalidRequest(str(exc), details={"repo": str(repo), + "output": str(output)}) from exc + + summary["schemaVersion"] = self.schema_version + return summary + + +__all__ = ["ExportService"] diff --git a/openmind/services/health_service.py b/openmind/services/health_service.py new file mode 100644 index 0000000..9f33f34 --- /dev/null +++ b/openmind/services/health_service.py @@ -0,0 +1,299 @@ +"""Runtime diagnostics — the machine-readable answer behind ``openmind doctor`` +and ``GET /api/health``. + +SEVERITY POLICY +--------------- +Only ``error`` fails ``doctor``. ``warn`` means degraded-but-usable, and the +most important instance of that is the local model: OpenMind ingests, extracts, +exports artifacts and answers glossary/structure queries with NO model at all. +An absent ``llama-server`` binary or an unloaded model is therefore a warning, +never a failure — a missing optional local model must not break diagnostics for +someone who never asked for a model-dependent operation. Callers that DO need a +model check :meth:`model_ready` explicitly. + +Every check is non-destructive and independently guarded: one probe raising +never prevents the others from reporting. A probe that cannot answer says so +(``error``/``warn`` with the reason) rather than reporting a healthy default. +""" +from __future__ import annotations + +import os +import shutil +import sqlite3 +import tempfile +from pathlib import Path +from typing import Any, Callable, Dict, List + +from .. import config +from .. import db as db_module +from ..domain.types import (STATUS_ERROR, STATUS_OK, STATUS_WARN, HealthCheck, + HealthReport) +from ..version import RUNTIME_VERSION + + +class HealthService: + """Non-destructive runtime diagnostics.""" + + def __init__(self, db: Any = None) -> None: + self._db: Any = db if db is not None else db_module + + # -- the report --------------------------------------------------------- + def report(self) -> HealthReport: + """Run every diagnostic and aggregate. Never raises.""" + checks: List[HealthCheck] = [ + self._runtime_version(), + self._data_dir(), + self._database(), + self._migrations(), + self._project_dir_permissions(), + self._vectorstore_backend(), + self._embedding_backend(), + self._mcp_dependency(), + self._model_config(), + self._model_readiness(), + self._network_policy(), + ] + return HealthReport(version=RUNTIME_VERSION, checks=checks) + + def summary(self) -> Dict[str, Any]: + """The report as a plain dict, for ``--json`` and the HTTP adapter.""" + return self.report().as_dict() + + def model_ready(self) -> bool: + """Whether a local model is actually reachable right now. + + Separate from :meth:`report` on purpose: this is the gate for a + model-dependent operation, not a health signal. + """ + try: + from .. import llm_client + return bool(llm_client.is_ready()) + except Exception: + return False + + # -- individual probes -------------------------------------------------- + @staticmethod + def _guard(name: str, probe: Callable[[], HealthCheck]) -> HealthCheck: + """Run one probe; convert an unexpected failure into an error check. + + This is the one place a broad except is right: a diagnostic tool that + crashes because a diagnostic crashed is useless. The exception type and + message are reported, not swallowed. + """ + try: + return probe() + except Exception as exc: + return HealthCheck(name, STATUS_ERROR, + f"probe failed: {type(exc).__name__}: {exc}") + + def _runtime_version(self) -> HealthCheck: + return HealthCheck("runtime_version", STATUS_OK, RUNTIME_VERSION, + {"version": RUNTIME_VERSION}) + + def _data_dir(self) -> HealthCheck: + def probe() -> HealthCheck: + path = Path(config.DATA_DIR) + data = {"path": str(path)} + if not path.exists(): + return HealthCheck("data_dir", STATUS_ERROR, + f"data directory does not exist: {path}", data) + if not os.access(path, os.W_OK): + return HealthCheck("data_dir", STATUS_ERROR, + f"data directory is not writable: {path}", data) + usage = shutil.disk_usage(str(path)) + data["free_mb"] = round(usage.free / (1024 * 1024)) + if usage.free < 100 * 1024 * 1024: + return HealthCheck("data_dir", STATUS_WARN, + f"less than 100 MB free at {path}", data) + return HealthCheck("data_dir", STATUS_OK, str(path), data) + + return self._guard("data_dir", probe) + + def _database(self) -> HealthCheck: + def probe() -> HealthCheck: + path = Path(config.DB_PATH) + data: Dict[str, Any] = {"path": str(path), "exists": path.exists()} + if path.exists(): + data["size_bytes"] = path.stat().st_size + conn = sqlite3.connect(str(path)) + try: + tables = {r[0] for r in conn.execute( + "SELECT name FROM sqlite_master WHERE type='table'").fetchall()} + data["tables"] = sorted(tables) + journal = conn.execute("PRAGMA journal_mode").fetchone() + if journal: + data["journal_mode"] = journal[0] + finally: + conn.close() + missing = {"projects", "jobs", "file_index"} - set(data["tables"]) + if missing: + return HealthCheck( + "database", STATUS_ERROR, + "core tables missing: " + ", ".join(sorted(missing)), data) + return HealthCheck("database", STATUS_OK, + f"{len(data['tables'])} tables", data) + + return self._guard("database", probe) + + def _migrations(self) -> HealthCheck: + def probe() -> HealthCheck: + status = self._db.migration_status() + version = status.get("version", 0) + data = dict(status) + if not version: + return HealthCheck("migrations", STATUS_ERROR, + "database has no applied migrations", data) + unknown = status.get("unknown_applied") or [] + if unknown: + return HealthCheck( + "migrations", STATUS_WARN, + f"schema version {version}, but this build does not know " + f"migration(s) {unknown} — the database was written by a " + f"newer OpenMind", data) + return HealthCheck("migrations", STATUS_OK, + f"schema version {version}", data) + + return self._guard("migrations", probe) + + def _project_dir_permissions(self) -> HealthCheck: + def probe() -> HealthCheck: + path = Path(config.DATA_DIR) + data = {"path": str(path)} + if not path.exists(): + return HealthCheck("project_dirs", STATUS_ERROR, + f"data directory does not exist: {path}", data) + # Prove writability by actually writing; an ACL can make os.access + # optimistic on Windows. + try: + handle, probe_path = tempfile.mkstemp(prefix=".om_doctor_", + dir=str(path)) + os.close(handle) + os.unlink(probe_path) + except OSError as exc: + return HealthCheck("project_dirs", STATUS_ERROR, + f"cannot create files under {path}: {exc}", data) + data["writable"] = True + return HealthCheck("project_dirs", STATUS_OK, + f"writable: {path}", data) + + return self._guard("project_dirs", probe) + + def _vectorstore_backend(self) -> HealthCheck: + def probe() -> HealthCheck: + from .. import vectorstore + name = vectorstore.backend_name() + data = {"backend": name} + if not name: + return HealthCheck("vectorstore", STATUS_ERROR, + "no vector-store backend available", data) + # The numpy fallback is fully functional, just slower and in-process. + if "numpy" in name.lower(): + return HealthCheck( + "vectorstore", STATUS_WARN, + f"{name} (in-process fallback; chromadb is not available)", data) + return HealthCheck("vectorstore", STATUS_OK, name, data) + + return self._guard("vectorstore", probe) + + def _embedding_backend(self) -> HealthCheck: + def probe() -> HealthCheck: + from .. import embeddings + name = embeddings.backend_name() + data = {"backend": name, "dim": embeddings.dim(), + "offline": os.environ.get("OPENMIND_EMBED_OFFLINE") == "1"} + if not name: + return HealthCheck("embeddings", STATUS_ERROR, + "no embedding backend available", data) + # The hashing embedder keeps ingestion working offline, but recall + # is materially worse than a real model — say so plainly. + if "hash" in name.lower(): + return HealthCheck( + "embeddings", STATUS_WARN, + f"{name} (deterministic fallback; semantic recall is reduced)", + data) + return HealthCheck("embeddings", STATUS_OK, name, data) + + return self._guard("embeddings", probe) + + def _mcp_dependency(self) -> HealthCheck: + def probe() -> HealthCheck: + import importlib.util + spec = importlib.util.find_spec("mcp") + if spec is None: + return HealthCheck( + "mcp", STATUS_WARN, + "the 'mcp' package is not installed; " + "'openmind mcp serve' will not start (pip install mcp)", + {"available": False}) + return HealthCheck("mcp", STATUS_OK, "the 'mcp' package is available", + {"available": True}) + + return self._guard("mcp", probe) + + def _model_config(self) -> HealthCheck: + def probe() -> HealthCheck: + cfg = self._db.get_model_config() + model_path = str(cfg.get("model_path") or "") + server = str(cfg.get("llama_server_path") or "") + data = {"configured": bool(model_path), + "host": cfg.get("host"), "port": cfg.get("port"), + "llama_server_path": server} + if not model_path: + return HealthCheck( + "model_config", STATUS_WARN, + "no local model is configured; ingestion, extraction and " + "artifact export do not need one", data) + if not Path(model_path).exists(): + data["model_path"] = model_path + return HealthCheck( + "model_config", STATUS_WARN, + f"configured model file is missing: {model_path}", data) + data["model_path"] = model_path + return HealthCheck("model_config", STATUS_OK, + f"model configured: {Path(model_path).name}", data) + + return self._guard("model_config", probe) + + def _model_readiness(self) -> HealthCheck: + def probe() -> HealthCheck: + from .. import llm_client + base = llm_client.base_url() + data = {"base_url": base, "loopback": llm_client.is_local_endpoint()} + if not llm_client.is_local_endpoint(): + # A non-loopback model endpoint would send project content off + # the machine. That IS an error. + return HealthCheck( + "model_server", STATUS_ERROR, + f"model endpoint is not loopback: {base}", data) + ready = self.model_ready() + data["ready"] = ready + if not ready: + return HealthCheck( + "model_server", STATUS_WARN, + "no local model server is responding; model-dependent " + "features (Ask) are unavailable, everything else works", data) + return HealthCheck("model_server", STATUS_OK, f"ready at {base}", data) + + return self._guard("model_server", probe) + + def _network_policy(self) -> HealthCheck: + def probe() -> HealthCheck: + from .. import netguard + data = { + "enrich_egress": config.ENRICH_EGRESS, + "sourcelink_egress": config.SOURCELINK_EGRESS, + "allowed_hosts": sorted(config.ALLOWED_HOSTS), + "outbound_calls_logged": len(netguard.get_log(1000)), + "audit_log": str(config.OUTBOUND_LOG), + } + enabled = [n for n, on in (("enrichment", config.ENRICH_EGRESS), + ("source-link", config.SOURCELINK_EGRESS)) + if on] + detail = ("loopback only" if not enabled + else "loopback + audited egress: " + ", ".join(enabled)) + return HealthCheck("network_policy", STATUS_OK, detail, data) + + return self._guard("network_policy", probe) + + +__all__ = ["HealthService"] diff --git a/openmind/services/ingest_service.py b/openmind/services/ingest_service.py new file mode 100644 index 0000000..b1abeae --- /dev/null +++ b/openmind/services/ingest_service.py @@ -0,0 +1,108 @@ +"""Ingest use cases. + +Thin on purpose. The ingestion ALGORITHM is not rewritten in this phase — the +incremental walk, per-file hashing, semantic chunking, glossary and structure +builds, embedding batching and file-boundary checkpoints all stay exactly where +they are, in the persistent job engine. This service only owns the use-case +shape around them: validate the workspace, resolve an optional path filter, +enqueue, and optionally wait. + +The one thing it does add is the requirement that a worker actually exists. +Before this phase, ``jobs.start_worker()`` was called only from the FastAPI +lifespan, so an ingest enqueued from any other process sat ``queued`` forever. +The runtime now starts the worker on demand, and ``start(wait=True)`` asks for +it explicitly. +""" +from __future__ import annotations + +from typing import Any, Callable, Dict, Optional + +from .. import machine +from ..domain.errors import InvalidRequest +from ..domain.types import JobWaitResult +from .job_service import DEFAULT_TIMEOUT, JobService +from .workspace_service import WorkspaceService + + +class IngestService: + """Start and observe ingestion for a workspace.""" + + def __init__(self, workspaces: WorkspaceService, jobs: JobService, + ensure_worker: Optional[Callable[[], None]] = None) -> None: + self._workspaces = workspaces + self._jobs = jobs + # Supplied by the runtime. Without it a --wait ingest would block on a + # job no worker will ever pick up. + self._ensure_worker = ensure_worker + + def start(self, workspace_id: str, path: Optional[str] = None, + wait: bool = False, + timeout: float = DEFAULT_TIMEOUT) -> Dict[str, Any]: + """Enqueue an ingest, optionally waiting for it to finish. + + *path* restricts the ingest to a subtree of the workspace's registered + source. It is stored relative to the workspace root by the engine, so an + absolute machine path never reaches the portable database. + + Without *wait* this returns as soon as the job row is persisted, so the + caller gets a durable job id it can poll later. + """ + workspace = self._workspaces.get(workspace_id) + if not self._workspaces.paths(workspace_id): + raise InvalidRequest( + "workspace has no registered source path; add one before " + "ingesting", + details={"workspace_id": workspace_id}) + + if path is not None and not str(path).strip(): + raise InvalidRequest("path filter must not be blank", + details={"field": "path"}) + + if wait and self._ensure_worker is not None: + self._ensure_worker() + + job = self._jobs.enqueue_ingest(workspace_id, path) + result: Dict[str, Any] = { + "workspace_id": workspace_id, + "workspace": workspace.get("name"), + "job_id": job["job_id"], + "job": job, + "waited": False, + } + if not wait: + return result + + outcome: JobWaitResult = self._jobs.wait_for_terminal( + job["job_id"], timeout=timeout) + result.update({ + "waited": True, + "job": outcome.job, + "status": outcome.status, + "completed": outcome.completed, + "waited_seconds": round(outcome.waited_seconds, 3), + }) + return result + + def status(self, workspace_id: str) -> Dict[str, Any]: + """Ingest state for a workspace: its own state plus its ingest jobs, + newest first.""" + workspace = self._workspaces.get(workspace_id) + ingest_jobs = [j for j in self._jobs.list([workspace_id]) + if j.get("type") == "ingest"] + active = [j for j in ingest_jobs + if j.get("status") in ("queued", "running")] + return { + "workspace_id": workspace_id, + "state": workspace.get("state"), + "source_root": self._workspaces.source_root(workspace_id), + "active_job": active[0] if active else None, + "jobs": ingest_jobs, + } + + def relative_path(self, workspace_id: str, path: str) -> str: + """A workspace-relative form of *path*, for callers that hold an + absolute machine path and must not leak it into stored state.""" + return machine.to_rel(workspace_id, path) + + +__all__ = ["IngestService"] diff --git a/openmind/services/job_service.py b/openmind/services/job_service.py new file mode 100644 index 0000000..618d9b2 --- /dev/null +++ b/openmind/services/job_service.py @@ -0,0 +1,152 @@ +"""Job use cases — enqueue, inspect, control, and bounded waiting. + +This wraps the EXISTING single-worker job engine; it does not replace it. Phase +1 explicitly preserves persistent job state, resumable interrupted jobs, pause +and resume, cancellation, file-boundary checkpoints, asynchronous deletion, +restart recovery and SSE compatibility. The value added here is a stable +surface that the CLI, MCP and tests can drive, plus a bounded wait that does not +exist anywhere today. + +The reads deliberately go through the job *reader* (the database) rather than +through :mod:`openmind.jobs`, because that is where job state actually lives — +there is no public ``get_job`` in the engine module. +""" +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +from .. import db as db_module +from .. import jobs as jobs_module +from ..domain.errors import InvalidRequest, JobNotFound, OperationTimeout +from ..domain.types import (ACTIVE_JOB_STATUSES, SETTLED_JOB_STATUSES, + TERMINAL_JOB_STATUSES, JOB_STATUS_DONE, + JobWaitResult) +from ..ports.job_repository import JobEngine, JobReader +from ..ports.runtime_ports import Clock, SystemClock + +#: How often a bounded wait re-reads persisted job state. Job progress is +#: written on every tick, and the SSE stream polls at 0.8s; matching that keeps +#: CLI latency comparable to the UI without adding database pressure. +DEFAULT_POLL_INTERVAL = 0.5 + +#: Default ceiling for ``wait_for_terminal``. Generous, because a first ingest +#: of a large repository legitimately takes a long time; callers that want a +#: tighter bound pass one. +DEFAULT_TIMEOUT = 3600.0 + + +class JobService: + """Use cases over the persistent job engine.""" + + def __init__(self, reader: Optional[JobReader] = None, + engine: Optional[JobEngine] = None, + clock: Optional[Clock] = None) -> None: + self._reader: Any = reader if reader is not None else db_module + self._engine: Any = engine if engine is not None else jobs_module + self._clock: Clock = clock if clock is not None else SystemClock() + + # -- enqueue ------------------------------------------------------------ + def enqueue_ingest(self, workspace_id: str, + path: Optional[str] = None) -> Dict[str, Any]: + """Queue an ingest. The engine dedupes: an ingest already queued or + running for this workspace is returned instead of a second one.""" + return self._engine.enqueue_ingest(workspace_id, path) + + def enqueue_gendocs(self, workspace_id: str) -> Dict[str, Any]: + """Queue a generated-documentation build.""" + return self._engine.enqueue_gendocs(workspace_id) + + # -- reads -------------------------------------------------------------- + def get(self, job_id: str) -> Dict[str, Any]: + job = self._reader.get_job(job_id) + if not job: + raise JobNotFound(job_id) + return job + + def list(self, workspace_ids: Optional[List[str]] = None, + status: Optional[str] = None) -> List[Dict[str, Any]]: + return self._reader.list_jobs(workspace_ids, status) + + def active(self) -> List[Dict[str, Any]]: + """Every queued or running job, across all workspaces.""" + return self._reader.active_jobs() + + # -- control ------------------------------------------------------------ + # The engine raises KeyError for an unknown job id; translate it once here + # so every adapter gets a typed error instead of re-deriving the mapping. + def pause(self, job_id: str) -> Dict[str, Any]: + try: + return self._engine.pause(job_id) + except KeyError: + raise JobNotFound(job_id) from None + + def resume(self, job_id: str) -> Dict[str, Any]: + try: + return self._engine.resume(job_id) + except KeyError: + raise JobNotFound(job_id) from None + + def cancel(self, job_id: str) -> Dict[str, Any]: + try: + return self._engine.cancel_job(job_id) + except KeyError: + raise JobNotFound(job_id) from None + + # -- bounded waiting ---------------------------------------------------- + def wait_for_terminal(self, job_id: str, + timeout: float = DEFAULT_TIMEOUT, + poll_interval: float = DEFAULT_POLL_INTERVAL + ) -> JobWaitResult: + """Poll persisted job state until the job settles or *timeout* expires. + + Returns when the job is: + + * ``done`` -> ``completed=True`` + * ``failed`` -> ``completed=False`` (terminal, unsuccessful) + * ``paused`` / ``interrupted`` -> ``completed=False`` + + The last case matters: those are NOT terminal, but the worker will not + advance them without an explicit resume, so blocking until timeout + would be a hang, not a wait. The caller gets the real status and can + decide. + + Never caches: every poll re-reads the row, because job state is written + by the worker, request threads and cleanup threads alike. + + On timeout this raises :class:`OperationTimeout` — the job is NOT + cancelled and keeps running; it can still be polled or waited on again. + """ + if timeout <= 0: + raise InvalidRequest("timeout must be greater than zero", + details={"timeout": timeout}) + if poll_interval <= 0: + raise InvalidRequest("poll interval must be greater than zero", + details={"poll_interval": poll_interval}) + + started = self._clock.monotonic() + # Check before sleeping: an already-finished job must return instantly. + while True: + job = self.get(job_id) + status = str(job.get("status") or "") + if status in TERMINAL_JOB_STATUSES or status in SETTLED_JOB_STATUSES: + return JobWaitResult( + job=job, status=status, + completed=(status == JOB_STATUS_DONE), + waited_seconds=self._clock.monotonic() - started) + + elapsed = self._clock.monotonic() - started + if elapsed >= timeout: + raise OperationTimeout( + f"job {job_id} did not finish within {timeout:g}s " + f"(last status: {status or 'unknown'}); it is still running " + f"and can be polled again", + timeout=timeout, + details={"job_id": job_id, "status": status, + "waited_seconds": round(elapsed, 3)}) + self._clock.sleep(min(poll_interval, max(0.0, timeout - elapsed))) + + def is_active(self, job_id: str) -> bool: + return str(self.get(job_id).get("status") or "") in ACTIVE_JOB_STATUSES + + +__all__ = ["JobService", "DEFAULT_POLL_INTERVAL", "DEFAULT_TIMEOUT"] diff --git a/openmind/services/service_container.py b/openmind/services/service_container.py new file mode 100644 index 0000000..d91027d --- /dev/null +++ b/openmind/services/service_container.py @@ -0,0 +1,70 @@ +"""Lazily-constructed, cached application services. + +The container exists so every adapter shares one set of service instances +instead of each constructing its own. It is intentionally not a DI framework: +services are constructed by name, dependencies are wired explicitly in one +readable place, and the whole thing is about forty lines. + +Construction is lazy because the surfaces need different subsets. +``openmind export`` must not touch the vector store or the job engine, and +``doctor`` must not spawn a worker thread. A container that eagerly built +everything would make those commands pay for services they never call. +""" +from __future__ import annotations + +from typing import Callable, Optional + +from ..ports.runtime_ports import Clock +from .export_service import ExportService +from .health_service import HealthService +from .ingest_service import IngestService +from .job_service import JobService +from .workspace_service import WorkspaceService + + +class ServiceContainer: + """Holds the application services for one runtime.""" + + def __init__(self, ensure_worker: Optional[Callable[[], None]] = None, + clock: Optional[Clock] = None) -> None: + self._ensure_worker = ensure_worker + self._clock = clock + self._workspaces: Optional[WorkspaceService] = None + self._jobs: Optional[JobService] = None + self._ingest: Optional[IngestService] = None + self._export: Optional[ExportService] = None + self._health: Optional[HealthService] = None + + @property + def workspaces(self) -> WorkspaceService: + if self._workspaces is None: + self._workspaces = WorkspaceService() + return self._workspaces + + @property + def jobs(self) -> JobService: + if self._jobs is None: + self._jobs = JobService(clock=self._clock) + return self._jobs + + @property + def ingest(self) -> IngestService: + if self._ingest is None: + self._ingest = IngestService(self.workspaces, self.jobs, + ensure_worker=self._ensure_worker) + return self._ingest + + @property + def export(self) -> ExportService: + if self._export is None: + self._export = ExportService() + return self._export + + @property + def health(self) -> HealthService: + if self._health is None: + self._health = HealthService() + return self._health + + +__all__ = ["ServiceContainer"] diff --git a/openmind/services/workspace_service.py b/openmind/services/workspace_service.py new file mode 100644 index 0000000..1fc238a --- /dev/null +++ b/openmind/services/workspace_service.py @@ -0,0 +1,211 @@ +"""Workspace use cases — create, inspect, path registration, template +selection, termination and deletion. + +VOCABULARY: "workspace" is internal naming only. The stored entity is still a +project, the REST API still says ``/projects``, and ``workspace_id`` IS the +existing ``p_*`` project id. Nothing about the persisted shape changes. + +This service holds the orchestration that used to live inside FastAPI route +bodies, so the same use cases are reachable from the CLI, MCP and tests without +constructing an HTTP request. It returns plain dictionaries — the same shapes +the REST API already publishes. + +NOT here, on purpose: + +* Template AUTO-selection. The ingest job already scores templates against the + walked file list (``jobs._run_ingest``). Re-implementing it here would create + a second, divergent selector. This service only reads the recorded result and + owns the explicit user OVERRIDE. +* Path normalization. The existing REST routes store the path exactly as given, + and path matching elsewhere depends on that. Callers that need a resolved + absolute path (the CLI does) normalize before calling. +""" +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +from .. import db as db_module +from .. import jobs as jobs_module +from .. import machine, mapio, templates, vectorstore +from ..domain.errors import InvalidRequest, WorkspaceNotFound +from ..ports.job_repository import JobEngine +from ..ports.workspace_repository import WorkspaceRepository + + +class WorkspaceService: + """Use cases over the project/workspace entity.""" + + def __init__(self, repo: Optional[WorkspaceRepository] = None, + engine: Optional[JobEngine] = None) -> None: + self._repo: Any = repo if repo is not None else db_module + self._engine: Any = engine if engine is not None else jobs_module + + # -- reads -------------------------------------------------------------- + def get(self, workspace_id: str) -> Dict[str, Any]: + """The workspace record, or raise :class:`WorkspaceNotFound`. + + A workspace being deleted is already reported as gone by the repository + (its storage is reclaimed in the background), so this raises for it too + rather than returning a half-alive record. + """ + record = self._repo.get_project(workspace_id) + if not record: + raise WorkspaceNotFound(workspace_id) + return record + + def list(self, state: Optional[str] = None) -> List[Dict[str, Any]]: + return self._repo.list_projects(state) + + def exists(self, workspace_id: str) -> bool: + return self._repo.get_project(workspace_id) is not None + + def describe(self, workspace_id: str) -> Dict[str, Any]: + """The workspace record decorated with live store counts — the shape + ``GET /projects/{id}`` returns.""" + record = self.get(workspace_id) + record["code_chunks"] = vectorstore.get_code_store(workspace_id).count() + record["cases_count"] = vectorstore.get_cases_store(workspace_id).count() + record["files_indexed"] = len(self._repo.get_file_index(workspace_id)) + return record + + # -- creation ----------------------------------------------------------- + def create(self, name: str, path: Optional[str] = None, + exclude: Optional[List[str]] = None) -> Dict[str, Any]: + """Create a workspace and optionally register its first source path. + + Creation never ingests. Learning is an explicit, separate step + (``POST /ingest`` or ``openmind ingest``), so creating a workspace stays + instant and side-effect-light. + """ + clean = (name or "").strip() + if not clean: + raise InvalidRequest("workspace name must not be empty", + details={"field": "name"}) + record = self._repo.create_project(clean) + if path: + self._repo.upsert_project_path(record["id"], path, list(exclude or [])) + record = self.get(record["id"]) + return record + + # -- paths -------------------------------------------------------------- + def add_path(self, workspace_id: str, path: str, + exclude: Optional[List[str]] = None) -> Dict[str, Any]: + """Register or update a source path and its exclude set. + + The path is stored in the machine-local sidecar, never in the portable + database — that is what keeps a copied ``data/`` folder free of an + absolute origin path. The value is stored verbatim; see the module + docstring on normalization. + """ + self.get(workspace_id) # 404 before touching the sidecar + if not (path or "").strip(): + raise InvalidRequest("path must not be empty", details={"field": "path"}) + self._repo.upsert_project_path(workspace_id, path, list(exclude or [])) + return self.get(workspace_id) + + def remove_path(self, workspace_id: str, path: str) -> Dict[str, Any]: + self.get(workspace_id) + if not (path or "").strip(): + raise InvalidRequest("path must not be empty", details={"field": "path"}) + self._repo.remove_project_path(workspace_id, path) + return self.get(workspace_id) + + def paths(self, workspace_id: str) -> List[Dict[str, Any]]: + self.get(workspace_id) + return self._repo.get_project_paths(workspace_id) + + def source_root(self, workspace_id: str) -> str: + """The machine-local absolute root the workspace's relative paths hang + off, or '' when unset (data copied to a machine that has not pointed the + root yet).""" + return machine.project_root(workspace_id) + + # -- template selection ------------------------------------------------- + def template_selection(self, workspace_id: str) -> Dict[str, Any]: + """What the user chose, what detection recorded, and which is effective.""" + return templates.selection_info(self.get(workspace_id)) + + def set_template(self, workspace_id: str, + name: Optional[str]) -> Dict[str, Any]: + """Set or clear the explicit template override. + + An empty/None name CLEARS the override and falls back to the recorded + auto-selection. A name that does not resolve is rejected here rather + than silently ignored — the caller said "use THIS one". Names are + lower-cased, matching the existing REST behaviour. + """ + record = self.get(workspace_id) + clean = (name or "").strip().lower() + if clean and not templates.get_template(clean): + raise InvalidRequest( + f"unknown or invalid template: {clean!r} " + "(GET /templates lists what is available)", + details={"template": clean, + "available": [t["name"] for t in templates.list_templates()]}) + meta = dict(record.get("meta") or {}) + if clean: + meta["template"] = clean + else: + meta.pop("template", None) + self._repo.update_project_meta(workspace_id, meta) + return templates.selection_info(self.get(workspace_id)) + + # -- status ------------------------------------------------------------- + def status(self, workspace_id: str) -> Dict[str, Any]: + """Everything ``openmind status`` reports for one workspace. + + Counts come from the live stores. A store that cannot be opened reports + ``None`` rather than 0 — a false zero would read as "nothing indexed" + when the truth is "could not tell". ``doctor`` reports backend health + separately, so the reason is never lost. + """ + record = self.get(workspace_id) + return { + "workspace": record, + "paths": self._repo.get_project_paths(workspace_id), + "state": record.get("state"), + "template": templates.selection_info(record), + "counts": { + "indexed_files": _count( + lambda: len(self._repo.get_file_index(workspace_id))), + "code_chunks": _count( + lambda: vectorstore.get_code_store(workspace_id).count()), + "solved_cases": _count( + lambda: vectorstore.get_cases_store(workspace_id).count()), + "glossary_terms": _count( + lambda: len(mapio.load_glossary(workspace_id).get("terms") or {})), + }, + } + + # -- lifecycle ---------------------------------------------------------- + def terminate(self, workspace_id: str, + clear_cases: bool = False) -> Dict[str, Any]: + """Drop everything learned about the workspace, keeping the workspace + itself. Synchronous: it stops running jobs first (bounded wait).""" + self.get(workspace_id) + return self._engine.terminate_project(workspace_id, clear_cases=clear_cases) + + def request_delete(self, workspace_id: str) -> Dict[str, Any]: + """Tombstone the workspace and reclaim its storage in the background. + + Returns immediately. The workspace is treated as gone from this point; + the row is dropped for real when cleanup finishes. + """ + self.get(workspace_id) + return self._engine.request_delete(workspace_id) + + +def _count(fn) -> Optional[int]: + """A store count, or None when the store cannot be read. + + Broad by design and narrow in effect: the ONLY thing it hides is "this + optional backend would not open", and it reports that as ``None`` (unknown) + rather than 0 (empty). Every caller renders None distinctly. + """ + try: + return int(fn()) + except Exception: + return None + + +__all__ = ["WorkspaceService"] diff --git a/openmind/version.py b/openmind/version.py new file mode 100644 index 0000000..30a055e --- /dev/null +++ b/openmind/version.py @@ -0,0 +1,23 @@ +"""The single runtime version constant. + +Everything that reports a version — the CLI ``--version``, ``doctor``, the +FastAPI app + ``GET /api/health``, and the ``.openmind`` artifact manifest's +``generator.version`` — reads it from here, so the value can never drift +between surfaces. + +This is a PRE-v2 development version on purpose. OpenMind v2's enterprise +knowledge features (Asset/Revision/Claim tables, the engineering Knowledge +Graph, cloud providers, traceability) are NOT implemented; labelling the +current build ``2.0.0`` would be a false claim. ``1.1.0-dev`` is the honest +reading: the shipped artifact contract is schema 1.1.x, and this is +development work on top of it. + +The artifact ``schemaVersion`` is deliberately NOT derived from this constant — +it is a separate, frozen integration contract owned by +:mod:`openmind.artifacts`. +""" +from __future__ import annotations + +RUNTIME_VERSION = "1.1.0-dev" + +__all__ = ["RUNTIME_VERSION"] diff --git a/package.json b/package.json index 66ed01b..b1f6b1e 100644 --- a/package.json +++ b/package.json @@ -1,12 +1,14 @@ { "name": "open-mind", - "version": "0.1.0", + "version": "1.1.0-dev", "private": true, "description": "Task-runner shims for Open Mind (a Python project). Each script delegates to the Python implementation so the documented cross-repo commands work verbatim; Node.js is never required to use Open Mind itself.", "scripts": { "analyze": "python -m openmind.artifacts", "build": "python -m compileall -q openmind", "test": "python tests/verify_artifacts.py", + "acceptance": "python scripts/run_acceptance.py", + "doctor": "python -m openmind.cli doctor", "clean": "python -c \"import shutil; shutil.rmtree('.openmind', ignore_errors=True)\"" } } diff --git a/scripts/run_acceptance.py b/scripts/run_acceptance.py new file mode 100644 index 0000000..8d1bfd0 --- /dev/null +++ b/scripts/run_acceptance.py @@ -0,0 +1,294 @@ +"""Run the OpenMind acceptance suite. + + python scripts/run_acceptance.py # the CI gate (core tier) + python scripts/run_acceptance.py --all # every supported script + python scripts/run_acceptance.py --tier local + python scripts/run_acceptance.py --list + python scripts/run_acceptance.py --json + +Each script runs in its own process with its own isolated ``OPENMIND_DATA_DIR`` +and ``OPENMIND_MACHINE_DIR``, and with egress and embeddings forced offline, so +a run is deterministic, touches no live data, and calls no paid or public API. + +A SKIP IS NEVER A PASS +---------------------- +Every script on disk must appear in :data:`SUITE`. A ``tests/verify_*.py`` that +is not listed makes the runner fail with "not in the manifest" rather than +quietly going unrun — that is the failure mode this file exists to prevent. A +script that cannot execute is reported as ``skipped`` and counted separately; +a skipped CORE script fails the run. +""" +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from typing import Any, Dict, List, Optional + +REPO_ROOT = Path(__file__).resolve().parent.parent +TESTS_DIR = REPO_ROOT / "tests" + +CORE = "core" +LOCAL = "local" + + +class Script: + """One acceptance script and why it is in the tier it is in.""" + + def __init__(self, name: str, tier: str, summary: str, + reason: str = "", timeout: int = 1800) -> None: + self.name = name + self.tier = tier + self.summary = summary + self.reason = reason # required for LOCAL: why it is not in CI + self.timeout = timeout + + @property + def path(self) -> Path: + return TESTS_DIR / f"{self.name}.py" + + +#: The supported acceptance suite. Ordered cheapest-first so an obvious +#: breakage surfaces in seconds rather than minutes. +SUITE: List[Script] = [ + # -- deterministic, dependency-light ------------------------------------ + Script("verify_migrations", CORE, "schema migrations: empty, legacy, checksum, rollback"), + Script("verify_structure", CORE, "detection + structure extraction, incremental"), + Script("verify_glossary", CORE, "deterministic glossary extraction and lookup"), + Script("verify_router", CORE, "capability routing and graceful degradation"), + Script("verify_diagrams", CORE, "deterministic diagram projection"), + Script("verify_grounding", CORE, "grounded answers and honest not-found"), + Script("verify_artifacts", CORE, ".openmind export contract (schema 1.1.x)"), + Script("verify_runtime", CORE, "runtime bootstrap, idempotency, worker opt-in"), + Script("verify_services", CORE, "application service layer"), + Script("verify_cli", CORE, "CLI contract: JSON, exit codes, commands"), + Script("verify_adapters", CORE, "FastAPI route + MCP + skill-bridge compatibility"), + + # -- template / docs pipeline ------------------------------------------- + Script("verify_templates", CORE, "template profile selection and validation"), + Script("verify_facets", CORE, "template facets, roles and projections"), + Script("verify_guide", CORE, "generated guide sections and citations"), + + # -- app-level behaviour ------------------------------------------------- + Script("verify", CORE, "end-to-end ingest, search and job lifecycle"), + Script("verify_fixes", CORE, "regression fixes (batch 1)"), + Script("verify_fixes2", CORE, "regression fixes (batch 2)"), + Script("verify_resources", CORE, "ingest RAM governor"), + Script("verify_ask", CORE, "Ask gating when no model is ready"), + Script("verify_ask2", CORE, "Ask history, cancellation and case saving"), + Script("verify_async_delete", CORE, "asynchronous project delete"), + Script("verify_source_link", CORE, + "GitHub link parsing + egress policy (no live network call)"), + Script("verify_modelserver", CORE, + "model-server attach/start logic against a local stub server"), + + # -- excluded from the CI gate, with a reason --------------------------- + Script("verify_delete_responsive", LOCAL, + "delete stays responsive under load", + reason="asserts sub-2-second API latency while a delete drains; a " + "shared CI runner's scheduling jitter makes that flaky, and a " + "flaky gate is worse than an honest exclusion. Run it locally " + "before releasing a change to the delete path."), +] + +BY_NAME = {s.name: s for s in SUITE} + + +def base_env() -> Dict[str, str]: + """A deterministic, offline environment. + + Embeddings fall back to the in-process hashing embedder, enrichment and + source-link egress are off, and the GPU is never touched. No paid API and no + public LLM endpoint is reachable from a run configured this way. + """ + env = dict(os.environ) + env.update({ + "OPENMIND_EMBED_OFFLINE": "1", + "OPENMIND_EMBED_DEVICE": "cpu", + "OPENMIND_INGEST_FREE_GPU": "0", + "OPENMIND_ENRICH_EGRESS": "0", + "OPENMIND_SOURCELINK_EGRESS": "0", + "PYTHONIOENCODING": "utf-8", + "PYTHONUTF8": "1", + }) + # Never inherit the developer's live data dir; tests/_isolate.py also + # guards this, but the runner must not depend on the script doing so. + env.pop("OPENMIND_DATA_DIR", None) + env.pop("OPENMIND_MACHINE_DIR", None) + return env + + +def run_script(script: Script, verbose: bool = False) -> Dict[str, Any]: + """Run one script in an isolated data directory. Never raises.""" + if not script.path.is_file(): + return {"name": script.name, "tier": script.tier, "status": "skipped", + "reason": f"script not found: {script.path}", "seconds": 0.0} + + env = base_env() + env["OPENMIND_DATA_DIR"] = tempfile.mkdtemp(prefix=f"om_acc_{script.name}_") + env["OPENMIND_MACHINE_DIR"] = tempfile.mkdtemp(prefix=f"om_mach_{script.name}_") + + started = time.time() + try: + proc = subprocess.run( + [sys.executable, str(script.path)], + cwd=str(REPO_ROOT), env=env, timeout=script.timeout, + capture_output=True, text=True, errors="replace") + except subprocess.TimeoutExpired: + return {"name": script.name, "tier": script.tier, "status": "failed", + "reason": f"timed out after {script.timeout}s", + "seconds": round(time.time() - started, 1)} + except OSError as exc: + return {"name": script.name, "tier": script.tier, "status": "skipped", + "reason": f"could not start: {exc}", + "seconds": round(time.time() - started, 1)} + + elapsed = round(time.time() - started, 1) + # The verdict line ("N passed, M failed") is on STDOUT. stderr carries + # library warnings, so summarizing the combined tail would report a + # third-party warning where the result belongs. + out_lines = [ln for ln in (proc.stdout or "").splitlines() if ln.strip()] + err_lines = [ln for ln in (proc.stderr or "").splitlines() if ln.strip()] + result = { + "name": script.name, + "tier": script.tier, + "status": "passed" if proc.returncode == 0 else "failed", + "exit_code": proc.returncode, + "seconds": elapsed, + "summary": (out_lines[-1] if out_lines + else err_lines[-1] if err_lines else "").strip(), + } + if proc.returncode != 0 or verbose: + # Keep enough output to diagnose without dumping a whole ingest log. + # On failure the FAIL lines are what matter, so show those first. + failures = [ln for ln in out_lines if ln.startswith("FAIL")] + result["output"] = "\n".join(failures[-20:] + out_lines[-20:] + + err_lines[-10:]) + return result + + +def check_manifest() -> List[str]: + """Every ``tests/verify_*.py`` must be in :data:`SUITE`. + + Without this, adding a test file and forgetting to register it would mean + the suite silently stops covering it while still reporting success. + """ + on_disk = {p.stem for p in TESTS_DIR.glob("verify*.py")} + return sorted(on_disk - set(BY_NAME)) + + +def main(argv: Optional[List[str]] = None) -> int: + parser = argparse.ArgumentParser( + prog="python scripts/run_acceptance.py", + description="Run the OpenMind acceptance suite in isolated, offline " + "environments.") + parser.add_argument("--tier", choices=[CORE, LOCAL], default=CORE, + help="which tier to run (default: core, the CI gate)") + parser.add_argument("--all", action="store_true", + help="run every tier") + parser.add_argument("--only", action="append", metavar="NAME", + help="run only this script (repeatable)") + parser.add_argument("--fail-fast", action="store_true", + help="stop at the first failure") + parser.add_argument("--json", action="store_true", dest="as_json", + help="print one machine-readable JSON object on stdout") + parser.add_argument("--list", action="store_true", dest="list_only", + help="list the suite and exit") + parser.add_argument("--verbose", "-v", action="store_true", + help="include output for passing scripts too") + args = parser.parse_args(argv) + + log = (lambda m: print(m, file=sys.stderr, flush=True)) if args.as_json \ + else (lambda m: print(m, flush=True)) + + unregistered = check_manifest() + if unregistered: + message = ("acceptance manifest is out of date; these scripts exist but " + "are not listed in SUITE: " + ", ".join(unregistered)) + if args.as_json: + print(json.dumps({"ok": False, "error": message}, indent=2)) + else: + log("error: " + message) + log("Add them to SUITE in scripts/run_acceptance.py " + "(with a tier and, for 'local', a reason).") + return 2 + + if args.list_only: + for script in SUITE: + log(f"{script.tier:6} {script.name:26} {script.summary}") + if script.reason: + log(f"{'':6} {'':26} excluded from CI: {script.reason}") + return 0 + + if args.only: + unknown = [n for n in args.only if n not in BY_NAME] + if unknown: + log(f"error: unknown script(s): {', '.join(unknown)}") + return 2 + selected = [BY_NAME[n] for n in args.only] + elif args.all: + selected = list(SUITE) + else: + selected = [s for s in SUITE if s.tier == args.tier] + + log(f"Running {len(selected)} acceptance script(s) " + f"[{'all tiers' if args.all else args.tier}] with " + f"Python {sys.version.split()[0]}") + log("") + + results: List[Dict[str, Any]] = [] + for script in selected: + log(f" {script.name:26} ... ") + result = run_script(script, verbose=args.verbose) + results.append(result) + mark = {"passed": "PASS", "failed": "FAIL", "skipped": "SKIP"}[result["status"]] + log(f" {script.name:26} {mark} {result['seconds']:>6.1f}s " + f"{result.get('summary', result.get('reason', ''))}") + if result["status"] == "failed" and result.get("output"): + for line in result["output"].splitlines(): + log(f" | {line}") + if args.fail_fast and result["status"] != "passed": + log("\nstopping at first failure (--fail-fast)") + break + + passed = [r for r in results if r["status"] == "passed"] + failed = [r for r in results if r["status"] == "failed"] + skipped = [r for r in results if r["status"] == "skipped"] + # A skipped CORE script is a failure: the gate must never report success + # for coverage it did not actually run. + skipped_core = [r for r in skipped if r["tier"] == CORE] + not_run = [s.name for s in selected + if s.name not in {r["name"] for r in results}] + + ok = not failed and not skipped_core and not not_run + + log("") + log(f"{len(passed)} passed, {len(failed)} failed, {len(skipped)} skipped" + + (f", {len(not_run)} not run" if not_run else "")) + if failed: + log("failed: " + ", ".join(r["name"] for r in failed)) + if skipped_core: + log("SKIPPED CORE SCRIPTS (counted as failure): " + + ", ".join(f"{r['name']} ({r.get('reason', '')})" for r in skipped_core)) + if skipped and not skipped_core: + log("skipped (non-core): " + ", ".join(r["name"] for r in skipped)) + + if args.as_json: + print(json.dumps({ + "ok": ok, + "tier": "all" if args.all else args.tier, + "counts": {"passed": len(passed), "failed": len(failed), + "skipped": len(skipped), "not_run": len(not_run)}, + "results": results, + }, indent=2)) + + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/verify_adapters.py b/tests/verify_adapters.py new file mode 100644 index 0000000..52c64df --- /dev/null +++ b/tests/verify_adapters.py @@ -0,0 +1,341 @@ +"""Adapter compatibility — FastAPI routes, MCP construction, and the skill bridge. + +Phase 1 moved orchestration out of the route bodies and into the service layer. +The point of this suite is that NOTHING externally visible changed: the same +paths, the same methods, the same status codes, the same response keys, the same +nine MCP tools, and the same skill-bridge protocol. +""" +import json +import os +import subprocess +import sys +import tempfile + +os.environ.setdefault("OPENMIND_DATA_DIR", tempfile.mkdtemp()) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import _isolate # noqa: E402,F401 — forces an isolated data dir (never the live one) + +from fastapi.testclient import TestClient # noqa: E402 + +from openmind.main import app # noqa: E402 +from openmind.version import RUNTIME_VERSION # noqa: E402 + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +FIXTURE = os.path.join(REPO, "fixtures", "sample-repo") + +_results = [] + + +def check(desc, cond, extra=""): + _results.append((desc, bool(cond))) + print(("PASS" if cond else "FAIL") + " - " + desc + + (f" [{extra}]" if extra and not cond else "")) + + +# --------------------------------------------------------------------------- +# The published route table must not shrink or drift +# --------------------------------------------------------------------------- +ROUTES = {(m, r.path) for r in app.routes + for m in getattr(r, "methods", set()) or set()} + +EXPECTED = [ + ("GET", "/"), ("GET", "/api/health"), ("GET", "/netlog"), + ("GET", "/system/load"), + ("GET", "/projects"), ("POST", "/projects"), + ("GET", "/projects/{project_id}"), ("DELETE", "/projects/{project_id}"), + ("POST", "/projects/{project_id}/paths"), + ("DELETE", "/projects/{project_id}/paths"), + ("POST", "/projects/{project_id}/selection"), + ("GET", "/projects/{project_id}/template"), + ("POST", "/projects/{project_id}/template"), + ("POST", "/projects/{project_id}/source-link"), + ("POST", "/projects/{project_id}/terminate"), + ("GET", "/templates"), ("GET", "/scope/{scope_id}"), + ("GET", "/fs/tree"), ("GET", "/fs/list"), ("GET", "/fs/pick-folder"), + ("GET", "/model-config"), ("POST", "/model-config"), + ("POST", "/server/start"), ("POST", "/server/stop"), + ("POST", "/server/restart"), ("GET", "/server/status"), + ("POST", "/ingest"), ("POST", "/gendocs"), + ("GET", "/jobs"), ("GET", "/jobs/{job_id}"), + ("POST", "/jobs/{job_id}/pause"), ("POST", "/jobs/{job_id}/resume"), + ("POST", "/jobs/{job_id}/cancel"), ("GET", "/jobs/{job_id}/stream"), + ("GET", "/glossary"), ("POST", "/glossary/enrich"), + ("POST", "/glossary/enrich/auto"), + ("GET", "/source"), ("GET", "/structure"), ("GET", "/route"), + ("GET", "/dispatch"), ("GET", "/graph"), ("GET", "/graph/children"), + ("GET", "/graph/node"), + ("POST", "/search"), ("POST", "/ocr"), + ("POST", "/ask"), ("GET", "/ask/history"), + ("GET", "/ask/exchange/{exchange_id}"), ("POST", "/ask/clear"), + ("POST", "/ask/save-case"), + ("POST", "/cases"), ("GET", "/cases"), + ("GET", "/docs"), ("GET", "/docs/{page}"), +] +missing = [f"{m} {p}" for m, p in EXPECTED if (m, p) not in ROUTES] +check(f"all {len(EXPECTED)} pre-existing routes are still registered", + not missing, "missing: " + ", ".join(missing)) +check("/projects was NOT renamed to /workspaces in the public API", + not any(p.startswith("/workspaces") for _m, p in ROUTES)) + +with TestClient(app) as c: + # ----------------------------------------------------------------------- + # Health — additive only + # ----------------------------------------------------------------------- + r = c.get("/api/health") + health = r.json() + check("GET /api/health is 200", r.status_code == 200) + for key in ("ok", "embeddings_backend", "vectorstore_backend", + "llm_base_url", "llm_local", "outbound_calls_logged", "server"): + check(f"/api/health still returns the pre-existing key '{key}'", + key in health) + check("/api/health now also reports the runtime version", + health.get("version") == RUNTIME_VERSION) + check("/api/health now also reports the schema version", + health.get("schema_version", 0) >= 2) + check("/api/health stays cheap by default (no diagnostics probe)", + "diagnostics" not in health) + + r = c.get("/api/health", params={"diagnostics": "1"}) + full = r.json() + check("/api/health?diagnostics=1 exposes the doctor report", + isinstance(full.get("diagnostics", {}).get("checks"), list)) + check("the HTTP diagnostics report matches the doctor contract", + full["diagnostics"].get("version") == RUNTIME_VERSION + and "ok" in full["diagnostics"]) + + # ----------------------------------------------------------------------- + # Projects — created through the service, unchanged over HTTP + # ----------------------------------------------------------------------- + r = c.post("/projects", json={"name": "adapter demo"}) + pid = r.json()["id"] + check("POST /projects is 200 and returns a project record", + r.status_code == 200 and pid.startswith("p_")) + for key in ("id", "name", "state", "paths", "created_at", "updated_at", "meta"): + check(f"POST /projects still returns '{key}'", key in r.json()) + + r = c.post("/projects", json={"name": "with path", "path": FIXTURE, + "exclude": ["target"]}) + check("POST /projects registers an initial path", + r.status_code == 200 and len(r.json()["paths"]) == 1) + check("POST /projects records the exclude set", + r.json()["paths"][0]["exclude"] == ["target"]) + pid_with_path = r.json()["id"] + + r = c.get("/projects") + check("GET /projects lists projects", + r.status_code == 200 and isinstance(r.json()["projects"], list)) + + r = c.get(f"/projects/{pid}") + check("GET /projects/{id} is 200", r.status_code == 200) + for key in ("code_chunks", "cases_count", "files_indexed"): + check(f"GET /projects/{{id}} still decorates the record with '{key}'", + key in r.json()) + + r = c.get("/projects/p_does_not_exist") + check("GET /projects/{id} 404s for an unknown project", r.status_code == 404) + check("the 404 body still carries 'detail' (FastAPI's shape)", + "detail" in r.json()) + check("the 404 body ALSO carries a machine-readable error code", + r.json().get("error", {}).get("code") == "workspace_not_found") + + # -- paths + r = c.post(f"/projects/{pid}/paths", json={"path": FIXTURE, "exclude": []}) + check("POST /projects/{id}/paths is 200", r.status_code == 200) + check("POST /projects/{id}/paths returns the updated project", + len(r.json()["paths"]) == 1) + r = c.post("/projects/p_nope/paths", json={"path": FIXTURE, "exclude": []}) + check("POST /projects/{id}/paths 404s for an unknown project", + r.status_code == 404) + + r = c.post(f"/projects/{pid}/selection", + json={"path": FIXTURE, "exclude": ["build"]}) + check("POST /projects/{id}/selection still works", r.status_code == 200) + check("selection updates the exclude set", + r.json()["paths"][0]["exclude"] == ["build"]) + + r = c.request("DELETE", f"/projects/{pid}/paths", params={"path": FIXTURE}) + check("DELETE /projects/{id}/paths is 200", r.status_code == 200) + check("DELETE /projects/{id}/paths removes the path", r.json()["paths"] == []) + # Before Phase 1 this returned None from a handler annotated -> Dict, which + # FastAPI's response validation turned into a 500 ResponseValidationError. + r = c.request("DELETE", "/projects/p_nope/paths", params={"path": FIXTURE}) + check("DELETE /projects/{id}/paths now 404s instead of raising a 500 " + "(documented behaviour fix)", r.status_code == 404, str(r.status_code)) + + # -- templates + r = c.get("/templates") + check("GET /templates lists profiles", + r.status_code == 200 and "templates" in r.json() and "user_dir" in r.json()) + + r = c.get(f"/projects/{pid}/template") + check("GET /projects/{id}/template is 200", r.status_code == 200) + for key in ("override", "override_error", "auto", "effective", + "effective_source"): + check(f"template selection still returns '{key}'", key in r.json()) + check("GET /projects/{id}/template 404s for an unknown project", + c.get("/projects/p_nope/template").status_code == 404) + + r = c.post(f"/projects/{pid}/template", json={"name": "nope"}) + check("POST an unknown template name is still a 400", r.status_code == 400, + str(r.status_code)) + check("the 400 body still carries 'detail'", "detail" in r.json()) + r = c.post(f"/projects/{pid}/template", json={"name": None}) + check("POST template name=null clears the override", + r.status_code == 200 and r.json()["override"] is None) + + # ----------------------------------------------------------------------- + # Ingest + jobs + # ----------------------------------------------------------------------- + r = c.post("/ingest", json={"project_id": pid_with_path}) + check("POST /ingest is 200", r.status_code == 200) + check("POST /ingest still returns {job_id, job}", + "job_id" in r.json() and "job" in r.json()) + job_id = r.json()["job_id"] + check("the enqueued job carries the pre-existing job-record keys", + all(k in r.json()["job"] for k in + ("job_id", "project_id", "type", "status", "step", "progress", + "log_tail", "control", "error", "created_at", "updated_at"))) + + check("POST /ingest 404s for an unknown project", + c.post("/ingest", json={"project_id": "p_nope"}).status_code == 404) + check("POST /gendocs 404s for an unknown project", + c.post("/gendocs", json={"project_id": "p_nope"}).status_code == 404) + + r = c.get("/jobs", params={"scope": pid_with_path}) + check("GET /jobs?scope= lists that project's jobs", + r.status_code == 200 and any(j["job_id"] == job_id + for j in r.json()["jobs"])) + r = c.get("/jobs", params={"scope": "p_nope"}) + check("GET /jobs with an unresolvable scope lists nothing " + "(does not fall back to every project)", + r.status_code == 200 and r.json()["jobs"] == []) + + r = c.get(f"/jobs/{job_id}") + check("GET /jobs/{id} is 200", r.status_code == 200) + check("GET /jobs/{id} 404s for an unknown job", + c.get("/jobs/job_nope").status_code == 404) + check("POST /jobs/{id}/pause 404s for an unknown job", + c.post("/jobs/job_nope/pause").status_code == 404) + check("POST /jobs/{id}/resume 404s for an unknown job", + c.post("/jobs/job_nope/resume").status_code == 404) + check("POST /jobs/{id}/cancel 404s for an unknown job", + c.post("/jobs/job_nope/cancel").status_code == 404) + + r = c.post(f"/jobs/{job_id}/cancel") + check("POST /jobs/{id}/cancel is 200 for a real job", r.status_code == 200) + + # ----------------------------------------------------------------------- + # Terminate / delete + # ----------------------------------------------------------------------- + check("POST /projects/{id}/terminate 404s for an unknown project", + c.post("/projects/p_nope/terminate", + json={"clear_cases": False}).status_code == 404) + check("DELETE /projects/{id} 404s for an unknown project", + c.delete("/projects/p_nope").status_code == 404) + + r = c.delete(f"/projects/{pid}") + check("DELETE /projects/{id} still returns {deleting} immediately", + r.status_code == 200 and r.json().get("deleting") == pid) + check("a deleted project is 404 at once", + c.get(f"/projects/{pid}").status_code == 404) + +# --------------------------------------------------------------------------- +# MCP construction +# --------------------------------------------------------------------------- +import asyncio # noqa: E402 + +from openmind import mcp_server # noqa: E402 +from openmind.runtime import get_runtime # noqa: E402 + +REQUIRED_TOOLS = ("search", "route", "dispatch", "get_glossary", + "find_similar_cases", "save_case", "get_doc", "propose_fix", + "apply_fix") + +check("the MCP module publishes its tool set", len(mcp_server.TOOLS) == 9) +check("every documented MCP tool name is present", + set(REQUIRED_TOOLS) == set(mcp_server.TOOL_NAMES), + str(mcp_server.TOOL_NAMES)) + +server = mcp_server.create_mcp_server(get_runtime()) +check("create_mcp_server(runtime) builds a server", server is not None) +check("the server keeps the published name", server.name == "open-mind") + +registered = sorted(t.name for t in asyncio.run(server.list_tools())) +check("all nine tools are registered on the server", + registered == sorted(REQUIRED_TOOLS), str(registered)) + +descriptions = {t.name: (t.description or "") for t in asyncio.run(server.list_tools())} +check("every tool still carries its docstring description", + all(descriptions[n].strip() for n in REQUIRED_TOOLS)) + +second = mcp_server.create_mcp_server(get_runtime()) +check("create_mcp_server can be called repeatedly (independent instances)", + second is not server) +check("the lazy module-level `mcp` object still resolves", + mcp_server.mcp is not None) + +# Importing the module must not have side effects on the database. +probe = subprocess.run( + [sys.executable, "-c", + "import os, tempfile;" + "os.environ['OPENMIND_DATA_DIR']=tempfile.mkdtemp();" + "os.environ['OPENMIND_MACHINE_DIR']=tempfile.mkdtemp();" + "from openmind import config;" + "import openmind.mcp_server;" + "import pathlib;" + "print(pathlib.Path(config.DB_PATH).exists())"], + cwd=REPO, capture_output=True, text=True) +check("importing openmind.mcp_server no longer opens the database at import time", + probe.stdout.strip() == "False", probe.stdout + probe.stderr) + +# The documented command must still exist and be runnable. +probe = subprocess.run( + [sys.executable, "-c", + "import openmind.mcp_server as m; print(callable(m.main))"], + cwd=REPO, capture_output=True, text=True) +check("python -m openmind.mcp_server still has a main() entry point", + probe.stdout.strip() == "True", probe.stdout + probe.stderr) + +# --------------------------------------------------------------------------- +# Skill bridge — must still run the REAL implementation over the JSON protocol +# --------------------------------------------------------------------------- +env = dict(os.environ) +env.update({"OPENMIND_EMBED_OFFLINE": "1", "OPENMIND_ENRICH_EGRESS": "0", + "PYTHONIOENCODING": "utf-8"}) +proc = subprocess.Popen( + [sys.executable, "-m", "openmind.skill_bridge", "--root", FIXTURE], + cwd=REPO, env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, bufsize=1) +try: + ready = json.loads(proc.stdout.readline()) + check("skill bridge starts and prints a ready line", ready.get("ready") is True) + check("skill bridge walked the corpus", ready.get("files", 0) > 0) + check("skill bridge built the structure artifact", + ready.get("definitions", 0) > 0) + + proc.stdin.write(json.dumps({"id": 1, "op": "route", + "arg": "what does ISR mean"}) + "\n") + proc.stdin.flush() + reply = json.loads(proc.stdout.readline()) + check("skill bridge answers a route request", + reply.get("id") == 1 and reply.get("ok") is True) + check("the route reply carries the deterministic capability trace", + all(k in reply["result"] for k in + ("capability", "decided_by", "deterministic_fallback", "reason"))) + + proc.stdin.write(json.dumps({"id": 2, "op": "nonsense", "arg": "x"}) + "\n") + proc.stdin.flush() + reply = json.loads(proc.stdout.readline()) + check("skill bridge reports an unknown op as an error, and keeps serving", + reply.get("id") == 2 and reply.get("ok") is False and reply.get("error")) +finally: + proc.stdin.close() + try: + proc.wait(timeout=20) + except subprocess.TimeoutExpired: + proc.kill() + +bad = [d for d, ok in _results if not ok] +print(f"\n{len(_results) - len(bad)} passed, {len(bad)} failed") +sys.exit(1 if bad else 0) diff --git a/tests/verify_cli.py b/tests/verify_cli.py new file mode 100644 index 0000000..23c44ed --- /dev/null +++ b/tests/verify_cli.py @@ -0,0 +1,354 @@ +"""CLI contract — JSON output, exit codes, stream separation, and every command. + +Each case runs the CLI as a REAL subprocess. Calling ``cli.main()`` in-process +would test the parser but not the contract that actually matters: the exit code +the shell sees, and the guarantee that stdout carries JSON and nothing else. +""" +import json +import os +import subprocess +import sys +import tempfile + +os.environ.setdefault("OPENMIND_DATA_DIR", tempfile.mkdtemp()) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import _isolate # noqa: E402,F401 — forces an isolated data dir (never the live one) + +from openmind.version import RUNTIME_VERSION # noqa: E402 + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +FIXTURE = os.path.join(REPO, "fixtures", "sample-repo") + +_results = [] + + +def check(desc, cond, extra=""): + _results.append((desc, bool(cond))) + print(("PASS" if cond else "FAIL") + " - " + desc + + (f" [{extra}]" if extra and not cond else "")) + + +ENV = dict(os.environ) +ENV.update({ + "OPENMIND_DATA_DIR": tempfile.mkdtemp(prefix="om_cli_"), + "OPENMIND_MACHINE_DIR": tempfile.mkdtemp(prefix="om_cli_machine_"), + "OPENMIND_EMBED_OFFLINE": "1", + "OPENMIND_EMBED_DEVICE": "cpu", + "OPENMIND_INGEST_FREE_GPU": "0", + "OPENMIND_ENRICH_EGRESS": "0", + "OPENMIND_SOURCELINK_EGRESS": "0", + "PYTHONIOENCODING": "utf-8", +}) + + +def run(*args, timeout=900): + """Run the CLI and return (exit_code, stdout, stderr).""" + proc = subprocess.run([sys.executable, "-m", "openmind.cli", *args], + cwd=REPO, env=ENV, capture_output=True, text=True, + errors="replace", timeout=timeout) + return proc.returncode, proc.stdout, proc.stderr + + +def as_json(stdout): + try: + return json.loads(stdout) + except (ValueError, TypeError): + return None + + +# --------------------------------------------------------------------------- +# Global behaviour +# --------------------------------------------------------------------------- +code, out, err = run("--version") +check("--version exits 0", code == 0) +check("--version prints the runtime version", RUNTIME_VERSION in out, out.strip()) + +code, out, err = run("--help") +check("--help exits 0", code == 0) +for command in ("doctor", "init", "add", "ingest", "status", "export", "mcp", "serve"): + check(f"--help documents the '{command}' command", command in out) +check("--help documents the exit codes", "Exit codes" in out) + +code, out, err = run() +check("no command exits 2 (invalid usage)", code == 2, str(code)) +check("no command prints usage on stderr, not stdout", + "usage" in err.lower() and out.strip() == "") + +code, out, err = run("mcp") +check("a command group with no subcommand exits 2", code == 2, str(code)) + +code, out, err = run("status", "--json") +check("a missing required argument exits 2", code == 2, str(code)) + +# --------------------------------------------------------------------------- +# doctor +# --------------------------------------------------------------------------- +code, out, err = run("doctor", "--json") +payload = as_json(out) +check("doctor --json exits 0 on a healthy runtime", code == 0, f"{code}: {err[-200:]}") +check("doctor --json prints exactly one JSON object on stdout", payload is not None) +check("doctor --json reports ok", payload and payload.get("ok") is True) +check("doctor --json reports the runtime version", + payload and payload.get("version") == RUNTIME_VERSION) +check("doctor --json lists checks", payload and len(payload.get("checks", [])) >= 8) +check("doctor --json includes runtime info", + payload and "schema_version" in (payload.get("runtime") or {})) +check("doctor --json emits no ANSI escape codes", "\x1b[" not in out) + +names = {c["name"] for c in (payload or {}).get("checks", [])} +for required in ("data_dir", "database", "migrations", "project_dirs", + "vectorstore", "embeddings", "mcp", "model_config", + "model_server", "runtime_version"): + check(f"doctor checks '{required}'", required in names) + +check("doctor does not fail because no local model is configured", + code == 0 and any(c["name"] == "model_config" and c["status"] == "warn" + for c in (payload or {}).get("checks", []))) + +code, human_out, err = run("doctor") +check("doctor without --json prints a human report on stdout", + code == 0 and "OpenMind" in human_out) +check("the human doctor report is not JSON", as_json(human_out) is None) + +# Global flags must work on BOTH sides of the subcommand. argparse `parents=` +# shares Action objects with every subparser, so a default on the sub-parser +# silently clobbers a flag given before the subcommand — regression-tested here +# in both orders for each flag. +code, quiet_out, quiet_err = run("--quiet", "doctor") +check("--quiet BEFORE the subcommand suppresses the human report", + quiet_out.strip() == "", repr(quiet_out[:120])) +code, quiet_out, quiet_err = run("doctor", "--quiet") +check("--quiet AFTER the subcommand suppresses the human report", + quiet_out.strip() == "", repr(quiet_out[:120])) +code, jout, _ = run("--json", "doctor") +check("--json BEFORE the subcommand still emits JSON", as_json(jout) is not None) +code, jout, _ = run("doctor", "--json") +check("--json AFTER the subcommand still emits JSON", as_json(jout) is not None) + +# --------------------------------------------------------------------------- +# init +# --------------------------------------------------------------------------- +code, out, err = run("init", "--name", "cli demo", "--path", FIXTURE, "--json") +payload = as_json(out) +check("init --json exits 0", code == 0, f"{code}: {err[-300:]}") +check("init --json returns the workspace id", + payload and str(payload.get("workspace_id", "")).startswith("p_")) +check("init --json reports ok", payload and payload["ok"] is True) +check("init registers the given path", + payload and len(payload["workspace"]["paths"]) == 1) +check("init resolves the path to an absolute location", + payload and os.path.isabs(payload["workspace"]["paths"][0]["path"])) +check("init does NOT ingest unless asked", + payload and payload["workspace"]["state"] == "init") +check("init emits nothing but JSON on stdout", "\x1b[" not in out) + +WS = payload["workspace_id"] + +code, out, err = run("init", "--name", "bad path", "--path", + os.path.join(REPO, "no-such-directory"), "--json") +check("init with a non-existent path exits 2", code == 2, str(code)) +check("the error is machine-readable", + (as_json(out) or {}).get("error", {}).get("code") == "invalid_request") + +code, out, err = run("init", "--name", "no path", "--json") +check("init without --path succeeds (a path can be added later)", code == 0) + +# --------------------------------------------------------------------------- +# add +# --------------------------------------------------------------------------- +code, out, err = run("add", "--workspace", WS, "--path", FIXTURE, + "--exclude", "target", "--exclude", "build", "--json") +payload = as_json(out) +check("add --json exits 0", code == 0, f"{code}: {err[-200:]}") +check("add updates the existing path rather than duplicating it", + payload and len(payload["paths"]) == 1) +check("add records the exclude set", + payload and payload["paths"][0]["exclude"] == ["build", "target"]) + +code, out, err = run("add", "--workspace", "p_nope", "--path", FIXTURE, "--json") +check("add to an unknown workspace exits 1", code == 1, str(code)) +check("the unknown-workspace error is machine-readable", + (as_json(out) or {}).get("error", {}).get("code") == "workspace_not_found") + +# --------------------------------------------------------------------------- +# ingest +# --------------------------------------------------------------------------- +code, out, err = run("ingest", "--workspace", "p_nope", "--json") +check("ingest on an unknown workspace exits 1", code == 1, str(code)) + +code, out, err = run("ingest", "--workspace", WS, "--wait", "--timeout", "600", + "--json") +payload = as_json(out) +check("ingest --wait --json exits 0", code == 0, f"{code}: {err[-400:]}") +check("ingest --wait reports completion", + payload and payload.get("completed") is True and payload.get("status") == "done") +check("ingest --wait returns the persisted job id", + payload and str(payload.get("job_id", "")).startswith("job_")) +check("ingest --wait reports progress counters", + payload and payload.get("progress", {}).get("files_indexed", 0) > 0) +check("ingest --wait reports how long it took", + payload and payload.get("waited_seconds") is not None) + +# --------------------------------------------------------------------------- +# status +# +# Checked BEFORE the non-waiting ingest below: that one leaves a job in flight, +# which legitimately puts the workspace back into 'learning'. +# --------------------------------------------------------------------------- +code, out, err = run("status", "--workspace", WS, "--json") +payload = as_json(out) +check("status --json exits 0", code == 0, f"{code}: {err[-200:]}") +for key in ("workspace", "state", "paths", "template", "counts", + "active_jobs", "recent_jobs", "schema_version", "version"): + check(f"status --json reports '{key}'", payload and key in payload) +check("status reports the workspace reached 'ready'", + payload and payload["state"] == "ready", str(payload and payload["state"])) +check("status reports indexed files after an ingest", + payload and payload["counts"]["indexed_files"] > 0) +check("status reports code chunks after an ingest", + payload and payload["counts"]["code_chunks"] > 0) +check("status reports the migration schema version", + payload and payload["schema_version"] >= 2) +check("status reports the runtime version", + payload and payload["version"] == RUNTIME_VERSION) +check("status lists the ingest job", + payload and any(j["type"] == "ingest" for j in payload["recent_jobs"])) +check("status does not require a running FastAPI server", code == 0) + +code, out, err = run("status", "--workspace", "p_nope", "--json") +check("status on an unknown workspace exits 1", code == 1, str(code)) +payload = as_json(out) +check("a failing command still prints parseable JSON", payload is not None) +check("the failure payload sets ok=false", payload and payload["ok"] is False) +check("the failure payload carries a code and message", + payload and payload["error"]["code"] == "workspace_not_found" + and payload["error"]["message"]) + +code, human_out, err = run("status", "--workspace", "p_nope") +check("a human-mode failure prints to stderr, not stdout", + human_out.strip() == "" and "error" in err.lower()) + +# -------------------------------------------------------------------------- +# Cases that deliberately leave a job in flight — kept last, because they put +# the workspace back into 'learning' and would break the status checks above. +# -------------------------------------------------------------------------- +code, out, err = run("ingest", "--workspace", WS, "--json") +payload = as_json(out) +check("ingest without --wait exits 0 immediately", code == 0) +check("ingest without --wait returns a job id and does not wait", + payload and payload["waited"] is False and payload["job_id"]) + +# A --wait that times out must still print exactly ONE JSON object, report +# ok=false, exit 5, and hand back a pollable job id (the job is NOT cancelled). +code, out, err = run("ingest", "--workspace", WS, "--wait", "--timeout", "0.05", + "--json") +payload = as_json(out) +check("an ingest --wait timeout exits 5", code == 5, str(code)) +check("a timed-out ingest prints exactly one JSON object (not two)", + payload is not None and out.count('"ok"') == 1, repr(out[:200])) +check("a timed-out ingest reports ok=false", payload and payload["ok"] is False) +check("a timed-out ingest uses the timeout error code", + payload and payload["error"]["code"] == "timeout") +check("a timed-out ingest still returns a pollable job id", + payload and str(payload.get("job_id", "")).startswith("job_")) +check("the timeout message says the job is still running", + payload and "still running" in payload["error"]["message"]) + +# --------------------------------------------------------------------------- +# export +# --------------------------------------------------------------------------- +out_dir = os.path.join(tempfile.mkdtemp(prefix="om_cli_export_"), ".openmind") +code, out, err = run("export", "--repo", FIXTURE, "--output", out_dir, "--json") +payload = as_json(out) +check("export --json exits 0", code == 0, f"{code}: {err[-300:]}") +check("export writes a manifest", + os.path.isfile(os.path.join(out_dir, "manifest.json"))) +check("export reports the frozen schema version", + payload and payload["schemaVersion"] == "1.1.0") + +manifest = json.load(open(os.path.join(out_dir, "manifest.json"), encoding="utf-8")) +check("the manifest keeps schema 1.1.0", manifest["schemaVersion"] == "1.1.0") +check("the manifest still names the generator 'open-mind'", + manifest["generator"]["name"] == "open-mind") +check("the manifest reports the runtime version as the generator version", + manifest["generator"]["version"] == RUNTIME_VERSION) +for artifact in ("glossary.json", "architecture.json", "flows.json", + "source-index.json", "metadata.json"): + check(f"export wrote {artifact}", + os.path.isfile(os.path.join(out_dir, artifact))) + +code, out, err = run("export", "--repo", os.path.join(REPO, "nope"), + "--output", out_dir, "--json") +check("export with a missing repo exits 2", code == 2, str(code)) + +code, out, err = run("export", "--repo", FIXTURE, "--output", out_dir, + "--template", "spring-boot", "--no-template", "--json") +check("export rejects --template with --no-template (exit 2)", code == 2, str(code)) + +# Export must stay STANDALONE: no database, no vector store, no web app, and no +# heavy dependency. The dependency-free artifact-contract CI job runs the CLI +# exporter with nothing installed, so an eager import anywhere on this path +# would break it. Simulate the absent dependencies rather than trusting layering. +probe = subprocess.run( + [sys.executable, "-c", ''' +import builtins, os, tempfile +os.environ["OPENMIND_DATA_DIR"] = tempfile.mkdtemp() +os.environ["OPENMIND_MACHINE_DIR"] = tempfile.mkdtemp() +BLOCKED = {"chromadb", "fastapi", "uvicorn", "fastembed", "onnxruntime", "mcp", + "httpx", "tree_sitter", "tree_sitter_java", "sqlglot", "psutil", + "numpy", "pydantic", "yaml"} +_real = builtins.__import__ +def guard(name, *a, **kw): + if name.split(".")[0] in BLOCKED: + raise ImportError("No module named %r (simulated)" % name.split(".")[0]) + return _real(name, *a, **kw) +builtins.__import__ = guard +from openmind.services.export_service import ExportService +summary = ExportService().export("./fixtures/sample-repo", tempfile.mkdtemp()) +print(summary["schemaVersion"], summary["filesIndexed"]) +'''], + cwd=REPO, env=ENV, capture_output=True, text=True) +parts = probe.stdout.split() +check("artifact export runs with NO heavy dependencies installed " + "(no chroma, numpy, fastapi, mcp)", + probe.returncode == 0 and parts and parts[0] == "1.1.0", + (probe.stdout + probe.stderr)[-400:]) +check("the dependency-free export still indexes the fixture", + len(parts) > 1 and int(parts[1]) > 0, probe.stdout) + +# --------------------------------------------------------------------------- +# serve — argument validation only (never actually bind) +# --------------------------------------------------------------------------- +code, out, err = run("serve", "--host", "0.0.0.0", "--json") +check("serve refuses a non-loopback bind by default (exit 2)", code == 2, str(code)) +check("the refusal explains the risk and the override", + "--allow-non-loopback" in ((as_json(out) or {}).get("error", {}) + .get("message", ""))) + +code, out, err = run("serve", "--help") +check("serve documents the loopback default", code == 0 and "127.0.0.1" in out) + +# --------------------------------------------------------------------------- +# mcp serve — the CLI must reach the SAME implementation, not a copy +# --------------------------------------------------------------------------- +code, out, err = run("mcp", "--help") +check("'mcp' exposes a serve subcommand", code == 0 and "serve" in out) + +probe = subprocess.run( + [sys.executable, "-c", + "import openmind.cli as c, openmind.mcp_server as m;" + "import inspect;" + "src = inspect.getsource(c.cmd_mcp_serve);" + "print('create_mcp_server' in src);" + "print(len(m.TOOLS))"], + cwd=REPO, env=ENV, capture_output=True, text=True) +lines = probe.stdout.split() +check("the CLI delegates to create_mcp_server rather than redefining tools", + lines and lines[0] == "True", probe.stdout + probe.stderr) +check("the MCP tool set has all 9 tools", len(lines) > 1 and lines[1] == "9", + probe.stdout) + +bad = [d for d, ok in _results if not ok] +print(f"\n{len(_results) - len(bad)} passed, {len(bad)} failed") +sys.exit(1 if bad else 0) diff --git a/tests/verify_migrations.py b/tests/verify_migrations.py new file mode 100644 index 0000000..5a81e37 --- /dev/null +++ b/tests/verify_migrations.py @@ -0,0 +1,255 @@ +"""Migration runner — empty database, legacy baseline, idempotency, checksum +immutability, and transactional rollback. + +Pure stdlib + sqlite3: no embeddings, no vector store, no model. Every case +builds its own throwaway database file so nothing here can touch real data. +""" +import os +import sqlite3 +import sys +import tempfile + +os.environ.setdefault("OPENMIND_DATA_DIR", tempfile.mkdtemp()) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import _isolate # noqa: E402,F401 — forces an isolated data dir (never the live one) + +from openmind import migrations # noqa: E402 +from openmind.migrations import runner # noqa: E402 + +_results = [] + + +def check(desc, cond): + _results.append((desc, bool(cond))) + print(("PASS" if cond else "FAIL") + " - " + desc) + + +def _db(): + """A fresh, empty sqlite database file.""" + path = os.path.join(tempfile.mkdtemp(prefix="om_mig_"), "openmind.db") + conn = sqlite3.connect(path, check_same_thread=False) + conn.row_factory = sqlite3.Row + return conn + + +def _tables(conn): + return {r[0] for r in conn.execute( + "SELECT name FROM sqlite_master WHERE type='table'").fetchall()} + + +def _fake(version, name, statements, payload=None): + """A migration built from literal SQL, for cases that must not depend on + the real (frozen) migration set.""" + sql = tuple(statements) + + def apply(conn): + for s in sql: + conn.execute(s) + + return runner.Migration(version, name, payload or "\n".join(sql), apply) + + +# --------------------------------------------------------------------------- +# 1. Empty database +# --------------------------------------------------------------------------- +conn = _db() +result = migrations.migrate(conn) +tables = _tables(conn) + +check("empty db: runner reports the head version", result.version == 2) +check("empty db: both migrations are applied", + result.applied == ["0001_baseline", "0002_paths_sidecar"]) +check("empty db: nothing was reported as already applied", result.already_applied == []) +check("empty db: not flagged as a legacy baseline", result.baselined_legacy is False) +check("empty db: schema_migrations ledger exists", "schema_migrations" in tables) +for t in ("projects", "jobs", "model_config", "file_index", "kv", "ask_history"): + check(f"empty db: table '{t}' created", t in tables) +check("empty db: ask_history index created", + conn.execute("SELECT 1 FROM sqlite_master WHERE type='index' AND " + "name='idx_ask_scope'").fetchone() is not None) +check("empty db: ledger rows carry a checksum and a timestamp", + all(r["checksum"] and r["applied_at"] for r in + conn.execute("SELECT * FROM schema_migrations").fetchall())) + +# --------------------------------------------------------------------------- +# 2. Repeat run is a no-op +# --------------------------------------------------------------------------- +again = migrations.migrate(conn) +check("repeat run: applies nothing", again.applied == []) +check("repeat run: reports both as already applied", + again.already_applied == ["0001_baseline", "0002_paths_sidecar"]) +check("repeat run: version is unchanged", again.version == result.version) + +# --------------------------------------------------------------------------- +# 3. Legacy database — has the tables, no ledger. Must be baselined, not wiped. +# --------------------------------------------------------------------------- +legacy = _db() +legacy.executescript( + """ + CREATE TABLE projects ( + id TEXT PRIMARY KEY, name TEXT NOT NULL, + state TEXT NOT NULL DEFAULT 'init', + paths_json TEXT NOT NULL DEFAULT '[]', + created_at TEXT NOT NULL, updated_at TEXT NOT NULL, + meta_json TEXT NOT NULL DEFAULT '{}'); + CREATE TABLE jobs ( + job_id TEXT PRIMARY KEY, project_id TEXT NOT NULL, type TEXT NOT NULL, + path TEXT, status TEXT NOT NULL, step TEXT DEFAULT '', + progress_json TEXT NOT NULL DEFAULT '{}', + log_tail_json TEXT NOT NULL DEFAULT '[]', + control_json TEXT NOT NULL DEFAULT '{}', error TEXT DEFAULT '', + created_at TEXT NOT NULL, updated_at TEXT NOT NULL, + started_at TEXT, finished_at TEXT); + CREATE TABLE file_index ( + project_id TEXT NOT NULL, file_path TEXT NOT NULL, + file_hash TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'indexed', + chunk_ids_json TEXT NOT NULL DEFAULT '[]', service TEXT DEFAULT '', + topics_json TEXT NOT NULL DEFAULT '[]', updated_at TEXT NOT NULL, + PRIMARY KEY (project_id, file_path)); + CREATE TABLE kv (key TEXT PRIMARY KEY, value TEXT); + INSERT INTO projects VALUES + ('p_legacy0001','Legacy Project','ready','[]','2026-01-01T00:00:00', + '2026-01-01T00:00:00','{"template":"spring-boot"}'); + INSERT INTO jobs VALUES + ('job_legacy01','p_legacy0001','ingest',NULL,'done','', + '{"files_done":7}','[]','{}','','2026-01-01T00:00:00', + '2026-01-01T00:00:00',NULL,NULL); + INSERT INTO file_index VALUES + ('p_legacy0001','src/Main.java','abc123','indexed','["c1"]','svc','[]', + '2026-01-01T00:00:00'); + INSERT INTO kv VALUES ('last_project_folder','/somewhere'); + """ +) +legacy.commit() + +check("legacy db: detected as legacy before migrating", runner.detect_legacy(legacy) is True) + +legacy_result = migrations.migrate(legacy) + +check("legacy db: reported as a legacy baseline", legacy_result.baselined_legacy is True) +check("legacy db: brought to head version", legacy_result.version == 2) +check("legacy db: baseline recorded, not skipped", + "0001_baseline" in legacy_result.applied) +check("legacy db: project row survived", + legacy.execute("SELECT COUNT(*) FROM projects").fetchone()[0] == 1) +check("legacy db: project meta survived intact", + legacy.execute("SELECT meta_json FROM projects WHERE id='p_legacy0001'" + ).fetchone()[0] == '{"template":"spring-boot"}') +check("legacy db: project state survived intact", + legacy.execute("SELECT state FROM projects WHERE id='p_legacy0001'" + ).fetchone()[0] == "ready") +check("legacy db: job row survived", + legacy.execute("SELECT COUNT(*) FROM jobs").fetchone()[0] == 1) +check("legacy db: file index row survived", + legacy.execute("SELECT COUNT(*) FROM file_index").fetchone()[0] == 1) +check("legacy db: kv row survived", + legacy.execute("SELECT value FROM kv WHERE key='last_project_folder'" + ).fetchone()[0] == "/somewhere") +check("legacy db: the missing ask_history table was added", + "ask_history" in _tables(legacy)) +check("legacy db: no longer detected as legacy once baselined", + runner.detect_legacy(legacy) is False) + +# --------------------------------------------------------------------------- +# 4. Checksum immutability +# --------------------------------------------------------------------------- +drift = _db() +original = _fake(1, "widgets", ["CREATE TABLE widgets (id TEXT PRIMARY KEY)"]) +migrations.migrate(drift, migrations=[original]) + +edited = _fake(1, "widgets", + ["CREATE TABLE widgets (id TEXT PRIMARY KEY, extra TEXT)"]) +mismatch = None +try: + migrations.migrate(drift, migrations=[edited]) +except migrations.MigrationChecksumMismatch as exc: + mismatch = exc + +check("checksum drift: an edited applied migration raises", mismatch is not None) +check("checksum drift: the error names the version", getattr(mismatch, "version", None) == 1) +check("checksum drift: the error names the migration", + getattr(mismatch, "name", None) == "widgets") +check("checksum drift: the error reports both checksums", + bool(getattr(mismatch, "stored", "")) and bool(getattr(mismatch, "computed", "")) + and mismatch.stored != mismatch.computed) +check("checksum drift: the message is actionable", + "immutable" in str(mismatch).lower()) +check("checksum drift: an identical re-run still passes", + migrations.migrate(drift, migrations=[original]).applied == []) + +# --------------------------------------------------------------------------- +# 5. Transactional rollback — a failing migration leaves NOTHING behind +# --------------------------------------------------------------------------- +rollback = _db() +good = _fake(1, "good", ["CREATE TABLE good (id TEXT PRIMARY KEY)"]) +bad = _fake(2, "bad", [ + "CREATE TABLE bad_partial (id TEXT PRIMARY KEY)", + "THIS IS NOT VALID SQL", +]) + +failure = None +try: + migrations.migrate(rollback, migrations=[good, bad]) +except migrations.MigrationApplyError as exc: + failure = exc + +rb_tables = _tables(rollback) +check("rollback: the failing migration raises MigrationApplyError", failure is not None) +check("rollback: the error names the failing version", + getattr(failure, "version", None) == 2) +check("rollback: the earlier migration's table survived", "good" in rb_tables) +check("rollback: the failing migration's partial table was rolled back", + "bad_partial" not in rb_tables) +check("rollback: no ledger row was written for the failed migration", + rollback.execute("SELECT COUNT(*) FROM schema_migrations WHERE version=2" + ).fetchone()[0] == 0) +check("rollback: the successful migration stays recorded", + rollback.execute("SELECT COUNT(*) FROM schema_migrations WHERE version=1" + ).fetchone()[0] == 1) +check("rollback: version reflects only what actually applied", + runner.current_version(rollback) == 1) +check("rollback: retrying after the fix applies the remainder", + migrations.migrate(rollback, migrations=[ + good, _fake(2, "bad", ["CREATE TABLE fixed (id TEXT PRIMARY KEY)"]) + ]).applied == ["0002_bad"]) + +# --------------------------------------------------------------------------- +# 6. Ordering, duplicate detection, and a newer-than-code database +# --------------------------------------------------------------------------- +order = _db() +applied_order = migrations.migrate(order, migrations=[ + _fake(3, "third", ["CREATE TABLE t3 (id TEXT)"]), + _fake(1, "first", ["CREATE TABLE t1 (id TEXT)"]), + _fake(2, "second", ["CREATE TABLE t2 (id TEXT)"]), +]).applied +check("ordering: migrations apply in ascending version order", + applied_order == ["0001_first", "0002_second", "0003_third"]) + +ahead = _db() +migrations.migrate(ahead, migrations=[ + _fake(1, "one", ["CREATE TABLE t1 (id TEXT)"]), + _fake(2, "two", ["CREATE TABLE t2 (id TEXT)"]), +]) +ahead_result = migrations.migrate(ahead, migrations=[ + _fake(1, "one", ["CREATE TABLE t1 (id TEXT)"]), +]) +check("newer db: an applied version unknown to this build is reported", + ahead_result.unknown_applied == [2]) +check("newer db: reporting it does not destroy or re-run anything", + ahead_result.applied == [] and "t2" in _tables(ahead)) + +# --------------------------------------------------------------------------- +# 7. The real migration set is well-formed +# --------------------------------------------------------------------------- +real = migrations.discover() +check("real set: migrations are discovered", len(real) >= 2) +check("real set: versions are unique and ascending", + [m.version for m in real] == sorted({m.version for m in real})) +check("real set: every migration has a non-empty checksum", + all(len(m.checksum) == 64 for m in real)) +check("real set: v0001 is the baseline", real[0].label == "0001_baseline") + +bad_results = [d for d, ok in _results if not ok] +print(f"\n{len(_results) - len(bad_results)} passed, {len(bad_results)} failed") +sys.exit(1 if bad_results else 0) diff --git a/tests/verify_runtime.py b/tests/verify_runtime.py new file mode 100644 index 0000000..46c592f --- /dev/null +++ b/tests/verify_runtime.py @@ -0,0 +1,142 @@ +"""Runtime composition root — bootstrap idempotency, worker opt-in, and the +single shared version. + +The property that matters most here: bootstrap must be idempotent and must NOT +start a background worker. Before Phase 1, ``jobs.start_worker()`` ran only +under FastAPI, so a job enqueued from anywhere else sat queued forever; the fix +must not overcorrect into spawning a thread for every ``doctor`` invocation. +""" +import os +import sys +import tempfile + +os.environ.setdefault("OPENMIND_DATA_DIR", tempfile.mkdtemp()) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import _isolate # noqa: E402,F401 — forces an isolated data dir (never the live one) + +import threading # noqa: E402 + +from openmind import config, db, jobs # noqa: E402 +from openmind.runtime import OpenMindRuntime, get_runtime, reset_runtime # noqa: E402 +from openmind.services.service_container import ServiceContainer # noqa: E402 +from openmind.version import RUNTIME_VERSION # noqa: E402 + +_results = [] + + +def check(desc, cond, extra=""): + _results.append((desc, bool(cond))) + print(("PASS" if cond else "FAIL") + " - " + desc + (f" [{extra}]" if extra and not cond else "")) + + +def _worker_threads(): + return [t for t in threading.enumerate() if t.name == "openmind-job-worker"] + + +# --------------------------------------------------------------------------- +# Version is single-sourced +# --------------------------------------------------------------------------- +import openmind # noqa: E402 +from openmind import artifacts # noqa: E402 + +check("runtime version is a pre-v2 development version, not a false 2.0.0", + RUNTIME_VERSION.startswith("1.") and "dev" in RUNTIME_VERSION) +check("openmind.__version__ is the runtime version", + openmind.__version__ == RUNTIME_VERSION) +check("the artifact generator reports the runtime version", + artifacts._generator_version() == RUNTIME_VERSION) +check("the artifact SCHEMA version is NOT tied to the runtime version " + "(it is a frozen integration contract)", + artifacts.SCHEMA_VERSION == "1.1.0") + +# --------------------------------------------------------------------------- +# Bootstrap +# --------------------------------------------------------------------------- +runtime = OpenMindRuntime() +before_workers = len(_worker_threads()) + +container = runtime.bootstrap() +check("bootstrap returns a service container", + isinstance(container, ServiceContainer)) +check("bootstrap is idempotent: the same container comes back", + runtime.bootstrap() is container) +check("bootstrap is idempotent a third time", + runtime.bootstrap() is container) +check("bootstrap created the data directory", os.path.isdir(str(config.DATA_DIR))) +check("bootstrap created the database", os.path.isfile(str(config.DB_PATH))) +check("bootstrap migrated the schema to head", + db.migration_status()["version"] >= 2) + +check("bootstrap does NOT start the job worker", + len(_worker_threads()) == before_workers and not runtime.worker_running) + +# --------------------------------------------------------------------------- +# Services are cached and wired +# --------------------------------------------------------------------------- +check("workspace service is cached", runtime.workspaces is runtime.workspaces) +check("job service is cached", runtime.jobs is runtime.jobs) +check("ingest service is cached", runtime.ingest is runtime.ingest) +check("export service is cached", runtime.export is runtime.export) +check("health service is cached", runtime.health is runtime.health) +check("the runtime exposes its version", runtime.version == RUNTIME_VERSION) + +info = runtime.info() +for key in ("version", "data_dir", "database", "schema_version", "worker_running"): + check(f"runtime.info() reports {key}", key in info) +check("runtime.info() reports the isolated data dir, not the live one", + str(config.DATA_DIR) in info["data_dir"]) + +# --------------------------------------------------------------------------- +# Worker is opt-in, and idempotent once asked for +# --------------------------------------------------------------------------- +runtime.ensure_worker() +check("ensure_worker() starts the worker", runtime.worker_running) +check("ensure_worker() actually spawned the worker thread", + len(_worker_threads()) == before_workers + 1 + or jobs._worker_started) # engine flag: a prior test may have started it +runtime.ensure_worker() +runtime.ensure_worker() +check("ensure_worker() is idempotent: no duplicate worker threads", + len(_worker_threads()) <= before_workers + 1) + +# --------------------------------------------------------------------------- +# Process-wide runtime +# --------------------------------------------------------------------------- +shared = get_runtime() +check("get_runtime() returns the same instance every time", get_runtime() is shared) +check("get_runtime() returns a bootstrapped runtime", + isinstance(shared.services, ServiceContainer)) +check("get_runtime() shares one service container", + get_runtime().workspaces is shared.workspaces) + +reset_runtime() +fresh = get_runtime() +check("reset_runtime() lets a test rebuild the process-wide runtime", + fresh is not shared) + +# --------------------------------------------------------------------------- +# Thread safety: concurrent bootstrap must produce ONE container +# --------------------------------------------------------------------------- +racy = OpenMindRuntime() +seen = [] +barrier = threading.Barrier(8) + + +def _boot(): + barrier.wait() + seen.append(racy.bootstrap()) + + +threads = [threading.Thread(target=_boot) for _ in range(8)] +for t in threads: + t.start() +for t in threads: + t.join() + +check("concurrent bootstrap yields exactly one container", + len(seen) == 8 and len({id(c) for c in seen}) == 1) + +bad = [d for d, ok in _results if not ok] +print(f"\n{len(_results) - len(bad)} passed, {len(bad)} failed") +sys.exit(1 if bad else 0) diff --git a/tests/verify_services.py b/tests/verify_services.py new file mode 100644 index 0000000..3282346 --- /dev/null +++ b/tests/verify_services.py @@ -0,0 +1,375 @@ +"""Application service layer — workspace, job, ingest, export and health. + +The bounded-wait cases use the fake clock and a fake job reader from +``openmind.ports``. That is what those ports are for: driving the wait state +machine through queued -> running -> done (and into a timeout) deterministically +and in milliseconds, instead of spinning a real worker and sleeping. +""" +import os +import sys +import tempfile + +os.environ.setdefault("OPENMIND_DATA_DIR", tempfile.mkdtemp()) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import _isolate # noqa: E402,F401 — forces an isolated data dir (never the live one) + +from typing import Any, Dict, List, Optional # noqa: E402 + +from openmind.domain.errors import (InvalidRequest, JobNotFound, # noqa: E402 + OperationTimeout, WorkspaceNotFound) +from openmind.domain.types import HealthCheck, HealthReport # noqa: E402 +from openmind.ports.runtime_ports import FakeClock # noqa: E402 +from openmind.runtime import OpenMindRuntime # noqa: E402 +from openmind.services.job_service import JobService # noqa: E402 + +_results = [] + + +def check(desc, cond, extra=""): + _results.append((desc, bool(cond))) + print(("PASS" if cond else "FAIL") + " - " + desc + + (f" [{extra}]" if extra and not cond else "")) + + +def raises(exc_type, fn, *a, **kw): + try: + fn(*a, **kw) + except exc_type as exc: + return exc + except Exception as exc: # wrong type — report it, don't pass + return ("WRONG", type(exc).__name__, str(exc)) + return None + + +FIXTURE = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "fixtures", "sample-repo") + +runtime = OpenMindRuntime() +runtime.bootstrap() +ws = runtime.workspaces + +# --------------------------------------------------------------------------- +# WorkspaceService — creation +# --------------------------------------------------------------------------- +created = ws.create("Service Demo") +check("create() returns a workspace record", bool(created.get("id"))) +check("create() uses the existing project id shape", + created["id"].startswith("p_")) +check("a new workspace starts in 'init'", created["state"] == "init") +check("create() does not register a path when none is given", + created["paths"] == []) + +err = raises(InvalidRequest, ws.create, " ") +check("create() rejects a blank name", isinstance(err, InvalidRequest), repr(err)) +check("the blank-name error names the field", + isinstance(err, InvalidRequest) and err.details.get("field") == "name") + +with_path = ws.create("With Path", path=FIXTURE) +check("create(path=...) registers the initial path", + len(with_path["paths"]) == 1 + and with_path["paths"][0]["path"] == FIXTURE) + +# --------------------------------------------------------------------------- +# WorkspaceService — reads and honest not-found +# --------------------------------------------------------------------------- +check("get() round-trips", ws.get(created["id"])["id"] == created["id"]) +check("exists() is True for a real workspace", ws.exists(created["id"])) +check("exists() is False for an unknown id", not ws.exists("p_does_not_exist")) + +err = raises(WorkspaceNotFound, ws.get, "p_does_not_exist") +check("get() raises WorkspaceNotFound, not a bare KeyError/None", + isinstance(err, WorkspaceNotFound), repr(err)) +check("WorkspaceNotFound maps to HTTP 404", getattr(err, "http_status", None) == 404) +check("WorkspaceNotFound maps to CLI exit 1", getattr(err, "exit_code", None) == 1) +check("WorkspaceNotFound is machine-readable", + err.as_dict()["code"] == "workspace_not_found" + and err.as_dict()["details"]["workspace_id"] == "p_does_not_exist") + +ids = {w["id"] for w in ws.list()} +check("list() includes every created workspace", + {created["id"], with_path["id"]} <= ids) + +# --------------------------------------------------------------------------- +# WorkspaceService — path registration +# --------------------------------------------------------------------------- +updated = ws.add_path(created["id"], FIXTURE, ["target", "build"]) +check("add_path() registers the path", len(updated["paths"]) == 1) +check("add_path() stores the exclude set", + updated["paths"][0]["exclude"] == ["build", "target"]) + +re_added = ws.add_path(created["id"], FIXTURE, ["node_modules"]) +check("add_path() UPDATES an existing path instead of duplicating it", + len(re_added["paths"]) == 1 + and re_added["paths"][0]["exclude"] == ["node_modules"]) + +err = raises(WorkspaceNotFound, ws.add_path, "p_nope", FIXTURE) +check("add_path() 404s before touching the sidecar", + isinstance(err, WorkspaceNotFound)) +err = raises(InvalidRequest, ws.add_path, created["id"], " ") +check("add_path() rejects a blank path", isinstance(err, InvalidRequest)) + +check("paths() reads back what add_path wrote", + ws.paths(created["id"])[0]["path"] == FIXTURE) +check("source_root() resolves the machine-local root", + ws.source_root(created["id"]) == FIXTURE.replace("\\", "/")) + +removed = ws.remove_path(created["id"], FIXTURE) +check("remove_path() removes the path", removed["paths"] == []) +err = raises(WorkspaceNotFound, ws.remove_path, "p_nope", FIXTURE) +check("remove_path() 404s for an unknown workspace", + isinstance(err, WorkspaceNotFound)) + +# --------------------------------------------------------------------------- +# WorkspaceService — machine-local storage (zero origin traces) +# --------------------------------------------------------------------------- +import sqlite3 # noqa: E402 + +from openmind import config # noqa: E402 + +raw = sqlite3.connect(str(config.DB_PATH)) +rows = raw.execute("SELECT paths_json FROM projects").fetchall() +raw.close() +check("the portable database stores NO absolute source path", + all((r[0] or "[]") == "[]" for r in rows), + f"rows={rows}") + +# --------------------------------------------------------------------------- +# WorkspaceService — template selection +# --------------------------------------------------------------------------- +selection = ws.template_selection(created["id"]) +check("template_selection() reports no override on a fresh workspace", + selection["override"] is None and selection["effective"] is None) + +err = raises(InvalidRequest, ws.set_template, created["id"], "no-such-template") +check("set_template() rejects an unknown template", + isinstance(err, InvalidRequest), repr(err)) +check("the unknown-template error lists what IS available", + isinstance(err, InvalidRequest) and err.details.get("available")) +check("InvalidRequest maps to HTTP 400 / CLI exit 2", + err.http_status == 400 and err.exit_code == 2) + +from openmind import templates # noqa: E402 + +available = [t["name"] for t in templates.list_templates() if t.get("valid", True)] +if available: + picked = available[0] + after = ws.set_template(created["id"], picked.upper()) + check("set_template() accepts a valid template and lower-cases it", + after["override"] == picked.lower(), str(after)) + check("set_template() makes the override effective", + after["effective"] == picked.lower() + and after["effective_source"] == "override") + cleared = ws.set_template(created["id"], None) + check("set_template(None) clears the override", + cleared["override"] is None) +else: + check("template profiles are available to test selection", False, + "no valid built-in templates found") + +# --------------------------------------------------------------------------- +# WorkspaceService — status +# --------------------------------------------------------------------------- +status = ws.status(with_path["id"]) +check("status() reports the workspace record", status["workspace"]["id"] == with_path["id"]) +check("status() reports registered paths", len(status["paths"]) == 1) +check("status() reports state", status["state"] == "init") +check("status() reports template selection", "effective" in status["template"]) +for key in ("indexed_files", "code_chunks", "solved_cases", "glossary_terms"): + check(f"status() reports the {key} count", key in status["counts"]) +check("status() counts are integers (or None for an unreadable store)", + all(v is None or isinstance(v, int) for v in status["counts"].values())) + + +# --------------------------------------------------------------------------- +# JobService — bounded waiting, driven by fakes +# --------------------------------------------------------------------------- +class FakeReader: + """A job reader that walks a scripted sequence of statuses.""" + + def __init__(self, statuses: List[str]) -> None: + self.statuses = list(statuses) + self.reads = 0 + + def get_job(self, job_id: str) -> Optional[Dict[str, Any]]: + if job_id != "job_1": + return None + self.reads += 1 + index = min(self.reads - 1, len(self.statuses) - 1) + return {"job_id": job_id, "status": self.statuses[index], + "type": "ingest", "progress": {"files_done": self.reads}, + "error": ""} + + def list_jobs(self, project_ids=None, status=None): + return [] + + def active_jobs(self): + return [] + + +clock = FakeClock() +svc = JobService(reader=FakeReader(["queued", "running", "running", "done"]), + engine=None, clock=clock) +outcome = svc.wait_for_terminal("job_1", timeout=60, poll_interval=0.5) +check("wait_for_terminal() returns when the job reaches 'done'", + outcome.status == "done" and outcome.completed) +check("wait_for_terminal() polled until the terminal state", clock.sleeps == [0.5] * 3) +check("wait_for_terminal() reports how long it waited", + abs(outcome.waited_seconds - 1.5) < 1e-9, str(outcome.waited_seconds)) +check("the wait result is machine-readable", + outcome.as_dict()["job_id"] == "job_1" + and outcome.as_dict()["completed"] is True) + +svc = JobService(reader=FakeReader(["running", "failed"]), engine=None, + clock=FakeClock()) +outcome = svc.wait_for_terminal("job_1", timeout=60) +check("a failed job is terminal but NOT completed", + outcome.status == "failed" and outcome.completed is False) + +# The case that would otherwise hang: paused/interrupted never progress. +for stalled in ("paused", "interrupted"): + svc = JobService(reader=FakeReader(["running", stalled]), engine=None, + clock=FakeClock()) + outcome = svc.wait_for_terminal("job_1", timeout=60) + check(f"a '{stalled}' job returns instead of blocking until timeout", + outcome.status == stalled and outcome.completed is False) + +svc = JobService(reader=FakeReader(["running"]), engine=None, clock=FakeClock()) +err = raises(OperationTimeout, svc.wait_for_terminal, "job_1", timeout=3, + poll_interval=1) +check("a job that never finishes raises OperationTimeout", + isinstance(err, OperationTimeout), repr(err)) +check("OperationTimeout maps to CLI exit 5", getattr(err, "exit_code", None) == 5) +check("the timeout error says the job is still running", + isinstance(err, OperationTimeout) and "still running" in str(err)) +check("the timeout error carries the job id and last status", + isinstance(err, OperationTimeout) + and err.details.get("job_id") == "job_1" + and err.details.get("status") == "running") + +svc = JobService(reader=FakeReader(["done"]), engine=None, clock=FakeClock()) +instant = svc.wait_for_terminal("job_1", timeout=60) +check("an already-finished job returns without sleeping at all", + instant.completed and instant.waited_seconds == 0) + +err = raises(JobNotFound, svc.get, "job_missing") +check("get() raises JobNotFound for an unknown job", isinstance(err, JobNotFound)) +check("JobNotFound maps to HTTP 404", getattr(err, "http_status", None) == 404) + +svc = JobService(reader=FakeReader(["running"]), engine=None, clock=FakeClock()) +err = raises(InvalidRequest, svc.wait_for_terminal, "job_1", timeout=0) +check("wait_for_terminal() rejects a non-positive timeout", + isinstance(err, InvalidRequest)) + + +# KeyError from the engine must become a typed JobNotFound. +class ExplodingEngine: + def pause(self, job_id): raise KeyError(job_id) + def resume(self, job_id): raise KeyError(job_id) + def cancel_job(self, job_id): raise KeyError(job_id) + + +svc = JobService(reader=FakeReader(["running"]), engine=ExplodingEngine(), + clock=FakeClock()) +for name in ("pause", "resume", "cancel"): + err = raises(JobNotFound, getattr(svc, name), "job_gone") + check(f"{name}() translates the engine's KeyError into JobNotFound", + isinstance(err, JobNotFound), repr(err)) + +# --------------------------------------------------------------------------- +# IngestService +# --------------------------------------------------------------------------- +ingest = runtime.ingest +err = raises(WorkspaceNotFound, ingest.start, "p_nope") +check("ingest.start() 404s for an unknown workspace", + isinstance(err, WorkspaceNotFound)) + +no_paths = ws.create("No Paths") +err = raises(InvalidRequest, ingest.start, no_paths["id"]) +check("ingest.start() refuses a workspace with no registered source path", + isinstance(err, InvalidRequest), repr(err)) +check("the no-path error explains what to do", + isinstance(err, InvalidRequest) and "add one" in str(err)) + +queued = ingest.start(with_path["id"]) +check("ingest.start() returns a persisted job id", + queued["job_id"].startswith("job_")) +check("ingest.start() without wait does not block", queued["waited"] is False) +check("the enqueued job is an ingest for this workspace", + queued["job"]["type"] == "ingest" + and queued["job"]["project_id"] == with_path["id"]) + +again = ingest.start(with_path["id"]) +check("the engine dedupes a second ingest for the same workspace", + again["job_id"] == queued["job_id"]) + +state = ingest.status(with_path["id"]) +check("ingest.status() reports the workspace state", "state" in state) +check("ingest.status() lists ingest jobs", + any(j["job_id"] == queued["job_id"] for j in state["jobs"])) +check("ingest.status() surfaces the active job", + (state["active_job"] or {}).get("job_id") == queued["job_id"]) + +# --------------------------------------------------------------------------- +# ExportService +# --------------------------------------------------------------------------- +export = runtime.export +check("export service reports the frozen artifact schema version", + export.schema_version == "1.1.0") + +err = raises(InvalidRequest, export.export, "./definitely-not-here", "out") +check("export() rejects a missing repository", isinstance(err, InvalidRequest)) +err = raises(InvalidRequest, export.export, "", "out") +check("export() rejects an empty repo path", isinstance(err, InvalidRequest)) +err = raises(InvalidRequest, export.export, FIXTURE, "out", + template="spring-boot", no_template=True) +check("export() rejects --template together with --no-template", + isinstance(err, InvalidRequest)) + +out_dir = tempfile.mkdtemp(prefix="om_export_") +summary = export.export(FIXTURE, out_dir) +check("export() writes the artifact directory", + os.path.isfile(os.path.join(out_dir, "manifest.json"))) +check("export() reports the schema version", summary["schemaVersion"] == "1.1.0") +check("export() indexed the fixture files", summary["filesIndexed"] > 0) + +# --------------------------------------------------------------------------- +# HealthService +# --------------------------------------------------------------------------- +report = runtime.health.report() +check("health report is a HealthReport", isinstance(report, HealthReport)) +check("health report carries the runtime version", + report.version == runtime.version) +check("health report has checks", len(report.checks) >= 8) +check("every health check is a HealthCheck", + all(isinstance(c, HealthCheck) for c in report.checks)) + +names = {c.name for c in report.checks} +for required in ("data_dir", "database", "migrations", "project_dirs", + "vectorstore", "embeddings", "mcp", "model_config", + "model_server", "runtime_version"): + check(f"health reports the '{required}' check", required in names) + +check("no health check is in an unknown state", + all(c.status in ("ok", "warn", "error") for c in report.checks)) +check("a missing optional local model does NOT fail doctor", + report.ok is True, f"failures={[c.name for c in report.failures()]}") +check("the migrations check reports the schema version", + any(c.name == "migrations" and c.data.get("version", 0) >= 2 + for c in report.checks)) +check("the health summary is JSON-serializable", + isinstance(runtime.health.summary()["checks"], list)) + +# An error-severity check must fail the report; a warning must not. +check("a report with only warnings is ok", + HealthReport("x", [HealthCheck("a", "ok"), HealthCheck("b", "warn")]).ok) +check("a report with an error is not ok", + not HealthReport("x", [HealthCheck("a", "ok"), HealthCheck("b", "error")]).ok) +check("report status is the worst severity present", + HealthReport("x", [HealthCheck("a", "ok"), + HealthCheck("b", "warn")]).status == "warn") + +bad = [d for d, ok in _results if not ok] +print(f"\n{len(_results) - len(bad)} passed, {len(bad)} failed") +sys.exit(1 if bad else 0)