diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9e2f5ef..7bcf0f0 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -8,7 +8,25 @@ permissions: contents: read jobs: + verify: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: python -m pip install --upgrade pip && python -m pip install '.[dev,anthropic]' + - name: Verify release candidate + run: | + python -m pytest -q + ruff check . + ruff format --check . + mypy src/threadlang + bandit -q -r src/threadlang + pip-audit + build: + needs: verify runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -16,20 +34,42 @@ jobs: with: python-version: "3.12" - run: python -m pip install --upgrade pip build - - name: Verify tag matches pyproject version + - name: Verify tag matches package versions run: | PKG=$(python -c "import tomllib;print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])") - TAG="${GITHUB_REF_NAME#v}" - [ "$PKG" = "$TAG" ] || { echo "tag $GITHUB_REF_NAME != pyproject version $PKG"; exit 1; } + RUNTIME=$(PYTHONPATH=src python -c "import threadlang;print(threadlang.__version__)") + [ "$GITHUB_REF_NAME" = "v$PKG" ] || { echo "tag $GITHUB_REF_NAME != v$PKG"; exit 1; } + [ "$RUNTIME" = "$PKG" ] || { echo "runtime version $RUNTIME != pyproject version $PKG"; exit 1; } - run: python -m build - uses: actions/upload-artifact@v4 with: name: dist path: dist/ - publish: + verify_artifact: needs: build runs-on: ubuntu-latest + steps: + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - uses: actions/download-artifact@v4 + with: + name: dist + path: dist/ + - run: python -m pip install --upgrade pip twine + - run: python -m twine check dist/* + - name: Smoke-test built wheel + run: | + python -m venv "$RUNNER_TEMP/threadlang-smoke" + "$RUNNER_TEMP/threadlang-smoke/bin/pip" install dist/*.whl + "$RUNNER_TEMP/threadlang-smoke/bin/threadlang" --version + "$RUNNER_TEMP/threadlang-smoke/bin/threadlang-serve" --help >/dev/null + "$RUNNER_TEMP/threadlang-smoke/bin/support-triage" --help >/dev/null + + publish: + needs: verify_artifact + runs-on: ubuntu-latest environment: pypi permissions: id-token: write # trusted publishing (OIDC) — no API token stored anywhere diff --git a/README.md b/README.md index 6123e70..ad35599 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ workflows**. ThreadLang validates a workflow graph, executes model and tool calls within explicit limits, and records every binding, step, model turn, tool call, and result as a structured trace. -The current source version is **v0.13.2 (alpha)**. It includes: +The current source version is **v0.13.3 (alpha)**. It includes: - model and allow-listed [agentic tool-use steps](#agentic-steps-v03); - durable SQLite execution with [checkpoint, resume, and replay](#durability-v04); @@ -50,7 +50,14 @@ thread TwoStep { ## Install -ThreadLang is not published on PyPI yet. Install the current source: +ThreadLang requires Python 3.11 or newer. Install the published package: + +```bash +python -m pip install threadlang # core + OpenAI-compatible backend, zero runtime deps +python -m pip install 'threadlang[anthropic]' # optional Anthropic client +``` + +Or install the current source: ```bash git clone https://github.com/minglong51/threadlang.git @@ -120,6 +127,16 @@ vLLM, Together, …) over stdlib HTTP — no SDK. Tool-calling rides the OpenAI of emitting native `tool_calls` — use them for `llm`/`complete` steps, and DeepSeek (or Claude) for `agent` steps. +Provider selection fails closed: a requested real backend never silently +downgrades to dry-run output. `OPENAI_API_KEY` is read only for the official +`https://api.openai.com` endpoint; compatible endpoints use +`THREADLANG_API_KEY` or an explicit programmatic key. Keys require HTTPS unless +the endpoint is loopback; loopback HTTP bypasses environment proxies, and +unkeyed local HTTP endpoints remain supported. Provider redirects are refused, +base URLs cannot embed credentials, query parameters, or fragments, and +OpenAI-compatible response bodies are capped at 8 MiB. Invalid Unicode and +malformed tool-call payloads fail closed before execution. + ## Agentic steps (v0.3) An `agent` step is a model that can *act*. It runs a tool-use loop — model → @@ -154,23 +171,27 @@ status. If it crashes, resume it from the last completed step — no re-running finished work. ```bash -# Persist a run; prints run_id and (on failure) the resume command +# Persist a run; retryable provider-call failures print a resume command threadlang examples/two_step.thread --input text="..." --backend openai --store runs.db -# A crash prints: run failed; resume with: --store runs.db --resume -threadlang examples/two_step.thread --input text="..." --backend openai \ - --store runs.db --resume # skips completed steps, continues +# A retryable provider-call failure prints a copyable command with the original backend settings. +# Persisted inputs are loaded from the run; completed steps are skipped. +threadlang examples/two_step.thread --store runs.db --resume \ + --backend openai ``` ```python from threadlang import parse_program, run_durable, RunStore store = RunStore("runs.db") -durable = run_durable(parse_program(src), {"text": "..."}, store) -durable.run_id # the run's id -store.get_run(durable.run_id).status # 'completed' | 'failed' | 'running' -store.load_events(durable.run_id) # the persisted trace -store.list_runs() # all runs (for a dashboard) +try: + durable = run_durable(parse_program(src), {"text": "..."}, store) + durable.run_id # the run's id + store.get_run(durable.run_id).status # 'completed' | 'failed' | 'running' + store.load_events(durable.run_id) # the persisted trace + store.list_runs() # all runs (for a dashboard) +finally: + store.close() ``` The runtime stays storage-agnostic — `run_durable` hands it a write-through @@ -202,10 +223,10 @@ curl localhost:8765/runs # list all runs | Method / path | Does | |---|---| -| `POST /runs` | enqueue `{source, inputs}` (program validated first) → `run_id` | -| `GET /runs` | list runs (id, status, program, output) | +| `POST /runs` | enqueue exactly one of `{source, inputs}` or `{ir, inputs}` (validated first) → `run_id` | +| `GET /runs?limit=&offset=` | paginated run summaries | | `GET /runs/{id}` | one run: status, output, error, and the persisted trace | -| `GET /healthz` | liveness | +| `GET /healthz` / `GET /readyz` | database liveness / worker readiness + queue depth | Built from `process_one` (claim + run one queued run) and a `WorkerPool` of threads; the claim is atomic so no run executes twice. Details: @@ -250,7 +271,7 @@ app-specific support. # one ticket, durably, in-process — deterministic, no key support-triage run --ticket "The dashboard is down with 500s, urgent" --dry-run -# real model (DeepSeek native tool-calling, or local Ollama) +# real model with native tool-calling (DeepSeek, Claude, or a compatible local model) support-triage run --ticket "..." --backend openai # or serve the API + workers + dashboard with the app's tool registry wired in @@ -265,8 +286,10 @@ entrypoint. The one core change is `serve(tools=...)`, so any app can serve its own programs over the same API. `classify_priority` is deterministic keyword rules (no model call — cheap, inspectable); the model is spent only on the draft. Because the tools are pure and `DryRunClient` fires the first -allow-listed tool, the whole product runs end-to-end under `--dry-run` and is -golden-tested. Details: +allow-listed tool with placeholder arguments, the plumbing runs end-to-end +under `--dry-run` and is golden-tested. Dry-run does not validate ticket +classification, KB-search coverage, or reply quality; use a real tool-calling +model for those semantics. Details: [`docs/design/phase-5-vertical-slice.md`](docs/design/phase-5-vertical-slice.md). ## Metrics (v0.8) diff --git a/RELEASING.md b/RELEASING.md new file mode 100644 index 0000000..ad091e5 --- /dev/null +++ b/RELEASING.md @@ -0,0 +1,123 @@ +# Releasing ThreadLang + +ThreadLang publishes to PyPI from a GitHub Release through +[`.github/workflows/publish.yml`](.github/workflows/publish.yml). The workflow +uses trusted publishing; no PyPI API token is stored in the repository. + +## Prepare and verify + +Merge the release pull request only after all CI checks are green. Then start +from the exact, clean `main` commit that will be tagged. Run the verification +lane with Python 3.12, matching the publish workflow: + +```bash +set -euo pipefail + +git switch main +git pull --ff-only origin main + +VERSION="$(python3.12 -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')" +test "$(PYTHONPATH=src python3.12 -c 'import threadlang; print(threadlang.__version__)')" = "$VERSION" +test -z "$(git status --short)" + +RELEASE_ENV="$(mktemp -d)/venv" +python3.12 -m venv "$RELEASE_ENV" +"$RELEASE_ENV/bin/pip" install '.[dev,anthropic]' +"$RELEASE_ENV/bin/python" -m pytest -q +"$RELEASE_ENV/bin/ruff" check . +"$RELEASE_ENV/bin/ruff" format --check . +"$RELEASE_ENV/bin/mypy" src/threadlang +"$RELEASE_ENV/bin/bandit" -q -r src/threadlang +"$RELEASE_ENV/bin/pip-audit" + +ARTIFACT_DIR="$(mktemp -d)" +"$RELEASE_ENV/bin/python" -m build --outdir "$ARTIFACT_DIR" +"$RELEASE_ENV/bin/python" -m twine check "$ARTIFACT_DIR"/* + +SMOKE_ROOT="$(mktemp -d)" +python3.12 -m venv "$SMOKE_ROOT/venv" +"$SMOKE_ROOT/venv/bin/pip" install \ + "$ARTIFACT_DIR/threadlang-$VERSION-py3-none-any.whl" +( + cd "$SMOKE_ROOT" + test "$(env -u PYTHONPATH -u PYTHONHOME "$SMOKE_ROOT/venv/bin/threadlang" --version)" = \ + "threadlang $VERSION" + env -u PYTHONPATH -u PYTHONHOME \ + "$SMOKE_ROOT/venv/bin/threadlang-serve" --help >/dev/null + env -u PYTHONPATH -u PYTHONHOME \ + "$SMOKE_ROOT/venv/bin/support-triage" run \ + --ticket "release smoke" --dry-run --store "$SMOKE_ROOT/triage.db" >/dev/null +) +``` + +The project version in `pyproject.toml` and `threadlang.__version__` must match. +The release tag must exactly equal `v`; the publish workflow +rejects any other tag or a runtime/project version mismatch. + +## Tag and publish + +Confirm `HEAD` is the intended commit on `origin/main`, then create and push a +new lightweight tag and create a draft GitHub Release from that existing tag: + +```bash +set -euo pipefail + +VERSION="$(python3.12 -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')" +test "$(PYTHONPATH=src python3.12 -c 'import threadlang; print(threadlang.__version__)')" = "$VERSION" +test -z "$(git status --short)" +git fetch origin main +test "$(git rev-parse HEAD)" = "$(git rev-parse origin/main)" +git tag "v$VERSION" +git push origin "v$VERSION" +gh release create "v$VERSION" --verify-tag --generate-notes --draft +``` + +Inspect the draft release's generated notes and tag target: + +```bash +set -euo pipefail + +VERSION="$(python3.12 -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')" +git show --no-patch --format=fuller "v$VERSION" +gh release view "v$VERSION" --json isDraft,tagName,targetCommitish,name,body +``` + +If the draft is correct, publish it explicitly: + +```bash +set -euo pipefail + +VERSION="$(python3.12 -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')" +gh release edit "v$VERSION" --draft=false +``` + +Publishing starts, but does not immediately complete, the PyPI release. The +workflow runs `verify`, `build`, and `verify_artifact` first. Its final +`publish` job targets the protected `pypi` environment and waits for the +configured maintainer approval before exchanging its OIDC identity for a PyPI +upload. + +## Verify the release + +Watch the complete `Publish to PyPI` workflow and approve the `pypi` +deployment only after the preceding jobs pass. After PyPI reports the new +version, verify it from a fresh environment outside the repository: + +```bash +set -euo pipefail + +VERSION="$(python3.12 -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')" +VERIFY_ROOT="$(mktemp -d)" +VERIFY_ENV="$VERIFY_ROOT/venv" +python3.12 -m venv "$VERIFY_ENV" +"$VERIFY_ENV/bin/pip" install "threadlang==$VERSION" +cd "$VERIFY_ROOT" +test "$(env -u PYTHONPATH -u PYTHONHOME "$VERIFY_ENV/bin/threadlang" --version)" = \ + "threadlang $VERSION" +env -u PYTHONPATH -u PYTHONHOME "$VERIFY_ENV/bin/threadlang-serve" --help >/dev/null +env -u PYTHONPATH -u PYTHONHOME "$VERIFY_ENV/bin/support-triage" run \ + --ticket "release smoke" --dry-run --store "$VERIFY_ROOT/triage.db" >/dev/null +``` + +PyPI versions are immutable. Never move or reuse a release tag. If a published +artifact needs correction, prepare and release a new patch version. diff --git a/docs/benchmarks/dsl-comparison.md b/docs/benchmarks/dsl-comparison.md index 1a0e2a5..27512f4 100644 --- a/docs/benchmarks/dsl-comparison.md +++ b/docs/benchmarks/dsl-comparison.md @@ -2,7 +2,7 @@ This is a primary-source **semantics matrix**, not a model-quality or performance score. `Full`, `Partial`, and `No` describe documented behavior; they are not weighted or aggregated. -| Capability | ThreadLang v0.12 | Temporal | AWS Step Functions / ASL | LangGraph | AutoGen GraphFlow | Dapr Workflow | +| Capability | ThreadLang v0.13 | Temporal | AWS Step Functions / ASL | LangGraph | AutoGen GraphFlow | Dapr Workflow | |---|---|---|---|---|---|---| | Deterministic control structure | Full: parsed forward-only graph | Full: replay-compatible workflow commands | Full: declarative state-machine interpreter | Partial: graph/super-step control, not Temporal replay | Partial: deterministic routing structure; experimental | Full: replay-compatible workflow code | | Crash durability | Partial: SQLite step-boundary checkpoint | Full event-history reconstruction | Full for Standard Workflows | Full with persistent checkpointer | Application-managed save/load | Full event-sourced state via Dapr | @@ -17,7 +17,7 @@ This is a primary-source **semantics matrix**, not a model-quality or performanc ## Design conclusions -1. ThreadLang should describe v0.12 as **step-checkpoint durability**, not durable replay. +1. ThreadLang should describe v0.13 as **step-checkpoint durability**, not durable replay. 2. LLM and agent calls are nondeterministic activities and may execute more than once after a hard crash. 3. The current product boundary is intentionally smaller than Temporal, ASL, LangGraph, or Dapr: a compact textual agent DSL with local execution and inspection. 4. Durable external events, version pinning/migrations, typed state, and call-level idempotency keys belong in later versioned designs, not undocumented semantics. diff --git a/docs/design/HLD.md b/docs/design/HLD.md index 25b67b6..d6071ce 100644 --- a/docs/design/HLD.md +++ b/docs/design/HLD.md @@ -1,6 +1,6 @@ # ThreadLang — High-Level Design -*Refreshed for v0.12. Source line numbers are intentionally omitted where they +*Refreshed for v0.13.3. Source line numbers are intentionally omitted where they would make the design document brittle.* The supported production boundary is one POSIX process and one local SQLite @@ -9,115 +9,127 @@ deterministic event-history replay. See [`../production.md`](../production.md). ## Purpose -ThreadLang is a small DSL for **deterministic, fully-traceable LLM and agent -workflows** — the authoring layer of an agent platform whose bet is that every -run is a replayable, inspectable trace (`README.md:5`). A `.thread` program -declares a `context` block of fixed values, an ordered `steps` block where each -step is either a single-shot `llm` call or a tool-using `agent` loop, and a -final `emit` (`text` concat or one more `llm` call). Execution is -**parse → AST → runtime → emit**, and every phase — context binding, step call, -agent turn, tool call, tool result, denial — appends a structured `TraceEvent` -(`src/threadlang/runtime.py:14`). Around that core the repo has grown the full -platform stack: a sqlite-backed **durable run store** with checkpoint/resume +ThreadLang is a small DSL for **bounded, fully-traceable LLM and agent +workflows**. A `.thread` program declares fixed context, a forward-only graph +of `llm`, tool-using `agent`, and closed-label `route` steps, optional hard +output contracts, and a final `emit`. Source is parsed to the frozen AST and +compiled to canonical Workflow IR; validated IR executes through the strict +IR→AST compatibility bridge, with the established AST interpreter remaining +authoritative. Canonical bytes and a SHA-256 definition fingerprint give +durable runs a stable, reviewable identity. Every execution phase appends a +structured `TraceEvent`. Around that core the repo provides a sqlite-backed +**durable run store** with checkpoint/resume and definition fencing (`store.py`), a **control plane** (HTTP API + worker pool draining a pending-run queue, `control.py`/`server.py`), a read-only **observability dashboard** (`dashboard.py`), **metrics derived purely from the trace** (`metrics.py`), and a first vertical-slice product, a **support-triage app** (`apps/support_triage/`). Zero required runtime dependencies -(`pyproject.toml:13`); `anthropic` is an optional extra (`pyproject.toml:16`). +(`pyproject.toml`); `anthropic` is an optional extra (`pyproject.toml`). ## System Context ``` - .thread source ──► threadlang CLI (cli.py:26) ─────────────┐ - │ parse_program (parser.py:74) - HTTP client ──► POST /runs (server.py:115) ─► RunStore ▼ - (curl / app) GET /runs, /metrics, /ui (sqlite) run_program (runtime.py:51) + .thread source ──► threadlang CLI (cli.py) ─────────────┐ + │ parse_program (parser.py) + HTTP client ──► POST /runs (server.py) ─► RunStore ▼ + (curl / app) GET /runs, /metrics, /ui (sqlite) run_program (runtime.py) ▲ ▲ │ │ .complete / .agent_step │ │ │ ▼ - Browser ──► dashboard HTML (dashboard.py) ──────┘ │ LLMClient (llm.py:28) + Browser ──► dashboard HTML (dashboard.py) ──────┘ │ LLMClient (llm.py) │ ├─ DryRunClient (echo, no network) - WorkerPool threads (control.py:61) ────────────────┘ ├─ OpenAICompatClient ─► any /v1/chat/completions - claim pending → run_durable (store.py:317) │ (DeepSeek hosted, local Ollama, vLLM…) + WorkerPool threads (control.py) ────────────────┘ ├─ OpenAICompatClient ─► any /v1/chat/completions + claim pending → run_durable (store.py) │ (DeepSeek hosted, local Ollama, vLLM…) └─ AnthropicClient ──► Anthropic SDK ─► Claude API - support-triage CLI (apps/support_triage/app.py:60) (optional extra) + support-triage CLI (apps/support_triage/app.py) (optional extra) wraps the same store/serve/run_durable paths ``` +The source-oriented labels show the original entry path. CLI and HTTP callers +may also submit canonical IR; both forms converge on the same Workflow IR +identity and AST compatibility runtime. + External dependencies: - **OpenAI-compatible endpoints** (DeepSeek hosted by default, local Ollama, vLLM, Together…) — reached over **stdlib `urllib`**, no SDK - (`llm.py:232`, base URL default `llm.py:229`). Auth via `THREADLANG_API_KEY` - (or `OPENAI_API_KEY`), optional for local servers (`llm.py:261`). + (`llm.py`). Auth uses an explicit key or `THREADLANG_API_KEY`; + `OPENAI_API_KEY` is accepted only for the official HTTPS OpenAI host. Keys + are optional for local servers and are refused on plain HTTP except for + loopback endpoints, whose HTTP connections bypass environment proxies. + Provider redirects are refused, and endpoint URLs cannot carry embedded + credentials, query parameters, or fragments. Compatible-provider response + bodies are capped at 8 MiB, and malformed tool-call payloads fail closed + before the tool execution boundary. - **Anthropic SDK + Claude API** — only when `AnthropicClient` is used; gated - behind the `anthropic` extra and `ANTHROPIC_API_KEY` (`llm.py:137`, - `llm.py:143`). + behind the `anthropic` extra and `ANTHROPIC_API_KEY` (`llm.py`). - **sqlite** (stdlib `sqlite3`) — the run store file passed via `--store` - (`store.py:94`). No external database server. -- **CI**: GitHub Actions runs `pip install -e . pytest && pytest` on Python 3.12 - (`.github/workflows/ci.yml:17`). + (`store.py`). No external database server. +- **CI**: GitHub Actions covers Python 3.11–3.13 plus Ruff, mypy, Bandit, + packaging checks, dependency audit, and a container smoke test + (`.github/workflows/ci.yml`). There is no message broker (pending rows in sqlite *are* the queue, -`control.py:6`), no web framework (stdlib `http.server`, `server.py:21`), no -JS build (server-rendered HTML with inline CSS, `dashboard.py:9`). +`control.py`), no web framework (stdlib `http.server`, `server.py`), no +JS build (server-rendered HTML with inline CSS, `dashboard.py`). ## Component Map | Path | Responsibility | |------|----------------| -| `src/threadlang/__init__.py` | Public API surface — re-exports the whole stack: parse/run, store/durable, control plane, server, dashboard renderers, metrics, clients, tools (`__init__.py:28`). | -| `src/threadlang/ast.py` | Frozen-dataclass source AST: `Program`, `ContextBlock`, `Step` (llm) and `AgentStep` as distinct node types, `EmitBlock`, expression terms. Parser↔current-runtime contract. | -| `src/threadlang/ir.py` | Experimental non-executing Workflow IR v1 compiler. Produces immutable tagged nodes, canonical JSON bytes, and a SHA-256 definition fingerprint without changing the current AST runtime. See `v013-architecture-proposal.md`. | +| `src/threadlang/__init__.py` | Public API surface — re-exports the whole stack: parse/run, store/durable, control plane, server, dashboard renderers, metrics, clients, tools (`__init__.py`). | +| `src/threadlang/ast.py` | Frozen-dataclass source AST: `Program`, context, expressions (including optional branch references), `Step` + `ExpectRule`, `AgentStep`, `RouteStep`/arms, explicit next targets, and `EmitBlock`. Parser↔current-runtime contract. | +| `src/threadlang/ir.py` | Load-bearing Workflow IR v1 contract: AST compilation, strict untrusted-JSON loading, canonical JSON bytes, SHA-256 fingerprints, and compatibility execution through `program_from_ir`/`run_ir`. It deliberately does not introduce a second interpreter. | | `src/threadlang/parser.py` | Position-aware lexer and recursive-descent parser. It consumes all source, handles strings/comments structurally, validates graph/reference availability, and raises line/column `ParseError` diagnostics. | -| `src/threadlang/runtime.py` | Deterministic interpreter. `run_program(...) -> RuntimeResult`; runs llm steps, agent tool-use loops (allow-list enforced, denials traced), and emit. Storage-agnostic durability hooks (`trace`, `resume_outputs`, `on_step_complete`, `runtime.py:57`). | -| `src/threadlang/llm.py` | Client backends behind two protocols: `LLMClient.complete` and `AgentLLMClient.agent_step`. `DryRunClient` (deterministic echo + two-phase agent stub), `OpenAICompatClient` (stdlib HTTP), `AnthropicClient` (SDK). | -| `src/threadlang/tools.py` | The agent execution boundary: `ToolSpec`/`Tool`/`FunctionTool`, `ToolRegistry` allow-list, deterministic built-ins `echo` + `calculator` (AST-walked arithmetic, no `eval`, no `**`, `tools.py:111`). | +| `src/threadlang/runtime.py` | Deterministic control-flow interpreter. `run_program(...) -> RuntimeResult`; runs llm steps with contracts, agent tool-use loops, forward routing, and emit. Storage-agnostic durability hooks carry traces, checkpoints, and resume outputs. | +| `src/threadlang/llm.py` | Client backends behind a baseline protocol plus optional capabilities: `LLMClient.complete`, `AgentLLMClient.agent_step`, and `RouteLLMClient.route`. `DryRunClient` (deterministic echo + two-phase agent stub), `OpenAICompatClient` (stdlib HTTP), `AnthropicClient` (SDK). | +| `src/threadlang/tools.py` | The agent execution boundary: `ToolSpec`/`Tool`/`FunctionTool`, `ToolRegistry` allow-list, deterministic built-ins `echo` + `calculator` (AST-walked arithmetic, no `eval`, no `**`, `tools.py`). | | `src/threadlang/trace.py` | `TraceEvent(phase, message, data)`, `Trace` alias, `DenialCode` enum. The durable record's unit. | -| `src/threadlang/store.py` | Durability (L3): `RunStore` (sqlite tables `runs`/`events`/`step_outputs`, autocommit), `run_durable` (write-through trace + step checkpoints + resume/replay), per-run and aggregate metrics queries. | -| `src/threadlang/control.py` | Control plane workers (L4): `process_one` (atomic claim + execute one pending run) and `WorkerPool` (threads, per-thread stores, shared client). | -| `src/threadlang/server.py` | Stdlib `http.server` JSON API + dashboard host: `POST /runs`, `GET /runs[/{id}[/metrics]]`, `GET /metrics`, `GET /healthz`, `GET /` + `/ui/runs/{id}` (HTML). `serve()` starts pool + server together; `main()` is the `threadlang-serve` script. | +| `src/threadlang/store.py` | Durability (L3): `RunStore` (sqlite tables `runs`/`events`/`step_outputs`, WAL/autocommit), canonical definition/input binding with legacy source fencing, bounded queue/retention, CAS resume, write-through traces, step checkpoints, replay, and metrics queries. | +| `src/threadlang/control.py` | Control plane workers (L4): exclusive per-store process lock, orphan requeue, atomic claim, source-or-IR execution, per-thread stores, exception-contained worker loops, and readiness state. | +| `src/threadlang/server.py` | Authenticated stdlib JSON API + dashboard host: source-or-IR `POST /runs`, paginated run queries, metrics, liveness/readiness, Host/origin/body/input admission checks, and HTML views. `serve()` starts the exclusive worker pool and server together. | | `src/threadlang/dashboard.py` | Observability (L5): pure `(record, events, metrics) -> HTML` renderers for the run list (with aggregate panel) and per-run trace timeline; everything `html.escape`d; meta-refresh while a run is in flight. | | `src/threadlang/metrics.py` | Metrics (v0.8): `compute_metrics` — a pure fold over the trace into `RunMetrics` (deterministic control-flow counts vs observational latency/tokens, kept apart); `aggregate` rolls runs up per-program. | | `src/threadlang/apps/support_triage/` | Vertical slice (v0.7): `triage.thread` (agent classify+KB-search → llm draft), app tools `classify_priority`/`search_kb` over a bundled in-process KB (`kb.py`), and the `support-triage` entrypoint (`app.py`). Adds no core machinery. | | `docs/spec.md`, `docs/grammar.ebnf` | Language spec + EBNF grammar. | -| `docs/design/phase-*.md` | The per-phase build plans (agentic core → durability → control plane → observability → vertical slice). | -| `examples/*.thread` | Runnable samples: `hello`, `summarize`, `two_step`, `agent`, `release_report`. | -| `tests/` | Golden + per-version suites (`test_golden_hello.py`, `test_v1_llm.py`, `test_v03_agent.py` … `test_v08_metrics.py`), all runnable without a key via `DryRunClient`. | +| `docs/design/phase-*.md` | Historical per-phase build plans (agentic core → durability → control plane → observability → vertical slice → routing → probes → contracts); HLD/LLD are the live architecture contracts. | +| `examples/*.thread` | Runnable samples spanning interpolation, llm chains, agents, routing, contracts, and the release-report pipeline. | +| `tests/` | Golden, per-version, IR, durability-policy, parser-pressure, provider-security, and server-hardening suites; live provider credentials are not required. | ## Runtime / Deploy Model -Three console scripts, all defined in `pyproject.toml:25`: +Three console scripts, all defined in `pyproject.toml`: - **`threadlang`** (`threadlang.cli:main`) — one-shot CLI. Reads a `.thread` - file, picks a backend (`--backend dry-run|anthropic|openai`, default - anthropic with soft dry-run fallback, `cli.py:98`), runs synchronously, + file or canonical IR, picks a backend (`--backend + dry-run|anthropic|openai`, default anthropic), runs synchronously and fails + closed if a required real provider is unavailable, prints output to stdout; `--store PATH` makes the run durable/resumable and `--metrics`/`--trace` print derived views to stderr. - **`threadlang-serve`** (`threadlang.server:main`) — the long-running control plane: one process hosting a `ThreadingHTTPServer` **and** a `WorkerPool` - against the same sqlite file (`server.py:181`). Deploy is "run the process + against the same sqlite file (`server.py`). Deploy is "run the process with a store path"; default bind `127.0.0.1:8765`, default backend dry-run - (`server.py:203`). Restart-safe: the queue is `pending` rows in sqlite, and a - worker crash mid-run resumes via `run_durable`'s step checkpoints - (`control.py:11`). + (`server.py`). Restart-safe: the queue is `pending` rows in sqlite, and a + process restart requeues orphaned work and resumes from `run_durable`'s step + checkpoints. An advisory lock prevents two worker pools from owning one + store. - **`support-triage`** (`apps.support_triage.app:main`) — the vertical-slice product with two subcommands: `run --ticket ...` (one durable in-process run) and `serve` (the same API/workers/dashboard with the app's tool registry - wired in via `serve(tools=...)`, `app.py:83`). + wired in via `serve(tools=...)`, `app.py`). It is also a plain **library**: `from threadlang import parse_program, run_program, run_durable, RunStore, ...` with any object satisfying the client protocols. Concurrency model: threads only (workers + per-request handler threads), each opening its own sqlite connection; claims are serialized with -`BEGIN IMMEDIATE` so no run executes twice (`store.py:168`). Determinism: the +`BEGIN IMMEDIATE` so no run executes twice (`store.py`). Determinism: the model is the only non-deterministic part; `DryRunClient` makes even agent loops -reproducible end-to-end (`llm.py:99`). +reproducible end-to-end (`llm.py`). ## How It's Used ```bash -# One-shot CLI (see README.md:62) +# One-shot CLI (see README.md) threadlang examples/hello.thread --input name=world threadlang examples/two_step.thread --input text="..." --dry-run --trace threadlang examples/agent.thread --input task="what is 21*2?" --backend openai # DeepSeek, THREADLANG_API_KEY @@ -129,7 +141,8 @@ threadlang --store runs.db --resume # Control plane + dashboard threadlang-serve --store runs.db --port 8765 --workers 2 --backend openai -curl -X POST localhost:8765/runs -d '{"source":"thread T {...}","inputs":{"x":"hi"}}' +curl -X POST localhost:8765/runs -H 'content-type: application/json' \ + -d '{"source":"thread T {...}","inputs":{"x":"hi"}}' curl localhost:8765/runs/ # status + output + persisted trace curl localhost:8765/metrics # aggregate rollup open http://localhost:8765/ # run list; /ui/runs/ for the timeline @@ -139,10 +152,10 @@ support-triage run --ticket "The dashboard is down with 500s, urgent" --dry-run support-triage serve --store runs.db --backend openai ``` -- `--input key=value` is repeatable and becomes `inputs.` (`cli.py:30`). +- `--input key=value` is repeatable and becomes `inputs.` (`cli.py`). - Per-step model names in the `.thread` file are the cost-routing lever — a cheap open model for easy steps, a strong model only where it earns it - (`README.md:350`). + (`README.md`). - Library use mirrors the tests: `run_program(program, inputs, llm_client=..., tools=my_registry)` or `run_durable(..., store)`; see - `README.md:163` and `tests/`. + `README.md` and `tests/`. diff --git a/docs/design/LLD.md b/docs/design/LLD.md index 1f09444..92bd7c1 100644 --- a/docs/design/LLD.md +++ b/docs/design/LLD.md @@ -1,14 +1,12 @@ # ThreadLang — Low-Level Design -Refreshed for v0.12. The supported boundary is one POSIX process and one local +Refreshed for v0.13.3. The supported boundary is one POSIX process and one local SQLite store; see [`../production.md`](../production.md). Historical line references elsewhere in this document are explanatory and not API contracts. -> **Refreshed 2026-08-09.** Adds `ir.py` (flagged by the drift check) and the five -> `runs` columns the documented schema was missing — `program_sha256`, -> `inputs_sha256`, `definition_json`, `definition_sha256`, `ir_version`. The -> columns arrived by editing `store.py` in place, which the drift check cannot see; -> only the new module flagged. Where this disagrees with the code, the code wins. +> **Refreshed 2026-08-19.** Covers the shipped canonical-IR execution and +> durable-binding path plus the v0.12 admission, recovery, and ownership +> hardening. Where this disagrees with the code, the code wins. ## Module Breakdown @@ -16,19 +14,23 @@ references elsewhere in this document are explanatory and not API contracts. Frozen dataclasses (immutable) shared by parser and runtime: -- `ContextAssignment(name: str, value: str)` (`ast.py:7`); - `ContextBlock(assignments: List[ContextAssignment])` (`ast.py:13`) -- Expression terms (`ExpressionTerm` union, `ast.py:40`): `StringLiteral(value)` - (`ast.py:18`), `ContextRef(name)` (`ast.py:23`), `InputsRef(name)` - (`ast.py:28`), `StepsRef(step_name)` — `steps..output` (`ast.py:33`) -- `Expression(terms: List[ExpressionTerm])` (`ast.py:43`) -- `Step(name, model, prompt: Expression)` — single-shot llm step (`ast.py:48`) -- `AgentStep(name, model, prompt, tools: Tuple[str, ...] = (), max_iters: int = 6)` - — tool-use loop (`ast.py:61`); `StepNode = Union[Step, AgentStep]` (`ast.py:78`) -- `StepsBlock(steps: List[StepNode] = [])` (`ast.py:81`) +- `ContextAssignment(name: str, value: str)` (`ast.py`); + `ContextBlock(assignments: List[ContextAssignment])` (`ast.py`) +- Expression terms: `StringLiteral(value)`, `ContextRef(name)`, + `InputsRef(name)`, and `StepsRef(step_name, optional=False)`; the optional + form represents `steps..output?` on branch joins. +- `Expression(terms: List[ExpressionTerm])` (`ast.py`) +- `ExpectRule(kind, values, pattern, limit)` plus `Step(name, model, prompt, + next_target=None, expect=())` — contracted single-shot llm step. +- `AgentStep(name, model, prompt, tools=(), max_iters=6, next_target=None)` — + bounded tool-use loop with an optional explicit edge. +- `RouteArm(label, target)` and `RouteStep(name, model, prompt, arms, + else_target=None)` — closed-label forward dispatch. +- `StepNode = Union[Step, AgentStep, RouteStep]`. +- `StepsBlock(steps: List[StepNode] = [])` (`ast.py`) - `EmitBlock(kind: str, expression: Expression, model: Optional[str] = None)` — - `kind ∈ {"text","llm"}`; `model` only for `llm` (`ast.py:86`) -- `Program(thread_name, context, steps, emit)` (`ast.py:100`) + `kind ∈ {"text","llm"}`; `model` only for `llm` (`ast.py`) +- `Program(thread_name, context, steps, emit)` (`ast.py`) ### `parser.py` — position-aware recursive-descent parser @@ -46,230 +48,229 @@ Frozen dataclasses (immutable) shared by parser and runtime: - `run_program(program, inputs, llm_client=None, tools=None, *, trace=None, resume_outputs=None, on_step_complete=None) -> RuntimeResult` - (`runtime.py:51`). Defaults: `DryRunClient()` (`runtime.py:84`), - `default_registry()` (`runtime.py:85`). The three keyword hooks are the + (`runtime.py`). Defaults: `DryRunClient()` and `default_registry()`. The three keyword hooks are the durability seam used by `store.run_durable` — the runtime never sees storage - (`runtime.py:71`). + (`runtime.py`). - `RuntimeResult(output: str, trace: Trace, step_outputs: Dict[str, str])` - (`runtime.py:44`). -- `_build_context(program, trace) -> Dict[str, str]` (`runtime.py:99`) — one + (`runtime.py`). +- `_build_context(program, trace) -> Dict[str, str]` (`runtime.py`) — one `context` trace event per assignment. -- `_run_steps(...)` (`runtime.py:113`) — declaration order. If a step name is - in `resume_outputs` it is skipped, its stored output reused, and the skip - traced with `resumed: True` (`runtime.py:125`). Dispatches `AgentStep` vs - `Step`; calls `on_step_complete(name, output)` after each fresh step - (`runtime.py:145`). -- `_run_llm_step(...)` (`runtime.py:150`) — render prompt, trace - "Calling LLM for step", `client.complete(model, prompt)` (any exception - wrapped in `RuntimeError`, `runtime.py:168`), trace "produced output". -- `_run_agent_step(...) -> str` (`runtime.py:182`) — the tool-use loop: +- `_run_steps(...)` walks the forward-only graph from the first declaration. + A checkpointed step is skipped, its stored output reused, and the resume is + traced. Fresh `Step`, `AgentStep`, and `RouteStep` nodes dispatch to their + specific executor; routing or `then ->` selects the next index, and each + fresh output is checkpointed through `on_step_complete`. +- `_run_llm_step(...)` renders the prompt, calls `complete`, and enforces every + `expect` rule. A violation is traced and retried once with feedback; a second + violation fails the run. `one_of` may use the optional `route` client method. +- `_run_route_step(...)` asks for one label from the declared arm set, retries + one invalid answer, then follows the matching arm, `else`, or fails closed. +- `_run_agent_step(...) -> str` (`runtime.py`) — the tool-use loop: 1. Requires the client to expose `agent_step` (duck-typed via `getattr`; - `RuntimeError` otherwise, `runtime.py:196`). + `RuntimeError` otherwise, `runtime.py`). 2. Validates every allow-listed tool exists in the registry - (`runtime.py:203`); collects `specs` and the `allowed` set. + (`runtime.py`); collects `specs` and the `allowed` set. 3. Seeds `messages = [{"role": "user", "content": prompt}]`, traces - "Agent ... started" with tools + max_iters (`runtime.py:213`). + "Agent ... started" with tools + max_iters (`runtime.py`). 4. Up to `max_iters` turns: call `agent_step(model, messages, tools=specs)`; trace the turn (text + tool_calls). Empty `tool_calls` → trace - "finished", return `response.text` (`runtime.py:251`). + "finished", return `response.text` (`runtime.py`). 5. Otherwise append the assistant message and, per call: if allowed **and** registered, `registry.get(name).run(arguments)` — a raising tool becomes - an observable `"error: ..."` string, not a crash (`runtime.py:267`); + an observable `"error: ..."` string, not a crash (`runtime.py`); else a `phase="denial"` event with `DenialCode.TOOL_NOT_ALLOWED` / - `TOOL_NOT_REGISTERED` (`runtime.py:283`). Results feed back as - `{"role": "tool", "tool_call_id": ...}` messages (`runtime.py:302`). + `TOOL_NOT_REGISTERED` (`runtime.py`). Results feed back as + `{"role": "tool", "tool_call_id": ...}` messages (`runtime.py`). 6. Loop exhaustion raises `RuntimeError("... exceeded max_iters ...")` - (`runtime.py:306`). -- `_evaluate_emit(...)` (`runtime.py:311`) — `text` → concat with per-term - tracing; `llm` → render, assert model (parser invariant, `runtime.py:323`), + (`runtime.py`). +- `_evaluate_emit(...)` (`runtime.py`) — `text` → concat with per-term + tracing; `llm` → render, assert model (parser invariant, `runtime.py`), call the client (wrapped on failure); unknown kind raises. - `_render_expression(expression, context, inputs, step_outputs, trace=None)` - (`runtime.py:340`) — resolves terms; missing context/input or a - forward/unknown step reference raises `RuntimeError` (`runtime.py:354`, - `runtime.py:359`, `runtime.py:363`). Per-term trace events only when `trace` - is passed — only `emit text` passes it (`runtime.py:320`). -- `RuntimeError(ValueError)` (`runtime.py:40`) — shadows the builtin - deliberately; re-exported by the package (`__init__.py:23`). + (`runtime.py`) — resolves terms; missing context/input or a + forward/unknown step reference raises `RuntimeError`. Per-term trace events only when `trace` + is passed — only `emit text` passes it (`runtime.py`). +- `RuntimeError(ValueError)` (`runtime.py`) — shadows the builtin + deliberately; re-exported by the package (`__init__.py`). ### `llm.py` — client backends -Two protocols, both implemented by every backend: +One baseline protocol plus two step-specific capabilities: -- `LLMClient.complete(model: str, prompt: str) -> str` (`llm.py:28`) — plain +- `LLMClient.complete(model: str, prompt: str) -> str` (`llm.py`) — plain `llm` steps and `emit llm`. - `AgentLLMClient.agent_step(model, messages: Sequence[Message], - tools: Sequence[ToolSpec]) -> AgentTurn` (`llm.py:63`). + tools: Sequence[ToolSpec]) -> AgentTurn` (`llm.py`) — required by `agent` + steps. +- `RouteLLMClient.route(model, prompt, options: Sequence[str]) -> str` + (`llm.py`) — optional closed-label capability; the runtime falls back to + `complete` when it is absent. + +All built-in backends implement `complete` and `agent_step`; `DryRunClient` +also implements `route` directly. The CLI's lazy wrapper exposes `route` and +delegates to the selected backend's `complete` fallback when necessary. Shapes: -- `ToolCall(id, name, arguments: Dict[str, object])` (`llm.py:35`) +- `ToolCall(id, name, arguments: Dict[str, object])` (`llm.py`) - `AgentTurn(text, tool_calls: Sequence[ToolCall] = ())` — empty `tool_calls` - means done (`llm.py:45`) + means done (`llm.py`) - `Message = Dict[str, object]` — the runtime-owned normalized message; roles `user` / `assistant` (text + tool_calls) / `tool` (tool_call_id + content) - (`llm.py:56`) -- `LLMError(RuntimeError)` (`llm.py:73`) + (`llm.py`) +- `LLMError(RuntimeError)` (`llm.py`) Backends: -- `DryRunClient` (`llm.py:92`) — `complete` returns +- `DryRunClient` (`llm.py`) — `complete` returns `f"[dry-run:{model}] {prompt}"`. `agent_step` is a deterministic two-phase loop: if tools exist and no tool has run yet, call the *first* tool with placeholder args deterministically derived from its JSON schema - (`_placeholder_args`, `llm.py:77`); otherwise finalize, echoing the latest - tool/user observation (`llm.py:99`). Makes agent programs golden-testable. -- `AnthropicClient(api_key=None, max_tokens=1024)` (`llm.py:128`) — lazy-imports - the SDK (`LLMError` if absent, `llm.py:136`), reads `ANTHROPIC_API_KEY` - (`llm.py:143`). `agent_step` maps normalized messages ↔ Anthropic content + (`_placeholder_args`, `llm.py`); otherwise finalize, echoing the latest + tool/user observation (`llm.py`). Makes agent programs golden-testable. +- `AnthropicClient(api_key=None, max_tokens=1024)` (`llm.py`) — lazy-imports + the SDK (`LLMError` if absent, `llm.py`), reads `ANTHROPIC_API_KEY` + (`llm.py`). `agent_step` maps normalized messages ↔ Anthropic content blocks (`tool_use` / `tool_result`) via `_to_anthropic_messages` - (`llm.py:194`). + (`llm.py`). - `OpenAICompatClient(base_url=None, api_key=None, max_tokens=1024, - timeout=120.0)` (`llm.py:232`) — any `/v1/chat/completions` server over - stdlib `urllib` (`_post`, `llm.py:269`). Defaults: `THREADLANG_BASE_URL` or - DeepSeek (`llm.py:229`); key from `THREADLANG_API_KEY` or `OPENAI_API_KEY`, - optional for local servers (`llm.py:261`). HTTP/URL/JSON failures raise - `LLMError` with truncated detail (`llm.py:280`). Tool-calling rides the - OpenAI `tools`/`tool_calls` shape (`_to_openai_messages`, `llm.py:351`; + timeout=120.0)` (`llm.py`) — any `/v1/chat/completions` server over + stdlib `urllib` (`_post`, `llm.py`). Defaults: `THREADLANG_BASE_URL` or + DeepSeek. An explicit key or `THREADLANG_API_KEY` applies to compatible + endpoints; ambient `OPENAI_API_KEY` is used only when the endpoint is + `https://api.openai.com`. Keys are optional for local servers and keyed HTTP + is accepted only for loopback endpoints, using a proxy-disabled opener. + Endpoint URLs containing userinfo, a query, or a fragment are rejected, and + provider redirects are refused. HTTP/URL/JSON failures raise `LLMError`; + upstream HTTP bodies and endpoint details are never copied into durable + errors. + Tool-calling rides the + OpenAI `tools`/`tool_calls` shape (`_to_openai_messages`, `llm.py`; defensive `choices[0].message` extraction in `_openai_message`, - `llm.py:340`; malformed tool-call arguments degrade to `{}`, `llm.py:326`). -- `default_client() -> LLMClient` returns `AnthropicClient()` (`llm.py:387`). + `llm.py`). Responses over 8 MiB, invalid Unicode/text shapes, and malformed + or non-object tool-call arguments raise `LLMError` before tool execution. +- `default_client() -> LLMClient` returns `AnthropicClient()` (`llm.py`). Both HTTP clients hold no mutable per-call state — the thread-safety contract -the shared-client `WorkerPool` relies on (`control.py:66`). +the shared-client `WorkerPool` relies on (`control.py`). ### `tools.py` — the execution boundary - `ToolSpec(name, description, parameters: Dict[str, object])` — the JSON - schema the model sees (`tools.py:32`). -- `Tool` Protocol: `.spec` + `.run(args: Mapping) -> str` (`tools.py:43`); - `FunctionTool(spec, _fn)` wraps a plain callable (`tools.py:49`). -- `ToolRegistry` (`tools.py:61`) — `register` (rejects duplicates), `has`, + schema the model sees (`tools.py`). +- `Tool` Protocol: `.spec` + `.run(args: Mapping) -> str` (`tools.py`); + `FunctionTool(spec, _fn)` wraps a plain callable (`tools.py`). +- `ToolRegistry` (`tools.py`) — `register` (rejects duplicates), `has`, `get`, `specs(names)`, `names()`. An agent step references tools by name; only the registry turns a name into code. -- Built-ins (`default_registry()`, `tools.py:181`): `echo` (`tools.py:95`) and - `calculator` (`tools.py:162`). The calculator parses with `ast.parse` and - walks the tree (`_eval_arithmetic`, `tools.py:125`) — only numeric literals +- Built-ins (`default_registry()`, `tools.py`): `echo` (`tools.py`) and + `calculator` (`tools.py`). The calculator parses with `ast.parse` and + walks the tree (`_eval_arithmetic`, `tools.py`) — only numeric literals and whitelisted operators (`+ - * / // % `, unary `+/-`); `**` is excluded as - a DoS vector (`tools.py:108`). Errors return `"error: ..."` strings. + a DoS vector (`tools.py`). Errors return `"error: ..."` strings. ### `trace.py` - `TraceEvent(phase: str, message: str, data: Dict[str, Any] = {})` - (`trace.py:13`); `Trace = List[TraceEvent]` (`trace.py:20`). + (`trace.py`); `Trace = List[TraceEvent]` (`trace.py`). - `DenialCode(str, Enum)`: `TOOL_NOT_ALLOWED = "tool-not-allowed"`, - `TOOL_NOT_REGISTERED = "tool-not-registered"` (`trace.py:8`). -- Phases in use: `context`, `step`, `agent`, `denial`, `runtime`, `emit` - (colors in `dashboard.py:80`). + `TOOL_NOT_REGISTERED = "tool-not-registered"` (`trace.py`). +- Phases in use: `context`, `step`, `agent`, `route`, `contract`, `denial`, + `runtime`, `emit` + (colors in `dashboard.py`). ### `store.py` — durable run store -- `RunStore(path)` (`store.py:86`) — stdlib sqlite, `isolation_level=None` - (autocommit: every write durable immediately, `store.py:93`), - `busy_timeout = 5000` for cross-thread claims (`store.py:98`), schema - applied idempotently plus `_migrate()` which `ALTER TABLE`s `events.ts` onto - pre-v0.8 stores (`store.py:102`). -- `RunRecord(id, program_name, status, inputs, output, error, source=None)` - (`store.py:75`). -- Runs: `create_run(program_name, inputs) -> run_id` (status `running`, - `store.py:130`), `get_run`, `list_runs()` (newest first, `store.py:144`), - `mark_running/mark_completed/mark_failed` (`store.py:201`). -- Queue: `enqueue_run(program_name, source, inputs)` inserts `pending` with the - program source (`store.py:153`); `claim_next_pending()` takes the oldest - pending under `BEGIN IMMEDIATE` and flips it to `running` — the atomic claim - that guarantees single execution (`store.py:168`). -- Events: `append_event(run_id, event)` assigns `seq = MAX(seq)+1` and stamps - wall-clock `ts` (`store.py:221`); `load_events(run_id) -> Trace` - (`store.py:232`). -- Checkpoints: `save_step_output` (upsert, `store.py:244`) / - `load_step_outputs` (`store.py:251`). -- Metrics queries: `run_metrics(run_id) -> Optional[RunMetrics]` (fold of the - persisted trace + timestamp span, `store.py:265`); `aggregate_metrics()` - (`store.py:277`). -- `_WriteThroughTrace(List[TraceEvent])` (`store.py:292`) — overrides `append` - to also persist; the runtime appends through it unknowingly. -- `run_durable(program, inputs, store, *, llm_client=None, tools=None, - run_id=None) -> DurableRun` (`store.py:317`): - - `run_id` of a **completed** run → replay: return the stored output, events, - and step outputs with no model calls (`store.py:340`). - - `run_id` of a failed/running run → resume: preload `step_outputs` as - `resume_outputs`, `mark_running` (`store.py:350`). - - No `run_id` → `create_run`. Then execute `run_program` with the - write-through trace and a checkpoint closure; any exception → - `mark_failed` + re-raise (`store.py:370`); success → `mark_completed`. - - `DurableRun(run_id, result: RuntimeResult)` (`store.py:308`). +- `RunStore(path)` opens a per-thread stdlib sqlite connection in WAL mode, + enables foreign keys and a five-second busy timeout, applies the schema, and + performs additive migrations. Writes use autocommit. +- `RunRecord` includes status/output/error plus optional source, source/input + digests, canonical `definition_json`, definition digest, IR version, and + timestamps. +- `create_run(...)` inserts `created`; `mark_running(expected=...)` is a + compare-and-swap transition that fences concurrent CLI resumes. +- `enqueue_run(...)` and `enqueue_ir(...)` bind canonical definition and input identity, + enforce the pending limit, prune terminal retention, and insert `pending` + under `BEGIN IMMEDIATE`. `claim_next_pending()` atomically claims the oldest + row. `requeue_orphans()` moves restart-stranded sourced/IR runs back to + `pending`. +- Events are sequenced and timestamped; step outputs are upserted checkpoints. + Per-run and aggregate metrics are folds over those persisted events. +- `run_durable(...)` compiles the current program to canonical IR and binds its + digest with canonical inputs. The source digest remains metadata and the + identity fence for legacy rows lacking canonical definition identity. Resume + verifies stored IR integrity, definition/input identity, IR version, and + eligible status before it loads checkpoints. A completed run replays without + model calls; a fresh run moves `created→running`; any execution exception + marks `failed`; success marks `completed`. ### `control.py` — worker pool - `process_one(store, *, llm_client=None, tools=None) -> Optional[DurableRun]` - (`control.py:31`) — claim, `parse_program(claimed.source)`, `run_durable` - with the claimed id. A raising run is already marked failed; the exception is - swallowed so one bad run never kills a worker (`control.py:55`). Returns - `None` on empty queue. + claims one row, loads its bound canonical IR when present (or parses legacy + source), and calls `run_durable` with the claimed id. Every malformed or + raising run is marked failed and contained so one job cannot kill a worker. - `WorkerPool(store_path, *, n_workers=2, llm_client=None, tools=None, - poll_interval=0.05)` (`control.py:61`) — `start()` spawns daemon threads - (`control.py:87`); each `_loop` opens its **own** `RunStore` (sqlite - connections are per-thread, `control.py:94`) and polls `process_one`, - waiting `poll_interval` on empty. `stop(timeout=5.0)` joins; `drain(store, - max_runs=10_000)` processes synchronously in the current thread (tests / - batch mode, `control.py:112`). + poll_interval=0.05)` acquires `.worker.lock`, requeues orphaned + `running` rows, then starts daemon threads with one `RunStore` each. The loop + contains store/provider infrastructure errors; `is_healthy()`/`status()` + expose thread liveness. `stop()` joins and releases the lock; `drain()` is + the synchronous batch/test path and continues past failed jobs. ### `server.py` — HTTP API + dashboard host -- `_Handler(BaseHTTPRequestHandler)` (`server.py:32`) on a - `ThreadingHTTPServer`; each request opens/closes its own `RunStore` - (`server.py:37`, `server.py:112`). -- `do_GET` (`server.py:59`): `/` and `/ui` → `render_run_list(list_runs, - aggregate_metrics)`; `/ui/runs/{id}` → `render_run_detail(record, events, - run_metrics)` (HTML 404 for unknown); `/healthz` → `{"ok": true}`; - `/metrics` → `aggregate_metrics().to_dict()`; `/runs/{id}/metrics`; - `/runs` (summaries); `/runs/{id}` (summary + full `trace` array). JSON 404 - otherwise. -- `do_POST /runs` (`server.py:115`): validates JSON body, requires non-empty - string `source` and dict `inputs`, **parses the program before enqueuing** - (400 with `parse error: ...` on `ParseError`, `server.py:134`), stringifies - input keys/values, returns `201 {"run_id", "status": "pending"}`. -- `make_server(store_path, host="127.0.0.1", port=8765)` (`server.py:159`) — - builds the server, stashing `store_path` on it. -- `serve(store_path, *, host, port, n_workers=2, llm_client=None, tools=None)` - (`server.py:166`) — starts the `WorkerPool` then blocks in - `serve_forever()`; `tools` is the seam apps use to serve their own registries - (`server.py:177`). `main()` (`server.py:197`) is the `threadlang-serve` - script: `--store` (required), `--host`, `--port`, `--workers`, - `--backend dry-run|anthropic|openai` (default **dry-run**), `--base-url`. +- `_Handler(BaseHTTPRequestHandler)` runs on `ThreadingHTTPServer`; each data + request opens and closes its own `RunStore`. JSON and HTML responses set + no-store, nosniff, and frame-denial headers; HTML also has a restrictive CSP. +- Tokenless mode admits only loopback Host/origin traffic. A configured bearer + token gates every data, dashboard, metrics, and submission route; + `/healthz` and `/readyz` intentionally reveal only database, worker, and + pending/running queue state. +- `GET /runs` is bounded and paginated; run detail includes the trace. Metrics, + dashboard list/detail, liveness, and readiness have dedicated routes. +- `POST /runs` requires JSON content type and bounded length, exactly one of + non-empty UTF-8 `.thread` source or an IR object, and bounded string inputs. + Source is parsed+compiled and IR is strictly loaded before the canonical + definition is enqueued. Capacity exhaustion returns 429; validation returns + 4xx without creating a row. +- `make_server(...)` validates bind/auth/admission settings. `serve(...)` + starts the exclusive `WorkerPool` and HTTP server together and stops both on + shutdown. The `threadlang-serve` CLI defaults to loopback + dry-run and + exposes provider, worker, queue, retention, timeout, and auth-token-env knobs. ### `dashboard.py` — pure HTML renderers - `render_run_list(runs: List[RunRecord], aggregate: Optional[AggregateMetrics]) - -> str` (`dashboard.py:167`) — table of id/program/status/output-or-error - with an aggregate metrics chip panel (`_aggregate_panel`, `dashboard.py:149`); - meta-refresh every 1s while any run is pending/running (`dashboard.py:195`). -- `render_run_detail(record, events, metrics=None) -> str` (`dashboard.py:199`) + -> str` (`dashboard.py`) — table of id/program/status/output-or-error + with an aggregate metrics chip panel (`_aggregate_panel`, `dashboard.py`); + meta-refresh every 1s while any run is pending/running (`dashboard.py`). +- `render_run_detail(record, events, metrics=None) -> str` (`dashboard.py`) — header (status badge, inputs, output/error), per-run metric chips - (`_run_metrics_panel`, `dashboard.py:129`; warn styling for tool errors / + (`_run_metrics_panel`, `dashboard.py`; warn styling for tool errors / denials / resumed steps), then the phase-colored `TraceEvent` timeline - (`_PHASE_COLOR`, `dashboard.py:80`). If `metrics` is omitted it is derived - from `events` alone (`dashboard.py:208`). + (`_PHASE_COLOR`, `dashboard.py`). If `metrics` is omitted it is derived + from `events` alone (`dashboard.py`). - Every interpolated value passes `_esc` = `html.escape(..., quote=True)` - (`dashboard.py:90`) — model output and trace data are untrusted. + (`dashboard.py`) — model output and trace data are untrusted. ### `metrics.py` — derived metrics -- `RunMetrics` (`metrics.py:46`) — deterministic block: `context_vars`, +- `RunMetrics` (`metrics.py`) — deterministic block: `context_vars`, `steps_completed`, `agent_steps`, `agent_turns`, `model_calls` - (= complete calls + agent turns, `metrics.py:169`), `tool_calls`, + (= complete calls + agent turns, `metrics.py`), `tool_calls`, `tool_errors`, `denials`, `resumed_steps`, `status`; observational block: `duration_ms`, `input_tokens`, `output_tokens` (all Optional — `None` means "not recorded", not zero). Properties `ok`, `total_tokens`; `to_dict()` - nests `{deterministic, observational}` (`metrics.py:82`). + nests `{deterministic, observational}` (`metrics.py`). - `compute_metrics(trace, *, status=None, duration_ms=None) -> RunMetrics` - (`metrics.py:105`) — a pure fold matching exactly the event shapes - `runtime.run_program` appends (phase/message patterns, `metrics.py:128`). + (`metrics.py`) — a pure fold matching exactly the event shapes + `runtime.run_program` appends (phase/message patterns, `metrics.py`). Token usage is read from any event `data.usage` dict; the built-in clients - don't emit it yet (`metrics.py:27`). -- `trace_span_ms(timestamps) -> Optional[float]` (`metrics.py:181`) — first-to- + don't emit it yet (`metrics.py`). +- `trace_span_ms(timestamps) -> Optional[float]` (`metrics.py`) — first-to- last ISO timestamp span; `None` under two parseable stamps (pre-v0.8 rows). -- `AggregateMetrics` (`metrics.py:198`) + `aggregate(items: - Sequence[Tuple[str, RunMetrics]])` (`metrics.py:228`) — `by_status`, +- `AggregateMetrics` (`metrics.py`) + `aggregate(items: + Sequence[Tuple[str, RunMetrics]])` (`metrics.py`) — `by_status`, `success_rate` = completed/(completed+failed) over terminal runs only, `avg_duration_ms`, call/error/denial totals, and a `by_program` breakdown. @@ -277,30 +278,29 @@ the shared-client `WorkerPool` relies on (`control.py:66`). - `triage.thread` — `SupportTriage`: agent step `investigate` (`deepseek-chat`, `tools [ classify_priority, search_kb ]`, `max_iters 5`) - → llm step `draft` → `emit text { steps.draft.output }` - (`triage.thread:1`). -- `app.py` — `PROGRAM_PATH` points at the bundled program (`app.py:34`; - shipped via package-data, `pyproject.toml:23`); `load_program()` - (`app.py:37`); `main()` (`app.py:60`) with subcommands: + → llm step `draft` → `emit text { steps.draft.output }`. +- `app.py` — `PROGRAM_PATH` points at the bundled program (`app.py`; + shipped via package-data, `pyproject.toml`); `load_program()` + (`app.py`); `main()` (`app.py`) with subcommands: - `serve --store ... [--host --port --workers --backend --base-url]` → - `serve(..., tools=build_registry())` (`app.py:83`). + `serve(..., tools=build_registry())` (`app.py`). - `run --ticket ... [--store triage-runs.db] [--dry-run --backend --base-url]` → `run_durable(load_program(), {"ticket": ...}, store, tools=registry)`; - prints `run_id`/status + output; exit 0 iff completed (`app.py:107`). + prints `run_id`/status + output; exit 0 iff completed (`app.py`). - `tools.py` — `classify_priority`: deterministic keyword rules mapping ticket - text to P0/P1/P2 (`_P0_SIGNALS`/`_P1_SIGNALS`, `tools.py:28`; `_classify`, - `tools.py:45`). `search_kb`: token-overlap scoring over the bundled articles, - tag hits weighted double, top 2 returned (`_score`, `tools.py:65`; - `_search_kb`, `tools.py:73`). `build_registry()` = `default_registry()` + - both (`tools.py:128`). -- `kb.py` — `Article(id, title, body, tags)` (`kb.py:17`) and the four-article - in-process `ARTICLES` list (`kb.py:25`). Swapping in a real store is a - tool-implementation detail (`kb.py:6`). + text to P0/P1/P2 (`_P0_SIGNALS`/`_P1_SIGNALS`, `tools.py`; `_classify`, + `tools.py`). `search_kb`: token-overlap scoring over the bundled articles, + tag hits weighted double, top 2 returned (`_score`, `tools.py`; + `_search_kb`, `tools.py`). `build_registry()` = `default_registry()` + + both (`tools.py`). +- `kb.py` — `Article(id, title, body, tags)` (`kb.py`) and the four-article + in-process `ARTICLES` list (`kb.py`). Swapping in a real store is a + tool-implementation detail (`kb.py`). -### `ir.py` — versioned canonical IR (724 lines) +### `ir.py` — versioned canonical IR `IR_VERSION = "threadlang.ir/v1"`, `LANGUAGE_VERSION = "threadlang/v0.12"` -(`ir.py:47-48`). IR v1 losslessly represents the v0.12 source AST for inspection, +(`ir.py`). IR v1 losslessly represents the v0.12 source AST for inspection, stable serialization, and definition fingerprints. **It is not a second interpreter.** The docstring is explicit: the existing runtime @@ -310,65 +310,77 @@ separately reviewed and verified. Read that as a deliberate refusal, not a gap. - Frozen node types mirroring the AST: `IRContextEntry`, `IRTerm`, `IRExpression`, `IRExpectation`, `IRRouteArm`, `IRLLMStep`, `IRAgentStep`, `IRRouteStep`, - `IREmit`, and the `WorkflowIR` root (`ir.py:56-138`). -- `compile_program(program: Program) -> WorkflowIR` (`:207`) — AST → IR. -- `program_from_ir(workflow: WorkflowIR) -> Program` (`:279`) — the bridge back; + `IREmit`, and the `WorkflowIR` root (`ir.py`). +- `compile_program(program: Program) -> WorkflowIR` (`ir.py`) — AST → IR. +- `program_from_ir(workflow: WorkflowIR) -> Program` (`ir.py`) — the bridge back; this is what lets a stored definition execute on the existing runtime. -- `load_ir_bytes(payload: bytes) -> WorkflowIR` (`:659`) — parse + validate; - raises `IRCompileError` (`:51`). -- `canonical_ir_bytes(workflow) -> bytes` (`:712`) — UTF-8 JSON with `sort_keys` and +- `run_ir(workflow, inputs, llm_client=None, tools=None) -> RuntimeResult` + (`ir.py`) is the explicit compatibility execution entry point. +- `load_ir_bytes(payload: bytes) -> WorkflowIR` (`ir.py`) — parse + validate; + raises `IRCompileError`. +- `canonical_ir_bytes(workflow) -> bytes` (`ir.py`) — UTF-8 JSON with `sort_keys` and `(",", ":")` separators. The canonicalization is the point: identity must not change because a dict happened to iterate differently. -- `workflow_fingerprint(workflow) -> str` (`:722`) — SHA-256 of those bytes. +- `workflow_fingerprint(workflow) -> str` (`ir.py`) — SHA-256 of those bytes. Imported by `store.py`, `server.py`, `control.py`, `cli.py`, and re-exported from `__init__.py`, which makes it a load-bearing contract rather than a utility. ### `cli.py` — `threadlang` entry point -- `main() -> int` (`cli.py:26`) — args: `source` (positional Path), - `--input k=v` (repeatable, `_parse_inputs` splits on first `=`, `cli.py:16`), +- `main() -> int` (`cli.py`) — args: `source` (positional Path), + `--input k=v` (repeatable, `_parse_inputs` splits on first `=`, `cli.py`), `--backend dry-run|anthropic|openai` (default **anthropic**), `--dry-run` - (shorthand), `--base-url`, `--store PATH`, `--resume RUN_ID` (requires - `--store`, exit 2 otherwise, `cli.py:83`), `--trace`, `--metrics`. -- Client selection (`cli.py:92`): dry-run / openai / anthropic; a failed - `AnthropicClient()` soft-falls-back to `DryRunClient`, warning only if the - program actually needs a model (`cli.py:100`). + (shorthand), `--base-url`, provider limits, `--store PATH`, `--resume + RUN_ID`, `--probe N`, `--trace`, `--metrics`, `--from-ir`, and `--emit-ir`. +- Client selection is dry-run / OpenAI-compatible / Anthropic. A workflow that + needs a model fails closed when the selected real client is unavailable; a + pure `emit text` workflow can continue because it never calls the client. - With `--store`: establishes the run id up front so it can be reported even on - a crash (`cli.py:119`), runs `run_durable`; on `LLMError`/`RuntimeError` - prints the exact resume command and exits 1 (`cli.py:126`). Without: + a crash, then runs `run_durable`; provider-call failures print a shell-quoted + resume command preserving source/IR mode, provider, endpoint, token limit, + and timeout, then exit 1. Deterministic runtime failures exit 1 without an + unusable retry hint. Definition/input/status refusals exit 2. Without a store: plain `run_program`, metrics computed from the in-memory trace with - `status="completed"` (`cli.py:144`). + `status="completed"` (`cli.py`). - Output to stdout; `run_id`, trace lines, and metrics JSON to stderr. ## Data Models ### `.thread` source contract -Per `docs/grammar.ebnf` and `docs/spec.md` (spec text predates v0.3 — the -grammar file and parser are current): +The normative grammar is `docs/grammar.ebnf`; this excerpt shows the execution +shape: ``` program = "thread" name "{" context [ steps ] emit "}" context = "context" "{" { name "=" string } "}" steps = "steps" "{" { step } "}" -step = "step" name "{" llm_body | agent_body "}" -llm_body = "llm" string "{" expression "}" +step = "step" name "{" llm_body | agent_body | route_body "}" +llm_body = "llm" string "{" expression [ expect ] [ then_decl ] "}" agent_body = "agent" string "{" [ "tools" "[" name {"," name} "]" ] - [ "max_iters" int ] expression "}" + [ "max_iters" int ] expression [ then_decl ] "}" +route_body = "route" string "{" expression arm { arm } [ else_decl ] "}" +arm = "on" string "->" target +expect = "expect" "{" expect_rule { expect_rule } "}" +expect_rule = one_of | matches | max_chars | nonempty +then_decl = "then" "->" target +target = name | "end" emit = "emit" "text" "{" expression "}" | "emit" "llm" string "{" expression "}" expression = term { "+" term } -term = string | "context." name | "inputs." name | "steps." name ".output" +term = string | "context." name | "inputs." name + | "steps." name ".output" [ "?" ] ``` Constraints: `context` required, `steps` optional, exactly one `emit`, unique -step names, `max_iters >= 1`, tool names must be identifiers. +step names, bounded `max_iters`, tool names must be identifiers, and all graph +edges/references must be statically valid and forward-only. -### sqlite schema (`store.py:41`) +### sqlite schema (`store.py`) ```sql -runs (id TEXT PK, program_name TEXT, status TEXT, -- pending|running|completed|failed +runs (id TEXT PK, program_name TEXT, status TEXT, -- created|pending|running|completed|failed inputs_json TEXT, source TEXT, -- source set when enqueued via the API program_sha256 TEXT, inputs_sha256 TEXT, -- reproducibility of what ran definition_json TEXT, -- the canonical IR (ir.py) @@ -381,104 +393,114 @@ step_outputs (run_id TEXT, step_name TEXT, output TEXT, PRIMARY KEY (run_id, step_name)) ``` -### HTTP JSON contracts (`server.py:5`) - -- `POST /runs` body `{"source": "", "inputs": {str: str}}` → - `201 {"run_id", "status": "pending"}`; `400` on bad JSON / missing source / - non-object inputs / parse error. -- Run summary (`_run_summary`, `server.py:148`): - `{id, program_name, status, inputs, output, error}`; `GET /runs/{id}` adds - `trace: [{phase, message, data}]`. +### HTTP JSON contracts (`server.py`) + +- `POST /runs` accepts exactly one of `{"source": ""}` or + `{"ir": }`, plus optional `{str: str}` inputs. It returns + `201 {"run_id", "status": "pending"}`; malformed, oversized, + non-UTF-8-encodable, or policy-invalid submissions fail before enqueue. +- Run summary (`_run_summary`, `server.py`): + `{id, program_name, status, inputs, output, error, created_at, updated_at, + program_sha256, inputs_sha256, definition_sha256, ir_version}`; + `GET /runs/{id}` adds `trace: [{phase, message, data}]`. +- `GET /runs?limit=N&offset=N` is bounded/paginated. `/healthz` verifies the + store; `/readyz` also verifies worker liveness and reports queue depth. - `GET /runs/{id}/metrics` → `{"run_id", "metrics": {deterministic: {...}, - observational: {...}}}` (shape in `metrics.py:82`). + observational: {...}}}` (shape in `metrics.py`). - `GET /metrics` → `{total_runs, by_status, success_rate, avg_duration_ms, total_model_calls, total_tool_calls, total_tool_errors, total_denials, by_program: {name: {runs, completed, failed, success_rate, - avg_duration_ms}}}` (`metrics.py:214`). + avg_duration_ms}}}` (`metrics.py`). ### TraceEvent payloads (by phase) -- `context` — `{name, value}` (`runtime.py:103`) +- `context` — `{name, value}` (`runtime.py`) - `step` — call `{step, model, prompt}`; output `{step, output}`; resume - `{step, output, resumed: true}` (`runtime.py:129`) + `{step, output, resumed: true}` (`runtime.py`) - `agent` — started `{step, model, prompt, tools, max_iters}`; turn `{step, turn, text, tool_calls: [{name, arguments}]}`; tool call `{step, tool, arguments, result}`; finished `{step, turns, output}` -- `denial` — `{step, tool, arguments, code, result}` (`runtime.py:289`) -- `runtime` — term eval `{source, value}` (emit-text only, `runtime.py:373`) -- `emit` — llm call `{model, prompt}`; final `{output}` (`runtime.py:93`) +- `route` — decision attempts/violations and chosen `{step, label, target}` +- `contract` — rejected llm output plus the violated rules and retry attempt +- `denial` — `{step, tool, arguments, code, result}` (`runtime.py`) +- `runtime` — term eval `{source, value}` (emit-text only, `runtime.py`) +- `emit` — llm call `{model, prompt}`; final `{output}` (`runtime.py`) -The metrics fold (`metrics.py:128`) and the dashboard timeline both consume +The metrics fold (`metrics.py`) and the dashboard timeline both consume exactly these shapes — change them in lockstep. ## Main Control Flow ### Queued path (control plane, the production shape) -1. `threadlang-serve --store runs.db ...` (`server.py:197`) builds a backend +1. `threadlang-serve --store runs.db ...` (`server.py`) builds a backend client, starts `WorkerPool.start()` + `ThreadingHTTPServer` - (`server.py:181`). -2. `POST /runs` (`server.py:115`) validates + parses the source, then - `store.enqueue_run(...)` inserts a `pending` row → `201 {run_id}`. -3. A worker's `_loop` (`control.py:93`) calls `process_one`: - `claim_next_pending()` atomically flips pending→running (`store.py:168`); - the source is re-parsed (`control.py:45`); `run_durable` executes with a + (`server.py`). +2. `POST /runs` validates exactly one source/IR definition, canonicalizes it, + binds its digest with the inputs, then inserts a `pending` row. +3. A worker's `_loop` (`control.py`) calls `process_one`: + `claim_next_pending()` atomically flips pending→running; the canonical IR + is loaded through the compatibility bridge; `run_durable` executes with a `_WriteThroughTrace` (every event lands in `events` as it happens) and a - step-checkpoint hook (`store.py:355`). + step-checkpoint hook (`store.py`). 4. Inside `run_program`: context → steps (llm calls and/or agent tool-use loops) → emit, as detailed above. Success → `mark_completed`; failure → `mark_failed` with the error string. 5. Clients poll `GET /runs/{id}` (status + trace) or watch `/ui/runs/{id}` — which meta-refreshes until the run settles - (`dashboard.py:244`). Metrics on `/metrics` are recomputed from the same - rows on each request (`store.py:277`). -6. Crash recovery: a worker death leaves the run `running` with checkpoints - intact; calling `run_durable` with the same id resumes, skipping - checkpointed steps (`store.py:350`, `runtime.py:125`). A completed id - replays without model calls (`store.py:340`). + (`dashboard.py`). Metrics on `/metrics` are recomputed from the same + rows on each request (`store.py`). +6. Crash recovery: after exclusive store ownership is reacquired on process + restart, sourced/IR `running` rows are requeued. The next claim resumes with + bound-definition checks and skips checkpointed steps. A completed id + replays without model calls. ### One-shot CLI path `threadlang file.thread --input k=v [--store runs.db]` → read + parse → select client → `run_durable` (durable) or `run_program` (ephemeral) → print -output; on durable failure print `resume with: --store ... --resume ` -(`cli.py:126`). +output; on a retryable durable provider-call failure print `resume with: +--store ... --resume ` with the original backend/endpoint/limit settings +and a shell-quoted source. ## Error Handling -- **Parse** → `ParseError(ValueError)`: bad thread wrapper (`parser.py:77`), - missing context (`parser.py:96`), invalid assignment (`parser.py:106`), - unbalanced braces (`parser.py:126`), duplicate step (`parser.py:146`), - invalid step body (`parser.py:180`), bad tool name / `max_iters < 1` - (`parser.py:194`, `parser.py:203`), missing prompt/emit/term - (`parser.py:173`, `parser.py:243`, `parser.py:264`). The API converts these - to HTTP 400 before enqueuing (`server.py:135`). +- **Parse** → `ParseError(ValueError)`: bad thread wrapper (`parser.py`), + missing context, invalid assignments, unbalanced braces, duplicate or invalid + steps, bad tool names or iteration bounds, and missing prompt/emit/terms. The + API converts these to HTTP 400 before enqueuing (`server.py`). - **Runtime** → `RuntimeError(ValueError)`: non-agent client on an agent step - (`runtime.py:197`), unknown allow-listed tool (`runtime.py:205`), max_iters - exhaustion (`runtime.py:306`), unknown context/input/step refs - (`runtime.py:354`–`runtime.py:363`), unknown emit kind (`runtime.py:337`). + (`runtime.py`), unknown allow-listed tools, iteration exhaustion, unknown + context/input/step references, and unknown emit kinds. - **Model-call failures are wrapped**: exceptions from `complete`/`agent_step` re-raise as `RuntimeError` naming the step/phase, original chained via - `from exc` (`runtime.py:168`, `runtime.py:230`, `runtime.py:333`). + `from exc` (`runtime.py`). - **Tool failures are observable, not fatal**: a raising tool yields an - `"error: ..."` result string fed back to the model (`runtime.py:268`); - disallowed/unregistered tools yield traced denials (`runtime.py:282`). + `"error: ..."` result string fed back to the model (`runtime.py`); + disallowed/unregistered tools yield traced denials (`runtime.py`). - **Client construction/transport** → `LLMError(RuntimeError)`: SDK missing / - no key (`llm.py:138`, `llm.py:145`); HTTP status, unreachable host, non-JSON - body, malformed choices (`llm.py:280`–`llm.py:288`, `llm.py:344`). -- **Durable runs**: any exception → `mark_failed` + re-raise (`store.py:370`); - `process_one` swallows it so the worker survives (`control.py:55`); the CLI - catches it and prints the resume command (`cli.py:123`). -- **CLI**: `--resume` without `--store` → exit 2 (`cli.py:83`); run failures → - exit 1; malformed `--input` → `ValueError` (`cli.py:20`). Parse errors - surface as tracebacks (no top-level catch in `main`). + no key, invalid or insecure keyed endpoint, HTTP status, unreachable host, + non-JSON body, or malformed choices (`llm.py`). +- **Durable runs**: any exception → `mark_failed` + re-raise (`store.py`); + `process_one` swallows it so the worker survives (`control.py`); the CLI + prints a resume command only for provider-call failures, where retrying the + same bound definition and inputs can make progress. +- **CLI boundaries**: provider/runtime failures exit 1; invalid arguments, + source/IR, resume identity, filesystem, and sqlite failures exit 2. Common + operator errors are rendered as one-line diagnostics, not tracebacks. +- **HTTP boundaries**: malformed request targets, JSON/Unicode, wrong content + type, invalid Host/origin/auth, oversized bodies/inputs, invalid source/IR, + and capacity exhaustion become explicit 4xx responses. A bad submission + creates no run; an execution failure becomes a failed run without killing + its worker. ## Config / Env Surface ### `policy.py` — fail-closed resource limits -Module-level constants, no env override, imported by `ir.py`, `parser.py`, -`runtime.py` and `server.py`. Its docstring is the scope statement worth keeping: +Module-level constants, no env override, imported by `ir.py`, `llm.py`, +`parser.py`, `runtime.py`, and `server.py`. Its docstring is the scope +statement worth keeping: these are **conservative defaults for the single-node runtime, not distributed-runtime service-level guarantees** — a workload that needs more should split programs or put an authenticated admission layer in front of the server, @@ -493,6 +515,7 @@ rather than raise the numbers. | `MAX_REGEX_PATTERN_CHARS` / `MAX_REGEX_INPUT_CHARS` | 512 / 64 Ki | regex surface | | `REGEX_TIMEOUT_SECONDS` | 1.0 | per-match wall clock — the ReDoS floor | | `MAX_REQUEST_BYTES` | 1 MiB | HTTP body | +| `MAX_PROVIDER_RESPONSE_BYTES` | 8 MiB | OpenAI-compatible response body | | `MAX_INPUTS` / `MAX_INPUT_KEY_CHARS` / `MAX_INPUT_VALUE_CHARS` | 128 / 128 / 64 Ki | run inputs | | `DEFAULT_MAX_PENDING_RUNS` / `DEFAULT_MAX_RETAINED_RUNS` | 1 000 / 10 000 | queue + retention | | `DEFAULT_LIST_LIMIT` / `MAX_LIST_LIMIT` | 100 / 1 000 | list pagination | @@ -500,19 +523,25 @@ rather than raise the numbers. Fail-closed means a value over the limit is rejected, never truncated — a silently clipped program would execute something the author did not write. -### Environment variables (all read in `llm.py`) +### Environment variables - `ANTHROPIC_API_KEY` — `AnthropicClient` when no `api_key` passed - (`llm.py:143`). + (`llm.py`). - `THREADLANG_BASE_URL` — default endpoint for `OpenAICompatClient` - (falls back to DeepSeek, `llm.py:256`). -- `THREADLANG_API_KEY`, then `OPENAI_API_KEY` — bearer token for - `OpenAICompatClient`; optional for local servers (`llm.py:261`). - -CLI flags: see `cli.py` (`threadlang`), `server.py:202` (`threadlang-serve`), -`app.py:64` (`support-triage`). Backend defaults differ deliberately: -`threadlang` defaults to `anthropic` with soft fallback; `threadlang-serve` -and `support-triage` default to `dry-run`. + (falls back to DeepSeek, `llm.py`). +- `THREADLANG_API_KEY` — generic bearer token for the configured + OpenAI-compatible endpoint; optional for local servers and refused over + plain HTTP except on loopback. +- `OPENAI_API_KEY` — fallback only for `https://api.openai.com`, never for + DeepSeek or an arbitrary compatible host. +- `THREADLANG_AUTH_TOKEN` — default control-plane bearer-token variable; + `--auth-token-env` selects a different variable name. + +CLI flags: see `cli.py` (`threadlang`), `server.py` (`threadlang-serve`), +`app.py` (`support-triage`). Backend defaults differ deliberately: +`threadlang` defaults to `anthropic` and fails closed when a model workflow +cannot construct it; `threadlang-serve` and `support-triage` default to +explicit `dry-run`. Programmatic knobs: `AnthropicClient(api_key, max_tokens=1024)`; `OpenAICompatClient(base_url, api_key, max_tokens=1024, timeout=120.0)`; @@ -520,7 +549,7 @@ Programmatic knobs: `AnthropicClient(api_key, max_tokens=1024)`; port=8765, n_workers=2, llm_client, tools)`; `run_program(tools=...)` / `run_durable(run_id=...)`. -Packaging: zero runtime deps (`pyproject.toml:13`); optional extra -`anthropic>=0.40,<1.0` (`pyproject.toml:16`); `requires-python >= 3.10` -(`pyproject.toml:10`); `triage.thread` ships as package data -(`pyproject.toml:23`). No settings file or dotenv loading exists. +Packaging: zero runtime deps (`pyproject.toml`); optional extra +`anthropic>=0.40,<1.0` (`pyproject.toml`); `requires-python >= 3.11` +(`pyproject.toml`); `triage.thread` ships as package data +(`pyproject.toml`). No settings file or dotenv loading exists. diff --git a/docs/ir-production.md b/docs/ir-production.md index 73dae4e..95b0028 100644 --- a/docs/ir-production.md +++ b/docs/ir-production.md @@ -83,7 +83,8 @@ Resume fails closed when: - stored canonical definition cannot be parsed and validated; - stored bytes do not match the persisted digest; - the supplied workflow definition differs from the original run; -- source or inputs violate the existing v0.12 resume fence. +- canonical inputs differ, or a legacy row without definition identity fails + its source/program fence. Older databases migrate additively. Existing rows keep nullable definition fields and are bound only when the current program identity can be proved under the v0.12 migration rules. diff --git a/docs/production.md b/docs/production.md index 5ba65c0..59ed955 100644 --- a/docs/production.md +++ b/docs/production.md @@ -1,14 +1,14 @@ -# ThreadLang v0.12 production profile +# ThreadLang single-node production profile -ThreadLang v0.12 supports a deliberately narrow **single-node, POSIX, local-filesystem** production profile. It is not a distributed durable-execution engine and does not claim Temporal/Dapr-style event-history replay. +ThreadLang v0.13 retains the deliberately narrow **single-node, POSIX, local-filesystem** production profile introduced in v0.12. It is not a distributed durable-execution engine and does not claim Temporal/Dapr-style event-history replay. ## Supported boundary - One `threadlang-serve` process per SQLite store. - Linux or macOS/POSIX filesystem with working advisory file locks. - SQLite WAL on a local disk; network filesystems are unsupported. -- Step-boundary checkpoints. A process death can rerun the current incomplete LLM/agent step. -- LLM calls are therefore at-least-once. Durable runs reject custom tools declared as both side-effecting and non-idempotent. +- Step-boundary checkpoints. A process death can repeat model/tool calls in the current incomplete step, or an incomplete `emit llm`. +- Model and tool calls are therefore at-least-once. Durable runs reject custom tools declared as both side-effecting and non-idempotent. - Forward-only graphs only; `max_iters` is capped by runtime policy. ## Start safely @@ -19,13 +19,15 @@ Loopback development mode needs no token: threadlang-serve --store ./runs.db --backend dry-run ``` -Any non-loopback bind requires a bearer token supplied through an environment variable, never argv: +The built-in listener is plaintext HTTP and does not terminate TLS. For remote access, keep it behind a TLS-terminating reverse proxy or on another trusted, access-controlled transport; never expose the raw listener to an untrusted network. Supply the bearer token through an environment variable, never argv: ```bash export THREADLANG_AUTH_TOKEN="$(python -c 'import secrets; print(secrets.token_urlsafe(32))')" -threadlang-serve --store /data/threadlang.db --host 0.0.0.0 --backend anthropic +threadlang-serve --store /data/threadlang.db --host 127.0.0.1 --backend anthropic ``` +If a reverse proxy in another container or host requires a non-loopback bind, use `--host 0.0.0.0` only on a firewalled private link and keep the external leg under TLS. + All data, dashboard, metrics, and submission routes require `Authorization: Bearer …` when a token is configured. `/healthz` and `/readyz` expose only health/queue state. ## Operational controls @@ -35,7 +37,9 @@ All data, dashboard, metrics, and submission routes require `Authorization: Bear - `--max-tokens` and `--timeout`: provider response and deadline policy. - `--workers`: local worker-thread count. - `THREADLANG_AUTH_TOKEN`: default bearer-token environment variable. -- `THREADLANG_API_KEY`, `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`: provider credentials. Never store these in source or inputs. +- `THREADLANG_API_KEY`: bearer credential for DeepSeek or another configured OpenAI-compatible endpoint; keyed endpoints require HTTPS except on loopback. +- `OPENAI_API_KEY`: used only when the configured endpoint is the official `https://api.openai.com` host. +- `ANTHROPIC_API_KEY`: Anthropic credential. Never store provider credentials in source or inputs. The worker pool owns `.worker.lock`. A second process fails startup rather than requeueing work active in the first process. Kernel lock release after process death makes startup orphan requeue safe within the supported local-filesystem boundary. @@ -46,25 +50,26 @@ The worker pool owns `.worker.lock`. A second process fails startup rathe ## Durability contract -- Program source and canonical inputs are SHA-256 bound to a run. -- Resume rejects changed source/inputs and uses a compare-and-swap status transition. -- Only validated step outputs and resolved route labels are checkpointed. +- Canonical Workflow IR and canonical inputs are SHA-256 bound to a v0.13 run; a source digest is retained as metadata and for legacy rows without IR identity. +- Resume verifies stored IR integrity and rejects changed canonical definitions or inputs using a compare-and-swap status transition. Formatting- or comment-only source changes that compile to identical IR do not change execution identity. +- Completed step outputs are checkpointed only after their outgoing edge has resolved; emit output is not a step checkpoint. - Regex output contracts execute in a killable isolated interpreter with size and time limits. - Non-idempotent side-effecting tools are rejected on the durable path. -LLM/agent steps remain at-least-once across a hard crash. Tool authors must truthfully declare `ToolSpec.side_effects` and `ToolSpec.idempotent`. Exactly-once external effects are out of scope. +Model and tool calls, including `emit llm`, remain at-least-once across a hard crash. Tool authors must truthfully declare `ToolSpec.side_effects` and `ToolSpec.idempotent`. Exactly-once external effects are out of scope. ## Data and secrets -SQLite stores inputs, outputs, traces, and tool observations in plaintext. Protect the database and lock file using OS permissions and encrypted storage where required. Provider HTTP bodies are not copied into durable errors. Retention is count-based; legal/time-based erasure remains an operator responsibility. +SQLite stores inputs, outputs, traces, and tool observations in plaintext. Protect the database and lock file using OS permissions and encrypted storage where required. For the OpenAI-compatible client, HTTP response bodies and endpoint details are not copied into durable errors, redirects are refused, endpoint URLs cannot embed credentials, queries, or fragments, keyed non-loopback endpoints require HTTPS, loopback HTTP bypasses environment proxies, and responses are capped at 8 MiB. Malformed OpenAI-compatible text and tool-call payloads fail before persistence or tool execution. Retention is count-based; legal/time-based erasure remains an operator responsibility. -## Upgrade from v0.11 +## Upgrade to v0.13 1. Stop every writer and back up the SQLite database using the online backup API or a clean shutdown/copy. -2. Install v0.12 and start exactly one process. `RunStore` performs additive, idempotent column/index migrations. +2. Install v0.13 and start exactly one process. `RunStore` performs additive, idempotent column/index migrations, including direct upgrades from older stores. 3. Runs created before source hashing cannot be safely resumed if their original source is unavailable. Inspect or fail them explicitly rather than fabricating source. -4. Revalidate custom programs: v0.12 rejects ignored tokens, malformed strings/comments, backward references, unavailable `steps.*` references, duplicate route labels, and out-of-policy sizes that older parsers could accept or misparse. -5. Downgrade after opening a store with v0.12 is not supported without restoring the backup. +4. Revalidate custom programs: the stricter parser rejects ignored tokens, malformed strings/comments, backward references, unavailable `steps.*` references, duplicate route labels, and out-of-policy sizes that older versions could accept or misparse. +5. New runs bind canonical Workflow IR and its digest. Older rows retain nullable definition fields and can resume only when their source/input identity is provable; see [`ir-production.md`](ir-production.md). +6. Downgrade after opening a store with v0.13 is not supported without restoring the backup. ## Backup and restore @@ -72,7 +77,11 @@ Use SQLite's online backup API or stop the server before copying the database. D ## Container -The supplied image runs as a non-root user and stores state under `/data`. Because its default bind is `0.0.0.0`, `THREADLANG_AUTH_TOKEN` is mandatory at startup. +The supplied image runs as a non-root user and stores state under `/data`. Because its default bind is `0.0.0.0`, `THREADLANG_AUTH_TOKEN` is mandatory at startup and the exposed port must remain behind TLS or a trusted private transport. + +Maintainers publish packages through the gated procedure in +[`RELEASING.md`](../RELEASING.md); production operators do not need that +workflow. ## Explicit non-goals / future work diff --git a/docs/spec.md b/docs/spec.md index 1702446..a463bbd 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -12,20 +12,19 @@ every phase. - Explicit, inspectable AST nodes (frozen dataclasses). - Runtime traceability — every step appends a `TraceEvent`. - No hidden magic; clarity over cleverness. -- Zero runtime dependencies for `emit text` programs; one optional dep - (`anthropic`) for programs that call a real model. +- Zero required runtime dependencies. The OpenAI-compatible backend uses + stdlib HTTP; the `anthropic` extra is required only for `AnthropicClient`. ## Non-goals in v1 -- Loops. -- Recursion. -- Branching / conditionals. *(Since v0.9: forward-only routing — see below. - Cyclic control flow remains out.)* +- Cyclic control flow, loops, and recursion. +- Parallel step scheduling. - Streaming output. -- Tool use / function calling. *(Since v0.3: `agent` steps run a tool-use - loop over an allow-listed registry.)* -- System prompts (LLM calls send a single user-role message). -- Advanced type system (terms are strings, period). +- External events, human approval, cancellation, and in-flight migration. +- Distributed execution. +- Source-level system prompts (plain llm/emit calls send one user-role prompt; + agent message history is runtime-owned). +- An advanced type system (expression values are strings). These are deliberate. The point of v1 is that the workflow shape (context → steps → emit) actually executes; the surface area is held narrow on @@ -35,36 +34,41 @@ extension. ## Supported syntax ``` -program = "thread" name "{" context [ steps ] emit "}" +program = "thread" name "{" context [ steps ] emit_block "}" context = "context" "{" { name "=" string } "}" steps = "steps" "{" { step } "}" step = "step" name "{" ( llm_body | agent_body | route_body ) "}" llm_body = "llm" string "{" expression [ expect ] [ then ] "}" agent_body = "agent" string "{" [ tools ] [ max_iters ] expression [ then ] "}" +tools = "tools" "[" [ name { "," name } ] "]" +max_iters = "max_iters" integer expect = "expect" "{" rule { rule } "}" rule = "one_of" string { "," string } | "matches" string - | "max_chars" number | "nonempty" + | "max_chars" integer | "nonempty" route_body = "route" string "{" expression arm { arm } [ "else" "->" target ] "}" arm = "on" string "->" target then = "then" "->" target target = name | "end" +emit_block = emit_text | emit_llm emit_text = "emit" "text" "{" expression "}" emit_llm = "emit" "llm" string "{" expression "}" expression = term { "+" term } term = string | "context." name | "inputs." name | "steps." name ".output" [ "?" ] +integer = digit { digit } ``` - `context` block: name → string-literal map. Required. -- `steps` block: zero or more `step` definitions. Optional. Each step - calls an LLM and binds the response to `steps..output`. +- `steps` block: zero or more forward-only `llm`, `agent`, or `route` step + definitions. Optional. Each executed step binds its output to + `steps..output`; route edges can skip later steps. - `emit` block: required. Either `emit text` (string concatenation over expression terms) or `emit llm "" { ... }` (rendered prompt sent to the model; response becomes the program output). - Step names within a single `steps` block must be unique. `end` is a reserved jump target and cannot be a step name. -- Source, string, step-count, expression, contract, and `max_iters` limits are - normative fail-closed runtime policy. Programs exceeding them are invalid. +- Source bytes, string literals, regex patterns, and `max_iters` are bounded by + normative fail-closed policy; values over those limits are rejected. - Comments and delimiters inside quoted strings are lexical content, not structure. The parser consumes all input and reports line/column errors. @@ -72,11 +76,13 @@ term = string | "context." name | "inputs." name The language semantics are independent of storage. The bundled durable runtime provides step-boundary checkpoints on one POSIX process and one local SQLite -store. It binds a run to hashes of its source and canonical inputs and rejects -concurrent resume with a compare-and-swap transition. A hard crash may repeat -the current incomplete LLM/agent step; this is not deterministic event-history -replay. Side-effecting tools must be declared idempotent to run durably. The -full operational contract is [`production.md`](production.md). +store. It binds a v0.13 run to its canonical Workflow IR and canonical inputs; +the source digest is retained as metadata and as the legacy resume fence for +rows without IR identity. Concurrent resume is rejected with a compare-and-swap +transition. A hard crash may repeat the current incomplete `llm`, `agent`, or +`route` step, or an incomplete `emit llm`; this is not deterministic +event-history replay. Side-effecting tools must be declared idempotent to run +durably. The full operational contract is [`production.md`](production.md). ### Step graph (v0.9) @@ -138,49 +144,106 @@ final answer is shaped by its tool loop. ## Runtime behavior -1. Build context (deterministic map). -2. For each step in declaration order: - a. Render its prompt expression against (context, inputs, prior step outputs). - b. Call `client.complete(model, prompt)` on the provided `LLMClient`. - c. Bind the response to `steps..output`. -3. Evaluate the emit block: +1. Execute source through `run_program`, or strictly load Workflow IR v1 and + bridge it through `program_from_ir`/`run_ir` to the same authoritative AST + interpreter. +2. Build the deterministic context map. +3. Traverse the forward-only step graph from the first declared step: + - `llm` renders its prompt and calls `complete` (or optional `route` for a + `one_of` contract). Contracts are validated, retried once with feedback, + and then fail closed. + - `agent` calls `agent_step` in a bounded loop, executes only allow-listed + tools, feeds observations back to the client, and binds the tool-free + final answer. + - `route` calls optional `route` or falls back to `complete`, normalizes the + closed-label result, retries one rejection, takes the matching forward + edge, uses `else` after a second miss, or fails if no `else` is declared. + - After step completion and outgoing-edge resolution, bind the output and + invoke the optional checkpoint callback. Resumed outputs skip their + model/tool work. +4. Evaluate the emit block: - `emit text` — concatenate expression terms. - `emit llm` — render prompt expression, call model, return response. -4. Return `RuntimeResult(output, trace, step_outputs)`. +5. Append the final emit event and return + `RuntimeResult(output, trace, step_outputs)`. -Trace events are appended at each context binding, each step (one for -"calling", one for "produced output"), each rendered expression term in -`emit text`, and on the final emit. +Trace phases cover context construction, llm and agent turns, tool calls and +denials, routing, contract rejection, `emit text` term evaluation, checkpoint +reuse, and final emission. A resumed route re-derives its edge from the stored +output without another model call. ## LLM client protocol ```python class LLMClient(Protocol): def complete(self, model: str, prompt: str) -> str: ... + +class AgentLLMClient(Protocol): + def agent_step( + self, + model: str, + messages: Sequence[Message], + tools: Sequence[ToolSpec], + ) -> AgentTurn: ... + +class RouteLLMClient(Protocol): + def route( + self, + model: str, + prompt: str, + options: Sequence[str], + ) -> str: ... ``` +`complete` is the baseline protocol for `llm`, `emit llm`, and the fallback +route path. `agent_step` is required only by `agent` steps. `route` is optional; +when absent, the runtime sends the same closed-label contract through +`complete`. + Built-in implementations: -- `DryRunClient` — returns `f"[dry-run:{model}] {prompt}"`. Used by tests - and by `threadlang --dry-run`. -- `AnthropicClient` — real Claude calls via the `anthropic` SDK. Requires - the optional install (`pip install 'threadlang[anthropic]'`) and - `ANTHROPIC_API_KEY` in env. +- `DryRunClient` — deterministic `complete`, first-option routing, and a + two-phase agent stub. Used by tests and explicit `threadlang --dry-run`. +- `AnthropicClient` — Claude `complete` and native tool use via the optional + `anthropic` SDK. Requires `pip install 'threadlang[anthropic]'` and + `ANTHROPIC_API_KEY`. +- `OpenAICompatClient` — dependency-free stdlib HTTP client implementing + `complete` and `agent_step` through OpenAI `tools`/`tool_calls`. It defaults + to DeepSeek and can target OpenAI, Ollama, vLLM, or another compatible `/v1` + endpoint. -Any object satisfying the `complete(model, prompt) -> str` protocol works; -plug in OpenAI, Ollama, etc., as needed. +Any object satisfying only `complete(model, prompt) -> str` can run llm/emit +work and routes through the fallback. Agent programs require `agent_step`. ## CLI ``` -threadlang [--input k=v ...] [--dry-run] [--trace] +threadlang [--version] [--from-ir] [--emit-ir PATH] + [--input k=v ...] + [--backend {dry-run,anthropic,openai}] [--dry-run] + [--base-url URL] [--max-tokens N] [--timeout SECONDS] + [--store PATH] [--resume RUN_ID] [--probe N] + [--trace] [--metrics] + ``` - `--input` is repeatable. Keys are referenced as `inputs.`. -- `--dry-run` uses `DryRunClient` even if the Anthropic SDK + API key are - available. -- If the Anthropic SDK / key are missing and the program needs an LLM - call, the CLI falls back to `DryRunClient` with a warning rather than - erroring out — useful for "does my program parse and route values - correctly" checks. -- `--trace` prints structured trace events to stderr after the output. +- `--from-ir` strictly loads canonical Workflow IR instead of source; + `--emit-ir PATH` compiles or normalizes IR and exits (`-` writes stdout). It + cannot combine with `--store`, `--resume`, or `--probe`. +- `--backend` selects a real provider or deterministic dry-run and defaults to + `anthropic`; `--max-tokens` and `--timeout` configure provider calls. + `--dry-run` is shorthand for `--backend dry-run`. `--base-url` configures + the OpenAI-compatible endpoint. +- `--store` enables durable trace/checkpoint persistence. `--resume` requires + `--store`, verifies the current definition and effective inputs against + stored identity, and reuses completed checkpoints. `--probe N` also requires + `--store`, cannot combine with `--resume`, and prints a persisted stability + report. +- `--trace` prints structured trace events to stderr; `--metrics` prints + metrics derived from that trace. +- If the selected real provider cannot be constructed and the program needs a + model call, the CLI exits with an error. It never turns a real run into + synthetic dry-run output; use `--dry-run` explicitly for plumbing checks. +- A program with no model steps and `emit text` still runs without a provider, + because its selected client is never called. diff --git a/examples/release_report.thread b/examples/release_report.thread index 8283458..cdc66c0 100644 --- a/examples/release_report.thread +++ b/examples/release_report.thread @@ -7,7 +7,7 @@ thread ReleaseReport { steps { step compute { agent "deepseek-chat" { - tools [ calculator, echo ] + tools [ echo, calculator ] max_iters 4 "Work out the percentage change implied by these release stats. Use the calculator tool for any arithmetic; do not estimate. Report each figure on its own line. Stats:\n" + inputs.stats } diff --git a/pyproject.toml b/pyproject.toml index ef6b66f..6125fa3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "threadlang" -version = "0.13.2" +version = "0.13.3" description = "A compact DSL and single-node runtime for bounded, traceable LLM and agent workflows" readme = "README.md" requires-python = ">=3.11" diff --git a/src/threadlang/__init__.py b/src/threadlang/__init__.py index 820aff0..be7e80e 100644 --- a/src/threadlang/__init__.py +++ b/src/threadlang/__init__.py @@ -37,7 +37,7 @@ from .store import DurableRun, RunRecord, RunStore, RunStoreCapacityError, run_durable from .tools import FunctionTool, Tool, ToolRegistry, ToolSpec, default_registry -__version__ = "0.13.2" +__version__ = "0.13.3" __all__ = [ "parse_program", diff --git a/src/threadlang/apps/support_triage/app.py b/src/threadlang/apps/support_triage/app.py index a2c9bef..26588af 100644 --- a/src/threadlang/apps/support_triage/app.py +++ b/src/threadlang/apps/support_triage/app.py @@ -21,14 +21,17 @@ from __future__ import annotations import argparse +import math import os +import sqlite3 import sys from pathlib import Path from typing import Optional from ...llm import AnthropicClient, DryRunClient, LLMClient, LLMError, OpenAICompatClient from ...parser import parse_program -from ...server import serve +from ...policy import DEFAULT_MAX_PENDING_RUNS, DEFAULT_MAX_RETAINED_RUNS +from ...server import _validate_server_options, serve from ...store import RunStore, run_durable from .tools import build_registry @@ -83,8 +86,23 @@ def main(argv: Optional[list] = None) -> int: _add_backend_args(p_run) args = parser.parse_args(argv) - if args.max_tokens < 1 or args.timeout <= 0: + if args.max_tokens < 1 or not math.isfinite(args.timeout) or args.timeout <= 0: parser.error("--max-tokens and --timeout must be positive") + auth_token = None + if args.cmd == "serve": + auth_token = os.environ.get(args.auth_token_env) + try: + _validate_server_options( + args.host, + args.port, + auth_token, + DEFAULT_MAX_PENDING_RUNS, + DEFAULT_MAX_RETAINED_RUNS, + ) + if args.workers < 1: + raise ValueError("workers must be >= 1") + except ValueError as exc: + parser.error(str(exc)) registry = build_registry() try: @@ -97,7 +115,7 @@ def main(argv: Optional[list] = None) -> int: n_workers=args.workers, llm_client=client, tools=registry, - auth_token=os.environ.get(args.auth_token_env), + auth_token=auth_token, ) return 0 @@ -123,7 +141,7 @@ def main(argv: Optional[list] = None) -> int: print(f"run_id: {durable.run_id} status: {final.status if final else '?'}\n") print(durable.result.output) return 0 if final and final.status == "completed" else 1 - except (LLMError, FileNotFoundError) as exc: + except (LLMError, OSError, RuntimeError, sqlite3.Error, ValueError) as exc: print(f"error: {exc}", file=sys.stderr) return 1 diff --git a/src/threadlang/cli.py b/src/threadlang/cli.py index 6017851..553e0fa 100644 --- a/src/threadlang/cli.py +++ b/src/threadlang/cli.py @@ -3,19 +3,32 @@ from __future__ import annotations import argparse +import math +import shlex +import sqlite3 import sys +from collections.abc import Callable, Sequence from pathlib import Path from typing import Dict, List from . import __version__ from .ir import canonical_ir_bytes, compile_program, load_ir_bytes, program_from_ir -from .llm import AnthropicClient, DryRunClient, LLMClient, LLMError, OpenAICompatClient +from .llm import ( + AgentTurn, + AnthropicClient, + DryRunClient, + LLMClient, + LLMError, + Message, + OpenAICompatClient, +) from .metrics import compute_metrics from .parser import parse_program from .probe import ProbeRunData, probe_report from .runtime import RuntimeError as TLRuntimeError from .runtime import RuntimeResult, run_program from .store import RunStore, run_durable +from .tools import ToolSpec def _parse_inputs(input_flags: List[str]) -> Dict[str, str]: @@ -24,10 +37,48 @@ def _parse_inputs(input_flags: List[str]) -> Dict[str, str]: if "=" not in item: raise ValueError(f"Invalid --input format: {item!r}; expected key=value") key, value = item.split("=", 1) + if not key: + raise ValueError(f"Invalid --input format: {item!r}; key cannot be empty") parsed[key] = value return parsed +class _LazyClient: + def __init__(self, factory: Callable[[], LLMClient]) -> None: + self._factory = factory + self._client: LLMClient | None = None + + def _get(self) -> LLMClient: + if self._client is None: + self._client = self._factory() + return self._client + + def complete(self, model: str, prompt: str) -> str: + return self._get().complete(model, prompt) + + def agent_step( + self, model: str, messages: Sequence[Message], tools: Sequence[ToolSpec] + ) -> AgentTurn: + agent_step = getattr(self._get(), "agent_step", None) + if agent_step is None: + raise LLMError("selected backend does not support agent steps") + return agent_step(model, messages, tools) + + def route(self, model: str, prompt: str, options: Sequence[str]) -> str: + client = self._get() + route = getattr(client, "route", None) + if route is None: + return client.complete(model, prompt) + return route(model, prompt, options) + + +def _should_offer_resume(exc: BaseException) -> bool: + return isinstance(exc, LLMError) or ( + isinstance(exc, TLRuntimeError) + and str(exc).startswith(("LLM call failed", "Agent call failed")) + ) + + def main() -> int: parser = argparse.ArgumentParser(description="Run a ThreadLang source file.") parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}") @@ -116,7 +167,7 @@ def main() -> int: file=sys.stderr, ) return 2 - if args.max_tokens < 1 or args.timeout <= 0: + if args.max_tokens < 1 or not math.isfinite(args.timeout) or args.timeout <= 0: print("error: --max-tokens and --timeout must be positive", file=sys.stderr) return 2 if args.probe is not None: @@ -135,7 +186,10 @@ def main() -> int: try: return _run(args) - except (OSError, ValueError) as exc: + except LLMError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + except (OSError, sqlite3.Error, ValueError) as exc: print(f"error: {exc}", file=sys.stderr) return 2 @@ -158,74 +212,79 @@ def _run(args: argparse.Namespace) -> int: Path(args.emit_ir).write_bytes(payload + b"\n") return 0 + inputs = _parse_inputs(args.input) backend = "dry-run" if args.dry_run else args.backend client: LLMClient if backend == "dry-run": client = DryRunClient() elif backend == "openai": - client = OpenAICompatClient( - base_url=args.base_url, - max_tokens=args.max_tokens, - timeout=args.timeout, + client = _LazyClient( + lambda: OpenAICompatClient( + base_url=args.base_url, + max_tokens=args.max_tokens, + timeout=args.timeout, + ) ) else: - try: - client = AnthropicClient(max_tokens=args.max_tokens, timeout=args.timeout) - except LLMError as exc: - # Soft fallback: a program with no steps and `emit text` doesn't - # need an LLM client. Warn only if the program will actually - # need one and we couldn't build it. - client = DryRunClient() - if program.steps.steps or program.emit.kind == "llm": - print( - f"warning: {exc}\n (falling back to dry-run; output is the echoed prompt, not real LLM output)", - file=sys.stderr, - flush=True, - ) + client = _LazyClient( + lambda: AnthropicClient(max_tokens=args.max_tokens, timeout=args.timeout) + ) - inputs = _parse_inputs(args.input) if args.probe is not None: return _probe(args, program, inputs, client) result: RuntimeResult if args.store: store = RunStore(args.store) - if args.resume: - prior = store.get_run(args.resume) - if prior is not None: - inputs = {**prior.inputs, **inputs} - # Establish the run id up front (create fresh, or reuse the one being - # resumed) so we can report it even if the run crashes mid-flight. - run_id = args.resume or store.create_run(program.thread_name, inputs) - print(f"run_id: {run_id}", file=sys.stderr) try: - durable = run_durable( - program, - inputs, - store, - llm_client=client, - run_id=run_id, - source=source_text, - ) - except ValueError as exc: - print(f"error: {exc}", file=sys.stderr) - store.close() - return 2 - except (LLMError, TLRuntimeError) as exc: - # The run is marked 'failed' and its completed steps are checkpointed; - # tell the user how to resume from exactly where it died. - print(f"error: {exc}", file=sys.stderr) - print( - f" run failed; resume with: threadlang {args.source} --store {args.store} --resume {run_id}", - file=sys.stderr, - ) + if args.resume: + prior = store.get_run(args.resume) + if prior is not None: + inputs = {**prior.inputs, **inputs} + run_id = args.resume or store.create_run(program.thread_name, inputs) + print(f"run_id: {run_id}", file=sys.stderr) + try: + durable = run_durable( + program, + inputs, + store, + llm_client=client, + run_id=run_id, + source=source_text, + ) + except (LLMError, TLRuntimeError) as exc: + print(f"error: {exc}", file=sys.stderr) + if _should_offer_resume(exc): + resume_args = [ + "threadlang", + "--store", + str(args.store), + "--resume", + run_id, + "--backend", + backend, + "--max-tokens", + str(args.max_tokens), + "--timeout", + str(args.timeout), + ] + if args.from_ir: + resume_args.append("--from-ir") + if backend == "openai" and args.base_url: + resume_args.extend(("--base-url", args.base_url)) + resume_args.extend(("--", str(args.source))) + print( + f" run failed; resume with: {shlex.join(resume_args)}", + file=sys.stderr, + ) + return 1 + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + result = durable.result + run_metrics = store.run_metrics(run_id) if args.metrics else None + finally: store.close() - return 1 - result = durable.result - # Compute metrics before closing — the --store path can include - # wall-clock latency from the persisted event timestamps. - run_metrics = store.run_metrics(run_id) if args.metrics else None - store.close() else: try: result = run_program(program, inputs=inputs, llm_client=client) @@ -252,28 +311,30 @@ def _probe(args: argparse.Namespace, program, inputs: Dict[str, str], client: LL import json store = RunStore(args.store) - runs = [] - for i in range(args.probe): - run_id = store.create_run(program.thread_name, inputs) - print(f"probe run {i + 1}/{args.probe}: {run_id}", file=sys.stderr) - try: - run_durable(program, inputs, store, llm_client=client, run_id=run_id) - except (LLMError, TLRuntimeError) as exc: - print(f" failed: {exc}", file=sys.stderr) - record = store.get_run(run_id) - metrics = store.run_metrics(run_id) - if record is None or metrics is None: - raise RuntimeError(f"probe run disappeared from store: {run_id}") - runs.append( - ProbeRunData( - status=record.status, - output=record.output, - step_outputs=store.load_step_outputs(run_id), - metrics=metrics, + try: + runs = [] + for i in range(args.probe): + run_id = store.create_run(program.thread_name, inputs) + print(f"probe run {i + 1}/{args.probe}: {run_id}", file=sys.stderr) + try: + run_durable(program, inputs, store, llm_client=client, run_id=run_id) + except (LLMError, TLRuntimeError) as exc: + print(f" failed: {exc}", file=sys.stderr) + record = store.get_run(run_id) + metrics = store.run_metrics(run_id) + if record is None or metrics is None: + raise RuntimeError(f"probe run disappeared from store: {run_id}") + runs.append( + ProbeRunData( + status=record.status, + output=record.output, + step_outputs=store.load_step_outputs(run_id), + metrics=metrics, + ) ) - ) - report = probe_report(program, runs) - store.close() + report = probe_report(program, runs) + finally: + store.close() print(json.dumps(report.to_dict(), indent=2)) return 0 diff --git a/src/threadlang/llm.py b/src/threadlang/llm.py index 8d14936..2933776 100644 --- a/src/threadlang/llm.py +++ b/src/threadlang/llm.py @@ -15,6 +15,8 @@ from __future__ import annotations +import http.client +import ipaddress import json import os import urllib.error @@ -23,6 +25,7 @@ from dataclasses import dataclass, field from typing import Dict, List, Mapping, Protocol, Sequence +from .policy import MAX_PROVIDER_RESPONSE_BYTES from .tools import ToolSpec @@ -85,6 +88,36 @@ class LLMError(RuntimeError): """Raised when an LLM call fails.""" +def _validated_provider_text(text: str) -> str: + try: + text.encode("utf-8") + except UnicodeEncodeError as exc: + raise LLMError("provider returned invalid Unicode text") from exc + return text + + +def _is_loopback_hostname(hostname: str) -> bool: + if hostname.casefold() == "localhost": + return True + try: + return ipaddress.ip_address(hostname).is_loopback + except ValueError: + return False + + +class _NoRedirectHandler(urllib.request.HTTPRedirectHandler): + def redirect_request( + self, + req: urllib.request.Request, + fp: object, + code: int, + msg: str, + headers: Mapping[str, str], + newurl: str, + ) -> None: + return None + + def _placeholder_args(schema: Mapping[str, object]) -> Dict[str, object]: """Deterministically fill a tool's required arguments from its JSON schema. Used by the dry-run client so the agent loop is reproducible without a @@ -176,7 +209,7 @@ def complete(self, model: str, prompt: str) -> str: if getattr(block, "type", None) == "text": text = block.text # type: ignore[attr-defined] if text: - return text + return _validated_provider_text(text) raise LLMError("provider returned no text content") def agent_step( @@ -200,7 +233,7 @@ def agent_step( for block in resp.content: btype = getattr(block, "type", None) if btype == "text": - text_parts.append(block.text) # type: ignore[attr-defined] + text_parts.append(_validated_provider_text(block.text)) # type: ignore[attr-defined] elif btype == "tool_use": calls.append( ToolCall( @@ -280,44 +313,69 @@ def __init__( self._base_url = ( base_url or os.environ.get("THREADLANG_BASE_URL") or DEEPSEEK_BASE_URL ).rstrip("/") - parsed_base = urllib.parse.urlsplit(self._base_url) - if parsed_base.scheme not in ("http", "https") or not parsed_base.hostname: + try: + parsed_base = urllib.parse.urlsplit(self._base_url) + hostname = parsed_base.hostname + parsed_base.port + except ValueError as exc: + raise LLMError( + "OpenAI-compatible base URL must use http or https and include a host" + ) from exc + if parsed_base.scheme not in ("http", "https") or not hostname: raise LLMError("OpenAI-compatible base URL must use http or https and include a host") + if any(ord(char) <= 32 or ord(char) == 127 for char in hostname): + raise LLMError("OpenAI-compatible base URL must include a valid host") + if parsed_base.username is not None or parsed_base.password is not None: + raise LLMError("OpenAI-compatible base URL must not include credentials") + if parsed_base.query or parsed_base.fragment: + raise LLMError("OpenAI-compatible base URL must not include a query or fragment") # A key is optional: local servers (Ollama) ignore it. Hosted providers # 401 without one, which surfaces as a clear LLMError at call time. - self._api_key = ( - api_key or os.environ.get("THREADLANG_API_KEY") or os.environ.get("OPENAI_API_KEY") - ) + resolved_key = api_key or os.environ.get("THREADLANG_API_KEY") + if not resolved_key and parsed_base.scheme == "https" and hostname == "api.openai.com": + resolved_key = os.environ.get("OPENAI_API_KEY") + if resolved_key and parsed_base.scheme != "https" and not _is_loopback_hostname(hostname): + raise LLMError("refusing to send an API key to a non-HTTPS, non-loopback endpoint") + self._api_key = resolved_key + bypass_proxy = parsed_base.scheme == "http" and _is_loopback_hostname(hostname) + handlers: List[urllib.request.BaseHandler] = [_NoRedirectHandler()] + if bypass_proxy: + handlers.insert(0, urllib.request.ProxyHandler({})) + self._opener = urllib.request.build_opener(*handlers) self._max_tokens = max_tokens self._timeout = timeout def _post(self, payload: Dict[str, object]) -> Dict[str, object]: data = json.dumps(payload).encode("utf-8") headers = {"Content-Type": "application/json"} - if self._api_key: - headers["Authorization"] = f"Bearer {self._api_key}" - request = urllib.request.Request( - f"{self._base_url}/chat/completions", data=data, headers=headers, method="POST" - ) try: - # URL scheme and host are validated in __init__; urllib is used to - # keep the OpenAI-compatible adapter dependency-free. - with urllib.request.urlopen( # nosec B310 - request, timeout=self._timeout - ) as resp: - body = resp.read().decode("utf-8") + request = urllib.request.Request( + f"{self._base_url}/chat/completions", data=data, headers=headers, method="POST" + ) + if self._api_key: + request.add_unredirected_header("Authorization", f"Bearer {self._api_key}") + response = self._opener.open(request, timeout=self._timeout) # nosec B310 + with response as resp: + body_bytes = resp.read(MAX_PROVIDER_RESPONSE_BYTES + 1) + if len(body_bytes) > MAX_PROVIDER_RESPONSE_BYTES: + raise LLMError( + f"provider response exceeds {MAX_PROVIDER_RESPONSE_BYTES} byte limit" + ) + body = body_bytes.decode("utf-8") except urllib.error.HTTPError as exc: # Provider bodies may contain request fragments, account metadata, # or echoed secrets. Persist only the status and endpoint; operators # can correlate provider-side logs without leaking the body into the # durable run record or dashboard. - raise LLMError(f"HTTP {exc.code} from {self._base_url}") from exc + raise LLMError(f"HTTP {exc.code} from provider endpoint") from exc except urllib.error.URLError as exc: - raise LLMError(f"could not reach {self._base_url}: {exc.reason}") from exc + raise LLMError(f"could not reach provider endpoint: {exc.reason}") from exc + except (http.client.HTTPException, UnicodeError, ValueError) as exc: + raise LLMError("invalid OpenAI-compatible provider endpoint") from exc try: return json.loads(body) except json.JSONDecodeError as exc: - raise LLMError(f"non-JSON response from {self._base_url}") from exc + raise LLMError("non-JSON response from provider endpoint") from exc def complete(self, model: str, prompt: str) -> str: resp = self._post( @@ -333,7 +391,7 @@ def complete(self, model: str, prompt: str) -> str: content = _openai_message(resp).get("content") if not isinstance(content, str) or not content: raise LLMError("provider returned no text content") - return content + return _validated_provider_text(content) def agent_step( self, model: str, messages: Sequence[Message], tools: Sequence[ToolSpec] @@ -360,23 +418,51 @@ def agent_step( if choice.get("finish_reason") == "length": raise LLMError("provider response was truncated at max_tokens") message = _openai_message(response) - text = message.get("content") or "" + raw_text = message.get("content") + if raw_text is not None and not isinstance(raw_text, str): + raise LLMError("provider returned invalid text content") + text = _validated_provider_text(raw_text) if raw_text else "" calls: List[ToolCall] = [] - for raw in message.get("tool_calls") or (): - fn = raw.get("function", {}) if isinstance(raw, Mapping) else {} - raw_args = fn.get("arguments", "{}") + raw_calls = message.get("tool_calls") + if raw_calls is None: + raw_calls = [] + elif not isinstance(raw_calls, list): + raise LLMError("provider returned malformed tool calls") + for raw in raw_calls: + if not isinstance(raw, Mapping): + raise LLMError("provider returned malformed tool call") + fn = raw.get("function") + if not isinstance(fn, Mapping): + raise LLMError("provider returned malformed tool call") + call_name = fn.get("name") + if not isinstance(call_name, str) or not call_name: + raise LLMError("provider returned tool call without a valid name") + call_id = raw.get("id") + if call_id is None: + call_id = f"call_{len(calls)}" + if not isinstance(call_id, str) or not call_id: + raise LLMError("provider returned tool call without a valid id") + raw_args = fn.get("arguments") try: - arguments = json.loads(raw_args) if isinstance(raw_args, str) else dict(raw_args) - except (json.JSONDecodeError, TypeError): - arguments = {} + if isinstance(raw_args, str): + arguments = json.loads(raw_args) + elif isinstance(raw_args, Mapping): + arguments = dict(raw_args) + else: + raise TypeError + if not isinstance(arguments, dict): + raise TypeError + json.dumps(arguments, ensure_ascii=False, allow_nan=False).encode("utf-8") + except (json.JSONDecodeError, TypeError, ValueError, UnicodeEncodeError) as exc: + raise LLMError("provider returned malformed tool-call arguments") from exc calls.append( ToolCall( - id=str(raw.get("id") or f"call_{len(calls)}"), - name=str(fn.get("name", "")), - arguments=arguments if isinstance(arguments, dict) else {}, + id=_validated_provider_text(call_id), + name=_validated_provider_text(call_name), + arguments=arguments, ) ) - turn = AgentTurn(text=str(text), tool_calls=tuple(calls)) + turn = AgentTurn(text=text, tool_calls=tuple(calls)) if not turn.text and not turn.tool_calls: raise LLMError("provider returned neither text nor tool calls") return turn @@ -434,7 +520,5 @@ def _to_openai_messages(messages: Sequence[Message]) -> List[Dict[str, object]]: def default_client() -> LLMClient: - """Used by CLI when --dry-run is not passed. Raises if the SDK or key - are missing — call sites should catch and fall back to DryRunClient - if they want a soft mode.""" + """Return the default Anthropic client, failing closed when unavailable.""" return AnthropicClient() diff --git a/src/threadlang/policy.py b/src/threadlang/policy.py index 659ce37..f0cae5d 100644 --- a/src/threadlang/policy.py +++ b/src/threadlang/policy.py @@ -13,6 +13,7 @@ MAX_REGEX_INPUT_CHARS = 64 * 1024 REGEX_TIMEOUT_SECONDS = 1.0 MAX_REQUEST_BYTES = 1024 * 1024 +MAX_PROVIDER_RESPONSE_BYTES = 8 * 1024 * 1024 MAX_INPUTS = 128 MAX_INPUT_KEY_CHARS = 128 MAX_INPUT_VALUE_CHARS = 64 * 1024 diff --git a/src/threadlang/server.py b/src/threadlang/server.py index 87f27a0..a50f0e6 100644 --- a/src/threadlang/server.py +++ b/src/threadlang/server.py @@ -10,11 +10,13 @@ import hmac import ipaddress import json +import math import os +import sqlite3 import sys from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from typing import Optional -from urllib.parse import parse_qs, urlsplit +from urllib.parse import SplitResult, parse_qs, urlsplit from .control import WorkerPool from .dashboard import render_run_detail, render_run_list @@ -110,29 +112,46 @@ def _valid_browser_origin(self) -> bool: if self.server.auth_token is not None: # type: ignore[attr-defined] return True host = self.headers.get("Host", "") - hostname = urlsplit(f"http://{host}").hostname + try: + hostname = urlsplit(f"http://{host}").hostname + except ValueError: + hostname = None if hostname is None or not _is_loopback_host(hostname): self._send(421, {"error": "invalid Host for loopback-only server"}) return False if self.command == "POST": origin = self.headers.get("Origin") if origin: - origin_host = urlsplit(origin).hostname + try: + origin_host = urlsplit(origin).hostname + except ValueError: + origin_host = None if origin_host is None or not _is_loopback_host(origin_host): self._send(403, {"error": "cross-origin POST denied"}) return False return True + def _request_target(self) -> Optional[SplitResult]: + try: + return urlsplit(self.path) + except ValueError: + self._send(400, {"error": "invalid request target"}) + return None + def log_message(self, format: str, *args: object) -> None: # Structured logs intentionally exclude headers, bodies, inputs, traces, # and auth material. + try: + path = urlsplit(self.path).path + except ValueError: + path = "" print( json.dumps( { "component": "threadlang-http", "client": self.client_address[0], "method": self.command, - "path": urlsplit(self.path).path, + "path": path, "message": format % args, }, separators=(",", ":"), @@ -144,7 +163,9 @@ def log_message(self, format: str, *args: object) -> None: def do_GET(self) -> None: if not self._valid_browser_origin(): return - parsed = urlsplit(self.path) + parsed = self._request_target() + if parsed is None: + return path = parsed.path if path == "/healthz": try: @@ -159,11 +180,22 @@ def do_GET(self) -> None: if path == "/readyz": pool = self.server.worker_pool # type: ignore[attr-defined] healthy = pool is not None and pool.is_healthy() - store = self._store() try: - queue = store.counts_by_status() - finally: - store.close() + store = self._store() + try: + queue = store.counts_by_status() + finally: + store.close() + except Exception: + self._send( + 503, + { + "ok": False, + "database": "unavailable", + "workers": pool.status() if pool is not None else None, + }, + ) + return self._send( 200 if healthy else 503, { @@ -256,7 +288,10 @@ def do_GET(self) -> None: def do_POST(self) -> None: if not self._valid_browser_origin(): return - path = urlsplit(self.path).path + parsed = self._request_target() + if parsed is None: + return + path = parsed.path if path != "/runs": self._send(404, {"error": "not found"}) return @@ -284,7 +319,7 @@ def do_POST(self) -> None: return try: body = json.loads(self.rfile.read(length) or b"{}") - except (json.JSONDecodeError, UnicodeDecodeError): + except (ValueError, UnicodeDecodeError, RecursionError): self._send(400, {"error": "body must be valid JSON"}) return if not isinstance(body, dict): @@ -317,6 +352,9 @@ def do_POST(self) -> None: ).encode("utf-8") workflow = load_ir_bytes(ir_bytes) program = program_from_ir(workflow) + except UnicodeEncodeError: + self._send(400, {"error": "workflow text must be valid Unicode"}) + return except (ParseError, IRCompileError) as exc: self._send(400, {"error": f"invalid workflow: {exc}"}) return @@ -366,6 +404,11 @@ def _validate_inputs(value: object) -> Optional[str]: for key, item in value.items(): if not isinstance(key, str) or not isinstance(item, str): return "input keys and values must be strings" + try: + key.encode("utf-8") + item.encode("utf-8") + except UnicodeEncodeError: + return "input keys and values must be valid Unicode" if not key or len(key) > MAX_INPUT_KEY_CHARS: return f"input keys must be 1..{MAX_INPUT_KEY_CHARS} characters" if len(item) > MAX_INPUT_VALUE_CHARS: @@ -401,12 +444,7 @@ def make_server( max_retained: int = DEFAULT_MAX_RETAINED_RUNS, ) -> ThreadingHTTPServer: """Build but do not start the control-plane server.""" - if not _is_loopback_host(host) and not auth_token: - raise ValueError("non-loopback bind requires a bearer auth token") - if auth_token is not None and len(auth_token) < 16: - raise ValueError("auth token must be at least 16 characters") - if max_pending < 1 or max_retained < 0: - raise ValueError("max_pending must be >= 1 and max_retained must be >= 0") + _validate_server_options(host, port, auth_token, max_pending, max_retained) httpd = ThreadingHTTPServer((host, port), _Handler) httpd.store_path = store_path # type: ignore[attr-defined] httpd.auth_token = auth_token # type: ignore[attr-defined] @@ -416,6 +454,23 @@ def make_server( return httpd +def _validate_server_options( + host: str, + port: int, + auth_token: Optional[str], + max_pending: int, + max_retained: int, +) -> None: + if not _is_loopback_host(host) and not auth_token: + raise ValueError("non-loopback bind requires a bearer auth token") + if auth_token is not None and len(auth_token) < 16: + raise ValueError("auth token must be at least 16 characters") + if not 0 <= port <= 65535: + raise ValueError("port must be between 0 and 65535") + if max_pending < 1 or max_retained < 0: + raise ValueError("max_pending must be >= 1 and max_retained must be >= 0") + + def serve( store_path: str, *, @@ -428,6 +483,9 @@ def serve( max_pending: int = DEFAULT_MAX_PENDING_RUNS, max_retained: int = DEFAULT_MAX_RETAINED_RUNS, ) -> None: + _validate_server_options(host, port, auth_token, max_pending, max_retained) + if n_workers < 1: + raise ValueError("n_workers must be >= 1") pool = WorkerPool(store_path, n_workers=n_workers, llm_client=llm_client, tools=tools) pool.start() try: @@ -481,26 +539,40 @@ def main() -> int: parser.add_argument("--max-retained", type=int, default=DEFAULT_MAX_RETAINED_RUNS) args = parser.parse_args() - client: LLMClient - if args.backend == "dry-run": - client = DryRunClient() - elif args.backend == "openai": - client = OpenAICompatClient( - base_url=args.base_url, - max_tokens=args.max_tokens, - timeout=args.timeout, - ) - else: - try: - client = AnthropicClient(max_tokens=args.max_tokens, timeout=args.timeout) - except LLMError as exc: - print(f"error: {exc}", file=sys.stderr, flush=True) - return 1 - - if args.max_tokens < 1 or args.timeout <= 0: + if args.max_tokens < 1 or not math.isfinite(args.timeout) or args.timeout <= 0: print("error: --max-tokens and --timeout must be positive", file=sys.stderr) return 2 auth_token = os.environ.get(args.auth_token_env) + try: + _validate_server_options( + args.host, + args.port, + auth_token, + args.max_pending, + args.max_retained, + ) + if args.workers < 1: + raise ValueError("workers must be >= 1") + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr, flush=True) + return 2 + + try: + client: LLMClient + if args.backend == "dry-run": + client = DryRunClient() + elif args.backend == "openai": + client = OpenAICompatClient( + base_url=args.base_url, + max_tokens=args.max_tokens, + timeout=args.timeout, + ) + else: + client = AnthropicClient(max_tokens=args.max_tokens, timeout=args.timeout) + except LLMError as exc: + print(f"error: {exc}", file=sys.stderr, flush=True) + return 1 + try: serve( args.store, @@ -512,7 +584,7 @@ def main() -> int: max_pending=args.max_pending, max_retained=args.max_retained, ) - except ValueError as exc: + except (OSError, RuntimeError, sqlite3.Error, ValueError) as exc: print(f"error: {exc}", file=sys.stderr, flush=True) return 2 return 0 diff --git a/src/threadlang/store.py b/src/threadlang/store.py index 6936449..b9337bb 100644 --- a/src/threadlang/store.py +++ b/src/threadlang/store.py @@ -703,9 +703,9 @@ def _checkpoint(step_name: str, output: str) -> None: resume_outputs=resume_outputs, on_step_complete=_checkpoint, ) + store.mark_completed(run_id, result.output) except Exception as exc: store.mark_failed(run_id, f"{type(exc).__name__}: {exc}") raise - store.mark_completed(run_id, result.output) return DurableRun(run_id=run_id, result=result) diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..875ade6 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,342 @@ +from __future__ import annotations + +import shlex +import sqlite3 +import sys +from pathlib import Path +from typing import NoReturn + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "src")) + +from threadlang import cli # noqa: E402 +from threadlang.ir import canonical_ir_bytes, compile_program # noqa: E402 +from threadlang.llm import LLMError # noqa: E402 +from threadlang.parser import parse_program # noqa: E402 +from threadlang.store import RunStore # noqa: E402 + + +def _invoke_cli(monkeypatch: pytest.MonkeyPatch, *args: object) -> int: + monkeypatch.setattr(sys, "argv", ["threadlang", *(str(arg) for arg in args)]) + return cli.main() + + +def _unavailable(*args: object, **kwargs: object) -> NoReturn: + raise LLMError("provider unavailable") + + +@pytest.mark.parametrize( + "source_text", + [ + 'thread T { context {} emit llm "m" { "secret" } }', + 'thread T { context {} steps { step choose { route "m" { "pick" on "yes" -> end else -> end } } } emit text { steps.choose.output } }', + 'thread T { context {} steps { step act { agent "m" { tools [ echo ] max_iters 2 "act" } } } emit text { steps.act.output } }', + ], +) +def test_anthropic_unavailable_fails_closed( + source_text: str, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + source = tmp_path / "model.thread" + source.write_text(source_text, encoding="utf-8") + monkeypatch.setattr(cli, "AnthropicClient", _unavailable) + + assert _invoke_cli(monkeypatch, source) == 1 + captured = capsys.readouterr() + assert captured.out == "" + assert "provider unavailable" in captured.err + assert "dry-run" not in captured.out + assert "falling back" not in captured.err + + +def test_unused_provider_configuration_is_lazy( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + source = tmp_path / "text.thread" + source.write_text('thread T { context {} emit text { "ok" } }', encoding="utf-8") + calls = 0 + + def unexpected_provider(*args: object, **kwargs: object) -> NoReturn: + nonlocal calls + calls += 1 + raise AssertionError("provider should not be constructed") + + monkeypatch.setattr(cli, "OpenAICompatClient", unexpected_provider) + + assert ( + _invoke_cli( + monkeypatch, + source, + "--backend", + "openai", + "--base-url", + "not-a-url", + ) + == 0 + ) + captured = capsys.readouterr() + assert captured.out == "ok\n" + assert captured.err == "" + assert calls == 0 + + +def test_completed_replay_does_not_require_anthropic( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + source = tmp_path / "model.thread" + source.write_text('thread T { context {} emit llm "m" { "prompt" } }', encoding="utf-8") + store_path = tmp_path / "runs.db" + + assert _invoke_cli(monkeypatch, source, "--dry-run", "--store", store_path) == 0 + first = capsys.readouterr() + store = RunStore(str(store_path)) + try: + run_id = store.list_runs()[0].id + finally: + store.close() + + def unexpected_provider(*args: object, **kwargs: object) -> NoReturn: + raise AssertionError("provider should not be constructed during replay") + + monkeypatch.setattr(cli, "AnthropicClient", unexpected_provider) + assert _invoke_cli(monkeypatch, source, "--store", store_path, "--resume", run_id) == 0 + replay = capsys.readouterr() + assert replay.out == first.out + assert "provider unavailable" not in replay.err + + +@pytest.mark.parametrize("timeout", ["nan", "inf", "-inf"]) +def test_cli_rejects_non_finite_timeout( + timeout: str, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + source = tmp_path / "text.thread" + source.write_text('thread T { context {} emit text { "ok" } }', encoding="utf-8") + + assert _invoke_cli(monkeypatch, source, "--dry-run", f"--timeout={timeout}") == 2 + captured = capsys.readouterr() + assert captured.out == "" + assert "must be positive" in captured.err + + +@pytest.mark.parametrize("bad_input", ["broken", "=value"]) +def test_malformed_input_precedes_provider_setup( + bad_input: str, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + source = tmp_path / "model.thread" + source.write_text('thread T { context {} emit llm "m" { "prompt" } }', encoding="utf-8") + store_path = tmp_path / "runs.db" + calls = 0 + + def counted_provider(*args: object, **kwargs: object) -> NoReturn: + nonlocal calls + calls += 1 + raise LLMError("provider should not be constructed") + + monkeypatch.setattr(cli, "AnthropicClient", counted_provider) + assert ( + _invoke_cli( + monkeypatch, + source, + "--input", + bad_input, + "--store", + store_path, + ) + == 2 + ) + captured = capsys.readouterr() + assert "Invalid --input format" in captured.err + assert calls == 0 + assert not store_path.exists() + + +def test_deterministic_durable_failure_has_no_resume_hint( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + source = tmp_path / "input.thread" + source.write_text("thread T { context {} emit text { inputs.required } }", encoding="utf-8") + store_path = tmp_path / "runs.db" + monkeypatch.setattr(cli, "AnthropicClient", _unavailable) + + assert _invoke_cli(monkeypatch, source, "--store", store_path) == 1 + captured = capsys.readouterr() + assert "Missing input value: required" in captured.err + assert "resume with:" not in captured.err + store = RunStore(str(store_path)) + try: + assert store.list_runs()[0].status == "failed" + finally: + store.close() + + +def test_cli_formats_store_open_errors( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + source = tmp_path / "text.thread" + source.write_text('thread T { context {} emit text { "ok" } }', encoding="utf-8") + store_path = tmp_path / "missing" / "runs.db" + monkeypatch.setattr(cli, "AnthropicClient", _unavailable) + + assert _invoke_cli(monkeypatch, source, "--store", store_path) == 2 + captured = capsys.readouterr() + assert captured.out == "" + assert captured.err.startswith("error:") + assert "Traceback" not in captured.err + + +def test_cli_closes_store_when_durable_run_raises( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + source = tmp_path / "text.thread" + source.write_text('thread T { context {} emit text { "ok" } }', encoding="utf-8") + stores: list[RunStore] = [] + + class TrackingStore(RunStore): + closed = False + + def close(self) -> None: + self.closed = True + super().close() + + def open_store(path: str) -> TrackingStore: + store = TrackingStore(path) + stores.append(store) + return store + + def fail_run(*args: object, **kwargs: object) -> NoReturn: + raise sqlite3.OperationalError("write failed") + + monkeypatch.setattr(cli, "RunStore", open_store) + monkeypatch.setattr(cli, "run_durable", fail_run) + + assert _invoke_cli(monkeypatch, source, "--store", tmp_path / "runs.db") == 2 + assert len(stores) == 1 + assert isinstance(stores[0], TrackingStore) and stores[0].closed + assert "error: write failed" in capsys.readouterr().err + + +def test_probe_closes_store_when_report_fails( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + source = tmp_path / "text.thread" + source.write_text('thread T { context {} emit text { "ok" } }', encoding="utf-8") + stores: list[RunStore] = [] + + class TrackingStore(RunStore): + closed = False + + def close(self) -> None: + self.closed = True + super().close() + + def open_store(path: str) -> TrackingStore: + store = TrackingStore(path) + stores.append(store) + return store + + def fail_report(*args: object, **kwargs: object) -> NoReturn: + raise OSError("report failed") + + monkeypatch.setattr(cli, "RunStore", open_store) + monkeypatch.setattr(cli, "probe_report", fail_report) + + assert ( + _invoke_cli( + monkeypatch, + source, + "--dry-run", + "--store", + tmp_path / "runs.db", + "--probe", + "1", + ) + == 2 + ) + assert len(stores) == 1 + assert isinstance(stores[0], TrackingStore) and stores[0].closed + assert "error: report failed" in capsys.readouterr().err + + +def test_provider_failure_resume_hint_round_trips_and_runs( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + monkeypatch.chdir(tmp_path) + ir_path = Path("-workflow with ' quote.ir.json") + workflow = compile_program( + parse_program('thread T { context {} emit llm "m" { "prompt:" + inputs.value } }') + ) + ir_path.write_bytes(canonical_ir_bytes(workflow)) + store_dir = Path("state with ' quote") + store_dir.mkdir() + store_path = store_dir / "runs.db" + base_url = "http://127.0.0.1:9/v1?x=1&y=2" + + class FlakyClient: + def __init__(self) -> None: + self.calls = 0 + + def complete(self, model: str, prompt: str) -> str: + self.calls += 1 + if self.calls == 1: + raise LLMError("temporary provider failure") + return f"recovered:{prompt}" + + client = FlakyClient() + monkeypatch.setattr(cli, "OpenAICompatClient", lambda **kwargs: client) + assert ( + _invoke_cli( + monkeypatch, + "--backend", + "openai", + "--base-url", + base_url, + "--max-tokens", + "7", + "--timeout", + "2.5", + "--store", + store_path, + "--input", + "value=kept", + "--from-ir", + "--", + ir_path, + ) + == 1 + ) + failed = capsys.readouterr() + hint = next( + line.split("resume with:", 1)[1].strip() + for line in failed.err.splitlines() + if "resume with:" in line + ) + argv = shlex.split(hint) + assert argv[0] == "threadlang" + assert argv[argv.index("--store") + 1] == str(store_path) + assert argv[argv.index("--backend") + 1] == "openai" + assert argv[argv.index("--max-tokens") + 1] == "7" + assert argv[argv.index("--timeout") + 1] == "2.5" + assert argv[argv.index("--base-url") + 1] == base_url + assert "--from-ir" in argv + assert "--input" not in argv + assert argv[-2:] == ["--", str(ir_path)] + + assert _invoke_cli(monkeypatch, *argv[1:]) == 0 + resumed = capsys.readouterr() + assert resumed.out == "recovered:prompt:kept\n" + assert client.calls == 2 + store = RunStore(str(store_path)) + try: + assert store.list_runs()[0].status == "completed" + finally: + store.close() diff --git a/tests/test_provider_security.py b/tests/test_provider_security.py index c707f45..00a061b 100644 --- a/tests/test_provider_security.py +++ b/tests/test_provider_security.py @@ -3,10 +3,12 @@ from __future__ import annotations import io +import json from email.message import Message from pathlib import Path import sys import urllib.error +import urllib.request import pytest @@ -14,6 +16,7 @@ sys.path.insert(0, str(REPO_ROOT / "src")) from threadlang.llm import LLMError, OpenAICompatClient # noqa: E402 +import threadlang.llm as llm_module # noqa: E402 class _StaticOpenAI(OpenAICompatClient): @@ -27,6 +30,7 @@ def _post(self, payload: dict[str, object]) -> dict[str, object]: def test_openai_http_error_body_is_redacted(monkeypatch: pytest.MonkeyPatch) -> None: secret = "fake-secret-that-must-not-be-persisted" + endpoint_secret = "endpoint-secret-that-must-not-be-persisted" def fail(*args, **kwargs): raise urllib.error.HTTPError( @@ -37,12 +41,19 @@ def fail(*args, **kwargs): io.BytesIO(secret.encode()), ) - monkeypatch.setattr("urllib.request.urlopen", fail) - client = OpenAICompatClient(base_url="https://provider.invalid/v1", api_key="placeholder") + class FailingOpener: + def open(self, *args: object, **kwargs: object) -> io.BytesIO: + return fail(*args, **kwargs) + + monkeypatch.setattr(urllib.request, "build_opener", lambda *handlers: FailingOpener()) + client = OpenAICompatClient( + base_url=f"https://provider.invalid/{endpoint_secret}", api_key="placeholder" + ) with pytest.raises(LLMError) as raised: client.complete("m", "hello") assert "HTTP 500" in str(raised.value) assert secret not in str(raised.value) + assert endpoint_secret not in str(raised.value) def test_openai_truncation_and_empty_text_fail_closed() -> None: @@ -55,3 +66,219 @@ def test_openai_truncation_and_empty_text_fail_closed() -> None: empty = _StaticOpenAI({"choices": [{"finish_reason": "stop", "message": {"content": ""}}]}) with pytest.raises(LLMError, match="no text"): empty.complete("m", "hello") + + +@pytest.mark.parametrize( + "arguments", + ["{", "[]", None, {"value": float("nan")}, {"value": "\ud800"}], +) +def test_openai_malformed_tool_arguments_fail_closed(arguments: object) -> None: + client = _StaticOpenAI( + { + "choices": [ + { + "finish_reason": "tool_calls", + "message": { + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "record", "arguments": arguments}, + } + ], + }, + } + ] + } + ) + + with pytest.raises(LLMError, match="malformed tool-call arguments"): + client.agent_step("m", [], []) + + +def test_openai_provider_response_size_is_bounded(monkeypatch: pytest.MonkeyPatch) -> None: + read_sizes: list[int] = [] + + class OversizedResponse(io.BytesIO): + def read(self, size: int = -1) -> bytes: + read_sizes.append(size) + return super().read(size) + + class OversizedOpener: + def open(self, *args: object, **kwargs: object) -> io.BytesIO: + return OversizedResponse(b"x" * 17) + + monkeypatch.setattr(llm_module, "MAX_PROVIDER_RESPONSE_BYTES", 16) + monkeypatch.setattr(urllib.request, "build_opener", lambda *handlers: OversizedOpener()) + + with pytest.raises(LLMError, match="provider response exceeds 16 byte limit"): + OpenAICompatClient(base_url="https://provider.invalid/v1").complete("m", "hello") + assert read_sizes == [17] + + +@pytest.mark.parametrize("content", ["\ud800", {"unexpected": "object"}]) +def test_openai_invalid_agent_text_fails_closed(content: object) -> None: + client = _StaticOpenAI( + {"choices": [{"finish_reason": "stop", "message": {"content": content}}]} + ) + + with pytest.raises(LLMError, match="invalid Unicode|invalid text"): + client.agent_step("m", [], []) + + +@pytest.mark.parametrize("message", [{"content": "done"}, {"content": "done", "tool_calls": None}]) +def test_openai_content_only_agent_turn_accepts_missing_tool_calls( + message: dict[str, object], +) -> None: + client = _StaticOpenAI({"choices": [{"finish_reason": "stop", "message": message}]}) + + assert client.agent_step("m", [], []).text == "done" + + +def test_openai_valid_tool_call_is_preserved() -> None: + client = _StaticOpenAI( + { + "choices": [ + { + "finish_reason": "tool_calls", + "message": { + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "record", + "arguments": '{"value":"ok"}', + }, + } + ], + }, + } + ] + } + ) + + turn = client.agent_step("m", [], []) + assert len(turn.tool_calls) == 1 + assert turn.tool_calls[0].id == "call_1" + assert turn.tool_calls[0].name == "record" + assert turn.tool_calls[0].arguments == {"value": "ok"} + + +@pytest.mark.parametrize( + ("base_url", "threadlang_key", "expected_authorization"), + [ + ("https://api.openai.com/v1", None, "Bearer ambient-openai-key"), + ("http://api.openai.com/v1", None, None), + ("https://api.openai.com.evil.invalid/v1", None, None), + ("https://provider.invalid/v1", None, None), + ("https://provider.invalid/v1", "generic-provider-key", "Bearer generic-provider-key"), + ("http://127.0.0.1:11434/v1", "local-provider-key", "Bearer local-provider-key"), + ("http://localhost:11434/v1", "local-provider-key", "Bearer local-provider-key"), + ("http://[::1]:11434/v1", "local-provider-key", "Bearer local-provider-key"), + ], +) +def test_openai_api_key_is_scoped_to_official_https_host( + base_url: str, + threadlang_key: str | None, + expected_authorization: str | None, + monkeypatch: pytest.MonkeyPatch, +) -> None: + requests: list[urllib.request.Request] = [] + configured_handlers: list[object] = [] + response = json.dumps( + {"choices": [{"finish_reason": "stop", "message": {"content": "ok"}}]} + ).encode() + + def capture(request: urllib.request.Request, **kwargs: object) -> io.BytesIO: + requests.append(request) + return io.BytesIO(response) + + class CapturingOpener: + def open(self, request: urllib.request.Request, **kwargs: object) -> io.BytesIO: + return capture(request, **kwargs) + + def capture_opener(*handlers: object) -> CapturingOpener: + configured_handlers.extend(handlers) + return CapturingOpener() + + monkeypatch.setenv("OPENAI_API_KEY", "ambient-openai-key") + if threadlang_key is None: + monkeypatch.delenv("THREADLANG_API_KEY", raising=False) + else: + monkeypatch.setenv("THREADLANG_API_KEY", threadlang_key) + monkeypatch.setattr(urllib.request, "build_opener", capture_opener) + + assert OpenAICompatClient(base_url=base_url).complete("m", "hello") == "ok" + original = requests[0] + assert original.get_header("Authorization") == expected_authorization + redirect_handlers = [ + handler + for handler in configured_handlers + if isinstance(handler, urllib.request.HTTPRedirectHandler) + ] + assert len(redirect_handlers) == 1 + assert ( + redirect_handlers[0].redirect_request( + original, + None, + 302, + "Found", + Message(), + "https://redirect.invalid/chat/completions", + ) + is None + ) + proxy_handlers = [ + handler + for handler in configured_handlers + if isinstance(handler, urllib.request.ProxyHandler) + ] + if base_url.startswith(("http://127.0.0.1", "http://localhost", "http://[::1]")): + assert len(proxy_handlers) == 1 + assert proxy_handlers[0].proxies == {} + else: + assert proxy_handlers == [] + + +@pytest.mark.parametrize("key_source", ["explicit", "environment"]) +def test_api_key_is_rejected_for_remote_http_endpoint( + key_source: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("THREADLANG_API_KEY", raising=False) + kwargs: dict[str, str] = {} + if key_source == "explicit": + kwargs["api_key"] = "secret" + else: + monkeypatch.setenv("THREADLANG_API_KEY", "secret") + + with pytest.raises(LLMError, match="non-HTTPS, non-loopback"): + OpenAICompatClient(base_url="http://provider.invalid/v1", **kwargs) + + +@pytest.mark.parametrize( + "base_url", + [ + "https://user:secret@provider.invalid/v1", + "https://provider.invalid/v1?api_key=secret", + "https://provider.invalid/v1#secret", + ], +) +def test_openai_rejects_credential_bearing_base_urls(base_url: str) -> None: + with pytest.raises(LLMError, match="credentials|query or fragment"): + OpenAICompatClient(base_url=base_url) + + +@pytest.mark.parametrize( + "base_url", + [ + "https://provider.invalid:notaport/v1", + "https://provider.invalid:70000/v1", + "https://bad host.invalid/v1", + ], +) +def test_openai_rejects_malformed_base_urls(base_url: str) -> None: + with pytest.raises(LLMError, match="base URL must|provider endpoint"): + OpenAICompatClient(base_url=base_url) diff --git a/tests/test_release_metadata.py b/tests/test_release_metadata.py new file mode 100644 index 0000000..7ff3b1f --- /dev/null +++ b/tests/test_release_metadata.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +from pathlib import Path +import sys +import tomllib + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "src")) + +import threadlang # noqa: E402 + + +def test_package_versions_match() -> None: + with (REPO_ROOT / "pyproject.toml").open("rb") as pyproject: + metadata = tomllib.load(pyproject) + assert metadata["project"]["version"] == threadlang.__version__ diff --git a/tests/test_server_hardening.py b/tests/test_server_hardening.py index 7746769..141a64c 100644 --- a/tests/test_server_hardening.py +++ b/tests/test_server_hardening.py @@ -5,6 +5,7 @@ import http.client import json from pathlib import Path +import socket import sys import threading @@ -18,6 +19,7 @@ from threadlang.llm import DryRunClient # noqa: E402 from threadlang.parser import parse_program # noqa: E402 from threadlang.policy import MAX_REQUEST_BYTES # noqa: E402 +import threadlang.server as server_module # noqa: E402 from threadlang.server import make_server # noqa: E402 from threadlang.store import RunStore # noqa: E402 @@ -76,12 +78,19 @@ def test_non_loopback_bind_requires_auth_token(tmp_path: Path) -> None: make_server(str(tmp_path / "x.db"), "0.0.0.0", 0) +@pytest.mark.parametrize("port", [-1, 65536]) +def test_make_server_rejects_out_of_range_ports(tmp_path: Path, port: int) -> None: + with pytest.raises(ValueError, match="port must be between 0 and 65535"): + make_server(str(tmp_path / "x.db"), "127.0.0.1", port) + + def test_unauthenticated_loopback_rejects_dns_rebinding_and_cross_origin(tmp_path: Path) -> None: server = make_server(str(tmp_path / "local.db"), "127.0.0.1", 0) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() try: port = server.server_address[1] + body = json.dumps({"source": SOURCE, "inputs": {}}).encode() conn = http.client.HTTPConnection("127.0.0.1", port, timeout=5) conn.putrequest("GET", "/healthz", skip_host=True) conn.putheader("Host", "evil.example") @@ -91,7 +100,31 @@ def test_unauthenticated_loopback_rejects_dns_rebinding_and_cross_origin(tmp_pat response.read() conn.close() - body = json.dumps({"source": SOURCE, "inputs": {}}).encode() + conn = http.client.HTTPConnection("127.0.0.1", port, timeout=5) + conn.request( + "POST", + "/runs", + body=body, + headers={ + "Host": f"127.0.0.1:{port}", + "Origin": "[", + "Content-Type": "application/json", + }, + ) + response = conn.getresponse() + assert response.status == 403 + response.read() + conn.close() + + conn = http.client.HTTPConnection("127.0.0.1", port, timeout=5) + conn.putrequest("GET", "/healthz", skip_host=True) + conn.putheader("Host", "[") + conn.endheaders() + response = conn.getresponse() + assert response.status == 421 + response.read() + conn.close() + conn = http.client.HTTPConnection("127.0.0.1", port, timeout=5) conn.request( "POST", @@ -113,6 +146,23 @@ def test_unauthenticated_loopback_rejects_dns_rebinding_and_cross_origin(tmp_pat thread.join(timeout=5) +def test_malformed_request_target_returns_400_and_server_survives(tmp_path: Path) -> None: + running = _RunningServer(tmp_path) + try: + with socket.create_connection(("127.0.0.1", running.port), timeout=5) as connection: + connection.sendall( + b"GET http://[ HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n" + ) + response = b"" + while chunk := connection.recv(4096): + response += chunk + assert response.startswith(b"HTTP/1.0 400") + assert b"invalid request target" in response + assert running.request("GET", "/healthz") == (200, {"ok": True, "database": "ok"}) + finally: + running.close() + + def test_health_is_public_but_data_routes_require_bearer(tmp_path: Path) -> None: running = _RunningServer(tmp_path) try: @@ -190,6 +240,65 @@ def test_post_requires_exactly_one_workflow_representation(tmp_path: Path) -> No running.close() +@pytest.mark.parametrize( + ("payload", "message"), + [ + ({"source": "\ud800", "inputs": {}}, "workflow text must be valid Unicode"), + ( + {"source": SOURCE, "inputs": {"x": "\ud800"}}, + "input keys and values must be valid Unicode", + ), + ( + {"source": SOURCE, "inputs": {"\ud800": "x"}}, + "input keys and values must be valid Unicode", + ), + ({"ir": {"name": "\ud800"}, "inputs": {}}, "workflow text must be valid Unicode"), + ], +) +def test_malformed_unicode_returns_400_and_server_survives( + tmp_path: Path, payload: dict[str, object], message: str +) -> None: + running = _RunningServer(tmp_path) + try: + status, response = running.request("POST", "/runs", payload=payload, authorized=True) + assert status == 400 + assert response == {"error": message} + assert running.request("GET", "/healthz") == (200, {"ok": True, "database": "ok"}) + finally: + running.close() + + +@pytest.mark.parametrize( + "body", + [ + b'{"source":"thread T { context {} emit text { \\"ok\\" } }","inputs":{},"n":' + + b"9" * 5000 + + b"}", + ], +) +def test_json_parser_failures_return_400_and_server_survives(tmp_path: Path, body: bytes) -> None: + running = _RunningServer(tmp_path) + try: + conn = http.client.HTTPConnection("127.0.0.1", running.port, timeout=5) + conn.request( + "POST", + "/runs", + body=body, + headers={ + "Authorization": f"Bearer {TOKEN}", + "Content-Type": "application/json", + "Content-Length": str(len(body)), + }, + ) + response = conn.getresponse() + assert response.status == 400 + assert json.loads(response.read()) == {"error": "body must be valid JSON"} + conn.close() + assert running.request("GET", "/healthz") == (200, {"ok": True, "database": "ok"}) + finally: + running.close() + + def test_run_list_is_paginated_and_bounded(tmp_path: Path) -> None: running = _RunningServer(tmp_path) try: @@ -240,3 +349,125 @@ def test_readiness_reflects_live_workers(tmp_path: Path) -> None: finally: running.close() pool.stop() + + +def test_readiness_returns_503_when_database_is_unavailable( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + running = _RunningServer(tmp_path) + + class UnavailableStore: + def __init__(self, path: str) -> None: + raise OSError("database unavailable") + + try: + monkeypatch.setattr(server_module, "RunStore", UnavailableStore) + status, payload = running.request("GET", "/readyz") + assert status == 503 + assert payload == {"ok": False, "database": "unavailable", "workers": None} + finally: + running.close() + + +def test_server_main_formats_bind_errors( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + def fail_serve(*args: object, **kwargs: object) -> None: + raise OSError("bind failed") + + monkeypatch.setattr(server_module, "serve", fail_serve) + monkeypatch.setattr(sys, "argv", ["threadlang-serve", "--store", str(tmp_path / "runs.db")]) + assert server_module.main() == 2 + captured = capsys.readouterr() + assert captured.out == "" + assert captured.err == "error: bind failed\n" + + +@pytest.mark.parametrize("base_url", ["not-a-url", "http://[::1"]) +def test_server_main_formats_provider_setup_errors( + base_url: str, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + monkeypatch.setattr( + sys, + "argv", + [ + "threadlang-serve", + "--store", + str(tmp_path / "runs.db"), + "--backend", + "openai", + "--base-url", + base_url, + ], + ) + assert server_module.main() == 1 + captured = capsys.readouterr() + assert captured.out == "" + assert captured.err.startswith("error: OpenAI-compatible base URL") + assert "Traceback" not in captured.err + + +@pytest.mark.parametrize("timeout", ["nan", "inf", "-inf"]) +def test_server_main_rejects_non_finite_timeout_before_provider_setup( + timeout: str, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + provider_calls = 0 + serve_calls = 0 + + def unexpected_provider(*args: object, **kwargs: object) -> None: + nonlocal provider_calls + provider_calls += 1 + + def unexpected_serve(*args: object, **kwargs: object) -> None: + nonlocal serve_calls + serve_calls += 1 + raise AssertionError("server should not start") + + monkeypatch.setattr("threadlang.llm.OpenAICompatClient", unexpected_provider) + monkeypatch.setattr(server_module, "serve", unexpected_serve) + monkeypatch.setattr( + sys, + "argv", + [ + "threadlang-serve", + "--store", + str(tmp_path / "runs.db"), + "--backend", + "openai", + f"--timeout={timeout}", + ], + ) + assert server_module.main() == 2 + assert provider_calls == 0 + assert serve_calls == 0 + assert "must be positive" in capsys.readouterr().err + + +def test_server_main_validates_bounds_before_provider_setup( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + monkeypatch.setattr( + sys, + "argv", + [ + "threadlang-serve", + "--store", + str(tmp_path / "runs.db"), + "--backend", + "openai", + "--base-url", + "not-a-url", + "--max-tokens", + "0", + ], + ) + assert server_module.main() == 2 + captured = capsys.readouterr() + assert "must be positive" in captured.err + assert "base URL" not in captured.err diff --git a/tests/test_v04_durability.py b/tests/test_v04_durability.py index 632d97c..2521cf5 100644 --- a/tests/test_v04_durability.py +++ b/tests/test_v04_durability.py @@ -76,6 +76,24 @@ def test_durable_run_persists_events_status_and_checkpoints(tmp_path: Path) -> N store.close() +def test_completion_persistence_failure_marks_run_failed(tmp_path: Path) -> None: + class InvalidUnicodeClient: + def complete(self, model: str, prompt: str) -> str: + return "\ud800" + + source = 'thread T { context {} steps { step x { llm "m" { "prompt" } } } emit text { steps.x.output } }' + store = RunStore(str(tmp_path / "runs.db")) + + with pytest.raises(UnicodeEncodeError): + run_durable(parse_program(source), {}, store, llm_client=InvalidUnicodeClient()) + + runs = store.list_runs() + assert len(runs) == 1 + assert runs[0].status == "failed" + assert runs[0].error is not None and "UnicodeEncodeError" in runs[0].error + store.close() + + def test_resume_skips_completed_step_after_crash(tmp_path: Path) -> None: store = RunStore(str(tmp_path / "runs.db")) program = parse_program(_TWO_STEP) diff --git a/tests/test_v07_triage.py b/tests/test_v07_triage.py index 950890c..c801b74 100644 --- a/tests/test_v07_triage.py +++ b/tests/test_v07_triage.py @@ -18,10 +18,12 @@ from pathlib import Path import sys +import pytest + REPO_ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(REPO_ROOT / "src")) -from threadlang.apps.support_triage.app import load_program # noqa: E402 +from threadlang.apps.support_triage.app import load_program, main as app_main # noqa: E402 from threadlang.apps.support_triage.tools import build_registry # noqa: E402 from threadlang.ast import AgentStep, Step # noqa: E402 from threadlang.control import WorkerPool # noqa: E402 @@ -106,3 +108,60 @@ def test_durable_queued_path(tmp_path) -> None: ) finally: store.close() + + +def test_run_formats_store_open_errors(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + store_path = tmp_path / "missing" / "runs.db" + assert app_main(["run", "--ticket", "test", "--store", str(store_path), "--dry-run"]) == 1 + captured = capsys.readouterr() + assert captured.out == "" + assert captured.err.startswith("error:") + assert "Traceback" not in captured.err + + +@pytest.mark.parametrize("timeout", ["nan", "inf", "-inf"]) +def test_rejects_non_finite_timeout(timeout: str, tmp_path: Path) -> None: + with pytest.raises(SystemExit) as raised: + app_main( + [ + "run", + "--ticket", + "x", + "--store", + str(tmp_path / "runs.db"), + "--dry-run", + f"--timeout={timeout}", + ] + ) + assert raised.value.code == 2 + + +def test_serve_validates_bounds_before_provider_setup( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + calls = 0 + + def unexpected_provider(*args: object, **kwargs: object) -> None: + nonlocal calls + calls += 1 + + monkeypatch.setattr("threadlang.apps.support_triage.app._make_client", unexpected_provider) + with pytest.raises(SystemExit) as raised: + app_main( + [ + "serve", + "--store", + "x.db", + "--workers", + "0", + "--backend", + "openai", + "--base-url", + "not-a-url", + ] + ) + assert raised.value.code == 2 + assert calls == 0 + captured = capsys.readouterr() + assert "workers must be >= 1" in captured.err + assert "base URL" not in captured.err diff --git a/tests/test_v08_metrics.py b/tests/test_v08_metrics.py index a14caba..6fc8d27 100644 --- a/tests/test_v08_metrics.py +++ b/tests/test_v08_metrics.py @@ -116,6 +116,16 @@ def test_agent_step_metrics() -> None: assert m.steps_completed == 1 +def test_release_report_dry_run_has_no_tool_errors() -> None: + source = (REPO_ROOT / "examples" / "release_report.thread").read_text(encoding="utf-8") + result = run_program( + parse_program(source), + inputs={"stats": "errors fell from 40 to 10", "notes": "shipped"}, + llm_client=DryRunClient(), + ) + assert compute_metrics(result.trace).tool_errors == 0 + + def test_tool_errors_are_counted() -> None: trace = [ TraceEvent("agent", "Agent step 's' started", {"step": "s"}),