diff --git a/CLAUDE.md b/CLAUDE.md index 78af1f0..1c79c90 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -147,6 +147,12 @@ docs/WORKSPACES.md, and docs/proposals/server-workspaces.md for the later stages rooted at the library rather than the workflow file. The library is `DW_PROMPT_DIR` / `--prompt-dir`, else `./prompts` if it exists, else found by walking up from the workflow file's directory +- Every run directory holds `workflow.json` beside its manifest: the *realized* + workflow, with the run's arguments folded into the variable defaults, the seed + it used, stored prompt text inlined and `output:.../latest/...` pinned to the + run it resolved to. Written by `realize_workflow` (`dw/realize.py`) at run + start, best effort. Over MCP, `get_job_workflow` reads it back and + `save_workflow` names it; `export_job` bundles the run The same conventions, written for an agent composing a workflow over MCP, are the `Authoring a workflow from an agent` section of docs/WORKFLOW_GUIDE.md; @@ -202,7 +208,14 @@ All entry points use `dw/security.py`. When adding features: A sub-workflow inherits the parent's run directory and writes no manifest of its own. `--output-layout flat` / `DW_OUTPUT_LAYOUT` / the `output_layout` setting restores the old layout. The gallery groups a workflow's runs under one folder by stripping the run - id (`strip_run_id`) + id (`strip_run_id`). The realized workflow is written into the same directory as + `workflow.json` (`dw/realize.py`, `write_realized_workflow`), and the manifest's + `workflow` block carries `realized`, `prompts` (the stored prompts inlined) and + `sub_workflows` (path -> SHA-256). A job records the run it was + (`run_id`/`run_dir` on `Job` and in `jobs.sqlite`), which is how + `JobManager.realized` finds the file. `exports` is a reserved workspace name: + `POST /api/jobs/{id}/export` gathers one finished job into + `/exports//` and `GET /exports/.zip` streams it. - **Step cache**: a process-wide singleton (`dw/step_cache.py`) consulted by every `Workflow.run`, including server jobs; entries are keyed by `(workflow id, step name)` and validated against the output *root*, never the per-run directory - a run directory is new every execution and would defeat the cache; disabled entirely when the workflow sets no `seed`; a hit reports the earlier run's files with `reused: true` and writes nothing new; `memory clear` drops it diff --git a/docs/MCP.md b/docs/MCP.md index 01449ca..81efcf8 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -280,6 +280,8 @@ references written in the same session. | --- | --- | --- | | `run_workflow(workflow_path=None, inline_workflow=None, arguments=None, acknowledged_cost=False)` | exactly one of `workflow_path` (a catalog name from `list_workflows`, with or without `.json`, or a path to a workflow file on the server) or `inline_workflow`, optional `arguments`, `acknowledged_cost` | Queue a workflow for generation. Returns as soon as the job is queued | | `get_job(job_id)` | `job_id` | Get a job's status, warnings, output manifest, error and traceback | +| `get_job_workflow(job_id)` | `job_id` | The workflow the job actually ran. `realized: true` means every mutable input is pinned (arguments, seed, prompts, `output:latest`); `false` means the job predates run tracking and this is the definition as submitted. Pass it to `save_workflow` to keep it under a name | +| `export_job(job_id, overwrite=False)` | `job_id`, `overwrite` | Gather one finished job into `/exports//` on the server: the realized workflow, the run's manifest, the job row, a README, and copies of the assets, earlier-run inputs and outputs. Returns the directory, a zip URL, the file list with sizes and the three JSON files inline. **The directory is on the machine running the server**, like `download_output`'s destination - fetch the zip URL and unpack it into `exports/` under the session's working directory (a deliverable, not a temp file); the archive already unpacks into one folder named after the job id | | `get_job_events(job_id, after=-1, limit=200)` | `job_id`, `after`, `limit` | Get a page of a job's progress events | | `wait_for_job(job_id, timeout_seconds=20)` | `job_id`, `timeout_seconds` | Block until a job reaches a terminal status, or `timeout_seconds` elapses (capped well under a generation's real runtime). Use instead of hand-polling `get_job`/`get_job_events` in a loop; if it returns `still_running: true`, call it again | | `cancel_job(job_id)` | `job_id` | Ask a queued or running job to stop | diff --git a/docs/SERVER.md b/docs/SERVER.md index 4a2945a..75b9808 100644 --- a/docs/SERVER.md +++ b/docs/SERVER.md @@ -135,7 +135,9 @@ from another machine: | `POST /api/jobs` | Queue a run: `{"workflow_path": ...}` or an inline `{"workflow": {...}, "base_dir": ...}`, plus `arguments` for variable overrides. `workflow_path` accepts a stored workflow name as listed by `/api/workflows` (with or without `.json`, nested names included), or a relative/absolute path that still resolves under `--workflow-dir` - confined the same way the `/api/workflows` CRUD routes are; a path that names a real file outside that directory is rejected with 400, not opened. Answers with argument warnings from signature checking. | | `GET /api/jobs` | Queue + history summaries | | `GET /api/jobs/{id}` | Full detail: spec, events, manifest, error. A manifest entry for a step served from the step cache carries `reused: true` | -| `GET /api/jobs/{id}/workflow` | The definition the job ran, for the job page's read-only flow graph: `{id, definition}`. An inline definition comes from the job's own spec; a job launched from a path is re-read from the root it was confined to, so 404 means the file has since moved or changed - the job itself is still readable | +| `GET /api/jobs/{id}/workflow` | The workflow the job ran: `{id, definition, realized}`. `realized: true` is the copy the run itself wrote (`workflow.json` in its run directory), with arguments, seed, prompts and `output:latest` pinned; `false` falls back to the submitted definition, which is what a job from before run tracking has. 404 means neither is readable - the job itself still is | +| `POST /api/jobs/{id}/export?workspace=&overwrite=` | Gather one finished job into `/exports//`: `workflow.json`, `manifest.json`, `job.json`, `README.md`, `assets/`, `inputs/`, `outputs/`. 201 with the file list, total bytes, anything it could not find, a `zip_url`, and the three JSON files inline. 404 unknown job, 409 for a job still running or an existing export without `overwrite` | +| `GET /exports/{id}.zip?workspace=` | The same tree as one archive, built on request rather than kept as a second copy. Entries are named `/`. Ungated exactly as `/outputs` is | | `GET /api/jobs/{id}/events` | Server-sent events stream; `?after=N` / `Last-Event-ID` replay missed events, so reconnects are lossless | | `GET /api/jobs/{id}/event-log?after=-1&limit=200` | The same events as the SSE stream, as one JSON page: `{id, status, events, last_seq, truncated, note}`. `after` is exclusive; page by passing back the previous `last_seq`. A job restored from history serves the bounded event tail persisted with it; a job that finished before events were retained returns an empty list and a `note` saying so. | | `POST /api/jobs/{id}/cancel` | Cooperative cancel (takes effect at the next step boundary or denoise step) | @@ -154,6 +156,7 @@ Every event in the stream carries a `seq` and an `event` name: | `job_status` | queued/running/terminal transitions | `status` | | `log` | worker output lines | `message` | | `memory` | device memory after a run | `info` | +| `run_start` | the run directory is chosen, before the first step | `run_id`, `identity`, `run_dir` | | `workflow_start` | the run begins | `workflow`, `total_steps`, `steps`, `seed` | | `step_start` / `step_end` | each step | `step`, `index`, `total_steps`; `files` at the end. A step served from the step cache adds `reused: true` to `step_end`, and its `files` are the earlier run's files rather than newly written ones | | `iteration_start` | each argument combination in a step | `step`, `iteration`, `total_iterations` | @@ -259,6 +262,10 @@ The editor's forms come from these; they are just as usable from scripts: refuses until acknowledged, refuses the default, and refuses a workspace with jobs still queued. A workspace is a namespace, **not** a security boundary: the API token is all-or-nothing + + `exports/` sits beside the workspace's own folders, holding one directory per + exported job. It is a reserved name: no workspace can be called `exports`, and + the folder is never listed as one. - `GET /api/assets` — the asset library: input media, each with the `asset:` reference a workflow carries rather than a path, since a path only means something on the server's own machine. Empty rather than an diff --git a/docs/WORKFLOW_GUIDE.md b/docs/WORKFLOW_GUIDE.md index 43aa717..ee2f072 100644 --- a/docs/WORKFLOW_GUIDE.md +++ b/docs/WORKFLOW_GUIDE.md @@ -270,6 +270,14 @@ not validation. `text`, rooted at the prompt library. That text may not itself begin with any of these prefixes; the engine rejects such a prompt rather than resolving twice. +After a long inline run that is worth keeping, `get_job_workflow(job_id)` +returns the realized workflow — the definition with the arguments, seed and +prompts of that run pinned into it — and `save_workflow` gives it a name, so +the next run is by name rather than by pasting JSON again. `export_job(job_id)` +bundles the whole run (workflow, manifest, job row, the media on both sides) +into a directory on the server plus a zip URL, for a run worth committing or +handing to someone else. + A reference is resolved wherever it appears in the arguments, including inside a nested object or list — not only at the top level. It is always the *whole* value: `"variable:base_prompt"` resolves, `"variable:base_prompt, in fog"` asks @@ -999,6 +1007,26 @@ Omit `seed` entirely to let the workflow draw a random one at run time. The seed actually used - drawn or named - is recorded in its `manifest.json`, so a run you liked can be reproduced after the fact. +Beside that manifest the run also writes `workflow.json` — the *realized* +workflow, meaning the one that actually ran. Every mutable input is pinned into +it: the caller's `arguments` folded into the `variables` defaults, the seed the +run used, each `prompt:` reference replaced by the stored text, and each +`output:/latest/` rewritten to the run id it resolved to. +`asset:`, `constant:`, `previous_result:` and `builtin:` are kept as written — +each already names something pinned by the asset library or by the manifest's +`dw_version` — and a sub-workflow named by local path is kept with its file's +SHA-256 recorded in the manifest. The manifest also lists which stored prompts +were inlined, since inlining loses the name. + +The file is a valid workflow, and running it again is `python -m dw.run +workflow.json` or handing its contents to `run_workflow` as `inline_workflow` +— but either way the `asset:` and `output:` names in it resolve against the +server's or CLI's own libraries, not against the run directory, so doing this +from inside that directory reproduces the run only when its libraries are the +ones the original run used too. Writing the file is best effort, exactly like +the manifest — a run that produced its files has succeeded either way — and +`--output-layout flat` writes no run directory, so it writes neither file. + Any of the three levels accepts a `variable:` reference, which is how a seed becomes settable per run without editing the file: diff --git a/docs/WORKSPACES.md b/docs/WORKSPACES.md index 079092a..b06f47f 100644 --- a/docs/WORKSPACES.md +++ b/docs/WORKSPACES.md @@ -176,6 +176,7 @@ outputs/ Gyre-still.0-0.0.png Gyre-video.1-0.0.mp4 manifest.json + workflow.json ``` The folder is the workflow's identity — its path under a `workflows/` tree @@ -185,6 +186,10 @@ runs of the same workflow sort by time and a rerun of an edited workflow is visibly different; a second run of the same spec in the same second takes a counter rather than sharing a directory. +`workflow.json` is the realized workflow — the definition with this run's +arguments, seed and stored prompts pinned into it, so the directory reproduces +itself. `manifest.json` points at it and lists which prompts were inlined. + `manifest.json` records the run beside what it made — status, seed, arguments, device, dw version, and each step's files, named relative to the directory so it keeps describing itself if you move or copy it. It is written even when a @@ -229,6 +234,10 @@ reference, and a prompt duplicated per workspace would resolve to different text depending on where a workflow happened to be saved. `workflows`, `prompts`, `assets` and `outputs` are reserved names for that reason. +A fifth name is reserved beside `workflows`, `prompts`, `assets` and `outputs`: +`exports`. `POST /api/jobs/{id}/export` gathers one finished job into +`/exports//`, and that folder is never mistaken for a workspace. + This is what lets two agents share one GPU without sharing a namespace: each takes a workspace, and neither can save over the other's workflows or delete the other's renders. diff --git a/docs/proposals/codeql-sanitizer-model.md b/docs/proposals/codeql-sanitizer-model.md new file mode 100644 index 0000000..5444802 --- /dev/null +++ b/docs/proposals/codeql-sanitizer-model.md @@ -0,0 +1,114 @@ +# Proposal: making CodeQL see the path validators + +Status: draft, 2026-09-08. No implementation. + +## Problem + +Every release adds a few `py/path-injection` alerts from code scanning, and +every one so far has had the same shape: a path is joined and then confined +by `validate_path` in `dw/security.py` (or one of the wrappers around it), +and CodeQL flags the `os.*` or `shutil.*` call downstream because it does +not treat the validator as a sanitizer. The count on 2026-09-08 is 28 open +and 14 dismissed as "mitigated", all of that rule, plus the ten on #56 that +were dismissed the same way after a review found one real gap among them +(an `export_directory` job id of `.` named the exports root itself; fixed +in 7ead046 with a shape check). + +Dismissing by hand works but scales badly, and it teaches the reviewer to +skim: the one real finding on #56 was flagged in exactly the same words as +the nine that were not. + +## What already exists, and why it is not working + +`.github/codeql/extensions/dw-models/` is a CodeQL model pack, added with +the workspaces PR (#37, 2026-09-05), that declares the validators' return +values as `path-injection` barriers with the `barrierModel` extensible +predicate: + +```yaml +- ["dw.security", "Member[validate_path].ReturnValue", "path-injection"] +``` + +The approach is right. `barrierModel` and `barrierGuardModel` were added +to models-as-data for Python in CodeQL 2.25.2 (April 2026), the path +injection query's `SanitizerFromModel` reads barriers of kind +`path-injection`, code scanning default setup picks up model packs from +`.github/codeql/extensions/` without a workflow, and the repository is +scanned with CodeQL 2.26.4. Yet every alert listed above was created +*after* the pack landed: 26 to 43 on 2026-09-06 and 07, 44 to 53 on the +08th. The pack is not taking effect, and nothing in the scanning UI says +why. + +Hypotheses, most likely first: + +1. **The type string does not resolve.** Every caller imports the + validators relatively (`from ..security import validate_path`, + `from .security import ...`). The `type` column of a Python model is + matched through the API graph's module-import resolution, which is + built for absolute imports of library modules; whether a relative + import inside the analysed package resolves to `dw.security` is not + documented and is the first thing to test. +2. **The pack is not being loaded at all.** A malformed + `codeql-pack.yml`, a name collision, or a default-setup restriction + the docs do not spell out. Default setup exposes no log for this. +3. **The barrier is on the wrong node.** A barrier on the return value + stops flow that *passes through* the call. The flow CodeQL reports may + instead go around it: `validate_path` returns a value computed from + its argument, and if the analysed body contributes a second path + (the argument itself flowing to a sink inside `validate_path`, then + summarised) the return-value barrier would not cut it. Unlikely, but + the alert's path view would show it. +4. **`base_dir=None` is a real hole in the model.** `validate_path(p, + None)` resolves and pattern-checks but confines to nothing, so a + barrier on its return value claims more than the function guarantees. + That does not explain the alerts, but it is why the model should sit + on the wrappers that always pass a root (`validate_output_path`, + `validate_workflow_path`, `validate_prompt_path`) and on + `validate_path` only when the root argument is present, which the + model language cannot express. Decide whether to accept the + over-claim or narrow the model. + +## Proposed shape + +**Reproduce locally, then fix the model, then let the scanner confirm.** +Default setup gives no feedback loop; the CodeQL CLI does. + +1. Install the CodeQL CLI (`gh extension install github/gh-codeql`, which + also fetches the bundle) on the Mac. +2. `codeql database create --language=python dw-db` at the repo root, + then `codeql database analyze dw-db codeql/python-queries:codeql-suites/python-code-scanning.qls --model-packs dkackman/dw-models --format=sarif-latest --output=out.sarif` + with the pack path supplied through `--additional-packs + .github/codeql/extensions`. Count the `py/path-injection` results. +3. Iterate on `dw-security.model.yml` until the count drops to the + findings that are real: try an absolute-import form of the type, try + the wrappers rather than `validate_path`, try `barrierGuardModel` on + the `SecurityError` branch. Each try is one edit and one analyze. +4. Commit the working model. Code scanning re-runs on the next push and + the open alerts close as "fixed" on their own, which is the only + confirmation that matters. +5. Record in `dw/security.py`'s module docstring that the validators are + modelled, and where, so the next person adding a validator adds a row. + +A `scripts/codeql-local.sh` that does steps 2 and 3 in one command is +worth keeping, since the model will need the same loop each time a +validator is added. + +## What this is not + +- Not a workflow migration. Default setup honours repository model packs, + and advanced setup is only needed if step 3 shows the fix requires a + custom query, which nothing so far suggests. +- Not a reason to stop reading the alerts. Once the model holds, an alert + of this rule means a path that did not go through a validator, which is + exactly the signal the rule is for. + +## Open questions + +- Whether the fourteen "mitigated" dismissals and the ten from #56 should + be reopened once the model works, so the scanner re-evaluates them. They + would close as fixed if the model is right and stay open if it is not, + which is a useful check; but reopening is a manual click each. +- Whether `validate_path` with no root should keep returning a path at + all, or whether every caller should be made to pass one. That is a + security question independent of CodeQL, and the model's over-claim in + hypothesis 4 is the argument for looking at it. diff --git a/docs/proposals/job-record-and-export.md b/docs/proposals/job-record-and-export.md new file mode 100644 index 0000000..992f5dc --- /dev/null +++ b/docs/proposals/job-record-and-export.md @@ -0,0 +1,165 @@ +# Proposal: the realized workflow as a job's record, and exporting a job + +Status: implemented 2026-09-08 (design: +`docs/superpowers/specs/2026-09-08-job-record-and-export-design.md`, plan: +`docs/superpowers/plans/2026-09-08-job-record-and-export.md`). Supplies the +"manifest carries step identity" stage of [resume.md](resume.md), which stays +the design for resuming a run; this proposal is the record that resume reads, +plus the two ways to get it off the server. + +## Problem + +On 2026-09-08 an MCP session assembled a six-shot MiniMax H3 short as an +inline workflow, ran it for most of an hour, and lost it at the last step to +a bad argument. The retry regenerated every shot. Three things about that +run are worth fixing, and none of them is the bad argument: + +- **The workflow that produced the cut existed nowhere the user could read.** + The job store keeps an inline definition in its spec column and the job + page can draw it, but no MCP tool returns it, and the run directory holds + a manifest that says only `"file": ".../__inline__.json"`. +- **A template job records less than that.** Its spec holds a path and the + arguments. Edit the template, or a stored prompt it names, or make a newer + run that `output:latest` now picks, and the record no longer describes what + ran. The manifest pins the seed and the arguments, but not the definition. +- **Nothing packages a run.** Manifest, workflow, the assets it referenced + and the files it made are four places on one machine, and `download_output` + moves one file at a time onto that same machine. A run that is worth + keeping, sharing or committing has to be gathered by hand. + +MCP is turning out to be the primary way work is submitted, and an agent +writes inline workflows freely: an inline job is now the common case, not +the exception the job store treated it as. + +## What exists today + +| Piece | Where | What it records | +|---|---|---| +| Submitted definition | `jobs.sqlite` `spec` (`workflow` for inline, `workflow_path` for a file) | What was sent, not what ran | +| Read-only view of it | `GET /api/jobs/{id}/workflow`, `JobManager.definition` | UI only; no MCP tool | +| Rerun | `POST /api/jobs/{id}/rerun`, `rerun_job` | Resubmits the spec verbatim, arguments included | +| Run directory | `///` (`dw/runs.py`) | Every file a step wrote | +| Manifest | `manifest.json` beside them | Status, times, engine version, device, seed actually used, arguments, per-step files | +| Run id | Directory name only | Never reported to the job or the job store | +| Reuse of a file | `output://` | Resolves out of that tree | +| Keeping a file | `keep_output` | Copies one output into the assets library | + +The record of a job is therefore split between a database row that knows the +definition and a directory that knows the outcome, and neither is complete. + +## Proposed shape + +### 1. Every run writes its realized workflow beside its manifest + +`Workflow.run` writes `workflow.json` into the run directory at the start of +the run, before the first step, so a crash or a cancel still leaves it. The +manifest points at it (`"workflow": {..., "realized": "workflow.json"}`). +CLI runs and server jobs get it alike, since the manifest is written there +too. A sub-workflow inherits its parent's run directory and writes none of +its own, as with the manifest. + +*Realized* means every mutable input is pinned, so the file reproduces the +run whatever changes later: + +| Reference | In the realized file | +|---|---| +| `variables` and the job's `arguments` | Arguments folded into the variable defaults; `variable:` references stay, so the file remains runnable with overrides | +| `seed` | The seed the run actually used (drawn if the workflow named none) | +| `prompt:name` | The stored prompt's text, inlined | +| `output:/latest/` | Rewritten to the concrete run id it resolved to | +| `output:` with an explicit run id | Kept | +| `asset:name` | Kept: a name in the library, which the export bundles | +| `constant:` | Kept: a value in the Python module the manifest's `dw_version` pins | +| `builtin:` sub-workflow | Kept: packaged with the engine, pinned by `dw_version` | +| sub-workflow by local path | Kept; the manifest records the file's SHA-256 | +| `previous_result:` | Kept: it names a step in the same file | + +The realized file is a valid workflow. `validate_workflow` accepts it and +`run_workflow(inline_workflow=...)` reproduces the run, which is the +simplest possible resume-by-hand and needs no new engine path. + +### 2. The job knows its run + +The worker reports the run id and the run directory (relative to the output +root) in a `run_start` event; `Job` records them and `jobs.sqlite` gains a +`run_id` column. `JobManager.realized(job_id)` reads the run's +`workflow.json`. A job from before this feature has no run id: the manager +falls back to the submitted definition and says so, rather than deriving a +run directory from file paths. + +### 3. MCP can read it + +`get_job_workflow(job_id)` returns the realized workflow, with `realized: +true`, or the submitted definition with `realized: false` and a sentence +saying why. Small enough to return inline: a workflow is text. + +### 4. Exporting a job + +`POST /api/jobs/{id}/export` gathers one job into +`/exports//`: + +``` +exports// + README.md what this is, how it was made, how to run it again + workflow.json the realized workflow (section 1) + manifest.json the run's manifest + job.json the job row: status, times, arguments, warnings, error + assets/ every asset:name the workflow references, by name + inputs/ every file an output: reference named from another run + outputs/ every file the manifest lists, in the run's layout +``` + +The tree is git-ready as it stands: text at the top, media in folders, no +absolute paths (the README notes that large media belongs in Git LFS). +`GET /exports/.zip` streams the same tree as one archive for a +browser or a laptop, built on request from the directory rather than kept as +a second copy. `exports` joins the reserved workspace names. + +MCP `export_job(job_id)` runs the export and returns the directory, the zip +URL, and the file list with sizes; the JSON files it also returns inline. +Like `download_output`, the directory is on the machine running the server, +and the tool says so. + +### 5. Recoverability + +This proposal writes the record; [resume.md](resume.md) is the design for +acting on it. Its stage 2, a per-step identity in the manifest, is served by +the realized file: each step's definition is there to compare, and the +manifest's per-step files say what it made. Its stage 3, rehydrating the +step cache from a previous run, is what turns the lighthouse retry into a +one-step rerun, and is planned as the next piece after this one lands. Until +then the realized file is what a user hands back to `run_workflow` to +reproduce a run, and what an agent edits to change one step of it. + +## Staging + +1. **The realized file.** Realization rules, written at run start, manifest + pointer, tests for every row of the table above. CLI and server alike. +2. **Run id on the job, and the MCP read.** The `run_start` event, the + column, `JobManager.realized`, `get_job_workflow`. +3. **Export.** The route, the directory layout, the README, the zip, the MCP + tool, the reserved name. +4. **Resume**, as resume.md stage 3, its own proposal and plan. + +Stages 1 and 2 are the useful unit; 3 stands on them. + +## Open questions + +- **Sub-workflows by local path.** Inlining them would make the realized file + self-contained, but the schema's `workflow` step takes a path, not a + definition. Decided in the design: keep the path, record the file's + digest in the manifest, and inline only if the digest turns out to be + what people trip over. +- **Realizing `prompt:` loses the name.** Decided in the design: the + manifest lists the stored prompts the run inlined, so the realized file + stays schema-clean and the name is not lost. +- **Export size.** A six-shot H3 run is hundreds of megabytes of video. The + export copies rather than hard-links, since `exports/` is what a user moves + or deletes; the response reports the total so an agent can quote it. +- **`inputs/` and the realized file.** The export copies what an `output:` + reference named from another run, but does not rewrite the reference, + since the realized file is the immutable record of the run. The README + says where each input came from. +- **History rebuild.** A run directory with `workflow.json` and + `manifest.json` is enough to reconstruct a job row without the database. + Not proposed here; noted because this is what makes it possible. diff --git a/docs/proposals/resume.md b/docs/proposals/resume.md index 7690bee..4c62f83 100644 --- a/docs/proposals/resume.md +++ b/docs/proposals/resume.md @@ -128,9 +128,13 @@ version is already scoped to a session where the user can see what happened. 1. **Prerequisite (done): seed.** `seed` accepts a `variable:` reference, and the manifest records the seed actually used. Without both, a seedless workflow has nothing to resume against. -2. **Manifest carries step identity.** Add a per-step digest of the resolved - `step_data` to the manifest. Inert on its own — nothing reads it yet — and - independently useful for answering "did this run actually run the same thing". +2. **Manifest carries step identity.** *Satisfied by the realized workflow* + ([job-record-and-export.md](job-record-and-export.md)): every run now writes + `workflow.json` beside its manifest with every mutable input pinned, so each + step's definition as it actually ran is on disk to compare against, and the + manifest's per-step files say what it made. A per-step digest may still be + worth adding for a cheaper comparison, but the information is no longer + missing. 3. **Rehydration.** A loader that turns manifests into `StepCache` entries, plus whatever `StepCache` needs to compare by digest. Behind `--resume`. 4. **Reload saved media into results.** Widen resumability past files-only diff --git a/docs/superpowers/plans/2026-09-08-job-record-and-export.md b/docs/superpowers/plans/2026-09-08-job-record-and-export.md new file mode 100644 index 0000000..f1aa086 --- /dev/null +++ b/docs/superpowers/plans/2026-09-08-job-record-and-export.md @@ -0,0 +1,2791 @@ +# Job Record and Export Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Every run leaves a realized copy of the workflow that produced it beside its manifest; a server job knows which run is its own; an MCP client can read that workflow and export one job as a git-ready directory and a zip. + +**Architecture:** A new pure module `dw/realize.py` walks a workflow definition and pins every mutable input (arguments folded into variable defaults, the drawn seed, stored prompt text inlined, `output:.../latest/...` rewritten to a concrete run id), returning a schema-clean copy plus annotations for the manifest. `Workflow.run` writes that copy as `workflow.json` in the run directory and emits a `run_start` event carrying the run id; the server's `JobManager` records that on the job and in `jobs.sqlite`, which lets it read the realized file back. On top of that record, `dw/server/exports.py` gathers one finished job into `/exports//` and a route streams the same tree as a zip; two MCP tools expose the read and the export. + +**Tech Stack:** Python 3, FastAPI/Starlette, sqlite3, pytest, httpx `MockTransport` (MCP tests), the MCP Python SDK (`dw_mcp/server.py`). + +**Spec:** `docs/superpowers/specs/2026-09-08-job-record-and-export-design.md` +(Context: `docs/proposals/job-record-and-export.md`, `docs/proposals/resume.md`.) + +**Running tests:** the suite needs the project venv. Every pytest command in this +plan is written as `source ./activate && python -m pytest ...` and must be run +from the repo root (`/Users/don/src/dkackman/diffusers-workflow`). + +## Global Constraints + +Copied verbatim from the spec's "Global constraints", and implicitly part of +every task's requirements: + +- Model knowledge stays out of engine code; nothing here names a model. +- Every path read or written goes through `dw/security.py` validators: + `validate_output_path`, `validate_path`, the asset and output resolvers. +- The realized file must validate against `dw/workflow_schema.json` + unchanged: annotations that the schema would reject live in the manifest. +- Writing the realized file and the manifest is best effort: a run that + produced its files has succeeded whether or not the record landed + (`write_manifest`'s rule). +- Nothing bundled requires `--trust-workflows`. +- `exports` is a reserved workspace name, beside `workflows`, `prompts`, + `assets`, `outputs`. + +Plus the repo rules in `CLAUDE.md` that bear on this work: never `eval()`, +`exec()` or `shell=True`; path traversal is blocked; schema validation runs +before variable substitution. + +## File Structure + +**Created** + +- `dw/realize.py` — the realization rules. Pure: no I/O beyond the resolvers it + calls, never mutates its input, never raises for an unresolvable reference. +- `dw/server/exports.py` — gathering one finished job into a directory. +- `dw_mcp/exports.py` — the MCP handler over the export route. +- `tests/test_realize.py`, `tests/test_server_jobs.py`, + `tests/test_server_exports.py`, `tests/test_mcp_exports.py`. + +**Modified** + +- `dw/runs.py` — `REALIZED_FILE_NAME`, `write_realized_workflow`. +- `dw/workflow.py` — call realization at run start, emit `run_start`, carry the + two new manifest keys. +- `dw/workspace.py` — `EXPORTS_SUBDIR`, reserved name, `_foreign_entries`. +- `dw/server/jobs.py` — `Job.run_id`/`run_dir`, the sqlite columns, + `JobManager.realized`. +- `dw/server/app.py` — the `realized` flag on the workflow route, the export + route, the zip route. +- `dw_mcp/diagnose.py`, `dw_mcp/client.py`, `dw_mcp/server.py` — the two tools. +- Docs and plugin skills (Task 7). + +--- + +### Task 1: Realization module (`sonnet`) + +*Model rationale: one new file, but the semantics span four resolver modules +(`variables`, `prompts`, `runs`, `security`) and the tree walk must be exactly +right — integration judgement, not typing.* + +**Files:** +- Create: `dw/realize.py` +- Test: `tests/test_realize.py` + +**Interfaces:** +- Consumes: `dw.variables.set_variables(values, variables)` (mutates + `variables` in place); `dw.prompts.fetch_prompt(reference, prompt_dir=None, + base_dir=None) -> str` and `dw.prompts.PROMPT_PREFIX`; + `dw.runs.resolve_output_reference(reference, root=None) -> str`, + `dw.runs.output_root() -> str`, `dw.runs.OUTPUT_PREFIX`, `dw.runs.LATEST`, + `dw.runs.is_output_reference(value)`; + `dw.security.validate_workflow_path(path, confine_to)`, + `dw.security.SecurityError`. +- Produces: `realize_workflow(definition, arguments, seed, base_dir=None, + prompt_dir=None, output_root=None, workflow_dir=None) -> (dict, dict)`. + The second element is + `{"prompts": [str, ...], "sub_workflows": {str: str | None}}`. + +- [ ] **Step 1: Write the failing tests** + +Create `tests/test_realize.py`: + +```python +"""Realization: a copy of a workflow with every mutable input pinned, so the +file beside a run's manifest reproduces that run whatever changes later.""" + +import copy +import hashlib +import json +import os + +import pytest + +from dw.realize import realize_workflow +from dw.runs import new_run_id +from dw.schema import load_schema, validate_data + + +def definition(): + return { + "id": "realize_test", + "variables": {"prompt": "a default", "steps": 25}, + "steps": [ + { + "name": "gen", + "pipeline": { + "configuration": {"component_type": "{Fake}"}, + "from_pretrained_arguments": {"model_name": "m"}, + "arguments": { + "prompt": "variable:prompt", + "num_inference_steps": "variable:steps", + }, + }, + } + ], + } + + +@pytest.fixture +def prompt_library(tmp_path): + library = tmp_path / "prompts" + (library / "scenic").mkdir(parents=True) + (library / "scenic" / "dusk.json").write_text( + json.dumps({"text": "a harbour at dusk"}) + ) + return str(library) + + +@pytest.fixture +def output_root(tmp_path): + """An output root holding one finished run of 'ltx2/Gyre'.""" + root = tmp_path / "outputs" + run_id = new_run_id({"a": 1}) + run = root / "ltx2" / "Gyre" / run_id + run.mkdir(parents=True) + (run / "still.png").write_bytes(b"not really a png") + return str(root), run_id + + +class TestVariablesAndSeed: + def test_arguments_become_the_variable_defaults(self): + realized, _ = realize_workflow( + definition(), {"prompt": "a cat", "steps": 4}, 7 + ) + assert realized["variables"] == {"prompt": "a cat", "steps": 4} + + def test_variable_references_are_left_alone(self): + realized, _ = realize_workflow(definition(), {"prompt": "a cat"}, 7) + arguments = realized["steps"][0]["pipeline"]["arguments"] + assert arguments["prompt"] == "variable:prompt" + + def test_the_seed_is_written_even_when_the_definition_had_none(self): + realized, _ = realize_workflow(definition(), {}, 991) + assert realized["seed"] == 991 + + def test_the_input_definition_is_not_mutated(self): + original = definition() + before = copy.deepcopy(original) + realize_workflow(original, {"prompt": "a cat"}, 7) + assert original == before + + +class TestPrompts: + def test_a_stored_prompt_is_inlined_and_annotated(self, prompt_library): + source = definition() + source["steps"][0]["pipeline"]["arguments"]["prompt"] = "prompt:scenic/dusk" + + realized, annotations = realize_workflow( + source, {}, 7, prompt_dir=prompt_library + ) + + arguments = realized["steps"][0]["pipeline"]["arguments"] + assert arguments["prompt"] == "a harbour at dusk" + assert annotations["prompts"] == ["scenic/dusk"] + + def test_a_name_is_annotated_once_in_first_seen_order(self, prompt_library): + source = definition() + source["steps"][0]["pipeline"]["arguments"]["prompt"] = "prompt:scenic/dusk" + source["steps"][0]["pipeline"]["arguments"]["negative_prompt"] = ( + "prompt:scenic/dusk" + ) + + _, annotations = realize_workflow(source, {}, 7, prompt_dir=prompt_library) + + assert annotations["prompts"] == ["scenic/dusk"] + + def test_an_unresolvable_prompt_is_left_as_written(self, prompt_library): + source = definition() + source["steps"][0]["pipeline"]["arguments"]["prompt"] = "prompt:missing" + + realized, annotations = realize_workflow( + source, {}, 7, prompt_dir=prompt_library + ) + + assert realized["steps"][0]["pipeline"]["arguments"]["prompt"] == ( + "prompt:missing" + ) + assert annotations["prompts"] == [] + + +class TestOutputReferences: + def test_latest_is_pinned_to_the_run_it_resolved_to(self, output_root): + root, run_id = output_root + source = definition() + source["steps"][0]["pipeline"]["arguments"]["image"] = ( + "output:ltx2/Gyre/latest/still.png" + ) + + realized, _ = realize_workflow(source, {}, 7, output_root=root) + + assert realized["steps"][0]["pipeline"]["arguments"]["image"] == ( + f"output:ltx2/Gyre/{run_id}/still.png" + ) + + def test_an_explicit_run_id_is_kept_as_written(self, output_root): + root, run_id = output_root + written = f"output:ltx2/Gyre/{run_id}/still.png" + source = definition() + source["steps"][0]["pipeline"]["arguments"]["image"] = written + + realized, _ = realize_workflow(source, {}, 7, output_root=root) + + assert realized["steps"][0]["pipeline"]["arguments"]["image"] == written + + def test_an_unresolvable_output_is_left_as_written(self, output_root): + root, _ = output_root + written = "output:ltx2/Nothing/latest/still.png" + source = definition() + source["steps"][0]["pipeline"]["arguments"]["image"] = written + + realized, _ = realize_workflow(source, {}, 7, output_root=root) + + assert realized["steps"][0]["pipeline"]["arguments"]["image"] == written + + +class TestReferencesThatAreKept: + @pytest.mark.parametrize( + "value", + [ + "asset:iris.png", + "constant:diffusers.pipelines.ltx2.utils.DISTILLED_SIGMA_VALUES", + "previous_result:gen", + ], + ) + def test_kept_verbatim(self, value): + source = definition() + source["steps"][0]["pipeline"]["arguments"]["thing"] = value + + realized, _ = realize_workflow(source, {}, 7) + + assert realized["steps"][0]["pipeline"]["arguments"]["thing"] == value + + +class TestSubWorkflows: + def test_a_local_path_is_kept_and_digested(self, tmp_path): + tree = tmp_path / "workflows" + (tree / "steps").mkdir(parents=True) + child = tree / "steps" / "upscale.json" + child.write_text(json.dumps({"id": "child", "steps": []})) + + source = definition() + source["steps"].append( + {"name": "up", "workflow": {"path": "steps/upscale.json"}} + ) + + realized, annotations = realize_workflow( + source, {}, 7, base_dir=str(tree), workflow_dir=str(tree) + ) + + assert realized["steps"][1]["workflow"]["path"] == "steps/upscale.json" + digest = annotations["sub_workflows"]["steps/upscale.json"] + assert digest == hashlib.sha256(child.read_bytes()).hexdigest() + + def test_an_unreadable_sub_workflow_digests_to_null(self, tmp_path): + tree = tmp_path / "workflows" + tree.mkdir() + source = definition() + source["steps"].append({"name": "up", "workflow": {"path": "gone.json"}}) + + _, annotations = realize_workflow( + source, {}, 7, base_dir=str(tree), workflow_dir=str(tree) + ) + + assert annotations["sub_workflows"] == {"gone.json": None} + + def test_a_builtin_is_not_digested(self): + source = definition() + source["steps"].append( + {"name": "up", "workflow": {"path": "builtin:upscale.json"}} + ) + + realized, annotations = realize_workflow(source, {}, 7) + + assert realized["steps"][1]["workflow"]["path"] == "builtin:upscale.json" + assert annotations["sub_workflows"] == {} + + +def test_the_realized_file_validates_against_the_schema(prompt_library): + source = definition() + source["steps"][0]["pipeline"]["arguments"]["prompt"] = "prompt:scenic/dusk" + + realized, _ = realize_workflow( + source, {"steps": 4}, 991, prompt_dir=prompt_library + ) + + ok, message = validate_data(realized, load_schema("workflow")) + assert ok, message +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `source ./activate && python -m pytest tests/test_realize.py -v` +Expected: FAIL — `ModuleNotFoundError: No module named 'dw.realize'`. + +- [ ] **Step 3: Write the module** + +Create `dw/realize.py`: + +```python +"""Realizing a workflow: a copy of a definition with every mutable input +pinned, so the file left beside a run's manifest reproduces that run however +the library, the catalog or the output tree change afterwards. + +What "mutable" means here is precisely the set of things that can differ +between two runs of the same file: the arguments a caller passed, the seed a +seedless workflow drew, the text a stored prompt held at the time, and which +run 'output:.../latest/...' picked. Everything else - 'asset:', 'constant:', +'previous_result:', 'builtin:' and a sub-workflow's path - is a name whose +meaning is pinned by something already recorded (the asset library, the +manifest's dw_version, the file itself), so it is kept as written and, for a +local sub-workflow, digested into the manifest instead. + +Two rules hold this module together. It never mutates its input: the caller +hands it the definition the run is about to work from. And it never fails a +run: a reference that will not resolve is left exactly as written, so the +engine raises its own error at the point it would have raised anyway. +""" + +import copy +import hashlib +import logging +import os + +from .prompts import PROMPT_PREFIX, fetch_prompt +from .runs import ( + LATEST, + OUTPUT_PREFIX, + is_output_reference, + output_root as default_output_root, + resolve_output_reference, +) +from .security import SecurityError, validate_workflow_path +from .variables import set_variables + +logger = logging.getLogger("dw") + +BUILTIN_PREFIX = "builtin:" + + +def realize_workflow( + definition, + arguments, + seed, + base_dir=None, + prompt_dir=None, + output_root=None, + workflow_dir=None, +): + """A copy of `definition` with every mutable input pinned. + + Args: + definition: The workflow as loaded, before Workflow.run's deep copy. + Never mutated. + arguments: The run's argument dict, folded into the variable defaults + exactly as `set_variables` folds them for the run itself. + seed: The seed the run resolved - an integer, never None, because + `Workflow.run` draws one when the workflow names none. + base_dir: The workflow file's directory, anchoring prompt discovery + and a sub-workflow's relative path. + prompt_dir: The prompt library, for `prompt:` inlining. + output_root: The output directory `output:` names resolve against. + workflow_dir: The root a sub-workflow path is confined to, as + `Workflow` confines it; None for an unconfined CLI run. + + Returns: + (realized, annotations) - the pinned copy, and + {"prompts": [name, ...], "sub_workflows": {path: sha256 or None}} + for the manifest to carry, since the schema has nowhere to put them. + """ + annotations = {"prompts": [], "sub_workflows": {}} + realized = copy.deepcopy(definition) + + variables = realized.get("variables") + if isinstance(variables, dict): + # Exactly what the run computed: set_variables coerces each value to + # the type of the declared default and rejects an undeclared name + set_variables(arguments or {}, variables) + + realized["seed"] = seed + realized = _pin(realized, annotations, base_dir, prompt_dir, output_root) + _record_sub_workflows( + realized.get("steps"), annotations, base_dir, workflow_dir + ) + return realized, annotations + + +def _pin(value, annotations, base_dir, prompt_dir, output_root): + """Rebuild a value with prompt and output references pinned. + + One recursive function over dicts, lists and strings - the shape + `referenced_result_names` in dw/step_cache.py walks - so a reference is + found wherever it sits: a pipeline argument, a task argument, a + sub-workflow's argument map, an element of a list. + """ + if isinstance(value, str): + if value.startswith(PROMPT_PREFIX): + return _inline_prompt(value, annotations, prompt_dir, base_dir) + if is_output_reference(value): + return _pin_output(value, output_root) + return value + if isinstance(value, dict): + return { + key: _pin(item, annotations, base_dir, prompt_dir, output_root) + for key, item in value.items() + } + if isinstance(value, list): + return [ + _pin(item, annotations, base_dir, prompt_dir, output_root) + for item in value + ] + return value + + +def _inline_prompt(reference, annotations, prompt_dir, base_dir): + """The stored text, and the name recorded for the manifest. + + A stored prompt's text may not itself begin with a reference prefix (an + engine rule `fetch_prompt` enforces), so inlining cannot introduce a + second resolution. + """ + try: + text = fetch_prompt(reference, prompt_dir, base_dir) + except (SecurityError, OSError, ValueError) as e: + logger.warning(f"Realization kept {reference} as written: {e}") + return reference + name = reference.removeprefix(PROMPT_PREFIX).strip() + if name not in annotations["prompts"]: + annotations["prompts"].append(name) + return text + + +def _pin_output(reference, output_root): + """'output:/latest/' rewritten to the run it resolved to. + + An explicit run id is already pinned, so it is returned untouched without + touching the disk - realizing must not fail on a reference the run has + not reached yet. + """ + name = reference.removeprefix(OUTPUT_PREFIX).strip() + if LATEST not in name.split("/"): + return reference + root = output_root or default_output_root() + try: + resolved = resolve_output_reference(reference, root) + relative = os.path.relpath(resolved, root).replace(os.sep, "/") + except (SecurityError, OSError, ValueError) as e: + logger.warning(f"Realization kept {reference} as written: {e}") + return reference + return f"{OUTPUT_PREFIX}{relative}" + + +def _record_sub_workflows(steps, annotations, base_dir, workflow_dir): + """Digest every sub-workflow a step names by local path. + + The schema's 'workflow' step takes a path, not a definition, so the + realized file keeps the path and the manifest records what the file held. + A builtin is packaged with the engine and pinned by the manifest's + dw_version, so it is not digested. + """ + + def scan(value): + if isinstance(value, dict): + reference = value.get("workflow") + if isinstance(reference, dict): + path = reference.get("path") + if isinstance(path, str) and not path.startswith(BUILTIN_PREFIX): + annotations["sub_workflows"][path] = _digest( + path, base_dir, workflow_dir + ) + for item in value.values(): + scan(item) + elif isinstance(value, list): + for item in value: + scan(item) + + for step in steps or []: + scan(step) + + +def _digest(path, base_dir, workflow_dir): + """The SHA-256 of a sub-workflow file, or None when it cannot be read. + + Resolved the way `Workflow.create_step_action` resolves it - relative to + the referencing file's directory, then through `validate_workflow_path` + confined to `workflow_dir` - so a path this run could not have loaded is + not one realization reads either. + """ + try: + candidate = ( + path + if os.path.isabs(path) + else os.path.normpath(os.path.join(base_dir or ".", path)) + ) + validated = validate_workflow_path(candidate, workflow_dir) + with open(validated, "rb") as file: + return hashlib.sha256(file.read()).hexdigest() + except (SecurityError, OSError, ValueError) as e: + logger.debug(f"No digest for sub-workflow {path}: {e}") + return None +``` + +- [ ] **Step 4: Run the tests to verify they pass** + +Run: `source ./activate && python -m pytest tests/test_realize.py -v` +Expected: PASS (all tests). + +- [ ] **Step 5: Commit** + +```bash +git add dw/realize.py tests/test_realize.py +cat > /tmp/dw-commit-1.txt <<'EOF' +Realization: pin a workflow's mutable inputs into a runnable copy + +realize_workflow folds the run's arguments into the variable defaults, +writes the seed the run resolved, inlines stored prompt text and rewrites +'output:.../latest/...' to the run id it picked. asset:, constant:, +previous_result: and builtin: are kept; a local sub-workflow path is kept +and digested into the annotations the manifest will carry. + +Never mutates its input and never fails a run: a reference that will not +resolve is left exactly as written, so the engine raises its own error. + +Co-Authored-By: Claude Fable 5.1 +EOF +git commit -F /tmp/dw-commit-1.txt +``` + +--- + +### Task 2: Write the realized file from `Workflow.run` (`sonnet`) + +*Model rationale: touches the engine's run loop, `dw/runs.py` and the manifest +shape at once, and the placement inside `run`'s try/finally has to be exactly +right — multi-file integration.* + +**Files:** +- Modify: `dw/runs.py` (add `REALIZED_FILE_NAME`, `write_realized_workflow` + after `write_manifest`) +- Modify: `dw/workflow.py` (imports; `Workflow.run` ~line 405-425; + `_write_run_manifest` ~line 668) +- Test: `tests/test_runs.py` + +**Interfaces:** +- Consumes: `dw.realize.realize_workflow(definition, arguments, seed, + base_dir=None, prompt_dir=None, output_root=None, workflow_dir=None) -> + (dict, dict)` from Task 1. +- Produces: `dw.runs.REALIZED_FILE_NAME == "workflow.json"`; + `dw.runs.write_realized_workflow(run_dir, realized) -> str | None`; a + manifest whose `workflow` block carries `realized`, `prompts` and + `sub_workflows`. + +- [ ] **Step 1: Write the failing tests** + +Append to `tests/test_runs.py` (a new class at the end of the file): + +```python +class TestRealizedWorkflow: + def test_the_run_directory_holds_a_realized_copy(self, tmp_path, fake_pipeline): + from dw.workflow import Workflow + + definition = _workflow_definition() + definition["variables"] = {"prompt": "a default"} + definition["steps"][0]["pipeline"]["arguments"]["prompt"] = "variable:prompt" + Workflow(definition, str(tmp_path), "/w/workflows/ltx2/Gyre.json").run( + {"prompt": "a cat"} + ) + + run_dir = next((tmp_path / "ltx2" / "Gyre").iterdir()) + realized = json.loads((run_dir / "workflow.json").read_text()) + assert realized["variables"] == {"prompt": "a cat"} + assert realized["seed"] == 7 + # the reference stays, so the file is still runnable with overrides + assert realized["steps"][0]["pipeline"]["arguments"]["prompt"] == ( + "variable:prompt" + ) + + def test_the_manifest_points_at_it(self, tmp_path, fake_pipeline): + from dw.workflow import Workflow + + Workflow( + _workflow_definition(), str(tmp_path), "/w/workflows/ltx2/Gyre.json" + ).run({}) + + run_dir = next((tmp_path / "ltx2" / "Gyre").iterdir()) + manifest = json.loads((run_dir / "manifest.json").read_text()) + assert manifest["workflow"]["realized"] == "workflow.json" + assert manifest["workflow"]["prompts"] == [] + assert manifest["workflow"]["sub_workflows"] == {} + + def test_a_seedless_run_pins_the_seed_it_drew(self, tmp_path, fake_pipeline): + from dw.workflow import Workflow + + definition = _workflow_definition() + del definition["seed"] + Workflow(definition, str(tmp_path), "/w/workflows/ltx2/Gyre.json").run({}) + + run_dir = next((tmp_path / "ltx2" / "Gyre").iterdir()) + realized = json.loads((run_dir / "workflow.json").read_text()) + manifest = json.loads((run_dir / "manifest.json").read_text()) + assert isinstance(realized["seed"], int) + assert realized["seed"] == manifest["seed"] + + def test_the_flat_layout_writes_none(self, tmp_path, fake_pipeline, monkeypatch): + from dw.workflow import Workflow + + monkeypatch.setenv(OUTPUT_LAYOUT_ENV_VAR, FLAT_LAYOUT) + Workflow( + _workflow_definition(), str(tmp_path), "/w/workflows/ltx2/Gyre.json" + ).run({}) + + assert not list(tmp_path.rglob("workflow.json")) + + def test_writing_it_is_best_effort(self, tmp_path): + from dw.runs import write_realized_workflow + + # a run directory that cannot be made - the run still succeeded + blocker = tmp_path / "blocker" + blocker.write_text("not a directory") + assert write_realized_workflow(str(blocker / "run"), {"id": "x"}) is None + + def test_writing_it_returns_the_path(self, tmp_path): + from dw.runs import write_realized_workflow + + path = write_realized_workflow(str(tmp_path / "run"), {"id": "x"}) + assert path == str(tmp_path / "run" / "workflow.json") + assert json.loads(open(path).read()) == {"id": "x"} +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `source ./activate && python -m pytest tests/test_runs.py::TestRealizedWorkflow -v` +Expected: FAIL — `ImportError: cannot import name 'write_realized_workflow'` +and `KeyError: 'realized'`. + +- [ ] **Step 3: Add `write_realized_workflow` to `dw/runs.py`** + +Add beside `MANIFEST_FILE_NAME` (after the `MANIFEST_FILE_NAME = "manifest.json"` +line): + +```python +# The workflow a run actually ran, written beside its manifest. Named +# 'workflow.json' rather than something run-specific because the directory +# already says which run it is, and 'python -m dw.run workflow.json' from +# inside it is the whole reproduction story +REALIZED_FILE_NAME = "workflow.json" +``` + +Add after `write_manifest`: + +```python +def write_realized_workflow(run_dir, realized): + """Leave the workflow that produced a run beside what it produced. + + The submitted definition says what was asked for; this says what ran - + arguments folded in, the seed pinned, stored prompts inlined, + 'output:latest' resolved. Best effort, exactly like write_manifest: a run + that produced its files has succeeded whether or not this lands. + """ + path = os.path.join(run_dir, REALIZED_FILE_NAME) + try: + os.makedirs(run_dir, exist_ok=True) + with open(path, "w") as file: + json.dump(realized, file, indent=2, default=str) + except (OSError, TypeError, ValueError) as e: + logger.warning(f"Could not write {path}: {e}") + return None + return path +``` + +- [ ] **Step 4: Call it from `Workflow.run`** + +In `dw/workflow.py`, extend the `.runs` import block (currently +`FLAT_LAYOUT, activate_output_root, ...`) with the two new names, keeping +alphabetical order within the block: + +```python +from .runs import ( + FLAT_LAYOUT, + REALIZED_FILE_NAME, + activate_output_root, + deactivate_output_root, + workflow_identity, + manifest_relative_files, + new_run_id, + output_layout, + run_directory, + write_manifest, + write_realized_workflow, +) +from .realize import realize_workflow +``` + +`REALIZED_FILE_NAME` is imported here because the manifest records the name. + +In `Workflow.run`, beside the other pre-`try` locals (next to +`resolved_seed = None`), add: + +```python + # What the run recorded about itself, read by _write_run_manifest in + # the finally below - initialized here so a failure before the run + # directory exists still writes a well-formed manifest + realized_name = None + annotations = {"prompts": [], "sub_workflows": {}} +``` + +Then, immediately after the `if not self._run_dir_inherited:` block that sets +`self._run_dir` (just before `# Initialize collections for sharing state +between steps`), insert: + +```python + # The record of what actually ran, written before the first step + # so a crash or a cancel still leaves it. A sub-workflow inherits + # the parent's directory and writes none of its own, as with the + # manifest, and the flat layout has no directory to write into + if self._run_dir and not self._run_dir_inherited: + try: + realized, annotations = realize_workflow( + self.workflow_definition, + arguments, + default_seed, + base_dir=base_dir, + output_root=self.output_dir, + workflow_dir=self.workflow_dir, + ) + if write_realized_workflow(self._run_dir, realized): + realized_name = REALIZED_FILE_NAME + except Exception as e: + # Never fatal: the record is worth less than the run + logger.warning( + f"Could not realize workflow {workflow_id}: {e}" + ) +``` + +Note the arguments: `self.workflow_definition` (the original, before `run`'s +deep copy), the local `base_dir` computed earlier in `run`, `default_seed` +(which at this point is `resolved_seed`), and `self.output_dir` as the output +root. `prompt_dir` is left to `fetch_prompt`'s own discovery, which is what the +run itself uses. + +- [ ] **Step 5: Carry the annotations into the manifest** + +In `dw/workflow.py`, change the `finally` block's call: + +```python + if self._run_dir and not self._run_dir_inherited: + self._write_run_manifest( + run_id, + status, + started_at, + arguments, + resolved_seed, + realized_name, + annotations, + ) +``` + +and the method: + +```python + def _write_run_manifest( + self, + run_id, + status, + started_at, + arguments, + seed, + realized_name=None, + annotations=None, + ): +``` + +with its `workflow` block becoming: + +```python + "workflow": { + "id": self.name, + "file": self.file_spec, + "identity": workflow_identity(self.file_spec, self.name), + # The realized copy beside this manifest, or null when + # writing it did not land - the manifest is the only + # place that difference is visible + "realized": realized_name, + # Annotations the schema has nowhere to put: which + # stored prompts were inlined, and what each local + # sub-workflow file held when it ran + "prompts": (annotations or {}).get("prompts", []), + "sub_workflows": (annotations or {}).get("sub_workflows", {}), + }, +``` + +- [ ] **Step 6: Run the tests to verify they pass** + +Run: `source ./activate && python -m pytest tests/test_runs.py tests/test_realize.py tests/test_workflow.py -v` +Expected: PASS. + +- [ ] **Step 7: Commit** + +```bash +git add dw/runs.py dw/workflow.py tests/test_runs.py +cat > /tmp/dw-commit-2.txt <<'EOF' +Every run writes its realized workflow beside its manifest + +Workflow.run realizes the definition once the seed and the run directory +have settled and writes it as workflow.json before the first step, so a +crash or a cancel still leaves it. The manifest's workflow block gains +'realized', 'prompts' and 'sub_workflows'. + +Best effort throughout: a failed realization warns and the run continues. +A sub-workflow inherits the parent's directory and writes none of its own; +the flat layout has no run directory and so writes none either. + +Co-Authored-By: Claude Fable 5.1 +EOF +git commit -F /tmp/dw-commit-2.txt +``` + +--- + +### Task 3: The job knows its run (`sonnet`) + +*Model rationale: one event in the engine, a schema migration and two new +fields in the job store, and a route flag — four files that only make sense +together.* + +**Files:** +- Modify: `dw/workflow.py` (emit `run_start` after the realization block) +- Modify: `dw/server/jobs.py` (imports; migration ~line 77-115; `record`, + `recent_summaries`, `get`, `_to_detail` ~line 118-300; `Job.__init__`, + `summary`, `detail` ~line 306-375; new `JobManager.realized`; the progress + branch of `_consume_results` ~line 800-830) +- Modify: `dw/server/app.py` (`GET /api/jobs/{job_id}/workflow` ~line 834) +- Test: `tests/test_server_jobs.py` (new) + +**Interfaces:** +- Consumes: `dw.runs.REALIZED_FILE_NAME`, `dw.runs.workflow_identity(file_spec, + workflow_id)` from Task 2. +- Produces: a `progress` event `{"event": "run_start", "run_id": str, + "identity": str, "run_dir": str}`; `Job.run_id`, `Job.run_dir`; + `JobManager.realized(job_id) -> dict | None`; `GET + /api/jobs/{id}/workflow` answering `{"id", "definition", "realized": bool}`. + +- [ ] **Step 1: Write the failing tests** + +Create `tests/test_server_jobs.py`: + +```python +"""The job store's record of which run a job was: the run_start event, the +two columns, and reading the realized workflow back off disk. + +The manager tests proper live in tests/test_server.py; this file is the run +record, which is a store concern rather than a routing one. +""" + +import json +import time + +import pytest + +from dw.runs import REALIZED_FILE_NAME, new_run_id +from dw.server.jobs import TERMINAL_STATES, JobHistory, JobManager + +from .test_server import ScriptedWorkerManager, valid_workflow + + +RUN_ID = new_run_id({"workflow": "spec"}) +RUN_DIR = f"server_test/{RUN_ID}" + + +def tracked_script(command): + """A worker that reports its run before doing anything else - what + Workflow.run emits once the run directory is chosen.""" + yield { + "type": "progress", + "event": "run_start", + "run_id": RUN_ID, + "identity": "server_test", + "run_dir": RUN_DIR, + } + yield {"type": "success", "message": "ok", "run_count": 1, "manifest": []} + + +@pytest.fixture +def manager(tmp_path): + made = JobManager( + str(tmp_path / "outputs"), + worker_manager=ScriptedWorkerManager(tracked_script), + history_path=str(tmp_path / "jobs.sqlite"), + workflow_dir=str(tmp_path), + ) + yield made + made.shutdown() + + +def finished_job(manager): + job = manager.submit(workflow=valid_workflow(), base_dir=None) + deadline = time.time() + 5 + while job.status not in TERMINAL_STATES and time.time() < deadline: + time.sleep(0.01) + assert job.status == "succeeded", job.error + return job + + +def test_run_start_populates_the_job(manager): + job = finished_job(manager) + assert job.run_id == RUN_ID + assert job.run_dir == RUN_DIR + assert job.summary()["run_id"] == RUN_ID + assert job.detail()["run_dir"] == RUN_DIR + + +def test_both_persist_and_read_back(manager): + job = finished_job(manager) + historical = manager.history.get(job.id) + assert historical["run_id"] == RUN_ID + assert historical["run_dir"] == RUN_DIR + + +def test_realized_reads_the_file_the_run_wrote(manager, tmp_path): + job = finished_job(manager) + run_dir = tmp_path / "outputs" / "server_test" / RUN_ID + run_dir.mkdir(parents=True) + (run_dir / REALIZED_FILE_NAME).write_text(json.dumps({"id": "realized"})) + + assert manager.realized(job.id) == {"id": "realized"} + + +def test_realized_is_none_without_the_file(manager): + job = finished_job(manager) + assert manager.realized(job.id) is None + + +def test_realized_is_none_for_a_pre_tracking_row(manager): + """A job recorded before run tracking has no run_dir, and the manager + does not guess one from file paths.""" + job = finished_job(manager) + with manager.history._connect() as connection: + connection.execute( + "UPDATE jobs SET run_id = NULL, run_dir = NULL WHERE id = ?", (job.id,) + ) + manager.jobs.pop(job.id) + + assert manager.realized(job.id) is None + + +def test_realized_refuses_a_run_dir_that_escapes_the_output_root(manager, tmp_path): + job = finished_job(manager) + job.run_dir = "../../etc" + assert manager.realized(job.id) is None + + +def test_a_database_without_the_columns_is_migrated(tmp_path): + import sqlite3 + + path = str(tmp_path / "old.sqlite") + # A store written before run tracking: the ALTER is the whole migration + with sqlite3.connect(path) as connection: + connection.execute( + "CREATE TABLE jobs (id TEXT PRIMARY KEY, workflow TEXT, status TEXT," + " created_at REAL, started_at REAL, finished_at REAL, arguments TEXT," + " spec TEXT, manifest TEXT, warnings TEXT, error TEXT)" + ) + connection.execute( + "INSERT INTO jobs (id, status) VALUES ('old-1', 'succeeded')" + ) + + history = JobHistory(path) + row = history.get("old-1") + assert row["run_id"] is None and row["run_dir"] is None +``` + +Also add to `tests/test_server.py`, beside the other workflow-route tests: + +```python +def test_job_workflow_reports_whether_it_is_realized(server): + """A job with no run record falls back to the submitted definition and + says so - the flag is what tells a client which it is looking at.""" + with server(success_script) as client: + submitted = client.post( + "/api/jobs", json={"workflow": valid_workflow(), "arguments": {}} + ).json() + wait_for_status(client, submitted["id"], TERMINAL_STATES) + body = client.get(f"/api/jobs/{submitted['id']}/workflow").json() + + assert body["realized"] is False + assert body["definition"]["id"] == "server_test" +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `source ./activate && python -m pytest tests/test_server_jobs.py "tests/test_server.py::test_job_workflow_reports_whether_it_is_realized" -v` +Expected: FAIL — `AttributeError: 'Job' object has no attribute 'run_id'` and +`KeyError: 'realized'`. + +- [ ] **Step 3: Emit `run_start` from `Workflow.run`** + +In `dw/workflow.py`, immediately after the realization block added in Task 2 +(still inside `if self._run_dir and not self._run_dir_inherited:`, at the same +indentation as the `try:` it contains), add: + +```python + # Which run this is, so a server job can find the directory + # it wrote. Emitted even when the realized file did not land: + # the manifest is still there, and so are the files + run_context.emit( + "run_start", + run_id=run_id, + identity=workflow_identity(self.file_spec, workflow_id), + run_dir=os.path.relpath( + self._run_dir, self.output_dir + ).replace(os.sep, "/"), + ) +``` + +The worker already forwards every emitted event as a `progress` message, so no +worker change is needed. + +- [ ] **Step 4: Add the fields, the columns and `realized()` to `dw/server/jobs.py`** + +Extend the security import block: + +```python +from ..security import ( + SecurityError, + validate_json_size, + validate_output_path, + validate_path, + validate_workflow_path, +) +from ..runs import REALIZED_FILE_NAME +``` + +In `JobHistory.__init__`, after the `workflow_name` migration: + +```python + # Which run of the workflow this job was - the directory under the + # output root that holds its manifest and its realized workflow. + # NULL for every row predating run tracking, and the manager + # refuses to guess one from file paths + if "run_id" not in columns: + connection.execute("ALTER TABLE jobs ADD COLUMN run_id TEXT") + if "run_dir" not in columns: + connection.execute("ALTER TABLE jobs ADD COLUMN run_dir TEXT") +``` + +In `record`, extend the column list, the placeholders and the values tuple: + +```python + "INSERT OR REPLACE INTO jobs (id, workflow, status, created_at," + " started_at, finished_at, arguments, spec, manifest, warnings," + " error, events, workspace, workflow_name, run_id, run_dir) VALUES" + " (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", +``` + +and, after `job.catalog_name` in the tuple: + +```python + job.run_id, + job.run_dir, +``` + +In `recent_summaries`, add `run_id` to the SELECT and the dict, so a live +summary and a historical one keep the same shape: + +```python + query = ( + "SELECT id, workflow, status, created_at, started_at, finished_at," + " workspace, workflow_name, run_id FROM jobs" + ) +``` + +```python + "workflow_name": row[7], + "run_id": row[8], + "historical": True, +``` + +In `get`, extend the SELECT: + +```python + "SELECT id, workflow, status, created_at, started_at, finished_at," + " arguments, spec, manifest, warnings, error, workspace," + " workflow_name, run_id, run_dir FROM jobs WHERE id = ?", +``` + +and in `_to_detail`, after `"workflow_name": row[12],`: + +```python + "run_id": row[13], + "run_dir": row[14], +``` + +In `Job.__init__`, after `self.traceback = None`: + +```python + # Which run this job turned out to be - reported by the worker's + # run_start event, unknown until then and forever for a job that + # never got that far + self.run_id = None + self.run_dir = None +``` + +In `Job.summary`, after the `"workspace"` entry: + +```python + "run_id": self.run_id, +``` + +In `Job.detail`, after `"event_count": len(self.events),`: + +```python + "run_dir": self.run_dir, +``` + +In `JobManager._consume_results`, inside the `if message_type == "progress":` +branch, between building `event` and `job.add_event(event)`: + +```python + if event.get("event") == "run_start": + job.run_id = event.get("run_id") + job.run_dir = event.get("run_dir") +``` + +And add `realized` beside `definition`: + +```python + def realized(self, job_id): + """The realized workflow a job ran, or None when the job predates + run tracking or its run directory no longer holds the file. + + Read from the job's own output directory, not the manager's: one + server holds several workspaces, and a job carries the root it ran + against. The join is confined to that root, so a run_dir read back + out of the database cannot name anything outside it. + """ + job = self.jobs.get(job_id) + if job is not None: + run_dir = job.run_dir + output_dir = job.spec.get("output_dir") or self.output_dir + else: + historical = self.history.get(job_id) + if historical is None: + return None + run_dir = historical.get("run_dir") + output_dir = ( + historical.get("spec") or {} + ).get("output_dir") or self.output_dir + if not run_dir: + return None + try: + root = validate_output_path(output_dir, None) + path = validate_path( + os.path.join(root, run_dir, REALIZED_FILE_NAME), root + ) + validate_json_size(path) + with open(path, "r") as file: + return json.load(file) + except (SecurityError, OSError, ValueError) as e: + logger.debug(f"No realized workflow for job {job_id}: {e}") + return None +``` + +- [ ] **Step 5: Add the flag to the route** + +In `dw/server/app.py`, replace the body of `get_job_workflow`: + +```python + @app.get("/api/jobs/{job_id}/workflow") + def get_job_workflow(job_id: str): + """The workflow this job ran, for the read-only graph on the job page + and for `get_job_workflow` over MCP. + + `realized: true` means every mutable input is pinned - the copy the + run itself wrote. `false` means the job predates run tracking (or its + run directory is gone) and this is the definition as submitted. 404 + when neither is readable - the job itself still is.""" + if manager.get(job_id) is None: + raise HTTPException(status_code=404, detail="Unknown job") + realized = manager.realized(job_id) + definition = realized if realized is not None else manager.definition(job_id) + if definition is None: + raise HTTPException( + status_code=404, detail="No workflow definition for this job" + ) + return { + "id": job_id, + "definition": definition, + "realized": realized is not None, + } +``` + +- [ ] **Step 6: Run the tests to verify they pass** + +Run: `source ./activate && python -m pytest tests/test_server_jobs.py tests/test_server.py tests/test_server_workspaces.py -v` +Expected: PASS. + +- [ ] **Step 7: Commit** + +```bash +git add dw/workflow.py dw/server/jobs.py dw/server/app.py tests/test_server_jobs.py tests/test_server.py +cat > /tmp/dw-commit-3.txt <<'EOF' +A job knows which run it was, and can read that run's workflow + +Workflow.run emits a run_start event carrying the run id, the workflow +identity and the run directory relative to the output root. The Job records +both, jobs.sqlite gains run_id and run_dir (ALTER when absent, NULL for +older rows), and JobManager.realized reads workflow.json back out of the +job's own output root, confined to it. + +GET /api/jobs/{id}/workflow now answers {id, definition, realized}: the +realized copy when there is one, the submitted definition otherwise. + +Co-Authored-By: Claude Fable 5.1 +EOF +git commit -F /tmp/dw-commit-3.txt +``` + +--- + +### Task 4: `get_job_workflow` MCP tool (`haiku`) + +*Model rationale: one handler, one registration, one test file — mechanical, +with the complete code below.* + +**Files:** +- Modify: `dw_mcp/diagnose.py` (after `get_job`) +- Modify: `dw_mcp/server.py` (the diagnose group, ~line 605-660) +- Test: `tests/test_mcp_diagnose.py` + +**Interfaces:** +- Consumes: `GET /api/jobs/{id}/workflow` answering `{"id", "definition", + "realized": bool}` from Task 3; `dw_mcp.client.api_path(*segments)`; + `DwClient.get_json(path, params=None)`. +- Produces: `dw_mcp.diagnose.get_job_workflow(client, job_id) -> {"job_id", + "realized", "workflow", "next"}`, registered as the `get_job_workflow` MCP + tool with `READ_ONLY` annotations. + +- [ ] **Step 1: Write the failing tests** + +Append to `tests/test_mcp_diagnose.py`: + +```python +class TestGetJobWorkflow: + def test_a_realized_workflow_comes_back_with_the_flag_set(self): + client, seen = scripted( + { + ("GET", "/api/jobs/job-1/workflow"): ( + 200, + {"id": "job-1", "definition": WORKFLOW, "realized": True}, + ) + } + ) + + result = diagnose.get_job_workflow(client, "job-1") + + assert result["job_id"] == "job-1" + assert result["realized"] is True + assert result["workflow"] == WORKFLOW + assert len(seen) == 1 + + def test_a_pre_tracking_job_reports_the_submitted_definition(self): + client, _ = scripted( + { + ("GET", "/api/jobs/job-1/workflow"): ( + 200, + {"id": "job-1", "definition": WORKFLOW, "realized": False}, + ) + } + ) + + result = diagnose.get_job_workflow(client, "job-1") + + assert result["realized"] is False + assert result["workflow"] == WORKFLOW + + def test_next_names_the_two_tools_that_use_it(self): + client, _ = scripted( + { + ("GET", "/api/jobs/job-1/workflow"): ( + 200, + {"id": "job-1", "definition": WORKFLOW, "realized": True}, + ) + } + ) + + result = diagnose.get_job_workflow(client, "job-1") + + assert "save_workflow" in result["next"] + assert "run_workflow" in result["next"] + + def test_an_unknown_job_raises_the_client_error(self): + client, _ = scripted({}) + + with pytest.raises(DwApiError): + diagnose.get_job_workflow(client, "nope") +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `source ./activate && python -m pytest tests/test_mcp_diagnose.py::TestGetJobWorkflow -v` +Expected: FAIL — `AttributeError: module 'dw_mcp.diagnose' has no attribute 'get_job_workflow'`. + +- [ ] **Step 3: Write the handler** + +In `dw_mcp/diagnose.py`, after `get_job`: + +```python +def get_job_workflow(client, job_id): + """The workflow a job ran. `realized: true` means every mutable input + is pinned (arguments, seed, prompts, output:latest); false means the + job predates run tracking and this is the definition as submitted. + Pass it to save_workflow to rerun it by name, or edit it and pass it + to run_workflow as inline_workflow.""" + body = client.get_json(api_path("api", "jobs", job_id, "workflow")) + return { + "job_id": job_id, + "realized": bool(body.get("realized")), + "workflow": body.get("definition"), + "next": "Pass `workflow` to save_workflow to keep it in the catalog " + "under a name, or edit it and pass it to run_workflow as " + "inline_workflow.", + } +``` + +- [ ] **Step 4: Register the tool** + +In `dw_mcp/server.py`, in the diagnose group, after the `get_job` definition: + +```python + def get_job_workflow(job_id: str) -> dict: + """Get the workflow a job actually ran. When `realized` is true every + mutable input is pinned - the caller's arguments folded into the + variables, the seed the run used, stored prompt text inlined, and any + `output:.../latest/...` rewritten to the run it resolved to - so the + definition reproduces that run however the library changes. When it is + false the job predates run tracking and this is the definition as + submitted. After a long inline run worth keeping, this then + `save_workflow` is how it gets a name.""" + return diagnose.get_job_workflow(client, job_id) +``` + +and add it to the read-only registrations: + +```python + tool(get_job, READ_ONLY) + tool(get_job_workflow, READ_ONLY) + tool(get_job_events, READ_ONLY) +``` + +- [ ] **Step 5: Run the tests to verify they pass** + +Run: `source ./activate && python -m pytest tests/test_mcp_diagnose.py tests/test_mcp_server.py -v` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add dw_mcp/diagnose.py dw_mcp/server.py tests/test_mcp_diagnose.py +cat > /tmp/dw-commit-4.txt <<'EOF' +MCP: get_job_workflow returns the workflow a job ran + +A read-only tool over GET /api/jobs/{id}/workflow. 'realized: true' means +every mutable input is pinned; false means the job predates run tracking. +The 'next' sentence points at save_workflow and run_workflow, which is how +an inline run worth keeping gets a name. + +Co-Authored-By: Claude Fable 5.1 +EOF +git commit -F /tmp/dw-commit-4.txt +``` + +--- + +### Task 5: Export a job (`opus`) + +*Model rationale: the largest and most judgement-heavy piece — a new module +with a generated README, two routes with distinct failure codes, a reserved +workspace name that ripples through workspace listing, and copy semantics that +have to hold for both a run-directory job and a pre-tracking one.* + +**Files:** +- Create: `dw/server/exports.py` +- Modify: `dw/workspace.py` (`EXPORTS_SUBDIR`, `RESERVED_WORKSPACE_NAMES`, + `_foreign_entries`) +- Modify: `dw/server/app.py` (import `exports`; the export route beside + `rerun_job` ~line 846; the zip route beside `/outputs` ~line 2252) +- Test: `tests/test_server_exports.py` (new) + +**Interfaces:** +- Consumes: `JobManager.get(job_id)` (a live `Job` or a historical detail + dict), `JobManager.describe(job)`, `JobManager.definition(job_id)`, + `JobManager.realized(job_id)` from Task 3; `dw.runs.MANIFEST_FILE_NAME`, + `dw.runs.REALIZED_FILE_NAME`, `dw.runs.OUTPUT_PREFIX`, + `dw.runs.resolve_output_reference(reference, root)`; + `dw.assets.ASSET_PREFIX`; `dw.security.validate_asset_reference`, + `validate_output_path`, `validate_path`; `_asset_roots(ws)` and + `_served_url(path, ws)` in `dw/server/app.py`. +- Produces: `dw.workspace.EXPORTS_SUBDIR == "exports"`; + `dw.server.exports.ExportSummary` (dataclass with `job_id`, `directory`, + `files`, `total_bytes`, `missing`, and `as_dict()`); + `dw.server.exports.export_directory(workspace_root, job_id) -> str`; + `dw.server.exports.export_job(manager, job_id, workspace_root, asset_roots, + overwrite=False) -> ExportSummary`; + `POST /api/jobs/{job_id}/export` and `GET /exports/{job_id}.zip`. + +- [ ] **Step 1: Write the failing tests** + +Create `tests/test_server_exports.py`: + +```python +"""Exporting one finished job: a directory that stands on its own, and the +same tree as a zip.""" + +import io +import json +import os +import zipfile + +import pytest +from fastapi.testclient import TestClient + +from dw.runs import REALIZED_FILE_NAME, new_run_id +from dw.server.app import create_app +from dw.server.jobs import JobManager, TERMINAL_STATES +from dw.workspace import Workspace + +from .test_server import ( + ScriptedWorkerManager, + hanging_script, + valid_workflow, + wait_for_status, +) + +RUN_ID = new_run_id({"workflow": "export"}) +RUN_DIR = f"server_test/{RUN_ID}" + + +def exporting_script(command): + """A run that reports its directory and writes one file.""" + output_dir = command["output_dir"] + run_dir = os.path.join(output_dir, "server_test", RUN_ID) + os.makedirs(run_dir, exist_ok=True) + with open(os.path.join(run_dir, "still.png"), "wb") as file: + file.write(b"an image") + with open(os.path.join(run_dir, REALIZED_FILE_NAME), "w") as file: + json.dump( + { + "id": "server_test", + "seed": 7, + "steps": [ + { + "name": "gen", + "pipeline": { + "configuration": {"component_type": "{Fake}"}, + "from_pretrained_arguments": {"model_name": "m"}, + "arguments": {"image": "asset:iris.png"}, + }, + } + ], + }, + file, + ) + with open(os.path.join(run_dir, "manifest.json"), "w") as file: + json.dump( + { + "run_id": RUN_ID, + "status": "completed", + "seed": 7, + "steps": [{"step": "gen", "files": ["still.png"]}], + }, + file, + ) + yield { + "type": "progress", + "event": "run_start", + "run_id": RUN_ID, + "identity": "server_test", + "run_dir": RUN_DIR, + } + yield { + "type": "success", + "message": "ok", + "run_count": 1, + "manifest": [ + {"step": "gen", "files": [os.path.join(run_dir, "still.png")]} + ], + } + + +@pytest.fixture +def workspace_root(tmp_path): + root = Workspace(tmp_path / "studio", "flag").ensure() + with open(os.path.join(root.assets, "iris.png"), "wb") as file: + file.write(b"an iris") + return root + + +@pytest.fixture +def server(workspace_root, tmp_path): + def make(script=exporting_script): + manager = JobManager( + workspace_root.outputs, + worker_manager=ScriptedWorkerManager(script), + history_path=str(tmp_path / "jobs.sqlite"), + workflow_dir=workspace_root.workflows, + ) + app = create_app( + workflow_dir=workspace_root.workflows, + output_dir=workspace_root.outputs, + job_manager=manager, + prompt_dir=workspace_root.prompts, + asset_dir=workspace_root.assets, + workspace=workspace_root.root, + ) + return TestClient(app, base_url="http://localhost") + + return make + + +def finished(client): + submitted = client.post( + "/api/jobs", json={"workflow": valid_workflow(), "arguments": {}} + ).json() + wait_for_status(client, submitted["id"], TERMINAL_STATES) + return submitted["id"] + + +class TestExportDirectory: + def test_it_gathers_the_whole_run(self, server, workspace_root): + with server() as client: + job_id = finished(client) + response = client.post(f"/api/jobs/{job_id}/export") + + assert response.status_code == 201 + body = response.json() + directory = body["directory"] + assert directory == os.path.join(workspace_root.root, "exports", job_id) + for name in ("README.md", "workflow.json", "manifest.json", "job.json"): + assert os.path.isfile(os.path.join(directory, name)) + assert os.path.isfile(os.path.join(directory, "assets", "iris.png")) + assert os.path.isfile(os.path.join(directory, "outputs", "still.png")) + assert body["total_bytes"] > 0 + assert body["missing"] == [] + assert body["zip_url"] == f"/exports/{job_id}.zip" + + def test_the_workflow_is_the_realized_one(self, server): + with server() as client: + job_id = finished(client) + body = client.post(f"/api/jobs/{job_id}/export").json() + + recorded = json.loads( + open(os.path.join(body["directory"], "job.json")).read() + ) + assert recorded["realized"] is True + assert "traceback" not in recorded and "event_count" not in recorded + assert body["workflow"]["seed"] == 7 + + def test_the_readme_names_the_job_and_says_how_to_run_it(self, server): + with server() as client: + job_id = finished(client) + body = client.post(f"/api/jobs/{job_id}/export").json() + + readme = open(os.path.join(body["directory"], "README.md")).read() + assert job_id in readme + assert "python -m dw.run workflow.json" in readme + assert "Git LFS" in readme + + def test_an_unresolvable_asset_is_reported_missing(self, server, workspace_root): + os.unlink(os.path.join(workspace_root.assets, "iris.png")) + with server() as client: + job_id = finished(client) + body = client.post(f"/api/jobs/{job_id}/export").json() + + assert body["missing"] == ["asset:iris.png"] + + def test_an_unknown_job_is_404(self, server): + with server() as client: + assert client.post("/api/jobs/nope/export").status_code == 404 + + def test_a_live_job_is_409(self, server): + with server(hanging_script) as client: + submitted = client.post( + "/api/jobs", json={"workflow": valid_workflow(), "arguments": {}} + ).json() + wait_for_status(client, submitted["id"], ("running",)) + response = client.post(f"/api/jobs/{submitted['id']}/export") + client.post(f"/api/jobs/{submitted['id']}/cancel") + + assert response.status_code == 409 + + def test_a_second_export_without_overwrite_is_409(self, server): + with server() as client: + job_id = finished(client) + assert client.post(f"/api/jobs/{job_id}/export").status_code == 201 + again = client.post(f"/api/jobs/{job_id}/export") + assert again.status_code == 409 + forced = client.post(f"/api/jobs/{job_id}/export?overwrite=true") + assert forced.status_code == 201 + + +class TestExportZip: + def test_it_lists_the_same_entries_as_the_directory(self, server): + with server() as client: + job_id = finished(client) + body = client.post(f"/api/jobs/{job_id}/export").json() + response = client.get(f"/exports/{job_id}.zip") + + assert response.status_code == 200 + archive = zipfile.ZipFile(io.BytesIO(response.content)) + assert sorted(archive.namelist()) == sorted( + f"{job_id}/{entry['path']}" for entry in body["files"] + ) + + def test_no_export_is_404(self, server): + with server() as client: + job_id = finished(client) + assert client.get(f"/exports/{job_id}.zip").status_code == 404 + + +class TestReservedName: + def test_exports_cannot_name_a_workspace(self, server): + with server() as client: + response = client.post("/api/workspaces", json={"name": "exports"}) + assert response.status_code == 400 + assert "cannot name a workspace" in response.json()["detail"] + + def test_an_exports_folder_is_not_listed_as_a_workspace( + self, server, workspace_root + ): + with server() as client: + job_id = finished(client) + client.post(f"/api/jobs/{job_id}/export") + names = [w["name"] for w in client.get("/api/workspaces").json()["workspaces"]] + assert names == ["default"] +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `source ./activate && python -m pytest tests/test_server_exports.py -v` +Expected: FAIL — `assert 404 == 201` for the export route (it does not exist). + +- [ ] **Step 3: Reserve the name in `dw/workspace.py`** + +Beside the other subdir constants (after `SUBDIRS = (...)`): + +```python +# Where a job export lands: '/exports//'. Not a workspace +# folder - it is beside them, holding gathered copies rather than working +# content - but it is a name a workspace may not take, and workspace_names +# must not mistake it for one +EXPORTS_SUBDIR = "exports" +``` + +and: + +```python +RESERVED_WORKSPACE_NAMES = SUBDIRS + (EXPORTS_SUBDIR,) +``` + +and in `_foreign_entries`: + +```python +def _foreign_entries(path): + """What a directory holds besides a workspace's own three folders and + the exports it may have gathered.""" + ignored = NAMED_SUBDIRS + (EXPORTS_SUBDIR,) + try: + return sorted(entry for entry in os.listdir(path) if entry not in ignored) + except OSError: + return [] +``` + +`workspace_names` already skips `RESERVED_WORKSPACE_NAMES`, so an `exports/` +folder at the root stops being a candidate workspace as soon as the constant +lands. + +- [ ] **Step 4: Write `dw/server/exports.py`** + +```python +"""Gathering one finished job into a directory that stands on its own. + +The record of a run is split across four places on one machine: the job row, +the run's manifest, the workflow that ran, and the media on either side of it. +This module puts them in one tree - text at the top, media in folders, no +absolute paths inside the files - so the run can be committed, moved or handed +to someone else. + + exports// + README.md what this is, how it was made, how to run it again + workflow.json the realized workflow, when the run wrote one + manifest.json the run's manifest, or a synthesized stand-in + job.json the job row: status, times, arguments, warnings, error + assets/ every asset: the workflow names, under its own name + inputs/ every file an output: reference named, by run + outputs/ every file the manifest lists + +Copies are copies, never hard links: exports/ is what a user moves or deletes, +and a hard link would make deleting it look like deleting the original. +""" + +import json +import logging +import os +import shutil +from dataclasses import dataclass, field + +from ..assets import ASSET_PREFIX +from ..runs import ( + MANIFEST_FILE_NAME, + OUTPUT_PREFIX, + is_output_reference, + resolve_output_reference, +) +from ..security import ( + SecurityError, + validate_asset_reference, + validate_output_path, + validate_path, +) +from ..workspace import EXPORTS_SUBDIR +from .jobs import TERMINAL_STATES + +logger = logging.getLogger("dw") + +WORKFLOW_FILE_NAME = "workflow.json" +JOB_FILE_NAME = "job.json" +README_FILE_NAME = "README.md" + +# The job detail keys an export does not carry: a traceback is a developer's +# artifact of one process, and an event count describes a log this tree does +# not hold +OMITTED_JOB_KEYS = ("traceback", "event_count") + +README_TEMPLATE = """# {workflow_name} - job {job_id} + +{realized_sentence} + +## The run + +| | | +| --- | --- | +| Job | `{job_id}` | +| Workflow | `{workflow_name}` | +| Catalog entry | {catalog_name} | +| Status | {status} | +| Started | {started_at} | +| Finished | {finished_at} | +| Device | {device} | +| Engine | dw {dw_version} | +| Seed | `{seed}` | + +Arguments as submitted: + +```json +{arguments} +``` + +## Stored prompts + +{prompts} + +## Sub-workflows + +{sub_workflows} + +## Inputs + +{inputs} + +## Running it again + +From a checkout of diffusers-workflow, with this directory as the working +directory: + +```bash +python -m dw.run workflow.json +``` + +Over MCP, pass the contents of `workflow.json` to `run_workflow` as +`inline_workflow`. Either way the `asset:` and `output:` names in it resolve +against the server's libraries, not against this directory - the copies under +`assets/` and `inputs/` are the record of what those names meant, not a +substitute for them. + +## Missing + +{missing} + +## A note on committing this + +`assets/`, `inputs/` and `outputs/` hold generated and source media, which is +usually large and always binary. If this tree goes into Git, put those three +directories in Git LFS; the four files at the top are text and belong in Git +proper. +""" + + +@dataclass +class ExportSummary: + """What an export produced, computed from what actually landed.""" + + job_id: str + directory: str + files: list = field(default_factory=list) + total_bytes: int = 0 + missing: list = field(default_factory=list) + + def as_dict(self): + return { + "job_id": self.job_id, + "directory": self.directory, + "files": self.files, + "total_bytes": self.total_bytes, + "missing": self.missing, + } + + +def export_directory(workspace_root, job_id): + """Where one job's export lives, confined to the workspace root. + + The job id is caller input - it arrives in a URL - so it is joined and + then validated rather than trusted to be the hex string the manager + generates. + """ + root = validate_output_path(os.path.join(workspace_root, EXPORTS_SUBDIR), None) + return validate_path(os.path.join(root, job_id), root) + + +def export_job(manager, job_id, workspace_root, asset_roots, overwrite=False): + """Gather one finished job into '/exports//'. + + Args: + manager: The JobManager holding the job (live or historical). + job_id: The job to export. + workspace_root: The workspace the export lands in. + asset_roots: The workspace's asset search path, in order - the same + order 'asset:' resolves in, so what is copied is what the run + loaded. + overwrite: Replace an existing export rather than refusing. + + Returns: + An ExportSummary. + + Raises: + ValueError: Unknown job, or a job that has not finished. + FileExistsError: The export exists and overwrite is False. + """ + job = manager.get(job_id) + if job is None: + raise ValueError(f"Unknown job {job_id}") + detail = job if isinstance(job, dict) else manager.describe(job) + status = detail.get("status") + if status not in TERMINAL_STATES: + raise ValueError( + f"Job {job_id} is {status} - only a finished job can be exported" + ) + + spec = job.get("spec", {}) if isinstance(job, dict) else job.spec + run_dir = job.get("run_dir") if isinstance(job, dict) else job.run_dir + output_root = validate_output_path( + spec.get("output_dir") or manager.output_dir, None + ) + + target = export_directory(workspace_root, job_id) + if os.path.exists(target): + if not overwrite: + raise FileExistsError( + f"An export of job {job_id} already exists - pass overwrite to " + f"replace it" + ) + shutil.rmtree(target) + os.makedirs(target) + + summary = ExportSummary(job_id=job_id, directory=target) + + realized = manager.realized(job_id) + workflow = realized if realized is not None else (manager.definition(job_id) or {}) + _write_json(summary, target, WORKFLOW_FILE_NAME, workflow) + + manifest = _run_manifest(output_root, run_dir) + if manifest is None: + # No run directory, or it no longer holds a manifest: the job row's + # own per-step file list is what is left, and it says so + manifest = { + "synthesized": True, + "run_id": None, + "status": status, + "steps": detail.get("manifest") or [], + } + _write_json(summary, target, MANIFEST_FILE_NAME, manifest) + + record = { + key: value for key, value in detail.items() if key not in OMITTED_JOB_KEYS + } + record["realized"] = realized is not None + _write_json(summary, target, JOB_FILE_NAME, record) + + _copy_assets(summary, workflow, target, asset_roots) + _copy_inputs(summary, workflow, target, output_root) + _copy_outputs(summary, manifest, target, output_root, run_dir) + + readme = _readme( + job_id, detail, manifest, workflow, summary, realized is not None + ) + _write_text(summary, target, README_FILE_NAME, readme) + return summary + + +# -------------------------------------------------------------- the pieces + + +def _run_manifest(output_root, run_dir): + """The manifest the run left, or None when there is no reading it.""" + if not run_dir: + return None + try: + path = validate_path( + os.path.join(output_root, run_dir, MANIFEST_FILE_NAME), output_root + ) + with open(path, "r") as file: + return json.load(file) + except (SecurityError, OSError, ValueError) as e: + logger.debug(f"No run manifest to export from {run_dir}: {e}") + return None + + +def _references(value, prefix): + """Every string under `value` that begins with `prefix`, deduplicated. + + The same recursive shape realization walks, so a reference is found + wherever it sits in the tree. + """ + found = set() + + def scan(item): + if isinstance(item, str): + if item.startswith(prefix): + found.add(item) + elif isinstance(item, dict): + for child in item.values(): + scan(child) + elif isinstance(item, list): + for child in item: + scan(child) + + scan(value) + return sorted(found) + + +def _copy_assets(summary, workflow, target, asset_roots): + """Every 'asset:' the workflow names, under its own name in assets/.""" + for reference in _references(workflow, ASSET_PREFIX): + try: + name = validate_asset_reference( + reference.removeprefix(ASSET_PREFIX).strip() + ) + except SecurityError: + summary.missing.append(reference) + continue + source = None + for root in asset_roots: + try: + candidate = validate_path(os.path.join(root, name), root) + except SecurityError: + continue + if os.path.isfile(candidate): + source = candidate + break + if source is None: + summary.missing.append(reference) + continue + _copy(summary, source, target, os.path.join("assets", *name.split("/"))) + + +def _copy_inputs(summary, workflow, target, output_root): + """Every 'output:' the workflow names, kept under the run it came from. + + The reference itself is not rewritten - the realized workflow is the + immutable record of the run - so the directory name is the reference's + own name, and the README says where each one came from. + """ + for reference in _references(workflow, OUTPUT_PREFIX): + if not is_output_reference(reference): + continue + name = reference.removeprefix(OUTPUT_PREFIX).strip() + try: + source = resolve_output_reference(reference, output_root) + except (SecurityError, OSError, ValueError): + summary.missing.append(reference) + continue + _copy(summary, source, target, os.path.join("inputs", *name.split("/"))) + + +def _copy_outputs(summary, manifest, target, output_root, run_dir): + """Every file the manifest lists, under outputs/. + + A manifest entry names a file relative to the run directory when the run + wrote it, and absolutely when a step-cache hit republished an earlier + run's file. Both land here: the first under its own relative name, the + second under its path relative to the output root, which keeps the + identity and run id that say where it really came from. + """ + run_root = os.path.join(output_root, run_dir) if run_dir else output_root + for entry in manifest.get("steps") or []: + if not isinstance(entry, dict): + continue + for recorded in entry.get("files") or []: + source = ( + recorded + if os.path.isabs(recorded) + else os.path.join(run_root, recorded) + ) + try: + source = validate_path(source, output_root) + except SecurityError: + summary.missing.append(recorded) + continue + if not os.path.isfile(source): + summary.missing.append(recorded) + continue + try: + relative = os.path.relpath(source, run_root) + except ValueError: # different drive on Windows + relative = os.path.basename(source) + if relative.startswith(os.pardir): + relative = os.path.relpath(source, output_root) + _copy( + summary, + source, + target, + os.path.join("outputs", *relative.split(os.sep)), + ) + + +def _readme(job_id, detail, manifest, workflow, summary, realized): + workflow_block = manifest.get("workflow") or {} + prompts = workflow_block.get("prompts") or [] + sub_workflows = workflow_block.get("sub_workflows") or {} + inputs = [ + entry["path"] + for entry in summary.files + if entry["path"].startswith("inputs/") + ] + return README_TEMPLATE.format( + job_id=job_id, + workflow_name=detail.get("workflow") or "unknown", + catalog_name=f"`{detail['workflow_name']}`" + if detail.get("workflow_name") + else "none - an inline definition", + realized_sentence=( + "`workflow.json` is the *realized* workflow: every mutable input " + "is pinned, so it reproduces this run whatever changes afterwards." + if realized + else "`workflow.json` is the definition as submitted - this job " + "predates run tracking, so its arguments and prompts are not " + "pinned into it." + ), + status=detail.get("status"), + started_at=detail.get("started_at"), + finished_at=detail.get("finished_at"), + device=manifest.get("device", "unknown"), + dw_version=manifest.get("dw_version", "unknown"), + seed=manifest.get("seed", workflow.get("seed", "not recorded")), + arguments=json.dumps(detail.get("arguments") or {}, indent=2), + prompts=_bullets( + f"`{name}` - inlined into `workflow.json`" for name in prompts + ) + or "None: this workflow named no stored prompt.", + sub_workflows=_bullets( + f"`{path}` - sha256 `{digest}`" if digest else f"`{path}` - unreadable" + for path, digest in sorted(sub_workflows.items()) + ) + or "None: this workflow composed no other workflow by path.", + inputs=_bullets( + f"`{path}` - copied from the run named in its own path" + for path in inputs + ) + or "None: this workflow named no file from an earlier run.", + missing=_bullets(f"`{name}`" for name in summary.missing) + or "Nothing: every file this run referenced was found and copied.", + ) + + +def _bullets(lines): + rendered = "\n".join(f"- {line}" for line in lines) + return rendered + + +# ------------------------------------------------------------------- files + + +def _copy(summary, source, target, relative): + """Copy one file into the export and record it.""" + destination = os.path.join(target, relative) + try: + os.makedirs(os.path.dirname(destination), exist_ok=True) + shutil.copyfile(source, destination) + except OSError as e: + logger.warning(f"Could not copy {source} into the export: {e}") + summary.missing.append(source) + return + _record(summary, destination, target) + + +def _write_json(summary, target, name, payload): + _write_text(summary, target, name, json.dumps(payload, indent=2, default=str)) + + +def _write_text(summary, target, name, text): + path = os.path.join(target, name) + try: + with open(path, "w", encoding="utf-8") as file: + file.write(text) + except OSError as e: + logger.warning(f"Could not write {path}: {e}") + return + _record(summary, path, target) + + +def _record(summary, path, target): + try: + size = os.path.getsize(path) + except OSError: + return + summary.files.append( + {"path": os.path.relpath(path, target).replace(os.sep, "/"), "bytes": size} + ) + summary.total_bytes += size +``` + +- [ ] **Step 5: Add the routes to `dw/server/app.py`** + +Add to the imports, beside `from .enhancers import ...`: + +```python +from .exports import export_directory, export_job +``` + +Add after the `rerun_job` route: + +```python + @app.post("/api/jobs/{job_id}/export", status_code=201) + def export_job_route( + job_id: str, + overwrite: bool = False, + ws: Workspace = Depends(selected_workspace), + ): + """Gather one finished job into '/exports//': the + workflow it ran, the run's manifest, the job row, the media it used + and the media it made, plus a README. 404 for an unknown job, 409 for + one still running or for an export that already exists without + `overwrite`. + + The three JSON files come back inline as well as on disk - the + directory is on the server, and a client on another machine has no + other way to read them without fetching the zip.""" + try: + summary = export_job( + manager, job_id, ws.root, _asset_roots(ws), overwrite=overwrite + ) + except FileExistsError as e: + raise HTTPException(status_code=409, detail=str(e)) + except ValueError as e: + message = str(e) + if message.startswith("Unknown job"): + raise HTTPException(status_code=404, detail=message) + raise HTTPException(status_code=409, detail=message) + body = summary.as_dict() + body["zip_url"] = _served_url(f"/exports/{quote(job_id)}.zip", ws) + for key, name in ( + ("workflow", "workflow.json"), + ("manifest", "manifest.json"), + ("job", "job.json"), + ): + try: + with open(os.path.join(summary.directory, name), "r") as file: + body[key] = json.load(file) + except (OSError, ValueError): + body[key] = None + return body +``` + +Add beside the `/outputs` route (ungated for the same reason it is: the auth +middleware only gates `/api/`): + +```python + @app.get("/exports/{job_id}.zip") + def export_zip(job_id: str, ws: Workspace = Depends(selected_workspace)): + """One job's export as a zip, built on request from the directory + rather than kept as a second copy. Entries are named + '/', so unzipping anywhere gives the same tree + the server holds.""" + try: + directory = export_directory(ws.root, job_id) + except SecurityError: + raise HTTPException(status_code=404, detail="No export for this job") + if not os.path.isdir(directory): + raise HTTPException(status_code=404, detail="No export for this job") + + handle = tempfile.NamedTemporaryFile(suffix=".zip", delete=False) + handle.close() + with zipfile.ZipFile(handle.name, "w", zipfile.ZIP_DEFLATED) as archive: + for current, _dirs, names in os.walk(directory): + for name in sorted(names): + path = os.path.join(current, name) + entry = os.path.relpath(path, directory).replace(os.sep, "/") + archive.write(path, f"{job_id}/{entry}") + + def stream(): + with open(handle.name, "rb") as file: + while True: + chunk = file.read(64 * 1024) + if not chunk: + return + yield chunk + + return StreamingResponse( + stream(), + media_type="application/zip", + headers={ + "content-disposition": f'attachment; filename="{job_id}.zip"' + }, + # The archive is a temp file, not a second permanent copy - it + # goes as soon as the response has been sent + background=BackgroundTask(os.unlink, handle.name), + ) +``` + +- [ ] **Step 6: Run the tests to verify they pass** + +Run: `source ./activate && python -m pytest tests/test_server_exports.py tests/test_server_workspaces.py tests/test_workspace.py -v` +Expected: PASS. + +- [ ] **Step 7: Run the whole server and engine suite for regressions** + +Run: `source ./activate && python -m pytest tests/test_server.py tests/test_server_jobs.py tests/test_runs.py tests/test_realize.py -v` +Expected: PASS. + +- [ ] **Step 8: Commit** + +```bash +git add dw/server/exports.py dw/workspace.py dw/server/app.py tests/test_server_exports.py +cat > /tmp/dw-commit-5.txt <<'EOF' +Export one finished job as a git-ready directory and a zip + +POST /api/jobs/{id}/export gathers the realized workflow, the run's +manifest, the job row, every asset: and output: file the workflow named and +every file the manifest lists into /exports//, with a +generated README saying what the run was, where each input came from, how +to run it again, and that the media belongs in Git LFS. 404 for an unknown +job, 409 for a live one or an existing export without overwrite. + +GET /exports/{id}.zip streams the same tree, built on request. 'exports' +joins the reserved workspace names, so the folder is never mistaken for a +workspace. + +Co-Authored-By: Claude Fable 5.1 +EOF +git commit -F /tmp/dw-commit-5.txt +``` + +--- + +### Task 6: `export_job` MCP tool (`haiku`) + +*Model rationale: one handler module, one client method tweak, one +registration, one test file — mechanical, with complete code below.* + +**Files:** +- Create: `dw_mcp/exports.py` +- Modify: `dw_mcp/client.py` (`post_json` gains `params`) +- Modify: `dw_mcp/server.py` (import `exports`; register in the jobs group) +- Test: `tests/test_mcp_exports.py` (new) + +**Interfaces:** +- Consumes: `POST /api/jobs/{id}/export?overwrite=` answering the + `ExportSummary` fields plus `zip_url`, `workflow`, `manifest`, `job` from + Task 5; `dw_mcp.client.api_path`, `DwApiError`. +- Produces: `dw_mcp.exports.export_job(client, job_id, overwrite=False) -> + dict` with keys `job_id`, `where`, `directory`, `zip_url`, `files`, + `total_bytes`, `missing`, `workflow`, `manifest`, `job`, `next`; + `DwClient.post_json(path, payload=None, params=None)`. + +- [ ] **Step 1: Write the failing tests** + +Create `tests/test_mcp_exports.py`: + +```python +"""Exporting a job over MCP: the directory is on the server, and the tool +says so - the lesson download_output taught.""" + +import httpx +import pytest + +from dw_mcp import exports +from dw_mcp.client import DwApiError, DwClient + +SUMMARY = { + "job_id": "job-1", + "directory": "/srv/studio/exports/job-1", + "files": [ + {"path": "workflow.json", "bytes": 412}, + {"path": "outputs/still.png", "bytes": 90210}, + ], + "total_bytes": 90622, + "missing": [], + "zip_url": "/exports/job-1.zip", + "workflow": {"id": "w", "steps": []}, + "manifest": {"run_id": "20260908-120000-abcdef01"}, + "job": {"id": "job-1", "status": "succeeded"}, +} + + +def scripted(routes): + seen = [] + + def handler(request): + key = (request.method, request.url.path) + seen.append({"key": key, "params": dict(request.url.params)}) + if key not in routes: + return httpx.Response(404, json={"detail": f"unrouted {key}"}) + status, body = routes[key] + return httpx.Response(status, json=body) + + return DwClient(transport=httpx.MockTransport(handler)), seen + + +def exporting(status=201, body=None): + return scripted( + {("POST", "/api/jobs/job-1/export"): (status, body or SUMMARY)} + ) + + +def test_it_returns_the_directory_the_zip_and_the_file_list(): + client, seen = exporting() + + result = exports.export_job(client, "job-1") + + assert result["job_id"] == "job-1" + assert result["directory"] == "/srv/studio/exports/job-1" + assert result["zip_url"] == "/exports/job-1.zip" + assert result["total_bytes"] == 90622 + assert [entry["path"] for entry in result["files"]] == [ + "workflow.json", + "outputs/still.png", + ] + assert len(seen) == 1 + + +def test_the_three_json_files_come_back_inline(): + client, _ = exporting() + + result = exports.export_job(client, "job-1") + + assert result["workflow"] == {"id": "w", "steps": []} + assert result["manifest"]["run_id"] == "20260908-120000-abcdef01" + assert result["job"]["status"] == "succeeded" + + +def test_it_says_where_the_directory_is(): + client, _ = exporting() + + result = exports.export_job(client, "job-1") + + assert result["where"] == ( + "/srv/studio/exports/job-1 on the machine running the MCP server" + ) + + +def test_overwrite_travels_as_a_query_parameter(): + client, seen = exporting() + + exports.export_job(client, "job-1", overwrite=True) + + assert seen[0]["params"]["overwrite"] == "true" + + +def test_a_409_reaches_the_model_as_a_readable_refusal(): + client, _ = exporting(status=409, body={"detail": "An export already exists"}) + + with pytest.raises(DwApiError) as caught: + exports.export_job(client, "job-1") + + assert "already exists" in str(caught.value) +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `source ./activate && python -m pytest tests/test_mcp_exports.py -v` +Expected: FAIL — `ModuleNotFoundError: No module named 'dw_mcp.exports'`. + +- [ ] **Step 3: Let `post_json` carry query parameters** + +In `dw_mcp/client.py`: + +```python + def post_json(self, path, payload=None, params=None): + """`params` is for a route whose options are query parameters rather + than a body - the export route, which takes `overwrite` beside the + workspace selector `_scoped` adds.""" + return self._json( + self._request("POST", path, json=payload or {}, params=params), path + ) +``` + +- [ ] **Step 4: Write `dw_mcp/exports.py`** + +```python +"""Bundling a finished job so it can leave the server. + +The one thing this module has to keep saying: the directory it makes is on +the machine running dw.serve, which over a `dw.serve --mcp` endpoint is the +GPU box and not where the agent is. The zip URL is the way to it from +anywhere else. +""" + +from dw_mcp.client import api_path + + +def export_job(client, job_id, overwrite=False): + """Gather one finished job into a directory on the machine running + dw.serve: workflow.json (realized), manifest.json, job.json, README, + assets/, inputs/, outputs/. Returns the directory, the zip URL, the + file list with sizes and the total, and the three JSON files inline. + The directory is on the server machine, not this one - use the zip + URL to fetch it elsewhere.""" + body = client.post_json( + api_path("api", "jobs", job_id, "export"), + params={"overwrite": "true" if overwrite else "false"}, + ) + directory = body.get("directory") + return { + "job_id": job_id, + "where": f"{directory} on the machine running the MCP server", + "directory": directory, + "zip_url": body.get("zip_url"), + "files": body.get("files") or [], + "total_bytes": body.get("total_bytes"), + "missing": body.get("missing") or [], + "workflow": body.get("workflow"), + "manifest": body.get("manifest"), + "job": body.get("job"), + "next": "Report the directory as a path on the server, and hand the " + "user the zip URL if they want the files locally.", + } +``` + +- [ ] **Step 5: Register the tool** + +In `dw_mcp/server.py`, add `exports` to the `from dw_mcp import (...)` block +(alphabetically, after `diagnose`), and in the diagnose/jobs group after +`move_job`: + +```python + def export_job(job_id: str, overwrite: bool = False) -> dict: + """Gather one finished job into a directory on the server: the + realized workflow, the run's manifest, the job row, a README, and + copies of every asset it used, every earlier run's file it read and + every file it made. Returns the directory, a zip URL, the file list + with sizes and the total, and the three JSON files inline. THE + DIRECTORY IS ON THE MACHINE RUNNING THE SERVER, not on yours - report + it as a server path and hand the user the zip URL if they want the + files locally. Refuses a job that is still running; refuses an + existing export unless overwrite=true.""" + return exports.export_job(client, job_id, overwrite=overwrite) +``` + +and register it beside the other writers: + +```python + for fn in (run_workflow, cancel_job, rerun_job, move_job, export_job): + tool(fn, WRITES) +``` + +- [ ] **Step 6: Run the tests to verify they pass** + +Run: `source ./activate && python -m pytest tests/test_mcp_exports.py tests/test_mcp_diagnose.py tests/test_mcp_server.py tests/test_mcp_media.py -v` +Expected: PASS. + +- [ ] **Step 7: Commit** + +```bash +git add dw_mcp/exports.py dw_mcp/client.py dw_mcp/server.py tests/test_mcp_exports.py +cat > /tmp/dw-commit-6.txt <<'EOF' +MCP: export_job bundles a finished job for git + +A handler over POST /api/jobs/{id}/export returning the directory, the zip +URL, the file list with sizes and the three JSON documents inline. The +'where' sentence says the directory is on the machine running the server - +the lesson download_output taught. + +DwClient.post_json gains an optional params argument, since the route's +overwrite flag is a query parameter beside the workspace selector. + +Co-Authored-By: Claude Fable 5.1 +EOF +git commit -F /tmp/dw-commit-6.txt +``` + +--- + +### Task 7: Docs and skills (`sonnet`) + +*Model rationale: eight prose files that must stay consistent with each other +and with the code — judgement about wording, but no design decisions left.* + +**Files:** +- Modify: `docs/WORKFLOW_GUIDE.md` (the seed/run-directory prose ~line 995-1000 + and the "Authoring a workflow from an agent" section ~line 237) +- Modify: `CLAUDE.md` (the type-system list and the run-directories gotcha) +- Modify: `docs/MCP.md` (the Diagnose table ~line 277) +- Modify: `docs/SERVER.md` (the Jobs API table ~line 131, and Files ~line 201) +- Modify: `docs/WORKSPACES.md` (the Runs section ~line 167) +- Modify: `plugins/dw/skills/minimax-h3/SKILL.md`, + `plugins/dw/skills/minimax-music3/SKILL.md`, + `plugins/dw/skills/ltx-2.5/SKILL.md` ("Run and judge") +- Modify: `docs/proposals/job-record-and-export.md` (status line), + `docs/proposals/resume.md` (stage 2) + +**Interfaces:** +- Consumes: everything Tasks 1-6 produced. No new code. + +- [ ] **Step 1: `docs/WORKFLOW_GUIDE.md`** + +After the paragraph ending "…can be reproduced after the fact." (the seed +prose), add: + +```markdown +Beside that manifest the run also writes `workflow.json` — the *realized* +workflow, meaning the one that actually ran. Every mutable input is pinned into +it: the caller's `arguments` folded into the `variables` defaults, the seed the +run used, each `prompt:` reference replaced by the stored text, and each +`output:/latest/` rewritten to the run id it resolved to. +`asset:`, `constant:`, `previous_result:` and `builtin:` are kept as written — +each already names something pinned by the asset library or by the manifest's +`dw_version` — and a sub-workflow named by local path is kept with its file's +SHA-256 recorded in the manifest. The manifest also lists which stored prompts +were inlined, since inlining loses the name. + +The file is a valid workflow: `python -m dw.run workflow.json` from inside the +run directory reproduces the run, and so does handing it to `run_workflow` as +`inline_workflow`. Writing it is best effort, exactly like the manifest — a run +that produced its files has succeeded either way — and `--output-layout flat` +writes no run directory, so it writes neither file. +``` + +In the "Authoring a workflow from an agent" section, at the end of the +References list, add: + +```markdown +After a long inline run that is worth keeping, `get_job_workflow(job_id)` +returns the realized workflow — the definition with the arguments, seed and +prompts of that run pinned into it — and `save_workflow` gives it a name, so +the next run is by name rather than by pasting JSON again. `export_job(job_id)` +bundles the whole run (workflow, manifest, job row, the media on both sides) +into a directory on the server plus a zip URL, for a run worth committing or +handing to someone else. +``` + +- [ ] **Step 2: `CLAUDE.md`** + +In the type-system bullet list, after the `prompt:` bullet, add: + +```markdown +- Every run directory holds `workflow.json` beside its manifest: the *realized* + workflow, with the run's arguments folded into the variable defaults, the seed + it used, stored prompt text inlined and `output:.../latest/...` pinned to the + run it resolved to. Written by `realize_workflow` (`dw/realize.py`) at run + start, best effort. Over MCP, `get_job_workflow` reads it back and + `save_workflow` names it; `export_job` bundles the run +``` + +In the **Run directories** gotcha, after the sentence ending "…and a sub-workflow +inherits the parent's run directory and writes no manifest of its own", add: + +```markdown + The realized workflow is written into the same directory as `workflow.json` + (`dw/realize.py`, `write_realized_workflow`), and the manifest's `workflow` + block carries `realized`, `prompts` (the stored prompts inlined) and + `sub_workflows` (path -> SHA-256). A job records the run it was + (`run_id`/`run_dir` on `Job` and in `jobs.sqlite`), which is how + `JobManager.realized` finds the file. `exports` is a reserved workspace name: + `POST /api/jobs/{id}/export` gathers one finished job into + `/exports//` and `GET /exports/.zip` streams it +``` + +- [ ] **Step 3: `docs/MCP.md`** + +In the Diagnose table, after the `get_job` row: + +```markdown +| `get_job_workflow(job_id)` | `job_id` | The workflow the job actually ran. `realized: true` means every mutable input is pinned (arguments, seed, prompts, `output:latest`); `false` means the job predates run tracking and this is the definition as submitted. Pass it to `save_workflow` to keep it under a name | +| `export_job(job_id, overwrite=False)` | `job_id`, `overwrite` | Gather one finished job into `/exports//` on the server: the realized workflow, the run's manifest, the job row, a README, and copies of the assets, earlier-run inputs and outputs. Returns the directory, a zip URL, the file list with sizes and the three JSON files inline. **The directory is on the machine running the server**, like `download_output`'s destination - report it as a server path and hand the user the zip URL for a local copy | +``` + +- [ ] **Step 4: `docs/SERVER.md`** + +Replace the `GET /api/jobs/{id}/workflow` row and add two more: + +```markdown +| `GET /api/jobs/{id}/workflow` | The workflow the job ran: `{id, definition, realized}`. `realized: true` is the copy the run itself wrote (`workflow.json` in its run directory), with arguments, seed, prompts and `output:latest` pinned; `false` falls back to the submitted definition, which is what a job from before run tracking has. 404 means neither is readable - the job itself still is | +| `POST /api/jobs/{id}/export?workspace=&overwrite=` | Gather one finished job into `/exports//`: `workflow.json`, `manifest.json`, `job.json`, `README.md`, `assets/`, `inputs/`, `outputs/`. 201 with the file list, total bytes, anything it could not find, a `zip_url`, and the three JSON files inline. 404 unknown job, 409 for a job still running or an existing export without `overwrite` | +| `GET /exports/{id}.zip?workspace=` | The same tree as one archive, built on request rather than kept as a second copy. Entries are named `/`. Ungated exactly as `/outputs` is | +``` + +In the "Files and models" section, add a sentence after the outputs paragraph: + +```markdown +`exports/` sits beside the workspace's own folders, holding one directory per +exported job. It is a reserved name: no workspace can be called `exports`, and +the folder is never listed as one. +``` + +- [ ] **Step 5: `docs/WORKSPACES.md`** + +In the Runs section, update the tree and add a sentence: + +```markdown +``` +outputs/ + ltx2/Gyre/ + 20260905-181530-a1b2c3d4/ + Gyre-still.0-0.0.png + Gyre-video.1-0.0.mp4 + manifest.json + workflow.json +``` +``` + +and after the paragraph describing the run id: + +```markdown +`workflow.json` is the realized workflow — the definition with this run's +arguments, seed and stored prompts pinned into it, so the directory reproduces +itself. `manifest.json` points at it and lists which prompts were inlined. +``` + +At the end of the "Several workspaces on one server" section, add: + +```markdown +A fifth name is reserved beside `workflows`, `prompts`, `assets` and `outputs`: +`exports`. `POST /api/jobs/{id}/export` gathers one finished job into +`/exports//`, and that folder is never mistaken for a workspace. +``` + +- [ ] **Step 6: The three plugin skills** + +In each of `plugins/dw/skills/minimax-h3/SKILL.md`, +`plugins/dw/skills/minimax-music3/SKILL.md` and +`plugins/dw/skills/ltx-2.5/SKILL.md`, append one numbered bullet to the end of +the "Run and judge" list (renumbering is not needed — append after the last +item): + +```markdown +- After an inline run worth keeping, `get_job_workflow` and `save_workflow` it, + so the next run is by name rather than by pasting JSON; `export_job` bundles + the run — workflow, manifest, job row and media — for git. +``` + +- [ ] **Step 7: The proposals** + +In `docs/proposals/job-record-and-export.md`, change the status line to: + +```markdown +Status: implemented 2026-09-08 (design: +`docs/superpowers/specs/2026-09-08-job-record-and-export-design.md`, plan: +`docs/superpowers/plans/2026-09-08-job-record-and-export.md`). Supplies the +"manifest carries step identity" stage of [resume.md](resume.md), which stays +the design for resuming a run; this proposal is the record that resume reads, +plus the two ways to get it off the server. +``` + +In `docs/proposals/resume.md`, in the Staging list, replace stage 2 with: + +```markdown +2. **Manifest carries step identity.** *Satisfied by the realized workflow* + ([job-record-and-export.md](job-record-and-export.md)): every run now writes + `workflow.json` beside its manifest with every mutable input pinned, so each + step's definition as it actually ran is on disk to compare against, and the + manifest's per-step files say what it made. A per-step digest may still be + worth adding for a cheaper comparison, but the information is no longer + missing. +``` + +- [ ] **Step 8: Verify the docs still pass their own test** + +Run: `source ./activate && python -m pytest tests/test_docs_links.py tests/test_plugin_skills.py -v` +Expected: PASS. `test_docs_links.py` checks that every `workflows/...` path a +doc names exists — none of the additions above names one, so a failure here +means a typo introduced a path. + +- [ ] **Step 9: Run the whole suite** + +Run: `source ./activate && python -m pytest tests/ -q` +Expected: PASS. + +- [ ] **Step 10: Commit** + +```bash +git add docs CLAUDE.md plugins +cat > /tmp/dw-commit-7.txt <<'EOF' +Document the realized workflow, the run record and the export + +WORKFLOW_GUIDE gets the realized file beside the manifest and, in the +agent-authoring section, the get_job_workflow -> save_workflow loop after a +long inline run. CLAUDE.md mirrors it in the type-system list and the +run-directories gotcha. MCP.md documents the two tools with the +server-machine caveat, SERVER.md the three routes, WORKSPACES.md the +workflow.json in a run directory and the reserved 'exports' name. + +Each family skill's "Run and judge" gains the same one-line habit. The +proposal moves to implemented, and resume.md records that its stage 2 is +satisfied by the realized file. + +Co-Authored-By: Claude Fable 5.1 +EOF +git commit -F /tmp/dw-commit-7.txt +``` + +--- + +## Self-review notes + +Recorded here so an executor knows which parts of the spec were interpreted +rather than transcribed: + +1. **`realize_workflow`'s signature gains `workflow_dir=None`.** The spec's + sub-workflow rule says the digest resolves "as `Workflow` resolves it + (relative to `base_dir`, confined to `workflow_dir`)", but the signature it + prints has no such parameter. The added keyword defaults to `None` (an + unconfined CLI run), so the spec's signature remains callable as written. +2. **`base_dir=self.base_dir` in the spec's call site does not exist.** + `Workflow` has no `base_dir` attribute; `run` computes it as a local. The + plan passes that local. +3. **Asset resolution in the export.** The spec names + `resolve_asset_reference`, whose search path is derived from process-wide + discovery. The plan instead walks the `asset_roots` the caller passed, using + `validate_asset_reference` + `validate_path` — the same algorithm, restricted + to exactly the roots `_asset_roots(ws)` built, which is what the spec's + parenthetical actually asks for and what makes the function testable. +4. **`EXPORTS_SUBDIR` lives in `dw/workspace.py`, not `dw/server/exports.py`.** + `RESERVED_WORKSPACE_NAMES` and `_foreign_entries` need it, and + `dw/workspace.py` must not import from `dw.server`. `dw/server/exports.py` + re-imports it from there, so the spec's name is still available at the + spec's module. +5. **The export route returns more than `ExportSummary`.** The spec's MCP tool + promises "the three JSON files it also returns inline", and the MCP client + cannot read the server's disk — so the route's body is the summary plus + `zip_url` plus `workflow`, `manifest` and `job`. `ExportSummary` itself keeps + the fields the spec gives it. +6. **`overwrite` reaches the route as a query parameter**, as the spec writes + it, which meant giving `DwClient.post_json` an optional `params` argument. +7. **`recent_summaries` gains `run_id` too.** The spec only names `summary()`, + but `JobManager.list` mixes live summaries and historical rows, and a field + present in one and absent in the other is a shape a client cannot rely on. +8. **Which manifest the export copies its outputs from.** The spec says "every + file the manifest lists, copied with the manifest's relative names" without + saying which manifest. The plan uses the run's own manifest when the run + directory is known (names relative to the run directory, absolute for a + step-cache hit republishing an earlier run) and the job row's synthesized + one otherwise; an absolute path is named by its path relative to the output + root, which keeps the identity and run id that explain where it came from. diff --git a/docs/superpowers/specs/2026-09-08-job-record-and-export-design.md b/docs/superpowers/specs/2026-09-08-job-record-and-export-design.md new file mode 100644 index 0000000..976c670 --- /dev/null +++ b/docs/superpowers/specs/2026-09-08-job-record-and-export-design.md @@ -0,0 +1,310 @@ +# Job record and export: design + +Date: 2026-09-08. Proposal: [docs/proposals/job-record-and-export.md](../../proposals/job-record-and-export.md). +Companion: [docs/proposals/resume.md](../../proposals/resume.md), which this +design serves but does not implement. + +## Goal + +Every run leaves a realized copy of the workflow that produced it beside its +manifest; a server job knows which run is its own; an MCP client can read +that workflow and can export one job as a git-ready directory and a zip. + +## Non-goals + +- Resuming a run from a previous run's files (resume.md stage 3). +- Rewriting references inside an exported workflow. +- Rebuilding job history from run directories. +- Widening the schema's `workflow` step to hold an inline definition. + +## Global constraints + +- Model knowledge stays out of engine code; nothing here names a model. +- Every path read or written goes through `dw/security.py` validators: + `validate_output_path`, `validate_path`, the asset and output resolvers. +- The realized file must validate against `dw/workflow_schema.json` + unchanged: annotations that the schema would reject live in the manifest. +- Writing the realized file and the manifest is best effort: a run that + produced its files has succeeded whether or not the record landed + (`write_manifest`'s rule). +- Nothing bundled requires `--trust-workflows`. +- `exports` is a reserved workspace name, beside `workflows`, `prompts`, + `assets`, `outputs`. + +## 1. Realization + +### Module + +New `dw/realize.py`: + +```python +def realize_workflow(definition, arguments, seed, base_dir=None, + prompt_dir=None, output_root=None): + """A copy of `definition` with every mutable input pinned. + Returns (realized, annotations).""" +``` + +- `definition` is the workflow as loaded (before `Workflow.run`'s deep copy + mutates it); the function never mutates its input. +- `arguments` is the run's argument dict; `seed` is the seed the run + resolved (`resolved_seed` in `Workflow.run`), never `None` here because + `run` draws one when the workflow names none. +- `annotations` is a dict the manifest carries: + `{"prompts": [...names...], "sub_workflows": {path: sha256_hex}}`. + +### Rules + +| Reference | Realized as | +|---|---| +| `variables` | The defaults after `set_variables(arguments, variables)`, which is exactly what the run computed. `variable:` references elsewhere in the file are left alone. | +| `seed` | The integer `seed`, written at the top level even when the definition had none. | +| `prompt:name` (any string value, anywhere in the tree) | `fetch_prompt(reference, prompt_dir, base_dir)`'s text. `name` is appended to `annotations["prompts"]` (deduplicated, in first-seen order). | +| `output:/latest/` | `output://` where run id comes from `resolve_output_reference(reference, output_root)` relative to the root. An explicit run id is kept as written. | +| `asset:`, `constant:`, `previous_result:`, `builtin:` | Kept. | +| `workflow.path` on a step, not `builtin:` | Kept; `annotations["sub_workflows"][path]` is the SHA-256 of the file, resolved as `Workflow` resolves it (relative to `base_dir`, confined to `workflow_dir`). Unreadable file: the digest is `null`, no error. | +| A prompt or output reference that fails to resolve | Left as written; the run will raise on it later with the engine's own error. Realization never fails a run. | + +The tree walk is one recursive function over dicts, lists and strings, the +shape `referenced_result_names` in `dw/step_cache.py` uses. A stored +prompt's text may not itself begin with a reference prefix (an existing +engine rule), so inlining cannot introduce a second resolution. + +### When and where it is written + +In `Workflow.run`, immediately after the run directory is chosen +(`self._run_dir` set, `run_id` known) and before the step loop, when +`self._run_dir` is set and not inherited: + +```python +realized, annotations = realize_workflow( + self.workflow_definition, arguments, resolved_seed, + base_dir=self.base_dir, output_root=self.output_dir) +write_realized_workflow(self._run_dir, realized) +``` + +New in `dw/runs.py`: `REALIZED_FILE_NAME = "workflow.json"` and +`write_realized_workflow(run_dir, realized)`, the same best-effort shape as +`write_manifest`, returning the path or `None`. + +The manifest's `workflow` block gains `"realized": "workflow.json"` (or +`null` when the write failed) and the two annotation keys: + +```json +"workflow": { + "id": "...", "file": "...", "identity": "...", + "realized": "workflow.json", + "prompts": ["scenic/dusk"], + "sub_workflows": {"steps/upscale.json": "ab12..."} +} +``` + +`realized_seed` is already the manifest's `seed`. A sub-workflow inherits +the run directory and writes neither file, as today. + +Flat output layout writes no run directory, so it writes no realized file; +the CLI does not warn, matching how the manifest behaves there. + +## 2. The job knows its run + +### Event + +`Workflow.run` emits, right after the realized file is written (also when +the write failed): + +```python +run_context.emit("run_start", run_id=run_id, + identity=workflow_identity(self.file_spec, workflow_id), + run_dir=) +``` + +The worker already forwards every emitted event as a `progress` message, +so the server sees it with no worker change. + +### Job and store + +`Job` gains `run_id = None` and `run_dir = None`; `JobManager`'s progress +branch sets them when `event["event"] == "run_start"`. `summary()` includes +`run_id`; `detail()` includes `run_id` and `run_dir`. + +`jobs.sqlite` gains `run_id TEXT` and `run_dir TEXT`, migrated like +`workflow_name` (ALTER when absent, NULL for older rows). Persisted at job +finish with the rest of the row; read back into the history dict. + +### Reading the realized workflow + +```python +def realized(self, job_id): + """The realized workflow a job ran, or None when the job predates + run tracking or its run directory no longer holds the file.""" +``` + +Reads `//workflow.json` where `output_dir` is the +job's own (`spec["output_dir"]`, or the manager's default for history +rows without one), after `validate_output_path` confines the join. + +`GET /api/jobs/{job_id}/workflow` returns +`{"id", "definition", "realized": bool}`: the realized workflow when +`realized()` finds one, else the submitted definition as today. The UI's +graph keeps working unchanged; a later UI change may label it. + +### MCP + +New tool in `dw_mcp/diagnose.py`, registered in `dw_mcp/server.py` beside +`get_job`: + +```python +def get_job_workflow(client, job_id): + """The workflow a job ran. `realized: true` means every mutable input + is pinned (arguments, seed, prompts, output:latest); false means the + job predates run tracking and this is the definition as submitted. + Pass it to save_workflow to rerun it by name, or edit it and pass it + to run_workflow as inline_workflow.""" +``` + +Returns `{"job_id", "realized", "workflow", "next"}` where `next` is one +sentence naming `save_workflow` and `run_workflow`. + +## 3. Export + +### Server module + +New `dw/server/exports.py`: + +```python +EXPORTS_SUBDIR = "exports" + +@dataclass +class ExportSummary: + job_id: str + directory: str # absolute, on the server + files: list[dict] # [{"path": "outputs/x.mp4", "bytes": n}, ...] + total_bytes: int + missing: list[str] # references the export could not find + +def export_job(manager, job_id, workspace_root, asset_roots, + overwrite=False) -> ExportSummary +``` + +Behaviour: + +1. `manager.get(job_id)` must exist and be terminal; a live or unknown job + is a `ValueError` the route turns into 409 or 404. +2. Target is `/exports//`, validated with + `validate_output_path` against the workspace root. Existing target and + `overwrite=False` raises `FileExistsError` (409); `overwrite=True` + removes it first. +3. `workflow.json`: `manager.realized(job_id)`, else the submitted + definition, and `job.json` records which (`"realized": bool`). +4. `manifest.json`: copied from the run directory when `run_dir` is known; + else synthesized from the job row's `manifest` list, marked + `"synthesized": true`. +5. `job.json`: the job's `detail()` minus `traceback` and `event_count`. +6. `assets/`: for every `asset:` string in `workflow.json`, the file + `resolve_asset_reference` finds on `asset_roots` (the workspace's asset + search path, in the order `_asset_roots` in `app.py` builds it), copied + under its reference name (folders preserved). Unresolvable: added to + `missing`. +7. `inputs///`: for every `output:` string, the + file `resolve_output_reference` finds under the job's output root. + Unresolvable: `missing`. +8. `outputs/`: every file the manifest lists, copied with the + manifest's relative names. +9. `README.md`: generated text (a module constant with format fields): + what the job was (workflow id, catalog name, status, started and finished + times, device, engine version), the seed, the arguments as a JSON block, + the stored prompts inlined by name, the sub-workflow digests, where each + `inputs/` file came from, how to run it again + (`python -m dw.run workflow.json` from a checkout, or `run_workflow` + with the file as `inline_workflow`), the `missing` list, and one + paragraph saying media under `assets/`, `inputs/` and `outputs/` belongs + in Git LFS if the tree is committed. + +Copies are copies, never hard links; the file list and `total_bytes` are +computed from what landed. + +### Routes + +- `POST /api/jobs/{job_id}/export?workspace=&overwrite=` → 201 with the + `ExportSummary` as JSON plus `"zip_url"` built by `_served_url` for + `/exports/.zip`. 404 unknown job, 409 live job or existing + export without `overwrite`. +- `GET /exports/{job_id}.zip?workspace=` → `StreamingResponse` of a zip + built on the fly from the export directory with `zipfile` at + `ZIP_DEFLATED`, entries named `/`. 404 when the + directory does not exist. Same auth treatment as `/outputs`: gated only + where `/outputs` is. +- `GET /exports/{job_id}/{path}` is not added; the zip and the directory + are the two forms. + +`RESERVED_WORKSPACE_NAMES` gains `exports`; `_foreign_entries` skips it +when listing workspaces. + +### MCP + +New `dw_mcp/exports.py`: + +```python +def export_job(client, job_id, overwrite=False): + """Gather one finished job into a directory on the machine running + dw.serve: workflow.json (realized), manifest.json, job.json, README, + assets/, inputs/, outputs/. Returns the directory, the zip URL, the + file list with sizes and the total, and the three JSON files inline. + The directory is on the server machine, not this one - use the zip + URL to fetch it elsewhere.""" +``` + +Registered in `dw_mcp/server.py` in the jobs group. The response includes +`"where": " on the machine running the MCP server"` so an +agent does not report a local path, the lesson `download_output` taught. + +## 4. Tests + +- `tests/test_realize.py`: one test per row of the realization table, + using a temporary prompt library and output root; a test that the input + definition is not mutated; a test that an unresolvable prompt is left as + written; a test that the realized file validates against the schema + (`dw.validate`). +- `tests/test_runs.py` (existing): `write_realized_workflow` best-effort + return and the manifest's `realized`, `prompts`, `sub_workflows` keys + after a task-only workflow runs; flat layout writes none. +- `tests/test_server_jobs.py` (new; the manager tests today live in `tests/test_server.py`): `run_start` populates + `run_id`/`run_dir`; both persist and read back; `realized()` returns the + file, `None` for a pre-tracking row. +- `tests/test_server_exports.py` (new, on the `tests/test_server.py` client fixture): the workflow route's + `realized` flag; export 201 tree contents; 409 on a live job; 409 without + overwrite; the zip lists the same entries as the directory; `exports` is + refused as a workspace name. +- `tests/test_mcp_diagnose.py`: `get_job_workflow` shape and `next`. +- `tests/test_mcp_exports.py`: `export_job` shape, `where` sentence, the + refusal path when the server answers 409. +- `tests/test_docs_links.py` keeps passing for every path the docs name. + +## 5. Docs + +- `docs/WORKFLOW_GUIDE.md`, run directories: `workflow.json` beside the + manifest, what "realized" means, the reproduction sentence. The + "Authoring a workflow from an agent" section: after a long inline run, + `get_job_workflow` then `save_workflow` so the next run is by name; the + same sentence in `CLAUDE.md`'s type-system list where the conventions + are mirrored. +- `CLAUDE.md` run-directories gotcha: the realized file and the manifest + keys; `exports` reserved. +- `docs/MCP.md`: the two tools, with the server-machine caveat. +- `docs/SERVER.md`: the export routes and the `exports/` directory. +- `docs/WORKSPACES.md`: `exports/` beside the four folders. +- `plugins/dw/skills/*/SKILL.md`, "Run and judge": one bullet each, "After + an inline run worth keeping, `get_job_workflow` and `save_workflow` it; + `export_job` bundles the run for git." +- `docs/proposals/job-record-and-export.md` status line moves to + "implemented" with the PR; `resume.md` gains a sentence that its stage 2 + is satisfied by the realized file. + +## Sequence for the plan + +1. Realization module and tests. +2. Writing it from `Workflow.run`, manifest keys, runs tests. +3. `run_start`, job fields, store migration, `realized()`, route flag. +4. `get_job_workflow` tool. +5. Export module, routes, reserved name, tests. +6. `export_job` tool. +7. Docs and skills. diff --git a/dw/realize.py b/dw/realize.py new file mode 100644 index 0000000..5368f21 --- /dev/null +++ b/dw/realize.py @@ -0,0 +1,229 @@ +"""Realizing a workflow: a copy of a definition with every mutable input +pinned, so the file left beside a run's manifest reproduces that run however +the library, the catalog or the output tree change afterwards. + +What "mutable" means here is precisely the set of things that can differ +between two runs of the same file: the arguments a caller passed, the seed a +seedless workflow drew, the text a stored prompt held at the time, and which +run 'output:.../latest/...' picked. Everything else - 'asset:', 'constant:', +'previous_result:', 'builtin:' and a sub-workflow's path - is a name whose +meaning is pinned by something already recorded (the asset library, the +manifest's dw_version, the file itself), so it is kept as written and, for a +local sub-workflow, digested into the manifest instead. + +Two rules hold this module together. It never mutates its input: the caller +hands it the definition the run is about to work from. And it never fails a +run: a reference that will not resolve is left exactly as written, so the +engine raises its own error at the point it would have raised anyway. +""" + +import copy +import hashlib +import logging +import os + +from .prompts import PROMPT_PREFIX, fetch_prompt +from .runs import ( + LATEST, + OUTPUT_PREFIX, + is_output_reference, + output_root as default_output_root, + resolve_output_reference, +) +from .security import SecurityError, validate_workflow_path +from .variables import set_variables + +logger = logging.getLogger("dw") + +BUILTIN_PREFIX = "builtin:" +VARIABLE_PREFIX = "variable:" + + +def realize_workflow( + definition, + arguments, + seed, + base_dir=None, + prompt_dir=None, + output_root=None, + workflow_dir=None, +): + """A copy of `definition` with every mutable input pinned. + + Args: + definition: The workflow as loaded, before Workflow.run's deep copy. + Never mutated. + arguments: The run's argument dict, folded into the variable defaults + exactly as `set_variables` folds them for the run itself. + seed: The seed the run resolved - an integer, never None, because + `Workflow.run` draws one when the workflow names none. + base_dir: The workflow file's directory, anchoring prompt discovery + and a sub-workflow's relative path. + prompt_dir: The prompt library, for `prompt:` inlining. + output_root: The output directory `output:` names resolve against. + workflow_dir: The root a sub-workflow path is confined to, as + `Workflow` confines it; None for an unconfined CLI run. + + Returns: + (realized, annotations) - the pinned copy, and + {"prompts": [name, ...], "sub_workflows": {path: sha256 or None}} + for the manifest to carry, since the schema has nowhere to put them. + """ + annotations = {"prompts": [], "sub_workflows": {}} + realized = copy.deepcopy(definition) + + variables = realized.get("variables") + if isinstance(variables, dict): + # Exactly what the run computed: set_variables coerces each value to + # the type of the declared default and rejects an undeclared name + set_variables(arguments or {}, variables) + + realized["seed"] = seed + # A definition can point its top-level seed at a declared variable + # ('"seed": "variable:seed_arg"') rather than an integer, so the run's + # resolved seed can also be read wherever else the workflow names that + # variable. Pinning the top-level field alone would leave the variable's + # own default whatever it was written as (typically none) - and a rerun + # of this realized copy with no seed argument would put that null + # default back over the pinned integer everywhere but the top level. + definition_seed = definition.get("seed") + if isinstance(definition_seed, str) and definition_seed.startswith(VARIABLE_PREFIX): + seed_variable = definition_seed.removeprefix(VARIABLE_PREFIX) + if isinstance(variables, dict) and seed_variable in variables: + variables[seed_variable] = seed + realized = _pin(realized, annotations, base_dir, prompt_dir, output_root) + _record_sub_workflows(realized.get("steps"), annotations, base_dir, workflow_dir) + return realized, annotations + + +def strings_with_prefix(tree, prefix): + """Every string in a nested dict/list tree that starts with `prefix`, in + first-seen order, deduplicated.""" + found = [] + + def collect(value): + if value.startswith(prefix) and value not in found: + found.append(value) + return value + + _map_strings(tree, collect) + return found + + +def _map_strings(value, transform): + """Rebuild `value`, replacing every string it contains with + `transform(string)`. One recursive walk over dicts, lists and strings - + the shape `referenced_result_names` in dw/step_cache.py walks - shared by + `_pin` (which rewrites matching references) and `strings_with_prefix` + (which only collects them), so a reference is found wherever it sits: a + pipeline argument, a task argument, a sub-workflow's argument map, an + element of a list. + """ + if isinstance(value, str): + return transform(value) + if isinstance(value, dict): + return {key: _map_strings(item, transform) for key, item in value.items()} + if isinstance(value, list): + return [_map_strings(item, transform) for item in value] + return value + + +def _pin(value, annotations, base_dir, prompt_dir, output_root): + """Rebuild a value with prompt and output references pinned.""" + + def transform(string): + if string.startswith(PROMPT_PREFIX): + return _inline_prompt(string, annotations, prompt_dir, base_dir) + if is_output_reference(string): + return _pin_output(string, output_root) + return string + + return _map_strings(value, transform) + + +def _inline_prompt(reference, annotations, prompt_dir, base_dir): + """The stored text, and the name recorded for the manifest. + + A stored prompt's text may not itself begin with a reference prefix (an + engine rule `fetch_prompt` enforces), so inlining cannot introduce a + second resolution. + """ + try: + text = fetch_prompt(reference, prompt_dir, base_dir) + except (SecurityError, OSError, ValueError) as e: + logger.warning(f"Realization kept {reference} as written: {e}") + return reference + name = reference.removeprefix(PROMPT_PREFIX).strip() + if name not in annotations["prompts"]: + annotations["prompts"].append(name) + return text + + +def _pin_output(reference, output_root): + """'output:/latest/' rewritten to the run it resolved to. + + An explicit run id is already pinned, so it is returned untouched without + touching the disk - realizing must not fail on a reference the run has + not reached yet. + """ + name = reference.removeprefix(OUTPUT_PREFIX).strip() + if LATEST not in name.split("/"): + return reference + root = output_root or default_output_root() + try: + resolved = resolve_output_reference(reference, root) + relative = os.path.relpath(resolved, root).replace(os.sep, "/") + except (SecurityError, OSError, ValueError) as e: + logger.warning(f"Realization kept {reference} as written: {e}") + return reference + return f"{OUTPUT_PREFIX}{relative}" + + +def _record_sub_workflows(steps, annotations, base_dir, workflow_dir): + """Digest every sub-workflow a step names by local path. + + The schema's 'workflow' step takes a path, not a definition, so the + realized file keeps the path and the manifest records what the file held. + A builtin is packaged with the engine and pinned by the manifest's + dw_version, so it is not digested. + """ + + def scan(value): + if isinstance(value, dict): + reference = value.get("workflow") + if isinstance(reference, dict): + path = reference.get("path") + if isinstance(path, str) and not path.startswith(BUILTIN_PREFIX): + annotations["sub_workflows"][path] = _digest( + path, base_dir, workflow_dir + ) + for item in value.values(): + scan(item) + elif isinstance(value, list): + for item in value: + scan(item) + + for step in steps or []: + scan(step) + + +def _digest(path, base_dir, workflow_dir): + """The SHA-256 of a sub-workflow file, or None when it cannot be read. + + Resolved the way `Workflow.create_step_action` resolves it - relative to + the referencing file's directory, then through `validate_workflow_path` + confined to `workflow_dir` - so a path this run could not have loaded is + not one realization reads either. + """ + try: + candidate = ( + path + if os.path.isabs(path) + else os.path.normpath(os.path.join(base_dir or ".", path)) + ) + validated = validate_workflow_path(candidate, workflow_dir) + with open(validated, "rb") as file: + return hashlib.sha256(file.read()).hexdigest() + except (SecurityError, OSError, ValueError) as e: + logger.debug(f"No digest for sub-workflow {path}: {e}") + return None diff --git a/dw/runs.py b/dw/runs.py index d848e67..0587ac2 100644 --- a/dw/runs.py +++ b/dw/runs.py @@ -40,6 +40,12 @@ MANIFEST_FILE_NAME = "manifest.json" +# The workflow a run actually ran, written beside its manifest. Named +# 'workflow.json' rather than something run-specific because the directory +# already says which run it is, and 'python -m dw.run workflow.json' from +# inside it is the whole reproduction story +REALIZED_FILE_NAME = "workflow.json" + # The prefix marking a value as a reference to a file an earlier run wrote. # Like 'asset:', it stands for a path - what a previous run made is an input # like any other, and multi-stage work is what a workflow engine is for @@ -352,6 +358,25 @@ def write_manifest(run_dir, manifest): return path +def write_realized_workflow(run_dir, realized): + """Leave the workflow that produced a run beside what it produced. + + The submitted definition says what was asked for; this says what ran - + arguments folded in, the seed pinned, stored prompts inlined, + 'output:latest' resolved. Best effort, exactly like write_manifest: a run + that produced its files has succeeded whether or not this lands. + """ + path = os.path.join(run_dir, REALIZED_FILE_NAME) + try: + os.makedirs(run_dir, exist_ok=True) + with open(path, "w") as file: + json.dump(realized, file, indent=2, default=str) + except (OSError, TypeError, ValueError) as e: + logger.warning(f"Could not write {path}: {e}") + return None + return path + + def manifest_relative_files(files, run_dir): """A run's file paths as the manifest records them: relative to the run directory, so the directory can be moved or copied and still describe diff --git a/dw/server/CLAUDE.md b/dw/server/CLAUDE.md index ef2ea2e..330188e 100644 --- a/dw/server/CLAUDE.md +++ b/dw/server/CLAUDE.md @@ -15,4 +15,14 @@ guide file resolves to the checkout's `docs/` first, else the packaged `dw/docs/` copy `scripts/build_dist.sh` makes, the same rule `default_ui_dir` uses for the SPA. `dw_mcp/guides.py` is a proxy of these routes. +`exports.py` gathers one finished job into a standalone directory - +`workflow.json` (the realized copy when the run wrote one), `manifest.json`, +`job.json`, a README, and copies of the assets, earlier-run inputs and outputs +it referenced - under `/exports//`. `EXPORTS_SUBDIR` lives +in `dw/workspace.py` rather than here, since `RESERVED_WORKSPACE_NAMES` needs +it and `dw/workspace.py` must not import from `dw.server`; this module +re-imports it. `app.py`'s `POST /api/jobs/{id}/export` calls it and returns +the summary plus a `zip_url`; `GET /exports/{id}.zip` builds the archive on +request from the same directory rather than keeping a second copy. + See docs/SERVER.md. diff --git a/dw/server/app.py b/dw/server/app.py index 47a057d..89f45c7 100644 --- a/dw/server/app.py +++ b/dw/server/app.py @@ -53,6 +53,7 @@ from ..prompts import PROMPT_PREFIX, RESERVED_TEXT_PREFIXES from ..workflow import Workflow, workflow_from_definition, workflow_from_file from .enhancers import build_enhance_workflow, preset_descriptions +from .exports import export_directory, export_job from ..result import read_embedded_metadata from ..hub_cache import scan_models, delete_model, DownloadManager from ..runs import strip_run_id @@ -833,17 +834,26 @@ def get_job(job_id: str): @app.get("/api/jobs/{job_id}/workflow") def get_job_workflow(job_id: str): - """The workflow definition this job ran, for the read-only graph on - the job page. 404 when the job named a file that is no longer - readable - the job itself still is.""" + """The workflow this job ran, for the read-only graph on the job page + and for `get_job_workflow` over MCP. + + `realized: true` means every mutable input is pinned - the copy the + run itself wrote. `false` means the job predates run tracking (or its + run directory is gone) and this is the definition as submitted. 404 + when neither is readable - the job itself still is.""" if manager.get(job_id) is None: raise HTTPException(status_code=404, detail="Unknown job") - definition = manager.definition(job_id) + realized = manager.realized(job_id) + definition = realized if realized is not None else manager.definition(job_id) if definition is None: raise HTTPException( status_code=404, detail="No workflow definition for this job" ) - return {"id": job_id, "definition": definition} + return { + "id": job_id, + "definition": definition, + "realized": realized is not None, + } @app.post("/api/jobs/{job_id}/rerun", status_code=201) def rerun_job(job_id: str): @@ -856,6 +866,50 @@ def rerun_job(job_id: str): raise HTTPException(status_code=404, detail="Unknown job") return manager.describe(job) + @app.post("/api/jobs/{job_id}/export", status_code=201) + def export_job_route( + job_id: str, + overwrite: bool = False, + ws: Workspace = Depends(selected_workspace), + ): + """Gather one finished job into '/exports//': the + workflow it ran, the run's manifest, the job row, the media it used + and the media it made, plus a README. 404 for an unknown job, 409 for + one still running or for an export that already exists without + `overwrite`. + + The three JSON files come back inline as well as on disk - the + directory is on the server, and a client on another machine has no + other way to read them without fetching the zip.""" + try: + summary = export_job( + manager, + job_id, + ws.root, + _asset_roots_for_job(job_id, ws), + overwrite=overwrite, + ) + except FileExistsError as e: + raise HTTPException(status_code=409, detail=str(e)) + except ValueError as e: + message = str(e) + if message.startswith("Unknown job"): + raise HTTPException(status_code=404, detail=message) + raise HTTPException(status_code=409, detail=message) + body = summary.as_dict() + body["zip_url"] = _served_url(f"/exports/{quote(job_id)}.zip", ws) + for key, name in ( + ("workflow", "workflow.json"), + ("manifest", "manifest.json"), + ("job", "job.json"), + ): + try: + with open(os.path.join(summary.directory, name), "r") as file: + body[key] = json.load(file) + except (OSError, ValueError): + body[key] = None + return body + class MoveRequest(BaseModel): direction: str = Field(description="up, down, front, or back") @@ -1581,6 +1635,35 @@ def _asset_roots(ws): roots.append(root) return roots + def _asset_roots_for_job(job_id, ws): + """The asset search path a job's own run used, for export: its spec's + `asset_dir` (or the historical row's), then the read-only example + libraries an --examples-dir tree brought with it - the same shape + `_asset_roots` builds for the selected workspace, but rooted at + wherever the job actually ran rather than at the workspace the + caller happens to be scoped to now. A job that ran in one workspace + while the caller exports it scoped to another must still find its + own 'asset:' files, not the other workspace's. + + Falls back to `_asset_roots(ws)` when the job carries no asset_dir + of its own - an inline-workflow job, or one recorded before this + field existed.""" + job = manager.get(job_id) + if job is None: + return _asset_roots(ws) + spec = (job.get("spec") or {}) if isinstance(job, dict) else job.spec + asset_dir = spec.get("asset_dir") + if not asset_dir: + return _asset_roots(ws) + roots = [] + for root in [asset_dir, *app.state.example_asset_dirs]: + if not root: + continue + root = os.path.abspath(root) + if root not in roots and os.path.isdir(root): + roots.append(root) + return roots + def _served_url(path, ws, version=None): """The URL a served file is reachable at: the default workspace's files keep the URL they have always had, a named one carries the @@ -2281,6 +2364,47 @@ async def input_file( files = _static_files_for(roots[0]) return await files.get_response(name, request.scope) + # Ungated for the same reason the two above are: a download link cannot + # attach an Authorization header either + @app.get("/exports/{job_id}.zip") + def export_zip(job_id: str, ws: Workspace = Depends(selected_workspace)): + """One job's export as a zip, built on request from the directory + rather than kept as a second copy. Entries are named + '/', so unzipping anywhere gives the same tree + the server holds.""" + try: + directory = export_directory(ws.root, job_id) + except SecurityError: + raise HTTPException(status_code=404, detail="No export for this job") + if not os.path.isdir(directory): + raise HTTPException(status_code=404, detail="No export for this job") + + handle = tempfile.NamedTemporaryFile(suffix=".zip", delete=False) + try: + with handle: + with zipfile.ZipFile(handle, "w", zipfile.ZIP_DEFLATED) as archive: + for current, _dirs, names in os.walk(directory): + for name in sorted(names): + path = os.path.join(current, name) + entry = os.path.relpath(path, directory).replace( + os.sep, "/" + ) + archive.write(path, f"{job_id}/{entry}") + except BaseException: + # Nothing is going to attach the background unlink now, so the + # half-written archive has to go here + os.unlink(handle.name) + raise + + return FileResponse( + handle.name, + media_type="application/zip", + filename=f"{job_id}.zip", + # The archive is a temp file, not a second permanent copy - it + # goes as soon as the response has been sent + background=BackgroundTask(os.unlink, handle.name), + ) + # ---------------------------------------------------------------- the UI resolved_ui = ui_dir or default_ui_dir() diff --git a/dw/server/exports.py b/dw/server/exports.py new file mode 100644 index 0000000..7953f7a --- /dev/null +++ b/dw/server/exports.py @@ -0,0 +1,468 @@ +"""Gathering one finished job into a directory that stands on its own. + +The record of a run is split across four places on one machine: the job row, +the run's manifest, the workflow that ran, and the media on either side of it. +This module puts them in one tree - text at the top, media in folders, no +absolute paths inside the files - so the run can be committed, moved or handed +to someone else. + + exports// + README.md what this is, how it was made, how to run it again + workflow.json the realized workflow, when the run wrote one + manifest.json the run's manifest, or a synthesized stand-in + job.json the job row: status, times, arguments, warnings, error + assets/ every asset: the workflow names, under its own name + inputs/ every file an output: reference named, by run + outputs/ every file the manifest lists + +Copies are copies, never hard links: exports/ is what a user moves or deletes, +and a hard link would make deleting it look like deleting the original. +""" + +import json +import logging +import os +import re +import shutil +from dataclasses import dataclass, field + +from ..assets import ASSET_PREFIX +from ..realize import strings_with_prefix +from ..runs import ( + MANIFEST_FILE_NAME, + OUTPUT_PREFIX, + is_output_reference, + resolve_output_reference, +) +from ..security import ( + PathTraversalError, + SecurityError, + validate_asset_reference, + validate_output_path, + validate_path, +) +from ..workspace import EXPORTS_SUBDIR +from .jobs import TERMINAL_STATES + +# A job id is one path segment of the manager's making - hex today, but any +# name without a separator or a leading dot is accepted so history stays +# readable if the shape ever changes +JOB_ID_SEGMENT = re.compile(r"[A-Za-z0-9_][A-Za-z0-9_.-]*") + +logger = logging.getLogger("dw") + +WORKFLOW_FILE_NAME = "workflow.json" +JOB_FILE_NAME = "job.json" +README_FILE_NAME = "README.md" + +# What job.json holds, in this order, whatever the job was read from. A live +# job's detail and a historical one's differ in shape - history carries the +# submitted spec and a 'historical' flag, a live job a traceback and an event +# count - and an export is a record, so it takes one fixed key set: a +# traceback is a developer's artifact of one process, an event count +# describes a log this tree does not hold, and the spec is what +# workflow.json already is. 'realized' is added beside them +JOB_RECORD_KEYS = ( + "id", + "workflow", + "workflow_name", + "status", + "created_at", + "started_at", + "finished_at", + "workspace", + "arguments", + "warnings", + "manifest", + "error", + "run_id", + "run_dir", +) + +README_TEMPLATE = """# {workflow_name} - job {job_id} + +{realized_sentence} + +## The run + +| | | +| --- | --- | +| Job | `{job_id}` | +| Workflow | `{workflow_name}` | +| Catalog entry | {catalog_name} | +| Status | {status} | +| Started | {started_at} | +| Finished | {finished_at} | +| Device | {device} | +| Engine | dw {dw_version} | +| Seed | `{seed}` | + +Arguments as submitted: + +```json +{arguments} +``` + +## Stored prompts + +{prompts} + +## Sub-workflows + +{sub_workflows} + +## Inputs + +{inputs} + +## Running it again + +From a checkout of diffusers-workflow, with this directory as the working +directory: + +```bash +python -m dw.run workflow.json +``` + +Over MCP, pass the contents of `workflow.json` to `run_workflow` as +`inline_workflow`. Either way the `asset:` and `output:` names in it resolve +against the server's libraries, not against this directory - the copies under +`assets/` and `inputs/` are the record of what those names meant, not a +substitute for them. + +## Missing + +{missing} + +## A note on committing this + +`assets/`, `inputs/` and `outputs/` hold generated and source media, which is +usually large and always binary. If this tree goes into Git, put those three +directories in Git LFS; the four files at the top are text and belong in Git +proper. +""" + + +@dataclass +class ExportSummary: + """What an export produced, computed from what actually landed.""" + + job_id: str + directory: str + files: list = field(default_factory=list) + total_bytes: int = 0 + missing: list = field(default_factory=list) + + def as_dict(self): + return { + "job_id": self.job_id, + "directory": self.directory, + "files": self.files, + "total_bytes": self.total_bytes, + "missing": self.missing, + } + + +def export_directory(workspace_root, job_id): + """Where one job's export lives, confined to the workspace root. + + The job id is caller input - it arrives in a URL - so it is joined and + then validated rather than trusted to be the hex string the manager + generates. + """ + if not workspace_root: + raise ValueError("This server has no workspace root to export into") + # One plain segment, before the join: validate_path accepts a path equal + # to its root, so '.' or '' would otherwise name the exports root itself + # and the zip route would archive every export in the workspace + if not JOB_ID_SEGMENT.fullmatch(job_id or ""): + raise PathTraversalError(f"Not a job id: {job_id!r}") + root = validate_output_path(os.path.join(workspace_root, EXPORTS_SUBDIR), None) + return validate_path(os.path.join(root, job_id), root) + + +def export_job(manager, job_id, workspace_root, asset_roots, overwrite=False): + """Gather one finished job into '/exports//'. + + Args: + manager: The JobManager holding the job (live or historical). + job_id: The job to export. + workspace_root: The workspace the export lands in. + asset_roots: The workspace's asset search path, in order - the same + order 'asset:' resolves in, so what is copied is what the run + loaded. + overwrite: Replace an existing export rather than refusing. + + Returns: + An ExportSummary. + + Raises: + ValueError: Unknown job, or a job that has not finished. + FileExistsError: The export exists and overwrite is False. + """ + job = manager.get(job_id) + if job is None: + raise ValueError(f"Unknown job {job_id}") + detail = job if isinstance(job, dict) else manager.describe(job) + status = detail.get("status") + if status not in TERMINAL_STATES: + raise ValueError( + f"Job {job_id} is {status} - only a finished job can be exported" + ) + + spec = job.get("spec", {}) if isinstance(job, dict) else job.spec + run_dir = job.get("run_dir") if isinstance(job, dict) else job.run_dir + output_root = validate_output_path( + spec.get("output_dir") or manager.output_dir, None + ) + + target = export_directory(workspace_root, job_id) + if os.path.exists(target): + if not overwrite: + raise FileExistsError( + f"An export of job {job_id} already exists - pass overwrite to " + f"replace it" + ) + shutil.rmtree(target) + os.makedirs(target) + + summary = ExportSummary(job_id=job_id, directory=target) + + realized = manager.realized(job_id) + workflow = realized if realized is not None else (manager.definition(job_id) or {}) + _write_json(summary, target, WORKFLOW_FILE_NAME, workflow) + + manifest = _run_manifest(output_root, run_dir) + if manifest is None: + # No run directory, or it no longer holds a manifest: the job row's + # own per-step file list is what is left, and it says so + manifest = { + "synthesized": True, + "run_id": None, + "status": status, + "steps": detail.get("manifest") or [], + } + _write_json(summary, target, MANIFEST_FILE_NAME, manifest) + + record = {key: detail.get(key) for key in JOB_RECORD_KEYS} + record["realized"] = realized is not None + _write_json(summary, target, JOB_FILE_NAME, record) + + _copy_assets(summary, workflow, target, asset_roots) + _copy_inputs(summary, workflow, target, output_root) + _copy_outputs(summary, manifest, target, output_root, run_dir) + + readme = _readme(job_id, detail, manifest, workflow, summary, realized is not None) + _write_text(summary, target, README_FILE_NAME, readme) + return summary + + +# -------------------------------------------------------------- the pieces + + +def _run_manifest(output_root, run_dir): + """The manifest the run left, or None when there is no reading it.""" + if not run_dir: + return None + try: + path = validate_path( + os.path.join(output_root, run_dir, MANIFEST_FILE_NAME), output_root + ) + with open(path, "r") as file: + return json.load(file) + except (SecurityError, OSError, ValueError) as e: + logger.debug(f"No run manifest to export from {run_dir}: {e}") + return None + + +def _copy_assets(summary, workflow, target, asset_roots): + """Every 'asset:' the workflow names, under its own name in assets/.""" + for reference in strings_with_prefix(workflow, ASSET_PREFIX): + try: + name = validate_asset_reference( + reference.removeprefix(ASSET_PREFIX).strip() + ) + except SecurityError: + summary.missing.append(reference) + continue + source = None + for root in asset_roots: + try: + candidate = validate_path(os.path.join(root, name), root) + except SecurityError: + continue + if os.path.isfile(candidate): + source = candidate + break + if source is None: + summary.missing.append(reference) + continue + _copy( + summary, source, target, os.path.join("assets", *name.split("/")), reference + ) + + +def _copy_inputs(summary, workflow, target, output_root): + """Every 'output:' the workflow names, kept under the run it came from. + + The reference itself is not rewritten - the realized workflow is the + immutable record of the run - so the directory name is the reference's + own name, and the README says where each one came from. + """ + for reference in strings_with_prefix(workflow, OUTPUT_PREFIX): + if not is_output_reference(reference): + continue + name = reference.removeprefix(OUTPUT_PREFIX).strip() + try: + source = resolve_output_reference(reference, output_root) + except (SecurityError, OSError, ValueError): + summary.missing.append(reference) + continue + _copy( + summary, source, target, os.path.join("inputs", *name.split("/")), reference + ) + + +def _copy_outputs(summary, manifest, target, output_root, run_dir): + """Every file the manifest lists, under outputs/. + + A manifest entry names a file relative to the run directory when the run + wrote it, and absolutely when a step-cache hit republished an earlier + run's file. Both land here: the first under its own relative name, the + second under its path relative to the output root, which keeps the + identity and run id that say where it really came from. + """ + run_root = os.path.join(output_root, run_dir) if run_dir else output_root + # One file can be listed by two steps (a chain's output is the next + # step's input); it is one file in the export, copied and counted once + copied = set() + for entry in manifest.get("steps") or []: + if not isinstance(entry, dict): + continue + for recorded in entry.get("files") or []: + source = ( + recorded + if os.path.isabs(recorded) + else os.path.join(run_root, recorded) + ) + try: + source = validate_path(source, output_root) + except SecurityError: + summary.missing.append(recorded) + continue + if not os.path.isfile(source): + summary.missing.append(recorded) + continue + if source in copied: + continue + copied.add(source) + try: + relative = os.path.relpath(source, run_root) + except ValueError: # different drive on Windows + relative = os.path.basename(source) + if relative == os.pardir or relative.startswith(os.pardir + os.sep): + relative = os.path.relpath(source, output_root) + _copy( + summary, + source, + target, + os.path.join("outputs", *relative.split(os.sep)), + recorded, + ) + + +def _readme(job_id, detail, manifest, workflow, summary, realized): + workflow_block = manifest.get("workflow") or {} + prompts = workflow_block.get("prompts") or [] + sub_workflows = workflow_block.get("sub_workflows") or {} + inputs = [ + entry["path"] for entry in summary.files if entry["path"].startswith("inputs/") + ] + return README_TEMPLATE.format( + job_id=job_id, + workflow_name=detail.get("workflow") or "unknown", + catalog_name=( + f"`{detail['workflow_name']}`" + if detail.get("workflow_name") + else "none - an inline definition" + ), + realized_sentence=( + "`workflow.json` is the *realized* workflow: every mutable input " + "is pinned, so it reproduces this run whatever changes afterwards." + if realized + else "`workflow.json` is the definition as submitted - this job " + "predates run tracking, so its arguments and prompts are not " + "pinned into it." + ), + status=detail.get("status"), + started_at=detail.get("started_at"), + finished_at=detail.get("finished_at"), + device=manifest.get("device", "unknown"), + dw_version=manifest.get("dw_version", "unknown"), + seed=manifest.get("seed", workflow.get("seed", "not recorded")), + arguments=json.dumps(detail.get("arguments") or {}, indent=2), + prompts=_bullets(f"`{name}` - inlined into `workflow.json`" for name in prompts) + or "None: this workflow named no stored prompt.", + sub_workflows=_bullets( + f"`{path}` - sha256 `{digest}`" if digest else f"`{path}` - unreadable" + for path, digest in sorted(sub_workflows.items()) + ) + or "None: this workflow composed no other workflow by path.", + inputs=_bullets( + f"`{path}` - copied from the run named in its own path" for path in inputs + ) + or "None: this workflow named no file from an earlier run.", + missing=_bullets(f"`{name}`" for name in summary.missing) + or "Nothing: every file this run referenced was found and copied.", + ) + + +def _bullets(lines): + return "\n".join(f"- {line}" for line in lines) + + +# ------------------------------------------------------------------- files + + +def _copy(summary, source, target, relative, name): + """Copy one file into the export and record it. + + `name` is how a failure is reported - the reference or the manifest's own + entry, never the absolute path, because `missing` is read out of the + README on another machine where that path means nothing. + """ + destination = os.path.join(target, relative) + try: + os.makedirs(os.path.dirname(destination), exist_ok=True) + shutil.copyfile(source, destination) + except OSError as e: + logger.warning(f"Could not copy {source} into the export: {e}") + summary.missing.append(name) + return + _record(summary, destination, target) + + +def _write_json(summary, target, name, payload): + _write_text(summary, target, name, json.dumps(payload, indent=2, default=str)) + + +def _write_text(summary, target, name, text): + path = os.path.join(target, name) + try: + with open(path, "w", encoding="utf-8") as file: + file.write(text) + except OSError as e: + logger.warning(f"Could not write {path}: {e}") + return + _record(summary, path, target) + + +def _record(summary, path, target): + try: + size = os.path.getsize(path) + except OSError: + return + summary.files.append( + {"path": os.path.relpath(path, target).replace(os.sep, "/"), "bytes": size} + ) + summary.total_bytes += size diff --git a/dw/server/jobs.py b/dw/server/jobs.py index b65823f..55cde2f 100644 --- a/dw/server/jobs.py +++ b/dw/server/jobs.py @@ -23,8 +23,10 @@ SecurityError, validate_json_size, validate_output_path, + validate_path, validate_workflow_path, ) +from ..runs import REALIZED_FILE_NAME from ..settings import resolve_path from ..workspace import DEFAULT_WORKSPACE_NAME @@ -110,6 +112,14 @@ def __init__(self, db_path): # unjoinable, new history is exact if "workflow_name" not in columns: connection.execute("ALTER TABLE jobs ADD COLUMN workflow_name TEXT") + # Which run of the workflow this job was - the directory under the + # output root that holds its manifest and its realized workflow. + # NULL for every row predating run tracking, and the manager + # refuses to guess one from file paths + if "run_id" not in columns: + connection.execute("ALTER TABLE jobs ADD COLUMN run_id TEXT") + if "run_dir" not in columns: + connection.execute("ALTER TABLE jobs ADD COLUMN run_dir TEXT") def _connect(self): return sqlite3.connect(self.db_path, timeout=5) @@ -121,8 +131,8 @@ def record(self, job): connection.execute( "INSERT OR REPLACE INTO jobs (id, workflow, status, created_at," " started_at, finished_at, arguments, spec, manifest, warnings," - " error, events, workspace, workflow_name) VALUES" - " (?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + " error, events, workspace, workflow_name, run_id, run_dir) VALUES" + " (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", ( job.id, job.workflow_name, @@ -138,6 +148,8 @@ def record(self, job): json.dumps(job.events[-MAX_PERSISTED_EVENTS:], default=str), job.spec.get("workspace") or DEFAULT_WORKSPACE_NAME, job.catalog_name, + job.run_id, + job.run_dir, ), ) @@ -150,7 +162,7 @@ def recent_summaries(self, limit=200, workspace=None): """ query = ( "SELECT id, workflow, status, created_at, started_at, finished_at," - " workspace, workflow_name FROM jobs" + " workspace, workflow_name, run_id FROM jobs" ) params = [] if workspace: @@ -170,6 +182,7 @@ def recent_summaries(self, limit=200, workspace=None): "finished_at": row[5], "workspace": row[6] or DEFAULT_WORKSPACE_NAME, "workflow_name": row[7], + "run_id": row[8], "historical": True, } for row in rows @@ -179,8 +192,8 @@ def get(self, job_id): with self._lock, self._connect() as connection: row = connection.execute( "SELECT id, workflow, status, created_at, started_at, finished_at," - " arguments, spec, manifest, warnings, error, workspace, workflow_name" - " FROM jobs WHERE id = ?", + " arguments, spec, manifest, warnings, error, workspace," + " workflow_name, run_id, run_dir FROM jobs WHERE id = ?", (job_id,), ).fetchone() return self._to_detail(row) if row else None @@ -294,6 +307,8 @@ def parse(text, fallback): "error": row[10], "workspace": row[11] or DEFAULT_WORKSPACE_NAME, "workflow_name": row[12], + "run_id": row[13], + "run_dir": row[14], "traceback": None, "event_count": 0, "historical": True, @@ -316,6 +331,11 @@ def __init__(self, spec): self.warnings = spec.get("warnings", []) self.error = None self.traceback = None + # Which run this job turned out to be - reported by the worker's + # run_start event, unknown until then and forever for a job that + # never got that far + self.run_id = None + self.run_dir = None self.events = [] self.condition = threading.Condition() @@ -359,6 +379,7 @@ def summary(self): # carry one yet (e.g. a caller that never named a workspace), # so it defaults the same way history's column does "workspace": self.spec.get("workspace") or DEFAULT_WORKSPACE_NAME, + "run_id": self.run_id, } def detail(self): @@ -370,6 +391,7 @@ def detail(self): "error": self.error, "traceback": self.traceback, "event_count": len(self.events), + "run_dir": self.run_dir, } @@ -549,6 +571,39 @@ def definition(self, job_id): logger.debug(f"No workflow definition available for job {job_id}") return None + def realized(self, job_id): + """The realized workflow a job ran, or None when the job predates + run tracking or its run directory no longer holds the file. + + Read from the job's own output directory, not the manager's: one + server holds several workspaces, and a job carries the root it ran + against. The join is confined to that root, so a run_dir read back + out of the database cannot name anything outside it. + """ + job = self.jobs.get(job_id) + if job is not None: + run_dir = job.run_dir + output_dir = job.spec.get("output_dir") or self.output_dir + else: + historical = self.history.get(job_id) + if historical is None: + return None + run_dir = historical.get("run_dir") + output_dir = (historical.get("spec") or {}).get( + "output_dir" + ) or self.output_dir + if not run_dir: + return None + try: + root = validate_output_path(output_dir, None) + path = validate_path(os.path.join(root, run_dir, REALIZED_FILE_NAME), root) + validate_json_size(path) + with open(path, "r") as file: + return json.load(file) + except (SecurityError, OSError, ValueError) as e: + logger.debug(f"No realized workflow for job {job_id}: {e}") + return None + def rerun(self, job_id): """Queue a fresh job from a previous job's spec. @@ -809,6 +864,9 @@ def _consume_results(self, job): event["files"] = self._relative_output_names( event["files"], job.spec.get("output_dir") ) + if event.get("event") == "run_start": + job.run_id = event.get("run_id") + job.run_dir = event.get("run_dir") job.add_event(event) elif message_type in ("output", "workflow_loaded"): text = message.get("message") or message.get("workflow_name", "") diff --git a/dw/workflow.py b/dw/workflow.py index 65bbcae..e367aff 100644 --- a/dw/workflow.py +++ b/dw/workflow.py @@ -25,6 +25,7 @@ ) from .runs import ( FLAT_LAYOUT, + REALIZED_FILE_NAME, activate_output_root, deactivate_output_root, workflow_identity, @@ -33,7 +34,9 @@ output_layout, run_directory, write_manifest, + write_realized_workflow, ) +from .realize import realize_workflow from .schema import validate_data_all, format_validation_errors, load_schema from .variables import replace_variables, set_variables from .pipeline_processors.pipeline import Pipeline @@ -334,6 +337,11 @@ def run( # manifest of every seedless run and lose the only record of what produced # its files - the seed is what makes a run repeatable resolved_seed = None + # What the run recorded about itself, read by _write_run_manifest in + # the finally below - initialized here so a failure before the run + # directory exists still writes a well-formed manifest + realized_name = None + annotations = {"prompts": [], "sub_workflows": {}} started_at = datetime.now(timezone.utc).isoformat() try: # CRITICAL: Work on a copy to avoid mutating the original workflow definition @@ -422,6 +430,38 @@ def run( ) logger.debug(f"Run directory: {self._run_dir}") + # The record of what actually ran, written before the first step + # so a crash or a cancel still leaves it. A sub-workflow inherits + # the parent's directory and writes none of its own, as with the + # manifest, and the flat layout has no directory to write into + if self._run_dir and not self._run_dir_inherited: + try: + realized, annotations = realize_workflow( + self.workflow_definition, + arguments, + default_seed, + base_dir=base_dir, + output_root=self.output_dir, + workflow_dir=self.workflow_dir, + ) + if write_realized_workflow(self._run_dir, realized): + realized_name = REALIZED_FILE_NAME + except Exception as e: + # Never fatal: the record is worth less than the run + logger.warning(f"Could not realize workflow {workflow_id}: {e}") + + # Which run this is, so a server job can find the directory + # it wrote. Emitted even when the realized file did not land: + # the manifest is still there, and so are the files + run_context.emit( + "run_start", + run_id=run_id, + identity=workflow_identity(self.file_spec, workflow_id), + run_dir=os.path.relpath(self._run_dir, self.output_dir).replace( + os.sep, "/" + ), + ) + # Initialize collections for sharing state between steps results = {} # Stores results from each step shared_components = {} # Shared resources between steps @@ -660,12 +700,27 @@ def run( # what a failed run needs to explain itself if self._run_dir and not self._run_dir_inherited: self._write_run_manifest( - run_id, status, started_at, arguments, resolved_seed + run_id, + status, + started_at, + arguments, + resolved_seed, + realized_name, + annotations, ) deactivate_output_root(output_root_token) deactivate_context(context_token) - def _write_run_manifest(self, run_id, status, started_at, arguments, seed): + def _write_run_manifest( + self, + run_id, + status, + started_at, + arguments, + seed, + realized_name=None, + annotations=None, + ): """Leave a record of the run beside the files it wrote. A server run is in jobs.sqlite as well, but a CLI run has never been @@ -688,6 +743,15 @@ def _write_run_manifest(self, run_id, status, started_at, arguments, seed): "id": self.name, "file": self.file_spec, "identity": workflow_identity(self.file_spec, self.name), + # The realized copy beside this manifest, or null when + # writing it did not land - the manifest is the only + # place that difference is visible + "realized": realized_name, + # Annotations the schema has nowhere to put: which + # stored prompts were inlined, and what each local + # sub-workflow file held when it ran + "prompts": (annotations or {}).get("prompts", []), + "sub_workflows": (annotations or {}).get("sub_workflows", {}), }, "seed": seed, "arguments": arguments or {}, diff --git a/dw/workspace.py b/dw/workspace.py index d669542..1b2dc10 100644 --- a/dw/workspace.py +++ b/dw/workspace.py @@ -60,6 +60,12 @@ SUBDIRS = (WORKFLOWS_SUBDIR, PROMPTS_SUBDIR, ASSETS_SUBDIR, OUTPUTS_SUBDIR) +# Where a job export lands: '/exports//'. Not a workspace +# folder - it is beside them, holding gathered copies rather than working +# content - but it is a name a workspace may not take, and workspace_names +# must not mistake it for one +EXPORTS_SUBDIR = "exports" + # What makes a directory recognizable as a workspace. assets/ is deliberately # not a marker - a bare assets/ folder is a common thing to have lying around, # where these three together say "content lives here" @@ -74,8 +80,8 @@ DEFAULT_WORKSPACE_NAME = "default" # Names a workspace cannot take, because the root's own folders already -# use them -RESERVED_WORKSPACE_NAMES = SUBDIRS +# use them - its four content folders, and the exports gathered beside them +RESERVED_WORKSPACE_NAMES = SUBDIRS + (EXPORTS_SUBDIR,) # What a named workspace holds - prompts excluded, per above NAMED_SUBDIRS = (WORKFLOWS_SUBDIR, ASSETS_SUBDIR, OUTPUTS_SUBDIR) @@ -384,9 +390,11 @@ def _holds_a_workspace(path): def _foreign_entries(path): - """What a directory holds besides a workspace's own three folders.""" + """What a directory holds besides a workspace's own three folders and + the exports it may have gathered.""" + ignored = NAMED_SUBDIRS + (EXPORTS_SUBDIR,) try: - return sorted(entry for entry in os.listdir(path) if entry not in NAMED_SUBDIRS) + return sorted(entry for entry in os.listdir(path) if entry not in ignored) except OSError: return [] diff --git a/dw_mcp/client.py b/dw_mcp/client.py index 025f075..1dda9f4 100644 --- a/dw_mcp/client.py +++ b/dw_mcp/client.py @@ -126,8 +126,13 @@ def close(self): def get_json(self, path, params=None): return self._json(self._request("GET", path, params=params), path) - def post_json(self, path, payload=None): - return self._json(self._request("POST", path, json=payload or {}), path) + def post_json(self, path, payload=None, params=None): + """`params` is for a route whose options are query parameters rather + than a body - the export route, which takes `overwrite` beside the + workspace selector `_scoped` adds.""" + return self._json( + self._request("POST", path, json=payload or {}, params=params), path + ) def put_json(self, path, payload): return self._json(self._request("PUT", path, json=payload), path) diff --git a/dw_mcp/diagnose.py b/dw_mcp/diagnose.py index 43f47fc..8397308 100644 --- a/dw_mcp/diagnose.py +++ b/dw_mcp/diagnose.py @@ -71,6 +71,23 @@ def get_job(client, job_id): return client.get_json(api_path("api", "jobs", job_id)) +def get_job_workflow(client, job_id): + """The workflow a job ran. `realized: true` means every mutable input + is pinned (arguments, seed, prompts, output:latest); false means the + job predates run tracking and this is the definition as submitted. + Pass it to save_workflow to rerun it by name, or edit it and pass it + to run_workflow as inline_workflow.""" + body = client.get_json(api_path("api", "jobs", job_id, "workflow")) + return { + "job_id": job_id, + "realized": bool(body.get("realized")), + "workflow": body.get("definition"), + "next": "Pass `workflow` to save_workflow to keep it in the catalog " + "under a name, or edit it and pass it to run_workflow as " + "inline_workflow.", + } + + def get_job_events(client, job_id, after=-1, limit=200): """One page of a job's progress events. `after` is exclusive - pass back the previous call's `last_seq` to continue.""" diff --git a/dw_mcp/exports.py b/dw_mcp/exports.py new file mode 100644 index 0000000..4d5ec9e --- /dev/null +++ b/dw_mcp/exports.py @@ -0,0 +1,44 @@ +"""Bundling a finished job so it can leave the server. + +The one thing this module has to keep saying: the directory it makes is on +the machine running dw.serve, which over a `dw.serve --mcp` endpoint is the +GPU box and not where the agent is. The zip URL is the way to it from +anywhere else. +""" + +from dw_mcp.client import api_path + + +def export_job(client, job_id, overwrite=False): + """Gather one finished job into a directory on the machine running + dw.serve: workflow.json (realized), manifest.json, job.json, README, + assets/, inputs/, outputs/. The export copies every output and input + file rather than linking them, so a video job's export costs its size + again on the server's disk; `total_bytes` in the result reports what + was copied. Returns the directory, the zip URL, the file list with + sizes and the total, and the three JSON files inline. The directory is + on the server machine, not this one - use the zip URL to fetch it + elsewhere.""" + body = client.post_json( + api_path("api", "jobs", job_id, "export"), + params={"overwrite": "true" if overwrite else "false"}, + ) + directory = body.get("directory") + return { + "job_id": job_id, + "where": f"{directory} on the machine running the MCP server", + "directory": directory, + "zip_url": body.get("zip_url"), + "files": body.get("files") or [], + "total_bytes": body.get("total_bytes"), + "missing": body.get("missing") or [], + "workflow": body.get("workflow"), + "manifest": body.get("manifest"), + "job": body.get("job"), + "next": "The directory is on the server. To give the user the files, " + "fetch zip_url and unpack it into exports/ under the session's " + "working directory - it is the user's deliverable, not a temporary " + "file, so not a scratch or temp directory. The archive already " + "unpacks into one folder named after the job id; do not create " + "that folder first or the id is doubled in the path.", + } diff --git a/dw_mcp/server.py b/dw_mcp/server.py index 4d3c7ba..409979d 100644 --- a/dw_mcp/server.py +++ b/dw_mcp/server.py @@ -17,6 +17,7 @@ authoring, catalog, diagnose, + exports, guides, media, models, @@ -612,6 +613,17 @@ def get_job(job_id: str) -> dict: are what to read before changing anything.""" return diagnose.get_job(client, job_id) + def get_job_workflow(job_id: str) -> dict: + """Get the workflow a job actually ran. When `realized` is true every + mutable input is pinned - the caller's arguments folded into the + variables, the seed the run used, stored prompt text inlined, and any + `output:.../latest/...` rewritten to the run it resolved to - so the + definition reproduces that run however the library changes. When it is + false the job predates run tracking and this is the definition as + submitted. After a long inline run worth keeping, this then + `save_workflow` is how it gets a name.""" + return diagnose.get_job_workflow(client, job_id) + def get_job_events(job_id: str, after: int = -1, limit: int = 200) -> dict: """Get a page of a job's progress events - phase transitions, memory readings and log lines. `after` is exclusive: pass back the previous @@ -648,10 +660,29 @@ def move_job( already running cannot.""" return diagnose.move_job(client, job_id, direction) + def export_job(job_id: str, overwrite: bool = False) -> dict: + """Gather one finished job into a directory on the server: the + realized workflow, the run's manifest, the job row, a README, and + copies of every asset it used, every earlier run's file it read and + every file it made. The export copies every output and input file + rather than linking them, so a video job's export costs its size + again on the server's disk; `total_bytes` in the result reports + what was copied. Returns the directory, a zip URL, the file list + with sizes and the total, and the three JSON files inline. THE + DIRECTORY IS ON THE MACHINE RUNNING THE SERVER, not on yours. To give + the user the files, fetch the zip URL and unpack it into exports/ + under the session's working directory - it is the user's deliverable, + not a temp file; the archive already unpacks into one folder named + after the job id, so do not create that folder first. Refuses a job + that is still running; refuses an existing export unless + overwrite=true.""" + return exports.export_job(client, job_id, overwrite=overwrite) + tool(get_job, READ_ONLY) + tool(get_job_workflow, READ_ONLY) tool(get_job_events, READ_ONLY) tool(wait_for_job, READ_ONLY) - for fn in (run_workflow, cancel_job, rerun_job, move_job): + for fn in (run_workflow, cancel_job, rerun_job, move_job, export_job): tool(fn, WRITES) # -------------------------------------------------------------- models diff --git a/plugins/dw/skills/ltx-2.5/SKILL.md b/plugins/dw/skills/ltx-2.5/SKILL.md index 6f17736..a6b2d01 100644 --- a/plugins/dw/skills/ltx-2.5/SKILL.md +++ b/plugins/dw/skills/ltx-2.5/SKILL.md @@ -134,6 +134,13 @@ AESTHETIC QUALITY (in addition to the above, without breaking the objective capt where the prompt contradicted the image; softness where the refine pass was skipped; a near-silent soundtrack where the caption gave the sound nothing to do. +5. After an inline run worth keeping, `get_job_workflow` and `save_workflow` it, + so the next run is by name rather than by pasting JSON; `export_job` bundles + the run — workflow, manifest, job row and media — for git. The bundle is on + the server: fetch its zip URL and unpack it into `exports/` under the + session's working directory, never a temp directory, and do not make a + folder named after the job id first, since the archive already unpacks + into one. ## Sources diff --git a/plugins/dw/skills/minimax-h3/SKILL.md b/plugins/dw/skills/minimax-h3/SKILL.md index 16e564b..d2e23ab 100644 --- a/plugins/dw/skills/minimax-h3/SKILL.md +++ b/plugins/dw/skills/minimax-h3/SKILL.md @@ -150,6 +150,13 @@ inherits the portrait's composition. failure modes: a character that changes between shots (reference the same portraits in every shot), a reference portrait imposing its framing on every shot, a storyboard skipped, drift sharpening into noise late in a chain. +5. After an inline run worth keeping, `get_job_workflow` and `save_workflow` it, + so the next run is by name rather than by pasting JSON; `export_job` bundles + the run — workflow, manifest, job row and media — for git. The bundle is on + the server: fetch its zip URL and unpack it into `exports/` under the + session's working directory, never a temp directory, and do not make a + folder named after the job id first, since the archive already unpacks + into one. ## Sources diff --git a/plugins/dw/skills/minimax-music3/SKILL.md b/plugins/dw/skills/minimax-music3/SKILL.md index f06db19..fe1f575 100644 --- a/plugins/dw/skills/minimax-music3/SKILL.md +++ b/plugins/dw/skills/minimax-music3/SKILL.md @@ -119,6 +119,13 @@ Control" section. the tags (fewer sections, plainer directions). 5. To use the track in a later workflow, `keep_output` makes it an `asset:`; to trim it in the same run, chain `templates/audio-trim-fade` on the output. +6. After an inline run worth keeping, `get_job_workflow` and `save_workflow` it, + so the next run is by name rather than by pasting JSON; `export_job` bundles + the run — workflow, manifest, job row and media — for git. The bundle is on + the server: fetch its zip URL and unpack it into `exports/` under the + session's working directory, never a temp directory, and do not make a + folder named after the job id first, since the archive already unpacks + into one. ## Sources diff --git a/tests/test_events.py b/tests/test_events.py index 161de25..aa3ebf4 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -85,13 +85,14 @@ def test_progress_event_sequence(): _run(_workflow_def(), context) names = [event["event"] for event in events] - assert names[0] == "workflow_start" + assert names[0] == "run_start" + assert names[1] == "workflow_start" assert names[-1] == "workflow_end" assert "step_start" in names and "step_end" in names assert "iteration_start" in names assert names.count("pipeline_step") == 3 - start = events[0] + start = events[1] assert start["workflow"] == "events_test" assert start["total_steps"] == 1 and start["steps"] == ["gen0"] assert isinstance(start["seed"], int) diff --git a/tests/test_mcp_diagnose.py b/tests/test_mcp_diagnose.py index bf2ee84..cc2aee4 100644 --- a/tests/test_mcp_diagnose.py +++ b/tests/test_mcp_diagnose.py @@ -332,3 +332,58 @@ def test_wait_for_job_does_not_require_acknowledged_cost(): result = diagnose.wait_for_job(client, "job-1") assert result["status"] == "succeeded" + + +class TestGetJobWorkflow: + def test_a_realized_workflow_comes_back_with_the_flag_set(self): + client, seen = scripted( + { + ("GET", "/api/jobs/job-1/workflow"): ( + 200, + {"id": "job-1", "definition": WORKFLOW, "realized": True}, + ) + } + ) + + result = diagnose.get_job_workflow(client, "job-1") + + assert result["job_id"] == "job-1" + assert result["realized"] is True + assert result["workflow"] == WORKFLOW + assert len(seen) == 1 + + def test_a_pre_tracking_job_reports_the_submitted_definition(self): + client, _ = scripted( + { + ("GET", "/api/jobs/job-1/workflow"): ( + 200, + {"id": "job-1", "definition": WORKFLOW, "realized": False}, + ) + } + ) + + result = diagnose.get_job_workflow(client, "job-1") + + assert result["realized"] is False + assert result["workflow"] == WORKFLOW + + def test_next_names_the_two_tools_that_use_it(self): + client, _ = scripted( + { + ("GET", "/api/jobs/job-1/workflow"): ( + 200, + {"id": "job-1", "definition": WORKFLOW, "realized": True}, + ) + } + ) + + result = diagnose.get_job_workflow(client, "job-1") + + assert "save_workflow" in result["next"] + assert "run_workflow" in result["next"] + + def test_an_unknown_job_raises_the_client_error(self): + client, _ = scripted({}) + + with pytest.raises(DwApiError): + diagnose.get_job_workflow(client, "nope") diff --git a/tests/test_mcp_exports.py b/tests/test_mcp_exports.py new file mode 100644 index 0000000..cf1876e --- /dev/null +++ b/tests/test_mcp_exports.py @@ -0,0 +1,112 @@ +"""Exporting a job over MCP: the directory is on the server, and the tool +says so - the lesson download_output taught.""" + +import httpx +import pytest + +from dw_mcp import exports +from dw_mcp.client import DwApiError, DwClient + +SUMMARY = { + "job_id": "job-1", + "directory": "/srv/studio/exports/job-1", + "files": [ + {"path": "workflow.json", "bytes": 412}, + {"path": "outputs/still.png", "bytes": 90210}, + ], + "total_bytes": 90622, + "missing": [], + "zip_url": "/exports/job-1.zip", + "workflow": {"id": "w", "steps": []}, + "manifest": {"run_id": "20260908-120000-abcdef01"}, + "job": {"id": "job-1", "status": "succeeded"}, +} + + +def scripted(routes): + seen = [] + + def handler(request): + key = (request.method, request.url.path) + seen.append({"key": key, "params": dict(request.url.params)}) + if key not in routes: + return httpx.Response(404, json={"detail": f"unrouted {key}"}) + status, body = routes[key] + return httpx.Response(status, json=body) + + return DwClient(transport=httpx.MockTransport(handler)), seen + + +def exporting(status=201, body=None): + return scripted({("POST", "/api/jobs/job-1/export"): (status, body or SUMMARY)}) + + +def test_it_returns_the_directory_the_zip_and_the_file_list(): + client, seen = exporting() + + result = exports.export_job(client, "job-1") + + assert result["job_id"] == "job-1" + assert result["directory"] == "/srv/studio/exports/job-1" + assert result["zip_url"] == "/exports/job-1.zip" + assert result["total_bytes"] == 90622 + assert [entry["path"] for entry in result["files"]] == [ + "workflow.json", + "outputs/still.png", + ] + assert len(seen) == 1 + + +def test_the_three_json_files_come_back_inline(): + client, _ = exporting() + + result = exports.export_job(client, "job-1") + + assert result["workflow"] == {"id": "w", "steps": []} + assert result["manifest"]["run_id"] == "20260908-120000-abcdef01" + assert result["job"]["status"] == "succeeded" + + +def test_it_says_where_the_directory_is(): + client, _ = exporting() + + result = exports.export_job(client, "job-1") + + assert result["where"] == ( + "/srv/studio/exports/job-1 on the machine running the MCP server" + ) + + +def test_overwrite_travels_as_a_query_parameter(): + client, seen = exporting() + + exports.export_job(client, "job-1", overwrite=True) + + assert seen[0]["params"]["overwrite"] == "true" + + +def test_a_409_reaches_the_model_as_a_readable_refusal(): + client, _ = exporting(status=409, body={"detail": "An export already exists"}) + + with pytest.raises(DwApiError) as caught: + exports.export_job(client, "job-1") + + assert "already exists" in str(caught.value) + + +def test_the_docstring_says_copying_costs_disk_and_names_total_bytes(): + # A caller reading only the handler's docstring has to learn this before + # exporting a video job fills the server's disk a second time - the + # export copies files rather than linking them. + assert "copies every output and input" in exports.export_job.__doc__ + assert "total_bytes" in exports.export_job.__doc__ + + +def test_the_next_hint_sends_the_zip_to_the_working_directory(): + """The drill showed an agent unpacking the export into its scratchpad + and doubling the job id in the path; the hint is where that is steered.""" + client, _ = exporting() + hint = exports.export_job(client, "job-1")["next"] + assert "working directory" in hint + assert "temp" in hint + assert "do not create that folder" in hint diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 5458c45..9ffb10f 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -39,11 +39,13 @@ "delete_workflow", "run_workflow", "get_job", + "get_job_workflow", "get_job_events", "wait_for_job", "cancel_job", "rerun_job", "move_job", + "export_job", "download_model", "list_downloads", "cancel_download", @@ -76,6 +78,7 @@ "cancel_job", "rerun_job", "move_job", + "export_job", "download_model", "cancel_download", "delete_model", @@ -343,6 +346,7 @@ def _a_file_to_upload(): "/api/jobs", ), ("get_job", {"job_id": "j1"}, "GET", "/api/jobs/j1"), + ("get_job_workflow", {"job_id": "j1"}, "GET", "/api/jobs/j1/workflow"), ("get_job_events", {"job_id": "j1"}, "GET", "/api/jobs/j1/event-log"), ( # timeout_seconds=0 keeps this to the single poll the wiring test @@ -363,6 +367,7 @@ def _a_file_to_upload(): "/api/jobs/j1/rerun", ), ("move_job", {"job_id": "j1", "direction": "up"}, "POST", "/api/jobs/j1/move"), + ("export_job", {"job_id": "j1"}, "POST", "/api/jobs/j1/export"), ( "download_model", {"repo_id": "org/model", "acknowledged_cost": True}, @@ -602,6 +607,24 @@ async def test_rerun_job_advertises_its_cost(): assert "acknowledged_cost" in description +@pytest.mark.asyncio +async def test_export_job_warns_the_copy_costs_disk_and_names_total_bytes(): + tools = await tools_of(server_over(ok({}))) + + description = tools["export_job"].description + assert "copies every output and input" in description + assert "total_bytes" in description + + +@pytest.mark.asyncio +async def test_export_job_sends_the_zip_to_the_working_directory(): + tools = await tools_of(server_over(ok({}))) + + description = tools["export_job"].description + assert "working directory" in description + assert "do not create that folder first" in description + + @pytest.mark.asyncio async def test_rerun_job_refuses_without_acknowledgement_and_sends_nothing(): seen = [] diff --git a/tests/test_realize.py b/tests/test_realize.py new file mode 100644 index 0000000..db76cf3 --- /dev/null +++ b/tests/test_realize.py @@ -0,0 +1,244 @@ +"""Realization: a copy of a workflow with every mutable input pinned, so the +file beside a run's manifest reproduces that run whatever changes later.""" + +import copy +import hashlib +import json +import os + +import pytest + +from dw.realize import realize_workflow, strings_with_prefix +from dw.runs import new_run_id +from dw.schema import load_schema, validate_data + + +def definition(): + return { + "id": "realize_test", + "variables": {"prompt": "a default", "steps": 25}, + "steps": [ + { + "name": "gen", + "pipeline": { + "configuration": {"component_type": "{Fake}"}, + "from_pretrained_arguments": {"model_name": "m"}, + "arguments": { + "prompt": "variable:prompt", + "num_inference_steps": "variable:steps", + }, + }, + } + ], + } + + +@pytest.fixture +def prompt_library(tmp_path): + library = tmp_path / "prompts" + (library / "scenic").mkdir(parents=True) + (library / "scenic" / "dusk.json").write_text( + json.dumps({"text": "a harbour at dusk"}) + ) + return str(library) + + +@pytest.fixture +def output_root(tmp_path): + """An output root holding one finished run of 'ltx2/Gyre'.""" + root = tmp_path / "outputs" + run_id = new_run_id({"a": 1}) + run = root / "ltx2" / "Gyre" / run_id + run.mkdir(parents=True) + (run / "still.png").write_bytes(b"not really a png") + return str(root), run_id + + +class TestVariablesAndSeed: + def test_arguments_become_the_variable_defaults(self): + realized, _ = realize_workflow(definition(), {"prompt": "a cat", "steps": 4}, 7) + assert realized["variables"] == {"prompt": "a cat", "steps": 4} + + def test_variable_references_are_left_alone(self): + realized, _ = realize_workflow(definition(), {"prompt": "a cat"}, 7) + arguments = realized["steps"][0]["pipeline"]["arguments"] + assert arguments["prompt"] == "variable:prompt" + + def test_the_seed_is_written_even_when_the_definition_had_none(self): + realized, _ = realize_workflow(definition(), {}, 991) + assert realized["seed"] == 991 + + def test_a_seed_pinned_through_a_variable_reference_updates_that_variable(self): + source = definition() + source["seed"] = "variable:seed_arg" + source["variables"]["seed_arg"] = None + + realized, _ = realize_workflow(source, {}, 991) + + assert realized["seed"] == 991 + assert realized["variables"]["seed_arg"] == 991 + + def test_the_input_definition_is_not_mutated(self): + original = definition() + before = copy.deepcopy(original) + realize_workflow(original, {"prompt": "a cat"}, 7) + assert original == before + + +class TestPrompts: + def test_a_stored_prompt_is_inlined_and_annotated(self, prompt_library): + source = definition() + source["steps"][0]["pipeline"]["arguments"]["prompt"] = "prompt:scenic/dusk" + + realized, annotations = realize_workflow( + source, {}, 7, prompt_dir=prompt_library + ) + + arguments = realized["steps"][0]["pipeline"]["arguments"] + assert arguments["prompt"] == "a harbour at dusk" + assert annotations["prompts"] == ["scenic/dusk"] + + def test_a_name_is_annotated_once_in_first_seen_order(self, prompt_library): + source = definition() + source["steps"][0]["pipeline"]["arguments"]["prompt"] = "prompt:scenic/dusk" + source["steps"][0]["pipeline"]["arguments"][ + "negative_prompt" + ] = "prompt:scenic/dusk" + + _, annotations = realize_workflow(source, {}, 7, prompt_dir=prompt_library) + + assert annotations["prompts"] == ["scenic/dusk"] + + def test_an_unresolvable_prompt_is_left_as_written(self, prompt_library): + source = definition() + source["steps"][0]["pipeline"]["arguments"]["prompt"] = "prompt:missing" + + realized, annotations = realize_workflow( + source, {}, 7, prompt_dir=prompt_library + ) + + assert realized["steps"][0]["pipeline"]["arguments"]["prompt"] == ( + "prompt:missing" + ) + assert annotations["prompts"] == [] + + +class TestOutputReferences: + def test_latest_is_pinned_to_the_run_it_resolved_to(self, output_root): + root, run_id = output_root + source = definition() + source["steps"][0]["pipeline"]["arguments"][ + "image" + ] = "output:ltx2/Gyre/latest/still.png" + + realized, _ = realize_workflow(source, {}, 7, output_root=root) + + assert realized["steps"][0]["pipeline"]["arguments"]["image"] == ( + f"output:ltx2/Gyre/{run_id}/still.png" + ) + + def test_an_explicit_run_id_is_kept_as_written(self, output_root): + root, run_id = output_root + written = f"output:ltx2/Gyre/{run_id}/still.png" + source = definition() + source["steps"][0]["pipeline"]["arguments"]["image"] = written + + realized, _ = realize_workflow(source, {}, 7, output_root=root) + + assert realized["steps"][0]["pipeline"]["arguments"]["image"] == written + + def test_an_unresolvable_output_is_left_as_written(self, output_root): + root, _ = output_root + written = "output:ltx2/Nothing/latest/still.png" + source = definition() + source["steps"][0]["pipeline"]["arguments"]["image"] = written + + realized, _ = realize_workflow(source, {}, 7, output_root=root) + + assert realized["steps"][0]["pipeline"]["arguments"]["image"] == written + + +class TestReferencesThatAreKept: + @pytest.mark.parametrize( + "value", + [ + "asset:iris.png", + "constant:diffusers.pipelines.ltx2.utils.DISTILLED_SIGMA_VALUES", + "previous_result:gen", + ], + ) + def test_kept_verbatim(self, value): + source = definition() + source["steps"][0]["pipeline"]["arguments"]["thing"] = value + + realized, _ = realize_workflow(source, {}, 7) + + assert realized["steps"][0]["pipeline"]["arguments"]["thing"] == value + + +class TestSubWorkflows: + def test_a_local_path_is_kept_and_digested(self, tmp_path): + tree = tmp_path / "workflows" + (tree / "steps").mkdir(parents=True) + child = tree / "steps" / "upscale.json" + child.write_text(json.dumps({"id": "child", "steps": []})) + + source = definition() + source["steps"].append( + {"name": "up", "workflow": {"path": "steps/upscale.json"}} + ) + + realized, annotations = realize_workflow( + source, {}, 7, base_dir=str(tree), workflow_dir=str(tree) + ) + + assert realized["steps"][1]["workflow"]["path"] == "steps/upscale.json" + digest = annotations["sub_workflows"]["steps/upscale.json"] + assert digest == hashlib.sha256(child.read_bytes()).hexdigest() + + def test_an_unreadable_sub_workflow_digests_to_null(self, tmp_path): + tree = tmp_path / "workflows" + tree.mkdir() + source = definition() + source["steps"].append({"name": "up", "workflow": {"path": "gone.json"}}) + + _, annotations = realize_workflow( + source, {}, 7, base_dir=str(tree), workflow_dir=str(tree) + ) + + assert annotations["sub_workflows"] == {"gone.json": None} + + def test_a_builtin_is_not_digested(self): + source = definition() + source["steps"].append( + {"name": "up", "workflow": {"path": "builtin:upscale.json"}} + ) + + realized, annotations = realize_workflow(source, {}, 7) + + assert realized["steps"][1]["workflow"]["path"] == "builtin:upscale.json" + assert annotations["sub_workflows"] == {} + + +def test_the_realized_file_validates_against_the_schema(prompt_library): + source = definition() + source["steps"][0]["pipeline"]["arguments"]["prompt"] = "prompt:scenic/dusk" + + realized, _ = realize_workflow(source, {"steps": 4}, 991, prompt_dir=prompt_library) + + ok, message = validate_data(realized, load_schema("workflow")) + assert ok, message + + +class TestStringsWithPrefix: + def test_finds_every_matching_string_deduplicated_in_first_seen_order(self): + tree = { + "a": "asset:iris.png", + "b": ["asset:iris.png", "output:x/y/still.png"], + "c": {"d": "asset:mask.png", "e": 5, "f": None}, + } + + assert strings_with_prefix(tree, "asset:") == [ + "asset:iris.png", + "asset:mask.png", + ] diff --git a/tests/test_runs.py b/tests/test_runs.py index 4baf4b3..6fe3139 100644 --- a/tests/test_runs.py +++ b/tests/test_runs.py @@ -340,3 +340,75 @@ def test_empty_steps_workflow_records_completed_status( manifest = json.loads((run_dir / "manifest.json").read_text()) assert manifest["status"] == "completed" assert manifest["workflow"]["id"] == "empty_steps_test" + + +class TestRealizedWorkflow: + def test_the_run_directory_holds_a_realized_copy(self, tmp_path, fake_pipeline): + from dw.workflow import Workflow + + definition = _workflow_definition() + definition["variables"] = {"prompt": "a default"} + definition["steps"][0]["pipeline"]["arguments"]["prompt"] = "variable:prompt" + Workflow(definition, str(tmp_path), "/w/workflows/ltx2/Gyre.json").run( + {"prompt": "a cat"} + ) + + run_dir = next((tmp_path / "ltx2" / "Gyre").iterdir()) + realized = json.loads((run_dir / "workflow.json").read_text()) + assert realized["variables"] == {"prompt": "a cat"} + assert realized["seed"] == 7 + # the reference stays, so the file is still runnable with overrides + assert realized["steps"][0]["pipeline"]["arguments"]["prompt"] == ( + "variable:prompt" + ) + + def test_the_manifest_points_at_it(self, tmp_path, fake_pipeline): + from dw.workflow import Workflow + + Workflow( + _workflow_definition(), str(tmp_path), "/w/workflows/ltx2/Gyre.json" + ).run({}) + + run_dir = next((tmp_path / "ltx2" / "Gyre").iterdir()) + manifest = json.loads((run_dir / "manifest.json").read_text()) + assert manifest["workflow"]["realized"] == "workflow.json" + assert manifest["workflow"]["prompts"] == [] + assert manifest["workflow"]["sub_workflows"] == {} + + def test_a_seedless_run_pins_the_seed_it_drew(self, tmp_path, fake_pipeline): + from dw.workflow import Workflow + + definition = _workflow_definition() + del definition["seed"] + Workflow(definition, str(tmp_path), "/w/workflows/ltx2/Gyre.json").run({}) + + run_dir = next((tmp_path / "ltx2" / "Gyre").iterdir()) + realized = json.loads((run_dir / "workflow.json").read_text()) + manifest = json.loads((run_dir / "manifest.json").read_text()) + assert isinstance(realized["seed"], int) + assert realized["seed"] == manifest["seed"] + + def test_the_flat_layout_writes_none(self, tmp_path, fake_pipeline, monkeypatch): + from dw.workflow import Workflow + + monkeypatch.setenv(OUTPUT_LAYOUT_ENV_VAR, FLAT_LAYOUT) + Workflow( + _workflow_definition(), str(tmp_path), "/w/workflows/ltx2/Gyre.json" + ).run({}) + + assert not list(tmp_path.rglob("workflow.json")) + + def test_writing_it_is_best_effort(self, tmp_path): + from dw.runs import write_realized_workflow + + # a run directory that cannot be made - the run still succeeded + blocker = tmp_path / "blocker" + blocker.write_text("not a directory") + assert write_realized_workflow(str(blocker / "run"), {"id": "x"}) is None + + def test_writing_it_returns_the_path(self, tmp_path): + from dw.runs import write_realized_workflow + + path = write_realized_workflow(str(tmp_path / "run"), {"id": "x"}) + assert path == str(tmp_path / "run" / "workflow.json") + assert json.loads(open(path).read()) == {"id": "x"} diff --git a/tests/test_server.py b/tests/test_server.py index 3c8f7bf..919bfd3 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -432,6 +432,20 @@ def test_job_workflow_404s_when_the_file_is_gone(server, tmp_path): assert client.get(f"/api/jobs/{job['id']}").status_code == 200 +def test_job_workflow_reports_whether_it_is_realized(server): + """A job with no run record falls back to the submitted definition and + says so - the flag is what tells a client which it is looking at.""" + with server(success_script) as client: + submitted = client.post( + "/api/jobs", json={"workflow": valid_workflow(), "arguments": {}} + ).json() + wait_for_status(client, submitted["id"], TERMINAL_STATES) + body = client.get(f"/api/jobs/{submitted['id']}/workflow").json() + + assert body["realized"] is False + assert body["definition"]["id"] == "server_test" + + def test_submit_rejects_a_real_path_outside_the_workflow_dir(server, tmp_path): """workflow_path is confined to --workflow-dir, the same as the /api/workflows CRUD routes - a real, existing file elsewhere on disk @@ -2626,6 +2640,8 @@ class FinishedJob: manifest = [] warnings = [] error = None + run_id = None + run_dir = None spec = {"arguments": {}, "workflow_path": "w.json"} job = FinishedJob() diff --git a/tests/test_server_exports.py b/tests/test_server_exports.py new file mode 100644 index 0000000..6727648 --- /dev/null +++ b/tests/test_server_exports.py @@ -0,0 +1,416 @@ +"""Exporting one finished job: a directory that stands on its own, and the +same tree as a zip.""" + +import io +import json +import os +import zipfile + +import pytest +from fastapi.testclient import TestClient + +from dw.runs import REALIZED_FILE_NAME, new_run_id +from dw.server.app import create_app +from dw.server.jobs import JobManager, TERMINAL_STATES +from dw.workspace import Workspace, create_workspace + +from .test_server import ( + ScriptedWorkerManager, + hanging_script, + server as workspace_less_server, + success_script, + valid_workflow, + wait_for_status, +) + +RUN_ID = new_run_id({"workflow": "export"}) +RUN_DIR = f"server_test/{RUN_ID}" + + +def exporting_script(command): + """A run that reports its directory and writes one file.""" + output_dir = command["output_dir"] + run_dir = os.path.join(output_dir, "server_test", RUN_ID) + os.makedirs(run_dir, exist_ok=True) + with open(os.path.join(run_dir, "still.png"), "wb") as file: + file.write(b"an image") + with open(os.path.join(run_dir, REALIZED_FILE_NAME), "w") as file: + json.dump( + { + "id": "server_test", + "seed": 7, + "steps": [ + { + "name": "gen", + "pipeline": { + "configuration": {"component_type": "{Fake}"}, + "from_pretrained_arguments": {"model_name": "m"}, + "arguments": {"image": "asset:iris.png"}, + }, + } + ], + }, + file, + ) + with open(os.path.join(run_dir, "manifest.json"), "w") as file: + json.dump( + { + "run_id": RUN_ID, + "status": "completed", + "seed": 7, + "steps": [{"step": "gen", "files": ["still.png"]}], + }, + file, + ) + yield { + "type": "progress", + "event": "run_start", + "run_id": RUN_ID, + "identity": "server_test", + "run_dir": RUN_DIR, + } + yield { + "type": "success", + "message": "ok", + "run_count": 1, + "manifest": [{"step": "gen", "files": [os.path.join(run_dir, "still.png")]}], + } + + +PRIOR_RUN_ID = new_run_id({"workflow": "prior"}) +PRIOR_REFERENCE = f"output:prior/{PRIOR_RUN_ID}/prior.png" + + +def untracked_script(command): + """A run from before run tracking: no run_start event, so the job never + learns a run directory and its own row is the only manifest there is.""" + output_dir = command["output_dir"] + os.makedirs(output_dir, exist_ok=True) + path = os.path.join(output_dir, "legacy.png") + with open(path, "wb") as file: + file.write(b"an older image") + yield { + "type": "success", + "message": "ok", + "run_count": 1, + "manifest": [{"step": "gen", "files": [path]}], + } + + +def chained_script(command): + """A run whose realized workflow names an earlier run's file.""" + output_dir = command["output_dir"] + prior_dir = os.path.join(output_dir, "prior", PRIOR_RUN_ID) + os.makedirs(prior_dir, exist_ok=True) + with open(os.path.join(prior_dir, "prior.png"), "wb") as file: + file.write(b"the first stage") + + run_dir = os.path.join(output_dir, "server_test", RUN_ID) + os.makedirs(run_dir, exist_ok=True) + with open(os.path.join(run_dir, "still.png"), "wb") as file: + file.write(b"an image") + with open(os.path.join(run_dir, REALIZED_FILE_NAME), "w") as file: + json.dump( + { + "id": "server_test", + "seed": 7, + "steps": [ + { + "name": "gen", + "pipeline": { + "configuration": {"component_type": "{Fake}"}, + "from_pretrained_arguments": {"model_name": "m"}, + "arguments": {"image": PRIOR_REFERENCE}, + }, + } + ], + }, + file, + ) + with open(os.path.join(run_dir, "manifest.json"), "w") as file: + json.dump( + { + "run_id": RUN_ID, + "status": "completed", + "seed": 7, + # The same file twice: a chain's step names the one before + # it, and the export holds one copy of it + "steps": [ + {"step": "gen", "files": ["still.png"]}, + {"step": "post", "files": ["still.png"]}, + ], + }, + file, + ) + yield { + "type": "progress", + "event": "run_start", + "run_id": RUN_ID, + "identity": "server_test", + "run_dir": RUN_DIR, + } + yield { + "type": "success", + "message": "ok", + "run_count": 1, + "manifest": [{"step": "gen", "files": [os.path.join(run_dir, "still.png")]}], + } + + +@pytest.fixture +def workspace_root(tmp_path): + root = Workspace(tmp_path / "studio", "flag").ensure() + with open(os.path.join(root.assets, "iris.png"), "wb") as file: + file.write(b"an iris") + return root + + +@pytest.fixture +def server(workspace_root, tmp_path): + def make(script=exporting_script): + manager = JobManager( + workspace_root.outputs, + worker_manager=ScriptedWorkerManager(script), + history_path=str(tmp_path / "jobs.sqlite"), + workflow_dir=workspace_root.workflows, + ) + # The test's handle on the manager the app is talking to, so a test + # can evict a finished job and make the server answer from history + make.manager = manager + app = create_app( + workflow_dir=workspace_root.workflows, + output_dir=workspace_root.outputs, + job_manager=manager, + prompt_dir=workspace_root.prompts, + asset_dir=workspace_root.assets, + workspace=workspace_root.root, + ) + return TestClient(app, base_url="http://localhost") + + return make + + +def finished(client): + submitted = client.post( + "/api/jobs", json={"workflow": valid_workflow(), "arguments": {}} + ).json() + wait_for_status(client, submitted["id"], TERMINAL_STATES) + return submitted["id"] + + +class TestExportDirectory: + def test_it_gathers_the_whole_run(self, server, workspace_root): + with server() as client: + job_id = finished(client) + response = client.post(f"/api/jobs/{job_id}/export") + + assert response.status_code == 201 + body = response.json() + directory = body["directory"] + assert directory == os.path.join(workspace_root.root, "exports", job_id) + for name in ("README.md", "workflow.json", "manifest.json", "job.json"): + assert os.path.isfile(os.path.join(directory, name)) + assert os.path.isfile(os.path.join(directory, "assets", "iris.png")) + assert os.path.isfile(os.path.join(directory, "outputs", "still.png")) + assert body["total_bytes"] > 0 + assert body["missing"] == [] + assert body["zip_url"] == f"/exports/{job_id}.zip" + + def test_the_workflow_is_the_realized_one(self, server): + with server() as client: + job_id = finished(client) + body = client.post(f"/api/jobs/{job_id}/export").json() + + recorded = json.loads(open(os.path.join(body["directory"], "job.json")).read()) + assert recorded["realized"] is True + assert "traceback" not in recorded and "event_count" not in recorded + assert body["workflow"]["seed"] == 7 + + def test_a_historical_job_records_the_same_keys(self, server): + with server() as client: + live_id = finished(client) + live = client.post(f"/api/jobs/{live_id}/export").json()["job"] + + job_id = finished(client) + # Evicted from memory: the server has to answer from the + # history store, whose dict is a different shape + server.manager.jobs.clear() + historical = client.post(f"/api/jobs/{job_id}/export").json()["job"] + + assert sorted(historical) == sorted(live) + assert "spec" not in historical and "historical" not in historical + assert historical["id"] == job_id + assert historical["run_dir"] == RUN_DIR + + def test_a_job_with_no_run_directory_gets_a_synthesized_manifest(self, server): + with server(untracked_script) as client: + job_id = finished(client) + body = client.post(f"/api/jobs/{job_id}/export").json() + + directory = body["directory"] + assert body["manifest"]["synthesized"] is True + assert body["manifest"]["steps"] == [{"step": "gen", "files": ["legacy.png"]}] + # The job predates run tracking, so the workflow is the definition + # as submitted rather than a realized one - and the file the row + # names still lands, resolved against the output root itself + assert ( + json.loads(open(os.path.join(directory, "job.json")).read())["realized"] + is False + ) + assert os.path.isfile(os.path.join(directory, "outputs", "legacy.png")) + assert body["missing"] == [] + + def test_an_output_reference_is_copied_under_the_run_it_names(self, server): + with server(chained_script) as client: + job_id = finished(client) + body = client.post(f"/api/jobs/{job_id}/export").json() + + copied = os.path.join("inputs", "prior", PRIOR_RUN_ID, "prior.png") + assert os.path.isfile(os.path.join(body["directory"], copied)) + paths = [entry["path"] for entry in body["files"]] + assert copied.replace(os.sep, "/") in paths + # Listed by two steps, copied and counted once + assert paths.count("outputs/still.png") == 1 + assert body["missing"] == [] + + def test_the_readme_names_the_job_and_says_how_to_run_it(self, server): + with server() as client: + job_id = finished(client) + body = client.post(f"/api/jobs/{job_id}/export").json() + + readme = open(os.path.join(body["directory"], "README.md")).read() + assert job_id in readme + assert "python -m dw.run workflow.json" in readme + assert "Git LFS" in readme + + def test_the_job_s_own_asset_dir_is_used_not_the_export_s_workspace( + self, server, workspace_root + ): + # The job ran in workspace 'a', whose asset library holds iris.png; + # exporting it while scoped to workspace 'b' - which has no assets + # of its own - must still find iris.png through the job's own + # asset_dir, not report it missing because 'b' lacks it. + a = create_workspace(workspace_root, "a") + create_workspace(workspace_root, "b") + with open(os.path.join(a.assets, "iris.png"), "wb") as file: + file.write(b"an iris") + + with server() as client: + submitted = client.post( + "/api/jobs", + json={ + "workflow": valid_workflow(), + "arguments": {}, + "workspace": "a", + }, + ).json() + wait_for_status(client, submitted["id"], TERMINAL_STATES) + response = client.post(f"/api/jobs/{submitted['id']}/export?workspace=b") + + assert response.status_code == 201 + body = response.json() + assert body["directory"] == os.path.join( + workspace_root.root, "b", "exports", submitted["id"] + ) + assert os.path.isfile(os.path.join(body["directory"], "assets", "iris.png")) + assert body["missing"] == [] + + def test_an_unresolvable_asset_is_reported_missing(self, server, workspace_root): + os.unlink(os.path.join(workspace_root.assets, "iris.png")) + with server() as client: + job_id = finished(client) + body = client.post(f"/api/jobs/{job_id}/export").json() + + assert body["missing"] == ["asset:iris.png"] + + def test_an_unknown_job_is_404(self, server): + with server() as client: + assert client.post("/api/jobs/nope/export").status_code == 404 + + def test_a_live_job_is_409(self, server): + with server(hanging_script) as client: + submitted = client.post( + "/api/jobs", json={"workflow": valid_workflow(), "arguments": {}} + ).json() + wait_for_status(client, submitted["id"], ("running",)) + response = client.post(f"/api/jobs/{submitted['id']}/export") + client.post(f"/api/jobs/{submitted['id']}/cancel") + + assert response.status_code == 409 + + def test_a_second_export_without_overwrite_is_409(self, server): + with server() as client: + job_id = finished(client) + assert client.post(f"/api/jobs/{job_id}/export").status_code == 201 + again = client.post(f"/api/jobs/{job_id}/export") + assert again.status_code == 409 + forced = client.post(f"/api/jobs/{job_id}/export?overwrite=true") + assert forced.status_code == 201 + + +class TestExportWithoutAWorkspace: + def test_a_server_with_no_workspace_root_answers_409_not_a_crash( + self, workspace_less_server + ): + # create_app(workspace=None) - the fixture in test_server.py, used + # by everything that predates workspaces - leaves the default + # workspace's 'root' None. export_directory used to hand that + # straight to os.path.join and blow up with a TypeError; it must + # answer 409 like any other export that cannot proceed. + with workspace_less_server(success_script) as client: + submitted = client.post( + "/api/jobs", json={"workflow": valid_workflow(), "arguments": {}} + ).json() + wait_for_status(client, submitted["id"], TERMINAL_STATES) + response = client.post(f"/api/jobs/{submitted['id']}/export") + + assert response.status_code == 409 + assert "workspace" in response.json()["detail"] + + +class TestExportZip: + def test_it_lists_the_same_entries_as_the_directory(self, server): + with server() as client: + job_id = finished(client) + body = client.post(f"/api/jobs/{job_id}/export").json() + response = client.get(f"/exports/{job_id}.zip") + + assert response.status_code == 200 + archive = zipfile.ZipFile(io.BytesIO(response.content)) + assert sorted(archive.namelist()) == sorted( + f"{job_id}/{entry['path']}" for entry in body["files"] + ) + + def test_a_dot_does_not_archive_the_whole_exports_folder(self, server): + # validate_path accepts a path equal to its root, so without a + # shape check '.' would resolve to exports/ itself and zip every job + with server() as client: + job_id = finished(client) + client.post(f"/api/jobs/{job_id}/export") + response = client.get("/exports/..zip") + + assert response.status_code == 404 + + def test_no_export_is_404(self, server): + with server() as client: + job_id = finished(client) + assert client.get(f"/exports/{job_id}.zip").status_code == 404 + + +class TestReservedName: + def test_exports_cannot_name_a_workspace(self, server): + with server() as client: + response = client.post("/api/workspaces", json={"name": "exports"}) + assert response.status_code == 400 + assert "cannot name a workspace" in response.json()["detail"] + + def test_an_exports_folder_is_not_listed_as_a_workspace( + self, server, workspace_root + ): + with server() as client: + job_id = finished(client) + client.post(f"/api/jobs/{job_id}/export") + names = [ + w["name"] for w in client.get("/api/workspaces").json()["workspaces"] + ] + assert names == ["default"] diff --git a/tests/test_server_jobs.py b/tests/test_server_jobs.py new file mode 100644 index 0000000..c02ed4c --- /dev/null +++ b/tests/test_server_jobs.py @@ -0,0 +1,121 @@ +"""The job store's record of which run a job was: the run_start event, the +two columns, and reading the realized workflow back off disk. + +The manager tests proper live in tests/test_server.py; this file is the run +record, which is a store concern rather than a routing one. +""" + +import json +import time + +import pytest + +from dw.runs import REALIZED_FILE_NAME, new_run_id +from dw.server.jobs import TERMINAL_STATES, JobHistory, JobManager + +from .test_server import ScriptedWorkerManager, valid_workflow + +RUN_ID = new_run_id({"workflow": "spec"}) +RUN_DIR = f"server_test/{RUN_ID}" + + +def tracked_script(command): + """A worker that reports its run before doing anything else - what + Workflow.run emits once the run directory is chosen.""" + yield { + "type": "progress", + "event": "run_start", + "run_id": RUN_ID, + "identity": "server_test", + "run_dir": RUN_DIR, + } + yield {"type": "success", "message": "ok", "run_count": 1, "manifest": []} + + +@pytest.fixture +def manager(tmp_path): + made = JobManager( + str(tmp_path / "outputs"), + worker_manager=ScriptedWorkerManager(tracked_script), + history_path=str(tmp_path / "jobs.sqlite"), + workflow_dir=str(tmp_path), + ) + yield made + made.shutdown() + + +def finished_job(manager): + job = manager.submit(workflow=valid_workflow(), base_dir=None) + deadline = time.time() + 5 + while job.status not in TERMINAL_STATES and time.time() < deadline: + time.sleep(0.01) + assert job.status == "succeeded", job.error + return job + + +def test_run_start_populates_the_job(manager): + job = finished_job(manager) + assert job.run_id == RUN_ID + assert job.run_dir == RUN_DIR + assert job.summary()["run_id"] == RUN_ID + assert job.detail()["run_dir"] == RUN_DIR + + +def test_both_persist_and_read_back(manager): + job = finished_job(manager) + historical = manager.history.get(job.id) + assert historical["run_id"] == RUN_ID + assert historical["run_dir"] == RUN_DIR + + +def test_realized_reads_the_file_the_run_wrote(manager, tmp_path): + job = finished_job(manager) + run_dir = tmp_path / "outputs" / "server_test" / RUN_ID + run_dir.mkdir(parents=True) + (run_dir / REALIZED_FILE_NAME).write_text(json.dumps({"id": "realized"})) + + assert manager.realized(job.id) == {"id": "realized"} + + +def test_realized_is_none_without_the_file(manager): + job = finished_job(manager) + assert manager.realized(job.id) is None + + +def test_realized_is_none_for_a_pre_tracking_row(manager): + """A job recorded before run tracking has no run_dir, and the manager + does not guess one from file paths.""" + job = finished_job(manager) + with manager.history._connect() as connection: + connection.execute( + "UPDATE jobs SET run_id = NULL, run_dir = NULL WHERE id = ?", (job.id,) + ) + manager.jobs.pop(job.id) + + assert manager.realized(job.id) is None + + +def test_realized_refuses_a_run_dir_that_escapes_the_output_root(manager, tmp_path): + job = finished_job(manager) + job.run_dir = "../../etc" + assert manager.realized(job.id) is None + + +def test_a_database_without_the_columns_is_migrated(tmp_path): + import sqlite3 + + path = str(tmp_path / "old.sqlite") + # A store written before run tracking: the ALTER is the whole migration + with sqlite3.connect(path) as connection: + connection.execute( + "CREATE TABLE jobs (id TEXT PRIMARY KEY, workflow TEXT, status TEXT," + " created_at REAL, started_at REAL, finished_at REAL, arguments TEXT," + " spec TEXT, manifest TEXT, warnings TEXT, error TEXT)" + ) + connection.execute( + "INSERT INTO jobs (id, status) VALUES ('old-1', 'succeeded')" + ) + + history = JobHistory(path) + row = history.get("old-1") + assert row["run_id"] is None and row["run_dir"] is None diff --git a/ui/src/lib/api.ts b/ui/src/lib/api.ts index cffae84..6539ec6 100644 --- a/ui/src/lib/api.ts +++ b/ui/src/lib/api.ts @@ -265,9 +265,12 @@ export const api = { ), getJob: (id: string) => request(`/api/jobs/${id}`), /** The definition a job ran, for the job page's read-only flow view. - * 404s when the job named a workflow file that is no longer readable. */ + * `realized` true means `definition` is the realized copy the run itself + * wrote - every mutable input pinned - rather than the definition as + * submitted. 404s when the job named a workflow file that is no longer + * readable. */ getJobWorkflow: (id: string) => - request<{ id: string; definition: Record }>( + request<{ id: string; definition: Record; realized: boolean }>( `/api/jobs/${id}/workflow`, ), rerunJob: (id: string) =>