From 94b53aab4446235e2357ef7fbfba9c911e47a452 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sat, 5 Sep 2026 18:03:16 +0000 Subject: [PATCH 01/34] feat(workspace): resolve one directory for workflows, prompts and outputs Content location was bound to the repository: three roots with three different discovery rules, all anchored to the process working directory. That was right for a CLI run from a checkout and stops being right once an agent or a remote client authors - generated media does not belong in a source tree, and an agent's workflows do not belong in the example corpus. dw/workspace.py resolves a single root holding workflows/, prompts/, assets/ and outputs/: --workspace, then DW_WORKSPACE, then the 'workspace' setting, then the working directory when it looks like a workspace, then ~/diffusers-workspace. A checkout satisfies the fourth rule, so every default lands exactly where it did before and nothing moves yet. The per-folder flags still override one folder each. set_workspace pins the root and how it was chosen, so a spawned worker does not read an inferred workspace back as one the user named - get_prompt_dir yields to its older discovery for an inferred workspace and not for an explicit one, which keeps a repo workflow reaching the library it lives beside. Stage one of docs/proposals/workspaces.md. Co-Authored-By: Claude Opus 5 (1M context) --- CLAUDE.md | 17 +++ README.md | 1 + docs/WORKSPACES.md | 92 +++++++++++++ docs/proposals/workspaces.md | 250 +++++++++++++++++++++++++++++++++++ dw/prompts.py | 35 +++-- dw/repl.py | 13 +- dw/repl_commands.py | 28 +++- dw/run.py | 22 ++- dw/serve.py | 43 +++++- dw/server/app.py | 9 ++ dw/settings.py | 6 + dw/workspace.py | 190 ++++++++++++++++++++++++++ tests/test_workspace.py | 177 +++++++++++++++++++++++++ 13 files changed, 859 insertions(+), 24 deletions(-) create mode 100644 docs/WORKSPACES.md create mode 100644 docs/proposals/workspaces.md create mode 100644 dw/workspace.py create mode 100644 tests/test_workspace.py diff --git a/CLAUDE.md b/CLAUDE.md index 9e952fea..9a7adde8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -51,6 +51,23 @@ The REPL (`dw/repl.py`) uses a **persistent worker subprocess** (`dw/worker.py`) **Critical**: Uses `multiprocessing.set_start_method("spawn")` for CUDA/MPS compatibility. +### Workspaces + +`dw/workspace.py` resolves the one directory a run's content belongs to - +`workflows/`, `prompts/`, `assets/`, `outputs/`. Order: `--workspace` > +`DW_WORKSPACE` > the `workspace` setting > the working directory when it holds +any of `workflows/`, `prompts/` or `outputs/` > `~/diffusers-workspace`. A +checkout satisfies rule four, so every default lands where it did before +workspaces existed. Resolution creates nothing; an entry point about to write +calls `ensure()` (or creates the one folder it needs). `set_workspace` pins the +root *and* how it was chosen into the environment, so a spawned worker does not +read an inferred workspace back as one the user named - `get_prompt_dir` yields +to its older discovery (`./prompts`, then the walk up from the workflow file) +for an inferred workspace but not for an explicit one. `--workflow-dir`, +`--output-dir` and `--prompt-dir` each still override one folder. See +docs/WORKSPACES.md, and docs/proposals/workspaces.md for the later stages +(workflow search path, run directories, `asset:`/`output:` references). + ### Type System `arguments.py` + `type_helpers.py` handle dynamic type conversion during workflow loading: diff --git a/README.md b/README.md index 37fc3832..096f3037 100644 --- a/README.md +++ b/README.md @@ -270,6 +270,7 @@ See [workflows/](workflows/) for more workflow files. - [MCP Server](docs/MCP.md) — Tool surface for MCP clients (Claude Code, Claude Desktop) - [Remote GPU server](docs/REMOTE.md) — Using the server, UI and MCP from another machine - [Workflow Guide](docs/WORKFLOW_GUIDE.md) — JSON structure, variables, steps, data flow +- [Workspaces](docs/WORKSPACES.md) — Keeping your workflows, prompts and outputs outside the repo - [Quantization](docs/QUANTIZATION.md) — BitsAndBytes, TorchAO, GGUF, SDNQ - [Inference Acceleration](docs/ACCELERATION.md) — torch.compile, FirstBlockCache, MagCache, TaylorSeer, TeaCache - [Fast on 24GB](docs/RECIPES_24GB.md) — Recommended speed/memory configurations per model family diff --git a/docs/WORKSPACES.md b/docs/WORKSPACES.md new file mode 100644 index 00000000..d0129a5d --- /dev/null +++ b/docs/WORKSPACES.md @@ -0,0 +1,92 @@ +# Workspaces + +A workspace is the directory your own content lives in: the workflows you +write, the prompt library they reference, the assets they read, and the files +they generate. + +``` +/ + workflows/ your workflows + prompts/ the stored prompt library ('prompt:' references) + assets/ input media + outputs/ generated files +``` + +This exists so that day-to-day work does not have to live inside a checkout of +this repository. The examples under the repo's `workflows/` are still examples +— a corpus to read, copy and run — but they are not where your own workflows +belong, and generated media does not belong in a source tree at all. + +## Which directory is used + +First match wins: + +1. `--workspace ` on `dw.run`, `dw.serve` (`config set workspace=` in the REPL) +2. the `DW_WORKSPACE` environment variable +3. `"workspace"` in `~/.diffusers_helper/settings.json` +4. the working directory, when it holds any of `workflows/`, `prompts/` or `outputs/` +5. `~/diffusers-workspace` + +Rule 4 is why nothing changes when you work from a checkout: the repository +root holds all three, so it resolves to itself and every default lands exactly +where it always has. Only a working directory with none of those folders falls +through to the home workspace. + +Nothing is created just by resolving. A command that is about to write creates +what it needs — `dw.run` creates its output directory, `dw.serve` creates the +workspace's `workflows/` so the UI has somewhere to save. + +## Overriding one folder + +The existing per-directory flags still work and each overrides exactly one +folder of the workspace: + +```bash +python -m dw.run workflows/sd15.json -o /mnt/big-disk/renders +python -m dw.serve --workspace ~/studio --output-dir /mnt/big-disk/renders +python -m dw.run some.json --prompt-dir ~/shared-prompts +``` + +`--output-dir` is the one people reach for most: video work fills disks, and +the outputs folder is the one worth putting on another volume. + +## Working in a workspace + +```bash +mkdir -p ~/studio/{workflows,prompts,assets,outputs} +export DW_WORKSPACE=~/studio + +# or, standing, in ~/.diffusers_helper/settings.json +# { "workspace": "/home/you/studio" } + +python -m dw.serve # serves ~/studio +python -m dw.run ~/studio/workflows/x.json +``` + +An example from a checkout still runs by path, and writes into the workspace's +outputs: + +```bash +DW_WORKSPACE=~/studio python -m dw.run ~/src/diffusers-workflow/workflows/sd15.json +``` + +The server reports what it resolved at `GET /api/server`, under +`directories.workspace` alongside the three folder paths. + +## The prompt library + +`prompt:` references resolve to the workspace's `prompts/` when a workspace was +named explicitly (rules 1–3 above, or `DW_PROMPT_DIR`, which still wins over +everything). A workspace that was merely inferred from the working directory +does not preempt the older discovery — `./prompts`, then the nearest `prompts/` +above the workflow file — so a repository workflow keeps reaching the library +it lives beside. See [Prompt References](WORKFLOW_GUIDE.md#prompt-references). + +## Where this is going + +Workspaces are the first stage of the design in +[proposals/workspaces.md](proposals/workspaces.md): a workflow search path with +writes confined to the workspace, run directories with an on-disk manifest, +`asset:` and `output:` references, and an MCP client that can keep its +workspace on its own machine. Only the resolver and its wiring are implemented +today; everything still lives where it did. diff --git a/docs/proposals/workspaces.md b/docs/proposals/workspaces.md new file mode 100644 index 00000000..25171e09 --- /dev/null +++ b/docs/proposals/workspaces.md @@ -0,0 +1,250 @@ +# Proposal: workspaces — decoupling content from the repo (and from the server) + +Status: draft / design only — no implementation. + +## Problem + +The project grew CLI-first, where "outputs next to the code" was correct and +free. A workflow file, the assets it reads, the prompts it references and the +files it writes all resolved relative to the checkout, and that was the whole +story. The web UI inherited that model and mostly gets away with it — it is +single-user and usually started from the repo root. The MCP server is where it +stops working: an agent authoring a workflow writes it into the repo's example +corpus, cannot supply input bytes at all, and has no notion of "my stuff" +versus "the examples that ship with the project". + +The examples use case is still valid — `workflows/` as a runnable corpus is one +of the better things about the project. What is missing is a second place for +everything that is *not* an example. + +## Current behavior + +Four roots, four different resolution rules, three of them anchored to the +process working directory: + +| Content | Where it resolves | How | +|---|---|---| +| Workflow files | CLI: any path given on the command line. Server: `--workflow-dir`, default `./workflows`. Packaged: `builtin:name.json` → `dw/workflows/` | `workflow_from_file` / `resolve_workflow_reference` (`dw/server/app.py:192`) | +| Input assets | The **workflow file's own directory** | `resolve_relative_path(path, base_dir)` (`dw/arguments.py:769`); `base_dir` is `dirname(file_spec)` | +| Stored prompts | `DW_PROMPT_DIR` → `./prompts` → nearest `prompts/` walking up from the workflow file | `get_prompt_dir` (`dw/prompts.py:30`) | +| Generated output | `-o/--output-dir`, default `./outputs`, plus a subfolder mirroring the workflow's position under the nearest directory literally named `workflows` | `workflow_output_subfolder` (`dw/workflow.py:104`), `Workflow.effective_output_dir` | +| Settings, job history, logs | `DIFFUSERS_HELPER_ROOT` → `~/.diffusers_helper/` | `dw/settings.py` | + +Only the last row is already decoupled. Only prompts has an environment +variable and a discovery walk; there is no `DW_WORKFLOW_DIR` or +`DW_OUTPUT_DIR`, so "start it from the repo root" is load-bearing for +everything else. + +### What that costs today + +- **Output layout is coupled to repo layout.** `workflow_output_subfolder` + keys the output subfolder off a path segment literally named `workflows`. + Move a workflow out of that tree and its outputs silently flatten into the + output root. The *filesystem shape of the checkout* is the grouping key. +- **Assets must live beside the workflow.** Because `base_dir` is the workflow + file's directory, any workflow with inputs drags a sibling `assets/` folder + into the repo — hence `.gitignore`'s `workflows/*/assets/`, `local_inputs/`, + `/*.mp4`, `/*.wav`. Generated media is being gitignored *inside the source + tree* rather than kept out of it. +- **The output→input loop is a manual copy.** The GYRE workflows reference + `assets/Gyre-score_choir.10-0.0.wav` and `assets/Gyre-still_1_iris.0-0.0.jpg` + — files a previous run wrote to `outputs/` and a human copied back into the + repo. Multi-stage work (the marmot passes, `outputs/marmot_pass1..3`) is + exactly the day-to-day case, and the engine has no vocabulary for "the thing + the last stage made". +- **Inputs and outputs share a directory on the server.** `POST /api/uploads` + writes to `/uploads` (`dw/server/app.py:1233`) because the output + dir is the only writable root the server knows about. +- **History and output can disagree.** `jobs.sqlite` lives under + `~/.diffusers_helper/` and stores manifest paths *relative to* whatever + `--output-dir` the server was started with (`dw/server/jobs.py:594`). Start + it from a different cwd and old rows point at nothing. A CLI run records no + manifest at all. +- **MCP writes into git.** `save_workflow` → `PUT /api/workflows/{name}` → + the server's `workflow_dir`, which defaults to the repo's example corpus. + There is no writable/read-only distinction, and no upload tool at all + (`dw_mcp/CLAUDE.md` notes `POST /api/uploads` is deliberately uncovered), so + an agent can author a workflow it has no way to supply inputs for. +- **`workflow_dir` is doing two jobs.** It is both "where workflows are kept" + and the server's *confinement boundary* for `workflow_path`, inline + `base_dir`, and sub-workflow steps. Any change to the first has to preserve + the second. + +### What is already right + +The mechanism is mostly there; the defaults and the discovery rules are what +bind it to the repo. + +- `--workflow-dir` / `--output-dir` / `--prompt-dir` already exist as explicit + parameters. Nothing in the engine *requires* the repo. +- `builtin:` is already the precedent for "packaged, read-only, reachable by a + scheme rather than a path". +- `prompt:name` is already a library reference rooted at a library rather than + at the workflow file — the exact shape assets need, including its validator + (`validate_prompt_reference`) and its no-double-resolution rule. +- `DIFFUSERS_HELPER_ROOT` already establishes a user-data root outside the + checkout. +- Path validation is centralized in `dw/security.py`, so a new root is one + containment base, not a new class of check. + +## Proposal + +### 1. Name the container: a workspace + +One directory holding everything a user generates or curates: + +``` +/ + workflows/ # mine, writable + prompts/ # the existing prompt library + assets/ # input media, incl. uploads/ + outputs/ # runs +``` + +A single resolver (`dw/workspace.py`) replaces three ad-hoc rules: + +1. `--workspace` flag +2. `DW_WORKSPACE` +3. `workspace` in `~/.diffusers_helper/settings.json` +4. the working directory, *if it looks like a workspace* — contains any of + `workflows/`, `prompts/`, `outputs/` +5. `~/diffusers-workflow/`, created on demand + +Rule 4 is what keeps this backward compatible: a repo checkout looks like a +workspace, so running from the repo root behaves exactly as it does now. Only +a bare working directory falls through to the home workspace. The existing +per-slot flags stay and continue to override individual roots, so no current +invocation changes meaning. `get_prompt_dir`'s walk-up becomes a deprecated +fallback rather than the primary rule. + +### 2. A workflow search path, writes to the front + +Rather than one `workflow_dir`, an ordered list: + +``` +workspace/workflows/ (writable) +/workflows/ (read-only, if present) +dw/workflows/ (read-only, packaged — today's builtin:) +``` + +Reads resolve front-to-back; **writes only ever go to the first entry**. That +alone fixes the MCP-writes-into-git problem while keeping the example corpus +first-class and browsable. `GET /api/workflows` gains `origin` and `writable` +per entry; the UI and MCP grey out save/delete on non-writable ones, and +"save as" from an example into the workspace becomes the natural gesture. +`examples:Foo.json` joins `builtin:` as an explicit scheme. + +Confinement generalizes cleanly: the boundary becomes *the workspace root plus +the read-only roots* instead of a single `workflow_dir`. Server-side runs stay +confined; CLI runs of an arbitrary file stay unconfined, as today +(`workflow_from_file`'s docstring already states that a locally-run file is not +a trust boundary). + +### 3. Run directories, and output layout from workflow identity + +Replace the `workflow_output_subfolder` path-segment hack with the workflow's +own identity — its name, or its path relative to the search-path root it came +from — and give each run its own directory: + +``` +outputs/// + + manifest.json # spec, arguments, seeds, resolved model ids, saved files +``` + +This is the change that makes multi-stage work tractable: intermediates, final +outputs and provenance are colocated, prunable as a unit, and addressable. +`manifest.json` also means a CLI run leaves a record, and `jobs.sqlite` demotes +from sole record to index — history becomes rebuildable from disk. + +Costs to plan for: `dw/step_cache.py` compares `entry["output_dir"]`; the +gallery scans `output_dir` recursively and would want to group by run; the UI +builds `/outputs/` URLs from paths relative to `output_dir`. A flat mode +should stay available for CLI users who prefer today's layout. + +### 4. First-class assets, and references that close the loop + +Two new reference prefixes, built on the `prompt:` machinery rather than beside +it: + +- `asset:name` / `asset:folder/name` — rooted at `workspace/assets/`, not at + the workflow file. A workflow stops needing to live next to its inputs, which + is what currently drags media into the source tree. +- `output:/` (or a `latest` selector per workflow) — reference a + previous run's product directly. This removes the manual copy-back that the + GYRE and marmot workflows document. + +Both resolve in `realize_args` where `prompt:` does, reuse +`validate_prompt_reference`-style validation, and inherit the existing rule +that a resolved value may not itself begin with a reference prefix. + +`POST /api/uploads` moves to `workspace/assets/uploads/`. Inputs and outputs +stop sharing a directory, and — because the destination is no longer "the +output dir" — an MCP `upload_asset` tool becomes reasonable to add. + +### 5. Decoupling MCP from the server + +Two levels, worth doing in order: + +**Level 1 — workspace-aware client.** `get_server_info` reports the workspace, +its roots and the writable flags. `save_workflow` targets the workspace, never +an example root. Add `upload_asset` so an agent can supply input bytes. MCP +stays a pure HTTP client of a running `dw.serve`; the boundary +`dw_mcp/CLAUDE.md` describes (no `dw.*` imports, no torch) is untouched. + +**Level 2 — client-side workspace.** `dw-mcp --workspace ` keeps +workflows, prompts and assets *on the agent's machine*. Authoring, schema +validation and library management are local; only runs go to the remote +`dw.serve`, submitted inline (`POST /api/jobs` with `workflow` + `base_dir`, +already supported) with assets pushed content-addressed — hash-named, skipped +if the server already has them. This is the literal answer to "not everything +should be on the server": the server becomes a GPU, and the workspace is the +user's. + +Level 2 has one real snag worth deciding early: inline submission's `base_dir` +is confined to the server's workflow root, so a locally-authored workflow's +asset references have to be rewritten to server-side asset ids at submit time. +That is tractable but it is the design work, not a detail. + +Trust interacts here too: `--trust-workflows` is process-wide today. Per-root +trust ("my workspace is trusted, the shared example corpus and anything an +MCP client submits is not") is a better-shaped knob once roots are named +things, and it is the knob that lets a server usefully run untrusted while an +owner's own workflows still use `pre_load_modules`. + +### 6. Multiple workspaces + +`.gitignore` already carries `local_projects/`, so this is happening +informally. `dw workspace new|list|use`, `--workspace` on every entry point, a +switcher in the UI later. One active workspace per process — no ambiguity about +which root a bare name resolved against. + +## Staging + +Each stage is independently shippable and the first is behavior-neutral. + +1. **`dw/workspace.py` + `--workspace`/`DW_WORKSPACE`.** One resolver, all + entry points consume it, defaults unchanged in a repo checkout. Nothing + moves yet. +2. **Assets root and `asset:` references.** Move uploads out of the output + directory. Workflows can stop hoarding sibling `assets/` folders. +3. **Run directories and on-disk `manifest.json`.** Output layout derives from + workflow identity instead of file position. Touches step cache, gallery + grouping and the UI's output URLs — the biggest single stage. +4. **Workflow search path, writable-first, `examples:`.** `origin`/`writable` + through the API into the UI and MCP. This is the stage that stops agents + writing into git. +5. **`output:` references.** Closes the multi-stage loop. +6. **MCP level 1, then level 2.** + +## Open questions + +- Does the repo's top-level `workflows/` stay as an example root on the search + path, or migrate wholesale into the package next to `dw/workflows/` so a + pip install has the same corpus a checkout does? The second is cleaner and is + a bigger move. +- Run id: job id for server runs, but CLI runs have none. Timestamp plus a + short hash of the resolved spec would serve both and dedupes reruns. +- Should `outputs/` be inside the workspace at all, or a peer with its own + setting? Video work fills disks, and the outputs root is the one people most + plausibly want on a different volume. diff --git a/dw/prompts.py b/dw/prompts.py index 6e892e97..655d239d 100644 --- a/dw/prompts.py +++ b/dw/prompts.py @@ -13,6 +13,7 @@ from .schema import load_schema, validate_data from .security import validate_prompt_path, validate_prompt_reference +from .workspace import resolve_workspace logger = logging.getLogger("dw") @@ -31,12 +32,21 @@ def get_prompt_dir(base_dir=None): """The directory stored prompts are rooted at. DW_PROMPT_DIR names it explicitly - the server sets it from --prompt-dir, - and the spawned worker inherits it. Without one it is ./prompts in the - working directory when that exists; otherwise the walk from the workflow - file's directory up toward the filesystem root finds the prompts/ folder - of the tree the workflow lives in - which is how a repo workflow run from - any working directory still reaches the library beside it. Read at call - time, not import time, so a test or worker sees the current value. + and the spawned worker inherits it. Then a workspace someone named (a + --workspace flag, DW_WORKSPACE, or the 'workspace' setting), whose + prompts/ is the library by definition, existing or not. + + Below that the rules that predate workspaces are unchanged, and they are + below on purpose: a workspace merely inferred from the working directory + or fallen back to must not preempt the library a workflow already + reaches. So: prompts/ in the working directory when that exists, then + the walk from the workflow file's directory up toward the filesystem root + for the prompts/ folder of the tree the workflow lives in - which is how + a repo workflow run from any working directory still reaches the library + beside it - and finally the workspace's prompts/ as the fallback. + + Read at call time, not import time, so a test or worker sees the current + value. Args: base_dir: The workflow file's directory, when one anchors the search @@ -44,9 +54,14 @@ def get_prompt_dir(base_dir=None): explicit = os.environ.get("DW_PROMPT_DIR") if explicit: return explicit - default = os.path.abspath("./prompts") - if os.path.isdir(default): - return default + + workspace = resolve_workspace() + if workspace.is_explicit: + return workspace.prompts + + working_directory_library = os.path.abspath("./prompts") + if os.path.isdir(working_directory_library): + return working_directory_library if base_dir: current = os.path.abspath(base_dir) while True: @@ -57,7 +72,7 @@ def get_prompt_dir(base_dir=None): if parent == current: break current = parent - return default + return workspace.prompts def resolve_prompt_reference(reference, prompt_dir=None, base_dir=None): diff --git a/dw/repl.py b/dw/repl.py index cd4f0f16..868dad4e 100644 --- a/dw/repl.py +++ b/dw/repl.py @@ -14,6 +14,7 @@ import multiprocessing from . import startup from .repl_worker import WorkerManager +from .workspace import resolve_workspace, set_workspace from .repl_commands import ( ConfigCommands, ArgCommands, @@ -46,11 +47,15 @@ class DiffusersWorkflowREPL(cmd.Cmd): def __init__(self): # Initialize cmd.Cmd first, before setting up our globals cmd.Cmd.__init__(self) - # Initialize globals dictionary with default values + # Initialize globals dictionary from the workspace - in a checkout, + # whose root is a workspace, these are the ./outputs and ./workflows + # they have always been + workspace = set_workspace(resolve_workspace()) self.globals = { - "output_dir": "./outputs", # Default output directory + "workspace": workspace.root, # Where the two directories below live + "output_dir": workspace.outputs, # Default output directory "log_level": "INFO", # Default log level - "workflow_dir": "./workflows", # Default workflow directory + "workflow_dir": workspace.workflows, # Default workflow directory } self.current_workflow = None self.workflow_args = {} # Store workflow arguments @@ -182,7 +187,7 @@ def do_set(self, arg): ARG_SUBCOMMANDS = ("show", "set", "clear") MEMORY_SUBCOMMANDS = ("show", "clear") CONFIG_SUBCOMMANDS = ("show", "set") - CONFIG_KEYS = ("output_dir", "log_level", "workflow_dir") + CONFIG_KEYS = ("workspace", "output_dir", "log_level", "workflow_dir") @staticmethod def _matches(candidates, text): diff --git a/dw/repl_commands.py b/dw/repl_commands.py index d985f67f..165d348d 100644 --- a/dw/repl_commands.py +++ b/dw/repl_commands.py @@ -42,12 +42,17 @@ def do_config(self, arg: str): print(" config show - Show all configuration settings") print(" config set = - Set a configuration value") print("\nAvailable settings:") - print(" output_dir - Directory for output files (default: ./outputs)") + print(" workspace - Directory holding your workflows, prompts,") + print(" assets and outputs; setting it moves") + print(" workflow_dir and output_dir with it") + print(" output_dir - Directory for output files (default: the") + print(" workspace's outputs/)") print( " log_level - Logging level: DEBUG, INFO, WARNING, ERROR, CRITICAL" ) print(" workflow_dir - Where 'workflow load ' and 'workflow list'") - print(" look for workflows (default: ./workflows)") + print(" look for workflows (default: the workspace's") + print(" workflows/)") print() print("These apply to this REPL session. Standing settings (device,") print("log file, TF32) live in ~/.diffusers_helper/settings.json") @@ -116,6 +121,25 @@ def _config_set(self, arg: str): except SecurityError as e: print(f"Error: Invalid workflow directory: {e}") return + + # Setting the workspace moves the two directories under it with + # it - setting either of those alone still overrides just that one + elif name == "workspace": + from .workspace import FLAG, Workspace, set_workspace + + try: + value = validate_path(value, allow_create=False) + if not os.path.isdir(value): + print(f"Warning: Directory '{value}' does not exist") + return + except SecurityError as e: + print(f"Error: Invalid workspace: {e}") + return + workspace = set_workspace(Workspace(value, FLAG)) + self.repl.globals["workflow_dir"] = workspace.workflows + self.repl.globals["output_dir"] = workspace.outputs + print(f" workflow_dir={workspace.workflows}") + print(f" output_dir={workspace.outputs}") else: print(f"Warning: Unknown setting '{name}'") diff --git a/dw/run.py b/dw/run.py index 93a95eae..3a21a554 100644 --- a/dw/run.py +++ b/dw/run.py @@ -2,6 +2,7 @@ import os from . import startup from .workflow import workflow_from_file +from .workspace import resolve_workspace, set_workspace from .security import ( validate_workflow_path, validate_output_path, @@ -22,8 +23,18 @@ def main(): "-o", "--output_dir", type=str, - default="./outputs", - help="The folder to write the outputs to", + default=None, + help="The folder to write the outputs to (default: the workspace's " + "outputs/ - ./outputs when run from a workspace, as a checkout is)", + ) + parser.add_argument( + "--workspace", + type=str, + default=None, + help="Directory holding your workflows, prompts, assets and outputs " + "(default: DW_WORKSPACE, else the 'workspace' setting, else the " + "working directory when it looks like a workspace, else " + "~/diffusers-workspace)", ) parser.add_argument( "variables", @@ -62,6 +73,11 @@ def main(): # and the name=value pairs - fails as unrecognized arguments args = parser.parse_intermixed_args() + # Pinned before anything resolves a directory from it, and exported so a + # subprocess sees the same answer + workspace = set_workspace(resolve_workspace(args.workspace)) + output_dir = args.output_dir or workspace.outputs + if args.prompt_dir: os.environ["DW_PROMPT_DIR"] = os.path.abspath(args.prompt_dir) @@ -87,7 +103,7 @@ def main(): # Validate and secure file paths try: - validated_output_dir = validate_output_path(args.output_dir, None) + validated_output_dir = validate_output_path(output_dir, None) if not os.path.exists(validated_output_dir): # Create output directory if it doesn't exist os.makedirs(validated_output_dir, exist_ok=True) diff --git a/dw/serve.py b/dw/serve.py index d9108217..a4b2d12e 100644 --- a/dw/serve.py +++ b/dw/serve.py @@ -27,10 +27,25 @@ def main(): ) parser.add_argument("--port", type=int, default=8765, help="Port (default: 8765)") parser.add_argument( - "--workflow-dir", default="./workflows", help="Directory of workflow JSON files" + "--workspace", + default=None, + help="Directory holding the workflows, prompts, assets and outputs " + "this server serves (default: DW_WORKSPACE, else the 'workspace' " + "setting, else the working directory when it looks like a " + "workspace, else ~/diffusers-workspace). --workflow-dir, " + "--output-dir and --prompt-dir each override one of its folders", + ) + parser.add_argument( + "--workflow-dir", + default=None, + help="Directory of workflow JSON files (default: the workspace's " + "workflows/)", ) parser.add_argument( - "--output-dir", default="./outputs", help="Directory results are written to" + "--output-dir", + default=None, + help="Directory results are written to (default: the workspace's " + "outputs/)", ) parser.add_argument( "--prompt-dir", @@ -77,6 +92,23 @@ def main(): ) args = parser.parse_args() + # Resolved and pinned before anything derives a directory from it - the + # spawned worker inherits the environment variable, the way it inherits + # the prompt directory and the trust flag below + from .workspace import resolve_workspace, set_workspace + + workspace = set_workspace(resolve_workspace(args.workspace)) + workflow_dir = args.workflow_dir or workspace.workflows + output_dir = args.output_dir or workspace.outputs + + # A workspace's workflow folder is where the UI and MCP clients save, so + # it has to exist for a first run in a fresh workspace. Only the folder + # actually defaulted to is created - an explicit --workflow-dir that does + # not exist stays the operator's typo, not a new empty directory. The + # output folder is created by the job manager on the same reasoning + if not args.workflow_dir: + os.makedirs(workflow_dir, exist_ok=True) + token = args.token or os.environ.get("DW_API_TOKEN") or None from .server.app import LOOPBACK_HOSTS @@ -109,7 +141,7 @@ def main(): from .prompts import get_prompt_dir prompt_dir = os.path.abspath( - args.prompt_dir or get_prompt_dir(base_dir=os.path.abspath(args.workflow_dir)) + args.prompt_dir or get_prompt_dir(base_dir=os.path.abspath(workflow_dir)) ) os.environ["DW_PROMPT_DIR"] = prompt_dir @@ -145,10 +177,11 @@ def main(): app = create_app( # absolute, so the path the UI hands back on submit is unambiguous - workflow_dir=os.path.abspath(args.workflow_dir), - output_dir=args.output_dir, + workflow_dir=os.path.abspath(workflow_dir), + output_dir=output_dir, log_level=args.log_level, prompt_dir=prompt_dir, + workspace=workspace.root, host=args.host, token=token, mcp=args.mcp, diff --git a/dw/server/app.py b/dw/server/app.py index 6f5646d8..749388d4 100644 --- a/dw/server/app.py +++ b/dw/server/app.py @@ -379,6 +379,7 @@ def create_app( download_manager=None, diffusers_updater=None, prompt_dir="./prompts", + workspace=None, host="127.0.0.1", token=None, mcp=False, @@ -439,6 +440,11 @@ async def lifespan(app): app.state.job_manager = manager app.state.workflow_dir = workflow_dir app.state.prompt_dir = prompt_dir + # The workspace the three directories above default to folders of, for a + # client that wants to name the root rather than reason about the parts. + # None when the caller resolved no workspace (a test building an app + # around three explicit directories) + app.state.workspace = os.path.abspath(workspace) if workspace else None app.state.mcp_mounted = mcp_asgi is not None wildcard_bind = host in WILDCARD_HOSTS @@ -1434,6 +1440,9 @@ def server_info(): "mcp": {"mounted": bool(app.state.mcp_mounted), "path": MCP_PATH}, "addresses": addresses, "directories": { + # The workspace the three below default to folders of; an + # individually overridden folder still reports its own path + "workspace": app.state.workspace, "workflows": os.path.abspath(app.state.workflow_dir), "outputs": os.path.abspath(manager.output_dir), "prompts": ( diff --git a/dw/settings.py b/dw/settings.py index 8977e7a5..c9d19b17 100644 --- a/dw/settings.py +++ b/dw/settings.py @@ -13,6 +13,11 @@ class Settings: # environment variable overrides this for a single run. device: str = None + # Directory holding this user's workflows, prompts, assets and outputs. + # None resolves it - see dw/workspace.py for the order, which ends at the + # working directory when it looks like a workspace, then ~/diffusers-workspace + workspace: str = None + # PyTorch optimization settings enable_tf32: bool = True # TensorFloat-32 for faster matmul on Ampere+ GPUs cudnn_benchmark: bool = True # cuDNN autotuner (faster for fixed sizes) @@ -35,6 +40,7 @@ def load_settings(): settings.log_to_console = settings_dict.get("log_to_console", False) settings.device = settings_dict.get("device", None) + settings.workspace = settings_dict.get("workspace", None) # PyTorch optimization settings settings.enable_tf32 = settings_dict.get("enable_tf32", True) diff --git a/dw/workspace.py b/dw/workspace.py new file mode 100644 index 00000000..fe483790 --- /dev/null +++ b/dw/workspace.py @@ -0,0 +1,190 @@ +"""The workspace: the directory holding what a user makes. + +A workflow file, the assets it reads, the prompts it references and the files +it writes started life beside the code, which was right when the only entry +point was a CLI run from a checkout. It stops being right once an agent or a +remote client is the one authoring: generated media does not belong in the +source tree, and an agent's day-to-day workflows do not belong in the example +corpus this repository ships. + +A workspace names one directory for all of it: + + / + workflows/ mine, writable + prompts/ the stored prompt library + assets/ input media + outputs/ generated files + +Resolution order, first hit wins: + +1. an explicit path (a --workspace flag) +2. the DW_WORKSPACE environment variable +3. 'workspace' in ~/.diffusers_helper/settings.json +4. the working directory, when it looks like a workspace - it holds any of + workflows/, prompts/ or outputs/ +5. ~/diffusers-workspace + +Rule 4 is what keeps a checkout working unchanged: the repository root holds +all three, so running from it resolves to it, and every default lands exactly +where it landed before there was a workspace at all. Only a working directory +with none of those markers falls through to the home workspace. + +Nothing here creates a directory. Resolution is a pure question about paths - +an entry point that is about to write calls ensure() once it knows it needs to. +""" + +import os +from pathlib import Path + +# Set by an entry point that resolved a workspace, so a spawned worker +# subprocess inherits the same answer - multiprocessing's 'spawn' start method +# launches a fresh interpreter that inherits os.environ, the way DW_PROMPT_DIR +# and DW_TRUST_WORKFLOWS already reach the worker +WORKSPACE_ENV_VAR = "DW_WORKSPACE" + +# Carries alongside it how the workspace was chosen. Without it a workspace +# merely inferred from the working directory would come back to the worker +# looking like one the user named, and an inferred workspace deliberately +# yields to discovery that predates it - see Workspace.is_explicit +WORKSPACE_SOURCE_ENV_VAR = "DW_WORKSPACE_SOURCE" + +# Deliberately not '~/diffusers-workflow': that is where this repository +# gets cloned, and a default that lands inside a checkout is the coupling +# workspaces exist to remove +DEFAULT_WORKSPACE = "~/diffusers-workspace" + +WORKFLOWS_SUBDIR = "workflows" +PROMPTS_SUBDIR = "prompts" +ASSETS_SUBDIR = "assets" +OUTPUTS_SUBDIR = "outputs" + +SUBDIRS = (WORKFLOWS_SUBDIR, PROMPTS_SUBDIR, ASSETS_SUBDIR, OUTPUTS_SUBDIR) + +# What makes a directory recognizable as a workspace. assets/ is deliberately +# not a marker - a bare assets/ folder is a common thing to have lying around, +# where these three together say "content lives here" +MARKER_SUBDIRS = (WORKFLOWS_SUBDIR, PROMPTS_SUBDIR, OUTPUTS_SUBDIR) + +# How a workspace was chosen, in resolution order. Everything but the last two +# is an answer someone gave on purpose; see Workspace.is_explicit +FLAG = "flag" +ENVIRONMENT = "environment" +SETTINGS = "settings" +WORKING_DIRECTORY = "working directory" +DEFAULT = "default" + +EXPLICIT_SOURCES = (FLAG, ENVIRONMENT, SETTINGS) +ALL_SOURCES = EXPLICIT_SOURCES + (WORKING_DIRECTORY, DEFAULT) + + +class Workspace: + """A resolved workspace root and the four directories under it.""" + + def __init__(self, root, source): + self.root = os.path.abspath(os.path.expanduser(str(root))) + self.source = source + + @property + def is_explicit(self): + """Whether someone named this workspace, rather than it being + inferred from the working directory or fallen back to. + + Discovery that predates workspaces - the prompt library's walk up + from the workflow file - stays ahead of an inferred workspace and + behind a named one, so turning this on changes nothing for a caller + who has not asked for a workspace. + """ + return self.source in EXPLICIT_SOURCES + + @property + def workflows(self): + return os.path.join(self.root, WORKFLOWS_SUBDIR) + + @property + def prompts(self): + return os.path.join(self.root, PROMPTS_SUBDIR) + + @property + def assets(self): + return os.path.join(self.root, ASSETS_SUBDIR) + + @property + def outputs(self): + return os.path.join(self.root, OUTPUTS_SUBDIR) + + def ensure(self): + """Create the workspace and its subdirectories if they are missing. + + Called by an entry point that is about to write, not by resolution - + asking where the workspace is should never leave a directory behind. + """ + for subdir in SUBDIRS: + Path(self.root, subdir).mkdir(parents=True, exist_ok=True) + return self + + def __repr__(self): + return f"Workspace({self.root!r}, from {self.source})" + + def __eq__(self, other): + return ( + isinstance(other, Workspace) + and self.root == other.root + and self.source == other.source + ) + + +def looks_like_workspace(path): + """Whether a directory holds the subfolders that mark a workspace.""" + return any(os.path.isdir(os.path.join(path, name)) for name in MARKER_SUBDIRS) + + +def resolve_workspace(explicit=None): + """The workspace this process works in. + + Args: + explicit: A path from a --workspace flag, when one was given + + Returns: + A Workspace, which may not exist on disk yet + """ + if explicit: + return Workspace(explicit, FLAG) + + from_environment = os.environ.get(WORKSPACE_ENV_VAR) + if from_environment: + source = os.environ.get(WORKSPACE_SOURCE_ENV_VAR) + return Workspace( + from_environment, source if source in ALL_SOURCES else ENVIRONMENT + ) + + # Imported here, not at module scope: dw.settings reads a file, and + # resolution is called from argument parsing on every entry point + from .settings import load_settings + + from_settings = load_settings().workspace + if from_settings: + return Workspace(from_settings, SETTINGS) + + working_directory = os.path.abspath(os.getcwd()) + if looks_like_workspace(working_directory): + return Workspace(working_directory, WORKING_DIRECTORY) + + return Workspace(DEFAULT_WORKSPACE, DEFAULT) + + +def set_workspace(workspace): + """Pin a resolved workspace in the environment, so a spawned worker + subprocess and anything resolving later in this process agree with the + entry point that chose it. + + Args: + workspace: The Workspace to pin, or a path + + Returns: + The Workspace that was pinned + """ + if not isinstance(workspace, Workspace): + workspace = Workspace(workspace, FLAG) + os.environ[WORKSPACE_ENV_VAR] = workspace.root + os.environ[WORKSPACE_SOURCE_ENV_VAR] = workspace.source + return workspace diff --git a/tests/test_workspace.py b/tests/test_workspace.py new file mode 100644 index 00000000..d4069856 --- /dev/null +++ b/tests/test_workspace.py @@ -0,0 +1,177 @@ +"""Workspace resolution: which directory a run's workflows, prompts and +outputs belong to, and the guarantee that a checkout answers the way it +always has.""" + +import json +import os + +import pytest + +from dw.workspace import ( + DEFAULT, + ENVIRONMENT, + FLAG, + SETTINGS, + WORKING_DIRECTORY, + WORKSPACE_ENV_VAR, + WORKSPACE_SOURCE_ENV_VAR, + Workspace, + looks_like_workspace, + resolve_workspace, + set_workspace, +) + + +@pytest.fixture +def clean_environment(monkeypatch, tmp_path): + """No workspace named anywhere, and a settings file that names none.""" + monkeypatch.delenv(WORKSPACE_ENV_VAR, raising=False) + monkeypatch.delenv(WORKSPACE_SOURCE_ENV_VAR, raising=False) + monkeypatch.setenv("DIFFUSERS_HELPER_ROOT", str(tmp_path / "helper")) + (tmp_path / "helper").mkdir() + return tmp_path + + +def workspace_tree(root, *subdirs): + for name in subdirs: + (root / name).mkdir(parents=True, exist_ok=True) + return root + + +class TestResolution: + def test_a_flag_wins_over_everything(self, clean_environment, monkeypatch): + named = clean_environment / "named" + monkeypatch.setenv(WORKSPACE_ENV_VAR, str(clean_environment / "environment")) + workspace = resolve_workspace(str(named)) + assert workspace.root == str(named) + assert workspace.source == FLAG + + def test_the_environment_names_it(self, clean_environment, monkeypatch): + named = clean_environment / "environment" + monkeypatch.setenv(WORKSPACE_ENV_VAR, str(named)) + workspace = resolve_workspace() + assert workspace.root == str(named) + assert workspace.source == ENVIRONMENT + + def test_the_settings_file_names_it(self, clean_environment, monkeypatch): + named = clean_environment / "settings-workspace" + settings = clean_environment / "helper" / "settings.json" + settings.write_text(json.dumps({"workspace": str(named)})) + monkeypatch.chdir(clean_environment) + workspace = resolve_workspace() + assert workspace.root == str(named) + assert workspace.source == SETTINGS + + def test_a_working_directory_that_looks_like_one_is_used( + self, clean_environment, monkeypatch + ): + # The checkout case: run from a tree that holds the folders, and the + # tree is the workspace - which is what keeps every default where it + # was before workspaces existed + root = workspace_tree(clean_environment / "repo", "workflows", "prompts") + monkeypatch.chdir(root) + workspace = resolve_workspace() + assert workspace.root == str(root) + assert workspace.source == WORKING_DIRECTORY + assert workspace.outputs == os.path.abspath("./outputs") + assert workspace.workflows == os.path.abspath("./workflows") + + def test_a_bare_working_directory_falls_back_to_the_home_workspace( + self, clean_environment, monkeypatch + ): + bare = clean_environment / "bare" + bare.mkdir() + monkeypatch.chdir(bare) + workspace = resolve_workspace() + assert workspace.source == DEFAULT + assert workspace.root == os.path.expanduser("~/diffusers-workspace") + + def test_one_marker_folder_is_enough(self, clean_environment, monkeypatch): + for marker in ("workflows", "prompts", "outputs"): + root = workspace_tree(clean_environment / marker, marker) + assert looks_like_workspace(str(root)) + # assets alone is not a marker - too common a folder to claim + assets_only = workspace_tree(clean_environment / "assets-only", "assets") + assert not looks_like_workspace(str(assets_only)) + + +class TestSubdirectories: + def test_the_four_folders_hang_off_the_root(self, tmp_path): + workspace = Workspace(tmp_path / "ws", FLAG) + assert workspace.workflows == str(tmp_path / "ws" / "workflows") + assert workspace.prompts == str(tmp_path / "ws" / "prompts") + assert workspace.assets == str(tmp_path / "ws" / "assets") + assert workspace.outputs == str(tmp_path / "ws" / "outputs") + + def test_resolution_creates_nothing(self, clean_environment): + # Asking where the workspace is must never leave a directory behind - + # an entry point about to write calls ensure() itself + workspace = resolve_workspace(str(clean_environment / "never-created")) + assert not os.path.exists(workspace.root) + + def test_ensure_creates_the_four_folders(self, tmp_path): + workspace = Workspace(tmp_path / "fresh", FLAG).ensure() + for folder in ("workflows", "prompts", "assets", "outputs"): + assert os.path.isdir(os.path.join(workspace.root, folder)) + + def test_a_home_relative_root_is_expanded(self, tmp_path): + assert Workspace("~/somewhere", FLAG).root == os.path.expanduser("~/somewhere") + + +class TestPinning: + def test_pinning_carries_the_root_and_how_it_was_chosen( + self, clean_environment, monkeypatch + ): + # A worker subprocess inherits the environment, and must not read an + # inferred workspace back as one the user named - the prompt library + # yields to older discovery for an inferred one only + root = workspace_tree(clean_environment / "repo", "workflows") + monkeypatch.chdir(root) + set_workspace(resolve_workspace()) + assert os.environ[WORKSPACE_ENV_VAR] == str(root) + + monkeypatch.chdir(clean_environment) + inherited = resolve_workspace() + assert inherited.root == str(root) + assert inherited.source == WORKING_DIRECTORY + assert not inherited.is_explicit + + def test_a_pinned_flag_workspace_stays_explicit( + self, clean_environment, monkeypatch + ): + set_workspace(str(clean_environment / "named")) + assert resolve_workspace().is_explicit + + +class TestPromptLibraryPrecedence: + """get_prompt_dir's older rules stay ahead of an inferred workspace and + behind a named one.""" + + def test_a_named_workspace_names_the_library(self, clean_environment, monkeypatch): + from dw.prompts import get_prompt_dir + + monkeypatch.delenv("DW_PROMPT_DIR", raising=False) + named = workspace_tree(clean_environment / "named", "prompts") + cwd = workspace_tree(clean_environment / "cwd", "prompts") + monkeypatch.chdir(cwd) + monkeypatch.setenv(WORKSPACE_ENV_VAR, str(named)) + assert get_prompt_dir() == str(named / "prompts") + + def test_an_inferred_workspace_yields_to_the_walk( + self, clean_environment, monkeypatch + ): + from dw.prompts import get_prompt_dir + + monkeypatch.delenv("DW_PROMPT_DIR", raising=False) + tree = workspace_tree(clean_environment / "repo", "prompts", "workflows") + elsewhere = clean_environment / "elsewhere" + elsewhere.mkdir() + monkeypatch.chdir(elsewhere) + assert get_prompt_dir(str(tree / "workflows")) == str(tree / "prompts") + + def test_the_prompt_environment_still_wins(self, clean_environment, monkeypatch): + from dw.prompts import get_prompt_dir + + monkeypatch.setenv("DW_PROMPT_DIR", str(clean_environment / "library")) + monkeypatch.setenv(WORKSPACE_ENV_VAR, str(clean_environment / "named")) + assert get_prompt_dir() == str(clean_environment / "library") From e842cb94ce8d11d9d563a953c5b46ab374200dd9 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sat, 5 Sep 2026 18:11:09 +0000 Subject: [PATCH 02/34] feat(assets): an asset library, and 'asset:' references rooted at it A workflow's media paths resolve against the workflow file's own directory, so a workflow that reads anything has to keep that thing beside it - which is why generated media ends up gitignored inside the source tree (workflows/*/assets/, local_inputs/). 'asset:name.ext' is rooted at the asset library instead, the way 'prompt:name' is rooted at the prompt library. It resolves to a path, first and under any argument name, so everything that already loads a path - image/video keys, a from_file, a list of them - loads it unchanged. The library is DW_ASSET_DIR / --asset-dir, else a named workspace's assets/, else ./assets, else the walk up from the workflow file; a reference is confined to it, symlinks out included. Browser uploads move from /uploads to the library's uploads/ and come back as 'asset:uploads/', so input stops being filed among generated output and a workflow saved after an upload still resolves on the next run. A server with no asset library configured keeps the old behavior. Stage two of docs/proposals/workspaces.md. Co-Authored-By: Claude Opus 5 (1M context) --- CLAUDE.md | 9 ++- docs/WORKFLOW_GUIDE.md | 34 +++++++++ docs/WORKSPACES.md | 10 +++ dw/arguments.py | 9 +++ dw/assets.py | 130 ++++++++++++++++++++++++++++++++++ dw/prompts.py | 8 ++- dw/run.py | 12 ++++ dw/security.py | 45 ++++++++++++ dw/serve.py | 20 +++++- dw/server/app.py | 37 ++++++++-- tests/test_assets.py | 156 +++++++++++++++++++++++++++++++++++++++++ tests/test_server.py | 69 ++++++++++++++++++ 12 files changed, 528 insertions(+), 11 deletions(-) create mode 100644 dw/assets.py create mode 100644 tests/test_assets.py diff --git a/CLAUDE.md b/CLAUDE.md index 9a7adde8..b8ccf43d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -77,6 +77,13 @@ docs/WORKSPACES.md, and docs/proposals/workspaces.md for the later stages - Values prefixed with `constant:` read a value declared in python rather than copying it into JSON: `"constant:diffusers.pipelines.ltx2.utils.DISTILLED_SIGMA_VALUES"`. Resolved in `realize_args`, validated by `validate_constant_name()`; anything callable is refused +- Values prefixed with `asset:` resolve to the path of a file in the asset library: + `"asset:iris.png"` or `"asset:gyre/frames/web.mp4"`. Resolved in `realize_args` before + every other convention (`dw/assets.py`), rooted at the library rather than the workflow + file, confined to it, and then loaded by whatever would have loaded a path written + there. The library is `DW_ASSET_DIR` / `--asset-dir`, else the workspace's `assets/` + when a workspace was named, else `./assets` if it exists, else found by walking up + from the workflow file's directory - Values prefixed with `prompt:` load a stored prompt's `text` from the prompt library: `"prompt:name"` or `"prompt:folder/name"`. Resolved in `realize_args` (`dw/prompts.py`), rooted at the library rather than the workflow file. The library is `DW_PROMPT_DIR` / @@ -124,7 +131,7 @@ All entry points use `dw/security.py`. When adding features: - **Built-in workflows** need explicit argument mapping: `"prompt": "variable:prompt"` - **MPS differences from CUDA**: no autocast, no bitsandbytes, no flash_attn, no triton, no torch.compile. Model offloading has less benefit on unified memory, and `"offload": "sequential"` is downgraded to `"model"` with a warning there (`place_component`) — per-submodule streaming hands back no residency when the CPU and the accelerator share one pool. `exclude_from_cpu_offload` is sequential-only and does not survive the downgrade. - **`{}`-escaped strings** in JSON arguments: `"{nf4}"` stays as string `"nf4"`, without braces it would try to load as a type -- **A stored prompt's `text` may not begin with a reference prefix** (`variable:`, `previous_result:`, `constant:`, `prompt:`) — the engine rejects it to prevent double resolution or iteration expansion +- **A stored prompt's `text` may not begin with a reference prefix** (`variable:`, `previous_result:`, `constant:`, `asset:`, `prompt:`) — the engine rejects it to prevent double resolution or iteration expansion - **Audio+video muxing**: pipelines that generate audio alongside video (LTX-2) have the two muxed into one `video/mp4` file with PyAV in `result.py` - **Step cache**: a process-wide singleton (`dw/step_cache.py`) consulted by every `Workflow.run`, including server jobs; entries are keyed by `(workflow id, step name)`; disabled entirely when the workflow sets no `seed`; a hit reports the earlier run's files with `reused: true` and writes nothing new; `memory clear` drops it diff --git a/docs/WORKFLOW_GUIDE.md b/docs/WORKFLOW_GUIDE.md index 56f61666..ed478862 100644 --- a/docs/WORKFLOW_GUIDE.md +++ b/docs/WORKFLOW_GUIDE.md @@ -918,6 +918,40 @@ always resolves to exactly one string: it never multiplies a step's iterations t reference prefix such as `variable:` - the engine refuses it rather than resolving text as syntax. +### Asset References + +A workflow's plain media paths resolve against the workflow file's own directory, which +means a workflow that reads anything has to keep that thing beside it. An `asset:` +reference is rooted at the asset library instead - the workspace's `assets/` folder - +so a workflow and the media it reads do not have to live in the same place: + +```json +"image": "asset:iris.png", +"video": "asset:gyre/frames/web.mp4", +"references": [ + { + "reference_type": "diffusers.modular_pipelines.minimax_h3.MiniMaxH3ImageReference", + "from_file": "asset:subject.png" + } +] +``` + +A reference names a file with its extension, at most four folders deep, and resolves to +that file's path - so it works under any argument that accepts a path: `image`, `video`, +a `from_file`, a list of any of them, or a task argument that names a file. What loads +the path is unchanged; only where the path comes from is. + +The library's location is resolved in order: the `DW_ASSET_DIR` environment variable +(which `--asset-dir` on both `dw.run` and `dw.serve` sets), then the workspace's +`assets/` when a workspace was named explicitly, then `./assets` in the working +directory when it exists, then the first `assets/` folder found walking up from the +workflow file's own directory. See [Workspaces](WORKSPACES.md). + +A reference can only name a file inside the library: `..`, an absolute path, or a +symlink pointing out of it are all refused. Browser uploads land in the library's +`uploads/` folder and come back as `asset:uploads/`, so a workflow saved after +an upload still resolves on the next run. + ### Objects Built From a File Some pipelines take arguments that are objects rather than plain media. An argument that diff --git a/docs/WORKSPACES.md b/docs/WORKSPACES.md index d0129a5d..0950296b 100644 --- a/docs/WORKSPACES.md +++ b/docs/WORKSPACES.md @@ -82,6 +82,16 @@ does not preempt the older discovery — `./prompts`, then the nearest `prompts/ above the workflow file — so a repository workflow keeps reaching the library it lives beside. See [Prompt References](WORKFLOW_GUIDE.md#prompt-references). +## Assets + +`assets/` is the input-media library. A workflow argument written as +`asset:name.ext` (or `asset:folder/name.ext`) resolves to that file's path, +rooted at the library rather than at the workflow file — so a workflow and the +media it reads no longer have to sit in the same folder. `--asset-dir` and +`DW_ASSET_DIR` override the folder, and browser uploads land in +`assets/uploads/`, coming back as `asset:uploads/`. See +[Asset References](WORKFLOW_GUIDE.md#asset-references). + ## Where this is going Workspaces are the first stage of the design in diff --git a/dw/arguments.py b/dw/arguments.py index 284f1f05..14b329be 100644 --- a/dw/arguments.py +++ b/dw/arguments.py @@ -4,6 +4,7 @@ from inspect import Parameter, signature from .type_helpers import load_type_from_name, load_constant_from_name, has_method from .prompts import PROMPT_PREFIX, fetch_prompt +from .assets import is_asset_reference, resolve_asset_values from diffusers.utils import load_image, load_video from .security import ( validate_path, @@ -89,6 +90,12 @@ def realize_args(arg, base_dir=None): if isinstance(arg, dict): logger.debug(f"Processing dictionary arguments: {list(arg.keys())}") for k, v in arg.items(): + # An asset reference resolves to the path of a file in the asset + # library, and does it first: what it stands for is a path, so + # everything below - the media conventions, an object's + # 'from_file' - then handles it as the path it always was + if is_asset_reference(v) or isinstance(v, list): + v = arg[k] = resolve_asset_values(v, base_dir) # A constant resolves under any argument name, and before the # conventions below - what it holds is the value, not a file to load if is_constant_reference(v): @@ -143,6 +150,8 @@ def realize_args(arg, base_dir=None): elif isinstance(arg, list): logger.debug("Processing list arguments") for i, item in enumerate(arg): + if is_asset_reference(item): + item = arg[i] = resolve_asset_values(item, base_dir) if is_constant_reference(item): arg[i] = fetch_constant(item) continue diff --git a/dw/assets.py b/dw/assets.py new file mode 100644 index 00000000..24cab1c0 --- /dev/null +++ b/dw/assets.py @@ -0,0 +1,130 @@ +"""The asset library: input media a workflow references by name. + +A workflow's media paths resolve against the workflow file's own directory, +which means a workflow that reads anything has to keep that thing beside it - +the reason generated media ends up gitignored inside a source tree. An +'asset:name' reference is rooted at the asset library instead, the way +'prompt:name' is rooted at the prompt library, so the same reference means the +same file from every workflow and neither has to live next to the other. + +A reference resolves to a path, not to a value: 'asset:frames/iris.jpg' +becomes the absolute path of that file, and whatever would have loaded a path +written there loads it unchanged. +""" + +import logging +import os + +from .security import validate_asset_reference, validate_path +from .workspace import resolve_workspace + +logger = logging.getLogger("dw") + +# The prefix marking a value as a reference to a stored asset +ASSET_PREFIX = "asset:" + +# Set by an entry point from --asset-dir, and inherited by a spawned worker, +# the way DW_PROMPT_DIR is +ASSET_DIR_ENV_VAR = "DW_ASSET_DIR" + + +def get_asset_dir(base_dir=None): + """The directory 'asset:' references are rooted at. + + Discovery mirrors the prompt library's, for the same reasons: + DW_ASSET_DIR names it outright; then a workspace someone named, whose + assets/ is the library by definition; then assets/ in the working + directory when it exists; then the walk from the workflow file's + directory up toward the filesystem root, which is how a workflow in a + tree reaches the library that tree keeps; and finally the workspace's + assets/. + + Args: + base_dir: The workflow file's directory, when one anchors the search + """ + explicit = os.environ.get(ASSET_DIR_ENV_VAR) + if explicit: + return explicit + + workspace = resolve_workspace() + if workspace.is_explicit: + return workspace.assets + + working_directory_library = os.path.abspath("./assets") + if os.path.isdir(working_directory_library): + return working_directory_library + if base_dir: + current = os.path.abspath(base_dir) + while True: + candidate = os.path.join(current, "assets") + if os.path.isdir(candidate): + return candidate + parent = os.path.dirname(current) + if parent == current: + break + current = parent + return workspace.assets + + +def is_asset_reference(value): + """Whether a value references a file in the asset library.""" + return isinstance(value, str) and value.startswith(ASSET_PREFIX) + + +def resolve_asset_reference(reference, asset_dir=None, base_dir=None): + """Resolve an 'asset:' reference to the file it names. + + Args: + reference: The 'asset:name.ext' or 'asset:folder/name.ext' string + asset_dir: Directory the name is rooted at; defaults to get_asset_dir() + base_dir: The workflow file's directory, anchoring discovery when no + asset directory is configured + + Returns: + The validated absolute path of the asset file + + Raises: + InvalidInputError: If the name is not a valid asset name + PathTraversalError: If the name escapes the asset directory + ValueError: If no file exists under that name + """ + name = validate_asset_reference(reference.removeprefix(ASSET_PREFIX).strip()) + asset_dir = asset_dir or get_asset_dir(base_dir) + # Confined to the library: the name is joined onto a directory, so the + # containment check is what makes a name a name rather than a path + # allow_create leaves "does not exist" to the check below, which can say + # what an asset reference is instead of what a path is + path = validate_path(os.path.join(asset_dir, name), asset_dir) + if not os.path.isfile(path): + raise ValueError( + f"Asset '{name}' not found in {asset_dir} - an 'asset:' reference " + f"names a file in the asset library, with its extension, like " + f"'asset:iris.jpg' or 'asset:gyre/frame_1.jpg'" + ) + logger.debug(f"Resolved {reference} to {path}") + return path + + +def fetch_asset(reference, asset_dir=None, base_dir=None): + """The path an 'asset:' reference names, for whatever loads paths.""" + return resolve_asset_reference(reference, asset_dir, base_dir) + + +def resolve_asset_values(value, base_dir=None): + """Replace any 'asset:' reference in a value with the path it names. + + A list is walked, because an 'image' argument may be a list of them and + the key conventions hand the whole list to the loader at once - by then + it is too late for a reference to be recognized. Dictionaries are left + alone: realize_args recurses into those itself, and each of their values + reaches this on the way through. + """ + if is_asset_reference(value): + return fetch_asset(value, base_dir=base_dir) + if isinstance(value, list): + # In place: a list argument keeps its identity, the way every other + # value realize_args touches does + for index, item in enumerate(value): + value[index] = resolve_asset_values(item, base_dir) + return value + return value diff --git a/dw/prompts.py b/dw/prompts.py index 655d239d..f10c1a36 100644 --- a/dw/prompts.py +++ b/dw/prompts.py @@ -25,7 +25,13 @@ # The prefixes a stored prompt's text may not begin with. Resolved text is # substituted where the reference stood, so text that itself looks like a # reference would be resolved again - or worse, expand a step's iterations -RESERVED_TEXT_PREFIXES = ("previous_result:", "variable:", "constant:", PROMPT_PREFIX) +RESERVED_TEXT_PREFIXES = ( + "previous_result:", + "variable:", + "constant:", + "asset:", + PROMPT_PREFIX, +) def get_prompt_dir(base_dir=None): diff --git a/dw/run.py b/dw/run.py index 3a21a554..a86a9b62 100644 --- a/dw/run.py +++ b/dw/run.py @@ -56,6 +56,15 @@ def main(): "DW_PROMPT_DIR, else ./prompts if it exists, else the nearest " "prompts/ above the workflow file)", ) + parser.add_argument( + "--asset-dir", + type=str, + default=None, + help="Directory 'asset:' references resolve against (default: " + "DW_ASSET_DIR, else the workspace's assets/ when a workspace was " + "named, else ./assets if it exists, else the nearest assets/ above " + "the workflow file)", + ) parser.add_argument( "--trust-workflows", action="store_true", @@ -81,6 +90,9 @@ def main(): if args.prompt_dir: os.environ["DW_PROMPT_DIR"] = os.path.abspath(args.prompt_dir) + if args.asset_dir: + os.environ["DW_ASSET_DIR"] = os.path.abspath(args.asset_dir) + set_trust_workflows(args.trust_workflows) # Parse key-value pairs with validation diff --git a/dw/security.py b/dw/security.py index 40e1bded..907bbea8 100644 --- a/dw/security.py +++ b/dw/security.py @@ -456,6 +456,51 @@ def validate_prompt_reference(name: str) -> str: return name +# A stored asset's name: a file name with its extension, optionally under +# folders. Each segment starts with a word character, which precludes '..', +# hidden files and absolute paths; the depth cap keeps a name a name. A prompt +# is named without its extension and lives at most one folder deep - an asset +# carries its extension, because which file it is depends on it, and media +# libraries nest deeper than prompt libraries do +ASSET_REFERENCE_PATTERN = r"^[\w][\w.-]*(/[\w][\w.-]*){0,4}\Z" +MAX_ASSET_REFERENCE_LENGTH = 400 + + +def validate_asset_reference(name: str) -> str: + """ + Validate the name an 'asset:' reference points at. + + The name is joined onto the asset directory to find the file, so it is + checked before anything touches the filesystem. Containment in the + library is checked separately, by the validate_path call that joins it. + + Args: + name: Asset name to validate + + Returns: + The validated name + + Raises: + InvalidInputError: If name is invalid + """ + if not name: + raise InvalidInputError("Asset name cannot be empty") + + if not re.match(ASSET_REFERENCE_PATTERN, name): + raise InvalidInputError( + f"Invalid asset name: {name} - an asset is named by its file under " + f"the asset directory, with its extension and at most four folders " + f"deep, like 'iris.jpg' or 'gyre/frames/iris.jpg'" + ) + + if len(name) > MAX_ASSET_REFERENCE_LENGTH: + raise InvalidInputError( + f"Asset name too long: {len(name)} > {MAX_ASSET_REFERENCE_LENGTH}" + ) + + return name + + # A dotted python name: identifiers separated by dots, and nothing else CONSTANT_NAME_PATTERN = r"^[a-zA-Z_][a-zA-Z0-9_]*(\.[a-zA-Z_][a-zA-Z0-9_]*)*\Z" diff --git a/dw/serve.py b/dw/serve.py index a4b2d12e..b4138c55 100644 --- a/dw/serve.py +++ b/dw/serve.py @@ -44,8 +44,7 @@ def main(): parser.add_argument( "--output-dir", default=None, - help="Directory results are written to (default: the workspace's " - "outputs/)", + help="Directory results are written to (default: the workspace's " "outputs/)", ) parser.add_argument( "--prompt-dir", @@ -54,6 +53,13 @@ def main(): "a CLI run discovers it - DW_PROMPT_DIR, else ./prompts if it " "exists, else the nearest prompts/ above the workflow directory)", ) + parser.add_argument( + "--asset-dir", + default=None, + help="Directory of input media 'asset:' references resolve against, " + "and where browser uploads are saved (default: the workspace's " + "assets/)", + ) parser.add_argument( "-l", "--log_level", @@ -100,6 +106,7 @@ def main(): workspace = set_workspace(resolve_workspace(args.workspace)) workflow_dir = args.workflow_dir or workspace.workflows output_dir = args.output_dir or workspace.outputs + asset_dir = os.path.abspath(args.asset_dir or workspace.assets) # A workspace's workflow folder is where the UI and MCP clients save, so # it has to exist for a first run in a fresh workspace. Only the folder @@ -109,6 +116,14 @@ def main(): if not args.workflow_dir: os.makedirs(workflow_dir, exist_ok=True) + # Created either way: it is the upload destination and a static mount, + # both of which need it to exist before the first request + os.makedirs(asset_dir, exist_ok=True) + + # Pinned like the prompt directory, so 'asset:' resolves to the same + # library in the worker that the upload route writes into + os.environ["DW_ASSET_DIR"] = asset_dir + token = args.token or os.environ.get("DW_API_TOKEN") or None from .server.app import LOOPBACK_HOSTS @@ -181,6 +196,7 @@ def main(): output_dir=output_dir, log_level=args.log_level, prompt_dir=prompt_dir, + asset_dir=asset_dir, workspace=workspace.root, host=args.host, token=token, diff --git a/dw/server/app.py b/dw/server/app.py index 749388d4..fb05c21c 100644 --- a/dw/server/app.py +++ b/dw/server/app.py @@ -379,6 +379,7 @@ def create_app( download_manager=None, diffusers_updater=None, prompt_dir="./prompts", + asset_dir=None, workspace=None, host="127.0.0.1", token=None, @@ -440,6 +441,10 @@ async def lifespan(app): app.state.job_manager = manager app.state.workflow_dir = workflow_dir app.state.prompt_dir = prompt_dir + # Where uploads land and 'asset:' references resolve. None when the + # caller configured no asset library: uploads then fall back to the + # output directory's uploads/ subfolder, as they did before there was one + app.state.asset_dir = os.path.abspath(asset_dir) if asset_dir else None # The workspace the three directories above default to folders of, for a # client that wants to name the root rather than reason about the parts. # None when the caller resolved no workspace (a test building an app @@ -1206,12 +1211,17 @@ def delete_output(name: str): @app.post("/api/uploads", status_code=201) async def upload_media(request: Request, filename: str): - """Save a browser-picked image or video into the output directory's - uploads/ subfolder and hand back its path - the same string shape a - workflow's 'image'/'video' arguments already accept (a plain path, - resolved absolute so it works regardless of the workflow file's own - directory). The body is the raw file bytes: no multipart parser - dependency needed for a single-file upload. + """Save a browser-picked image or video into the asset library's + uploads/ subfolder and hand back the reference a workflow argument + can carry. + + An upload is input, so it belongs in the asset library rather than + among generated output, and the reference handed back is + 'asset:uploads/' - portable, and meaningful in a workflow that + is saved and rerun later. A server with no asset library configured + keeps the old behavior, writing to the output directory's uploads/ + and returning an absolute path. The body is the raw file bytes: no + multipart parser dependency needed for a single-file upload. """ extension = os.path.splitext(os.path.basename(filename))[1].lower() if extension not in ALLOWED_UPLOAD_EXTENSIONS: @@ -1236,7 +1246,8 @@ async def upload_media(request: Request, filename: str): detail=f"Upload too large: {len(body)} > {MAX_UPLOAD_BYTES}", ) - uploads_dir = os.path.join(manager.output_dir, UPLOADS_SUBDIR) + library = app.state.asset_dir or manager.output_dir + uploads_dir = os.path.join(library, UPLOADS_SUBDIR) os.makedirs(uploads_dir, exist_ok=True) name = f"{uuid.uuid4().hex}{extension}" try: @@ -1248,6 +1259,11 @@ async def upload_media(request: Request, filename: str): # stream and poll for its duration await run_in_threadpool(_write_bytes, dest, body) logger.info(f"Saved upload {filename!r} -> {dest}") + if app.state.asset_dir: + return { + "path": f"asset:{UPLOADS_SUBDIR}/{name}", + "url": f"/assets/{UPLOADS_SUBDIR}/{quote(name)}", + } return { "path": dest, "url": f"/outputs/{UPLOADS_SUBDIR}/{quote(name)}", @@ -1444,6 +1460,7 @@ def server_info(): # individually overridden folder still reports its own path "workspace": app.state.workspace, "workflows": os.path.abspath(app.state.workflow_dir), + "assets": app.state.asset_dir, "outputs": os.path.abspath(manager.output_dir), "prompts": ( os.path.abspath(app.state.prompt_dir) @@ -1473,6 +1490,12 @@ def server_info(): ) app.mount("/outputs", StaticFiles(directory=manager.output_dir), name="outputs") + # Input media, served for the editor's preview of an uploaded or chosen + # asset. Ungated like /outputs, and for the same reason: an tag + # cannot attach an Authorization header + if app.state.asset_dir: + os.makedirs(app.state.asset_dir, exist_ok=True) + app.mount("/assets", StaticFiles(directory=app.state.asset_dir), name="assets") # ---------------------------------------------------------------- the UI diff --git a/tests/test_assets.py b/tests/test_assets.py new file mode 100644 index 00000000..d6763eb3 --- /dev/null +++ b/tests/test_assets.py @@ -0,0 +1,156 @@ +"""The asset library: 'asset:' references resolve to files under the asset +directory, and never to anything outside it.""" + +import os + +import pytest +from PIL import Image + +from dw.arguments import realize_args +from dw.assets import ( + ASSET_DIR_ENV_VAR, + get_asset_dir, + is_asset_reference, + resolve_asset_reference, +) +from dw.security import InvalidInputError, SecurityError +from dw.workspace import WORKSPACE_ENV_VAR, WORKSPACE_SOURCE_ENV_VAR + + +@pytest.fixture +def asset_dir(tmp_path, monkeypatch): + """An asset library with a top-level and a nested image.""" + library = tmp_path / "assets" + (library / "gyre" / "frames").mkdir(parents=True) + Image.new("RGB", (8, 8), "red").save(library / "iris.png") + Image.new("RGB", (8, 8), "blue").save(library / "gyre" / "frames" / "web.png") + monkeypatch.setenv(ASSET_DIR_ENV_VAR, str(library)) + return library + + +@pytest.fixture +def no_library(monkeypatch, tmp_path): + monkeypatch.delenv(ASSET_DIR_ENV_VAR, raising=False) + monkeypatch.delenv(WORKSPACE_ENV_VAR, raising=False) + monkeypatch.delenv(WORKSPACE_SOURCE_ENV_VAR, raising=False) + monkeypatch.setenv("DIFFUSERS_HELPER_ROOT", str(tmp_path / "helper")) + (tmp_path / "helper").mkdir() + return tmp_path + + +class TestAssetDir: + def test_the_environment_names_the_library(self, asset_dir): + assert get_asset_dir() == str(asset_dir) + + def test_a_named_workspace_names_the_library(self, no_library, monkeypatch): + monkeypatch.setenv(WORKSPACE_ENV_VAR, str(no_library / "studio")) + assert get_asset_dir() == str(no_library / "studio" / "assets") + + def test_the_working_directory_library_is_used(self, no_library, monkeypatch): + (no_library / "here" / "assets").mkdir(parents=True) + monkeypatch.chdir(no_library / "here") + assert get_asset_dir() == str(no_library / "here" / "assets") + + def test_the_walk_from_the_workflow_dir_finds_the_tree_s_library( + self, no_library, monkeypatch + ): + (no_library / "repo" / "assets").mkdir(parents=True) + workflow_dir = no_library / "repo" / "workflows" / "gyre" + workflow_dir.mkdir(parents=True) + elsewhere = no_library / "elsewhere" + elsewhere.mkdir() + monkeypatch.chdir(elsewhere) + assert get_asset_dir(str(workflow_dir)) == str(no_library / "repo" / "assets") + + +class TestReferences: + def test_a_reference_resolves_to_the_file(self, asset_dir): + assert resolve_asset_reference("asset:iris.png") == str(asset_dir / "iris.png") + + def test_a_nested_reference_resolves(self, asset_dir): + assert resolve_asset_reference("asset:gyre/frames/web.png") == str( + asset_dir / "gyre" / "frames" / "web.png" + ) + + def test_a_missing_asset_says_so(self, asset_dir): + with pytest.raises(ValueError, match="not found"): + resolve_asset_reference("asset:nothing.png") + + @pytest.mark.parametrize( + "reference", + [ + "asset:../outside.png", + "asset:/etc/passwd", + "asset:gyre/../../outside.png", + "asset:", + ], + ) + def test_a_reference_cannot_leave_the_library(self, asset_dir, reference): + with pytest.raises(SecurityError): + resolve_asset_reference(reference) + + def test_a_symlink_out_of_the_library_is_refused(self, asset_dir, tmp_path): + outside = tmp_path / "outside.png" + Image.new("RGB", (8, 8)).save(outside) + os.symlink(outside, asset_dir / "link.png") + with pytest.raises(SecurityError): + resolve_asset_reference("asset:link.png") + + def test_the_prefix_is_recognized(self): + assert is_asset_reference("asset:x.png") + assert not is_asset_reference("prompt:x") + assert not is_asset_reference(3) + + +class TestRealizedArguments: + def test_an_image_argument_loads_the_asset(self, asset_dir): + args = {"image": "asset:iris.png"} + realize_args(args) + assert args["image"].size == (8, 8) + + def test_a_plain_argument_becomes_the_path(self, asset_dir): + args = {"reference_file": "asset:gyre/frames/web.png"} + realize_args(args) + assert args["reference_file"] == str(asset_dir / "gyre" / "frames" / "web.png") + + def test_a_list_of_assets_resolves(self, asset_dir): + args = {"image": ["asset:iris.png", "asset:gyre/frames/web.png"]} + realize_args(args) + assert [image.size for image in args["image"]] == [(8, 8), (8, 8)] + + def test_an_object_built_from_an_asset_resolves(self, asset_dir): + args = {"reference": {"from_file": "asset:iris.png"}} + realize_args(args) + assert args["reference"]["from_file"] == str(asset_dir / "iris.png") + + def test_a_reference_is_rooted_at_the_library_not_the_workflow( + self, asset_dir, tmp_path + ): + # base_dir names an unrelated directory: an asset reference ignores it, + # which is the whole point - the workflow does not have to live beside + # the media it reads + elsewhere = tmp_path / "workflows" / "gyre" + elsewhere.mkdir(parents=True) + args = {"image": "asset:iris.png"} + realize_args(args, base_dir=str(elsewhere)) + assert args["image"].size == (8, 8) + + def test_a_bad_name_is_refused_during_realization(self, asset_dir): + with pytest.raises(InvalidInputError): + realize_args({"image": "asset:../escape.png"}) + + +class TestStoredPromptText: + def test_a_prompt_may_not_masquerade_as_an_asset_reference( + self, tmp_path, monkeypatch + ): + import json + + from dw.prompts import fetch_prompt + + library = tmp_path / "prompts" + library.mkdir() + (library / "sneaky.json").write_text(json.dumps({"text": "asset:iris.png"})) + monkeypatch.setenv("DW_PROMPT_DIR", str(library)) + with pytest.raises(ValueError, match="reference prefix"): + fetch_prompt("prompt:sneaky") diff --git a/tests/test_server.py b/tests/test_server.py index d311571b..73291ef8 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -1056,6 +1056,75 @@ def test_upload_media_saves_file_and_returns_absolute_path(server, tmp_path): assert fetched.content == b"not-really-png-bytes" +@pytest.fixture +def asset_server(tmp_path): + """A server with an asset library configured, which is where uploads go.""" + workflow_dir = tmp_path / "workflows" + workflow_dir.mkdir() + assets = tmp_path / "assets" + assets.mkdir() + + def make(script): + manager = JobManager( + str(tmp_path / "outputs"), + worker_manager=ScriptedWorkerManager(script), + history_path=str(tmp_path / "jobs.sqlite"), + ) + app = create_app( + workflow_dir=str(workflow_dir), + output_dir=str(tmp_path / "outputs"), + job_manager=manager, + asset_dir=str(assets), + ) + return TestClient(app, base_url="http://localhost") + + return make + + +def test_upload_media_lands_in_the_asset_library(asset_server, tmp_path): + """An upload is input, so it belongs in the asset library rather than + among generated output - and comes back as the reference a saved workflow + can carry, not as a path that only means something on this machine.""" + with asset_server(success_script) as client: + response = client.post( + "/api/uploads", + params={"filename": "source-image.png"}, + content=b"not-really-png-bytes", + ) + assert response.status_code == 201 + body = response.json() + + saved = tmp_path / "assets" / "uploads" + files = list(saved.iterdir()) + assert len(files) == 1 + assert files[0].read_bytes() == b"not-really-png-bytes" + assert body["path"] == f"asset:uploads/{files[0].name}" + assert body["url"] == f"/assets/uploads/{files[0].name}" + assert not (tmp_path / "outputs" / "uploads").exists() + + # served back for the editor's preview, through its own static mount + fetched = client.get(body["url"]) + assert fetched.status_code == 200 + assert fetched.content == b"not-really-png-bytes" + + +def test_the_asset_library_is_reported(asset_server, tmp_path): + with asset_server(success_script) as client: + directories = client.get("/api/server").json()["directories"] + assert directories["assets"] == str(tmp_path / "assets") + + +def test_without_an_asset_library_uploads_keep_the_old_shape(server, tmp_path): + with server(success_script) as client: + body = client.post( + "/api/uploads", + params={"filename": "source-image.png"}, + content=b"bytes", + ).json() + assert os.path.isabs(body["path"]) + assert body["url"].startswith("/outputs/uploads/") + + def test_upload_media_rejects_disallowed_extension(server): with server(success_script) as client: response = client.post( From 020104809f63994db90c179a6541604bb3cc8fba Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sat, 5 Sep 2026 18:25:46 +0000 Subject: [PATCH 03/34] feat(runs): one directory per run, with a manifest beside its files Output layout was keyed off a path segment literally named 'workflows', so the shape of a checkout was the grouping key and a workflow moved out of that tree silently flattened. It is now keyed off the workflow's own identity, and each execution gets its own directory: /// + manifest.json The run id is a UTC timestamp plus eight hex of the spec that ran, with a counter when two runs of the same spec start in the same second - one execution is one directory. A sub-workflow is part of its parent's run: it writes into the same directory and rolls up into the same manifest. manifest.json records status, seed, arguments, device, version and each step's files, relative to the directory so it keeps describing itself when moved. It is written for a failed run too - the files it wrote are on disk either way. A CLI run has never been recorded anywhere before this. The step cache now validates against the output *root* rather than the per-run directory, which is new every execution and would have defeated it; an unchanged rerun still writes nothing and reports the earlier run's files as reused. The gallery groups a workflow's runs under one folder by stripping the run id, so the folder filter does not grow one entry per run. --output-layout flat / DW_OUTPUT_LAYOUT / the output_layout setting keep the previous layout for scripts that glob the output directory. Stage three of docs/proposals/workspaces.md. Co-Authored-By: Claude Opus 5 (1M context) --- CLAUDE.md | 12 +- docs/SERVER.md | 23 +- docs/WORKSPACES.md | 36 +++ dw/run.py | 13 + dw/runs.py | 212 ++++++++++++++ dw/serve.py | 13 + dw/server/app.py | 22 +- dw/settings.py | 6 + dw/step_cache.py | 3 + dw/workflow.py | 125 ++++++++- .../w/20260905-181615-c771d89c/manifest.json | 16 ++ .../w/20260905-181615-fb21c742/manifest.json | 16 ++ .../w/20260905-181835-c771d89c/manifest.json | 16 ++ .../w/20260905-181835-f4f03ac4/manifest.json | 16 ++ .../w/20260905-182441-2a068c96/manifest.json | 16 ++ .../w/20260905-182441-c771d89c/manifest.json | 16 ++ tests/test_events.py | 16 +- tests/test_runs.py | 260 ++++++++++++++++++ 18 files changed, 811 insertions(+), 26 deletions(-) create mode 100644 dw/runs.py create mode 100644 output/w/20260905-181615-c771d89c/manifest.json create mode 100644 output/w/20260905-181615-fb21c742/manifest.json create mode 100644 output/w/20260905-181835-c771d89c/manifest.json create mode 100644 output/w/20260905-181835-f4f03ac4/manifest.json create mode 100644 output/w/20260905-182441-2a068c96/manifest.json create mode 100644 output/w/20260905-182441-c771d89c/manifest.json create mode 100644 tests/test_runs.py diff --git a/CLAUDE.md b/CLAUDE.md index b8ccf43d..ec19aec2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -133,7 +133,17 @@ All entry points use `dw/security.py`. When adding features: - **`{}`-escaped strings** in JSON arguments: `"{nf4}"` stays as string `"nf4"`, without braces it would try to load as a type - **A stored prompt's `text` may not begin with a reference prefix** (`variable:`, `previous_result:`, `constant:`, `asset:`, `prompt:`) — the engine rejects it to prevent double resolution or iteration expansion - **Audio+video muxing**: pipelines that generate audio alongside video (LTX-2) have the two muxed into one `video/mp4` file with PyAV in `result.py` -- **Step cache**: a process-wide singleton (`dw/step_cache.py`) consulted by every `Workflow.run`, including server jobs; entries are keyed by `(workflow id, step name)`; disabled entirely when the workflow sets no `seed`; a hit reports the earlier run's files with `reused: true` and writes nothing new; `memory clear` drops it +- **Run directories**: each execution writes `///` + with a `manifest.json` beside its files (`dw/runs.py`, `Workflow.effective_output_dir`). + Identity is the workflow's path under a `workflows/` tree, else its file name, else its + `id`; the run id is `-<8 hex of the spec>`, with a `-N` counter if taken. + A sub-workflow inherits the parent's run directory and writes no manifest of its own. + `--output-layout flat` / `DW_OUTPUT_LAYOUT` / the `output_layout` setting restores the + old layout. The gallery groups a workflow's runs under one folder by stripping the run + id (`strip_run_id`) +- **Step cache**: a process-wide singleton (`dw/step_cache.py`) consulted by every `Workflow.run`, including server jobs; entries are keyed by `(workflow id, step name)` and validated against the output + *root*, never the per-run directory - a run directory is new every execution and would + defeat the cache; disabled entirely when the workflow sets no `seed`; a hit reports the earlier run's files with `reused: true` and writes nothing new; `memory clear` drops it ## JSON Workflow Structure diff --git a/docs/SERVER.md b/docs/SERVER.md index ea8c43d3..3043d541 100644 --- a/docs/SERVER.md +++ b/docs/SERVER.md @@ -8,6 +8,9 @@ past generations in a gallery, and manage the models on disk. ```bash python -m dw.serve # http://127.0.0.1:8765 python -m dw.serve --port 8000 --workflow-dir ./workflows --output-dir ./outputs --prompt-dir ./prompts + +# or point it at a workspace, which supplies all four directories +python -m dw.serve --workspace ~/studio python -m dw.serve --host 0.0.0.0 --token "some-long-random-string" # reachable off this machine python -m dw.serve --trust-workflows # only if nothing untrusted can reach POST /api/jobs - see Security model ``` @@ -58,7 +61,11 @@ load entirely. literal `num_images_per_prompt` on both producers). It's a diagram of the JSON, not a second way to edit it; clicking a step jumps to it in the form view. -- **Gallery** — everything in the output directory. Images generated with +- **Gallery** — everything in the output directory, which the engine lays out + as `//`. The folder filter groups a workflow's runs + together rather than listing each run separately, and each run directory + also holds a `manifest.json` describing what produced it (see + [Workspaces](WORKSPACES.md#runs)). Images generated with `embed_metadata` carry their full workflow definition and seed; **open as workflow** loads that definition into the editor with the seed pinned, so any image can be reproduced or riffed on. Each tile carries a checkbox @@ -173,12 +180,14 @@ The editor's forms come from these; they are just as usable from scripts: - `POST /api/uploads?filename=...` — the raw bytes of one image or video (200MB ceiling, checked from `Content-Length` before a byte is read, and again on the body; extension held to the allowed image/video list), saved - into the output directory's `uploads/` subfolder under a generated name. - Answers 201 with `path` - the absolute path, which is the string shape a - workflow's `image`/`video` argument already takes - and `url`, the same - file under the `/outputs` mount. This is how the UI's file pickers get a - local file onto the machine that will run the workflow. The body is the - file itself, so no multipart parser is needed for a single-file upload + into the asset library's `uploads/` subfolder under a generated name. + Answers 201 with `path` - `asset:uploads/`, the reference a saved + workflow can carry and still resolve on a later run - and `url`, the same + file under the `/assets` mount, for the editor's preview. A server started + without an asset library falls back to the output directory's `uploads/` + and an absolute path. This is how the UI's file pickers get a local file + onto the machine that will run the workflow. The body is the file itself, + so no multipart parser is needed for a single-file upload - `GET /api/models`, `DELETE /api/models?repo={repo_id}` — hub cache inventory and deletion - `POST /api/models/download` (`{"repo_id": ...}`), `GET /api/models/downloads`, diff --git a/docs/WORKSPACES.md b/docs/WORKSPACES.md index 0950296b..0fe2adad 100644 --- a/docs/WORKSPACES.md +++ b/docs/WORKSPACES.md @@ -92,6 +92,42 @@ media it reads no longer have to sit in the same folder. `--asset-dir` and `assets/uploads/`, coming back as `asset:uploads/`. See [Asset References](WORKFLOW_GUIDE.md#asset-references). +## Runs + +Each execution writes its own directory under the output folder, named by the +workflow and the run: + +``` +outputs/ + ltx2/Gyre/ + 20260905-181530-a1b2c3d4/ + Gyre-still.0-0.0.png + Gyre-video.1-0.0.mp4 + manifest.json +``` + +The folder is the workflow's identity — its path under a `workflows/` tree +when it has one, its file name otherwise, its `id` for an inline definition. +The run id is a timestamp plus a short digest of what actually ran, so two +runs of the same workflow sort by time and a rerun of an edited workflow is +visibly different; a second run of the same spec in the same second takes a +counter rather than sharing a directory. + +`manifest.json` records the run beside what it made — status, seed, arguments, +device, dw version, and each step's files, named relative to the directory so +it keeps describing itself if you move or copy it. It is written even when a +run fails part way, since the files it did write are on disk either way. A +sub-workflow is part of its parent's run: it writes into the same directory and +rolls up into the same manifest. + +An unchanged rerun still reuses the step cache: it writes no new files and its +manifest reports the earlier run's, marked `"reused": true`. + +To keep the previous layout — everything at the output root, with only a +`workflows/`-mirroring subfolder — use `--output-layout flat`, `DW_OUTPUT_LAYOUT=flat`, +or `"output_layout": "flat"` in settings. Scripts that glob the output directory +are the reason to. + ## Where this is going Workspaces are the first stage of the design in diff --git a/dw/run.py b/dw/run.py index a86a9b62..33788799 100644 --- a/dw/run.py +++ b/dw/run.py @@ -2,6 +2,7 @@ import os from . import startup from .workflow import workflow_from_file +from .runs import set_output_layout from .workspace import resolve_workspace, set_workspace from .security import ( validate_workflow_path, @@ -65,6 +66,15 @@ def main(): "named, else ./assets if it exists, else the nearest assets/ above " "the workflow file)", ) + parser.add_argument( + "--output-layout", + type=str, + choices=("run", "flat"), + default=None, + help="'run' (default) gives each run its own directory under " + "//, with a manifest.json beside its files; " + "'flat' writes the way it did before run directories", + ) parser.add_argument( "--trust-workflows", action="store_true", @@ -93,6 +103,9 @@ def main(): if args.asset_dir: os.environ["DW_ASSET_DIR"] = os.path.abspath(args.asset_dir) + if args.output_layout: + set_output_layout(args.output_layout) + set_trust_workflows(args.trust_workflows) # Parse key-value pairs with validation diff --git a/dw/runs.py b/dw/runs.py new file mode 100644 index 00000000..f43da209 --- /dev/null +++ b/dw/runs.py @@ -0,0 +1,212 @@ +"""A run: the directory one execution of a workflow writes into, and the +manifest it leaves behind. + +Output used to be laid out by where the workflow file sits - the subfolder +mirrored its position under the nearest directory literally named 'workflows', +so the *shape of a checkout* was the grouping key, and a workflow moved out of +that tree silently flattened. A run directory replaces that with the +workflow's own identity plus one directory per execution: + + /// + + manifest.json + +Everything one execution produced - intermediates, finals, and the record of +what made them - lands in one place, prunable and addressable as a unit, and +a rerun can no longer interleave its files with an earlier one's. + +The old flat-ish layout stays available: DW_OUTPUT_LAYOUT=flat, an +'output_layout' setting of "flat", or --output-layout flat on dw.run and +dw.serve, for a caller whose scripts glob the output directory. +""" + +import hashlib +import json +import logging +import os +import re +from datetime import datetime + +logger = logging.getLogger("dw") + +RUN_LAYOUT = "run" +FLAT_LAYOUT = "flat" +LAYOUTS = (RUN_LAYOUT, FLAT_LAYOUT) + +# Set by an entry point, and inherited by a spawned worker the way +# DW_PROMPT_DIR and DW_ASSET_DIR are +OUTPUT_LAYOUT_ENV_VAR = "DW_OUTPUT_LAYOUT" + +MANIFEST_FILE_NAME = "manifest.json" + +# What a run id looks like: a UTC timestamp and a short digest of the spec. +# The pattern is not only documentation - the gallery reads it to group a +# workflow's runs under one folder rather than listing every run separately +# The trailing counter appears only when two runs of the same spec start in +# the same second - see run_directory +RUN_ID_PATTERN = re.compile(r"^\d{8}-\d{6}-[0-9a-f]{8}(-\d+)?$") + +# Characters allowed in a path segment derived from a workflow's name or file +_UNSAFE_SEGMENT_CHARACTERS = re.compile(r"[^A-Za-z0-9_.-]+") + +# The synthetic file name workflow_from_definition gives an inline workflow - +# it carries a directory, not an identity +INLINE_FILE_NAME = "__inline__" + + +def output_layout(): + """Whether runs get their own directory ('run') or write into the output + directory the way they did before ('flat'). + + Read at call time so a worker subprocess and a test see the current + value, the same as every other directory question. + """ + from_environment = os.environ.get(OUTPUT_LAYOUT_ENV_VAR) + if from_environment in LAYOUTS: + return from_environment + + from .settings import load_settings + + from_settings = load_settings().output_layout + return from_settings if from_settings in LAYOUTS else RUN_LAYOUT + + +def set_output_layout(layout): + """Pin the layout for this process and anything it spawns.""" + if layout not in LAYOUTS: + raise ValueError(f"Unknown output layout: {layout}") + os.environ[OUTPUT_LAYOUT_ENV_VAR] = layout + return layout + + +def _safe_segment(text): + cleaned = _UNSAFE_SEGMENT_CHARACTERS.sub("_", str(text)).strip("._") + return cleaned or "workflow" + + +def workflow_identity(file_spec, workflow_id=None): + """What names this workflow's outputs, as a relative path. + + A workflow's position under a 'workflows' tree still reads as its + identity when it has one - 'workflows/ltx2/Gyre.json' is 'ltx2/Gyre' - + because that is the organization a user already chose. Outside such a + tree the file's own name is the identity, and an inline definition, + which has no file, is named by its workflow id. + + The result is always a relative path of safe segments: it is joined onto + the output directory, and nothing about it is allowed to leave. + """ + name = None + subfolder = "" + if file_spec: + base = os.path.basename(file_spec) + stem = os.path.splitext(base)[0] + if stem and stem != INLINE_FILE_NAME: + name = stem + directory = os.path.dirname(os.path.abspath(file_spec)) + parts = os.path.normpath(directory).split(os.sep) + try: + # The last 'workflows' segment wins, matching the packaged + # dw/workflows tree when a checkout has a top-level one too + index = len(parts) - 1 - parts[::-1].index("workflows") + except ValueError: + index = None + if index is not None and index + 1 < len(parts): + subfolder = os.path.join(*(_safe_segment(p) for p in parts[index + 1 :])) + + name = _safe_segment(name or workflow_id or "workflow") + return os.path.join(subfolder, name) if subfolder else name + + +def new_run_id(spec=None, now=None): + """An identifier for one execution: a UTC timestamp, then eight hex + digits of the spec that produced it. + + The timestamp is what sorts and what a person reads; the digest is what + tells two runs of the same second apart and makes a rerun of an edited + workflow visibly different from a rerun of the same one. A server job + could have used its job id, but a CLI run has none, and one scheme + everywhere is what lets anything reading the directory tree - the + gallery, a future history rebuild - understand both. + """ + stamp = (now or datetime.now()).strftime("%Y%m%d-%H%M%S") + try: + material = json.dumps(spec, sort_keys=True, default=str) + except (TypeError, ValueError): + material = repr(spec) + digest = hashlib.sha256(material.encode("utf-8", "replace")).hexdigest()[:8] + return f"{stamp}-{digest}" + + +def is_run_id(segment): + """Whether a path segment is a run id this module generated.""" + return bool(RUN_ID_PATTERN.match(segment or "")) + + +def strip_run_id(relative_path): + """The workflow identity a run-relative path belongs to. + + 'ltx2/Gyre/20260905-181530-a1b2c3d4/still-0.png' -> 'ltx2/Gyre'. A path + with no run id in it comes back with its own directory unchanged, which + is what a flat-layout output does. + """ + parts = [part for part in (relative_path or "").split("/") if part] + directory = parts[:-1] + if directory and is_run_id(directory[-1]): + directory = directory[:-1] + return "/".join(directory) + + +def run_directory(output_dir, file_spec, workflow_id, run_id): + """Where one execution writes: //. + + One execution gets one directory, so a run id already taken - two runs + of the same spec started in the same second, which is what a quick + rerun is - takes a counter rather than writing into the earlier run's + directory and burying its manifest. + """ + base = os.path.join(output_dir, workflow_identity(file_spec, workflow_id), run_id) + candidate = base + counter = 1 + while os.path.exists(candidate): + counter += 1 + candidate = f"{base}-{counter}" + return candidate + + +def write_manifest(run_dir, manifest): + """Record what a run did, beside what it made. + + A server run is already in jobs.sqlite, but a CLI run has never been + recorded anywhere, and history that lives only in a database cannot + survive the directory being moved to another machine. Never fatal: a + run that produced its files has succeeded whether or not this lands. + """ + path = os.path.join(run_dir, MANIFEST_FILE_NAME) + try: + os.makedirs(run_dir, exist_ok=True) + with open(path, "w") as file: + json.dump(manifest, file, indent=2, default=str) + except OSError as e: + logger.warning(f"Could not write {path}: {e}") + return None + return path + + +def manifest_relative_files(files, run_dir): + """A run's file paths as the manifest records them: relative to the run + directory, so the directory can be moved or copied and still describe + itself. A file from an earlier run - what a step cache hit republishes - + is outside this directory and stays absolute. + """ + recorded = [] + for path in files or []: + try: + relative = os.path.relpath(path, run_dir) + except ValueError: # different drive on Windows + recorded.append(path) + continue + recorded.append( + path if relative.startswith(os.pardir) else relative.replace(os.sep, "/") + ) + return recorded diff --git a/dw/serve.py b/dw/serve.py index b4138c55..8d32b5e9 100644 --- a/dw/serve.py +++ b/dw/serve.py @@ -60,6 +60,14 @@ def main(): "and where browser uploads are saved (default: the workspace's " "assets/)", ) + parser.add_argument( + "--output-layout", + choices=("run", "flat"), + default=None, + help="'run' (default) gives each job its own directory under " + "//, with a manifest.json beside its files; " + "'flat' writes the way it did before run directories", + ) parser.add_argument( "-l", "--log_level", @@ -124,6 +132,11 @@ def main(): # library in the worker that the upload route writes into os.environ["DW_ASSET_DIR"] = asset_dir + if args.output_layout: + from .runs import set_output_layout + + set_output_layout(args.output_layout) + token = args.token or os.environ.get("DW_API_TOKEN") or None from .server.app import LOOPBACK_HOSTS diff --git a/dw/server/app.py b/dw/server/app.py index fb05c21c..3e921ae9 100644 --- a/dw/server/app.py +++ b/dw/server/app.py @@ -53,6 +53,7 @@ from .enhancers import build_enhance_workflow, preset_descriptions from ..result import read_embedded_metadata from ..hub_cache import scan_models, delete_model, DownloadManager +from ..runs import strip_run_id from .jobs import JobManager, MAX_PERSISTED_EVENTS, TERMINAL_STATES from .netinfo import local_addresses from .updater import DiffusersUpdater @@ -1016,20 +1017,26 @@ def _output_file(name): def _iter_gallery_files(): """Every media file under the output directory, recursing into the - per-workflow subfolders (dw/workflow.py's effective_output_dir mirrors - a workflow's position under a 'workflows' tree into the output dir). + per-workflow subfolders (dw/workflow.py's effective_output_dir writes + each run under '//', and mirrors a + workflow's position under a 'workflows' tree in the flat layout). Yields (relative_name, folder, kind, path) - relative_name always uses '/' so it round-trips through a URL the same way on every - platform.""" + platform. + + The folder a file is grouped under drops the run id: a workflow run + fifty times is one folder in the filter, not fifty. Which run a file + came from is still in its name, and in the manifest beside it.""" for root, _dirs, names in os.walk(manager.output_dir): rel_root = os.path.relpath(root, manager.output_dir) - folder = "" if rel_root == "." else rel_root.replace(os.sep, "/") + directory = "" if rel_root == "." else rel_root.replace(os.sep, "/") for name in names: extension = os.path.splitext(name)[1].lower() kind = MEDIA_KINDS.get(extension) if kind is None: continue - relative_name = name if not folder else f"{folder}/{name}" + relative_name = name if not directory else f"{directory}/{name}" + folder = strip_run_id(relative_name) yield relative_name, folder, kind, os.path.join(root, name) def _gallery_entries(): @@ -1073,8 +1080,9 @@ def gallery(limit: int = 200, offset: int = 0, folder: Optional[str] = None): """A page of media files in the output directory, newest first. Stateless by design - the gallery survives server restarts because it reads the directory tree, not job history. 'folders' lists every - distinct workflow subfolder present (over the whole directory, not - just this page), for the UI's folder filter - '' stands for files + distinct workflow folder present (over the whole directory, not just + this page), for the UI's folder filter - a run id is not a folder of + its own, so a workflow's runs group together; '' stands for files saved directly at the output root, and is itself always a member so that folder-less outputs stay selectable once anything is nested.""" entries = _gallery_entries() diff --git a/dw/settings.py b/dw/settings.py index c9d19b17..b6fa36cd 100644 --- a/dw/settings.py +++ b/dw/settings.py @@ -18,6 +18,11 @@ class Settings: # working directory when it looks like a workspace, then ~/diffusers-workspace workspace: str = None + # How generated files are laid out under the output directory: "run" + # gives each execution its own directory, "flat" keeps the pre-workspace + # layout. See dw/runs.py + output_layout: str = "run" + # PyTorch optimization settings enable_tf32: bool = True # TensorFloat-32 for faster matmul on Ampere+ GPUs cudnn_benchmark: bool = True # cuDNN autotuner (faster for fixed sizes) @@ -41,6 +46,7 @@ def load_settings(): settings.device = settings_dict.get("device", None) settings.workspace = settings_dict.get("workspace", None) + settings.output_layout = settings_dict.get("output_layout", "run") # PyTorch optimization settings settings.enable_tf32 = settings_dict.get("enable_tf32", True) diff --git a/dw/step_cache.py b/dw/step_cache.py index 9a7eec9b..ee472a63 100644 --- a/dw/step_cache.py +++ b/dw/step_cache.py @@ -157,6 +157,9 @@ def get( return None if entry["step_seed"] != step_seed: return None + # The output *root* a run was told to write to. A run directory is + # new every execution and would defeat the cache; the root changing + # means the caller asked for output somewhere the cached files are not if entry["output_dir"] != output_dir: return None if needs_result and not entry["retained"]: diff --git a/dw/workflow.py b/dw/workflow.py index 219d0880..10aac5c6 100644 --- a/dw/workflow.py +++ b/dw/workflow.py @@ -6,6 +6,7 @@ import gc import hashlib import logging +from datetime import datetime, timezone from .arguments import realize_args, realize_constants from .events import ( RunContext, @@ -22,6 +23,15 @@ referenced_result_names, reference_resolves_to, ) +from .runs import ( + FLAT_LAYOUT, + workflow_identity, + manifest_relative_files, + new_run_id, + output_layout, + run_directory, + write_manifest, +) from .schema import validate_data, load_schema from .variables import replace_variables, set_variables from .pipeline_processors.pipeline import Pipeline @@ -180,6 +190,17 @@ class Workflow: # to hand down to a sub-workflow _cache_enabled_this_run = True + # The directory the run in progress writes into, set by run() and, for a + # sub-workflow, handed down by the parent - one execution is one + # directory, whichever workflow inside it did the writing. None in the + # flat layout, and before a run starts + _run_dir = None + # Whether that directory came from a parent workflow. A sub-workflow is + # part of the parent's execution: it writes into the same directory and + # leaves no manifest of its own, since its steps are already rolled up + # into the parent's + _run_dir_inherited = False + def __init__(self, workflow_definition, output_dir, file_spec, workflow_dir=None): self.workflow_definition = workflow_definition self.output_dir = output_dir @@ -209,19 +230,24 @@ def step_file_prefix(self, step_name): @property def effective_output_dir(self): - """Where this workflow's own results are written: output_dir, plus a - subfolder mirroring the workflow file's position under a 'workflows' - directory, if it has one. + """Where this workflow's own results are written. + + In the default layout that is the run directory run() opened - + '///' - shared by every step of the + run, sub-workflows included, so one execution leaves one directory. - A workflow at 'workflows/ltx/Foo.json' writes under + In the flat layout it is output_dir plus a subfolder mirroring the + workflow file's position under a 'workflows' directory, if it has + one: a workflow at 'workflows/ltx/Foo.json' writes under '/ltx/'; one directly inside a 'workflows' folder (or a builtin, which always resolves to dw/workflows/.json) writes flat at '/', same as one outside any 'workflows' tree - entirely (an inline definition, say). self.output_dir itself always - stays the plain root - this is derived fresh from it every time, so - a sub-workflow computes its own subfolder from its own file, not the - parent's. + entirely. self.output_dir itself always stays the plain root - this + is derived fresh from it every time, so a flat-layout sub-workflow + computes its own subfolder from its own file, not the parent's. """ + if self._run_dir: + return self._run_dir subfolder = workflow_output_subfolder(self.file_spec) return ( os.path.join(self.output_dir, subfolder) if subfolder else self.output_dir @@ -263,6 +289,11 @@ def run( # BEFORE its replacement loads, or the transition holds both at once self._prior_step_keys = prior_step_keys or {} self.manifest = [] + # Overwritten on the way out of the try below - a run that leaves + # this alone died on an exception the manifest should say so about + status = "failed" + run_id = None + started_at = datetime.now(timezone.utc).isoformat() try: # CRITICAL: Work on a copy to avoid mutating the original workflow definition # This allows the workflow to be run multiple times with different arguments @@ -317,6 +348,24 @@ def run( default_seed = torch.Generator().seed() workflow_def["seed"] = default_seed + # One execution, one directory - opened here, after variable + # substitution and the seed have settled, so the run's identity + # covers what actually ran rather than what was written down. A + # sub-workflow inherits the parent's and never opens its own + started_at = datetime.now(timezone.utc).isoformat() + run_id = None + if not self._run_dir_inherited: + if output_layout() == FLAT_LAYOUT: + self._run_dir = None + else: + run_id = new_run_id( + {"workflow": workflow_def, "arguments": arguments} + ) + self._run_dir = run_directory( + self.output_dir, self.file_spec, workflow_id, run_id + ) + logger.debug(f"Run directory: {self._run_dir}") + # Initialize collections for sharing state between steps results = {} # Stores results from each step shared_components = {} # Shared resources between steps @@ -413,7 +462,13 @@ def run( step_data_snapshot, step_seed, hits_this_run, - self.effective_output_dir, + # The root, not this run's directory: a hit reports + # the earlier run's files and writes nothing new, so + # keying on a directory that is new every run would + # mean the cache could never hit again. What the root + # still guards is a run redirected somewhere else, + # where the earlier files are not what the caller asked for + self.output_dir, needs_result=result_needed, ) if is_cacheable @@ -451,7 +506,7 @@ def run( step_data_snapshot, step_seed, result, - self.effective_output_dir, + self.output_dir, retain_result=result_needed, ) @@ -519,12 +574,14 @@ def run( "workflow_end", workflow=workflow_id, manifest=self.manifest ) # Return only the last step's results for child workflows + status = "completed" return last_result.result_list if last_result is not None else [] except WorkflowCancelled: # The user asked for this - report it without an error traceback workflow_id = self.workflow_definition.get("id", "unknown") logger.info(f"Workflow {workflow_id} cancelled") + status = "cancelled" raise except (SecurityError, PathTraversalError, InvalidInputError) as e: # Security validation failures - these should fail fast, without the @@ -541,8 +598,51 @@ def run( ) raise finally: + # Recorded even for a run that failed part way: the files it did + # write are on disk either way, and what produced them is exactly + # what a failed run needs to explain itself + if self._run_dir and not self._run_dir_inherited: + self._write_run_manifest(run_id, status, started_at, arguments) deactivate_context(context_token) + def _write_run_manifest(self, run_id, status, started_at, arguments): + """Leave a record of the run beside the files it wrote. + + A server run is in jobs.sqlite as well, but a CLI run has never been + recorded anywhere, and a database on one machine cannot describe a + directory copied to another. Paths are relative to the run directory + so the directory keeps describing itself wherever it goes. + """ + from . import __version__ + + write_manifest( + self._run_dir, + { + "run_id": run_id, + "status": status, + "started_at": started_at, + "finished_at": datetime.now(timezone.utc).isoformat(), + "dw_version": __version__, + "device": str(get_device()), + "workflow": { + "id": self.name, + "file": self.file_spec, + "identity": workflow_identity(self.file_spec, self.name), + }, + "seed": self.workflow_definition.get("seed"), + "arguments": arguments or {}, + "steps": [ + { + **entry, + "files": manifest_relative_files( + entry.get("files"), self._run_dir + ), + } + for entry in self.manifest + ], + }, + ) + def _step_pipeline_key(self, step_name, cache_key): """Record which cache key a step's pipeline lives under this run.""" if not hasattr(self, "_pipeline_keys_by_step"): @@ -723,6 +823,11 @@ def create_step_action( # step of the child can ever hit - the child must not pay the # cache's deepcopy and Result pinning for it workflow._cache_enabled_by_parent = self._cache_enabled_this_run + # One execution, one directory: the child writes into the + # parent's run directory and leaves no manifest of its own - its + # steps roll up into the parent's manifest already + workflow._run_dir = self._run_dir + workflow._run_dir_inherited = self._run_dir is not None workflow.validate() return workflow diff --git a/output/w/20260905-181615-c771d89c/manifest.json b/output/w/20260905-181615-c771d89c/manifest.json new file mode 100644 index 00000000..a20fb0ad --- /dev/null +++ b/output/w/20260905-181615-c771d89c/manifest.json @@ -0,0 +1,16 @@ +{ + "run_id": "20260905-181615-c771d89c", + "status": "failed", + "started_at": "2026-09-05T18:16:15.072353+00:00", + "finished_at": "2026-09-05T18:16:15.072502+00:00", + "dw_version": "0.4.0-beta.1", + "device": "cuda", + "workflow": { + "id": "w", + "file": "", + "identity": "w" + }, + "seed": 42, + "arguments": {}, + "steps": [] +} \ No newline at end of file diff --git a/output/w/20260905-181615-fb21c742/manifest.json b/output/w/20260905-181615-fb21c742/manifest.json new file mode 100644 index 00000000..477180d3 --- /dev/null +++ b/output/w/20260905-181615-fb21c742/manifest.json @@ -0,0 +1,16 @@ +{ + "run_id": "20260905-181615-fb21c742", + "status": "failed", + "started_at": "2026-09-05T18:16:15.075168+00:00", + "finished_at": "2026-09-05T18:16:15.075304+00:00", + "dw_version": "0.4.0-beta.1", + "device": "cuda", + "workflow": { + "id": "w", + "file": "", + "identity": "w" + }, + "seed": null, + "arguments": {}, + "steps": [] +} \ No newline at end of file diff --git a/output/w/20260905-181835-c771d89c/manifest.json b/output/w/20260905-181835-c771d89c/manifest.json new file mode 100644 index 00000000..677e8ca4 --- /dev/null +++ b/output/w/20260905-181835-c771d89c/manifest.json @@ -0,0 +1,16 @@ +{ + "run_id": "20260905-181835-c771d89c", + "status": "failed", + "started_at": "2026-09-05T18:18:35.971638+00:00", + "finished_at": "2026-09-05T18:18:35.971851+00:00", + "dw_version": "0.4.0-beta.1", + "device": "cuda", + "workflow": { + "id": "w", + "file": "", + "identity": "w" + }, + "seed": 42, + "arguments": {}, + "steps": [] +} \ No newline at end of file diff --git a/output/w/20260905-181835-f4f03ac4/manifest.json b/output/w/20260905-181835-f4f03ac4/manifest.json new file mode 100644 index 00000000..2f7b637c --- /dev/null +++ b/output/w/20260905-181835-f4f03ac4/manifest.json @@ -0,0 +1,16 @@ +{ + "run_id": "20260905-181835-f4f03ac4", + "status": "failed", + "started_at": "2026-09-05T18:18:35.975347+00:00", + "finished_at": "2026-09-05T18:18:35.975536+00:00", + "dw_version": "0.4.0-beta.1", + "device": "cuda", + "workflow": { + "id": "w", + "file": "", + "identity": "w" + }, + "seed": null, + "arguments": {}, + "steps": [] +} \ No newline at end of file diff --git a/output/w/20260905-182441-2a068c96/manifest.json b/output/w/20260905-182441-2a068c96/manifest.json new file mode 100644 index 00000000..7cd19e51 --- /dev/null +++ b/output/w/20260905-182441-2a068c96/manifest.json @@ -0,0 +1,16 @@ +{ + "run_id": "20260905-182441-2a068c96", + "status": "failed", + "started_at": "2026-09-05T18:24:41.520769+00:00", + "finished_at": "2026-09-05T18:24:41.520962+00:00", + "dw_version": "0.4.0-beta.1", + "device": "cuda", + "workflow": { + "id": "w", + "file": "", + "identity": "w" + }, + "seed": null, + "arguments": {}, + "steps": [] +} \ No newline at end of file diff --git a/output/w/20260905-182441-c771d89c/manifest.json b/output/w/20260905-182441-c771d89c/manifest.json new file mode 100644 index 00000000..448224af --- /dev/null +++ b/output/w/20260905-182441-c771d89c/manifest.json @@ -0,0 +1,16 @@ +{ + "run_id": "20260905-182441-c771d89c", + "status": "failed", + "started_at": "2026-09-05T18:24:41.517070+00:00", + "finished_at": "2026-09-05T18:24:41.517303+00:00", + "dw_version": "0.4.0-beta.1", + "device": "cuda", + "workflow": { + "id": "w", + "file": "", + "identity": "w" + }, + "seed": 42, + "arguments": {}, + "steps": [] +} \ No newline at end of file diff --git a/tests/test_events.py b/tests/test_events.py index 70284b68..161de258 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -2,6 +2,8 @@ import json import logging +import os +import pathlib import pytest from unittest.mock import patch @@ -10,6 +12,7 @@ from dw.events import RunContext, WorkflowCancelled, get_context, current_context from dw.log_setup import setup_logging from dw.result import Result +from dw.runs import is_run_id from dw.workflow import Workflow, pipeline_cache_key from dw.pipeline_processors.pipeline import Pipeline @@ -150,7 +153,18 @@ def mock_load(self, shared_components): assert entry["step"] == "gen0" assert len(entry["files"]) == 1 saved = entry["files"][0] - assert saved.endswith(".png") and (tmp_path / saved.split("/")[-1]).exists() + # The manifest names the file by the absolute path it was written to, + # which the default layout puts in this run's own directory under the + # workflow's identity - '/test//' + assert saved.endswith(".png") and os.path.exists(saved) + run_dir = os.path.dirname(saved) + assert os.path.dirname(run_dir) == str(tmp_path / "test") + assert is_run_id(os.path.basename(run_dir)) + # and the run records itself beside what it made + manifest = json.loads((pathlib.Path(run_dir) / "manifest.json").read_text()) + assert manifest["status"] == "completed" + assert manifest["workflow"]["identity"] == "test" + assert manifest["steps"][0]["files"] == [os.path.basename(saved)] def test_result_save_returns_json_paths(tmp_path): diff --git a/tests/test_runs.py b/tests/test_runs.py new file mode 100644 index 00000000..8c94e19d --- /dev/null +++ b/tests/test_runs.py @@ -0,0 +1,260 @@ +"""Run directories: one execution writes one directory, named by the +workflow's identity, and leaves a manifest describing itself.""" + +import json +import os +from unittest.mock import patch + +import pytest + +from dw.runs import ( + FLAT_LAYOUT, + OUTPUT_LAYOUT_ENV_VAR, + RUN_LAYOUT, + is_run_id, + manifest_relative_files, + new_run_id, + output_layout, + strip_run_id, + workflow_identity, +) + + +class TestIdentity: + @pytest.mark.parametrize( + "file_spec,expected", + [ + ("/x/workflows/ltx2/Gyre.json", "ltx2/Gyre"), + ("/x/workflows/Gyre.json", "Gyre"), + ("/x/anywhere/Gyre.json", "Gyre"), + ("/x/workflows/a/b/Gyre.json", "a/b/Gyre"), + ], + ) + def test_a_workflow_is_named_by_its_file(self, file_spec, expected): + assert workflow_identity(file_spec, "id").replace(os.sep, "/") == expected + + def test_an_inline_definition_is_named_by_its_id(self): + # The synthetic file name carries a directory, not an identity + assert workflow_identity("/x/workflows/__inline__.json", "my-flow") == "my-flow" + + def test_a_hostile_id_cannot_escape_the_output_directory(self): + # The identity is joined onto the output directory, so nothing in it + # may traverse: separators and dot segments do not survive + identity = workflow_identity(None, "../../etc/passwd") + assert "/" not in identity and "\\" not in identity and ".." not in identity + + def test_something_is_always_named(self): + assert workflow_identity(None, None) == "workflow" + + +class TestRunIds: + def test_a_run_id_is_a_timestamp_and_a_digest(self): + run_id = new_run_id({"workflow": "spec"}) + assert is_run_id(run_id) + + def test_the_digest_is_of_the_spec(self): + # Two runs of the same spec share a digest, which is what makes a + # rerun of an edited workflow visibly different in the directory list + from datetime import datetime + + when = datetime(2026, 9, 5, 12, 0, 0) + assert new_run_id({"a": 1}, now=when) == new_run_id({"a": 1}, now=when) + assert new_run_id({"a": 1}, now=when) != new_run_id({"a": 2}, now=when) + + def test_an_unserializable_spec_still_yields_an_id(self): + assert is_run_id(new_run_id({"generator": object()})) + + def test_stripping_a_run_id_gives_the_workflow_folder(self): + run_id = new_run_id({}) + assert strip_run_id(f"ltx2/Gyre/{run_id}/still.png") == "ltx2/Gyre" + assert strip_run_id(f"{run_id}/still.png") == "" + + def test_a_path_with_no_run_id_keeps_its_folder(self): + assert strip_run_id("ltx2/still.png") == "ltx2" + assert strip_run_id("still.png") == "" + + +class TestLayoutResolution: + def test_run_is_the_default(self, monkeypatch, tmp_path): + monkeypatch.delenv(OUTPUT_LAYOUT_ENV_VAR, raising=False) + monkeypatch.setenv("DIFFUSERS_HELPER_ROOT", str(tmp_path)) + assert output_layout() == RUN_LAYOUT + + def test_the_environment_selects_flat(self, monkeypatch): + monkeypatch.setenv(OUTPUT_LAYOUT_ENV_VAR, FLAT_LAYOUT) + assert output_layout() == FLAT_LAYOUT + + def test_the_setting_selects_flat(self, monkeypatch, tmp_path): + monkeypatch.delenv(OUTPUT_LAYOUT_ENV_VAR, raising=False) + monkeypatch.setenv("DIFFUSERS_HELPER_ROOT", str(tmp_path)) + (tmp_path / "settings.json").write_text(json.dumps({"output_layout": "flat"})) + assert output_layout() == FLAT_LAYOUT + + def test_nonsense_falls_back_to_run(self, monkeypatch, tmp_path): + monkeypatch.setenv(OUTPUT_LAYOUT_ENV_VAR, "sideways") + monkeypatch.setenv("DIFFUSERS_HELPER_ROOT", str(tmp_path)) + assert output_layout() == RUN_LAYOUT + + +class TestManifestPaths: + def test_files_inside_the_run_are_recorded_relative(self, tmp_path): + run_dir = str(tmp_path / "run") + files = [os.path.join(run_dir, "a.png"), os.path.join(run_dir, "sub", "b.png")] + assert manifest_relative_files(files, run_dir) == ["a.png", "sub/b.png"] + + def test_a_file_from_an_earlier_run_stays_absolute(self, tmp_path): + # What a step cache hit republishes: the file is real, but it is not + # this run's to describe relatively + earlier = str(tmp_path / "earlier" / "a.png") + assert manifest_relative_files([earlier], str(tmp_path / "run")) == [earlier] + + +def _workflow_definition(): + return { + "id": "runs_test", + "seed": 7, + "steps": [ + { + "name": "gen0", + "result": {"content_type": "image/png"}, + "pipeline": { + "configuration": { + "component_type": "{FakePipeline}", + "no_generator": True, + }, + "from_pretrained_arguments": {"model_name": "model-0"}, + "arguments": {"prompt": "p", "num_inference_steps": 1}, + }, + } + ], + } + + +@pytest.fixture +def fake_pipeline(): + """A workflow run whose pipeline yields one small image.""" + from PIL import Image + + from dw.pipeline_processors.pipeline import Pipeline + + class FakePipeline: + def __call__(self, *args, **kwargs): + class Output: + images = [Image.new("RGB", (8, 8), "green")] + + return Output() + + def to(self, *args, **kwargs): + return self + + @property + def components(self): + return {} + + def mock_load(self, shared_components): + self.pipeline = FakePipeline() + + with patch.object(Pipeline, "load", mock_load): + with patch("dw.workflow.empty_device_cache"): + yield + + +class TestRunDirectories: + def test_each_run_writes_its_own_directory(self, tmp_path, fake_pipeline): + from dw.workflow import Workflow + + first = Workflow( + _workflow_definition(), str(tmp_path), "/w/workflows/Gyre.json" + ) + first.run({}) + second = Workflow( + _workflow_definition(), str(tmp_path), "/w/workflows/Gyre.json" + ) + second.run({}) + + runs = sorted((tmp_path / "Gyre").iterdir()) + # Even started in the same second with the same spec, which is what a + # quick rerun is: the second run never writes into the first's + # directory + assert len(runs) == 2 + assert all(is_run_id(run.name) for run in runs) + + first_manifest = json.loads((runs[0] / "manifest.json").read_text()) + second_manifest = json.loads((runs[1] / "manifest.json").read_text()) + # The first run wrote its image; the second was an unchanged rerun, + # so the step cache served it - it writes nothing new and reports the + # earlier run's file, by the absolute path that is not its own to + # describe relatively + assert first_manifest["steps"][0]["files"] == ["runs_test-gen0.0-0.0.png"] + assert not first_manifest["steps"][0].get("reused") + assert second_manifest["steps"][0]["reused"] is True + reused = second_manifest["steps"][0]["files"][0] + assert os.path.isabs(reused) and os.path.exists(reused) + assert os.path.dirname(reused) == str(runs[0]) + + def test_a_changed_run_writes_its_own_files(self, tmp_path, fake_pipeline): + from dw.workflow import Workflow + + first = Workflow( + _workflow_definition(), str(tmp_path), "/w/workflows/Gyre.json" + ) + first.run({}) + changed = _workflow_definition() + changed["steps"][0]["pipeline"]["arguments"]["prompt"] = "different" + Workflow(changed, str(tmp_path), "/w/workflows/Gyre.json").run({}) + + runs = sorted((tmp_path / "Gyre").iterdir()) + assert len(runs) == 2 + for run in runs: + assert any(name.suffix == ".png" for name in run.iterdir()) + + def test_the_manifest_describes_the_run(self, tmp_path, fake_pipeline): + from dw.workflow import Workflow + + workflow = Workflow( + _workflow_definition(), str(tmp_path), "/w/workflows/ltx2/Gyre.json" + ) + workflow.run({"prompt": "a cat"}) + + run_dir = next((tmp_path / "ltx2" / "Gyre").iterdir()) + manifest = json.loads((run_dir / "manifest.json").read_text()) + assert manifest["status"] == "completed" + assert manifest["workflow"]["identity"] == "ltx2/Gyre" + assert manifest["workflow"]["id"] == "runs_test" + assert manifest["seed"] == 7 + assert manifest["arguments"] == {"prompt": "a cat"} + assert is_run_id(manifest["run_id"]) + assert manifest["steps"][0]["step"] == "gen0" + # relative to the directory that describes itself + for name in manifest["steps"][0]["files"]: + assert not os.path.isabs(name) + assert (run_dir / name).exists() + + def test_a_failed_run_still_records_what_it_wrote(self, tmp_path, fake_pipeline): + from dw.workflow import Workflow + + definition = _workflow_definition() + definition["steps"].append({"name": "boom", "task": {"command": "no_such"}}) + workflow = Workflow(definition, str(tmp_path), "/w/workflows/Gyre.json") + with pytest.raises(Exception): + workflow.run({}) + + run_dir = next((tmp_path / "Gyre").iterdir()) + manifest = json.loads((run_dir / "manifest.json").read_text()) + assert manifest["status"] == "failed" + assert manifest["steps"][0]["step"] == "gen0" + + def test_the_flat_layout_writes_where_it_always_did( + self, tmp_path, fake_pipeline, monkeypatch + ): + from dw.workflow import Workflow + + monkeypatch.setenv(OUTPUT_LAYOUT_ENV_VAR, FLAT_LAYOUT) + workflow = Workflow( + _workflow_definition(), str(tmp_path), "/w/workflows/ltx2/Gyre.json" + ) + workflow.run({}) + # the pre-run-directory layout: the workflow's position under a + # 'workflows' tree, and no run directory or manifest + written = list((tmp_path / "ltx2").iterdir()) + assert [path.suffix for path in written] == [".png"] From f87102c27d505b3613e3d3df00435c838e709552 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sat, 5 Sep 2026 18:34:40 +0000 Subject: [PATCH 04/34] chore: ignore the workspace asset library, keep tests out of the checkout outputs/ already covers run directories and their manifests - the rule matches at any depth. The asset library is new and was not covered: with the checkout as the workspace, dw.serve creates /assets/ and browser uploads land in it. Anchored at the root so the SPA's own assets/ folders are untouched. test_serve_main was creating that directory in the working tree on every run, because main() with no --workspace resolves the working directory - give it a workspace of its own. Co-Authored-By: Claude Opus 5 (1M context) --- .gitignore | 5 +++++ output/w/20260905-183112-c771d89c/manifest.json | 16 ++++++++++++++++ output/w/20260905-183112-fe03bfa5/manifest.json | 16 ++++++++++++++++ output/w/20260905-183411-3b6e1739/manifest.json | 16 ++++++++++++++++ output/w/20260905-183411-c771d89c/manifest.json | 16 ++++++++++++++++ tests/test_serve_main.py | 5 +++++ 6 files changed, 74 insertions(+) create mode 100644 output/w/20260905-183112-c771d89c/manifest.json create mode 100644 output/w/20260905-183112-fe03bfa5/manifest.json create mode 100644 output/w/20260905-183411-3b6e1739/manifest.json create mode 100644 output/w/20260905-183411-c771d89c/manifest.json diff --git a/.gitignore b/.gitignore index 8eeb3768..44c2405c 100644 --- a/.gitignore +++ b/.gitignore @@ -153,5 +153,10 @@ venv # generated media - regenerable, archived outside the repo local_inputs/ workflows/*/assets/ +# the workspace's asset library: input media, and where browser uploads land +# when the checkout is the workspace (dw/workspace.py). Anchored at the root +# so the SPA's own assets/ folders are not swept up. Generated output is +# covered by the outputs/ rule above, run directories and manifests included +/assets/ /*.mp4 /*.wav diff --git a/output/w/20260905-183112-c771d89c/manifest.json b/output/w/20260905-183112-c771d89c/manifest.json new file mode 100644 index 00000000..919af432 --- /dev/null +++ b/output/w/20260905-183112-c771d89c/manifest.json @@ -0,0 +1,16 @@ +{ + "run_id": "20260905-183112-c771d89c", + "status": "failed", + "started_at": "2026-09-05T18:31:12.269757+00:00", + "finished_at": "2026-09-05T18:31:12.269987+00:00", + "dw_version": "0.4.0-beta.1", + "device": "cuda", + "workflow": { + "id": "w", + "file": "", + "identity": "w" + }, + "seed": 42, + "arguments": {}, + "steps": [] +} \ No newline at end of file diff --git a/output/w/20260905-183112-fe03bfa5/manifest.json b/output/w/20260905-183112-fe03bfa5/manifest.json new file mode 100644 index 00000000..cb9bb904 --- /dev/null +++ b/output/w/20260905-183112-fe03bfa5/manifest.json @@ -0,0 +1,16 @@ +{ + "run_id": "20260905-183112-fe03bfa5", + "status": "failed", + "started_at": "2026-09-05T18:31:12.273450+00:00", + "finished_at": "2026-09-05T18:31:12.273646+00:00", + "dw_version": "0.4.0-beta.1", + "device": "cuda", + "workflow": { + "id": "w", + "file": "", + "identity": "w" + }, + "seed": null, + "arguments": {}, + "steps": [] +} \ No newline at end of file diff --git a/output/w/20260905-183411-3b6e1739/manifest.json b/output/w/20260905-183411-3b6e1739/manifest.json new file mode 100644 index 00000000..b1f6c5b4 --- /dev/null +++ b/output/w/20260905-183411-3b6e1739/manifest.json @@ -0,0 +1,16 @@ +{ + "run_id": "20260905-183411-3b6e1739", + "status": "failed", + "started_at": "2026-09-05T18:34:11.688969+00:00", + "finished_at": "2026-09-05T18:34:11.689167+00:00", + "dw_version": "0.4.0-beta.1", + "device": "cuda", + "workflow": { + "id": "w", + "file": "", + "identity": "w" + }, + "seed": null, + "arguments": {}, + "steps": [] +} \ No newline at end of file diff --git a/output/w/20260905-183411-c771d89c/manifest.json b/output/w/20260905-183411-c771d89c/manifest.json new file mode 100644 index 00000000..c9f2228e --- /dev/null +++ b/output/w/20260905-183411-c771d89c/manifest.json @@ -0,0 +1,16 @@ +{ + "run_id": "20260905-183411-c771d89c", + "status": "failed", + "started_at": "2026-09-05T18:34:11.685177+00:00", + "finished_at": "2026-09-05T18:34:11.685470+00:00", + "dw_version": "0.4.0-beta.1", + "device": "cuda", + "workflow": { + "id": "w", + "file": "", + "identity": "w" + }, + "seed": 42, + "arguments": {}, + "steps": [] +} \ No newline at end of file diff --git a/tests/test_serve_main.py b/tests/test_serve_main.py index e14f1447..6a403d8a 100644 --- a/tests/test_serve_main.py +++ b/tests/test_serve_main.py @@ -26,6 +26,11 @@ def fake_run(app, **kwargs): monkeypatch.delenv("DW_API_TOKEN", raising=False) # main() pins DW_PROMPT_DIR in os.environ; monkeypatch restores it monkeypatch.setenv("DW_PROMPT_DIR", str(tmp_path / "prompts")) + # And a workspace of its own: without one main() resolves the working + # directory, which for the test suite is the checkout - and then creates + # the asset library inside it + monkeypatch.setenv("DW_WORKSPACE", str(tmp_path / "workspace")) + monkeypatch.setenv("DW_WORKSPACE_SOURCE", "flag") (tmp_path / "workflows").mkdir() def run(*argv): From a005f14b507639a54165b85c7366262973576c71 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sat, 5 Sep 2026 18:40:45 +0000 Subject: [PATCH 05/34] feat(ui): show the workspace and asset library on the Server page /api/server has reported both since the workspace and asset work landed; the page listed only workflows, outputs and prompts, so the root they are folders of was invisible in the UI. Co-Authored-By: Claude Opus 5 (1M context) --- ui/src/lib/pages/ServerPage.svelte | 16 ++++++++++++++++ ui/src/lib/pages/ServerPage.test.ts | 2 ++ ui/src/lib/serverinfo.test.ts | 8 +++++++- ui/src/lib/types.ts | 5 +++++ 4 files changed, 30 insertions(+), 1 deletion(-) diff --git a/ui/src/lib/pages/ServerPage.svelte b/ui/src/lib/pages/ServerPage.svelte index 740b5c6e..e6f3590f 100644 --- a/ui/src/lib/pages/ServerPage.svelte +++ b/ui/src/lib/pages/ServerPage.svelte @@ -278,6 +278,14 @@

Directories

+
Workspace
+
+ {#if info.directories.workspace} + {info.directories.workspace} + {:else} + none — directories set individually + {/if} +
Workflows
{info.directories.workflows}
Outputs
@@ -290,6 +298,14 @@ none configured {/if} +
Assets
+
+ {#if info.directories.assets} + {info.directories.assets} + {:else} + none configured + {/if} +
{:else if !error} diff --git a/ui/src/lib/pages/ServerPage.test.ts b/ui/src/lib/pages/ServerPage.test.ts index b362fd1b..f93cd43e 100644 --- a/ui/src/lib/pages/ServerPage.test.ts +++ b/ui/src/lib/pages/ServerPage.test.ts @@ -18,9 +18,11 @@ const base: ServerInfo = { mcp: { mounted: true, path: '/mcp' }, addresses: [{ address: '192.168.1.50', family: 'IPv4', interface: 'enp6s0' }], directories: { + workspace: '/home/don/studio', workflows: '/home/don/workflows', outputs: '/home/don/outputs', prompts: null, + assets: '/home/don/studio/assets', }, } diff --git a/ui/src/lib/serverinfo.test.ts b/ui/src/lib/serverinfo.test.ts index caad9b7a..b47cc30b 100644 --- a/ui/src/lib/serverinfo.test.ts +++ b/ui/src/lib/serverinfo.test.ts @@ -21,7 +21,13 @@ const info = (overrides: Partial = {}): ServerInfo => ({ auth_required: true, mcp: { mounted: true, path: '/mcp' }, addresses: [{ address: '192.168.1.50', family: 'IPv4', interface: 'enp6s0' }], - directories: { workflows: '/w', outputs: '/o', prompts: null }, + directories: { + workspace: '/ws', + workflows: '/w', + outputs: '/o', + prompts: null, + assets: null, + }, ...overrides, }) diff --git a/ui/src/lib/types.ts b/ui/src/lib/types.ts index 74d09d19..33b854b9 100644 --- a/ui/src/lib/types.ts +++ b/ui/src/lib/types.ts @@ -57,9 +57,14 @@ export interface ServerInfo { mcp: { mounted: boolean; path: string } addresses: ServerAddress[] directories: { + /** The workspace the folders below are folders of, when the server + * resolved one; an individually overridden folder still reports its + * own path. */ + workspace: string | null workflows: string outputs: string prompts: string | null + assets: string | null } } From b6809de9f5bb6f9a2c2a2e8dabef77d4fe8b1348 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sat, 5 Sep 2026 18:54:02 +0000 Subject: [PATCH 06/34] feat(workflows): a search path, with saves confined to the writable root The workflow directory was both the library and the place saves landed, so with a checkout as the workspace every save from the editor or an MCP client wrote into the example corpus. Reads now span a search path - the workspace's workflows/ first, then any --examples-dir, read-only - while writes only ever reach the front. A name in an earlier root shadows the same name later, so "open an example, change it, save" writes a copy into the user's library and shadows the example instead of overwriting it. Deleting from a read-only root answers 403 with the source it came from rather than a 404 that reads like a missing file. A job now carries the root it is confined to, so a workflow read from an examples directory runs confined to that directory - its sub-workflow steps and relative assets resolve where it lives, not in a writable root it will never be saved to. GET /api/workflows reports the path as `sources` and tags every entry with origin and writable. The UI hides delete and marks a read-only workflow, and submits by name rather than by a path composed from the workflow directory, which only ever named the writable root. MCP passes both through. The packaged dw/workflows/ stay off the path: they are the pieces a 'builtin:' sub-workflow step names, resolved by the engine where that step is read, not workflows anyone browses. Stage four of docs/proposals/workspaces.md. Co-Authored-By: Claude Opus 5 (1M context) --- CLAUDE.md | 13 ++ docs/MCP.md | 4 +- docs/SERVER.md | 3 + docs/WORKSPACES.md | 34 +++ dw/serve.py | 11 + dw/server/app.py | 204 ++++++++++++++---- dw/server/jobs.py | 26 ++- dw/workflow_sources.py | 161 ++++++++++++++ dw_mcp/authoring.py | 7 +- dw_mcp/catalog.py | 6 +- .../w/20260905-184853-b9666c68/manifest.json | 16 ++ .../w/20260905-184853-c771d89c/manifest.json | 16 ++ .../w/20260905-185329-6569942c/manifest.json | 16 ++ .../w/20260905-185329-c771d89c/manifest.json | 16 ++ tests/test_server.py | 108 ++++++++++ tests/test_workflow_sources.py | 121 +++++++++++ ui/src/lib/api.ts | 8 + ui/src/lib/pages/WorkflowPage.svelte | 44 +++- ui/src/lib/pages/WorkflowsPage.svelte | 8 + 19 files changed, 759 insertions(+), 63 deletions(-) create mode 100644 dw/workflow_sources.py create mode 100644 output/w/20260905-184853-b9666c68/manifest.json create mode 100644 output/w/20260905-184853-c771d89c/manifest.json create mode 100644 output/w/20260905-185329-6569942c/manifest.json create mode 100644 output/w/20260905-185329-c771d89c/manifest.json create mode 100644 tests/test_workflow_sources.py diff --git a/CLAUDE.md b/CLAUDE.md index ec19aec2..58a89316 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -51,6 +51,19 @@ The REPL (`dw/repl.py`) uses a **persistent worker subprocess** (`dw/worker.py`) **Critical**: Uses `multiprocessing.set_start_method("spawn")` for CUDA/MPS compatibility. +### Workflow sources + +`dw/workflow_sources.py` is the server's workflow search path: the writable +directory first (the workspace's `workflows/`), then any `--examples-dir`, each +read-only. Reads (`listing`, `find_workflow`) span every root front-to-back so an +earlier name shadows a later one; `PUT /api/workflows` always resolves through +`writable_source`, so saving something opened from a read-only root writes a copy +rather than overwriting it, and `DELETE` on a read-only root answers 403. A job +carries the root it is confined to (`JobManager.submit(workflow_dir=...)`), so an +examples workflow runs confined to the examples directory rather than to the +writable one. Packaged `dw/workflows/` is off the path - it is what `builtin:` +sub-workflow steps name, resolved in `dw/workflow.py`. + ### Workspaces `dw/workspace.py` resolves the one directory a run's content belongs to - diff --git a/docs/MCP.md b/docs/MCP.md index 0e1d549d..1fd35430 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -189,7 +189,7 @@ workflow is a preference, not a rule; `run_workflow` still takes an | Tool | Arguments | Purpose | | --- | --- | --- | -| `list_workflows()` | — | List stored workflows, each with its description, output kinds, step count, variable names and the stored prompts it references. The first call to make for a request an existing workflow might cover | +| `list_workflows()` | — | List stored workflows, each with its description, output kinds, step count, variable names, the stored prompts it references, and its `origin`/`writable` - a workflow from a read-only examples directory can be read and run but not saved over or deleted. The first call to make for a request an existing workflow might cover | | `get_workflow(name)` | `name` | Get one stored workflow's full JSON definition | | `get_schema()` | — | Get the JSON schema every workflow definition must satisfy | | `list_pipelines()` | — | List every diffusers pipeline class this installation provides | @@ -220,7 +220,7 @@ workflow is a preference, not a rule; `run_workflow` still takes an | Tool | Arguments | Purpose | | --- | --- | --- | | `validate_workflow(workflow=None, name=None)` | exactly one of `workflow` (inline definition) or `name` (a stored workflow, as `list_workflows` reports it) | Check a workflow against the schema and against real pipeline signatures. Free and instant. Validating by name uses the workflow file's own directory as the base directory, so it sees what a run would | -| `save_workflow(name, workflow)` | `name`, `workflow` | Save a workflow to the server, overwriting any existing workflow of that name | +| `save_workflow(name, workflow)` | `name`, `workflow` | Save a workflow into the server's writable workflow directory, overwriting any existing workflow of that name there. A name that currently resolves to a read-only source (an examples directory) is not overwritten - the copy lands in the writable directory and shadows it | | `delete_workflow(name)` | `name` | Permanently delete a stored workflow | ### Prompts diff --git a/docs/SERVER.md b/docs/SERVER.md index 3043d541..1a21632c 100644 --- a/docs/SERVER.md +++ b/docs/SERVER.md @@ -11,6 +11,9 @@ python -m dw.serve --port 8000 --workflow-dir ./workflows --output-dir ./outputs # or point it at a workspace, which supplies all four directories python -m dw.serve --workspace ~/studio + +# your own workflows, with a checkout's examples alongside them read-only +python -m dw.serve --workspace ~/studio --examples-dir ~/src/diffusers-workflow/workflows python -m dw.serve --host 0.0.0.0 --token "some-long-random-string" # reachable off this machine python -m dw.serve --trust-workflows # only if nothing untrusted can reach POST /api/jobs - see Security model ``` diff --git a/docs/WORKSPACES.md b/docs/WORKSPACES.md index 0fe2adad..d9180f55 100644 --- a/docs/WORKSPACES.md +++ b/docs/WORKSPACES.md @@ -92,6 +92,40 @@ media it reads no longer have to sit in the same folder. `--asset-dir` and `assets/uploads/`, coming back as `asset:uploads/`. See [Asset References](WORKFLOW_GUIDE.md#asset-references). +## Where workflows are read from, and written to + +The server reads workflows from a search path and writes them to exactly one +place — the front: + +``` +/workflows/ yours, writable — every save lands here +<--examples-dir> read-only, repeatable +``` + +A name found in an earlier root shadows the same name in a later one, so a +workspace copy of an example is the one that runs. Reads — listing, opening, +downloading, validating, running — span the whole path. Saves and deletes do +not: `PUT` always writes into the writable root, and deleting something from a +read-only root is refused with a 403 that says where it came from. + +That makes "open an example, change it, save" do the obvious thing: the copy +lands in your library and shadows the example from then on, and the example +itself is never touched. It is also what stops an agent's saves landing in a +checkout — point `--workflow-dir` (or `--workspace`) at your own directory and +the repository's workflows at `--examples-dir`: + +```bash +python -m dw.serve --workspace ~/studio --examples-dir ~/src/diffusers-workflow/workflows +``` + +`GET /api/workflows` reports the path as `sources` and tags every entry with +its `origin` and `writable`, which is how the UI knows to hide delete and how +an MCP client can tell what it may change. + +The packaged workflows in `dw/workflows/` are deliberately *not* on the path. +They are the pieces a `builtin:` sub-workflow step names, resolved by the +engine where that step is read — not workflows to browse or run on their own. + ## Runs Each execution writes its own directory under the output folder, named by the diff --git a/dw/serve.py b/dw/serve.py index 8d32b5e9..fb217116 100644 --- a/dw/serve.py +++ b/dw/serve.py @@ -53,6 +53,16 @@ def main(): "a CLI run discovers it - DW_PROMPT_DIR, else ./prompts if it " "exists, else the nearest prompts/ above the workflow directory)", ) + parser.add_argument( + "--examples-dir", + action="append", + default=None, + dest="examples_dirs", + metavar="DIR", + help="A read-only directory of workflows to offer alongside the " + "workspace's own - a checkout's workflows/ tree, say. Repeatable. " + "Saves never go here: they always land in --workflow-dir", + ) parser.add_argument( "--asset-dir", default=None, @@ -210,6 +220,7 @@ def main(): log_level=args.log_level, prompt_dir=prompt_dir, asset_dir=asset_dir, + examples_dirs=args.examples_dirs, workspace=workspace.root, host=args.host, token=token, diff --git a/dw/server/app.py b/dw/server/app.py index 3e921ae9..3cb84a61 100644 --- a/dw/server/app.py +++ b/dw/server/app.py @@ -54,6 +54,15 @@ from ..result import read_embedded_metadata from ..hub_cache import scan_models, delete_model, DownloadManager from ..runs import strip_run_id +from ..workflow_sources import ( + find_workflow, + listing, + resolve_in_source, + source_for_path, + workflow_names, + workflow_sources, + writable_source, +) from .jobs import JobManager, MAX_PERSISTED_EVENTS, TERMINAL_STATES from .netinfo import local_addresses from .updater import DiffusersUpdater @@ -80,27 +89,23 @@ class JobRequest(BaseModel): ) -def workflow_names(workflow_dir): - """Workflow names under workflow_dir, as relative paths without .json.""" - names = [] - if not os.path.isdir(workflow_dir): - return names - for root, _dirs, files in os.walk(workflow_dir): - for file_name in files: - if file_name.endswith(".json"): - relative = os.path.relpath(os.path.join(root, file_name), workflow_dir) - names.append(relative[: -len(".json")]) - return sorted(names) - - # What each workflow produces and takes, for listing cards - cached by mtime _workflow_detail_cache = {} def _prune_detail_cache(cache, directory, names): """Forget files a listing no longer names - a long-lived server that - creates and deletes scratch files would otherwise grow the cache forever.""" - live = {os.path.join(directory, f"{name}.json") for name in names} + creates and deletes scratch files would otherwise grow the cache forever. + + `names` are relative names under `directory`, or - when `directory` is + None, as it is for a listing spanning several roots - the absolute paths + themselves. + """ + live = ( + set(names) + if directory is None + else {os.path.join(directory, f"{name}.json") for name in names} + ) for stale in [path for path in cache if path not in live]: del cache[stale] @@ -121,15 +126,21 @@ def collect_prompt_references(value): return references -def workflow_details(workflow_dir, names): +def workflow_details(sources_by_name): """Per-workflow card metadata: output kinds, step and variable counts, and the variable names themselves - enough for an agent to pick a workflow and know what to pass it without fetching each candidate. The names but not their defaults: across the workflows on disk the defaults - are an order of magnitude more payload, on a listing the UI reloads.""" + are an order of magnitude more payload, on a listing the UI reloads. + + Takes the name -> source mapping the search path produced, so each + entry also says where it came from and whether it can be written to - + what a client needs to decide between offering save and offering + save-a-copy. + """ details = {} - for name in names: - path = os.path.join(workflow_dir, f"{name}.json") + for name, source in sources_by_name.items(): + path = os.path.join(source.root, f"{name}.json") try: mtime = os.path.getmtime(path) except OSError: @@ -168,8 +179,21 @@ def workflow_details(workflow_dir, names): "prompt_refs": [], } _workflow_detail_cache[path] = (mtime, detail) - details[name] = detail - _prune_detail_cache(_workflow_detail_cache, workflow_dir, names) + # Cached by content, not by placement: the same file listed from a + # different source keeps its parsed detail and gets fresh origins + details[name] = { + **detail, + "origin": source.origin, + "writable": source.writable, + } + _prune_detail_cache( + _workflow_detail_cache, + None, + { + os.path.join(source.root, f"{name}.json") + for name, source in sources_by_name.items() + }, + ) return details @@ -190,7 +214,51 @@ def resolve_workflow_name(workflow_dir, name, allow_create=False): raise HTTPException(status_code=404, detail=f"Unknown workflow: {e}") -def resolve_workflow_reference(workflow_dir, workflow_path): +def _confinement_root(sources, path): + """The source root a resolved workflow path is confined to. + + A workflow read from an examples directory is confined to that + directory - its sub-workflow steps and relative assets resolve there, + not in the writable root it will never be saved to. A path in no source + keeps the writable root's confinement, which is what refuses it. + """ + source = source_for_path(sources, path) if path else None + return source.root if source else writable_source(sources).root + + +def resolve_readable_workflow(sources, name): + """The path a name has anywhere on the search path, and its source. + + Reads span every root - the workspace's own workflows, any examples + directory, and the packaged builtins - front to back, so a workspace + copy shadows the example it came from. + """ + path, source = find_workflow(sources, name) + if path is None: + raise HTTPException(status_code=404, detail=f"Unknown workflow: {name}") + return path, source + + +def resolve_writable_workflow(sources, name): + """Where a save goes: always the writable source, whatever the name + currently resolves to. + + Saving a workflow opened from an example is not an overwrite of that + example - it is a copy into the user's own library, which is what makes + the read-only roots safe to browse and edit from. + """ + source = writable_source(sources) + if source is None: + raise HTTPException( + status_code=409, detail="This server has no writable workflow directory" + ) + path = resolve_in_source(source, name, allow_create=True) + if path is None: + raise HTTPException(status_code=404, detail=f"Unknown workflow: {name}") + return path, source + + +def resolve_workflow_reference(workflow_dir, workflow_path, sources=None): """A submitted workflow_path, resolved to a file on disk, confined to workflow_dir - the same confinement the /api/workflows CRUD routes already enforce via resolve_workflow_name. @@ -208,6 +276,21 @@ def resolve_workflow_reference(workflow_dir, workflow_path): """ if workflow_path is None: return workflow_path + # Every root on the search path, when the caller has one: a run of an + # example is a read, and reads are not confined to the writable root + if sources is not None: + path, _source = find_workflow(sources, workflow_path) + if path is not None: + return path + for source in sources: + candidate = os.path.abspath(workflow_path) + if source.contains(candidate) and os.path.isfile(candidate): + return candidate + raise HTTPException( + status_code=400, + detail=f"workflow_path must name a workflow the server can reach: " + f"{workflow_path}", + ) try: return resolve_workflow_name(workflow_dir, workflow_path) except HTTPException: @@ -381,6 +464,7 @@ def create_app( diffusers_updater=None, prompt_dir="./prompts", asset_dir=None, + examples_dirs=None, workspace=None, host="127.0.0.1", token=None, @@ -441,6 +525,10 @@ async def lifespan(app): ) app.state.job_manager = manager app.state.workflow_dir = workflow_dir + # The search path: the writable directory first, then read-only roots - + # any --examples-dir, then the packaged builtins. Reads span all of it, + # saves only ever reach the front + app.state.workflow_sources = workflow_sources(workflow_dir, examples_dirs) app.state.prompt_dir = prompt_dir # Where uploads land and 'asset:' references resolve. None when the # caller configured no asset library: uploads then fall back to the @@ -559,13 +647,24 @@ async def require_bearer_token(request: Request, call_next): @app.post("/api/jobs", status_code=201) def submit_job(request: JobRequest): try: + resolved = resolve_workflow_reference( + app.state.workflow_dir, + request.workflow_path, + app.state.workflow_sources, + ) job = manager.submit( - workflow_path=resolve_workflow_reference( - app.state.workflow_dir, request.workflow_path - ), + workflow_path=resolved, workflow=request.workflow, arguments=request.arguments, base_dir=request.base_dir, + # The root this run is confined to: the source the workflow + # came from, so an example runs where it lives while an + # inline definition stays held to the writable root + workflow_dir=( + _confinement_root(app.state.workflow_sources, resolved) + if resolved + else None + ), ) except HTTPException: raise @@ -766,12 +865,17 @@ def validate_workflow(request: JobRequest): if request.workflow_path is not None: # Built from the file so relative paths inside it resolve # against its own directory, exactly as a run would + resolved = resolve_workflow_reference( + app.state.workflow_dir, + request.workflow_path, + app.state.workflow_sources, + ) candidate = workflow_from_file( - resolve_workflow_reference( - app.state.workflow_dir, request.workflow_path - ), + resolved, manager.output_dir, - app.state.workflow_dir, + # Confined to the source it came from, not to the + # writable root - an example is read where it lives + _confinement_root(app.state.workflow_sources, resolved), ) definition = candidate.workflow_definition else: @@ -810,21 +914,30 @@ def validate_workflow(request: JobRequest): @app.get("/api/workflows") def list_workflows(): - names = workflow_names(app.state.workflow_dir) + """Every workflow the search path offers, each detail saying which + source it came from and whether it can be written to. 'workflow_dir' + stays the writable one - what a save targets.""" + found = listing(app.state.workflow_sources) return { "workflow_dir": app.state.workflow_dir, - "workflows": names, - "details": workflow_details(app.state.workflow_dir, names), + "sources": [source.to_dict() for source in app.state.workflow_sources], + "workflows": list(found), + "details": workflow_details(found), } @app.put("/api/workflows/{name:path}") def save_workflow(name: str, request: JobRequest): - """Write a workflow into the workflow directory. The definition must - be schema-valid - the editor validates before saving, and a save that - silently wrote a broken file would betray both.""" + """Write a workflow into the writable workflow directory. The + definition must be schema-valid - the editor validates before saving, + and a save that silently wrote a broken file would betray both. + + A name that currently resolves to a read-only source (an example, a + builtin) is not overwritten: the copy lands in the writable source + and shadows it from then on. + """ if request.workflow is None: raise HTTPException(status_code=400, detail="Provide an inline workflow") - path = resolve_workflow_name(app.state.workflow_dir, name, allow_create=True) + path, _source = resolve_writable_workflow(app.state.workflow_sources, name) candidate = Workflow( copy.deepcopy(request.workflow), manager.output_dir, @@ -848,8 +961,19 @@ def save_workflow(name: str, request: JobRequest): @app.delete("/api/workflows/{name:path}") def delete_workflow(name: str): - """Remove a workflow file from the workflow directory.""" - path = resolve_workflow_name(app.state.workflow_dir, name) + """Remove a workflow file from the writable workflow directory. + + A read-only source is refused rather than silently ignored: an + example or a builtin is not the caller's to delete, and saying so + is more useful than a 404 that reads like the file is missing. + """ + path, source = resolve_readable_workflow(app.state.workflow_sources, name) + if not source.writable: + raise HTTPException( + status_code=403, + detail=f"'{name}' comes from the read-only {source.origin} " + f"directory {source.root} and cannot be deleted", + ) os.remove(path) logger.info(f"Deleted workflow {name} ({path})") return {"name": name, "deleted": True} @@ -858,14 +982,14 @@ def delete_workflow(name: str): @query_token_ok def download_workflow(name: str): """Serve a workflow definition as a forced download.""" - path = resolve_workflow_name(app.state.workflow_dir, name) + path, _source = resolve_readable_workflow(app.state.workflow_sources, name) return FileResponse( path, filename=os.path.basename(path), media_type="application/json" ) @app.get("/api/workflows/{name:path}") def get_workflow(name: str): - path = resolve_workflow_name(app.state.workflow_dir, name) + path, _source = resolve_readable_workflow(app.state.workflow_sources, name) try: with open(path, "r") as file: return JSONResponse(json.load(file)) diff --git a/dw/server/jobs.py b/dw/server/jobs.py index 0dccf501..860b4228 100644 --- a/dw/server/jobs.py +++ b/dw/server/jobs.py @@ -343,25 +343,39 @@ def __init__( # ------------------------------------------------------------- submission - def submit(self, workflow_path=None, workflow=None, arguments=None, base_dir=None): + def submit( + self, + workflow_path=None, + workflow=None, + arguments=None, + base_dir=None, + workflow_dir=None, + ): """Validate a job request and queue it. Raises ValueError on a bad - request so the HTTP layer can answer 400 before anything runs.""" + request so the HTTP layer can answer 400 before anything runs. + + `workflow_dir` overrides this job's confinement root for a workflow + that lives outside the writable directory - an example or a builtin, + which the caller has already resolved against the search path. The + worker re-validates against whatever this job records, so the + override travels with the job rather than widening the manager. + """ arguments = arguments or {} if (workflow_path is None) == (workflow is None): raise ValueError("Provide exactly one of workflow_path or workflow") + confinement = workflow_dir or self.workflow_dir + if workflow_path is not None: # Loads and schema-validates now - a bad path or file fails the # request, not the queue - loaded = workflow_from_file( - workflow_path, self.output_dir, self.workflow_dir - ) + loaded = workflow_from_file(workflow_path, self.output_dir, confinement) loaded.validate() spec = { "workflow_path": workflow_path, "workflow_name": loaded.name, "arguments": arguments, - "workflow_dir": self.workflow_dir, + "workflow_dir": confinement, } else: # workflow_from_definition validates base_dir - it is HTTP-supplied diff --git a/dw/workflow_sources.py b/dw/workflow_sources.py new file mode 100644 index 00000000..56134e8e --- /dev/null +++ b/dw/workflow_sources.py @@ -0,0 +1,161 @@ +"""Where workflows are read from, and the one place they are written to. + +A workflow directory used to be a single directory that was both the library +and the place saves landed. With the repository's own workflows/ as that +directory - the default when a checkout is the workspace - every save from +the editor or an MCP client wrote into the example corpus. + +A search path separates the two. Reads resolve front to back; writes only +ever go to the front: + + /workflows/ the user's own, writable + read-only, --examples-dir + +A name found in an earlier source shadows the same name in a later one, so a +workspace copy of an example is the one that runs. Saving over a read-only +workflow is not an error and not an overwrite: it writes a copy into the +writable source, which is what "open an example, change it, save" should do. +""" + +import logging +import os + +from .security import SecurityError, validate_path + +logger = logging.getLogger("dw") + +# What a source is, for a client deciding whether to offer save or delete +WORKSPACE_ORIGIN = "workspace" +EXAMPLES_ORIGIN = "examples" +BUILTIN_ORIGIN = "builtin" + + +def builtin_root(): + """The packaged workflows that ship inside dw/ - what 'builtin:' names.""" + return os.path.join(os.path.dirname(os.path.abspath(__file__)), "workflows") + + +class WorkflowSource: + """One root on the search path.""" + + def __init__(self, root, origin, writable): + self.root = os.path.abspath(os.path.expanduser(str(root))) + self.origin = origin + self.writable = writable + + def contains(self, path): + """Whether a path resolves inside this root - the containment check + the security layer already implements, asked as a question rather + than raised as an error.""" + try: + validate_path(path, self.root) + return True + except SecurityError: + return False + + def names(self): + """Workflow names under this root, as relative paths without .json.""" + return workflow_names(self.root) + + def to_dict(self): + return {"root": self.root, "origin": self.origin, "writable": self.writable} + + def __repr__(self): + return f"WorkflowSource({self.root!r}, {self.origin}, writable={self.writable})" + + +def workflow_names(root): + """Workflow names under a root, as relative paths without .json.""" + names = [] + if not os.path.isdir(root): + return names + for directory, _dirs, files in os.walk(root): + for file_name in files: + if file_name.endswith(".json"): + relative = os.path.relpath(os.path.join(directory, file_name), root) + names.append(relative[: -len(".json")].replace(os.sep, "/")) + return sorted(names) + + +def workflow_sources(workflow_dir, examples_dirs=None, include_builtin=False): + """The search path: the writable directory first, then read-only roots. + + A read-only root that is the writable one - a checkout whose workflows/ + is both the workspace library and the examples - appears once, writable, + rather than twice with two different answers about whether it can be + saved to. + + The packaged workflows are off the path by default. They are the pieces + a 'builtin:' sub-workflow step names, resolved by the engine where that + step is read (dw/workflow.py) - not workflows anyone browses or runs on + their own, and listing them would put a handful of fragments in front of + every user who never asked for them. + """ + sources = [WorkflowSource(workflow_dir, WORKSPACE_ORIGIN, True)] + candidates = [(directory, EXAMPLES_ORIGIN) for directory in examples_dirs or []] + if include_builtin: + candidates.append((builtin_root(), BUILTIN_ORIGIN)) + + seen = {sources[0].root} + for root, origin in candidates: + source = WorkflowSource(root, origin, False) + if source.root in seen: + continue + seen.add(source.root) + sources.append(source) + return sources + + +def writable_source(sources): + """The source saves go to: the front of the path.""" + for source in sources: + if source.writable: + return source + return None + + +def source_for_path(sources, path): + """Which source a resolved path belongs to, or None if it is outside + every root - which is what makes a path a workflow rather than an + arbitrary file.""" + for source in sources: + if source.contains(path): + return source + return None + + +def resolve_in_source(source, name, allow_create=False): + """The on-disk path a name has in one source, or None when the name does + not resolve inside it. Containment is the security layer's, so a name + that tries to traverse simply does not resolve.""" + if not name.endswith(".json"): + name = f"{name}.json" + try: + return validate_path( + os.path.join(source.root, name), source.root, allow_create=allow_create + ) + except SecurityError: + return None + + +def find_workflow(sources, name): + """The first source that has this name, as (path, source). + + Front to back, so a workspace copy shadows the example it was copied + from. (None, None) when no source has it. + """ + for source in sources: + path = resolve_in_source(source, name) + if path and os.path.isfile(path): + return path, source + return None, None + + +def listing(sources): + """Every name the search path offers, each with the source it comes + from - a name in an earlier source shadowing the same name later.""" + found = {} + for source in sources: + for name in source.names(): + found.setdefault(name, source) + return dict(sorted(found.items())) diff --git a/dw_mcp/authoring.py b/dw_mcp/authoring.py index b131bb43..aff2311b 100644 --- a/dw_mcp/authoring.py +++ b/dw_mcp/authoring.py @@ -27,8 +27,11 @@ def validate_workflow(client, workflow=None, name=None): def save_workflow(client, name, workflow): - """Write a workflow into the server's workflow directory, overwriting any - file already under that name. The server validates before writing.""" + """Write a workflow into the server's writable workflow directory, + overwriting any file already under that name there. A name that resolves + to one of the server's read-only sources (an examples directory) is not + overwritten: the copy lands in the writable directory and shadows it from + then on. The server validates before writing.""" return client.put_json(api_path("api", "workflows", name), {"workflow": workflow}) diff --git a/dw_mcp/catalog.py b/dw_mcp/catalog.py index d0fff870..08bc5303 100644 --- a/dw_mcp/catalog.py +++ b/dw_mcp/catalog.py @@ -7,8 +7,10 @@ def list_workflows(client): - """Workflow names in the server's workflow directory, with details - - description, output kinds and variable names per workflow.""" + """Workflow names the server can reach, with details - description, + output kinds and variable names per workflow, plus which source each came + from and whether it can be written to (a read-only examples directory + can be read and run, but not saved over or deleted).""" return client.get_json("/api/workflows") diff --git a/output/w/20260905-184853-b9666c68/manifest.json b/output/w/20260905-184853-b9666c68/manifest.json new file mode 100644 index 00000000..86034f07 --- /dev/null +++ b/output/w/20260905-184853-b9666c68/manifest.json @@ -0,0 +1,16 @@ +{ + "run_id": "20260905-184853-b9666c68", + "status": "failed", + "started_at": "2026-09-05T18:48:53.257365+00:00", + "finished_at": "2026-09-05T18:48:53.257560+00:00", + "dw_version": "0.4.0-beta.1", + "device": "cuda", + "workflow": { + "id": "w", + "file": "", + "identity": "w" + }, + "seed": null, + "arguments": {}, + "steps": [] +} \ No newline at end of file diff --git a/output/w/20260905-184853-c771d89c/manifest.json b/output/w/20260905-184853-c771d89c/manifest.json new file mode 100644 index 00000000..6f4db329 --- /dev/null +++ b/output/w/20260905-184853-c771d89c/manifest.json @@ -0,0 +1,16 @@ +{ + "run_id": "20260905-184853-c771d89c", + "status": "failed", + "started_at": "2026-09-05T18:48:53.253656+00:00", + "finished_at": "2026-09-05T18:48:53.253871+00:00", + "dw_version": "0.4.0-beta.1", + "device": "cuda", + "workflow": { + "id": "w", + "file": "", + "identity": "w" + }, + "seed": 42, + "arguments": {}, + "steps": [] +} \ No newline at end of file diff --git a/output/w/20260905-185329-6569942c/manifest.json b/output/w/20260905-185329-6569942c/manifest.json new file mode 100644 index 00000000..3d97bb0c --- /dev/null +++ b/output/w/20260905-185329-6569942c/manifest.json @@ -0,0 +1,16 @@ +{ + "run_id": "20260905-185329-6569942c", + "status": "failed", + "started_at": "2026-09-05T18:53:29.041945+00:00", + "finished_at": "2026-09-05T18:53:29.042143+00:00", + "dw_version": "0.4.0-beta.1", + "device": "cuda", + "workflow": { + "id": "w", + "file": "", + "identity": "w" + }, + "seed": null, + "arguments": {}, + "steps": [] +} \ No newline at end of file diff --git a/output/w/20260905-185329-c771d89c/manifest.json b/output/w/20260905-185329-c771d89c/manifest.json new file mode 100644 index 00000000..5f3aa651 --- /dev/null +++ b/output/w/20260905-185329-c771d89c/manifest.json @@ -0,0 +1,16 @@ +{ + "run_id": "20260905-185329-c771d89c", + "status": "failed", + "started_at": "2026-09-05T18:53:29.038211+00:00", + "finished_at": "2026-09-05T18:53:29.038451+00:00", + "dw_version": "0.4.0-beta.1", + "device": "cuda", + "workflow": { + "id": "w", + "file": "", + "identity": "w" + }, + "seed": 42, + "arguments": {}, + "steps": [] +} \ No newline at end of file diff --git a/tests/test_server.py b/tests/test_server.py index 73291ef8..58a54ca1 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -1176,10 +1176,118 @@ def test_workflow_listing_carries_details(server): "variable_names": ["prompt"], "description": "Renders a small test image.", "prompt_refs": [], + # where it came from, and whether a client should offer save and + # delete for it or only save-a-copy + "origin": "workspace", + "writable": True, } assert listing["details"]["Basic"]["kinds"] == [] +@pytest.fixture +def examples_server(tmp_path): + """A server whose workspace library is empty and whose examples come + from a second, read-only directory.""" + workflows = tmp_path / "workflows" + workflows.mkdir() + examples = tmp_path / "examples" + (examples / "ltx2").mkdir(parents=True) + (examples / "ltx2" / "Gyre.json").write_text(json.dumps(valid_workflow("gyre"))) + + def make(script): + manager = JobManager( + str(tmp_path / "outputs"), + worker_manager=ScriptedWorkerManager(script), + history_path=str(tmp_path / "jobs.sqlite"), + workflow_dir=str(workflows), + ) + app = create_app( + workflow_dir=str(workflows), + output_dir=str(tmp_path / "outputs"), + job_manager=manager, + examples_dirs=[str(examples)], + ) + return TestClient(app, base_url="http://localhost") + + return make + + +def test_examples_are_listed_read_only(examples_server): + with examples_server(success_script) as client: + listing = client.get("/api/workflows").json() + assert listing["workflows"] == ["ltx2/Gyre"] + assert listing["details"]["ltx2/Gyre"]["origin"] == "examples" + assert listing["details"]["ltx2/Gyre"]["writable"] is False + # the writable root is still what a save targets, and is named first + assert listing["sources"][0]["writable"] is True + assert listing["sources"][1]["origin"] == "examples" + + # and it reads like any other workflow + assert client.get("/api/workflows/ltx2/Gyre").status_code == 200 + + +def test_saving_an_example_copies_it_into_the_writable_library( + examples_server, tmp_path +): + """Open an example, change it, save: the copy lands in the user's own + library and shadows the example from then on - the example itself is + untouched.""" + with examples_server(success_script) as client: + original = json.loads( + (tmp_path / "examples" / "ltx2" / "Gyre.json").read_text() + ) + edited = json.loads(json.dumps(original)) + edited["description"] = "my version" + + response = client.put("/api/workflows/ltx2/Gyre", json={"workflow": edited}) + assert response.status_code == 200 + assert response.json()["path"] == str( + tmp_path / "workflows" / "ltx2" / "Gyre.json" + ) + # the example on disk did not move or change + assert ( + json.loads((tmp_path / "examples" / "ltx2" / "Gyre.json").read_text()) + == original + ) + + listing = client.get("/api/workflows").json() + assert listing["details"]["ltx2/Gyre"]["origin"] == "workspace" + assert client.get("/api/workflows/ltx2/Gyre").json()["description"] == ( + "my version" + ) + + +def test_an_example_cannot_be_deleted(examples_server, tmp_path): + with examples_server(success_script) as client: + response = client.delete("/api/workflows/ltx2/Gyre") + assert response.status_code == 403 + assert "read-only" in response.json()["detail"] + assert (tmp_path / "examples" / "ltx2" / "Gyre.json").exists() + + +def test_an_example_can_be_validated_and_run(examples_server): + with examples_server(success_script) as client: + assert client.post("/api/validate", json={"workflow_path": "ltx2/Gyre"}).json()[ + "valid" + ] + + response = client.post("/api/jobs", json={"workflow_path": "ltx2/Gyre"}) + assert response.status_code == 201 + detail = wait_for_status(client, response.json()["id"], {"succeeded", "failed"}) + assert detail["status"] == "succeeded" + + +def test_a_workflow_outside_every_source_is_refused(examples_server, tmp_path): + outside = tmp_path / "outside.json" + outside.write_text(json.dumps(valid_workflow("outside"))) + with examples_server(success_script) as client: + assert ( + client.post("/api/jobs", json={"workflow_path": str(outside)}).status_code + == 400 + ) + assert client.get("/api/workflows/outside").status_code == 404 + + def test_workflow_details_name_their_prompt_references(server): """The listing says which stored prompts a workflow leans on - deleting a prompt warns from exactly this.""" diff --git a/tests/test_workflow_sources.py b/tests/test_workflow_sources.py new file mode 100644 index 00000000..07bcc927 --- /dev/null +++ b/tests/test_workflow_sources.py @@ -0,0 +1,121 @@ +"""The workflow search path: reads span every root, writes reach only the +front, and a read-only source cannot be written to or deleted from.""" + +import json +import os + +import pytest + +from dw.workflow_sources import ( + BUILTIN_ORIGIN, + EXAMPLES_ORIGIN, + WORKSPACE_ORIGIN, + builtin_root, + find_workflow, + listing, + resolve_in_source, + source_for_path, + workflow_names, + workflow_sources, + writable_source, +) + + +@pytest.fixture +def roots(tmp_path): + """A writable workspace library and a read-only examples tree, with one + name present in both.""" + workspace = tmp_path / "studio" / "workflows" + (workspace / "mine").mkdir(parents=True) + (workspace / "Shared.json").write_text(json.dumps({"id": "mine-shared"})) + (workspace / "mine" / "Solo.json").write_text(json.dumps({"id": "solo"})) + + examples = tmp_path / "repo" / "workflows" + (examples / "ltx2").mkdir(parents=True) + (examples / "Shared.json").write_text(json.dumps({"id": "example-shared"})) + (examples / "ltx2" / "Gyre.json").write_text(json.dumps({"id": "gyre"})) + return workspace, examples + + +class TestSources: + def test_the_writable_root_comes_first(self, roots): + workspace, examples = roots + sources = workflow_sources(str(workspace), [str(examples)]) + assert [s.origin for s in sources] == [WORKSPACE_ORIGIN, EXAMPLES_ORIGIN] + assert [s.writable for s in sources] == [True, False] + assert writable_source(sources).root == str(workspace) + + def test_a_repeated_root_is_writable_once(self, roots): + # The checkout-as-workspace case: the same directory named as both + # the library and the examples must not answer two ways + workspace, _examples = roots + sources = workflow_sources(str(workspace), [str(workspace)]) + assert len(sources) == 1 + assert sources[0].writable + + def test_the_packaged_workflows_are_off_the_path_by_default(self, roots): + workspace, _examples = roots + assert builtin_root() not in [s.root for s in workflow_sources(str(workspace))] + with_builtins = workflow_sources(str(workspace), include_builtin=True) + assert [s.origin for s in with_builtins][-1] == BUILTIN_ORIGIN + + def test_a_missing_root_is_simply_empty(self, tmp_path): + sources = workflow_sources(str(tmp_path / "nothing-here")) + assert workflow_names(sources[0].root) == [] + + +class TestResolution: + def test_reads_span_every_root(self, roots): + workspace, examples = roots + sources = workflow_sources(str(workspace), [str(examples)]) + path, source = find_workflow(sources, "ltx2/Gyre") + assert source.origin == EXAMPLES_ORIGIN + assert path == str(examples / "ltx2" / "Gyre.json") + + def test_the_front_of_the_path_shadows_the_rest(self, roots): + workspace, examples = roots + sources = workflow_sources(str(workspace), [str(examples)]) + path, source = find_workflow(sources, "Shared") + assert source.origin == WORKSPACE_ORIGIN + assert json.loads(open(path).read())["id"] == "mine-shared" + + def test_a_listing_names_each_workflow_once(self, roots): + workspace, examples = roots + found = listing(workflow_sources(str(workspace), [str(examples)])) + assert sorted(found) == ["Shared", "ltx2/Gyre", "mine/Solo"] + assert found["Shared"].origin == WORKSPACE_ORIGIN + assert found["ltx2/Gyre"].origin == EXAMPLES_ORIGIN + + def test_an_unknown_name_resolves_nowhere(self, roots): + workspace, examples = roots + assert ( + find_workflow(workflow_sources(str(workspace), [str(examples)]), "Nope")[0] + is None + ) + + @pytest.mark.parametrize("name", ["../outside", "/etc/passwd", "a/../../escape"]) + def test_a_name_cannot_traverse_out_of_a_source(self, roots, name): + workspace, examples = roots + sources = workflow_sources(str(workspace), [str(examples)]) + assert find_workflow(sources, name) == (None, None) + assert resolve_in_source(sources[0], name, allow_create=True) is None + + def test_a_path_knows_which_source_it_belongs_to(self, roots, tmp_path): + workspace, examples = roots + sources = workflow_sources(str(workspace), [str(examples)]) + assert source_for_path( + sources, str(examples / "ltx2" / "Gyre.json") + ).origin == (EXAMPLES_ORIGIN) + assert source_for_path(sources, str(tmp_path / "elsewhere.json")) is None + + +class TestNames: + def test_names_are_relative_and_slash_separated(self, roots): + _workspace, examples = roots + assert workflow_names(str(examples)) == ["Shared", "ltx2/Gyre"] + + def test_non_json_files_are_not_workflows(self, roots): + _workspace, examples = roots + (examples / "notes.txt").write_text("hello") + assert "notes" not in workflow_names(str(examples)) + assert os.path.isfile(examples / "notes.txt") diff --git a/ui/src/lib/api.ts b/ui/src/lib/api.ts index eea97e0a..5299138e 100644 --- a/ui/src/lib/api.ts +++ b/ui/src/lib/api.ts @@ -99,7 +99,11 @@ async function downloadResponse( export const api = { listWorkflows: () => request<{ + /** The writable directory - where a save lands, whatever source a + * workflow was read from. */ workflow_dir: string + /** The search path, writable root first. */ + sources?: { root: string; origin: string; writable: boolean }[] workflows: string[] details: Record< string, @@ -109,6 +113,10 @@ export const api = { variables: number description: string prompt_refs?: string[] + /** Which source it came from: 'workspace', 'examples', 'builtin'. */ + origin?: string + /** False for a read-only source: offer save-a-copy, not delete. */ + writable?: boolean } > }>('/api/workflows'), diff --git a/ui/src/lib/pages/WorkflowPage.svelte b/ui/src/lib/pages/WorkflowPage.svelte index dbd87f65..4aa16444 100644 --- a/ui/src/lib/pages/WorkflowPage.svelte +++ b/ui/src/lib/pages/WorkflowPage.svelte @@ -14,6 +14,10 @@ let workflow = $state(null) let workflowDir = $state('') + /** Where this workflow was read from, and whether it is the user's to + * change - an examples or builtin source is read-only. */ + let origin = $state('') + let writable = $state(true) let overrides = $state>({}) let error = $state('') let submitting = $state(false) @@ -22,7 +26,11 @@ $effect(() => { overrides = {} workflow = null - api.listWorkflows().then((r) => (workflowDir = r.workflow_dir)) + api.listWorkflows().then((r) => { + workflowDir = r.workflow_dir + origin = r.details[name]?.origin ?? '' + writable = r.details[name]?.writable ?? true + }) loadPromptLibrary() api .getWorkflow(name) @@ -59,7 +67,9 @@ if (value !== '') args[key] = value } const job = await api.submitJob({ - workflow_path: `${workflowDir}/${name}.json`, + // By name, not by composed path: the server resolves a name across + // every source it can read, so an example runs where it lives + workflow_path: name, arguments: args, }) go('jobs', job.id) @@ -100,19 +110,26 @@ New from + {#if !writable} + + read-only{origin ? ` (${origin})` : ''} + + {/if} - + {#if writable} + + {/if} + {/if} + + {/each} + + {/if} + {#if workspace.root} +
+ event.key === 'Enter' && addWorkspace()} + /> + +
+ {:else} +

+ This server was started with individual directory overrides, so it has + one workspace and cannot create others. +

+ {/if} + {#if workspaceError}

{workspaceError}

{/if} + {:else if !error}

loading server details…

{/if}