diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 35938933..21436dbd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,9 @@ jobs: pip install -r requirements.txt -r requirements-test.txt pip install git+https://github.com/huggingface/diffusers - name: Format check - run: black --check dw dw_mcp tests + run: ruff format --check dw dw_mcp tests + - name: Lint + run: ruff check dw dw_mcp tests - name: Tests run: pytest -q diff --git a/docs/SECURITY_QUICKREF.md b/docs/SECURITY_QUICKREF.md index ccaeb634..79bff9a1 100644 --- a/docs/SECURITY_QUICKREF.md +++ b/docs/SECURITY_QUICKREF.md @@ -13,9 +13,16 @@ and [REMOTE.md](REMOTE.md). ```python from dw.security import ( - validate_path, validate_workflow_path, validate_output_path, - validate_url, validate_variable_name, validate_string_input, - sanitize_command_args, SecurityError, PathTraversalError, InvalidInputError + validate_path, + validate_workflow_path, + validate_output_path, + validate_url, + validate_variable_name, + validate_string_input, + sanitize_command_args, + SecurityError, + PathTraversalError, + InvalidInputError, ) ``` @@ -28,11 +35,11 @@ anything a workflow names: ```python from dw.locations import ( - validate_media_path, # confined to the workflow dir / assets / outputs - validate_media_glob, # the same, on a pattern's fixed prefix - contained_matches, # each match re-checked on its real path - validate_media_url, # no loopback / link-local / private host - validate_model_name, # a Hub repo id, or a contained path + validate_media_path, # confined to the workflow dir / assets / outputs + validate_media_glob, # the same, on a pattern's fixed prefix + contained_matches, # each match re-checked on its real path + validate_media_url, # no loopback / link-local / private host + validate_model_name, # a Hub repo id, or a contained path ) ``` @@ -49,13 +56,14 @@ workflow_path = validate_workflow_path("workflow.json") output_path = validate_output_path(user_path, base_output_dir) # User input -var_name = validate_variable_name("prompt") # OK -var_name = validate_variable_name("bad;name") # raises InvalidInputError +var_name = validate_variable_name("prompt") # OK +var_name = validate_variable_name("bad;name") # raises InvalidInputError value = validate_string_input(user_input, max_length=1000) -url = validate_url(user_url) # http/https only +url = validate_url(user_url) # http/https only # Subprocess (dw/ doesn't currently shell out anywhere - pattern for if/when it does) import subprocess + cmd = sanitize_command_args(["python", "-m", "dw.run", validated_path]) subprocess.Popen(cmd, shell=False) ``` diff --git a/docs/superpowers/plans/2026-03-26-ecosystem-utilities.md b/docs/superpowers/plans/2026-03-26-ecosystem-utilities.md index d2da6433..53ee7f0c 100644 --- a/docs/superpowers/plans/2026-03-26-ecosystem-utilities.md +++ b/docs/superpowers/plans/2026-03-26-ecosystem-utilities.md @@ -229,7 +229,9 @@ class TestSegmentImage: torch.zeros(1, 1, 480, 640) ] # Set the mask area to 1 - mock_sam_proc_instance.post_process_masks.return_value[0][0, 0, 100:300, 100:300] = 1.0 + mock_sam_proc_instance.post_process_masks.return_value[0][ + 0, 0, 100:300, 100:300 + ] = 1.0 image = _make_test_image() result = segment_image(image, "dog") @@ -787,9 +789,7 @@ def interpolate_frames(video, device="cpu", **kwargs): ) if len(video) < 2: - raise ValueError( - f"Need at least 2 frames to interpolate, got {len(video)}" - ) + raise ValueError(f"Need at least 2 frames to interpolate, got {len(video)}") logger.info( f"Interpolating {len(video)} frames with {multiplier}x multiplier on {device}" @@ -836,7 +836,9 @@ def _load_rife_model(device, model_name=None): ) if model_name is None: - model_name = "skytnt/anime-seg" # Placeholder — replace with actual RIFE HF repo + model_name = ( + "skytnt/anime-seg" # Placeholder — replace with actual RIFE HF repo + ) logger.info(f"Loading RIFE model from {model_name} to {device}") @@ -1026,14 +1028,20 @@ class TestMetadataEmbedding: def test_png_metadata_embedded(self): """When embed_metadata is true, PNG should contain parameters text chunk.""" with tempfile.TemporaryDirectory() as temp_dir: - result_def = {"content_type": "image/png", "save": True, "embed_metadata": True} + result_def = { + "content_type": "image/png", + "save": True, + "embed_metadata": True, + } result = Result(result_def) - result.set_metadata({ - "workflow_id": "test_workflow", - "step_name": "generate", - "model_name": "test/model", - "arguments": {"prompt": "a cat", "num_inference_steps": 25}, - }) + result.set_metadata( + { + "workflow_id": "test_workflow", + "step_name": "generate", + "model_name": "test/model", + "arguments": {"prompt": "a cat", "num_inference_steps": 25}, + } + ) # Add a real PIL image img = Image.new("RGB", (64, 64), color=(128, 64, 32)) @@ -1163,45 +1171,45 @@ Replace it with metadata-aware saving: Add the `_save_image_with_metadata` method to the Result class (after `save_artifact`): ```python - def _save_image_with_metadata(self, image, output_path, content_type): - """Save an image with embedded generation metadata. +def _save_image_with_metadata(self, image, output_path, content_type): + """Save an image with embedded generation metadata. - Args: - image: PIL Image to save - output_path: File path to save to - content_type: MIME type (determines embedding method) - """ - metadata_json = json.dumps(self.metadata, default=str) - - if content_type == "image/png": - from PIL.PngImagePlugin import PngInfo - - png_info = PngInfo() - png_info.add_text("parameters", metadata_json) - image.save(output_path, pnginfo=png_info) - logger.debug(f"Embedded PNG metadata in {output_path}") - elif content_type in ("image/jpeg", "image/webp"): - try: - import piexif - - exif_dict = {"0th": {}, "Exif": {}, "GPS": {}, "1st": {}} - if hasattr(image, "info") and "exif" in image.info: - exif_dict = piexif.load(image.info["exif"]) - exif_dict["Exif"][ - piexif.ExifIFD.UserComment - ] = piexif.helper.UserComment.dump(metadata_json) - exif_bytes = piexif.dump(exif_dict) - image.save(output_path, exif=exif_bytes) - logger.debug(f"Embedded EXIF metadata in {output_path}") - except ImportError: - logger.warning( - "piexif not installed - saving without metadata. " - "Install with: pip install piexif" - ) - image.save(output_path) - else: - # Unsupported image format for metadata - save normally + Args: + image: PIL Image to save + output_path: File path to save to + content_type: MIME type (determines embedding method) + """ + metadata_json = json.dumps(self.metadata, default=str) + + if content_type == "image/png": + from PIL.PngImagePlugin import PngInfo + + png_info = PngInfo() + png_info.add_text("parameters", metadata_json) + image.save(output_path, pnginfo=png_info) + logger.debug(f"Embedded PNG metadata in {output_path}") + elif content_type in ("image/jpeg", "image/webp"): + try: + import piexif + + exif_dict = {"0th": {}, "Exif": {}, "GPS": {}, "1st": {}} + if hasattr(image, "info") and "exif" in image.info: + exif_dict = piexif.load(image.info["exif"]) + exif_dict["Exif"][piexif.ExifIFD.UserComment] = ( + piexif.helper.UserComment.dump(metadata_json) + ) + exif_bytes = piexif.dump(exif_dict) + image.save(output_path, exif=exif_bytes) + logger.debug(f"Embedded EXIF metadata in {output_path}") + except ImportError: + logger.warning( + "piexif not installed - saving without metadata. " + "Install with: pip install piexif" + ) image.save(output_path) + else: + # Unsupported image format for metadata - save normally + image.save(output_path) ``` - [ ] **Step 3: Run metadata tests** diff --git a/docs/superpowers/plans/2026-09-01-desktop-installers.md b/docs/superpowers/plans/2026-09-01-desktop-installers.md index 64b2ee2d..4c91ccee 100644 --- a/docs/superpowers/plans/2026-09-01-desktop-installers.md +++ b/docs/superpowers/plans/2026-09-01-desktop-installers.md @@ -139,7 +139,9 @@ def settings_dir(): runs dw/__init__.py and pulls in torch, which this pure HTTP client has no use for (test_mcp_server.py::TestStartupWeight guards the boundary). """ - return Path(os.environ.get("DIFFUSERS_HELPER_ROOT") or "~/.diffusers_helper/").expanduser() + return Path( + os.environ.get("DIFFUSERS_HELPER_ROOT") or "~/.diffusers_helper/" + ).expanduser() def _base_url_from_server_file(): diff --git a/docs/superpowers/plans/2026-09-01-mcp-server.md b/docs/superpowers/plans/2026-09-01-mcp-server.md index 0715a23b..62bcde11 100644 --- a/docs/superpowers/plans/2026-09-01-mcp-server.md +++ b/docs/superpowers/plans/2026-09-01-mcp-server.md @@ -208,27 +208,25 @@ In `JobHistory.__init__`, add `events TEXT` to the `CREATE TABLE` and migrate databases that predate it: ```python - connection.execute("""CREATE TABLE IF NOT EXISTS jobs ( - id TEXT PRIMARY KEY, - workflow TEXT, - status TEXT, - created_at REAL, - started_at REAL, - finished_at REAL, - arguments TEXT, - spec TEXT, - manifest TEXT, - warnings TEXT, - error TEXT, - events TEXT - )""") - # Databases written before events were persisted are missing the - # column; ALTER is the whole migration, and rows keep NULL - columns = { - row[1] for row in connection.execute("PRAGMA table_info(jobs)") - } - if "events" not in columns: - connection.execute("ALTER TABLE jobs ADD COLUMN events TEXT") +connection.execute("""CREATE TABLE IF NOT EXISTS jobs ( + id TEXT PRIMARY KEY, + workflow TEXT, + status TEXT, + created_at REAL, + started_at REAL, + finished_at REAL, + arguments TEXT, + spec TEXT, + manifest TEXT, + warnings TEXT, + error TEXT, + events TEXT + )""") +# Databases written before events were persisted are missing the +# column; ALTER is the whole migration, and rows keep NULL +columns = {row[1] for row in connection.execute("PRAGMA table_info(jobs)")} +if "events" not in columns: + connection.execute("ALTER TABLE jobs ADD COLUMN events TEXT") ``` Update the class docstring, which currently says events are not persisted: @@ -385,9 +383,7 @@ def test_event_log_pages_with_after_and_limit(client_and_manager): assert first["truncated"] is True assert first["last_seq"] == 1 - rest = client.get( - f"/api/jobs/{job_id}/event-log?after={first['last_seq']}" - ).json() + rest = client.get(f"/api/jobs/{job_id}/event-log?after={first['last_seq']}").json() assert rest["events"][0]["seq"] == 2 assert rest["truncated"] is False @@ -423,9 +419,7 @@ def test_event_log_serves_a_historical_jobs_persisted_events(client_and_manager) def test_event_log_pages_a_historical_jobs_events(client_and_manager): client, manager = client_and_manager manager.get = lambda job_id: {"id": job_id, "status": "complete"} - manager.history.events_for = lambda job_id: [ - {"seq": index} for index in range(5) - ] + manager.history.events_for = lambda job_id: [{"seq": index} for index in range(5)] body = client.get("/api/jobs/historical/event-log?after=1&limit=2").json() @@ -920,11 +914,15 @@ def recording_client(body=None, status=200): (lambda c: catalog.get_workflow(c, "folder/w"), "/api/workflows/folder/w"), (lambda c: catalog.get_schema(c), "/api/schema"), (lambda c: catalog.list_pipelines(c), "/api/pipelines"), - (lambda c: catalog.get_pipeline_signature(c, "FluxPipeline"), - "/api/pipelines/FluxPipeline"), + ( + lambda c: catalog.get_pipeline_signature(c, "FluxPipeline"), + "/api/pipelines/FluxPipeline", + ), (lambda c: catalog.list_classes(c, "schedulers"), "/api/classes"), - (lambda c: catalog.get_class(c, "diffusers.AutoencoderKL"), - "/api/classes/diffusers.AutoencoderKL"), + ( + lambda c: catalog.get_class(c, "diffusers.AutoencoderKL"), + "/api/classes/diffusers.AutoencoderKL", + ), (lambda c: catalog.list_tasks(c), "/api/tasks"), (lambda c: catalog.get_task(c, "upscale"), "/api/tasks/upscale"), (lambda c: catalog.list_models(c), "/api/models"), @@ -932,8 +930,10 @@ def recording_client(body=None, status=200): (lambda c: catalog.get_health(c), "/api/health"), (lambda c: catalog.list_jobs(c), "/api/jobs"), (lambda c: catalog.list_gallery(c), "/api/gallery"), - (lambda c: catalog.get_gallery_metadata(c, "a.png"), - "/api/gallery/a.png/metadata"), + ( + lambda c: catalog.get_gallery_metadata(c, "a.png"), + "/api/gallery/a.png/metadata", + ), ], ) def test_each_catalog_tool_calls_its_route(call, path): @@ -1410,7 +1410,12 @@ def scripted(routes): def test_validate_posts_an_inline_workflow(): client, seen = scripted( - {("POST", "/api/validate"): (200, {"valid": True, "error": None, "warnings": []})} + { + ("POST", "/api/validate"): ( + 200, + {"valid": True, "error": None, "warnings": []}, + ) + } ) result = authoring.validate_workflow(client, workflow=WORKFLOW) @@ -1475,8 +1480,9 @@ def test_save_puts_the_definition_under_its_name(): body_seen["method"] = request.method body_seen["path"] = request.url.path body_seen["body"] = request.read() - return httpx.Response(200, json={"name": "mine", "path": "/w/mine.json", - "warnings": []}) + return httpx.Response( + 200, json={"name": "mine", "path": "/w/mine.json", "warnings": []} + ) client = DwClient(transport=httpx.MockTransport(handler)) @@ -1633,8 +1639,9 @@ def scripted(routes): def handler(request): key = (request.method, request.url.path) - seen.append({"key": key, "body": request.read(), - "params": dict(request.url.params)}) + seen.append( + {"key": key, "body": request.read(), "params": dict(request.url.params)} + ) if key not in routes: return httpx.Response(404, json={"detail": f"unrouted {key}"}) status, body = routes[key] @@ -1688,9 +1695,7 @@ def test_run_returns_immediately_rather_than_waiting_for_the_job(): def test_run_sends_an_inline_workflow_when_given_one(): client, seen = submitting() - diagnose.run_workflow( - client, inline_workflow=WORKFLOW, acknowledged_cost=True - ) + diagnose.run_workflow(client, inline_workflow=WORKFLOW, acknowledged_cost=True) assert b'"workflow"' in seen[0]["body"] @@ -1700,9 +1705,7 @@ def test_run_never_sends_base_dir(): a path-authority parameter the tool surface deliberately withholds.""" client, seen = submitting() - diagnose.run_workflow( - client, inline_workflow=WORKFLOW, acknowledged_cost=True - ) + diagnose.run_workflow(client, inline_workflow=WORKFLOW, acknowledged_cost=True) assert b"base_dir" not in seen[0]["body"] @@ -1747,15 +1750,17 @@ def test_run_surfaces_a_rejected_workflow(): ) with pytest.raises(DwApiError, match="steps must not be empty"): - diagnose.run_workflow( - client, inline_workflow=WORKFLOW, acknowledged_cost=True - ) + diagnose.run_workflow(client, inline_workflow=WORKFLOW, acknowledged_cost=True) def test_get_job_returns_the_detail_payload(): client, _seen = scripted( - {("GET", "/api/jobs/job-1"): (200, {"id": "job-1", "status": "failed", - "error": "CUDA out of memory"})} + { + ("GET", "/api/jobs/job-1"): ( + 200, + {"id": "job-1", "status": "failed", "error": "CUDA out of memory"}, + ) + } ) assert diagnose.get_job(client, "job-1")["error"] == "CUDA out of memory" @@ -1766,9 +1771,14 @@ def test_get_job_events_pages_from_the_event_log(): { ("GET", "/api/jobs/job-1/event-log"): ( 200, - {"id": "job-1", "status": "running", - "events": [{"seq": 3, "event": "phase"}], - "last_seq": 3, "truncated": True, "note": None}, + { + "id": "job-1", + "status": "running", + "events": [{"seq": 3, "event": "phase"}], + "last_seq": 3, + "truncated": True, + "note": None, + }, ) } ) @@ -1793,10 +1803,14 @@ def test_get_job_events_defaults_to_the_whole_log(): def test_cancel_rerun_and_move_call_their_routes(): client, seen = scripted( { - ("POST", "/api/jobs/job-1/cancel"): (200, {"id": "job-1", - "status": "cancelled"}), - ("POST", "/api/jobs/job-1/rerun"): (201, {"id": "job-2", - "status": "queued"}), + ("POST", "/api/jobs/job-1/cancel"): ( + 200, + {"id": "job-1", "status": "cancelled"}, + ), + ("POST", "/api/jobs/job-1/rerun"): ( + 201, + {"id": "job-2", "status": "queued"}, + ), ("POST", "/api/jobs/job-1/move"): (200, {"id": "job-1", "queue": []}), } ) @@ -2032,7 +2046,9 @@ async def test_every_designed_tool_is_registered(): async def test_every_tool_has_a_description(): tools = await tools_of(server_over(ok({}))) - missing = [name for name, tool in tools.items() if not (tool.description or "").strip()] + missing = [ + name for name, tool in tools.items() if not (tool.description or "").strip() + ] assert missing == [] diff --git a/docs/superpowers/plans/2026-09-03-memory-manager-step-cache-downloads.md b/docs/superpowers/plans/2026-09-03-memory-manager-step-cache-downloads.md index d85a0022..447ca82e 100644 --- a/docs/superpowers/plans/2026-09-03-memory-manager-step-cache-downloads.md +++ b/docs/superpowers/plans/2026-09-03-memory-manager-step-cache-downloads.md @@ -195,7 +195,9 @@ class FakeModel: self.oom_on = None def to(self, device): - device = torch.device(device) if not isinstance(device, torch.device) else device + device = ( + torch.device(device) if not isinstance(device, torch.device) else device + ) if self.oom_on is not None and str(device) == str(self.oom_on): self.oom_on = None # only OOM once, so a retry after eviction succeeds raise torch.OutOfMemoryError(f"{self.name} cannot fit on {device}") @@ -301,6 +303,7 @@ makes for its node graph. It is not consulted by 'offload: model/sequential' or 'group_offload', which install diffusers' own hooks - dw does not own the individual .to() calls those make, so there is nothing to intercept. """ + import time import logging @@ -334,10 +337,16 @@ class MemoryManager: try: component.to(device) if entry is not None: - entry["device"] = torch.device(device) if not isinstance(device, torch.device) else device + entry["device"] = ( + torch.device(device) + if not isinstance(device, torch.device) + else device + ) return except torch.OutOfMemoryError: - victim_id = self._pick_eviction_candidate(device, exclude_id=id(component)) + victim_id = self._pick_eviction_candidate( + device, exclude_id=id(component) + ) if victim_id is None: raise self._evict(victim_id, offload_device) @@ -346,7 +355,11 @@ class MemoryManager: """Record that `component` has been moved back to `offload_device`.""" entry = self._entries.get(id(component)) if entry is not None: - entry["device"] = torch.device(offload_device) if not isinstance(offload_device, torch.device) else offload_device + entry["device"] = ( + torch.device(offload_device) + if not isinstance(offload_device, torch.device) + else offload_device + ) def _pick_eviction_candidate(self, device, exclude_id): device = str(device) @@ -364,9 +377,15 @@ class MemoryManager: entry = self._entries.get(comp_id) if entry is None: return - logger.debug(f"Evicting a lower-priority on-demand component to free {entry['device']}") + logger.debug( + f"Evicting a lower-priority on-demand component to free {entry['device']}" + ) entry["component"].to(offload_device) - entry["device"] = torch.device(offload_device) if not isinstance(offload_device, torch.device) else offload_device + entry["device"] = ( + torch.device(offload_device) + if not isinstance(offload_device, torch.device) + else offload_device + ) memory_manager = MemoryManager() @@ -752,6 +771,7 @@ A step is safe to skip only if: 3. every previous_result: it reads was ITSELF served from cache this run - otherwise a change upstream leaves this step's cached output stale """ + import logging from .workflow import referenced_result_names @@ -896,43 +916,41 @@ from .step_cache import step_cache Then in `Workflow.run`, initialize `hits_this_run = set()` immediately before the `for i, step_data in enumerate(steps):` loop (`dw/workflow.py:337-338`), and replace the loop body from `step = Step(step_data, step_seed, self.workflow_definition)` through the `saved_files = result.save(...)` / `self.manifest.append(...)` block (`dw/workflow.py:352-370`) with: ```python - step = Step(step_data, step_seed, self.workflow_definition) - - is_cacheable = "workflow" not in step_data - cached_result = ( - step_cache.get(step_data, step_seed, hits_this_run) - if is_cacheable - else None - ) - - if cached_result is not None: - logger.info(f"Step '{step.name}' unchanged - reusing cached result") - result = cached_result - saved_files = result.saved_files - hits_this_run.add(step.name) - step_action = None - else: - step_action = self.create_step_action( - step_data, - shared_components, - pipelines, - step_seed, - get_device(), - ) - result = step.run(results, pipelines, step_action) - saved_files = result.save( - self.effective_output_dir, f"{workflow_id}-{step.name}.{i}" - ) - if is_cacheable: - step_cache.put(step_data, step_seed, result) - - last_result = result - results[step.name] = result - self.manifest.append({"step": step.name, "files": saved_files}) - # A sub-workflow's saves land in the child's manifest - roll - # them up so job history and the gallery see every file - if isinstance(step_action, Workflow): - self.manifest.extend(getattr(step_action, "manifest", [])) +step = Step(step_data, step_seed, self.workflow_definition) + +is_cacheable = "workflow" not in step_data +cached_result = ( + step_cache.get(step_data, step_seed, hits_this_run) if is_cacheable else None +) + +if cached_result is not None: + logger.info(f"Step '{step.name}' unchanged - reusing cached result") + result = cached_result + saved_files = result.saved_files + hits_this_run.add(step.name) + step_action = None +else: + step_action = self.create_step_action( + step_data, + shared_components, + pipelines, + step_seed, + get_device(), + ) + result = step.run(results, pipelines, step_action) + saved_files = result.save( + self.effective_output_dir, f"{workflow_id}-{step.name}.{i}" + ) + if is_cacheable: + step_cache.put(step_data, step_seed, result) + +last_result = result +results[step.name] = result +self.manifest.append({"step": step.name, "files": saved_files}) +# A sub-workflow's saves land in the child's manifest - roll +# them up so job history and the gallery see every file +if isinstance(step_action, Workflow): + self.manifest.extend(getattr(step_action, "manifest", [])) ``` Leave everything below this (the `run_context.emit("step_end", ...)` call and everything after, `dw/workflow.py:371-395`) unchanged — `saved_files` is defined on both branches so it still works. @@ -1305,14 +1323,18 @@ def test_download_output_writes_bytes_to_explicit_file_path(tmp_path): def test_download_output_into_a_directory_uses_the_output_basename(tmp_path): client = serving(png_bytes(10, 10), "image/png") - result = download_output(client, "sub/run-step.0-0.0.png", destination=str(tmp_path)) + result = download_output( + client, "sub/run-step.0-0.0.png", destination=str(tmp_path) + ) saved = tmp_path / "run-step.0-0.0.png" assert saved.read_bytes() == png_bytes(10, 10) assert result["saved_to"] == str(saved) -def test_download_output_with_no_destination_saves_to_current_directory(tmp_path, monkeypatch): +def test_download_output_with_no_destination_saves_to_current_directory( + tmp_path, monkeypatch +): monkeypatch.chdir(tmp_path) client = serving(png_bytes(10, 10), "image/png") diff --git a/docs/superpowers/plans/2026-09-04-remote-gpu-server.md b/docs/superpowers/plans/2026-09-04-remote-gpu-server.md index 4f209836..035e24fb 100644 --- a/docs/superpowers/plans/2026-09-04-remote-gpu-server.md +++ b/docs/superpowers/plans/2026-09-04-remote-gpu-server.md @@ -364,7 +364,12 @@ def no_stdio(monkeypatch): def healthy(request): return httpx.Response( 200, - json={"status": "ok", "hostname": "gpu-box", "version": "1.2.3", "device": "cuda"}, + json={ + "status": "ok", + "hostname": "gpu-box", + "version": "1.2.3", + "device": "cuda", + }, ) @@ -434,7 +439,9 @@ def test_a_successful_probe_prints_the_server_identity(no_stdio, capsys): seen = [] def handler(request): - seen.append((request.method, request.url.path, request.headers.get("authorization"))) + seen.append( + (request.method, request.url.path, request.headers.get("authorization")) + ) return healthy(request) code = cli.main( @@ -710,7 +717,9 @@ def test_mcp_is_not_mounted_by_default(tmp_path): app = make_app(tmp_path, mcp=False) with TestClient(app, base_url="http://localhost") as client: assert client.get("/api/health").json()["mcp"] is False - assert client.post("/mcp", json=INITIALIZE, headers=MCP_HEADERS).status_code == 404 + assert ( + client.post("/mcp", json=INITIALIZE, headers=MCP_HEADERS).status_code == 404 + ) def test_mcp_mount_answers_initialize(tmp_path): @@ -726,12 +735,16 @@ def test_mcp_mount_answers_initialize(tmp_path): def test_mcp_mount_is_gated_by_the_bearer_token(tmp_path): app = make_app(tmp_path, token="s3cr3t") with TestClient(app, base_url="http://localhost") as client: - assert client.post("/mcp", json=INITIALIZE, headers=MCP_HEADERS).status_code == 401 + assert ( + client.post("/mcp", json=INITIALIZE, headers=MCP_HEADERS).status_code == 401 + ) wrong = {**MCP_HEADERS, "Authorization": "Bearer nope"} assert client.post("/mcp", json=INITIALIZE, headers=wrong).status_code == 401 # never as a query parameter - that allowance is for / only assert ( - client.post("/mcp?token=s3cr3t", json=INITIALIZE, headers=MCP_HEADERS).status_code + client.post( + "/mcp?token=s3cr3t", json=INITIALIZE, headers=MCP_HEADERS + ).status_code == 401 ) right = {**MCP_HEADERS, "Authorization": "Bearer s3cr3t"} @@ -759,7 +772,9 @@ async def test_a_real_mcp_client_lists_every_tool_over_http(tmp_path): async with streamable_http_client( "http://localhost/mcp", http_client=http ) as streams: - async with ClientSession(streams.read_stream, streams.write_stream) as session: + async with ClientSession( + streams.read_stream, streams.write_stream + ) as session: await session.initialize() tools = await session.list_tools() assert {t.name for t in tools.tools} == EXPECTED_TOOLS diff --git a/docs/superpowers/plans/2026-09-06-catalog-metadata.md b/docs/superpowers/plans/2026-09-06-catalog-metadata.md index 4cb684cb..ffa1acd6 100644 --- a/docs/superpowers/plans/2026-09-06-catalog-metadata.md +++ b/docs/superpowers/plans/2026-09-06-catalog-metadata.md @@ -72,7 +72,9 @@ from dw.server.catalog_shape import ( ) -def pipeline_step(name, content_type, component_type="{Fake}", arguments=None, chain=None): +def pipeline_step( + name, content_type, component_type="{Fake}", arguments=None, chain=None +): pipeline = { "configuration": {"component_type": component_type}, "from_pretrained_arguments": {"model_name": "m"}, @@ -80,7 +82,11 @@ def pipeline_step(name, content_type, component_type="{Fake}", arguments=None, c } if chain is not None: pipeline["chain"] = chain - return {"name": name, "pipeline": pipeline, "result": {"content_type": content_type}} + return { + "name": name, + "pipeline": pipeline, + "result": {"content_type": content_type}, + } def task_step(name, command, arguments, content_type=None): @@ -95,13 +101,32 @@ def definition(*steps, **top): def test_vocabularies_are_closed_and_stable(): - assert SHAPES == ("image", "image-set", "image-edit", "shot", "sequence", "audio", "text", "utility") + assert SHAPES == ( + "image", + "image-set", + "image-edit", + "shot", + "sequence", + "audio", + "text", + "utility", + ) assert TRAITS == ( - "speech", "chained", "image-conditioned", "identity-referenced", - "needs-input-media", "composes-workflows", + "speech", + "chained", + "image-conditioned", + "identity-referenced", + "needs-input-media", + "composes-workflows", ) assert GENERATIVE_TASKS == frozenset( - {"generate_speech", "text_generation", "image_to_text", "diffusion_upscale", "interpolate_frames"} + { + "generate_speech", + "text_generation", + "image_to_text", + "diffusion_upscale", + "interpolate_frames", + } ) @@ -130,22 +155,36 @@ def test_a_workflow_step_emitting_images_is_an_image_set(): @pytest.mark.parametrize( - "component_type", ["FluxImg2ImgPipeline", "StableDiffusionInpaintPipeline", "QwenImageEditPipeline", "FluxKontextPipeline", "StableDiffusionUpscalePipeline"] + "component_type", + [ + "FluxImg2ImgPipeline", + "StableDiffusionInpaintPipeline", + "QwenImageEditPipeline", + "FluxKontextPipeline", + "StableDiffusionUpscalePipeline", + ], ) def test_an_editing_pipeline_is_image_edit(component_type): - meta = derive_catalog_metadata(definition(pipeline_step("gen", "image/jpeg", component_type))) + meta = derive_catalog_metadata( + definition(pipeline_step("gen", "image/jpeg", component_type)) + ) assert meta["shape"] == "image-edit" def test_an_image_argument_on_an_image_pipeline_is_image_edit(): - step = pipeline_step("gen", "image/jpeg", arguments={"prompt": "p", "image": "variable:image"}) + step = pipeline_step( + "gen", "image/jpeg", arguments={"prompt": "p", "image": "variable:image"} + ) meta = derive_catalog_metadata(definition(step)) assert meta["shape"] == "image-edit" assert "needs-input-media" in meta["traits"] def test_one_clip_is_a_shot(): - assert derive_catalog_metadata(definition(pipeline_step("v", "video/mp4")))["shape"] == "shot" + assert ( + derive_catalog_metadata(definition(pipeline_step("v", "video/mp4")))["shape"] + == "shot" + ) def test_a_concat_fed_by_two_steps_is_a_sequence(): @@ -153,7 +192,12 @@ def test_a_concat_fed_by_two_steps_is_a_sequence(): definition( pipeline_step("a", "video/mp4"), pipeline_step("b", "video/mp4"), - task_step("cut", "concat_videos", {"videos": ["previous_result:a", "previous_result:b"]}, "video/mp4"), + task_step( + "cut", + "concat_videos", + {"videos": ["previous_result:a", "previous_result:b"]}, + "video/mp4", + ), ) ) assert meta["shape"] == "sequence" @@ -164,7 +208,12 @@ def test_a_dissolve_fed_by_two_steps_is_a_sequence(): definition( pipeline_step("a", "video/mp4"), pipeline_step("b", "video/mp4"), - task_step("cut", "dissolve_videos", {"videos": ["previous_result:a", "previous_result:b"]}, "video/mp4"), + task_step( + "cut", + "dissolve_videos", + {"videos": ["previous_result:a", "previous_result:b"]}, + "video/mp4", + ), ) ) assert meta["shape"] == "sequence" @@ -174,7 +223,12 @@ def test_a_concat_fed_by_one_step_is_still_a_shot(): meta = derive_catalog_metadata( definition( pipeline_step("a", "video/mp4"), - task_step("cut", "concat_videos", {"videos": ["previous_result:a", "previous_result:a"]}, "video/mp4"), + task_step( + "cut", + "concat_videos", + {"videos": ["previous_result:a", "previous_result:a"]}, + "video/mp4", + ), ) ) assert meta["shape"] == "shot" @@ -187,15 +241,21 @@ def test_a_chain_is_a_chained_shot_not_a_sequence(): assert "chained" in meta["traits"] -@pytest.mark.parametrize("name", ["last_frame", "last_segment", "last_image", "match_audio"]) +@pytest.mark.parametrize( + "name", ["last_frame", "last_segment", "last_image", "match_audio"] +) def test_continuation_arguments_are_chained(name): - step = pipeline_step("v", "video/mp4", arguments={"prompt": "p", name: "previous_result:x"}) + step = pipeline_step( + "v", "video/mp4", arguments={"prompt": "p", name: "previous_result:x"} + ) assert "chained" in derive_catalog_metadata(definition(step))["traits"] def test_video_outranks_image_when_both_are_produced(): meta = derive_catalog_metadata( - definition(pipeline_step("board", "image/jpeg"), pipeline_step("v", "video/mp4")) + definition( + pipeline_step("board", "image/jpeg"), pipeline_step("v", "video/mp4") + ) ) assert meta["shape"] == "shot" @@ -208,7 +268,9 @@ def test_audio_only_is_audio(): def test_text_only_is_text(): - step = task_step("expand", "text_generation", {"prompt": "variable:prompt"}, "text/plain") + step = task_step( + "expand", "text_generation", {"prompt": "variable:prompt"}, "text/plain" + ) assert derive_catalog_metadata(definition(step))["shape"] == "text" @@ -225,13 +287,19 @@ def test_a_generative_task_is_not_utility(): def test_a_video_pipeline_emitting_audio_speaks(): - step = pipeline_step("v", "video/mp4", arguments={"prompt": "p", "output": ["videos", "audio"]}) + step = pipeline_step( + "v", "video/mp4", arguments={"prompt": "p", "output": ["videos", "audio"]} + ) assert "speech" in derive_catalog_metadata(definition(step))["traits"] def test_a_video_pipeline_with_an_image_argument_is_image_conditioned(): - step = pipeline_step("v", "video/mp4", arguments={"prompt": "p", "image": "previous_result:still"}) - meta = derive_catalog_metadata(definition(pipeline_step("still", "image/jpeg"), step)) + step = pipeline_step( + "v", "video/mp4", arguments={"prompt": "p", "image": "previous_result:still"} + ) + meta = derive_catalog_metadata( + definition(pipeline_step("still", "image/jpeg"), step) + ) assert "image-conditioned" in meta["traits"] # previous_result is not supplied media assert "needs-input-media" not in meta["traits"] @@ -243,26 +311,42 @@ def test_an_image_to_video_component_is_image_conditioned(): def test_references_are_identity_referenced(): - step = pipeline_step("v", "video/mp4", arguments={"prompt": "p", "references": ["previous_result:face"]}) + step = pipeline_step( + "v", + "video/mp4", + arguments={"prompt": "p", "references": ["previous_result:face"]}, + ) assert "identity-referenced" in derive_catalog_metadata(definition(step))["traits"] def test_a_location_argument_needs_input_media(): - step = pipeline_step("v", "video/mp4", arguments={"prompt": "p", "image": {"location": "https://x/y.png"}}) + step = pipeline_step( + "v", + "video/mp4", + arguments={"prompt": "p", "image": {"location": "https://x/y.png"}}, + ) assert "needs-input-media" in derive_catalog_metadata(definition(step))["traits"] def test_a_pipeline_reference_step_counts_as_generation(): ref = { "name": "shot_2", - "pipeline_reference": {"reference_name": "shot_1", "arguments": {"prompt": "p", "references": ["asset:face.png"]}}, + "pipeline_reference": { + "reference_name": "shot_1", + "arguments": {"prompt": "p", "references": ["asset:face.png"]}, + }, "result": {"content_type": "video/mp4"}, } meta = derive_catalog_metadata( definition( pipeline_step("shot_1", "video/mp4"), ref, - task_step("cut", "concat_videos", {"videos": ["previous_result:shot_1", "previous_result:shot_2"]}, "video/mp4"), + task_step( + "cut", + "concat_videos", + {"videos": ["previous_result:shot_1", "previous_result:shot_2"]}, + "video/mp4", + ), ) ) assert meta["shape"] == "sequence" @@ -271,19 +355,27 @@ def test_a_pipeline_reference_step_counts_as_generation(): def test_summary_is_the_first_sentence(): - meta = derive_catalog_metadata(definition(pipeline_step("g", "image/jpeg"), description="Makes a cat. Then more.")) + meta = derive_catalog_metadata( + definition( + pipeline_step("g", "image/jpeg"), description="Makes a cat. Then more." + ) + ) assert meta["summary"] == "Makes a cat." assert meta["summary_truncated"] is False def test_summary_splits_on_newline_too(): - meta = derive_catalog_metadata(definition(pipeline_step("g", "image/jpeg"), description="Line one\nLine two.")) + meta = derive_catalog_metadata( + definition(pipeline_step("g", "image/jpeg"), description="Line one\nLine two.") + ) assert meta["summary"] == "Line one" def test_a_long_first_sentence_is_truncated_at_a_word_boundary(): words = " ".join(["word"] * 40) + "." - meta = derive_catalog_metadata(definition(pipeline_step("g", "image/jpeg"), description=words)) + meta = derive_catalog_metadata( + definition(pipeline_step("g", "image/jpeg"), description=words) + ) assert len(meta["summary"]) <= SUMMARY_LIMIT assert meta["summary"].endswith("…") assert not meta["summary"][:-1].endswith(" ") @@ -342,7 +434,16 @@ read structure (a concat step, a `references` argument), never checkpoints. import re -SHAPES = ("image", "image-set", "image-edit", "shot", "sequence", "audio", "text", "utility") +SHAPES = ( + "image", + "image-set", + "image-edit", + "shot", + "sequence", + "audio", + "text", + "utility", +) TRAITS = ( "speech", "chained", @@ -354,13 +455,21 @@ TRAITS = ( # Tasks that create content rather than process it. A workflow made only # of processing tasks is a utility. GENERATIVE_TASKS = frozenset( - {"generate_speech", "text_generation", "image_to_text", "diffusion_upscale", "interpolate_frames"} + { + "generate_speech", + "text_generation", + "image_to_text", + "diffusion_upscale", + "interpolate_frames", + } ) SUMMARY_LIMIT = 120 _KIND_PRECEDENCE = ("video", "audio", "image", "text") _EDIT_PIPELINE = re.compile(r"inpaint|img2img|edit|upscale|outpaint|kontext", re.I) -_CHAIN_ARGUMENTS = frozenset({"last_frame", "last_segment", "last_image", "match_audio"}) +_CHAIN_ARGUMENTS = frozenset( + {"last_frame", "last_segment", "last_image", "match_audio"} +) _MEDIA_ARGUMENTS = frozenset({"image", "video", "audio", "mask_image"}) _CUT_TASKS = frozenset({"concat_videos", "dissolve_videos"}) _SENTENCE_END = re.compile(r"(?<=[.!?])\s|\n") @@ -368,7 +477,11 @@ _SENTENCE_END = re.compile(r"(?<=[.!?])\s|\n") def _steps(definition): steps = definition.get("steps") if isinstance(definition, dict) else None - return [step for step in steps if isinstance(step, dict)] if isinstance(steps, list) else [] + return ( + [step for step in steps if isinstance(step, dict)] + if isinstance(steps, list) + else [] + ) def _block(step): @@ -440,7 +553,11 @@ def _needs_input_media(steps): for step in steps: arguments = _arguments(step) for name, value in arguments.items(): - if name in _MEDIA_ARGUMENTS and isinstance(value, str) and value.startswith("variable:"): + if ( + name in _MEDIA_ARGUMENTS + and isinstance(value, str) + and value.startswith("variable:") + ): return True for value in _walk(arguments): if isinstance(value, str) and value.startswith("asset:"): @@ -464,13 +581,17 @@ def _derive_shape(steps, kind): if len(_fed_by(_arguments(step).get("videos"))) >= 2: return "sequence" return "shot" - image_steps = [step for step in steps if _generates(step) and _kind(step) == "image"] + image_steps = [ + step for step in steps if _generates(step) and _kind(step) == "image" + ] for step in image_steps: if _EDIT_PIPELINE.search(_component_type(step)): return "image-edit" if _MEDIA_ARGUMENTS & set(_arguments(step)) & {"image", "mask_image"}: return "image-edit" - if len(image_steps) >= 2 or any(_block(step)[0] == "workflow" for step in image_steps): + if len(image_steps) >= 2 or any( + _block(step)[0] == "workflow" for step in image_steps + ): return "image-set" return "image" @@ -489,7 +610,9 @@ def _derive_traits(steps): traits.add("chained") if _CHAIN_ARGUMENTS & set(arguments): traits.add("chained") - if _kind(step) == "video" and ("image" in arguments or "ImageToVideo" in _component_type(step)): + if _kind(step) == "video" and ( + "image" in arguments or "ImageToVideo" in _component_type(step) + ): traits.add("image-conditioned") if "references" in arguments: traits.add("identity-referenced") @@ -599,9 +722,19 @@ def test_the_schema_declares_the_vocabulary(): def test_a_declared_cost_validates_and_a_bad_one_does_not(): schema = load_schema("workflow") base = definition(pipeline_step("g", "image/jpeg")) - ok, _ = validate_data({**base, "cost": [{"device": "cuda", "name": "RTX 4090", "vram_gb": 22, "minutes": 3}]}, schema) + ok, _ = validate_data( + { + **base, + "cost": [ + {"device": "cuda", "name": "RTX 4090", "vram_gb": 22, "minutes": 3} + ], + }, + schema, + ) assert ok - bad, message = validate_data({**base, "cost": [{"device": "tpu", "vram_gb": 1, "minutes": 1}]}, schema) + bad, message = validate_data( + {**base, "cost": [{"device": "tpu", "vram_gb": 1, "minutes": 1}]}, schema + ) assert not bad and "cost" in message bad, _ = validate_data({**base, "shape": "cinematic"}, schema) assert not bad @@ -693,20 +826,28 @@ def video_workflow(job_id, with_cost=False): "pipeline": { "configuration": {"component_type": "{Fake}", "no_generator": True}, "from_pretrained_arguments": {"model_name": "m"}, - "arguments": {"prompt": "variable:prompt", "output": ["videos", "audio"]}, + "arguments": { + "prompt": "variable:prompt", + "output": ["videos", "audio"], + }, }, "result": {"content_type": "video/mp4"}, } ], } if with_cost: - workflow["cost"] = [{"device": "cuda", "name": "RTX 4090", "vram_gb": 20, "minutes": 2}] + workflow["cost"] = [ + {"device": "cuda", "name": "RTX 4090", "vram_gb": 20, "minutes": 2} + ] return workflow def test_the_listing_carries_derived_metadata(server): with server(success_script) as client: - client.put("/api/workflows/templates/clip", json={"workflow": video_workflow("clip", with_cost=True)}) + client.put( + "/api/workflows/templates/clip", + json={"workflow": video_workflow("clip", with_cost=True)}, + ) tuned = video_workflow("tuned") tuned["configures"] = "templates/clip" tuned["description"] = "The same clip on a bigger checkpoint." @@ -718,7 +859,9 @@ def test_the_listing_carries_derived_metadata(server): assert clip["shape"] == "shot" assert clip["traits"] == ["speech"] assert clip["summary"] == "One clip from a prompt." - assert clip["cost"] == [{"device": "cuda", "name": "RTX 4090", "vram_gb": 20, "minutes": 2}] + assert clip["cost"] == [ + {"device": "cuda", "name": "RTX 4090", "vram_gb": 20, "minutes": 2} + ] tuned = details["models/tuned"] assert tuned["shape"] == "shot" and tuned["traits"] == ["speech"] @@ -729,7 +872,17 @@ def test_the_listing_carries_derived_metadata(server): assert basic["shape"] == "utility" # no result block, so no kind assert basic["summary"] == "" and basic["cost"] is None # nothing the UI reads went away - assert {"kinds", "steps", "variables", "variable_names", "description", "configures", "prompt_refs", "origin", "writable"} <= set(basic) + assert { + "kinds", + "steps", + "variables", + "variable_names", + "description", + "configures", + "prompt_refs", + "origin", + "writable", + } <= set(basic) ``` - [ ] **Step 2: Run to verify failure** @@ -833,10 +986,19 @@ from dw.server.catalog_shape import COMPACT_FIELDS, project_listing def entry(shape, traits=(), configures="", **extra): return { - "kinds": [], "steps": 1, "variables": 0, "variable_names": [], - "description": "long text", "configures": configures, "prompt_refs": [], - "origin": "workspace", "writable": True, - "shape": shape, "traits": sorted(traits), "summary": "short", "cost": None, + "kinds": [], + "steps": 1, + "variables": 0, + "variable_names": [], + "description": "long text", + "configures": configures, + "prompt_refs": [], + "origin": "workspace", + "writable": True, + "shape": shape, + "traits": sorted(traits), + "summary": "short", + "cost": None, **extra, } @@ -859,8 +1021,13 @@ def test_shape_filters(): def test_traits_must_all_match(): - assert set(project_listing(LISTING, traits=["speech"])) == {"templates/talk", "templates/clip"} - assert set(project_listing(LISTING, traits=["speech", "identity-referenced"])) == {"templates/talk"} + assert set(project_listing(LISTING, traits=["speech"])) == { + "templates/talk", + "templates/clip", + } + assert set(project_listing(LISTING, traits=["speech", "identity-referenced"])) == { + "templates/talk" + } def test_configures_filters_to_a_templates_configs(): @@ -875,15 +1042,23 @@ def test_compact_drops_prose_and_model_configs_and_keeps_user_workflows(): def test_compact_with_include_models_keeps_them(): - assert "models/flux" in project_listing(LISTING, view="compact", include_models=True) + assert "models/flux" in project_listing( + LISTING, view="compact", include_models=True + ) def test_compact_with_configures_implies_models(): - assert set(project_listing(LISTING, view="compact", configures="templates/tti")) == {"models/flux"} + assert set( + project_listing(LISTING, view="compact", configures="templates/tti") + ) == {"models/flux"} def test_compact_keeps_configures_missing_when_set(): - listing = {"models/typo": entry("image", configures="", configures_missing="templates/nope")} + listing = { + "models/typo": entry( + "image", configures="", configures_missing="templates/nope" + ) + } compact = project_listing(listing, view="compact", include_models=True) assert compact["models/typo"]["configures_missing"] == "templates/nope" @@ -918,7 +1093,15 @@ COMPACT_FIELDS = ( ) -def project_listing(details, *, shape=None, traits=None, configures=None, include_models=False, view=None): +def project_listing( + details, + *, + shape=None, + traits=None, + configures=None, + include_models=False, + view=None, +): """The listing an agent asked for: filtered by shape and traits, and in the compact view stripped to what choosing a template needs. @@ -929,11 +1112,15 @@ def project_listing(details, *, shape=None, traits=None, configures=None, includ found. The full view never drops entries or fields. """ if shape is not None and shape not in SHAPES: - raise ValueError(f"Unknown shape {shape!r}. The shapes are: {', '.join(SHAPES)}.") + raise ValueError( + f"Unknown shape {shape!r}. The shapes are: {', '.join(SHAPES)}." + ) traits = list(traits or []) unknown = [t for t in traits if t not in TRAITS] if unknown: - raise ValueError(f"Unknown trait(s) {', '.join(unknown)}. The traits are: {', '.join(TRAITS)}.") + raise ValueError( + f"Unknown trait(s) {', '.join(unknown)}. The traits are: {', '.join(TRAITS)}." + ) if view not in (None, "compact"): raise ValueError("view must be 'compact' or omitted") @@ -970,7 +1157,9 @@ Expected: all pass. ```python def test_the_listing_filters_and_compacts(server): with server(success_script) as client: - client.put("/api/workflows/templates/clip", json={"workflow": video_workflow("clip")}) + client.put( + "/api/workflows/templates/clip", json={"workflow": video_workflow("clip")} + ) tuned = video_workflow("tuned") tuned["configures"] = "templates/clip" client.put("/api/workflows/models/tuned", json={"workflow": tuned}) @@ -985,12 +1174,18 @@ def test_the_listing_filters_and_compacts(server): compact = client.get("/api/workflows", params={"view": "compact"}).json() assert set(compact["details"]) == {"Basic", "templates/clip"} assert "description" not in compact["details"]["templates/clip"] - assert compact["details"]["templates/clip"]["summary"] == "One clip from a prompt." + assert ( + compact["details"]["templates/clip"]["summary"] == "One clip from a prompt." + ) - with_models = client.get("/api/workflows", params={"view": "compact", "include_models": "true"}).json() + with_models = client.get( + "/api/workflows", params={"view": "compact", "include_models": "true"} + ).json() assert "models/tuned" in with_models["details"] - configs = client.get("/api/workflows", params={"configures": "templates/clip"}).json() + configs = client.get( + "/api/workflows", params={"configures": "templates/clip"} + ).json() assert set(configs["details"]) == {"models/tuned"} by_trait = client.get("/api/workflows", params={"traits": "speech"}).json() @@ -1101,7 +1296,11 @@ def test_list_workflows_passes_its_filters_through(): client, seen = recording_client({"workflows": [], "details": {}}) catalog.list_workflows( - client, shape="sequence", traits=["speech", "chained"], configures="templates/x", include_models=True + client, + shape="sequence", + traits=["speech", "chained"], + configures="templates/x", + include_models=True, ) assert seen["params"] == { @@ -1120,7 +1319,9 @@ In `tests/test_mcp_server.py`, add: async def test_list_workflows_takes_shape_and_traits(): tools = await tools_of(server_over(ok({}))) schema = tools["list_workflows"].inputSchema - assert {"shape", "traits", "configures", "include_models"} <= set(schema["properties"]) + assert {"shape", "traits", "configures", "include_models"} <= set( + schema["properties"] + ) assert "shape" in tools["list_workflows"].description @@ -1141,7 +1342,9 @@ Expected: `seen["params"]` is `{}`; the schema lacks `shape`. `dw_mcp/catalog.py`: ```python -def list_workflows(client, shape=None, traits=None, configures=None, include_models=False): +def list_workflows( + client, shape=None, traits=None, configures=None, include_models=False +): """Workflow names the server can reach, in the compact view: summary, shape, traits, cost, output kinds and variable names per workflow - what choosing one needs and nothing that reading one needs. Templates @@ -1151,7 +1354,9 @@ def list_workflows(client, shape=None, traits=None, configures=None, include_mod if shape: params["shape"] = shape if traits: - params["traits"] = ",".join(traits) if isinstance(traits, (list, tuple)) else traits + params["traits"] = ( + ",".join(traits) if isinstance(traits, (list, tuple)) else traits + ) if configures: params["configures"] = configures if include_models: @@ -1194,15 +1399,16 @@ Add `from typing import Optional` at the top of `dw_mcp/server.py` if it is not Instructions: replace the paragraph beginning `"Start from \`list_workflows\`: ..."` with: ```python - "Start from `list_workflows(shape=...)`: the server keeps a " - "large catalog, and its compact listing carries each " - "workflow's summary, shape, traits, cost and variable names - " - "run what is already there, with `arguments` overriding its " - "variables, rather than authoring a new workflow for a " - "request an existing one covers. Shapes: image, image-set, " - "image-edit, shot, sequence, audio, text, utility. Traits: " - "speech, chained, image-conditioned, identity-referenced, " - "needs-input-media, composes-workflows.\n" +"Start from `list_workflows(shape=...)`: the server keeps a" + +"large catalog, and its compact listing carries each " +"workflow's summary, shape, traits, cost and variable names - " +"run what is already there, with `arguments` overriding its " +"variables, rather than authoring a new workflow for a " +"request an existing one covers. Shapes: image, image-set, " +"image-edit, shot, sequence, audio, text, utility. Traits: " +"speech, chained, image-conditioned, identity-referenced, " +"needs-input-media, composes-workflows.\n" ``` and in the next paragraph change `"Decide which shape the deliverable is first, then match the catalog against that; "` to `"Decide which shape the deliverable is first, then call `list_workflows` with it; "`. @@ -1263,7 +1469,9 @@ Co-Authored-By: Claude Fable 5.1 " ```python def test_saving_reports_how_the_workflow_will_be_matched(server): with server(success_script) as client: - saved = client.put("/api/workflows/clip", json={"workflow": video_workflow("clip")}).json() + saved = client.put( + "/api/workflows/clip", json={"workflow": video_workflow("clip")} + ).json() assert saved["shape"] == "shot" assert saved["traits"] == ["speech"] assert saved["summary"] == "One clip from a prompt." @@ -1340,14 +1548,16 @@ Note the existing attribute `Job.workflow_name` holds the definition's **`id`** def test_a_job_remembers_the_catalog_name_it_ran_from(server, tmp_path): with server(success_script) as client: job = client.post("/api/jobs", json={"workflow_path": "Basic"}).json() - assert job["workflow"] == "basic" # the definition's id, as before - assert job["workflow_name"] == "Basic" # the catalog name + assert job["workflow"] == "basic" # the definition's id, as before + assert job["workflow_name"] == "Basic" # the catalog name wait_for_status(client, job["id"], TERMINAL_STATES) listed = {j["id"]: j for j in client.get("/api/jobs").json()["jobs"]} assert listed[job["id"]]["workflow_name"] == "Basic" - inline = client.post("/api/jobs", json={"workflow": valid_workflow("inline")}).json() + inline = client.post( + "/api/jobs", json={"workflow": valid_workflow("inline")} + ).json() assert inline["workflow_name"] is None wait_for_status(client, inline["id"], TERMINAL_STATES) @@ -1364,6 +1574,7 @@ def test_a_job_remembers_the_catalog_name_it_ran_from(server, tmp_path): def test_an_old_history_database_gains_the_column(tmp_path): import sqlite3 + db = tmp_path / "old.sqlite" with sqlite3.connect(db) as connection: connection.execute( @@ -1371,8 +1582,14 @@ def test_an_old_history_database_gains_the_column(tmp_path): " started_at REAL, finished_at REAL, arguments TEXT, spec TEXT, manifest TEXT," " warnings TEXT, error TEXT)" ) - connection.execute("INSERT INTO jobs (id, workflow, status) VALUES ('old1', 'sd', 'finished')") - manager = JobManager(str(tmp_path / "outputs"), worker_manager=ScriptedWorkerManager(success_script), history_path=str(db)) + connection.execute( + "INSERT INTO jobs (id, workflow, status) VALUES ('old1', 'sd', 'finished')" + ) + manager = JobManager( + str(tmp_path / "outputs"), + worker_manager=ScriptedWorkerManager(success_script), + history_path=str(db), + ) assert manager.get("old1")["workflow_name"] is None ``` @@ -1412,9 +1629,9 @@ Migration — after the `workspace` block in `JobHistory.__init__`: `dw/server/app.py` `submit_job` — add to the `manager.submit(` call: ```python - # The listing name, when the request came as one - what a - # later runtime-by-workflow report joins on - catalog_name=request.workflow_path if source else None, +# The listing name, when the request came as one - what a +# later runtime-by-workflow report joins on +catalog_name = (request.workflow_path if source else None,) ``` Check `rerun` passes `spec` through to `submit(**spec, ...)`; if it enumerates keyword arguments instead, add `catalog_name=spec.get("catalog_name")`. @@ -1460,13 +1677,31 @@ def load(path): EXPECTED_SHAPES = { "workflows/templates/text-to-image.json": ("image", []), "workflows/templates/minimax/storyboard.json": ("shot", ["identity-referenced"]), - "workflows/templates/minimax/dialogue-short.json": ("sequence", ["identity-referenced", "speech"]), - "workflows/templates/minimax/music-video.json": ("sequence", ["identity-referenced", "speech"]), - "workflows/templates/minimax/chained-segments.json": ("shot", ["chained", "image-conditioned", "needs-input-media", "speech"]), - "workflows/templates/ltx2/chained-segments.json": ("shot", ["chained", "image-conditioned", "needs-input-media", "speech"]), + "workflows/templates/minimax/dialogue-short.json": ( + "sequence", + ["identity-referenced", "speech"], + ), + "workflows/templates/minimax/music-video.json": ( + "sequence", + ["identity-referenced", "speech"], + ), + "workflows/templates/minimax/chained-segments.json": ( + "shot", + ["chained", "image-conditioned", "needs-input-media", "speech"], + ), + "workflows/templates/ltx2/chained-segments.json": ( + "shot", + ["chained", "image-conditioned", "needs-input-media", "speech"], + ), "workflows/templates/image-variation.json": ("image-edit", ["needs-input-media"]), - "workflows/templates/segment-and-inpaint.json": ("image-edit", ["needs-input-media"]), - "workflows/templates/describe-and-regenerate.json": ("image-set", ["composes-workflows", "needs-input-media"]), + "workflows/templates/segment-and-inpaint.json": ( + "image-edit", + ["needs-input-media"], + ), + "workflows/templates/describe-and-regenerate.json": ( + "image-set", + ["composes-workflows", "needs-input-media"], + ), "workflows/templates/compose-workflows.json": ("image-set", ["composes-workflows"]), "workflows/templates/generate-speech.json": ("audio", ["speech"]), "workflows/templates/assemble-and-score.json": ("sequence", ["needs-input-media"]), @@ -1495,9 +1730,13 @@ def test_the_rules_read_these_templates_as_expected(path, expected): def test_no_template_falls_through_to_utility(path): meta = derive_catalog_metadata(load(path)) if meta["shape"] == "utility": - assert path in UTILITIES, f"{path} derived 'utility' - a rule missed it, or add it to UTILITIES" + assert path in UTILITIES, ( + f"{path} derived 'utility' - a rule missed it, or add it to UTILITIES" + ) else: - assert path not in UTILITIES, f"{path} is listed as a utility but derives {meta['shape']}" + assert path not in UTILITIES, ( + f"{path} is listed as a utility but derives {meta['shape']}" + ) @pytest.mark.parametrize("path", TEMPLATES + MODEL_CONFIGS) @@ -1506,10 +1745,14 @@ def test_a_declaration_must_differ_from_the_derivation(path): rules or the file change. Declare only what derivation gets wrong.""" definition = load(path) meta = derive_catalog_metadata(definition) - stripped = {k: v for k, v in definition.items() if k not in ("shape", "traits", "summary")} + stripped = { + k: v for k, v in definition.items() if k not in ("shape", "traits", "summary") + } derived = derive_catalog_metadata(stripped) for key in meta["declared"]: - assert meta[key] != derived[key], f"{path} declares {key}={meta[key]!r}, which derivation already produces" + assert meta[key] != derived[key], ( + f"{path} declares {key}={meta[key]!r}, which derivation already produces" + ) @pytest.mark.parametrize("path", TEMPLATES) @@ -1580,7 +1823,9 @@ def test_workflow_ids_are_unique_across_the_catalog(): for path in TEMPLATES + MODEL_CONFIGS + BUILTINS: identity = load(path).get("id") assert identity, f"{path} has no id" - assert identity not in seen, f"{path} and {seen[identity]} share id {identity!r}" + assert identity not in seen, ( + f"{path} and {seen[identity]} share id {identity!r}" + ) seen[identity] = path @@ -1589,11 +1834,17 @@ def test_a_declared_cost_is_well_formed(path): cost = load(path).get("cost") if cost is None: return - assert isinstance(cost, list) and cost, f"{path}: cost must be a non-empty list or absent" + assert isinstance(cost, list) and cost, ( + f"{path}: cost must be a non-empty list or absent" + ) for entry in cost: assert entry["device"] in ("cuda", "mps", "cpu"), path - assert isinstance(entry["vram_gb"], (int, float)) and entry["vram_gb"] >= 0, path - assert isinstance(entry["minutes"], (int, float)) and entry["minutes"] >= 0, path + assert isinstance(entry["vram_gb"], (int, float)) and entry["vram_gb"] >= 0, ( + path + ) + assert isinstance(entry["minutes"], (int, float)) and entry["minutes"] >= 0, ( + path + ) BACKTICKED = re.compile(r"`([a-z_][a-z0-9_]*)`") @@ -1617,7 +1868,9 @@ def test_a_description_names_only_variables_the_workflow_declares(path): catalog_variables = _variable_names_in_catalog() mentioned = set(BACKTICKED.findall(definition.get("description", ""))) undeclared = (mentioned & catalog_variables) - declared - assert not undeclared, f"{path} describes {sorted(undeclared)} but declares no such variable" + assert not undeclared, ( + f"{path} describes {sorted(undeclared)} but declares no such variable" + ) ``` - [ ] **Step 2: Run and resolve** @@ -1669,15 +1922,21 @@ def _tokens(payload): def test_the_compact_listing_fits_the_budget(): - found = listing([WorkflowSource(os.path.join(REPO_ROOT, "workflows"), "workspace", True)]) + found = listing( + [WorkflowSource(os.path.join(REPO_ROOT, "workflows"), "workspace", True)] + ) details = workflow_details(found) compact = project_listing(details, view="compact") - assert _tokens(compact) <= COMPACT_BUDGET, f"compact listing is {_tokens(compact):.0f} tokens" + assert _tokens(compact) <= COMPACT_BUDGET, ( + f"compact listing is {_tokens(compact):.0f} tokens" + ) sequences = project_listing(details, view="compact", shape="sequence") assert sequences, "no template derives 'sequence'" - assert _tokens(sequences) <= FILTERED_BUDGET, f"shape=sequence is {_tokens(sequences):.0f} tokens" + assert _tokens(sequences) <= FILTERED_BUDGET, ( + f"shape=sequence is {_tokens(sequences):.0f} tokens" + ) ``` - [ ] **Step 2: Run** diff --git a/docs/superpowers/plans/2026-09-06-workflow-catalog-restructure.md b/docs/superpowers/plans/2026-09-06-workflow-catalog-restructure.md index b8d833e3..c290be38 100644 --- a/docs/superpowers/plans/2026-09-06-workflow-catalog-restructure.md +++ b/docs/superpowers/plans/2026-09-06-workflow-catalog-restructure.md @@ -381,9 +381,7 @@ def test_every_model_config_names_the_template_it_configures(path): definition = json.load(open(path, encoding="utf-8")) configures = definition.get("configures", "") - assert configures, ( - f"{os.path.relpath(path, REPO_ROOT)} has no 'configures'" - ) + assert configures, f"{os.path.relpath(path, REPO_ROOT)} has no 'configures'" target = os.path.join(REPO_ROOT, "workflows", f"{configures}.json") assert os.path.isfile(target), ( f"{os.path.relpath(path, REPO_ROOT)} configures '{configures}', " diff --git a/docs/superpowers/plans/2026-09-07-dw-plugin-skills.md b/docs/superpowers/plans/2026-09-07-dw-plugin-skills.md index 4d7563eb..c81266ef 100644 --- a/docs/superpowers/plans/2026-09-07-dw-plugin-skills.md +++ b/docs/superpowers/plans/2026-09-07-dw-plugin-skills.md @@ -109,7 +109,12 @@ def _pyproject_version(): def test_the_marketplace_names_the_plugin(): import json - manifest = json.load(open(os.path.join(REPO_ROOT, ".claude-plugin", "marketplace.json"), encoding="utf-8")) + manifest = json.load( + open( + os.path.join(REPO_ROOT, ".claude-plugin", "marketplace.json"), + encoding="utf-8", + ) + ) assert manifest["name"] == "diffusers-workflow" (plugin,) = manifest["plugins"] @@ -122,7 +127,11 @@ def test_the_plugin_version_is_the_engine_version(): this number, so the release script bumps both in one commit.""" import json - plugin = json.load(open(os.path.join(PLUGIN_DIR, ".claude-plugin", "plugin.json"), encoding="utf-8")) + plugin = json.load( + open( + os.path.join(PLUGIN_DIR, ".claude-plugin", "plugin.json"), encoding="utf-8" + ) + ) assert plugin["name"] == "dw" assert plugin["version"] == _pyproject_version() @@ -132,25 +141,35 @@ def test_there_are_skills(): assert SKILLS, "the plugin ships at least one skill" -@pytest.mark.parametrize("path", SKILLS, ids=lambda p: os.path.basename(os.path.dirname(p))) +@pytest.mark.parametrize( + "path", SKILLS, ids=lambda p: os.path.basename(os.path.dirname(p)) +) def test_a_skill_has_a_triggering_description_under_the_size_cap(path): text = skill_text(path) fields = frontmatter(text) assert fields["name"] == os.path.basename(os.path.dirname(path)) assert "description" in fields and len(fields["description"]) > 40 - assert len(text.encode("utf-8")) <= SKILL_SIZE_LIMIT, f"{path} is over {SKILL_SIZE_LIMIT} bytes" + assert len(text.encode("utf-8")) <= SKILL_SIZE_LIMIT, ( + f"{path} is over {SKILL_SIZE_LIMIT} bytes" + ) -@pytest.mark.parametrize("path", SKILLS, ids=lambda p: os.path.basename(os.path.dirname(p))) +@pytest.mark.parametrize( + "path", SKILLS, ids=lambda p: os.path.basename(os.path.dirname(p)) +) def test_every_catalog_name_a_skill_quotes_resolves(path): """A renamed template fails here rather than in a cold session.""" names = CATALOG_NAME.findall(skill_text(path)) assert names, f"{path} quotes no catalog names" for name in names: - target = os.path.join(REPO_ROOT, "workflows", name.removesuffix(".json") + ".json") - assert os.path.isfile(target), f"{path} quotes {name}, which is not a workflow ({target})" + target = os.path.join( + REPO_ROOT, "workflows", name.removesuffix(".json") + ".json" + ) + assert os.path.isfile(target), ( + f"{path} quotes {name}, which is not a workflow ({target})" + ) ``` - [ ] **Step 2: Run it to see it fail** @@ -314,7 +333,10 @@ class TestMiniMaxH3Skill: def test_the_frame_rule_and_bounds_are_the_pipeline_s(self): import inspect - from diffusers.modular_pipelines.minimax_h3 import before_encoder, modular_pipeline + from diffusers.modular_pipelines.minimax_h3 import ( + before_encoder, + modular_pipeline, + ) text = skill_text(H3_SKILL) assert "17n + 5" in text or "17 * n + 5" in text @@ -323,12 +345,18 @@ class TestMiniMaxH3Skill: # 124 and 345 are the smallest and largest 17n + 5 inside 5 to 15 seconds at 24 fps assert "124" in text and "345" in text assert 124 == 17 * 7 + 5 and 345 == 17 * 20 + 5 - assert 124 / modular_pipeline.MINIMAX_H3_FPS >= 5 and 345 / modular_pipeline.MINIMAX_H3_FPS <= 15 + assert ( + 124 / modular_pipeline.MINIMAX_H3_FPS >= 5 + and 345 / modular_pipeline.MINIMAX_H3_FPS <= 15 + ) def test_the_canvas_rules_are_the_pipeline_s(self): import inspect - from diffusers.modular_pipelines.minimax_h3 import before_encoder, modular_pipeline + from diffusers.modular_pipelines.minimax_h3 import ( + before_encoder, + modular_pipeline, + ) text = skill_text(H3_SKILL) source = inspect.getsource(before_encoder) @@ -548,7 +576,10 @@ class TestLtx25Skill: and the one vendor text it quotes is the library's own constant.""" def test_the_schedule_is_the_library_s(self): - from diffusers.pipelines.ltx2.utils import DISTILLED_SIGMA_VALUES, STAGE_2_DISTILLED_SIGMA_VALUES + from diffusers.pipelines.ltx2.utils import ( + DISTILLED_SIGMA_VALUES, + STAGE_2_DISTILLED_SIGMA_VALUES, + ) text = skill_text(LTX_SKILL) assert len(DISTILLED_SIGMA_VALUES) == 8 and "eight" in text @@ -573,9 +604,13 @@ class TestLtx25Skill: ships it so it cannot drift.""" from diffusers.pipelines.ltx2.utils import LTX2_5_T2V_DEFAULT_SYSTEM_PROMPT - quoted = _fenced_block_after(skill_text(LTX_SKILL), "## The trained caption spec") + quoted = _fenced_block_after( + skill_text(LTX_SKILL), "## The trained caption spec" + ) - assert " ".join(quoted.split()) == " ".join(LTX2_5_T2V_DEFAULT_SYSTEM_PROMPT.split()) + assert " ".join(quoted.split()) == " ".join( + LTX2_5_T2V_DEFAULT_SYSTEM_PROMPT.split() + ) def test_the_skill_starts_with_the_server(self): text = skill_text(LTX_SKILL) diff --git a/docs/superpowers/plans/2026-09-07-guides-and-validation.md b/docs/superpowers/plans/2026-09-07-guides-and-validation.md index 43b923d1..ff5b4b3f 100644 --- a/docs/superpowers/plans/2026-09-07-guides-and-validation.md +++ b/docs/superpowers/plans/2026-09-07-guides-and-validation.md @@ -116,9 +116,7 @@ class TestListing: def test_each_guide_carries_a_summary_and_its_sections(self, checkout): # The summary is what an agent matches a request against; the # section headings are the routing table - tasks = next( - g for g in guides.list_guides()["guides"] if g["name"] == "tasks" - ) + tasks = next(g for g in guides.list_guides()["guides"] if g["name"] == "tasks") assert tasks["summary"].strip() assert tasks["file"] == "TASKS.md" @@ -207,9 +205,7 @@ class TestTheRealDocs: assert guides.read_guide(name).strip() def test_the_tasks_guide_indexes_speech_generation(self): - tasks = next( - g for g in guides.list_guides()["guides"] if g["name"] == "tasks" - ) + tasks = next(g for g in guides.list_guides()["guides"] if g["name"] == "tasks") assert "Speech Generation" in tasks["sections"] ``` @@ -479,26 +475,28 @@ from .guides import GuideError Directly after the `/api/schema` route (the `workflow_schema` function, ~line 1005), before `@app.post("/api/validate")`: ```python - # ------------------------------------------------------------ guides - - @app.get("/api/guides") - def list_guides(): - """The documentation that bears on choosing a capability: each - guide's name, what it covers, and its section headings. Served by - the engine rather than read from an MCP client's install, so the - guides an agent reads are the guides for the engine it drives.""" - return guides.list_guides() - - @app.get("/api/guides/{name}") - def get_guide(name: str, section: Optional[str] = None): - """One guide from /api/guides, whole or one section of it. A - section name is matched loosely - case and punctuation dropped - - so a heading copied approximately still resolves. An unknown name - or section is a 404 whose detail lists what exists.""" - try: - return guides.get_guide(name, section=section) - except GuideError as e: - raise HTTPException(status_code=404, detail=str(e)) +# ------------------------------------------------------------ guides + + +@app.get("/api/guides") +def list_guides(): + """The documentation that bears on choosing a capability: each + guide's name, what it covers, and its section headings. Served by + the engine rather than read from an MCP client's install, so the + guides an agent reads are the guides for the engine it drives.""" + return guides.list_guides() + + +@app.get("/api/guides/{name}") +def get_guide(name: str, section: Optional[str] = None): + """One guide from /api/guides, whole or one section of it. A + section name is matched loosely - case and punctuation dropped - + so a heading copied approximately still resolves. An unknown name + or section is a 404 whose detail lists what exists.""" + try: + return guides.get_guide(name, section=section) + except GuideError as e: + raise HTTPException(status_code=404, detail=str(e)) ``` A missing guide *file* (an install with no `dw/docs/`) raises `FileNotFoundError` and surfaces as a 500 — that is a broken install, not a client error, and the log carries the message. @@ -833,7 +831,10 @@ class TestEveryError: _status, message = validate_data(workflow, schema) assert len(errors) == 1 - assert message == f"Validation error at {errors[0]['path']}: {errors[0]['message']}" + assert ( + message + == f"Validation error at {errors[0]['path']}: {errors[0]['message']}" + ) def test_a_valid_definition_yields_no_errors(self): assert validate_data_all(_pipeline_step({}), load_schema("workflow")) == [] @@ -871,7 +872,9 @@ class TestFormatting: assert text == "Validation error at steps[0].seed: 'x' is not of type 'integer'" def test_one_root_error_has_no_location(self): - text = format_validation_errors([{"path": None, "message": "'steps' is a required property"}]) + text = format_validation_errors( + [{"path": None, "message": "'steps' is a required property"}] + ) assert text == "Validation error: 'steps' is a required property" @@ -892,7 +895,10 @@ class TestFormatting: assert text.count("Validation error") == 1 def test_a_capped_list_says_so(self): - errors = [{"path": f"steps[{i}]", "message": "bad"} for i in range(MAX_VALIDATION_ERRORS)] + errors = [ + {"path": f"steps[{i}]", "message": "bad"} + for i in range(MAX_VALIDATION_ERRORS) + ] text = format_validation_errors(errors) @@ -1059,12 +1065,14 @@ def test_validate_endpoint_lists_every_schema_error(server): In `tests/test_mcp_authoring.py`, change the scripted body in `test_validate_returns_an_invalid_verdict_rather_than_raising` to the new shape (the test's assertions stay): ```python - { - "valid": False, - "error": "Validation error: steps must not be empty", - "errors": [{"path": None, "message": "steps must not be empty"}], - "warnings": [], - }, +( + { + "valid": False, + "error": "Validation error: steps must not be empty", + "errors": [{"path": None, "message": "steps must not be empty"}], + "warnings": [], + }, +) ``` - [ ] **Step 2: Run the tests to verify they fail** @@ -1191,34 +1199,51 @@ Co-Authored-By: Claude Fable 5.1 " Append to `TestTheRealDocs` in `tests/test_server_guides.py`: ```python - def test_the_authoring_section_is_reachable_by_name(self): - guide = guides.get_guide("workflows", section="authoring-a-workflow-from-an-agent") - - assert guide["section"] == "Authoring a workflow from an agent" - - def test_the_authoring_section_names_every_reference_prefix(self): - """The prefixes the engine reserves are the ones the section has to - explain; a new prefix added to the engine fails here until it is - written up.""" - from dw.prompts import RESERVED_TEXT_PREFIXES - - content = guides.get_guide( - "workflows", section="Authoring a workflow from an agent" - )["content"] - - for prefix in RESERVED_TEXT_PREFIXES: - assert f"`{prefix}`" in content, prefix - - def test_the_authoring_section_states_the_cartesian_rule_and_the_loop(self): - content = guides.get_guide( - "workflows", section="Authoring a workflow from an agent" - )["content"] - - assert "cartesian" in content.lower() - for tool in ("validate_workflow", "save_workflow", "run_workflow", "wait_for_job", "get_output_image"): - assert f"`{tool}`" in content, tool - for shape in ("image", "image-set", "image-edit", "shot", "sequence", "audio", "text", "utility"): - assert f"`{shape}`" in content, shape +def test_the_authoring_section_is_reachable_by_name(self): + guide = guides.get_guide("workflows", section="authoring-a-workflow-from-an-agent") + + assert guide["section"] == "Authoring a workflow from an agent" + + +def test_the_authoring_section_names_every_reference_prefix(self): + """The prefixes the engine reserves are the ones the section has to + explain; a new prefix added to the engine fails here until it is + written up.""" + from dw.prompts import RESERVED_TEXT_PREFIXES + + content = guides.get_guide( + "workflows", section="Authoring a workflow from an agent" + )["content"] + + for prefix in RESERVED_TEXT_PREFIXES: + assert f"`{prefix}`" in content, prefix + + +def test_the_authoring_section_states_the_cartesian_rule_and_the_loop(self): + content = guides.get_guide( + "workflows", section="Authoring a workflow from an agent" + )["content"] + + assert "cartesian" in content.lower() + for tool in ( + "validate_workflow", + "save_workflow", + "run_workflow", + "wait_for_job", + "get_output_image", + ): + assert f"`{tool}`" in content, tool + for shape in ( + "image", + "image-set", + "image-edit", + "shot", + "sequence", + "audio", + "text", + "utility", + ): + assert f"`{shape}`" in content, shape ``` - [ ] **Step 2: Run the tests to verify they fail** diff --git a/docs/superpowers/plans/2026-09-07-ltx-h3-catalog-repair.md b/docs/superpowers/plans/2026-09-07-ltx-h3-catalog-repair.md index 7673e888..3a0859c9 100644 --- a/docs/superpowers/plans/2026-09-07-ltx-h3-catalog-repair.md +++ b/docs/superpowers/plans/2026-09-07-ltx-h3-catalog-repair.md @@ -81,7 +81,9 @@ class TestLatentHandoff: def _base_result(self): result = Result({"content_type": "video/mp4", "save": False}) result.add_result( - _LatentOutput(frames=torch.zeros(1, 128, 16, 14, 24), audio=torch.zeros(1, 8, 50, 16)) + _LatentOutput( + frames=torch.zeros(1, 128, 16, 14, 24), audio=torch.zeros(1, 8, 50, 16) + ) ) return result @@ -147,7 +149,9 @@ class TestLtxTwoStage: name, so the template carries the literal and this test ties it to the library.""" def _definition(self): - path = os.path.join(REPO_ROOT, "workflows", "templates", "ltx2", "two-stage.json") + path = os.path.join( + REPO_ROOT, "workflows", "templates", "ltx2", "two-stage.json" + ) return json.load(open(path, encoding="utf-8")) def test_the_renoise_scale_is_the_first_stage_two_sigma(self): @@ -155,13 +159,19 @@ class TestLtxTwoStage: refine = _step(self._definition(), "refine") - assert refine["pipeline"]["arguments"]["noise_scale"] == STAGE_2_DISTILLED_SIGMA_VALUES[0] + assert ( + refine["pipeline"]["arguments"]["noise_scale"] + == STAGE_2_DISTILLED_SIGMA_VALUES[0] + ) def test_the_refine_pass_runs_the_stage_two_schedule_on_the_upsampled_latents(self): refine = _step(self._definition(), "refine") arguments = refine["pipeline"]["arguments"] - assert arguments["sigmas"] == "constant:diffusers.pipelines.ltx2.utils.STAGE_2_DISTILLED_SIGMA_VALUES" + assert ( + arguments["sigmas"] + == "constant:diffusers.pipelines.ltx2.utils.STAGE_2_DISTILLED_SIGMA_VALUES" + ) assert arguments["latents"] == "previous_result:upscale.frames" assert arguments["audio_latents"] == "previous_result:base.audio" @@ -597,7 +607,9 @@ def test_a_prompt_is_one_paragraph_of_caption_length(path): assert "\n" not in text.strip(), f"{path} is more than one paragraph" words = len(text.split()) - assert 140 <= words <= 240, f"{path} is {words} words; the trained caption is 150-220" + assert 140 <= words <= 240, ( + f"{path} is {words} words; the trained caption is 150-220" + ) @pytest.mark.parametrize("path", PROMPTS, ids=os.path.basename) @@ -614,7 +626,9 @@ def test_a_prompt_names_the_model_it_is_for(path): def test_no_ltx_template_summary_names_the_older_model(): - templates = glob.glob(os.path.join(REPO_ROOT, "workflows", "templates", "ltx2", "*.json")) + templates = glob.glob( + os.path.join(REPO_ROOT, "workflows", "templates", "ltx2", "*.json") + ) for path in templates: summary = json.load(open(path, encoding="utf-8")).get("summary", "") assert "LTX-2 " not in summary and not summary.endswith("LTX-2"), path @@ -719,7 +733,10 @@ def test_silent_audio_fields_are_written_as_not_applicable(): prompt = _system_prompt() assert "N/A" in prompt - assert "overall_soundscape" in prompt[prompt.index("N/A") - 600 : prompt.index("N/A") + 600] + assert ( + "overall_soundscape" + in prompt[prompt.index("N/A") - 600 : prompt.index("N/A") + 600] + ) def test_video_and_audio_references_are_numbered_within_their_own_category(): @@ -868,7 +885,10 @@ Append to `tests/test_catalog_structure.py`: LINK_PATTERN = re.compile(r"\]\(([^)]+)\)") READMES = sorted( os.path.relpath(path, REPO_ROOT) - for path in glob.glob(os.path.join(REPO_ROOT, "workflows", "templates", "**", "README.md"), recursive=True) + for path in glob.glob( + os.path.join(REPO_ROOT, "workflows", "templates", "**", "README.md"), + recursive=True, + ) ) @@ -884,7 +904,9 @@ def test_every_readme_link_resolves(path): if target.startswith(("http://", "https://", "#")): continue target = target.split("#", 1)[0] - assert os.path.exists(os.path.join(base, target)), f"{path} links to {target}, which does not exist" + assert os.path.exists(os.path.join(base, target)), ( + f"{path} links to {target}, which does not exist" + ) ``` and add `import glob` to the file's imports. diff --git a/docs/superpowers/plans/2026-09-08-job-record-and-export.md b/docs/superpowers/plans/2026-09-08-job-record-and-export.md index f1aa086f..694339df 100644 --- a/docs/superpowers/plans/2026-09-08-job-record-and-export.md +++ b/docs/superpowers/plans/2026-09-08-job-record-and-export.md @@ -149,9 +149,7 @@ def output_root(tmp_path): class TestVariablesAndSeed: def test_arguments_become_the_variable_defaults(self): - realized, _ = realize_workflow( - definition(), {"prompt": "a cat", "steps": 4}, 7 - ) + realized, _ = realize_workflow(definition(), {"prompt": "a cat", "steps": 4}, 7) assert realized["variables"] == {"prompt": "a cat", "steps": 4} def test_variable_references_are_left_alone(self): @@ -309,9 +307,7 @@ def test_the_realized_file_validates_against_the_schema(prompt_library): source = definition() source["steps"][0]["pipeline"]["arguments"]["prompt"] = "prompt:scenic/dusk" - realized, _ = realize_workflow( - source, {"steps": 4}, 991, prompt_dir=prompt_library - ) + realized, _ = realize_workflow(source, {"steps": 4}, 991, prompt_dir=prompt_library) ok, message = validate_data(realized, load_schema("workflow")) assert ok, message @@ -408,9 +404,7 @@ def realize_workflow( realized["seed"] = seed realized = _pin(realized, annotations, base_dir, prompt_dir, output_root) - _record_sub_workflows( - realized.get("steps"), annotations, base_dir, workflow_dir - ) + _record_sub_workflows(realized.get("steps"), annotations, base_dir, workflow_dir) return realized, annotations @@ -435,8 +429,7 @@ def _pin(value, annotations, base_dir, prompt_dir, output_root): } if isinstance(value, list): return [ - _pin(item, annotations, base_dir, prompt_dir, output_root) - for item in value + _pin(item, annotations, base_dir, prompt_dir, output_root) for item in value ] return value @@ -738,27 +731,25 @@ Then, immediately after the `if not self._run_dir_inherited:` block that sets between steps`), insert: ```python - # The record of what actually ran, written before the first step - # so a crash or a cancel still leaves it. A sub-workflow inherits - # the parent's directory and writes none of its own, as with the - # manifest, and the flat layout has no directory to write into - if self._run_dir and not self._run_dir_inherited: - try: - realized, annotations = realize_workflow( - self.workflow_definition, - arguments, - default_seed, - base_dir=base_dir, - output_root=self.output_dir, - workflow_dir=self.workflow_dir, - ) - if write_realized_workflow(self._run_dir, realized): - realized_name = REALIZED_FILE_NAME - except Exception as e: - # Never fatal: the record is worth less than the run - logger.warning( - f"Could not realize workflow {workflow_id}: {e}" - ) +# The record of what actually ran, written before the first step +# so a crash or a cancel still leaves it. A sub-workflow inherits +# the parent's directory and writes none of its own, as with the +# manifest, and the flat layout has no directory to write into +if self._run_dir and not self._run_dir_inherited: + try: + realized, annotations = realize_workflow( + self.workflow_definition, + arguments, + default_seed, + base_dir=base_dir, + output_root=self.output_dir, + workflow_dir=self.workflow_dir, + ) + if write_realized_workflow(self._run_dir, realized): + realized_name = REALIZED_FILE_NAME + except Exception as e: + # Never fatal: the record is worth less than the run + logger.warning(f"Could not realize workflow {workflow_id}: {e}") ``` Note the arguments: `self.workflow_definition` (the original, before `run`'s @@ -1028,17 +1019,15 @@ In `dw/workflow.py`, immediately after the realization block added in Task 2 indentation as the `try:` it contains), add: ```python - # Which run this is, so a server job can find the directory - # it wrote. Emitted even when the realized file did not land: - # the manifest is still there, and so are the files - run_context.emit( - "run_start", - run_id=run_id, - identity=workflow_identity(self.file_spec, workflow_id), - run_dir=os.path.relpath( - self._run_dir, self.output_dir - ).replace(os.sep, "/"), - ) +# Which run this is, so a server job can find the directory +# it wrote. Emitted even when the realized file did not land: +# the manifest is still there, and so are the files +run_context.emit( + "run_start", + run_id=run_id, + identity=workflow_identity(self.file_spec, workflow_id), + run_dir=os.path.relpath(self._run_dir, self.output_dir).replace(os.sep, "/"), +) ``` The worker already forwards every emitted event as a `progress` message, so no @@ -1075,17 +1064,18 @@ In `JobHistory.__init__`, after the `workflow_name` migration: In `record`, extend the column list, the placeholders and the values tuple: ```python - "INSERT OR REPLACE INTO jobs (id, workflow, status, created_at," - " started_at, finished_at, arguments, spec, manifest, warnings," - " error, events, workspace, workflow_name, run_id, run_dir) VALUES" - " (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", +"INSERT OR REPLACE INTO jobs (id, workflow, status, created_at," + +" started_at, finished_at, arguments, spec, manifest, warnings," +" error, events, workspace, workflow_name, run_id, run_dir) VALUES" +(" (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",) ``` and, after `job.catalog_name` in the tuple: ```python - job.run_id, - job.run_dir, +(job.run_id,) +(job.run_dir,) ``` In `recent_summaries`, add `run_id` to the SELECT and the dict, so a live @@ -1107,9 +1097,10 @@ summary and a historical one keep the same shape: In `get`, extend the SELECT: ```python - "SELECT id, workflow, status, created_at, started_at, finished_at," - " arguments, spec, manifest, warnings, error, workspace," - " workflow_name, run_id, run_dir FROM jobs WHERE id = ?", +"SELECT id, workflow, status, created_at, started_at, finished_at," + +" arguments, spec, manifest, warnings, error, workspace," +(" workflow_name, run_id, run_dir FROM jobs WHERE id = ?",) ``` and in `_to_detail`, after `"workflow_name": row[12],`: @@ -1153,40 +1144,36 @@ branch, between building `event` and `job.add_event(event)`: And add `realized` beside `definition`: ```python - def realized(self, job_id): - """The realized workflow a job ran, or None when the job predates - run tracking or its run directory no longer holds the file. - - Read from the job's own output directory, not the manager's: one - server holds several workspaces, and a job carries the root it ran - against. The join is confined to that root, so a run_dir read back - out of the database cannot name anything outside it. - """ - job = self.jobs.get(job_id) - if job is not None: - run_dir = job.run_dir - output_dir = job.spec.get("output_dir") or self.output_dir - else: - historical = self.history.get(job_id) - if historical is None: - return None - run_dir = historical.get("run_dir") - output_dir = ( - historical.get("spec") or {} - ).get("output_dir") or self.output_dir - if not run_dir: - return None - try: - root = validate_output_path(output_dir, None) - path = validate_path( - os.path.join(root, run_dir, REALIZED_FILE_NAME), root - ) - validate_json_size(path) - with open(path, "r") as file: - return json.load(file) - except (SecurityError, OSError, ValueError) as e: - logger.debug(f"No realized workflow for job {job_id}: {e}") +def realized(self, job_id): + """The realized workflow a job ran, or None when the job predates + run tracking or its run directory no longer holds the file. + + Read from the job's own output directory, not the manager's: one + server holds several workspaces, and a job carries the root it ran + against. The join is confined to that root, so a run_dir read back + out of the database cannot name anything outside it. + """ + job = self.jobs.get(job_id) + if job is not None: + run_dir = job.run_dir + output_dir = job.spec.get("output_dir") or self.output_dir + else: + historical = self.history.get(job_id) + if historical is None: return None + run_dir = historical.get("run_dir") + output_dir = (historical.get("spec") or {}).get("output_dir") or self.output_dir + if not run_dir: + return None + try: + root = validate_output_path(output_dir, None) + path = validate_path(os.path.join(root, run_dir, REALIZED_FILE_NAME), root) + validate_json_size(path) + with open(path, "r") as file: + return json.load(file) + except (SecurityError, OSError, ValueError) as e: + logger.debug(f"No realized workflow for job {job_id}: {e}") + return None ``` - [ ] **Step 5: Add the flag to the route** @@ -1510,9 +1497,7 @@ def exporting_script(command): "type": "success", "message": "ok", "run_count": 1, - "manifest": [ - {"step": "gen", "files": [os.path.join(run_dir, "still.png")]} - ], + "manifest": [{"step": "gen", "files": [os.path.join(run_dir, "still.png")]}], } @@ -1577,9 +1562,7 @@ class TestExportDirectory: job_id = finished(client) body = client.post(f"/api/jobs/{job_id}/export").json() - recorded = json.loads( - open(os.path.join(body["directory"], "job.json")).read() - ) + recorded = json.loads(open(os.path.join(body["directory"], "job.json")).read()) assert recorded["realized"] is True assert "traceback" not in recorded and "event_count" not in recorded assert body["workflow"]["seed"] == 7 @@ -1659,7 +1642,9 @@ class TestReservedName: with server() as client: job_id = finished(client) client.post(f"/api/jobs/{job_id}/export") - names = [w["name"] for w in client.get("/api/workspaces").json()["workspaces"]] + names = [ + w["name"] for w in client.get("/api/workspaces").json()["workspaces"] + ] assert names == ["default"] ``` @@ -2213,46 +2198,44 @@ Add beside the `/outputs` route (ungated for the same reason it is: the auth middleware only gates `/api/`): ```python - @app.get("/exports/{job_id}.zip") - def export_zip(job_id: str, ws: Workspace = Depends(selected_workspace)): - """One job's export as a zip, built on request from the directory - rather than kept as a second copy. Entries are named - '/', so unzipping anywhere gives the same tree - the server holds.""" - try: - directory = export_directory(ws.root, job_id) - except SecurityError: - raise HTTPException(status_code=404, detail="No export for this job") - if not os.path.isdir(directory): - raise HTTPException(status_code=404, detail="No export for this job") - - handle = tempfile.NamedTemporaryFile(suffix=".zip", delete=False) - handle.close() - with zipfile.ZipFile(handle.name, "w", zipfile.ZIP_DEFLATED) as archive: - for current, _dirs, names in os.walk(directory): - for name in sorted(names): - path = os.path.join(current, name) - entry = os.path.relpath(path, directory).replace(os.sep, "/") - archive.write(path, f"{job_id}/{entry}") - - def stream(): - with open(handle.name, "rb") as file: - while True: - chunk = file.read(64 * 1024) - if not chunk: - return - yield chunk - - return StreamingResponse( - stream(), - media_type="application/zip", - headers={ - "content-disposition": f'attachment; filename="{job_id}.zip"' - }, - # The archive is a temp file, not a second permanent copy - it - # goes as soon as the response has been sent - background=BackgroundTask(os.unlink, handle.name), - ) +@app.get("/exports/{job_id}.zip") +def export_zip(job_id: str, ws: Workspace = Depends(selected_workspace)): + """One job's export as a zip, built on request from the directory + rather than kept as a second copy. Entries are named + '/', so unzipping anywhere gives the same tree + the server holds.""" + try: + directory = export_directory(ws.root, job_id) + except SecurityError: + raise HTTPException(status_code=404, detail="No export for this job") + if not os.path.isdir(directory): + raise HTTPException(status_code=404, detail="No export for this job") + + handle = tempfile.NamedTemporaryFile(suffix=".zip", delete=False) + handle.close() + with zipfile.ZipFile(handle.name, "w", zipfile.ZIP_DEFLATED) as archive: + for current, _dirs, names in os.walk(directory): + for name in sorted(names): + path = os.path.join(current, name) + entry = os.path.relpath(path, directory).replace(os.sep, "/") + archive.write(path, f"{job_id}/{entry}") + + def stream(): + with open(handle.name, "rb") as file: + while True: + chunk = file.read(64 * 1024) + if not chunk: + return + yield chunk + + return StreamingResponse( + stream(), + media_type="application/zip", + headers={"content-disposition": f'attachment; filename="{job_id}.zip"'}, + # The archive is a temp file, not a second permanent copy - it + # goes as soon as the response has been sent + background=BackgroundTask(os.unlink, handle.name), + ) ``` - [ ] **Step 6: Run the tests to verify they pass** @@ -2355,9 +2338,7 @@ def scripted(routes): def exporting(status=201, body=None): - return scripted( - {("POST", "/api/jobs/job-1/export"): (status, body or SUMMARY)} - ) + return scripted({("POST", "/api/jobs/job-1/export"): (status, body or SUMMARY)}) def test_it_returns_the_directory_the_zip_and_the_file_list(): diff --git a/docs/superpowers/plans/2026-09-10-mcp-field-report-followups.md b/docs/superpowers/plans/2026-09-10-mcp-field-report-followups.md index 5e21c69c..78f52638 100644 --- a/docs/superpowers/plans/2026-09-10-mcp-field-report-followups.md +++ b/docs/superpowers/plans/2026-09-10-mcp-field-report-followups.md @@ -338,19 +338,20 @@ git commit -m "run_workflow/validate_workflow: pin one call to a named workspace In `tests/test_mcp_workspaces.py` `TestLifecycle`, add beside `test_creating_one_does_not_switch_to_it`: ```python - def test_creating_one_says_it_did_not_switch(self): - """A create-then-run sequence landed a five-shot job in the wrong - workspace; the result now says where the session still is.""" - client, _ = recording({"name": "shots"}) - result = create_workspace(client, "shots") - assert result["current"] == DEFAULT_WORKSPACE - assert "use_workspace" in result["next"] - - def test_creating_with_use_switches_to_it(self): - client, seen = recording({"name": "shots"}) - result = create_workspace(client, "shots", use=True) - assert client.workspace == "shots" - assert result["current"] == "shots" +def test_creating_one_says_it_did_not_switch(self): + """A create-then-run sequence landed a five-shot job in the wrong + workspace; the result now says where the session still is.""" + client, _ = recording({"name": "shots"}) + result = create_workspace(client, "shots") + assert result["current"] == DEFAULT_WORKSPACE + assert "use_workspace" in result["next"] + + +def test_creating_with_use_switches_to_it(self): + client, seen = recording({"name": "shots"}) + result = create_workspace(client, "shots", use=True) + assert client.workspace == "shots" + assert result["current"] == "shots" ``` `recording` scripts one response for every request; `use_workspace` GETs the listing and checks the name is in it. If the recorded body `{"name": "shots"}` makes `use_workspace` raise "No workspace named", script the listing instead: look at how `test_deleting_the_current_one_falls_back_to_the_default` uses `listing("default", "shots")` and use that body for the `use=True` test. @@ -791,26 +792,24 @@ Expected: FAIL with `KeyError: 'media'`. In `dw/server/app.py`, import `from dw.media_info import probe_media` beside the other `dw.` imports, and change `gallery_metadata`: ```python - @app.get("/api/gallery/{name:path}/metadata") - def gallery_metadata(name: str, ws: Workspace = Depends(selected_workspace)): - """Generation metadata embedded in a saved image ('workflow' inside - it is the full definition the editor can reopen), plus the job that - produced the file when history remembers one, plus - for audio and - video - what the file itself holds: duration, format and level, - which is how an agent that cannot listen checks a track.""" - path = _output_file(name, ws.outputs) - metadata = read_embedded_metadata(path) - try: - job = manager.history.job_for_file(name, workspace=ws.name) - except Exception: - job = None - extension = os.path.splitext(path)[1].lower() - media = ( - probe_media(path) - if MEDIA_KINDS.get(extension) in ("audio", "video") - else None - ) - return {"name": name, "metadata": metadata, "job": job, "media": media} +@app.get("/api/gallery/{name:path}/metadata") +def gallery_metadata(name: str, ws: Workspace = Depends(selected_workspace)): + """Generation metadata embedded in a saved image ('workflow' inside + it is the full definition the editor can reopen), plus the job that + produced the file when history remembers one, plus - for audio and + video - what the file itself holds: duration, format and level, + which is how an agent that cannot listen checks a track.""" + path = _output_file(name, ws.outputs) + metadata = read_embedded_metadata(path) + try: + job = manager.history.job_for_file(name, workspace=ws.name) + except Exception: + job = None + extension = os.path.splitext(path)[1].lower() + media = ( + probe_media(path) if MEDIA_KINDS.get(extension) in ("audio", "video") else None + ) + return {"name": name, "metadata": metadata, "job": job, "media": media} ``` Keep the existing "Scoped to this workspace" comment above the `job_for_file` call. diff --git a/docs/superpowers/plans/2026-09-11-list-driven-stage-3.md b/docs/superpowers/plans/2026-09-11-list-driven-stage-3.md index 25708a89..0b54d3ab 100644 --- a/docs/superpowers/plans/2026-09-11-list-driven-stage-3.md +++ b/docs/superpowers/plans/2026-09-11-list-driven-stage-3.md @@ -75,17 +75,25 @@ class TestListFields: def test_a_bare_item_means_the_entry_is_a_value(self): fields = list_fields( definition( - {"name": "say", "for_each": "variable:lines", "task": {"arguments": {"text": "item:"}}} + { + "name": "say", + "for_each": "variable:lines", + "task": {"arguments": {"text": "item:"}}, + } ) ) assert fields == {"lines": {"fields": None, "steps": ["say"]}} def test_a_literal_list_is_not_an_argument(self): - assert list_fields(definition({"name": "s", "for_each": ["a"], "task": {}})) == {} + assert ( + list_fields(definition({"name": "s", "for_each": ["a"], "task": {}})) == {} + ) def test_a_step_reading_no_field_still_lists_name(self): fields = list_fields( - definition({"name": "s", "for_each": "variable:xs", "task": {"arguments": {}}}) + definition( + {"name": "s", "for_each": "variable:xs", "task": {"arguments": {}}} + ) ) assert fields == {"xs": {"fields": ["name"], "steps": ["s"]}} @@ -192,7 +200,9 @@ def for_each_step(name, variable, arguments): def test_lists_name_the_fields_an_entry_takes_and_the_default_length(): meta = derive_catalog_metadata( definition( - for_each_step("shot", "shots", {"prompt": "item:prompt", "n": "item:num_frames"}), + for_each_step( + "shot", "shots", {"prompt": "item:prompt", "n": "item:num_frames"} + ), variables={"shots": [{"name": "a", "prompt": "p", "num_frames": 1}] * 3}, ) ) @@ -206,7 +216,10 @@ def test_lists_name_the_fields_an_entry_takes_and_the_default_length(): def test_lists_is_empty_without_for_each_and_entries_is_none_without_a_list_default(): - assert derive_catalog_metadata(definition(pipeline_step("g", "image/jpeg")))["lists"] == {} + assert ( + derive_catalog_metadata(definition(pipeline_step("g", "image/jpeg")))["lists"] + == {} + ) meta = derive_catalog_metadata( definition(for_each_step("shot", "shots", {"prompt": "item:prompt"})) ) @@ -218,7 +231,9 @@ def test_compact_carries_lists_only_when_there_are_any(): "plain": entry("image"), "cut": entry( "sequence", - lists={"shots": {"fields": ["name", "prompt"], "steps": ["shot"], "entries": 2}}, + lists={ + "shots": {"fields": ["name", "prompt"], "steps": ["shot"], "entries": 2} + }, ), } compact = project_listing(listing, view="compact") @@ -240,7 +255,9 @@ def test_workflow_details_describe_their_lists(server): "cut.json", { "id": "cut", - "variables": {"shots": [{"name": "a", "prompt": "p"}, {"name": "b", "prompt": "q"}]}, + "variables": { + "shots": [{"name": "a", "prompt": "p"}, {"name": "b", "prompt": "q"}] + }, "steps": [ { "name": "shot", @@ -355,7 +372,9 @@ class TestEntryFieldWarnings: def test_a_caller_s_list_is_reported_under_arguments(self): warnings = entry_field_warnings( self.workflow(), - arguments={"shots": [{"name": "a", "prompt": "p", "num_frames": 1, "note": "x"}]}, + arguments={ + "shots": [{"name": "a", "prompt": "p", "num_frames": 1, "note": "x"}] + }, ) assert warnings == [ "arguments.shots[0]: entry 'a' carries 'note', which no step reads; " @@ -368,7 +387,11 @@ class TestEntryFieldWarnings: def test_value_entries_and_clean_entries_warn_about_nothing(self): assert entry_field_warnings(self.workflow()) == [] w = definition( - {"name": "say", "for_each": "variable:lines", "task": {"arguments": {"t": "item:"}}}, + { + "name": "say", + "for_each": "variable:lines", + "task": {"arguments": {"t": "item:"}}, + }, variables={"lines": ["a", "b"]}, ) assert entry_field_warnings(w) == [] @@ -458,11 +481,17 @@ def test_an_entry_key_no_step_reads_is_a_warning_not_an_error(client_and_workspa } response = client.post( "/api/validate", - json={"workflow": workflow, "arguments": {"shots": [{"name": "a", "text": "p", "txt": "q"}]}}, + json={ + "workflow": workflow, + "arguments": {"shots": [{"name": "a", "text": "p", "txt": "q"}]}, + }, ) body = response.json() assert body["valid"] is True - assert any(w.startswith("arguments.shots[0]: entry 'a' carries 'txt'") for w in body["warnings"]) + assert any( + w.startswith("arguments.shots[0]: entry 'a' carries 'txt'") + for w in body["warnings"] + ) ``` Use whatever `task` command the file's other tests use for a no-op step so the schema and the task registry accept it. @@ -523,12 +552,24 @@ def test_a_per_entry_cost_validates_and_a_partial_one_does_not(): base = definition(pipeline_step("g", "image/jpeg")) cost = {"device": "cuda", "vram_gb": 24, "minutes": 42} ok, _ = validate_data( - {**base, "cost": [{**cost, "per_entry": {"variable": "shots", "minutes": 7.2, "entries": 5}}]}, + { + **base, + "cost": [ + { + **cost, + "per_entry": {"variable": "shots", "minutes": 7.2, "entries": 5}, + } + ], + }, schema, ) assert ok bad, message = validate_data( - {**base, "cost": [{**cost, "per_entry": {"variable": "shots", "minutes": 7.2}}]}, schema + { + **base, + "cost": [{**cost, "per_entry": {"variable": "shots", "minutes": 7.2}}], + }, + schema, ) assert not bad and "per_entry" in message ``` @@ -550,7 +591,9 @@ def test_a_per_entry_cost_names_a_list_the_steps_read(path): if per_entry is None: continue lists = list_fields(definition) - assert per_entry["variable"] in lists, f"{path}: {per_entry['variable']} is not a for_each list" + assert per_entry["variable"] in lists, ( + f"{path}: {per_entry['variable']} is not a for_each list" + ) default = (definition.get("variables") or {}).get(per_entry["variable"]) assert isinstance(default, list) and len(default) == per_entry["entries"], path ``` @@ -595,20 +638,21 @@ def test_workflow_variables_preview_inside_list_entries(server): `dw/server/app.py` `get_workflow_variables`: replace the loop with a recursive preview: ```python - def preview(value, path): - if isinstance(value, str) and len(value) > VARIABLE_VALUE_PREVIEW: - truncated.append(path) - return value[:VARIABLE_VALUE_PREVIEW] - if isinstance(value, list): - return [preview(item, f"{path}[{i}]") for i, item in enumerate(value)] - if isinstance(value, dict): - return {key: preview(item, f"{path}.{key}") for key, item in value.items()} - return value - - variables = definition.get("variables") or {} - values, truncated = {}, [] - for variable, value in variables.items(): - values[variable] = value if full else preview(value, variable) +def preview(value, path): + if isinstance(value, str) and len(value) > VARIABLE_VALUE_PREVIEW: + truncated.append(path) + return value[:VARIABLE_VALUE_PREVIEW] + if isinstance(value, list): + return [preview(item, f"{path}[{i}]") for i, item in enumerate(value)] + if isinstance(value, dict): + return {key: preview(item, f"{path}.{key}") for key, item in value.items()} + return value + + +variables = definition.get("variables") or {} +values, truncated = {}, [] +for variable, value in variables.items(): + values[variable] = value if full else preview(value, variable) ``` Update the docstring: "Long strings - a shot's prompt runs to kilobytes, and a list-driven workflow's default list holds several - are cut to their first 200 characters wherever they sit and named in `truncated` (`shots[0].prompt`)". @@ -659,7 +703,10 @@ def test_validation_realizes_a_constant_default_list(tmp_path): definition["variables"]["shots"] = "constant:tests.test_workflow.CONSTANT_SHOTS" workflow = _workflow_from(definition, tmp_path) assert workflow.validation_errors() == [] - assert [s["name"] for s in workflow.expanded_definition()["steps"]][:2] == ["shot@a", "shot@b"] + assert [s["name"] for s in workflow.expanded_definition()["steps"]][:2] == [ + "shot@a", + "shot@b", + ] ``` with `CONSTANT_SHOTS = [{"name": "a", "text": "1"}, {"name": "b", "text": "2"}]` at module level (match `_for_each_workflow`'s entry field name). `validate_constant_name` (`dw/security.py`) must accept a `tests.` module path — check its rules; if it refuses, put the constant on a module it accepts and say which. diff --git a/docs/superpowers/plans/2026-09-11-list-driven-steps.md b/docs/superpowers/plans/2026-09-11-list-driven-steps.md index e92eb789..f13cc3e8 100644 --- a/docs/superpowers/plans/2026-09-11-list-driven-steps.md +++ b/docs/superpowers/plans/2026-09-11-list-driven-steps.md @@ -456,7 +456,8 @@ def _entry_keys(entries, path): else: hint = "" raise ForEachError( - _render_path(path), f"for_each must be a list, got {type(entries).__name__}{hint}" + _render_path(path), + f"for_each must be a list, got {type(entries).__name__}{hint}", ) if len(entries) > MAX_FOR_EACH_ENTRIES: raise ForEachError( @@ -707,7 +708,10 @@ class TestGather: self.group( { "name": "score", - "workflow": {"path": "builtin:x.json", "arguments": {"clips": "gather:shot"}}, + "workflow": { + "path": "builtin:x.json", + "arguments": {"clips": "gather:shot"}, + }, } ) ) @@ -784,7 +788,9 @@ class TestGroupReferences: definition( { "name": "edit", - "task": {"arguments": {"r": [{"from_previous_result": "variable:x"}]}}, + "task": { + "arguments": {"r": [{"from_previous_result": "variable:x"}]} + }, } ) ) @@ -956,8 +962,16 @@ class TestMusicVideoTemplate: template = load_template("music-video.json") today = steps_by_name(template) shots = [ - {"name": "wide_open", "prompt": "variable:shot_1_wide_open", "start_frame": 0}, - {"name": "closeup", "prompt": "variable:shot_2_closeup", "start_frame": 124}, + { + "name": "wide_open", + "prompt": "variable:shot_1_wide_open", + "start_frame": 0, + }, + { + "name": "closeup", + "prompt": "variable:shot_2_closeup", + "start_frame": 124, + }, {"name": "room", "prompt": "variable:shot_3_room", "start_frame": 248}, {"name": "finale", "prompt": "variable:shot_4_finale", "start_frame": 372}, ] @@ -979,8 +993,13 @@ class TestMusicVideoTemplate: expanded = expand_for_each( definition( - today["draw_singer"], today["write_song"], slice_template, - today["soundtrack"], shot_template, edit, today["music_video"], + today["draw_singer"], + today["write_song"], + slice_template, + today["soundtrack"], + shot_template, + edit, + today["music_video"], ) ) got = steps_by_name(expanded) @@ -993,8 +1012,15 @@ class TestMusicVideoTemplate: # Each expanded shot is today's shot (as a full pipeline block) with # the new name and its slice renamed - hand_written = ["shot_1_wide_open", "shot_2_closeup", "shot_3_room", "shot_4_finale"] - for key, old, index in zip(["wide_open", "closeup", "room", "finale"], hand_written, range(1, 5)): + hand_written = [ + "shot_1_wide_open", + "shot_2_closeup", + "shot_3_room", + "shot_4_finale", + ] + for key, old, index in zip( + ["wide_open", "closeup", "room", "finale"], hand_written, range(1, 5) + ): step = today[old] if "pipeline_reference" in step: step = without_pipeline_reference(step, today["shot_1_wide_open"]) @@ -1006,7 +1032,8 @@ class TestMusicVideoTemplate: assert got[f"shot@{key}"] == expected assert got["edit"]["task"]["arguments"]["videos"] == [ - f"previous_result:shot@{k}" for k in ["wide_open", "closeup", "room", "finale"] + f"previous_result:shot@{k}" + for k in ["wide_open", "closeup", "room", "finale"] ] ``` @@ -1040,16 +1067,27 @@ class TestDialogueShortTemplate: ] first = today["shot_1_cold_open"] full = { - old: (step if "pipeline_reference" not in step else without_pipeline_reference(step, first)) + old: ( + step + if "pipeline_reference" not in step + else without_pipeline_reference(step, first) + ) for old, step in today.items() if old.startswith("shot_") } shots = [] for key, old in hand_written: arguments = full[old]["pipeline"]["arguments"] - entry = {"name": key, "prompt": arguments["prompt"], "references": arguments["references"]} + entry = { + "name": key, + "prompt": arguments["prompt"], + "references": arguments["references"], + } # Only the tag shot has its own frame count in the template - if arguments.get("num_frames") != first["pipeline"]["arguments"]["num_frames"]: + if ( + arguments.get("num_frames") + != first["pipeline"]["arguments"]["num_frames"] + ): entry["num_frames"] = arguments["num_frames"] shots.append(entry) @@ -1060,7 +1098,9 @@ class TestDialogueShortTemplate: shot_template["pipeline"]["arguments"]["references"] = "item:references" expanded = expand_for_each( - definition(today["draw_character_a"], today["draw_character_b"], shot_template) + definition( + today["draw_character_a"], today["draw_character_b"], shot_template + ) ) got = steps_by_name(expanded) for key, old in hand_written: @@ -1071,8 +1111,14 @@ class TestDialogueShortTemplate: # what the hand-written shot has too pass got_step = got[f"shot@{key}"] - assert got_step["pipeline"]["arguments"]["prompt"] == expected["pipeline"]["arguments"]["prompt"] - assert got_step["pipeline"]["arguments"]["references"] == expected["pipeline"]["arguments"]["references"] + assert ( + got_step["pipeline"]["arguments"]["prompt"] + == expected["pipeline"]["arguments"]["prompt"] + ) + assert ( + got_step["pipeline"]["arguments"]["references"] + == expected["pipeline"]["arguments"]["references"] + ) ``` Then read `dialogue-short.json` (`python3 -c "import json; d=json.load(open('workflows/templates/minimax/dialogue-short.json')); [print(s['name'], json.dumps(s)[:600]) for s in d['steps']]"`) and extend the entry to carry every argument that differs between shots (the template's `num_frames` vs `tag_num_frames` is one; there may be others), mapping each to an `item:` in `shot_template`. The final assertion should compare the *whole* step: `assert got[f"shot@{key}"] == expected`, with `expected["name"]` set as above. Replace the two field assertions with that once the entry shape is right. @@ -1175,17 +1221,25 @@ Append to `tests/test_workflow.py` (it already imports `Workflow`/`workflow_from def _for_each_workflow(**overrides): definition = { "id": "fe", - "variables": {"shots": [{"name": "a", "text": "A"}, {"name": "b", "text": "B"}]}, + "variables": { + "shots": [{"name": "a", "text": "A"}, {"name": "b", "text": "B"}] + }, "steps": [ { "name": "shot", "for_each": "variable:shots", - "task": {"command": "compose_text", "arguments": {"parts": ["item:text"]}}, + "task": { + "command": "compose_text", + "arguments": {"parts": ["item:text"]}, + }, "result": {"content_type": "text/plain"}, }, { "name": "edit", - "task": {"command": "compose_text", "arguments": {"parts": "gather:shot"}}, + "task": { + "command": "compose_text", + "arguments": {"parts": "gather:shot"}, + }, "result": {"content_type": "text/plain"}, }, ], @@ -1272,42 +1326,43 @@ from .variables import ( Replace `validation_errors`: ```python - def expanded_definition(self, arguments=None): - """The definition as the run will see it: variables substituted - - the caller's `arguments` folded in when they are all good, else the - declared defaults - and every for_each step expanded. - - Raises ForEachError for a for_each that cannot be expanded. A - 'variable:' that names nothing is left in place rather than raised: - validate_workflow already reports that as a warning, and the - reference check is happy to skip a reference it cannot read. - """ - definition = copy.deepcopy(self.workflow_definition) - variables = definition.get("variables") - if isinstance(variables, dict): - if arguments and not argument_errors(definition, arguments): - set_variables(arguments, variables) - try: - definition = replace_variables(definition, variables) - except VariableNotFoundError: - pass - return expand_for_each(definition) - - def validation_errors(self, arguments=None): - """Every schema violation in the definition, as [{path, message}]; - empty when it validates. `arguments` are the caller's, so a - for_each over a list the caller supplies is checked as it will run.""" - errors = validate_data_all(self.workflow_definition, load_schema("workflow")) - # Only once the shape is known good: the passes below walk the - # steps array and a definition that fails the schema may have no - # such array to walk - if errors: - return errors +def expanded_definition(self, arguments=None): + """The definition as the run will see it: variables substituted - + the caller's `arguments` folded in when they are all good, else the + declared defaults - and every for_each step expanded. + + Raises ForEachError for a for_each that cannot be expanded. A + 'variable:' that names nothing is left in place rather than raised: + validate_workflow already reports that as a warning, and the + reference check is happy to skip a reference it cannot read. + """ + definition = copy.deepcopy(self.workflow_definition) + variables = definition.get("variables") + if isinstance(variables, dict): + if arguments and not argument_errors(definition, arguments): + set_variables(arguments, variables) try: - expanded = self.expanded_definition(arguments) - except ForEachError as e: - return [{"path": e.path, "message": str(e)}] - return previous_result_reference_errors(expanded) + definition = replace_variables(definition, variables) + except VariableNotFoundError: + pass + return expand_for_each(definition) + + +def validation_errors(self, arguments=None): + """Every schema violation in the definition, as [{path, message}]; + empty when it validates. `arguments` are the caller's, so a + for_each over a list the caller supplies is checked as it will run.""" + errors = validate_data_all(self.workflow_definition, load_schema("workflow")) + # Only once the shape is known good: the passes below walk the + # steps array and a definition that fails the schema may have no + # such array to walk + if errors: + return errors + try: + expanded = self.expanded_definition(arguments) + except ForEachError as e: + return [{"path": e.path, "message": str(e)}] + return previous_result_reference_errors(expanded) ``` `set_variables` may need `realize_constants` first when a default is `constant:` - it does not for the list case, and `argument_errors` already runs `set_variables` on the raw declared block, so this mirrors it. @@ -1367,12 +1422,18 @@ def test_validate_expands_for_each_with_the_callers_list(client): { "name": "shot", "for_each": "variable:shots", - "task": {"command": "compose_text", "arguments": {"parts": ["item:text"]}}, + "task": { + "command": "compose_text", + "arguments": {"parts": ["item:text"]}, + }, "result": {"content_type": "text/plain"}, }, { "name": "edit", - "task": {"command": "compose_text", "arguments": {"parts": "gather:shot"}}, + "task": { + "command": "compose_text", + "arguments": {"parts": "gather:shot"}, + }, "result": {"content_type": "text/plain"}, }, ], @@ -1382,7 +1443,10 @@ def test_validate_expands_for_each_with_the_callers_list(client): bad = client.post( "/api/validate", - json={"workflow": workflow, "arguments": {"shots": [{"name": "x"}, {"name": "x"}]}}, + json={ + "workflow": workflow, + "arguments": {"shots": [{"name": "x"}, {"name": "x"}]}, + }, ).json() assert bad["valid"] is False assert bad["errors"][0]["path"] == "steps[0].for_each[1].name" diff --git a/docs/superpowers/plans/2026-09-11-list-driven-templates.md b/docs/superpowers/plans/2026-09-11-list-driven-templates.md index 0529c3d1..877111e2 100644 --- a/docs/superpowers/plans/2026-09-11-list-driven-templates.md +++ b/docs/superpowers/plans/2026-09-11-list-driven-templates.md @@ -104,7 +104,9 @@ class TestResolveVariableValues: class TestUndeclaredReferencesInsideVariableValues: def test_a_reference_inside_a_list_value_is_found_with_its_path(self): definition = { - "variables": {"shots": [{"references": [{}, {"from_file": "variable:nope"}]}]}, + "variables": { + "shots": [{"references": [{}, {"from_file": "variable:nope"}]}] + }, "steps": [], } assert undeclared_variable_references(definition) == [ @@ -533,12 +535,17 @@ class TestMusicVideoTemplate: def expanded(self): from dw.workflow import Workflow - return Workflow(load_template("music-video.json"), TEMPLATES).expanded_definition() + return Workflow( + load_template("music-video.json"), TEMPLATES + ).expanded_definition() def test_the_template_validates_as_it_will_run(self): from dw.workflow import Workflow - assert Workflow(load_template("music-video.json"), TEMPLATES).validation_errors() == [] + assert ( + Workflow(load_template("music-video.json"), TEMPLATES).validation_errors() + == [] + ) def test_one_slice_and_one_shot_per_entry_in_list_order(self): names = [s["name"] for s in self.expanded()["steps"]] @@ -552,7 +559,9 @@ class TestMusicVideoTemplate: def test_each_slice_starts_where_its_entry_says(self): got = steps_by_name(self.expanded()) - starts = [got[f"slice@{k}"]["task"]["arguments"]["start_frame"] for k in self.KEYS] + starts = [ + got[f"slice@{k}"]["task"]["arguments"]["start_frame"] for k in self.KEYS + ] assert starts == [0, 124, 248, 372] def test_each_shot_reads_its_own_slice_and_the_one_portrait(self): @@ -592,7 +601,10 @@ class TestMusicVideoTemplate: 'shots' variable, not the expanded members.""" template = load_template("music-video.json") assert "shots" in template["variables"] - assert [s["name"] for s in template["steps"] if "for_each" in s] == ["slice", "shot"] + assert [s["name"] for s in template["steps"] if "for_each" in s] == [ + "slice", + "shot", + ] ``` Check `Workflow.__init__`'s signature (`dw/workflow.py`, `class Workflow`) before writing: if it takes `(workflow_definition, base_dir, ...)` in a different order or by keyword, match it. If the file's `without_pipeline_reference` helper is now unused by any test, delete it. @@ -767,11 +779,11 @@ class TestOptionalVoices: def test_every_entry_names_the_voice_variables_rather_than_a_file(self, definition): for entry in definition["variables"]["shots"]: - voices = [ - r["from_file"] for r in entry["references"] if "from_file" in r - ] + voices = [r["from_file"] for r in entry["references"] if "from_file" in r] assert voices, entry["name"] - assert all(v.startswith("variable:character_") for v in voices), entry["name"] + assert all(v.startswith("variable:character_") for v in voices), entry[ + "name" + ] def test_no_voice_named_leaves_only_the_portraits(self, definition): for name, references in shot_references(definition, {}).items(): @@ -798,20 +810,30 @@ class TestOptionalVoices: def test_the_tag_runs_longer(self, definition): frames = {e["name"]: e["num_frames"] for e in definition["variables"]["shots"]} assert frames == { - "cold_open": 124, "deflect": 124, "react": 124, "button": 124, "tag": 141 + "cold_open": 124, + "deflect": 124, + "react": 124, + "button": 124, + "tag": 141, } def test_the_variable_names_are_roles_rather_than_a_cast(self, definition): """Every run carried howie_portrait_prompt and shot_3_howie_incredulous through its arguments, manifest and export whatever the cast was.""" - names = " ".join(definition["variables"]) + " ".join( - step["name"] for step in definition["steps"] - ) + " ".join(e["name"] for e in definition["variables"]["shots"]) + names = ( + " ".join(definition["variables"]) + + " ".join(step["name"] for step in definition["steps"]) + + " ".join(e["name"] for e in definition["variables"]["shots"]) + ) assert "howie" not in names.lower() assert "pat_" not in names.lower() assert "character_a_portrait_prompt" in definition["variables"] assert [e["name"] for e in definition["variables"]["shots"]] == [ - "cold_open", "deflect", "react", "button", "tag" + "cold_open", + "deflect", + "react", + "button", + "tag", ] ``` @@ -830,12 +852,19 @@ class TestDialogueShortTemplate: def expanded(self): from dw.workflow import Workflow - return Workflow(load_template("dialogue-short.json"), TEMPLATES).expanded_definition() + return Workflow( + load_template("dialogue-short.json"), TEMPLATES + ).expanded_definition() def test_the_template_validates_as_it_will_run(self): from dw.workflow import Workflow - assert Workflow(load_template("dialogue-short.json"), TEMPLATES).validation_errors() == [] + assert ( + Workflow( + load_template("dialogue-short.json"), TEMPLATES + ).validation_errors() + == [] + ) def test_one_shot_per_entry_between_the_cast_and_the_edit(self): names = [s["name"] for s in self.expanded()["steps"]] @@ -873,14 +902,19 @@ class TestDialogueShortTemplate: def test_the_tag_runs_longer(self): got = steps_by_name(self.expanded()) - frames = [got[f"shot@{k}"]["pipeline"]["arguments"]["num_frames"] for k in self.KEYS] + frames = [ + got[f"shot@{k}"]["pipeline"]["arguments"]["num_frames"] for k in self.KEYS + ] assert frames == [124, 124, 124, 124, 141] def test_every_shot_is_the_same_pipeline(self): from dw.workflow import pipeline_cache_key got = steps_by_name(self.expanded()) - assert len({pipeline_cache_key(got[f"shot@{k}"]["pipeline"]) for k in self.KEYS}) == 1 + assert ( + len({pipeline_cache_key(got[f"shot@{k}"]["pipeline"]) for k in self.KEYS}) + == 1 + ) def test_the_episode_gathers_the_shots_in_order(self): got = steps_by_name(self.expanded()) diff --git a/docs/superpowers/plans/2026-09-12-output-subfolders-stage-1-engine.md b/docs/superpowers/plans/2026-09-12-output-subfolders-stage-1-engine.md index dbbc4867..31066ef1 100644 --- a/docs/superpowers/plans/2026-09-12-output-subfolders-stage-1-engine.md +++ b/docs/superpowers/plans/2026-09-12-output-subfolders-stage-1-engine.md @@ -53,39 +53,41 @@ Add to `tests/test_runs.py`, importing `split_run_path` alongside `strip_run_id` in the `from dw.runs import (...)` block, and inside the class that holds `test_stripping_a_run_id_gives_the_workflow_folder`: ```python - def test_splitting_a_run_path_names_its_three_parts(self): - run_id = new_run_id({"id": "x"}) - assert split_run_path(f"ltx2/Gyre/{run_id}/still.png") == ( - "ltx2/Gyre", - run_id, - "", - ) - assert split_run_path(f"ltx2/Gyre/{run_id}/final/still.png") == ( - "ltx2/Gyre", - run_id, - "final", - ) - assert split_run_path(f"ltx2/Gyre/{run_id}/shots/act-1/x.mp4") == ( - "ltx2/Gyre", - run_id, - "shots/act-1", - ) - # A counter suffix is still a run id - assert split_run_path(f"Gyre/{run_id}-2/final/x.mp4") == ( - "Gyre", - f"{run_id}-2", - "final", - ) +def test_splitting_a_run_path_names_its_three_parts(self): + run_id = new_run_id({"id": "x"}) + assert split_run_path(f"ltx2/Gyre/{run_id}/still.png") == ( + "ltx2/Gyre", + run_id, + "", + ) + assert split_run_path(f"ltx2/Gyre/{run_id}/final/still.png") == ( + "ltx2/Gyre", + run_id, + "final", + ) + assert split_run_path(f"ltx2/Gyre/{run_id}/shots/act-1/x.mp4") == ( + "ltx2/Gyre", + run_id, + "shots/act-1", + ) + # A counter suffix is still a run id + assert split_run_path(f"Gyre/{run_id}-2/final/x.mp4") == ( + "Gyre", + f"{run_id}-2", + "final", + ) + - def test_a_path_with_no_run_id_splits_to_its_directory(self): - # Flat layout: nothing to anchor on, so the directory is the identity - assert split_run_path("ltx2/final/still.png") == ("ltx2/final", "", "") - assert split_run_path("still.png") == ("", "", "") +def test_a_path_with_no_run_id_splits_to_its_directory(self): + # Flat layout: nothing to anchor on, so the directory is the identity + assert split_run_path("ltx2/final/still.png") == ("ltx2/final", "", "") + assert split_run_path("still.png") == ("", "", "") - def test_stripping_a_run_id_ignores_what_follows_it(self): - run_id = new_run_id({"id": "x"}) - assert strip_run_id(f"ltx2/Gyre/{run_id}/final/still.png") == "ltx2/Gyre" - assert strip_run_id(f"{run_id}/final/still.png") == "" + +def test_stripping_a_run_id_ignores_what_follows_it(self): + run_id = new_run_id({"id": "x"}) + assert strip_run_id(f"ltx2/Gyre/{run_id}/final/still.png") == "ltx2/Gyre" + assert strip_run_id(f"{run_id}/final/still.png") == "" ``` - [ ] **Step 2: Run the tests to verify they fail** @@ -361,7 +363,9 @@ class TestSubfolderErrors: errors = subfolder_errors(definition) assert len(errors) == 1 assert errors[0]["path"] == "steps[0].result.subfolder" - assert "Subfolder" in errors[0]["message"] or "subfolder" in errors[0]["message"] + assert ( + "Subfolder" in errors[0]["message"] or "subfolder" in errors[0]["message"] + ) def test_a_separator_in_file_base_name_is_reported_at_its_path(self): definition = {"steps": [_step("a", {"file_base_name": "final/"})]} @@ -552,7 +556,10 @@ class TestValidationErrorsIntegration: { "name": "a", "task": {"command": "noop", "arguments": {}}, - "result": {"content_type": "image/png", "subfolder": "variable:dest"}, + "result": { + "content_type": "image/png", + "subfolder": "variable:dest", + }, } ], } @@ -602,9 +609,9 @@ Change the last line of `validation_errors` from: to: ```python - return previous_result_reference_errors( - expanded, source_indices - ) + subfolder_errors(expanded, source_indices) +return previous_result_reference_errors(expanded, source_indices) + subfolder_errors( + expanded, source_indices +) ``` - [ ] **Step 7: Run the tests to verify they pass** @@ -653,7 +660,10 @@ class TestResultSubfolder: { "name": "a", "task": {"command": "noop", "arguments": {}}, - "result": {"content_type": "image/png", "subfolder": "variable:dest"}, + "result": { + "content_type": "image/png", + "subfolder": "variable:dest", + }, } ], } @@ -758,7 +768,9 @@ class TestSubfolders: def test_step_output_dir_is_the_run_directory_without_a_subfolder(self, tmp_path): from dw.workflow import Workflow - workflow = Workflow(_workflow_definition(), str(tmp_path), "/w/workflows/Gyre.json") + workflow = Workflow( + _workflow_definition(), str(tmp_path), "/w/workflows/Gyre.json" + ) workflow._run_dir = str(tmp_path / "Gyre" / "run") step = workflow.workflow_definition["steps"][0] assert workflow.step_output_dir(step) == workflow.effective_output_dir @@ -774,12 +786,16 @@ class TestSubfolders: with pytest.raises(SecurityError): workflow.step_output_dir(workflow.workflow_definition["steps"][0]) - def test_the_pipeline_wrapper_is_pointed_at_the_subfolder(self, tmp_path, fake_pipeline): + def test_the_pipeline_wrapper_is_pointed_at_the_subfolder( + self, tmp_path, fake_pipeline + ): # A chain step's save_segments spill writes through the pipeline's # output_dir, so it has to be the step's directory, not the run's from dw.workflow import Workflow - workflow = Workflow(_foldered_definition(), str(tmp_path), "/w/workflows/Gyre.json") + workflow = Workflow( + _foldered_definition(), str(tmp_path), "/w/workflows/Gyre.json" + ) workflow._run_dir = str(tmp_path / "Gyre" / "run") step = workflow.workflow_definition["steps"][0] action = workflow.create_step_action(step, {}, {}, 7, "cpu") @@ -832,9 +848,7 @@ Confirm `os` is imported in `dw/workflow.py` (it is used by `effective_output_di In the step loop (~line 743), change: ```python - saved_files = result.save( - self.effective_output_dir, f"{workflow_id}-{step.name}.{i}" - ) +saved_files = result.save(self.effective_output_dir, f"{workflow_id}-{step.name}.{i}") ``` to: @@ -849,7 +863,7 @@ to: In `create_step_action`, replace each of the three `output_dir=self.effective_output_dir,` kwargs (the cached-pipeline reuse branch ~962, the fresh `Pipeline(...)` ~1003, and the `pipeline_reference` branch ~1032) with: ```python - output_dir=self.step_output_dir(step_definition), +output_dir = (self.step_output_dir(step_definition),) ``` (`step_definition` is the parameter name in `create_step_action`; keep each line's existing indentation.) Do not touch the `workflow` (sub-workflow) branch: a child's steps compute their own directories against the inherited run directory, and the parent's `subfolder` governs only what the parent saves from the child's return value — which the `result.save` change above already covers. @@ -924,58 +938,53 @@ Co-Authored-By: Claude Opus 5 (1M context) " Add to `TestSubfolders` in `tests/test_runs.py`: ```python - def test_the_manifest_entry_carries_the_subfolder(self, tmp_path, fake_pipeline): - from dw.workflow import Workflow - - Workflow(_foldered_definition(), str(tmp_path), "/w/workflows/Gyre.json").run( - {} - ) - (run,) = (tmp_path / "Gyre").iterdir() - manifest = json.loads((run / "manifest.json").read_text()) - (entry,) = manifest["steps"] - assert entry["subfolder"] == "final" - assert entry["files"] == ["final/runs_test-gen0.0-0.0.png"] - - def test_an_unfoldered_entry_carries_the_empty_string(self, tmp_path, fake_pipeline): - from dw.workflow import Workflow - - Workflow(_workflow_definition(), str(tmp_path), "/w/workflows/Gyre.json").run( - {} - ) - (run,) = (tmp_path / "Gyre").iterdir() - manifest = json.loads((run / "manifest.json").read_text()) - assert manifest["steps"][0]["subfolder"] == "" - - def test_a_reused_entry_carries_the_definitions_subfolder( - self, tmp_path, fake_pipeline - ): - from dw.workflow import Workflow - - Workflow(_foldered_definition(), str(tmp_path), "/w/workflows/Gyre.json").run( - {} - ) - Workflow(_foldered_definition(), str(tmp_path), "/w/workflows/Gyre.json").run( - {} - ) - first, second = sorted((tmp_path / "Gyre").iterdir()) - manifest = json.loads((second / "manifest.json").read_text()) - (entry,) = manifest["steps"] - assert entry["reused"] is True - assert entry["subfolder"] == "final" - # The file is the first run's, absolute, inside its 'final' - assert os.path.dirname(entry["files"][0]) == str(first / "final") - - def test_the_step_end_event_carries_the_subfolder(self, tmp_path, fake_pipeline): - from dw.events import RunContext - from dw.workflow import Workflow - - seen = [] - context = RunContext(on_event=seen.append) - Workflow(_foldered_definition(), str(tmp_path), "/w/workflows/Gyre.json").run( - {}, previous_pipelines={}, context=context - ) - step_ends = [event for event in seen if event.get("event") == "step_end"] - assert step_ends and step_ends[0]["subfolder"] == "final" +def test_the_manifest_entry_carries_the_subfolder(self, tmp_path, fake_pipeline): + from dw.workflow import Workflow + + Workflow(_foldered_definition(), str(tmp_path), "/w/workflows/Gyre.json").run({}) + (run,) = (tmp_path / "Gyre").iterdir() + manifest = json.loads((run / "manifest.json").read_text()) + (entry,) = manifest["steps"] + assert entry["subfolder"] == "final" + assert entry["files"] == ["final/runs_test-gen0.0-0.0.png"] + + +def test_an_unfoldered_entry_carries_the_empty_string(self, tmp_path, fake_pipeline): + from dw.workflow import Workflow + + Workflow(_workflow_definition(), str(tmp_path), "/w/workflows/Gyre.json").run({}) + (run,) = (tmp_path / "Gyre").iterdir() + manifest = json.loads((run / "manifest.json").read_text()) + assert manifest["steps"][0]["subfolder"] == "" + + +def test_a_reused_entry_carries_the_definitions_subfolder( + self, tmp_path, fake_pipeline +): + from dw.workflow import Workflow + + Workflow(_foldered_definition(), str(tmp_path), "/w/workflows/Gyre.json").run({}) + Workflow(_foldered_definition(), str(tmp_path), "/w/workflows/Gyre.json").run({}) + first, second = sorted((tmp_path / "Gyre").iterdir()) + manifest = json.loads((second / "manifest.json").read_text()) + (entry,) = manifest["steps"] + assert entry["reused"] is True + assert entry["subfolder"] == "final" + # The file is the first run's, absolute, inside its 'final' + assert os.path.dirname(entry["files"][0]) == str(first / "final") + + +def test_the_step_end_event_carries_the_subfolder(self, tmp_path, fake_pipeline): + from dw.events import RunContext + from dw.workflow import Workflow + + seen = [] + context = RunContext(on_event=seen.append) + Workflow(_foldered_definition(), str(tmp_path), "/w/workflows/Gyre.json").run( + {}, previous_pipelines={}, context=context + ) + step_ends = [event for event in seen if event.get("event") == "step_end"] + assert step_ends and step_ends[0]["subfolder"] == "final" ``` (`RunContext(on_event=...)` and `run(..., context=context)` is the same setup `tests/test_events.py::_run` uses.) @@ -1056,20 +1065,21 @@ Co-Authored-By: Claude Opus 5 (1M context) " Add to `TestSubfolders` in `tests/test_runs.py` (import `resolve_output_reference` from `dw.runs` in the file's import block if it is not already there): ```python - def test_an_output_reference_reaches_into_a_subfolder(self, tmp_path, fake_pipeline): - from dw.workflow import Workflow +def test_an_output_reference_reaches_into_a_subfolder(self, tmp_path, fake_pipeline): + from dw.workflow import Workflow - Workflow(_foldered_definition(), str(tmp_path), "/w/workflows/Gyre.json").run( - {} - ) - (run,) = (tmp_path / "Gyre").iterdir() - resolved = resolve_output_reference( - "output:Gyre/latest/final/runs_test-gen0.0-0.0.png", root=str(tmp_path) - ) - assert resolved == str(run / "final" / "runs_test-gen0.0-0.0.png") - assert resolve_output_reference( + Workflow(_foldered_definition(), str(tmp_path), "/w/workflows/Gyre.json").run({}) + (run,) = (tmp_path / "Gyre").iterdir() + resolved = resolve_output_reference( + "output:Gyre/latest/final/runs_test-gen0.0-0.0.png", root=str(tmp_path) + ) + assert resolved == str(run / "final" / "runs_test-gen0.0-0.0.png") + assert ( + resolve_output_reference( f"output:Gyre/{run.name}/final/runs_test-gen0.0-0.0.png", root=str(tmp_path) - ) == resolved + ) + == resolved + ) ``` - [ ] **Step 2: Run it** diff --git a/docs/superpowers/plans/2026-09-12-output-subfolders-stage-2-server.md b/docs/superpowers/plans/2026-09-12-output-subfolders-stage-2-server.md index 262cd470..49aa39e8 100644 --- a/docs/superpowers/plans/2026-09-12-output-subfolders-stage-2-server.md +++ b/docs/superpowers/plans/2026-09-12-output-subfolders-stage-2-server.md @@ -84,7 +84,9 @@ def test_gallery_reports_and_filters_by_subfolder(server, tmp_path): final = by_name[f"dialogue/{run_id}/final/dialogue-assemble.0-0.0.png"] assert final["folder"] == "dialogue" assert final["subfolder"] == "final" - nested = by_name[f"dialogue/{run_id}/intermediate/shots/dialogue-slice.0-0.0.png"] + nested = by_name[ + f"dialogue/{run_id}/intermediate/shots/dialogue-slice.0-0.0.png" + ] assert nested["subfolder"] == "intermediate/shots" assert by_name[f"dialogue/{run_id}/dialogue-still.0-0.0.png"]["subfolder"] == "" assert by_name["ltx/flat.png"]["folder"] == "ltx" @@ -115,7 +117,12 @@ def test_job_for_file_attributes_a_file_in_a_subfolder(tmp_path): history = JobHistory(str(tmp_path / "jobs.sqlite")) name = "dialogue/20260912-120000-abcdef01/final/dialogue-assemble.0-0.0.png" - _record(history, "writer", 1.0, [{"step": "assemble", "files": [name], "subfolder": "final"}]) + _record( + history, + "writer", + 1.0, + [{"step": "assemble", "files": [name], "subfolder": "final"}], + ) assert history.job_for_file(name)["id"] == "writer" ``` @@ -499,56 +506,62 @@ No engine change. Both tests pin behaviour that is correct by inspection; if eit Add to `TestSubfolders`: ```python - def test_a_parents_subfolder_does_not_move_a_childs_files( - self, tmp_path, fake_pipeline - ): - # A 'workflow' step's own result block governs what the parent saves - # from the child's return value; the child's steps place their own - # files, into the run directory they inherit - from dw.workflow import Workflow - - tree = tmp_path / "workflows" - tree.mkdir() - (tree / "child.json").write_text(json.dumps(_workflow_definition())) - parent = { - "id": "parent", - "seed": 7, - "steps": [ - { - "name": "child", - "workflow": {"path": "child.json"}, - "result": {"subfolder": "final"}, - } - ], - } - Workflow(parent, str(tmp_path / "out"), str(tree / "Parent.json")).run({}) - (run,) = (tmp_path / "out" / "Parent").iterdir() - assert (run / "runs_test-gen0.0-0.0.png").is_file() - assert not (run / "final" / "runs_test-gen0.0-0.0.png").exists() - - def test_a_chain_spill_lands_in_the_steps_subfolder(self, tmp_path, fake_pipeline): - # save_segments writes through the pipeline wrapper's output_dir, - # which create_step_action points at the step's subfolder - from dw.pipeline_processors.chain import run_chain - from dw.workflow import Workflow - from tests.test_chain import FakePipeline, video_output - - workflow = Workflow(_foldered_definition(), str(tmp_path), "/w/workflows/Gyre.json") - workflow._run_dir = str(tmp_path / "Gyre" / "run") - action = workflow.create_step_action( - workflow.workflow_definition["steps"][0], {}, {}, 7, "cpu" - ) - spilling = FakePipeline( - video_output, output_dir=action.output_dir, file_prefix=action.file_prefix - ) - run_chain( - spilling, - {"segments": 2, "trim_frames": 1, "fps": 4, "save_segments": True, - "keep_segments": True}, - {}, - ) - segments = sorted((tmp_path / "Gyre" / "run" / "final").glob("*.segment-*.mp4")) - assert len(segments) == 2 +def test_a_parents_subfolder_does_not_move_a_childs_files( + self, tmp_path, fake_pipeline +): + # A 'workflow' step's own result block governs what the parent saves + # from the child's return value; the child's steps place their own + # files, into the run directory they inherit + from dw.workflow import Workflow + + tree = tmp_path / "workflows" + tree.mkdir() + (tree / "child.json").write_text(json.dumps(_workflow_definition())) + parent = { + "id": "parent", + "seed": 7, + "steps": [ + { + "name": "child", + "workflow": {"path": "child.json"}, + "result": {"subfolder": "final"}, + } + ], + } + Workflow(parent, str(tmp_path / "out"), str(tree / "Parent.json")).run({}) + (run,) = (tmp_path / "out" / "Parent").iterdir() + assert (run / "runs_test-gen0.0-0.0.png").is_file() + assert not (run / "final" / "runs_test-gen0.0-0.0.png").exists() + + +def test_a_chain_spill_lands_in_the_steps_subfolder(self, tmp_path, fake_pipeline): + # save_segments writes through the pipeline wrapper's output_dir, + # which create_step_action points at the step's subfolder + from dw.pipeline_processors.chain import run_chain + from dw.workflow import Workflow + from tests.test_chain import FakePipeline, video_output + + workflow = Workflow(_foldered_definition(), str(tmp_path), "/w/workflows/Gyre.json") + workflow._run_dir = str(tmp_path / "Gyre" / "run") + action = workflow.create_step_action( + workflow.workflow_definition["steps"][0], {}, {}, 7, "cpu" + ) + spilling = FakePipeline( + video_output, output_dir=action.output_dir, file_prefix=action.file_prefix + ) + run_chain( + spilling, + { + "segments": 2, + "trim_frames": 1, + "fps": 4, + "save_segments": True, + "keep_segments": True, + }, + {}, + ) + segments = sorted((tmp_path / "Gyre" / "run" / "final").glob("*.segment-*.mp4")) + assert len(segments) == 2 ``` Before running, check: (a) that `Workflow` resolves a sub-workflow's `path` relative to the parent's `file_spec` directory (read the `workflow` branch of `create_step_action`); if it needs the parent file to exist, write `parent` to `tree / "Parent.json"` and load it with `workflow_from_file`; (b) the exact child file name — the child's id is `runs_test`, its step `gen0` — and adjust the asserted name to what the run wrote if the workflow-step naming differs, keeping the assertion that it is at the run root and not under `final/`; (c) that `tests/test_chain.py` exports `FakePipeline` and `video_output` at module level and `FakePipeline` accepts `output_dir`/`file_prefix` kwargs (it does in `TestSaveSegments.make_pipeline`); (d) the attribute name the `Pipeline` wrapper stores its file prefix under (`file_prefix` per the constructor kwarg — confirm in `dw/pipeline_processors/pipeline.py`). diff --git a/docs/superpowers/plans/2026-09-12-output-subfolders-stage-3-steering.md b/docs/superpowers/plans/2026-09-12-output-subfolders-stage-3-steering.md index 42c8796b..88782913 100644 --- a/docs/superpowers/plans/2026-09-12-output-subfolders-stage-3-steering.md +++ b/docs/superpowers/plans/2026-09-12-output-subfolders-stage-3-steering.md @@ -131,7 +131,8 @@ def test_the_subfolder_is_the_last_key_of_the_result(template): @pytest.mark.parametrize( - "builtin", sorted(name for name in os.listdir(BUILTIN_DIR) if name.endswith(".json")) + "builtin", + sorted(name for name in os.listdir(BUILTIN_DIR) if name.endswith(".json")), ) def test_the_packaged_builtins_stay_unmarked(builtin): with open(os.path.join(BUILTIN_DIR, builtin), encoding="utf-8") as file: @@ -141,7 +142,9 @@ def test_the_packaged_builtins_stay_unmarked(builtin): for step in definition.get("steps", []) if "subfolder" in (step.get("result") or {}) ] - assert not marked, f"dw/workflows/{builtin} marks {marked}; a role is the parent's to assign" + assert not marked, ( + f"dw/workflows/{builtin} marks {marked}; a role is the parent's to assign" + ) ``` - [ ] **Step 2: Run it to verify it fails** @@ -279,7 +282,9 @@ def test_the_h3_skill_names_the_intermediate_steps_the_templates_mark(): must change with it.""" text = skill_text(H3_SKILL) for name in ("dialogue-short", "music-video", "storyboard"): - path = os.path.join(REPO_ROOT, "workflows", "templates", "minimax", name + ".json") + path = os.path.join( + REPO_ROOT, "workflows", "templates", "minimax", name + ".json" + ) with open(path, encoding="utf-8") as f: spec = json.load(f) finals = [ @@ -288,7 +293,9 @@ def test_the_h3_skill_names_the_intermediate_steps_the_templates_mark(): if (step.get("result") or {}).get("subfolder") == "final" ] assert len(finals) == 1, (name, finals) - assert f"`{finals[0]}`" in text, f"the skill does not name {name}'s final step {finals[0]}" + assert f"`{finals[0]}`" in text, ( + f"the skill does not name {name}'s final step {finals[0]}" + ) ``` - [ ] **Step 2: Run them to verify they fail** diff --git a/docs/superpowers/plans/2026-09-13-acknowledged-cost-stage-1-plan.md b/docs/superpowers/plans/2026-09-13-acknowledged-cost-stage-1-plan.md index 7346e25d..c824214e 100644 --- a/docs/superpowers/plans/2026-09-13-acknowledged-cost-stage-1-plan.md +++ b/docs/superpowers/plans/2026-09-13-acknowledged-cost-stage-1-plan.md @@ -58,9 +58,7 @@ class TestUnpinnedOutputs: spec["steps"][0]["pipeline"]["arguments"]["image"] = ( "output:ltx2/Gyre/latest/still.png" ) - realized, _ = realize_workflow( - spec, {}, 7, output_root=root, pin_outputs=False - ) + realized, _ = realize_workflow(spec, {}, 7, output_root=root, pin_outputs=False) assert ( realized["steps"][0]["pipeline"]["arguments"]["image"] == "output:ltx2/Gyre/latest/still.png" @@ -225,14 +223,14 @@ Co-Authored-By: Claude Opus 5 (1M context) " ```python def build_plan( - candidate, # a dw.workflow.Workflow, as the route constructed it - arguments, # the caller's dict, already past argument_errors + candidate, # a dw.workflow.Workflow, as the route constructed it + arguments, # the caller's dict, already past argument_errors *, - device, # "cuda" | "mps" | "cpu" - the serving backend + device, # "cuda" | "mps" | "cpu" - the serving backend prompt_dir=None, cache_dir=None, lookup_sizes=True, - cache_probe=None, # stage 2; ignored here, cached_steps is always None + cache_probe=None, # stage 2; ignored here, cached_steps is always None ): """What a run of `candidate` with `arguments` will execute and cost.""" ``` @@ -245,8 +243,8 @@ returning "steps": int, "list_entries": {variable: int}, "cached_steps": None, - "downloads_required": [...], # Task 4; [] until then - "estimate": {...}, # Task 3; placeholder until then + "downloads_required": [...], # Task 4; [] until then + "estimate": {...}, # Task 3; placeholder until then } ``` @@ -312,7 +310,9 @@ def definition(): def prompt_library(tmp_path): library = tmp_path / "prompts" (library / "scenic").mkdir(parents=True) - (library / "scenic" / "dusk.json").write_text(json.dumps({"text": "a harbour at dusk"})) + (library / "scenic" / "dusk.json").write_text( + json.dumps({"text": "a harbour at dusk"}) + ) return library @@ -333,7 +333,9 @@ def plan(tmp_path, prompt_library, output_root, monkeypatch): monkeypatch.setattr(dw.plan, "scan_models", lambda cache_dir=None: {"repos": []}) monkeypatch.setattr( - dw.plan, "model_info", lambda *a, **k: (_ for _ in ()).throw(RuntimeError("offline")) + dw.plan, + "model_info", + lambda *a, **k: (_ for _ in ()).throw(RuntimeError("offline")), ) def make(spec=None, arguments=None, **overrides): @@ -352,8 +354,12 @@ class TestShape: def test_the_documented_keys(self, plan): answer = plan() assert set(answer) == { - "fingerprint", "steps", "list_entries", "cached_steps", - "downloads_required", "estimate", + "fingerprint", + "steps", + "list_entries", + "cached_steps", + "downloads_required", + "estimate", } assert answer["fingerprint"].startswith("sha256:") assert len(answer["fingerprint"]) == len("sha256:") + 64 @@ -443,9 +449,20 @@ class TestFingerprintChangesWith: removed = definition() del removed["steps"][1] added = definition() - added["steps"].append({"name": "extra", "task": {"command": "x", "arguments": {}}}) - assert len({base, plan(renamed)["fingerprint"], plan(removed)["fingerprint"], - plan(added)["fingerprint"]}) == 4 + added["steps"].append( + {"name": "extra", "task": {"command": "x", "arguments": {}}} + ) + assert ( + len( + { + base, + plan(renamed)["fingerprint"], + plan(removed)["fingerprint"], + plan(added)["fingerprint"], + } + ) + == 4 + ) ``` - [ ] **Step 2: Run the tests to verify they fail** @@ -479,7 +496,12 @@ import os from huggingface_hub import model_info from .hub_cache import scan_models -from .realize import BUILTIN_PREFIX, VARIABLE_PREFIX, read_sub_workflow, realize_workflow +from .realize import ( + BUILTIN_PREFIX, + VARIABLE_PREFIX, + read_sub_workflow, + realize_workflow, +) from .security import validate_url from .workflow import Workflow @@ -643,8 +665,11 @@ class TestEstimate: spec = definition() del spec["cost"] assert plan(spec)["estimate"] == { - "minutes": None, "basis": "unknown", "device": "cuda", - "measured_on": None, "partial": False, + "minutes": None, + "basis": "unknown", + "device": "cuda", + "measured_on": None, + "partial": False, } def test_an_empty_cost_list_is_unknown(self, plan): @@ -656,22 +681,30 @@ class TestEstimate: spec = definition() spec["cost"] = [cost("mps", 40, name="M2"), cost("cuda", 10, name="4090")] assert plan(spec)["estimate"] == { - "minutes": 10.0, "basis": "catalog", "device": "cuda", - "measured_on": "4090", "partial": False, + "minutes": 10.0, + "basis": "catalog", + "device": "cuda", + "measured_on": "4090", + "partial": False, } def test_another_devices_entry_is_reported_as_such(self, plan): spec = definition() spec["cost"] = [cost("mps", 40, name="M2")] assert plan(spec)["estimate"] == { - "minutes": 40.0, "basis": "other_device", "device": "cuda", - "measured_on": "M2", "partial": False, + "minutes": 40.0, + "basis": "other_device", + "device": "cuda", + "measured_on": "M2", + "partial": False, } def test_per_entry_scales_by_the_callers_list(self, plan): spec = definition() # 10 minutes for the 2-entry default, of which 3 per entry: 4 fixed - spec["cost"] = [cost("cuda", 10, {"variable": "shots", "minutes": 3, "entries": 2})] + spec["cost"] = [ + cost("cuda", 10, {"variable": "shots", "minutes": 3, "entries": 2}) + ] shots = [{"name": n, "prompt": n} for n in "abcde"] answer = plan(spec, arguments={"shots": shots})["estimate"] assert answer["minutes"] == 4 + 3 * 5 @@ -679,18 +712,24 @@ class TestEstimate: def test_per_entry_floors_at_zero(self, plan): spec = definition() - spec["cost"] = [cost("cuda", 1, {"variable": "shots", "minutes": 3, "entries": 2})] + spec["cost"] = [ + cost("cuda", 1, {"variable": "shots", "minutes": 3, "entries": 2}) + ] assert plan(spec)["estimate"]["minutes"] == 0.0 def test_per_entry_naming_no_list_falls_back_to_catalog(self, plan): spec = definition() - spec["cost"] = [cost("cuda", 10, {"variable": "other", "minutes": 3, "entries": 2})] + spec["cost"] = [ + cost("cuda", 10, {"variable": "other", "minutes": 3, "entries": 2}) + ] answer = plan(spec)["estimate"] assert (answer["minutes"], answer["basis"]) == (10.0, "catalog") def test_other_device_beats_per_entry(self, plan): spec = definition() - spec["cost"] = [cost("mps", 10, {"variable": "shots", "minutes": 3, "entries": 2})] + spec["cost"] = [ + cost("mps", 10, {"variable": "shots", "minutes": 3, "entries": 2}) + ] answer = plan(spec)["estimate"] assert (answer["minutes"], answer["basis"]) == (10.0, "other_device") @@ -820,7 +859,11 @@ def _price(cost, device, list_entries): basis = OTHER_DEVICE minutes = float(chosen.get("minutes", 0)) per = chosen.get("per_entry") - if basis == CATALOG and isinstance(per, dict) and per.get("variable") in list_entries: + if ( + basis == CATALOG + and isinstance(per, dict) + and per.get("variable") in list_entries + ): count = list_entries[per["variable"]] each = float(per.get("minutes", 0)) measured_with = int(per.get("entries", 0)) @@ -868,7 +911,8 @@ class TestDownloadsRequired: import dw.plan monkeypatch.setattr( - dw.plan, "scan_models", + dw.plan, + "scan_models", lambda cache_dir=None: {"repos": [{"repo_id": "org/still-model"}]}, ) assert plan()["downloads_required"] == [] @@ -878,7 +922,9 @@ class TestDownloadsRequired: seen = [] monkeypatch.setattr( - dw.plan, "scan_models", lambda cache_dir=None: seen.append(cache_dir) or {"repos": []} + dw.plan, + "scan_models", + lambda cache_dir=None: seen.append(cache_dir) or {"repos": []}, ) plan(cache_dir="/somewhere") assert seen == ["/somewhere"] @@ -887,7 +933,9 @@ class TestDownloadsRequired: local = tmp_path / "weights" local.mkdir() spec = definition() - spec["steps"][0]["pipeline"]["from_pretrained_arguments"]["model_name"] = str(local) + spec["steps"][0]["pipeline"]["from_pretrained_arguments"]["model_name"] = str( + local + ) assert plan(spec)["downloads_required"] == [] def test_a_single_file_url_is_listed_without_a_size(self, plan): @@ -913,16 +961,24 @@ class TestDownloadsRequired: } child = { "id": "child", - "steps": [{"name": "c", "pipeline": { - "configuration": {"component_type": "{Fake}"}, - "from_pretrained_arguments": {"model_name": "org/still-model"}, - "arguments": {}, - }}], + "steps": [ + { + "name": "c", + "pipeline": { + "configuration": {"component_type": "{Fake}"}, + "from_pretrained_arguments": {"model_name": "org/still-model"}, + "arguments": {}, + }, + } + ], } (tmp_path / "child.json").write_text(json.dumps(child)) - spec["steps"].append({"name": "sub", "workflow": {"path": "child.json", "arguments": {}}}) + spec["steps"].append( + {"name": "sub", "workflow": {"path": "child.json", "arguments": {}}} + ) assert [d["repo"] for d in plan(spec)["downloads_required"]] == [ - "org/still-model", "org/vae", + "org/still-model", + "org/vae", ] def test_sizes_come_from_the_hub_in_gib(self, plan, monkeypatch): @@ -1101,7 +1157,9 @@ class TestValidatePlan: def test_a_valid_answer_carries_a_plan(self, server, monkeypatch): import dw.plan - monkeypatch.setattr(dw.plan, "scan_models", lambda cache_dir=None: {"repos": []}) + monkeypatch.setattr( + dw.plan, "scan_models", lambda cache_dir=None: {"repos": []} + ) with server(success_script) as client: result = client.post( "/api/validate?sizes=false", @@ -1110,8 +1168,12 @@ class TestValidatePlan: assert result["valid"] is True plan = result["plan"] assert set(plan) == { - "fingerprint", "steps", "list_entries", "cached_steps", - "downloads_required", "estimate", + "fingerprint", + "steps", + "list_entries", + "cached_steps", + "downloads_required", + "estimate", } assert plan["steps"] == 1 assert plan["estimate"]["basis"] in {"catalog", "other_device"} @@ -1147,19 +1209,29 @@ class TestValidatePlan: def spy(candidate, arguments, **kwargs): seen.append(kwargs["lookup_sizes"]) - return {"fingerprint": "sha256:0", "steps": 0, "list_entries": {}, - "cached_steps": None, "downloads_required": [], "estimate": None} + return { + "fingerprint": "sha256:0", + "steps": 0, + "list_entries": {}, + "cached_steps": None, + "downloads_required": [], + "estimate": None, + } monkeypatch.setattr(app_module, "build_plan", spy) with server(success_script) as client: client.post("/api/validate", json={"workflow": valid_workflow("v")}) - client.post("/api/validate?sizes=false", json={"workflow": valid_workflow("v")}) + client.post( + "/api/validate?sizes=false", json={"workflow": valid_workflow("v")} + ) assert seen == [True, False] def test_the_plan_sees_the_callers_arguments(self, server, monkeypatch): import dw.plan - monkeypatch.setattr(dw.plan, "scan_models", lambda cache_dir=None: {"repos": []}) + monkeypatch.setattr( + dw.plan, "scan_models", lambda cache_dir=None: {"repos": []} + ) with server(success_script) as client: body = {"workflow": valid_workflow("v")} one = client.post("/api/validate?sizes=false", json=body).json()["plan"] diff --git a/docs/superpowers/plans/2026-09-13-acknowledged-cost-stage-2-plan.md b/docs/superpowers/plans/2026-09-13-acknowledged-cost-stage-2-plan.md index f5bebfac..d1404f89 100644 --- a/docs/superpowers/plans/2026-09-13-acknowledged-cost-stage-2-plan.md +++ b/docs/superpowers/plans/2026-09-13-acknowledged-cost-stage-2-plan.md @@ -76,7 +76,9 @@ class TestCacheHits: workflow.run({}) probe = workflow.cache_hits({}) workflow.run({}) - reused = [entry["step"] for entry in workflow.manifest if entry.get("reused")] + reused = [ + entry["step"] for entry in workflow.manifest if entry.get("reused") + ] assert probe == ["generate"] assert probe == reused assert call_count() == 1, "the probe executed nothing" @@ -127,167 +129,166 @@ Expected: FAIL - `AttributeError: 'Workflow' object has no attribute 'cache_hits In `dw/workflow.py`, inside `class Workflow`, add two methods (place them just above `run`): ```python - def _prepare_definition(self, workflow_def, arguments, base_dir): - """The definition as a run works from it: constants realized, - arguments folded into the variables, list entries' own references - resolved, variable values realized (assets loaded), every - 'variable:' substituted, every for_each expanded, and the seed read - and coerced. Returns (workflow_def, default_seed) - the seed is - None when the workflow names none, and the caller decides what - that means (run() draws one; cache_hits() reports no hits). - - Shared by run() and cache_hits() so the probe prepares exactly what - the run prepares - the step cache keys on the realized step, and a - probe that prepared it differently would answer for a run that - never happens. - """ - workflow_id = workflow_def["id"] - variables = workflow_def.get("variables", None) - if variables is not None: - logger.debug(f"Setting variables for workflow: {workflow_id}") - # a constant is the value a variable declares, so it resolves before - # anything is converted to the type of that declaration - realize_constants(variables) - # first set variable values base don the arguments passed to the workflow - # these may come form the command line or form a parent workflow - set_variables(arguments, variables) - # an entry of a list-valued variable may name another - # variable; resolve those before anything inside it is - # realized, so a reference type in an entry is a type name - variables = resolve_variable_values(variables) - # realize the variables, initialiting downloads of images etc - realize_args(variables, base_dir) - ## then replace any variable references in the workflow definition with the actual values - # replace_variables returns a new structure rather than mutating in - # place, so the result must be captured here - workflow_def = replace_variables(workflow_def, variables) - - # One ordinary step per entry of every for_each list, before the - # seed, the run id and the realized workflow are computed, so - # each covers what actually runs. A ForEachError here fails the - # run before anything loads - workflow_def = expand_for_each(workflow_def) - - # Set up random seed for reproducibility. Resolved lazily - as a - # dict.get default, torch.seed() would run on every call and reseed - # the global RNG even when the workflow names an explicit seed - default_seed = workflow_def.get("seed") - # The schema lets 'seed' be a string so it can hold a 'variable:' - # reference, which the substitution above has already resolved - - # but a variable overridden from the command line arrives as a - # string whenever the workflow declared no integer default to - # coerce against, and manual_seed would fail deep inside the run - if isinstance(default_seed, str): - try: - default_seed = int(default_seed) - except ValueError: - raise ValueError( - f"Workflow {workflow_id} seed must be an integer, " - f"got {default_seed!r}" - ) - workflow_def["seed"] = default_seed - return workflow_def, default_seed +def _prepare_definition(self, workflow_def, arguments, base_dir): + """The definition as a run works from it: constants realized, + arguments folded into the variables, list entries' own references + resolved, variable values realized (assets loaded), every + 'variable:' substituted, every for_each expanded, and the seed read + and coerced. Returns (workflow_def, default_seed) - the seed is + None when the workflow names none, and the caller decides what + that means (run() draws one; cache_hits() reports no hits). + + Shared by run() and cache_hits() so the probe prepares exactly what + the run prepares - the step cache keys on the realized step, and a + probe that prepared it differently would answer for a run that + never happens. + """ + workflow_id = workflow_def["id"] + variables = workflow_def.get("variables", None) + if variables is not None: + logger.debug(f"Setting variables for workflow: {workflow_id}") + # a constant is the value a variable declares, so it resolves before + # anything is converted to the type of that declaration + realize_constants(variables) + # first set variable values base don the arguments passed to the workflow + # these may come form the command line or form a parent workflow + set_variables(arguments, variables) + # an entry of a list-valued variable may name another + # variable; resolve those before anything inside it is + # realized, so a reference type in an entry is a type name + variables = resolve_variable_values(variables) + # realize the variables, initialiting downloads of images etc + realize_args(variables, base_dir) + ## then replace any variable references in the workflow definition with the actual values + # replace_variables returns a new structure rather than mutating in + # place, so the result must be captured here + workflow_def = replace_variables(workflow_def, variables) + + # One ordinary step per entry of every for_each list, before the + # seed, the run id and the realized workflow are computed, so + # each covers what actually runs. A ForEachError here fails the + # run before anything loads + workflow_def = expand_for_each(workflow_def) + + # Set up random seed for reproducibility. Resolved lazily - as a + # dict.get default, torch.seed() would run on every call and reseed + # the global RNG even when the workflow names an explicit seed + default_seed = workflow_def.get("seed") + # The schema lets 'seed' be a string so it can hold a 'variable:' + # reference, which the substitution above has already resolved - + # but a variable overridden from the command line arrives as a + # string whenever the workflow declared no integer default to + # coerce against, and manual_seed would fail deep inside the run + if isinstance(default_seed, str): + try: + default_seed = int(default_seed) + except ValueError: + raise ValueError( + f"Workflow {workflow_id} seed must be an integer, got {default_seed!r}" + ) + workflow_def["seed"] = default_seed + return workflow_def, default_seed + + +def _cache_lookup( + self, workflow_id, steps, index, step_data, step_seed, hits_this_run, cache_enabled +): + """Whether the step cache serves step `index`, as + (cached_result or None, the step_data snapshot the entry is keyed + on or None). Shared by run() and cache_hits() - see + _prepare_definition for why. + """ + # What later steps still read, which decides both whether this + # step's result has to be kept alive after the step and whether a + # cached entry that kept none can serve this run + remaining_refs = referenced_result_names(steps[index + 1 :]) + result_needed = index == len(steps) - 1 or any( + reference_resolves_to(ref, step_data["name"]) for ref in remaining_refs + ) + # create_step_action (and the pipeline load it triggers) mutates + # step_data in place - injecting a "generator" key - so the cache + # must key off a snapshot taken before that happens, and that same + # snapshot must be reused for the put() later. A sub-workflow step + # is never cacheable: its files roll up from the child's own + # manifest, which a hit does not rebuild. + is_cacheable = "workflow" not in step_data and cache_enabled + # The last step of a composed child whose parent does the saving + # (#92) - its files are the parent step's, written once, under the + # parent's name and subfolder + parent_saves_this = self._final_save_owned_by_parent and index == len(steps) - 1 + step_data_snapshot = None + if is_cacheable: + try: + step_data_snapshot = copy.deepcopy(step_data) + if parent_saves_this: + # Keyed apart from the same step run standalone: this + # entry's result was never saved here, so a standalone + # hit on it would report no files + step_data_snapshot["__saved_by_parent__"] = True + except Exception as ex: + # A realized argument that cannot be deep-copied (an open + # handle, a live model object) just means this step is not + # cacheable - never a failed run + logger.debug( + f"Step '{step_data['name']}' arguments are not copyable " + f"({ex}) - skipping the step cache for it" + ) + is_cacheable = False + if not is_cacheable: + return None, None + cached_result = step_cache.get( + workflow_id, + step_data_snapshot, + step_seed, + hits_this_run, + # The root, not this run's directory: a hit reports the earlier + # run's files and writes nothing new, so keying on a directory + # that is new every run would mean the cache could never hit + # again. What the root still guards is a run redirected + # somewhere else, where the earlier files are not what the + # caller asked for + self.output_dir, + needs_result=result_needed, + ) + return cached_result, step_data_snapshot - def _cache_lookup( - self, workflow_id, steps, index, step_data, step_seed, hits_this_run, cache_enabled - ): - """Whether the step cache serves step `index`, as - (cached_result or None, the step_data snapshot the entry is keyed - on or None). Shared by run() and cache_hits() - see - _prepare_definition for why. - """ - # What later steps still read, which decides both whether this - # step's result has to be kept alive after the step and whether a - # cached entry that kept none can serve this run - remaining_refs = referenced_result_names(steps[index + 1 :]) - result_needed = index == len(steps) - 1 or any( - reference_resolves_to(ref, step_data["name"]) for ref in remaining_refs - ) - # create_step_action (and the pipeline load it triggers) mutates - # step_data in place - injecting a "generator" key - so the cache - # must key off a snapshot taken before that happens, and that same - # snapshot must be reused for the put() later. A sub-workflow step - # is never cacheable: its files roll up from the child's own - # manifest, which a hit does not rebuild. - is_cacheable = "workflow" not in step_data and cache_enabled - # The last step of a composed child whose parent does the saving - # (#92) - its files are the parent step's, written once, under the - # parent's name and subfolder - parent_saves_this = self._final_save_owned_by_parent and index == len(steps) - 1 - step_data_snapshot = None - if is_cacheable: - try: - step_data_snapshot = copy.deepcopy(step_data) - if parent_saves_this: - # Keyed apart from the same step run standalone: this - # entry's result was never saved here, so a standalone - # hit on it would report no files - step_data_snapshot["__saved_by_parent__"] = True - except Exception as ex: - # A realized argument that cannot be deep-copied (an open - # handle, a live model object) just means this step is not - # cacheable - never a failed run - logger.debug( - f"Step '{step_data['name']}' arguments are not copyable " - f"({ex}) - skipping the step cache for it" - ) - is_cacheable = False - if not is_cacheable: - return None, None - cached_result = step_cache.get( - workflow_id, - step_data_snapshot, - step_seed, - hits_this_run, - # The root, not this run's directory: a hit reports the earlier - # run's files and writes nothing new, so keying on a directory - # that is new every run would mean the cache could never hit - # again. What the root still guards is a run redirected - # somewhere else, where the earlier files are not what the - # caller asked for - self.output_dir, - needs_result=result_needed, - ) - return cached_result, step_data_snapshot - def cache_hits(self, arguments): - """The steps the step cache would serve for a run with `arguments`, - in step order - what the plan reports as cached_steps (#85). +def cache_hits(self, arguments): + """The steps the step cache would serve for a run with `arguments`, + in step order - what the plan reports as cached_steps (#85). - Prepares the definition exactly as run() does and asks the cache the - question run() asks, step by step with the hits so far, and executes - nothing: no run directory, no events, no pipeline. An unseeded - workflow has no cache, so it answers [] without asking. - """ - output_root_token = activate_output_root(self.output_dir) - try: - workflow_def = copy.deepcopy(self.workflow_definition) - workflow_id = workflow_def["id"] - base_dir = ( - os.path.dirname(os.path.abspath(self.file_spec)) - if self.file_spec - else None - ) - workflow_def, default_seed = self._prepare_definition( - workflow_def, arguments or {}, base_dir + Prepares the definition exactly as run() does and asks the cache the + question run() asks, step by step with the hits so far, and executes + nothing: no run directory, no events, no pipeline. An unseeded + workflow has no cache, so it answers [] without asking. + """ + output_root_token = activate_output_root(self.output_dir) + try: + workflow_def = copy.deepcopy(self.workflow_definition) + workflow_id = workflow_def["id"] + base_dir = ( + os.path.dirname(os.path.abspath(self.file_spec)) if self.file_spec else None + ) + workflow_def, default_seed = self._prepare_definition( + workflow_def, arguments or {}, base_dir + ) + if default_seed is None or not self._cache_enabled_by_parent: + return [] + steps = workflow_def.get("steps", []) + realize_args(steps, base_dir) + hits_this_run = set() + hits = [] + for index, step_data in enumerate(steps): + step_seed = step_data.get("seed", default_seed) + cached_result, _ = self._cache_lookup( + workflow_id, steps, index, step_data, step_seed, hits_this_run, True ) - if default_seed is None or not self._cache_enabled_by_parent: - return [] - steps = workflow_def.get("steps", []) - realize_args(steps, base_dir) - hits_this_run = set() - hits = [] - for index, step_data in enumerate(steps): - step_seed = step_data.get("seed", default_seed) - cached_result, _ = self._cache_lookup( - workflow_id, steps, index, step_data, step_seed, hits_this_run, True - ) - if cached_result is not None: - hits_this_run.add(step_data["name"]) - hits.append(step_data["name"]) - return hits - finally: - deactivate_output_root(output_root_token) + if cached_result is not None: + hits_this_run.add(step_data["name"]) + hits.append(step_data["name"]) + return hits + finally: + deactivate_output_root(output_root_token) ``` Then in `run()`: @@ -295,9 +296,7 @@ Then in `run()`: - Replace the block from `# Handle variable substitution if variables are defined` through the `workflow_def["seed"] = default_seed` line that follows the `int(default_seed)` coercion (lines ~710-752, ending just before `# A workflow that names no seed gets a fresh one every run`) with: ```python - workflow_def, default_seed = self._prepare_definition( - workflow_def, arguments, base_dir - ) +workflow_def, default_seed = self._prepare_definition(workflow_def, arguments, base_dir) ``` Keep everything from `cache_enabled_this_run = (...)` onward unchanged (the random draw, `workflow_def["seed"] = default_seed`, `resolved_seed`). @@ -381,7 +380,12 @@ def test_probe_cache_reports_a_failure_as_unknown_not_as_a_crash(): worker = _make_worker() with patch("dw.worker.workflow_from_file", side_effect=ValueError("bad file")): worker._handle_probe_cache( - {"type": "probe_cache", "workflow_path": "x.json", "arguments": {}, "output_dir": "/tmp"} + { + "type": "probe_cache", + "workflow_path": "x.json", + "arguments": {}, + "output_dir": "/tmp", + } ) [answer] = _drain(worker.result_queue) assert answer["type"] == "probe_cache" @@ -444,7 +448,12 @@ class TestProbeCache: with server(success_script) as client: manager = client.app.state.job_manager assert manager.worker_manager.worker_active is False - assert manager.probe_cache({"workflow_path": "x.json", "arguments": {}, "output_dir": "/tmp"}) == [] + assert ( + manager.probe_cache( + {"workflow_path": "x.json", "arguments": {}, "output_dir": "/tmp"} + ) + == [] + ) assert manager.worker_manager.commands == [] def test_a_running_job_means_unknown(self, server): @@ -453,7 +462,12 @@ class TestProbeCache: job_id = response.json()["id"] wait_for_status(client, job_id, ("running",)) manager = client.app.state.job_manager - assert manager.probe_cache({"workflow_path": "x.json", "arguments": {}, "output_dir": "/tmp"}) is None + assert ( + manager.probe_cache( + {"workflow_path": "x.json", "arguments": {}, "output_dir": "/tmp"} + ) + is None + ) client.post(f"/api/jobs/{job_id}/cancel") def test_an_unanswered_probe_is_unknown(self, server): @@ -461,9 +475,13 @@ class TestProbeCache: manager = client.app.state.job_manager manager.worker_manager.ensure_worker() manager.worker_manager.send_command = lambda command: None # swallow it - assert manager.probe_cache( - {"workflow_path": "x.json", "arguments": {}, "output_dir": "/tmp"}, timeout=0.05 - ) is None + assert ( + manager.probe_cache( + {"workflow_path": "x.json", "arguments": {}, "output_dir": "/tmp"}, + timeout=0.05, + ) + is None + ) ``` - [ ] **Step 2: Run the tests to verify they fail** @@ -483,31 +501,29 @@ Expected: FAIL - no `_handle_probe_cache`, no `probe_cache`. and beside `_handle_memory_status`: ```python - def _handle_probe_cache(self, command: Dict[str, Any]): - """Which steps the step cache would serve for a run of this command - - the plan's cached_steps (#85). Same fields as an execute command; - loads the workflow, executes nothing. A failure answers - cached: null with the reason rather than an error message, since - an unknown answer is a valid plan and a crashed probe is not. - """ +def _handle_probe_cache(self, command: Dict[str, Any]): + """Which steps the step cache would serve for a run of this command + - the plan's cached_steps (#85). Same fields as an execute command; + loads the workflow, executes nothing. A failure answers + cached: null with the reason rather than an error message, since + an unknown answer is a valid plan and a crashed probe is not. + """ + try: + workflow, _ = self._load_workflow(command, command["output_dir"]) + asset_token = ( + activate_asset_dir(command["asset_dir"]) + if command.get("asset_dir") + else None + ) try: - workflow, _ = self._load_workflow(command, command["output_dir"]) - asset_token = ( - activate_asset_dir(command["asset_dir"]) - if command.get("asset_dir") - else None - ) - try: - cached = workflow.cache_hits(command.get("arguments") or {}) - finally: - if asset_token is not None: - deactivate_asset_dir(asset_token) - self.result_queue.put({"type": "probe_cache", "cached": cached}) - except Exception as e: - logger.debug(f"Cache probe failed: {e}") - self.result_queue.put( - {"type": "probe_cache", "cached": None, "error": str(e)} - ) + cached = workflow.cache_hits(command.get("arguments") or {}) + finally: + if asset_token is not None: + deactivate_asset_dir(asset_token) + self.result_queue.put({"type": "probe_cache", "cached": cached}) + except Exception as e: + logger.debug(f"Cache probe failed: {e}") + self.result_queue.put({"type": "probe_cache", "cached": None, "error": str(e)}) ``` `dw/server/jobs.py`, beside `memory_status`: @@ -615,38 +631,40 @@ class TestCachedSteps: Append to `tests/test_server.py` inside `TestValidatePlan`: ```python - def test_cached_steps_comes_from_the_worker(self, server, monkeypatch): - import dw.plan - - monkeypatch.setattr(dw.plan, "scan_models", lambda cache_dir=None: {"repos": []}) - with server(success_script) as client: - manager = client.app.state.job_manager - manager.worker_manager.ensure_worker() - manager.worker_manager.cached_steps = ["gen"] - seeded = valid_workflow("seeded") - seeded["seed"] = 7 - result = client.post( - "/api/validate?sizes=false", json={"workflow": seeded, "arguments": {"prompt": "x"}} - ).json() - assert result["plan"]["cached_steps"] == 1 - probe = [c for c in manager.worker_manager.commands if c["type"] == "probe_cache"] - assert len(probe) == 1 - assert probe[0]["arguments"] == {"prompt": "x"} - assert probe[0]["workflow"] == seeded - assert probe[0]["output_dir"] == manager.output_dir +def test_cached_steps_comes_from_the_worker(self, server, monkeypatch): + import dw.plan - def test_an_unseeded_workflow_does_not_probe(self, server, monkeypatch): - import dw.plan + monkeypatch.setattr(dw.plan, "scan_models", lambda cache_dir=None: {"repos": []}) + with server(success_script) as client: + manager = client.app.state.job_manager + manager.worker_manager.ensure_worker() + manager.worker_manager.cached_steps = ["gen"] + seeded = valid_workflow("seeded") + seeded["seed"] = 7 + result = client.post( + "/api/validate?sizes=false", + json={"workflow": seeded, "arguments": {"prompt": "x"}}, + ).json() + assert result["plan"]["cached_steps"] == 1 + probe = [c for c in manager.worker_manager.commands if c["type"] == "probe_cache"] + assert len(probe) == 1 + assert probe[0]["arguments"] == {"prompt": "x"} + assert probe[0]["workflow"] == seeded + assert probe[0]["output_dir"] == manager.output_dir + + +def test_an_unseeded_workflow_does_not_probe(self, server, monkeypatch): + import dw.plan - monkeypatch.setattr(dw.plan, "scan_models", lambda cache_dir=None: {"repos": []}) - with server(success_script) as client: - manager = client.app.state.job_manager - manager.worker_manager.ensure_worker() - result = client.post( - "/api/validate?sizes=false", json={"workflow": valid_workflow("v")} - ).json() - assert result["plan"]["cached_steps"] == 0 - assert all(c["type"] != "probe_cache" for c in manager.worker_manager.commands) + monkeypatch.setattr(dw.plan, "scan_models", lambda cache_dir=None: {"repos": []}) + with server(success_script) as client: + manager = client.app.state.job_manager + manager.worker_manager.ensure_worker() + result = client.post( + "/api/validate?sizes=false", json={"workflow": valid_workflow("v")} + ).json() + assert result["plan"]["cached_steps"] == 0 + assert all(c["type"] != "probe_cache" for c in manager.worker_manager.commands) ``` - [ ] **Step 2: Run the tests to verify they fail** @@ -768,7 +786,10 @@ class TestAcknowledgementRecord: manager = client.app.state.job_manager bound = {"fingerprint": "sha256:abc", "minutes": 3.0, "downloads": []} job = manager.submit( - workflow=valid_workflow(), arguments={}, acknowledged="bound", acknowledged_cost=bound + workflow=valid_workflow(), + arguments={}, + acknowledged="bound", + acknowledged_cost=bound, ) detail = manager.describe(job) assert detail["acknowledged"] == "bound" @@ -778,15 +799,24 @@ class TestAcknowledgementRecord: def test_history_keeps_the_form_and_the_object(self, server, tmp_path): with server(success_script) as client: manager = client.app.state.job_manager - bound = {"fingerprint": "sha256:abc", "minutes": 3.0, "downloads": ["org/x"]} + bound = { + "fingerprint": "sha256:abc", + "minutes": 3.0, + "downloads": ["org/x"], + } job = manager.submit( - workflow=valid_workflow(), arguments={}, acknowledged="bound", acknowledged_cost=bound + workflow=valid_workflow(), + arguments={}, + acknowledged="bound", + acknowledged_cost=bound, ) wait_for_status(client, job.id, TERMINAL_STATES) row = manager.history.get(job.id) assert row["acknowledged"] == "bound" assert row["spec"]["acknowledged_cost"] == bound - listed = [s for s in manager.history.recent_summaries() if s["id"] == job.id] + listed = [ + s for s in manager.history.recent_summaries() if s["id"] == job.id + ] assert listed[0]["acknowledged"] == "bound" def test_a_database_without_the_column_is_migrated(self, tmp_path): @@ -813,7 +843,10 @@ class TestAcknowledgementRecord: manager = client.app.state.job_manager bound = {"fingerprint": "sha256:abc", "minutes": 3.0, "downloads": []} job = manager.submit( - workflow=valid_workflow(), arguments={}, acknowledged="bound", acknowledged_cost=bound + workflow=valid_workflow(), + arguments={}, + acknowledged="bound", + acknowledged_cost=bound, ) wait_for_status(client, job.id, TERMINAL_STATES) rerun = manager.rerun(job.id) @@ -848,12 +881,10 @@ ACK_BOUND = "bound" Schema migration, after the `run_dir` block: ```python - # Which form of cost acknowledgement queued the job. Rows before - # the column are 'none' - nothing recorded is nothing recorded - if "acknowledged" not in columns: - connection.execute( - "ALTER TABLE jobs ADD COLUMN acknowledged TEXT DEFAULT 'none'" - ) +# Which form of cost acknowledgement queued the job. Rows before +# the column are 'none' - nothing recorded is nothing recorded +if "acknowledged" not in columns: + connection.execute("ALTER TABLE jobs ADD COLUMN acknowledged TEXT DEFAULT 'none'") ``` `record`: add `acknowledged` to the column list and `job.acknowledged` to the values (17 placeholders). `recent_summaries`: add `acknowledged` to the SELECT and `"acknowledged": row[9] or ACK_NONE` to each summary. `get`: add `acknowledged` to the SELECT; `_to_detail`: `"acknowledged": row[15] or ACK_NONE, "acknowledged_cost": (parse(row[7], {}) or {}).get("acknowledged_cost")` - reuse the parsed spec rather than parsing twice. @@ -963,7 +994,8 @@ class TestBoundAcknowledgement: with server(success_script) as client: plan = plan_for(client, list_workflow()) response = client.post( - "/api/jobs", json={"workflow": list_workflow(), "acknowledged_cost": bound(plan)} + "/api/jobs", + json={"workflow": list_workflow(), "acknowledged_cost": bound(plan)}, ) assert response.status_code == 201, response.json() assert response.json()["acknowledged"] == "bound" @@ -975,7 +1007,11 @@ class TestBoundAcknowledgement: longer = {"shots": [{"name": n, "prompt": n} for n in "abc"]} response = client.post( "/api/jobs", - json={"workflow": list_workflow(), "arguments": longer, "acknowledged_cost": bound(plan)}, + json={ + "workflow": list_workflow(), + "arguments": longer, + "acknowledged_cost": bound(plan), + }, ) assert response.status_code == 409 detail = response.json()["detail"] @@ -991,7 +1027,11 @@ class TestBoundAcknowledgement: plan = plan_for(client, list_workflow()) response = client.post( "/api/jobs", - json={"workflow": list_workflow(), "arguments": {"seed": 99}, "acknowledged_cost": bound(plan)}, + json={ + "workflow": list_workflow(), + "arguments": {"seed": 99}, + "acknowledged_cost": bound(plan), + }, ) assert response.status_code == 201 @@ -1001,28 +1041,39 @@ class TestBoundAcknowledgement: acknowledgement = bound(plan) acknowledgement["downloads"] = [] # the caller left the repo out response = client.post( - "/api/jobs", json={"workflow": list_workflow(), "acknowledged_cost": acknowledgement} + "/api/jobs", + json={ + "workflow": list_workflow(), + "acknowledged_cost": acknowledgement, + }, ) assert response.status_code == 409 detail = response.json()["detail"] assert detail["reason"] == "downloads" assert "m" in detail["message"] - def test_a_download_that_vanished_is_not_a_refusal(self, server, no_hub, monkeypatch): + def test_a_download_that_vanished_is_not_a_refusal( + self, server, no_hub, monkeypatch + ): with server(success_script) as client: plan = plan_for(client, list_workflow()) assert bound(plan)["downloads"] == ["m"] import dw.plan monkeypatch.setattr( - dw.plan, "scan_models", lambda cache_dir=None: {"repos": [{"repo_id": "m"}]} + dw.plan, + "scan_models", + lambda cache_dir=None: {"repos": [{"repo_id": "m"}]}, ) response = client.post( - "/api/jobs", json={"workflow": list_workflow(), "acknowledged_cost": bound(plan)} + "/api/jobs", + json={"workflow": list_workflow(), "acknowledged_cost": bound(plan)}, ) assert response.status_code == 201 - def test_an_unplannable_run_is_refused_not_passed(self, server, no_hub, monkeypatch): + def test_an_unplannable_run_is_refused_not_passed( + self, server, no_hub, monkeypatch + ): import dw.server.app as app_module with server(success_script) as client: @@ -1033,7 +1084,8 @@ class TestBoundAcknowledgement: monkeypatch.setattr(app_module, "build_plan", boom) response = client.post( - "/api/jobs", json={"workflow": list_workflow(), "acknowledged_cost": bound(plan)} + "/api/jobs", + json={"workflow": list_workflow(), "acknowledged_cost": bound(plan)}, ) assert response.status_code == 409 assert response.json()["detail"]["reason"] == "unplannable" @@ -1049,10 +1101,12 @@ class TestBoundAcknowledgement: with server(success_script) as client: plain = client.post("/api/jobs", json={"workflow": valid_workflow("p")}) flagged = client.post( - "/api/jobs", json={"workflow": valid_workflow("f"), "acknowledged_cost": True} + "/api/jobs", + json={"workflow": valid_workflow("f"), "acknowledged_cost": True}, ) off = client.post( - "/api/jobs", json={"workflow": valid_workflow("o"), "acknowledged_cost": False} + "/api/jobs", + json={"workflow": valid_workflow("o"), "acknowledged_cost": False}, ) assert plain.json()["acknowledged"] == "none" assert flagged.json()["acknowledged"] == "boolean" @@ -1061,11 +1115,17 @@ class TestBoundAcknowledgement: def test_a_bound_form_without_a_fingerprint_is_a_422(self, server): with server(success_script) as client: response = client.post( - "/api/jobs", json={"workflow": valid_workflow(), "acknowledged_cost": {"minutes": 3}} + "/api/jobs", + json={ + "workflow": valid_workflow(), + "acknowledged_cost": {"minutes": 3}, + }, ) assert response.status_code == 422 - def test_a_stored_prompt_edited_after_validation_is_refused(self, server, no_hub, tmp_path): + def test_a_stored_prompt_edited_after_validation_is_refused( + self, server, no_hub, tmp_path + ): (tmp_path / "prompts" / "p.json").write_text(json.dumps({"text": "before"})) workflow = valid_workflow("prompted") workflow["variables"]["prompt"] = "prompt:p" @@ -1073,18 +1133,22 @@ class TestBoundAcknowledgement: plan = plan_for(client, workflow) (tmp_path / "prompts" / "p.json").write_text(json.dumps({"text": "after"})) response = client.post( - "/api/jobs", json={"workflow": workflow, "acknowledged_cost": bound(plan)} + "/api/jobs", + json={"workflow": workflow, "acknowledged_cost": bound(plan)}, ) assert response.status_code == 409 assert response.json()["detail"]["reason"] == "fingerprint" class TestBoundRerun: - def test_a_rerun_with_the_original_plan_queues_even_with_a_new_seed(self, server, no_hub): + def test_a_rerun_with_the_original_plan_queues_even_with_a_new_seed( + self, server, no_hub + ): with server(success_script) as client: plan = plan_for(client, list_workflow()) first = client.post( - "/api/jobs", json={"workflow": list_workflow(), "acknowledged_cost": bound(plan)} + "/api/jobs", + json={"workflow": list_workflow(), "acknowledged_cost": bound(plan)}, ).json() wait_for_status(client, first["id"], TERMINAL_STATES) response = client.post( @@ -1100,7 +1164,8 @@ class TestBoundRerun: wait_for_status(client, first["id"], TERMINAL_STATES) other = plan_for(client, valid_workflow("other")) response = client.post( - f"/api/jobs/{first['id']}/rerun", json={"acknowledged_cost": bound(other)} + f"/api/jobs/{first['id']}/rerun", + json={"acknowledged_cost": bound(other)}, ) assert response.status_code == 409 assert response.json()["detail"]["reason"] == "fingerprint" @@ -1109,14 +1174,17 @@ class TestBoundRerun: with server(success_script) as client: first = client.post("/api/jobs", json={"workflow": list_workflow()}).json() wait_for_status(client, first["id"], TERMINAL_STATES) - response = client.post(f"/api/jobs/{first['id']}/rerun", json={"acknowledged_cost": True}) + response = client.post( + f"/api/jobs/{first['id']}/rerun", json={"acknowledged_cost": True} + ) assert response.status_code == 201 assert response.json()["acknowledged"] == "boolean" def test_an_unknown_job_is_still_404(self, server): with server(success_script) as client: response = client.post( - "/api/jobs/nope/rerun", json={"acknowledged_cost": {"fingerprint": "sha256:0"}} + "/api/jobs/nope/rerun", + json={"acknowledged_cost": {"fingerprint": "sha256:0"}}, ) assert response.status_code == 404 ``` @@ -1165,95 +1233,100 @@ ACKNOWLEDGED_COST_FIELD = Field( Helpers inside `create_app`, beside `_argument_reference_errors`: ```python - def _acknowledgement_form(value): - """none | boolean | bound - classified once, here, so the check and - the record agree.""" - if isinstance(value, AcknowledgedCost): - return ACK_BOUND - return ACK_BOOLEAN if value is True else ACK_NONE - - def _check_bound_acknowledgement(candidate, arguments, acknowledged, workspace): - """Refuse with 409 when the run `candidate` + `arguments` will - execute is not the one `acknowledged` was bound to: a different - fingerprint, or a download the caller did not acknowledge. The body - carries the current plan so the agent re-quotes from it without a - second validate call. A plan that cannot be built is a refusal too - - never a silent pass (#85). - """ - record = acknowledged.model_dump() - try: - from .. import get_device, get_device_type +def _acknowledgement_form(value): + """none | boolean | bound - classified once, here, so the check and + the record agree.""" + if isinstance(value, AcknowledgedCost): + return ACK_BOUND + return ACK_BOOLEAN if value is True else ACK_NONE + + +def _check_bound_acknowledgement(candidate, arguments, acknowledged, workspace): + """Refuse with 409 when the run `candidate` + `arguments` will + execute is not the one `acknowledged` was bound to: a different + fingerprint, or a download the caller did not acknowledge. The body + carries the current plan so the agent re-quotes from it without a + second validate call. A plan that cannot be built is a refusal too - + never a silent pass (#85). + """ + record = acknowledged.model_dump() + try: + from .. import get_device, get_device_type + + current = build_plan( + candidate, + arguments, + device=get_device_type(get_device()), + prompt_dir=workspace.prompts, + lookup_sizes=False, + ) + except Exception: + logger.exception("Plan could not be built for a bound acknowledgement") + raise HTTPException( + status_code=409, + detail={ + "message": "The run could not be planned, so a bound " + "acknowledgement cannot be checked; acknowledge with true " + "or validate again", + "reason": "unplannable", + "acknowledged": record, + "plan": None, + }, + ) + if current["fingerprint"] != acknowledged.fingerprint: + raise HTTPException( + status_code=409, + detail={ + "message": "The run's shape changed since it was acknowledged: " + "the workflow or its arguments differ from what was validated", + "reason": "fingerprint", + "acknowledged": record, + "plan": current, + }, + ) + missing = [ + entry["repo"] + for entry in current["downloads_required"] + if entry.get("repo") and entry["repo"] not in acknowledged.downloads + ] + if missing: + raise HTTPException( + status_code=409, + detail={ + "message": "The run's shape changed since it was acknowledged: " + f"it now has to download {', '.join(missing)} first", + "reason": "downloads", + "acknowledged": record, + "plan": current, + }, + ) - current = build_plan( - candidate, - arguments, - device=get_device_type(get_device()), - prompt_dir=workspace.prompts, - lookup_sizes=False, - ) - except Exception: - logger.exception("Plan could not be built for a bound acknowledgement") - raise HTTPException( - status_code=409, - detail={ - "message": "The run could not be planned, so a bound " - "acknowledgement cannot be checked; acknowledge with true " - "or validate again", - "reason": "unplannable", - "acknowledged": record, - "plan": None, - }, - ) - if current["fingerprint"] != acknowledged.fingerprint: - raise HTTPException( - status_code=409, - detail={ - "message": "The run's shape changed since it was acknowledged: " - "the workflow or its arguments differ from what was validated", - "reason": "fingerprint", - "acknowledged": record, - "plan": current, - }, - ) - missing = [ - entry["repo"] - for entry in current["downloads_required"] - if entry.get("repo") and entry["repo"] not in acknowledged.downloads - ] - if missing: - raise HTTPException( - status_code=409, - detail={ - "message": "The run's shape changed since it was acknowledged: " - f"it now has to download {', '.join(missing)} first", - "reason": "downloads", - "acknowledged": record, - "plan": current, - }, - ) - def _candidate_for(workflow_path, workflow, base_dir, output_dir, workflow_dir): - """The Workflow a job spec names, built as the worker will build it.""" - if workflow_path is not None: - return workflow_from_file(workflow_path, output_dir, workflow_dir) - return workflow_from_definition( - copy.deepcopy(workflow), output_dir, base_dir, workflow_dir - ) +def _candidate_for(workflow_path, workflow, base_dir, output_dir, workflow_dir): + """The Workflow a job spec names, built as the worker will build it.""" + if workflow_path is not None: + return workflow_from_file(workflow_path, output_dir, workflow_dir) + return workflow_from_definition( + copy.deepcopy(workflow), output_dir, base_dir, workflow_dir + ) ``` In `submit_job`, after the `reference_problems` check and before `manager.submit(...)`: ```python - form = _acknowledgement_form(request.acknowledged_cost) - if form == ACK_BOUND: - confinement = source.root if source else workspace.workflows - candidate = _candidate_for( - resolved, request.workflow, request.base_dir, - workspace.outputs, confinement, - ) - _check_bound_acknowledgement( - candidate, request.arguments, request.acknowledged_cost, workspace - ) +form = _acknowledgement_form(request.acknowledged_cost) +if form == ACK_BOUND: + confinement = source.root if source else workspace.workflows + candidate = _candidate_for( + resolved, + request.workflow, + request.base_dir, + workspace.outputs, + confinement, + ) + _check_bound_acknowledgement( + candidate, request.arguments, request.acknowledged_cost, workspace + ) ``` and pass to `manager.submit`: `acknowledged=form, acknowledged_cost=(request.acknowledged_cost.model_dump() if form == ACK_BOUND else None)`. The `except HTTPException: raise` already precedes the catch-all, so the 409 passes through. @@ -1261,36 +1334,41 @@ and pass to `manager.submit`: `acknowledged=form, acknowledged_cost=(request.ack In `rerun_job`: ```python - form = _acknowledgement_form(body.acknowledged_cost) - if form == ACK_BOUND: - prepared = manager.rerun_spec(job_id) - if prepared is None: - raise HTTPException(status_code=404, detail="Unknown job") - spec, arguments = prepared - try: - candidate = _candidate_for( - spec.get("workflow_path"), spec.get("workflow"), spec.get("base_dir"), - spec.get("output_dir") or manager.output_dir, spec.get("workflow_dir"), - ) - except Exception as e: - raise HTTPException(status_code=400, detail=str(e)) - _check_bound_acknowledgement( - candidate, arguments, body.acknowledged_cost, - _workspace_for(spec.get("workspace")), - ) - try: - job = manager.rerun( - job_id, - new_seed=body.new_seed, - acknowledged=form, - acknowledged_cost=( - body.acknowledged_cost.model_dump() if form == ACK_BOUND else None - ), - ) - except HTTPException: - raise - except Exception as e: - raise HTTPException(status_code=400, detail=str(e)) +form = _acknowledgement_form(body.acknowledged_cost) +if form == ACK_BOUND: + prepared = manager.rerun_spec(job_id) + if prepared is None: + raise HTTPException(status_code=404, detail="Unknown job") + spec, arguments = prepared + try: + candidate = _candidate_for( + spec.get("workflow_path"), + spec.get("workflow"), + spec.get("base_dir"), + spec.get("output_dir") or manager.output_dir, + spec.get("workflow_dir"), + ) + except Exception as e: + raise HTTPException(status_code=400, detail=str(e)) + _check_bound_acknowledgement( + candidate, + arguments, + body.acknowledged_cost, + _workspace_for(spec.get("workspace")), + ) +try: + job = manager.rerun( + job_id, + new_seed=body.new_seed, + acknowledged=form, + acknowledged_cost=( + body.acknowledged_cost.model_dump() if form == ACK_BOUND else None + ), + ) +except HTTPException: + raise +except Exception as e: + raise HTTPException(status_code=400, detail=str(e)) ``` (`_workspace_for(None)` is the default workspace - see its first line.) Update the route docstring: `Pass acknowledged_cost as on POST /api/jobs; a bound one is checked against the stored spec's plan - the fresh seed of new_seed does not change a fingerprint.` @@ -1350,7 +1428,9 @@ def test_run_does_not_send_a_bare_true(): def test_run_refuses_a_bound_form_without_a_fingerprint(): client, seen = submitting() with pytest.raises(DwApiError, match="fingerprint"): - diagnose.run_workflow(client, workflow_path="w.json", acknowledged_cost={"minutes": 4}) + diagnose.run_workflow( + client, workflow_path="w.json", acknowledged_cost={"minutes": 4} + ) assert seen == [] @@ -1377,7 +1457,13 @@ def test_a_409_surfaces_with_the_new_estimate(): "list_entries": {"shots": 5}, "cached_steps": None, "downloads_required": [{"repo": "org/y", "gb": 3.5}], - "estimate": {"minutes": 19.0, "basis": "per_entry", "device": "cuda", "measured_on": "card", "partial": False}, + "estimate": { + "minutes": 19.0, + "basis": "per_entry", + "device": "cuda", + "measured_on": "card", + "partial": False, + }, }, } }, @@ -1427,8 +1513,8 @@ COST_REFUSAL = ( "with (free): its `plan` says what will execute - `estimate.minutes` with " "its `basis`, and any weights in `downloads_required` this box has to " "fetch first. Tell the user that number, get their go-ahead, then call " - "again with acknowledged_cost bound to the plan: {\"fingerprint\": " - "plan.fingerprint, \"minutes\": plan.estimate.minutes, \"downloads\": " + 'again with acknowledged_cost bound to the plan: {"fingerprint": ' + 'plan.fingerprint, "minutes": plan.estimate.minutes, "downloads": ' "[each downloads_required repo]} - the server then refuses (409) if the " "run's shape changed since. Pass true instead only when `plan` was null." ) diff --git a/docs/superpowers/specs/2026-03-26-ecosystem-utilities-design.md b/docs/superpowers/specs/2026-03-26-ecosystem-utilities-design.md index 1c28fb0d..8ccdbcb7 100644 --- a/docs/superpowers/specs/2026-03-26-ecosystem-utilities-design.md +++ b/docs/superpowers/specs/2026-03-26-ecosystem-utilities-design.md @@ -46,6 +46,7 @@ A new `segment` task command that takes an image + text prompt and returns a bin # In task.py from .segment import segment_image + @register_command("segment") def _handle_segment(task, arguments, previous_pipelines): """Segment objects in an image using text prompt""" @@ -160,6 +161,7 @@ A new `interpolate_frames` task that takes video frames and returns interpolated # In task.py from .interpolate_frames import interpolate_frames + @register_command("interpolate_frames") def _handle_interpolate_frames(task, arguments, previous_pipelines): """Interpolate video frames to increase frame rate""" diff --git a/docs/superpowers/specs/2026-09-04-remote-gpu-server-design.md b/docs/superpowers/specs/2026-09-04-remote-gpu-server-design.md index 1df0dd20..5e9416e0 100644 --- a/docs/superpowers/specs/2026-09-04-remote-gpu-server-design.md +++ b/docs/superpowers/specs/2026-09-04-remote-gpu-server-design.md @@ -110,19 +110,23 @@ there is no UI-style "paste the token" gate in front of it. ```python def mount_mcp(app, *, port, token): - from dw_mcp.client import DwClient # dw_mcp is a pure HTTP client - from dw_mcp.server import build_server # only module importing the SDK + from dw_mcp.client import DwClient # dw_mcp is a pure HTTP client + from dw_mcp.server import build_server # only module importing the SDK from mcp.server.transport_security import TransportSecuritySettings + client = DwClient(base_url=f"http://127.0.0.1:{port}", token=token) server = build_server(client) asgi = server.streamable_http_app( - streamable_http_path="/", # the SDK app routes at "/" ... + streamable_http_path="/", # the SDK app routes at "/" ... stateless_http=True, transport_security=TransportSecuritySettings( - enable_dns_rebinding_protection=False), # app.py's checks own this + enable_dns_rebinding_protection=False + ), # app.py's checks own this ) - app.mount("/mcp", asgi) # ... so the mount makes it "/mcp" (verified against mcp 2.1.1) - return client # closed in the app's lifespan + app.mount( + "/mcp", asgi + ) # ... so the mount makes it "/mcp" (verified against mcp 2.1.1) + return client # closed in the app's lifespan ``` The import of `mcp` is inside the function; a missing package produces diff --git a/docs/superpowers/specs/2026-09-08-job-record-and-export-design.md b/docs/superpowers/specs/2026-09-08-job-record-and-export-design.md index 976c670f..1cf0a030 100644 --- a/docs/superpowers/specs/2026-09-08-job-record-and-export-design.md +++ b/docs/superpowers/specs/2026-09-08-job-record-and-export-design.md @@ -38,8 +38,9 @@ that workflow and can export one job as a git-ready directory and a zip. New `dw/realize.py`: ```python -def realize_workflow(definition, arguments, seed, base_dir=None, - prompt_dir=None, output_root=None): +def realize_workflow( + definition, arguments, seed, base_dir=None, prompt_dir=None, output_root=None +): """A copy of `definition` with every mutable input pinned. Returns (realized, annotations).""" ``` @@ -77,8 +78,12 @@ In `Workflow.run`, immediately after the run directory is chosen ```python realized, annotations = realize_workflow( - self.workflow_definition, arguments, resolved_seed, - base_dir=self.base_dir, output_root=self.output_dir) + self.workflow_definition, + arguments, + resolved_seed, + base_dir=self.base_dir, + output_root=self.output_dir, +) write_realized_workflow(self._run_dir, realized) ``` diff --git a/docs/superpowers/specs/2026-09-12-acknowledged-cost-binding-design.md b/docs/superpowers/specs/2026-09-12-acknowledged-cost-binding-design.md index 6cf5f3d0..54c4b84e 100644 --- a/docs/superpowers/specs/2026-09-12-acknowledged-cost-binding-design.md +++ b/docs/superpowers/specs/2026-09-12-acknowledged-cost-binding-design.md @@ -65,7 +65,7 @@ def build_plan( workflow_dir=None, device, cache_dir=None, - cache_probe=None, # stage 2 + cache_probe=None, # stage 2 lookup_sizes=True, ): """What a run of `definition` with `arguments` will execute and cost.""" @@ -281,9 +281,9 @@ acknowledged_cost: bool | AcknowledgedCost | None = None ```python class AcknowledgedCost(BaseModel): - fingerprint: str # "sha256:…" from plan.fingerprint - minutes: float | None = None # plan.estimate.minutes, recorded only - downloads: list[str] = [] # plan.downloads_required[*].repo, non-null ones + fingerprint: str # "sha256:…" from plan.fingerprint + minutes: float | None = None # plan.estimate.minutes, recorded only + downloads: list[str] = [] # plan.downloads_required[*].repo, non-null ones ``` The form is classified once, in the route: diff --git a/dw/arguments.py b/dw/arguments.py index 4d57ff4b..a0da8751 100644 --- a/dw/arguments.py +++ b/dw/arguments.py @@ -906,7 +906,7 @@ def fetch_image(img_spec, base_dir=None): # If already a PIL Image, return as-is (allows multiple realize_args calls) if hasattr(img_spec, "mode") and hasattr(img_spec, "size"): - logger.debug(f"Image already loaded, returning as-is") + logger.debug("Image already loaded, returning as-is") return img_spec # Handle dict format: {"location": "url_or_path"} @@ -997,12 +997,12 @@ def fetch_video(video_spec, base_dir=None): return [fetch_video(vid, base_dir) for vid in video_spec] # Otherwise assume it's already loaded video frames else: - logger.debug(f"Video frames already loaded, returning as-is") + logger.debug("Video frames already loaded, returning as-is") return video_spec # If already loaded video frames (tuple), return as-is if isinstance(video_spec, tuple): - logger.debug(f"Video frames already loaded, returning as-is") + logger.debug("Video frames already loaded, returning as-is") return video_spec # Handle dict format: {"location": "url_or_path"} diff --git a/dw/repl.py b/dw/repl.py index ca52c4bd..75412691 100644 --- a/dw/repl.py +++ b/dw/repl.py @@ -261,7 +261,7 @@ def _print_memory_info(self, info): self._print_host_memory(info) return - print(f"\nGPU Memory Status:") + print("\nGPU Memory Status:") print(f" Device: {info.get('gpu_device_name', 'Unknown')}") print(f" Allocated: {info.get('gpu_memory_allocated_mb', 0):.1f} MB") print(f" Reserved: {info.get('gpu_memory_reserved_mb', 0):.1f} MB") @@ -279,7 +279,7 @@ def _print_host_memory(self, info): weights the card says very little about what a run is holding.""" if "host_memory_rss_mb" not in info: return - print(f"\nHost Memory:") + print("\nHost Memory:") print(f" Worker RSS: {info['host_memory_rss_mb']:.1f} MB") if "host_memory_peak_rss_mb" in info: print(f" Worker peak RSS: {info['host_memory_peak_rss_mb']:.1f} MB") diff --git a/dw/serve.py b/dw/serve.py index caed92cf..8c471eac 100644 --- a/dw/serve.py +++ b/dw/serve.py @@ -38,13 +38,12 @@ def main(): parser.add_argument( "--workflow-dir", default=None, - help="Directory of workflow JSON files (default: the workspace's " - "workflows/)", + help="Directory of workflow JSON files (default: the workspace's workflows/)", ) parser.add_argument( "--output-dir", default=None, - help="Directory results are written to (default: the workspace's " "outputs/)", + help="Directory results are written to (default: the workspace's outputs/)", ) parser.add_argument( "--prompt-dir", diff --git a/dw/server/app.py b/dw/server/app.py index 48a2b263..ac297b29 100644 --- a/dw/server/app.py +++ b/dw/server/app.py @@ -2108,8 +2108,6 @@ def enhance(request: EnhanceRequest, ws: Workspace = Depends(selected_workspace) # exactly once - the gallery had already drifted (.bmp, .mkv, .mov) from ..security import ( ALLOWED_AUDIO_EXTENSIONS, - ALLOWED_IMAGE_EXTENSIONS, - ALLOWED_VIDEO_EXTENSIONS, ) MEDIA_KINDS = { @@ -2267,8 +2265,12 @@ def _iter_gallery_files(root, group_runs=True): folder, _run_id, subfolder = split_run_path(relative_name) else: folder, subfolder = directory, "" - yield relative_name, folder, subfolder, kind, os.path.join( - current, name + yield ( + relative_name, + folder, + subfolder, + kind, + os.path.join(current, name), ) def _gallery_entries(root, ws): diff --git a/dw/server/guides.py b/dw/server/guides.py index 69a799c6..d0660e48 100644 --- a/dw/server/guides.py +++ b/dw/server/guides.py @@ -63,7 +63,7 @@ ), "prompt-weighting": ( "PROMPT_WEIGHTING.md", - "Emphasis and de-emphasis syntax in prompts, and which pipelines " "honour it.", + "Emphasis and de-emphasis syntax in prompts, and which pipelines honour it.", ), "ip-adapter": ( "IP_ADAPTER.md", diff --git a/dw/tasks/audio_utils.py b/dw/tasks/audio_utils.py index 29614f9e..89d101a5 100644 --- a/dw/tasks/audio_utils.py +++ b/dw/tasks/audio_utils.py @@ -16,8 +16,6 @@ from ..events import emit_warning from ..security import ( - validate_path, - validate_url, validate_file_extension, ALLOWED_AUDIO_EXTENSIONS, ) @@ -677,8 +675,7 @@ def match_levels(waveforms, measure, target_dbfs=None, command="concat_videos"): """ if measure not in MATCH_MEASURES: raise ValueError( - f"{command} 'match_levels' must be one of {MATCH_MEASURES}, " - f"got '{measure}'" + f"{command} 'match_levels' must be one of {MATCH_MEASURES}, got '{measure}'" ) if target_dbfs is None: target_dbfs = DEFAULT_MATCH_DBFS[measure] diff --git a/dw/tasks/image_utils.py b/dw/tasks/image_utils.py index 7ca0dd54..e31647b7 100644 --- a/dw/tasks/image_utils.py +++ b/dw/tasks/image_utils.py @@ -498,8 +498,8 @@ def add_watermark( "add_border_and_mask": lambda image, device, kwargs: add_border_and_mask( image, **kwargs ), - "add_border_and_mask_with_size": lambda image, device, kwargs: add_border_and_mask_with_size( - image, **kwargs + "add_border_and_mask_with_size": lambda image, device, kwargs: ( + add_border_and_mask_with_size(image, **kwargs) ), "remove_background": _remove_background_handler, # Raw cv2 Canny at native resolution - see image_to_canny() docstring diff --git a/dw/teacache.py b/dw/teacache.py index 0c4f8a32..c919809a 100644 --- a/dw/teacache.py +++ b/dw/teacache.py @@ -110,7 +110,12 @@ def teacache_forward( return_dict: bool = True, controlnet_blocks_repeat: bool = False, ) -> typing.Union[torch.FloatTensor, Transformer2DModelOutput]: - nonlocal cnt, accumulated_rel_l1_distance, previous_modulated_input, previous_residual, previous_timestep + nonlocal \ + cnt, \ + accumulated_rel_l1_distance, \ + previous_modulated_input, \ + previous_residual, \ + previous_timestep # TeaCache assumes exactly one transformer forward call per denoising # step. Pipelines running true classifier-free guidance (e.g. Flux with diff --git a/dw/worker.py b/dw/worker.py index 8e0533a5..52317726 100644 --- a/dw/worker.py +++ b/dw/worker.py @@ -19,7 +19,6 @@ from dw.assets import activate_asset_dir, deactivate_asset_dir from dw.log_setup import setup_logging, set_log_level from dw.settings import load_settings, resolve_path -from dw.security import validate_output_path from dw.events import RunContext, WorkflowCancelled from dw import get_device_type, empty_device_cache, device_memory_stats from dw.host_memory import ( diff --git a/dw/workflow.py b/dw/workflow.py index c81f5eb0..aebe8ee2 100644 --- a/dw/workflow.py +++ b/dw/workflow.py @@ -1009,7 +1009,7 @@ def run( # Execute each step in sequence for i, step_data in enumerate(steps): run_context.check_cancelled() - logger.debug(f"Running step {i+1}/{len(steps)}: {step_data['name']}") + logger.debug(f"Running step {i + 1}/{len(steps)}: {step_data['name']}") run_context.emit( "step_start", workflow=workflow_id, @@ -1364,11 +1364,11 @@ def create_step_action( logger.debug( "Setting up generator for cached pipeline with new arguments" ) - new_pipeline_wrapper.argument_template[ - "generator" - ] = torch.Generator(new_pipeline_wrapper.device).manual_seed( - new_pipeline_wrapper.pipeline_definition.get( - "seed", default_seed + new_pipeline_wrapper.argument_template["generator"] = ( + torch.Generator(new_pipeline_wrapper.device).manual_seed( + new_pipeline_wrapper.pipeline_definition.get( + "seed", default_seed + ) ) ) diff --git a/dw_mcp/diagnose.py b/dw_mcp/diagnose.py index 13a5d09c..75b39aed 100644 --- a/dw_mcp/diagnose.py +++ b/dw_mcp/diagnose.py @@ -256,8 +256,7 @@ def wait_for_job(client, job_id, timeout_seconds=20): remaining = deadline - time.monotonic() if remaining <= 0: next_step = ( - "Call wait_for_job again, or get_job_events for incremental " - "progress." + "Call wait_for_job again, or get_job_events for incremental progress." ) if capped: next_step = ( diff --git a/preflight.sh b/preflight.sh new file mode 100755 index 00000000..4c0f1570 --- /dev/null +++ b/preflight.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -uo pipefail + +cd "$(dirname "$0")" || exit 1 + +failures=() + +run_step() { + local name="$1" + shift + echo "==> $name" + if ! "$@"; then + failures+=("$name") + fi +} + +run_step "ruff format" ruff format . +run_step "ruff check" ruff check . --fix +run_step "pytest" python -m pytest +run_step "ui preflight" bash -c 'cd ui && npm run preflight' + +echo +if [ ${#failures[@]} -eq 0 ]; then + echo "All preflight checks passed." + exit 0 +else + echo "Preflight failures:" + for f in "${failures[@]}"; do + echo " - $f" + done + exit 1 +fi diff --git a/pyproject.toml b/pyproject.toml index 99177a80..11b1a16b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -95,7 +95,7 @@ dev = [ "pytest-mock>=3.15.1", "pytest-asyncio>=1.4.0", "httpx>=0.28.1", - "black>=26.5.1", + "ruff>=0.15.0", "build", "mcp>=2.1.1", ] @@ -111,8 +111,24 @@ dw-serve = "dw.serve:main" dw-test = "dw.test:main" dw-mcp = "dw_mcp.__main__:main" -[tool.black] -target-version = ["py310"] +[tool.ruff] +target-version = "py310" + +[tool.ruff.lint] +# Start conservative: pyflakes + pycodestyle only. Widen (e.g. add "I" for +# import sorting) once the existing codebase is clean under those too. +select = ["E", "F"] +# The formatter already wraps code to the line length; E501 mostly flags +# comments/strings/URLs that can't be split, so it's noise on top of that. +ignore = ["E501"] + +[tool.ruff.lint.per-file-ignores] +# These import a pytest fixture by name and reuse that name as a test +# function's parameter (the idiomatic way to share a fixture across +# modules) - pyflakes/ruff can't tell that from a genuine redefinition. +"tests/test_jobs_listing.py" = ["F811"] +"tests/test_server_downloads.py" = ["F811"] +"tests/test_server_exports.py" = ["F811"] [tool.setuptools.packages.find] # tasks/, pipeline_processors/ and community_pipelines/ are namespace diff --git a/requirements-test.txt b/requirements-test.txt index 410a50b8..db1d71aa 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,4 +1,4 @@ -# Test and development tooling (pytest and friends, black, flake8, build) +# Test and development tooling (pytest and friends, ruff, build) # # The list lives in pyproject.toml's [project.optional-dependencies] dev # extra - this file just points pip at it. diff --git a/tests/README.md b/tests/README.md index 2fe25d7e..be3d87bb 100644 --- a/tests/README.md +++ b/tests/README.md @@ -141,11 +141,12 @@ Example: import pytest from dw.my_module import my_function + class TestMyFunction: def test_normal_case(self): result = my_function("input") assert result == "expected" - + def test_error_case(self): with pytest.raises(ValueError): my_function("invalid") diff --git a/tests/conftest.py b/tests/conftest.py index 38affd26..4d4db639 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,5 @@ import pytest import os -import json import tempfile import warnings from PIL import Image diff --git a/tests/run_tests.py b/tests/run_tests.py index f216630a..5c99e279 100644 --- a/tests/run_tests.py +++ b/tests/run_tests.py @@ -4,7 +4,6 @@ """ import sys -import subprocess def main(): @@ -33,7 +32,7 @@ def main(): # Add coverage if pytest-cov is available try: - import pytest_cov + import pytest_cov # noqa: F401 - presence is the check args.extend( [ diff --git a/tests/test_argument_updates.py b/tests/test_argument_updates.py index 2c3b98d6..c6055914 100644 --- a/tests/test_argument_updates.py +++ b/tests/test_argument_updates.py @@ -7,7 +7,7 @@ import os import sys import logging -from unittest.mock import Mock, patch, MagicMock +from unittest.mock import patch, MagicMock # Add parent directory to path sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) @@ -52,8 +52,6 @@ def test_cached_pipeline_uses_new_arguments(): captured_arguments = [] original_pipeline_init = Pipeline.__init__ - original_pipeline_load = Pipeline.load - original_pipeline_run = Pipeline.run def mock_pipeline_init(self, *args, **kwargs): original_pipeline_init(self, *args, **kwargs) @@ -73,7 +71,6 @@ def mock_pipeline_run(self, arguments, *args, **kwargs): with patch.object(Pipeline, "__init__", mock_pipeline_init): with patch.object(Pipeline, "load", mock_pipeline_load): with patch.object(Pipeline, "run", mock_pipeline_run): - pipeline_cache = {} # First run - should create and cache pipeline @@ -88,7 +85,7 @@ def mock_pipeline_run(self, arguments, *args, **kwargs): # Simulate step.run() calling action.run() action1.run({"prompt": "a cat", "num_inference_steps": 20}, {}) - logger.info(f"✅ Run 1 complete") + logger.info("✅ Run 1 complete") logger.info(f" Prompt passed: '{captured_arguments[-1]['prompt']}'") logger.info( f" Steps passed: {captured_arguments[-1]['num_inference_steps']}" @@ -112,7 +109,7 @@ def mock_pipeline_run(self, arguments, *args, **kwargs): # Simulate step.run() calling action.run() action2.run({"prompt": "a dog", "num_inference_steps": 30}, {}) - logger.info(f"✅ Run 2 complete") + logger.info("✅ Run 2 complete") logger.info(f" Prompt passed: '{captured_arguments[-1]['prompt']}'") logger.info( f" Steps passed: {captured_arguments[-1]['num_inference_steps']}" @@ -123,28 +120,28 @@ def mock_pipeline_run(self, arguments, *args, **kwargs): logger.info("VERIFICATION") logger.info("=" * 60) - assert ( - len(captured_arguments) == 2 - ), f"Expected 2 runs, got {len(captured_arguments)}" - logger.info(f"✅ Both runs executed") + assert len(captured_arguments) == 2, ( + f"Expected 2 runs, got {len(captured_arguments)}" + ) + logger.info("✅ Both runs executed") run1_prompt = captured_arguments[0].get("prompt") run2_prompt = captured_arguments[1].get("prompt") - assert ( - run1_prompt == "a cat" - ), f"Run 1 should have 'a cat', got '{run1_prompt}'" + assert run1_prompt == "a cat", ( + f"Run 1 should have 'a cat', got '{run1_prompt}'" + ) logger.info(f"✅ Run 1 used correct prompt: '{run1_prompt}'") - assert ( - run2_prompt == "a dog" - ), f"Run 2 should have 'a dog', got '{run2_prompt}'" + assert run2_prompt == "a dog", ( + f"Run 2 should have 'a dog', got '{run2_prompt}'" + ) logger.info(f"✅ Run 2 used NEW prompt: '{run2_prompt}'") - assert ( - run1_prompt != run2_prompt - ), "Arguments should be different between runs!" - logger.info(f"✅ Arguments changed between runs") + assert run1_prompt != run2_prompt, ( + "Arguments should be different between runs!" + ) + logger.info("✅ Arguments changed between runs") run1_steps = captured_arguments[0].get("num_inference_steps") run2_steps = captured_arguments[1].get("num_inference_steps") @@ -184,7 +181,6 @@ def test_generator_seed_updates(): pipeline_cache = {} original_pipeline_init = Pipeline.__init__ - original_pipeline_load = Pipeline.load def mock_pipeline_init(self, *args, **kwargs): original_pipeline_init(self, *args, **kwargs) @@ -195,13 +191,12 @@ def mock_pipeline_load(self, *args, **kwargs): with patch.object(Pipeline, "__init__", mock_pipeline_init): with patch.object(Pipeline, "load", mock_pipeline_load): - logger.info("\n" + "=" * 60) logger.info("SEED TEST") logger.info("=" * 60) # Run 1 with seed 100 - action1 = workflow.create_step_action( + workflow.create_step_action( workflow_def["steps"][0], {}, pipeline_cache, 42, get_device() ) seed1 = workflow_def["steps"][0]["pipeline"].get("seed", 42) @@ -216,9 +211,9 @@ def mock_pipeline_load(self, *args, **kwargs): logger.info(f"Run 2: seed={seed2}") # Check that action2 has the new pipeline definition with seed 200 - assert ( - action2.pipeline_definition["seed"] == 200 - ), f"Expected seed 200, got {action2.pipeline_definition.get('seed')}" + assert action2.pipeline_definition["seed"] == 200, ( + f"Expected seed 200, got {action2.pipeline_definition.get('seed')}" + ) logger.info("✅ Pipeline wrapper gets updated seed") diff --git a/tests/test_cache_blocks.py b/tests/test_cache_blocks.py index e57d9ce1..c068a896 100644 --- a/tests/test_cache_blocks.py +++ b/tests/test_cache_blocks.py @@ -7,7 +7,6 @@ """ import inspect -import json import os import sys @@ -236,7 +235,6 @@ def _state_manager(model): def test_cache_state_needs_a_context(): """Without the context, the hook raises - the failure the wrapper prevents.""" - from dw.pipeline_processors.pipeline import stateful_cache_context manager = _state_manager(_cached_tiny_model()) diff --git a/tests/test_catalog_structure.py b/tests/test_catalog_structure.py index c11a107d..e7804cbf 100644 --- a/tests/test_catalog_structure.py +++ b/tests/test_catalog_structure.py @@ -54,9 +54,9 @@ def test_every_model_config_names_the_template_it_configures(path): assert configures, f"{path} has no 'configures'" target = os.path.join(REPO_ROOT, "workflows", f"{configures}.json") - assert os.path.isfile( - target - ), f"{path} configures '{configures}', which is not a workflow ({target})" + assert os.path.isfile(target), ( + f"{path} configures '{configures}', which is not a workflow ({target})" + ) def load(path): @@ -130,13 +130,13 @@ def test_the_rules_read_these_templates_as_expected(path, expected): def test_no_template_falls_through_to_utility(path): meta = derive_catalog_metadata(load(path)) if meta["shape"] == "utility": - assert ( - path in UTILITIES - ), f"{path} derived 'utility' - a rule missed it, or add it to UTILITIES" + assert path in UTILITIES, ( + f"{path} derived 'utility' - a rule missed it, or add it to UTILITIES" + ) else: - assert ( - path not in UTILITIES - ), f"{path} is listed as a utility but derives {meta['shape']}" + assert path not in UTILITIES, ( + f"{path} is listed as a utility but derives {meta['shape']}" + ) @pytest.mark.parametrize("path", TEMPLATES + MODEL_CONFIGS) @@ -150,9 +150,9 @@ def test_a_declaration_must_differ_from_the_derivation(path): } derived = derive_catalog_metadata(stripped) for key in meta["declared"]: - assert ( - meta[key] != derived[key] - ), f"{path} declares {key}={meta[key]!r}, which derivation already produces" + assert meta[key] != derived[key], ( + f"{path} declares {key}={meta[key]!r}, which derivation already produces" + ) @pytest.mark.parametrize("path", TEMPLATES) @@ -160,9 +160,9 @@ def test_every_template_has_a_summary_that_fits(path): meta = derive_catalog_metadata(load(path)) assert meta["summary"], f"{path}: description has no first sentence" assert len(meta["summary"]) <= SUMMARY_LIMIT - assert not meta[ - "summary_truncated" - ], f"{path}: first sentence runs past {SUMMARY_LIMIT} chars - shorten it or declare 'summary': {meta['summary']!r}" + assert not meta["summary_truncated"], ( + f"{path}: first sentence runs past {SUMMARY_LIMIT} chars - shorten it or declare 'summary': {meta['summary']!r}" + ) BUILTINS = sorted( @@ -179,9 +179,9 @@ def test_workflow_ids_are_unique_across_the_catalog(): for path in TEMPLATES + MODEL_CONFIGS + BUILTINS: identity = load(path).get("id") assert identity, f"{path} has no id" - assert ( - identity not in seen - ), f"{path} and {seen[identity]} share id {identity!r}" + assert identity not in seen, ( + f"{path} and {seen[identity]} share id {identity!r}" + ) seen[identity] = path @@ -190,17 +190,17 @@ def test_a_declared_cost_is_well_formed(path): cost = load(path).get("cost") if cost is None: return - assert ( - isinstance(cost, list) and cost - ), f"{path}: cost must be a non-empty list or absent" + assert isinstance(cost, list) and cost, ( + f"{path}: cost must be a non-empty list or absent" + ) for entry in cost: assert entry["device"] in ("cuda", "mps", "cpu"), path - assert ( - isinstance(entry["vram_gb"], (int, float)) and entry["vram_gb"] >= 0 - ), path - assert ( - isinstance(entry["minutes"], (int, float)) and entry["minutes"] >= 0 - ), path + assert isinstance(entry["vram_gb"], (int, float)) and entry["vram_gb"] >= 0, ( + path + ) + assert isinstance(entry["minutes"], (int, float)) and entry["minutes"] >= 0, ( + path + ) def per_entry_problems(definition): @@ -318,9 +318,9 @@ def test_a_description_names_only_variables_the_workflow_declares(path): catalog_variables = _variable_names_in_catalog() allowed = LEGITIMATE_MENTIONS.get(path, set()) undeclared = (_mentioned(definition) & catalog_variables) - declared - allowed - assert ( - not undeclared - ), f"{path} describes {sorted(undeclared)} but declares no such variable" + assert not undeclared, ( + f"{path} describes {sorted(undeclared)} but declares no such variable" + ) def test_the_drift_check_actually_matches_something(): @@ -331,19 +331,19 @@ def test_the_drift_check_actually_matches_something(): for path in TEMPLATES if _mentioned(load(path)) & _variable_names_in_catalog() ] - assert ( - matched - ), "no template description quotes a catalog variable name - the pattern is wrong" + assert matched, ( + "no template description quotes a catalog variable name - the pattern is wrong" + ) def test_no_stale_entry_in_the_allowlist(): for path, names in LEGITIMATE_MENTIONS.items(): - assert ( - path in TEMPLATES + MODEL_CONFIGS - ), f"{path} is allowlisted but not in the catalog" - assert names <= _mentioned( - load(path) - ), f"{path} no longer mentions {sorted(names - _mentioned(load(path)))}" + assert path in TEMPLATES + MODEL_CONFIGS, ( + f"{path} is allowlisted but not in the catalog" + ) + assert names <= _mentioned(load(path)), ( + f"{path} no longer mentions {sorted(names - _mentioned(load(path)))}" + ) # Spec targets, as chars / 4. The listing is the first thing an agent reads; @@ -363,15 +363,15 @@ def test_the_compact_listing_fits_the_budget(): details = workflow_details(found) compact = project_listing(details, view="compact") - assert ( - _tokens(compact) <= COMPACT_BUDGET - ), f"compact listing is {_tokens(compact):.0f} tokens" + assert _tokens(compact) <= COMPACT_BUDGET, ( + f"compact listing is {_tokens(compact):.0f} tokens" + ) sequences = project_listing(details, view="compact", shape="sequence") assert sequences, "no template derives 'sequence'" - assert ( - _tokens(sequences) <= FILTERED_BUDGET - ), f"shape=sequence is {_tokens(sequences):.0f} tokens" + assert _tokens(sequences) <= FILTERED_BUDGET, ( + f"shape=sequence is {_tokens(sequences):.0f} tokens" + ) def _walk(value): @@ -490,9 +490,9 @@ def test_every_readme_link_resolves(path): if target.startswith(("http://", "https://", "#")): continue target = target.split("#", 1)[0] - assert os.path.exists( - os.path.join(base, target) - ), f"{path} links to {target}, which does not exist" + assert os.path.exists(os.path.join(base, target)), ( + f"{path} links to {target}, which does not exist" + ) COSTED = { diff --git a/tests/test_chain.py b/tests/test_chain.py index 102caaaf..b504ce82 100644 --- a/tests/test_chain.py +++ b/tests/test_chain.py @@ -10,14 +10,12 @@ from dataclasses import dataclass from types import SimpleNamespace -import numpy import pytest import torch from PIL import Image from dw.pipeline_processors.chain import ( ChainConfig, - Segment, plan_segments, run_chain, snap_frames, diff --git a/tests/test_configuration_schema.py b/tests/test_configuration_schema.py index 07da0355..c13cb271 100644 --- a/tests/test_configuration_schema.py +++ b/tests/test_configuration_schema.py @@ -61,7 +61,9 @@ def configuration_keys_read_by_the_code(): name = ( target.id if isinstance(target, ast.Name) - else target.attr if isinstance(target, ast.Attribute) else None + else target.attr + if isinstance(target, ast.Attribute) + else None ) if name not in ("configuration", "component_configuration"): continue diff --git a/tests/test_device.py b/tests/test_device.py index 729f25ef..860ec66f 100644 --- a/tests/test_device.py +++ b/tests/test_device.py @@ -3,7 +3,6 @@ Tests the configured device override and device type resolution """ -import pytest import dw diff --git a/tests/test_diffusion_upscale.py b/tests/test_diffusion_upscale.py index b7c3d830..591c1896 100644 --- a/tests/test_diffusion_upscale.py +++ b/tests/test_diffusion_upscale.py @@ -48,9 +48,7 @@ def test_default_mode_is_x4(self, mock_diffusers): @patch("dw.tasks.diffusion_upscale.diffusers") def test_x2_mode_uses_latent_pipeline(self, mock_diffusers): mock_pipe = self._make_mock_pipeline() - mock_diffusers.StableDiffusionLatentUpscalePipeline.from_pretrained.return_value = ( - mock_pipe - ) + mock_diffusers.StableDiffusionLatentUpscalePipeline.from_pretrained.return_value = mock_pipe diffusion_upscale(self._make_image(), device="cpu", mode="x2") @@ -72,9 +70,7 @@ def test_x4_includes_noise_level(self, mock_diffusers): @patch("dw.tasks.diffusion_upscale.diffusers") def test_x2_excludes_noise_level(self, mock_diffusers): mock_pipe = self._make_mock_pipeline() - mock_diffusers.StableDiffusionLatentUpscalePipeline.from_pretrained.return_value = ( - mock_pipe - ) + mock_diffusers.StableDiffusionLatentUpscalePipeline.from_pretrained.return_value = mock_pipe diffusion_upscale(self._make_image(), device="cpu", mode="x2") diff --git a/tests/test_gather.py b/tests/test_gather.py index 957f230a..a084a630 100644 --- a/tests/test_gather.py +++ b/tests/test_gather.py @@ -6,7 +6,7 @@ import pytest import os import tempfile -from unittest.mock import patch, MagicMock +from unittest.mock import patch from PIL import Image from dw.tasks.gather import gather_images, gather_videos, gather_inputs from dw.result import AudioVideo @@ -207,7 +207,7 @@ def test_a_gathered_video_is_one_artifact(self, tmp_path): step that consumed them out over frames instead of videos.""" from dw.result import get_artifact_list - path = write_video(tmp_path / "shot.mp4") + write_video(tmp_path / "shot.mp4") videos = gather_videos(glob=os.path.join(str(tmp_path), "*.mp4")) @@ -218,7 +218,7 @@ def test_a_gathered_video_is_one_artifact(self, tmp_path): def test_a_gathered_video_keeps_its_audio(self, tmp_path): """The audio muxed into the file is what an earlier run generated alongside the picture - gathering it silent loses that run's work.""" - path = write_video(tmp_path / "shot.mp4", sample_rate=8000) + write_video(tmp_path / "shot.mp4", sample_rate=8000) video = gather_videos(glob=os.path.join(str(tmp_path), "*.mp4"))[0] diff --git a/tests/test_integration.py b/tests/test_integration.py index 0d22f501..612663d8 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -8,7 +8,6 @@ import json import tempfile from dw.workflow import Workflow, workflow_from_file -from dw.result import Result @pytest.fixture diff --git a/tests/test_introspection.py b/tests/test_introspection.py index eb4f1024..9e7616b3 100644 --- a/tests/test_introspection.py +++ b/tests/test_introspection.py @@ -87,8 +87,7 @@ def test_a_reference_naming_no_variable_is_warned_about_before_anything_loads(): warnings = workflow_argument_warnings(workflow) assert len(warnings) == 2 assert warnings[0] == ( - "seed: 'variable:seed' names no declared variable; " - "declared: base_prompt, steps" + "seed: 'variable:seed' names no declared variable; declared: base_prompt, steps" ) assert warnings[1].startswith( "steps[0].pipeline.arguments.prompt: 'variable:base_prompt, clear sky' " diff --git a/tests/test_ltx_prompt_library.py b/tests/test_ltx_prompt_library.py index b0f091c4..eb72c457 100644 --- a/tests/test_ltx_prompt_library.py +++ b/tests/test_ltx_prompt_library.py @@ -45,9 +45,9 @@ def test_a_prompt_is_one_paragraph_of_caption_length(path): assert "\n" not in text.strip(), f"{path} is more than one paragraph" words = len(text.split()) - assert ( - 140 <= words <= 240 - ), f"{path} is {words} words; the trained caption is 150-220" + assert 140 <= words <= 240, ( + f"{path} is {words} words; the trained caption is 150-220" + ) @pytest.mark.parametrize("path", PROMPTS, ids=os.path.basename) diff --git a/tests/test_mcp_main.py b/tests/test_mcp_main.py index f900f813..e1cd1944 100644 --- a/tests/test_mcp_main.py +++ b/tests/test_mcp_main.py @@ -1,7 +1,5 @@ """`dw-mcp` startup: what it refuses, what it checks, what it prints.""" -import json - import httpx import pytest diff --git a/tests/test_mcp_media.py b/tests/test_mcp_media.py index b207bdcb..65388ed4 100644 --- a/tests/test_mcp_media.py +++ b/tests/test_mcp_media.py @@ -3,6 +3,7 @@ import base64 import io +import os import httpx import numpy as np @@ -11,7 +12,7 @@ import dw_mcp.media as media from dw_mcp.client import DwApiError, DwClient -from dw_mcp.media import MAX_RETURNED_BYTES, get_output_image +from dw_mcp.media import MAX_RETURNED_BYTES, download_output, get_output_image def noise_png_bytes(width, height, seed=0): @@ -380,11 +381,6 @@ def handler(request): # --------------------------------------------------------- output download -import os - -from dw_mcp.media import download_output - - def test_download_output_writes_bytes_to_explicit_file_path(tmp_path): client = serving(png_bytes(64, 48), "image/png") destination = tmp_path / "saved.png" diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index cce69812..45231dd2 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -590,9 +590,9 @@ async def test_optional_parameters_are_declared_nullable(): for name, tool in tools.items(): for parameter, schema in tool.input_schema["properties"].items(): if schema.get("default", "missing") is None: - assert ( - "anyOf" in schema or schema.get("type") == "null" - ), f"{name}.{parameter} defaults to null but is not nullable" + assert "anyOf" in schema or schema.get("type") == "null", ( + f"{name}.{parameter} defaults to null but is not nullable" + ) @pytest.mark.asyncio diff --git a/tests/test_pipeline_caching.py b/tests/test_pipeline_caching.py index f51413ca..5a53dacb 100644 --- a/tests/test_pipeline_caching.py +++ b/tests/test_pipeline_caching.py @@ -8,7 +8,7 @@ import sys import logging import pytest -from unittest.mock import Mock, patch, MagicMock +from unittest.mock import patch, MagicMock # Add parent directory to path sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) @@ -57,7 +57,6 @@ def test_pipeline_caching(): loaded_models = {} # Track loaded models by step name original_pipeline_init = Pipeline.__init__ - original_pipeline_load = Pipeline.load def mock_pipeline_init(self, *args, **kwargs): # Extract the pipeline argument before calling original init @@ -116,22 +115,22 @@ def mock_pipeline_load(self, *args, **kwargs): logger.info("VERIFICATION") logger.info("=" * 60) - assert ( - first_load_count == 1 - ), f"Expected 1 load on first run, got {first_load_count}" - logger.info(f"✅ First run loaded exactly once") + assert first_load_count == 1, ( + f"Expected 1 load on first run, got {first_load_count}" + ) + logger.info("✅ First run loaded exactly once") - assert ( - second_load_count == 1 - ), f"Expected no additional loads on second run, got {second_load_count}" - logger.info(f"✅ Second run reused cached pipeline (no reload)") + assert second_load_count == 1, ( + f"Expected no additional loads on second run, got {second_load_count}" + ) + logger.info("✅ Second run reused cached pipeline (no reload)") # Note: We now create a new wrapper but reuse the underlying model - assert ( - action1.pipeline is action2.pipeline - ), "Expected same underlying pipeline model to be reused" + assert action1.pipeline is action2.pipeline, ( + "Expected same underlying pipeline model to be reused" + ) logger.info( - f"✅ Both runs reused the same underlying model (pipeline.pipeline)" + "✅ Both runs reused the same underlying model (pipeline.pipeline)" ) logger.info("\n" + "=" * 60) @@ -170,7 +169,6 @@ def test_pipeline_caching_different_steps(): load_call_count = 0 original_pipeline_init = Pipeline.__init__ - original_pipeline_load = Pipeline.load def mock_pipeline_init(self, *args, **kwargs): # Check if pipeline is being reused @@ -210,15 +208,15 @@ def mock_pipeline_load(self, *args, **kwargs): ) logger.info(f"✅ Step1 reused: load_count={load_call_count}") - assert ( - load_call_count == 2 - ), f"Expected 2 loads (one per step), got {load_call_count}" - assert ( - action1.pipeline is action1_reuse.pipeline - ), "Step1 underlying model should be reused from cache" - assert ( - action1.pipeline is not action2.pipeline - ), "Step1 and step2 should have different underlying models" + assert load_call_count == 2, ( + f"Expected 2 loads (one per step), got {load_call_count}" + ) + assert action1.pipeline is action1_reuse.pipeline, ( + "Step1 underlying model should be reused from cache" + ) + assert action1.pipeline is not action2.pipeline, ( + "Step1 and step2 should have different underlying models" + ) logger.info("\n" + "=" * 60) logger.info("🎉 MULTI-STEP TEST PASSED!") @@ -441,9 +439,9 @@ def mock_load(self, shared_components): with patch.object(Pipeline, "load", mock_load): workflow.create_step_action(new_step, {}, cache, 1, "cpu") - assert ( - seen_at_load["old_still_cached"] is False - ), "the redefined step's previous model must be evicted before load" + assert seen_at_load["old_still_cached"] is False, ( + "the redefined step's previous model must be evicted before load" + ) def test_pipeline_released_is_reported_on_the_event_stream(): diff --git a/tests/test_pipeline_components.py b/tests/test_pipeline_components.py index ba0a2ab1..f254f9ec 100644 --- a/tests/test_pipeline_components.py +++ b/tests/test_pipeline_components.py @@ -94,7 +94,9 @@ def test_callback_is_wired_to_the_pipeline(self): def test_an_explicit_callback_is_left_alone(self): from diffusers import FasterCacheConfig - callback = lambda: 3 + def callback(): + return 3 + config = FasterCacheConfig(current_timestep_callback=callback) enable_cache_on_transformer(MagicMock(), config) diff --git a/tests/test_plan.py b/tests/test_plan.py index 4a853706..d9cfaae5 100644 --- a/tests/test_plan.py +++ b/tests/test_plan.py @@ -448,9 +448,9 @@ def test_a_local_path_is_not_a_download_and_is_not_probed( names = (str(tmp_path / "weights"), "/Users/someone/.ssh", "./weights") for local in names: spec = definition() - spec["steps"][0]["pipeline"]["from_pretrained_arguments"][ - "model_name" - ] = local + spec["steps"][0]["pipeline"]["from_pretrained_arguments"]["model_name"] = ( + local + ) assert plan(spec)["downloads_required"] == [], local assert not set(names) & set(probed) diff --git a/tests/test_plugin_skills.py b/tests/test_plugin_skills.py index ba3fd663..e05e3533 100644 --- a/tests/test_plugin_skills.py +++ b/tests/test_plugin_skills.py @@ -90,9 +90,9 @@ def test_a_skill_has_a_triggering_description_under_the_size_cap(path): assert fields["name"] == os.path.basename(os.path.dirname(path)) assert "description" in fields and len(fields["description"]) > 40 - assert ( - len(text.encode("utf-8")) <= SKILL_SIZE_LIMIT - ), f"{path} is over {SKILL_SIZE_LIMIT} bytes" + assert len(text.encode("utf-8")) <= SKILL_SIZE_LIMIT, ( + f"{path} is over {SKILL_SIZE_LIMIT} bytes" + ) @pytest.mark.parametrize( @@ -107,9 +107,9 @@ def test_every_catalog_name_a_skill_quotes_resolves(path): target = os.path.join( REPO_ROOT, "workflows", name.removesuffix(".json") + ".json" ) - assert os.path.isfile( - target - ), f"{path} quotes {name}, which is not a workflow ({target})" + assert os.path.isfile(target), ( + f"{path} quotes {name}, which is not a workflow ({target})" + ) H3_SKILL = os.path.join(PLUGIN_DIR, "skills", "minimax-h3", "SKILL.md") @@ -241,9 +241,9 @@ def test_the_lora_coupling_is_scoped_to_the_turbo_templates(self): ) spec = open(path, encoding="utf-8").read() assert "lora_model_name" not in spec, f"{name} now loads a LoRA" - assert ( - '"num_inference_steps": 20' in spec - ), f"{name} no longer runs 20 steps" + assert '"num_inference_steps": 20' in spec, ( + f"{name} no longer runs 20 steps" + ) json.loads(spec) def test_the_skill_defers_prompt_format_to_minimax(self): @@ -431,9 +431,9 @@ def test_a_skill_states_the_subfolder_convention(path): composing a new workflow needs to keep it.""" text = skill_text(path) assert "`subfolder`" in text, f"{path} does not name the subfolder field" - assert ( - "`final`" in text and "`intermediate`" in text - ), f"{path} does not state the final/intermediate convention" + assert "`final`" in text and "`intermediate`" in text, ( + f"{path} does not state the final/intermediate convention" + ) # the convention is stated where the manifest is read assert text.index("`subfolder`") > text.index("## Run and judge") @@ -455,6 +455,6 @@ def test_the_h3_skill_names_each_cut_templates_final_step(): if (step.get("result") or {}).get("subfolder") == "final" ] assert len(finals) == 1, (name, finals) - assert ( - f"`{finals[0]}`" in text - ), f"the skill does not name {name}'s final step {finals[0]}" + assert f"`{finals[0]}`" in text, ( + f"the skill does not name {name}'s final step {finals[0]}" + ) diff --git a/tests/test_realize.py b/tests/test_realize.py index 6740d8f9..cec60148 100644 --- a/tests/test_realize.py +++ b/tests/test_realize.py @@ -4,7 +4,6 @@ import copy import hashlib import json -import os import pytest @@ -101,9 +100,9 @@ def test_a_stored_prompt_is_inlined_and_annotated(self, prompt_library): def test_a_name_is_annotated_once_in_first_seen_order(self, prompt_library): source = definition() source["steps"][0]["pipeline"]["arguments"]["prompt"] = "prompt:scenic/dusk" - source["steps"][0]["pipeline"]["arguments"][ - "negative_prompt" - ] = "prompt:scenic/dusk" + source["steps"][0]["pipeline"]["arguments"]["negative_prompt"] = ( + "prompt:scenic/dusk" + ) _, annotations = realize_workflow(source, {}, 7, prompt_dir=prompt_library) @@ -127,9 +126,9 @@ class TestOutputReferences: def test_latest_is_pinned_to_the_run_it_resolved_to(self, output_root): root, run_id = output_root source = definition() - source["steps"][0]["pipeline"]["arguments"][ - "image" - ] = "output:ltx2/Gyre/latest/still.png" + source["steps"][0]["pipeline"]["arguments"]["image"] = ( + "output:ltx2/Gyre/latest/still.png" + ) realized, _ = realize_workflow(source, {}, 7, output_root=root) @@ -248,9 +247,9 @@ class TestUnpinnedOutputs: def test_pin_outputs_false_leaves_latest_as_written(self, output_root): root, _ = output_root spec = definition() - spec["steps"][0]["pipeline"]["arguments"][ - "image" - ] = "output:ltx2/Gyre/latest/still.png" + spec["steps"][0]["pipeline"]["arguments"]["image"] = ( + "output:ltx2/Gyre/latest/still.png" + ) realized, _ = realize_workflow(spec, {}, 7, output_root=root, pin_outputs=False) assert ( realized["steps"][0]["pipeline"]["arguments"]["image"] @@ -269,9 +268,9 @@ def test_pin_outputs_false_still_inlines_prompts(self, prompt_library): def test_the_default_still_pins(self, output_root): root, run_id = output_root spec = definition() - spec["steps"][0]["pipeline"]["arguments"][ - "image" - ] = "output:ltx2/Gyre/latest/still.png" + spec["steps"][0]["pipeline"]["arguments"]["image"] = ( + "output:ltx2/Gyre/latest/still.png" + ) realized, _ = realize_workflow(spec, {}, 7, output_root=root) assert ( realized["steps"][0]["pipeline"]["arguments"]["image"] diff --git a/tests/test_repl_commands.py b/tests/test_repl_commands.py index 4bf046ae..429b6038 100644 --- a/tests/test_repl_commands.py +++ b/tests/test_repl_commands.py @@ -36,10 +36,10 @@ def test_repl_commands(): print("=" * 70) for cmd, description in test_commands: - print(f"\n{'='*70}") + print(f"\n{'=' * 70}") print(f"Test: {description}") print(f"Command: {cmd}") - print(f"{'='*70}") + print(f"{'=' * 70}") repl.onecmd(cmd) print("\n" + "=" * 70) diff --git a/tests/test_repl_reorganization.py b/tests/test_repl_reorganization.py index 3e0869ed..f50408da 100644 --- a/tests/test_repl_reorganization.py +++ b/tests/test_repl_reorganization.py @@ -32,11 +32,11 @@ def test_command_equivalence(): for test in tests: repl = DiffusersWorkflowREPL() # Fresh REPL for each test - print(f"\n{'-'*70}") + print(f"\n{'-' * 70}") print(f"Test: {test['description']}") print(f"Old command: {test['old']}") print(f"New command: {test['new']}") - print(f"{'-'*70}") + print(f"{'-' * 70}") # The output should be the same print("Old command output:") @@ -64,9 +64,9 @@ def test_help_system(): commands = ["workflow", "arg", "model", "memory", "config"] for cmd in commands: - print(f"\n{'-'*70}") + print(f"\n{'-' * 70}") print(f"Testing: {cmd} ?") - print(f"{'-'*70}") + print(f"{'-' * 70}") repl.onecmd(f"{cmd} ?") print(f"✅ Help for '{cmd}' works") @@ -93,10 +93,10 @@ def test_command_flow(): ] for cmd, description in flow: - print(f"\n{'-'*70}") + print(f"\n{'-' * 70}") print(f"Step: {description}") print(f"Command: {cmd}") - print(f"{'-'*70}") + print(f"{'-' * 70}") repl.onecmd(cmd) print(f"✅ {description} - OK") diff --git a/tests/test_result_output_naming.py b/tests/test_result_output_naming.py index fa251a1b..70d0cfa7 100644 --- a/tests/test_result_output_naming.py +++ b/tests/test_result_output_naming.py @@ -1,5 +1,3 @@ -import os - from dw.result import output_file_path diff --git a/tests/test_segment.py b/tests/test_segment.py index fb1bba5c..854a42bd 100644 --- a/tests/test_segment.py +++ b/tests/test_segment.py @@ -1,4 +1,3 @@ -import pytest from unittest.mock import patch, MagicMock import torch import numpy as np diff --git a/tests/test_server.py b/tests/test_server.py index 3f3bc917..4cec7dbb 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -112,9 +112,7 @@ class DyingWorkerManager(ScriptedWorkerManager): """A worker killed by a signal: it sends nothing, and get_result raises the way the real WorkerManager's liveness poll does.""" - def __init__( - self, detail="killed by SIGKILL (typically the " "out-of-memory killer)" - ): + def __init__(self, detail="killed by SIGKILL (typically the out-of-memory killer)"): super().__init__(script=lambda command: []) self.detail = detail self.crashed = False @@ -514,9 +512,9 @@ def test_validate_accepts_a_stored_workflow_name(server, tmp_path): # the stored file is what gets checked, warnings and all typo = valid_workflow("typo") - typo["steps"][0]["pipeline"]["configuration"][ - "component_type" - ] = "ZImagePipeline" + typo["steps"][0]["pipeline"]["configuration"]["component_type"] = ( + "ZImagePipeline" + ) typo["steps"][0]["pipeline"]["arguments"]["guidance_scael"] = 3 (tmp_path / "workflows" / "Typo.json").write_text(json.dumps(typo)) @@ -863,9 +861,9 @@ def test_introspection_endpoints(server): def test_validate_endpoint_flags_signature_typos(server): with server(success_script) as client: workflow = valid_workflow() - workflow["steps"][0]["pipeline"]["configuration"][ - "component_type" - ] = "ZImagePipeline" + workflow["steps"][0]["pipeline"]["configuration"]["component_type"] = ( + "ZImagePipeline" + ) workflow["steps"][0]["pipeline"]["arguments"]["guidance_scael"] = 3 result = client.post("/api/validate", json={"workflow": workflow}).json() @@ -897,9 +895,9 @@ def test_validate_explains_why_an_unseeded_workflow_caches_nothing(server): def test_submission_carries_argument_warnings(server): with server(success_script) as client: workflow = valid_workflow() - workflow["steps"][0]["pipeline"]["configuration"][ - "component_type" - ] = "ZImagePipeline" + workflow["steps"][0]["pipeline"]["configuration"]["component_type"] = ( + "ZImagePipeline" + ) workflow["steps"][0]["pipeline"]["arguments"]["guidance_scael"] = 3 job = client.post("/api/jobs", json={"workflow": workflow}).json() assert any("guidance_scael" in w for w in job["warnings"]) diff --git a/tests/test_server_exports.py b/tests/test_server_exports.py index 6727648a..ef23ebe6 100644 --- a/tests/test_server_exports.py +++ b/tests/test_server_exports.py @@ -17,7 +17,7 @@ from .test_server import ( ScriptedWorkerManager, hanging_script, - server as workspace_less_server, + server as workspace_less_server, # noqa: F401 - used as a fixture by name success_script, valid_workflow, wait_for_status, diff --git a/tests/test_server_workspaces.py b/tests/test_server_workspaces.py index 2038d962..1408e271 100644 --- a/tests/test_server_workspaces.py +++ b/tests/test_server_workspaces.py @@ -2,7 +2,6 @@ root, each with its own workflows, assets and outputs, all sharing the one prompt library.""" -import json import os import pytest diff --git a/tests/test_step.py b/tests/test_step.py index 5b893037..78b8e015 100644 --- a/tests/test_step.py +++ b/tests/test_step.py @@ -4,7 +4,7 @@ """ import pytest -from unittest.mock import Mock, MagicMock +from unittest.mock import Mock from dw.step import Step from dw.result import Result @@ -141,7 +141,7 @@ def capture_args(args, pipelines): previous_results = {"images": images_result, "prompts": prompts_result} previous_pipelines = {} - result = step.run(previous_results, previous_pipelines, mock_action) + step.run(previous_results, previous_pipelines, mock_action) # Should create 2x2 = 4 combinations assert mock_action.run.call_count == 4 diff --git a/tests/test_task.py b/tests/test_task.py index 707e03d6..1fd38bb1 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -98,9 +98,9 @@ def test_format_chat_message_task(): result = task.run(task_def["arguments"]) # Check the overall structure - assert isinstance( - result, dict - ), "Expected a formatted dict from format_chat_message" + assert isinstance(result, dict), ( + "Expected a formatted dict from format_chat_message" + ) assert "text_inputs" in result, "Result should contain text_inputs key" # Check the text_inputs array structure @@ -110,9 +110,9 @@ def test_format_chat_message_task(): # Check system message assert text_inputs[0]["role"] == "system", "First message should have role 'system'" - assert ( - text_inputs[0]["content"] == "Hello, world!" - ), "System message content mismatch" + assert text_inputs[0]["content"] == "Hello, world!", ( + "System message content mismatch" + ) # Check user message assert text_inputs[1]["role"] == "user", "Second message should have role 'user'" diff --git a/tests/test_template_subfolders.py b/tests/test_template_subfolders.py index 860a696f..c366f42c 100644 --- a/tests/test_template_subfolders.py +++ b/tests/test_template_subfolders.py @@ -60,9 +60,9 @@ def test_every_saving_step_of_a_multi_step_template_names_its_role(template): roles = {step["name"]: step["result"].get("subfolder") for step in steps} unmarked = sorted(name for name, role in roles.items() if role not in CONVENTION) - assert ( - not unmarked - ), f"{template}: saving steps without a final/intermediate subfolder: {unmarked}" + assert not unmarked, ( + f"{template}: saving steps without a final/intermediate subfolder: {unmarked}" + ) assert "final" in roles.values(), f"{template}: no step is marked final" @@ -70,9 +70,9 @@ def test_every_saving_step_of_a_multi_step_template_names_its_role(template): def test_the_subfolder_is_the_last_key_of_the_result(template): """One added line per step, and every template reads alike.""" for step in saving_steps(load(template)): - assert ( - list(step["result"])[-1] == "subfolder" - ), f"{template}: step {step['name']!r} does not end its result with subfolder" + assert list(step["result"])[-1] == "subfolder", ( + f"{template}: step {step['name']!r} does not end its result with subfolder" + ) @pytest.mark.parametrize( @@ -87,6 +87,6 @@ def test_the_packaged_builtins_stay_unmarked(builtin): for step in definition.get("steps", []) if "subfolder" in (step.get("result") or {}) ] - assert ( - not marked - ), f"dw/workflows/{builtin} marks {marked}; a role is the parent's to assign" + assert not marked, ( + f"dw/workflows/{builtin} marks {marked}; a role is the parent's to assign" + ) diff --git a/tests/test_tensor_image.py b/tests/test_tensor_image.py index 647f875a..20ad2a60 100644 --- a/tests/test_tensor_image.py +++ b/tests/test_tensor_image.py @@ -1,5 +1,4 @@ import numpy as np -import pytest import torch from PIL import Image diff --git a/tests/test_type_helpers.py b/tests/test_type_helpers.py index 57ad8782..c8208024 100644 --- a/tests/test_type_helpers.py +++ b/tests/test_type_helpers.py @@ -6,7 +6,6 @@ import pytest from dw.type_helpers import ( get_type, - load_type_from_name, load_type_from_full_name, has_method, ) @@ -18,7 +17,6 @@ class TestGetType: def test_get_type_from_diffusers(self): # This would work if diffusers is installed # For testing, we'll use a built-in type - import sys result = get_type("sys", "version") assert result is not None diff --git a/tests/test_video_utils.py b/tests/test_video_utils.py index 86b3684d..fd59d335 100644 --- a/tests/test_video_utils.py +++ b/tests/test_video_utils.py @@ -53,8 +53,9 @@ def test_get_last_frame(self, video): def test_get_last_frame_ignores_a_frame_index(self, video): # get_last_frame computes its own index; a stray argument must not win - assert process_video(video, "get_last_frame", "cpu", {"frame_index": 0}) is ( - video[3] + assert ( + process_video(video, "get_last_frame", "cpu", {"frame_index": 0}) + is (video[3]) ) @pytest.mark.parametrize("name", ["GET_LAST_FRAME", "Get_Last_Frame"]) diff --git a/tests/test_workflow_step_cache.py b/tests/test_workflow_step_cache.py index 5c80250f..35862e0a 100644 --- a/tests/test_workflow_step_cache.py +++ b/tests/test_workflow_step_cache.py @@ -7,7 +7,6 @@ import os from unittest.mock import MagicMock, patch -import pytest from dw.events import RunContext from dw.step_cache import step_cache