diff --git a/docs/MCP.md b/docs/MCP.md index d8f93e3..fd3937a 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -225,7 +225,7 @@ when no single workflow covers it. | `get_server_info()` | — | What this installation can do and where it keeps things: `device` (the accelerator a run will use), `version`, the `workspace` this session is working in and the workflow/asset/output/prompt `directories` of *that* workspace, the bind address and port, whether a token is required, and whether MCP is mounted. Check the device before authoring - a CUDA-only choice (bitsandbytes, `torch.compile`, flash attention) is not available on an `mps` or `cpu` server | | `list_jobs()` | — | List queued, running and recent jobs. In a named workspace, that workspace's jobs; in the default one, every job the server holds | | `list_gallery(limit=50)` | `limit` | List generated output files, newest first. A name is `//`; each entry also carries a ready-made `url`, already scoped to the workspace that made it - a hand-built `/outputs/` URL 404s for anything but the default workspace | -| `get_gallery_metadata(name)` | `name` | Get the metadata embedded in a generated file: the exact workflow and arguments that produced it | +| `get_gallery_metadata(name)` | `name` | Get the metadata embedded in a generated file: the exact workflow and arguments that produced it, and, for audio/video, a `media` block (duration, rate, channels, fps, size, peak/mean dBFS) | ### Media diff --git a/dw/media_info.py b/dw/media_info.py new file mode 100644 index 0000000..ebeeb67 --- /dev/null +++ b/dw/media_info.py @@ -0,0 +1,114 @@ +"""What the server knows about a generated media file and would otherwise +not say. An agent cannot listen: duration against a ceiling, peak against +a normalization target and the level at a seam are the only checks it can +make on an audio deliverable, and every one of them was being made by +fetching the file and running ffprobe by hand. +""" + +import logging +import math + +import av +import numpy + +logger = logging.getLogger("dw") + +# The floor a level is reported at rather than -inf, which JSON cannot carry +SILENCE_DBFS = -120.0 + + +def _dbfs(value): + if value <= 0: + return SILENCE_DBFS + return max(SILENCE_DBFS, 20.0 * math.log10(float(value))) + + +def probe_media(path): + """Duration, format and level of an audio or video file, or None. + + Video answers fps, frame_count, width and height, plus the soundtrack's + sample_rate, channels, peak_dbfs and mean_dbfs when it carries one; + audio answers the soundtrack fields. Levels come from decoding the + whole track, which is cheap next to generating it. + + When a frame count still needs counting and/or a soundtrack still needs + its levels measured, both are gathered from a single decode pass over + whichever streams are involved - `container.decode()` demuxes to EOF, so + two separate passes (count video, then decode audio) would leave the + second one nothing to read. + """ + try: + container = av.open(path) + except Exception as e: + logger.debug(f"Not probeable as media: {path}: {e}") + return None + with container: + video = container.streams.video[0] if container.streams.video else None + audio = container.streams.audio[0] if container.streams.audio else None + if video is None and audio is None: + return None + info = {} + if video is not None: + info["kind"] = "video" + info["fps"] = float(video.average_rate) if video.average_rate else None + info["width"] = int(video.width) + info["height"] = int(video.height) + else: + info["kind"] = "audio" + if container.duration is not None: + info["duration_seconds"] = float(container.duration / av.time_base) + if audio is not None: + info["sample_rate"] = int(audio.rate) + info["channels"] = int(audio.channels) + + # Some muxers don't write a frame count up front (0 means "count + # them"); a soundtrack always needs decoding to measure its level. + # Do both together, since decoding is a one-way trip through the file. + need_frame_count = video is not None and not video.frames + if video is not None and not need_frame_count: + info["frame_count"] = int(video.frames) + + if need_frame_count or audio is not None: + frame_count = 0 + peak = 0.0 + total = 0.0 + count = 0 + streams = [ + s + for s in ((video if need_frame_count else None), audio) + if s is not None + ] + try: + for frame in container.decode(*streams): + if isinstance(frame, av.VideoFrame): + frame_count += 1 + elif isinstance(frame, av.AudioFrame): + samples = frame.to_ndarray() + if samples.dtype.kind == "u": + iinfo = numpy.iinfo(samples.dtype) + half = (iinfo.max + 1) / 2 + samples = (samples.astype(numpy.float32) - half) / half + elif samples.dtype.kind == "i": + samples = ( + samples.astype(numpy.float32) + / numpy.iinfo(samples.dtype).max + ) + samples = samples.astype(numpy.float32) + peak = max(peak, float(numpy.abs(samples).max(initial=0.0))) + total += float(numpy.square(samples).sum()) + count += samples.size + except Exception as e: + # A track that opens fine can still fail mid-decode (damage + # past the header); the fields already gathered - duration, + # format - are still true, so report those rather than + # failing the whole probe. Matches read_embedded_metadata's + # precedent of degrading rather than raising. + logger.debug(f"Decode failed partway through {path}: {e}") + return info + if need_frame_count: + info["frame_count"] = frame_count + if audio is not None: + rms = math.sqrt(total / count) if count else 0.0 + info["peak_dbfs"] = _dbfs(peak) + info["mean_dbfs"] = _dbfs(rms) + return info diff --git a/dw/server/app.py b/dw/server/app.py index f6456ec..195936d 100644 --- a/dw/server/app.py +++ b/dw/server/app.py @@ -55,6 +55,7 @@ from .enhancers import build_enhance_workflow, preset_descriptions from .exports import export_directory, export_job from ..result import read_embedded_metadata +from ..media_info import probe_media from ..hub_cache import scan_models, delete_model, DownloadManager from ..runs import strip_run_id from ..workspace import ( @@ -1821,7 +1822,9 @@ def gallery( def gallery_metadata(name: str, ws: Workspace = Depends(selected_workspace)): """Generation metadata embedded in a saved image ('workflow' inside it is the full definition the editor can reopen), plus the job that - produced the file when history remembers one.""" + produced the file when history remembers one, plus - for audio and + video - what the file itself holds: duration, format and level, + which is how an agent that cannot listen checks a track.""" path = _output_file(name, ws.outputs) metadata = read_embedded_metadata(path) try: @@ -1831,7 +1834,13 @@ def gallery_metadata(name: str, ws: Workspace = Depends(selected_workspace)): job = manager.history.job_for_file(name, workspace=ws.name) except Exception: job = None - return {"name": name, "metadata": metadata, "job": job} + extension = os.path.splitext(path)[1].lower() + media = ( + probe_media(path) + if MEDIA_KINDS.get(extension) in ("audio", "video") + else None + ) + return {"name": name, "metadata": metadata, "job": job, "media": media} @app.get("/api/gallery/{name:path}/thumbnail") @query_token_ok diff --git a/dw_mcp/catalog.py b/dw_mcp/catalog.py index 9a5a3ac..2a48d1f 100644 --- a/dw_mcp/catalog.py +++ b/dw_mcp/catalog.py @@ -103,5 +103,17 @@ def list_gallery(client, limit=50): def get_gallery_metadata(client, name): """Metadata embedded in a saved file: the full workflow that made it, - plus the job that produced it when history remembers one.""" - return client.get_json(api_path("api", "gallery", name, "metadata")) + plus the job that produced it when history remembers one, plus for + audio and video what the file holds - duration, sample rate, channels, + fps, size, peak and mean level in dBFS.""" + body = client.get_json(api_path("api", "gallery", name, "metadata")) + media = body.get("media") + if media and media.get("kind") in ("audio", "video"): + body["next"] = ( + "Check duration_seconds against what was asked for: a Music 3 " + "track that lands within 0.2 s of its audio_duration ceiling was " + "cut off, one well short of it finished naturally. peak_dbfs is " + "the level normalize_audio would be given; mean_dbfs below -40 " + "on a track that should be full is a near-silent render." + ) + return body diff --git a/dw_mcp/server.py b/dw_mcp/server.py index 533b6c1..61401e2 100644 --- a/dw_mcp/server.py +++ b/dw_mcp/server.py @@ -289,7 +289,9 @@ def get_gallery_metadata(name: str) -> dict: workflow, arguments and seed that produced it. Use this to reproduce a result, or to see what a run that went wrong actually ran - it is the definition, not a summary, so it can be edited and - re-run.""" + re-run. For audio and video the `media` block carries duration, + sample rate, channels, fps, size and level - the checks an agent + that cannot listen makes on a deliverable.""" return catalog.get_gallery_metadata(client, name) def list_guides() -> dict: diff --git a/plugins/dw/skills/minimax-music3/SKILL.md b/plugins/dw/skills/minimax-music3/SKILL.md index d164017..b0d5f85 100644 --- a/plugins/dw/skills/minimax-music3/SKILL.md +++ b/plugins/dw/skills/minimax-music3/SKILL.md @@ -116,8 +116,10 @@ Control" section. 4. You cannot listen: no tool returns audio inline. Hand the user the gallery `url` (`list_gallery`, or the manifest's file name) and check what you can yourself - `get_gallery_metadata` for the file's duration against the - ceiling (a track well short of it stopped on its own; one exactly at it was - cut) and the sample rate. Ask the user to listen for the family's failure + ceiling: `media.duration_seconds` within 0.2 s of `audio_duration` means + the ceiling cut the track (raise it and rerun); well short of it means the + song finished on its own. Also check the sample rate. Ask the user to + listen for the family's failure modes: a song that went instrumental (name the vocals in the caption), an ending cut mid-note (raise the ceiling, then trim), a structure that ignored the tags (fewer sections, plainer directions). diff --git a/tests/test_mcp_catalog.py b/tests/test_mcp_catalog.py index fc3f776..38a6ca0 100644 --- a/tests/test_mcp_catalog.py +++ b/tests/test_mcp_catalog.py @@ -21,6 +21,20 @@ def handler(request): return DwClient(transport=httpx.MockTransport(handler)), seen +def scripted(routes): + """A client whose transport answers a fixed map of (method, path) -> + (status, body), for a route the recording_client above can't.""" + + def handler(request): + key = (request.method, request.url.path) + if key not in routes: + return httpx.Response(404, json={"detail": f"unrouted {key}"}) + status, body = routes[key] + return httpx.Response(status, json=body) + + return DwClient(transport=httpx.MockTransport(handler)), None + + @pytest.mark.parametrize( "call, path", [ @@ -146,3 +160,18 @@ def handler(request): catalog.get_workflow(client, "../escape") assert seen["raw_path"] == b"/api/workflows/..%2Fescape" + + +def test_gallery_metadata_passes_the_media_block_through_and_says_how_to_read_it(): + body = { + "name": "score.mp3", + "metadata": None, + "job": {"id": "job-1", "status": "succeeded"}, + "media": {"kind": "audio", "duration_seconds": 45.05, "peak_dbfs": -1.0}, + } + client, _ = scripted({("GET", "/api/gallery/score.mp3/metadata"): (200, body)}) + + result = catalog.get_gallery_metadata(client, "score.mp3") + + assert result["media"]["duration_seconds"] == 45.05 + assert "audio_duration" in result["next"] diff --git a/tests/test_media_info.py b/tests/test_media_info.py new file mode 100644 index 0000000..aef76bd --- /dev/null +++ b/tests/test_media_info.py @@ -0,0 +1,181 @@ +"""probe_media reports what the server knows about a generated file and +would otherwise not say - an agent cannot listen, so duration and level +are the only way it checks an audio deliverable.""" + +import math + +import numpy +import pytest + +from dw.media_info import probe_media + + +def write_wav(path, seconds=2.0, sample_rate=8000, amplitude=0.5): + import wave + + t = numpy.arange(int(seconds * sample_rate)) / sample_rate + samples = (numpy.sin(2 * numpy.pi * 220 * t) * amplitude * 32767).astype("