Gallery metadata describes audio and video

docs/MCP.md

@@ -225,7 +225,7 @@ when no single workflow covers it.
 | `get_server_info()` | — | What this installation can do and where it keeps things: `device` (the accelerator a run will use), `version`, the `workspace` this session is working in and the workflow/asset/output/prompt `directories` of *that* workspace, the bind address and port, whether a token is required, and whether MCP is mounted. Check the device before authoring - a CUDA-only choice (bitsandbytes, `torch.compile`, flash attention) is not available on an `mps` or `cpu` server |
 | `list_jobs()` | — | List queued, running and recent jobs. In a named workspace, that workspace's jobs; in the default one, every job the server holds |
 | `list_gallery(limit=50)` | `limit` | List generated output files, newest first. A name is `<workflow>/<run id>/<file>`; each entry also carries a ready-made `url`, already scoped to the workspace that made it - a hand-built `/outputs/<name>` URL 404s for anything but the default workspace |
-| `get_gallery_metadata(name)` | `name` | Get the metadata embedded in a generated file: the exact workflow and arguments that produced it |
+| `get_gallery_metadata(name)` | `name` | Get the metadata embedded in a generated file: the exact workflow and arguments that produced it, and, for audio/video, a `media` block (duration, rate, channels, fps, size, peak/mean dBFS) |
 
 ### Media
 

dw/media_info.py

@@ -0,0 +1,114 @@
+"""What the server knows about a generated media file and would otherwise
+not say. An agent cannot listen: duration against a ceiling, peak against
+a normalization target and the level at a seam are the only checks it can
+make on an audio deliverable, and every one of them was being made by
+fetching the file and running ffprobe by hand.
+"""
+
+import logging
+import math
+
+import av
+import numpy
+
+logger = logging.getLogger("dw")
+
+# The floor a level is reported at rather than -inf, which JSON cannot carry
+SILENCE_DBFS = -120.0
+
+
+def _dbfs(value):
+    if value <= 0:
+        return SILENCE_DBFS
+    return max(SILENCE_DBFS, 20.0 * math.log10(float(value)))
+
+
+def probe_media(path):
+    """Duration, format and level of an audio or video file, or None.
+
+    Video answers fps, frame_count, width and height, plus the soundtrack's
+    sample_rate, channels, peak_dbfs and mean_dbfs when it carries one;
+    audio answers the soundtrack fields. Levels come from decoding the
+    whole track, which is cheap next to generating it.
+
+    When a frame count still needs counting and/or a soundtrack still needs
+    its levels measured, both are gathered from a single decode pass over
+    whichever streams are involved - `container.decode()` demuxes to EOF, so
+    two separate passes (count video, then decode audio) would leave the
+    second one nothing to read.
+    """
+    try:
+        container = av.open(path)
+    except Exception as e:
+        logger.debug(f"Not probeable as media: {path}: {e}")
+        return None
+    with container:
+        video = container.streams.video[0] if container.streams.video else None
+        audio = container.streams.audio[0] if container.streams.audio else None
+        if video is None and audio is None:
+            return None
+        info = {}
+        if video is not None:
+            info["kind"] = "video"
+            info["fps"] = float(video.average_rate) if video.average_rate else None
+            info["width"] = int(video.width)
+            info["height"] = int(video.height)
+        else:
+            info["kind"] = "audio"
+        if container.duration is not None:
+            info["duration_seconds"] = float(container.duration / av.time_base)
+        if audio is not None:
+            info["sample_rate"] = int(audio.rate)
+            info["channels"] = int(audio.channels)
+
+        # Some muxers don't write a frame count up front (0 means "count
+        # them"); a soundtrack always needs decoding to measure its level.
+        # Do both together, since decoding is a one-way trip through the file.
+        need_frame_count = video is not None and not video.frames
+        if video is not None and not need_frame_count:
+            info["frame_count"] = int(video.frames)
+
+        if need_frame_count or audio is not None:
+            frame_count = 0
+            peak = 0.0
+            total = 0.0
+            count = 0
+            streams = [
+                s
+                for s in ((video if need_frame_count else None), audio)
+                if s is not None
+            ]
+            try:
+                for frame in container.decode(*streams):
+                    if isinstance(frame, av.VideoFrame):
+                        frame_count += 1
+                    elif isinstance(frame, av.AudioFrame):
+                        samples = frame.to_ndarray()
+                        if samples.dtype.kind == "u":
+                            iinfo = numpy.iinfo(samples.dtype)
+                            half = (iinfo.max + 1) / 2
+                            samples = (samples.astype(numpy.float32) - half) / half
+                        elif samples.dtype.kind == "i":
+                            samples = (
+                                samples.astype(numpy.float32)
+                                / numpy.iinfo(samples.dtype).max
+                            )
+                        samples = samples.astype(numpy.float32)
+                        peak = max(peak, float(numpy.abs(samples).max(initial=0.0)))
+                        total += float(numpy.square(samples).sum())
+                        count += samples.size
+            except Exception as e:
+                # A track that opens fine can still fail mid-decode (damage
+                # past the header); the fields already gathered - duration,
+                # format - are still true, so report those rather than
+                # failing the whole probe. Matches read_embedded_metadata's
+                # precedent of degrading rather than raising.
+                logger.debug(f"Decode failed partway through {path}: {e}")
+                return info
+            if need_frame_count:
+                info["frame_count"] = frame_count
+            if audio is not None:
+                rms = math.sqrt(total / count) if count else 0.0
+                info["peak_dbfs"] = _dbfs(peak)
+                info["mean_dbfs"] = _dbfs(rms)
+        return info

dw/server/app.py

@@ -55,6 +55,7 @@
 from .enhancers import build_enhance_workflow, preset_descriptions
 from .exports import export_directory, export_job
 from ..result import read_embedded_metadata
+from ..media_info import probe_media
 from ..hub_cache import scan_models, delete_model, DownloadManager
 from ..runs import strip_run_id
 from ..workspace import (
@@ -1821,7 +1822,9 @@ def gallery(
     def gallery_metadata(name: str, ws: Workspace = Depends(selected_workspace)):
         """Generation metadata embedded in a saved image ('workflow' inside
         it is the full definition the editor can reopen), plus the job that
-        produced the file when history remembers one."""
+        produced the file when history remembers one, plus - for audio and
+        video - what the file itself holds: duration, format and level,
+        which is how an agent that cannot listen checks a track."""
         path = _output_file(name, ws.outputs)
         metadata = read_embedded_metadata(path)
         try:
@@ -1831,7 +1834,13 @@ def gallery_metadata(name: str, ws: Workspace = Depends(selected_workspace)):
             job = manager.history.job_for_file(name, workspace=ws.name)
         except Exception:
             job = None
-        return {"name": name, "metadata": metadata, "job": job}
+        extension = os.path.splitext(path)[1].lower()
+        media = (
+            probe_media(path)
+            if MEDIA_KINDS.get(extension) in ("audio", "video")
+            else None
+        )
+        return {"name": name, "metadata": metadata, "job": job, "media": media}
 
     @app.get("/api/gallery/{name:path}/thumbnail")
     @query_token_ok

dw_mcp/catalog.py

@@ -103,5 +103,17 @@ def list_gallery(client, limit=50):
 
 def get_gallery_metadata(client, name):
     """Metadata embedded in a saved file: the full workflow that made it,
-    plus the job that produced it when history remembers one."""
-    return client.get_json(api_path("api", "gallery", name, "metadata"))
+    plus the job that produced it when history remembers one, plus for
+    audio and video what the file holds - duration, sample rate, channels,
+    fps, size, peak and mean level in dBFS."""
+    body = client.get_json(api_path("api", "gallery", name, "metadata"))
+    media = body.get("media")
+    if media and media.get("kind") in ("audio", "video"):
+        body["next"] = (
+            "Check duration_seconds against what was asked for: a Music 3 "
+            "track that lands within 0.2 s of its audio_duration ceiling was "
+            "cut off, one well short of it finished naturally. peak_dbfs is "
+            "the level normalize_audio would be given; mean_dbfs below -40 "
+            "on a track that should be full is a near-silent render."
+        )
+    return body

dw_mcp/server.py

@@ -289,7 +289,9 @@ def get_gallery_metadata(name: str) -> dict:
         workflow, arguments and seed that produced it. Use this to
         reproduce a result, or to see what a run that went wrong actually
         ran - it is the definition, not a summary, so it can be edited and
-        re-run."""
+        re-run. For audio and video the `media` block carries duration,
+        sample rate, channels, fps, size and level - the checks an agent
+        that cannot listen makes on a deliverable."""
         return catalog.get_gallery_metadata(client, name)
 
     def list_guides() -> dict:

plugins/dw/skills/minimax-music3/SKILL.md

@@ -116,8 +116,10 @@ Control" section.
 4. You cannot listen: no tool returns audio inline. Hand the user the gallery
    `url` (`list_gallery`, or the manifest's file name) and check what you can
    yourself - `get_gallery_metadata` for the file's duration against the
-   ceiling (a track well short of it stopped on its own; one exactly at it was
-   cut) and the sample rate. Ask the user to listen for the family's failure
+   ceiling: `media.duration_seconds` within 0.2 s of `audio_duration` means
+   the ceiling cut the track (raise it and rerun); well short of it means the
+   song finished on its own. Also check the sample rate. Ask the user to
+   listen for the family's failure
    modes: a song that went instrumental (name the vocals in the caption), an
    ending cut mid-note (raise the ceiling, then trim), a structure that ignored
    the tags (fewer sections, plainer directions).

tests/test_mcp_catalog.py

@@ -21,6 +21,20 @@ def handler(request):
     return DwClient(transport=httpx.MockTransport(handler)), seen
 
 
+def scripted(routes):
+    """A client whose transport answers a fixed map of (method, path) ->
+    (status, body), for a route the recording_client above can't."""
+
+    def handler(request):
+        key = (request.method, request.url.path)
+        if key not in routes:
+            return httpx.Response(404, json={"detail": f"unrouted {key}"})
+        status, body = routes[key]
+        return httpx.Response(status, json=body)
+
+    return DwClient(transport=httpx.MockTransport(handler)), None
+
+
 @pytest.mark.parametrize(
     "call, path",
     [
@@ -146,3 +160,18 @@ def handler(request):
     catalog.get_workflow(client, "../escape")
 
     assert seen["raw_path"] == b"/api/workflows/..%2Fescape"
+
+
+def test_gallery_metadata_passes_the_media_block_through_and_says_how_to_read_it():
+    body = {
+        "name": "score.mp3",
+        "metadata": None,
+        "job": {"id": "job-1", "status": "succeeded"},
+        "media": {"kind": "audio", "duration_seconds": 45.05, "peak_dbfs": -1.0},
+    }
+    client, _ = scripted({("GET", "/api/gallery/score.mp3/metadata"): (200, body)})
+
+    result = catalog.get_gallery_metadata(client, "score.mp3")
+
+    assert result["media"]["duration_seconds"] == 45.05
+    assert "audio_duration" in result["next"]