diff --git a/cli/ffrwd/cli.py b/cli/ffrwd/cli.py index af0763e..1b197d1 100644 --- a/cli/ffrwd/cli.py +++ b/cli/ffrwd/cli.py @@ -70,10 +70,10 @@ package segment is the directory's name unless ``--name`` says otherwise; the namespace is ``--namespace``'s, or derived from the git remote's owner, or required. Refuses to overwrite any file it would write. ``--rust`` - scaffolds a wasm module package instead of the bare one: a cargo crate - whose ``build.rs`` finds the wit, an ``invert`` module in Rust, the lib SQL - declaring it and a recipe calling it. ``cargo build --target wasm32-wasip2 - --release`` then ``ffrwd publish`` is the whole path from there. + scaffolds a node module package instead of the bare one: a cargo crate on + the ffrwd-node SDK holding a ``passthrough`` node, the lib SQL declaring it + and a recipe calling it. ``cargo build --release --target wasm32-wasip2`` + then ``ffrwd publish`` is the whole path from there. * ``search [TERM] [--json]`` -- ask the registry what it ranks for TERM and print it, most relevant first. No term browses everything; a term matching nothing is an empty table, exit 0. @@ -191,7 +191,19 @@ from pathlib import Path from typing import Any -from . import binaries, credentials, diagram, loudnorm, nn, redact, remote, show, store, wasm +from . import ( + binaries, + credentials, + diagram, + loudnorm, + nn, + redact, + remote, + show, + store, + timing, + wasm, +) from . import packages as packages_module from . import publish as publish_module from . import registry as registry_module @@ -541,6 +553,11 @@ def _build_parser() -> argparse.ArgumentParser: action="store_true", help="render the flowchart in the terminal (needs the diagram extra)", ) + explain_view.add_argument( + "--delays", + action="store_true", + help="print each node's window and how far behind the source each output runs", + ) validate_p = subparsers.add_parser("validate", help="check that a query compiles") _add_query_arguments(validate_p) _add_quiet_argument(validate_p) @@ -685,7 +702,7 @@ def _build_parser() -> argparse.ArgumentParser: init_p.add_argument( "--rust", action="store_true", - help="scaffold a wasm module package: the Rust crate that builds it, " + help="scaffold a node module package: the Rust crate that builds it, " "the SQL declaring it, and a recipe calling it", ) @@ -1474,6 +1491,10 @@ def _cmd_explain(args: argparse.Namespace, on_warning: OnWarning) -> int: return 1 graphs = compiled.graphs + if args.delays: + if compiled.timing is not None: + print(timing.summary(compiled.timing)) + return 0 if args.mermaid or args.diagram: text = diagram.render_diagram(graphs, compiled.plan) if args.mermaid: @@ -1489,8 +1510,12 @@ def _cmd_explain(args: argparse.Namespace, on_warning: OnWarning) -> int: payload: object = graphs[0].to_dict() if len(graphs) == 1 else [ graph.to_dict() for graph in graphs ] - if compiled.plan is not None: - payload = {"graph": payload, "plan": compiled.plan.to_dict()} + if compiled.plan is not None or compiled.timing is not None: + payload = {"graph": payload} + if compiled.plan is not None: + payload["plan"] = compiled.plan.to_dict() + if compiled.timing is not None: + payload["timing"] = compiled.timing.to_dict() print(json.dumps(payload, indent=2)) return 0 @@ -2413,36 +2438,39 @@ def _cmd_list(args: argparse.Namespace, on_warning: OnWarning) -> int: ) TO :'dest' """ -# The wasm module scaffold `init --rust` writes on top of the manifest and the +# The node module scaffold `init --rust` writes on top of the manifest and the # lockfile. One export and one recipe, both named for what the module does. -_RUST_EXPORT = "invert" +_RUST_EXPORT = "passthrough" _RUST_EXPORT_FILE = f"src/{_RUST_EXPORT}.sql" -_RUST_RECIPE = "invert" +_RUST_RECIPE = "passthrough" _RUST_RECIPE_FILE = f"recipes/{_RUST_RECIPE}.sql" -_RUST_BUILD_FILE = "build.rs" _RUST_CARGO_FILE = "Cargo.toml" _RUST_SOURCE_FILE = "src/lib.rs" _RUST_IGNORE_FILE = store.IGNORE_NAME _RUST_GITIGNORE_FILE = store.GITIGNORE_NAME -# The wit-bindgen the in-repo modules build with; the scaffold pins the same one. -_WIT_BINDGEN_VERSION = "0.57.1" +# The releases the scaffold's crate pins; ffrwd-node's speaks the world +# `wasm.WORLD_VERSION` names, which the manifest depends on. +_NODE_SDK_TAG = "v0.2.0" +_FRAME_TAG = "v0.1.1" + +_RUST_BUILD_LINE = "cargo build --release --target wasm32-wasip2" # Where a built module lands, and what the lib SQL therefore names. _RUST_ARTIFACT = "target/wasm32-wasip2/release/{crate}.wasm" -_RUST_CARGO = f"""\ +_RUST_CARGO = """\ [package] -name = "{{crate}}" +name = "{crate}" version = "0.1.0" edition = "2021" -# A wasm component: one cdylib, no binary. [lib] crate-type = ["cdylib"] [dependencies] -wit-bindgen = "{_WIT_BINDGEN_VERSION}" +ffrwd-node = {{ git = "https://github.com/imbcmdth/ffrwd-node", tag = "{sdk_tag}" }} +ffrwd-frame = {{ git = "https://github.com/imbcmdth/ffrwd-frame", tag = "{frame_tag}" }} [profile.release] opt-level = 3 @@ -2450,131 +2478,42 @@ def _cmd_list(args: argparse.Namespace, on_warning: OnWarning) -> int: strip = true """ -_RUST_BUILD = '''\ -// Puts the `ffrwd:av` wit where `wit_bindgen::generate!` reads it, from -// whichever of the two sources is available: FFRWD_WIT_DIR when the -// environment names one, else the `ffrwd/wasm` package installed here. -use std::env; -use std::fs; -use std::path::PathBuf; -use std::process::Command; - -const WIT_DIR_ENV: &str = "FFRWD_WIT_DIR"; -const WIT_PACKAGE: &str = "ffrwd/wasm"; -const WIT_FILE: &str = "av.wit"; - -fn main() { - println!("cargo::rerun-if-env-changed={WIT_DIR_ENV}"); - let source = match env::var_os(WIT_DIR_ENV) { - Some(named) => PathBuf::from(named), - None => installed_wit_dir(), - } - .join(WIT_FILE); - println!("cargo::rerun-if-changed={}", source.display()); - - let manifest = PathBuf::from(env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR")); - let wit = manifest.join("wit"); - fs::create_dir_all(&wit).expect("create wit/"); - fs::copy(&source, wit.join(WIT_FILE)) - .unwrap_or_else(|err| panic!("copy {}: {err}", source.display())); -} - -/// The `wit` directory of the installed `ffrwd/wasm` package, asked of ffrwd. -fn installed_wit_dir() -> PathBuf { - let asked = Command::new("ffrwd") - .args(["path", WIT_PACKAGE]) - .output() - .unwrap_or_else(|err| { - panic!("`ffrwd path {WIT_PACKAGE}` could not be run ({err}); set {WIT_DIR_ENV} instead") - }); - if !asked.status.success() { - panic!( - "`ffrwd path {WIT_PACKAGE}` failed: {}", - String::from_utf8_lossy(&asked.stderr).trim() - ); - } - let printed = String::from_utf8(asked.stdout).expect("a path, in utf-8"); - PathBuf::from(printed.trim()).join("wit") -} -''' - _RUST_SOURCE = '''\ -wit_bindgen::generate!({ - path: "wit", - world: "video-module", -}); +use ffrwd_node::{Bound, Init, Input, NoParams, Node, Out, Output, Result, Shape, Tick}; -use exports::ffrwd::av::filter::{FrameInfo, Guest, Meta, Outcome, Output, StreamInfo}; - -struct Invert; - -// JSON Schema for the `params` string a call passes; this module takes none. -const PARAMS_SCHEMA: &str = r#"{"type":"object","properties":{},"additionalProperties":false}"#; - -fn validate_params(params: &str) -> Result<(), String> { - match params.trim() { - "" | "{}" => Ok(()), - other => Err(format!("invert takes no params, got: {other}")), - } +struct Passthrough { + v: u32, } -impl Guest for Invert { - // What the module is, read before anything runs. `pixel_formats` empty - // would make this an audio module, and the two are never both. - fn describe() -> Meta { - Meta { - name: "invert".to_string(), - version: "0.1.0".to_string(), - params_schema: PARAMS_SCHEMA.to_string(), - rows_schema: String::new(), - pixel_formats: vec!["rgba".to_string()], - sample_formats: vec![], - sample_rates: vec![], - channel_counts: vec![], - rows_language: vec![], - } - } - - // Once per instance, before any frame. The frame size and pixel format - // are fixed from here on, so state sized to them is built here. - fn init( - _width: u32, - _height: u32, - _pix_fmt: String, - _stream_info: StreamInfo, - params: String, - ) -> Result<(), String> { - validate_params(¶ms) +impl Node for Passthrough { + const NAME: &'static str = "passthrough"; + const VERSION: &'static str = "0.1.0"; + type Params = NoParams; + + fn shape(_: &NoParams, _: &Bound) -> Result { + Ok(Shape::new() + .input(Input::video("v").clock().pixel_formats(&["rgba"])) + .output(Output::like("v")) + .pure() + .one_to_one()) } - // New parameters between frames; rejecting them leaves the old ones in force. - fn set_params(params: String) -> Result<(), String> { - validate_params(¶ms) + fn init(_: NoParams, init: &Init) -> Result { + Ok(Passthrough { + v: init.stream("v")?.id, + }) } - // True lets the host run frames in parallel, so it must be a promise. - fn frame_independent() -> bool { - true - } - - // One frame in, one frame out. `Output::Passthrough` would hand the input - // back uncopied; this one rewrites the bytes it was given. - fn process(_info: FrameInfo, frame: Vec) -> Outcome { - let mut out = frame; - let (pixels, _) = out.as_chunks_mut::<4>(); - for pixel in pixels { - pixel[0] = 255 - pixel[0]; - pixel[1] = 255 - pixel[1]; - pixel[2] = 255 - pixel[2]; - } - Outcome { - output: Output::Frame(out), - rows: vec![], - } + fn process(&mut self, tick: &Tick, out: &mut Out) -> Result<()> { + let Some(frame) = tick.frame(self.v) else { + return Ok(()); + }; + // Your work goes here: `tick.fetch` reads the picture, `out.frame` sends a new one. + Ok(out.pass("v", self.v, &frame)?) } } -export!(Invert); +ffrwd_node::export!(Passthrough); ''' _RUST_EXPORT_SQL = """\ @@ -2585,7 +2524,7 @@ def _cmd_list(args: argparse.Namespace, on_warning: OnWarning) -> int: """ _RUST_RECIPE_QUERY = """\ --- Invert a file's picture, its audio carried through untouched. +-- Run a file's picture through the module, its audio carried through untouched. -- variables: source (input media path), dest (output path) -- example: ffrwd run {recipe} -v source=in.mp4 -v dest=out.mp4 COPY ( @@ -2601,22 +2540,26 @@ def _cmd_list(args: argparse.Namespace, on_warning: OnWarning) -> int: # Build input, and the output it produces. The one wasm the lib SQL names # ships out of target/ regardless; nothing else here belongs in the archive. Cargo.toml -build.rs src/*.rs target/ -wit/ """ -# `wit/` is written by build.rs, not by hand. _RUST_GITIGNORE = """\ target/ -wit/ """ -_RUST_README = """\ -# {name} +_RUST_README = f"""\ +# {{name}} TODO: what this package does, and how a query calls it. + +Written on the [ffrwd-node](https://github.com/imbcmdth/ffrwd-node) SDK, whose +README is the reference for what a node declares and emits. + +``` +{_RUST_BUILD_LINE} +ffrwd run {_RUST_RECIPE} -v source=in.mp4 -v dest=out.mp4 +``` """ @@ -2624,8 +2567,9 @@ def _rust_scaffold(name: str) -> dict[str, str]: """The module scaffold's files, keyed by their path under the project.""" segment = name.partition("/")[2] return { - _RUST_CARGO_FILE: _RUST_CARGO.format(crate=segment), - _RUST_BUILD_FILE: _RUST_BUILD, + _RUST_CARGO_FILE: _RUST_CARGO.format( + crate=segment, sdk_tag=_NODE_SDK_TAG, frame_tag=_FRAME_TAG + ), _RUST_SOURCE_FILE: _RUST_SOURCE, _RUST_EXPORT_FILE: _RUST_EXPORT_SQL.format( export=_RUST_EXPORT, artifact=_RUST_ARTIFACT.format(crate=segment) @@ -2643,7 +2587,6 @@ def _rust_scaffold(name: str) -> dict[str, str]: # refuses to overwrite, checked before a name is worked out. _RUST_PATHS = ( _RUST_CARGO_FILE, - _RUST_BUILD_FILE, _RUST_SOURCE_FILE, _RUST_EXPORT_FILE, _RUST_RECIPE_FILE, @@ -2794,7 +2737,7 @@ def _cmd_init(args: argparse.Namespace, on_warning: OnWarning) -> int: """Write the files a project starts as, into the working directory. The manifest, an empty lockfile and a starter recipe; ``--rust`` writes a - wasm module package instead of the bare one -- the crate that builds the + node module package instead of the bare one -- the crate that builds the module, the SQL declaring it, and a recipe calling it. """ directory = Path.cwd() @@ -2872,8 +2815,7 @@ def _cmd_init(args: argparse.Namespace, on_warning: OnWarning) -> int: f"it calls its functions as {name.replace('/', '.')}.name()" ) if rust: - print(f"take the wit the module builds against: ffrwd install {wasm.WIT_PACKAGE}") - print("build the module: cargo build --target wasm32-wasip2 --release") + print(f"build the module: {_RUST_BUILD_LINE}") print( f"then run the recipe: ffrwd run {_RUST_RECIPE} " f"-v source=in.mp4 -v dest=out.mp4" diff --git a/cli/ffrwd/compiler.py b/cli/ffrwd/compiler.py index 7ad11da..99b1eb6 100644 --- a/cli/ffrwd/compiler.py +++ b/cli/ffrwd/compiler.py @@ -44,6 +44,7 @@ from dataclasses import dataclass, replace from . import registry as registry_module +from . import shapes as shapes_module from . import wasm from .emit import Emitted, emit from .errors import ErrorCode, FfrwdError @@ -64,14 +65,17 @@ check_spellable, external_filters, from_commands, + is_live, + is_live_probe, partition, ) from .project import ModelPin, PackageSet from .pts import insert_pts_resets from .split import insert_splits from .table import TableSink +from .timing import HOLD_LIMIT, Timing, check_live_leads, timing from .vars import substitute -from .warnings import OnWarning +from .warnings import FfrwdWarning, OnWarning, WarningCode from .wasm import Described __all__ = [ @@ -354,6 +358,8 @@ def _negotiable( if declared.module not in describes or declared.reads_rows_from_select: continue # a row-reading sink has no single kind; its pads are the rows' described = describes[declared.module] + if described.node: + continue # its formats are its shape's, per call and per edge if _declared_kind(declared, described) != kind: continue if described.packet_sink or described.packet_filter: @@ -477,7 +483,7 @@ def _module_shapes( return { declared.module: describes[declared.module].shape for declared in declared_stream.values() - if declared.module in describes + if declared.module in describes and not describes[declared.module].node } @@ -526,6 +532,9 @@ class Compiled: plan: ProcessPlan | None = None default_timeout: float | None = None duration: float | None = None + # How late each node module's outputs run behind the source, for a query + # that calls one (:mod:`ffrwd.timing`). + timing: Timing | None = None def _input_duration(probes: Mapping[str, ProbeResult | None]) -> float | None: @@ -655,6 +664,7 @@ def compile_all( unset: Mapping[tuple[int, int], str] | None = None, describe: wasm.Describe = wasm.describe, invoke: wasm.Invoke = wasm.invoke, + shape: shapes_module.Shape = shapes_module.shape, ) -> Compiled: """Compile SQL `text` into its commands, and the plan that runs them. @@ -668,7 +678,8 @@ def compile_all( lowering's `describes` is: a caller with no sidecar can still compile. `invoke` is the same for a VALUE-returning module: lowering runs it once per call site to fold the result, and a test hands over its own so - folding spawns nothing. + folding spawns nothing. `shape` is the same for a node module's shape, + asked once per distinct call. Raises ``FfrwdError`` — and nothing else — on every rejection. """ @@ -685,6 +696,7 @@ def compile_all( describes=describes, invoke=invoke, probe_failures=probe_failures, + shapes=shapes_module.ShapeCache(shape), ) ready = [insert_splits(insert_pts_resets(graph)) for graph in graphs] ready[0] = replace( @@ -694,22 +706,32 @@ def compile_all( for lateral in ready[0].laterals ], ) + timed = timing(ready[0], probes, {m: a[2] for m, a in _module_anchors(res).items()}) + if timed is not None: + if _runs_live(res, probes, ready[0]): + check_live_leads(ready[0], probes, _module_anchors(res)) + _warn_held(timed, on_warning) budget = _default_timeout(_input_duration(probes)) span = _run_duration(ready, _probed_paths(res, probes)) stream_wasm = _stream_wasm(res) hosted = _hosted_wasm(res) + sourced = {ready[0].nodes[name].filter for name in ready[0].node_sources.values()} leaky = any(node.filter == LEAKY for node in ready[0].nodes.values()) - if not hosted and not ready[0].module_sources and not leaky: - return Compiled(graphs=ready, default_timeout=budget, duration=span) + if not hosted and not ready[0].module_sources and not leaky and not sourced: + return Compiled( + graphs=ready, default_timeout=budget, duration=span, timing=timed + ) try: plan = partition( ready[0], - external=external_filters(*sorted({d.module for d in hosted.values()})), + external=external_filters( + *sorted({d.module for d in hosted.values()} | sourced) + ), probes=probes, pix_fmts=_wire_formats(stream_wasm, describes), shapes=_module_shapes(stream_wasm, describes), audio_wires=_audio_wires(stream_wasm, describes), - models=_nn_models(hosted, describes, packages), + models=_nn_models(hosted | _source_wasm(res), describes, packages), effects=_effect_grants(stream_wasm | _source_wasm(res), describes), anchors=res.input_anchors, ) @@ -717,7 +739,7 @@ def compile_all( except FfrwdError as err: raise _anchored(err, stream_wasm) from err return Compiled( - graphs=ready, plan=plan, default_timeout=budget, duration=span + graphs=ready, plan=plan, default_timeout=budget, duration=span, timing=timed ) except FfrwdError: raise @@ -739,6 +761,48 @@ def compile_all( ) from err +def _runs_live( + res: Resolved, probes: Mapping[str, ProbeResult | None], graph: Graph +) -> bool: + """Whether the query reads anything that does not end: a live input, or a + node read in FROM that never finishes by itself.""" + for alias, index in res.sources.items(): + options = graph.input_options.get(alias) + if is_live(res.input_paths[index], options) or is_live_probe(probes.get(alias)): + return True + return any( + graph.node_shapes.get(name, {}).get("bounded") is False + for name in graph.node_sources.values() + ) + + +def _module_anchors(res: Resolved) -> dict[str, tuple[int, int, str]]: + """Each module path -> where the query declared it, and what it is called.""" + found: dict[str, tuple[int, int, str]] = {} + for declared in res.wasm.values(): + found.setdefault(declared.module, (declared.line, declared.col, declared.called)) + return found + + +def _warn_held(timed: Timing, on_warning: OnWarning | None) -> None: + """Say so where a stream written beside a later one holds a lot of it.""" + if on_warning is None: + return + for output in timed.outputs: + if output.held is None or output.held <= HOLD_LIMIT: + continue + on_warning( + FfrwdWarning( + WarningCode.HELD_STREAM, + "", + f"'{output.ref}' waits {round(output.holds, 3):g} s for the stream " + f"written beside it, about {output.held // (1024 * 1024)} MiB of it", + hint="the stream waits in its pipe until the later one catches up; " + "a shorter window or latency on the later path holds less", + ) + ) + + # What a run-time lateral's body is resolved with at compile time, a value of # each declared type standing in for what a message will bind. _STAND_INS = {"number": "1", "text": "x", "boolean": "true"} diff --git a/cli/ffrwd/emit.py b/cli/ffrwd/emit.py index 7cc8d9d..d63ede3 100644 --- a/cli/ffrwd/emit.py +++ b/cli/ffrwd/emit.py @@ -425,7 +425,8 @@ class OutputGroup: each takes an output stream index after every map of this group. `wire` is the stream format set when this group writes a pipe edge rather than a file; its codec and pixel format ride `options`, and what an audio - edge is conformed to is rendered off it. + edge is conformed to is rendered off it. `wires` is every stream's, in + map order, where the edge is one NUT carrying several. """ maps: list[OutputMap] @@ -437,6 +438,7 @@ class OutputGroup: metadata: int | None = None attachments: list[Attachment] = field(default_factory=list) wire: StreamFormat | None = None + wires: tuple[StreamFormat, ...] = () @dataclass @@ -506,7 +508,7 @@ def emit(g: Graph, *, network: bool = False, separate: bool | None = None) -> Em g = dedup_inputs(g) g = _drop_unused_url_inputs(g) g = _drop_dropped_branch_inputs(g) - _verify_topological(g) + _verify_topological(g, network) nodes = list(g.nodes.values()) pads = {node.id: _out_pad_count(node) for node in nodes} @@ -924,7 +926,7 @@ def build_process_args( g: Graph, *, pipe_inputs: Sequence[tuple[str, str]] = (), - pipe_outputs: Sequence[tuple[str, StreamFormat]] = (), + pipe_outputs: Sequence[tuple[str, StreamFormat | Sequence[StreamFormat]]] = (), pipe_buffers: Sequence[EdgeBuffer | None] = (), pipe_live: Sequence[bool] = (), live: bool = False, @@ -1006,15 +1008,27 @@ def build_process_args( f"process writes {len(pipes)} pipes but {len(pipe_outputs)} were wired" ) groups = list(e.groups) - for slot, (index, (spelling, wire)) in enumerate(zip(pipes, pipe_outputs)): + for slot, (index, (spelling, carried)) in enumerate(zip(pipes, pipe_outputs)): group = groups[index] buffer = pipe_buffers[slot] if slot < len(pipe_buffers) else None live = pipe_live[slot] if slot < len(pipe_live) else False + wires: tuple[StreamFormat, ...] = ( + (carried,) + if isinstance(carried, VideoFormat | AudioFormat | DataFormat) + else tuple(carried) + ) + written = _bundle_options(wires, buffer, live=live) + # The container comes after every stream's codec, as one wire's does. + for key in (FIFO_FORMAT, QUEUE_SIZE, _FORMAT): + if key in written: + written[key] = written.pop(key) + wire = next((one for one in wires if isinstance(one, AudioFormat)), wires[0]) groups[index] = replace( group, path=spelling, wire=wire, - options={**_wire_options(wire, buffer, live=live), **group.options}, + wires=wires if len(wires) > 1 else (), + options={**written, **group.options}, ) return build_ffmpeg_args(replace(e, groups=groups)) @@ -1048,6 +1062,48 @@ def build_network_graph( return e.filter_complex, [m.target for group in e.groups for m in group.maps] +def build_node_network( + g: Graph, *, pipe_inputs: Sequence[str] = () +) -> tuple[str, list[list[str]]]: + """:func:`build_network_graph` for a region holding node modules. + + Its sinks are NUTs of several streams each, so the ``-map`` targets come + back grouped one list per sink, in sink order. + """ + slots = [index for index, path in enumerate(g.input_paths) if path == PIPE] + if len(slots) != len(pipe_inputs): + raise _internal( + f"network reads {len(slots)} pipes but {len(pipe_inputs)} were wired" + ) + paths = list(g.input_paths) + for slot, spelling in zip(slots, pipe_inputs): + paths[slot] = spelling + e = emit(replace(g, input_paths=paths), network=True) + return e.filter_complex, [[m.target for m in group.maps] for group in e.groups] + + +def _bundle_options( + wires: Sequence[StreamFormat], buffer: EdgeBuffer | None, *, live: bool +) -> dict[str, object]: + """The sink options writing every stream one edge carries. + + Streams of one kind share an option where they agree on it; where they + differ it is given per track, in map order, as a WITH list is. + """ + written: dict[str, object] = {} + kinds: dict[type, list[dict[str, object]]] = {} + for one in wires: + options = _wire_options(one, buffer, live=live) + written.update(options) + kinds.setdefault(type(one), []).append(options) + for tracks in kinds.values(): + for key in dict.fromkeys(key for options in tracks for key in options): + values = [options.get(key) for options in tracks] + if any(value != values[0] for value in values): + written[key] = values + return written + + def _render_conformance(group: OutputGroup) -> list[str]: """``-ar:``/``-ac:`` for each audio stream a wire edge constrains. @@ -1055,13 +1111,16 @@ def _render_conformance(group: OutputGroup) -> list[str]: and ahead of the codec that follows it. A module naming neither a rate nor a channel count renders nothing and the stream is left alone. """ - wire = group.wire - if not isinstance(wire, AudioFormat): + if not isinstance(group.wire, AudioFormat): return [] + audio = [one for one in group.wires if isinstance(one, AudioFormat)] or [group.wire] args: list[str] = [] + taken = 0 for index, mapping in enumerate(group.maps): if mapping.type != "audio": continue + wire = audio[min(taken, len(audio) - 1)] + taken += 1 if wire.required_rate is not None: args += [f"{SAMPLE_RATE_FLAG}:{index}", str(wire.required_rate)] if wire.required_channels is not None: @@ -1461,8 +1520,9 @@ def _src_spec(g: Graph, ref: FrameRef) -> str: return f"{g.sources[alias]}:{_TYPE_MARKERS[stream_type]}:{index}" -def _verify_topological(g: Graph) -> None: - """Check every ref resolves and points backwards; a cycle cannot pass.""" +def _verify_topological(g: Graph, network: bool = False) -> None: + """Check every ref resolves and points backwards; a cycle cannot pass. + A network may have no output: a node sink's is its own effect.""" defined: set[str] = set() for node_id, node in g.nodes.items(): if node.id != node_id: @@ -1470,7 +1530,7 @@ def _verify_topological(g: Graph) -> None: for ref in node.inputs: _check_ref(g, ref, defined, f"node {node.id!r}") defined.add(node_id) - if not g.outputs: + if not g.outputs and not network: raise _internal("graph has no outputs") for index, output in enumerate(g.outputs): if not output.ref: @@ -1750,20 +1810,45 @@ def _render_chain( network: bool = False, ) -> str: head, tail = chain[0], chain[-1] + if network and (head.ports or head.out_ports): + return _render_node(head, g, labels, measure) prefix = "".join(f"[{_input_label(g, ref, labels, network)}]" for ref in head.inputs) body = ",".join(_render_filter(node, measure) for node in chain) suffix = "".join(f"[{labels[f'{tail.id}:{pad}']}]" for pad in range(pads[tail.id])) return f"{prefix}{body}{suffix}" +def _render_node(node: Node, g: Graph, labels: dict[str, str], measure: bool) -> str: + """A node module's chain: each pad names the port it binds. + + An output nothing in the network or its maps reads is left off, which + is how the node is told the query does not latch it. + """ + read = {_slot(ref) for other in g.nodes.values() for ref in other.inputs} + read |= {_slot(output.ref) for output in g.outputs if not is_src(output.ref)} + prefix = "".join( + f"[{port}={_input_label(g, ref, labels, True)}]" + for port, ref in zip(node.ports, node.inputs) + ) + suffix = "".join( + f"[{port}={labels[f'{node.id}:{pad}']}]" + for pad, port in enumerate(node.out_ports) + if f"{node.id}:{pad}" in read + ) + return f"{prefix}{_render_filter(node, measure)}{suffix}" + + def _input_label( g: Graph, ref: FrameRef, labels: dict[str, str], network: bool = False ) -> str: if is_src(ref): spec = _src_spec(g, ref) - # A network input carries one stream, so its per-type index says - # nothing and the subset grammar leaves it off. - return spec.rpartition(":")[0] if network else spec + # A network input's first stream of a kind is that kind alone; the + # subset grammar spells an index past it, which only a NUT carrying + # several streams of one kind has. + if network and spec.endswith(":0"): + return spec.rpartition(":")[0] + return spec slot = _slot(ref) label = labels.get(slot) if label is None: diff --git a/cli/ffrwd/errors.py b/cli/ffrwd/errors.py index 26a5cfe..d9eac9d 100644 --- a/cli/ffrwd/errors.py +++ b/cli/ffrwd/errors.py @@ -36,6 +36,7 @@ class ErrorCode(str, Enum): PLAYER_NOT_FOUND = "PLAYER_NOT_FOUND" # --show asked for, ffplay not on PATH RUNTIME_NOT_FOUND = "RUNTIME_NOT_FOUND" # setup nn: no ONNX Runtime pinned for this platform UNBOUNDED_LIVE_INPUT = "UNBOUNDED_LIVE_INPUT" # one-open input, uncountable paths + LIVE_LEAD = "LIVE_LEAD" # a live node fed later than the lead it needs BUFFER_OVERFLOW = "BUFFER_OVERFLOW" # a run-time edge outgrew its computed bound INPUT_NEVER_OPENED = "INPUT_NEVER_OPENED" # a run-time consumer never opened its end STARTUP_DEADLOCK = "STARTUP_DEADLOCK" # no pipe order lets every process start @@ -72,6 +73,10 @@ def __init__( self.hint = hint super().__init__(str(self)) + def __reduce__(self) -> tuple[object, ...]: + """Copied and pickled whole: a refusal can wait inside a query's tree.""" + return (_rebuilt, (self.code, self.message, self.line, self.col, self.hint)) + def to_dict(self) -> dict[str, object]: return { "line": self.line, @@ -91,3 +96,9 @@ def __str__(self) -> str: if self.hint is not None: parts.append(f" (hint: {self.hint})") return "".join(parts) + + +def _rebuilt( + code: ErrorCode, message: str, line: int | None, col: int | None, hint: str | None +) -> FfrwdError: + return FfrwdError(code, message, line=line, col=col, hint=hint) diff --git a/cli/ffrwd/execute.py b/cli/ffrwd/execute.py index 90e4cad..186f534 100644 --- a/cli/ffrwd/execute.py +++ b/cli/ffrwd/execute.py @@ -146,7 +146,9 @@ from .errors import ErrorCode, FfrwdError from .ir import ( FEEDER_HOST, + PARAMS_FILE, STDERR_ROW, + TAP_DOCUMENT, Lateral, LateralValue, feeder_path, @@ -166,6 +168,8 @@ StreamEdge, VideoFormat, encoded, + once_per_pipe, + pipe_key, ) from .relay import Relay, RelayEdge from .vars import substitute @@ -637,6 +641,10 @@ def _stop_player(player: subprocess.Popen[bytes]) -> None: # placeholders as they are, which is what a printed command shows. RowsNamer = Callable[[str], str] +# Writes one node's params, as JSON, to the file its placeholder stands for, +# and names that file. +ParamsNamer = Callable[[str, str], str] + # Renders one sidecar process as the argv that runs it, given the path each # stream it reads arrives on and the path each rows document it writes goes # to. The real one lands with the sidecar itself; until then a caller @@ -809,6 +817,58 @@ def unheard_error( ) +def unstarted_error(writer: str, readers: Sequence[str], port: int) -> FfrwdError: + """The typed failure for a feeder's writer the stage ended without starting.""" + return FfrwdError( + ErrorCode.INPUT_NEVER_OPENED, + f"{writer}, which writes {feeder_path(port)} for {' and '.join(readers)}, " + "was never started: its readers ended without listening there, and " + "nothing it writes was written", + hint="the module listens on its port when it is opened; check that it " + "imports wasi:sockets/tcp and opens the port its describe names", + ) + + +def held_writers( + writers: Mapping[str, tuple[int, list[str]]], feeds: Iterable[tuple[str, str]] +) -> dict[str, tuple[int, list[str]]]: + """The feeders' writers a stage holds until their port accepts. + + A writer one of whose readers is itself, or reads what it writes, is + started with the rest: its readers would never listen while it waited + for them. Its lateral's instances wait for the port instead. + """ + downstream: dict[str, set[str]] = {} + for source, target in feeds: + downstream.setdefault(source, set()).add(target) + + def reached(start: str) -> set[str]: + seen = {start} + todo = [start] + while todo: + for after in downstream.get(todo.pop(), ()): + if after not in seen: + seen.add(after) + todo.append(after) + return seen + + return { + pid: (port, readers) + for pid, (port, readers) in writers.items() + if not reached(pid) & set(readers) + } + + +def never_started( + held: Mapping[str, tuple[int, list[str]]], started: Collection[str], local: Collection[str] +) -> tuple[str, FfrwdError] | None: + """The first writer held here that a stage ended without starting, and why.""" + for pid, (port, readers) in held.items(): + if pid in local and pid not in started: + return pid, unstarted_error(pid, readers, port) + return None + + def _await_port( port: int, readers: Sequence[_Member], @@ -970,9 +1030,10 @@ def _pipe_edges(plan: ProcessPlan) -> tuple[PipeEdge, ...]: The order is what pairs an edge with the ``pipe:`` slot it fills: a reading process's own inputs come first in its ``-i`` list, and a rows - track is one of those, where a frame edge is appended after them. + track is one of those, where a frame edge is appended after them. The + streams riding one NUT are one pipe, which the first of them stands for. """ - return (*plan.rows_edges, *plan.stream_edges) + return (*plan.rows_edges, *once_per_pipe(plan.stream_edges)) def plan_argv( @@ -982,6 +1043,8 @@ def plan_argv( pipe_path: PipeNamer | None = None, rows_path: RowsNamer | None = None, apart: Callable[[PipeEdge], bool] | None = None, + params_path: ParamsNamer | None = None, + tap_path: Callable[[int], str] | None = None, ) -> dict[str, list[str]]: """The argv that runs each process of `plan`, keyed by process id. @@ -1001,6 +1064,13 @@ def plan_argv( `apart` says which edges a placed run cuts between nodes; each is named at both ends (:func:`wires`). + + `params_path` writes a node's params to a file and names it, for each + params placeholder; without it the placeholders stay, as a printed + command shows them. + + `tap_path` names the pipe a node region writes a run-time lateral's + messages to, for each tap placeholder; without it they stay too. """ read: dict[PipeEdge, str] = {} write: dict[PipeEdge, str] = {} @@ -1011,17 +1081,22 @@ def plan_argv( write[wire.edge] = ( STDOUT if wire.write_stdio else _named(pipe_path, wire.edge, "write") ) + first = {pipe_key(edge): edge for edge in reversed(plan.stream_edges)} + for edge in plan.stream_edges: + if edge.nut: + read[edge] = read[first[edge.nut]] + write[edge] = write[first[edge.nut]] argv: dict[str, list[str]] = {} for process in plan.processes: - incoming = _once_per_ref( - [e for e in plan.stream_edges if e.target == process.id] - ) - outgoing = [e for e in plan.stream_edges if e.source == process.id] + incoming = _once_per_ref([e for e in plan.stream_edges if e.target == process.id]) + carried = [e for e in plan.stream_edges if e.source == process.id] + outgoing = once_per_pipe(carried) if isinstance(process, SidecarProcess): # A sidecar's reads are its pads, in the order its module takes # them, whatever order the startup walk put the edges in. incoming.sort(key=lambda edge: _pad_of(process, edge)) + incoming = once_per_pipe(incoming) argv[process.id] = _sidecar_args( process, sidecar_argv, @@ -1030,18 +1105,41 @@ def plan_argv( len(outgoing), ) continue + incoming = once_per_pipe(incoming) rows_in = _rows_inputs(process, plan) argv[process.id] = build_process_args( process.graph, pipe_inputs=[(read[edge], edge.container) for edge in rows_in] + [(read[edge], edge.format.container) for edge in incoming], - pipe_outputs=[(write[edge], edge.format) for edge in outgoing], + pipe_outputs=[ + ( + write[edge], + tuple(one.format for one in carried if pipe_key(one) == pipe_key(edge)), + ) + for edge in outgoing + ], pipe_buffers=[edge.buffer for edge in outgoing], pipe_live=[edge.live for edge in outgoing], live=any(edge.live for edge in incoming), copyts=all(keeps_clock(edge, plan) for edge in incoming), ) - return _resolve_rows_documents(argv, rows_path) + argv = _resolve_taps(argv, tap_path) + return _resolve_params_files(_resolve_rows_documents(argv, rows_path), plan, params_path) + + +def _resolve_taps( + argv: dict[str, list[str]], tap_path: Callable[[int], str] | None +) -> dict[str, list[str]]: + """Every tap placeholder in `argv` replaced by the pipe it is written to.""" + if tap_path is None: + return argv + + def resolve(token: str) -> str: + if not token.startswith(TAP_DOCUMENT): + return token + return tap_path(int(token[len(TAP_DOCUMENT) :])) + + return {pid: [resolve(token) for token in args] for pid, args in argv.items()} def _pad_of(process: SidecarProcess, edge: StreamEdge) -> int: @@ -1065,6 +1163,8 @@ def keeps_clock(edge: StreamEdge, plan: ProcessPlan) -> bool: reorder delay to take out. A codec package's encoder writes each packet at the time of the frame it coded, and its header says how deep it reorders rather than shifting anything, so its packets keep the clock. + A node network's packets keep the clock its inputs had, and a node + reading nothing writes its own, as a source module does. """ if isinstance(edge.format, DataFormat) or not encoded(edge.format): return True @@ -1073,13 +1173,39 @@ def keeps_clock(edge: StreamEdge, plan: ProcessPlan) -> bool: return False if producer.packet_source or producer.codec == "encode": return True - if not producer.packet_filter: + if not producer.packet_filter and not producer.node_network: return False return all( keeps_clock(e, plan) for e in plan.stream_edges if e.target == producer.id ) +def _resolve_params_files( + argv: dict[str, list[str]], plan: ProcessPlan, params_path: ParamsNamer | None +) -> dict[str, list[str]]: + """Each node's params placeholder replaced by the file its params were + written to; unchanged without a namer.""" + if params_path is None: + return argv + graphs = { + process.id: process.graph + for process in plan.processes + if isinstance(process, SidecarProcess) and process.graph is not None + } + + def resolve(token: str) -> str: + name, sep, rest = token.partition("=") + if not sep or not rest.startswith(PARAMS_FILE): + return token + pid, _, node = rest[len(PARAMS_FILE) :].partition(":") + graph = graphs.get(pid) + if graph is None or node not in graph.nodes: + return token + return f"{name}={params_path(rest, json.dumps(graph.nodes[node].args, sort_keys=True))}" + + return {pid: [resolve(token) for token in args] for pid, args in argv.items()} + + def _resolve_rows_documents( argv: dict[str, list[str]], rows_path: RowsNamer | None ) -> dict[str, list[str]]: @@ -1118,7 +1244,12 @@ def _sidecar_writes( order it reads them, and then its rows documents. Everything else hands its frames on over one output: its stdout where the edge chains, named here only where it does not, the pipe the relay serves.""" - several = process.packet_source or process.packet_filter or process.data_filter + several = ( + process.packet_source + or process.packet_filter + or process.data_filter + or process.node_network + ) if several: streams = [write[edge] for edge in outgoing] else: @@ -1181,7 +1312,7 @@ def render_plan( run = plan_argv(plan, sidecar_argv=sidecar_argv, pipe_path=pipe_path or _placeholder_pipe) argv = {pid: redact.argv(words) for pid, words in run.items()} if _is_pipeline(plan) and not plan.feeder_edges: - return _render_pipeline(plan, argv) + return "\n".join([_render_pipeline(plan, argv), *_listen_lines(plan)]) return _render_listing(plan, argv) @@ -1234,6 +1365,7 @@ def _render_listing(plan: ProcessPlan, argv: Mapping[str, list[str]]) -> str: for index, process in enumerate(plan.processes, start=1) ] lines += _feeder_lines(plan) + lines += _listen_lines(plan) lines += _lateral_lines(plan) lines.append(_COURTESY_NOTE) return "\n".join(lines) @@ -1261,6 +1393,17 @@ def _feeder_lines(plan: ProcessPlan) -> list[str]: ] +def _listen_lines(plan: ProcessPlan) -> list[str]: + """One line per port a process listens on for an input a node holds: + whatever connects there is shown, and the query writes nothing to it.""" + return [ + f"# listens: {process.id} at {feeder_path(port)} for {name}({port_name})" + for process in plan.processes + if isinstance(process, SidecarProcess) + for port, name, port_name in process.listens + ] + + def _lateral_lines(plan: ProcessPlan) -> list[str]: """Each run-time lateral as a block of its own: the data stream it is started from, the connections its instances write, what binds each value @@ -1269,7 +1412,8 @@ def _lateral_lines(plan: ProcessPlan) -> list[str]: for lateral in plan.laterals: lines.append( f"# run-time: for each message of {lateral.stream} ({lateral.writer} " - f"writes it to {feeder_path(lateral.tap)} for the host), {lateral.function}" + + ("writes it" if lateral.pipe else f"writes it to {feeder_path(lateral.tap)}") + + f" for the host), {lateral.function}" ) for connection in lateral.connections: found = [ @@ -1454,17 +1598,37 @@ def rows_path(placeholder: str) -> str: name = placeholder.rpartition(":")[2] or "0" return str(workspace() / f"rows-{name}.ndjson") + def params_path(placeholder: str, content: str) -> str: + # A node's params, written once where the run keeps its files. + path = workspace() / f"params-{len(list(workspace().glob('params-*')))}.json" + path.write_text(content, encoding="utf-8") + return str(path) + def pipe_path(edge: PipeEdge, side: Side) -> str: # Named here, made by the relay when the edge's stage starts. path = pipes.path(workspace(), str(len(named))) named[(edge, side)] = path return path + taps: dict[int, tuple[str, str]] = {} + + def tap_path(tap: int) -> str: + # The region writes one end and the host reads the other; the + # relay makes both when the writer's stage starts. + if tap not in taps: + taps[tap] = ( + pipes.path(workspace(), f"tap{tap}"), + pipes.path(workspace(), f"tap{tap}-host"), + ) + return taps[tap][0] + argv = plan_argv( plan, sidecar_argv=sidecar_argv, pipe_path=pipe_path, rows_path=rows_path, + params_path=params_path, + tap_path=tap_path, ) assigned = wires(plan) terminal = terminal_member(plan) if work is not None else None @@ -1496,7 +1660,7 @@ def pipe_path(edge: PipeEdge, side: Side) -> str: work=work, terminal=terminal, laterals=_Laterals( - compile_instance, sidecar_argv, rows or _print_row, dump + compile_instance, sidecar_argv, rows or _print_row, dump, taps ), stop=stop, ) @@ -1606,7 +1770,10 @@ def _sidecar_args( hint="pass sidecar_argv, which renders one sidecar process as argv", ) if streams > 1 and not ( - process.packet_source or process.packet_filter or process.data_filter + process.packet_source + or process.packet_filter + or process.data_filter + or process.node_network ): raise FfrwdError( ErrorCode.INTERNAL, @@ -1616,7 +1783,10 @@ def _sidecar_args( "spell a named pipe path", ) if len(reads) > 1 and not ( - process.packet_sink or process.packet_filter or process.data_filter + process.packet_sink + or process.packet_filter + or process.data_filter + or process.node_network ): raise FfrwdError( ErrorCode.INTERNAL, @@ -1789,6 +1959,9 @@ class _LateralRun: at a time. A message that arrives while another's instance runs waits its turn, and one whose programme time would overlap the running or a waiting one is refused with a row. Each instance ends with a row saying how. + + `ports` are the connections whose readers were not listening when the + writer started: the first instance waits for each to accept. """ def __init__( @@ -1799,8 +1972,14 @@ def __init__( rows: RowSink, dump: Path | None, echo: Callable[[str, list[str]], None] | None, + path: str | None = None, + ports: Sequence[int] = (), ) -> None: self.lateral = lateral + self._ports = list(ports) + # The pipe the host reads the messages off, where the writer's region + # writes them itself; None where an ffmpeg dials the tap's port. + self._path = path self._compile = compile_instance self._sidecar_argv = sidecar_argv self._rows = rows @@ -1812,6 +1991,10 @@ def __init__( self._running: _Launch | None = None self._read_all = False self._count = 0 + self._listener: socket.socket | None = None + if path is not None: + self._threads = [_start(self._read_pipe), _start(self._launch)] + return try: self._listener = socket.create_server((FEEDER_HOST, lateral.tap)) except OSError as err: @@ -1834,8 +2017,9 @@ def stop(self) -> None: self._turn.notify_all() for launch in left: self._row(launch.row, launch.start, refused=_ENDED) - with contextlib.suppress(OSError): - self._listener.close() + if self._listener is not None: + with contextlib.suppress(OSError): + self._listener.close() def join(self) -> None: for thread in self._threads: @@ -1873,7 +2057,41 @@ def _read(self) -> None: self._read_all = True self._turn.notify_all() + def _read_pipe(self) -> None: + """Read every message off the pipe the relay hands the host, once the + relay has made it.""" + assert self._path is not None # only started for a piped tap + try: + stream = None + while stream is None and not self._stop.is_set(): + try: + stream = open(self._path, "rb", buffering=0) # noqa: SIM115 + except OSError: + time.sleep(_FEEDER_POLL) + if stream is None: + return + messages = _Messages() + with stream: + while not self._stop.is_set(): + try: + chunk = stream.read(_CHUNK) + except OSError: + break + if not chunk: + break + for message in messages.feed(chunk): + self._arrived(message) + rest = messages.rest() + if rest: + self._count += 1 + self._row(self._count, None, refused=f"an unreadable message: {rest[:80]}") + finally: + with self._turn: + self._read_all = True + self._turn.notify_all() + def _accept(self) -> socket.socket | None: + assert self._listener is not None # only asked for a tap on a port while not self._stop.is_set(): try: connection, _ = self._listener.accept() @@ -1930,6 +2148,18 @@ def _launch(self) -> None: def _run(self, launch: _Launch) -> None: """One instance: compiled from the template as ``ffrwd run -v`` would, run by the plan runner, its members' stderr kept, and a row said.""" + unheard = _unheard_port(self._ports, self._stop) + if unheard is not None: + self._row( + launch.row, + launch.start, + refused=_ENDED + if self._stop.is_set() + else f"nothing listened on {feeder_path(unheard)} for its feeder in " + f"{FEEDER_WAIT:.0f}s", + ) + return + self._ports.clear() lateral = self.lateral filled = substitute(lateral.template, dict(launch.variables)) # The template follows the definitions, so what an unset variable's @@ -1970,6 +2200,26 @@ def _run(self, launch: _Launch) -> None: self._row(launch.row, launch.start, exit=_instance_exit(result)) +def _unheard_port(ports: Sequence[int], stop: threading.Event) -> int | None: + """The first of `ports` that accepted no connection in `FEEDER_WAIT`, or + before `stop`; None once every one has.""" + for port in ports: + until = time.monotonic() + FEEDER_WAIT + while True: + if stop.is_set() or time.monotonic() >= until: + return port + try: + with socket.create_connection((FEEDER_HOST, port), timeout=_FEEDER_POLL * 5): + break + except OSError: + time.sleep(_FEEDER_POLL) + return None + + +def _ignore_flow(row: Mapping[str, object]) -> None: + """A tap's copy has no flow a stall or overflow is read off.""" + + def _instance_exit(result: PlanResult) -> int: """How an instance ended: its failing member's code, else the code of a member the run's end stopped, else 0.""" @@ -1995,6 +2245,9 @@ class _Laterals: sidecar_argv: SidecarArgv | None rows: RowSink dump: Path | None + # Each piped lateral's tap: the pipe its region writes and the one the + # host reads, which the relay joins. + taps: Mapping[int, tuple[str, str]] = field(default_factory=dict) @dataclass @@ -2106,6 +2359,10 @@ def _run_stage( run.feeding, stop, ) + if failed is None and not show_only and not (stop is not None and stop.is_set()): + missing = never_started(run.held, run.members, run.local) + if missing is not None: + failed, timed_out, wedge = missing[0], True, missing[1] except KeyboardInterrupt: # `failed`/`timed_out`/`wedge` stay at their unstruck defaults: the # stage below reads as a clean stop, not a failure. @@ -2211,6 +2468,7 @@ def __init__( self.stage_wires, self.writers = stage_wires(plan, stage, assigned) self.feeds = [(wire.edge.source, wire.edge.target) for wire in self.stage_wires] self.feeding = {pid: readers for pid, (_, readers) in self.writers.items()} + self.held = held_writers(self.writers, self.feeds) self.deadline = math.inf if timeout is None else time.monotonic() + timeout self.members: dict[str, _Member] = {} self.watching: dict[str, subprocess.Popen[bytes]] = {} @@ -2232,6 +2490,7 @@ def start(self) -> None: for lateral in self.plan.laterals: if lateral.writer in self.local and laterals is not None: assert laterals.compile_instance is not None # execute_plan checks + tap = laterals.taps.get(lateral.tap) if lateral.pipe else None self.runs.append( _LateralRun( lateral, @@ -2240,12 +2499,16 @@ def start(self) -> None: laterals.rows, laterals.dump, self._echo, + tap[1] if tap is not None else None, + () + if lateral.writer in self.held + else [one.port for one in lateral.connections], ) ) self._relay_wires() for pid in _spawn_order(self.ids, self.stage_wires): - if pid in self.local and pid not in self.writers: + if pid in self.local and pid not in self.held: self._spawn(pid) for pid, window in self.watching.items(): @@ -2265,7 +2528,7 @@ def feed_writers( time; None otherwise. """ live = self.live + list(elsewhere) - for pid, (port, readers) in self.writers.items(): + for pid, (port, readers) in self.held.items(): if pid not in self.local: continue heard = _await_port( @@ -2351,6 +2614,25 @@ def _relay_wires(self) -> None: ) heard[name] = functools.partial(_heard, flow) self.relayed.append(name) + taps = self._laterals.taps if self._laterals is not None else {} + for lateral in self.plan.laterals: + tap = taps.get(lateral.tap) if lateral.pipe else None + if tap is None or lateral.writer not in self.local: + continue + # The messages the region writes, carried to the host's own read. + name = f"s{self._stage}tap{lateral.tap}" + edges.append( + RelayEdge( + id=name, + source=tap[0], + dest=tap[1], + depth=_read_ahead(None), + buffer=pipes.DEFAULT_BUFFER, + spool=True, + ) + ) + heard[name] = _ignore_flow + self.relayed.append(name) if edges: self._relay().open(edges, heard, self.deadline) @@ -2455,7 +2737,7 @@ def stage_result( if timed_out: failure = next((r for r in results if r.id == failed), None) failures = [r for r in results if r.id == failed] - code = 0 if failure is None else _FAILED + code = 0 if failure is None and wedge is None else _FAILED elif failed is not None: failure, consequences = _attribute(results, ended, feeds, writers) blamed = {r.id for r in consequences} @@ -2859,11 +3141,11 @@ def _cpu_seconds(proc: subprocess.Popen[bytes] | RemoteProcess) -> float | None: def _writes_rows_to_stdout(process: Process) -> bool: - """True for a sink region or a data filter whose rows name no file: they - ride its stdout.""" + """True for a sink region, a data filter or a node network whose rows + name no file: they ride its stdout, and its streams take named pipes.""" return ( isinstance(process, SidecarProcess) - and (process.sink or process.data_filter) + and (process.sink or process.data_filter or process.node_network) and any( not document.sink.alias and not document.sink.path for document in process.rows diff --git a/cli/ffrwd/functions.py b/cli/ffrwd/functions.py index adc13fc..6746050 100644 --- a/cli/ffrwd/functions.py +++ b/cli/ffrwd/functions.py @@ -112,6 +112,8 @@ FILTER_NAMESPACE, MACRO_NAMESPACE, MERGE_CUES, + NODE_REFUSAL, + OLDER_WORLD, SINK_ALIAS, SINK_STREAMS, ModuleExport, @@ -149,17 +151,22 @@ "SINK_ALIAS", "SINK_STREAMS", "WASM_DATA", + "WASM_NODE", "WASM_STREAM_NAMES", "WASM_STREAM_TYPES", "Annotation", "AnnotationField", "Parameter", + "STAR_FIELD", "RuntimeLateral", "Script", "Signature", "WasmFunction", "expanded", "is_number_argument", + "is_port", + "spread", + "struct_entries", "package_modules", "package_signatures", "package_sources", @@ -337,6 +344,16 @@ class RuntimeLateral: **WASM_STREAM_TYPES, WASM_DATA: "data", } +# What a declaration RETURNS when its signature is one only a node module +# has: kinds mixed in any order, a stream left out, an array of streams, rows +# beside several outputs. Which it is waits on the module's describe, so the +# declaration keeps the refusal an older world's module earns (`refusal`). +WASM_NODE = "node" +# Marks a field read `().*` wrote, which a call whose shape makes no +# such output leaves out rather than refuses. +STAR_FIELD = "star_field" +# The types a node reads as an input port rather than as a value. +_PORT_TYPES = frozenset({_WASM_STREAM, _WASM_AUDIO_STREAM, WASM_DATA}) _WASM_DATA_HINT = ( "a data filter reads its streams first -- data_stream for messages, " "video_stream or audio_stream for the time alone -- then the values it is " @@ -372,6 +389,14 @@ class RuntimeLateral: "write RETURNS TABLE( , ...), one name per column the body selects" ) _VALUE_BODY_HINT = "a value-returning function's body is one SELECT of one column" +_STRUCT_BODY_HINT = ( + "a struct-returning function's body is one SELECT, one column per field, in order" +) +_STRUCT_RETURN_HINT = ( + "a function returning a struct names a stream and rows beside it: RETURNS " + "STRUCT( video_stream, STRUCT( , ...)[]), or " + "audio_stream and cue[] the same way" +) _TABLE_BODY_HINT = ( "a table-returning function's body is one SELECT, one column per RETURNS TABLE column" ) @@ -583,6 +608,20 @@ def _leading_streams(params: tuple[Parameter, ...]) -> tuple[Parameter, ...]: return params[:end] +def is_port(param: Parameter) -> bool: + """Whether a node reads `param` as an input port: a stream, or rows.""" + return param.annotation is not None or element_type(param.type) in _PORT_TYPES + + +def _written_outputs(outputs: tuple[Parameter, ...]) -> str: + """A node's RETURNS as a signature spells it.""" + if not outputs: + return WASM_SOURCE + if len(outputs) == 1 and not outputs[0].name: + return outputs[0].type + return "STRUCT(" + ", ".join(f"{o.name} {o.type}" for o in outputs) + ")" + + @dataclass(frozen=True) class WasmFunction: """One ``LANGUAGE wasm`` declaration: a module, an export, and a signature. @@ -636,12 +675,30 @@ class WasmFunction: # data_stream, whose one output is the call itself, and for every other # kind. data_fields: tuple[str, ...] = () + # What a node module makes, read off the RETURNS: one unnamed entry for a + # stream or rows, one per field of a STRUCT, none for a source, whose + # outputs its shape names. None where no node returns that. + outputs: tuple[Parameter, ...] | None = None + # What a module of an older world refuses this declaration with, for one + # only a node can carry (`returns` is WASM_NODE). + refusal: FfrwdError | None = field(default=None, compare=False) + + @property + def is_node_only(self) -> bool: + """True for a declaration only a node module can carry.""" + return self.returns == WASM_NODE + + @property + def ports(self) -> tuple[Parameter, ...]: + """Every parameter a node reads as an input port, in declared order.""" + return tuple(param for param in self.params if is_port(param)) @property def is_value(self) -> bool: """True for a function returning a compile-time value, not a stream.""" return ( self.returns not in WASM_STREAM_TYPES + and not self.is_node_only and not self.is_data_filter and not self.is_sink and not self.is_packets @@ -774,6 +831,8 @@ def stream_kind(self) -> StreamType: ) if self.is_data_filter: return "data" + if self.is_node_only: + raise ValueError(f"'{self.name}' is a node; each port has a kind of its own") written = ( self.params[0].type if (self.is_sink or self.is_packets or self.is_packet_rows) and self.params @@ -834,6 +893,8 @@ def stream_params(self) -> tuple[Parameter, ...]: """ if self.is_value or self.is_rows: return () + if self.is_node_only: + return tuple(p for p in self.ports if p.annotation is None) return _leading_streams(self.params) @property @@ -858,7 +919,7 @@ def reads_params(self) -> tuple[Parameter, ...]: together, and a parameter one skipped would be a value parameter the other kept. """ - if self.is_value or self.is_rows or self.is_packet_rows: + if self.is_value or self.is_rows or self.is_packet_rows or self.is_node_only: return () found: list[Parameter] = [] for param in self.params[self.stream_arity :]: @@ -876,7 +937,7 @@ def reads(self) -> Annotation | None: riding one: :attr:`rows_param` is that one. None for a PACKET ROWS function too, whose rows are what it hands back. """ - if self.is_value or self.is_rows or self.is_packet_rows: + if self.is_value or self.is_rows or self.is_packet_rows or self.is_node_only: return None after = self.stream_arity return self.params[after].annotation if len(self.params) > after else None @@ -910,6 +971,8 @@ def value_params(self) -> tuple[Parameter, ...]: return () if self.is_value: return self.params + if self.is_node_only: + return tuple(param for param in self.params if not is_port(param)) skip = self.stream_arity + len(self.reads_params) return self.params[skip:] @@ -925,7 +988,7 @@ def written_params(self) -> tuple[Parameter, ...]: it and any producer, so the rows reach it as arguments of their own and every one of them is written at the call. """ - if self.is_value or self.is_rows: + if self.is_value or self.is_rows or self.is_node_only: return self.params if self.is_packets: return (*self.stream_params, *self.reads_params, *self.value_params) @@ -956,6 +1019,8 @@ def written_returns(self) -> str: if self.data_fields: written = ", ".join(f"{field} {WASM_DATA}" for field in self.data_fields) return f"STRUCT({written})" + if self.is_node_only and self.outputs is not None: + return _written_outputs(self.outputs) if self.emits is None: return self.returns stream, annotation = self.stream_field, self.emits @@ -1008,6 +1073,9 @@ class _Function: used: bool = False package: str = "" package_version: str = "" + # A ``RETURNS STRUCT( , )``'s fields, one per + # column the body selects. None for every other return. + fields: tuple[Parameter, ...] | None = None @property def returns_rows(self) -> bool: @@ -1099,6 +1167,7 @@ def expanded( yield script except FfrwdError as err: raise expander.translate(err) from err + expander.settle_older_world(expanded_tree) expander.settle(expanded_tree) @@ -1225,6 +1294,21 @@ def _annotation( return Annotation(name=column, fields=tuple(declared)) +def _many_annotation( + node: exp.Expr | None, column: str, name: str, anchor: exp.Expr +) -> Annotation | None: + """The record an array of record arrays declares, ``STRUCT(...)[][]``, or None. + + A node port that takes any number of rows streams, each of that record. + """ + if not isinstance(node, exp.DataType) or node.this is not exp.DataType.Type.ARRAY: + return None + inner = node.expressions[0] if len(node.expressions) == 1 else None + if not isinstance(inner, exp.DataType) or inner.this is not exp.DataType.Type.ARRAY: + return None + return _annotation(inner, column, name, anchor) + + def _record_array( node: exp.Expr | None, column: str, name: str, anchor: exp.Expr ) -> Annotation | None: @@ -1350,14 +1434,25 @@ def _reanchor(err: FfrwdError, name: str, anchor: exp.Expr) -> FfrwdError: def _body_select( - text: str, name: str, anchor: exp.Expr, columns: tuple[Parameter, ...] | None + text: str, + name: str, + anchor: exp.Expr, + columns: tuple[Parameter, ...] | None, + fields: tuple[Parameter, ...] | None = None, ) -> exp.Select: """Parse and shape-check one body: a single SELECT of the declared width. - A value's body is one column; a table's is one per ``RETURNS TABLE`` name. + A value's body is one column; a table's is one per ``RETURNS TABLE`` name, + and a struct's one per field. """ - wanted = 1 if columns is None else len(columns) - shape = _VALUE_BODY_HINT if columns is None else _TABLE_BODY_HINT + wanted = len(fields) if fields is not None else 1 if columns is None else len(columns) + shape = ( + _STRUCT_BODY_HINT + if fields is not None + else _VALUE_BODY_HINT + if columns is None + else _TABLE_BODY_HINT + ) try: parsed = parse(text) except FfrwdError as err: @@ -1397,7 +1492,9 @@ def _body_select( if written != wanted: plural = "" if written == 1 else "s" said = ( - "and a value is one column" + f"but its RETURNS STRUCT declares {wanted} fields" + if fields is not None + else "and a value is one column" if columns is None else f"but its RETURNS TABLE declares {wanted}" ) @@ -1570,6 +1667,23 @@ def _column_defs( if kind.allow_annotation else None ) + many = ( + _many_annotation(node.args.get("kind"), written, name, anchor) + if kind.allow_annotation and annotation is None + else None + ) + if many is not None: + if default is not None and not isinstance(default, exp.Null): + raise _error( + ErrorCode.UNSUPPORTED_SQL, + f"function '{name}' gives the {kind.noun} '{written}' a DEFAULT", + anchor, + fallback=create, + hint="rows are produced by the call that fills them; DEFAULT " + "NULL, the only default they can carry, makes them optional", + ) + declared.append(Parameter(written, f"{many.written}[]", default, many)) + continue if annotation is not None: # DEFAULT NULL makes the column optional: a call over a plain # stream wires no rows in. Any other default would be a value, @@ -1983,6 +2097,134 @@ def _define_wasm( params: tuple[Parameter, ...], returns_prop: exp.ReturnsProperty, body: exp.Expr | None, +) -> WasmFunction: + """One validated ``LANGUAGE wasm`` declaration, as every world reads it. + + The kinds an older world's module has are decided here, by the signature + alone (:func:`_define_wasm_kind`). A node module's reading rides beside + every one of them (`outputs`), since only its describe says a module is a + node. A signature no older kind has, and a node can, is a node's alone: + it keeps the refusal an older module earns, for lowering to raise once + the module turns out not to be a node. + """ + node = returns_prop.this if isinstance(returns_prop.this, exp.Expr) else None + try: + declared = _define_wasm_kind(create, name, identifier, params, returns_prop, body) + except FfrwdError as refusal: + found = _node_declaration( + create, name, identifier, params, returns_prop, body, refusal + ) + if found is None: + raise + return found + if declared.is_value or declared.is_codec or declared.is_sink: + return declared + if declared.is_packets: + # A node hands back the coded stream it was given, of that kind. + return replace(declared, outputs=(Parameter("", params[0].type),)) + outputs = _node_outputs(node, name, identifier) + return declared if outputs is None else replace(declared, outputs=outputs) + + +def _node_outputs( + node: exp.Expr | None, name: str, identifier: exp.Identifier +) -> tuple[Parameter, ...] | None: + """What a node module makes, as a RETURNS says it, or None where no node + returns that. + + A stream or a data stream is one output, and so are rows (``STRUCT(...)[]``, + ``cue[]``); a ``STRUCT`` of those names several, one per field; a + ``source`` names none, its shape naming them. + """ + written = _type_name(node) + if written in _PORT_TYPES: + return (Parameter("", written),) + if written == WASM_SOURCE: + return () + try: + rows = _annotation(node, _ROWS_RETURN, name, identifier) + except FfrwdError: + return None + if rows is not None: + return (Parameter("", written or rows.written, annotation=rows),) + fields = _struct_fields(node) + if not fields: + return None + outputs: list[Parameter] = [] + for field_node in fields: + field_name = ( + _ident_name(field_node.this) if isinstance(field_node.this, exp.Identifier) else "" + ) + if not field_name or any(o.name == field_name for o in outputs): + return None + kind = field_node.args.get("kind") + field_type = _type_name(kind) + if field_type in _PORT_TYPES: + outputs.append(Parameter(field_name, field_type)) + continue + try: + record = _annotation(kind, field_name, name, identifier) + except FfrwdError: + return None + if record is None: + return None + outputs.append(Parameter(field_name, field_type or record.written, annotation=record)) + return tuple(outputs) + + +def _node_declaration( + create: exp.Create, + name: str, + identifier: exp.Identifier, + params: tuple[Parameter, ...], + returns_prop: exp.ReturnsProperty, + body: exp.Expr | None, + refusal: FfrwdError, +) -> WasmFunction | None: + """The declaration as a node module reads it, or None where no node can be it. + + Its parameters are ports (streams, data streams and rows, each maybe an + array, each maybe ``DEFAULT NULL``) and values (text, number, boolean, + vector), in any order; its RETURNS is one a node makes, and names at + least one output, since a node that reads ports and makes nothing is a + sink. + """ + if returns_prop.args.get("is_table"): + return None + try: + module, export, _ = _module_export(body, name, identifier, create) + except FfrwdError: + return None + node = returns_prop.this if isinstance(returns_prop.this, exp.Expr) else None + outputs = _node_outputs(node, name, identifier) + if not outputs: + return None + if not any(is_port(param) for param in params): + return None + for param in params: + if not is_port(param) and param.type not in _ANNOTATION_FIELD_TYPES: + return None + line, col = _pos(identifier, create) + return WasmFunction( + name=name, + module=module, + export=export, + params=params, + returns=WASM_NODE, + line=line, + col=col, + outputs=outputs, + refusal=refusal, + ) + + +def _define_wasm_kind( + create: exp.Create, + name: str, + identifier: exp.Identifier, + params: tuple[Parameter, ...], + returns_prop: exp.ReturnsProperty, + body: exp.Expr | None, ) -> WasmFunction: """One validated ``LANGUAGE wasm`` declaration: a stream filter or a value. @@ -2509,14 +2751,18 @@ def _define(create: exp.Create, *, packaged: bool = False) -> _Function | WasmFu hint="a module is hosted, not inlined; say LANGUAGE wasm", ) columns: tuple[Parameter, ...] | None = None + fields: tuple[Parameter, ...] | None = None if returns_prop.args.get("is_table"): columns = _table_columns(returns_prop, name, identifier, create) returns = "TABLE(" + ", ".join(f"{c.name} {c.type}" for c in columns) + ")" else: node = returns_prop.this if isinstance(returns_prop.this, exp.Expr) else None - returns = _checked_type(node, name, identifier) + fields = _sql_struct_return(node, name, identifier) + returns = _written_outputs(fields) if fields else _checked_type(node, name, identifier) - body = _body_select(_body_text(create, name, identifier), name, identifier, columns) + body = _body_select( + _body_text(create, name, identifier), name, identifier, columns, fields + ) aliases = _body_aliases(body, name, params, identifier) _check_body_scope(body, name, params, aliases, identifier) return _Function( @@ -2528,9 +2774,34 @@ def _define(create: exp.Create, *, packaged: bool = False) -> _Function | WasmFu aliases=aliases, position=0, columns=columns, + fields=fields, ) +def _sql_struct_return( + node: exp.Expr | None, name: str, identifier: exp.Identifier +) -> tuple[Parameter, ...] | None: + """A sql function's ``RETURNS STRUCT( , )``. + + The fields a call over it is read as: the stream and the rows beside it, + which a node reading both takes as two arguments. None for a RETURNS that + is not a struct at all. + """ + fields = _struct_fields(node) + if fields is None: + return None + outputs = _node_outputs(node, name, identifier) + streams = [o for o in outputs or () if o.annotation is None] + if outputs is None or len(outputs) != 2 or len(streams) != 1 or outputs[0] != streams[0]: + raise _error( + ErrorCode.UNSUPPORTED_SQL, + f"function '{name}' returns a struct that is not a stream and rows", + identifier, + hint=_STRUCT_RETURN_HINT, + ) + return outputs + + def _in_lib( err: FfrwdError, name: str, path: Path, anchor: exp.Expr | None ) -> FfrwdError: @@ -3270,12 +3541,12 @@ def _call_alias(item: exp.Table, function: _Function) -> str: # What a run-time lateral's columns may be, and its values. -_RUNTIME_COLUMN_TYPES = (_WASM_STREAM, _WASM_AUDIO_STREAM) +_RUNTIME_COLUMN_TYPES = (_WASM_STREAM, _WASM_AUDIO_STREAM, WASM_DATA) _RUNTIME_VALUE_TYPES = ("number", "text", "boolean") _RUNTIME_HINT = ( "declare ( data_stream, , ...) RETURNS " - "TABLE(video video_stream, audio audio_stream), either column or both, " - "each value text, number or boolean" + "TABLE(video video_stream, audio audio_stream, rows data_stream), any of " + "the columns, each value text, number or boolean" ) @@ -3306,7 +3577,7 @@ def _check_runs_per_message(function: _Function, anchor: exp.Expr) -> None: ErrorCode.UNSUPPORTED_SQL, f"{function.qualified}() is started once per message of its data " f"stream, and returns '{wrong.name} {wrong.type}': it returns the " - "picture and sound a feeder takes, one of each at most", + "picture, sound and rows a feeder takes, one of each at most", anchor, hint=_RUNTIME_HINT, ) @@ -3568,7 +3839,13 @@ def _wasm_argument_kind(call: exp.Anonymous, wasm: Mapping[str, WasmFunction] | if _call_name(call) in _VECTOR_BUILTIN_ARITY: return "number" declared = wasm.get(_call_name(call)) if wasm is not None else None - return _declared_kind(declared.returns) if declared is not None else "stream" + if declared is None: + return "stream" + if declared.is_node_only: + outputs = declared.outputs or () + one = len(outputs) == 1 and not outputs[0].name + return _declared_kind(outputs[0].type) if one else "stream" + return _declared_kind(declared.returns) def _positional_and_named( @@ -3726,6 +4003,8 @@ def _struct_fields_of(declared: WasmFunction) -> tuple[str, ...]: return declared.data_fields if declared.emits is not None: return (declared.stream_field, declared.emits.name) + if declared.is_node_only and declared.outputs: + return tuple(output.name for output in declared.outputs if output.name) return () @@ -3808,6 +4087,73 @@ def _declared_kind(declared: str) -> str: return "stream" if TYPES[element].kind != "scalar" else declared +def _struct_of(fields: tuple[Parameter, ...], projections: Sequence[exp.Expr]) -> exp.Struct: + """A struct-returning call's value: each field, the column the body selects for it.""" + entries: list[exp.Expr] = [] + for field_param, projection in zip(fields, projections): + value = projection.this if isinstance(projection, exp.Alias) else projection + assert isinstance(value, exp.Expr) + entries.append( + exp.PropertyEQ(this=exp.to_identifier(field_param.name), expression=value) + ) + return exp.Struct(expressions=entries) + + +def struct_entries(node: exp.Expr) -> list[tuple[str, exp.Expr]] | None: + """A written ``STRUCT( AS , ...)`` as its fields in order, or None.""" + node = _unparen(node) + if not isinstance(node, exp.Struct): + return None + entries: list[tuple[str, exp.Expr]] = [] + for entry in node.expressions: + if not isinstance(entry, exp.PropertyEQ) or not isinstance(entry.expression, exp.Expr): + return None + entries.append((_ident_name(entry.this), entry.expression)) + return entries + + +def _struct_star(projection: exp.Expr) -> list[tuple[str, exp.Expr]] | None: + """``().*``: its fields, each read once, or None.""" + dot = _unparen(projection) + if not isinstance(dot, exp.Dot) or not isinstance(dot.expression, exp.Star): + return None + return struct_entries(dot.this) if isinstance(dot.this, exp.Expr) else None + + +def spread(arguments: Sequence[exp.Expr]) -> list[exp.Expr]: + """Positional arguments with each struct a call became read as its fields. + + A call over a stream and the rows beside it hands a node both, the way a + call over a two-part result always has: ``ring(spotted(v))`` is + ``ring(, )``. + """ + spread_out: list[exp.Expr] = [] + for argument in arguments: + entries = None if isinstance(argument, exp.Kwarg) else struct_entries(argument) + if entries is None: + spread_out.append(argument) + continue + spread_out.extend(value for _, value in entries) + return spread_out + + +def _struct_field_read(value: exp.Expr) -> tuple[exp.Expr, exp.Expr] | None: + """``.`` around a struct a call became: the read, and the field.""" + inner: exp.Expr = value + parent = value.parent + while isinstance(parent, exp.Paren): + inner, parent = parent, parent.parent + if not isinstance(parent, exp.Dot) or parent.this is not inner: + return None + field_node = parent.args.get("expression") + entries = struct_entries(value) + if entries is None or not isinstance(field_node, exp.Identifier): + return None + wanted = _ident_name(field_node) + found = next((entry for name, entry in entries if name == wanted), None) + return None if found is None else (parent, found) + + def _record_rows(argument: exp.Expr) -> list[exp.Expr] | None: """The rows a ``STRUCT(...)[]`` argument writes, or None for anything else.""" node = _unparen(argument) @@ -4378,6 +4724,11 @@ def _expand_within( site.node.replace(replacement) if site.node is root: root = replacement + read = _struct_field_read(replacement) + if read is not None: + if read[0] is root: + root = read[1] + read[0].replace(read[1]) def _next_call(self, root: exp.Expr, position: int) -> _CallSite | _WasmSite | None: """The first call to a defined function in `root`'s own query, if any. @@ -4499,6 +4850,11 @@ def _expand_data_stars(self, statement: exp.Expr) -> None: projections: list[exp.Expr] = [] expanded = False for projection in select.expressions: + entries = _struct_star(projection) + if entries is not None: + expanded = True + projections.extend(exp.alias_(value, name) for name, value in entries) + continue call = self._data_star_call(projection) if call is None: projections.append(projection) @@ -4507,6 +4863,7 @@ def _expand_data_stars(self, statement: exp.Expr) -> None: declared = self.wasm[_call_name(call)] for field_name in _struct_fields_of(declared): read = exp.Dot(this=call.copy(), expression=exp.to_identifier(field_name)) + read.meta[STAR_FIELD] = True projections.append(exp.alias_(read, field_name)) if expanded: select.set("expressions", projections) @@ -4666,8 +5023,9 @@ def _check_wasm_calls(self, statement: exp.Expr, position: int) -> None: if declared.is_packet_rows: # As above: a Table-position call is skipped before this # loop sees it, so anything reaching here is written where - # a stream or a value belongs. - raise _error( + # a stream or a value belongs. A node answers it with a data + # stream, so the refusal waits for the module's describe. + node.meta[OLDER_WORLD] = _error( ErrorCode.UNSUPPORTED_SQL, f"function '{declared.name}' returns rows read off a " "stream's packets, and this call is not in FROM", @@ -4678,9 +5036,111 @@ def _check_wasm_calls(self, statement: exp.Expr, position: int) -> None: arguments = [ argument for argument in node.expressions if isinstance(argument, exp.Expr) ] - self._check_wasm_arguments(declared, node, arguments) + if declared.is_node_only or OLDER_WORLD in node.meta: + try: + self._check_node_arguments(declared, node, arguments) + except FfrwdError as refusal: + node.meta[NODE_REFUSAL] = refusal + else: + self._check_any_world_arguments(declared, node, arguments) self.wasm_used.add(declared.name) + def _check_any_world_arguments( + self, declared: WasmFunction, call: exp.Anonymous, arguments: list[exp.Expr] + ) -> None: + """A call's arguments as an older world's module reads them, or as a node does. + + A call only a node reads keeps the older refusal for lowering, which + raises it once the module's describe says it is not a node. + """ + try: + self._check_wasm_arguments(declared, call, arguments) + except FfrwdError as refusal: + if declared.outputs is None: + raise + try: + self._check_node_arguments(declared, call, arguments) + except FfrwdError: + raise refusal from None + call.meta[OLDER_WORLD] = refusal + + def _check_node_arguments( + self, declared: WasmFunction, call: exp.Anonymous, arguments: list[exp.Expr] + ) -> None: + """A call to a node against its signature: ports and values alike. + + The positionals fill the parameters in declared order, whatever each + is, and a name fills the one it names; each parameter once, and one + with no DEFAULT always. A port takes a stream, rows, an array of + either or NULL, and a number where a held input may be given as a port; + which of those fits is the module's shape to say, in lowering. + """ + positional, named = _positional_and_named(spread(arguments)) + params = declared.params + plural = "" if len(positional) == 1 else "s" + if len(positional) > len(params): + raise _error( + ErrorCode.UDF_ARG_TYPE, + f"{declared.name}() got {len(positional)} argument{plural}, but it " + f"declares {len(params)}", + call, + hint=declared.signature, + ) + filled: dict[str, exp.Expr] = { + param.name: argument for param, argument in zip(params, positional) + } + for name, value in named: + param = next((p for p in params if p.name == name), None) + if param is None: + listed = ", ".join(f"'{p.name}'" for p in params) + raise _error( + ErrorCode.UDF_ARG_TYPE, + f"{declared.name}() has no parameter '{name}'", + value, + fallback=call, + hint=f"its parameters are {listed}", + ) + if name in filled: + raise _error( + ErrorCode.UDF_ARG_TYPE, + f"{declared.name}() gets '{name}' twice: positionally and by name", + value, + fallback=call, + hint=f"write '{name}' once: {declared.signature}", + ) + filled[name] = value + unfilled = next( + (p for p in params if p.default is None and p.name not in filled), None + ) + if unfilled is not None: + raise _error( + ErrorCode.UDF_ARG_TYPE, + f"{declared.name}() does not write '{unfilled.name}', which has no " + "DEFAULT", + call, + hint=declared.signature, + ) + for param in params: + argument = filled.get(param.name) + if argument is None: + continue + if not is_port(param): + self._check_wasm_argument(declared, call, param, argument) + continue + if _call_name(argument) == _INPUT: + self._check_wasm_argument(declared, call, param, argument) + written = _argument_kind(argument, self.wasm) + if written in (None, "stream", "number", _RECORD_KIND) or _is_null(argument): + continue + raise _error( + ErrorCode.UDF_ARG_TYPE, + f"{declared.name}() takes {param.type} as its '{param.name}' " + f"argument, got {_KIND_NAMES.get(written, written)}", + argument, + fallback=call, + hint=declared.signature, + ) + def _reject_wasm_row_source(self, item: exp.Table) -> None: """A wasm function in FROM: refused unless it is a source, which is exactly a table. @@ -5096,6 +5556,8 @@ def _expand_call( with self._scoped(function.identity): self._expand_within(body, host, position, (*stack, function.qualified)) _splice(host, body) + if function.fields is not None: + return _struct_of(function.fields, body.expressions) projection: exp.Expr = body.expressions[0] inner = projection.this if isinstance(projection, exp.Alias) else None return inner if isinstance(inner, exp.Expr) else projection @@ -5666,6 +6128,14 @@ def translate(self, err: FfrwdError) -> FfrwdError: hint=err.hint, ) + def settle_older_world(self, script: exp.Expr) -> None: + """Say every refusal kept for lowering at the call site, as one raised now is.""" + for node in script.walk(): + for key in (OLDER_WORLD, NODE_REFUSAL): + refusal = node.meta.get(key) + if isinstance(refusal, FfrwdError): + node.meta[key] = self.translate(refusal) + def settle(self, script: exp.Expr) -> None: """Flatten every stamped position onto its call site. diff --git a/cli/ffrwd/ir.py b/cli/ffrwd/ir.py index c45eb63..b796a65 100644 --- a/cli/ffrwd/ir.py +++ b/cli/ffrwd/ir.py @@ -79,6 +79,11 @@ # the same way, and its one argument is the gap rows still merge across. ROWMERGE = "rowmerge" MAX_DISTANCE = "max_distance" +# The same node over rows written once per tick, each carrying the pts its +# span began at as `start_t`: runs of one `start_t` become one span, cut at +# `max_span` seconds, which is also how late a span may leave. +MAX_SPAN = "max_span" +MERGE_SPANS = "merge_spans" # The node that drops the pictures of a live stream that fall too far behind # the wall clock. Hosted the same way; its arguments are how late, in seconds @@ -105,6 +110,17 @@ ROWS_DOCUMENT = "ffrwd:rows:" +# Where a node region writes a run-time lateral's messages, as NDJSON, for the +# host to read: ``ffrwd:tap:``, a placeholder a run resolves to a pipe. +TAP_DOCUMENT = "ffrwd:tap:" + + +# Where a node's params go when they are too long for a command line, or not +# a flat list of values: ``ffrwd:params::``. A placeholder +# like a rows document's, which a run resolves to a file of the params. +PARAMS_FILE = "ffrwd:params:" + + def is_rows_document(path: str) -> bool: """True for a path that is one of those placeholders.""" return path.startswith(ROWS_DOCUMENT) @@ -173,6 +189,13 @@ class Node: # carries rows and no frames. Only a ROWS MODULE has any: it reads rows # and writes rows, so it has no `inputs` and no `outputs` at all. rows_inputs: list[str] = field(default_factory=list) + # A node module's port each input binds, one per input, and the port + # each output is, one per output. Empty for every other node. + ports: list[str] = field(default_factory=list) + out_ports: list[str] = field(default_factory=list) + # A node module's bound list, as the JSON its shape was asked with; the + # host asks the same and hints each stream from it. Empty for any other. + bound: str = "" @property def rows_only(self) -> bool: @@ -191,6 +214,12 @@ def to_dict(self) -> dict[str, object]: written["reads_annotations"] = True if self.rows_inputs: written["rows_inputs"] = list(self.rows_inputs) + if self.ports: + written["ports"] = list(self.ports) + if self.out_ports: + written["out_ports"] = list(self.out_ports) + if self.bound: + written["bound"] = self.bound return written @classmethod @@ -204,9 +233,13 @@ def from_dict(cls, d: dict[str, object]) -> Node: assert isinstance(node_filter, str) assert isinstance(node_args, dict) raw_rows_inputs = d.get("rows_inputs") or [] + raw_ports = d.get("ports") or [] + raw_out_ports = d.get("out_ports") or [] assert isinstance(node_inputs, list) assert isinstance(node_outputs, list) assert isinstance(raw_rows_inputs, list) + assert isinstance(raw_ports, list) + assert isinstance(raw_out_ports, list) return cls( id=node_id, filter=node_filter, @@ -215,6 +248,9 @@ def from_dict(cls, d: dict[str, object]) -> Node: outputs=[_parse_stream_type(x) for x in node_outputs], reads_annotations=bool(d.get("reads_annotations", False)), rows_inputs=[str(x) for x in raw_rows_inputs], + ports=[str(x) for x in raw_ports], + out_ports=[str(x) for x in raw_out_ports], + bound=str(d.get("bound") or ""), ) @@ -783,6 +819,10 @@ class Lateral: line: int = 1 col: int = 1 writer: str = "" + # True where a node region writes the messages itself, as NDJSON on the + # pipe ``ffrwd:tap:`` names, rather than an ffmpeg copying them to + # the loopback port `tap`. + pipe: bool = False def to_dict(self) -> dict[str, object]: return { @@ -798,6 +838,7 @@ def to_dict(self) -> dict[str, object]: "line": self.line, "col": self.col, "writer": self.writer, + "pipe": self.pipe, } @classmethod @@ -828,6 +869,7 @@ def from_dict(cls, d: dict[str, object]) -> Lateral: line=raw_line, col=raw_col, writer=str(d.get("writer", "")), + pipe=d.get("pipe") is True, ) @@ -916,6 +958,15 @@ class Graph: # Each run-time lateral: its data stream is a sink of its own, written to # the loopback port `tap` names, and its instances are started per message. laterals: list[Lateral] = field(default_factory=list) + # Each node module's node id -> the shape its call was given, as the + # sidecar wrote it: its ports, their pairings and its clock. + node_shapes: dict[str, dict[str, object]] = field(default_factory=dict) + # A node read in FROM: its alias -> the node making the alias's streams, + # which every ``src:`` ref has been rewritten to a pad of. + node_sources: dict[str, str] = field(default_factory=dict) + # The tags the query wrote on a stream a node reads, by the ref read: + # what no edge into the node's sidecar carries. + stream_tags: dict[str, dict[str, str]] = field(default_factory=dict) @property def outputs(self) -> list[Output]: @@ -995,6 +1046,12 @@ def to_dict(self) -> dict[str, object]: } if self.laterals: d["laterals"] = [lateral.to_dict() for lateral in self.laterals] + if self.node_shapes: + d["node_shapes"] = {name: dict(shape) for name, shape in self.node_shapes.items()} + if self.node_sources: + d["node_sources"] = dict(self.node_sources) + if self.stream_tags: + d["stream_tags"] = {ref: dict(tags) for ref, tags in self.stream_tags.items()} return d @classmethod @@ -1026,10 +1083,7 @@ def from_dict(cls, d: dict[str, object]) -> Graph: for alias, bounds in raw_input_trims.items(): assert isinstance(bounds, list) start, end = bounds - input_trims[str(alias)] = ( - float(start) if start is not None else None, - float(end) if end is not None else None, - ) + input_trims[str(alias)] = (_seconds(start), _seconds(end)) raw_input_options = d.get("input_options") input_options: dict[str, dict[str, object]] = {} @@ -1130,6 +1184,24 @@ def from_dict(cls, d: dict[str, object]) -> Graph: assert isinstance(raw_laterals, list) laterals = [Lateral.from_dict(one) for one in raw_laterals if isinstance(one, dict)] + raw_node_shapes = d.get("node_shapes") or {} + assert isinstance(raw_node_shapes, dict) + node_shapes = { + str(name): dict(shape) + for name, shape in raw_node_shapes.items() + if isinstance(shape, dict) + } + raw_node_sources = d.get("node_sources") or {} + assert isinstance(raw_node_sources, dict) + node_sources = {str(alias): str(name) for alias, name in raw_node_sources.items()} + raw_stream_tags = d.get("stream_tags") or {} + assert isinstance(raw_stream_tags, dict) + stream_tags = { + str(ref): {str(key): str(value) for key, value in tags.items()} + for ref, tags in raw_stream_tags.items() + if isinstance(tags, dict) + } + return cls( input_paths=[str(p) for p in raw_inputs], sources={str(k): int(v) for k, v in raw_sources.items()}, @@ -1151,9 +1223,23 @@ def from_dict(cls, d: dict[str, object]) -> Graph: dropped_aliases=dropped_aliases, feeders=feeders, laterals=laterals, + node_shapes=node_shapes, + node_sources=node_sources, + stream_tags=stream_tags, ) +def _seconds(value: object) -> float | None: + """A written bound as it was: a whole number stays one, so it reads back + the way it was written.""" + if value is None: + return None + if isinstance(value, int) and not isinstance(value, bool): + return value + assert isinstance(value, int | float | str) + return float(value) + + _MergeKey = tuple[str, tuple[tuple[str, object], ...]] diff --git a/cli/ffrwd/lower.py b/cli/ffrwd/lower.py index 308fc29..7f5987f 100644 --- a/cli/ffrwd/lower.py +++ b/cli/ffrwd/lower.py @@ -286,6 +286,7 @@ import urllib.parse from collections.abc import Callable, Iterable, Mapping, Sequence from dataclasses import dataclass, field, replace +from fractions import Fraction from typing import Literal, cast from sqlglot import exp @@ -295,6 +296,7 @@ from ffrwd.functions import ( DECLARED_STREAM, SHARED_ARGUMENT, + STAR_FIELD, WASM_DATA, WASM_STREAM_NAMES, WASM_STREAM_TYPES, @@ -303,7 +305,9 @@ RuntimeLateral, WasmFunction, is_number_argument, + is_port, named_annotation_parameter, + spread, wasm_named_parameter, ) from ffrwd.inputs import render_options, rendered_options @@ -312,6 +316,8 @@ FEEDER_HOST, LEAKY, MAX_DISTANCE, + MAX_SPAN, + MERGE_SPANS, NO_CHAPTERS, NO_METADATA, PIPE, @@ -319,6 +325,7 @@ ROWFILTER, ROWMERGE, ROWS_DOCUMENT, + TAP_DOCUMENT, Attachment, FeederCall, FrameRef, @@ -356,6 +363,8 @@ MACRO_NAMESPACE, MAP_COLUMNS, MERGE_CUES, + NODE_REFUSAL, + OLDER_WORLD, ROW_MERGE, ROW_PREDICATE, ROW_STREAM, @@ -418,8 +427,30 @@ track_cues, ) from ffrwd.probe import probe as probe_one_path -from ffrwd.processes import CLOCK_SIZE, COPY_CODEC, NUT, RAWVIDEO, ref_type +from ffrwd.processes import ( + CLOCK_SIZE, + COPY_CODEC, + NUT, + RAWVIDEO, + SETPARAMS_COLOR, + ref_type, + stream_colorimetry, +) from ffrwd.registry import DynamicFilter, FilterOption, Registry, SourceFilter +from ffrwd.shapes import ( + Binding, + Hold, + InputPort, + NodeShape, + OutputPort, + Shape, + ShapeCache, + StreamHint, + Wants, + bound_json, + node_shape, + row_mismatch, +) from ffrwd.sink import ( CODEC_PARAMS_FLAGS, COLOR_OPTIONS, @@ -444,6 +475,7 @@ TableSink, VectorCell, ) +from ffrwd.timing import stream_rate from ffrwd.types import ( ATTACHMENT_TYPE, ATTACHMENTS_COLUMN, @@ -480,11 +512,14 @@ CODEC_ENCODERS, CODEC_WORLD, DATA_FILTER_WORLD, + EMITTED_ROWS_PORT, FFMPEG_SAMPLE_FMTS, PACKET_FILTER_WORLD, PACKET_SOURCE_WORLD, + SAMPLE_FMT_CODECS, WIRE_AUDIO_CODECS, WIRE_PIX_FMTS, + WIRE_SAMPLE_FMTS, WIRE_VIDEO_CODECS, WORLDS, Described, @@ -495,6 +530,9 @@ PacketRead, ProbeSource, ReadPackets, + SinkArity, + SinkWants, + _grant_args, audio_encoder_codec, catalog_as_probe, encoder_codec, @@ -1656,6 +1694,10 @@ class _Connection: # never written by a COPY, only a feeder's (:meth:`_Lowerer._feed`), so it # names no pad at all. ``lateral::``. _LATERAL_REF = "lateral:" + + +class _UnmadeField(Exception): + """A field `.*` expanded that this call's shape makes no output for.""" # How a run-time lateral's data stream reaches the host: ffmpeg's raw data # muxer, which writes each packet's bytes as they are, so the host reads one # JSON object after another and a heartbeat's blank payload is only space. @@ -1750,7 +1792,11 @@ def _lateral_column(lateral: RuntimeLateral, call: _Call) -> tuple[str, StreamTy written = call.args[1] if len(call.args) > 1 else None name = str(written.this) if isinstance(written, exp.Literal) else "" column = next(c for c in lateral.columns if c.name == name) - return name, WASM_STREAM_TYPES[column.type] + return name, _LATERAL_KINDS[column.type] + + +# The kind of each column a run-time lateral may return. +_LATERAL_KINDS: Mapping[str, StreamType] = {**WASM_STREAM_TYPES, WASM_DATA: "data"} def _instance_template( @@ -1759,7 +1805,8 @@ def _instance_template( """One instance of a run-time lateral, as SQL. One COPY per connection its streams go to, writing them there as the - NUT a feeder carries, picture then sound. The call writes NULL in the + NUT a feeder carries, picture, sound, then the rows a group's data + input reads. The call writes NULL in the data stream's place, what the call wrote itself, and every other value as a variable, ``:'name'`` for text and ``:name`` for the rest, which the host sets per message; one it leaves unset is NULL, and takes the @@ -1778,9 +1825,7 @@ def _instance_template( call = f"{declared.function}({', '.join(arguments)}) {alias}" statements: list[str] = [] for port, connection in connections: - streams = sorted( - connection.streams.values(), key=lambda s: 0 if s.type == "video" else 1 - ) + streams = sorted(connection.streams.values(), key=_feed_order) columns = [] for stream in streams: parts = _lateral_parts(stream.ref) @@ -1798,6 +1843,11 @@ def _instance_template( return ";\n".join(statements) +def _feed_order(stream: _Stream) -> int: + """Where a stream rides a feeder's NUT: picture, sound, then data.""" + return {"video": 0, "audio": 1}.get(stream.type, 2) + + def _namespaced_call(node: exp.Expr) -> exp.Anonymous | None: """The ``exp.Anonymous`` inside ``ffmpeg.(...)``, else None. @@ -2003,6 +2053,112 @@ def _sink_alias(argument: exp.Expr) -> str | None: return written if isinstance(written, str) and written else None +def _param_takes_list(described: Described, name: str) -> bool: + """Whether the module's params schema takes `name` as an array, alone or + among other types.""" + properties = described.params_schema.get("properties") + return "array" in _schema_types( + properties.get(name) if isinstance(properties, dict) else None + ) + + +def _as_scalar_type(value: RowValue, schema: object) -> RowValue: + """A number written as the schema's first scalar type: a whole number + where that is ``integer``, so a param taking one port or a list of them + is handed one port as the integer it is.""" + scalars = [kind for kind in _schema_types(schema) if kind != "array"] + if ( + scalars + and scalars[0] == "integer" + and isinstance(value, float) + and value.is_integer() + ): + return int(value) + return value + + +def _sink_port(shape: NodeShape, kind: StreamType) -> InputPort | None: + """The input of a node sink's shape that takes streams of `kind`: its data + port, or a coded port named for the kind, an unnamed one taking video.""" + if kind == "data": + return next((port for port in shape.inputs if port.kind == "data"), None) + coded = [port for port in shape.inputs if port.kind == "packets"] + named = next((port for port in coded if port.name == kind), None) + if named is not None or kind == "audio": + return named + return next((port for port in coded if port.name != "audio"), None) + + +def _sink_view(described: Described, ports: Mapping[str, InputPort | None]) -> Described: + """A node sink as the packet sink its ports describe: how many streams of + each kind it reads, the codecs it takes, and how much of each it wants.""" + + def arity(port: InputPort | None) -> SinkArity: + return "none" if port is None else "many" if port.many else "one" + + video, audio, data = ports["video"], ports["audio"], ports["data"] + first = next((port for port in (video, audio, data) if port is not None), None) + return replace( + described, + video_codecs=tuple(video.accepts.codecs) if video is not None else (), + audio_codecs=tuple(audio.accepts.codecs) if audio is not None else (), + video_streams=arity(video), + audio_streams=arity(audio), + data_streams=arity(data), + wants=_sink_wants(first.accepts.wants) if first is not None else "all", + packet_filter=False, + ) + + +def _group_hold(shape: NodeShape, port: InputPort) -> Hold | None: + """How `port` is held where a feed by port serves it: its own hold, or + for a data input naming a hold group, that group's hold on a port.""" + if port.pairing.hold is not None: + return port.pairing.hold + interval = port.pairing.interval + if port.kind != "data" or interval is None or interval.group is None: + return None + return next( + ( + one.pairing.hold + for one in shape.inputs + if one.pairing.hold is not None + and one.pairing.hold.group == interval.group + and one.pairing.hold.port_param is not None + ), + None, + ) + + +def _sink_wants(wants: Wants) -> SinkWants: + """What a port reading coded packets asks for; `timing` is a frame + port's, which the host refuses on any other.""" + return "all" if wants == "timing" else wants + + +def _conformed(shape: NodeShape, bound: Sequence[Binding]) -> list[Binding]: + """`bound` with each sound's rate the one its port conforms it to: the + first rate an audio port accepts, where it names any. A port read for + its timing is handed the sound as it is.""" + conformed: list[Binding] = [] + for binding in bound: + port = shape.input(binding.input) + rates = port.accepts.sample_rates if port is not None else () + if port is None or port.kind != "audio" or not rates or port.accepts.wants == "timing": + conformed.append(binding) + continue + conformed.append( + replace( + binding, + streams=tuple( + hint if hint.rate is None else StreamHint(Fraction(rates[0])) + for hint in binding.streams + ), + ) + ) + return conformed + + def _sink_stream_count(node: exp.Expr, arguments: int) -> int: """How many of a sink call's leading arguments came out of the SELECT list. @@ -2537,43 +2693,6 @@ def _chapter_title(value: RowValue, node: exp.Expr) -> str | None: } ) -# A stream's colorimetry: each option ffmpeg's output takes it by, against -# the field ffprobe reports it in and setparams' option for it. -_PROBED_COLOR: Mapping[str, str] = { - "color_range": "color_range", - "color_primaries": "color_primaries", - "color_trc": "color_transfer", - "colorspace": "color_space", - "chroma_sample_location": "chroma_location", -} -_SETPARAMS_COLOR: Mapping[str, str] = { - "color_range": "range", - "color_primaries": "color_primaries", - "color_trc": "color_trc", - "colorspace": "colorspace", - "chroma_sample_location": "chroma_location", -} -# What ffprobe and setparams write for a field nothing settles. -_UNSAID_COLOR = frozenset({"unknown", "unspecified", "reserved", "auto"}) -# The fields that describe YUV alone, which an RGB picture has none of. -_YUV_ONLY_COLOR = ("color_range", "colorspace", "chroma_sample_location") -_RGB_PREFIXES = ("rgb", "bgr", "gbr", "argb", "abgr") -# Filters that convert colour, past which a stream's colorimetry is no longer -# its input's. A `scale` does too where it names an in_ or out_ option. -_COLOR_CONVERTING_FILTERS = frozenset( - {"colorspace", "colormatrix", "zscale", "tonemap", "tonemap_opencl", "libplacebo"} -) - - -def _converts_colour(node: Node) -> bool: - """True where `node` converts the colour of the pictures through it.""" - if node.filter in _COLOR_CONVERTING_FILTERS: - return True - return node.filter == "scale" and any( - str(key).startswith(("in_", "out_")) for key in node.args - ) - - def _colorimetry_options( tagged: Mapping[int, Mapping[str, str]], outputs: Sequence[Output] ) -> dict[str, object]: @@ -3859,6 +3978,149 @@ class _Env: # ExpandCtx +@dataclass(frozen=True) +class _NodeInstance: + """One node module's instance in the graph, and the shape it was given.""" + + ref: str + shape: NodeShape + + +# The kind of port each node parameter is, and the type a declaration spells +# one of each kind with. +_PORT_KINDS: Mapping[str, StreamType] = { + "video_stream": "video", + "audio_stream": "audio", + WASM_DATA: "data", +} +_PORT_TYPE_NAMES: Mapping[str, str] = { + "video": "video_stream", + "audio": "audio_stream", + "data": "data_stream or STRUCT(...)[]", + "packets": "packets", +} + +# What a JSON schema calls each type an annotation field may be. +_SCHEMA_TYPES: Mapping[str, dict[str, object]] = { + "number": {"type": "number"}, + "text": {"type": "string"}, + "boolean": {"type": "boolean"}, + "vector": {"type": "array", "items": {"type": "number"}}, +} + + +def _node_source_probe(shape: NodeShape) -> ProbeResult: + """A node source's outputs as a :class:`~ffrwd.probe.ProbeResult`. + + One stream per output, counted per kind the way ffprobe counts a file's. + A rate clock is the pictures' rate. Outputs naming a relation row group + into one rendition per row, which carries that row's name, bandwidth, + codecs and language; a source naming none is one row, as a file is. + """ + fps = f"{shape.clock.rate[0]}/{shape.clock.rate[1]}" if shape.clock.rate else None + counted: dict[str, int] = {} + streams: list[StreamMeta] = [] + by_row: dict[int, list[StreamMeta]] = {} + for output in shape.outputs: + kind = _output_kind(shape, output, {}) + index = counted.get(kind, 0) + counted[kind] = index + 1 + found = output.format + coded = found is not None and found.kind == "packets" + video = found is not None and (found.kind == "video" or coded) and kind == "video" + audio = found is not None and (found.kind == "audio" or coded) and kind == "audio" + stream = StreamMeta( + type=kind, + index=index, + metadata={}, + width=found.width if found is not None and video else None, + height=found.height if found is not None and video else None, + fps=fps if kind == "video" else None, + sample_rate=found.sample_rate if found is not None and audio else None, + codec=found.codec + if found is not None and coded + else _NODE_SOURCE_CODECS.get( + (found.sample_format if found is not None and audio else None) or kind + ), + channels=found.channels if found is not None and audio else None, + ) + streams.append(stream) + if output.row is not None: + by_row.setdefault(output.row, []).append(stream) + renditions: list[RenditionMeta] = [] + for row, members in sorted(by_row.items()): + said = shape.relation[row] if row < len(shape.relation) else {} + picture = next((s for s in members if s.type == "video"), None) + bandwidth = said.get("bandwidth") + renditions.append( + RenditionMeta( + streams=members, + bandwidth=bandwidth if isinstance(bandwidth, int) else None, + width=picture.width if picture is not None else None, + height=picture.height if picture is not None else None, + codecs=_text_or_none(said.get("codecs")), + name=_text_or_none(said.get("name")), + language=_text_or_none(said.get("language")), + program_id=None, + ) + ) + return ProbeResult( + streams=streams, format_name="node", renditions=renditions, live=not shape.bounded + ) + + +# What a node source's stream is on the wire, by its kind or, for sound, its +# sample format: what the probe of a file would have said its codec is. +_NODE_SOURCE_CODECS: Mapping[str, str] = { + "video": RAWVIDEO, + "audio": SAMPLE_FMT_CODECS["f32"], + "data": JSON_CODEC, + **SAMPLE_FMT_CODECS, +} + + +def _output_kind( + shape: NodeShape, output: OutputPort, bound: Mapping[str, StreamType] +) -> StreamType: + """The kind of stream a node's output is in the graph. + + Coded packets are the kind they carry: what the coded stream says, or, + where the output follows an input's format, what is bound there. + """ + if output.kind != "packets": + return cast(StreamType, output.kind) + found = output.format + if found is not None and found.kind == "packets" and found.coded is not None: + return found.coded + follows = ( + found.port + if found is not None and found.kind == "like" + else shape.clock.port + if found is None and shape.clock.kind == "input" + else None + ) + return bound.get(follows or "", "video") + + +def _text_or_none(value: object) -> str | None: + return value if isinstance(value, str) else None + + +def _port_kind(param: Parameter) -> StreamType: + """The kind of stream a node reads or writes for `param`: rows are data.""" + if param.annotation is not None: + return "data" + return _PORT_KINDS.get(element_type(param.type), "data") + + +def _record_schema(record: Annotation) -> dict[str, object]: + """A declared record as the JSON schema of one row of it.""" + return { + "type": "object", + "properties": {f.name: dict(_SCHEMA_TYPES[f.type]) for f in record.fields}, + } + + class _NodeFactory: """Mints ``n1, n2, ...`` node ids into a graph, in creation order. @@ -3916,8 +4178,33 @@ def __init__( probe_source: ProbeSource = wasm_probe_source, probe_path: ProbePath = probe_one_path, read_packets: ReadPackets = wasm_read_packet_rows, + shapes: Shape | None = None, ) -> None: self.res = res + # Asks a node module for its shape, once per distinct call. + self.shapes: Shape = shapes if shapes is not None else ShapeCache() + # (call text, id(env)) -> the instances a node call lowered to, and + # whether it broadcast: every read of one call in one branch is one. + self._node_calls: dict[ + tuple[str, int], tuple[tuple[_NodeInstance, ...], bool] + ] = {} + # (module, export, bound inputs, params) -> the instance: one call is + # one node wherever the query writes it. + self._node_refs: dict[ + tuple[str, str, tuple[tuple[str, FrameRef], ...], str], _NodeInstance + ] = {} + # A node's data pad -> the function writing it and the schema of its + # rows, which a reader's port is matched against. + self._data_schemas: dict[FrameRef, tuple[str, Mapping[str, object]]] = {} + # A node's rows pad -> its declaration, its record and the call, for + # the track or rows file a COPY selecting it writes. + self._node_rows: dict[FrameRef, tuple[WasmFunction, Annotation, exp.Anonymous]] = {} + # A node at a COPY's TO, by module: the packet sink its shape reads as, + # and the shape. + self._node_sinks: dict[str, tuple[Described, NodeShape]] = {} + # Each node's shape by its id, read back off the graph for the rates + # its outputs run at. + self._made_shapes: dict[str, NodeShape] = {} self.probes = probes # Why an alias in `probes` maps to None, when there is a specific # answer -- unset (or no answer for this alias) reads the same as an @@ -4110,6 +4397,9 @@ def __init__( # rows off the SELECT list rather than named stream parameters: a # multi-row relation is accepted here too, the way a manifest's is. self.row_reading_sink = False + # True while a column handed straight to a node's DEFAULT NULL port + # lowers: a NULL there leaves the port unbound, as a NULL literal does. + self.null_port_read = False # True while a CTE body lowers. The bodies are lowered once, before # any COPY, so the reader is not known here: a stream column records # one cell per row of the body's relation -- NULL where the row @@ -4281,6 +4571,7 @@ def run(self) -> Graph: is None). When there are sinks they are just a mirror of ``sinks[0]`` and walking them again would lower the first group twice. """ + self._check_declared_worlds() self._lower_ctes() if self.res.sinks: self.graph.sinks = self._lower_sinks() @@ -4305,6 +4596,7 @@ def run(self) -> Graph: attachments=list(self.attachments), ) ] + self._place_node_sources() self._check_feeder_groups() self._place_feeders() self._place_laterals() @@ -4367,7 +4659,7 @@ def decoded(ref: FrameRef) -> FrameRef | None: self.graph.codec_formats[node] = said made[ref] = self.ctx.node( "setparams", - {_SETPARAMS_COLOR[option]: value for option, value in said.items()}, + {SETPARAMS_COLOR[option]: value for option, value in said.items()}, [node], ["video"], ) @@ -5012,35 +5304,12 @@ def _stream_colorimetry(self, ref: FrameRef, pix_fmt: str | None = None) -> dict names, as far as the query says it. A ``setparams`` on the way settles each field it names; past a filter - that converts colour (:data:`_COLOR_CONVERTING_FILTERS`) nothing else + that converts colour (:data:`ffrwd.processes.COLOR_CONVERTING_FILTERS`) nothing else does. Every field left is the input stream's, as probed. A field nothing settles is absent. Pictures in an RGB `pix_fmt` have no YUV matrix, range or chroma siting to state. """ - said: dict[str, str] = {} - seen: set[str] = set() - current: FrameRef | None = ref - while current is not None and not is_src(current): - node = self._upstream(current, seen) - if node is None or _converts_colour(node): - current = None - break - if node.filter == "setparams": - for option, param in _SETPARAMS_COLOR.items(): - value = node.args.get(param) - if value is not None and str(value) not in _UNSAID_COLOR: - said.setdefault(option, str(value)) - current = self._picture_input(node) - meta = self._source_meta(current) - if meta is not None: - for option, field_name in _PROBED_COLOR.items(): - value = getattr(meta, field_name) - if isinstance(value, str) and value not in _UNSAID_COLOR: - said.setdefault(option, value) - if pix_fmt is not None and pix_fmt.startswith(_RGB_PREFIXES): - for option in _YUV_ONLY_COLOR: - said.pop(option, None) - return {option: said[option] for option in COLOR_OPTIONS if option in said} + return stream_colorimetry(self.graph, ref, self._source_meta, pix_fmt) def _described_codec( self, declared: WasmFunction, node: exp.Expr, select: exp.Select @@ -5093,7 +5362,9 @@ def _rows_file(self, raw: RawSink) -> str: ] sole = _unwrap(written[0]) if len(written) == 1 else None if sole is not None and ( - self._rows_projection(sole) is not None or self._rows_call(sole) is not None + self._rows_projection(sole) is not None + or self._rows_call(sole) is not None + or self._node_rows_column(sole) ): return path raise _error( @@ -5120,7 +5391,10 @@ def _lower_module_sink( filter calls start. """ declared = self.res.wasm[raw.module_sink] - described = self.describes.get(declared.module) + node_sink = self._node_sinks.get(declared.module) + described = ( + node_sink[0] if node_sink is not None else self.describes.get(declared.module) + ) if described is None or not described.packet_sink: self._no_packets_here(raw, first_filter, declared) if raw.options: @@ -7357,6 +7631,10 @@ def _lower_branch(self, select: exp.Select, *, tags: _TagScope) -> list[_Column] written, self.rows_file = self.rows_file, document or self.rows_file try: value = self._branch_value(projection, env, select) + if tags == "sink": + value = self._node_rows_at_sink(value, projection, env, select) + except _UnmadeField: + continue # `.*` over a call whose shape makes no such output finally: self.rows_file = written column = _Column( @@ -8616,6 +8894,10 @@ def _add_module_source( """ call = _call_parts(inner) assert call is not None # inner is exp.Anonymous; _call_parts always answers + node_module = self._node_module(declared) + if node_module is not None: + self._add_node_source(alias, inner, declared, node_module, call, join, env, select) + return described = self._described_source(declared, inner, select) if not described.source: self._add_url_source( @@ -8661,6 +8943,80 @@ def _add_module_source( bounded=catalog.bounded, ) + def _add_node_source( + self, + alias: str, + inner: exp.Anonymous, + declared: WasmFunction, + described: Described, + call: _Call, + join: RawRowJoin | None, + env: _Env, + select: exp.Select, + ) -> None: + """``FROM () alias``: a node reading nothing, bound as a + probed input is. + + Its shape for the call's params names its outputs, and each is one of + the alias's streams, ``s.video[1]`` the first picture; its relation + rows are the alias's renditions. The node making them is in the graph + like any other, and every ``src:`` ref is a pad of it once + lowering ends (:meth:`_place_node_sources`). A source that never ends + reads as a live input does. + """ + self._check_node_export(declared, described, inner, select) + params = self._wasm_params(declared, described, call, inner, select, env, {}, first=0) + shape = self._node_shape(declared, described, params, [], inner, select) + required = next((port for port in shape.inputs if port.required), None) + if required is not None: + raise _error( + ErrorCode.UNSUPPORTED_SQL, + f"{declared.name}() is read in FROM, and the module " + f"'{declared.module}' requires an input '{required.name}'", + inner, + fallback=select, + hint="a source reads nothing; a node reading a stream is called " + "over that stream in the SELECT list", + ) + result = _node_source_probe(shape) + self.probes[alias] = result + env.bindings[alias] = _InputBinding(alias=alias) + self._bind_renditions(alias, join, env, select) + kinds = [_output_kind(shape, output, {}) for output in shape.outputs] + ref = self.ctx.node(declared.module, params, [], kinds) + self.graph.nodes[ref].out_ports = [output.name for output in shape.outputs] + self.graph.nodes[ref].bound = bound_json([]) + self.graph.node_shapes[ref] = dict(shape.raw) + self._emits_rows(ref, described) + self.graph.node_sources[alias] = ref + + def _place_node_sources(self) -> None: + """Every ``src:`` ref of a node read in FROM, as that node's pad. + + The alias binds as a probed input does, so what reads it reads a + source ref; the node is what makes the stream, and is what the ref + now names. + """ + if not self.graph.node_sources: + return + pads: dict[FrameRef, FrameRef] = {} + for alias, name in self.graph.node_sources.items(): + made = self.graph.nodes[name] + counted: dict[StreamType, int] = {} + for pad, kind in enumerate(made.outputs): + index = counted.get(kind, 0) + counted[kind] = index + 1 + ref = f"src:{alias}:{_TYPE_MARKERS[kind]}:{index}" + pads[ref] = name if len(made.outputs) == 1 else f"{name}:{pad}" + for node in self.graph.nodes.values(): + node.inputs = [pads.get(ref, ref) for ref in node.inputs] + for unit in self.graph.sinks: + for output in unit.outputs: + output.ref = pads.get(output.ref, output.ref) + self.graph.rows_sinks = { + pads.get(ref, ref): sink for ref, sink in self.graph.rows_sinks.items() + } + # -- FROM () alias over a VALUES module: a URL table ---- def _add_url_source( @@ -9498,6 +9854,12 @@ def _add_packet_rows( declared, described, call, inner, select, env, {}, first=1 ) params_json = json.dumps(params, sort_keys=True) if params else "" + read_ref = f"src:{raw.source}:{_TYPE_MARKERS[kind]}:{index}" + port, wants, bound = ( + self._node_rows_port(declared, described, params, read_ref, inner, select) + if described.node + else ("", described.wants, "") + ) read = PacketRead( spec=spec, input_args=flags, @@ -9505,7 +9867,9 @@ def _add_packet_rows( index=index, module=declared.module, params=params_json, - wants=described.wants, + wants=wants, + port=port, + bound=bound, ) key = packet_rows_key( spec, @@ -9515,7 +9879,7 @@ def _add_packet_rows( declared.module, module_digest(declared.module), params_json, - described.wants, + wants, ) written = cached_packet_rows(key) if written is None: @@ -9571,7 +9935,7 @@ def _described_packet_rows( hint="this is a compiler bug; please report the query that " "produced it", ) - if described.world not in WORLDS: + if described.world not in WORLDS and not described.node: raise _error( ErrorCode.UNSUPPORTED_SQL, f"the module '{declared.module}' targets {described.world}, and " @@ -9591,6 +9955,11 @@ def _described_packet_rows( hint=f"a module carries one export; write '{described.name}' as " "the export", ) + if described.node: + # A node reading the packets: what its shape says of the port is + # checked where the call's params are known. + self._check_packet_rows_schema(declared, described, node, select) + return described if not described.packet_sink: raise _error( ErrorCode.UNSUPPORTED_SQL, @@ -9625,6 +9994,46 @@ def _described_packet_rows( self._check_packet_rows_schema(declared, described, node, select) return described + def _node_rows_port( + self, + declared: WasmFunction, + described: Described, + params: Mapping[str, object], + read: FrameRef, + node: exp.Anonymous, + select: exp.Select, + ) -> tuple[str, SinkWants, str]: + """The port a node read in FROM is handed the stream `read` on, how + much of it the port asks for, and the bound list its shape was asked + with, as JSON. + + Its shape for the call's params, with that port bound, has to read + coded packets there and make no output: what the read binds is the + rows the node emits beside its ports. + """ + name = declared.params[0].name + bound = [Binding(name, (StreamHint(self._stream_rate(read)),))] + shape = self._node_shape(declared, described, params, bound, node, select) + port = shape.input(name) + if port is None or port.kind != "packets" or shape.outputs: + said = ( + f"has no input '{name}'" + if port is None + else f"reads {port.kind} on '{name}'" + if port.kind != "packets" + else "makes outputs of its own" + ) + raise _error( + ErrorCode.UNSUPPORTED_SQL, + f"function '{declared.name}' returns rows read off a stream's " + f"packets, and the module '{declared.module}' {said}", + node, + fallback=select, + hint=f"a node read in FROM takes coded packets on '{name}' and " + "emits rows alone; declare this one as what it is", + ) + return name, _sink_wants(port.accepts.wants), bound_json(bound) + def _check_packet_rows_schema( self, declared: WasmFunction, @@ -10033,7 +10442,7 @@ def _split_where( if isinstance(env.bindings.get(alias), _RowBinding) or _reads_cte_value(alias, conjunct, env) } - if not rows: + if not rows or self._ends_node_source(conjunct, aliases): time_conjuncts.append(conjunct) continue if aliases - rows and len(rows) == 1 and self._is_row_window(conjunct, env): @@ -10054,6 +10463,21 @@ def _split_where( row_conjuncts.append(conjunct) return time_conjuncts, row_conjuncts, assertion_conjuncts + def _ends_node_source(self, conjunct: exp.Expr, aliases: set[str]) -> bool: + """Whether `conjunct` bounds the time of a node read in FROM: its + relation rows are renditions, and its time is the node's own.""" + parsed = _time_bounds(conjunct) + if parsed is None: + return False + column = parsed[0] + table = column.args.get("table") + return ( + table is not None + and aliases == {_fold(table)} + and _fold(table) in self.graph.node_sources + and _fold(column.this) == TIME_COLUMN + ) + def _check_row_window_seeks_a_file( self, conjunct: exp.Expr, where: exp.Where, env: _Env ) -> None: @@ -11289,7 +11713,9 @@ def _collect_trims( hint=_TIME_HINT, ) column, low, high, strict = parsed - if strict: + table_node = column.args.get("table") + ticks = table_node is not None and _fold(table_node) in self.graph.node_sources + if strict and not ticks: raise _error( ErrorCode.UNSUPPORTED_SQL, "strict inequalities are not supported", @@ -11297,7 +11723,6 @@ def _collect_trims( fallback=where, hint=_TIME_HINT, ) - table_node = column.args.get("table") if table_node is None: raise _error( ErrorCode.UNSUPPORTED_SQL, @@ -11307,6 +11732,15 @@ def _collect_trims( hint=_TIME_HINT, ) alias = _fold(table_node) + if ticks and low is not None: + raise _error( + ErrorCode.UNSUPPORTED_SQL, + f"'{alias}' is a node read in FROM, so it starts at 0 and " + f"'WHERE {alias}.t' can only end it", + conjunct, + fallback=where, + hint=f"write an end alone, e.g. WHERE {alias}.t < 10", + ) if _fold(column.this) != TIME_COLUMN: raise _error( ErrorCode.UNSUPPORTED_SQL, @@ -11366,7 +11800,9 @@ def _collect_trims( self._window_of(alias, low_node, high_node, env, row, select) for row in rows ] - if isinstance(env.bindings[alias], _InputBinding): + if isinstance(env.bindings[alias], _InputBinding) or ( + alias in self.graph.node_sources + ): if any( opt.name == "seek_end" for opt in self.res.input_options.get(alias, ()) @@ -11660,6 +12096,12 @@ def _check_declared_stream( def _lower_stream_expr(self, node: exp.Expr, env: _Env, select: exp.Select) -> _Value: node = _unwrap(node) + self._check_older_world(node) + # A node's outputs are its own, whatever the declaration would mean to + # a module of an older world. + lowered = self._lower_node_expr(node, env, select) + if lowered is not None: + return lowered # An array of cue records IS a subtitle track, so it lowers here, in a # stream position, and not as an output column the way `chapters` does. cues = self._lower_cue_array(node, env, select) @@ -12911,7 +13353,12 @@ def _row_stream( select, ) return row.stream - if self.table_mode or self.manifest is not None or self.row_reading_sink: + if ( + self.table_mode + or self.manifest is not None + or self.row_reading_sink + or self.null_port_read + ): return _Stream(ref=_NULL_STREAM_REF, type=binding.type, source=None) fill = _FILL_SPELLINGS.get(binding.type) hint = ( @@ -13883,6 +14330,7 @@ def _null_cells_are_read(self) -> bool: or self.row_reading_sink or self.cte_body or self.array_agg_reads_nulls + or self.null_port_read ) def _cte_cell_column(self, binding: _CteBinding, column: _Column) -> bool: @@ -14546,6 +14994,8 @@ def _wasm_params( first: int, params_schema: Mapping[str, object] | None = None, ports: _Ports | None = None, + written: Mapping[str, exp.Expr] | None = None, + value_params: Sequence[Parameter] | None = None, ) -> dict[str, object]: """The value arguments as the module's own parameters, schema-checked. @@ -14561,18 +15011,23 @@ def _wasm_params( whose parameters belong to one FUNCTION of the module rather than to the module's single export. `ports` is what the call's feeders settled: a port the host picked is written here, whatever the - declaration's DEFAULT says. + declaration's DEFAULT says. `written` and `value_params` are a node + call's, whose values stand anywhere among its ports. """ schema_source = ( described.params_schema if params_schema is None else params_schema ) properties = schema_source.get("properties") known = properties if isinstance(properties, dict) else {} - self._check_value_param_schemas(declared, known, node, select) - written = self._wasm_written(declared, call, node, select, first=first, ports=ports) + values = declared.value_params if value_params is None else tuple(value_params) + self._check_value_param_schemas(declared, known, node, select, values) + if written is None: + written = self._wasm_written( + declared, call, node, select, first=first, ports=ports + ) owned = ports.owned if ports is not None else {} params: dict[str, object] = {} - for param in declared.value_params: + for param in values: argument = written.get(param.name) anchor = argument if argument is not None else node value: RowValue @@ -14596,7 +15051,7 @@ def _wasm_params( hint=_declares_params(known), ) self._check_wasm_param(param.name, value, schema, anchor, select) - params[param.name] = value + params[param.name] = _as_scalar_type(value, schema) return params def _wasm_written( @@ -14705,6 +15160,7 @@ def _check_value_param_schemas( known: Mapping[str, object], node: exp.Expr, select: exp.Select, + values: Sequence[Parameter] | None = None, ) -> None: """Every value parameter the declaration names, against what SQL can write. @@ -14715,7 +15171,7 @@ def _check_value_param_schemas( here beats the module failing on a key it never got. A schema naming nothing judgeable is left alone, as it is at the argument. """ - for param in declared.value_params: + for param in declared.value_params if values is None else values: kinds = _schema_types(known.get(param.name)) if not kinds or any(kind in _JSON_TYPES for kind in kinds): continue @@ -15309,6 +15765,12 @@ def _lower_wasm_call( :meth:`_written_annotation` lowers the producer once and hands back the pad the module reads. """ + if declared.is_sink and self._node_module(declared) is not None: + return self._lower_node_sink(node, declared, call, env, select) + if self._node_module(declared) is not None: + lowered = self._lower_node_expr(node, env, select) + assert lowered is not None # a call to a node module + return lowered described = self._described(declared, node, select) if declared.is_packets: return self._lower_packets_call( @@ -15476,6 +15938,925 @@ def build(values: list[object], element: int) -> FrameRef: ) return lowered + # -- nodes -- + + def _node_module(self, declared: WasmFunction) -> Described | None: + """The describe of `declared`'s module, where that module is a node.""" + described = self.describes.get(declared.module) + return described if described is not None and described.node else None + + def _check_declared_worlds(self) -> None: + """A declaration only a node can carry, naming a module that is not one.""" + for declared in self.res.wasm.values(): + if declared.refusal is not None and self._node_module(declared) is None: + raise declared.refusal + + def _node_call( + self, node: exp.Expr + ) -> tuple[exp.Anonymous, WasmFunction, str | None] | None: + """```` or ``.`` over a call to a node module: the + call, its declaration and the field read. None for anything else.""" + node = _unwrap(node) + field_name: str | None = None + base: exp.Expr = node + if isinstance(node, exp.Dot): + identifier = node.args.get("expression") + inner = _unwrap(node.this) if isinstance(node.this, exp.Expr) else None + if not isinstance(identifier, exp.Identifier) or not isinstance( + inner, exp.Anonymous + ): + return None + field_name, base = _fold(identifier), inner + if not isinstance(base, exp.Anonymous): + return None + declared = self.res.wasm.get(str(base.name).lower()) + if declared is None or declared.is_sink or self._node_module(declared) is None: + return None + return base, declared, field_name + + def _check_older_world(self, node: exp.Expr) -> None: + """What resolve kept for a module of an older world, raised for one. + + A call or field read only a node makes sense of carries the refusal + an older module earns; a node module reads it as written. + """ + refusal = _unwrap(node).meta.get(OLDER_WORLD) + if isinstance(refusal, FfrwdError) and self._node_call(node) is None: + raise refusal + + def _node_output( + self, + declared: WasmFunction, + field_name: str | None, + node: exp.Expr, + select: exp.Select, + ) -> Parameter: + """The declared output a call read whole, or the field it read, names.""" + outputs = declared.outputs or () + if field_name is not None: + found = next((o for o in outputs if o.name == field_name), None) + if found is not None: + return found + listed = ", ".join(f"'{o.name}'" for o in outputs if o.name) + raise _error( + ErrorCode.UNSUPPORTED_SQL, + f"{declared.name}() returns no field '{field_name}'", + node, + fallback=select, + hint=f"it returns {listed}" if listed else declared.signature, + ) + if len(outputs) == 1 and not outputs[0].name: + return outputs[0] + fields = ", ".join(f".{o.name}" for o in outputs if o.name) + raise _error( + ErrorCode.UNSUPPORTED_SQL, + f"{declared.name}() returns {declared.written_returns}, and a struct " + "is not a stream", + node, + fallback=select, + hint=f"read one output off the call: it writes {fields}" + if fields + else declared.signature, + ) + + def _node_output_kind(self, declared: WasmFunction, field_name: str | None) -> str: + """The kind of what a node call or field read is, for the classifier.""" + outputs = declared.outputs or () + if field_name is not None: + found = next((o for o in outputs if o.name == field_name), None) + else: + found = outputs[0] if len(outputs) == 1 and not outputs[0].name else None + return _UNSUPPORTED_KIND if found is None else _port_kind(found) + + def _lower_node_expr( + self, node: exp.Expr, env: _Env, select: exp.Select + ) -> _Value | None: + """A call to a node module, or a field read off one; None for anything else. + + Every read of one call is one instance (:meth:`_node_instances`), so a + caption track and a module reading the same rows read one node's + output, split. + """ + found = self._node_call(node) + if found is None: + return None + base, declared, field_name = found + refusal = base.meta.get(NODE_REFUSAL) + if isinstance(refusal, FfrwdError): + raise refusal + call = _call_parts(base) + assert call is not None # an Anonymous always splits + output = self._node_output(declared, field_name, node, select) + if declared.is_packets: + self._check_packets_position(declared, base, select) + made = len(self.graph.nodes) + instances, broadcast = self._node_instances(base, declared, call, env, select) + if declared.is_packets: + # Each new instance reads what the destination encodes, which the + # destination settles once its options are known. + self.packet_filter_calls.extend( + (instance.ref, declared, base, select) + for instance in instances + if instance.ref in list(self.graph.nodes)[made:] + ) + if ( + field_name is not None + and _unwrap(node).meta.get(STAR_FIELD) + and any(instance.shape.output(field_name) is None for instance in instances) + ): + raise _UnmadeField(field_name) + streams = tuple( + _Stream(ref=ref, type=kind) + for ref, kind in ( + self._node_pad(instance, declared, output, base, select) + for instance in instances + ) + ) + value = _Value( + type=streams[0].type, + streams=streams, + is_array=broadcast, + ) + return self._row_filtered(value, _unwrap(node)) + + def _node_written( + self, + declared: WasmFunction, + call: _Call, + node: exp.Expr, + select: exp.Select, + ) -> dict[str, exp.Expr]: + """Each parameter the call writes, ports and values alike, keyed by name. + + The positionals fill the signature in declared order and each name the + one it names, as resolve already checked. + """ + written: dict[str, exp.Expr] = {} + positional = self._spread_node_outputs(spread(call.args)) + for param, argument in zip(declared.params, positional): + written[param.name] = argument + for named in call.named: + if not any(p.name == named.name for p in declared.params): + raise _error( + ErrorCode.UDF_ARG_TYPE, + f"{declared.name}() has no parameter '{named.name}'", + named.value, + fallback=node, + hint=declared.signature, + ) + written[named.name] = named.value + if len(positional) > len(declared.params): + raise _error( + ErrorCode.UDF_ARG_TYPE, + f"{declared.name}() got {len(positional)} arguments, but it declares " + f"{len(declared.params)}", + node, + fallback=select, + hint=declared.signature, + ) + return written + + def _spread_node_outputs(self, arguments: Sequence[exp.Expr]) -> list[exp.Expr]: + """A node call read whole where it makes a stream and the rows beside it, + as those two fields read off it, in its own order.""" + spread_out: list[exp.Expr] = [] + for argument in arguments: + found = self._node_call(argument) + outputs = found[1].outputs or () if found is not None else () + two_part = ( + found is not None + and found[2] is None + and len(outputs) == 2 + and outputs[0].annotation is None + and outputs[1].annotation is not None + ) + if found is None or not two_part: + spread_out.append(argument) + continue + spread_out.extend( + exp.Dot(this=found[0].copy(), expression=exp.to_identifier(output.name)) + for output in outputs + ) + return spread_out + + def _lower_port( + self, + declared: WasmFunction, + param: Parameter, + argument: exp.Expr, + env: _Env, + select: exp.Select, + ) -> _Value: + """One port's argument: a stream, rows, or an array of either. + + A column handed to a port declared DEFAULT NULL may be NULL where an + outer join left it so; the instance it falls to leaves the port + unbound. + """ + inner = _unwrap(argument) + if isinstance(inner, exp.Array) and inner.expressions: + elements = [ + self._lower_expr(element, env, select) + for element in inner.expressions + if isinstance(element, exp.Expr) + ] + streams = tuple(stream for value in elements for stream in value.streams) + value = _Value(type=elements[0].type, streams=streams, is_array=True) + else: + reading = self.null_port_read + self.null_port_read = isinstance(param.default, exp.Null) and isinstance( + inner, exp.Column + ) + try: + value = self._lower_expr(argument, env, select) + finally: + self.null_port_read = reading + wanted = _port_kind(param) + got = next((s.type for s in value.streams if s.type != wanted), None) + if got is not None: + raise _error( + ErrorCode.UDF_ARG_TYPE, + f"{declared.name}() takes {param.type} as its '{param.name}' " + f"argument, and its argument is a {got} stream", + argument, + fallback=select, + hint=declared.signature, + ) + return value + + def _node_instances( + self, + base: exp.Anonymous, + declared: WasmFunction, + call: _Call, + env: _Env, + select: exp.Select, + ) -> tuple[tuple[_NodeInstance, ...], bool]: + """The node instances a call is: one, or one per element it broadcasts over, + and whether it broadcasts. + + A port declared without ``[]`` given an array is one instance per + element, as a filter is; a value read off a row is one per row. A + call written twice in one branch is lowered once, and two calls whose + module, inputs and params agree are one instance wherever they are + written (:meth:`_node_instance`). + """ + key = (base.sql(), id(env)) + found = self._node_calls.get(key) + if found is not None: + return found + described = self._node_module(declared) + assert described is not None # what `_node_call` checked + self._check_node_export(declared, described, base, select) + written = self._node_written(declared, call, base, select) + params_by_name = {param.name: param for param in declared.params} + ports: dict[str, exp.Expr] = {} + values: dict[str, exp.Expr] = {} + for name, argument in written.items(): + if not is_port(params_by_name[name]): + values[name] = argument + elif not isinstance(_unwrap(argument), exp.Null): + ports[name] = argument + node_values = tuple(param for param in declared.params if not is_port(param)) + numbers = {name: arg for name, arg in ports.items() if is_number_argument(arg)} + streams = { + name: value + for name, argument in ports.items() + if name not in numbers + and not _is_null( + value := self._lower_port(declared, params_by_name[name], argument, env, select) + ) + } + tuples = env.relation.tuples if env.relation is not None else [] + per_row = any(_reads_row_column(argument, env) for argument in values.values()) + broadcast = { + name: value + for name, value in streams.items() + if value.is_array and not is_array(params_by_name[name].type) + } + length = self._node_length(declared, base, broadcast, per_row, len(tuples), select) + instances: list[_NodeInstance] = [] + for element in range(1 if length is None else length): + row = tuples[element] if per_row and element < len(tuples) else ( + tuples[0] if len(tuples) == 1 else {} + ) + bound_values: dict[str, _Value] = {} + for name, value in streams.items(): + one = _scalar(value.at(element)) if name in broadcast else value + if not _is_null(one): + bound_values[name] = one + params = self._wasm_params( + declared, described, call, base, select, env, row, first=0, + written=values, value_params=node_values, + ) + shape_params = params + if per_row: + fixed = { + name: argument + for name, argument in values.items() + if not _reads_row_column(argument, env) + } + shape_params = self._wasm_params( + declared, described, call, base, select, env, {}, first=0, + written=fixed, value_params=node_values, + ) + instances.append( + self._node_instance( + declared, + described, + params, + shape_params, + bound_values, + numbers, + values, + base, + select, + ) + ) + self._node_calls[key] = (tuple(instances), length is not None) + return self._node_calls[key] + + def _check_node_export( + self, + declared: WasmFunction, + described: Described, + node: exp.Expr, + select: exp.Select, + ) -> None: + """The export a declaration names, against the one its module carries.""" + if not described.name or described.name == declared.export: + return + raise _error( + ErrorCode.UNSUPPORTED_SQL, + f"function '{declared.name}' names the export '{declared.export}', " + f"and '{declared.module}' exports '{described.name}'", + node, + fallback=select, + hint=f"a module carries one node; write '{described.name}' as the export", + ) + + def _node_length( + self, + declared: WasmFunction, + base: exp.Anonymous, + broadcast: Mapping[str, _Value], + per_row: bool, + rows: int, + select: exp.Select, + ) -> int | None: + """How many instances a call broadcasts to, or None for one.""" + lengths = {name: len(value.streams) for name, value in broadcast.items()} + if per_row: + lengths["the rows its values are read off"] = rows + if not lengths: + return None + distinct = sorted(set(lengths.values())) + if len(distinct) > 1: + named = ", ".join(f"{name} has {count}" for name, count in lengths.items()) + raise _error( + ErrorCode.BROADCAST_MISMATCH, + f"{declared.name}() cannot broadcast over arrays of different " + f"lengths: {named}", + base, + fallback=select, + hint=_ZIP_HINT, + ) + return distinct[0] + + def _node_shape( + self, + declared: WasmFunction, + described: Described, + params: Mapping[str, object], + bound: Sequence[Binding], + base: exp.Anonymous, + select: exp.Select, + ) -> NodeShape: + """The module's shape for this call, said at the call when it refuses.""" + try: + return self.shapes( + declared.module, + json.dumps(params, sort_keys=True), + bound, + _grant_args(described, declared.module), + ) + except FfrwdError as err: + raise _error( + err.code, f"{declared.name}(): {err.message}", base, fallback=select, + hint=err.hint, + ) from None + + def _node_instance( + self, + declared: WasmFunction, + described: Described, + params: dict[str, object], + shape_params: dict[str, object], + streams: Mapping[str, _Value], + numbers: Mapping[str, exp.Expr], + written: Mapping[str, exp.Expr], + base: exp.Anonymous, + select: exp.Select, + ) -> _NodeInstance: + """One instance: the shape for its params and bound ports, checked + against the declaration, and the node it is. `written` is the values + the call wrote, which a port number may not write again. + + A run-time lateral's stream on an input the module holds on a port + binds nothing: the port is one the host listens on for this compile, + written into the param, and the lateral's instances connect to it, + one connection per hold group, as a feeder's do. + """ + fed = { + name: value + for name, value in streams.items() + if value.streams + and all(_lateral_parts(stream.ref) is not None for stream in value.streams) + } + # A lateral's port is bound, to the host's own stream, so the shape + # is asked with it; what it binds in the network is nothing. + bound = [ + Binding( + param.name, + tuple( + StreamHint(None if param.name in fed else self._stream_rate(stream.ref)) + for stream in streams[param.name].streams + ), + ) + for param in declared.ports + if param.name in streams + ] + streams = {name: value for name, value in streams.items() if name not in fed} + held = { + name: self._held_port(declared, name, argument, base, select) + for name, argument in numbers.items() + } + shape = self._node_shape(declared, described, shape_params, bound, base, select) + rated = _conformed(shape, bound) + if rated != bound: + bound = rated + shape = self._node_shape(declared, described, shape_params, bound, base, select) + self._check_node_ports(declared, shape, {**streams, **fed}, numbers, base, select) + ports: dict[str, object] = {} + for name, argument in numbers.items(): + port = shape.input(name) + hold = port.pairing.hold if port is not None else None + if hold is None or hold.port_param is None: + raise _error( + ErrorCode.UDF_ARG_TYPE, + f"{declared.name}() takes '{name}' as a stream, and the call " + "writes a number there", + argument, + fallback=base, + hint="a number stands for a port only where the module holds " + "an input on a port it names; pass a stream here", + ) + if hold.port_param in written: + raise _error( + ErrorCode.UDF_ARG_TYPE, + f"{declared.name}() writes '{hold.port_param}' twice: as a " + f"number in '{name}' and as itself", + argument, + fallback=base, + hint=f"write the port once: {declared.signature}", + ) + ports[hold.port_param] = held[name] + self._check_grouped_data(declared, shape, streams, base, select) + feeds: list[tuple[int, _Fed, Described]] = [] + for name, lateral in fed.items(): + first_stream = lateral.streams[0] + port = shape.input(name) + hold = _group_hold(shape, port) if port is not None else None + if port is None or hold is None or hold.port_param is None: + raise self._lateral_refusal(first_stream.ref, f"{declared.name}()", base) + listed = _param_takes_list(described, hold.port_param) + if len(lateral.streams) > 1 and not listed: + raise _error( + ErrorCode.UNSUPPORTED_SQL, + f"{declared.name}() hands '{name}' {len(lateral.streams)} run-time " + f"laterals, each a connection on a port of its own, and the module " + f"'{declared.module}' takes one port in '{hold.port_param}'", + base, + fallback=select, + hint=f"rebuild the module taking '{hold.port_param}' as an array of " + "ports, or feed it one lateral", + ) + if hold.port_param in written: + raise _error( + ErrorCode.UDF_ARG_TYPE, + f"{declared.name}() writes '{hold.port_param}', and '{name}' is " + "fed by a run-time lateral, which takes a port of its own", + written[hold.port_param], + fallback=base, + hint=f"leave '{hold.port_param}' out of the call", + ) + # What the lateral's instances write is what the port takes, + # conformed by them as a feeder's are. + taken = port.accepts + conformed = replace( + described, + pixel_formats=taken.pixel_formats or WIRE_PIX_FMTS[:1], + sample_formats=taken.sample_formats or WIRE_SAMPLE_FMTS[:1], + sample_rates=taken.sample_rates, + channel_counts=taken.channel_counts, + ) + numbers_given: list[int] = [] + for stream in lateral.streams: + one = _Fed( + Feeder( + input=0, + port_param=hold.port_param, + kind=stream.type, + group=hold.group or hold.port_param, + ), + next(param for param in declared.ports if param.name == name), + stream, + self._stream_sources(stream.ref), + base, + ) + number = self._feeder_port(one, select) + if number not in numbers_given: + numbers_given.append(number) + feeds.append((number, one, conformed)) + ports[hold.port_param] = numbers_given if listed else numbers_given[0] + if ports: + # The port is one of the call's params, and the shape is asked + # for the params the call ends up with. + params.update(ports) + shape = self._node_shape( + declared, described, {**shape_params, **ports}, bound, base, select + ) + inputs: list[FrameRef] = [] + names: list[str] = [] + for port in shape.inputs: + value = streams.get(port.name) + if value is None: + continue + for stream in value.streams: + inputs.append(stream.ref) + names.append(port.name) + if port.kind == "data" and port.schema is not None: + self._match_rows(declared, port, value, base, select) + bound_kinds = { + name: value.streams[0].type for name, value in streams.items() if value.streams + } + kinds = [_output_kind(shape, output, bound_kinds) for output in shape.outputs] + key = ( + declared.module, + declared.export, + tuple(zip(names, inputs)), + json.dumps(params, sort_keys=True), + ) + found = self._node_refs.get(key) + if found is not None: + return found + ref = self.ctx.node(declared.module, params, inputs, kinds) + made = self.graph.nodes[ref] + made.ports = names + made.out_ports = [output.name for output in shape.outputs] + made.bound = bound_json(bound) + self.graph.node_shapes[ref] = dict(shape.raw) + self._emits_rows(ref, described) + clock = shape.clock.port if shape.clock.port in names else None + timed = inputs[names.index(clock)] if clock is not None else (inputs or [""])[0] + for number, one, feeding in feeds: + # The programme a feed is shaped by is the call's own stream of + # its kind, else the clock's. + programme = next( + (read for read in inputs if ref_type(self.graph, read) == one.stream.type), + timed, + ) + self._feed(number, one, ref, programme, declared, feeding) + for name, value in streams.items(): + for stream in value.streams: + said = self._written_tags(stream) + if said: + self.graph.stream_tags[stream.ref] = said + instance = _NodeInstance(ref=ref, shape=shape) + self._node_refs[key] = instance + return instance + + def _stream_rate(self, ref: FrameRef) -> Fraction | None: + """The rate a node's shape is told the stream `ref` runs at.""" + return stream_rate(self.graph, self.probes, self._made_shape, ref) + + def _made_shape(self, name: str) -> NodeShape | None: + found = self._made_shapes.get(name) + raw = self.graph.node_shapes.get(name) + if found is None and raw is not None: + found = self._made_shapes[name] = node_shape(self.graph.nodes[name].filter, raw) + return found + + def _check_grouped_data( + self, + declared: WasmFunction, + shape: NodeShape, + streams: Mapping[str, _Value], + base: exp.Anonymous, + select: exp.Select, + ) -> None: + """Refuse a stream bound to a data input whose hold group a port + serves: its rows arrive on that group's connection or not at all.""" + for port in shape.inputs: + interval = port.pairing.interval + if interval is None or interval.group is None or port.name not in streams: + continue + served = [ + one.name + for one in shape.inputs + if one.pairing.hold is not None + and one.pairing.hold.group == interval.group + and one.pairing.hold.port_param is not None + and one.name not in streams + ] + if not served: + continue + raise _error( + ErrorCode.UDF_ARG_TYPE, + f"{declared.name}() binds '{port.name}', which arrives on the " + f"connection of the group '{interval.group}', and '{served[0]}' of " + "that group is served by a port", + base, + fallback=select, + hint=f"hand '{port.name}' the rows of the run-time lateral feeding " + f"'{served[0]}', or bind the group's inputs to streams", + ) + + def _emits_rows(self, ref: FrameRef, described: Described) -> None: + """The rows a node emits beside its ports, where it says it does: one + more output, which the run writes as its rows, as a sink's are.""" + if described.rows_schema is None: + return + made = self.graph.nodes[ref] + pad = len(made.outputs) + made.outputs = [*made.outputs, "data"] + made.out_ports = [*made.out_ports, EMITTED_ROWS_PORT] + self.graph.rows_sinks[f"{ref}:{pad}"] = RowsSink(container=_ROWS_CONTAINER) + + def _written_tags(self, stream: _Stream) -> dict[str, str]: + """The tags the query wrote on `stream`, which nothing on the edge to + a node carries.""" + if stream.source is None: + return {} + written = self._layered_tags().get(id(stream.source), {}) + return {key: value for key, value in written.items() if value is not None} + + def _held_port( + self, + declared: WasmFunction, + name: str, + argument: exp.Expr, + base: exp.Anonymous, + select: exp.Select, + ) -> int: + """A port number written where a held input goes: a whole number in range.""" + value = _number(argument) + if isinstance(value, float) and value.is_integer(): + value = int(value) + if not isinstance(value, int) or not 0 < value < 65536: + raise _error( + ErrorCode.UDF_ARG_TYPE, + f"{declared.name}()'s '{name}' is a port, and {_sql_text(argument)} " + "is not one", + argument, + fallback=base, + hint="a port is a whole number from 1 to 65535", + ) + return value + + def _check_node_ports( + self, + declared: WasmFunction, + shape: NodeShape, + streams: Mapping[str, _Value], + numbers: Mapping[str, exp.Expr], + base: exp.Anonymous, + select: exp.Select, + ) -> None: + """The declaration's ports against the shape's inputs. + + A parameter the shape has no port for is fine left unbound and + refused bound, naming it; an input the module requires is bound; each + bound port is the kind the module reads there, and only a port taking + many is handed several. + """ + declared_ports = {param.name: param for param in declared.ports} + for name in [*streams, *numbers]: + if shape.input(name) is not None: + continue + known = ", ".join(f"'{port.name}'" for port in shape.inputs) or "none" + raise _error( + ErrorCode.UDF_ARG_TYPE, + f"{declared.name}() binds '{name}', and for these params the module " + f"'{declared.module}' reads no input '{name}'", + base, + fallback=select, + hint=f"leave '{name}' out of this call; the inputs it reads here " + f"are {known}", + ) + for port in shape.inputs: + param = declared_ports.get(port.name) + if param is None: + if port.required: + raise _error( + ErrorCode.UNSUPPORTED_SQL, + f"the module '{declared.module}' requires an input " + f"'{port.name}', and {declared.name}() declares none", + base, + fallback=select, + hint=f"declare '{port.name}' as a " + f"{_PORT_TYPE_NAMES[port.kind]} parameter", + ) + continue + if port.required and port.name not in streams and port.name not in numbers: + raise _error( + ErrorCode.UDF_ARG_TYPE, + f"{declared.name}() leaves '{port.name}' out, and the module " + f"'{declared.module}' requires it", + base, + fallback=select, + hint=f"pass a stream for '{port.name}': {declared.signature}", + ) + coded = port.kind == "packets" and _port_kind(param) in ("video", "audio") + if _port_kind(param) != port.kind and not coded: + raise _error( + ErrorCode.UNSUPPORTED_SQL, + f"{declared.name}() declares '{port.name}' as {param.type}, and " + f"the module '{declared.module}' reads {port.kind} there", + base, + fallback=select, + hint=f"declare '{port.name}' as a " + f"{_PORT_TYPE_NAMES[port.kind]} parameter", + ) + value = streams.get(port.name) + if value is not None and len(value.streams) > 1 and not port.many: + raise _error( + ErrorCode.UDF_ARG_TYPE, + f"{declared.name}() hands {len(value.streams)} streams to " + f"'{port.name}', and the module '{declared.module}' reads one " + "there", + base, + fallback=select, + hint=f"pass one stream for '{port.name}'", + ) + + def _match_rows( + self, + declared: WasmFunction, + port: InputPort, + value: _Value, + base: exp.Anonymous, + select: exp.Select, + ) -> None: + """Rows a port reads against the rows its producer writes, by their schemas. + + Every field the port's schema names has to be in the producer's, with + a type it takes; fields the producer writes beyond them pass. + """ + assert port.schema is not None # checked by the caller + for stream in value.streams: + written = self._written_rows(stream.ref) + if written is None: + continue + name, schema = written + mismatch = row_mismatch(port.schema, schema) + if mismatch is None: + continue + field_name, wants, writes = mismatch + said = "does not write it" if writes == "nothing" else f"writes {writes}" + raise _error( + ErrorCode.UDF_ARG_TYPE, + f"{declared.name}() reads '{field_name}' as {wants} on its " + f"'{port.name}' input, and {name}() {said}", + base, + fallback=select, + hint=f"pass rows carrying '{field_name}' as {wants}; a reader names " + "only the fields it reads, and a producer may write more", + ) + + def _written_rows(self, ref: FrameRef) -> tuple[str, Mapping[str, object]] | None: + """Who wrote the rows `ref` carries, and their schema as they arrive. + + A filter over them passes them as they are; a span merge adds the + span's end, and writes the id it groups by as a whole number. + """ + node = self.graph.nodes.get(ref) + if node is None or node.filter not in (ROWFILTER, ROWMERGE): + return self._data_schemas.get(ref) + under = self._written_rows(node.inputs[0]) + if under is None or node.filter != ROWMERGE or MAX_SPAN not in node.args: + return under + name, schema = under + fields = schema.get("properties") + spanned = dict(fields) if isinstance(fields, dict) else {} + spanned["end_t"] = {"type": "number"} + if "id" in spanned: + spanned["id"] = {"type": "integer"} + return name, {**schema, "properties": spanned} + + def _node_pad( + self, + instance: _NodeInstance, + declared: WasmFunction, + output: Parameter, + base: exp.Anonymous, + select: exp.Select, + ) -> tuple[FrameRef, StreamType]: + """The pad of `instance` a declared output reads, and its kind. + + A named output is the module's output of that name; one unnamed is the + module's one output, or its one of the kind the RETURNS says. + """ + shape = instance.shape + wanted = _port_kind(output) + made = self.graph.nodes[instance.ref].outputs + if output.name: + found = shape.output(output.name) + if found is None: + named = ", ".join(f"'{o.name}'" for o in shape.outputs) or "none" + raise _error( + ErrorCode.UNSUPPORTED_SQL, + f"{declared.name}() returns the field '{output.name}', and the " + f"module '{declared.module}' makes no output '{output.name}'", + base, + fallback=select, + hint=f"name the fields after the outputs it makes: {named}", + ) + else: + kinds = [o for i, o in enumerate(shape.outputs) if made[i] == wanted] + if len(shape.outputs) == 1: + found = shape.outputs[0] + elif len(kinds) == 1: + found = kinds[0] + else: + listed = ", ".join(f"'{o.name}' ({o.kind})" for o in shape.outputs) + raise _error( + ErrorCode.UNSUPPORTED_SQL, + f"{declared.name}() returns {output.type}, and the module " + f"'{declared.module}' makes " + + (f"{len(kinds)} outputs of that kind" if kinds else "none of that kind"), + base, + fallback=select, + hint=f"name the one to read with RETURNS STRUCT( " + f", ...); it makes {listed or 'nothing'}", + ) + index = shape.outputs.index(found) + if made[index] != wanted: + raise _error( + ErrorCode.UNSUPPORTED_SQL, + f"{declared.name}() returns {output.type}" + + (f" as '{output.name}'" if output.name else "") + + f", and the module '{declared.module}' writes {made[index]} there", + base, + fallback=select, + hint=f"declare it as {_PORT_TYPE_NAMES[made[index]]}", + ) + ref = instance.ref if len(shape.outputs) == 1 else f"{instance.ref}:{index}" + if found.kind == "data": + schema = found.schema + if schema is None and output.annotation is not None: + schema = _record_schema(output.annotation) + if schema is not None: + self._data_schemas[ref] = (declared.name, schema) + if output.annotation is not None: + self._node_rows[ref] = (declared, output.annotation, base) + return ref, made[index] + + def _node_rows_column(self, node: exp.Expr) -> bool: + """Whether `node` is a node's rows, through any spans reduced off them.""" + node = _unwrap(node) + call = _call_parts(node) + if call is not None and call.is_macro and call.name.lower() == MERGE_SPANS: + return bool(call.args) and self._node_rows_column(call.args[0]) + found = self._node_call(node) + if found is None: + return False + outputs = found[1].outputs or () + if found[2] is None: + return len(outputs) == 1 and outputs[0].annotation is not None + return any(o.name == found[2] and o.annotation is not None for o in outputs) + + def _node_rows_at_sink( + self, value: _Value, node: exp.Expr, env: _Env, select: exp.Select + ) -> _Value: + """A node's rows selected as a column: a track of their own, or the rows. + + Rows read by another node stay a data stream; the ones a COPY writes + become what a module's rows have always become there, a rows file at a + rows destination and a subtitle track anywhere else. A one-element + array, the call under a track row a WHERE pinned, is its element. + """ + if len(value.streams) != 1: + return value + stream = value.streams[0] + found = self._node_rows.get(self._under_rows_nodes(stream.ref)) + if stream.type != "data" or found is None: + return value + declared, annotation, call = found + return self._rows_output( + stream.ref, "data", declared, annotation, call, node, env, select + ) + # -- feeders -- def _check_feeders( @@ -15778,8 +17159,11 @@ def _feed( connection.pix_fmt = wire_pix_fmt(described) if port not in use.ports: use.ports.append(port) - for name, value in self._feed_shape(fed.stream.type, programme, described).items(): - use.shape.setdefault(name, value) + if fed.stream.type != "data": + for name, value in self._feed_shape( + fed.stream.type, programme, described + ).items(): + use.shape.setdefault(name, value) connection.streams[fed.stream.ref] = fed.stream return connection.streams[fed.stream.ref] = _Stream( @@ -15911,7 +17295,8 @@ def _lower_lateral( argument = call.args[0] got = self._classify(argument, env, select) value = self._lower_expr(argument, env, select) if got == "data" else None - if value is None or value.is_array or len(value.streams) != 1: + # A one-element array, a column of a one-row source, is its element. + if value is None or len(value.streams) != 1: if value is not None: shown = f"{len(value.streams)} of them" elif got in _STREAM_KINDS: @@ -15957,14 +17342,15 @@ def _lateral_refusal( else f"{use.declared.alias}.{column}" ) kind = next( - WASM_STREAM_TYPES[c.type] for c in use.declared.columns if c.name == column + _LATERAL_KINDS[c.type] for c in use.declared.columns if c.name == column ) return _error( ErrorCode.UNSUPPORTED_SQL, f"'{read}' comes from LATERAL {use.declared.call}, which starts a " "source per row of a data stream: it is empty between rows, and " f"{consumer} reads a frame on every tick. A run-time lateral's " - "streams can only go to a module input declared 'feeder'", + "streams can only go to a module input declared 'feeder', or a " + "node's input held on a port", anchor if anchor is not None else (reader or use.anchor), hint="pass it to a feeder, such as the second argument of " f"ffrwd.switch.{kind}(, {read})", @@ -15985,14 +17371,23 @@ def _place_laterals(self) -> None: if not use.ports: continue tap = self._free_port() - path = feeder_path(tap) - self.graph.sinks.append( - SinkUnit( - outputs=[Output(ref=use.stream.ref, type="data", name=None, metadata={})], - path=path, - options={"format": _TAP_FORMAT}, + # A node's messages are written by its own region, as NDJSON on a + # pipe the host reads; any other module's are copied to the port. + piped = use.stream.ref.partition(":")[0] in self.graph.node_shapes + if piped: + self.graph.rows_sinks[use.stream.ref] = RowsSink( + container=_ROWS_CONTAINER, path=f"{TAP_DOCUMENT}{tap}" + ) + else: + self.graph.sinks.append( + SinkUnit( + outputs=[ + Output(ref=use.stream.ref, type="data", name=None, metadata={}) + ], + path=feeder_path(tap), + options={"format": _TAP_FORMAT}, + ) ) - ) declared = use.declared line, col = _pos(use.anchor) shapes = { @@ -16027,6 +17422,7 @@ def _place_laterals(self) -> None: needs=declared.needs, line=line, col=col, + pipe=piped, ) ) @@ -16504,11 +17900,23 @@ def _packet_filter_pads( already encoded without touching a picture. """ described = self.describes[declared.module] - inputs = self.graph.nodes[ref].inputs + filtered = self.graph.nodes[ref] + shape = node_shape(filtered.filter, self.graph.node_shapes[ref]) if ( + ref in self.graph.node_shapes + ) else None seen: dict[str, int] = {} pads: list[dict[str, object]] = [] - for input_ref in inputs: + for position, input_ref in enumerate(filtered.inputs): kind = ref_type(self.graph, input_ref) + port = shape.input(filtered.ports[position]) if shape is not None else None + if shape is not None and (port is None or port.kind != "packets"): + # A node's other inputs are not encoded for it: rows, or + # frames it reads as they are. + pads.append({}) + continue + if port is not None: + described = replace(described, video_codecs=port.accepts.codecs or None) + described = replace(described, audio_codecs=port.accepts.codecs) index = seen.get(kind, 0) seen[kind] = index + 1 if kind not in asked and self._copies_onto_sink(input_ref, kind, described): @@ -16693,6 +18101,100 @@ def _lower_sink_call( # way a rows projection's does not. return _Value(type=pads[0].type, streams=(), is_array=False) + def _lower_node_sink( + self, + node: exp.Expr, + declared: WasmFunction, + call: _Call, + env: _Env, + select: exp.Select, + ) -> _Value: + """A node at a COPY's TO: a sink, read as a packet sink is. + + Its shape for the call's params has no outputs; its inputs take the + SELECT's streams by kind, coded packets fronted by the encoder the + COPY's WITH shapes, and data. Each input is a many-port, so how many + of each kind the query hands it is the query's to say. The node is + spelled by its ports in the network, and the rows it emits are the + run's, as a packet sink's are. + + Which ports there are is asked with nothing bound; once the SELECT's + streams are on them, the shape is asked again with each port and a + hint per stream bound there, which is what the host asks at run time. + """ + described = self._node_module(declared) + assert described is not None # what the caller checked + at = _sink_stream_count(node, len(call.args)) + anchor = _unwrap(node) + assert isinstance(anchor, exp.Anonymous) # a sink call is one + params = self._wasm_params( + declared, described, call, node, select, env, {}, first=at + ) + shape = self._node_shape(declared, described, params, [], anchor, select) + ports = self._node_sink_ports(declared, shape, node, select) + view = _sink_view(described, ports) + self._node_sinks[declared.module] = (view, shape) + value = self._lower_sink_call(node, declared, view, call, env, select) + ref = self.graph.module_sinks[-1] + made = self.graph.nodes[ref] + kinds = [ref_type(self.graph, read) for read in made.inputs] + named = {kind: port.name for kind, port in ports.items() if port is not None} + bound = [ + Binding( + name, + tuple( + StreamHint(self._stream_rate(read)) + for read, kind in zip(made.inputs, kinds) + if named.get(kind) == name + ), + ) + for name in dict.fromkeys(named[kind] for kind in kinds if kind in named) + ] + shape = self._node_shape(declared, described, params, bound, anchor, select) + ports = self._node_sink_ports(declared, shape, node, select) + named = {kind: port.name for kind, port in ports.items() if port is not None} + if any(kind not in named for kind in kinds): + raise _error( + ErrorCode.UNSUPPORTED_SQL, + f"{declared.name}() returns sink, and with the streams the query " + f"hands it the module '{declared.module}' reads none of " + + ", ".join(sorted({kind for kind in kinds if kind not in named})), + node, + fallback=select, + hint="hand the sink only the kinds of stream its shape reads", + ) + self._node_sinks[declared.module] = (_sink_view(described, ports), shape) + made.ports = [named[kind] for kind in kinds] + made.bound = bound_json(bound) + emits = described.rows_schema is not None + if emits: + made.outputs = ["data"] + made.out_ports = [EMITTED_ROWS_PORT] if emits else [] + self.graph.node_shapes[ref] = dict(shape.raw) + return value + + def _node_sink_ports( + self, + declared: WasmFunction, + shape: NodeShape, + node: exp.Expr, + select: exp.Select, + ) -> dict[str, InputPort | None]: + """The port a node sink's shape reads each kind of stream on, refused + where it makes outputs or reads no coded packets.""" + ports = {kind: _sink_port(shape, kind) for kind in ("video", "audio", "data")} + if shape.outputs or ports["video"] is None and ports["audio"] is None: + raise _error( + ErrorCode.UNSUPPORTED_SQL, + f"{declared.name}() returns sink, and the module '{declared.module}' " + + ("makes outputs" if shape.outputs else "reads no coded packets"), + node, + fallback=select, + hint="a node at a COPY's TO reads coded packets and data and makes " + "no output; declare this one as what it is", + ) + return ports + def _lower_row_reading_sink_call( self, node: exp.Expr, @@ -17054,7 +18556,8 @@ def _lower_macro_call( stream_param = macro.params[stream_pos] self._reject_null_stream(call.display, call.args[stream_pos], select) kind = self._classify(call.args[stream_pos], env, select) - self._reject_passthrough_args(call.display, [kind], call, call.args[stream_pos]) + if stream_param.stream_type != "data": + self._reject_passthrough_args(call.display, [kind], call, call.args[stream_pos]) if kind != stream_param.stream_type: hint = macro.kind_hints.get( kind, @@ -17137,6 +18640,16 @@ def _macro_options( fallback=node, hint=f"its signature is {macro.signature}", ) + written_names = {argument.name for argument in call.named} + missing = next((name for name in macro.required if name not in written_names), None) + if missing is not None: + raise _error( + ErrorCode.UDF_ARG_TYPE, + f"{call.display}() needs '{missing}'", + node, + fallback=select, + hint=f"write it by name: {macro.signature}", + ) per_row = any(_reads_row_column(argument.value, env) for argument in call.named) tuples = env.relation.tuples if per_row and env.relation is not None else [] cache: dict[int, dict[str, object]] = {} @@ -18549,6 +20062,9 @@ def _classify(self, node: exp.Expr, env: _Env, select: exp.Select) -> str: and not node.this.is_string ): return "num" + found = self._node_call(node) + if found is not None: + return self._node_output_kind(found[1], found[2]) # An output of a data filter is a data stream. if _data_projection(node, self.res.wasm) is not None: return "data" @@ -18643,6 +20159,7 @@ def _classify(self, node: exp.Expr, env: _Env, select: exp.Select) -> str: def run_table(self) -> list[TableSink]: """One :class:`~ffrwd.table.TableSink` per COPY, or one bare-select.""" + self._check_declared_worlds() self._lower_ctes() self.table_mode = True sinks: list[TableSink] = [] @@ -19811,6 +21328,7 @@ def lower( probe_source: ProbeSource = wasm_probe_source, read_packets: ReadPackets = wasm_read_packet_rows, probe_path: ProbePath = probe_one_path, + shapes: Shape | None = None, ) -> Graph: """Lower a resolved query into an IR graph -- its FIRST command's. @@ -19835,14 +21353,15 @@ def lower( call site's arguments, to fold its result -- a parameter for the same reason. `probe_source` runs one ``RETURNS source`` module's ``probe``, once per FROM alias that calls one, to bind its catalog -- a parameter - for the same reason. + for the same reason. `shapes` asks a node module for its shape per call, + and is one too. Raises ``FfrwdError`` — and nothing else — on every rejection. """ return lower_commands( res, probes, registry=registry, on_warning=on_warning, describes=describes, invoke=invoke, probe_failures=probe_failures, probe_source=probe_source, - read_packets=read_packets, probe_path=probe_path, + read_packets=read_packets, probe_path=probe_path, shapes=shapes, )[0] @@ -19858,6 +21377,7 @@ def lower_commands( probe_source: ProbeSource = wasm_probe_source, read_packets: ReadPackets = wasm_read_packet_rows, probe_path: ProbePath = probe_one_path, + shapes: Shape | None = None, ) -> list[Graph]: """Lower a resolved query into one IR graph per ffmpeg COMMAND. @@ -19874,11 +21394,13 @@ def lower_commands( Same probing/registry contract as :func:`lower`; raises ``FfrwdError`` -- and nothing else -- on every rejection. """ + asked = shapes if shapes is not None else ShapeCache() try: shared = _Lowerer( res, probes, registry, fanout_sinks=True, on_warning=on_warning, describes=describes, invoke=invoke, probe_failures=probe_failures, probe_source=probe_source, read_packets=read_packets, probe_path=probe_path, + shapes=asked, ) graph = shared.run() count = shared.fanout_count @@ -19894,6 +21416,7 @@ def lower_commands( res, probes, registry, fanout_index=index, on_warning=on_warning, describes=describes, invoke=invoke, probe_failures=probe_failures, probe_source=probe_source, read_packets=read_packets, probe_path=probe_path, + shapes=asked, ).run() for index in range(count) ] @@ -19977,6 +21500,7 @@ def lower_table( probe_source: ProbeSource = wasm_probe_source, read_packets: ReadPackets = wasm_read_packet_rows, probe_path: ProbePath = probe_one_path, + shapes: Shape | None = None, ) -> list[TableSink]: """Lower a resolved TABLE query into its printable result set(s). @@ -19991,7 +21515,7 @@ def lower_table( return _Lowerer( res, probes, registry, on_warning=on_warning, describes=describes, invoke=invoke, probe_failures=probe_failures, probe_source=probe_source, - read_packets=read_packets, probe_path=probe_path, + read_packets=read_packets, probe_path=probe_path, shapes=shapes, ).run_table() except FfrwdError: raise diff --git a/cli/ffrwd/macros.py b/cli/ffrwd/macros.py index 2972aaf..2e67e3f 100644 --- a/cli/ffrwd/macros.py +++ b/cli/ffrwd/macros.py @@ -26,7 +26,10 @@ DEFAULT_MAX_SPREAD, LEAKY, MAX_LATENESS, + MAX_SPAN, MAX_SPREAD, + MERGE_SPANS, + ROWMERGE, StreamType, ) @@ -90,6 +93,8 @@ class Macro: options: tuple[str, ...] = () positive: tuple[str, ...] = () nonnegative: tuple[str, ...] = () + # The options a call has to write: what the node cannot do without. + required: tuple[str, ...] = () @property def signature(self) -> str: @@ -148,6 +153,12 @@ def _leaky(values: list[object], node: NodeBuilder, options: dict[str, object]) return node(LEAKY, limits, [str(f)], ["video"]) +def _merge_spans(values: list[object], node: NodeBuilder, options: dict[str, object]) -> str: + """The host's own rows node, grouping per-tick rows into spans by `start_t`.""" + (rows,) = values + return node(ROWMERGE, dict(options), [str(rows)], ["data"]) + + _LEAKY_SOUND_HINT = ( "ffrwd.leaky() drops late pictures and never sound: pass the picture " "through it and the sound beside it, e.g. ffrwd.leaky(s.video[1]), s.audio[1]" @@ -196,6 +207,15 @@ def _leaky(values: list[object], node: NodeBuilder, options: dict[str, object]) positive=(MAX_LATENESS,), nonnegative=(MAX_SPREAD,), ), + MERGE_SPANS: Macro( + name=MERGE_SPANS, + params=(MacroParam("rows", "stream", "data"),), + output="data", + expand=_merge_spans, + options=(MAX_SPAN,), + positive=(MAX_SPAN,), + required=(MAX_SPAN,), + ), "loudnorm2": Macro( name="loudnorm2", params=(MacroParam("stream", "stream", "audio"),), diff --git a/cli/ffrwd/nodes.py b/cli/ffrwd/nodes.py index 535351e..f2961f6 100644 --- a/cli/ffrwd/nodes.py +++ b/cli/ffrwd/nodes.py @@ -100,6 +100,8 @@ _resolve_rows_documents, _StageRun, _watch, + held_writers, + never_started, plan_argv, stage_result, stage_wires, @@ -1216,6 +1218,13 @@ def run_stage( current.abort.set() if current.unheard is not None and not interrupted: failed, timed_out, wedge = current.unheard[0], True, current.unheard[1] + elif failed is None and not interrupted and not show_only: + if stop is None or not stop.is_set(): + missing = never_started( + held_writers(writers, feeds), current.started, stage.processes + ) + if missing is not None: + failed, timed_out, wedge = missing[0], True, missing[1] for node in involved: self._send(node, {"type": "stop"}) results = self._collect(stage, current, involved) diff --git a/cli/ffrwd/packages.py b/cli/ffrwd/packages.py index d87ee40..2702bfd 100644 --- a/cli/ffrwd/packages.py +++ b/cli/ffrwd/packages.py @@ -69,7 +69,7 @@ import urllib.error import urllib.parse import urllib.request -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Callable, Collection, Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timezone from email.utils import parsedate_to_datetime @@ -1677,6 +1677,35 @@ def _ensure( brought.append(release) +def _unlinked( + entries: Sequence[LockEntry], wanted: Mapping[str, str], linked: Collection[str | None] +) -> list[LockEntry]: + """`entries` less each version of a linked name nothing `wanted` reaches. + + The link answers for that name, so a version installed before it stays + only where another pinned package's own dependencies name it. + """ + reached: set[tuple[str, str]] = set() + stack = list(wanted.items()) + while stack: + pinned = stack.pop() + if pinned in reached: + continue + reached.add(pinned) + entry = _entry_for(pinned[0], pinned[1], entries) + if entry is not None: + stack.extend(entry.dependencies.items()) + return [ + entry + for entry in entries + if not ( + isinstance(entry, RegistryEntry) + and entry.name in linked + and (entry.name, entry.version) not in reached + ) + ] + + def _write_lockfile_migrating( lock: Path, entries: Sequence[LockEntry], wanted: Mapping[str, str] ) -> None: @@ -1800,7 +1829,9 @@ def install_project( A dependency this project links to a working directory is left as the link it is: that is what a link is for, and fetching the registry's copy beside it would shadow the link and refuse the whole install for a - package that is not published yet. + package that is not published yet. A pin of it from before the link + goes too, so the lock is the one an install with the link already + standing writes. """ package = read_manifest(manifest) current = read_lockfile(lock) if lock.is_file() else None @@ -1813,13 +1844,14 @@ def install_project( if name in linked: if detail is not None: detail(f"{name} is linked to a working directory") + wanted.pop(name, None) continue if detail is not None: detail(f"resolving {name} {version}") release = resolve(f"{name}@{version}") _ensure(release, entries, [], brought, announce, progress, detail) wanted[name] = release.version - _write_lockfile_migrating(lock, entries, wanted) + _write_lockfile_migrating(lock, _unlinked(entries, wanted, linked), wanted) _install_models(package, announce, progress, detail) _install_runtime(package, announce, progress) diff --git a/cli/ffrwd/parser.py b/cli/ffrwd/parser.py index 782faeb..b11c02d 100644 --- a/cli/ffrwd/parser.py +++ b/cli/ffrwd/parser.py @@ -2717,6 +2717,15 @@ def _projection_field_name(node: exp.Expr) -> str | None: # with, for lowering to mint the node that applies it. ROW_PREDICATE = "row_predicate" +# Where a call or a field read only a node module can make sense of keeps the +# refusal an older world's module earns, for lowering to raise once the +# module's describe says it is not a node. +OLDER_WORLD = "older_world" + +# The same for the refusal a node module earns: a call written for an older +# world's module, which only a node's describe can show to be wrong. +NODE_REFUSAL = "node_refusal" + # Where `merge_cues(...)` over a module's rows leaves the distance it was # written with, for lowering to mint the node that applies it. ROW_MERGE = "row_merge" @@ -2951,6 +2960,25 @@ def annotation_projection( return (base, declared) if _ident_name(field) == declared.emits.name else None +def node_rows_call(node: object, wasm: Mapping[str, WasmFunction]) -> Annotation | None: + """The record a call's rows carry, for a call a node may answer with rows alone. + + A declaration whose RETURNS is one array of records over a stream: a + packet sink's in FROM, a node's run-time data stream anywhere else. None + for every other expression. + """ + if not isinstance(node, exp.Expr): + return None + call = _unwrap_paren(node) + if not isinstance(call, exp.Anonymous): + return None + declared = wasm.get(str(call.name).lower()) + outputs = declared.outputs if declared is not None else None + if not outputs or len(outputs) != 1 or outputs[0].name: + return None + return outputs[0].annotation + + def is_annotation_argument(node: object, wasm: Mapping[str, WasmFunction]) -> bool: """Whether this argument WRITES a consumer's annotation column. @@ -5149,6 +5177,31 @@ def _check_wasm_field( A data filter's struct is its outputs, each a data stream of its own, and every field of it is one. """ + if declared.is_node_only: + names = tuple(o.name for o in declared.outputs or () if o.name) + if path in names: + return + raise _error( + ErrorCode.UNSUPPORTED_SQL, + f"{declared.name}() returns no field '{path}'", + sub, + fallback=select, + hint="it returns " + ", ".join(f"'{name}'" for name in names) + if names + else f"it returns one {declared.written_returns}: the call itself is " + "the stream", + ) + try: + self._check_older_wasm_field(declared, path, sub, select) + except FfrwdError as refusal: + if not any(o.name == path for o in declared.outputs or ()): + raise + sub.meta[OLDER_WORLD] = refusal + + def _check_older_wasm_field( + self, declared: WasmFunction, path: str, sub: exp.Dot, select: exp.Select + ) -> None: + """:meth:`_check_wasm_field` as a module of an older world reads it.""" if declared.is_data_filter: if path in declared.data_fields: return @@ -5860,11 +5913,17 @@ def _row_gather(self, array_node: exp.Array) -> bool: arguments = item.expressions source = arguments[0] if len(arguments) == 1 else None found = annotation_projection(source, self.wasm) - if found is None or not isinstance(source, exp.Expr): + rows = node_rows_call(source, self.wasm) + if (found is None and rows is None) or not isinstance(source, exp.Expr): self._no_gather_over_rows_function(source, array_node) return False - emits = found[1].emits - assert emits is not None # what annotation_projection matched on + if found is None: + try: + self._no_gather_over_rows_function(source, array_node) + except FfrwdError as refusal: + _unwrap_paren(source).meta.setdefault(OLDER_WORLD, refusal) + emits = found[1].emits if found is not None else rows + assert emits is not None # what annotation_projection or node_rows_call matched on _check_query_args( subquery, frozenset({"expressions", "from_", "where"}), "row gather" ) @@ -7930,7 +7989,11 @@ def _check_where( hint=_WHERE_HINT, ) column, low, high, strict = parsed - if strict: + table_node = column.args.get("table") + # A node read in FROM ends on a tick, so `t < 10` says where; + # which wasm sources are nodes is the module's to say, in lowering. + ticks = table_node is not None and _ident_name(table_node) in self.wasm_sources + if strict and not ticks: raise _error( ErrorCode.UNSUPPORTED_SQL, "strict inequalities are not supported", @@ -7938,7 +8001,6 @@ def _check_where( fallback=where, hint=_STRICT_HINT, ) - table_node = column.args.get("table") if table_node is None: raise _error( ErrorCode.UNSUPPORTED_SQL, diff --git a/cli/ffrwd/processes.py b/cli/ffrwd/processes.py index 30b91c1..237bff6 100644 --- a/cli/ffrwd/processes.py +++ b/cli/ffrwd/processes.py @@ -108,7 +108,7 @@ import json import math import re -from collections.abc import Callable, Iterable, Mapping, Sequence +from collections.abc import Callable, Collection, Iterable, Mapping, Sequence from dataclasses import dataclass, field, replace from fractions import Fraction from typing import Literal @@ -123,6 +123,7 @@ PIPE, ROWFILTER, ROWMERGE, + TAP_DOCUMENT, FeederCall, FrameRef, Graph, @@ -140,6 +141,9 @@ src_parts, ) from .probe import JSON_CODEC, ProbeResult, StreamMeta, is_url +from .shapes import Accepts, NodeShape, node_shape +from .sink import COLOR_OPTIONS +from .timing import Paths, paths_of __all__ = [ "COPY_CODEC", @@ -154,6 +158,9 @@ "PIPE_BUFFER_LIMIT", "RAWVIDEO", "SAFETY", + "SAMPLE_FMT_CODECS", + "WIRE_PIX_FMTS", + "WIRE_SAMPLE_FMTS", "CLOCK_SIZE", "AudioFormat", "DataFormat", @@ -188,7 +195,9 @@ "is_live_probe", "nothing_external", "check_spellable", + "once_per_pipe", "partition", + "pipe_key", ] # The container every raw stream edge is wrapped in, and the codecs inside it. @@ -205,6 +214,13 @@ # picks and the producing ffmpeg is told to write. DEFAULT_PIX_FMT = "yuv420p" +# The pixel formats a stream edge into or out of the sidecar can carry. +WIRE_PIX_FMTS: tuple[str, ...] = ("rgba", "yuv420p", "yuv422p", "yuv444p") + +# The sample formats one can carry, and the pcm each of them travels as. +WIRE_SAMPLE_FMTS: tuple[str, ...] = ("f32", "s16") +SAMPLE_FMT_CODECS: Mapping[str, str] = {"f32": "pcm_f32le", "s16": "pcm_s16le"} + # The width and height of the picture a data filter's CLOCK pad is handed: it # reads the pts and nothing else, so each frame is made as small as a frame # can usefully be before it crosses the pipe. @@ -252,6 +268,45 @@ # The colorimetry a codec's sidecar is told, by the names of its flags. SIDECAR_COLOR_FLAGS: tuple[str, ...] = ("color_range", "color_primaries", "color_trc", "colorspace") +# A stream's colorimetry: each option ffmpeg's output takes it by, against +# the field ffprobe reports it in and setparams' option for it. +PROBED_COLOR: Mapping[str, str] = { + "color_range": "color_range", + "color_primaries": "color_primaries", + "color_trc": "color_transfer", + "colorspace": "color_space", + "chroma_sample_location": "chroma_location", +} +SETPARAMS_COLOR: Mapping[str, str] = { + "color_range": "range", + "color_primaries": "color_primaries", + "color_trc": "color_trc", + "colorspace": "colorspace", + "chroma_sample_location": "chroma_location", +} +# What ffprobe and setparams write for a field nothing settles. +UNSAID_COLOR = frozenset({"unknown", "unspecified", "reserved", "auto"}) +# The fields that describe YUV alone, which an RGB picture has none of. +YUV_ONLY_COLOR = ("color_range", "colorspace", "chroma_sample_location") +RGB_PREFIXES = ("rgb", "bgr", "gbr", "argb", "abgr") +# Filters that convert colour, past which a stream's colorimetry is no longer +# its input's. A `scale` does too where it names an in_ or out_ option. +COLOR_CONVERTING_FILTERS = frozenset( + {"colorspace", "colormatrix", "zscale", "tonemap", "tonemap_opencl", "libplacebo"} +) + +# The colour a node network's input is told, by the ``-pad`` key each option +# goes under, and what a field nothing settles is called there. +_PAD_COLOR: Mapping[str, str] = { + "color_range": "range", + "color_primaries": "primaries", + "color_trc": "trc", + "colorspace": "space", +} +_UNKNOWN_COLOR = "unknown" +# What leaves a conversion to an RGB pixel format: full range, no YUV matrix. +_RGB_WIRE_COLOR: Mapping[str, str] = {"color_range": "pc", "colorspace": "gbr"} + # Bytes one pixel takes on the wire, per pixel format ffrwd carries. _PIXEL_BYTES: Mapping[str, int] = { "rgba": 4, @@ -690,6 +745,10 @@ class StreamEdge: `live` marks an edge the one reader of a live input writes. Its pictures reach the muxer as they come (:data:`PASSTHROUGH`), never duplicated or dropped to keep a constant rate, since the bound counts them one for one. + + `nut` names the one NUT this stream rides with every other stream between + the same two processes, where one end hosts node modules; "" for a stream + on a pipe of its own. """ source: str @@ -700,6 +759,7 @@ class StreamEdge: bound: int = 0 buffer: EdgeBuffer | None = None live: bool = False + nut: str = "" def to_dict(self) -> dict[str, object]: written: dict[str, object] = { @@ -717,6 +777,8 @@ def to_dict(self) -> dict[str, object]: written["buffer"] = self.buffer.to_dict() if self.live: written["live"] = True + if self.nut: + written["nut"] = self.nut return written @classmethod @@ -731,9 +793,28 @@ def from_dict(cls, d: Mapping[str, object]) -> StreamEdge: bound=_read_whole(d, "bound", 0), buffer=None if buffer is None else EdgeBuffer.from_dict(_read_object(d, "buffer")), live=d.get("live") is True, + nut=_read_text(d, "nut", ""), ) +def pipe_key(edge: StreamEdge) -> str: + """Which pipe `edge` rides: its NUT's, or one of its own.""" + return edge.nut or f"{edge.source}>{edge.target}:{edge.ref}" + + +def once_per_pipe(edges: Sequence[StreamEdge]) -> list[StreamEdge]: + """`edges` with one entry per pipe: the first edge of each NUT stands for it.""" + kept: list[StreamEdge] = [] + seen: set[str] = set() + for edge in edges: + key = pipe_key(edge) + if key in seen: + continue + seen.add(key) + kept.append(edge) + return kept + + @dataclass(frozen=True) class FileEdge: """A file handed from one process to another, ordering the two.""" @@ -1190,6 +1271,20 @@ class SidecarProcess: # (`color_range`, `color_primaries`, `color_trc`, `colorspace`), for the # fields the query settles: the NUT the codec reads carries none of it. color: tuple[tuple[str, str], ...] = () + # True for a region holding a node module: its pads name the ports they + # bind, and each of its edges is one NUT carrying every stream between it + # and the process at the far end. + node_network: bool = False + # The ports this region listens on for an input a node holds and the + # query left unbound: (port, the node's name in the network, its input). + listens: tuple[tuple[int, str, str], ...] = () + # What each ``-i`` of a node network carries of its picture's colour, by + # the ``-pad`` key, in ``-i`` order: empty for an input with no raw + # picture an ffmpeg wrote. + colors: tuple[tuple[tuple[str, str], ...], ...] = () + # The tags the query wrote on what each ``-i`` of a node network carries, + # in ``-i`` order: empty for an input it wrote none on. + tags: tuple[tuple[tuple[str, str], ...], ...] = () @property def nodes(self) -> tuple[str, ...]: @@ -1211,6 +1306,8 @@ def network(self) -> bool: """ if self.graph is None: return False + if self.node_network: + return True if self.packet_sink or self.packet_source or self.packet_filter: return False if self.data_filter or self.codec: @@ -1267,6 +1364,14 @@ def to_dict(self) -> dict[str, object]: written["color"] = dict(self.color) if self.pads: written["pads"] = [None if p is None else p.to_dict() for p in self.pads] + if self.node_network: + written["node_network"] = True + if self.listens: + written["listens"] = [list(one) for one in self.listens] + if self.colors: + written["colors"] = [dict(one) for one in self.colors] + if self.tags: + written["tags"] = [dict(one) for one in self.tags] if self.network and self.graph is not None: written["graph"] = self.graph.to_dict() return written @@ -1313,6 +1418,22 @@ def from_dict(cls, d: Mapping[str, object]) -> SidecarProcess: codec=_read_text(d, "codec", ""), frame_rate=_read_text(d, "frame_rate", ""), color=tuple((flag, str(value)) for flag, value in _read_pairs(d, "color")), + node_network=d.get("node_network") is True, + listens=tuple( + (int(str(one[0])), str(one[1]), str(one[2])) + for one in _read_list(d, "listens") + if isinstance(one, list) and len(one) == 3 + ), + colors=tuple( + tuple((str(key), str(value)) for key, value in one.items()) + for one in _read_list(d, "colors") + if isinstance(one, dict) + ), + tags=tuple( + tuple((str(key), str(value)) for key, value in one.items()) + for one in _read_list(d, "tags") + if isinstance(one, dict) + ), ) @@ -1527,6 +1648,91 @@ def _named_ref(ref: FrameRef) -> str: return f"the output of '{_ref_node(ref)}'" +def _accepted( + accepts: Accepts, written: StreamFormat +) -> tuple[tuple[str, ...], str | None]: + """The formats a port takes, and the one `written` carries, of its kind. + + Pixel formats for a picture, sample formats for sound; nothing for a + stream copied as it was coded, or one whose format is not known. + """ + if isinstance(written, VideoFormat) and written.codec != COPY_CODEC: + return accepts.pixel_formats, written.pix_fmt + if isinstance(written, AudioFormat): + made = next( + (sample for sample, codec in SAMPLE_FMT_CODECS.items() if codec == written.codec), + None, + ) + return accepts.sample_formats, made + return (), None + + +def converts_colour(node: Node) -> bool: + """True where `node` converts the colour of the pictures through it.""" + if node.filter in COLOR_CONVERTING_FILTERS: + return True + return node.filter == "scale" and any( + str(key).startswith(("in_", "out_")) for key in node.args + ) + + +def stream_colorimetry( + graph: Graph, + ref: FrameRef, + source_meta: Callable[[FrameRef], StreamMeta | None], + pix_fmt: str | None = None, + *, + opaque: Collection[str] = (), +) -> dict[str, str]: + """The colorimetry the pictures `ref` names carry, by ffmpeg's option + names, as far as the query says it. + + A ``setparams`` on the way settles each field it names; past a filter + that converts colour (:data:`COLOR_CONVERTING_FILTERS`), or a node named + in `opaque`, nothing else does. Every field left is the input stream's, + as `source_meta` says it was probed. A field nothing settles is absent. + Pictures in an RGB `pix_fmt` have no YUV matrix, range or chroma siting + to state. + """ + said: dict[str, str] = {} + seen: set[str] = set() + current: FrameRef | None = ref + while current is not None and not is_src(current): + name, _, pad = current.rpartition(":") + name = name if name and pad.isdigit() else current + node = graph.nodes.get(name) + if node is None or name in seen or name in opaque or converts_colour(node): + current = None + break + seen.add(name) + if node.filter == "setparams": + for option, param in SETPARAMS_COLOR.items(): + value = node.args.get(param) + if value is not None and str(value) not in UNSAID_COLOR: + said.setdefault(option, str(value)) + current = next((r for r in node.inputs if ref_type(graph, r) == "video"), None) + meta = source_meta(current) if current is not None else None + if meta is not None: + for option, field_name in PROBED_COLOR.items(): + value = getattr(meta, field_name) + if isinstance(value, str) and value not in UNSAID_COLOR: + said.setdefault(option, value) + if pix_fmt is not None and pix_fmt.startswith(RGB_PREFIXES): + for option in YUV_ONLY_COLOR: + said.pop(option, None) + return {option: said[option] for option in COLOR_OPTIONS if option in said} + + +def _converted(written: StreamFormat, wanted: str) -> str: + """The ffmpeg filter call that hands a port `wanted` instead of `written`.""" + if isinstance(written, AudioFormat): + from .wasm import FFMPEG_SAMPLE_FMTS # wasm reads this module's wire formats + + sample = FFMPEG_SAMPLE_FMTS.get(wanted, wanted) + return f"ffmpeg.aformat(, sample_fmts => '{sample}')" + return f"ffmpeg.format(, pix_fmts => '{wanted}')" + + def _bindings(paths: Iterable[str]) -> tuple[ModuleBinding, ...]: """One ``-m`` entry per distinct module path, named after the file. @@ -1885,6 +2091,18 @@ def __init__( ) | frozenset( alias for alias, source in g.module_sources.items() if not source.bounded ) + self.node_shapes: dict[str, NodeShape] = { + name: node_shape(g.nodes[name].filter, raw) + for name, raw in g.node_shapes.items() + if name in g.nodes + } + # How late each node's refs run, counted once it is first asked. + self._paths: Paths | None = None + self.live |= frozenset( + alias + for alias, name in g.node_sources.items() + if name in self.node_shapes and not self.node_shapes[name].bounded + ) self.order = _topological(g) # A hosted node is external whoever asked: only the sidecar runs it. self.external = { @@ -1900,6 +2118,9 @@ def __init__( self.sidecars: list[SidecarProcess] = [] self.sidecar_of: dict[str, str] = {} # node id -> process id self.members: dict[str, list[str]] = {} # process id -> its node ids + # A node region's read of a stream another of its reads already + # carries in the same format: the copy, and the read it binds to. + self.same_reads: dict[FrameRef, FrameRef] = {} self.consumer_of: dict[str, str] = {} # feeder process id -> its reader # The ffmpeg processes copying one module's data stream to each of its # readers, by the sidecar writing it and the stream's ref. @@ -2152,8 +2373,11 @@ def _check_handed_once(self) -> None: continue # each track is its own named pipe by construction handed: dict[FrameRef, set[str]] = {} for edge in self.edges: - if edge.source == sidecar.id: - handed.setdefault(edge.ref, set()).add(edge.target) + if edge.source != sidecar.id: + continue + if sidecar.node_network and ref_type(self.g, edge.ref) == "data": + continue # the network maps a label to every output reading it + handed.setdefault(edge.ref, set()).add(edge.target) for ref, targets in handed.items(): if len(targets) < 2: continue @@ -2567,7 +2791,8 @@ def join(*names: str, one_output: bool = False) -> bool: """Merge the groups these nodes are in, if the result stays convex. `one_output` also holds the merge to a region whose frames still - leave on one pad, which is all a module process can write. + leave on one pad, which is all a module process can write. A + region holding a node writes as many as it has readers. """ reps = {home.get(name) for name in names} if None not in reps and len(reps) == 1: @@ -2578,7 +2803,11 @@ def join(*names: str, one_output: bool = False) -> bool: joined = [n for n in self.order if n in wanted] if not self._convex(joined, reach): return False - if one_output and len(self._region_writes(joined)) > 1: + if ( + one_output + and not any(name in self.node_shapes for name in joined) + and len(self._region_writes(joined)) > 1 + ): return False for rep in reps: if rep is not None: @@ -2629,6 +2858,17 @@ def join(*names: str, one_output: bool = False) -> bool: continue if any(reader in alone for reader in reads): continue # a packet sink's edge stays an ffmpeg's to encode + if any(reader in self.node_shapes for reader in reads) and ( + len( + { + self._format(inputs[0], reader) + for reader in reads + if not self._reads_timing(reader, inputs[0]) + } + ) + > 1 + ): + continue # nodes taking it in different formats get a stream each # The split's producer joins too when it is a module; otherwise # the split's own input becomes a boundary read of the region. feeds = _ref_node(inputs[0]) @@ -2641,7 +2881,14 @@ def join(*names: str, one_output: bool = False) -> bool: return [groups[name] for name in self.order if name in groups] def _region_reads(self, members: Sequence[str]) -> list[tuple[FrameRef, str]]: - """Refs this region reads from outside, each with the node reading it.""" + """Refs this region reads from outside, each with the node reading it. + + A region holding a node reads each stream once per format its ports + take it in: copies a split outside made of it are one read, which the + network hands every port (:attr:`same_reads`). A port reading it for + its timing alone takes whichever of those reads there is, and the + stream as it is where there is none. + """ inside = set(members) wanted: list[tuple[FrameRef, str]] = [] seen: set[FrameRef] = set() @@ -2654,7 +2901,28 @@ def _region_reads(self, members: Sequence[str]) -> list[tuple[FrameRef, str]]: continue seen.add(ref) wanted.append((ref, name)) - return wanted + if not any(name in self.node_shapes for name in members) or any( + name in self.g.packet_sinks for name in members + ): + return wanted + kept: dict[tuple[FrameRef, StreamFormat], FrameRef] = {} + any_read: dict[FrameRef, FrameRef] = {} + bound: dict[FrameRef, FrameRef] = {} + timed = [(ref, reader) for ref, reader in wanted if self._reads_timing(reader, ref)] + for ref, reader in [one for one in wanted if one not in timed] + timed: + source = self._past_splits(ref) + first = any_read.get(source) if (ref, reader) in timed else None + if first is None: + first = kept.setdefault((source, self._format(ref, reader)), ref) + any_read.setdefault(source, first) + bound[ref] = first + shared: list[tuple[FrameRef, str]] = [] + for ref, reader in wanted: + if bound[ref] == ref: + shared.append((ref, reader)) + else: + self.same_reads[ref] = bound[ref] + return shared def _region_writes(self, members: Sequence[str]) -> list[tuple[FrameRef, StreamType]]: """Pads this region produces that something outside it reads.""" @@ -2680,6 +2948,11 @@ def _region_writes(self, members: Sequence[str]) -> list[tuple[FrameRef, StreamT return written def _shape(self, name: str) -> ModuleShape: + found = self.node_shapes.get(name) + if found is not None: + # Its waits are counted in seconds (:mod:`ffrwd.timing`), not + # here in frames. + return ModuleShape(one_to_one=found.one_to_one, pure=found.pure) return self.shapes.get(self.g.nodes[name].filter, ModuleShape()) def _lookahead(self, members: Sequence[str], *, frames: bool = False) -> int: @@ -2746,6 +3019,8 @@ def _node_delay(self, name: str) -> int | None: # It drops pictures, but holds none: what it costs its path is # counted in time instead (:meth:`_late_seconds`). return 0 + if name in self.node_shapes: + return self._node_frames(name) if self.external.get(name, False): return self._frames_ahead(name) if self._shape(name).one_to_one else None if node.filter in SPLIT_FILTERS: @@ -2760,6 +3035,27 @@ def _node_delay(self, name: str) -> int | None: size = written if isinstance(written, int) and not isinstance(written, bool) else default return max(size - 1, 0) + def _node_frames(self, name: str) -> int | None: + """The frames node `name` holds past its clock: its window, the waits + its interval inputs add and its outputs' latency, at its pictures' rate.""" + if self._paths is None: + self._paths = paths_of(self.g, self.probes) + paths = self._paths + node = self.g.nodes[name] + shape = self.node_shapes[name] + ready = paths.ready(name)[0] + clock = shape.clock_input + refs = [ref for port, ref in zip(node.ports, node.inputs) if clock and port == clock.name] + clock_delay = paths.latest(refs) if refs else 0.0 + if ready is None or clock_delay is None: + return None + own = ready - clock_delay + max((o.latency for o in shape.outputs), default=0.0) + if own <= 0: + return 0 + video = next((ref for ref in node.inputs if ref_type(self.g, ref) == "video"), None) + rate = paths.rate(video) if video is not None else None + return None if rate is None else math.ceil(own * rate) + def _node_delays(self, names: Sequence[str]) -> dict[str, int | None]: """Each node's delay from this process's own inputs, over its longest path.""" inside = set(names) @@ -3389,6 +3685,55 @@ def _anchor(self, ref: FrameRef) -> str: return slot ref = node.inputs[0] + def _check_node_accepts(self) -> None: + """Refuse a node handed another node's output in a format it does not take. + + An edge an ffmpeg writes is conformed to the port it feeds. Between + two nodes nothing converts: the reader gets what the writer settled + on, its own format or the one it follows, so a port naming the + formats it takes is held to them here. + """ + for name, shape in self.node_shapes.items(): + node = self.g.nodes[name] + for read, port_name in zip(node.inputs, node.ports): + port = shape.input(port_name) + source = self._past_splits(read) + producer = _ref_node(source) + if ( + port is None + or port.accepts.wants == "timing" + or producer is None + or producer not in self.node_shapes + ): + continue + written = self._node_output_wire(producer, _ref_pad(source), source) + taken, made = _accepted(port.accepts, written) + if made is None or not taken or made in taken: + continue + pad = _ref_pad(source) + outputs = self.node_shapes[producer].outputs + output = outputs[pad].name if pad < len(outputs) else str(pad) + writer = _bindings([self.g.nodes[producer].filter])[0].name + raise FfrwdError( + ErrorCode.UNSUPPORTED_SQL, + f"the module '{node.filter}' takes {', '.join(taken)} on " + f"'{port_name}', and the '{output}' output of {writer} hands " + f"it {made}", + hint="nothing converts between two nodes, and an ffmpeg filter " + f"between them does: {_converted(written, taken[0])}", + ) + + def _past_splits(self, ref: FrameRef) -> FrameRef: + """`ref`, or what the splits in front of it copy.""" + seen: set[str] = set() + while (producer := _ref_node(ref)) is not None and producer not in seen: + node = self.g.nodes.get(producer) + if node is None or node.filter not in SPLIT_FILTERS: + break + seen.add(producer) + ref = node.inputs[0] + return ref + def _check_lockstep(self) -> None: """Refuse a multi-input module whose inputs do not share a timeline. @@ -3410,12 +3755,11 @@ def _check_lockstep(self) -> None: ): continue node = self.g.nodes[name] - if len(node.inputs) < 2: + refs = self._lockstep_refs(name) + if len(refs) < 2: continue - first = self._anchor(node.inputs[0]) - offender = next( - (ref for ref in node.inputs[1:] if self._anchor(ref) != first), None - ) + first = self._anchor(refs[0]) + offender = next((ref for ref in refs[1:] if self._anchor(ref) != first), None) if offender is None: continue raise FfrwdError( @@ -3427,6 +3771,28 @@ def _check_lockstep(self) -> None: "stream, through modules that declare one frame out per frame in", ) + def _lockstep_refs(self, name: str) -> list[FrameRef]: + """The streams of node `name` that pair with its clock frame for frame. + + Every input of an older module. A node pairs only its clock and its + lockstep inputs of the clock's own kind that way: sound against a + picture is cut to each tick by time, and held, interval and arrival + inputs pair by time as well. + """ + node = self.g.nodes[name] + shape = self.node_shapes.get(name) + if shape is None: + return list(node.inputs) + clock = shape.clock_input + if clock is None: + return [] + same = { + port.name + for port in shape.inputs + if port.pairing.kind == "lockstep" and port.kind == clock.kind + } + return [ref for port, ref in zip(node.ports, node.inputs) if port in same] + def _check_rows_reach(self) -> None: """That the rows a module reads can reach it. @@ -3466,6 +3832,7 @@ def _check_rows_reach(self) -> None: def run(self) -> ProcessPlan: self._check_lockstep() + self._check_node_accepts() for members in self._regions(): # The ENTRY is the first node reading a stream: a rows module # reads none, so it is never one however early it sits. @@ -3504,6 +3871,7 @@ def run(self) -> ProcessPlan: color=self._codec_color(members), rows_in=self._region_rows_in(members), pads=self._region_pad_meta(members), + node_network=any(name in self.node_shapes for name in members), ) self.sidecars.append(sidecar) self.members[sidecar.id] = list(members) @@ -3549,9 +3917,14 @@ def run(self) -> ProcessPlan: demands.extend((process.id, ref, depth) for ref in self._consumed(process)) for sidecar in self.sidecars: + members = self.members[sidecar.id] + # A node region reads a source stream wherever a member does, so + # it reads every one at the region's own depth, and one feeder + # hands it all of them. + lowest = min(self.depth[member] for member in members) demands.extend( - (sidecar.id, ref, self.depth[reader]) - for ref, reader in self._region_reads(self.members[sidecar.id]) + (sidecar.id, ref, lowest if sidecar.node_network else self.depth[reader]) + for ref, reader in self._region_reads(members) ) while demands: @@ -3568,6 +3941,9 @@ def run(self) -> ProcessPlan: and producer not in self.g.packet_filters and producer not in self.g.data_filters and producer not in self.g.encoders + and not ( + producer in self.node_shapes and ref_type(self.g, ref) == "data" + ) ): # A packet sink or filter consumes the encoder's output, # and a module region emits decoded frames: an encoding @@ -3624,6 +4000,7 @@ def run(self) -> ProcessPlan: self._redirect_live_reads() self._add_rows_edges() self._add_rows_documents() + self._bundle_node_edges() self._bound_edges() self._mark_live_edges() self._check_handed_once() @@ -3634,11 +4011,47 @@ def run(self) -> ProcessPlan: processes=tuple(processes), edges=(*self.edges, *self.rows, *self.documents, *self._feeder_edges()), laterals=tuple( - replace(lateral, writer=self.feeding[feeder_path(lateral.tap)]) + replace(lateral, writer=self._tap_writer(lateral)) for lateral in self.g.laterals ), ) + def _tap_writer(self, lateral: Lateral) -> str: + """The process writing a run-time lateral's messages for the host: the + node region whose rows document is its tap, else the ffmpeg copying + them to its port.""" + if not lateral.pipe: + return self.feeding[feeder_path(lateral.tap)] + tap = f"{TAP_DOCUMENT}{lateral.tap}" + return next( + sidecar.id + for sidecar in self.sidecars + for document in sidecar.rows + if document.sink.path == tap + ) + + def _bundle_node_edges(self) -> None: + """Every stream between a node region and one other process, in one NUT. + + A node reads and writes its kinds side by side, so the streams one + process hands another travel interleaved on one pipe, and no stream of + it can wait on another pipe. + """ + # A sink's pads stay one input apiece, each with what it says of its row. + nodes = { + sidecar.id + for sidecar in self.sidecars + if sidecar.node_network and not sidecar.packet_sink + } + if not nodes: + return + sinks = {sidecar.id for sidecar in self.sidecars if sidecar.packet_sink} + for index, edge in enumerate(self.edges): + if edge.target in sinks: + continue + if edge.source in nodes or edge.target in nodes: + self.edges[index] = replace(edge, nut=f"{edge.source}>{edge.target}") + def _add_data_read(self, source: str, target: str, ref: FrameRef, depth: int) -> None: """One more reader of the data stream `ref`, which sidecar `source` writes. @@ -3649,6 +4062,13 @@ def _add_data_read(self, source: str, target: str, ref: FrameRef, depth: int) -> copy costs next to nothing, where calling the module again would run it twice. """ + if any(sidecar.id == source and sidecar.node_network for sidecar in self.sidecars): + # A node network writes a label to as many outputs as read it. + if not any( + (e.source, e.target, e.ref) == (source, target, ref) for e in self.edges + ): + self._add_edge(source, target, ref) + return relay = self.relays.get((source, ref)) if relay is not None: if not any( @@ -3691,7 +4111,7 @@ def _feeder_edges(self) -> list[FeederEdge]: if path in self.g.feeders: connections.append((source, feeder_port(path), self.g.feeders[path])) for lateral in self.g.laterals: - source = self.feeding[feeder_path(lateral.tap)] + source = self._tap_writer(lateral) connections.extend( (source, connection.port, connection.calls) for connection in lateral.connections @@ -3836,11 +4256,15 @@ def _region_rows( region, and each rows-bearing node is a document of its own. """ slots = self._module_slots(members, bindings) - return tuple( - RowsDocument(sink=self.g.rows_sinks[name], node=name, source=slots.get(name)) - for name in members - if name in self.g.rows_sinks - ) + found: list[RowsDocument] = [] + for name in members: + pads = [name] + [f"{name}:{pad}" for pad in range(len(self.g.nodes[name].outputs))] + for ref in pads: + if ref in self.g.rows_sinks: + found.append( + RowsDocument(sink=self.g.rows_sinks[ref], node=ref, source=slots.get(name)) + ) + return tuple(found) def _module_slots( self, members: Sequence[str], bindings: Sequence[ModuleBinding] @@ -3977,15 +4401,41 @@ def _wire_reader(self, target: str, reader: str | None) -> str | None: named = [ (name, wire) for name in self._behind_split(target, reader) - if (wire := ( - self.pix_fmts.get(self.g.nodes[name].filter), - self.audio_wires.get(self.g.nodes[name].filter), - )) != (None, None) + if (wire := self._named_wire(name, reader)) is not None ] if not named or any(wire != named[0][1] for _, wire in named): return reader return named[0][0] + def _named_wire(self, name: str, split: str | None) -> object: + """The wire module `name` names for what it reads off `split`, or None. + + A node's is what its port accepts; an older module's, the pixel + format and pcm it declared. + """ + node = self.g.nodes[name] + if name in self.node_shapes and split is not None: + ref = self.g.nodes[split].inputs[0] + position = self._read_position(node, ref) + if position is None or self._reads_timing(name, ref): + return None + return self._node_input_wire(name, node.ports[position], ref) + wire = (self.pix_fmts.get(node.filter), self.audio_wires.get(node.filter)) + return None if wire == (None, None) else wire + + def _read_position(self, node: Node, ref: FrameRef) -> int | None: + """Where `node` reads `ref`: itself, or a pad of a split of it.""" + for position, read in enumerate(node.inputs): + producer = _ref_node(read) + if read == ref or ( + producer is not None + and producer in self.g.nodes + and self.g.nodes[producer].filter in SPLIT_FILTERS + and self.g.nodes[producer].inputs[0] == ref + ): + return position + return None + def _carries_annotations(self, ref: FrameRef, consumer: str | None) -> bool: """Whether this edge's frames travel with the producer's rows. @@ -4101,6 +4551,9 @@ def _format(self, ref: FrameRef, target: str | None = None) -> StreamFormat: # Messages cross as they are, whoever wrote them and whoever reads # them: copied into NUT, one packet each. return DataFormat() + node_wire = self._node_wire(ref, target) + if node_wire is not None: + return node_wire meta = self._origin_meta(ref) producer = _ref_node(ref) if producer is not None and ( @@ -4193,6 +4646,225 @@ def _format(self, ref: FrameRef, target: str | None = None) -> StreamFormat: timebase=_timebase(meta.fps) if meta else None, ) + def _node_wire(self, ref: FrameRef, target: str | None) -> StreamFormat | None: + """What an edge carries where a node writes it or reads it, else None. + + A node's output says its own format, or that it is an input's or the + clock input's; a node's input says what it accepts. Size and time base + are the stream's own, as on every other edge. + """ + if target is not None and ( + target in self.g.packet_filters or target in self.g.packet_sinks + ): + return None # its destination settled what encodes for it + producer = _ref_node(ref) + if producer is not None and producer in self.node_shapes: + return self._node_output_wire(producer, _ref_pad(ref), ref) + if target is not None and target in self.node_shapes: + node = self.g.nodes[target] + position = self._read_position(node, ref) + if position is not None: + return self._node_input_wire(target, node.ports[position], ref) + return None + + def _node_input_wire(self, name: str, port_name: str, ref: FrameRef) -> StreamFormat: + """What node `name` takes on its port `port_name`.""" + port = self.node_shapes[name].input(port_name) + meta = self._origin_meta(ref) + accepts = port.accepts if port is not None else None + if port is not None and port.kind == "packets": + return self._node_packets_wire(name, port_name, ref) + if accepts is not None and accepts.wants == "timing": + return self._timing_wire(ref) + if ref_type(self.g, ref) == "audio": + formats = accepts.sample_formats if accepts is not None else () + sample = next((f for f in formats if f in WIRE_SAMPLE_FMTS), None) + if formats and sample is None: + raise FfrwdError( + ErrorCode.UNSUPPORTED_SQL, + f"the module '{self.g.nodes[name].filter}' takes " + f"{', '.join(formats)} on '{port_name}', and an edge carries " + f"{', '.join(WIRE_SAMPLE_FMTS)}", + hint="rebuild the module accepting one of those sample formats", + ) + return AudioFormat( + rate=meta.sample_rate if meta else None, + channels=meta.channels if meta else None, + codec=SAMPLE_FMT_CODECS[sample or WIRE_SAMPLE_FMTS[0]], + required_rate=accepts.sample_rates[0] if accepts and accepts.sample_rates else None, + required_channels=( + accepts.channel_counts[0] if accepts and accepts.channel_counts else None + ), + ) + formats = accepts.pixel_formats if accepts is not None else () + pix_fmt = next((f for f in formats if f in WIRE_PIX_FMTS), None) + if formats and pix_fmt is None: + raise FfrwdError( + ErrorCode.UNSUPPORTED_SQL, + f"the module '{self.g.nodes[name].filter}' takes {', '.join(formats)} " + f"on '{port_name}', and an edge carries {', '.join(WIRE_PIX_FMTS)}", + hint="rebuild the module accepting one of those pixel formats", + ) + size = self._picture_size(ref) + return VideoFormat( + pix_fmt=pix_fmt or DEFAULT_PIX_FMT, + width=size[0] if size else None, + height=size[1] if size else None, + timebase=_timebase(meta.fps) if meta else None, + ) + + def _timing_wire(self, ref: FrameRef) -> StreamFormat: + """A stream a node reads for its frames' times alone: in the format it + already has, converted to nothing and conformed to nothing.""" + meta = self._origin_meta(ref) + if ref_type(self.g, ref) == "audio": + return AudioFormat( + rate=meta.sample_rate if meta else None, + channels=meta.channels if meta else None, + codec=SAMPLE_FMT_CODECS[WIRE_SAMPLE_FMTS[0]], + ) + producer = _ref_node(ref) + own = self._carried_pix_fmt(ref) + pix_fmt = ( + self._pix_fmt(ref, None) + if producer is not None and self.external.get(producer, False) + else own + if own in WIRE_PIX_FMTS + else DEFAULT_PIX_FMT + ) + size = self._picture_size(ref) + return VideoFormat( + pix_fmt=pix_fmt, + width=size[0] if size else None, + height=size[1] if size else None, + timebase=_timebase(meta.fps) if meta else None, + ) + + def _carried_pix_fmt(self, ref: FrameRef) -> str | None: + """The pixel format the pictures `ref` carries already have: the one the + nearest ``format`` on their way from the input names, else the input's + own. None past a module, or a ``format`` naming several.""" + seen: set[str] = set() + current = ref + while not is_src(current): + name = _ref_node(current) + if name is None or name not in self.g.nodes or name in seen: + return None + seen.add(name) + node = self.g.nodes[name] + if node.filter == "format": + named = str(node.args.get("pix_fmts", "")) + return named if named and "|" not in named else None + if self.external.get(name, False) or not node.inputs: + return None + current = next( + (r for r in node.inputs if ref_type(self.g, r) == "video"), node.inputs[0] + ) + meta = self._origin_meta(current) + return meta.pix_fmt if meta is not None else None + + def _reads_timing(self, name: str, ref: FrameRef) -> bool: + """Whether node `name` reads `ref`, or a split's copy of it, for its + frames' times alone.""" + shape = self.node_shapes.get(name) + if shape is None: + return False + node = self.g.nodes[name] + position = self._read_position(node, ref) + port = shape.input(node.ports[position]) if position is not None else None + return port is not None and port.accepts.wants == "timing" + + def _node_packets_wire(self, name: str, port_name: str, ref: FrameRef) -> StreamFormat: + """A stream a node reads as coded packets: copied as it was coded. + + The module names the codecs it takes; a stream coded in another, or + not coded at all, is one this edge cannot hand it. + """ + port = self.node_shapes[name].input(port_name) + codecs = port.accepts.codecs if port is not None else () + producer = _ref_node(ref) + meta = self._origin_meta(ref) + coded = producer is None or ( + producer in self.node_shapes + and self.node_shapes[producer].outputs[_ref_pad(ref)].kind == "packets" + ) + codec = meta.codec if meta is not None and producer is None else None + if not coded or (codecs and codec is not None and codec not in codecs): + raise FfrwdError( + ErrorCode.UNSUPPORTED_SQL, + f"the module '{self.g.nodes[name].filter}' reads coded packets on " + f"'{port_name}', and {_named_ref(ref)} is " + + (f"coded as {codec}" if coded else "not coded"), + hint="hand it an input's own stream in a codec it takes" + + (f" ({', '.join(codecs)})" if codecs else ""), + ) + if ref_type(self.g, ref) == "audio": + return AudioFormat( + rate=meta.sample_rate if meta else None, + channels=meta.channels if meta else None, + codec=COPY_CODEC, + ) + return VideoFormat( + width=meta.width if meta else None, + height=meta.height if meta else None, + timebase=_timebase(meta.fps) if meta else None, + codec=COPY_CODEC, + ) + + def _node_output_wire(self, name: str, pad: int, ref: FrameRef) -> StreamFormat: + """What node `name` writes on its output `pad`.""" + shape = self.node_shapes[name] + node = self.g.nodes[name] + output = shape.outputs[pad] if pad < len(shape.outputs) else None + found = output.format if output is not None else None + if output is not None and output.kind == "packets": + # Coded: what the module writes crosses as it is, and is copied. + if node.outputs[pad] == "data": + return DataFormat() + if node.outputs[pad] == "audio": + return AudioFormat( + rate=found.sample_rate if found is not None else None, + channels=found.channels if found is not None else None, + codec=COPY_CODEC, + ) + return VideoFormat( + width=found.width if found is not None else None, + height=found.height if found is not None else None, + codec=COPY_CODEC, + ) + follows = ( + found.port + if found is not None and found.kind == "like" + else shape.clock.port + if found is None and shape.clock.kind == "input" + else None + ) + inherited: StreamFormat | None = None + if follows is not None and follows in node.ports: + read = node.inputs[node.ports.index(follows)] + inherited = self._format(read, name) + if node.outputs[pad] == "audio": + base = inherited if isinstance(inherited, AudioFormat) else AudioFormat() + sample = found.sample_format if found is not None else None + if found is not None and found.kind == "audio": + return AudioFormat( + rate=found.sample_rate, + channels=found.channels, + codec=SAMPLE_FMT_CODECS.get(sample or "", base.codec), + ) + return replace(base, codec=SAMPLE_FMT_CODECS.get(sample or "", base.codec)) + base_video = inherited if isinstance(inherited, VideoFormat) else VideoFormat() + if found is not None and found.kind == "video": + return VideoFormat( + pix_fmt=found.pixel_format or base_video.pix_fmt, + width=found.width, + height=found.height, + timebase=base_video.timebase, + ) + if found is not None and found.pixel_format: + return replace(base_video, pix_fmt=found.pixel_format) + return base_video + def _pix_fmt(self, ref: FrameRef, target: str | None) -> str: """The pixel format this edge carries. @@ -4312,25 +4984,60 @@ def substitute(ref: FrameRef) -> FrameRef: return keep, resized, substitute + def _pipe_bundles(self, process: _Pending) -> list[list[FrameRef]]: + """`process`'s pipe refs, those riding one NUT together, in pipe order.""" + nut_of = {edge.ref: edge.nut for edge in self.edges if edge.source == process.id} + bundles: list[list[FrameRef]] = [] + by_nut: dict[str, list[FrameRef]] = {} + for ref in process.pipes: + nut = nut_of.get(ref, "") + if not nut: + bundles.append([ref]) + continue + if nut not in by_nut: + by_nut[nut] = [] + bundles.append(by_nut[nut]) + by_nut[nut].append(ref) + return bundles + + def _read_aliases( + self, incoming: Sequence[StreamEdge], taken: set[str] + ) -> tuple[dict[FrameRef, str], dict[FrameRef, str], list[str]]: + """Each piped ref's input alias and the spec it is read under, and the + aliases in ``-i`` order: one per pipe, a NUT's streams counted per kind.""" + read_as: dict[FrameRef, str] = {} + alias_of: dict[FrameRef, str] = {} + order: list[str] = [] + by_pipe: dict[str, str] = {} + counted: dict[str, dict[str, int]] = {} + for edge in incoming: + if edge.ref in read_as: + continue + key = pipe_key(edge) + alias = by_pipe.get(key) + if alias is None: + alias = _unique_alias(edge.nut.replace(">", "_") or edge.ref, taken) + taken.add(alias) + by_pipe[key] = alias + counted[alias] = {} + order.append(alias) + marker = _marker(edge.format) + index = counted[alias].get(marker, 0) + counted[alias][marker] = index + 1 + alias_of[edge.ref] = alias + read_as[edge.ref] = f"src:{alias}:{marker}:{index}" + return read_as, alias_of, order + def _materialize(self, process: _Pending) -> FfmpegProcess: """`process` as a complete graph, its pipes now inputs and sinks.""" kept, resized, substitute = self._shrink_splits(process) incoming = [e for e in self.edges if e.target == process.id] - alias_of: dict[FrameRef, str] = {} - marker_of: dict[FrameRef, str] = {} taken = set(self.g.sources) - for edge in incoming: - if edge.ref in alias_of: - continue - alias = _unique_alias(edge.ref, taken) - taken.add(alias) - alias_of[edge.ref] = alias - marker_of[edge.ref] = _marker(edge.format) + read_as, alias_of, read_order = self._read_aliases(incoming, taken) def rewrite(ref: FrameRef) -> FrameRef: ref = substitute(ref) - alias = alias_of.get(ref) - return ref if alias is None else f"src:{alias}:{marker_of[ref]}:0" + return read_as.get(ref, ref) nodes: dict[str, Node] = {} for name in kept: @@ -4354,19 +5061,28 @@ def rewrite(ref: FrameRef) -> FrameRef: name=None, metadata={}, ) + for ref in bundle ], path=PIPE, ) - for ref in process.pipes + for bundle in self._pipe_bundles(process) ) paths, sources, trims, options = self._inputs(nodes, sinks) - for edge in incoming: - alias = alias_of[edge.ref] + for alias in read_order: if alias in sources: continue sources[alias] = len(paths) paths.append(PIPE) + # A node read in FROM ends where the query's own `WHERE .t` + # says: the reader takes that long of what it writes and closes. + for alias, name in self.g.node_sources.items(): + bounds = self.g.input_trims.get(alias) + if bounds is None: + continue + for ref, piped in alias_of.items(): + if _ref_node(ref) == name: + trims[piped] = bounds return FfmpegProcess( id=process.id, @@ -4396,26 +5112,25 @@ def _materialize_region(self, sidecar: SidecarProcess) -> SidecarProcess: if edge.target == sidecar.id and edge.ref not in seen: seen.add(edge.ref) incoming.append(edge) + # In the order its pads are read, which is the order the plan's argv + # writes its reads in. + incoming.sort( + key=lambda edge: sidecar.inputs.index(edge.ref) + if edge.ref in sidecar.inputs + else len(sidecar.inputs) + ) outgoing = [e for e in self.edges if e.source == sidecar.id] - alias_of: dict[FrameRef, str] = {} - marker_of: dict[FrameRef, str] = {} taken = set(self.g.sources) - for edge in incoming: - alias = _unique_alias(edge.ref, taken) - taken.add(alias) - alias_of[edge.ref] = alias - marker_of[edge.ref] = _marker(edge.format) + read_as, alias_of, read_order = self._read_aliases(incoming, taken) names = {binding.path: binding.name for binding in sidecar.modules} dissolved: dict[str, FrameRef] = {} def rewrite(ref: FrameRef) -> FrameRef: slot = ref if is_src(ref) else f"{_ref_node(ref)}:{_ref_pad(ref)}" - ref = dissolved.get(slot, ref) - if ref in alias_of: - return f"src:{alias_of[ref]}:{marker_of[ref]}:0" - return ref + ref = self.same_reads.get(ref, dissolved.get(slot, ref)) + return read_as.get(ref, ref) nodes: dict[str, Node] = {} for name in members: # topological: a split precedes its readers @@ -4439,7 +5154,13 @@ def rewrite(ref: FrameRef) -> FrameRef: inputs=[rewrite(ref) for ref in node.inputs], outputs=list(node.outputs), reads_annotations=node.reads_annotations, + ports=list(node.ports), + out_ports=list(node.out_ports), + bound=node.bound, ) + bundles: dict[str, list[StreamEdge]] = {} + for edge in outgoing: + bundles.setdefault(pipe_key(edge), []).append(edge) sinks = [ SinkUnit( outputs=[ @@ -4449,10 +5170,11 @@ def rewrite(ref: FrameRef) -> FrameRef: name=None, metadata={}, ) + for edge in bundle ], path=PIPE, ) - for edge in outgoing + for bundle in bundles.values() ] # The module whose rows leave is a sink of the region too: the network # string has to name the pad they were read off, even though its @@ -4477,8 +5199,8 @@ def rewrite(ref: FrameRef) -> FrameRef: # A SINK MODULE is one too: the network string names its pad, and the # null output that pad is mapped to carries nothing. for name in members: - if name not in self.g.module_sinks: - continue + if name not in self.g.module_sinks or name in self.node_shapes: + continue # a node sink makes no output but the rows it emits sinks.append( SinkUnit( outputs=[ @@ -4494,17 +5216,103 @@ def rewrite(ref: FrameRef) -> FrameRef: ) return replace( sidecar, + listens=self._region_listens(members, names), + colors=self._region_colors(incoming, alias_of, read_order) + if sidecar.node_network + else (), + tags=self._region_tags(incoming, alias_of, read_order) + if sidecar.node_network + else (), reads_rows=any(e.annotations for e in self.edges if e.target == sidecar.id), writes_rows=any(e.annotations for e in self.edges if e.source == sidecar.id), rows_modules=self._rows_modules(sidecar, members), graph=Graph( - input_paths=[PIPE] * len(incoming), - sources={alias_of[e.ref]: index for index, e in enumerate(incoming)}, + input_paths=[PIPE] * len(read_order), + sources={alias: index for index, alias in enumerate(read_order)}, nodes=nodes, sinks=sinks, ), ) + def _region_colors( + self, + incoming: Sequence[StreamEdge], + alias_of: Mapping[FrameRef, str], + order: Sequence[str], + ) -> tuple[tuple[tuple[str, str], ...], ...]: + """The colour each ``-i`` of a node network is told, in ``-i`` order. + + NUT writes none, so each input an ffmpeg writes a raw picture on is + told the first one's: what the probe and the filters on the way say + (:func:`stream_colorimetry`), and full-range RGB where the edge + converts it to an RGB format. A field nothing settles is "unknown". + """ + sidecars = {sidecar.id for sidecar in self.sidecars} + modules = {name for name, external in self.external.items() if external} + found: dict[str, tuple[tuple[str, str], ...]] = {} + for edge in incoming: + wire = edge.format + alias = alias_of.get(edge.ref) + if ( + alias is None + or alias in found + or edge.source in sidecars + or not isinstance(wire, VideoFormat) + or wire.codec != RAWVIDEO + ): + continue + said = stream_colorimetry( + self.g, edge.ref, self._origin_meta, wire.pix_fmt, opaque=modules + ) + if wire.pix_fmt.startswith(RGB_PREFIXES): + said.update(_RGB_WIRE_COLOR) + found[alias] = tuple( + (key, said.get(option, _UNKNOWN_COLOR)) for option, key in _PAD_COLOR.items() + ) + if not found: + return () + return tuple(found.get(alias, ()) for alias in order) + + def _region_tags( + self, + incoming: Sequence[StreamEdge], + alias_of: Mapping[FrameRef, str], + order: Sequence[str], + ) -> tuple[tuple[tuple[str, str], ...], ...]: + """The tags the query wrote on each ``-i`` of a node network's streams, + in ``-i`` order: an earlier stream's key wins over a later one's.""" + found: dict[str, dict[str, str]] = {} + for edge in incoming: + alias = alias_of.get(edge.ref) + if alias is None: + continue + for ref in (edge.ref, self._past_splits(edge.ref)): + for key, value in self.g.stream_tags.get(ref, {}).items(): + found.setdefault(alias, {}).setdefault(key, value) + if not found: + return () + return tuple(tuple(found.get(alias, {}).items()) for alias in order) + + def _region_listens( + self, members: Sequence[str], names: Mapping[str, str] + ) -> tuple[tuple[int, str, str], ...]: + """Each input a node of this region holds on a port of its own and the + query bound no stream to: the port its param carries, and the input.""" + found: list[tuple[int, str, str]] = [] + for name in members: + shape = self.node_shapes.get(name) + if shape is None: + continue + node = self.g.nodes[name] + for port in shape.inputs: + hold = port.pairing.hold + if hold is None or hold.port_param is None or port.name in node.ports: + continue + number = node.args.get(hold.port_param) + if isinstance(number, int) and not isinstance(number, bool): + found.append((number, names.get(node.filter, node.filter), port.name)) + return tuple(found) + def _rows_pad(self, name: str) -> str: """The node whose PAD the rows `name` writes were read off. @@ -4513,6 +5321,8 @@ def _rows_pad(self, name: str) -> str: and whose own output is not a pad at all. """ seen: set[str] = set() + if name not in self.g.nodes: + return name # a node's own pad: its rows leave on it while self.g.nodes[name].rows_only and name not in seen: seen.add(name) name = self.g.nodes[name].rows_inputs[0] @@ -4644,6 +5454,8 @@ def check_spellable(plan: ProcessPlan) -> None: A SOURCE MODULE is exempt: its several pads are each their own named pipe by construction, the same way a packet sink's several inputs are. So is a packet filter, and a data filter: each output is a pipe of its own. + So is a node network: the streams it hands one process ride one NUT, and + each process it hands streams to has an output of its own. A pad handed to two processes is refused while the plan is built, where what reads it still has a name (:meth:`_Partitioner._check_handed_once`). @@ -4653,6 +5465,7 @@ def check_spellable(plan: ProcessPlan) -> None: sidecar.packet_source or sidecar.packet_filter or sidecar.data_filter + or sidecar.node_network or len(sidecar.outputs) <= 1 ): continue diff --git a/cli/ffrwd/prompt.py b/cli/ffrwd/prompt.py index f0204a6..e6b37aa 100644 --- a/cli/ffrwd/prompt.py +++ b/cli/ffrwd/prompt.py @@ -1698,6 +1698,13 @@ def _examples() -> str: "two paths meet -- apply it after they meet, or drop it -- or record " "the input to a file first and write the query over the file." ), + ErrorCode.LIVE_LEAD: ( + "A live query feeds a node an input later than the lead the node " + "needs on it; `message` names the node, the input, the lead and how " + "far behind the path feeding it runs. Feed that input from a path " + "with less delay -- fewer windowed nodes on the way, shorter windows " + "-- or give the node a longer lead where it takes one as a parameter." + ), ErrorCode.STARTUP_DEADLOCK: ( "The query splits one input's streams apart and brings them back " "together in a way whose processes would each be waiting on the next " diff --git a/cli/ffrwd/pts.py b/cli/ffrwd/pts.py index 3b01fa9..7ec1235 100644 --- a/cli/ffrwd/pts.py +++ b/cli/ffrwd/pts.py @@ -100,6 +100,9 @@ def rewire(ref: FrameRef, *, protected: bool) -> FrameRef: # Rows carry no timestamps a reset could touch; the edge rides # through naming the same producer. rows_inputs=list(node.rows_inputs), + ports=list(node.ports), + out_ports=list(node.out_ports), + bound=node.bound, ) # A trim/atrim mapped straight to an output file, with no filter in diff --git a/cli/ffrwd/shapes.py b/cli/ffrwd/shapes.py new file mode 100644 index 0000000..6d677dd --- /dev/null +++ b/cli/ffrwd/shapes.py @@ -0,0 +1,791 @@ +"""A node module's shape for one call: its ports, how each pairs, its clock. + +A module exporting ``ffrwd:av@0.19.1``'s ``node`` says what it reads and +writes per call, not once: its ports and formats turn on its params and on +which inputs the call binds. The compiler asks the sidecar for each distinct +call, ``ffrwd-wasm --shape --params --bound ``, +which prints the WIT's ``node-shape`` record as JSON (:class:`NodeShape`). +The bound inputs go as JSON, one :class:`Binding` per input with a +:class:`StreamHint` per stream, so a shape may turn on the clock's rate. +:func:`shape` runs that, and like :func:`ffrwd.wasm.describe` it is a seam: a +lowering test hands over its own and nothing is spawned. + +A shape is a pure function of the module's bytes, the params and the bound +list, hints and all, so :class:`ShapeCache` asks once per distinct triple. + +Also here, since every reader of a shape needs them: the streaming words a +window is said in (:func:`window_words`), and structural row matching +(:func:`row_mismatch`), which compares the JSON schemas two data ports carry. +""" + +from __future__ import annotations + +import hashlib +import json +import subprocess +import tempfile +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field +from fractions import Fraction +from pathlib import Path +from typing import Literal, cast + +from . import binaries +from .errors import ErrorCode, FfrwdError + +__all__ = [ + "Accepts", + "Anchor", + "Binding", + "Clock", + "Hold", + "InputPort", + "Interval", + "NodeShape", + "OutputFormat", + "OutputPort", + "Pairing", + "Shape", + "ShapeCache", + "StreamHint", + "bound_json", + "node_shape", + "row_mismatch", + "shape", + "window_words", +] + +PortKind = Literal["video", "audio", "data", "packets"] +RowsUse = Literal["ignore", "per-frame", "state"] +PairingKind = Literal["lockstep", "hold", "interval", "arrival"] +AnchorKind = Literal["shared-clock", "first-frame", "tagged"] +ClockKind = Literal["input", "rate", "rate-of", "self-clocked"] +FormatKind = Literal["video", "audio", "data", "packets", "like"] +Wants = Literal["all", "keyframes", "first", "timing"] + +_PORT_KINDS: tuple[PortKind, ...] = ("video", "audio", "data", "packets") +_ROWS_USES: tuple[RowsUse, ...] = ("ignore", "per-frame", "state") +_WANTS: tuple[Wants, ...] = ("all", "keyframes", "first", "timing") +_ANCHORS: tuple[AnchorKind, ...] = ("shared-clock", "first-frame", "tagged") + +_SHAPE_FLAG = "--shape" +_PARAMS_FLAG = "--params" +_PARAMS_FROM_FLAG = "--params-from" +_BOUND_FLAG = "--bound" + +# Params longer than this go in a file rather than on the command line, which +# Windows caps at 32,767 characters for everything on it. +PARAMS_INLINE_LIMIT = 4096 + +_UNKNOWN_HINT = "the module may be built against a sidecar this ffrwd does not know" + + +@dataclass(frozen=True) +class Anchor: + """How a hold or interval input's offset is fixed; `tag` names the tag + `tagged` reads.""" + + kind: AnchorKind + tag: str = "" + + +_SHARED_CLOCK = Anchor("shared-clock") + + +@dataclass(frozen=True) +class Hold: + anchor: Anchor + lead: float + linger: float | None = None + timeout: float | None = None + group: str | None = None + port_param: str | None = None + + +@dataclass(frozen=True) +class Interval: + latency: float | None = None + ahead: float = 0.0 + anchor: Anchor = _SHARED_CLOCK + group: str | None = None + + @property + def retimed(self) -> bool: + """Whether the host re-stamps the stream onto the clock: its pts count + from an origin of their own, not the clock's.""" + return self.anchor.kind != "shared-clock" + + +@dataclass(frozen=True) +class Pairing: + kind: PairingKind + hold: Hold | None = None + interval: Interval | None = None + + +@dataclass(frozen=True) +class Accepts: + pixel_formats: tuple[str, ...] = () + sample_formats: tuple[str, ...] = () + sample_rates: tuple[int, ...] = () + channel_counts: tuple[int, ...] = () + codecs: tuple[str, ...] = () + wants: Wants = "all" + like: str | None = None + + +@dataclass(frozen=True) +class InputPort: + name: str + kind: PortKind + required: bool + many: bool + pairing: Pairing + rows: RowsUse + window: int + stride: int + accepts: Accepts + schema: Mapping[str, object] | None = None + + +@dataclass(frozen=True) +class OutputFormat: + """One arm of ``output-format``. + + `video`: width, height, `pixel_format`. `audio`: `sample_rate`, + `channels`, `sample_format`. `data`: `codec`. `packets`: the coded + stream, `codec`, `time_base`, what it carries as `coded` (video, audio or + data) with that kind's own fields, and `extradata` as hex. `like`: + `port`, with `pixel_format` or `sample_format` the field it overrides. + """ + + kind: FormatKind + width: int | None = None + height: int | None = None + pixel_format: str | None = None + sample_rate: int | None = None + channels: int | None = None + sample_format: str | None = None + codec: str | None = None + time_base: tuple[int, int] | None = None + port: str | None = None + coded: Literal["video", "audio", "data"] | None = None + extradata: str = "" + + +@dataclass(frozen=True) +class OutputPort: + name: str + kind: PortKind + format: OutputFormat | None = None + time_base: tuple[int, int] | None = None + latency: float = 0.0 + schema: Mapping[str, object] | None = None + row: int | None = None + + +@dataclass(frozen=True) +class Clock: + """`port` for `input` and `rate-of`, `rate` for `rate`.""" + + kind: ClockKind + port: str = "" + rate: tuple[int, int] | None = None + + +@dataclass(frozen=True) +class NodeShape: + inputs: tuple[InputPort, ...] + outputs: tuple[OutputPort, ...] + clock: Clock + pure: bool = True + one_to_one: bool = False + bounded: bool = True + relation: tuple[Mapping[str, object], ...] = () + # The JSON the sidecar printed, kept for `explain` and the IR. + raw: Mapping[str, object] = field(default_factory=dict, compare=False, repr=False) + + def input(self, name: str) -> InputPort | None: + return next((port for port in self.inputs if port.name == name), None) + + def output(self, name: str) -> OutputPort | None: + return next((port for port in self.outputs if port.name == name), None) + + @property + def clock_input(self) -> InputPort | None: + """The input the clock names, for an input clock; None otherwise.""" + return self.input(self.clock.port) if self.clock.kind == "input" else None + + def to_dict(self) -> dict[str, object]: + return dict(self.raw) + + +@dataclass(frozen=True) +class StreamHint: + """What the compiler knows of one bound stream before the run: a video + stream's frame rate or an audio stream's sample rate, None where nothing + settles it.""" + + rate: Fraction | None = None + + +@dataclass(frozen=True) +class Binding: + """One input a call binds, as `shape` is told it: a hint per stream, in + the order the call names them.""" + + input: str + streams: tuple[StreamHint, ...] = (StreamHint(),) + + +def bound_json(bound: Sequence[Binding]) -> str: + """`bound` as ``--bound`` takes it.""" + return json.dumps( + [ + { + "input": binding.input, + "streams": [ + { + "rate": None + if hint.rate is None + else {"num": hint.rate.numerator, "den": hint.rate.denominator} + } + for hint in binding.streams + ], + } + for binding in bound + ], + separators=(",", ":"), + ) + + +def _reject(message: str, hint: str = _UNKNOWN_HINT) -> FfrwdError: + return FfrwdError(ErrorCode.UNSUPPORTED_SQL, message, hint=hint) + + +# -- reading the JSON ------------------------------------------------------- + + +def _object(value: object, what: str, module: str) -> Mapping[str, object]: + if not isinstance(value, dict): + raise _reject(f"the sidecar's shape of {module} has {what} that is not an object") + return cast(Mapping[str, object], value) + + +def _kind(value: Mapping[str, object], what: str, module: str) -> str: + kind = value.get("kind") + if not isinstance(kind, str): + raise _reject(f"the sidecar's shape of {module} has {what} with no kind") + return kind + + +def _arm(value: Mapping[str, object], kind: str) -> object: + """A variant's payload: nested under its kind's own name, or the object itself.""" + nested = value.get(kind.replace("-", "_")) + if nested is None: + nested = value.get("value") + return value if nested is None else nested + + +def _text(value: object) -> str | None: + return value if isinstance(value, str) else None + + +def _number(value: object) -> float | None: + if isinstance(value, bool) or not isinstance(value, int | float): + return None + return float(value) + + +def _whole(value: object) -> int | None: + if isinstance(value, bool) or not isinstance(value, int): + return None + return value + + +def _texts(value: object) -> tuple[str, ...]: + if not isinstance(value, list): + return () + return tuple(item for item in value if isinstance(item, str)) + + +def _wholes(value: object) -> tuple[int, ...]: + if not isinstance(value, list): + return () + return tuple(found for item in value if (found := _whole(item)) is not None) + + +def _rational(value: object) -> tuple[int, int] | None: + if isinstance(value, dict): + num, den = _whole(value.get("num")), _whole(value.get("den")) + elif isinstance(value, list) and len(value) == 2: + num, den = _whole(value[0]), _whole(value[1]) + else: + return None + if num is None or den is None or den == 0: + return None + return (num, den) + + +def _schema(value: object) -> Mapping[str, object] | None: + """A port's JSON schema: written as a string of JSON, or as the object.""" + if isinstance(value, str): + if not value.strip(): + return None + try: + value = json.loads(value) + except ValueError: + return None + return cast(Mapping[str, object], value) if isinstance(value, dict) else None + + +def _choice(value: object, choices: tuple[str, ...], fallback: str) -> str: + if isinstance(value, str): + spelled = value.replace("_", "-") + if spelled in choices: + return spelled + return fallback + + +def _anchor(value: object, module: str) -> Anchor: + if isinstance(value, str): + kind = _choice(value, ("shared-clock", "first-frame"), "first-frame") + return Anchor(cast(AnchorKind, kind)) + raw = _object(value, "a hold anchor", module) + kind = _choice(_kind(raw, "a hold anchor", module), _ANCHORS, "") + if not kind: + raise _reject(f"the sidecar's shape of {module} names an anchor this ffrwd does not know") + tag = "" + if kind == "tagged": + arm = _arm(raw, kind) + tag = _text(arm) or (_text(raw.get("name")) or _text(raw.get("tag")) or "") + return Anchor(cast(AnchorKind, kind), tag) + + +def _pairing(value: object, module: str, port: str) -> Pairing: + what = f"input '{port}''s pairing" + if isinstance(value, str): + kind = _choice(value, ("lockstep", "arrival"), "") + if not kind: + raise _reject(f"the sidecar's shape of {module} gives {what} no fields") + return Pairing(cast(PairingKind, kind)) + raw = _object(value, what, module) + kind = _choice(_kind(raw, what, module), ("lockstep", "hold", "interval", "arrival"), "") + if kind == "hold": + arm = _object(_arm(raw, kind), what, module) + return Pairing( + "hold", + hold=Hold( + anchor=_anchor(arm.get("anchor"), module), + lead=_number(arm.get("lead")) or 0.0, + linger=_number(arm.get("linger")), + timeout=_number(arm.get("timeout")), + group=_text(arm.get("group")), + port_param=_text(arm.get("port_param")), + ), + ) + if kind == "interval": + arm = _object(_arm(raw, kind), what, module) + anchor = arm.get("anchor") + return Pairing( + "interval", + interval=Interval( + latency=_number(arm.get("latency")), + ahead=_number(arm.get("ahead")) or 0.0, + anchor=_SHARED_CLOCK if anchor is None else _anchor(anchor, module), + group=_text(arm.get("group")), + ), + ) + if kind in ("lockstep", "arrival"): + return Pairing(cast(PairingKind, kind)) + raise _reject( + f"the sidecar's shape of {module} pairs input '{port}' in a way this ffrwd does not know" + ) + + +def _accepts(value: object) -> Accepts: + if not isinstance(value, dict): + return Accepts() + return Accepts( + pixel_formats=_texts(value.get("pixel_formats")), + sample_formats=_texts(value.get("sample_formats")), + sample_rates=_wholes(value.get("sample_rates")), + channel_counts=_wholes(value.get("channel_counts")), + codecs=_texts(value.get("codecs")), + wants=cast(Wants, _choice(value.get("wants"), _WANTS, "all")), + like=_text(value.get("like")), + ) + + +def _port_kind(value: object, module: str, port: str) -> PortKind: + kind = _choice(value, _PORT_KINDS, "") + if not kind: + raise _reject( + f"the sidecar's shape of {module} gives port '{port}' a kind this ffrwd does not know" + ) + return cast(PortKind, kind) + + +def _name(raw: Mapping[str, object], what: str, module: str) -> str: + name = raw.get("name") + if not isinstance(name, str) or not name: + raise _reject(f"the sidecar's shape of {module} has {what} with no name") + return name + + +def _input_port(value: object, module: str) -> InputPort: + raw = _object(value, "an input port", module) + name = _name(raw, "an input port", module) + window = _whole(raw.get("window")) or 1 + stride = _whole(raw.get("stride")) or 1 + return InputPort( + name=name, + kind=_port_kind(raw.get("kind"), module, name), + required=raw.get("required") is True, + many=raw.get("many") is True, + pairing=_pairing(raw.get("pairing"), module, name), + rows=cast(RowsUse, _choice(raw.get("rows"), _ROWS_USES, "ignore")), + window=max(window, 1), + stride=max(stride, 1), + accepts=_accepts(raw.get("accepts")), + schema=_schema(raw.get("schema")), + ) + + +def _output_format(value: object, module: str, port: str) -> OutputFormat | None: + if value is None: + return None + what = f"output '{port}''s format" + raw = _object(value, what, module) + kind = _choice(_kind(raw, what, module), ("video", "audio", "data", "packets", "like"), "") + arm = _arm(raw, kind) + if kind == "data": + codec = _text(arm) if not isinstance(arm, dict) else _text(arm.get("codec")) + return OutputFormat("data", codec=codec or "json") + body = _object(arm, what, module) + if kind == "video": + return OutputFormat( + "video", + width=_whole(body.get("width")), + height=_whole(body.get("height")), + pixel_format=_text(body.get("pix_fmt")) or _text(body.get("pixel_format")), + ) + if kind == "audio": + return OutputFormat( + "audio", + sample_rate=_whole(body.get("sample_rate")), + channels=_whole(body.get("channels")), + sample_format=_text(body.get("sample_fmt")) or _text(body.get("sample_format")), + ) + if kind == "packets": + coded = body.get("format") + carried = coded if isinstance(coded, dict) else {} + coded_kind = _choice(carried.get("kind"), ("video", "audio", "data"), "") + return OutputFormat( + "packets", + codec=_text(body.get("codec")), + time_base=_rational(body.get("time_base")), + coded=cast(Literal["video", "audio", "data"], coded_kind) if coded_kind else None, + width=_whole(carried.get("width")), + height=_whole(carried.get("height")), + sample_rate=_whole(carried.get("sample_rate")), + channels=_whole(carried.get("channels")), + extradata=_text(body.get("extradata")) or "", + ) + if kind == "like": + return OutputFormat( + "like", + port=_text(body.get("port")), + pixel_format=_text(body.get("pixel_format")), + sample_format=_text(body.get("sample_format")), + ) + raise _reject( + f"the sidecar's shape of {module} gives output '{port}' a format this ffrwd does not know" + ) + + +def _output_port(value: object, module: str) -> OutputPort: + raw = _object(value, "an output port", module) + name = _name(raw, "an output port", module) + return OutputPort( + name=name, + kind=_port_kind(raw.get("kind"), module, name), + format=_output_format(raw.get("format"), module, name), + time_base=_rational(raw.get("time_base")), + latency=max(_number(raw.get("latency")) or 0.0, 0.0), + schema=_schema(raw.get("schema")), + row=_whole(raw.get("row")), + ) + + +def _clock(value: object, module: str) -> Clock: + if isinstance(value, str): + if value.replace("_", "-") == "self-clocked": + return Clock("self-clocked") + raise _reject(f"the sidecar's shape of {module} names a clock with no fields") + raw = _object(value, "a clock", module) + kind = _choice(_kind(raw, "a clock", module), ("input", "rate", "rate-of", "self-clocked"), "") + arm = _arm(raw, kind) + if kind in ("input", "rate-of"): + port = _text(arm) if not isinstance(arm, dict) else _text(arm.get("port")) + if not port: + raise _reject(f"the sidecar's shape of {module} names a clock with no input") + return Clock(cast(ClockKind, kind), port=port) + if kind == "rate": + nested = isinstance(arm, dict) and "num" not in arm + rate = _rational(arm.get("rate") if nested and isinstance(arm, dict) else arm) + if rate is None or rate[0] <= 0 or rate[1] <= 0: + raise _reject(f"the sidecar's shape of {module} names a rate clock with no rate") + return Clock("rate", rate=rate) + if kind == "self-clocked": + return Clock("self-clocked") + raise _reject(f"the sidecar's shape of {module} names a clock this ffrwd does not know") + + +def _relation(value: object) -> tuple[Mapping[str, object], ...]: + if not isinstance(value, list): + return () + rows: list[Mapping[str, object]] = [] + for item in value: + found = _schema(item) + rows.append(found if found is not None else {}) + return tuple(rows) + + +def node_shape(module: str, payload: object) -> NodeShape: + """One ``--shape`` document as a :class:`NodeShape`, or a rejection naming `module`.""" + raw = _object(payload, "a document", module) + inputs = raw.get("inputs") + outputs = raw.get("outputs") + if not isinstance(inputs, list) or not isinstance(outputs, list): + raise _reject(f"the sidecar's shape of {module} names no inputs or no outputs") + return NodeShape( + inputs=tuple(_input_port(port, module) for port in inputs), + outputs=tuple(_output_port(port, module) for port in outputs), + clock=_clock(raw.get("clock"), module), + pure=raw.get("pure") is not False, + one_to_one=raw.get("one_to_one") is True, + bounded=raw.get("bounded") is not False, + relation=_relation(raw.get("relation")), + raw=dict(raw), + ) + + +# -- asking the sidecar ----------------------------------------------------- + + +# Runs one module's `shape` for one call: :func:`shape` is the real one, and a +# lowering test passes its own. `grants` are the sidecar flags a module's own +# imports need to answer at all (a source reading the network for its +# outputs), ahead of the flag that dispatches the call. +Shape = Callable[[str, str, Sequence[Binding], Sequence[str]], NodeShape] + + +def shape( + module: str, params: str, bound: Sequence[Binding], grants: Sequence[str] = () +) -> NodeShape: + """Ask the sidecar for the shape of `module` under `params` with `bound` bound. + + Raises ``FfrwdError`` and nothing else, unanchored: the caller anchors it + on the call that named the module. + """ + from .wasm import ( # deferred: wasm imports processes, which reads shapes + INSTALL_HINT, + _first_line, + timeout_seconds, + ) + + sidecar = binaries.ffrwd_wasm_path() + if sidecar is None: + raise _reject( + f"the ffrwd-wasm sidecar is not installed, and the shape of '{module}' " + "needs it to read the module", + hint=INSTALL_HINT, + ) + argv = [sidecar, *grants, _SHAPE_FLAG, module] + if bound: + argv += [_BOUND_FLAG, bound_json(bound)] + budget = timeout_seconds() + with tempfile.TemporaryDirectory(prefix="ffrwd-shape-") as scratch: + if len(params) > PARAMS_INLINE_LIMIT: + written = Path(scratch) / "params.json" + written.write_text(params, encoding="utf-8") + argv += [_PARAMS_FROM_FLAG, str(written)] + elif params and params != "{}": + argv += [_PARAMS_FLAG, params] + done = _run_shape(sidecar, module, argv, budget) + if done.returncode != 0: + raise _reject( + f"the module '{module}' refused the shape for these params: " + f"{_first_line(done.stderr)}", + hint="check the arguments match what the module declares", + ) + try: + payload = json.loads(done.stdout) + except ValueError as err: + raise _reject( + f"the ffrwd-wasm sidecar's shape of {module} is not JSON", + hint="the sidecar on PATH may be a different version than this ffrwd", + ) from err + return node_shape(module, payload) + + +def _run_shape( + sidecar: str, module: str, argv: list[str], budget: float +) -> subprocess.CompletedProcess[str]: + """One ``--shape`` run, refused by name when it cannot be run at all.""" + from .wasm import INSTALL_HINT, _budget_hint # deferred: wasm imports processes + + try: + return subprocess.run( + argv, + capture_output=True, + encoding="utf-8", + errors="replace", + timeout=budget, + check=False, + ) + except (OSError, ValueError) as err: + raise _reject( + f"could not run the ffrwd-wasm sidecar at {sidecar}: " + f"{getattr(err, 'strerror', None) or err}", + hint=INSTALL_HINT, + ) from err + except subprocess.TimeoutExpired as err: + raise _reject( + f"the ffrwd-wasm sidecar did not shape {module} within {budget:g}s", + hint=_budget_hint(budget), + ) from err + + +def _module_hash(module: str) -> str: + """The module file's own digest, or its path where the file cannot be read. + + A path that names nothing readable still shapes the same within one + compile, which is all a test's fake module needs. + """ + try: + return hashlib.sha256(Path(module).read_bytes()).hexdigest() + except OSError: + return f"path:{module}" + + +class ShapeCache: + """One :data:`Shape` asked once per (module bytes, params, bound list).""" + + def __init__(self, ask: Shape = shape) -> None: + self._ask = ask + self._hashes: dict[str, str] = {} + self._shapes: dict[tuple[str, str, tuple[Binding, ...]], NodeShape] = {} + + def __call__( + self, module: str, params: str, bound: Sequence[Binding], grants: Sequence[str] = () + ) -> NodeShape: + digest = self._hashes.get(module) + if digest is None: + digest = self._hashes[module] = _module_hash(module) + key = (digest, params, tuple(bound)) + found = self._shapes.get(key) + if found is None: + found = self._shapes[key] = self._ask(module, params, bound, grants) + return found + + +# -- windows, in streaming words ------------------------------------------- + + +def window_words(port: InputPort, rate: Fraction | None = None) -> str: + """The clock input's window as streaming SQL says it. + + `rate` is items per second (frames, or samples for audio); with it the + window is said in seconds, without it in items. + """ + if port.window <= 1: + return "per-frame" + + def span(items: int) -> str: + if rate is None or rate <= 0: + unit = "samples" if port.kind == "audio" else "frames" + return f"{items} {unit}" + return f"{_seconds(Fraction(items) / rate)} s" + + if port.stride >= port.window: + return f"tumbling {span(port.window)}" + if port.stride == 1 and port.kind != "audio": + return f"sliding {span(port.window)}" + return f"hopping {span(port.window)} every {span(port.stride)}" + + +def _seconds(value: Fraction | float) -> str: + """Seconds as a reader writes them: 2, 0.5, 0.033.""" + rounded = round(float(value), 3) + return f"{rounded:g}" + + +# -- structural row matching ------------------------------------------------ + + +def _types(schema: Mapping[str, object]) -> frozenset[str] | None: + written = schema.get("type") + if isinstance(written, str): + return frozenset({written}) + if isinstance(written, list): + return frozenset(item for item in written if isinstance(item, str)) + return None + + +def _covers(reader: frozenset[str], producer: frozenset[str]) -> bool: + """Whether every type the producer may write is one the reader takes. + + JSON Schema's integer is a number, so a reader of numbers takes a + producer of integers; the reverse is not so. + """ + return all(t in reader or (t == "integer" and "number" in reader) for t in producer) + + +def _type_text(types: frozenset[str] | None) -> str: + if not types: + return "no type" + return " or ".join(sorted(types)) + + +def row_mismatch( + reader: Mapping[str, object], producer: Mapping[str, object] +) -> tuple[str, str, str] | None: + """The first field `reader` names that `producer` lacks or types otherwise. + + ``(field, what the reader wants, what the producer writes)``, or None + where every field the reader names is there with a type it takes. Fields + the producer writes beyond those pass. Nested objects and arrays are + compared the same way, a nested field named by its dotted path. + """ + wanted = reader.get("properties") + if not isinstance(wanted, dict): + return None + written = producer.get("properties") + given: Mapping[str, object] = written if isinstance(written, dict) else {} + for name, want in wanted.items(): + if not isinstance(want, dict): + continue + have = given.get(name) + want_types = _types(want) + if not isinstance(have, dict): + return (name, _type_text(want_types), "nothing") + have_types = _types(have) + if want_types is not None and (have_types is None or not _covers(want_types, have_types)): + return (name, _type_text(want_types), _type_text(have_types)) + nested = row_mismatch(want, have) + if nested is not None: + return (f"{name}.{nested[0]}", nested[1], nested[2]) + items_want, items_have = want.get("items"), have.get("items") + if isinstance(items_want, dict) and isinstance(items_have, dict): + item_types_want, item_types_have = _types(items_want), _types(items_have) + if item_types_want is not None and ( + item_types_have is None or not _covers(item_types_want, item_types_have) + ): + return ( + f"{name}[]", + _type_text(item_types_want), + _type_text(item_types_have), + ) + return None diff --git a/cli/ffrwd/split.py b/cli/ffrwd/split.py index 99291f5..e04849d 100644 --- a/cli/ffrwd/split.py +++ b/cli/ffrwd/split.py @@ -174,6 +174,9 @@ def rewire(ref: FrameRef) -> FrameRef: # A rows edge is not a pad and never fans out: nothing to split, # so it rides through naming the same producer it always did. rows_inputs=list(node.rows_inputs), + ports=list(node.ports), + out_ports=list(node.out_ports), + bound=node.bound, ) # Units in order, each unit's outputs in list order: pad assignment is diff --git a/cli/ffrwd/startup.py b/cli/ffrwd/startup.py index fa8ff3e..c92d68b 100644 --- a/cli/ffrwd/startup.py +++ b/cli/ffrwd/startup.py @@ -66,6 +66,7 @@ RowsEdge, SidecarProcess, StreamEdge, + once_per_pipe, ) __all__ = ["Milestone", "arrange", "check", "relation", "stalled"] @@ -121,6 +122,13 @@ def relation(plan: ProcessPlan) -> dict[Milestone, tuple[Milestone, ...]]: concurrent = { process.id for process in plan.processes if isinstance(process, SidecarProcess) } + # A node network writes each output from a thread and a queue of its own, + # so no output's frames wait on another's reader. + unqueued = { + process.id + for process in plan.processes + if isinstance(process, SidecarProcess) and process.node_network + } waits: dict[Milestone, tuple[Milestone, ...]] = {} for pid in ids: @@ -142,12 +150,13 @@ def relation(plan: ProcessPlan) -> dict[Milestone, tuple[Milestone, ...]]: for index, edge in enumerate(outputs[pid]): waits[("head", pid, index)] = (("run", pid, 0), *fed) after: list[Milestone] = [("head", pid, index)] - if index: + if index and pid not in unqueued: after.append(("write", pid, index - 1)) if held is not None: after.append(held) waits[("write", pid, index)] = tuple(after) - held = None if _carried(plan, edge) else ("run", edge.target, 0) + carried = _carried(plan, edge) or pid in unqueued + held = None if carried else ("run", edge.target, 0) return waits @@ -379,11 +388,12 @@ def _reads(plan: ProcessPlan, pid: str) -> list[StreamEdge]: def _writes(plan: ProcessPlan, pid: str) -> list[StreamEdge]: - return [e for e in plan.stream_edges if e.source == pid] + return once_per_pipe([e for e in plan.stream_edges if e.source == pid]) def _once_per_ref(edges: Sequence[StreamEdge]) -> list[StreamEdge]: - """One edge per ref: two edges of one ref share a single ``-i``.""" + """One edge per ref: two edges of one ref share a single ``-i``, and the + streams of one NUT one pipe.""" seen: set[str] = set() kept: list[StreamEdge] = [] for edge in edges: @@ -391,7 +401,7 @@ def _once_per_ref(edges: Sequence[StreamEdge]) -> list[StreamEdge]: continue seen.add(edge.ref) kept.append(edge) - return kept + return once_per_pipe(kept) # ---------------------------------------------------------------- reading a plan @@ -408,7 +418,7 @@ def _pipe_inputs(plan: ProcessPlan, pid: str) -> list[Edge]: def _pipe_outputs(plan: ProcessPlan, pid: str) -> list[Edge]: """What `pid` writes, in the order its outputs are rendered.""" - frames: list[Edge] = [e for e in plan.stream_edges if e.source == pid] + frames: list[Edge] = list(once_per_pipe([e for e in plan.stream_edges if e.source == pid])) rows: list[Edge] = [e for e in plan.rows_edges if e.source == pid] return frames + rows @@ -433,7 +443,7 @@ def _key(edge: Edge) -> Wire: if isinstance(edge, RowsEdge): return ("rows", edge.source, edge.target, edge.alias) if isinstance(edge, StreamEdge): - return ("stream", edge.source, edge.target, edge.ref) + return ("stream", edge.source, edge.target, edge.nut or edge.ref) return ("file", edge.source, edge.target) diff --git a/cli/ffrwd/timing.py b/cli/ffrwd/timing.py new file mode 100644 index 0000000..6538b7d --- /dev/null +++ b/cli/ffrwd/timing.py @@ -0,0 +1,639 @@ +"""How late each node module's outputs run behind the source, and why. + +Every node declares the window its clock input reads and how late each output +may leave; every input says how it pairs with the clock. Summed along the +paths of a graph, those say how far behind its source each stream a query +writes runs (:func:`timing`): + +- a source's stream runs at its source's time, 0; +- a node is ready for a tick once every input it waits for has arrived: + the clock, each lockstep input, and each interval input with its `ahead`, + an interval input waiting no longer than its own bound past the clock. + A held input and one delivered on arrival hold nothing up, and an + interval input the host re-times onto the clock (anchored `first-frame` + or `tagged`) waits its bound and no more, since its producer counts from + another origin; +- a node's window is how long its tick takes to fill, a tumbling 2 s + window 2 s; a node's output adds the output's declared latency; +- the host's span reducer adds its `max_span`. + +Where streams that run at different delays meet, in one node or in one file +written, the earlier one waits: :attr:`OutputTiming.holds` says for how long. +A live query cannot give a node an interval input later than the bound the +node set on it, which is :data:`~ffrwd.errors.ErrorCode.LIVE_LEAD` +(:func:`check_live_leads`). +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from fractions import Fraction + +from .errors import ErrorCode, FfrwdError +from .ir import ( + MAX_SPAN, + ROWMERGE, + FrameRef, + Graph, + StreamType, + is_src, + src_parts, +) +from .probe import ProbeResult +from .shapes import InputPort, NodeShape, node_shape, window_words + +__all__ = [ + "HOLD_LIMIT", + "NodeTiming", + "OutputTiming", + "Paths", + "Timing", + "check_live_leads", + "paths_of", + "stream_rate", + "summary", + "timing", +] + +# How many bytes a stream may wait for the one written beside it before the +# compile says so: a picture held behind 30 s of late words is about 2 GB of +# 1080p, and that is worth a line. +HOLD_LIMIT = 512 * 1024 * 1024 + +# Bytes per picture element of a raw picture on an edge, yuv420p's; and of +# one sample of one channel, f32's. +_PICTURE_BYTES = Fraction(3, 2) +_SAMPLE_BYTES = 4 + + +@dataclass(frozen=True) +class Wait: + """One input of a node: how it pairs, and how late what it reads runs. + + `timing` marks an input read for its frames' times alone; `retimed` an + interval input the host re-stamps onto the clock. + """ + + port: str + pairing: str + delay: float | None + bound: float | None = None + timing: bool = False + retimed: bool = False + + +@dataclass(frozen=True) +class NodeTiming: + """One node module: its window in streaming words, and when it is ready. + + `delay` is how far behind the source a tick's outputs leave, before each + output's own latency; None where something on the way has no size. + """ + + node: str + module: str + window: str + waits: tuple[Wait, ...] + delay: float | None + # What the query calls it. + called: str = "" + + def to_dict(self) -> dict[str, object]: + return { + "node": self.node, + "module": self.module, + "called": self.called, + "window": self.window, + "inputs": [ + { + "port": wait.port, + "pairing": wait.pairing, + "delay": wait.delay, + **({"bound": wait.bound} if wait.bound is not None else {}), + **({"wants": "timing"} if wait.timing else {}), + **({"retimed": True} if wait.retimed else {}), + } + for wait in self.waits + ], + "delay": self.delay, + } + + +@dataclass(frozen=True) +class OutputTiming: + """One stream a query writes: how late it runs, how long it waits for + the latest one written beside it, and about how many bytes that wait + holds, None where the stream's size is not known.""" + + ref: FrameRef + delay: float | None + holds: float + held: int | None = None + # Where it is written: the file, the stream's place in it and its kind; + # a rows file names no stream. + path: str = "" + index: int | None = None + kind: str = "" + + def to_dict(self) -> dict[str, object]: + written: dict[str, object] = { + "ref": self.ref, + "path": self.path, + "index": self.index, + "kind": self.kind, + "delay": self.delay, + "holds": self.holds, + } + if self.held: + written["held_bytes"] = self.held + return written + + +@dataclass(frozen=True) +class Timing: + nodes: tuple[NodeTiming, ...] + outputs: tuple[OutputTiming, ...] + + def to_dict(self) -> dict[str, object]: + return { + "nodes": [node.to_dict() for node in self.nodes], + "outputs": [output.to_dict() for output in self.outputs], + } + + +class Paths: + """Delays over one graph, each ref's counted once.""" + + def __init__(self, graph: Graph, probes: Mapping[str, ProbeResult | None]) -> None: + self.graph = graph + self.probes = probes + self.shapes = { + name: node_shape(graph.nodes[name].filter, raw) + for name, raw in graph.node_shapes.items() + if name in graph.nodes + } + self._delays: dict[FrameRef, float | None] = {} + self._ready: dict[str, tuple[float | None, tuple[Wait, ...]]] = {} + + def delay(self, ref: FrameRef) -> float | None: + if ref not in self._delays: + self._delays[ref] = self._count(ref) + return self._delays[ref] + + def _count(self, ref: FrameRef) -> float | None: + if is_src(ref): + return 0.0 + name, _, pad = ref.partition(":") + node = self.graph.nodes.get(name) + if node is None: + return 0.0 + shape = self.shapes.get(name) + if shape is not None: + ready = self.ready(name)[0] + output = shape.outputs[int(pad) if pad else 0] if shape.outputs else None + latency = output.latency if output is not None else 0.0 + return None if ready is None else ready + latency + found: list[float | None] = [self.delay(one) for one in node.inputs] + if any(one is None for one in found): + return None + above = max((one for one in found if one is not None), default=0.0) + span = node.args.get(MAX_SPAN) if node.filter == ROWMERGE else None + if isinstance(span, int | float) and not isinstance(span, bool): + return above + float(span) + return above + + def ready(self, name: str) -> tuple[float | None, tuple[Wait, ...]]: + """When node `name`'s tick has all it waits for, its window filled.""" + if name in self._ready: + return self._ready[name] + node = self.graph.nodes[name] + shape = self.shapes[name] + by_port: dict[str, list[FrameRef]] = {} + for bound, ref in zip(node.ports, node.inputs): + by_port.setdefault(bound, []).append(ref) + clock = shape.clock_input + clock_delay: float | None = 0.0 + if clock is not None: + clock_delay = self._latest(by_port.get(clock.name, [])) + waits: list[Wait] = [] + ready = clock_delay + for port in shape.inputs: + refs = by_port.get(port.name, []) + if not refs: + continue + arrived = self._latest(refs) + pairing = port.pairing + interval = pairing.interval + timed = port.accepts.wants == "timing" + if pairing.kind == "lockstep": + waited = arrived + said = "clock" if clock is not None and port.name == clock.name else "lockstep" + elif interval is not None and interval.retimed: + waits.append( + Wait(port.name, "interval", arrived, interval.latency, retimed=True) + ) + if interval.latency is None or clock_delay is None: + continue + waited = clock_delay + interval.latency + interval.ahead + ready = None if ready is None else max(ready, waited) + continue + elif interval is not None: + said = "interval" + waited = None if arrived is None else arrived + interval.ahead + if interval.latency is not None and clock_delay is not None: + limit = clock_delay + interval.latency + interval.ahead + waited = limit if waited is None else min(waited, limit) + else: + waits.append(Wait(port.name, pairing.kind, arrived, timing=timed)) + continue + waits.append( + Wait( + port.name, + said, + arrived, + interval.latency if interval else None, + timing=timed, + ) + ) + ready = None if ready is None or waited is None else max(ready, waited) + window = self.window_seconds(name, shape, by_port) + total = None if ready is None or window is None else ready + window + self._ready[name] = (total, tuple(waits)) + return self._ready[name] + + def latest(self, refs: list[FrameRef]) -> float | None: + """The latest of `refs`, or None where one of them has no size.""" + return self._latest(refs) + + def _latest(self, refs: list[FrameRef]) -> float | None: + found = [self.delay(ref) for ref in refs] + if any(one is None for one in found): + return None + return max((one for one in found if one is not None), default=0.0) + + def window_seconds( + self, name: str, shape: NodeShape, by_port: Mapping[str, list[FrameRef]] + ) -> float | None: + """How long the clock input's window takes to fill; 0 for one frame.""" + clock = shape.clock_input + if clock is None or clock.window <= 1: + return 0.0 + rate = self.clock_rate(clock, by_port.get(clock.name, [])) + return None if rate is None else float(Fraction(clock.window) / rate) + + def clock_rate(self, port: InputPort, refs: list[FrameRef]) -> Fraction | None: + """Items per second of what the clock input reads: frames, or samples.""" + if port.kind == "audio" and port.accepts.sample_rates: + return Fraction(port.accepts.sample_rates[0]) + return self.rate(refs[0]) if refs else None + + def bytes_per_second(self, ref: FrameRef) -> Fraction | None: + """About how many bytes a second of the raw stream `ref` is on an edge.""" + origin = self.origin(ref) + if origin is None: + return None + alias, kind, index = origin + probe = self.probes.get(alias) + streams = probe.by_type(kind) if probe is not None else [] + if index >= len(streams): + return None + stream = streams[index] + if kind == "video": + rate = _fraction(stream.fps) + if rate is None or not stream.width or not stream.height: + return None + return stream.width * stream.height * _PICTURE_BYTES * rate + if kind == "audio" and stream.sample_rate and stream.channels: + return Fraction(stream.sample_rate * stream.channels * _SAMPLE_BYTES) + return None + + def origin(self, ref: FrameRef) -> tuple[str, StreamType, int] | None: + """The source stream `ref` was made from, along first inputs.""" + while not is_src(ref): + node = self.graph.nodes.get(ref.partition(":")[0]) + if node is None or not node.inputs: + return None + ref = node.inputs[0] + alias, kind, index = src_parts(ref) + return alias, kind, index + + def rate(self, ref: FrameRef) -> Fraction | None: + """The rate of the stream `ref` is (:func:`stream_rate`).""" + return stream_rate(self.graph, self.probes, self.shapes.get, ref) + + +# Filters whose pictures or sound leave at a rate their args do not say. +_RATE_LOST = frozenset( + { + "ainterleave", + "aselect", + "asetpts", + "atempo", + "decimate", + "framestep", + "interleave", + "minterpolate", + "mpdecimate", + "select", + "setpts", + "thumbnail", + "tile", + } +) + +# The args a filter setting a rate says it in; a filter with no input is a +# source, and says its rate in one of `_SOURCE_RATE_ARGS`. +_RATE_ARGS: Mapping[str, tuple[str, ...]] = { + "fps": ("fps", "rate", "r"), + "framerate": ("fps", "rate", "r"), + "aresample": ("osr", "out_sample_rate", "sample_rate"), + "asetrate": ("sample_rate", "r"), +} +_SOURCE_RATE_ARGS = ("rate", "r", "framerate", "fps", "sample_rate") + + +def stream_rate( + graph: Graph, + probes: Mapping[str, ProbeResult | None], + shape_of: Callable[[str], NodeShape | None], + ref: FrameRef, +) -> Fraction | None: + """The rate of the stream `ref` names, as known before the run. + + A picture's frame rate, a sound's sample rate; None for data and coded + streams a node writes, and wherever nothing settles it: a self-clocked + node's output, a stream with no probe (a feed by port), a filter whose + rate its args do not say. A node's picture leaves once per tick, so at + its clock's rate: a rate clock's own, the rate of the port a `rate-of` + clock names, the clock input's rate over its stride. A node's sound is + at the rate its format says, else the rate of the input it follows. + """ + seen: set[str] = set() + while True: + if is_src(ref): + alias, kind, index = src_parts(ref) + probe = probes.get(alias) + streams = probe.by_type(kind) if probe is not None else [] + if index >= len(streams): + return None + stream = streams[index] + if kind == "audio": + return Fraction(stream.sample_rate) if stream.sample_rate else None + return _fraction(stream.fps) if kind == "video" else None + name, _, pad_text = ref.partition(":") + node = graph.nodes.get(name) + if node is None or name in seen: + return None + seen.add(name) + pad = int(pad_text) if pad_text.isdigit() else 0 + kind = node.outputs[pad] if pad < len(node.outputs) else "video" + if kind not in ("video", "audio"): + return None + shape = shape_of(name) + if shape is not None: + return _node_output_rate(graph, probes, shape_of, name, shape, pad, kind) + if node.filter in _RATE_LOST: + return None + said = _RATE_ARGS.get(node.filter, () if node.inputs else _SOURCE_RATE_ARGS) + for key in said: + rate = _rate_arg(node.args.get(key)) + if rate is not None: + return rate + if node.filter in _RATE_ARGS or not node.inputs: + return None + ref = next( + (one for one in node.inputs if _kind_of(graph, one) == kind), node.inputs[0] + ) + + +def _node_output_rate( + graph: Graph, + probes: Mapping[str, ProbeResult | None], + shape_of: Callable[[str], NodeShape | None], + name: str, + shape: NodeShape, + pad: int, + kind: str, +) -> Fraction | None: + """The rate of what node `name` writes on output `pad`, of `kind`.""" + node = graph.nodes[name] + output = shape.outputs[pad] if pad < len(shape.outputs) else None + if output is None or output.kind not in ("video", "audio"): + return None + + def first(port: str) -> FrameRef | None: + return next((ref for bound, ref in zip(node.ports, node.inputs) if bound == port), None) + + found = output.format + if kind == "audio": + if found is not None and found.kind == "audio": + return Fraction(found.sample_rate) if found.sample_rate else None + follows = ( + found.port + if found is not None and found.kind == "like" + else shape.clock.port + if shape.clock.kind == "input" + else None + ) + read = first(follows) if follows else None + if read is None or _kind_of(graph, read) != "audio": + return None + return stream_rate(graph, probes, shape_of, read) + clock = shape.clock + if clock.kind == "rate" and clock.rate is not None: + return Fraction(*clock.rate) + if clock.kind not in ("input", "rate-of"): + return None + read = first(clock.port) + rate = stream_rate(graph, probes, shape_of, read) if read is not None else None + port = shape.input(clock.port) + if rate is None or clock.kind == "rate-of" or port is None: + return rate + return rate / port.stride + + +def _kind_of(graph: Graph, ref: FrameRef) -> str: + if is_src(ref): + return src_parts(ref)[1] + name, _, pad_text = ref.partition(":") + node = graph.nodes.get(name) + pad = int(pad_text) if pad_text.isdigit() else 0 + if node is None or pad >= len(node.outputs): + return "video" + return node.outputs[pad] + + +def _rate_arg(value: object) -> Fraction | None: + """A rate as a filter's arg says it: ``30``, ``29.97`` or ``30000/1001``.""" + if isinstance(value, bool): + return None + if isinstance(value, int): + return Fraction(value) if value > 0 else None + if isinstance(value, float): + return Fraction(value).limit_denominator(1001) if value > 0 else None + if not isinstance(value, str): + return None + text = value.strip() + try: + found = Fraction(text) if "/" in text else Fraction(text).limit_denominator(1001) + except (ValueError, ZeroDivisionError): + return None + return found if found > 0 else None + + +def paths_of(graph: Graph, probes: Mapping[str, ProbeResult | None]) -> Paths: + """How late each ref of `graph` runs, counted as it is asked for.""" + return Paths(graph, probes) + + +def _fraction(fps: str | None) -> Fraction | None: + numerator, _, denominator = (fps or "").partition("/") + try: + found = Fraction(int(numerator), int(denominator) if denominator else 1) + except (ValueError, ZeroDivisionError): + return None + return found if found > 0 else None + + +def timing( + graph: Graph, + probes: Mapping[str, ProbeResult | None], + called: Mapping[str, str] | None = None, +) -> Timing | None: + """Each node module's window and readiness, and each written stream's + delay; None for a graph with no node module in it. `called` names each + module path the way the query calls it.""" + if not graph.node_shapes: + return None + paths = Paths(graph, probes) + names = called or {} + # A track minted from rows runs as late as the node writing them. + rows_of = {sink.alias: ref for ref, sink in graph.rows_sinks.items() if sink.alias} + nodes: list[NodeTiming] = [] + for name, shape in paths.shapes.items(): + node = graph.nodes[name] + by_port: dict[str, list[FrameRef]] = {} + for bound, ref in zip(node.ports, node.inputs): + by_port.setdefault(bound, []).append(ref) + clock = shape.clock_input + if clock is not None: + rate = paths.clock_rate(clock, by_port.get(clock.name, [])) + window = window_words(clock, rate) + elif shape.clock.kind == "rate" and shape.clock.rate is not None: + window = f"rate {_rate_words(shape.clock.rate)}" + elif shape.clock.kind == "rate-of": + window = f"at the rate of {shape.clock.port}" + else: + window = shape.clock.kind + ready, waits = paths.ready(name) + called_as = names.get(node.filter, node.filter) + nodes.append(NodeTiming(name, node.filter, window, waits, ready, called_as)) + outputs: list[OutputTiming] = [] + for unit in graph.sinks: + refs = [ + rows_of.get(src_parts(output.ref)[0], output.ref) if is_src(output.ref) + else output.ref + for output in unit.outputs + ] + delays = [paths.delay(ref) for ref in refs] + latest = max((d for d in delays if d is not None), default=0.0) + for index, (output, delay) in enumerate(zip(unit.outputs, delays)): + holds = 0.0 if delay is None else latest - delay + per_second = paths.bytes_per_second(output.ref) if holds else None + held = None if per_second is None else int(per_second * Fraction(holds)) + outputs.append( + OutputTiming( + output.ref, delay, holds, held, unit.path or "", index, output.type + ) + ) + for ref, sink in graph.rows_sinks.items(): + if sink.path: + outputs.append(OutputTiming(ref, paths.delay(ref), 0.0, path=sink.path, kind="rows")) + return Timing(tuple(nodes), tuple(outputs)) + + +def summary(timed: Timing) -> str: + """What ``explain --delays`` prints: a line per node, then per output. + + A node says its window in streaming words, each input it reads for its + timing alone, and how each input it waits for by interval is bounded, + one the host re-times said to be on its own clock; an output how far + behind the source it runs, and how long it waits for the latest stream + written beside it. + """ + lines: list[str] = [] + for node in timed.nodes: + said = [node.window] + for wait in node.waits: + if wait.timing: + said.append(f"{wait.port} for its timing") + if wait.pairing != "interval": + continue + bound = "no bound" if wait.bound is None else f"at most {_seconds(wait.bound)}" + own = " on its own clock" if wait.retimed else "" + said.append(f"{wait.port} by interval{own}, {bound}") + lines.append(f"{node.called}: {'; '.join(said)}") + for output in timed.outputs: + where = ( + f"{output.path} ({output.kind})" + if output.index is None + else f"{output.path} stream {output.index} ({output.kind})" + ) + late = "no known delay" if output.delay is None else ( + f"{_seconds(output.delay)} behind the source" + ) + waits = f", waits {_seconds(output.holds)}" if output.holds else "" + lines.append(f"{where}: {late}{waits}") + return "\n".join(lines) + + +def _rate_words(rate: tuple[int, int]) -> str: + num, den = rate + return f"{num}/s" if den == 1 else f"{num}/{den}/s" + + +def check_live_leads( + graph: Graph, + probes: Mapping[str, ProbeResult | None], + anchors: Mapping[str, tuple[int, int, str]], +) -> None: + """Refuse a node whose interval input runs later than the bound it set. + + A node bounding how long it waits for an input (`interval.latency`) is + one that acts ahead of the rows it reads, and in a live run what arrives + past the bound is late for good. `anchors` maps a module path to where + the query named it and the function's name. + """ + paths = Paths(graph, probes) + for name, shape in paths.shapes.items(): + node = graph.nodes[name] + clock = shape.clock_input + by_port: dict[str, list[FrameRef]] = {} + for bound, ref in zip(node.ports, node.inputs): + by_port.setdefault(bound, []).append(ref) + clock_delay = paths._latest(by_port.get(clock.name, [])) if clock is not None else 0.0 + for port in shape.inputs: + interval = port.pairing.interval + if interval is None or interval.latency is None or interval.retimed: + continue + arrived = paths._latest(by_port.get(port.name, [])) + if arrived is None or clock_delay is None: + continue + late = arrived - clock_delay + if late <= interval.latency: + continue + line, col, called = anchors.get(node.filter, (1, 1, node.filter)) + raise FfrwdError( + ErrorCode.LIVE_LEAD, + f"{called}() needs '{port.name}' {_seconds(interval.latency)} ahead of " + f"its clock, and the path feeding it runs {_seconds(late)} behind", + line=line, + col=col, + hint=f"feed '{port.name}' from a path no later than " + f"{_seconds(interval.latency)}, or give {called}() a longer lead", + ) + + +def _seconds(value: float) -> str: + return f"{round(value, 3):g} s" diff --git a/cli/ffrwd/warnings.py b/cli/ffrwd/warnings.py index c35330d..58121a8 100644 --- a/cli/ffrwd/warnings.py +++ b/cli/ffrwd/warnings.py @@ -38,6 +38,7 @@ class WarningCode(str, Enum): MISSING_LICENSE = "MISSING_LICENSE" # a package published with no "license" RECIPE_DOES_NOT_COMPILE = "RECIPE_DOES_NOT_COMPILE" # a recipe the probe compile rejected UNSHAPED_LISTENER = "UNSHAPED_LISTENER" # a listener input probed, for want of a shape + HELD_STREAM = "HELD_STREAM" # a stream waits long beside a later one it is written with @dataclass(frozen=True) diff --git a/cli/ffrwd/wasm.py b/cli/ffrwd/wasm.py index fd07e52..48e95ea 100644 --- a/cli/ffrwd/wasm.py +++ b/cli/ffrwd/wasm.py @@ -48,22 +48,24 @@ import json import os import subprocess +import tempfile import threading from collections.abc import Callable, Mapping, Sequence -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from pathlib import Path from typing import Literal, Protocol, cast from . import binaries, nn, probe -from .emit import build_network_graph +from .emit import build_network_graph, build_node_network from .errors import ErrorCode, FfrwdError from .execute import STDIN, STDOUT -from .ir import RowsSink, StreamType +from .ir import PARAMS_FILE, PIPE, Graph, Node, Output, RowsSink, SinkUnit, StreamType from .probe import ProbeResult, RenditionMeta, StreamMeta from .processes import ( NUT, - PCM_F32LE, - PCM_S16LE, + SAMPLE_FMT_CODECS, + WIRE_PIX_FMTS, + WIRE_SAMPLE_FMTS, AudioFormat, EffectGrant, ModelBinding, @@ -72,6 +74,7 @@ RowsDocument, SidecarProcess, ) +from .shapes import PARAMS_INLINE_LIMIT __all__ = [ "ANNOTATION_TYPES", @@ -83,6 +86,7 @@ "DEFAULT_TIMEOUT_SECONDS", "LANGUAGE_TAGS", "MODEL_SUFFIX", + "NODE_WORLD", "PACKET_FILTER_WORLD", "PACKET_SOURCE_WORLD", "FFMPEG_SAMPLE_FMTS", @@ -170,6 +174,7 @@ "ffrwd:av@0.16.0", "ffrwd:av@0.17.0", "ffrwd:av@0.18.0", + "ffrwd:av@0.19.1", ) # The world a module scaffolded today is built against: the newest of those, @@ -179,8 +184,6 @@ # The package carrying the wit, whose version is the world's. WIT_PACKAGE = "ffrwd/wasm" -# The pixel formats a stream edge into or out of the sidecar can carry. -WIRE_PIX_FMTS: tuple[str, ...] = ("rgba", "yuv420p", "yuv422p", "yuv444p") # The coded video streams a stream edge can carry to a packet sink: the ones # the sidecar's NUT reader hands through untouched. @@ -242,10 +245,13 @@ # The first world whose sidecar hosts a codec package's encoder and decoder. CODEC_WORLD = "ffrwd:av@0.18.0" +# What a module exporting the 0.19.1 world's `node` describes as its world: +# its ports are not in its describe at all but in its shape, per call +# (:mod:`ffrwd.shapes`). +NODE_WORLD = "node-module" + # The sample formats one can carry, the pcm each of them travels as, and # the name ffmpeg's own options spell it by. -WIRE_SAMPLE_FMTS: tuple[str, ...] = ("f32", "s16") -SAMPLE_FMT_CODECS: Mapping[str, str] = {"f32": PCM_F32LE, "s16": PCM_S16LE} FFMPEG_SAMPLE_FMTS: Mapping[str, str] = {"f32": "flt", "s16": "s16"} # The JSON Schema types each declared annotation field type covers. `number` @@ -382,6 +388,13 @@ def _budget_hint(budget: float) -> str: # The sidecar's worker-thread cap. Unwritten, the sidecar sizes its own pool. _JOBS_FLAG = "-jobs" +# Where a node's params are read whole from a file: ``=``. +_PARAMS_FROM_FLAG = "-params-from" + +# The bound list a node's shape was asked with: ``=``, one per +# call in the order its chain is written. +_BOUND_FLAG = "-bound" + # Which half of a codec package's module a run drives: "encode" or "decode". _CODEC_FLAG = "-codec" @@ -630,6 +643,8 @@ class Described: # reads. None for every other module; a codec module fills both. encoder: EncoderInfo | None = None decoder: DecoderInfo | None = None + # A node: what it reads and writes is its shape's to say, per call. + node: bool = False @property def packet_sink(self) -> bool: @@ -775,12 +790,13 @@ def _described(path: str, payload: object) -> Described: f"the sidecar described {path} with something that is not an object", hint="the module may be built against a sidecar this ffrwd does not know", ) - world = payload.get("world") - if not isinstance(world, str): + written = payload.get("world") + if not isinstance(written, str): raise _reject( f"the sidecar's description of {path} names no world", hint="the module may be built against a sidecar this ffrwd does not know", ) + world = _hosted_world(written) name = payload.get("name") functions = _functions(payload.get("functions")) if not isinstance(name, str) and not functions: @@ -846,9 +862,21 @@ def _described(path: str, payload: object) -> Described: feeders=_feeders(payload.get("feeders")), encoder=_encoder_info(payload.get("encoder")), decoder=_decoder_info(payload.get("decoder")), + node=written == NODE_WORLD or payload.get("node") is True, ) +def _hosted_world(world: str) -> str: + """The world the sidecar hosts a module in, as the compiler's checks read it. + + The sidecar describes a module by the world it hosts it in, not the one + it was built against: a node is `node-module`, which is 0.19.1's, and + every older module is adapted into the newest world it knows. Each + `hosts_*` check asks what that world can host. + """ + return WORLDS[-1] if world == NODE_WORLD else world + + def _encoder_info(value: object) -> EncoderInfo | None: """The describe's ``encoder`` object, or None where there is none.""" if not isinstance(value, dict): @@ -1483,6 +1511,10 @@ def _last_line(text: str) -> str: # The rows document a packet sink writes, and the specifier every flag on the # copy's one output stream carries. _ROWS_FORMAT = "ndjson" +# The output a node's emitted rows are labelled by, beside its ports. +EMITTED_ROWS_PORT = "@rows" +# What a compile-time read names the one node it hosts. +_READ_NODE = "read" _OUTPUT_STREAM = "0" # How a `-map` names one stream of a kind, ffmpeg's own letters. @@ -1543,6 +1575,12 @@ class PacketRead: module: str params: str wants: SinkWants + # The input of a node module the stream binds, whose rows are what the + # node emits beside its ports; empty for a packet sink. + port: str = "" + # The bound list the node's shape was asked with, as JSON; empty for a + # packet sink. + bound: str = "" # Runs one packet sink over one stream and returns the rows it wrote: @@ -1637,6 +1675,55 @@ def _reader_argv(binary: str, read: PacketRead, described: Described | None) -> return _argv(binary, process, reads=(STDIN,), writes=(STDOUT,)) +def _node_reader_argv( + binary: str, read: PacketRead, described: Described | None, scratch: Path +) -> list[str]: + """The sidecar command hosting a node that reads the stream and emits rows. + + One node bound by its port to the stream on stdin, its emitted rows + mapped to stdout as NDJSON: a network of one, as a query would spell + it. Params a filtergraph cannot spell go in a file in `scratch`. + """ + params = _params_object(read.params) + filed: list[str] = [] + if params and _params_filed(params): + written = scratch / "params.json" + written.write_text(json.dumps(params), encoding="utf-8") + filed = [_PARAMS_FROM_FLAG, f"{_READ_NODE}={written}"] + params = {} + node = Node( + id=_READ_NODE, + filter=_READ_NODE, + args=params, + inputs=[f"src:{_READ_NODE}:{_TYPE_SPECIFIERS[read.kind]}:0"], + outputs=["data"], + ports=[read.port], + out_ports=[EMITTED_ROWS_PORT], + ) + graph = Graph( + input_paths=[PIPE], + sources={_READ_NODE: 0}, + nodes={_READ_NODE: node}, + sinks=[ + SinkUnit( + outputs=[Output(ref=_READ_NODE, type="data", name=None, metadata={})], + path=PIPE, + ) + ], + ) + network, groups = build_node_network(graph, pipe_inputs=[STDIN]) + argv = [binary, "-f", EDGE_FORMAT, "-i", STDIN] + for effect in EFFECTS: + if described is not None and getattr(described, effect): + argv += [_GRANT_FLAGS[effect], read.module] + argv += ["-m", f"{_READ_NODE}={read.module}", "-filter_complex", network, *filed] + if read.bound: + argv += [_BOUND_FLAG, f"{_READ_NODE}={read.bound}"] + for target in groups[0]: + argv += ["-map", target] + return [*argv, "-f", _ROWS_FORMAT, STDOUT] + + def _params_object(params: str) -> dict[str, object]: """A marshalled params string back as the object the argv writes.""" if not params: @@ -1678,18 +1765,23 @@ def read_packet_rows( "needs it to host the module", hint=INSTALL_HINT, ) - sidecar_command = _reader_argv(binary, read, described) - failure: FfrwdError | None = None - for attempt in range(3): - command = copy_argv(ffmpeg, read, attempt) - if attempt and command == copy_argv(ffmpeg, read, attempt - 1): - continue # nothing left to widen; the previous attempt was this one - try: - return _run_read(command, sidecar_command, read) - except _CopyRefused as refused: - failure = refused.error - assert failure is not None # the loop runs at least one attempt - raise failure + with tempfile.TemporaryDirectory() as scratch: + sidecar_command = ( + _node_reader_argv(binary, read, described, Path(scratch)) + if read.port + else _reader_argv(binary, read, described) + ) + failure: FfrwdError | None = None + for attempt in range(3): + command = copy_argv(ffmpeg, read, attempt) + if attempt and command == copy_argv(ffmpeg, read, attempt - 1): + continue # nothing left to widen; the previous attempt was this one + try: + return _run_read(command, sidecar_command, read) + except _CopyRefused as refused: + failure = refused.error + assert failure is not None # the loop runs at least one attempt + raise failure class _CopyRefused(Exception): @@ -1926,6 +2018,8 @@ def _argv( SINK pad carrying rendition metadata (:attr:`SidecarProcess.pads`) gets a ``-pad ''`` right after its own ``-i``, ``{"row": ..., "rendition": {...}}`` with absent attributes omitted -- a pad with none gets no flag. + A node network's input carrying a raw picture gets its ``"color"`` there + too (:attr:`SidecarProcess.colors`), since NUT writes none. `writes` is the mirror on the other side: one path per rows document the process writes, in document order, since a process writing several of @@ -1960,12 +2054,22 @@ def _argv( "its outputs carries, one per output", ) argv = [binary] - if not process.packet_source: + # A source reads nothing: a packet source, or a network of nodes none of + # which is handed a stream. + reads_nothing = process.packet_source or (process.node_network and not process.inputs) + if not reads_nothing: for index, path in enumerate(reads or (STDIN,)): argv += ["-f", EDGE_FORMAT, "-i", path] meta: PadMeta | None = process.pads[index] if index < len(process.pads) else None - if meta is not None: - argv += ["-pad", json.dumps(meta.to_dict())] + pad = meta.to_dict() if meta is not None else {} + colour = process.colors[index] if index < len(process.colors) else () + if colour: + pad["color"] = dict(colour) + tags = process.tags[index] if index < len(process.tags) else () + if tags: + pad["tags"] = dict(tags) + if pad: + argv += ["-pad", json.dumps(pad)] if any(grant.effect == "gpu" for grant in process.grants): jobs = GPU_JOBS if jobs is None else min(jobs, GPU_JOBS) if jobs is not None: @@ -1982,7 +2086,9 @@ def _argv( argv += [_FRAME_RATE_FLAG, process.frame_rate] for flag, value in process.color: argv += [f"-{flag}", value] - if process.network: + if process.node_network: + argv += _node_network_args(process, reads, writes) + elif process.network: argv += _network_args(process, writes) else: argv += ["-m", process.module] @@ -2162,6 +2268,70 @@ def _network_args(process: SidecarProcess, writes: Sequence[str] = ()) -> list[s return argv +def _node_network_args( + process: SidecarProcess, reads: Sequence[str], writes: Sequence[str] +) -> list[str]: + """The ``-m`` table, the network string and the outputs of a region + holding node modules. + + Each output is one NUT carrying every stream the plan takes to one + process, a ``-map`` per stream; each rows document an output of its own. + `reads` names its inputs in ``-i`` order and `writes` its stream outputs + then its documents; a printed command given none reads stdin and writes + stdout, then numbered pipes. + """ + graph = process.graph + if graph is None: # `network` is True for every node region + raise _reject( + f"process '{process.id}' hosts node modules and carries no graph", + hint="the plan was built without partitioning; recompile the query", + ) + wanted = graph.input_paths.count(PIPE) + given = list(reads)[:wanted] or [STDIN] * min(wanted, 1) + given += [f"pipe:{index}" for index in range(len(given), wanted)] + filed: list[str] = [] + nodes = dict(graph.nodes) + for name, node in graph.nodes.items(): + if not node.ports and not node.out_ports or not _params_filed(node.args): + continue + filed += [_PARAMS_FROM_FLAG, f"{node.filter}={PARAMS_FILE}{process.id}:{name}"] + nodes[name] = replace(node, args={}) + network, groups = build_node_network(replace(graph, nodes=nodes), pipe_inputs=given) + streams = len(groups) - len(process.rows) + paths = list(writes[:streams]) + paths += [STDOUT if not paths and index == 0 else f"pipe:{index + 1}" + for index in range(len(paths), streams)] + documents = writes[streams:] + argv: list[str] = [] + for binding in process.modules: + argv += ["-m", f"{binding.name}={binding.path}"] + argv += ["-filter_complex", network, *filed] + for name, node in graph.nodes.items(): + if node.bound: + argv += [_BOUND_FLAG, f"{nodes[name].filter}={node.bound}"] + for targets, path in zip(groups[:streams], paths): + for target in targets: + argv += ["-map", target] + argv += ["-f", EDGE_FORMAT, path] + for index, (targets, document) in enumerate(zip(groups[streams:], process.rows)): + given_path = documents[index] if index < len(documents) else "" + # Rows nothing reads are the run's own, on stdout. + unread = STDOUT if not document.sink.alias else f"pipe:{streams + index + 1}" + path = document.sink.path or given_path or unread + for target in targets: + argv += ["-map", target] + argv += ["-f", document.sink.container, path] + return argv + + +def _params_filed(params: Mapping[str, object]) -> bool: + """Whether a node's params go in a file: too long for a command line, or + holding a list a filtergraph option cannot spell.""" + if any(not isinstance(value, str | int | float | bool) for value in params.values()): + return True + return len(json.dumps(params)) > PARAMS_INLINE_LIMIT + + def sidecar_argv( process: SidecarProcess, reads: Sequence[str] = (), diff --git a/cli/tests/conftest.py b/cli/tests/conftest.py index 9084fdf..8143cfa 100644 --- a/cli/tests/conftest.py +++ b/cli/tests/conftest.py @@ -31,6 +31,8 @@ import pytest from ffrwd import registry as registry_module +from ffrwd.errors import FfrwdError +from ffrwd.parser import parse, resolve from ffrwd.registry import Registry, load_reference SNAPSHOT_PATH = Path(__file__).resolve().parent / "data" / "reference_registry.json" @@ -73,6 +75,22 @@ def pinned_ffmpeg() -> None: pytest.skip(message) +def older_world_refusal(text: str) -> FfrwdError: + """What a module of an older world refuses `text`'s declaration with. + + Resolve refuses it outright, or, for a signature a node module may + carry, keeps the refusal on the declaration for lowering to raise once + the module's describe says it is no node. + """ + try: + resolved = resolve(parse(text)) + except FfrwdError as err: + return err + kept = [d.refusal for d in resolved.wasm.values() if d.refusal is not None] + assert kept, "the declaration was neither refused nor kept a refusal" + return kept[0] + + def clear_leaks(home: Path, paths: Iterable[Path]) -> None: """Remove each of `paths`, and refuse any that is not under `home`. diff --git a/cli/tests/exec/test_exec_scaffold.py b/cli/tests/exec/test_exec_scaffold.py index 9190ac3..01a4176 100644 --- a/cli/tests/exec/test_exec_scaffold.py +++ b/cli/tests/exec/test_exec_scaffold.py @@ -7,19 +7,19 @@ Nothing here is a fixture the repo holds: ``init --rust`` writes a package into a temporary directory, cargo builds it for ``wasm32-wasip2``, the sidecar -describes what came out, and the compiler compiles the recipe the scaffold -shipped against that module. What the scaffold claims -- build, then publish -- -is the test. - -``FFRWD_WIT_DIR`` is pointed at the sidecar's own wit, the route each in-repo -package workspace takes; the other route the scaffold's ``build.rs`` knows -asks ``ffrwd path ffrwd/wasm``, which needs the package installed. Requires -cargo with the ``wasm32-wasip2`` target, ffmpeg on PATH, and the ``ffrwd-wasm`` -sidecar; skips cleanly without any of them. +describes what came out, and the compiler compiles and runs the recipe the +scaffold shipped against that module. What the scaffold claims -- build, then +publish -- is the test. + +The crate takes ``ffrwd-node`` and ``ffrwd-frame`` from git, so building needs +the network or a warm cargo cache. Requires cargo with the ``wasm32-wasip2`` +target, ffmpeg on PATH, and the ``ffrwd-wasm`` sidecar; skips cleanly without +any of them. """ from __future__ import annotations +import json import os import shutil import subprocess @@ -31,13 +31,13 @@ pytestmark = pytest.mark.exec -_REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent -_SIDECAR_WIT = _REPO_ROOT / "sidecar" / "wit" +_SOURCE = Path(__file__).resolve().parent.parent / "fixtures" / "av.mp4" _NAMESPACE = "me" _SEGMENT = "scaffolded" _TARGET = "wasm32-wasip2" _BUILD_TIMEOUT = 600.0 +_PROBE_TIMEOUT = 60.0 @pytest.fixture(autouse=True) @@ -48,8 +48,8 @@ def _require_a_toolchain() -> None: pytest.skip("ffmpeg/ffprobe not found on PATH") if binaries.ffrwd_wasm_path() is None: pytest.skip("ffrwd-wasm not found (uv sync --extra wasm, or set FFRWD_WASM)") - if not (_SIDECAR_WIT / "av.wit").is_file(): - pytest.skip(f"the sidecar's wit is not there: {_SIDECAR_WIT}") + if not _SOURCE.is_file(): + pytest.skip(f"fixture missing: {_SOURCE} (run scripts/gen_fixtures.py first)") @pytest.fixture(scope="module") @@ -64,11 +64,9 @@ def scaffold(tmp_path_factory: pytest.TempPathFactory) -> Path: finally: os.chdir(here) - environment = dict(os.environ, FFRWD_WIT_DIR=str(_SIDECAR_WIT)) done = subprocess.run( - ["cargo", "build", "--target", _TARGET, "--release"], + ["cargo", "build", "--release", "--target", _TARGET], cwd=root, - env=environment, capture_output=True, text=True, timeout=_BUILD_TIMEOUT, @@ -81,17 +79,43 @@ def _module(scaffold: Path) -> Path: return scaffold / "target" / _TARGET / "release" / f"{_SEGMENT}.wasm" +def _video(path: Path) -> dict[str, int]: + done = subprocess.run( + [ + "ffprobe", "-v", "error", "-select_streams", "v:0", "-count_frames", + "-show_entries", "stream=width,height,nb_read_frames", "-of", "json", str(path), + ], + capture_output=True, + text=True, + timeout=_PROBE_TIMEOUT, + check=False, + ) + assert done.returncode == 0, done.stderr + stream = json.loads(done.stdout)["streams"][0] + return {name: int(value) for name, value in stream.items()} + + def test_the_scaffolded_crate_builds_a_module(scaffold: Path) -> None: assert _module(scaffold).is_file() - # build.rs put the wit where the bindings macro reads it. - assert (scaffold / "wit" / "av.wit").read_bytes() == (_SIDECAR_WIT / "av.wit").read_bytes() + # The SDK carries the wit: the crate neither writes one nor needs a script to. + assert not (scaffold / "build.rs").exists() and not (scaffold / "wit").exists() + +def test_the_sidecar_describes_it_as_the_node_it_declares(scaffold: Path) -> None: + sidecar = binaries.ffrwd_wasm_path() + assert sidecar is not None + done = subprocess.run( + [str(sidecar), "--describe", str(_module(scaffold))], + capture_output=True, + text=True, + timeout=_PROBE_TIMEOUT, + check=False, + ) + assert done.returncode == 0, done.stderr + assert json.loads(done.stdout)["world"] == wasm.NODE_WORLD -def test_the_sidecar_describes_it_as_the_video_module_it_declares(scaffold: Path) -> None: described = wasm.describe(str(_module(scaffold))) - assert described.world == f"ffrwd:av@{wasm.WORLD_VERSION}" - assert described.name == "invert" - assert wasm.wire_pix_fmt(described) in wasm.WIRE_PIX_FMTS + assert described.node and described.name == "passthrough" def test_the_recipe_the_scaffold_ships_compiles_against_that_module( @@ -99,7 +123,7 @@ def test_the_recipe_the_scaffold_ships_compiles_against_that_module( ) -> None: monkeypatch.chdir(scaffold) code = cli.main( - ["compile", "-f", "recipes/invert.sql", "-v", "source=in.mp4", "-v", "dest=out.mp4"] + ["compile", "-f", "recipes/passthrough.sql", "-v", "source=in.mp4", "-v", "dest=out.mp4"] ) printed = capsys.readouterr().out assert code == 0, printed @@ -107,3 +131,18 @@ def test_the_recipe_the_scaffold_ships_compiles_against_that_module( # calling one costs and what the printed line has to show. assert printed.count("ffmpeg ") == 2 assert str(_module(scaffold)) in printed + + +def test_the_recipe_the_scaffold_ships_runs_and_hands_the_picture_back( + scaffold: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + monkeypatch.chdir(scaffold) + dest = scaffold / "out.mkv" + code = cli.main( + [ + "run", "passthrough", "-y", + "-v", f"source={_SOURCE.as_posix()}", "-v", f"dest={dest.as_posix()}", + ] + ) + assert code == 0, capsys.readouterr().err + assert _video(dest) == _video(_SOURCE) diff --git a/cli/tests/test_data_filter.py b/cli/tests/test_data_filter.py index bbb1c64..eb0fd0a 100644 --- a/cli/tests/test_data_filter.py +++ b/cli/tests/test_data_filter.py @@ -40,6 +40,7 @@ from ffrwd.registry import Registry, load_reference from ffrwd.split import insert_splits from ffrwd.wasm import WORLDS, Described +from tests.conftest import older_world_refusal SNAPSHOT_PATH = Path(__file__).resolve().parent / "data" / "reference_registry.json" @@ -400,15 +401,12 @@ def test_the_shapes_a_data_filter_is_declared_in() -> None: def test_a_data_filter_declaration_is_refused_by_what_it_gets_wrong( signature: str, needle: str ) -> None: - with pytest.raises(FfrwdError) as caught: - resolve( - parse( - f"CREATE FUNCTION f{signature} AS 'm.wasm', 'm' LANGUAGE wasm;\n" - "COPY (SELECT f(f.data[1])" + _FROM - ) - ) - assert caught.value.code is ErrorCode.UNSUPPORTED_SQL - assert needle in caught.value.message + refusal = older_world_refusal( + f"CREATE FUNCTION f{signature} AS 'm.wasm', 'm' LANGUAGE wasm;\n" + "COPY (SELECT f(f.data[1])" + _FROM + ) + assert refusal.code is ErrorCode.UNSUPPORTED_SQL + assert needle in refusal.message def test_a_field_a_data_filter_does_not_return_is_refused_at_resolve() -> None: diff --git a/cli/tests/test_examples.py b/cli/tests/test_examples.py index b590b16..324edad 100644 --- a/cli/tests/test_examples.py +++ b/cli/tests/test_examples.py @@ -201,8 +201,9 @@ def wrap_command(line: str, width: int = _WRAP_WIDTH) -> str: continuations. Any other line (a table/CSV row, a `$ ...` line) is returned unchanged. - Two shapes are wrapped: a whole `ffmpeg ...` command, and one numbered - member of the listing a plan with a named pipe prints (`3. sidecar: + Two shapes are wrapped: a whole command (`ffmpeg ...`, or `ffrwd-wasm + ...` where a node source starts the pipeline), and one numbered member + of the listing a plan with a named pipe prints (`3. sidecar: ffrwd-wasm ...`), whose argv is wrapped and whose `N. role: ` lead stays on the first line. @@ -218,7 +219,7 @@ def wrap_command(line: str, width: int = _WRAP_WIDTH) -> str: A token with no safe split point is left long. """ listed = _LISTING_RE.match(line) - if listed is None and not line.startswith("ffmpeg "): + if listed is None and not line.startswith(("ffmpeg ", "ffrwd-wasm ")): return line lead = listed.group("lead") if listed is not None else "" tokens = [_quote(token) for token in shlex.split(line[len(lead):])] @@ -325,11 +326,11 @@ def _shell_tokens(text: str) -> list[str]: def _assert_shlex_invariant(actual: str, expected: str) -> None: - """For a code block that wrapped a single `ffmpeg` line, prove the wrap kept + """For a code block that wrapped a single command line, prove the wrap kept the same shell command: the wrapped block text and the original unwrapped line must tokenize identically.""" actual_line = actual.rstrip("\n") - if "\n" in actual_line or not actual_line.startswith("ffmpeg "): + if "\n" in actual_line or not actual_line.startswith(("ffmpeg ", "ffrwd-wasm ")): return assert _shell_tokens(expected) == shlex.split(actual_line) @@ -511,6 +512,16 @@ def test_wrap_command_leaves_a_token_with_no_safe_split_point_long() -> None: assert shlex.split(wrapped.replace("\\\n", "")) == shlex.split(line) +def test_wrap_command_wraps_a_pipeline_a_node_source_starts() -> None: + line = ( + "ffrwd-wasm -m ticker=ticker.wasm -filter_complex 'ticker=fps=30[video=out0]' " + "-map '[out0]' -f nut pipe:1 | ffmpeg -f nut -i pipe:0 -map 0:v:0 ticker.mp4" + ) + wrapped = wrap_command(line, width=60) + assert len(wrapped.split("\n")) > 1 + assert _shell_tokens(wrapped) == shlex.split(line) + + def test_wrap_command_is_deterministic() -> None: line = ( "ffmpeg -i song.m4a -filter_complex " diff --git a/cli/tests/test_feeders.py b/cli/tests/test_feeders.py index 4e2a2df..3a8d108 100644 --- a/cli/tests/test_feeders.py +++ b/cli/tests/test_feeders.py @@ -47,6 +47,7 @@ from ffrwd.relay import Relay from ffrwd.split import insert_splits from ffrwd.wasm import WORLDS, Described, Feeder +from tests.conftest import older_world_refusal execute = sys.modules["ffrwd.execute"] lower = sys.modules["ffrwd.lower"] @@ -413,11 +414,11 @@ def test_no_feeder_wires_nothing_and_the_port_keeps_its_default(call: str) -> No def test_only_a_feeder_may_default_to_null_at_the_declaration( declaration: str, needle: str ) -> None: - with pytest.raises(FfrwdError) as caught: - resolve(parse(declaration + "\nCOPY (SELECT f(p.video[1]) FROM input('p.mp4') p) " - "TO 'out.mp4'")) - assert caught.value.code is ErrorCode.UNSUPPORTED_SQL - assert needle in caught.value.message + refusal = older_world_refusal( + declaration + "\nCOPY (SELECT f(p.video[1]) FROM input('p.mp4') p) TO 'out.mp4'" + ) + assert refusal.code is ErrorCode.UNSUPPORTED_SQL + assert needle in refusal.message def test_a_null_default_the_module_reads_as_a_pad_is_refused() -> None: @@ -649,6 +650,57 @@ def test_a_writer_ends_nothing_once_its_reader_has_gone(code: int) -> None: ) == (None, False, None) +def test_a_writer_its_readers_wait_on_is_started_with_the_stage() -> None: + """Held for its port is only a writer whose readers can listen without it.""" + writers = { + "ffmpeg1": (9000, ["sidecar0"]), + "sidecar1": (9002, ["sidecar1", "sidecar2"]), + "ffmpeg3": (9004, ["sidecar3"]), + } + feeds = [("ffmpeg2", "sidecar0"), ("ffmpeg3", "sidecar4"), ("sidecar4", "sidecar3")] + assert execute.held_writers(writers, feeds) == {"ffmpeg1": (9000, ["sidecar0"])} + + +def test_a_stage_that_never_started_a_writer_fails() -> None: + """Whether its port was never heard or its readers ended first, nothing it + writes was written, and the run says so instead of exiting 0.""" + held = {"sidecar0": (9000, ["sidecar1"])} + assert execute.never_started(held, ["sidecar0", "sidecar1"], ["sidecar0", "sidecar1"]) is None + assert execute.never_started(held, ["sidecar1"], ["sidecar1"]) is None + pid, error = execute.never_started(held, ["sidecar1"], ["sidecar0", "sidecar1"]) + assert pid == "sidecar0" and error.code is ErrorCode.INPUT_NEVER_OPENED + assert error.message.startswith( + "sidecar0, which writes tcp://127.0.0.1:9000 for sidecar1, was never started" + ) + result = execute.stage_result( + 0, [], {}, [], failed=pid, timed_out=True, wedge=error, interrupted=False + ) + assert result.exit_code != 0 and result.overflow is error and not result.timed_out + + +def test_an_unheld_writers_first_instance_waits_for_its_port( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(execute, "FEEDER_WAIT", 0.3) + stop = threading.Event() + with _held_port() as listener: + port = listener.getsockname()[1] + assert execute._unheard_port([port], stop) == port + + def listen_late() -> None: + time.sleep(0.1) + listener.listen() + + monkeypatch.setattr(execute, "FEEDER_WAIT", 10.0) + opener = threading.Thread(target=listen_late) + opener.start() + assert execute._unheard_port([port], stop) is None + opener.join() + stop.set() + with _held_port() as listener: + assert execute._unheard_port([listener.getsockname()[1]], stop) is not None + + def test_a_writer_failing_while_its_reader_runs_ends_the_stage() -> None: members = _members(sidecar0=None, ffmpeg0=1) failed, _, _ = execute._watch( diff --git a/cli/tests/test_laterals.py b/cli/tests/test_laterals.py index c6b2595..b56ece1 100644 --- a/cli/tests/test_laterals.py +++ b/cli/tests/test_laterals.py @@ -10,24 +10,26 @@ from __future__ import annotations import functools +import json import socket import sys import threading import time -from collections.abc import Callable, Iterator, Mapping +from collections.abc import Callable, Iterator, Mapping, Sequence from dataclasses import replace +from fractions import Fraction from pathlib import Path import pytest -from ffrwd import wasm +from ffrwd import shapes, wasm from ffrwd.compiler import compile_all from ffrwd.errors import ErrorCode, FfrwdError from ffrwd.execute import render_plan -from ffrwd.ir import FeederCall, Graph, Lateral, LateralConnection, LateralValue +from ffrwd.ir import FeederCall, Graph, Lateral, LateralConnection, LateralValue, Node from ffrwd.parser import parse, resolve from ffrwd.probe import ProbeResult, StreamMeta -from ffrwd.processes import ProcessPlan +from ffrwd.processes import ProcessPlan, SidecarProcess from ffrwd.registry import Registry, load_reference from ffrwd.split import insert_splits from ffrwd.wasm import WORLDS, Described, Feeder @@ -421,7 +423,7 @@ def test_a_data_stream_parameter_is_refused_a_picture() -> None: "'ad.video' comes from LATERAL play(s.data[1]), which starts a source per " "row of a data stream: it is empty between rows, and scale reads a frame " "on every tick. A run-time lateral's streams can only go to a module " - "input declared 'feeder'", + "input declared 'feeder', or a node's input held on a port", ), ( _ADS.format(select="ad.audio", stream="s.data[1]"), @@ -429,7 +431,7 @@ def test_a_data_stream_parameter_is_refused_a_picture() -> None: "'ad.audio' comes from LATERAL play(s.data[1]), which starts a source per " "row of a data stream: it is empty between rows, and a COPY reads a frame " "on every tick. A run-time lateral's streams can only go to a module " - "input declared 'feeder'", + "input declared 'feeder', or a node's input held on a port", ), ( _ADS.format(select="probe(ad.video)", stream="s.data[1]"), @@ -437,7 +439,7 @@ def test_a_data_stream_parameter_is_refused_a_picture() -> None: "'ad.video' comes from LATERAL play(s.data[1]), which starts a source per " "row of a data stream: it is empty between rows, and probe reads a frame " "on every tick. A run-time lateral's streams can only go to a module " - "input declared 'feeder'", + "input declared 'feeder', or a node's input held on a port", ), ( _ADS.format(select="probe(s.video[1], ad.video)", stream="s.video[1]"), @@ -482,10 +484,11 @@ def test_a_run_time_lateral_is_refused_by_what_it_gets_wrong( [ ( "CREATE FUNCTION play(launch data_stream, url text) RETURNS TABLE(d " - "data_stream) AS $$ SELECT m.data[1] AS d FROM input(url) m $$ LANGUAGE sql;", + "data_stream, e data_stream) AS $$ SELECT m.data[1] AS d, m.data[1] AS e " + "FROM input(url) m $$ LANGUAGE sql;", "play() is started once per message of its data stream, and returns " - "'d data_stream': it returns the picture and sound a feeder takes, one " - "of each at most", + "'d data_stream': it returns the picture, sound and rows a feeder takes, " + "one of each at most", ), ( "CREATE FUNCTION play(launch data_stream, v video_stream) RETURNS " @@ -673,3 +676,583 @@ def test_laterals_sharing_an_alias_each_feed_a_connection_of_their_own() -> None assert len(ports) == 2 for port in ports: assert len([edge for edge in plan.feeder_edges if edge.port == port]) == 2 + + +# -- a node holding its feed on a port ---------------------------------------- + +SWITCH = "modules/switch_node.wasm" +_SWITCH = ( + "CREATE FUNCTION switch(v video_stream, a audio_stream DEFAULT NULL, " + "feed video_stream DEFAULT NULL, feed_audio audio_stream DEFAULT NULL, " + "port number DEFAULT 9000) RETURNS STRUCT(v video_stream, a audio_stream) " + f"AS '{SWITCH}', 'switch' LANGUAGE wasm;" +) +_HELD = { + "kind": "hold", + "anchor": {"kind": "tagged", "tag": "smart_timed"}, + "group": "switch", + "lead": 0.3, + "port_param": "port", +} + + +def _switch_shape( + module: str, params: str, bound: Sequence[shapes.Binding], grants: Sequence[str] = () +) -> shapes.NodeShape: + """ffrwd/switch 0.5.0's shape: the programme, and a feed group held on `port`.""" + + def port(name: str, kind: str, pairing: dict[str, object]) -> dict[str, object]: + return { + "name": name, "kind": kind, "required": name == "v", "many": False, + "pairing": pairing, "rows": "ignore", "window": 1, "stride": 1, "accepts": {}, + } + + lockstep: dict[str, object] = {"kind": "lockstep"} + return shapes.node_shape( + module, + { + "inputs": [ + port("v", "video", lockstep), + port("a", "audio", lockstep), + port("feed", "video", _HELD), + port("feed_audio", "audio", _HELD), + ], + "outputs": [ + {"name": "v", "kind": "video", "latency": 0, + "format": {"kind": "like", "port": "v"}}, + {"name": "a", "kind": "audio", "latency": 0, + "format": {"kind": "like", "port": "a"}}, + ], + "clock": {"kind": "input", "port": "v"}, + "pure": True, + "one_to_one": True, + "bounded": True, + "relation": [], + }, + ) + + +def test_a_lateral_feeds_a_nodes_held_inputs_on_one_connection() -> None: + """The lateral's picture and sound go to the port the host listens on for + the node's feed group, written into the node's port param, as a feeder's + connection is.""" + modules = { + **_MODULES, + SWITCH: Described( + world="node-module", + name="switch", + params_schema={"type": "object", "properties": {"port": {"type": "integer"}}}, + node=True, + ), + } + query = """COPY ( + WITH prog AS (SELECT s.video[1] AS v, s.audio[1] AS a, s.data[1] AS d + FROM input('leaf.nut') s), + awards AS (SELECT auction(prog.d, prog.v).d AS d, + auction(prog.d, prog.v).launch AS launch FROM prog), + sw AS (SELECT (switch(prog.v, prog.a, ad.video, ad.audio)).* + FROM prog, awards, LATERAL play(awards.launch) ad) + SELECT sw.v, sw.a, awards.d AS deal FROM sw, awards +) TO publish('https://relay.example', 'leaf')""" + plan = compile_all( + _declared(query).replace("COPY (", _SWITCH + "\nCOPY (", 1), + describe=lambda path: modules[path], + shape=_switch_shape, + ).plan + assert plan is not None + (lateral,) = plan.laterals + (connection,) = lateral.connections + assert [call.param for call in connection.calls] == ["feed", "feed_audio"] + (switch,) = [ + node + for process in plan.sidecars + if process.graph is not None + for node in process.graph.nodes.values() + if node.filter == "switch_node" + ] + assert switch.args["port"] == connection.port + assert switch.ports == ["v", "a"] + + +# -- a node's data output over a source read row by row -------------------------- + +SUB = "modules/sub_node.wasm" +NODE_AUCTION = "modules/auction_node.wasm" + + +def _source_and_auction_shape( + module: str, params: str, bound: Sequence[shapes.Binding], grants: Sequence[str] = () +) -> shapes.NodeShape: + """A subscription's one rendition, and an auction node reading its deals.""" + data = {"kind": "data", "codec": "json"} + if module == NODE_AUCTION: + def port(name: str, kind: str) -> dict[str, object]: + pairing = {"kind": "lockstep"} if kind == "video" else {"kind": "arrival"} + return { + "name": name, "kind": kind, "required": True, "many": False, + "pairing": pairing, "rows": "ignore", "window": 1, "stride": 1, + "accepts": {}, + } + + return shapes.node_shape(module, { + "inputs": [port("d", "data"), port("clock", "video")], + "outputs": [ + {"name": "d", "kind": "data", "latency": 0, "format": data}, + {"name": "launch", "kind": "data", "latency": 0, "format": data}, + ], + "clock": {"kind": "input", "port": "clock"}, + "pure": False, "one_to_one": False, "bounded": False, "relation": [], + }) + picture = {"kind": "video", "width": 1280, "height": 720, "pix_fmt": "yuv420p"} + return shapes.node_shape(module, { + "inputs": [], + "outputs": [ + {"name": "video", "kind": "video", "latency": 0, "row": 0, "format": picture}, + {"name": "deals", "kind": "data", "latency": 0, "row": 0, "format": data}, + ], + "clock": {"kind": "self_clocked"}, + "pure": False, "one_to_one": False, "bounded": False, + "relation": ['{"name": "720p"}'], + }) + + +def test_a_nodes_data_over_a_source_read_by_row_starts_a_lateral() -> None: + """The auction runs once over the one row the source reads, so its + launch column is one stream however the body reads it.""" + modules = { + **_MODULES, + SUB: Described( + world="node-module", name="sub", + params_schema={"type": "object", "properties": {"relay": {"type": "string"}}}, + node=True, + ), + NODE_AUCTION: Described( + world="node-module", name="auction", + params_schema={"type": "object", "properties": {}}, node=True, + ), + } + query = ( + f"CREATE FUNCTION sub(relay text) RETURNS source AS '{SUB}', 'sub' LANGUAGE wasm;\n" + "CREATE FUNCTION auction(d data_stream, clock video_stream) " + "RETURNS STRUCT(d data_stream, launch data_stream) " + f"AS '{NODE_AUCTION}', 'auction' LANGUAGE wasm;\n" + + _DECLARATIONS["play"] + "\n" + _DECLARATIONS["video"] + "\n" + + """COPY ( + WITH prog AS (SELECT s.video[1] AS v, s.data[1] AS d FROM sub('r') s), + awards AS (SELECT (auction(prog.d, prog.v)).* FROM prog), + ads AS (SELECT ad.video FROM awards, LATERAL play(awards.launch) ad) + SELECT video(prog.v, ads.video), awards.d AS deal FROM prog, ads, awards +) TO 'out.nut'""" + ) + plan = compile_all( + query, describe=lambda path: modules[path], shape=_source_and_auction_shape + ).plan + assert plan is not None + (lateral,) = plan.laterals + assert lateral.stream == "awards.launch" + + +# -- two laterals on one held many-port --------------------------------------- + +COMPOSE = "modules/compose_node.wasm" +_COMPOSE = ( + "CREATE FUNCTION compose(v video_stream, inputs video_stream[] DEFAULT NULL, " + "port number DEFAULT NULL) RETURNS video_stream " + f"AS '{COMPOSE}', 'compose' LANGUAGE wasm;" +) +_TWO_ADS = """COPY ( + WITH prog AS (SELECT s.video[1] AS v, s.data[1] AS d FROM input('leaf.nut') s) + SELECT compose(prog.v, ARRAY[ad.video, lbar.video]) + FROM prog, LATERAL play(prog.d) ad, LATERAL play(prog.d) lbar +) TO 'out.nut'""" + + +def _compose_shape( + module: str, params: str, bound: Sequence[shapes.Binding], grants: Sequence[str] = () +) -> shapes.NodeShape: + """ffrwd/blitz's compose: the programme, and pictures held on `port` when + `inputs` is bound, so a call binding none listens on nothing.""" + held = { + "kind": "hold", + "anchor": {"kind": "tagged", "tag": "smart_timed"}, + "lead": 0.3, + "port_param": "port" if any(one.input == "inputs" for one in bound) else None, + } + + def port(name: str, pairing: dict[str, object], many: bool) -> dict[str, object]: + return { + "name": name, "kind": "video", "required": name == "v", "many": many, + "pairing": pairing, "rows": "ignore", "window": 1, "stride": 1, "accepts": {}, + } + + return shapes.node_shape(module, { + "inputs": [port("v", {"kind": "lockstep"}, False), port("inputs", held, True)], + "outputs": [{"name": "v", "kind": "video", "latency": 0, + "format": {"kind": "like", "port": "v"}}], + "clock": {"kind": "input", "port": "v"}, + "pure": True, "one_to_one": True, "bounded": True, "relation": [], + }) + + +def _compose_modules(port: dict[str, object]) -> dict[str, Described]: + return { + **_MODULES, + COMPOSE: Described( + world="node-module", name="compose", + params_schema={"type": "object", "properties": {"port": port}}, node=True, + ), + } + + +def _composed( + port: dict[str, object], asked: list[Sequence[shapes.Binding]] | None = None +) -> ProcessPlan: + """The two ads into compose; `asked` collects each bound list its shape is asked for.""" + + def shape( + module: str, params: str, bound: Sequence[shapes.Binding], grants: Sequence[str] = () + ) -> shapes.NodeShape: + if asked is not None and module == COMPOSE: + asked.append(bound) + return _compose_shape(module, params, bound, grants) + + plan = compile_all( + _declared(_TWO_ADS).replace("COPY (", _COMPOSE + "\nCOPY (", 1), + describe=lambda path: _compose_modules(port)[path], + shape=shape, + ).plan + assert plan is not None + return plan + + +def test_two_laterals_on_one_held_many_port_are_two_connections() -> None: + """Each lateral is a connection of its own, so a module taking its port + param as an array is given one port per lateral, in the order written, + and its shape is told two streams there whose rate nothing settles.""" + asked: list[Sequence[shapes.Binding]] = [] + plan = _composed({"type": "array", "items": {"type": "integer"}}, asked) + assert {tuple(bound) for bound in asked} == { + ( + shapes.Binding("v", (shapes.StreamHint(Fraction(25)),)), + shapes.Binding("inputs", (shapes.StreamHint(None), shapes.StreamHint(None))), + ) + } + assert len(plan.laterals) == 2 + ports = [lateral.connections[0].port for lateral in plan.laterals] + assert len(set(ports)) == 2 + (compose,) = [ + node + for process in plan.sidecars + if process.graph is not None + for node in process.graph.nodes.values() + if node.filter == "compose_node" + ] + assert compose.args["port"] == ports + assert compose.ports == ["v"] + + +def test_two_laterals_on_a_held_many_port_taking_one_port_are_refused() -> None: + with pytest.raises(FfrwdError) as caught: + _composed({"type": "integer", "minimum": 1, "maximum": 65535}) + assert caught.value.message.endswith( + f"hands 'inputs' 2 run-time laterals, each a connection on a port of its own, " + f"and the module '{COMPOSE}' takes one port in 'port'" + ) + + +def test_a_lateral_counts_as_bound_where_the_shape_says_which_port_it_holds() -> None: + """compose names `port` as its feed's port only with `inputs` bound.""" + query = _TWO_ADS.replace("ARRAY[ad.video, lbar.video]", "ARRAY[ad.video]").replace( + ", LATERAL play(prog.d) lbar", "" + ) + modules = _compose_modules({"type": "integer"}) + plan = compile_all( + _declared(query).replace("COPY (", _COMPOSE + "\nCOPY (", 1), + describe=lambda path: modules[path], + shape=_compose_shape, + ).plan + assert plan is not None + (lateral,) = plan.laterals + (compose,) = [ + node + for process in plan.sidecars + if process.graph is not None + for node in process.graph.nodes.values() + if node.filter == "compose_node" + ] + assert compose.args["port"] == lateral.connections[0].port + + +# compose's own `port`: one port the host may be handed, or the compiler's list. +_ONE_OR_MANY_PORTS: dict[str, object] = { + "type": ["array", "integer"], + "items": {"type": "integer", "minimum": 1, "maximum": 65535}, + "minimum": 1, + "maximum": 65535, +} + + +def _compose_node(plan: ProcessPlan) -> Node: + (compose,) = [ + node + for process in plan.sidecars + if process.graph is not None + for node in process.graph.nodes.values() + if node.filter == "compose_node" + ] + return compose + + +def test_a_port_param_taking_one_or_many_is_handed_a_list_for_two_laterals() -> None: + plan = _composed(_ONE_OR_MANY_PORTS) + ports = [lateral.connections[0].port for lateral in plan.laterals] + assert len(set(ports)) == 2 + assert _compose_node(plan).args["port"] == ports + + +def test_a_port_written_by_hand_stays_one_integer_where_a_list_also_fits() -> None: + query = """COPY ( + SELECT compose(s.video[1], port => 9100.0) FROM input('leaf.nut') s +) TO 'out.nut'""" + modules = _compose_modules(_ONE_OR_MANY_PORTS) + plan = compile_all( + _COMPOSE + "\n" + query, + describe=lambda path: modules[path], + shape=_compose_shape, + ).plan + assert plan is not None + port = _compose_node(plan).args["port"] + assert port == 9100 and isinstance(port, int) + + +# -- a node's data, read by several and by a lateral, with no copying ffmpeg --- + +SELL = "modules/sell_node.wasm" +_SOLD = """COPY ( + WITH prog AS (SELECT s.video[1] AS v, s.data[1] AS d FROM input('leaf.nut') s), + sold AS (SELECT (sell(prog.d, prog.v)).* FROM prog), + ads AS (SELECT ad.video FROM sold, LATERAL play(sold.launch) ad) + SELECT video(prog.v, ads.video), + auction(sold.d, prog.v, cohort => 'es').d AS es, + auction(sold.d, prog.v, cohort => 'es').launch AS es_launch, + auction(sold.d, prog.v, cohort => 'fr').d AS fr, + auction(sold.d, prog.v, cohort => 'fr').launch AS fr_launch, + sold.d AS deal + FROM prog, ads, sold +) TO 'out.nut'""" + + +def test_a_nodes_data_reaches_every_reader_and_the_host_from_its_own_region() -> None: + """sell's deals go to two auctions and the destination, one output of the + network apiece, and its launches to the host as NDJSON: no ffmpeg copies a + message. The same plan had eleven processes, seven of them ffmpeg, with a + copy per reader and one to the tap.""" + modules = { + **_MODULES, + SELL: Described( + world="node-module", name="sell", + params_schema={"type": "object", "properties": {}}, node=True, + ), + } + plan = compile_all( + "CREATE FUNCTION sell(d data_stream, clock video_stream) " + "RETURNS STRUCT(d data_stream, launch data_stream) " + f"AS '{SELL}', 'sell' LANGUAGE wasm;\n" + _declared(_SOLD), + describe=lambda path: modules[path], + # sell takes deals and a clock and writes deals and launches, as + # the node auction does. + shape=lambda module, params, bound, grants=(): _source_and_auction_shape( + NODE_AUCTION, params, bound, grants + ), + ).plan + assert plan is not None + ffmpegs = [p for p in plan.processes if not isinstance(p, SidecarProcess)] + assert (len(plan.processes), len(ffmpegs)) == (9, 5) + assert all(process.graph.nodes or process.graph.sinks for process in ffmpegs) + (lateral,) = plan.laterals + seller = next( + process for process in plan.sidecars if process.module == SELL + ) + assert lateral.pipe and lateral.writer == seller.id + argv = render_plan(plan, sidecar_argv=wasm.shown_argv) + line = next(one for one in argv.splitlines() if "sell_node=" in one) + assert line.count("-map '[out2]'") == 3 + assert line.endswith(f"-f ndjson ffrwd:tap:{lateral.tap}") + + +def test_a_region_writing_a_laterals_rows_and_hosting_its_switch_starts_with_its_stage() -> None: + """sell and the switch its launches feed share the programme's region, so + the region both writes the lateral's rows and listens for its instances: + held until its own port accepted, it would never have started.""" + modules = { + **_MODULES, + SELL: Described( + world="node-module", name="sell", + params_schema={"type": "object", "properties": {}}, node=True, + ), + } + plan = compile_all( + "CREATE FUNCTION sell(d data_stream, clock video_stream) " + "RETURNS STRUCT(d data_stream, launch data_stream) " + f"AS '{SELL}', 'sell' LANGUAGE wasm;\n" + + _declared( + """COPY ( + WITH prog AS (SELECT setpts(s.video[1], 'PTS+1/TB') AS v, s.data[1] AS d + FROM input('leaf.nut') s), + sold AS (SELECT (sell(prog.d, prog.v)).* FROM prog) + SELECT video(prog.v, ad.video), sold.d AS deal + FROM prog, sold, LATERAL play(sold.launch) ad +) TO 'out.nut'""" + ), + describe=lambda path: modules[path], + shape=lambda module, params, bound, grants=(): _source_and_auction_shape( + NODE_AUCTION, params, bound, grants + ), + ).plan + assert plan is not None + (lateral,) = plan.laterals + (edge,) = plan.feeder_edges + assert edge.source == edge.target == lateral.writer + (stage,) = plan.stages + found, writers = execute.stage_wires(plan, stage, execute.wires(plan)) + assert list(writers) == [lateral.writer] + feeds = [(wire.edge.source, wire.edge.target) for wire in found] + assert execute.held_writers(writers, feeds) == {} + + +def test_a_piped_tap_is_read_off_the_pipe_the_relay_hands_the_host(tmp_path: Path) -> None: + """The host reads the region's NDJSON as it reads the messages off a port.""" + rows: list[Mapping[str, object]] = [] + + def compile_instance(text: str, unset: Mapping[tuple[int, int], str]) -> ProcessPlan: + raise FfrwdError(ErrorCode.UNSUPPORTED_SQL, "no such file") + + tap = tmp_path / "tap.ndjson" + tap.write_text( + '{"url": "a.mp4", "start_pts": 1.0, "duration": 1.0}\n' + '{"url": "b.mp4", "start_pts": 3.0, "duration": 1.0}\n', + encoding="utf-8", + ) + run = execute._LateralRun( + replace(_LATERAL, pipe=True), compile_instance, None, rows.append, None, None, + str(tap), + ) + try: + _until(lambda: len(rows) == 2) + finally: + run.stop() + run.join() + assert [(row["row"], row["refused"]) for row in rows] == [ + (1, "no such file"), + (2, "no such file"), + ] + + +# -- rows on a hold group's connection ------------------------------------------ + +PANEL = "modules/panel_node.wasm" +_PANEL = ( + "CREATE FUNCTION panel(v video_stream, feed video_stream DEFAULT NULL, " + "feed_audio audio_stream DEFAULT NULL, cues data_stream DEFAULT NULL, " + "port number DEFAULT NULL) RETURNS video_stream " + f"AS '{PANEL}', 'panel' LANGUAGE wasm;" +) +# What a feeder plays: the picture, the sound and the rows of one file. +_PLAY_ROWS = """CREATE FUNCTION play_rows(launch data_stream, url text) +RETURNS TABLE(video video_stream, audio audio_stream, cues data_stream) AS $$ + SELECT m.video[1] AS video, m.audio[1] AS audio, m.data[1] AS cues FROM input(url) m +$$ LANGUAGE sql;""" +_SHOWN = """COPY ( + WITH prog AS (SELECT s.video[1] AS v, s.data[1] AS d FROM input('leaf.nut') s) + SELECT panel(prog.v, ad.video, ad.audio, {cues}) + FROM prog, LATERAL play_rows(prog.d) ad +) TO 'out.nut'""" + + +def _show_shape( + module: str, params: str, bound: Sequence[shapes.Binding], grants: Sequence[str] = () +) -> shapes.NodeShape: + """A picture shown with a feed's picture, sound and cues, all of the group + `panel` and served on `port`.""" + held = {"kind": "hold", "anchor": {"kind": "shared_clock"}, "lead": 0.3, + "group": "panel", "port_param": "port"} + + def port(name: str, kind: str, pairing: dict[str, object]) -> dict[str, object]: + return { + "name": name, "kind": kind, "required": name == "v", "many": False, + "pairing": pairing, "rows": "per-frame" if kind == "data" else "ignore", + "window": 1, "stride": 1, "accepts": {}, + } + + return shapes.node_shape(module, { + "inputs": [ + port("v", "video", {"kind": "lockstep"}), + port("feed", "video", held), + port("feed_audio", "audio", held), + port("cues", "data", {"kind": "interval", "latency": 1, "ahead": 0, + "anchor": {"kind": "shared_clock"}, "group": "panel"}), + ], + "outputs": [{"name": "v", "kind": "video", "latency": 0, + "format": {"kind": "like", "port": "v"}}], + "clock": {"kind": "input", "port": "v"}, + "pure": True, "one_to_one": True, "bounded": True, "relation": [], + }) + + +def _shown(cues: str) -> ProcessPlan: + modules = { + **_MODULES, + PANEL: Described( + world="node-module", name="panel", + params_schema={"type": "object", "properties": {"port": {"type": "integer"}}}, + node=True, + ), + } + plan = compile_all( + _PANEL + "\n" + _PLAY_ROWS + "\n" + _SHOWN.format(cues=cues), + describe=lambda path: modules[path], + shape=_show_shape, + ).plan + assert plan is not None + return plan + + +def test_a_laterals_rows_ride_its_groups_connection_beside_the_picture_and_sound() -> None: + plan = _shown("ad.cues") + (lateral,) = plan.laterals + (connection,) = lateral.connections + (statement,) = lateral.template.split(";\n") + assert statement.startswith("COPY (SELECT ad.video, ad.audio, ad.cues FROM play_rows(") + assert f"TO 'tcp://127.0.0.1:{connection.port}'" in statement + (panel,) = [ + node + for process in plan.sidecars + if process.graph is not None + for node in process.graph.nodes.values() + if node.filter == "panel_node" + ] + assert panel.ports == ["v"] + assert panel.args["port"] == connection.port + unknown = {"rate": None} + assert json.loads(panel.bound) == [ + {"input": "v", "streams": [{"rate": {"num": 25, "den": 1}}]}, + {"input": "feed", "streams": [unknown]}, + {"input": "feed_audio", "streams": [unknown]}, + {"input": "cues", "streams": [unknown]}, + ] + instance = compile_all( + lateral.definitions + "\n" + statement.replace(":'url'", "'ad.nut'"), + describe=lambda path: _MODULES[path], + ) + ((sink,),) = [graph.sinks for graph in instance.graphs] + assert sink.path == f"tcp://127.0.0.1:{connection.port}" + assert [output.type for output in sink.outputs] == ["video", "audio", "data"] + + +def test_a_stream_bound_where_a_groups_connection_brings_the_rows_is_refused() -> None: + with pytest.raises(FfrwdError) as caught: + _shown("prog.d") + assert caught.value.message == ( + "panel() binds 'cues', which arrives on the connection of the group 'panel', " + "and 'feed' of that group is served by a port" + ) diff --git a/cli/tests/test_lower.py b/cli/tests/test_lower.py index f9f2c1c..1c85f5b 100644 --- a/cli/tests/test_lower.py +++ b/cli/tests/test_lower.py @@ -71,6 +71,7 @@ from ffrwd.warnings import FfrwdWarning, OnWarning, WarningCode from ffrwd.wasm import WORLDS, Described, SourceCatalog, SourceRendition from ffrwd.wasm import SourceTrack as WasmSourceTrack +from tests.conftest import older_world_refusal PROJECT_ROOT = Path(__file__).resolve().parent.parent REPO_ROOT = PROJECT_ROOT.parent @@ -14865,15 +14866,11 @@ def test_only_a_packet_filter_declares_several_annotation_columns() -> None: " RETURNS video_stream\n" f" AS '{ROWS_MODULE}', 'shots' LANGUAGE wasm;\n" ) - with pytest.raises(FfrwdError) as caught: - resolve( - parse( - declare - + "COPY (SELECT blur(f.video[1]) FROM input('f.mp4') f) TO 'o.mp4'" - ) - ) - assert caught.value.code is ErrorCode.UNSUPPORTED_SQL - assert "takes the annotation column 'b' in position 3" in caught.value.message + refusal = older_world_refusal( + declare + "COPY (SELECT blur(f.video[1]) FROM input('f.mp4') f) TO 'o.mp4'" + ) + assert refusal.code is ErrorCode.UNSUPPORTED_SQL + assert "takes the annotation column 'b' in position 3" in refusal.message def test_a_packets_call_inside_a_cte_body_is_refused() -> None: diff --git a/cli/tests/test_node_world.py b/cli/tests/test_node_world.py new file mode 100644 index 0000000..0730b0d --- /dev/null +++ b/cli/tests/test_node_world.py @@ -0,0 +1,1863 @@ +"""Tests for calls to node modules: declarations, shapes, ports and outputs. + +Bare-machine, as tests/test_data_filter.py is: every module is a synthetic +:class:`~ffrwd.wasm.Described` describing a node, every shape is a JSON +document read by :func:`ffrwd.shapes.node_shape` exactly as the sidecar's +``--shape`` answer is, and nothing is spawned. +""" + +from __future__ import annotations + +import functools +import json +import re +import subprocess +from collections.abc import Callable, Mapping, Sequence +from dataclasses import replace +from fractions import Fraction +from pathlib import Path + +import pytest + +from ffrwd import shapes, wasm +from ffrwd.compiler import compile_all +from ffrwd.errors import ErrorCode, FfrwdError +from ffrwd.execute import plan_argv +from ffrwd.ir import Graph +from ffrwd.lower import lower +from ffrwd.parser import parse, resolve +from ffrwd.probe import ProbeResult, StreamMeta +from ffrwd.processes import ProcessPlan +from ffrwd.registry import Registry, load_reference +from ffrwd.timing import check_live_leads, summary, timing +from ffrwd.warnings import FfrwdWarning, WarningCode +from ffrwd.wasm import WORLDS, Described + +SNAPSHOT_PATH = Path(__file__).resolve().parent / "data" / "reference_registry.json" + +_ROWS = { + "type": "object", + "properties": { + name: {"type": "number"} for name in ("start_t", "id", "x", "y", "w", "h") + }, +} +_BOX = { + "type": "object", + "properties": {name: {"type": "number"} for name in ("x", "y", "w", "h")}, +} +_CUE = { + "type": "object", + "properties": { + "text": {"type": "string"}, + "start_t": {"type": "number"}, + "end_t": {"type": "number"}, + }, +} + + +def _clock(name: str, kind: str = "video", window: int = 1) -> dict[str, object]: + return { + "name": name, + "kind": kind, + "required": True, + "many": False, + "pairing": {"kind": "lockstep"}, + "rows": "ignore", + "window": window, + "stride": window, + "accepts": {}, + } + + +def _input( + name: str, + kind: str, + pairing: dict[str, object], + *, + required: bool = False, + many: bool = False, + schema: Mapping[str, object] | None = None, +) -> dict[str, object]: + port: dict[str, object] = { + "name": name, + "kind": kind, + "required": required, + "many": many, + "pairing": pairing, + "rows": "per-frame" if kind == "data" else "ignore", + "window": 1, + "stride": 1, + "accepts": {}, + } + if schema is not None: + port["schema"] = json.dumps(schema) + return port + + +def _output( + name: str, kind: str, *, schema: Mapping[str, object] | None = None, latency: float = 0 +) -> dict[str, object]: + port: dict[str, object] = {"name": name, "kind": kind, "latency": latency} + if kind == "data": + port["format"] = {"kind": "data", "codec": "json"} + if schema is not None: + port["schema"] = json.dumps(schema) + return port + + +def _shape( + inputs: list[dict[str, object]], + outputs: list[dict[str, object]], + clock: dict[str, object], + **rest: object, +) -> dict[str, object]: + return { + "inputs": inputs, + "outputs": outputs, + "clock": clock, + "pure": True, + "one_to_one": False, + "bounded": True, + "relation": [], + **rest, + } + + +def _spot(params: Mapping[str, object], bound: Sequence[str]) -> dict[str, object]: + return _shape( + [_clock("v")], [_output("spots", "data", schema=_ROWS)], {"kind": "input", "port": "v"} + ) + + +def _reader(port: str, schema: Mapping[str, object]) -> Callable[..., dict[str, object]]: + def shaped(params: Mapping[str, object], bound: Sequence[str]) -> dict[str, object]: + return _shape( + [_clock("v"), _input(port, "data", {"kind": "lockstep"}, required=True, + schema=schema)], + [_output("v", "video")], + {"kind": "input", "port": "v"}, + ) + + return shaped + + +def _hear(params: Mapping[str, object], bound: Sequence[str]) -> dict[str, object]: + return _shape( + [_clock("a", "audio", window=96000)], + [_output("cues", "data", schema=_CUE)], + {"kind": "input", "port": "a"}, + ) + + +def _burn(params: Mapping[str, object], bound: Sequence[str]) -> dict[str, object]: + return _shape( + [ + _clock("v"), + _input("a", "audio", {"kind": "lockstep"}), + _input("words", "data", {"kind": "interval", "ahead": 0}, schema=_CUE), + ], + [_output("v", "video")], + {"kind": "input", "port": "v"}, + ) + + +def _inset(params: Mapping[str, object], bound: Sequence[str]) -> dict[str, object]: + hold = { + "kind": "hold", + "anchor": {"kind": "first-frame"}, + "lead": params.get("lead", 0.5), + "port_param": "port", + } + return _shape( + [_clock("v"), _input("feed", "video", hold)], + [_output("v", "video")], + {"kind": "input", "port": "v"}, + ) + + +def _tile(params: Mapping[str, object], bound: Sequence[str]) -> dict[str, object]: + hold = {"kind": "hold", "anchor": {"kind": "shared-clock"}, "lead": 0} + clock: dict[str, object] = ( + {"kind": "rate", "rate": {"num": int(str(params["fps"])), "den": 1}} + if "fps" in params + else {"kind": "rate-of", "port": "v"} + ) + return _shape( + [_input("v", "video", hold, required=True, many=True)], + [_output("v", "video")], + clock, + ) + + +def _matte(params: Mapping[str, object], bound: Sequence[str]) -> dict[str, object]: + return _shape( + [_clock("v")], + [_output("mask", "video"), _output("spots", "data", schema=_ROWS)], + {"kind": "input", "port": "v"}, + ) + + +def _ticker(params: Mapping[str, object], bound: Sequence[str]) -> dict[str, object]: + canvas = { + "kind": "video", + "width": params.get("width", 1280), + "height": params.get("height", 720), + "pix_fmt": "rgba", + } + return _shape( + [], + [{"name": "v", "kind": "video", "format": canvas, "latency": 0}], + {"kind": "rate", "rate": {"num": int(str(params.get("fps", 30))), "den": 1}}, + bounded=False, + ) + + +SHAPES: dict[str, Callable[..., dict[str, object]]] = { + "spot.wasm": _spot, + "ring.wasm": _reader("spots", _ROWS), + "dim.wasm": _reader("boxes", _BOX), + "hear.wasm": _hear, + "burn.wasm": _burn, + "inset.wasm": _inset, + "tile.wasm": _tile, + "matte.wasm": _matte, + "ticker.wasm": _ticker, +} + +_PARAMS = { + "spot.wasm": {"every": {"type": "number"}}, + "dim.wasm": {"amount": {"type": "number"}}, + "inset.wasm": {"port": {"type": "integer"}, "lead": {"type": "number"}}, + "tile.wasm": {"columns": {"type": "number"}, "fps": {"type": "number"}}, + "matte.wasm": {"every": {"type": "number"}}, + "ticker.wasm": { + name: {"type": "number" if name != "text" else "string"} + for name in ("text", "width", "height", "fps") + }, + "sub.wasm": {"relay": {"type": "string"}}, +} + + +def _node(path: str) -> Described: + return Described( + world="node-module", + name=path.removesuffix(".wasm"), + version="0.1.0", + params_schema={"type": "object", "properties": _PARAMS.get(path, {})}, + node=True, + ) + + +class _Asked: + """A fake `shape` seam that counts what it is asked: the bound inputs by + name in `asked`, and the bound list whole, hints and all, in `bound`.""" + + def __init__(self) -> None: + self.asked: list[tuple[str, dict[str, object], tuple[str, ...]]] = [] + self.bound: list[tuple[str, tuple[shapes.Binding, ...]]] = [] + + def __call__( + self, + module: str, + params: str, + bound: Sequence[shapes.Binding], + grants: Sequence[str] = (), + ) -> shapes.NodeShape: + decoded = json.loads(params) + names = tuple(binding.input for binding in bound) + self.asked.append((module, decoded, names)) + self.bound.append((module, tuple(bound))) + return shapes.node_shape(module, SHAPES[module](decoded, names)) + + +_DECLARATIONS = { + "spot": "CREATE FUNCTION spot(v video_stream, every number DEFAULT 30) " + "RETURNS STRUCT(start_t number, id number, x number, y number, w number, h number)[] " + "AS 'spot.wasm', 'spot' LANGUAGE wasm;", + "ring": "CREATE FUNCTION ring(v video_stream, spots STRUCT(start_t number, id number, " + "x number, y number, w number, h number)[]) RETURNS video_stream " + "AS 'ring.wasm', 'ring' LANGUAGE wasm;", + "dim": "CREATE FUNCTION dim(v video_stream, boxes STRUCT(x number, y number, " + "w number, h number)[], amount number DEFAULT 0.5) RETURNS video_stream " + "AS 'dim.wasm', 'dim' LANGUAGE wasm;", + "hear": "CREATE FUNCTION hear(a audio_stream) RETURNS cue[] " + "AS 'hear.wasm', 'hear' LANGUAGE wasm;", + "burn": "CREATE FUNCTION burn(v video_stream, a audio_stream DEFAULT NULL, " + "words cue[] DEFAULT NULL) RETURNS video_stream AS 'burn.wasm', 'burn' LANGUAGE wasm;", + "inset": "CREATE FUNCTION inset(v video_stream, feed video_stream DEFAULT NULL, " + "port number DEFAULT 9000, lead number DEFAULT 0.5) RETURNS video_stream " + "AS 'inset.wasm', 'inset' LANGUAGE wasm;", + "tile": "CREATE FUNCTION tile(v video_stream[], columns number DEFAULT 2, " + "fps number DEFAULT NULL) RETURNS video_stream AS 'tile.wasm', 'tile' LANGUAGE wasm;", + "matte": "CREATE FUNCTION matte(v video_stream, every number DEFAULT 30) " + "RETURNS STRUCT(mask video_stream, spots STRUCT(start_t number, id number, " + "x number, y number, w number, h number)[]) AS 'matte.wasm', 'matte' LANGUAGE wasm;", + "ticker": "CREATE FUNCTION ticker(text text, width number DEFAULT 1280, " + "height number DEFAULT 720, fps number DEFAULT 30) RETURNS source " + "AS 'ticker.wasm', 'ticker' LANGUAGE wasm;", +} + + +def _declared(query: str) -> str: + called = [text for name, text in _DECLARATIONS.items() if f"{name}(" in query] + return "\n".join([*called, query]) + + +@functools.cache +def _registry() -> Registry: + return load_reference(SNAPSHOT_PATH) + + +def _probes( + rate: int = 48000, colour: Mapping[str, str] | None = None, pix_fmt: str | None = None +) -> dict[str, ProbeResult | None]: + said = colour or {} + + def media() -> ProbeResult: + return ProbeResult( + streams=[ + StreamMeta( + type="video", index=0, metadata={}, width=320, height=240, + fps="25/1", sample_rate=None, codec="h264", pix_fmt=pix_fmt, + color_range=said.get("color_range"), + color_primaries=said.get("color_primaries"), + color_transfer=said.get("color_transfer"), + color_space=said.get("color_space"), + ), + StreamMeta( + type="audio", index=0, metadata={}, width=None, height=None, + fps=None, sample_rate=rate, codec="aac", channels=2, + ), + ] + ) + + return {"f": media(), "a": media(), "b": media(), "c": media()} + + +def _lowered( + query: str, + modules: Mapping[str, Described] | None = None, + asked: _Asked | None = None, + probes: Mapping[str, ProbeResult | None] | None = None, +) -> Graph: + return lower( + resolve(parse(_declared(query))), + dict(probes) if probes is not None else _probes(), + registry=_registry(), + describes=dict(modules) if modules is not None else {p: _node(p) for p in SHAPES}, + shapes=asked if asked is not None else _Asked(), + ) + + +def _refused(query: str, modules: Mapping[str, Described] | None = None) -> FfrwdError: + with pytest.raises(FfrwdError) as caught: + _lowered(query, modules) + return caught.value + + +_FROM = " FROM input('f.mp4') f) TO 'out.mkv'" + + +# -- the shape document ------------------------------------------------------ + + +def test_a_shape_document_reads_every_field_the_wit_names() -> None: + read = shapes.node_shape( + "m.wasm", + { + "inputs": [ + {**_clock("v"), "accepts": {"wants": "timing"}}, + _input( + "feed", + "video", + { + "kind": "hold", + "hold": { + "anchor": {"kind": "tagged", "tagged": "smart_timed"}, + "lead": 0.3, + "linger": 1.5, + "group": "switch", + "port_param": "port", + }, + }, + ), + _input( + "words", + "data", + { + "kind": "interval", + "latency": 2, + "ahead": 0.1, + "anchor": {"kind": "first_frame"}, + }, + schema=_CUE, + ), + _input("deals", "data", {"kind": "interval", "group": "switch"}), + ], + "outputs": [ + { + "name": "mask", + "kind": "video", + "format": {"kind": "like", "port": "v", "pixel_format": "gray"}, + "latency": 0, + }, + { + "name": "clock", + "kind": "data", + "format": {"kind": "data", "data": "json"}, + "time_base": {"num": 1, "den": 1000000}, + "latency": 0.25, + "row": 0, + }, + ], + "clock": {"kind": "rate", "rate": {"num": 30000, "den": 1001}}, + "pure": False, + "one_to_one": True, + "bounded": False, + "relation": ['{"name": "720p"}'], + }, + ) + clock, feed, words, deals = read.inputs + assert clock.accepts.wants == "timing" + assert feed.pairing.hold == shapes.Hold( + anchor=shapes.Anchor("tagged", "smart_timed"), + lead=0.3, + linger=1.5, + group="switch", + port_param="port", + ) + assert words.pairing.interval == shapes.Interval( + latency=2.0, ahead=0.1, anchor=shapes.Anchor("first-frame") + ) + assert deals.pairing.interval == shapes.Interval(group="switch") + assert (words.pairing.interval.retimed, deals.pairing.interval.retimed) == (True, False) + assert words.schema == _CUE + assert read.outputs[0].format == shapes.OutputFormat( + "like", port="v", pixel_format="gray" + ) + assert read.outputs[1].format == shapes.OutputFormat("data", codec="json") + assert (read.outputs[1].time_base, read.outputs[1].latency, read.outputs[1].row) == ( + (1, 1_000_000), + 0.25, + 0, + ) + assert read.clock == shapes.Clock("rate", rate=(30000, 1001)) + assert (read.pure, read.one_to_one, read.bounded) == (False, True, False) + assert read.relation == ({"name": "720p"},) + + +def test_a_shape_document_missing_its_ports_is_refused_naming_the_module() -> None: + with pytest.raises(FfrwdError) as caught: + shapes.node_shape("m.wasm", {"clock": {"kind": "self-clocked"}}) + assert "m.wasm" in caught.value.message + + +@pytest.mark.parametrize( + ("port", "words"), + [ + ((1, 1), "per-frame"), + ((96000, 96000), "tumbling 2 s"), + ((96000, 48000), "hopping 2 s every 1 s"), + ((15, 1), "sliding 0.6 s"), + ], +) +def test_a_window_is_said_in_streaming_words(port: tuple[int, int], words: str) -> None: + window, stride = port + kind = "audio" if window > 100 else "video" + rate = 48000 if kind == "audio" else 25 + read = shapes.node_shape( + "m.wasm", + _shape([{**_clock("x", kind, window), "stride": stride}], [], + {"kind": "input", "port": "x"}), + ) + from fractions import Fraction + + assert shapes.window_words(read.inputs[0], Fraction(rate)) == words + + +def test_rows_match_by_their_fields_and_extra_fields_pass() -> None: + assert shapes.row_mismatch(_BOX, _ROWS) is None + integer = {"type": "object", "properties": {"x": {"type": "integer"}}} + assert shapes.row_mismatch(_BOX, integer | {"properties": { + name: {"type": "integer"} for name in ("x", "y", "w", "h")}}) is None + assert shapes.row_mismatch(integer, _BOX) == ("x", "integer", "number") + assert shapes.row_mismatch(_CUE, _ROWS) == ("text", "string", "nothing") + + +def test_one_shape_is_asked_once_per_module_params_and_bound_list() -> None: + asked = _Asked() + cache = shapes.ShapeCache(asked) + at_25 = [shapes.Binding("v", (shapes.StreamHint(Fraction(25)),))] + cache("spot.wasm", "{}", at_25) + cache("spot.wasm", "{}", at_25) + cache("spot.wasm", '{"every": 5}', at_25) + cache("spot.wasm", "{}", [shapes.Binding("v", (shapes.StreamHint(Fraction(50)),))]) + assert [one[1] for one in asked.asked] == [{}, {"every": 5}, {}] + + +# -- declarations ------------------------------------------------------------ + + +def test_a_signature_only_a_node_reads_resolves_and_waits_for_the_module() -> None: + res = resolve(parse(_declared("COPY (SELECT burn(f.video[1], f.audio[1])" + _FROM))) + burn = res.wasm["burn"] + assert burn.is_node_only + assert [port.name for port in burn.ports] == ["v", "a", "words"] + assert burn.refusal is not None + old = Described(world=WORLDS[-1], name="burn", pixel_formats=("rgba",)) + error = _refused("COPY (SELECT burn(f.video[1], f.audio[1])" + _FROM, {"burn.wasm": old}) + assert error.message == burn.refusal.message + + +def test_a_module_of_an_older_world_refuses_a_rows_call_outside_from() -> None: + old = Described(world=WORLDS[-1], name="spot", video_codecs=("h264",)) + error = _refused("COPY (SELECT spot(f.video[1])" + _FROM, {"spot.wasm": old}) + assert "this call is not in FROM" in error.message + + +# -- ports and outputs ------------------------------------------------------- + + +def test_a_detector_returns_rows_and_the_reader_takes_the_picture_from_the_source() -> None: + graph = _lowered("COPY (SELECT ring(f.video[1], spot(f.video[1]))" + _FROM) + nodes = {node.filter: node for node in graph.nodes.values()} + spot, ring = nodes["spot.wasm"], nodes["ring.wasm"] + assert (spot.inputs, spot.ports, spot.outputs, spot.out_ports) == ( + ["src:f:v:0"], ["v"], ["data"], ["spots"] + ) + assert (ring.inputs, ring.ports) == (["src:f:v:0", spot.id], ["v", "spots"]) + assert graph.node_shapes[spot.id]["clock"] == {"kind": "input", "port": "v"} + + +def test_a_reader_naming_a_field_the_producer_lacks_is_refused_naming_both() -> None: + SHAPES["dim.wasm"] = _reader("boxes", { + "type": "object", "properties": {"x": {"type": "number"}, "z": {"type": "number"}}, + }) + try: + error = _refused("COPY (SELECT dim(f.video[1], spot(f.video[1]))" + _FROM) + finally: + SHAPES["dim.wasm"] = _reader("boxes", _BOX) + assert error.message == ( + "dim() reads 'z' as number on its 'boxes' input, and spot() does not write it" + ) + + +def test_one_call_read_twice_is_one_node() -> None: + graph = _lowered( + "COPY (SELECT burn(f.video[1], words => hear(f.audio[1])), f.audio[1], " + "hear(f.audio[1])" + _FROM + ) + hears = [node for node in graph.nodes.values() if node.filter == "hear.wasm"] + assert len(hears) == 1 + (burn,) = [node for node in graph.nodes.values() if node.filter == "burn.wasm"] + assert burn.inputs == ["src:f:v:0", hears[0].id] + assert burn.ports == ["v", "words"] + assert graph.rows_sinks[hears[0].id].container == "webvtt" + + +def test_a_nodes_rows_under_a_pinned_track_row_are_a_track_of_the_file() -> None: + graph = _lowered( + "COPY (SELECT v, spot(v) FROM input('f.mp4') f, unnest(f.video) v " + "WHERE v.index = 1) TO 'out.mkv'" + ) + (spot,) = [node for node in graph.nodes.values() if node.filter == "spot.wasm"] + assert graph.rows_sinks[spot.id].container == "webvtt" + assert [output.type for output in graph.sinks[0].outputs] == ["video", "subtitle"] + + +def test_kinds_mix_in_one_call_and_a_left_out_port_is_unbound() -> None: + asked = _Asked() + graph = _lowered( + "COPY (SELECT burn(f.video[1], f.audio[1], hear(f.audio[1])), " + "burn(f.video[1]) AS plain" + _FROM, + asked=asked, + ) + burns = [node for node in graph.nodes.values() if node.filter == "burn.wasm"] + assert [node.ports for node in burns] == [["v", "a", "words"], ["v"]] + assert [one[2] for one in asked.asked if one[0] == "burn.wasm"] == [ + ("v", "a", "words"), + ("v",), + ] + + +def _with_silent_source() -> dict[str, ProbeResult | None]: + probes = _probes() + media = probes["f"] + assert media is not None + probes["s"] = ProbeResult(streams=[one for one in media.streams if one.type == "video"]) + return probes + + +def _outer_joined(source: str, call: str) -> str: + return ( + f"COPY (SELECT {call} FROM input('{source}.mp4') {source}, unnest({source}.video) v " + f"LEFT JOIN unnest({source}.audio) a ON v.index = a.index) TO 'out.mkv'" + ) + + +@pytest.mark.parametrize( + ("source", "bound"), [("s", ["v"]), ("f", ["v", "a"])], ids=["silent", "with-sound"] +) +def test_a_stream_an_outer_join_leaves_null_leaves_a_default_null_port_unbound( + source: str, bound: list[str] +) -> None: + graph = _lowered(_outer_joined(source, "burn(v, a)"), probes=_with_silent_source()) + (burn,) = [node for node in graph.nodes.values() if node.filter == "burn.wasm"] + assert burn.ports == bound + + +def test_a_stream_an_outer_join_leaves_null_is_still_refused_to_a_required_port() -> None: + with pytest.raises(FfrwdError) as caught: + _lowered(_outer_joined("s", "burn(v, words => hear(a))"), probes=_with_silent_source()) + assert caught.value.message.startswith("'a' is NULL in row 1") + + +def test_a_held_input_left_unbound_keeps_its_port_param() -> None: + graph = _lowered("COPY (SELECT inset(f.video[1], port => 9100)" + _FROM) + (inset,) = [node for node in graph.nodes.values() if node.filter == "inset.wasm"] + assert (inset.ports, inset.args) == (["v"], {"port": 9100, "lead": 0.5}) + + +def test_a_port_number_where_a_held_input_goes_writes_its_port_param() -> None: + graph = _lowered("COPY (SELECT inset(f.video[1], 9200)" + _FROM) + (inset,) = [node for node in graph.nodes.values() if node.filter == "inset.wasm"] + assert (inset.ports, inset.args["port"]) == (["v"], 9200) + + +def test_an_array_fills_a_port_taking_many() -> None: + graph = _lowered( + "COPY (SELECT tile(ARRAY[a.video[1], b.video[1], c.video[1]], 3) " + "FROM input('a.mp4') a, input('b.mp4') b, input('c.mp4') c) TO 'out.mkv'" + ) + (tile,) = [node for node in graph.nodes.values() if node.filter == "tile.wasm"] + assert tile.inputs == ["src:a:v:0", "src:b:v:0", "src:c:v:0"] + assert tile.ports == ["v", "v", "v"] + + +def test_every_field_of_a_struct_return_is_an_output_of_one_node() -> None: + graph = _lowered( + "COPY (WITH m AS (SELECT (matte(f.video[1])).* FROM input('f.mp4') f) " + "SELECT dim(m.mask, m.spots), m.spots FROM m) TO 'out.mkv'" + ) + (matte,) = [node for node in graph.nodes.values() if node.filter == "matte.wasm"] + (dim,) = [node for node in graph.nodes.values() if node.filter == "dim.wasm"] + assert matte.out_ports == ["mask", "spots"] + assert dim.inputs == [f"{matte.id}:0", f"{matte.id}:1"] + assert graph.rows_sinks[f"{matte.id}:1"].container == "webvtt" + + +def test_a_declared_port_the_shape_has_none_for_is_refused_bound() -> None: + SHAPES["burn.wasm"] = lambda params, bound: _shape( + [_clock("v")], [_output("v", "video")], {"kind": "input", "port": "v"} + ) + try: + unbound = _lowered("COPY (SELECT burn(f.video[1])" + _FROM) + error = _refused("COPY (SELECT burn(f.video[1], f.audio[1])" + _FROM) + finally: + SHAPES["burn.wasm"] = _burn + assert any(node.filter == "burn.wasm" for node in unbound.nodes.values()) + assert error.message == ( + "burn() binds 'a', and for these params the module 'burn.wasm' reads no input 'a'" + ) + + +def test_a_port_the_module_requires_is_refused_unbound() -> None: + SHAPES["burn.wasm"] = lambda params, bound: _shape( + [_clock("v"), _input("a", "audio", {"kind": "lockstep"}, required=True)], + [_output("v", "video")], + {"kind": "input", "port": "v"}, + ) + try: + error = _refused("COPY (SELECT burn(f.video[1])" + _FROM) + finally: + SHAPES["burn.wasm"] = _burn + assert error.message == "burn() leaves 'a' out, and the module 'burn.wasm' requires it" + + +def test_a_gather_over_a_nodes_rows_narrows_them_on_the_data_edge() -> None: + graph = _lowered( + "COPY (SELECT dim(f.video[1], ARRAY(SELECT s FROM unnest(spot(f.video[1])) s " + "WHERE s.w > 20))" + _FROM + ) + (spot,) = [node for node in graph.nodes.values() if node.filter == "spot.wasm"] + (narrow,) = [node for node in graph.nodes.values() if node.filter == "rowfilter"] + (dim,) = [node for node in graph.nodes.values() if node.filter == "dim.wasm"] + assert (narrow.inputs, narrow.outputs) == ([spot.id], ["data"]) + assert dim.inputs == ["src:f:v:0", narrow.id] + + +def test_spans_reduce_a_nodes_rows_into_a_rows_file() -> None: + graph = _lowered( + "COPY (SELECT ffrwd.merge_spans(spot(f.video[1]), max_span => 10) " + "FROM input('f.mp4') f) TO 'spots.ndjson'" + ) + (spot,) = [node for node in graph.nodes.values() if node.filter == "spot.wasm"] + (spans,) = [node for node in graph.nodes.values() if node.filter == "rowmerge"] + assert (spans.inputs, spans.args) == ([spot.id], {"max_span": 10}) + assert graph.rows_sinks[spans.id].path == "spots.ndjson" + + +def test_a_node_reading_nothing_is_a_source_in_from() -> None: + graph = _lowered( + "COPY (SELECT s.video[1] FROM ticker('Nothing to see here') s WHERE s.t < 10) " + "TO 'ticker.mp4'" + ) + (ticker,) = [node for node in graph.nodes.values() if node.filter == "ticker.wasm"] + assert (ticker.inputs, ticker.out_ports) == ([], ["v"]) + assert ticker.args == {"text": "Nothing to see here", "width": 1280, "height": 720, + "fps": 30} + assert graph.node_sources == {"s": ticker.id} + assert [output.ref for output in graph.sinks[0].outputs] == [ticker.id] + + +def test_a_sql_function_returning_a_stream_and_rows_hands_a_node_both() -> None: + graph = _lowered( + "CREATE FUNCTION spotted(v video_stream) RETURNS STRUCT(v video_stream, " + "spots STRUCT(start_t number, id number, x number, y number, w number, h number)[]) " + "AS $$ SELECT v, spot(v) AS spots $$ LANGUAGE sql;\n" + "COPY (SELECT ring(spotted(f.video[1]))" + _FROM + ) + (spot,) = [node for node in graph.nodes.values() if node.filter == "spot.wasm"] + (ring,) = [node for node in graph.nodes.values() if node.filter == "ring.wasm"] + assert (ring.inputs, ring.ports) == (["src:f:v:0", spot.id], ["v", "spots"]) + + +def test_a_node_making_a_stream_and_its_rows_hands_a_reader_both() -> None: + graph = _lowered("COPY (SELECT ring(matte(f.video[1]))" + _FROM) + (matte,) = [node for node in graph.nodes.values() if node.filter == "matte.wasm"] + (ring,) = [node for node in graph.nodes.values() if node.filter == "ring.wasm"] + assert ring.inputs == [f"{matte.id}:0", f"{matte.id}:1"] + + +def test_a_struct_a_sql_function_returns_is_a_stream_and_rows_only() -> None: + with pytest.raises(FfrwdError) as caught: + resolve(parse( + "CREATE FUNCTION two(v video_stream) RETURNS STRUCT(a video_stream, " + "b video_stream) AS $$ SELECT v, v $$ LANGUAGE sql;\n" + "COPY (SELECT two(f.video[1]).a" + _FROM + )) + assert caught.value.message == "function 'two' returns a struct that is not a stream and rows" + + +# -- what each node waits for ------------------------------------------------ + + +def test_each_nodes_window_and_each_outputs_delay_add_up_along_the_path() -> None: + graph = _lowered( + "COPY (SELECT burn(f.video[1], f.audio[1], hear(f.audio[1])), f.audio[1]" + _FROM + ) + timed = timing(graph, _probes()) + assert timed is not None + by_module = {node.module: node for node in timed.nodes} + assert (by_module["hear.wasm"].window, by_module["hear.wasm"].delay) == ( + "tumbling 2 s", + 2.0, + ) + burn = by_module["burn.wasm"] + assert (burn.window, burn.delay) == ("per-frame", 2.0) + assert [(wait.port, wait.pairing, wait.delay) for wait in burn.waits] == [ + ("v", "clock", 0.0), + ("a", "lockstep", 0.0), + ("words", "interval", 2.0), + ] + picture, sound = timed.outputs + assert (picture.delay, picture.holds) == (2.0, 0.0) + assert (sound.ref, sound.delay, sound.holds, sound.held) == ("src:f:a:0", 0.0, 2.0, 768000) + + +def test_a_live_node_fed_later_than_its_bound_is_refused() -> None: + bounded = {"kind": "interval", "latency": 1.0, "ahead": 0} + + def burn(params: Mapping[str, object], bound: Sequence[str]) -> dict[str, object]: + return _shape( + [_clock("v"), _input("words", "data", bounded, schema=_CUE)], + [_output("v", "video")], + {"kind": "input", "port": "v"}, + ) + + SHAPES["burn.wasm"] = burn + try: + graph = _lowered("COPY (SELECT burn(f.video[1], words => hear(f.audio[1]))" + _FROM) + finally: + SHAPES["burn.wasm"] = _burn + with pytest.raises(FfrwdError) as caught: + check_live_leads(graph, _probes(), {"burn.wasm": (3, 4, "burn")}) + assert caught.value.code is ErrorCode.LIVE_LEAD + assert caught.value.message == ( + "burn() needs 'words' 1 s ahead of its clock, and the path feeding it runs 2 s behind" + ) + assert (caught.value.line, caught.value.col) == (3, 4) + + +# -- on the sidecar's command line ------------------------------------------- + + +def _plan( + query: str, + monkeypatch: pytest.MonkeyPatch, + rate: int = 48000, + colour: Mapping[str, str] | None = None, + describe: Callable[[str], Described] = _node, + pix_fmt: str | None = None, +) -> ProcessPlan: + probes = _probes(rate, colour, pix_fmt) + monkeypatch.setattr( + "ffrwd.compiler.probe_path", lambda path, args=(), **kw: probes[path[0]] + ) + compiled = compile_all(_declared(query), describe=describe, shape=_Asked()) + assert compiled.plan is not None + return compiled.plan + + +def _plan_argv( + query: str, + monkeypatch: pytest.MonkeyPatch, + rate: int = 48000, + colour: Mapping[str, str] | None = None, + describe: Callable[[str], Described] = _node, + pix_fmt: str | None = None, +) -> dict[str, list[str]]: + """Each process of the compiled plan as the printed command shows it.""" + return plan_argv( + _plan(query, monkeypatch, rate, colour, describe, pix_fmt), + sidecar_argv=wasm.shown_argv, + pipe_path=lambda edge, side: f"<{edge.source}-{edge.target} {side}>", + ) + + +def test_a_node_network_names_the_port_each_pad_binds(monkeypatch: pytest.MonkeyPatch) -> None: + argv = _plan_argv( + "COPY (SELECT ring(f.video[1], spot(f.video[1])) FROM input('f.mp4') f) " + "TO 'ringed.mp4'", + monkeypatch, + ) + sidecar = argv["sidecar0"] + assert sidecar[sidecar.index("-filter_complex") + 1] == ( + "[v=0:v]spot=every=30[spots=n1];[v=0:v][spots=n1]ring[v=out0]" + ) + assert sidecar[sidecar.index("-map") :] == ["-map", "[out0]", "-f", "nut", "pipe:1"] + at_25 = {"rate": {"num": 25, "den": 1}} + assert _bound_flags(sidecar) == [ + ("spot", [{"input": "v", "streams": [at_25]}]), + ("ring", [{"input": "v", "streams": [at_25]}, + {"input": "spots", "streams": [{"rate": None}]}]), + ] + + +def test_a_module_called_twice_carries_each_calls_list_in_chain_order( + monkeypatch: pytest.MonkeyPatch, +) -> None: + sidecar = _plan_argv( + "COPY (SELECT burn(f.video[1], f.audio[1]), burn(f.video[1]) AS plain " + "FROM input('f.mp4') f) TO 'both.mkv'", + monkeypatch, + )["sidecar0"] + chains = sidecar[sidecar.index("-filter_complex") + 1].split(";") + pads = [ + re.findall(r"\[(\w+)=[^\]]*\]", chain.split("burn")[0]) + for chain in chains + if "]burn" in chain + ] + bound = [ + [binding["input"] for binding in listed] + for name, listed in _bound_flags(sidecar) + if name == "burn" and isinstance(listed, list) + ] + assert pads == bound == [["v", "a"], ["v"]] + assert sidecar.index("-bound") > sidecar.index("-filter_complex") + + +def test_every_stream_one_process_hands_a_node_network_rides_one_nut( + monkeypatch: pytest.MonkeyPatch, +) -> None: + argv = _plan_argv( + "COPY (SELECT burn(f.video[1], f.audio[1], hear(f.audio[1])), f.audio[1] " + "FROM input('f.mp4') f) TO 'burned.mp4'", + monkeypatch, + ) + sidecar = argv["sidecar0"] + assert sidecar.count("-i") == 1 + assert sidecar[sidecar.index("-filter_complex") + 1] == ( + "[a=0:a]hear[cues=n1];[v=0:v][a=0:a][words=n1]burn[v=out0]" + ) + (feeder,) = [ + words for pid, words in argv.items() if pid.startswith("ffmpeg") and "nut" in words + and words[-1] == "pipe:1" + ] + assert feeder.count("-map") == 2, "both ports take the sound as it is, so it crosses once" + assert feeder[-3:] == ["-f", "nut", "pipe:1"] + assert "rate" not in _pad_after_input(sidecar) + assert _bound_flags(sidecar)[1] == ("burn", [ + {"input": "v", "streams": [{"rate": {"num": 25, "den": 1}}]}, + {"input": "a", "streams": [{"rate": {"num": 48000, "den": 1}}]}, + {"input": "words", "streams": [{"rate": None}]}, + ]) + + +def _taking( + shaped: Callable[..., dict[str, object]], kind: str, accepts: Mapping[str, object] +) -> Callable[..., dict[str, object]]: + """`shaped` with every input of `kind` accepting `accepts`.""" + + def taking(params: Mapping[str, object], bound: Sequence[str]) -> dict[str, object]: + shape = shaped(params, bound) + inputs = shape["inputs"] + assert isinstance(inputs, list) + for port in inputs: + if port["kind"] == kind: + port["accepts"] = dict(accepts) + return shape + + return taking + + +def _feeder(argv: Mapping[str, list[str]]) -> list[str]: + (feeder,) = [ + words for pid, words in argv.items() if pid.startswith("ffmpeg") and "nut" in words + and words[-1] == "pipe:1" + ] + return feeder + + +def test_a_stream_two_nodes_read_crosses_in_the_format_both_take( + monkeypatch: pytest.MonkeyPatch, +) -> None: + rgba = {"pixel_formats": ["rgba"]} + monkeypatch.setitem(SHAPES, "spot.wasm", _taking(_spot, "video", rgba)) + monkeypatch.setitem(SHAPES, "ring.wasm", _taking(_reader("spots", _ROWS), "video", rgba)) + feeder = _feeder(_plan_argv( + "COPY (SELECT ring(f.video[1], spot(f.video[1])) FROM input('f.mp4') f) " + "TO 'ringed.mp4'", + monkeypatch, + )) + assert feeder[feeder.index("-pix_fmt:0") + 1] == "rgba" + + +def test_a_picture_written_beside_the_nodes_reading_it_crosses_to_them_once( + monkeypatch: pytest.MonkeyPatch, +) -> None: + argv = _plan_argv( + "COPY (SELECT ring(f.video[1], spot(f.video[1])), f.video[1] " + "FROM input('f.mp4') f) TO 'both.mkv'", + monkeypatch, + ) + assert _feeder(argv).count("-map") == 1 + sidecar = argv["sidecar0"] + assert sidecar[sidecar.index("-filter_complex") + 1] == ( + "[v=0:v]spot=every=30[spots=n1];[v=0:v][spots=n1]ring[v=out0]" + ) + + +@pytest.mark.parametrize("beside", ["", ", f.video[1]"], ids=["alone", "written-beside"]) +def test_nodes_taking_one_picture_in_different_formats_get_a_stream_each( + monkeypatch: pytest.MonkeyPatch, beside: str +) -> None: + yuv, rgba = {"pixel_formats": ["yuv420p"]}, {"pixel_formats": ["rgba"]} + monkeypatch.setitem(SHAPES, "spot.wasm", _taking(_spot, "video", yuv)) + monkeypatch.setitem(SHAPES, "ring.wasm", _taking(_reader("spots", _ROWS), "video", rgba)) + argv = _plan_argv( + f"COPY (SELECT ring(f.video[1], spot(f.video[1])){beside} " + "FROM input('f.mp4') f) TO 'both.mkv'", + monkeypatch, + ) + feeder = _feeder(argv) + assert [feeder[at + 1] for at, word in enumerate(feeder) if word.startswith("-pix_fmt")] == [ + "yuv420p", + "rgba", + ] + sidecar = argv["sidecar0"] + assert sidecar[sidecar.index("-filter_complex") + 1] == ( + "[v=0:v]spot=every=30[spots=n1];[v=0:v:1][spots=n1]ring[v=out0]" + ) + + +def test_each_stream_of_one_nut_is_conformed_to_the_port_it_feeds( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem(SHAPES, "hear.wasm", _taking( + _hear, "audio", {"sample_formats": ["f32"], "sample_rates": [48000]} + )) + monkeypatch.setitem(SHAPES, "burn.wasm", _taking(_burn, "audio", {"sample_formats": ["f32"]})) + feeder = _feeder(_plan_argv( + "COPY (SELECT burn(f.video[1], f.audio[1], hear(f.audio[1])), f.audio[1] " + "FROM input('f.mp4') f) TO 'burned.mp4'", + monkeypatch, + rate=44100, + )) + assert [word for word in feeder if word.startswith("-ar")] == ["-ar:0"] + assert feeder[feeder.index("-ar:0") + 1] == "48000" + + +def test_a_node_read_in_from_has_no_input_and_its_reader_ends_it( + monkeypatch: pytest.MonkeyPatch, +) -> None: + argv = _plan_argv( + "COPY (SELECT s.video[1] FROM ticker('Nothing to see here') s WHERE s.t < 10) " + "TO 'ticker.mp4'", + monkeypatch, + ) + assert "-i" not in argv["sidecar0"] + reader = argv["ffmpeg0"] + assert reader[reader.index("-to") + 1] == "10" + + +def test_a_node_source_whose_relation_is_its_renditions_ends_where_its_reader_says( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def ticker(params: Mapping[str, object], bound: Sequence[str]) -> dict[str, object]: + shape = _ticker(params, bound) + outputs = shape["outputs"] + assert isinstance(outputs, list) + outputs[0]["row"] = 0 + shape["relation"] = [json.dumps({"width": 1280, "height": 720})] + return shape + + monkeypatch.setitem(SHAPES, "ticker.wasm", ticker) + argv = _plan_argv( + "COPY (SELECT s.video[1] FROM ticker('Nothing to see here') s " + "WHERE s.height = 720 AND s.t < 10) TO 'ticker.mp4'", + monkeypatch, + ) + reader = argv["ffmpeg0"] + assert reader[reader.index("-to") + 1] == "10" + + +_BT709_PC = { + "color_range": "pc", + "color_primaries": "bt709", + "color_transfer": "bt709", + "color_space": "bt709", +} +_RING = "COPY (SELECT ring(f.video[1], spot(f.video[1])) FROM input('f.mp4') f) TO 'ringed.mp4'" + + +def _pad_after_input(sidecar: Sequence[str]) -> dict[str, object]: + """The ``-pad`` JSON written right after the sidecar's one ``-i``.""" + at = sidecar.index("-i") + 2 + assert sidecar[at] == "-pad" + pad = json.loads(sidecar[at + 1]) + assert isinstance(pad, dict) + return pad + + +def _bound_flags(sidecar: Sequence[str]) -> list[tuple[str, object]]: + """Each ``-bound =`` of a sidecar's argv, in order.""" + found: list[tuple[str, object]] = [] + for at, word in enumerate(sidecar): + if word == "-bound": + name, _, written = sidecar[at + 1].partition("=") + found.append((name, json.loads(written))) + return found + + +def _taking_pictures(monkeypatch: pytest.MonkeyPatch, pixel_format: str) -> None: + taken = {"pixel_formats": [pixel_format]} + monkeypatch.setitem(SHAPES, "spot.wasm", _taking(_spot, "video", taken)) + monkeypatch.setitem(SHAPES, "ring.wasm", _taking(_reader("spots", _ROWS), "video", taken)) + + +def test_a_yuv_picture_into_a_node_network_carries_the_probed_colour( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _taking_pictures(monkeypatch, "yuv420p") + argv = _plan_argv(_RING, monkeypatch, colour=_BT709_PC) + assert _pad_after_input(argv["sidecar0"])["color"] == { + "range": "pc", "primaries": "bt709", "trc": "bt709", "space": "bt709" + } + + +def test_a_picture_the_probe_says_nothing_of_carries_unknown_colour( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _taking_pictures(monkeypatch, "yuv420p") + argv = _plan_argv(_RING, monkeypatch) + assert _pad_after_input(argv["sidecar0"])["color"] == { + "range": "unknown", "primaries": "unknown", "trc": "unknown", "space": "unknown" + } + + +def test_a_picture_converted_to_rgb_on_its_way_carries_what_the_conversion_wrote( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _taking_pictures(monkeypatch, "rgba") + argv = _plan_argv(_RING, monkeypatch, colour=_BT709_PC) + assert _pad_after_input(argv["sidecar0"])["color"] == { + "range": "pc", "primaries": "bt709", "trc": "bt709", "space": "gbr" + } + + +def test_the_tags_a_query_writes_on_a_stream_reach_the_node_reading_it( + monkeypatch: pytest.MonkeyPatch, +) -> None: + argv = _plan_argv( + "COPY (WITH ad AS (SELECT v AS v, STRUCT('1' AS smart_timed) AS tags " + "FROM input('a.mp4') a, unnest(a.video) v) " + "SELECT ring(ad.v[1], spot(ad.v[1])) FROM ad) TO 'ringed.mp4'", + monkeypatch, + ) + assert _pad_after_input(argv["sidecar0"])["tags"] == {"smart_timed": "1"} + + +def test_a_star_over_a_call_reads_the_outputs_its_shape_makes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def mask_alone(params: Mapping[str, object], bound: Sequence[str]) -> dict[str, object]: + return _shape([_clock("v")], [_output("mask", "video")], {"kind": "input", "port": "v"}) + + monkeypatch.setitem(SHAPES, "matte.wasm", mask_alone) + graph = _lowered( + "COPY (WITH m AS (SELECT (matte(f.video[1])).* FROM input('f.mp4') f) " + "SELECT m.mask FROM m) TO 'out.mkv'" + ) + (matte,) = [node for node in graph.nodes.values() if node.filter == "matte.wasm"] + assert [output.ref for output in graph.sinks[0].outputs] == [matte.id] + + +def test_a_node_network_hands_one_process_every_stream_it_reads_on_one_nut( + monkeypatch: pytest.MonkeyPatch, +) -> None: + argv = _plan_argv( + "COPY (WITH m AS (SELECT f.video[1] AS v, (matte(f.video[1])).* " + "FROM input('f.mp4') f) SELECT dim(m.v, m.spots), m.mask FROM m) TO 'matte.mkv'", + monkeypatch, + ) + sidecar = argv["sidecar0"] + assert sidecar.count("-i") == 1 + assert sidecar[sidecar.index("-map") :] == [ + "-map", "[out0]", "-map", "[out1]", "-f", "nut", "pipe:1", + ] + assert _feeder(argv).count("-map") == 1 + + +def _gray_matte(params: Mapping[str, object], bound: Sequence[str]) -> dict[str, object]: + shape = _matte(params, bound) + outputs = shape["outputs"] + assert isinstance(outputs, list) + outputs[0]["format"] = {"kind": "like", "port": "v", "pixel_format": "gray"} + return shape + + +def test_a_node_handed_another_nodes_output_in_a_format_it_does_not_take_is_refused( + monkeypatch: pytest.MonkeyPatch, +) -> None: + rgba = {"pixel_formats": ["rgba"]} + monkeypatch.setitem(SHAPES, "matte.wasm", _taking(_gray_matte, "video", rgba)) + monkeypatch.setitem(SHAPES, "dim.wasm", _taking(_reader("boxes", _BOX), "video", rgba)) + with pytest.raises(FfrwdError) as caught: + _plan_argv( + "COPY (WITH m AS (SELECT (matte(f.video[1])).* FROM input('f.mp4') f) " + "SELECT dim(m.mask, m.spots) FROM m) TO 'dimmed.mkv'", + monkeypatch, + ) + assert caught.value.message == ( + "function 'dim': the module 'dim.wasm' takes rgba on 'v', and the 'mask' " + "output of matte hands it gray" + ) + assert caught.value.hint is not None + assert "ffmpeg.format(, pix_fmts => 'rgba')" in caught.value.hint + + +def test_an_ffmpeg_filter_between_two_nodes_hands_the_reader_its_format( + monkeypatch: pytest.MonkeyPatch, +) -> None: + rgba = {"pixel_formats": ["rgba"]} + monkeypatch.setitem(SHAPES, "matte.wasm", _taking(_gray_matte, "video", rgba)) + monkeypatch.setitem(SHAPES, "dim.wasm", _taking(_reader("boxes", _BOX), "video", rgba)) + argv = _plan_argv( + "COPY (WITH m AS (SELECT (matte(f.video[1])).* FROM input('f.mp4') f) " + "SELECT dim(ffmpeg.format(m.mask, pix_fmts => 'rgba'), m.spots) FROM m) " + "TO 'dimmed.mkv'", + monkeypatch, + ) + (converts,) = [words for words in argv.values() if "format=pix_fmts=rgba" in str(words)] + assert converts[converts.index("-pix_fmt:0") + 1] == "rgba" + + +def test_spans_are_the_hosts_rowmerge_with_its_span_written_as_rows( + monkeypatch: pytest.MonkeyPatch, +) -> None: + argv = _plan_argv( + "COPY (SELECT ffrwd.merge_spans(spot(f.video[1]), max_span => 10) " + "FROM input('f.mp4') f) TO 'spots.ndjson'", + monkeypatch, + ) + sidecar = argv["sidecar0"] + assert sidecar[sidecar.index("-filter_complex") + 1] == ( + "[v=0:v]spot=every=30[spots=n1];[n1]rowmerge=max_span=10[out0]" + ) + assert sidecar[-5:] == ["-map", "[out0]", "-f", "ndjson", "spots.ndjson"] + + +def test_spans_without_a_span_are_refused() -> None: + with pytest.raises(FfrwdError) as caught: + _lowered("COPY (SELECT ffrwd.merge_spans(spot(f.video[1])) FROM input('f.mp4') f) " + "TO 'spots.ndjson'") + assert caught.value.message == "ffrwd.merge_spans() needs 'max_span'" + + +_SPANS = { + "type": "object", + "properties": {"start_t": {"type": "number"}, "end_t": {"type": "number"}, + "id": {"type": "integer"}}, +} +_MASK = ( + "CREATE FUNCTION mask(v video_stream, spans STRUCT(start_t number, end_t number, " + "id number)[]) RETURNS video_stream AS 'mask.wasm', 'mask' LANGUAGE wasm;\n" +) + + +def test_spans_are_read_with_the_end_and_id_the_merge_writes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem(SHAPES, "mask.wasm", _reader("spans", _SPANS)) + graph = _lowered( + _MASK + "COPY (SELECT mask(f.video[1], ffrwd.merge_spans(spot(f.video[1]), " + "max_span => 10))" + _FROM + ) + (spans,) = [node for node in graph.nodes.values() if node.filter == "rowmerge"] + (mask,) = [node for node in graph.nodes.values() if node.filter == "mask.wasm"] + assert mask.inputs == ["src:f:v:0", spans.id] + + +def test_rows_read_with_an_end_they_do_not_carry_are_still_refused( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem(SHAPES, "mask.wasm", _reader("spans", _SPANS)) + with pytest.raises(FfrwdError) as caught: + _lowered(_MASK + "COPY (SELECT mask(f.video[1], spot(f.video[1]))" + _FROM) + assert caught.value.message == ( + "mask() reads 'end_t' as number on its 'spans' input, and spot() does not write it" + ) + + +def test_explain_delays_says_each_window_and_each_outputs_delay() -> None: + graph = _lowered( + "COPY (SELECT burn(f.video[1], f.audio[1], hear(f.audio[1])), f.audio[1]" + _FROM + ) + timed = timing(graph, _probes(), {"hear.wasm": "hear", "burn.wasm": "burn"}) + assert timed is not None + assert summary(timed).splitlines() == [ + "hear: tumbling 2 s", + "burn: per-frame; words by interval, no bound", + "out.mkv stream 0 (video): 2 s behind the source", + "out.mkv stream 1 (audio): 0 s behind the source, waits 2 s", + ] + + +# -- coded packets ----------------------------------------------------------- + + +def _subscribe(params: Mapping[str, object], bound: Sequence[str]) -> dict[str, object]: + def coded(name: str, codec: str, row: int, carried: dict[str, object]) -> dict[str, object]: + return { + "name": name, + "kind": "packets", + "format": { + "kind": "packets", + "codec": codec, + "time_base": {"num": 1, "den": 90000}, + "format": carried, + "extradata": "", + "profile": None, + "level": None, + }, + "latency": 0, + "row": row, + } + + return _shape( + [], + [ + coded("hd", "h264", 0, {"kind": "video", "width": 1280, "height": 720}), + coded("hd_audio", "aac", 0, {"kind": "audio", "sample_rate": 48000, "channels": 2}), + coded("sd", "h264", 1, {"kind": "video", "width": 640, "height": 360}), + ], + {"kind": "self_clocked"}, + bounded=False, + relation=['{"name": "720p", "bandwidth": 3000000}', '{"name": "360p"}'], + ) + + +def _remux(params: Mapping[str, object], bound: Sequence[str]) -> dict[str, object]: + coded = {**_clock("v"), "kind": "packets", "accepts": {"codecs": ["h264"]}} + return _shape( + [coded], + [{"name": "v", "kind": "packets", "format": {"kind": "like", "port": "v"}, "latency": 0}], + {"kind": "input", "port": "v"}, + ) + + +def test_a_node_source_writing_coded_packets_binds_one_row_per_rendition() -> None: + SHAPES["sub.wasm"] = _subscribe + try: + graph = _lowered( + "CREATE FUNCTION sub(relay text) RETURNS source AS 'sub.wasm', 'sub' LANGUAGE wasm;\n" + "COPY (SELECT v.video[1] FROM sub('r') v WHERE v.height = 720) TO 'out.mkv'", + {"sub.wasm": _node("sub.wasm")}, + ) + finally: + del SHAPES["sub.wasm"] + (sub,) = [node for node in graph.nodes.values() if node.filter == "sub.wasm"] + assert sub.outputs == ["video", "audio", "video"] + assert [output.ref for output in graph.sinks[0].outputs] == [f"{sub.id}:0"] + + +def test_a_node_sources_coded_streams_keep_their_time_at_a_file( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem(SHAPES, "sub.wasm", _subscribe) + argv = _plan_argv( + "CREATE FUNCTION sub(relay text) RETURNS source AS 'sub.wasm', 'sub' LANGUAGE wasm;\n" + "COPY (SELECT v.video[1], v.audio[1] FROM sub('r') v WHERE v.height = 720) " + "TO 'out.mkv'", + monkeypatch, + ) + assert argv["ffmpeg0"][:2] == ["ffmpeg", "-copyts"] + + +def _arrival(name: str, kind: str, codecs: Sequence[str] = ()) -> dict[str, object]: + return { + "name": name, "kind": kind, "required": name == "video", "many": True, + "pairing": {"kind": "arrival"}, "rows": "ignore", "window": 1, "stride": 1, + "accepts": {"codecs": list(codecs)}, + } + + +def _publish(params: Mapping[str, object], bound: Sequence[str]) -> dict[str, object]: + """ffrwd/moq's publish: coded pictures, coded sound and data, by arrival.""" + return _shape( + [_arrival("video", "packets", ["h264"]), _arrival("audio", "packets", ["aac"]), + _arrival("data", "data")], + [], + {"kind": "rate", "num": 50, "den": 1}, + ) + + +_PUBLISH = ( + "CREATE FUNCTION publish(relay text, broadcast text) RETURNS sink " + "AS 'publish.wasm', 'publish' LANGUAGE wasm;\n" +) + + +def _reporting(*reporting: str) -> Callable[[str], Described]: + """`_node`, with the modules named emitting rows of their own.""" + + def describe(path: str) -> Described: + described = _node(path) + if path not in reporting: + return described + return replace(described, rows_schema={"type": "object"}) + + return describe + + +def test_a_node_at_a_copys_to_reads_the_select_as_a_packet_sink_did( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem(SHAPES, "publish.wasm", _publish) + monkeypatch.setitem( + _PARAMS, "publish.wasm", {"relay": {"type": "string"}, "broadcast": {"type": "string"}} + ) + argv = _plan_argv( + _PUBLISH + "COPY (SELECT f.video[1], f.audio[1] FROM input('f.mp4') f) " + "TO publish('https://relay', 'b') WITH (video_codec 'libx264', audio_codec 'aac')", + monkeypatch, + describe=_reporting("publish.wasm"), + ) + sidecar = argv["sidecar0"] + assert sidecar.count("-i") == 2 and sidecar.count("-pad") == 2 + network = sidecar[sidecar.index("-filter_complex") + 1] + assert network.startswith("[video=0:v][audio=") + assert network.endswith("]publish=relay=https\\\\://relay:broadcast=b[@rows=out0]") + assert sidecar[-5:] == ["-map", "[out0]", "-f", "ndjson", "pipe:1"] + (encoder,) = [words for pid, words in argv.items() if pid.startswith("ffmpeg")] + assert encoder[encoder.index("-c:0") + 1] == "libx264" + + +def test_a_nodes_picture_into_a_node_sink_is_encoded_and_its_data_is_not_copied( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """ring's raw picture reaches publish through the encoder the WITH shapes, + and spot's messages go straight from one network to the other, an input of + their own.""" + monkeypatch.setitem(SHAPES, "publish.wasm", _publish) + monkeypatch.setitem( + _PARAMS, "publish.wasm", {"relay": {"type": "string"}, "broadcast": {"type": "string"}} + ) + argv = _plan_argv( + _PUBLISH + + "CREATE FUNCTION spotted(v video_stream) RETURNS data_stream " + "AS 'spot.wasm', 'spot' LANGUAGE wasm;\n" + "COPY (SELECT ring(f.video[1], spot(f.video[1])), spotted(f.video[1]) AS spots " + "FROM input('f.mp4') f) TO publish('https://relay', 'b') " + "WITH (video_codec 'libx264')", + monkeypatch, + ) + (encoder,) = [ + words for pid, words in argv.items() if pid.startswith("ffmpeg") and "-c:0" in words + and words[words.index("-c:0") + 1] == "libx264" + ] + assert "-copyts" in encoder + sink = next(words for words in argv.values() if "publish=publish.wasm" in words) + assert sink.count("-i") == 2 and sink.count("-pad") == 2 + copies = [ + words for pid, words in argv.items() + if pid.startswith("ffmpeg") and "-map" in words + and words[words.index("-map") + 1].endswith(":d:0") + ] + assert copies == [] + + +def test_a_node_sink_is_shaped_again_with_every_stream_the_select_binds( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem(SHAPES, "publish.wasm", _publish) + monkeypatch.setitem( + _PARAMS, "publish.wasm", {"relay": {"type": "string"}, "broadcast": {"type": "string"}} + ) + asked = _Asked() + graph = _lowered( + _PUBLISH + + "CREATE FUNCTION spotted(v video_stream) RETURNS data_stream " + "AS 'spot.wasm', 'spot' LANGUAGE wasm;\n" + "COPY (SELECT f.video[1], f.audio[1], spotted(f.video[1]) AS spots " + "FROM input('f.mp4') f) TO publish('https://relay', 'b') " + "WITH (video_codec 'libx264', audio_codec 'aac')", + asked=asked, + ) + assert _hints(asked, "publish.wasm") == [ + {}, + {"video": [Fraction(25)], "audio": [Fraction(48000)], "data": [None]}, + ] + (sink,) = [node for node in graph.nodes.values() if node.filter == "publish.wasm"] + assert [binding["input"] for binding in json.loads(sink.bound)] == [ + "video", "audio", "data" + ] + + +def test_the_rows_a_node_emits_are_the_runs_on_its_stdout( + monkeypatch: pytest.MonkeyPatch, +) -> None: + argv = _plan_argv( + "COPY (SELECT ring(f.video[1], spot(f.video[1])) FROM input('f.mp4') f) " + "TO 'ringed.mp4'", + monkeypatch, + describe=_reporting("spot.wasm"), + ) + sidecar = argv["sidecar0"] + assert "[@rows=out1]" in sidecar[sidecar.index("-filter_complex") + 1] + assert sidecar[-5:] == ["-map", "[out1]", "-f", "ndjson", "pipe:1"] + + +def test_a_node_reading_coded_packets_is_handed_the_stream_as_it_was_coded( + monkeypatch: pytest.MonkeyPatch, +) -> None: + SHAPES["remux.wasm"] = _remux + _DECLARATIONS["remux"] = ( + "CREATE FUNCTION remux(v video_stream) RETURNS video_stream " + "AS 'remux.wasm', 'remux' LANGUAGE wasm;" + ) + try: + argv = _plan_argv( + "COPY (SELECT remux(f.video[1]) FROM input('f.mp4') f) TO 'out.mkv'", monkeypatch + ) + finally: + del SHAPES["remux.wasm"] + del _DECLARATIONS["remux"] + feeder = argv["ffmpeg1"] + assert feeder[feeder.index("-c:0") + 1] == "copy" + reader = argv["ffmpeg0"] + assert reader[reader.index("-c:0") + 1] == "copy" + + +def test_params_too_long_for_a_command_line_are_read_from_a_file( + monkeypatch: pytest.MonkeyPatch, +) -> None: + long = "x" * (shapes.PARAMS_INLINE_LIMIT + 1) + argv = _plan_argv( + f"COPY (SELECT s.video[1] FROM ticker('{long}') s) TO 'ticker.mp4'", monkeypatch + ) + sidecar = argv["sidecar0"] + assert sidecar[sidecar.index("-filter_complex") + 1] == "ticker[v=out0]" + assert sidecar[sidecar.index("-params-from") + 1] == "ticker=ffrwd:params:sidecar0:n1" + + +def test_a_packets_function_over_a_node_hands_back_the_stream_still_coded( + monkeypatch: pytest.MonkeyPatch, +) -> None: + SHAPES["remux.wasm"] = _remux + _DECLARATIONS["remux"] = ( + "CREATE FUNCTION remux(v video_stream) RETURNS packets " + "AS 'remux.wasm', 'remux' LANGUAGE wasm;" + ) + try: + argv = _plan_argv( + "COPY (SELECT remux(f.video[1]) FROM input('f.mp4') f) TO 'out.mkv'", monkeypatch + ) + finally: + del SHAPES["remux.wasm"] + del _DECLARATIONS["remux"] + sidecar = argv["sidecar0"] + assert sidecar[sidecar.index("-filter_complex") + 1] == "[v=0:v]remux[v=out0]" + assert argv["ffmpeg1"][argv["ffmpeg1"].index("-c:0") + 1] == "copy" + + +def test_a_live_query_feeding_a_node_later_than_its_bound_is_refused_at_compile( + monkeypatch: pytest.MonkeyPatch, +) -> None: + bounded = {"kind": "interval", "latency": 1.0, "ahead": 0} + + def burn(params: Mapping[str, object], bound: Sequence[str]) -> dict[str, object]: + return _shape( + [_clock("v"), _input("words", "data", bounded, schema=_CUE)], + [_output("v", "video")], + {"kind": "input", "port": "v"}, + ) + + live = _probes()["f"] + monkeypatch.setattr("ffrwd.compiler.probe_path", lambda path, args=(), **kw: live) + SHAPES["burn.wasm"] = burn + try: + with pytest.raises(FfrwdError) as caught: + compile_all( + _declared( + "COPY (SELECT burn(f.video[1], words => hear(f.audio[1])) " + "FROM input('srt://127.0.0.1:9000') f) TO 'out.mkv'" + ), + describe=_node, + shape=_Asked(), + ) + finally: + SHAPES["burn.wasm"] = _burn + assert caught.value.code is ErrorCode.LIVE_LEAD + assert (caught.value.line, caught.value.col) == (2, 17) + + +def test_a_stream_waiting_long_beside_a_later_one_is_warned_about( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def hear(params: Mapping[str, object], bound: Sequence[str]) -> dict[str, object]: + return _shape( + [_clock("a", "audio", window=48000 * 1500)], + [_output("cues", "data", schema=_CUE)], + {"kind": "input", "port": "a"}, + ) + + probes = _probes() + monkeypatch.setattr( + "ffrwd.compiler.probe_path", lambda path, args=(), **kw: probes[path[0]] + ) + said: list[FfrwdWarning] = [] + SHAPES["hear.wasm"] = hear + try: + compile_all( + _declared( + "COPY (SELECT burn(f.video[1], words => hear(f.audio[1])), f.audio[1] " + "FROM input('f.mp4') f) TO 'out.mkv'" + ), + describe=_node, + shape=_Asked(), + on_warning=said.append, + ) + finally: + SHAPES["hear.wasm"] = _hear + assert [warning.code for warning in said] == [WarningCode.HELD_STREAM] + + +def test_a_field_only_a_node_makes_is_refused_for_a_module_of_an_older_world() -> None: + old = Described(world=WORLDS[-1], name="matte", pixel_formats=("rgba",)) + error = _refused("COPY (SELECT matte(f.video[1]).mask" + _FROM, {"matte.wasm": old}) + assert error.message == ( + "'.mask' is the stream matte() was handed, and a stream is not read back off a struct" + ) + + +def _weave(params: Mapping[str, object], bound: Sequence[str]) -> dict[str, object]: + coded = {**_clock("v"), "kind": "packets", "accepts": {"codecs": ["h264"]}} + clip = _input("clip", "data", {"kind": "interval", "ahead": 0}, schema=_ROWS) + return _shape( + [coded, *([clip] if "clip" in bound else [])], + [{"name": "v", "kind": "packets", "format": {"kind": "like", "port": "v"}, "latency": 0}], + {"kind": "input", "port": "v"}, + ) + + +def test_a_node_reading_packets_reads_what_its_destination_encodes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + SHAPES["weave.wasm"] = _weave + _DECLARATIONS["weave"] = ( + "CREATE FUNCTION weave(v video_stream, clip STRUCT(start_t number, id number, " + "x number, y number, w number, h number)[] DEFAULT NULL) RETURNS packets " + "AS 'weave.wasm', 'weave' LANGUAGE wasm;" + ) + try: + argv = _plan_argv( + "COPY (SELECT weave(f.video[1], clip => spot(f.video[1])) FROM input('f.mp4') f) " + "TO 'out.mkv' WITH (video_codec 'libx264', crf 20)", + monkeypatch, + ) + finally: + del SHAPES["weave.wasm"] + del _DECLARATIONS["weave"] + (weave,) = [words for words in argv.values() if "weave=weave.wasm" in words] + assert weave[weave.index("-filter_complex") + 1] == "[v=0:v][clip=1:d]weave[v=out0]" + (encoder,) = [words for words in argv.values() if "libx264" in words] + assert encoder[encoder.index("-c:0") + 1] == "libx264" + (writer,) = [words for words in argv.values() if "out.mkv" in words] + assert writer[writer.index("-c:0") + 1] == "copy" + + +# -- 0.19.1: timing inputs, stream hints, re-timed and grouped data --------- + + +_BOXES_MASK = ( + "CREATE FUNCTION boxes_mask(v video_stream, boxes STRUCT(x number, y number, " + "w number, h number)[]) RETURNS video_stream " + "AS 'boxes_mask.wasm', 'boxes_mask' LANGUAGE wasm;\n" +) +_TIMING = {"wants": "timing", "pixel_formats": ["yuv420p"]} +_RGBA = {"pixel_formats": ["rgba"]} + + +def _with_boxes_mask(monkeypatch: pytest.MonkeyPatch) -> None: + """boxes_mask: the picture as its clock, read for its timing alone.""" + monkeypatch.setitem( + SHAPES, "boxes_mask.wasm", _taking(_reader("boxes", _BOX), "video", _TIMING) + ) + monkeypatch.setitem(SHAPES, "spot.wasm", _taking(_spot, "video", _RGBA)) + + +def test_a_timing_input_beside_a_reader_of_the_same_picture_binds_its_stream( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _with_boxes_mask(monkeypatch) + argv = _plan_argv( + _BOXES_MASK + "COPY (SELECT boxes_mask(f.video[1], spot(f.video[1])) " + "FROM input('f.mp4') f) TO 'mask.mkv'", + monkeypatch, + ) + feeder = _feeder(argv) + assert feeder.count("-map") == 1 + assert feeder[feeder.index("-pix_fmt:0") + 1] == "rgba" + sidecar = argv["sidecar0"] + assert sidecar[sidecar.index("-filter_complex") + 1] == ( + "[v=0:v]spot=every=30[spots=n1];[v=0:v][boxes=n1]boxes_mask[v=out0]" + ) + + +def test_a_timing_input_alone_takes_the_picture_in_the_format_it_has( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem(SHAPES, "spot.wasm", _taking(_spot, "video", _TIMING)) + feeder = _feeder(_plan_argv( + "COPY (SELECT spot(f.video[1]) FROM input('f.mp4') f) TO 'spots.ndjson'", + monkeypatch, + pix_fmt="yuv444p", + )) + assert feeder[feeder.index("-pix_fmt:0") + 1] == "yuv444p" + + +@pytest.mark.parametrize(("source", "named"), [("yuv420p", "yuv444p"), ("yuv444p", "yuv420p")]) +def test_a_timing_input_after_a_format_takes_the_format_it_names( + monkeypatch: pytest.MonkeyPatch, source: str, named: str +) -> None: + """The stream at that point of the plan is the formatted one, so it crosses + as it is: never converted back to the source's format for a port that + reads no pixels.""" + monkeypatch.setitem(SHAPES, "spot.wasm", _taking(_spot, "video", _TIMING)) + argv = _plan_argv( + "COPY (SELECT spot(ffmpeg.format(f.video[1], pix_fmts => '" + + named + + "')) FROM input('f.mp4') f) TO 'spots.ndjson'", + monkeypatch, + pix_fmt=source, + ) + (writer,) = [words for pid, words in argv.items() if pid.startswith("ffmpeg")] + assert writer[writer.index("-filter_complex") + 1].endswith(f"format=pix_fmts={named}[out0]") + assert writer[writer.index("-pix_fmt:0") + 1] == named + _with_boxes_mask(monkeypatch) + mixed = _plan_argv( + _BOXES_MASK + "COPY (SELECT boxes_mask(ffmpeg.format(f.video[1], pix_fmts => '" + + named + + "'), spot(f.video[1])) FROM input('f.mp4') f) TO 'mask.mkv'", + monkeypatch, + pix_fmt=source, + ) + (feeder,) = [ + words for pid, words in mixed.items() if pid.startswith("ffmpeg") and "-pix_fmt:1" in words + ] + assert feeder[feeder.index("-pix_fmt:0") + 1] == "rgba" + assert feeder[feeder.index("-pix_fmt:1") + 1] == named + + +def test_explain_says_timing_for_an_input_read_for_its_timing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _with_boxes_mask(monkeypatch) + graph = _lowered( + _BOXES_MASK + "COPY (SELECT boxes_mask(f.video[1], spot(f.video[1]))" + _FROM, + {path: _node(path) for path in SHAPES}, + ) + timed = timing(graph, _probes(), {"boxes_mask.wasm": "boxes_mask", "spot.wasm": "spot"}) + assert timed is not None + (masked,) = [node for node in timed.nodes if node.called == "boxes_mask"] + assert masked.to_dict()["inputs"] == [ + {"port": "v", "pairing": "clock", "delay": 0.0, "wants": "timing"}, + {"port": "boxes", "pairing": "lockstep", "delay": 0.0}, + ] + assert summary(timed).splitlines()[1] == "boxes_mask: per-frame; v for its timing" + + +def _hints(asked: _Asked, module: str) -> list[dict[str, list[Fraction | None]]]: + return [ + {binding.input: [hint.rate for hint in binding.streams] for binding in bound} + for called, bound in asked.bound + if called == module + ] + + +def test_the_shape_is_told_each_bound_streams_rate() -> None: + asked = _Asked() + _lowered( + "COPY (SELECT burn(f.video[1], f.audio[1], hear(f.audio[1])), f.audio[1]" + _FROM, + asked=asked, + ) + assert _hints(asked, "hear.wasm") == [{"a": [Fraction(48000)]}] + assert _hints(asked, "burn.wasm") == [ + {"v": [Fraction(25)], "a": [Fraction(48000)], "words": [None]} + ] + tiled = _Asked() + _lowered( + "COPY (SELECT tile(ARRAY[a.video[1], b.video[1], c.video[1]], 3) " + "FROM input('a.mp4') a, input('b.mp4') b, input('c.mp4') c) TO 'tiled.mp4'", + asked=tiled, + ) + assert _hints(tiled, "tile.wasm") == [{"v": [Fraction(25)] * 3}] + + +def test_a_nodes_picture_is_hinted_at_its_clock_over_its_stride( + monkeypatch: pytest.MonkeyPatch, +) -> None: + + def hopping(params: Mapping[str, object], bound: Sequence[str]) -> dict[str, object]: + shape = _matte(params, bound) + inputs = shape["inputs"] + assert isinstance(inputs, list) + inputs[0] = {**inputs[0], "window": 4, "stride": 2} + return shape + + monkeypatch.setitem(SHAPES, "matte.wasm", hopping) + asked = _Asked() + graph = _lowered( + "COPY (WITH m AS (SELECT (matte(f.video[1])).* FROM input('f.mp4') f) " + "SELECT dim(m.mask, m.spots) FROM m) TO 'dimmed.mkv'", + asked=asked, + ) + assert _hints(asked, "dim.wasm") == [{"v": [Fraction(25, 2)], "boxes": [None]}] + (dim,) = [node for node in graph.nodes.values() if node.filter == "dim.wasm"] + assert json.loads(dim.bound) == [ + {"input": "v", "streams": [{"rate": {"num": 25, "den": 2}}]}, + {"input": "boxes", "streams": [{"rate": None}]}, + ] + ticked = _Asked() + _lowered( + "COPY (SELECT ring(s.video[1], spot(s.video[1])) FROM ticker('hi', fps => 30) s) " + "TO 'ringed.mp4'", + asked=ticked, + ) + assert _hints(ticked, "spot.wasm") == [{"v": [Fraction(30)]}] + + +def test_a_sound_conformed_to_its_port_is_hinted_at_the_rate_it_arrives_at( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem(SHAPES, "hear.wasm", _taking(_hear, "audio", {"sample_rates": [48000]})) + asked = _Asked() + _lowered( + "COPY (SELECT hear(f.audio[1])" + _FROM.replace("out.mkv", "cues.ndjson"), + asked=asked, + probes=_probes(rate=44100), + ) + assert _hints(asked, "hear.wasm") == [{"a": [Fraction(44100)]}, {"a": [Fraction(48000)]}] + sidecar = _plan_argv( + "COPY (SELECT hear(f.audio[1]) FROM input('f.mp4') f) TO 'cues.ndjson'", + monkeypatch, + rate=44100, + )["sidecar0"] + assert _bound_flags(sidecar) == [ + ("hear", [{"input": "a", "streams": [{"rate": {"num": 48000, "den": 1}}]}]) + ] + + +def test_the_bound_list_reaches_the_sidecar_as_json(monkeypatch: pytest.MonkeyPatch) -> None: + ran: list[list[str]] = [] + + def run( + sidecar: str, module: str, argv: list[str], budget: float + ) -> subprocess.CompletedProcess[str]: + ran.append(argv) + return subprocess.CompletedProcess(argv, 0, json.dumps(_spot({}, ["v"])), "") + + monkeypatch.setattr(shapes.binaries, "ffrwd_wasm_path", lambda: "ffrwd-wasm") + monkeypatch.setattr(shapes, "_run_shape", run) + shapes.shape( + "spot.wasm", + "{}", + [ + shapes.Binding("v", (shapes.StreamHint(Fraction(30000, 1001)),)), + shapes.Binding("words", (shapes.StreamHint(),)), + ], + ) + (argv,) = ran + assert json.loads(argv[argv.index("--bound") + 1]) == [ + {"input": "v", "streams": [{"rate": {"num": 30000, "den": 1001}}]}, + {"input": "words", "streams": [{"rate": None}]}, + ] + + +def test_an_input_the_host_re_times_waits_its_bound_and_not_its_producer() -> None: + retimed = { + "kind": "interval", + "latency": 1.0, + "ahead": 0, + "anchor": {"kind": "first_frame"}, + } + + def burn(params: Mapping[str, object], bound: Sequence[str]) -> dict[str, object]: + return _shape( + [_clock("v"), _input("words", "data", retimed, schema=_CUE)], + [_output("v", "video")], + {"kind": "input", "port": "v"}, + ) + + SHAPES["burn.wasm"] = burn + try: + graph = _lowered("COPY (SELECT burn(f.video[1], words => hear(f.audio[1]))" + _FROM) + finally: + SHAPES["burn.wasm"] = _burn + check_live_leads(graph, _probes(), {"burn.wasm": (3, 4, "burn")}) + timed = timing(graph, _probes(), {"hear.wasm": "hear", "burn.wasm": "burn"}) + assert timed is not None + assert summary(timed).splitlines()[:2] == [ + "hear: tumbling 2 s", + "burn: per-frame; words by interval on its own clock, at most 1 s", + ] + assert timed.outputs[0].delay == 1.0 + + +def test_a_node_runs_on_one_worker_only_where_its_shape_says_it_is_impure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def impure(params: Mapping[str, object], bound: Sequence[str]) -> dict[str, object]: + return {**_matte(params, bound), "pure": False} + + monkeypatch.setitem(SHAPES, "matte.wasm", impure) + plan = _plan( + "COPY (WITH m AS (SELECT f.video[1] AS v, (matte(f.video[1])).* " + "FROM input('f.mp4') f) SELECT dim(m.v, m.spots), ring(m.v, spot(m.v)) FROM m) " + "TO 'both.mkv'", + monkeypatch, + ) + (sidecar,) = plan.sidecars + assert sidecar.impure == ("matte.wasm",) diff --git a/cli/tests/test_packages.py b/cli/tests/test_packages.py index 573cca1..30220c6 100644 --- a/cli/tests/test_packages.py +++ b/cli/tests/test_packages.py @@ -52,10 +52,11 @@ read_linksfile, read_lockfile, read_manifest, + write_linksfile, write_lockfile, ) -QUERY = "COPY (SELECT {call}(f.audio[1]) FROM input('film.mkv') f) TO 'out.mkv'" +QUERY ="COPY (SELECT {call}(f.audio[1]) FROM input('film.mkv') f) TO 'out.mkv'" def _quieter(factor: str) -> str: @@ -1005,6 +1006,40 @@ def test_bare_install_leaves_a_dependency_that_is_linked_to_a_directory( assert code == 0 and "volume=volume=0.5" in out +def test_bare_install_drops_a_pin_a_link_made_since_answers_for( + store_home: Path, + registry: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """The link wins on install as it does on resolve: the lock comes out as + one written with the link already standing.""" + monkeypatch.setenv(packages.REGISTRY_ENV, str(registry)) + _publish(registry, _package(tmp_path / "pub", factor="0.5")) + project = _package( + tmp_path / "work", + name="consumer/mine", + dependencies={"broadcast/tracks": "1.0.0"}, + ) + assert _run(project, monkeypatch, capsys, "install")[0] == 0 + lock = project / "ffrwd.lock" + assert read_lockfile(lock).dependencies == {"broadcast/tracks": "1.0.0"} + _package(tmp_path / "dev", factor="0.25") + write_linksfile(links_path(lock), [LinkEntry(path="../dev")]) + + code, _out, err = _run(project, monkeypatch, capsys, "install") + assert code == 0, err + held = read_lockfile(lock) + assert held.dependencies == {} + assert held.entries == () + + code, out, _err = _run( + project, monkeypatch, capsys, "compile", QUERY.format(call="broadcast.tracks.quieter") + ) + assert code == 0 and "volume=volume=0.25" in out + + def test_bare_install_moves_an_old_lockfile_link_into_the_links_file( store_home: Path, registry: Path, diff --git a/cli/tests/test_packet_rows.py b/cli/tests/test_packet_rows.py index 43da97d..2616b9a 100644 --- a/cli/tests/test_packet_rows.py +++ b/cli/tests/test_packet_rows.py @@ -17,13 +17,14 @@ import os import shutil import subprocess +from collections.abc import Sequence from dataclasses import replace from pathlib import Path from typing import cast import pytest -from ffrwd import binaries +from ffrwd import binaries, shapes from ffrwd.compiler import compile_sql, compile_table_sql from ffrwd.emit import build_ffmpeg_args, emit from ffrwd.errors import ErrorCode, FfrwdError @@ -41,6 +42,7 @@ DescribedFunction, PacketRead, SinkWants, + _node_reader_argv, copy_argv, read_packet_rows, ) @@ -638,6 +640,101 @@ def test_a_module_that_is_not_a_packet_sink_is_refused_by_name() -> None: assert "is not a packet sink" in error.message +_BOUND_15 = '[{"input":"v","streams":[{"rate":{"num":15,"den":1}}]}]' + + +def _node_described() -> Described: + """A node reading the packets: what its describe says, rows_schema and all.""" + return Described(world="node-module", name="keys", rows_schema=_ROWS_SCHEMA, node=True) + + +def _node_shape(*, kind: str = "packets", outputs: int = 0) -> shapes.Shape: + def asked( + module: str, params: str, bound: Sequence[shapes.Binding], grants: Sequence[str] = () + ) -> shapes.NodeShape: + port = { + "name": "v", + "kind": kind, + "required": True, + "many": False, + "pairing": {"kind": "lockstep"}, + "rows": "ignore", + "window": 1, + "stride": 1, + "accepts": {"codecs": ["h264"], "wants": "keyframes"}, + } + made = [{"name": "v", "kind": "video", "latency": 0}] * outputs + return shapes.node_shape( + module, + { + "inputs": [port], + "outputs": made, + "clock": {"kind": "input", "port": "v"}, + "pure": True, + "one_to_one": False, + "bounded": True, + "relation": [], + }, + ) + + return asked + + +def _node_rows(sql: str, reads: _Reads, shape: shapes.Shape) -> list[list[object]]: + sinks = lower_table( + resolve(parse(_DECLARE + sql)), + {"f": _probe()}, + registry=load_reference(_SNAPSHOT_PATH), + describes={_MODULE: _node_described()}, + read_packets=reads, + shapes=shape, + ) + return sinks[0].result.rows + + +def test_a_node_reading_the_packets_is_read_in_from_as_a_packet_sink_is() -> None: + """Its rows are the ones it emits beside its ports, and how much of the + stream it is handed is what its port asks for.""" + reads = _Reads() + assert _node_rows( + "SELECT v.index FROM input('f.mp4') f, keys(f.video[1]) v", reads, _node_shape() + ) == [[1], [2], [3]] + (read,) = reads.reads + assert (read.port, read.wants, read.bound) == ("v", "keyframes", _BOUND_15) + + +@pytest.mark.parametrize( + ("shape", "said"), + [ + (_node_shape(kind="video"), "reads video on 'v'"), + (_node_shape(outputs=1), "makes outputs of its own"), + ], + ids=["frames", "outputs"], +) +def test_a_node_that_does_not_only_read_packets_is_refused_in_from( + shape: shapes.Shape, said: str +) -> None: + with pytest.raises(FfrwdError) as caught: + _node_rows("SELECT v.index FROM input('f.mp4') f, keys(f.video[1]) v", _Reads(), shape) + assert caught.value.message == ( + f"function 'keys' returns rows read off a stream's packets, and the module " + f"'{_MODULE}' {said}" + ) + + +def test_a_node_read_binds_its_port_and_maps_the_rows_it_emits(tmp_path: Path) -> None: + read = PacketRead( + spec="f.mp4", input_args=(), kind="video", index=0, module=_MODULE, params="", + wants="keyframes", port="v", bound=_BOUND_15, + ) + assert _node_reader_argv("ffrwd-wasm", read, None, tmp_path) == [ + "ffrwd-wasm", "-f", "nut", "-i", "pipe:0", "-m", f"read={_MODULE}", + "-filter_complex", "[v=0:v]read[@rows=out0]", + "-bound", "read=" + _BOUND_15, + "-map", "[out0]", "-f", "ndjson", "pipe:1", + ] + + def test_a_declared_column_the_module_never_writes_is_refused() -> None: error = _refuses( "SELECT v.index FROM input('f.mp4') f, keys(f.video[1]) v", diff --git a/cli/tests/test_project.py b/cli/tests/test_project.py index 11138d4..ff80e63 100644 --- a/cli/tests/test_project.py +++ b/cli/tests/test_project.py @@ -27,7 +27,7 @@ import pytest -from ffrwd import cli, packages, store, wasm +from ffrwd import cli, packages, store from ffrwd.compiler import compile_commands, compile_sql, compile_table_sql from ffrwd.emit import build_ffmpeg_args, emit from ffrwd.errors import ErrorCode, FfrwdError @@ -3430,7 +3430,7 @@ def test_init_refuses_a_reserved_namespace( assert "reserved" in err -def test_init_rust_writes_a_module_package_that_reads_back( +def test_init_rust_writes_a_node_module_package_that_reads_back( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: root = tmp_path / "my-filter" @@ -3439,12 +3439,13 @@ def test_init_rust_writes_a_module_package_that_reads_back( root, monkeypatch, capsys, "init", "--verbose", "--rust", "--namespace", "me" ) assert code == 0 - assert "cargo build --target wasm32-wasip2 --release" in out + assert "cargo build --release --target wasm32-wasip2" in out + assert "ffrwd install" not in out package = read_manifest(root / "ffrwd.json") assert package.name == "me/my_filter" and package.version == "0.1.0" - assert list(package.exports) == ["invert"] and list(package.recipes) == ["invert"] - assert dict(package.dependencies) == {"ffrwd/wasm": wasm.WORLD_VERSION} + assert list(package.exports) == ["passthrough"] and list(package.recipes) == ["passthrough"] + assert dict(package.dependencies) == {"ffrwd/wasm": "0.19.1"} # Declared empty rather than absent: the scaffold shows its author where # they go. assert package.keywords == () and package.capabilities == () @@ -3459,12 +3460,11 @@ def test_init_rust_writes_a_module_package_that_reads_back( ".gitignore", "Cargo.toml", "README.md", - "build.rs", "ffrwd.json", "ffrwd.lock", - "recipes/invert.sql", - "src/invert.sql", + "recipes/passthrough.sql", "src/lib.rs", + "src/passthrough.sql", } @@ -3482,17 +3482,53 @@ def _text(name: str) -> str: return (root / name).read_text(encoding="utf-8") assert 'name = "my_filter"' in _text("Cargo.toml") - assert "target/wasm32-wasip2/release/my_filter.wasm" in _text("src/invert.sql") - assert "LANGUAGE wasm" in _text("src/invert.sql") - assert "me.my_filter.invert(" in _text("recipes/invert.sql") - assert "-- variables:" in _text("recipes/invert.sql") - assert "-- example: ffrwd run invert" in _text("recipes/invert.sql") - # The wit comes from the environment or from the installed package, and - # lands where the bindings macro reads it. - assert "FFRWD_WIT_DIR" in _text("build.rs") - assert 'args(["path", WIT_PACKAGE])' in _text("build.rs") - assert 'path: "wit"' in _text("src/lib.rs") - assert _text(".gitignore").split() == ["target/", "wit/"] + assert "target/wasm32-wasip2/release/my_filter.wasm" in _text("src/passthrough.sql") + assert "LANGUAGE wasm" in _text("src/passthrough.sql") + assert "me.my_filter.passthrough(" in _text("recipes/passthrough.sql") + assert "-- variables:" in _text("recipes/passthrough.sql") + assert "-- example: ffrwd run passthrough" in _text("recipes/passthrough.sql") + # The module's own name is the export the lib SQL names. + assert 'const NAME: &\'static str = "passthrough";' in _text("src/lib.rs") + assert "'passthrough' LANGUAGE wasm" in _text("src/passthrough.sql") + assert _text(".gitignore").split() == ["target/"] + + +def test_init_rust_writes_a_crate_on_the_node_sdk_with_no_build_script( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + root = tmp_path / "my-filter" + root.mkdir() + assert _run(root, monkeypatch, capsys, "init", "--rust", "--namespace", "me")[0] == 0 + cargo = (root / "Cargo.toml").read_text(encoding="utf-8") + source = (root / "src" / "lib.rs").read_text(encoding="utf-8") + + assert ( + 'ffrwd-node = { git = "https://github.com/imbcmdth/ffrwd-node", tag = "v0.2.0" }' + in cargo + ) + assert ( + 'ffrwd-frame = { git = "https://github.com/imbcmdth/ffrwd-frame", tag = "v0.1.1" }' + in cargo + ) + assert 'crate-type = ["cdylib"]' in cargo + assert "wit-bindgen" not in cargo + assert not (root / "build.rs").exists() and not (root / "wit").exists() + + # One video input is the clock; the one output is that input's picture. + assert "impl Node for" in source + assert 'Input::video("v").clock()' in source + assert 'Output::like("v")' in source + assert "ffrwd_node::export!(" in source + # The one comment says where the work goes. + comments = [line.strip() for line in source.splitlines() if line.strip().startswith("//")] + assert len(comments) == 1 and "work goes here" in comments[0] + + readme = (root / "README.md").read_text(encoding="utf-8") + assert "cargo build --release --target wasm32-wasip2" in readme + assert "install" not in readme + for path in root.rglob("*"): + if path.is_file(): + assert "\u2014" not in path.read_text(encoding="utf-8"), path.name def test_init_rust_refuses_to_overwrite_any_file_it_would_write( diff --git a/cli/tests/test_wasm.py b/cli/tests/test_wasm.py index 399893e..2646883 100644 --- a/cli/tests/test_wasm.py +++ b/cli/tests/test_wasm.py @@ -37,7 +37,7 @@ ) from ffrwd.errors import ErrorCode, FfrwdError from ffrwd.execute import CHAIN, PIPELINE, PipeEdge, plan_argv, render_plan -from ffrwd.functions import WasmFunction, package_modules +from ffrwd.functions import Parameter, WasmFunction, package_modules from ffrwd.ir import ROWFILTER, Graph, RowsSink from ffrwd.lower import lower, lower_table from ffrwd.parser import ModuleExport, Resolved, parse, resolve @@ -193,6 +193,7 @@ def test_a_declaration_rides_out_on_the_resolved_query() -> None: returns="video_stream", line=declared.line, col=declared.col, + outputs=(Parameter("", "video_stream"),), ) assert [(p.name, p.type) for p in declared.params] == [("v", "video_stream")] diff --git a/docs/corpus.md b/docs/corpus.md index bde8826..6993ee2 100644 --- a/docs/corpus.md +++ b/docs/corpus.md @@ -2276,7 +2276,9 @@ COPY ( ``` $ ffrwd compile -f query.sql -v dest=source.mp4 -ffrwd-wasm -m ../sidecar/modules/target/wasm32-wasip2/release/source_replay.wasm -track 0 -f nut pipe:1 | ffmpeg -copyts -f nut -analyzeduration 0 -fpsprobesize 3 -i pipe:0 -map 0:v:0 -c:0 libx264 -crf:0 20 source.mp4 +ffrwd-wasm -m ../sidecar/modules/target/wasm32-wasip2/release/source_replay.wasm -track \ + 0 -f nut pipe:1 | ffmpeg -copyts -f nut -analyzeduration 0 -fpsprobesize 3 -i pipe:0 \ + -map 0:v:0 -c:0 libx264 -crf:0 20 source.mp4 ``` Reach for this for a module that manufactures a stream rather than filtering one - a synthetic test pattern, a packet source pulled off a socket - anywhere a query wants to name it as an input alongside, or instead of, a probed file. diff --git a/docs/dialect.md b/docs/dialect.md index b5be84e..79d81b5 100644 --- a/docs/dialect.md +++ b/docs/dialect.md @@ -33,12 +33,19 @@ function := CREATE FUNCTION name(param ptype [DEFAULT literal], ...) RETURNS rty AS 'module', 'export' LANGUAGE wasm | CREATE FUNCTION name(rows annotation) RETURNS annotation AS 'module', 'export' LANGUAGE wasm + | CREATE FUNCTION name(port ntype [DEFAULT NULL] | param vtype + [DEFAULT literal], ...) + RETURNS nrtype AS 'module', 'export' LANGUAGE wasm ptype := text | number | boolean | vector | _stream | chapter | cue | attachment | any of those with [] | STRUCT(field vtype, ...)[] rtype := text | number | boolean | vector | _stream | chapter | cue | attachment | any of those with [] | TABLE(col type, ...) + | STRUCT(name wstype, name annotation) wstype := video_stream | audio_stream | either of those with [] wrtype := wstype | sink | packets | STRUCT(name wstype, name annotation) +ntype := wstype | data_stream | annotation | any of those with [] +nrtype := wstype | data_stream | annotation | source + | STRUCT(name wstype | data_stream | annotation, ...) annotation := STRUCT(field vtype, ...)[] | cue[] vtype := text | number | boolean | vector select := [WITH cte (, cte)*] SELECT columns FROM from [WHERE pred] @@ -161,14 +168,22 @@ dest := 'path' | STDOUT | ( value-expression ) | sink(value, ...) expanded at compile time: its body is compiled and run once per message, while the query runs. It is declared `play(launch data_stream, url text, start_pts number, ..., channels number DEFAULT 2) RETURNS - TABLE(video video_stream, audio audio_stream)`, either column or both; - every parameter after the data stream is a text, number or boolean - value, and the body does not read the data stream. + TABLE(video video_stream, audio audio_stream)`, either column or both, + and a `data_stream` column beside them for the rows a node's data input + reads on the same connection; every parameter after the data stream is + a text, number or boolean value, and the body does not read the data + stream. - Its streams are empty between messages, so they go to a feeder and nowhere else: `ffrwd.switch.video(prog.v, ads.video)`. A filter, a module's pad or a `COPY` reading one is refused. Feeders of one group fed by its picture and its sound share one connection, which an instance writes as one NUT, picture then sound. + - A node's data input whose `interval` names a hold group the lateral + feeds (`interval.group`) takes the lateral's rows on that group's + connection, after its picture and sound in the same NUT, and binds no + pad: `panel(prog.v, ad.video, ad.audio, ad.cues)`. A stream of the + query's own handed to such an input while a port serves the group is + refused; its rows would never arrive. - Each value is bound by name, per message, in this order: an argument the call wrote, which is a constant the same for every message; the message's field of that name (a number for a number, a string for @@ -205,10 +220,11 @@ dest := 'path' | STDOUT | ( value-expression ) | sink(value, ...) the feeder connections its instances write, what binds each value, and the instance as SQL with each value bound per message a hole, ``. -- **`--jobs N`**, on `compile` and `run`, caps the sidecar's worker - threads at N. The sidecar runs a pool sized to the machine's cores by - default, and a module that describes itself as pure spreads across it - with no flag at all; one that carries state between calls, and one +- **`--jobs N`**, on `compile` and `run`, sets the sidecar's worker + threads to N, never more than the machine's cores. The sidecar runs a + pool of 4 by default, and a module that describes itself as pure + spreads across it with no flag at all; one that carries state between + calls, and one reading encoded packets, run one call at a time whatever N is. `--jobs 1` hosts everything serially. The output is byte-identical at any N. @@ -499,6 +515,126 @@ dest := 'path' | STDOUT | ( value-expression ) | sink(value, ...) `moq.subscribe`'s rows); read the stream as `.v[1]` there. A `RETURNS sink` reading several streams takes no annotation column. Recipe [144](examples.md#144-hand-one-modules-rows-to-a-module-reading-two-streams). +- A **node `LANGUAGE wasm` function** names a module exporting + `ffrwd:av@0.19.1`'s `node`, which its describe says (`"world": + "node-module"`). What a node reads and writes is its SHAPE for each + call: `ffrwd-wasm --shape` with the call's params and the inputs it + binds, each with the rate of every stream bound there, asked once per + distinct module, params and bound list. Its parameters are ports and + values, in any order: a port is a + `video_stream`, an `audio_stream`, a `data_stream` or rows + (`STRUCT(...)[]`, `cue[]`), named as the module names its input; a + value is text, number, boolean or vector, as any module's. Arguments + fill the parameters by position and then by name, ports included: + `burn(f.video[1], words => hear(f.audio[1]))`. How each input pairs + with the node's clock is the module's to say, not the query's. Recipe + [148](examples.md#148-a-node-reads-the-picture-the-sound-and-the-words-at-once). + - `DEFAULT NULL` on any port makes it optional: a call that leaves it + off, or writes NULL, binds nothing there, and the shape is asked + without it. A column an outer join left NULL, handed straight to such + a port, binds nothing too; to any other port it is refused as it is + anywhere. A port the module requires is refused left off. A port + the declaration names and the shape for these params has none of is + fine unbound and refused bound, naming it. Recipe + [149](examples.md#149-leave-an-input-out). + - `[]` on a port takes every stream the argument holds, in order: + `tile(ARRAY[a.video[1], b.video[1]])`. A port without it given an + array is one call per element, as a filter's is. Recipe + [150](examples.md#150-tile-any-number-of-pictures). + - An input the module holds on a port of its own (`hold` with a + `port_param`) is given a stream, or a port number: written in its + place, or by the param's name, `inset(v, port => 9100)`. Given a + number it binds nothing, and whatever connects to that port is shown. + - `RETURNS` a stream; rows alone (`STRUCT(...)[]`, `cue[]`), a data + stream the query reads while it runs; `STRUCT( , ...)`, + one field per output the module makes, read off the call + (`matte(v).mask`) or every field at once with `.*` in a WITH body; or + `source`, a node that reads nothing, called in FROM. A field names + the module's output of that name, and a lone stream or rows the + module's one output of that kind. Recipes + [145](examples.md#145-a-detector-returns-its-rows-and-the-picture-stays-where-it-was), + [151](examples.md#151-a-node-makes-a-matte-and-the-rows-that-go-with-it). + - A call over a node making a stream and the rows beside it hands a + reader both, the stream into one port and the rows into the next: + `ring(matte(v))` is `ring(matte(v).mask, matte(v).spots)`. + - Rows match by field, compared as the JSON schemas of the two ports: + every field the reading port names is in the producer's, with a type + it takes (`integer` is a `number`), and fields beyond those pass. A + missing or mistyped field is refused naming both. Recipe + [146](examples.md#146-a-reader-names-only-the-fields-it-reads). + - One call is one node wherever the query writes it: the same module, + arguments and params are one instance, and each of its outputs goes + to every reader. Recipe + [147](examples.md#147-one-call-however-many-places-read-it). + - Rows a COPY selects are what a module's rows always are there: the + rows of a `.ndjson` destination, a WebVTT track anywhere else. A + gather, `ARRAY(SELECT r FROM unnest() r WHERE ...)`, narrows + them on their way, as it narrows a module's annotation column. + - A node read in FROM binds its outputs as an input binds its streams, + `s.video[1]` the first picture, and its relation rows as renditions, + so `WHERE s.height = 720` picks one. `WHERE s.t < 10` (or `<=`) ends + it: its reader takes that much and closes. One that never ends makes + the query live. Recipe + [154](examples.md#154-a-page-with-no-inputs-is-a-source). + - A node called over an input's stream in FROM, `records(f.video[1]) + v`, whose shape reads coded packets on that port and makes no + output, is read while the query compiles, as a packet sink is: the + stream is copied to it as much as its port `wants`, and the rows it + emits are the alias's rows. + - A port reading coded packets is handed an input's own stream, + copied as it was coded, in a codec the module takes; an output + writing them is copied by whatever reads it. + - The bound list is one binding per input the call binds, ports held + on a port included (a lateral's feed, one stream per lateral), and a + hint per stream: its rate as far as the compile knows it before the + run, a picture's frame rate, a sound's sample rate (the one its port + conforms it to). An input's is its probe's; a node's picture runs at + its clock, a rate clock's own, the rate of the input a `rate-of` + clock names, or its clock input's over its stride. A feed by port, a + self-clocked node's output and data have none. Each call carries the + list it was asked with (`-bound =`), which the host asks + the shape with again, so the shape a node runs with is the one the + query compiled against. A node at a COPY's TO is asked once for its + ports and again with the streams the SELECT binds there. + - An input the module reads for its timing alone (`wants` `timing`) + is handed the stream in the format it already has: nothing converts + or conforms it, and where another port of the region reads the same + stream, it binds that one. `boxes_mask(v, detect(v))` sends the + picture once, in the format `detect` takes. `explain` says `timing` + for it. + - One region of a sidecar holds the nodes the query wires together, + and everything one process hands another travels as one NUT. A + signature only a node can carry (kinds mixed, a stream left out, a + value among ports) is refused for a module of an older world with + the refusal such a signature always had. +- **`ffrwd.merge_spans(, max_span => )`** turns rows + written once per tick into spans. Rows sharing a `start_t` are one + span, which keeps the last row's fields and ends at the last tick that + carried it plus that tick's length; a tick with no row for it is a gap + inside it. A span still open after `max_span` seconds is written as it + stands and goes on as a new one, so `max_span` is also how late a span + row may leave. It is the host's own node and runs in the sidecar + beside the rows' producer. Recipe + [152](examples.md#152-spans-from-the-rows-that-said-so-frame-by-frame). +- **What each node waits for.** A node's clock input reads a window + (per-frame, tumbling, hopping, sliding), and each output may leave late + by a latency it declares; an input paired by interval waits for its + producer, at most its own bound past the clock, and one the host + re-times onto the clock (anchored `first-frame` or `tagged`) is on a + clock of its own and waits its bound alone. Summed along each path, + those say how far behind the source every stream a query writes runs. + `ffrwd explain --delays` prints a line per node (its window, and each + interval input's bound) and per output (its delay, and how long it + waits for the latest stream written beside it); `explain` carries the + same under `timing`. A stream waiting more than 512 MiB of itself is + warned about (`HELD_STREAM`). A live query feeding an input later than + the bound its node set on it is refused as `LIVE_LEAD`. Recipe + [153](examples.md#153-see-what-each-node-waits-for). +- A **sql function returning a stream and its rows**, `RETURNS + STRUCT( , )`, selects both in its body, and a + call over it reads as both: handed to a node it fills the port it + stands in and the rows port after it, `ring(spotted(v))`; read off it, + `spotted(v).spots` is the rows; `.*` in a WITH body names both. - Trailing `;` allowed; `--` and `/* */` comments allowed. Unquoted identifiers fold to lowercase. View, CTE, and alias names share one flat namespace across the whole script. @@ -758,20 +894,24 @@ The starter is a recipe, `recipes/resize.sql` declared as a map `bin` entry, not an export: a lib must name a file defining its export, so a fresh directory has nothing to declare one with. -`--rust` writes a wasm module package instead of the bare one. On top +`--rust` writes a node module package instead of the bare one. On top of the manifest and the lockfile: `Cargo.toml` for a `cdylib` crate -named for the package segment, `build.rs`, `src/lib.rs` holding an -`invert` module, `src/invert.sql` declaring it as an export, -`recipes/invert.sql` calling that export, `.ffrwdignore`, `.gitignore` -and a `README.md`. The manifest depends on `ffrwd/wasm` at the current -world, and declares `capabilities` and `keywords` empty for their -author to fill in. `cargo build --target wasm32-wasip2 --release` then -`ffrwd publish` is the whole path from there. - -`build.rs` puts the wit where `wit_bindgen::generate!({path: "wit"})` -reads it, from whichever source is available: `FFRWD_WIT_DIR` when the -environment names one, otherwise `ffrwd path ffrwd/wasm`. The crate's -`wit/` is build output and is gitignored. +named for the package segment, taking `ffrwd-node` and `ffrwd-frame` +by git tag; `src/lib.rs` holding a `passthrough` node, which reads one +video input as its clock and hands the picture back on one video +output like it; `src/passthrough.sql` declaring it as an export, +`recipes/passthrough.sql` calling that export, `.ffrwdignore`, +`.gitignore` and a `README.md`. The crate has no `build.rs` and no +`wit/`: `ffrwd-node` carries the world. The manifest depends on +`ffrwd/wasm` at the current world, and declares `capabilities` and +`keywords` empty for their author to fill in. +`cargo build --release --target wasm32-wasip2` then `ffrwd publish` is +the whole path from there. + +`process` in `src/lib.rs` is where the work goes; the +[ffrwd-node](https://github.com/imbcmdth/ffrwd-node) README is the +reference for what a node declares and emits, and the node +declaration rules are under [Statements](#statements). ### Running a recipe @@ -994,8 +1134,10 @@ every dependency its own manifest pins, at the written version, plus its pinned models and the runtime its modules load - everything installing this package from the registry would have fetched. A fresh clone of a package's repository builds and publishes after one bare -install. `-g` without a package is an error; machine-wide installs -name what to fetch. +install. A dependency the project links is left to its link, and a pin +of it from before the link leaves the lockfile, unless another pinned +package depends on that version. `-g` without a package is an error; +machine-wide installs name what to fetch. ### Where a package is diff --git a/docs/error-schema.json b/docs/error-schema.json index c5ace01..36f6ecd 100644 --- a/docs/error-schema.json +++ b/docs/error-schema.json @@ -42,6 +42,7 @@ "PLAYER_NOT_FOUND", "RUNTIME_NOT_FOUND", "UNBOUNDED_LIVE_INPUT", + "LIVE_LEAD", "BUFFER_OVERFLOW", "INPUT_NEVER_OPENED", "STARTUP_DEADLOCK", diff --git a/docs/errors.md b/docs/errors.md index 4729375..fd1afc3 100644 --- a/docs/errors.md +++ b/docs/errors.md @@ -729,6 +729,14 @@ The third message under this code refuses a live input to a packet sink read at {"line": 5, "col": 38, "code": "UNBOUNDED_LIVE_INPUT", "message": "'live.m3u8' never ends, and a compile-time read reads a stream to the end", "hint": "a live stream is read at run time, by the same module written as a destination: COPY (SELECT ...) TO records()"} ``` +## LIVE_LEAD + +**Meaning:** A node bounds how long it waits for one of its inputs: it acts some time ahead of what that input says (an ad decision announced before the break, a playout that starts its next item early), and its shape says so as the input's `interval.latency`. In a live run what arrives past that bound is late for good. The compiler adds up how far behind the source the path feeding that input runs - every window and declared latency on the way - and refuses the query when that is more than the bound. [Recipe 153](examples.md#153-see-what-each-node-waits-for) shows where the sums are printed. + +**Fires when:** the query has a live input, or a node read in FROM that never ends, and a node's input paired by interval with a bound is fed by a path that runs later than its clock by more than that bound. Never for a file run: nothing there is late, and the node waits for its input as long as it takes. + +The anchor is the declaration of the node that needs the lead. + ## BUFFER_OVERFLOW **Meaning:** Not a compile rejection - the one code a RUN produces. The buffers a plan sized from its bounds were not deep enough, and the pipeline wedged: nothing crossed any pipe of the stage, and no process of it used any CPU, while every one was still alive and one of them was still waiting to hand its bytes over. The CPU is half the test: a stage's pumped pipes are not all the pipes it has, so a process computing over what it read stands still on all of them without being wedged. `ffrwd run` reports it instead of letting the stage sit until the timeout, so the message names the edge, the depth it was given, and how long nothing moved - never a bare "timed out", and never a silently dropped frame. diff --git a/docs/examples.md b/docs/examples.md index a03ecb0..4acbb13 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -1117,3 +1117,386 @@ ffmpeg -i tests/fixtures/av.mp4 -map 0:v:0 -c:0 rawvideo -pix_fmt:0 rgba -f nut Both `notes` calls and `pad_rows` run in one sidecar, and the rows never leave it: `n`'s notes ride the frames on `pad_rows`' first pad, and `o`'s, on the second, are dropped before the module is called. `pad_rows` is a stand-in that writes one row per call saying what reached it, so `seen.ndjson` holds sixty rows, `"first-0"` on frame 0, nothing on frame 1, `"first-2"` on frame 2, and never a note from `o`. The first stream has to be the one those rows ride, untouched: `pad_rows(o.v, n.v, notes => n.notes)` is refused, and so is an ffmpeg filter between `n.v` and the call, since ffmpeg carries the frames on and drops the rows. Selecting both halves is a WITH body's business alone; a SELECT that writes its columns reads `.notes` on its own, as a track. + +## 145. A detector returns its rows, and the picture stays where it was + +A module reading one stream need not hand it back. `spot` finds a mark in each frame and returns rows alone - `RETURNS STRUCT(...)[]` with no stream beside it is a data stream the query reads while it runs - and `ring` draws them, reading the picture from the source and the rows from the detector. Both arguments descend from `f.video[1]`, so the rows pair with the frames by pts, frame for frame, and nothing is copied that the detector did not make: + +```pgsql +CREATE FUNCTION spot(v video_stream, every number DEFAULT 30) +RETURNS STRUCT(start_t number, id number, x number, y number, w number, h number)[] + AS '../sidecar/modules/target/wasm32-wasip2/release/spot.wasm', 'spot' + LANGUAGE wasm; + +CREATE FUNCTION ring(v video_stream, + spots STRUCT(start_t number, id number, + x number, y number, w number, h number)[]) +RETURNS video_stream + AS '../sidecar/modules/target/wasm32-wasip2/release/ring.wasm', 'ring' + LANGUAGE wasm; + +COPY ( + SELECT ring(f.video[1], spot(f.video[1])) + FROM input('tests/fixtures/testsrc.mp4') f +) TO 'ringed.mp4' WITH (video_codec 'libx264', crf 20) +``` + +``` +$ ffrwd compile -f query.sql +ffmpeg -i tests/fixtures/testsrc.mp4 -map 0:v:0 -c:0 rawvideo -pix_fmt:0 rgba -f nut \ + pipe:1 | ffrwd-wasm -f nut -i pipe:0 -pad \ + '{"color": {"range": "pc", "primaries": "unknown", "trc": "unknown", "space": "gbr"}}' \ + -m spot=../sidecar/modules/target/wasm32-wasip2/release/spot.wasm -m \ + ring=../sidecar/modules/target/wasm32-wasip2/release/ring.wasm -filter_complex \ + '[v=0:v]spot=every=30[spots=n1];[v=0:v][spots=n1]ring[v=out0]' -bound \ + 'spot=[{"input":"v","streams":[{"rate":{"num":15,"den":1}}]}]' -bound \ + 'ring=[{"input":"v","streams":[{"rate":{"num":15,"den":1}}]},{"input":"spots",'\ +'"streams":[{"rate":null}]}]' -map '[out0]' -f nut pipe:1 | ffmpeg -copyts -f nut \ + -analyzeduration 0 -fpsprobesize 3 -i pipe:0 -map 0:v:0 -c:0 libx264 -crf:0 20 \ + ringed.mp4 +``` + +`spot` writes one row per frame for as long as the mark is in view, and every row of one mark carries the pts it was first seen at as `start_t`: the row for frame t says what is true at t, so a reader needs no look-ahead and a run split across workers agrees on the ids. The old spelling, a module returning `STRUCT(v video_stream, spots ...)` with the picture untouched, is what a package keeps when its own module has not moved: inside the sidecar the rows ride the frames exactly as before. A migrated package that wants the old reading back writes it in SQL - `CREATE FUNCTION spotted(v video_stream) RETURNS STRUCT(v video_stream, spots STRUCT(...)[]) AS $$ SELECT v, spot(v) AS spots $$ LANGUAGE sql` - and `ring(spotted(v))` reads the record as the stream and the rows, as a call over a two-part result always has. + +## 146. A reader names only the fields it reads + +`dim` wants a box; it does not care who found it or when. Its parameter declares the four fields it reads, and `spot`'s six-field rows are accepted because every field `dim` names is there with the type it names - the others pass through untouched. Row matching is structural, so one reader serves every detector whose rows carry a box: + +```pgsql +CREATE FUNCTION spot(v video_stream, every number DEFAULT 30) +RETURNS STRUCT(start_t number, id number, x number, y number, w number, h number)[] + AS '../sidecar/modules/target/wasm32-wasip2/release/spot.wasm', 'spot' + LANGUAGE wasm; + +CREATE FUNCTION dim(v video_stream, boxes STRUCT(x number, y number, w number, h number)[], + amount number DEFAULT 0.5) +RETURNS video_stream + AS '../sidecar/modules/target/wasm32-wasip2/release/dim.wasm', 'dim' + LANGUAGE wasm; + +COPY ( + SELECT dim(f.video[1], + ARRAY(SELECT s FROM unnest(spot(f.video[1])) s WHERE s.w >= 20)) + FROM input('tests/fixtures/testsrc.mp4') f +) TO 'dimmed.mp4' WITH (video_codec 'libx264', crf 20) +``` + +``` +$ ffrwd compile -f query.sql +ffmpeg -i tests/fixtures/testsrc.mp4 -map 0:v:0 -c:0 rawvideo -pix_fmt:0 rgba -f nut \ + pipe:1 | ffrwd-wasm -f nut -i pipe:0 -pad \ + '{"color": {"range": "pc", "primaries": "unknown", "trc": "unknown", "space": "gbr"}}' \ + -m spot=../sidecar/modules/target/wasm32-wasip2/release/spot.wasm -m \ + dim=../sidecar/modules/target/wasm32-wasip2/release/dim.wasm -filter_complex \ + '[v=0:v]spot=every=30[spots=n1];'\ +'[n1]rowfilter=pred={"ge"\\:\[{"field"\\:"w"}\,{"lit"\\:20}\]}[n2];'\ +'[v=0:v][boxes=n2]dim=amount=0.5[v=out0]' -bound \ + 'spot=[{"input":"v","streams":[{"rate":{"num":15,"den":1}}]}]' -bound \ + 'dim=[{"input":"v","streams":[{"rate":{"num":15,"den":1}}]},{"input":"boxes",'\ +'"streams":[{"rate":null}]}]' -map '[out0]' -f nut pipe:1 | ffmpeg -copyts -f nut \ + -analyzeduration 0 -fpsprobesize 3 -i pipe:0 -map 0:v:0 -c:0 libx264 -crf:0 20 \ + dimmed.mp4 +``` + +A field the reader names that the producer's rows lack, or carries with another type, is still refused at compile time, naming both. The WHERE runs inside the sidecar as before, and it may test fields the reader never sees. + +## 147. One call, however many places read it + +`hear` listens to the sound and writes a cue per window. The query reads it twice, as a caption track of the file and as the words `burn` paints onto the picture, and that is one node: a call with the same arguments written anywhere in one query is evaluated once, and its data output is split to every reader. Today those were two runs of the model, since a caption track feeds the muxer and a module's rows feed a stage of its own: + +```pgsql +CREATE FUNCTION hear(a audio_stream) RETURNS cue[] + AS '../sidecar/modules/target/wasm32-wasip2/release/hear.wasm', 'hear' + LANGUAGE wasm; + +CREATE FUNCTION burn(v video_stream, a audio_stream DEFAULT NULL, + words cue[] DEFAULT NULL) +RETURNS video_stream + AS '../sidecar/modules/target/wasm32-wasip2/release/burn.wasm', 'burn' + LANGUAGE wasm; + +COPY ( + SELECT burn(f.video[1], words => hear(f.audio[1])), f.audio[1], hear(f.audio[1]) + FROM input('tests/fixtures/av.mp4') f +) TO 'heard.mkv' WITH (video_codec 'libx264', crf 20, audio_codec 'copy') +``` + +``` +$ ffrwd compile -f query.sql +# named pipes: ffmpeg0 reads sidecar0, sidecar0; sidecar0 feeds ffmpeg0, ffmpeg0 +1. ffmpeg: ffmpeg -i tests/fixtures/av.mp4 -f webvtt -i \ + '' -f nut -analyzeduration 0 \ + -fpsprobesize 3 -i '' -map 2:v:0 -map 0:a:0 -map \ + 1:s:0 -c:2 copy -c:0 libx264 -crf:0 20 -c:1 copy heard.mkv +2. ffmpeg: ffmpeg -i tests/fixtures/av.mp4 -map 0:a:0 -map 0:v:0 -ar:0 48000 -c:0 \ + pcm_f32le -c:1 rawvideo -pix_fmt:1 rgba -f nut pipe:1 +3. sidecar: ffrwd-wasm -f nut -i pipe:0 -pad \ + '{"color": {"range": "pc", "primaries": "unknown", "trc": "unknown", "space": "gbr"}}' \ + -m hear=../sidecar/modules/target/wasm32-wasip2/release/hear.wasm -m \ + burn=../sidecar/modules/target/wasm32-wasip2/release/burn.wasm -filter_complex \ + '[a=0:a]hear[cues=out1];[v=0:v][words=out1]burn[v=out0]' -bound \ + 'hear=[{"input":"a","streams":[{"rate":{"num":48000,"den":1}}]}]' -bound \ + 'burn=[{"input":"v","streams":[{"rate":{"num":15,"den":1}}]},{"input":"words",'\ +'"streams":[{"rate":null}]}]' -map '[out0]' -f nut \ + '' -map '[out1]' -f webvtt \ + '' +# this listing is not a shell command -- run the plan with `ffrwd run` +``` + +Two calls whose arguments differ are still two nodes. The split is of a data edge, so what it costs is a second copy of each row, not a second run. + +## 148. A node reads the picture, the sound and the words at once + +A module's inputs are any mix of kinds in any order: `burn` takes the picture as its clock, the sound beside it frame for frame, and the words by their time. Each is a port of its own with its own pairing, declared by the module and read by the compiler, so the query writes the call and nothing about how the three are lined up: + +```pgsql +CREATE FUNCTION hear(a audio_stream) RETURNS cue[] + AS '../sidecar/modules/target/wasm32-wasip2/release/hear.wasm', 'hear' + LANGUAGE wasm; + +CREATE FUNCTION burn(v video_stream, a audio_stream DEFAULT NULL, + words cue[] DEFAULT NULL) +RETURNS video_stream + AS '../sidecar/modules/target/wasm32-wasip2/release/burn.wasm', 'burn' + LANGUAGE wasm; + +COPY ( + SELECT burn(f.video[1], f.audio[1], hear(f.audio[1])), f.audio[1] + FROM input('tests/fixtures/av.mp4') f +) TO 'burned.mp4' WITH (video_codec 'libx264', crf 20, audio_codec 'aac') +``` + +``` +$ ffrwd compile -f query.sql +ffmpeg -i tests/fixtures/av.mp4 -filter_complex '[0:a:0]asplit=2[out0][out2]' -map \ + '[out0]' -map 0:v:0 -map '[out2]' -ar:0 48000 -c:0 pcm_f32le -c:2 pcm_f32le -c:1 \ + rawvideo -pix_fmt:1 rgba -f nut pipe:1 | ffrwd-wasm -f nut -i pipe:0 -pad \ + '{"color": {"range": "pc", "primaries": "unknown", "trc": "unknown", "space": "gbr"}}' \ + -m hear=../sidecar/modules/target/wasm32-wasip2/release/hear.wasm -m \ + burn=../sidecar/modules/target/wasm32-wasip2/release/burn.wasm -filter_complex \ + '[a=0:a]hear[cues=n1];[v=0:v][a=0:a:1][words=n1]burn[v=out0]' -bound \ + 'hear=[{"input":"a","streams":[{"rate":{"num":48000,"den":1}}]}]' -bound \ + 'burn=[{"input":"v","streams":[{"rate":{"num":15,"den":1}}]},{"input":"a",'\ +'"streams":[{"rate":{"num":44100,"den":1}}]},{"input":"words",'\ +'"streams":[{"rate":null}]}]' -map '[out0]' -f nut pipe:1 | ffmpeg -i \ + tests/fixtures/av.mp4 -f nut -analyzeduration 0 -fpsprobesize 3 -i pipe:0 -map 1:v:0 \ + -map 0:a:0 -c:0 libx264 -crf:0 20 -c:1 aac burned.mp4 +``` + +`hear` works two seconds of sound at a time and says so, and a window's cues leave with the window. `burn` reads them by interval, so the host holds each picture until the window holding its time is done, and the picture leaves `burn` two seconds behind the sound that enters it. [Recipe 153](#153-see-what-each-node-waits-for) shows where that number is printed. + +## 149. Leave an input out + +Any stream parameter may carry `DEFAULT NULL`, and a call leaves it off or writes `NULL`: the port is unbound and the module is told so. `burn` with the picture alone paints nothing; with the sound and no words it paints a level meter; `inset` shows a feed that connects later, by port, over the picture, and runs on the picture alone until something connects: + +```pgsql +CREATE FUNCTION burn(v video_stream, a audio_stream DEFAULT NULL, + words cue[] DEFAULT NULL) +RETURNS video_stream + AS '../sidecar/modules/target/wasm32-wasip2/release/burn.wasm', 'burn' + LANGUAGE wasm; + +CREATE FUNCTION inset(v video_stream, feed video_stream DEFAULT NULL, + port number DEFAULT 9000, lead number DEFAULT 0.5) +RETURNS video_stream + AS '../sidecar/modules/target/wasm32-wasip2/release/inset.wasm', 'inset' + LANGUAGE wasm; + +COPY ( + SELECT inset(burn(f.video[1], f.audio[1]), port => 9100), f.audio[1] + FROM input('tests/fixtures/av.mp4') f +) TO 'inset.mp4' WITH (video_codec 'libx264', crf 20, audio_codec 'aac') +``` + +``` +$ ffrwd compile -f query.sql +ffmpeg -i tests/fixtures/av.mp4 -map 0:v:0 -map 0:a:0 -c:0 rawvideo -pix_fmt:0 rgba -c:1 \ + pcm_f32le -f nut pipe:1 | ffrwd-wasm -f nut -i pipe:0 -pad \ + '{"color": {"range": "pc", "primaries": "unknown", "trc": "unknown", "space": "gbr"}}' \ + -m burn=../sidecar/modules/target/wasm32-wasip2/release/burn.wasm -m \ + inset=../sidecar/modules/target/wasm32-wasip2/release/inset.wasm -filter_complex \ + '[v=0:v][a=0:a]burn[v=n1];[v=n1]inset=port=9100:lead=0.5[v=out0]' -bound \ + 'burn=[{"input":"v","streams":[{"rate":{"num":15,"den":1}}]},{"input":"a",'\ +'"streams":[{"rate":{"num":44100,"den":1}}]}]' -bound \ + 'inset=[{"input":"v","streams":[{"rate":{"num":15,"den":1}}]}]' -map '[out0]' -f nut \ + pipe:1 | ffmpeg -i tests/fixtures/av.mp4 -f nut -analyzeduration 0 -fpsprobesize 3 -i \ + pipe:0 -map 1:v:0 -map 0:a:0 -c:0 libx264 -crf:0 20 -c:1 aac inset.mp4 +# listens: sidecar0 at tcp://127.0.0.1:9100 for inset(feed) +``` + +`feed` is a hold input: whatever connects to port 9100 is shown at the picture's pace from `lead` seconds after its first frame arrives, the last frame held while it runs late, and the picture alone again when it ends. The compile listing names the port and the process that owns it, as it does for a switch's feeders. An input with no `DEFAULT NULL` is required, and a call that leaves it off is refused. + +## 150. Tile any number of pictures + +`video_stream[]` declares a port that takes as many streams as the call gives it, each arriving with its own size, rate and tags. `tile` lays them out in a grid. It ticks at the first picture's rate, or at `fps` when the call says so, and at every tick shows each stream's newest frame, so the pictures need not share a rate or a start: + +```pgsql +CREATE FUNCTION tile(v video_stream[], columns number DEFAULT 2, fps number DEFAULT NULL) +RETURNS video_stream + AS '../sidecar/modules/target/wasm32-wasip2/release/tile.wasm', 'tile' + LANGUAGE wasm; + +COPY ( + SELECT tile(ARRAY[a.video[1], b.video[1], c.video[1]], 3) + FROM input('tests/fixtures/av.mp4') a, + input('tests/fixtures/av2.mp4') b, + input('tests/fixtures/testsrc.mp4') c +) TO 'tiled.mp4' WITH (video_codec 'libx264', crf 20) +``` + +``` +$ ffrwd compile -f query.sql +# named pipes: sidecar0 reads ffmpeg1, ffmpeg2, ffmpeg3 +1. ffmpeg: ffmpeg -copyts -f nut -analyzeduration 0 -fpsprobesize 3 -i pipe:0 -map 0:v:0 \ + -c:0 libx264 -crf:0 20 tiled.mp4 +2. ffmpeg: ffmpeg -i tests/fixtures/av.mp4 -map 0:v:0 -c:0 rawvideo -pix_fmt:0 rgba -f \ + nut '' +3. ffmpeg: ffmpeg -i tests/fixtures/av2.mp4 -map 0:v:0 -c:0 rawvideo -pix_fmt:0 rgba -f \ + nut '' +4. ffmpeg: ffmpeg -i tests/fixtures/testsrc.mp4 -map 0:v:0 -c:0 rawvideo -pix_fmt:0 rgba \ + -f nut '' +5. sidecar: ffrwd-wasm -f nut -i '' -pad \ + '{"color": {"range": "pc", "primaries": "unknown", "trc": "unknown", "space": "gbr"}}' \ + -f nut -i '' -pad \ + '{"color": {"range": "pc", "primaries": "unknown", "trc": "unknown", "space": "gbr"}}' \ + -f nut -i '' -pad \ + '{"color": {"range": "pc", "primaries": "unknown", "trc": "unknown", "space": "gbr"}}' \ + -m tile=../sidecar/modules/target/wasm32-wasip2/release/tile.wasm -filter_complex \ + '[v=0:v][v=1:v][v=2:v]tile=columns=3[v=out0]' -bound \ + 'tile=[{"input":"v","streams":[{"rate":{"num":15,"den":1}},{"rate":{"num":15,'\ +'"den":1}},{"rate":{"num":15,"den":1}}]}]' -map '[out0]' -f nut pipe:1 +# this listing is not a shell command -- run the plan with `ffrwd run` +``` + +A bare array column broadcasts over a filter, one call per element; over a module port declared as an array it is the port's whole list, and a module that wants one call per element is called under `unnest`. `audio_stream[]` and a rows parameter with `[]` on the record work the same way. A port that takes several streams cannot be the module's clock, which is why `tile` keeps time itself; the rate it keeps is read off the first picture by the compiler, which knows every stream's rate before anything runs. + +## 151. A node makes a matte and the rows that go with it + +A module that produces two things returns a record naming both: `matte` makes a gray matte of the mark it finds and a row per mark, and both leave the one node. Read either field off the call, or every field at once with `.*` in a WITH body; however the fields are read, the call is one instance. Here the rows dim the picture where the marks are and the matte is written beside it: + +```pgsql +CREATE FUNCTION matte(v video_stream, every number DEFAULT 30) +RETURNS STRUCT(mask video_stream, + spots STRUCT(start_t number, id number, + x number, y number, w number, h number)[]) + AS '../sidecar/modules/target/wasm32-wasip2/release/matte.wasm', 'matte' + LANGUAGE wasm; + +CREATE FUNCTION dim(v video_stream, boxes STRUCT(x number, y number, w number, h number)[], + amount number DEFAULT 0.5) +RETURNS video_stream + AS '../sidecar/modules/target/wasm32-wasip2/release/dim.wasm', 'dim' + LANGUAGE wasm; + +COPY ( + WITH m AS (SELECT f.video[1] AS v, (matte(f.video[1])).* + FROM input('tests/fixtures/testsrc.mp4') f) + SELECT dim(m.v, m.spots), m.mask + FROM m +) TO 'matte.mkv' WITH (video_codec 'ffv1') +``` + +``` +$ ffrwd compile -f query.sql +ffmpeg -i tests/fixtures/testsrc.mp4 -map 0:v:0 -c:0 rawvideo -pix_fmt:0 rgba -f nut \ + pipe:1 | ffrwd-wasm -f nut -i pipe:0 -pad \ + '{"color": {"range": "pc", "primaries": "unknown", "trc": "unknown", "space": "gbr"}}' \ + -m matte=../sidecar/modules/target/wasm32-wasip2/release/matte.wasm -m \ + dim=../sidecar/modules/target/wasm32-wasip2/release/dim.wasm -filter_complex \ + '[v=0:v]matte=every=30[mask=out1][spots=n11];[v=0:v][boxes=n11]dim=amount=0.5[v=out0]' \ + -bound 'matte=[{"input":"v","streams":[{"rate":{"num":15,"den":1}}]}]' -bound \ + 'dim=[{"input":"v","streams":[{"rate":{"num":15,"den":1}}]},{"input":"boxes",'\ +'"streams":[{"rate":null}]}]' -map '[out0]' -map '[out1]' -f nut pipe:1 | ffmpeg -copyts \ + -f nut -analyzeduration 0 -fpsprobesize 3 -i pipe:0 -map 0:v:0 -map 0:v:1 -c:0 ffv1 \ + -c:1 ffv1 matte.mkv +``` + +Each field is an output port with a format and a time base of its own, declared by the module for the call's parameters. [Recipe 94](#94-blur-the-people-and-only-the-people)'s `segment` is this shape, and its rows no longer ride the map's frames: they are a data stream beside it, which is why `mask_select` can read them from a call `segment` is not part of. + +## 152. Spans from the rows that said so, frame by frame + +A row per frame is the honest shape for a thing known as it happens, and a span is what a subtitle track or a record wants. `ffrwd.merge_spans` is the reducer between them, a node the host provides: it groups rows by `start_t`, keeps the last row's fields, and ends each span at the last frame that carried it plus that frame's duration. A frame that misses a row is a gap inside the span, not the end of it: + +```pgsql +CREATE FUNCTION spot(v video_stream, every number DEFAULT 30) +RETURNS STRUCT(start_t number, id number, x number, y number, w number, h number)[] + AS '../sidecar/modules/target/wasm32-wasip2/release/spot.wasm', 'spot' + LANGUAGE wasm; + +COPY ( + SELECT ffrwd.merge_spans(spot(f.video[1]), max_span => 10) + FROM input('tests/fixtures/testsrc.mp4') f +) TO 'spots.ndjson' +``` + +``` +$ ffrwd compile -f query.sql +ffmpeg -i tests/fixtures/testsrc.mp4 -map 0:v:0 -c:0 rawvideo -pix_fmt:0 rgba -f nut \ + pipe:1 | ffrwd-wasm -f nut -i pipe:0 -pad \ + '{"color": {"range": "pc", "primaries": "unknown", "trc": "unknown", "space": "gbr"}}' \ + -m spot=../sidecar/modules/target/wasm32-wasip2/release/spot.wasm -filter_complex \ + '[v=0:v]spot=every=30[spots=n1];[n1]rowmerge=max_span=10[out0]' -bound \ + 'spot=[{"input":"v","streams":[{"rate":{"num":15,"den":1}}]}]' -map '[out0]' -f ndjson \ + spots.ndjson +``` + +The rows out carry `start_t` and `end_t` beside the fields in, one row per span, so `spots.ndjson` holds one line per mark rather than one per frame. Rows whose fields are `start_t` and `text` reduce to cues, and selecting them beside a picture writes a subtitle track. A span row leaves when its span ends, so a span is as late as it is long; `max_span` bounds that, and it is what a reader pairing by time waits for. A span still open after ten seconds is written as it stands and goes on as a new one. The reducer closes a span on its producer's progress, not on the next row, so the last span of a run ends where the rows did. + +## 153. See what each node waits for + +Every node declares the window it works in and how late its rows may leave, and the compiler adds the waits up along each path. `explain` prints them: per node, its window in streaming SQL's words (per-frame, tumbling, hopping, sliding), and per output, how far behind the source it runs. The query is [recipe 148](#148-a-node-reads-the-picture-the-sound-and-the-words-at-once)'s: + +```pgsql +CREATE FUNCTION hear(a audio_stream) RETURNS cue[] + AS '../sidecar/modules/target/wasm32-wasip2/release/hear.wasm', 'hear' + LANGUAGE wasm; + +CREATE FUNCTION burn(v video_stream, a audio_stream DEFAULT NULL, + words cue[] DEFAULT NULL) +RETURNS video_stream + AS '../sidecar/modules/target/wasm32-wasip2/release/burn.wasm', 'burn' + LANGUAGE wasm; + +COPY ( + SELECT burn(f.video[1], f.audio[1], hear(f.audio[1])), f.audio[1] + FROM input('tests/fixtures/av.mp4') f +) TO 'burned.mp4' WITH (video_codec 'libx264', crf 20, audio_codec 'aac') +``` + +``` +$ ffrwd explain --delays -f query.sql +hear: tumbling 2 s +burn: per-frame; words by interval, no bound +burned.mp4 stream 0 (video): 2 s behind the source +burned.mp4 stream 1 (audio): 0 s behind the source, waits 2 s +``` + +`hear` is a tumbling window of 2 s, so its cues trail the sound by up to 2 s and nothing more; `burn`'s picture is 2 s behind the source, and the sound written beside it, read straight from the file, waits those 2 s at the muxer, which `compile` sizes. On a live input the same sums decide whether a query can run at all: a node that must act ahead of time (an ad decision that needs `announce_before_s`, a playout that needs `lead_s`) fed by a path later than that lead is refused at compile time as `LIVE_LEAD`, naming the node, the lead it needs and the delay of the path feeding it. A file run has no such rule, since nothing there is late. + +## 154. A page with no inputs is a source + +A module with no stream parameters is a source: it declares its outputs for its parameters, ticks at the rate it declares, and reads nothing. `ticker` draws a line of text crossing a canvas. `RETURNS source` puts it in FROM, where the alias carries the stream columns the module declared, and the query reads them as it reads a file's: + +```pgsql +CREATE FUNCTION ticker(text text, width number DEFAULT 1280, height number DEFAULT 720, + fps number DEFAULT 30) RETURNS source + AS '../sidecar/modules/target/wasm32-wasip2/release/ticker.wasm', 'ticker' + LANGUAGE wasm; + +COPY ( + SELECT s.video[1] + FROM ticker('Nothing to see here') s + WHERE s.t < 10 +) TO 'ticker.mp4' WITH (video_codec 'libx264', crf 20) +``` + +``` +$ ffrwd compile -f query.sql +ffrwd-wasm -m ticker=../sidecar/modules/target/wasm32-wasip2/release/ticker.wasm \ + -filter_complex \ + 'ticker=text=Nothing\ to\ see\ here:width=1280:height=720:fps=30[video=out0]' -bound \ + 'ticker=[]' -map '[out0]' -f nut pipe:1 | ffmpeg -copyts -f nut -analyzeduration 0 \ + -fpsprobesize 3 -to 10 -i pipe:0 -map 0:v:0 -c:0 libx264 -crf:0 20 ticker.mp4 +``` + +In a file run the source runs as fast as its reader drains it; in a live run it is paced to the wall clock. `WHERE s.t < 10` ends it after ten seconds, as it would any source. A network source is the same shape with a clock of its own: it emits when it has something, and `shape` may reach the network at compile time to learn its outputs, as a manifest is probed. `ffrwd.blitz.compose` with no streams, a page that animates on its own, is this recipe's shape too. diff --git a/sidecar/Cargo.lock b/sidecar/Cargo.lock index 55c190c..9e42805 100644 --- a/sidecar/Cargo.lock +++ b/sidecar/Cargo.lock @@ -596,6 +596,26 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fast_image_resize" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9c50201dc184ba6553da1695aac20a042efffbe2d84542cee31917c86c3ab1e" +dependencies = [ + "cfg-if", + "document-features", + "num-traits", + "thiserror 2.0.20", +] + +[[package]] +name = "ffrwd-frame" +version = "0.1.1" +source = "git+https://github.com/imbcmdth/ffrwd-frame?tag=v0.1.1#cf8887ce24bf5f5d570fed73b278f10ffee991f1" +dependencies = [ + "fast_image_resize", +] + [[package]] name = "ffrwd-nal" version = "0.1.0" @@ -603,8 +623,8 @@ source = "git+https://github.com/imbcmdth/ffrwd-nal?tag=v0.1.0#bd2ffcf559065b836 [[package]] name = "ffrwd-nut" -version = "0.1.4" -source = "git+https://github.com/imbcmdth/ffrwd-nut?tag=v0.1.4#138b4bbf9357e720459c504be40a50fc5000ce93" +version = "0.1.5" +source = "git+https://github.com/imbcmdth/ffrwd-nut?tag=v0.1.5#30ca1c6018c9309f6e3a47149ab4066da87db0e6" dependencies = [ "serde", "serde_json", @@ -615,6 +635,8 @@ name = "ffrwd-wasm" version = "0.3.0" dependencies = [ "anyhow", + "fast_image_resize", + "ffrwd-frame", "ffrwd-nal", "ffrwd-nut", "ffrwd-wasm-runtime", @@ -622,6 +644,7 @@ dependencies = [ "serde", "serde_json", "sha2", + "socket2", "windows-sys 0.61.2", ] diff --git a/sidecar/NODE-CLI.md b/sidecar/NODE-CLI.md new file mode 100644 index 0000000..b511203 --- /dev/null +++ b/sidecar/NODE-CLI.md @@ -0,0 +1,389 @@ +# Node calls on the sidecar's command line + +How a compiled query hands the sidecar a network that holds node modules +(`ffrwd:av@0.19.1`). Everything an older module takes stays as it is: the +same `-m`, the same positional pads, the same outputs. A node module is +told apart by its export, and its pads say which port they bind. + +What a node's ports are named, which are required, and what kind each +carries come from `ffrwd-wasm --shape` (`NODE-SHAPE.md`). Port names in +the examples below are the stand-in modules' own. + +## Inputs + + -f nut -i + +One `-i` is one edge: one NUT, carrying every stream that crosses it, +video, audio, coded packets and data (a JSON stream, codec `json`) in any +mix, interleaved by time. A producer writes all of an edge's streams into +that one pipe, so no stream of it can wait on another pipe. + +A pad names a stream of an input by class and position, as ffmpeg's +stream specifiers do: + +| Pad | Stream | +|---|---| +| `[0:v]`, `[0:v:1]` | input 0's first, second video stream, raw or coded | +| `[0:a]`, `[0:a:1]` | its audio streams, raw or coded | +| `[0:d]`, `[0:d:1]` | its data streams | + +`[N:v]` is `[N:v:0]`. A raw stream binds a video or audio port, a coded +one a packets port, a data stream a data port. + + -pad '{"color": {"range": "tv", "primaries": "bt709", "trc": "bt709", "space": "bt709"}, "tags": {"smart_timed": "1"}}' + +after an `-i` says what its NUT does not carry: a raw picture's colour, in +ffmpeg's names, and tags for its streams beside their own, which reach a +node in `stream-info.tags` (a hold input anchored `tagged` reads them). + +## Nodes + + -m = + -filter_complex ';;...' + -params-from = + +A chain is input pads, the node, output pads, as for any module. For a +node module every pad carries the port it binds before an `=`: + + [=]...==:=[=