test(verify): run both Windows decoders and flag silent decoder fallbacks

Windows now runs the Media Foundation default and the forced FFmpeg
decoder as separate cases (labels mf and ffmpeg). Every case records which
decoder each stream actually used and fails if it is not the requested one,
so a Media Foundation or AVAssetReader fallback to FFmpeg cannot pass under
the wrong label.

Adds a temporary workflow that runs the suite and the decoder model fuzz on
windows-2022 and ubuntu-24.04.

Author: richiemcilroy

.github/workflows/verify-platforms-tmp.yml

@@ -0,0 +1,258 @@
+name: cap-verify platforms (temporary triage)
+
+# Temporary workflow for the PLATFORMS track: runs the cap-verify suite and
+# the decoder model fuzz on Windows and Linux runners, several times, and
+# uploads the per-case JSON reports. Not for merging; the CI-INFRA track owns
+# engine-verify.yml.
+
+on:
+  push:
+    branches:
+      - ci/verify-platforms
+  workflow_dispatch:
+    inputs:
+      repeats:
+        description: "Times to run each test binary"
+        default: "3"
+      case:
+        description: "CAP_VERIFY_CASE filter (glob, comma separated)"
+        default: ""
+
+concurrency:
+  group: verify-platforms-${{ github.ref_name }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+env:
+  CARGO_TERM_COLOR: never
+  CARGO_INCREMENTAL: "0"
+  RUST_BACKTRACE: "1"
+
+jobs:
+  suite:
+    name: suite (${{ matrix.runner }})
+    strategy:
+      fail-fast: false
+      matrix:
+        runner:
+          - windows-2022
+          - ubuntu-24.04
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 150
+    env:
+      REPEATS: ${{ github.event.inputs.repeats || '3' }}
+      CAP_VERIFY_CASE: ${{ github.event.inputs.case || '' }}
+      CAP_VERIFY_FIXTURE_DIR: ${{ github.workspace }}/verify-fixtures
+      OUT: ${{ github.workspace }}/verify-out
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: dtolnay/rust-toolchain@1.88.0
+
+      - name: Rust cache (restore only)
+        uses: ./.github/actions/setup-rust-cache
+        with:
+          target: ${{ runner.os == 'Windows' && 'x86_64-pc-windows-msvc' || 'x86_64-unknown-linux-gnu' }}
+          prefix-key: v1-rust-sync
+          save-cache: "false"
+
+      - uses: ./.github/actions/install-desktop-deps
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 24
+
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.13"
+
+      - name: Native dependencies
+        env:
+          RUST_TARGET_TRIPLE: ${{ runner.os == 'Linux' && 'x86_64-unknown-linux-gnu' || 'x86_64-pc-windows-msvc' }}
+        run: node scripts/setup.js
+
+      - name: FFmpeg DLLs on PATH (Windows)
+        if: runner.os == 'Windows'
+        shell: pwsh
+        run: Add-Content -Path $env:GITHUB_PATH -Value "${{ github.workspace }}\\target\\ffmpeg\\bin"
+
+      - name: Native libs on LD_LIBRARY_PATH (Linux)
+        if: runner.os == 'Linux'
+        run: echo "LD_LIBRARY_PATH=${{ github.workspace }}/target/native-deps/lib:${{ github.workspace }}/target/debug" >> "$GITHUB_ENV"
+
+      - name: Software Vulkan (Linux)
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y mesa-vulkan-drivers libvulkan1 vulkan-tools
+          vulkaninfo --summary || true
+
+      - name: Virtual audio device (Windows)
+        if: runner.os == 'Windows'
+        timeout-minutes: 10
+        shell: pwsh
+        run: |
+          Invoke-WebRequest -Uri https://github.com/duncanthrax/scream/releases/download/3.6/Scream3.6.zip -OutFile Scream3.6.zip
+          $expected = "25ea5e778b4e6995a98d448b9b5f6d321f681663f1aeeec69d8e63183d008b19"
+          $actual = (Get-FileHash -Algorithm SHA256 Scream3.6.zip).Hash.ToLowerInvariant()
+          if ($actual -ne $expected) { throw "Scream3.6.zip SHA-256 mismatch: got $actual, expected $expected" }
+          Expand-Archive -Path Scream3.6.zip -DestinationPath Scream
+          $cert = (Get-AuthenticodeSignature Scream\Install\driver\Scream.sys).SignerCertificate
+          $store = [System.Security.Cryptography.X509Certificates.X509Store]::new("TrustedPublisher", "LocalMachine")
+          $store.Open("ReadWrite")
+          $store.Add($cert)
+          $store.Close()
+          Scream\Install\helpers\devcon.exe install Scream\Install\driver\Scream.inf *Scream
+          if ($LASTEXITCODE -gt 1) { throw "devcon install failed with exit code $LASTEXITCODE" }
+          Start-Service -Name Audiosrv -ErrorAction SilentlyContinue
+
+      - name: Runner facts
+        shell: bash
+        run: |
+          mkdir -p "$OUT"
+          {
+            echo "runner=${{ matrix.runner }}"
+            nproc 2>/dev/null || echo "NUMBER_OF_PROCESSORS=$NUMBER_OF_PROCESSORS"
+            rustc -V
+          } | tee "$OUT/runner.txt"
+
+      - name: Build tests
+        shell: bash
+        run: |
+          start=$(date +%s)
+          cargo test --locked -p cap-verify --no-run 2>&1 | tail -40
+          echo "build_seconds=$(( $(date +%s) - start ))" | tee -a "$OUT/timings.txt"
+
+      - name: Run suite (${{ env.REPEATS }}x)
+        shell: bash
+        run: |
+          set +e
+          status=0
+          for run in $(seq 1 "$REPEATS"); do
+            for test in codec export playback seek; do
+              export CAP_VERIFY_REPORT_DIR="$OUT/run$run/$test"
+              mkdir -p "$CAP_VERIFY_REPORT_DIR"
+              start=$(date +%s)
+              cargo test --locked -p cap-verify --test "$test" -- --nocapture > "$OUT/run$run/$test.log" 2>&1
+              code=$?
+              secs=$(( $(date +%s) - start ))
+              echo "run$run $test exit=$code seconds=$secs" | tee -a "$OUT/timings.txt"
+              grep -E "^\[cap-verify\] (pass|xfail|FAIL|XPASS)|^test .* \.\.\. |panicked|error" "$OUT/run$run/$test.log" | head -80
+              [ $code -ne 0 ] && status=1
+            done
+          done
+          exit $status
+
+      - name: Summarise
+        if: always()
+        shell: bash
+        run: |
+          python - "$OUT" >> "$GITHUB_STEP_SUMMARY" <<'PY'
+          import glob, json, os, sys, collections
+          out = sys.argv[1]
+          cases = collections.OrderedDict()
+          for path in sorted(glob.glob(os.path.join(out, "run*", "*", "*.jsonl"))):
+              run = path.split(os.sep)[-3]
+              for line in open(path, encoding="utf-8"):
+                  r = json.loads(line)
+                  cases.setdefault(r["id"], []).append((run, r["outcome"], r["seconds"], r["summary"].splitlines()[0][:160]))
+          print(f"## cap-verify on {os.environ.get('RUNNER_OS')}")
+          print()
+          print("| case | outcomes | seconds | first line |")
+          print("|---|---|---|---|")
+          for cid, rs in cases.items():
+              outs = " ".join(o for _, o, _, _ in rs)
+              secs = " ".join(f"{s:.1f}" for _, _, s, _ in rs)
+              print(f"| {cid} | {outs} | {secs} | {rs[0][3].replace('|', '/')} |")
+          print()
+          for line in open(os.path.join(out, "timings.txt"), encoding="utf-8"):
+              print(f"- {line.strip()}")
+          PY
+
+      - name: Upload reports
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: verify-${{ matrix.runner }}
+          path: ${{ github.workspace }}/verify-out
+          if-no-files-found: warn
+
+  decoder-model:
+    name: decoder model (${{ matrix.runner }})
+    strategy:
+      fail-fast: false
+      matrix:
+        runner:
+          - windows-2022
+          - ubuntu-24.04
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 120
+    env:
+      REPEATS: ${{ github.event.inputs.repeats || '3' }}
+      OUT: ${{ github.workspace }}/model-out
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ci/verify-decoder-model
+
+      - uses: dtolnay/rust-toolchain@1.88.0
+
+      - name: Rust cache (restore only)
+        uses: ./.github/actions/setup-rust-cache
+        with:
+          target: ${{ runner.os == 'Windows' && 'x86_64-pc-windows-msvc' || 'x86_64-unknown-linux-gnu' }}
+          prefix-key: v1-rust-sync
+          save-cache: "false"
+
+      - uses: ./.github/actions/install-desktop-deps
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 24
+
+      - name: Native dependencies
+        env:
+          RUST_TARGET_TRIPLE: ${{ runner.os == 'Linux' && 'x86_64-unknown-linux-gnu' || 'x86_64-pc-windows-msvc' }}
+        run: node scripts/setup.js
+
+      - name: FFmpeg DLLs on PATH (Windows)
+        if: runner.os == 'Windows'
+        shell: pwsh
+        run: Add-Content -Path $env:GITHUB_PATH -Value "${{ github.workspace }}\\target\\ffmpeg\\bin"
+
+      - name: Native libs on LD_LIBRARY_PATH (Linux)
+        if: runner.os == 'Linux'
+        run: echo "LD_LIBRARY_PATH=${{ github.workspace }}/target/native-deps/lib:${{ github.workspace }}/target/debug" >> "$GITHUB_ENV"
+
+      - name: Build
+        shell: bash
+        run: |
+          mkdir -p "$OUT"
+          start=$(date +%s)
+          cargo test --locked -p cap-rendering --lib --no-run 2>&1 | tail -20
+          echo "build_seconds=$(( $(date +%s) - start ))" | tee -a "$OUT/timings.txt"
+
+      - name: Run decoder model (${{ env.REPEATS }}x)
+        shell: bash
+        run: |
+          set +e
+          status=0
+          for run in $(seq 1 "$REPEATS"); do
+            start=$(date +%s)
+            cargo test --locked -p cap-rendering --lib decoder::model -- --nocapture > "$OUT/run$run.log" 2>&1
+            code=$?
+            echo "run$run exit=$code seconds=$(( $(date +%s) - start ))" | tee -a "$OUT/timings.txt"
+            tail -60 "$OUT/run$run.log"
+            [ $code -ne 0 ] && status=1
+          done
+          exit $status
+
+      - name: Upload logs
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: decoder-model-${{ matrix.runner }}
+          path: ${{ github.workspace }}/model-out
+          if-no-files-found: warn

crates/verify/tests/common/mod.rs

@@ -88,7 +88,8 @@ pub enum Format {
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub enum Decoder {
-    /// The platform default (AVAssetReader on macOS, FFmpeg elsewhere).
+    /// The platform default: AVAssetReader on macOS, Media Foundation on
+    /// Windows, FFmpeg on Linux.
     Default,
     /// `--force-ffmpeg-decoder`.
     Ffmpeg,
@@ -98,19 +99,69 @@ impl Decoder {
     pub fn label(self) -> &'static str {
         match self {
             Decoder::Default if cfg!(target_os = "macos") => "avf",
+            Decoder::Default if cfg!(target_os = "windows") => "mf",
             Decoder::Default => "ffmpeg",
             Decoder::Ffmpeg => "ffmpeg",
         }
     }
 
     /// Decoders that are actually distinct on this platform.
     pub fn all() -> Vec<Decoder> {
-        if cfg!(target_os = "macos") {
+        if cfg!(any(target_os = "macos", target_os = "windows")) {
             vec![Decoder::Default, Decoder::Ffmpeg]
         } else {
             vec![Decoder::Default]
         }
     }
+
+    /// The `DecoderType` display prefix this choice must produce. A silent
+    /// fallback (Media Foundation or AVAssetReader failing over to FFmpeg)
+    /// would otherwise test the wrong decoder under this label.
+    pub fn expected_type(self) -> &'static str {
+        match self {
+            Decoder::Default if cfg!(target_os = "macos") => "AVAssetReader",
+            Decoder::Default if cfg!(target_os = "windows") => "MediaFoundation",
+            _ => "FFmpeg",
+        }
+    }
+}
+
+/// Records which decoder each stream actually used and flags any stream
+/// that is not on the requested decoder.
+pub fn check_decoders(case: &mut Case, decoder: Decoder, segments: &[cap_editor::SegmentMedia]) {
+    let mut used = Vec::new();
+    for (i, segment) in segments.iter().enumerate() {
+        let streams =
+            std::iter::once(("screen", Some(segment.decoders.screen_decoder_status()))).chain(
+                std::iter::once(("camera", segment.decoders.camera_decoder_status())),
+            );
+        for (stream, status) in streams {
+            let Some(status) = status else { continue };
+            let kind = status.decoder_type.to_string();
+            used.push(format!("{stream}{i}={kind}"));
+            if !kind.starts_with(decoder.expected_type()) {
+                case.problem(format!(
+                    "segment {i} {stream} decoded by {kind}, expected {} (fallback reason: {})",
+                    decoder.expected_type(),
+                    status.fallback_reason.as_deref().unwrap_or("none")
+                ));
+            }
+        }
+    }
+    case.note(format!("decoders: {}", used.join(" ")));
+}
+
+/// Opens the project's decoders the way the exporter does and checks them.
+/// The exporter's own decoders are private, so this probes the same
+/// `create_segments` decision on the same files.
+pub async fn probe_decoders(case: &mut Case, project: &Project, decoder: Decoder) {
+    let Some(meta) = project.meta.studio_meta() else {
+        return;
+    };
+    match cap_editor::create_segments(&project.meta, meta, decoder == Decoder::Ffmpeg).await {
+        Ok(segments) => check_decoders(case, decoder, &segments),
+        Err(e) => case.problem(format!("decoder probe failed: {e}")),
+    }
 }
 
 pub struct ExportRun {
@@ -342,6 +393,12 @@ impl Player {
         })
     }
 
+    /// Flags streams that are not on `decoder` (checked after the run, so a
+    /// mid-run fallback is caught too).
+    pub fn check_decoders(&self, case: &mut Case, decoder: Decoder) {
+        check_decoders(case, decoder, &self.segments);
+    }
+
     pub fn prefer_sequential(&self) {
         for segment in &self.segments {
             segment.decoders.prefer_sequential_access();

crates/verify/tests/export.rs

@@ -59,6 +59,7 @@ fn run(
                     format,
                     decoder,
                 };
+                rt.block_on(common::probe_decoders(&mut case, &project, decoder));
                 let first = dir.path().join(format!("a.{ext}"));
                 match rt.block_on(common::export(&project, config.clone(), &run, &first)) {
                     Err(e) => case.problem(format!("export failed: {e}")),

crates/verify/tests/playback.rs

@@ -70,6 +70,7 @@ fn sequential(
             for &f in &frames {
                 readouts.push(player.frame(f, fps, f == first).await?);
             }
+            player.check_decoders(&mut case, decoder);
             Ok::<_, String>(readouts)
         });
         match result {

crates/verify/tests/seek.rs

@@ -46,6 +46,7 @@ fn run(
             for &(frame, seek) in &requests {
                 readouts.push(player.frame(frame, fps, seek).await?);
             }
+            player.check_decoders(&mut case, decoder);
             Ok::<_, String>(readouts)
         });
         let frames: Vec<u32> = requests.iter().map(|r| r.0).collect();
@@ -233,7 +234,9 @@ fn cold_open() {
         for &frame in &frames {
             let result = rt.block_on(async {
                 let mut player = Player::open(&project, config.clone(), decoder, RES).await?;
-                player.frame(frame, 30, true).await
+                let readout = player.frame(frame, 30, true).await;
+                player.check_decoders(&mut case, decoder);
+                readout
             });
             match result {
                 Ok(readout) => readouts.push(readout),