From 2f925fce5d51a14a541ff8e0af57be41484b310d Mon Sep 17 00:00:00 2001 From: Philipp Bischoff Date: Tue, 3 Nov 2026 22:52:06 +0100 Subject: [PATCH] feat: expose the displayed video frame through VideoSampleTap PlayerSession.videoSamples hands the host the frame the renderer last displayed, so Lume can sample it for Philips Hue light sync. The tap is lazy and costs nothing until a host attaches it. Adds quadrant fixtures (8-bit H.264 and 10-bit HEVC) that pin the colour conversion. --- Sources/LumeEngine/LumePlayer.swift | 9 + .../Render/SystemRenderer.swift | 20 +- .../Render/VideoSampleTap.swift | 421 ++++++++++++++++++ .../Session/PlayerSession.swift | 21 +- .../Fixtures/generate-fixtures.sh | 22 + .../FrameSampleTapTests.swift | 395 ++++++++++++++++ 6 files changed, 886 insertions(+), 2 deletions(-) create mode 100644 Sources/LumeEngineCore/Render/VideoSampleTap.swift create mode 100644 Tests/LumeEngineCoreTests/FrameSampleTapTests.swift diff --git a/Sources/LumeEngine/LumePlayer.swift b/Sources/LumeEngine/LumePlayer.swift index 51b4c76..f1fcff6 100644 --- a/Sources/LumeEngine/LumePlayer.swift +++ b/Sources/LumeEngine/LumePlayer.swift @@ -55,6 +55,15 @@ public final class LumePlayer { /// layer tree directly (UIKit/AppKit). public var displayLayer: LumeDisplayLayer? { session?.renderer.displayLayer } + /// The current session's colour sampler (see `VideoSampleTap`); nil with + /// nothing loaded. Each `load` starts a new one, enabled only when the + /// configuration's `videoSampling` is set. + public var videoSamples: VideoSampleTap? { session?.renderer.videoSamples } + + /// The sample of the frame on screen now; nil while sampling is off. + /// Main actor — prefer `videoSamples` for anything per frame. + public var currentVideoSample: VideoSampleTap.Sample? { session?.currentVideoSample() } + /// tvOS: switch the TV to each loaded stream's refresh rate and dynamic /// range (Match Content) while it plays in a `LumePlayerView`, which /// supplies the window; `stop()` and the view going away hand the display diff --git a/Sources/LumeEngineCore/Render/SystemRenderer.swift b/Sources/LumeEngineCore/Render/SystemRenderer.swift index b225949..221dd2f 100644 --- a/Sources/LumeEngineCore/Render/SystemRenderer.swift +++ b/Sources/LumeEngineCore/Render/SystemRenderer.swift @@ -98,8 +98,23 @@ public final class SystemRenderer: @unchecked Sendable { /// (see `pumpAudio`); nil feeds every frame. private let audioOutputRoute: AudioOutputRoute? - public init(muted: Bool = false, audioOutputRoute: AudioOutputRoute? = nil) { + /// Colour readings of the frames handed to the display, off unless + /// enabled (see `VideoSampleTap`). Read + /// `videoSamples.sample(atOrBefore: currentTime)` for what is on screen. + public let videoSamples: VideoSampleTap + + /// - Parameter videoSampling: starts `videoSamples` enabled with this + /// grid; nil leaves it off (it can still be enabled at runtime). + public init( + muted: Bool = false, + audioOutputRoute: AudioOutputRoute? = nil, + videoSampling: VideoSampleTap.Configuration? = nil + ) { self.audioOutputRoute = audioOutputRoute + videoSamples = VideoSampleTap( + configuration: videoSampling ?? VideoSampleTap.Configuration(), + enabled: videoSampling != nil + ) displayLayer = AVSampleBufferDisplayLayer() displayLayer.videoGravity = .resizeAspect videoRenderer = displayLayer.sampleBufferRenderer @@ -335,6 +350,7 @@ public final class SystemRenderer: @unchecked Sendable { videoCadence.restart() lock.unlock() + videoSamples.flush(acceptingSerial: serial) videoRenderer.flush() audioRenderer.flush() } @@ -408,6 +424,7 @@ public final class SystemRenderer: @unchecked Sendable { synchronizer.setRate(0, time: .zero) videoRenderer.flush() audioRenderer.flush() + videoSamples.flush() } // MARK: Pumps (render queues) @@ -432,6 +449,7 @@ public final class SystemRenderer: @unchecked Sendable { do { let sample = try SampleBufferBuilder.video(from: frame, formatCache: &videoFormatCache) videoRenderer.enqueue(sample) + videoSamples.offer(frame) lock.lock() lastEnqueuedVideoPTS = frame.pts if !MediaTime.isValid(firstEnqueuedVideoPTS) { firstEnqueuedVideoPTS = frame.pts } diff --git a/Sources/LumeEngineCore/Render/VideoSampleTap.swift b/Sources/LumeEngineCore/Render/VideoSampleTap.swift new file mode 100644 index 0000000..ae821f7 --- /dev/null +++ b/Sources/LumeEngineCore/Render/VideoSampleTap.swift @@ -0,0 +1,421 @@ +import CoreVideo +import Foundation +import Synchronization + +/// A coarse colour reading of the picture, for hosts that drive something +/// from what is on screen (ambient lighting, a UI tint). +/// +/// Off by default. When enabled, the video render pump offers every frame it +/// hands the display right after enqueueing it; the tap reads a sparse grid of +/// luma and chroma straight from the pixel buffer (locked read-only, never +/// retained), converts it to linear light, and keeps a short ring of samples +/// stamped with the frame's PTS. Frames are enqueued ahead of the clock, so a +/// host reads `sample(atOrBefore: renderer.currentTime)` — what is being shown +/// now — rather than the newest sample. +/// +/// The render queue never waits on it and it never fails playback: a buffer +/// it cannot read (an unexpected pixel format, a lock that fails) is skipped. +/// The ring is emptied on seek and whenever frames of a newer serial arrive, +/// so a reading never comes from before a seek. +/// +/// Colours are linear light on BT.709/sRGB primaries, 0...1, where 1 is SDR +/// reference white: PQ is scaled so 203 nits reads as 1, HLG so its +/// reference white (75 % signal) does; brighter highlights clip to 1. BT.2020 +/// and P3 primaries are converted, and out-of-gamut values clip to 0. +public final class VideoSampleTap: Sendable { + /// How finely the picture is sampled and how much is kept. + public struct Configuration: Sendable, Equatable { + /// Grid cells across the picture, 1...64. + public var columns: Int + /// Grid cells down the picture, 1...64. + public var rows: Int + /// Points read per cell along each axis (n×n per cell), 1...8. + public var pointsPerCell: Int + /// Samples kept, 1...240. Has to span how far the renderer runs + /// ahead of the clock, or `sample(atOrBefore:)` finds nothing old + /// enough; the default covers about a second of 50p. + public var capacity: Int + + public init(columns: Int = 16, rows: Int = 9, pointsPerCell: Int = 3, capacity: Int = 48) { + self.columns = min(max(columns, 1), 64) + self.rows = min(max(rows, 1), 64) + self.pointsPerCell = min(max(pointsPerCell, 1), 8) + self.capacity = min(max(capacity, 1), 240) + } + } + + /// Linear-light RGB on BT.709/sRGB primaries, each 0...1. + public struct Color: Sendable, Equatable { + public var red: Float + public var green: Float + public var blue: Float + + public init(red: Float, green: Float, blue: Float) { + self.red = red + self.green = green + self.blue = blue + } + } + + /// The grid read from one frame. + public struct Sample: Sendable, Equatable { + /// The frame's presentation time, engine µs (comparable to + /// `SystemRenderer.currentTime`). + public let pts: Int64 + public let columns: Int + public let rows: Int + /// Row-major, top-left cell first; `columns * rows` entries. The cell + /// grid covers the coded picture (letterbox bars included). + public let colors: [Color] + + public init(pts: Int64, columns: Int, rows: Int, colors: [Color]) { + self.pts = pts + self.columns = columns + self.rows = rows + self.colors = colors + } + + /// The cell at `column`, `row`; nil outside the grid. + public func color(column: Int, row: Int) -> Color? { + guard (0.. + + public init(configuration: Configuration = Configuration(), enabled: Bool = false) { + state = Mutex(State(enabled: enabled, configuration: configuration)) + } + + // MARK: Host side (any thread) + + /// Whether frames are sampled. Turning it off empties the ring. + public var isEnabled: Bool { + get { state.withLock { $0.enabled } } + set { + state.withLock { state in + state.enabled = newValue + if !newValue { state.ring.removeAll() } + } + } + } + + /// The grid in use. Changing it empties the ring. + public var configuration: Configuration { + get { state.withLock { $0.configuration } } + set { + state.withLock { state in + guard state.configuration != newValue else { return } + state.configuration = newValue + state.ring.removeAll() + } + } + } + + /// The newest sample presented at or before `pts` (engine µs) — pass + /// `renderer.currentTime`. Nil when sampling is off, nothing has been + /// sampled since the last seek, `pts` is invalid, or every sample kept is + /// still ahead of `pts`. + public func sample(atOrBefore pts: Int64) -> Sample? { + guard MediaTime.isValid(pts) else { return nil } + return state.withLock { state in + state.ring.last { $0.pts <= pts } + } + } + + /// The newest sample taken, whether or not it is on screen yet. + public var latestSample: Sample? { + state.withLock { $0.ring.last } + } + + /// Drops every sample kept. + public func flush() { + state.withLock { $0.ring.removeAll() } + } + + // MARK: Engine side + + /// Seek flush: drop what is kept and refuse frames older than `serial`. + func flush(acceptingSerial serial: UInt64) { + state.withLock { state in + state.ring.removeAll() + state.serial = max(state.serial, serial) + } + } + + /// Render queue: called with each frame right after it was enqueued. + /// Costs one lock while disabled. + func offer(_ frame: VideoFrame) { + let (enabled, configuration, serial) = state.withLock { ($0.enabled, $0.configuration, $0.serial) } + guard enabled, frame.serial >= serial, MediaTime.isValid(frame.pts), + let sample = Self.read(frame.pixelBuffer, pts: frame.pts, configuration: configuration) + else { return } + record(sample, serial: frame.serial, configuration: configuration) + } + + private func record(_ sample: Sample, serial: UInt64, configuration: Configuration) { + state.withLock { state in + // Disabled, reconfigured or flushed past while the frame was read. + guard state.enabled, state.configuration == configuration, serial >= state.serial else { return } + if serial > state.serial { + state.serial = serial + state.ring.removeAll() + } + // A timeline that went backwards within one serial: what is kept + // from "after" it would shadow the new frames. + if let last = state.ring.last, last.pts >= sample.pts { + state.ring.removeAll { $0.pts >= sample.pts } + } + state.ring.append(sample) + if state.ring.count > configuration.capacity { + state.ring.removeFirst(state.ring.count - configuration.capacity) + } + } + } + + // MARK: Reading a buffer + + /// Reads the grid from a 4:2:0 bi-planar buffer (8- or 10-bit, video or + /// full range); nil for any other layout or a buffer that cannot be + /// locked. + static func read(_ buffer: CVPixelBuffer, pts: Int64, configuration: Configuration) -> Sample? { + guard let layout = PlaneLayout(CVPixelBufferGetPixelFormatType(buffer)), + CVPixelBufferGetPlaneCount(buffer) == 2 + else { return nil } + let lumaWidth = CVPixelBufferGetWidthOfPlane(buffer, 0) + let lumaHeight = CVPixelBufferGetHeightOfPlane(buffer, 0) + let chromaWidth = CVPixelBufferGetWidthOfPlane(buffer, 1) + let chromaHeight = CVPixelBufferGetHeightOfPlane(buffer, 1) + guard lumaWidth > 0, lumaHeight > 0, chromaWidth > 0, chromaHeight > 0 else { return nil } + let colorimetry = Colorimetry(buffer, height: lumaHeight) + + guard CVPixelBufferLockBaseAddress(buffer, .readOnly) == kCVReturnSuccess else { return nil } + defer { CVPixelBufferUnlockBaseAddress(buffer, .readOnly) } + guard let lumaBase = CVPixelBufferGetBaseAddressOfPlane(buffer, 0), + let chromaBase = CVPixelBufferGetBaseAddressOfPlane(buffer, 1) + else { return nil } + let lumaStride = CVPixelBufferGetBytesPerRowOfPlane(buffer, 0) + let chromaStride = CVPixelBufferGetBytesPerRowOfPlane(buffer, 1) + + let columns = configuration.columns + let rows = configuration.rows + let points = configuration.pointsPerCell + let perCell = Float(points * points) + var colors: [Color] = [] + colors.reserveCapacity(columns * rows) + + for row in 0.. Int { + if bytesPerSample == 1 { + return Int(base.load(fromByteOffset: offset, as: UInt8.self)) + } + return Int(base.loadUnaligned(fromByteOffset: offset, as: UInt16.self) >> 6) + } + + private var scale: Float { Float(1 << (bitDepth - 8)) } + private var maxCode: Float { Float((1 << bitDepth) - 1) } + + /// 0...1 (unclamped). + func normalizedLuma(_ code: Int) -> Float { + fullRange ? Float(code) / maxCode : (Float(code) - 16 * scale) / (219 * scale) + } + + /// −0.5...0.5 (unclamped). + func normalizedChroma(_ code: Int) -> Float { + fullRange + ? (Float(code) - 128 * scale) / maxCode + : (Float(code) - 128 * scale) / (224 * scale) + } +} + +/// The buffer's colour tags, resolved once per frame. +private struct Colorimetry { + enum Transfer { case sdr, pq, hlg } + + let kr: Float + let kb: Float + let transfer: Transfer + /// Row-major 3×3 to BT.709 primaries; nil when already BT.709. + let toBT709: [Float]? + + init(_ buffer: CVPixelBuffer, height: Int) { + func tag(_ key: CFString) -> String? { + CVBufferCopyAttachment(buffer, key, nil) as? String + } + let matrix = tag(kCVImageBufferYCbCrMatrixKey) + let is2020 = matrix == kCVImageBufferYCbCrMatrix_ITU_R_2020 as String + if is2020 { + (kr, kb) = (0.2627, 0.0593) + } else if matrix == kCVImageBufferYCbCrMatrix_ITU_R_709_2 as String { + (kr, kb) = (0.2126, 0.0722) + } else if matrix == kCVImageBufferYCbCrMatrix_ITU_R_601_4 as String { + (kr, kb) = (0.299, 0.114) + } else if matrix == kCVImageBufferYCbCrMatrix_SMPTE_240M_1995 as String { + (kr, kb) = (0.212, 0.087) + } else { + // Untagged: what the engine and VideoToolbox assume (see + // `PixelBufferAttachments`). + (kr, kb) = height <= 576 ? (0.299, 0.114) : (0.2126, 0.0722) + } + + let transferFunction = tag(kCVImageBufferTransferFunctionKey) + if transferFunction == kCVImageBufferTransferFunction_SMPTE_ST_2084_PQ as String { + transfer = .pq + } else if transferFunction == kCVImageBufferTransferFunction_ITU_R_2100_HLG as String { + transfer = .hlg + } else { + transfer = .sdr + } + + let primaries = tag(kCVImageBufferColorPrimariesKey) + if primaries == kCVImageBufferColorPrimaries_ITU_R_2020 as String { + toBT709 = Self.bt2020ToBT709 + } else if primaries == kCVImageBufferColorPrimaries_P3_D65 as String { + toBT709 = Self.p3ToBT709 + } else if primaries == nil, is2020 { + // Untagged primaries follow the matrix, as the engine tags them. + toBT709 = Self.bt2020ToBT709 + } else { + toBT709 = nil + } + } + + private static let bt2020ToBT709: [Float] = [ + 1.6605, -0.5876, -0.0728, + -0.1246, 1.1329, -0.0083, + -0.0182, -0.1006, 1.1187, + ] + + private static let p3ToBT709: [Float] = [ + 1.2249, -0.2247, 0, + -0.0420, 1.0419, 0, + -0.0197, -0.0786, 1.0979, + ] + + /// One point: Y'CbCr → R'G'B' → linear light, still on the source's + /// primaries. + func linear(luma: Float, cb: Float, cr: Float) -> VideoSampleTap.Color { + let kg = 1 - kr - kb + let red = luma + 2 * (1 - kr) * cr + let blue = luma + 2 * (1 - kb) * cb + let green = (luma - kr * red - kb * blue) / kg + return VideoSampleTap.Color(red: decode(red), green: decode(green), blue: decode(blue)) + } + + /// A cell's average: onto BT.709 primaries and clamped. + func output(_ color: VideoSampleTap.Color) -> VideoSampleTap.Color { + var (red, green, blue) = (color.red, color.green, color.blue) + if let matrix = toBT709, matrix.count == 9 { + (red, green, blue) = ( + matrix[0] * red + matrix[1] * green + matrix[2] * blue, + matrix[3] * red + matrix[4] * green + matrix[5] * blue, + matrix[6] * red + matrix[7] * green + matrix[8] * blue + ) + } + return VideoSampleTap.Color(red: clamp(red), green: clamp(green), blue: clamp(blue)) + } + + private func decode(_ value: Float) -> Float { + let signal = clamp(value) + switch transfer { + case .sdr: + return powf(signal, 2.2) + case .pq: + // SMPTE ST 2084 EOTF, scaled so 203 nits (SDR reference white) is 1. + let m1: Float = 0.1593017578125, m2: Float = 78.84375 + let c1: Float = 0.8359375, c2: Float = 18.8515625, c3: Float = 18.6875 + let power = powf(signal, 1 / m2) + let nits = powf(max(power - c1, 0) / (c2 - c3 * power), 1 / m1) * 10_000 + return clamp(nits / 203) + case .hlg: + // BT.2100 inverse OETF, scaled so reference white (75 % signal, + // ≈ 0.265 scene light) is 1. + let a: Float = 0.17883277, b: Float = 0.28466892, c: Float = 0.55991073 + let scene = signal <= 0.5 ? signal * signal / 3 : (expf((signal - c) / a) + b) / 12 + return clamp(scene / 0.265) + } + } + + private func clamp(_ value: Float) -> Float { + value.isFinite ? min(max(value, 0), 1) : 0 + } +} diff --git a/Sources/LumeEngineCore/Session/PlayerSession.swift b/Sources/LumeEngineCore/Session/PlayerSession.swift index e9a39a0..8f9a5ed 100644 --- a/Sources/LumeEngineCore/Session/PlayerSession.swift +++ b/Sources/LumeEngineCore/Session/PlayerSession.swift @@ -75,6 +75,10 @@ public struct PlayerConfiguration: Sendable { /// Pass one to pin the width or drive it yourself, e.g. for a macOS /// multichannel interface. public var audioOutputRoute: AudioOutputRoute? + /// Sample the colours of presented frames from the start (see + /// `VideoSampleTap`); nil leaves sampling off — `renderer.videoSamples` + /// can still turn it on at runtime. + public var videoSampling: VideoSampleTap.Configuration? public init() {} } @@ -108,6 +112,17 @@ public actor PlayerSession { /// `activeCues(at: renderer.currentTime)` from any thread. public nonisolated let subtitles = SubtitleStore() + /// Colour readings of presented frames (see `VideoSampleTap`); enable + /// with `PlayerConfiguration.videoSampling` or `videoSamples.isEnabled`. + public nonisolated var videoSamples: VideoSampleTap { renderer.videoSamples } + + /// The sample of the frame on screen now: `videoSamples` read at the + /// renderer's clock. Any thread; nil while sampling is off or before a + /// sampled frame is shown. + public nonisolated func currentVideoSample() -> VideoSampleTap.Sample? { + renderer.videoSamples.sample(atOrBefore: renderer.currentTime) + } + private(set) var state: State = .idle { didSet { if state != oldValue { eventSink.yield(.stateChanged(state)) } @@ -185,7 +200,11 @@ public actor PlayerSession { public init(configuration: PlayerConfiguration = PlayerConfiguration()) { self.configuration = configuration audioRoute = configuration.audioOutputRoute ?? AudioOutputRoute.system() - renderer = SystemRenderer(muted: configuration.muted, audioOutputRoute: audioRoute) + renderer = SystemRenderer( + muted: configuration.muted, + audioOutputRoute: audioRoute, + videoSampling: configuration.videoSampling + ) var continuation: AsyncStream.Continuation! events = AsyncStream(bufferingPolicy: .unbounded) { continuation = $0 } eventSink = continuation diff --git a/Tests/LumeEngineCoreTests/Fixtures/generate-fixtures.sh b/Tests/LumeEngineCoreTests/Fixtures/generate-fixtures.sh index 30a27fe..60a19da 100644 --- a/Tests/LumeEngineCoreTests/Fixtures/generate-fixtures.sh +++ b/Tests/LumeEngineCoreTests/Fixtures/generate-fixtures.sh @@ -22,6 +22,8 @@ # captions.ts — H.264 TS with CEA-608 captions in A/53 SEI (1–3 s, 4–6 s; needs python3) # pgs.mkv, dvbsub.ts, dvdsub.mkv — the same two bitmap subtitles (1–3 s, 4–6 s) # as PGS, DVB and DVD subtitles on a 1920x1080 canvas (needs python3) +# quadrant.mp4 — 2 s H.264 8-bit BT.709, four flat quadrants: red, green / blue, white +# quadrant10.mp4 — the same picture as HEVC Main 10 (10-bit frames for the colour sampler) set -euo pipefail FFMPEG="${FFMPEG:-ffmpeg}" @@ -503,4 +505,24 @@ PY fi fi +# Four flat quadrants (red | green over blue | white) for the colour sampler +# (`VideoSampleTap`): the RGB → Y'CbCr conversion is pinned to BT.709 limited +# range and tagged so, which is what the sampler must undo. 320x180 keeps the +# quadrant edges on even rows/columns, so 4:2:0 chroma never straddles them. +QUADRANT_INPUTS=( + -f lavfi -i "color=c=red:size=160x90:rate=25:duration=2" + -f lavfi -i "color=c=lime:size=160x90:rate=25:duration=2" + -f lavfi -i "color=c=blue:size=160x90:rate=25:duration=2" + -f lavfi -i "color=c=white:size=160x90:rate=25:duration=2" +) +QUADRANT_FILTER="[0][1][2][3]xstack=inputs=4:layout=0_0|w0_0|0_h0|w0_h0,scale=out_color_matrix=bt709:out_range=tv" +QUADRANT_TAGS=(-colorspace bt709 -color_primaries bt709 -color_trc bt709 -color_range tv) +gen quadrant.mp4 \ + "${QUADRANT_INPUTS[@]}" -filter_complex "$QUADRANT_FILTER,format=yuv420p" \ + -c:v libx264 -preset ultrafast -crf 12 -g 25 "${QUADRANT_TAGS[@]}" +gen quadrant10.mp4 \ + "${QUADRANT_INPUTS[@]}" -filter_complex "$QUADRANT_FILTER,format=yuv420p10le" \ + -c:v libx265 -preset ultrafast -crf 12 -tag:v hvc1 "${QUADRANT_TAGS[@]}" \ + -x265-params "log-level=error:keyint=25" + echo "fixtures ready in $OUT" diff --git a/Tests/LumeEngineCoreTests/FrameSampleTapTests.swift b/Tests/LumeEngineCoreTests/FrameSampleTapTests.swift new file mode 100644 index 0000000..249109a --- /dev/null +++ b/Tests/LumeEngineCoreTests/FrameSampleTapTests.swift @@ -0,0 +1,395 @@ +import CoreVideo +import Foundation +import Testing +@testable import LumeEngineCore + +@Suite("Frame sample tap", .serialized) +struct FrameSampleTapTests { + // MARK: Synthetic buffers + + private typealias RGB = (red: Float, green: Float, blue: Float) + + private enum Matrix: String, CaseIterable { + case bt601, bt709, bt2020 + + var coefficients: (kr: Float, kb: Float) { + switch self { + case .bt601: (0.299, 0.114) + case .bt709: (0.2126, 0.0722) + case .bt2020: (0.2627, 0.0593) + } + } + + var tag: CFString { + switch self { + case .bt601: kCVImageBufferYCbCrMatrix_ITU_R_601_4 + case .bt709: kCVImageBufferYCbCrMatrix_ITU_R_709_2 + case .bt2020: kCVImageBufferYCbCrMatrix_ITU_R_2020 + } + } + } + + /// A 4:2:0 bi-planar buffer filled with one flat gamma-encoded colour, + /// converted with `matrix` and tagged with `tag` (nil leaves it untagged). + private func flatBuffer( + _ color: RGB, + matrix: Matrix, + tag: CFString?, + primaries: CFString? = kCVImageBufferColorPrimaries_ITU_R_709_2, + format: OSType, + width: Int = 64, + height: Int = 36 + ) throws -> CVPixelBuffer { + var created: CVPixelBuffer? + let status = CVPixelBufferCreate(kCFAllocatorDefault, width, height, format, nil, &created) + let buffer = try #require(status == kCVReturnSuccess ? created : nil) + + let tenBit = format == kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange + || format == kCVPixelFormatType_420YpCbCr10BiPlanarFullRange + let fullRange = format == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange + || format == kCVPixelFormatType_420YpCbCr10BiPlanarFullRange + let scale: Float = tenBit ? 4 : 1 + let maxCode: Float = tenBit ? 1023 : 255 + + let (kr, kb) = matrix.coefficients + let luma = kr * color.red + (1 - kr - kb) * color.green + kb * color.blue + let cb = (color.blue - luma) / (2 * (1 - kb)) + let cr = (color.red - luma) / (2 * (1 - kr)) + func code(_ value: Float, isLuma: Bool) -> Int { + let raw: Float + if fullRange { + raw = isLuma ? value * maxCode : value * maxCode + 128 * scale + } else { + raw = isLuma ? 16 * scale + value * 219 * scale : 128 * scale + value * 224 * scale + } + return Int(min(max(raw.rounded(), 0), maxCode)) + } + let lumaCode = code(luma, isLuma: true) + let cbCode = code(cb, isLuma: false) + let crCode = code(cr, isLuma: false) + + CVPixelBufferLockBaseAddress(buffer, []) + defer { CVPixelBufferUnlockBaseAddress(buffer, []) } + for plane in 0..<2 { + let base = try #require(CVPixelBufferGetBaseAddressOfPlane(buffer, plane)) + let stride = CVPixelBufferGetBytesPerRowOfPlane(buffer, plane) + let planeWidth = CVPixelBufferGetWidthOfPlane(buffer, plane) + let planeHeight = CVPixelBufferGetHeightOfPlane(buffer, plane) + let values = plane == 0 ? [lumaCode] : [cbCode, crCode] + for row in 0.. VideoFrame { + VideoFrame(pixelBuffer: buffer, pts: pts, duration: 40_000, serial: serial, isHardwareDecoded: false) + } + + private func linear(_ color: RGB) -> VideoSampleTap.Color { + VideoSampleTap.Color(red: powf(color.red, 2.2), green: powf(color.green, 2.2), blue: powf(color.blue, 2.2)) + } + + private func distance(_ lhs: VideoSampleTap.Color, _ rhs: VideoSampleTap.Color) -> Float { + max(abs(lhs.red - rhs.red), abs(lhs.green - rhs.green), abs(lhs.blue - rhs.blue)) + } + + private static let palette: [(name: String, color: (red: Float, green: Float, blue: Float))] = [ + ("red", (1, 0, 0)), + ("green", (0, 1, 0)), + ("blue", (0, 0, 1)), + ("white", (1, 1, 1)), + ("black", (0, 0, 0)), + ("mixed", (0.8, 0.4, 0.2)), + ] + + private static let formats: [OSType] = [ + kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, + kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, + kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange, + kCVPixelFormatType_420YpCbCr10BiPlanarFullRange, + ] + + // MARK: Colour conversion + + @Test("colours: each tagged matrix decodes to the source colour, 8- and 10-bit, both ranges", + arguments: Matrix.allCases, formats) + private func taggedMatrix(_ matrix: Matrix, format: OSType) throws { + let configuration = VideoSampleTap.Configuration(columns: 4, rows: 3, pointsPerCell: 2, capacity: 4) + for (name, color) in Self.palette { + let buffer = try flatBuffer(color, matrix: matrix, tag: matrix.tag, format: format) + let sample = try #require(VideoSampleTap.read(buffer, pts: 0, configuration: configuration)) + #expect(sample.colors.count == 12) + let expected = linear(color) + for cell in sample.colors { + #expect(distance(cell, expected) < 0.02, "\(matrix.rawValue) \(name): got \(cell), want \(expected)") + } + } + } + + @Test("colours: the matrix comes from the buffer's tag, not a fixed guess") + func matrixTagIsHonoured() throws { + let configuration = VideoSampleTap.Configuration(columns: 1, rows: 1, pointsPerCell: 1, capacity: 1) + // Unsaturated, so the wrong matrix cannot hide behind clamping. + let color: RGB = (0.2, 0.9, 0.3) + let right = try flatBuffer( + color, matrix: .bt601, tag: Matrix.bt601.tag, format: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + ) + let wrong = try flatBuffer( + color, matrix: .bt601, tag: Matrix.bt709.tag, format: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + ) + let decodedRight = try #require(VideoSampleTap.read(right, pts: 0, configuration: configuration)?.average) + let decodedWrong = try #require(VideoSampleTap.read(wrong, pts: 0, configuration: configuration)?.average) + #expect(distance(decodedRight, linear(color)) < 0.02) + #expect(distance(decodedWrong, linear(color)) > 0.05, "a 601 picture read as 709 must not come out right") + } + + @Test("colours: untagged SD reads as BT.601, untagged HD as BT.709") + func untaggedFallback() throws { + let configuration = VideoSampleTap.Configuration(columns: 1, rows: 1, pointsPerCell: 1, capacity: 1) + let color: RGB = (0.2, 0.9, 0.3) + let sd = try flatBuffer( + color, matrix: .bt601, tag: nil, format: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, + width: 720, height: 576 + ) + let hd = try flatBuffer( + color, matrix: .bt709, tag: nil, format: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, + width: 1280, height: 720 + ) + let sdColor = try #require(VideoSampleTap.read(sd, pts: 0, configuration: configuration)?.average) + let hdColor = try #require(VideoSampleTap.read(hd, pts: 0, configuration: configuration)?.average) + #expect(distance(sdColor, linear(color)) < 0.02) + #expect(distance(hdColor, linear(color)) < 0.02) + } + + @Test("colours: BT.2020 primaries map onto BT.709, white stays white") + func bt2020Primaries() throws { + let configuration = VideoSampleTap.Configuration(columns: 1, rows: 1, pointsPerCell: 1, capacity: 1) + let white = try flatBuffer( + (1, 1, 1), matrix: .bt2020, tag: Matrix.bt2020.tag, primaries: nil, + format: kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange + ) + let red = try flatBuffer( + (0.6, 0.2, 0.2), matrix: .bt2020, tag: Matrix.bt2020.tag, primaries: nil, + format: kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange + ) + let whiteColor = try #require(VideoSampleTap.read(white, pts: 0, configuration: configuration)?.average) + let redColor = try #require(VideoSampleTap.read(red, pts: 0, configuration: configuration)?.average) + #expect(distance(whiteColor, VideoSampleTap.Color(red: 1, green: 1, blue: 1)) < 0.02) + // A BT.2020 red is more saturated than the same code values in BT.709. + let unconverted = linear((0.6, 0.2, 0.2)) + #expect(redColor.red > unconverted.red) + #expect(redColor.green < unconverted.green) + } + + @Test("buffers it cannot read give no sample") + func unsupportedFormat() throws { + var created: CVPixelBuffer? + CVPixelBufferCreate(kCFAllocatorDefault, 16, 16, kCVPixelFormatType_32BGRA, nil, &created) + let buffer = try #require(created) + #expect(VideoSampleTap.read(buffer, pts: 0, configuration: .init()) == nil) + } + + // MARK: Ring + + @Test("ring: off by default, so nothing is kept") + func disabledByDefault() throws { + let tap = VideoSampleTap() + #expect(!tap.isEnabled) + let buffer = try flatBuffer( + (1, 0, 0), matrix: .bt709, tag: Matrix.bt709.tag, format: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + ) + tap.offer(frame(buffer, pts: 0)) + #expect(tap.latestSample == nil) + + tap.isEnabled = true + tap.offer(frame(buffer, pts: 40_000)) + #expect(tap.latestSample?.pts == 40_000) + + tap.isEnabled = false + #expect(tap.latestSample == nil, "disabling clears what was kept") + } + + @Test("ring: atOrBefore picks the newest sample not after the time, capacity trims the oldest") + func atOrBefore() throws { + let tap = VideoSampleTap(configuration: .init(columns: 2, rows: 2, pointsPerCell: 1, capacity: 3), enabled: true) + let buffer = try flatBuffer( + (0, 0, 1), matrix: .bt709, tag: Matrix.bt709.tag, format: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + ) + for pts in stride(from: Int64(0), through: 160_000, by: 40_000) { + tap.offer(frame(buffer, pts: pts)) + } + // Kept: 80_000, 120_000, 160_000. + #expect(tap.sample(atOrBefore: 79_999) == nil) + #expect(tap.sample(atOrBefore: 80_000)?.pts == 80_000) + #expect(tap.sample(atOrBefore: 150_000)?.pts == 120_000) + #expect(tap.sample(atOrBefore: 10_000_000)?.pts == 160_000) + #expect(tap.sample(atOrBefore: MediaTime.noTimestamp) == nil) + #expect(tap.latestSample?.color(column: 1, row: 1) != nil) + #expect(tap.latestSample?.color(column: 2, row: 0) == nil) + } + + @Test("ring: empty after flush; a stale serial is refused, a newer one replaces the ring") + func flushAndSerial() throws { + let tap = VideoSampleTap(enabled: true) + let buffer = try flatBuffer( + (1, 1, 1), matrix: .bt709, tag: Matrix.bt709.tag, format: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + ) + tap.offer(frame(buffer, pts: 0)) + tap.offer(frame(buffer, pts: 40_000)) + #expect(tap.latestSample?.pts == 40_000) + + tap.flush() + #expect(tap.latestSample == nil) + + tap.offer(frame(buffer, pts: 80_000)) + tap.flush(acceptingSerial: 2) + #expect(tap.latestSample == nil) + + // A frame decoded before the seek, still in flight. + tap.offer(frame(buffer, pts: 120_000, serial: 1)) + #expect(tap.latestSample == nil, "a pre-seek frame must not land after the flush") + + tap.offer(frame(buffer, pts: 5_000_000, serial: 2)) + #expect(tap.latestSample?.pts == 5_000_000) + + // A newer serial without a flush (serial change) starts a fresh ring. + tap.offer(frame(buffer, pts: 1_000_000, serial: 3)) + #expect(tap.sample(atOrBefore: 5_000_000)?.pts == 1_000_000) + #expect(tap.sample(atOrBefore: 999_999) == nil) + } + + @Test("ring: a timeline that steps back drops what it would shadow") + func backwardsPTS() throws { + let tap = VideoSampleTap(enabled: true) + let buffer = try flatBuffer( + (0, 1, 0), matrix: .bt709, tag: Matrix.bt709.tag, format: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + ) + for pts in [Int64(0), 40_000, 80_000, 120_000] { + tap.offer(frame(buffer, pts: pts)) + } + tap.offer(frame(buffer, pts: 60_000)) + #expect(tap.latestSample?.pts == 60_000) + #expect(tap.sample(atOrBefore: 200_000)?.pts == 60_000) + #expect(tap.sample(atOrBefore: 50_000)?.pts == 40_000) + } + + @Test("ring: reconfiguring clears it and changes the grid") + func reconfigure() throws { + let tap = VideoSampleTap(enabled: true) + let buffer = try flatBuffer( + (1, 0, 0), matrix: .bt709, tag: Matrix.bt709.tag, format: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + ) + tap.offer(frame(buffer, pts: 0)) + #expect(tap.latestSample?.colors.count == 16 * 9) + tap.configuration = .init(columns: 3, rows: 2, pointsPerCell: 1, capacity: 8) + #expect(tap.latestSample == nil) + tap.offer(frame(buffer, pts: 40_000)) + #expect(tap.latestSample?.colors.count == 6) + // Out-of-range values clamp rather than trap. + let clamped = VideoSampleTap.Configuration(columns: 0, rows: 1_000, pointsPerCell: -3, capacity: 0) + #expect(clamped.columns >= 1 && clamped.rows <= 64 && clamped.pointsPerCell >= 1 && clamped.capacity >= 1) + } + + // MARK: Decoded fixture + + @Test("fixture: decoded quadrants come out red, green, blue, white with monotonic PTS", + .timeLimit(.minutes(1)), arguments: ["quadrant.mp4", "quadrant10.mp4"]) + func quadrantFixture(_ fixture: String) async throws { + let demuxer = Demuxer(url: try Fixtures.path(fixture)) + defer { demuxer.shutdown() } + var demuxEvents = demuxer.events.makeAsyncIterator() + demuxer.start() + guard case .opened(let info)? = await demuxEvents.next() else { + Issue.record("fixture \(fixture) failed to open") + return + } + let track = try #require(info.videoTracks.first) + let parameters = try #require(demuxer.codecParameters(forStream: track.index)) + + let packets = Channel(capacity: 64) + let frames = Channel(capacity: 16, measure: { $0.duration }) + demuxer.attach(channel: packets, toStream: track.index) + let decoder = VideoDecoder(parameters: parameters, input: packets, output: frames) + var decodeEvents = decoder.events.makeAsyncIterator() + decoder.start() + demuxer.resume() + + let tap = VideoSampleTap( + configuration: .init(columns: 2, rows: 2, pointsPerCell: 3, capacity: 240), + enabled: true + ) + // Offer on the drain thread, as the render queue would; keep only PTS. + let collector = ChannelDrain(frames, into: [Int64]()) { collected, frame in + tap.offer(frame) + collected.append(frame.pts) + } + + while let event = await demuxEvents.next() { + if case .endOfStream = event { + decoder.signalEndOfStream() + break + } + } + guard case .endOfStream? = await decodeEvents.next() else { + Issue.record("expected decoder .endOfStream") + decoder.shutdown() + return + } + decoder.shutdown() + let offered = await collector.value + + #expect(offered.count >= 45, "2 s @ 25 fps ≈ 50 frames, got \(offered.count)") + #expect(zip(offered, offered.dropFirst()).allSatisfy { $0 < $1 }, "decoded PTS must be monotonic") + + // Every offered frame is kept and found at its own PTS. + for pts in offered { + #expect(tap.sample(atOrBefore: pts)?.pts == pts) + } + if let first = offered.first { + #expect(tap.sample(atOrBefore: first - 1) == nil) + } + + let sample = try #require(tap.latestSample) + let expected: [(column: Int, row: Int, name: String, color: VideoSampleTap.Color)] = [ + (0, 0, "red", .init(red: 1, green: 0, blue: 0)), + (1, 0, "green", .init(red: 0, green: 1, blue: 0)), + (0, 1, "blue", .init(red: 0, green: 0, blue: 1)), + (1, 1, "white", .init(red: 1, green: 1, blue: 1)), + ] + for quadrant in expected { + let got = try #require(sample.color(column: quadrant.column, row: quadrant.row)) + #expect( + distance(got, quadrant.color) < 0.08, + "\(fixture) \(quadrant.name) quadrant: got \(got)" + ) + } + + tap.flush() + #expect(tap.latestSample == nil) + #expect(tap.sample(atOrBefore: offered.last ?? 0) == nil) + } +}