diff --git a/Sources/LumeEngine/LumePlayer.swift b/Sources/LumeEngine/LumePlayer.swift index 51b4c76..f1fcff6 100644 --- a/Sources/LumeEngine/LumePlayer.swift +++ b/Sources/LumeEngine/LumePlayer.swift @@ -55,6 +55,15 @@ public final class LumePlayer { /// layer tree directly (UIKit/AppKit). public var displayLayer: LumeDisplayLayer? { session?.renderer.displayLayer } + /// The current session's colour sampler (see `VideoSampleTap`); nil with + /// nothing loaded. Each `load` starts a new one, enabled only when the + /// configuration's `videoSampling` is set. + public var videoSamples: VideoSampleTap? { session?.renderer.videoSamples } + + /// The sample of the frame on screen now; nil while sampling is off. + /// Main actor — prefer `videoSamples` for anything per frame. + public var currentVideoSample: VideoSampleTap.Sample? { session?.currentVideoSample() } + /// tvOS: switch the TV to each loaded stream's refresh rate and dynamic /// range (Match Content) while it plays in a `LumePlayerView`, which /// supplies the window; `stop()` and the view going away hand the display diff --git a/Sources/LumeEngineCore/Render/SystemRenderer.swift b/Sources/LumeEngineCore/Render/SystemRenderer.swift index b225949..221dd2f 100644 --- a/Sources/LumeEngineCore/Render/SystemRenderer.swift +++ b/Sources/LumeEngineCore/Render/SystemRenderer.swift @@ -98,8 +98,23 @@ public final class SystemRenderer: @unchecked Sendable { /// (see `pumpAudio`); nil feeds every frame. private let audioOutputRoute: AudioOutputRoute? - public init(muted: Bool = false, audioOutputRoute: AudioOutputRoute? = nil) { + /// Colour readings of the frames handed to the display, off unless + /// enabled (see `VideoSampleTap`). Read + /// `videoSamples.sample(atOrBefore: currentTime)` for what is on screen. + public let videoSamples: VideoSampleTap + + /// - Parameter videoSampling: starts `videoSamples` enabled with this + /// grid; nil leaves it off (it can still be enabled at runtime). + public init( + muted: Bool = false, + audioOutputRoute: AudioOutputRoute? = nil, + videoSampling: VideoSampleTap.Configuration? = nil + ) { self.audioOutputRoute = audioOutputRoute + videoSamples = VideoSampleTap( + configuration: videoSampling ?? VideoSampleTap.Configuration(), + enabled: videoSampling != nil + ) displayLayer = AVSampleBufferDisplayLayer() displayLayer.videoGravity = .resizeAspect videoRenderer = displayLayer.sampleBufferRenderer @@ -335,6 +350,7 @@ public final class SystemRenderer: @unchecked Sendable { videoCadence.restart() lock.unlock() + videoSamples.flush(acceptingSerial: serial) videoRenderer.flush() audioRenderer.flush() } @@ -408,6 +424,7 @@ public final class SystemRenderer: @unchecked Sendable { synchronizer.setRate(0, time: .zero) videoRenderer.flush() audioRenderer.flush() + videoSamples.flush() } // MARK: Pumps (render queues) @@ -432,6 +449,7 @@ public final class SystemRenderer: @unchecked Sendable { do { let sample = try SampleBufferBuilder.video(from: frame, formatCache: &videoFormatCache) videoRenderer.enqueue(sample) + videoSamples.offer(frame) lock.lock() lastEnqueuedVideoPTS = frame.pts if !MediaTime.isValid(firstEnqueuedVideoPTS) { firstEnqueuedVideoPTS = frame.pts } diff --git a/Sources/LumeEngineCore/Render/VideoSampleTap.swift b/Sources/LumeEngineCore/Render/VideoSampleTap.swift new file mode 100644 index 0000000..ae821f7 --- /dev/null +++ b/Sources/LumeEngineCore/Render/VideoSampleTap.swift @@ -0,0 +1,421 @@ +import CoreVideo +import Foundation +import Synchronization + +/// A coarse colour reading of the picture, for hosts that drive something +/// from what is on screen (ambient lighting, a UI tint). +/// +/// Off by default. When enabled, the video render pump offers every frame it +/// hands the display right after enqueueing it; the tap reads a sparse grid of +/// luma and chroma straight from the pixel buffer (locked read-only, never +/// retained), converts it to linear light, and keeps a short ring of samples +/// stamped with the frame's PTS. Frames are enqueued ahead of the clock, so a +/// host reads `sample(atOrBefore: renderer.currentTime)` — what is being shown +/// now — rather than the newest sample. +/// +/// The render queue never waits on it and it never fails playback: a buffer +/// it cannot read (an unexpected pixel format, a lock that fails) is skipped. +/// The ring is emptied on seek and whenever frames of a newer serial arrive, +/// so a reading never comes from before a seek. +/// +/// Colours are linear light on BT.709/sRGB primaries, 0...1, where 1 is SDR +/// reference white: PQ is scaled so 203 nits reads as 1, HLG so its +/// reference white (75 % signal) does; brighter highlights clip to 1. BT.2020 +/// and P3 primaries are converted, and out-of-gamut values clip to 0. +public final class VideoSampleTap: Sendable { + /// How finely the picture is sampled and how much is kept. + public struct Configuration: Sendable, Equatable { + /// Grid cells across the picture, 1...64. + public var columns: Int + /// Grid cells down the picture, 1...64. + public var rows: Int + /// Points read per cell along each axis (n×n per cell), 1...8. + public var pointsPerCell: Int + /// Samples kept, 1...240. Has to span how far the renderer runs + /// ahead of the clock, or `sample(atOrBefore:)` finds nothing old + /// enough; the default covers about a second of 50p. + public var capacity: Int + + public init(columns: Int = 16, rows: Int = 9, pointsPerCell: Int = 3, capacity: Int = 48) { + self.columns = min(max(columns, 1), 64) + self.rows = min(max(rows, 1), 64) + self.pointsPerCell = min(max(pointsPerCell, 1), 8) + self.capacity = min(max(capacity, 1), 240) + } + } + + /// Linear-light RGB on BT.709/sRGB primaries, each 0...1. + public struct Color: Sendable, Equatable { + public var red: Float + public var green: Float + public var blue: Float + + public init(red: Float, green: Float, blue: Float) { + self.red = red + self.green = green + self.blue = blue + } + } + + /// The grid read from one frame. + public struct Sample: Sendable, Equatable { + /// The frame's presentation time, engine µs (comparable to + /// `SystemRenderer.currentTime`). + public let pts: Int64 + public let columns: Int + public let rows: Int + /// Row-major, top-left cell first; `columns * rows` entries. The cell + /// grid covers the coded picture (letterbox bars included). + public let colors: [Color] + + public init(pts: Int64, columns: Int, rows: Int, colors: [Color]) { + self.pts = pts + self.columns = columns + self.rows = rows + self.colors = colors + } + + /// The cell at `column`, `row`; nil outside the grid. + public func color(column: Int, row: Int) -> Color? { + guard (0.. + + public init(configuration: Configuration = Configuration(), enabled: Bool = false) { + state = Mutex(State(enabled: enabled, configuration: configuration)) + } + + // MARK: Host side (any thread) + + /// Whether frames are sampled. Turning it off empties the ring. + public var isEnabled: Bool { + get { state.withLock { $0.enabled } } + set { + state.withLock { state in + state.enabled = newValue + if !newValue { state.ring.removeAll() } + } + } + } + + /// The grid in use. Changing it empties the ring. + public var configuration: Configuration { + get { state.withLock { $0.configuration } } + set { + state.withLock { state in + guard state.configuration != newValue else { return } + state.configuration = newValue + state.ring.removeAll() + } + } + } + + /// The newest sample presented at or before `pts` (engine µs) — pass + /// `renderer.currentTime`. Nil when sampling is off, nothing has been + /// sampled since the last seek, `pts` is invalid, or every sample kept is + /// still ahead of `pts`. + public func sample(atOrBefore pts: Int64) -> Sample? { + guard MediaTime.isValid(pts) else { return nil } + return state.withLock { state in + state.ring.last { $0.pts <= pts } + } + } + + /// The newest sample taken, whether or not it is on screen yet. + public var latestSample: Sample? { + state.withLock { $0.ring.last } + } + + /// Drops every sample kept. + public func flush() { + state.withLock { $0.ring.removeAll() } + } + + // MARK: Engine side + + /// Seek flush: drop what is kept and refuse frames older than `serial`. + func flush(acceptingSerial serial: UInt64) { + state.withLock { state in + state.ring.removeAll() + state.serial = max(state.serial, serial) + } + } + + /// Render queue: called with each frame right after it was enqueued. + /// Costs one lock while disabled. + func offer(_ frame: VideoFrame) { + let (enabled, configuration, serial) = state.withLock { ($0.enabled, $0.configuration, $0.serial) } + guard enabled, frame.serial >= serial, MediaTime.isValid(frame.pts), + let sample = Self.read(frame.pixelBuffer, pts: frame.pts, configuration: configuration) + else { return } + record(sample, serial: frame.serial, configuration: configuration) + } + + private func record(_ sample: Sample, serial: UInt64, configuration: Configuration) { + state.withLock { state in + // Disabled, reconfigured or flushed past while the frame was read. + guard state.enabled, state.configuration == configuration, serial >= state.serial else { return } + if serial > state.serial { + state.serial = serial + state.ring.removeAll() + } + // A timeline that went backwards within one serial: what is kept + // from "after" it would shadow the new frames. + if let last = state.ring.last, last.pts >= sample.pts { + state.ring.removeAll { $0.pts >= sample.pts } + } + state.ring.append(sample) + if state.ring.count > configuration.capacity { + state.ring.removeFirst(state.ring.count - configuration.capacity) + } + } + } + + // MARK: Reading a buffer + + /// Reads the grid from a 4:2:0 bi-planar buffer (8- or 10-bit, video or + /// full range); nil for any other layout or a buffer that cannot be + /// locked. + static func read(_ buffer: CVPixelBuffer, pts: Int64, configuration: Configuration) -> Sample? { + guard let layout = PlaneLayout(CVPixelBufferGetPixelFormatType(buffer)), + CVPixelBufferGetPlaneCount(buffer) == 2 + else { return nil } + let lumaWidth = CVPixelBufferGetWidthOfPlane(buffer, 0) + let lumaHeight = CVPixelBufferGetHeightOfPlane(buffer, 0) + let chromaWidth = CVPixelBufferGetWidthOfPlane(buffer, 1) + let chromaHeight = CVPixelBufferGetHeightOfPlane(buffer, 1) + guard lumaWidth > 0, lumaHeight > 0, chromaWidth > 0, chromaHeight > 0 else { return nil } + let colorimetry = Colorimetry(buffer, height: lumaHeight) + + guard CVPixelBufferLockBaseAddress(buffer, .readOnly) == kCVReturnSuccess else { return nil } + defer { CVPixelBufferUnlockBaseAddress(buffer, .readOnly) } + guard let lumaBase = CVPixelBufferGetBaseAddressOfPlane(buffer, 0), + let chromaBase = CVPixelBufferGetBaseAddressOfPlane(buffer, 1) + else { return nil } + let lumaStride = CVPixelBufferGetBytesPerRowOfPlane(buffer, 0) + let chromaStride = CVPixelBufferGetBytesPerRowOfPlane(buffer, 1) + + let columns = configuration.columns + let rows = configuration.rows + let points = configuration.pointsPerCell + let perCell = Float(points * points) + var colors: [Color] = [] + colors.reserveCapacity(columns * rows) + + for row in 0.. Int { + if bytesPerSample == 1 { + return Int(base.load(fromByteOffset: offset, as: UInt8.self)) + } + return Int(base.loadUnaligned(fromByteOffset: offset, as: UInt16.self) >> 6) + } + + private var scale: Float { Float(1 << (bitDepth - 8)) } + private var maxCode: Float { Float((1 << bitDepth) - 1) } + + /// 0...1 (unclamped). + func normalizedLuma(_ code: Int) -> Float { + fullRange ? Float(code) / maxCode : (Float(code) - 16 * scale) / (219 * scale) + } + + /// −0.5...0.5 (unclamped). + func normalizedChroma(_ code: Int) -> Float { + fullRange + ? (Float(code) - 128 * scale) / maxCode + : (Float(code) - 128 * scale) / (224 * scale) + } +} + +/// The buffer's colour tags, resolved once per frame. +private struct Colorimetry { + enum Transfer { case sdr, pq, hlg } + + let kr: Float + let kb: Float + let transfer: Transfer + /// Row-major 3×3 to BT.709 primaries; nil when already BT.709. + let toBT709: [Float]? + + init(_ buffer: CVPixelBuffer, height: Int) { + func tag(_ key: CFString) -> String? { + CVBufferCopyAttachment(buffer, key, nil) as? String + } + let matrix = tag(kCVImageBufferYCbCrMatrixKey) + let is2020 = matrix == kCVImageBufferYCbCrMatrix_ITU_R_2020 as String + if is2020 { + (kr, kb) = (0.2627, 0.0593) + } else if matrix == kCVImageBufferYCbCrMatrix_ITU_R_709_2 as String { + (kr, kb) = (0.2126, 0.0722) + } else if matrix == kCVImageBufferYCbCrMatrix_ITU_R_601_4 as String { + (kr, kb) = (0.299, 0.114) + } else if matrix == kCVImageBufferYCbCrMatrix_SMPTE_240M_1995 as String { + (kr, kb) = (0.212, 0.087) + } else { + // Untagged: what the engine and VideoToolbox assume (see + // `PixelBufferAttachments`). + (kr, kb) = height <= 576 ? (0.299, 0.114) : (0.2126, 0.0722) + } + + let transferFunction = tag(kCVImageBufferTransferFunctionKey) + if transferFunction == kCVImageBufferTransferFunction_SMPTE_ST_2084_PQ as String { + transfer = .pq + } else if transferFunction == kCVImageBufferTransferFunction_ITU_R_2100_HLG as String { + transfer = .hlg + } else { + transfer = .sdr + } + + let primaries = tag(kCVImageBufferColorPrimariesKey) + if primaries == kCVImageBufferColorPrimaries_ITU_R_2020 as String { + toBT709 = Self.bt2020ToBT709 + } else if primaries == kCVImageBufferColorPrimaries_P3_D65 as String { + toBT709 = Self.p3ToBT709 + } else if primaries == nil, is2020 { + // Untagged primaries follow the matrix, as the engine tags them. + toBT709 = Self.bt2020ToBT709 + } else { + toBT709 = nil + } + } + + private static let bt2020ToBT709: [Float] = [ + 1.6605, -0.5876, -0.0728, + -0.1246, 1.1329, -0.0083, + -0.0182, -0.1006, 1.1187, + ] + + private static let p3ToBT709: [Float] = [ + 1.2249, -0.2247, 0, + -0.0420, 1.0419, 0, + -0.0197, -0.0786, 1.0979, + ] + + /// One point: Y'CbCr → R'G'B' → linear light, still on the source's + /// primaries. + func linear(luma: Float, cb: Float, cr: Float) -> VideoSampleTap.Color { + let kg = 1 - kr - kb + let red = luma + 2 * (1 - kr) * cr + let blue = luma + 2 * (1 - kb) * cb + let green = (luma - kr * red - kb * blue) / kg + return VideoSampleTap.Color(red: decode(red), green: decode(green), blue: decode(blue)) + } + + /// A cell's average: onto BT.709 primaries and clamped. + func output(_ color: VideoSampleTap.Color) -> VideoSampleTap.Color { + var (red, green, blue) = (color.red, color.green, color.blue) + if let matrix = toBT709, matrix.count == 9 { + (red, green, blue) = ( + matrix[0] * red + matrix[1] * green + matrix[2] * blue, + matrix[3] * red + matrix[4] * green + matrix[5] * blue, + matrix[6] * red + matrix[7] * green + matrix[8] * blue + ) + } + return VideoSampleTap.Color(red: clamp(red), green: clamp(green), blue: clamp(blue)) + } + + private func decode(_ value: Float) -> Float { + let signal = clamp(value) + switch transfer { + case .sdr: + return powf(signal, 2.2) + case .pq: + // SMPTE ST 2084 EOTF, scaled so 203 nits (SDR reference white) is 1. + let m1: Float = 0.1593017578125, m2: Float = 78.84375 + let c1: Float = 0.8359375, c2: Float = 18.8515625, c3: Float = 18.6875 + let power = powf(signal, 1 / m2) + let nits = powf(max(power - c1, 0) / (c2 - c3 * power), 1 / m1) * 10_000 + return clamp(nits / 203) + case .hlg: + // BT.2100 inverse OETF, scaled so reference white (75 % signal, + // ≈ 0.265 scene light) is 1. + let a: Float = 0.17883277, b: Float = 0.28466892, c: Float = 0.55991073 + let scene = signal <= 0.5 ? signal * signal / 3 : (expf((signal - c) / a) + b) / 12 + return clamp(scene / 0.265) + } + } + + private func clamp(_ value: Float) -> Float { + value.isFinite ? min(max(value, 0), 1) : 0 + } +} diff --git a/Sources/LumeEngineCore/Session/PlayerSession.swift b/Sources/LumeEngineCore/Session/PlayerSession.swift index e9a39a0..8f9a5ed 100644 --- a/Sources/LumeEngineCore/Session/PlayerSession.swift +++ b/Sources/LumeEngineCore/Session/PlayerSession.swift @@ -75,6 +75,10 @@ public struct PlayerConfiguration: Sendable { /// Pass one to pin the width or drive it yourself, e.g. for a macOS /// multichannel interface. public var audioOutputRoute: AudioOutputRoute? + /// Sample the colours of presented frames from the start (see + /// `VideoSampleTap`); nil leaves sampling off — `renderer.videoSamples` + /// can still turn it on at runtime. + public var videoSampling: VideoSampleTap.Configuration? public init() {} } @@ -108,6 +112,17 @@ public actor PlayerSession { /// `activeCues(at: renderer.currentTime)` from any thread. public nonisolated let subtitles = SubtitleStore() + /// Colour readings of presented frames (see `VideoSampleTap`); enable + /// with `PlayerConfiguration.videoSampling` or `videoSamples.isEnabled`. + public nonisolated var videoSamples: VideoSampleTap { renderer.videoSamples } + + /// The sample of the frame on screen now: `videoSamples` read at the + /// renderer's clock. Any thread; nil while sampling is off or before a + /// sampled frame is shown. + public nonisolated func currentVideoSample() -> VideoSampleTap.Sample? { + renderer.videoSamples.sample(atOrBefore: renderer.currentTime) + } + private(set) var state: State = .idle { didSet { if state != oldValue { eventSink.yield(.stateChanged(state)) } @@ -185,7 +200,11 @@ public actor PlayerSession { public init(configuration: PlayerConfiguration = PlayerConfiguration()) { self.configuration = configuration audioRoute = configuration.audioOutputRoute ?? AudioOutputRoute.system() - renderer = SystemRenderer(muted: configuration.muted, audioOutputRoute: audioRoute) + renderer = SystemRenderer( + muted: configuration.muted, + audioOutputRoute: audioRoute, + videoSampling: configuration.videoSampling + ) var continuation: AsyncStream.Continuation! events = AsyncStream(bufferingPolicy: .unbounded) { continuation = $0 } eventSink = continuation diff --git a/Tests/LumeEngineCoreTests/Fixtures/generate-fixtures.sh b/Tests/LumeEngineCoreTests/Fixtures/generate-fixtures.sh index 30a27fe..60a19da 100644 --- a/Tests/LumeEngineCoreTests/Fixtures/generate-fixtures.sh +++ b/Tests/LumeEngineCoreTests/Fixtures/generate-fixtures.sh @@ -22,6 +22,8 @@ # captions.ts — H.264 TS with CEA-608 captions in A/53 SEI (1–3 s, 4–6 s; needs python3) # pgs.mkv, dvbsub.ts, dvdsub.mkv — the same two bitmap subtitles (1–3 s, 4–6 s) # as PGS, DVB and DVD subtitles on a 1920x1080 canvas (needs python3) +# quadrant.mp4 — 2 s H.264 8-bit BT.709, four flat quadrants: red, green / blue, white +# quadrant10.mp4 — the same picture as HEVC Main 10 (10-bit frames for the colour sampler) set -euo pipefail FFMPEG="${FFMPEG:-ffmpeg}" @@ -503,4 +505,24 @@ PY fi fi +# Four flat quadrants (red | green over blue | white) for the colour sampler +# (`VideoSampleTap`): the RGB → Y'CbCr conversion is pinned to BT.709 limited +# range and tagged so, which is what the sampler must undo. 320x180 keeps the +# quadrant edges on even rows/columns, so 4:2:0 chroma never straddles them. +QUADRANT_INPUTS=( + -f lavfi -i "color=c=red:size=160x90:rate=25:duration=2" + -f lavfi -i "color=c=lime:size=160x90:rate=25:duration=2" + -f lavfi -i "color=c=blue:size=160x90:rate=25:duration=2" + -f lavfi -i "color=c=white:size=160x90:rate=25:duration=2" +) +QUADRANT_FILTER="[0][1][2][3]xstack=inputs=4:layout=0_0|w0_0|0_h0|w0_h0,scale=out_color_matrix=bt709:out_range=tv" +QUADRANT_TAGS=(-colorspace bt709 -color_primaries bt709 -color_trc bt709 -color_range tv) +gen quadrant.mp4 \ + "${QUADRANT_INPUTS[@]}" -filter_complex "$QUADRANT_FILTER,format=yuv420p" \ + -c:v libx264 -preset ultrafast -crf 12 -g 25 "${QUADRANT_TAGS[@]}" +gen quadrant10.mp4 \ + "${QUADRANT_INPUTS[@]}" -filter_complex "$QUADRANT_FILTER,format=yuv420p10le" \ + -c:v libx265 -preset ultrafast -crf 12 -tag:v hvc1 "${QUADRANT_TAGS[@]}" \ + -x265-params "log-level=error:keyint=25" + echo "fixtures ready in $OUT" diff --git a/Tests/LumeEngineCoreTests/FrameSampleTapTests.swift b/Tests/LumeEngineCoreTests/FrameSampleTapTests.swift new file mode 100644 index 0000000..249109a --- /dev/null +++ b/Tests/LumeEngineCoreTests/FrameSampleTapTests.swift @@ -0,0 +1,395 @@ +import CoreVideo +import Foundation +import Testing +@testable import LumeEngineCore + +@Suite("Frame sample tap", .serialized) +struct FrameSampleTapTests { + // MARK: Synthetic buffers + + private typealias RGB = (red: Float, green: Float, blue: Float) + + private enum Matrix: String, CaseIterable { + case bt601, bt709, bt2020 + + var coefficients: (kr: Float, kb: Float) { + switch self { + case .bt601: (0.299, 0.114) + case .bt709: (0.2126, 0.0722) + case .bt2020: (0.2627, 0.0593) + } + } + + var tag: CFString { + switch self { + case .bt601: kCVImageBufferYCbCrMatrix_ITU_R_601_4 + case .bt709: kCVImageBufferYCbCrMatrix_ITU_R_709_2 + case .bt2020: kCVImageBufferYCbCrMatrix_ITU_R_2020 + } + } + } + + /// A 4:2:0 bi-planar buffer filled with one flat gamma-encoded colour, + /// converted with `matrix` and tagged with `tag` (nil leaves it untagged). + private func flatBuffer( + _ color: RGB, + matrix: Matrix, + tag: CFString?, + primaries: CFString? = kCVImageBufferColorPrimaries_ITU_R_709_2, + format: OSType, + width: Int = 64, + height: Int = 36 + ) throws -> CVPixelBuffer { + var created: CVPixelBuffer? + let status = CVPixelBufferCreate(kCFAllocatorDefault, width, height, format, nil, &created) + let buffer = try #require(status == kCVReturnSuccess ? created : nil) + + let tenBit = format == kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange + || format == kCVPixelFormatType_420YpCbCr10BiPlanarFullRange + let fullRange = format == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange + || format == kCVPixelFormatType_420YpCbCr10BiPlanarFullRange + let scale: Float = tenBit ? 4 : 1 + let maxCode: Float = tenBit ? 1023 : 255 + + let (kr, kb) = matrix.coefficients + let luma = kr * color.red + (1 - kr - kb) * color.green + kb * color.blue + let cb = (color.blue - luma) / (2 * (1 - kb)) + let cr = (color.red - luma) / (2 * (1 - kr)) + func code(_ value: Float, isLuma: Bool) -> Int { + let raw: Float + if fullRange { + raw = isLuma ? value * maxCode : value * maxCode + 128 * scale + } else { + raw = isLuma ? 16 * scale + value * 219 * scale : 128 * scale + value * 224 * scale + } + return Int(min(max(raw.rounded(), 0), maxCode)) + } + let lumaCode = code(luma, isLuma: true) + let cbCode = code(cb, isLuma: false) + let crCode = code(cr, isLuma: false) + + CVPixelBufferLockBaseAddress(buffer, []) + defer { CVPixelBufferUnlockBaseAddress(buffer, []) } + for plane in 0..<2 { + let base = try #require(CVPixelBufferGetBaseAddressOfPlane(buffer, plane)) + let stride = CVPixelBufferGetBytesPerRowOfPlane(buffer, plane) + let planeWidth = CVPixelBufferGetWidthOfPlane(buffer, plane) + let planeHeight = CVPixelBufferGetHeightOfPlane(buffer, plane) + let values = plane == 0 ? [lumaCode] : [cbCode, crCode] + for row in 0.. VideoFrame { + VideoFrame(pixelBuffer: buffer, pts: pts, duration: 40_000, serial: serial, isHardwareDecoded: false) + } + + private func linear(_ color: RGB) -> VideoSampleTap.Color { + VideoSampleTap.Color(red: powf(color.red, 2.2), green: powf(color.green, 2.2), blue: powf(color.blue, 2.2)) + } + + private func distance(_ lhs: VideoSampleTap.Color, _ rhs: VideoSampleTap.Color) -> Float { + max(abs(lhs.red - rhs.red), abs(lhs.green - rhs.green), abs(lhs.blue - rhs.blue)) + } + + private static let palette: [(name: String, color: (red: Float, green: Float, blue: Float))] = [ + ("red", (1, 0, 0)), + ("green", (0, 1, 0)), + ("blue", (0, 0, 1)), + ("white", (1, 1, 1)), + ("black", (0, 0, 0)), + ("mixed", (0.8, 0.4, 0.2)), + ] + + private static let formats: [OSType] = [ + kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, + kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, + kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange, + kCVPixelFormatType_420YpCbCr10BiPlanarFullRange, + ] + + // MARK: Colour conversion + + @Test("colours: each tagged matrix decodes to the source colour, 8- and 10-bit, both ranges", + arguments: Matrix.allCases, formats) + private func taggedMatrix(_ matrix: Matrix, format: OSType) throws { + let configuration = VideoSampleTap.Configuration(columns: 4, rows: 3, pointsPerCell: 2, capacity: 4) + for (name, color) in Self.palette { + let buffer = try flatBuffer(color, matrix: matrix, tag: matrix.tag, format: format) + let sample = try #require(VideoSampleTap.read(buffer, pts: 0, configuration: configuration)) + #expect(sample.colors.count == 12) + let expected = linear(color) + for cell in sample.colors { + #expect(distance(cell, expected) < 0.02, "\(matrix.rawValue) \(name): got \(cell), want \(expected)") + } + } + } + + @Test("colours: the matrix comes from the buffer's tag, not a fixed guess") + func matrixTagIsHonoured() throws { + let configuration = VideoSampleTap.Configuration(columns: 1, rows: 1, pointsPerCell: 1, capacity: 1) + // Unsaturated, so the wrong matrix cannot hide behind clamping. + let color: RGB = (0.2, 0.9, 0.3) + let right = try flatBuffer( + color, matrix: .bt601, tag: Matrix.bt601.tag, format: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + ) + let wrong = try flatBuffer( + color, matrix: .bt601, tag: Matrix.bt709.tag, format: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + ) + let decodedRight = try #require(VideoSampleTap.read(right, pts: 0, configuration: configuration)?.average) + let decodedWrong = try #require(VideoSampleTap.read(wrong, pts: 0, configuration: configuration)?.average) + #expect(distance(decodedRight, linear(color)) < 0.02) + #expect(distance(decodedWrong, linear(color)) > 0.05, "a 601 picture read as 709 must not come out right") + } + + @Test("colours: untagged SD reads as BT.601, untagged HD as BT.709") + func untaggedFallback() throws { + let configuration = VideoSampleTap.Configuration(columns: 1, rows: 1, pointsPerCell: 1, capacity: 1) + let color: RGB = (0.2, 0.9, 0.3) + let sd = try flatBuffer( + color, matrix: .bt601, tag: nil, format: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, + width: 720, height: 576 + ) + let hd = try flatBuffer( + color, matrix: .bt709, tag: nil, format: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, + width: 1280, height: 720 + ) + let sdColor = try #require(VideoSampleTap.read(sd, pts: 0, configuration: configuration)?.average) + let hdColor = try #require(VideoSampleTap.read(hd, pts: 0, configuration: configuration)?.average) + #expect(distance(sdColor, linear(color)) < 0.02) + #expect(distance(hdColor, linear(color)) < 0.02) + } + + @Test("colours: BT.2020 primaries map onto BT.709, white stays white") + func bt2020Primaries() throws { + let configuration = VideoSampleTap.Configuration(columns: 1, rows: 1, pointsPerCell: 1, capacity: 1) + let white = try flatBuffer( + (1, 1, 1), matrix: .bt2020, tag: Matrix.bt2020.tag, primaries: nil, + format: kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange + ) + let red = try flatBuffer( + (0.6, 0.2, 0.2), matrix: .bt2020, tag: Matrix.bt2020.tag, primaries: nil, + format: kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange + ) + let whiteColor = try #require(VideoSampleTap.read(white, pts: 0, configuration: configuration)?.average) + let redColor = try #require(VideoSampleTap.read(red, pts: 0, configuration: configuration)?.average) + #expect(distance(whiteColor, VideoSampleTap.Color(red: 1, green: 1, blue: 1)) < 0.02) + // A BT.2020 red is more saturated than the same code values in BT.709. + let unconverted = linear((0.6, 0.2, 0.2)) + #expect(redColor.red > unconverted.red) + #expect(redColor.green < unconverted.green) + } + + @Test("buffers it cannot read give no sample") + func unsupportedFormat() throws { + var created: CVPixelBuffer? + CVPixelBufferCreate(kCFAllocatorDefault, 16, 16, kCVPixelFormatType_32BGRA, nil, &created) + let buffer = try #require(created) + #expect(VideoSampleTap.read(buffer, pts: 0, configuration: .init()) == nil) + } + + // MARK: Ring + + @Test("ring: off by default, so nothing is kept") + func disabledByDefault() throws { + let tap = VideoSampleTap() + #expect(!tap.isEnabled) + let buffer = try flatBuffer( + (1, 0, 0), matrix: .bt709, tag: Matrix.bt709.tag, format: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + ) + tap.offer(frame(buffer, pts: 0)) + #expect(tap.latestSample == nil) + + tap.isEnabled = true + tap.offer(frame(buffer, pts: 40_000)) + #expect(tap.latestSample?.pts == 40_000) + + tap.isEnabled = false + #expect(tap.latestSample == nil, "disabling clears what was kept") + } + + @Test("ring: atOrBefore picks the newest sample not after the time, capacity trims the oldest") + func atOrBefore() throws { + let tap = VideoSampleTap(configuration: .init(columns: 2, rows: 2, pointsPerCell: 1, capacity: 3), enabled: true) + let buffer = try flatBuffer( + (0, 0, 1), matrix: .bt709, tag: Matrix.bt709.tag, format: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + ) + for pts in stride(from: Int64(0), through: 160_000, by: 40_000) { + tap.offer(frame(buffer, pts: pts)) + } + // Kept: 80_000, 120_000, 160_000. + #expect(tap.sample(atOrBefore: 79_999) == nil) + #expect(tap.sample(atOrBefore: 80_000)?.pts == 80_000) + #expect(tap.sample(atOrBefore: 150_000)?.pts == 120_000) + #expect(tap.sample(atOrBefore: 10_000_000)?.pts == 160_000) + #expect(tap.sample(atOrBefore: MediaTime.noTimestamp) == nil) + #expect(tap.latestSample?.color(column: 1, row: 1) != nil) + #expect(tap.latestSample?.color(column: 2, row: 0) == nil) + } + + @Test("ring: empty after flush; a stale serial is refused, a newer one replaces the ring") + func flushAndSerial() throws { + let tap = VideoSampleTap(enabled: true) + let buffer = try flatBuffer( + (1, 1, 1), matrix: .bt709, tag: Matrix.bt709.tag, format: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + ) + tap.offer(frame(buffer, pts: 0)) + tap.offer(frame(buffer, pts: 40_000)) + #expect(tap.latestSample?.pts == 40_000) + + tap.flush() + #expect(tap.latestSample == nil) + + tap.offer(frame(buffer, pts: 80_000)) + tap.flush(acceptingSerial: 2) + #expect(tap.latestSample == nil) + + // A frame decoded before the seek, still in flight. + tap.offer(frame(buffer, pts: 120_000, serial: 1)) + #expect(tap.latestSample == nil, "a pre-seek frame must not land after the flush") + + tap.offer(frame(buffer, pts: 5_000_000, serial: 2)) + #expect(tap.latestSample?.pts == 5_000_000) + + // A newer serial without a flush (serial change) starts a fresh ring. + tap.offer(frame(buffer, pts: 1_000_000, serial: 3)) + #expect(tap.sample(atOrBefore: 5_000_000)?.pts == 1_000_000) + #expect(tap.sample(atOrBefore: 999_999) == nil) + } + + @Test("ring: a timeline that steps back drops what it would shadow") + func backwardsPTS() throws { + let tap = VideoSampleTap(enabled: true) + let buffer = try flatBuffer( + (0, 1, 0), matrix: .bt709, tag: Matrix.bt709.tag, format: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + ) + for pts in [Int64(0), 40_000, 80_000, 120_000] { + tap.offer(frame(buffer, pts: pts)) + } + tap.offer(frame(buffer, pts: 60_000)) + #expect(tap.latestSample?.pts == 60_000) + #expect(tap.sample(atOrBefore: 200_000)?.pts == 60_000) + #expect(tap.sample(atOrBefore: 50_000)?.pts == 40_000) + } + + @Test("ring: reconfiguring clears it and changes the grid") + func reconfigure() throws { + let tap = VideoSampleTap(enabled: true) + let buffer = try flatBuffer( + (1, 0, 0), matrix: .bt709, tag: Matrix.bt709.tag, format: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + ) + tap.offer(frame(buffer, pts: 0)) + #expect(tap.latestSample?.colors.count == 16 * 9) + tap.configuration = .init(columns: 3, rows: 2, pointsPerCell: 1, capacity: 8) + #expect(tap.latestSample == nil) + tap.offer(frame(buffer, pts: 40_000)) + #expect(tap.latestSample?.colors.count == 6) + // Out-of-range values clamp rather than trap. + let clamped = VideoSampleTap.Configuration(columns: 0, rows: 1_000, pointsPerCell: -3, capacity: 0) + #expect(clamped.columns >= 1 && clamped.rows <= 64 && clamped.pointsPerCell >= 1 && clamped.capacity >= 1) + } + + // MARK: Decoded fixture + + @Test("fixture: decoded quadrants come out red, green, blue, white with monotonic PTS", + .timeLimit(.minutes(1)), arguments: ["quadrant.mp4", "quadrant10.mp4"]) + func quadrantFixture(_ fixture: String) async throws { + let demuxer = Demuxer(url: try Fixtures.path(fixture)) + defer { demuxer.shutdown() } + var demuxEvents = demuxer.events.makeAsyncIterator() + demuxer.start() + guard case .opened(let info)? = await demuxEvents.next() else { + Issue.record("fixture \(fixture) failed to open") + return + } + let track = try #require(info.videoTracks.first) + let parameters = try #require(demuxer.codecParameters(forStream: track.index)) + + let packets = Channel(capacity: 64) + let frames = Channel(capacity: 16, measure: { $0.duration }) + demuxer.attach(channel: packets, toStream: track.index) + let decoder = VideoDecoder(parameters: parameters, input: packets, output: frames) + var decodeEvents = decoder.events.makeAsyncIterator() + decoder.start() + demuxer.resume() + + let tap = VideoSampleTap( + configuration: .init(columns: 2, rows: 2, pointsPerCell: 3, capacity: 240), + enabled: true + ) + // Offer on the drain thread, as the render queue would; keep only PTS. + let collector = ChannelDrain(frames, into: [Int64]()) { collected, frame in + tap.offer(frame) + collected.append(frame.pts) + } + + while let event = await demuxEvents.next() { + if case .endOfStream = event { + decoder.signalEndOfStream() + break + } + } + guard case .endOfStream? = await decodeEvents.next() else { + Issue.record("expected decoder .endOfStream") + decoder.shutdown() + return + } + decoder.shutdown() + let offered = await collector.value + + #expect(offered.count >= 45, "2 s @ 25 fps ≈ 50 frames, got \(offered.count)") + #expect(zip(offered, offered.dropFirst()).allSatisfy { $0 < $1 }, "decoded PTS must be monotonic") + + // Every offered frame is kept and found at its own PTS. + for pts in offered { + #expect(tap.sample(atOrBefore: pts)?.pts == pts) + } + if let first = offered.first { + #expect(tap.sample(atOrBefore: first - 1) == nil) + } + + let sample = try #require(tap.latestSample) + let expected: [(column: Int, row: Int, name: String, color: VideoSampleTap.Color)] = [ + (0, 0, "red", .init(red: 1, green: 0, blue: 0)), + (1, 0, "green", .init(red: 0, green: 1, blue: 0)), + (0, 1, "blue", .init(red: 0, green: 0, blue: 1)), + (1, 1, "white", .init(red: 1, green: 1, blue: 1)), + ] + for quadrant in expected { + let got = try #require(sample.color(column: quadrant.column, row: quadrant.row)) + #expect( + distance(got, quadrant.color) < 0.08, + "\(fixture) \(quadrant.name) quadrant: got \(got)" + ) + } + + tap.flush() + #expect(tap.latestSample == nil) + #expect(tap.sample(atOrBefore: offered.last ?? 0) == nil) + } +}