From bd131e99556758a171b26f06328069ea6c1f03df Mon Sep 17 00:00:00 2001 From: Roger Hughes Date: Tue, 28 Jul 2026 14:26:47 +0100 Subject: [PATCH 1/2] feat(iris): orient injected frames like a real sensor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A camera is bolted into the device, so its buffers always arrive in the sensor's own orientation and the scene inside them turns as the device does. Consumers depend on that: they read the interface orientation and rotate by the matching quarter turns to get an upright picture. The host webcam is bolted to nothing, so its frames arrive already upright and the consumer's correction becomes the error — 90 degrees out in portrait, the other way upside down, 180 out in the opposite landscape. Only one orientation ever looked right, and which one depended on the sensor the consumer assumed. Counter-rotate by the correction the consumer is about to apply so the two cancel, leaving the injected device behaving like hardware from the outside. This belongs here rather than in any individual app: a device published by an injector is still an AVCaptureDevice, and an app that special-cased where its pixels came from would be working around a bug rather than consuming a camera. The buffer keeps the sensor's fixed dimensions. The fake device advertises one format, and a frame whose width and height swapped underneath it would contradict its own activeFormat — so a rotated frame is scaled to cover the sensor rectangle and centre-cropped, which is what a real sensor shows in portrait anyway: same optics, narrower slice of the world. Landscape-right needs no turn, and that path is untouched: the IOSurface still reaches the app zero-copy. Rotation goes through a recycled buffer pool rather than allocating per frame, and any failure falls through to the unrotated frame — a picture the wrong way up beats no picture. UIKit is read only on the main thread, with the orientation published to the delivery queue as an atomic; an orientation UIKit doesn't resolve (face up, unknown) leaves the last good value alone instead of snapping to a default. Co-Authored-By: Claude Opus 5 (1M context) --- Iris/Scripts/build.sh | 1 + .../Sources/IrisInject/SampleBufferFactory.mm | 173 ++++++++++++++++++ docs/SIM_CAM_ARCHITECTURE.md | 39 ++++ 3 files changed, 213 insertions(+) diff --git a/Iris/Scripts/build.sh b/Iris/Scripts/build.sh index 2f39eaa..cc6621f 100755 --- a/Iris/Scripts/build.sh +++ b/Iris/Scripts/build.sh @@ -138,6 +138,7 @@ compile_arch() { -framework CoreImage \ -framework CoreGraphics \ -framework IOSurface \ + -framework UIKit \ "${BUILD_DIR}/SharedFrameReader_${ARCH}.o" \ "${BUILD_DIR}/SampleBufferFactory_${ARCH}.o" \ "${BUILD_DIR}/CaptureHooks_${ARCH}.o" \ diff --git a/Iris/Sources/IrisInject/SampleBufferFactory.mm b/Iris/Sources/IrisInject/SampleBufferFactory.mm index 100fdba..0c5d5a1 100755 --- a/Iris/Sources/IrisInject/SampleBufferFactory.mm +++ b/Iris/Sources/IrisInject/SampleBufferFactory.mm @@ -1,12 +1,91 @@ // SampleBufferFactory.mm #import +#import #import #import #import +#import #import +#import #import "SampleBufferFactory.h" #import "SharedFrameReader.hpp" +// MARK: - Device orientation +// +// A real camera is bolted into the device, so its buffers always arrive in the +// sensor's own orientation and the scene inside them turns as the device does. +// Consumers rely on that: they read the interface orientation and rotate by the +// matching quarter turns to get an upright picture. +// +// The host webcam is bolted to nothing. Delivered as-is, its frames are already +// upright, so the consumer's correction becomes the error — 90 degrees out in +// portrait, the other way upside down, 180 out in the opposite landscape. +// +// So counter-rotate by the correction the consumer is about to apply, and the +// two cancel. The buffer keeps the sensor's fixed dimensions throughout: the +// device advertises one format (FakeCaptureObjects), and a frame whose width +// and height swapped underneath it would contradict its own activeFormat. + +/// Quarter turns clockwise a consumer applies for the current interface +/// orientation. Matches AVFoundation's convention for a back-facing sensor, +/// whose native orientation is landscape-right. +static std::atomic gConsumerQuarterTurns{0}; + +static int MSCQuarterTurnsForOrientation(UIInterfaceOrientation orientation) { + switch (orientation) { + case UIInterfaceOrientationLandscapeRight: return 0; + case UIInterfaceOrientationPortrait: return 1; + case UIInterfaceOrientationLandscapeLeft: return 2; + case UIInterfaceOrientationPortraitUpsideDown: return 3; + default: return -1; // unknown + } +} + +/// Track the interface orientation on the main thread, where UIKit may be read, +/// and publish it for the delivery queue. An orientation we don't recognise +/// (face up, unknown) leaves the last good value in place rather than snapping +/// the picture to some default. +static void MSCRefreshOrientation(void) { + UIWindowScene *scene = nil; + for (UIScene *candidate in UIApplication.sharedApplication.connectedScenes) { + if ([candidate isKindOfClass:[UIWindowScene class]]) { + scene = (UIWindowScene *)candidate; + break; + } + } + if (!scene) return; + int turns = MSCQuarterTurnsForOrientation(scene.interfaceOrientation); + if (turns >= 0) gConsumerQuarterTurns.store(turns, std::memory_order_relaxed); +} + +static void MSCStartOrientationTracking(void) { + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + dispatch_async(dispatch_get_main_queue(), ^{ + [UIDevice.currentDevice beginGeneratingDeviceOrientationNotifications]; + [NSNotificationCenter.defaultCenter + addObserverForName:UIDeviceOrientationDidChangeNotification + object:nil + queue:NSOperationQueue.mainQueue + usingBlock:^(NSNotification *_Nonnull note) { + MSCRefreshOrientation(); + }]; + MSCRefreshOrientation(); + }); + }); +} + +/// The orientation to stamp on the frame so that the consumer's own rotation +/// lands upright — the inverse of what it is about to apply. +static CGImagePropertyOrientation MSCInverseOrientation(int consumerQuarterTurns) { + switch (((consumerQuarterTurns % 4) + 4) % 4) { + case 1: return kCGImagePropertyOrientationLeft; // undo 90 clockwise + case 2: return kCGImagePropertyOrientationDown; // undo 180 + case 3: return kCGImagePropertyOrientationRight; // undo 90 anticlockwise + default: return kCGImagePropertyOrientationUp; + } +} + @implementation MSCSampleBufferFactory { CMVideoFormatDescriptionRef _formatDesc; uint32_t _descWidth; @@ -14,6 +93,10 @@ @implementation MSCSampleBufferFactory { int32_t _fps; CMTime _frameDuration; CMTime _startCMTime; + CIContext *_ciContext; + CVPixelBufferPoolRef _rotationPool; + uint32_t _poolWidth; + uint32_t _poolHeight; } - (instancetype)initWithFPS:(int32_t)fps { @@ -21,6 +104,7 @@ - (instancetype)initWithFPS:(int32_t)fps { _fps = fps; _frameDuration = CMTimeMake(1, fps); _startCMTime = CMClockGetTime(CMClockGetHostTimeClock()); + MSCStartOrientationTracking(); return self; } @@ -29,6 +113,81 @@ - (void)dealloc { CFRelease(_formatDesc); _formatDesc = nullptr; } + if (_rotationPool) { + CVPixelBufferPoolRelease(_rotationPool); + _rotationPool = nullptr; + } +} + +/// Pool of sensor-sized buffers to rotate into. Recycled rather than allocated +/// per frame — at 30fps a fresh IOSurface each time is a lot of churn. +- (CVPixelBufferRef)rotationBufferOfWidth:(uint32_t)w height:(uint32_t)h { + if (!_rotationPool || _poolWidth != w || _poolHeight != h) { + if (_rotationPool) CVPixelBufferPoolRelease(_rotationPool); + _rotationPool = nullptr; + + NSDictionary *attrs = @{ + (NSString *)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA), + (NSString *)kCVPixelBufferWidthKey: @(w), + (NSString *)kCVPixelBufferHeightKey: @(h), + (NSString *)kCVPixelBufferIOSurfacePropertiesKey: @{}, + }; + if (CVPixelBufferPoolCreate(kCFAllocatorDefault, NULL, + (__bridge CFDictionaryRef)attrs, + &_rotationPool) != kCVReturnSuccess) { + _rotationPool = nullptr; + return nullptr; + } + _poolWidth = w; + _poolHeight = h; + } + + CVPixelBufferRef out = nullptr; + if (CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, _rotationPool, + &out) != kCVReturnSuccess) { + return nullptr; + } + return out; +} + +/// Rotate `src` so a consumer's orientation correction cancels out, returning a +/// buffer of the same dimensions. Rotating 90 degrees turns a landscape frame +/// portrait, so it is scaled to cover the sensor rectangle and centre-cropped — +/// which is what a real sensor shows in portrait too: the same optics, a +/// narrower slice of the world. +/// +/// Returns NULL if anything fails, leaving the caller to send the frame +/// unrotated. A picture the wrong way up beats no picture at all. +- (nullable CVPixelBufferRef)rotatedBuffer:(CVPixelBufferRef)src + turns:(int)turns + width:(uint32_t)w + height:(uint32_t)h { + if (!_ciContext) { + _ciContext = [CIContext contextWithOptions:@{ + kCIContextWorkingColorSpace: [NSNull null], + }]; + } + if (!_ciContext) return nullptr; + + CIImage *image = [CIImage imageWithCVPixelBuffer:src]; + image = [image imageByApplyingCGOrientation:MSCInverseOrientation(turns)]; + + CGRect extent = image.extent; + if (CGRectIsEmpty(extent)) return nullptr; + + CGFloat scale = MAX(w / extent.size.width, h / extent.size.height); + image = [image imageByApplyingTransform:CGAffineTransformMakeScale(scale, scale)]; + + extent = image.extent; + image = [image imageByApplyingTransform:CGAffineTransformMakeTranslation( + (w - extent.size.width) / 2.0 - extent.origin.x, + (h - extent.size.height) / 2.0 - extent.origin.y)]; + + CVPixelBufferRef dst = [self rotationBufferOfWidth:w height:h]; + if (!dst) return nullptr; + + [_ciContext render:image toCVPixelBuffer:dst]; + return dst; } - (nullable CMSampleBufferRef)sampleBufferFromReader:(SharedFrameReader *)reader { @@ -61,6 +220,20 @@ - (nullable CMSampleBufferRef)sampleBufferFromReader:(SharedFrameReader *)reader return nil; } + // Landscape-right needs no turn, which is the common case and stays + // zero-copy — the IOSurface goes to the app untouched. + int turns = gConsumerQuarterTurns.load(std::memory_order_relaxed); + if (turns != 0) { + CVPixelBufferRef rotated = [self rotatedBuffer:pixBuf + turns:turns + width:snap.width + height:snap.height]; + if (rotated) { + CVPixelBufferRelease(pixBuf); + pixBuf = rotated; // owned by us; wrapPixelBuffer releases it + } + } + return [self wrapPixelBuffer:pixBuf pts:snap.ptsNs width:snap.width height:snap.height]; } diff --git a/docs/SIM_CAM_ARCHITECTURE.md b/docs/SIM_CAM_ARCHITECTURE.md index 9cc83f5..a6cdf9f 100644 --- a/docs/SIM_CAM_ARCHITECTURE.md +++ b/docs/SIM_CAM_ARCHITECTURE.md @@ -212,6 +212,45 @@ flowchart LR The host app receives the delegate callbacks exactly as it would from a hardware camera. +### Frame orientation + +A hardware camera is bolted into the device. Its buffers always arrive in the +sensor's own orientation — landscape-right for a back-facing sensor — and the +scene *inside* them turns as the device turns. Apps rely on that: they read the +interface orientation and rotate by the matching quarter turns to get an upright +picture. + +The host webcam is bolted to nothing, so its frames arrive already upright. +Delivered as-is, the app's correction becomes the error: + +| Interface orientation | App's correction | Result without counter-rotation | +| --- | --- | --- | +| `landscapeRight` | none | correct | +| `portrait` | 90° clockwise | 90° out | +| `portraitUpsideDown` | 90° anticlockwise | 90° out the other way | +| `landscapeLeft` | 180° | upside down | + +So `SampleBufferFactory` counter-rotates by whatever the app is about to apply, +and the two cancel. From outside, the injected device behaves like hardware. + +Two properties this preserves: + +- **Fixed dimensions.** The synthesised device advertises one `activeFormat`, + and a frame whose width and height swapped underneath it would contradict that + format. A rotated frame is scaled to cover the sensor rectangle and + centre-cropped — which is what a real sensor shows in portrait anyway: the + same optics over a narrower slice of the world. +- **Zero-copy in landscape-right.** No turn is needed there, so the `IOSurface` + still reaches the app untouched. Only the rotated orientations pay for a + render, and those recycle a `CVPixelBufferPool` rather than allocating per + frame. Any failure falls through to the unrotated frame — a picture the wrong + way up beats no picture. + +`UIKit` is read only on the main thread, with the resulting quarter-turn count +published to the delivery queue as an atomic. An orientation UIKit doesn't +resolve (face up, unknown) leaves the last good value in place rather than +snapping the picture to a default. + ### Metadata objects (QR codes and faces) Apps that scan codes rarely read pixels themselves — they attach an From 2ff4f31a51e6ccf828339833e21b6d6bfdc0c206 Mon Sep 17 00:00:00 2001 From: Roger Hughes Date: Wed, 29 Jul 2026 11:51:03 +0100 Subject: [PATCH 2/2] fix(iris): map the landscape interface orientations the right way round MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit UIKit's landscape constants are the mirror of the device orientations that produce them, so pairing them up by name gets the two landscape cases backwards. The sensor's native landscape — the case needing no turn — is UIInterfaceOrientationLandscapeLeft, not LandscapeRight. The effect was that landscapeLeft kept its 180-degree error and landscapeRight, correct before this branch, regressed to 180 out: it was computing 2 turns rather than taking the zero-turn path at all. Verified on an iPad (A16) against the multicamera plugin, filming all four orientations under this build and under 961abaf, with a test card as the frame source. All four are now upright; portrait and portraitUpsideDown are unchanged. Co-Authored-By: Claude Opus 5 (1M context) --- Iris/Sources/IrisInject/SampleBufferFactory.mm | 13 +++++++++---- docs/SIM_CAM_ARCHITECTURE.md | 11 ++++++++--- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/Iris/Sources/IrisInject/SampleBufferFactory.mm b/Iris/Sources/IrisInject/SampleBufferFactory.mm index 0c5d5a1..32a048e 100755 --- a/Iris/Sources/IrisInject/SampleBufferFactory.mm +++ b/Iris/Sources/IrisInject/SampleBufferFactory.mm @@ -29,13 +29,18 @@ /// Quarter turns clockwise a consumer applies for the current interface /// orientation. Matches AVFoundation's convention for a back-facing sensor, /// whose native orientation is landscape-right. +/// +/// The device holding the sensor that way reports UIInterfaceOrientationLandscape +/// *Left*: UIKit's landscape constants are the mirror of the device orientations +/// that produce them. So the no-turn case below is the one whose name reads +/// wrong, and pairing these up by name is what breaks it. static std::atomic gConsumerQuarterTurns{0}; static int MSCQuarterTurnsForOrientation(UIInterfaceOrientation orientation) { switch (orientation) { - case UIInterfaceOrientationLandscapeRight: return 0; + case UIInterfaceOrientationLandscapeLeft: return 0; case UIInterfaceOrientationPortrait: return 1; - case UIInterfaceOrientationLandscapeLeft: return 2; + case UIInterfaceOrientationLandscapeRight: return 2; case UIInterfaceOrientationPortraitUpsideDown: return 3; default: return -1; // unknown } @@ -220,8 +225,8 @@ - (nullable CMSampleBufferRef)sampleBufferFromReader:(SharedFrameReader *)reader return nil; } - // Landscape-right needs no turn, which is the common case and stays - // zero-copy — the IOSurface goes to the app untouched. + // The sensor's native landscape needs no turn, which is the common case and + // stays zero-copy — the IOSurface goes to the app untouched. int turns = gConsumerQuarterTurns.load(std::memory_order_relaxed); if (turns != 0) { CVPixelBufferRef rotated = [self rotatedBuffer:pixBuf diff --git a/docs/SIM_CAM_ARCHITECTURE.md b/docs/SIM_CAM_ARCHITECTURE.md index a6cdf9f..560798b 100644 --- a/docs/SIM_CAM_ARCHITECTURE.md +++ b/docs/SIM_CAM_ARCHITECTURE.md @@ -225,10 +225,15 @@ Delivered as-is, the app's correction becomes the error: | Interface orientation | App's correction | Result without counter-rotation | | --- | --- | --- | -| `landscapeRight` | none | correct | +| `landscapeLeft` | none | correct | | `portrait` | 90° clockwise | 90° out | | `portraitUpsideDown` | 90° anticlockwise | 90° out the other way | -| `landscapeLeft` | 180° | upside down | +| `landscapeRight` | 180° | upside down | + +Those two landscape rows read backwards on purpose. `UIInterfaceOrientation`'s +landscape constants are the mirror of the device orientations that produce them, +so the sensor's native landscape — the row needing no correction — is the one +UIKit calls `landscapeLeft`. So `SampleBufferFactory` counter-rotates by whatever the app is about to apply, and the two cancel. From outside, the injected device behaves like hardware. @@ -240,7 +245,7 @@ Two properties this preserves: format. A rotated frame is scaled to cover the sensor rectangle and centre-cropped — which is what a real sensor shows in portrait anyway: the same optics over a narrower slice of the world. -- **Zero-copy in landscape-right.** No turn is needed there, so the `IOSurface` +- **Zero-copy in the sensor's native landscape.** No turn is needed there, so the `IOSurface` still reaches the app untouched. Only the rotated orientations pay for a render, and those recycle a `CVPixelBufferPool` rather than allocating per frame. Any failure falls through to the unrotated frame — a picture the wrong