diff --git a/Iris/Scripts/build.sh b/Iris/Scripts/build.sh index 2f39eaa..cc6621f 100755 --- a/Iris/Scripts/build.sh +++ b/Iris/Scripts/build.sh @@ -138,6 +138,7 @@ compile_arch() { -framework CoreImage \ -framework CoreGraphics \ -framework IOSurface \ + -framework UIKit \ "${BUILD_DIR}/SharedFrameReader_${ARCH}.o" \ "${BUILD_DIR}/SampleBufferFactory_${ARCH}.o" \ "${BUILD_DIR}/CaptureHooks_${ARCH}.o" \ diff --git a/Iris/Sources/IrisInject/SampleBufferFactory.mm b/Iris/Sources/IrisInject/SampleBufferFactory.mm index 100fdba..32a048e 100755 --- a/Iris/Sources/IrisInject/SampleBufferFactory.mm +++ b/Iris/Sources/IrisInject/SampleBufferFactory.mm @@ -1,12 +1,96 @@ // SampleBufferFactory.mm #import +#import #import #import #import +#import #import +#import #import "SampleBufferFactory.h" #import "SharedFrameReader.hpp" +// MARK: - Device orientation +// +// A real camera is bolted into the device, so its buffers always arrive in the +// sensor's own orientation and the scene inside them turns as the device does. +// Consumers rely on that: they read the interface orientation and rotate by the +// matching quarter turns to get an upright picture. +// +// The host webcam is bolted to nothing. Delivered as-is, its frames are already +// upright, so the consumer's correction becomes the error — 90 degrees out in +// portrait, the other way upside down, 180 out in the opposite landscape. +// +// So counter-rotate by the correction the consumer is about to apply, and the +// two cancel. The buffer keeps the sensor's fixed dimensions throughout: the +// device advertises one format (FakeCaptureObjects), and a frame whose width +// and height swapped underneath it would contradict its own activeFormat. + +/// Quarter turns clockwise a consumer applies for the current interface +/// orientation. Matches AVFoundation's convention for a back-facing sensor, +/// whose native orientation is landscape-right. +/// +/// The device holding the sensor that way reports UIInterfaceOrientationLandscape +/// *Left*: UIKit's landscape constants are the mirror of the device orientations +/// that produce them. So the no-turn case below is the one whose name reads +/// wrong, and pairing these up by name is what breaks it. +static std::atomic gConsumerQuarterTurns{0}; + +static int MSCQuarterTurnsForOrientation(UIInterfaceOrientation orientation) { + switch (orientation) { + case UIInterfaceOrientationLandscapeLeft: return 0; + case UIInterfaceOrientationPortrait: return 1; + case UIInterfaceOrientationLandscapeRight: return 2; + case UIInterfaceOrientationPortraitUpsideDown: return 3; + default: return -1; // unknown + } +} + +/// Track the interface orientation on the main thread, where UIKit may be read, +/// and publish it for the delivery queue. An orientation we don't recognise +/// (face up, unknown) leaves the last good value in place rather than snapping +/// the picture to some default. +static void MSCRefreshOrientation(void) { + UIWindowScene *scene = nil; + for (UIScene *candidate in UIApplication.sharedApplication.connectedScenes) { + if ([candidate isKindOfClass:[UIWindowScene class]]) { + scene = (UIWindowScene *)candidate; + break; + } + } + if (!scene) return; + int turns = MSCQuarterTurnsForOrientation(scene.interfaceOrientation); + if (turns >= 0) gConsumerQuarterTurns.store(turns, std::memory_order_relaxed); +} + +static void MSCStartOrientationTracking(void) { + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + dispatch_async(dispatch_get_main_queue(), ^{ + [UIDevice.currentDevice beginGeneratingDeviceOrientationNotifications]; + [NSNotificationCenter.defaultCenter + addObserverForName:UIDeviceOrientationDidChangeNotification + object:nil + queue:NSOperationQueue.mainQueue + usingBlock:^(NSNotification *_Nonnull note) { + MSCRefreshOrientation(); + }]; + MSCRefreshOrientation(); + }); + }); +} + +/// The orientation to stamp on the frame so that the consumer's own rotation +/// lands upright — the inverse of what it is about to apply. +static CGImagePropertyOrientation MSCInverseOrientation(int consumerQuarterTurns) { + switch (((consumerQuarterTurns % 4) + 4) % 4) { + case 1: return kCGImagePropertyOrientationLeft; // undo 90 clockwise + case 2: return kCGImagePropertyOrientationDown; // undo 180 + case 3: return kCGImagePropertyOrientationRight; // undo 90 anticlockwise + default: return kCGImagePropertyOrientationUp; + } +} + @implementation MSCSampleBufferFactory { CMVideoFormatDescriptionRef _formatDesc; uint32_t _descWidth; @@ -14,6 +98,10 @@ @implementation MSCSampleBufferFactory { int32_t _fps; CMTime _frameDuration; CMTime _startCMTime; + CIContext *_ciContext; + CVPixelBufferPoolRef _rotationPool; + uint32_t _poolWidth; + uint32_t _poolHeight; } - (instancetype)initWithFPS:(int32_t)fps { @@ -21,6 +109,7 @@ - (instancetype)initWithFPS:(int32_t)fps { _fps = fps; _frameDuration = CMTimeMake(1, fps); _startCMTime = CMClockGetTime(CMClockGetHostTimeClock()); + MSCStartOrientationTracking(); return self; } @@ -29,6 +118,81 @@ - (void)dealloc { CFRelease(_formatDesc); _formatDesc = nullptr; } + if (_rotationPool) { + CVPixelBufferPoolRelease(_rotationPool); + _rotationPool = nullptr; + } +} + +/// Pool of sensor-sized buffers to rotate into. Recycled rather than allocated +/// per frame — at 30fps a fresh IOSurface each time is a lot of churn. +- (CVPixelBufferRef)rotationBufferOfWidth:(uint32_t)w height:(uint32_t)h { + if (!_rotationPool || _poolWidth != w || _poolHeight != h) { + if (_rotationPool) CVPixelBufferPoolRelease(_rotationPool); + _rotationPool = nullptr; + + NSDictionary *attrs = @{ + (NSString *)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA), + (NSString *)kCVPixelBufferWidthKey: @(w), + (NSString *)kCVPixelBufferHeightKey: @(h), + (NSString *)kCVPixelBufferIOSurfacePropertiesKey: @{}, + }; + if (CVPixelBufferPoolCreate(kCFAllocatorDefault, NULL, + (__bridge CFDictionaryRef)attrs, + &_rotationPool) != kCVReturnSuccess) { + _rotationPool = nullptr; + return nullptr; + } + _poolWidth = w; + _poolHeight = h; + } + + CVPixelBufferRef out = nullptr; + if (CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, _rotationPool, + &out) != kCVReturnSuccess) { + return nullptr; + } + return out; +} + +/// Rotate `src` so a consumer's orientation correction cancels out, returning a +/// buffer of the same dimensions. Rotating 90 degrees turns a landscape frame +/// portrait, so it is scaled to cover the sensor rectangle and centre-cropped — +/// which is what a real sensor shows in portrait too: the same optics, a +/// narrower slice of the world. +/// +/// Returns NULL if anything fails, leaving the caller to send the frame +/// unrotated. A picture the wrong way up beats no picture at all. +- (nullable CVPixelBufferRef)rotatedBuffer:(CVPixelBufferRef)src + turns:(int)turns + width:(uint32_t)w + height:(uint32_t)h { + if (!_ciContext) { + _ciContext = [CIContext contextWithOptions:@{ + kCIContextWorkingColorSpace: [NSNull null], + }]; + } + if (!_ciContext) return nullptr; + + CIImage *image = [CIImage imageWithCVPixelBuffer:src]; + image = [image imageByApplyingCGOrientation:MSCInverseOrientation(turns)]; + + CGRect extent = image.extent; + if (CGRectIsEmpty(extent)) return nullptr; + + CGFloat scale = MAX(w / extent.size.width, h / extent.size.height); + image = [image imageByApplyingTransform:CGAffineTransformMakeScale(scale, scale)]; + + extent = image.extent; + image = [image imageByApplyingTransform:CGAffineTransformMakeTranslation( + (w - extent.size.width) / 2.0 - extent.origin.x, + (h - extent.size.height) / 2.0 - extent.origin.y)]; + + CVPixelBufferRef dst = [self rotationBufferOfWidth:w height:h]; + if (!dst) return nullptr; + + [_ciContext render:image toCVPixelBuffer:dst]; + return dst; } - (nullable CMSampleBufferRef)sampleBufferFromReader:(SharedFrameReader *)reader { @@ -61,6 +225,20 @@ - (nullable CMSampleBufferRef)sampleBufferFromReader:(SharedFrameReader *)reader return nil; } + // The sensor's native landscape needs no turn, which is the common case and + // stays zero-copy — the IOSurface goes to the app untouched. + int turns = gConsumerQuarterTurns.load(std::memory_order_relaxed); + if (turns != 0) { + CVPixelBufferRef rotated = [self rotatedBuffer:pixBuf + turns:turns + width:snap.width + height:snap.height]; + if (rotated) { + CVPixelBufferRelease(pixBuf); + pixBuf = rotated; // owned by us; wrapPixelBuffer releases it + } + } + return [self wrapPixelBuffer:pixBuf pts:snap.ptsNs width:snap.width height:snap.height]; } diff --git a/docs/SIM_CAM_ARCHITECTURE.md b/docs/SIM_CAM_ARCHITECTURE.md index 9cc83f5..560798b 100644 --- a/docs/SIM_CAM_ARCHITECTURE.md +++ b/docs/SIM_CAM_ARCHITECTURE.md @@ -212,6 +212,50 @@ flowchart LR The host app receives the delegate callbacks exactly as it would from a hardware camera. +### Frame orientation + +A hardware camera is bolted into the device. Its buffers always arrive in the +sensor's own orientation — landscape-right for a back-facing sensor — and the +scene *inside* them turns as the device turns. Apps rely on that: they read the +interface orientation and rotate by the matching quarter turns to get an upright +picture. + +The host webcam is bolted to nothing, so its frames arrive already upright. +Delivered as-is, the app's correction becomes the error: + +| Interface orientation | App's correction | Result without counter-rotation | +| --- | --- | --- | +| `landscapeLeft` | none | correct | +| `portrait` | 90° clockwise | 90° out | +| `portraitUpsideDown` | 90° anticlockwise | 90° out the other way | +| `landscapeRight` | 180° | upside down | + +Those two landscape rows read backwards on purpose. `UIInterfaceOrientation`'s +landscape constants are the mirror of the device orientations that produce them, +so the sensor's native landscape — the row needing no correction — is the one +UIKit calls `landscapeLeft`. + +So `SampleBufferFactory` counter-rotates by whatever the app is about to apply, +and the two cancel. From outside, the injected device behaves like hardware. + +Two properties this preserves: + +- **Fixed dimensions.** The synthesised device advertises one `activeFormat`, + and a frame whose width and height swapped underneath it would contradict that + format. A rotated frame is scaled to cover the sensor rectangle and + centre-cropped — which is what a real sensor shows in portrait anyway: the + same optics over a narrower slice of the world. +- **Zero-copy in the sensor's native landscape.** No turn is needed there, so the `IOSurface` + still reaches the app untouched. Only the rotated orientations pay for a + render, and those recycle a `CVPixelBufferPool` rather than allocating per + frame. Any failure falls through to the unrotated frame — a picture the wrong + way up beats no picture. + +`UIKit` is read only on the main thread, with the resulting quarter-turn count +published to the delivery queue as an atomic. An orientation UIKit doesn't +resolve (face up, unknown) leaves the last good value in place rather than +snapping the picture to a default. + ### Metadata objects (QR codes and faces) Apps that scan codes rarely read pixels themselves — they attach an