From 820ea4617b6282903f1cb2677ed8ef1feca653a1 Mon Sep 17 00:00:00 2001 From: Roger Hughes Date: Mon, 27 Jul 2026 18:44:24 +0100 Subject: [PATCH 1/2] feat(iris): deliver QR/face metadata objects to AVCaptureMetadataOutput MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apps that scan codes rarely touch pixels themselves — they attach an AVCaptureMetadataOutput and wait for AVMetadataMachineReadableCodeObjects. Nothing in the injected session produces those, so QR scanners saw the injected feed render correctly and then never fire. This runs detection over the frame the injector already delivered and calls the app's metadata delegate directly. Hooks AVCaptureMetadataOutput's setMetadataObjectsDelegate:queue:, availableMetadataObjectTypes, and the metadataObjectTypes pair. setMetadataObjectTypes: is recorded rather than forwarded — the real setter validates against its own (empty) available list and raises NSInvalidArgumentException. Three things this ran into, all documented in SIM_CAM_ARCHITECTURE.md: - Vision does not work in the Simulator. Every VNImageRequestHandler fails with "Could not create inference context", so detection uses CIDetector, which covers QR codes and faces. availableMetadataObjectTypes advertises exactly those two rather than the full symbology set, so apps don't wait on callbacks that can never come. - AVMetadataObject's -dealloc segfaults on an instance built with class_createInstance — reproducibly, even with nothing assigned. The synthesised objects are therefore pooled: allocated once, recycled, never released, which keeps memory bounded and that dealloc unreached. - Detection is throttled to 200ms and single-flight on its own queue. Also fixes a crash independent of this feature: -[AVCaptureDeviceInput portsWithMediaType:sourceDeviceType:sourceDevicePosition:] on the synthesised input fell through to AVFoundation's implementation, which walks state class_createInstance never set up and segfaults in -[AVCaptureDeviceInput multiCamPorts]. Any app calling ports(for:) hits it. MSCFakeCaptureInput now answers typed port lookups itself. Verified end to end against a real Flutter app (multicamera plugin) in the Simulator: `sim cam start --image qr.png` and the app's barcode callback fires with the decoded payload. --- Iris/Scripts/build.sh | 1 + Iris/Shared/include/IrisConstants.h | 6 + Iris/Sources/IrisInject/CaptureHooks.mm | 257 ++++++++++++++++++ Iris/Sources/IrisInject/FakeCaptureObjects.h | 20 ++ Iris/Sources/IrisInject/FakeCaptureObjects.mm | 128 +++++++++ README.md | 10 + docs/SIM_CAM_ARCHITECTURE.md | 38 +++ 7 files changed, 460 insertions(+) diff --git a/Iris/Scripts/build.sh b/Iris/Scripts/build.sh index da5f2f3..2f39eaa 100755 --- a/Iris/Scripts/build.sh +++ b/Iris/Scripts/build.sh @@ -135,6 +135,7 @@ compile_arch() { -framework Foundation \ -framework QuartzCore \ -framework VideoToolbox \ + -framework CoreImage \ -framework CoreGraphics \ -framework IOSurface \ "${BUILD_DIR}/SharedFrameReader_${ARCH}.o" \ diff --git a/Iris/Shared/include/IrisConstants.h b/Iris/Shared/include/IrisConstants.h index 460a351..bca6b98 100755 --- a/Iris/Shared/include/IrisConstants.h +++ b/Iris/Shared/include/IrisConstants.h @@ -42,3 +42,9 @@ // If no frame is published within this many nanoseconds, the injector treats // the producer as stalled and may deliver a placeholder frame. #define IRIS_STALE_THRESHOLD_NS (500 * 1000 * 1000ull) // 500 ms + +// Recognition (barcode/face) cadence +// Vision runs over the injected frames to synthesise AVCaptureMetadataOutput +// results. Detection costs milliseconds per frame and no consumer benefits +// from scanning every one, so it is throttled to this interval (seconds). +#define IRIS_RECOGNITION_INTERVAL 0.2 diff --git a/Iris/Sources/IrisInject/CaptureHooks.mm b/Iris/Sources/IrisInject/CaptureHooks.mm index d202f29..ce2ad6b 100755 --- a/Iris/Sources/IrisInject/CaptureHooks.mm +++ b/Iris/Sources/IrisInject/CaptureHooks.mm @@ -2,6 +2,7 @@ #import #import #import +#import #import #import #import @@ -25,6 +26,17 @@ NSHashTable *previewLayers = nil; + // Metadata (barcode/face) delivery. The app never sees a real capture + // connection, so recognition runs here over the injected frames and the + // results are handed straight to its AVCaptureMetadataOutput delegate. + id metadataDelegate = nil; + dispatch_queue_t metadataDelegateQueue = nullptr; + AVCaptureMetadataOutput *metadataOutput = nil; + NSArray *requestedMetadataTypes = nil; + dispatch_queue_t recognitionQueue = nullptr; + bool recognitionInFlight = false; + CFAbsoluteTime lastRecognition = 0; + int32_t fps = 30; bool running = false; bool avSessionStarted = false; @@ -35,6 +47,7 @@ void startDelivery(void); void stopDelivery(void); void deliverFrame(void); +void runRecognition(CVPixelBufferRef pixelBuffer, CMTime time); void MSCAutoStartSessionIfNeeded(AVCaptureSession *session) { if (!session) @@ -145,6 +158,111 @@ - (void)iris_setSampleBufferDelegate: @end +// MARK: - AVCaptureMetadataOutput (IrisHook) + +namespace { +/// What we can actually synthesise in the Simulator. +/// +/// Vision is unavailable there — every request fails with "Could not create +/// inference context" (its ML models have no backing device), which rules out +/// VNDetectBarcodesRequest and the full symbology set. Core Image's older +/// CPU detectors do work, and they cover QR codes and faces. Advertise exactly +/// those: apps intersect their requested types against this list, so claiming +/// EAN/PDF417 here would just make them wait for callbacks that never come. +NSArray *MSCSupportedMetadataTypes(void) { + static NSArray *types = nil; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + types = @[ AVMetadataObjectTypeQRCode, AVMetadataObjectTypeFace ]; + }); + return types; +} + +/// CIDetector construction is expensive; -featuresInImage: is thread-safe, so +/// one detector per type is reused for the life of the process. +CIDetector *MSCDetector(NSString *type) { + static NSMutableDictionary *detectors = nil; + static os_unfair_lock detectorLock = OS_UNFAIR_LOCK_INIT; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + detectors = [NSMutableDictionary dictionary]; + }); + + os_unfair_lock_lock(&detectorLock); + CIDetector *detector = detectors[type]; + if (!detector) { + detector = [CIDetector + detectorOfType:type + context:nil + options:@{CIDetectorAccuracy : CIDetectorAccuracyHigh}]; + if (detector) + detectors[type] = detector; + } + os_unfair_lock_unlock(&detectorLock); + return detector; +} + +/// Core Image works in pixels with a bottom-left origin; AVMetadataObject uses +/// a 0-1 space with a top-left origin. +CGRect MSCNormalisedBounds(CGRect rect, CGRect extent) { + if (CGRectIsEmpty(extent)) + return CGRectZero; + return CGRectMake(rect.origin.x / extent.size.width, + 1.0 - CGRectGetMaxY(rect) / extent.size.height, + rect.size.width / extent.size.width, + rect.size.height / extent.size.height); +} + +NSDictionary *MSCNormalisedCorner(CGPoint point, CGRect extent) { + if (CGRectIsEmpty(extent)) + return @{}; + CGPoint normalised = CGPointMake(point.x / extent.size.width, + 1.0 - point.y / extent.size.height); + return (__bridge_transfer NSDictionary *) + CGPointCreateDictionaryRepresentation(normalised); +} +} // namespace + +@implementation AVCaptureMetadataOutput (IrisHook) + +- (void)iris_setMetadataObjectsDelegate: + (id)delegate + queue:(dispatch_queue_t)queue { + os_unfair_lock_lock(&gState.lock); + gState.metadataDelegate = delegate; + gState.metadataDelegateQueue = queue ?: dispatch_get_main_queue(); + gState.metadataOutput = self; + os_unfair_lock_unlock(&gState.lock); + + NSLog(@"[IrisInject] Captured metadata delegate=%@ queue=%@", delegate, queue); + [self iris_setMetadataObjectsDelegate:delegate queue:queue]; + startDelivery(); +} + +- (NSArray *)iris_availableMetadataObjectTypes { + return MSCSupportedMetadataTypes(); +} + +- (void)iris_setMetadataObjectTypes:(NSArray *)types { + os_unfair_lock_lock(&gState.lock); + gState.requestedMetadataTypes = [types copy]; + os_unfair_lock_unlock(&gState.lock); + NSLog(@"[IrisInject] Metadata types requested: %lu (%@)", + (unsigned long)types.count, [types componentsJoinedByString:@", "]); + // Deliberately NOT forwarded: the real setter validates against its own + // (empty) available list and raises NSInvalidArgumentException. The getter + // below answers from what was recorded here instead. +} + +- (NSArray *)iris_metadataObjectTypes { + os_unfair_lock_lock(&gState.lock); + NSArray *types = gState.requestedMetadataTypes; + os_unfair_lock_unlock(&gState.lock); + return types ?: @[]; +} + +@end + // Fix for AVCapturePhotoOutput crash on iOS Simulator @interface AVCaptureOutput (IrisHookFix) @end @@ -366,6 +484,126 @@ void stopDelivery(void) { NSLog(@"[IrisInject] Delivery stopped"); } +/// Runs Core Image detectors over an injected frame and hands the results to +/// the app's AVCaptureMetadataOutput delegate as synthesised metadata objects. +/// +/// Throttled and single-flight: frames arrive at up to 120fps while detection +/// costs milliseconds each, and nothing downstream benefits from scanning every +/// one. +void runRecognition(CVPixelBufferRef pixelBuffer, CMTime time) { + os_unfair_lock_lock(&gState.lock); + id delegate = gState.metadataDelegate; + dispatch_queue_t delegateQueue = gState.metadataDelegateQueue; + AVCaptureMetadataOutput *output = gState.metadataOutput; + NSArray *requested = gState.requestedMetadataTypes; + bool inFlight = gState.recognitionInFlight; + CFAbsoluteTime last = gState.lastRecognition; + dispatch_queue_t queue = gState.recognitionQueue; + os_unfair_lock_unlock(&gState.lock); + + if (!delegate || !delegateQueue || !queue || requested.count == 0) + return; + if (inFlight) + return; + + CFAbsoluteTime now = CFAbsoluteTimeGetCurrent(); + if (now - last < IRIS_RECOGNITION_INTERVAL) + return; + + bool wantsCodes = [requested containsObject:AVMetadataObjectTypeQRCode]; + bool wantsFaces = [requested containsObject:AVMetadataObjectTypeFace]; + if (!wantsCodes && !wantsFaces) + return; + + os_unfair_lock_lock(&gState.lock); + gState.recognitionInFlight = true; + gState.lastRecognition = now; + os_unfair_lock_unlock(&gState.lock); + + CVPixelBufferRetain(pixelBuffer); + dispatch_async(queue, ^{ + NSMutableArray *objects = [NSMutableArray array]; + + @autoreleasepool { + CIImage *image = [CIImage imageWithCVPixelBuffer:pixelBuffer]; + CGRect extent = image.extent; + + if (wantsCodes) { + CIDetector *detector = MSCDetector(CIDetectorTypeQRCode); + for (CIFeature *feature in [detector featuresInImage:image]) { + if (![feature isKindOfClass:[CIQRCodeFeature class]]) + continue; + CIQRCodeFeature *code = (CIQRCodeFeature *)feature; + if (code.messageString.length == 0) + continue; + + NSArray *corners = @[ + MSCNormalisedCorner(code.topLeft, extent), + MSCNormalisedCorner(code.topRight, extent), + MSCNormalisedCorner(code.bottomRight, extent), + MSCNormalisedCorner(code.bottomLeft, extent), + ]; + MSCFakeMachineReadableCodeObject *object = + [MSCFakeMachineReadableCodeObject + objectWithType:AVMetadataObjectTypeQRCode + stringValue:code.messageString + bounds:MSCNormalisedBounds(code.bounds, extent) + corners:corners + time:time]; + if (object) + [objects addObject:object]; + } + } + + if (wantsFaces) { + CIDetector *detector = MSCDetector(CIDetectorTypeFace); + NSInteger faceID = 1; + for (CIFeature *feature in [detector featuresInImage:image]) { + if (![feature isKindOfClass:[CIFaceFeature class]]) + continue; + MSCFakeFaceObject *object = [MSCFakeFaceObject + objectWithFaceID:faceID++ + bounds:MSCNormalisedBounds(feature.bounds, extent) + time:time]; + if (object) + [objects addObject:object]; + } + } + } + + CVPixelBufferRelease(pixelBuffer); + + os_unfair_lock_lock(&gState.lock); + gState.recognitionInFlight = false; + os_unfair_lock_unlock(&gState.lock); + + if (objects.count == 0) + return; + + NSLog(@"[IrisInject] Recognised %lu metadata object(s)", + (unsigned long)objects.count); + + dispatch_async(delegateQueue, ^{ + if ([delegate respondsToSelector:@selector(captureOutput: + didOutputMetadataObjects:fromConnection:)]) { + AVCaptureConnection *connection = output.connections.firstObject; + if (!connection) { + static MSCFakeConnection *fakeConnection = nil; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + fakeConnection = (MSCFakeConnection *)class_createInstance( + [MSCFakeConnection class], 0); + }); + connection = fakeConnection; + } + [delegate captureOutput:output + didOutputMetadataObjects:objects + fromConnection:connection]; + } + }); + }); +} + void deliverFrame(void) { if (!gState.reader) return; @@ -386,6 +624,10 @@ void deliverFrame(void) { CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer((__bridge CMSampleBufferRef)arcSampleBuf); if (pixelBuffer) { + runRecognition(pixelBuffer, + CMSampleBufferGetPresentationTimeStamp( + (__bridge CMSampleBufferRef)arcSampleBuf)); + CGImageRef cgImage = NULL; if (VTCreateCGImageFromCVPixelBuffer(pixelBuffer, NULL, &cgImage) == noErr && @@ -442,6 +684,8 @@ void deliverFrame(void) { void MSCInstallHooks(SharedFrameReader *reader, int32_t fps) { gState.deliveryQueue = dispatch_queue_create("com.iris.delivery", DISPATCH_QUEUE_SERIAL); + gState.recognitionQueue = + dispatch_queue_create("com.iris.recognition", DISPATCH_QUEUE_SERIAL); gState.reader = reader; gState.fps = fps; gState.factory = [[MSCSampleBufferFactory alloc] initWithFPS:fps]; @@ -466,6 +710,19 @@ void MSCInstallHooks(SharedFrameReader *reader, int32_t fps) { @selector(setSampleBufferDelegate:queue:), @selector(iris_setSampleBufferDelegate:queue:)); + swizzleInstance([AVCaptureMetadataOutput class], + @selector(setMetadataObjectsDelegate:queue:), + @selector(iris_setMetadataObjectsDelegate:queue:)); + swizzleInstance([AVCaptureMetadataOutput class], + @selector(availableMetadataObjectTypes), + @selector(iris_availableMetadataObjectTypes)); + swizzleInstance([AVCaptureMetadataOutput class], + @selector(setMetadataObjectTypes:), + @selector(iris_setMetadataObjectTypes:)); + swizzleInstance([AVCaptureMetadataOutput class], + @selector(metadataObjectTypes), + @selector(iris_metadataObjectTypes)); + Class layerCls = NSClassFromString(@"AVCaptureVideoPreviewLayer"); if (layerCls) { swizzleInstance(layerCls, @selector(setSession:), diff --git a/Iris/Sources/IrisInject/FakeCaptureObjects.h b/Iris/Sources/IrisInject/FakeCaptureObjects.h index 7d69f23..db2f366 100644 --- a/Iris/Sources/IrisInject/FakeCaptureObjects.h +++ b/Iris/Sources/IrisInject/FakeCaptureObjects.h @@ -16,3 +16,23 @@ @interface MSCFakeCaptureInput : AVCaptureDeviceInput @end + +/// Metadata objects synthesised from Vision results run over the injected +/// frames. AVFoundation's metadata classes have no public initialiser, so +/// these are built with class_createInstance and answer every accessor from +/// their own ivars — the same approach MSCFakeCaptureDevice takes. Real +/// subclasses, not duck types, because apps commonly downcast with +/// `as? AVMetadataMachineReadableCodeObject`. +@interface MSCFakeMachineReadableCodeObject : AVMetadataMachineReadableCodeObject ++ (instancetype)objectWithType:(AVMetadataObjectType)type + stringValue:(NSString *)stringValue + bounds:(CGRect)bounds + corners:(NSArray *)corners + time:(CMTime)time; +@end + +@interface MSCFakeFaceObject : AVMetadataFaceObject ++ (instancetype)objectWithFaceID:(NSInteger)faceID + bounds:(CGRect)bounds + time:(CMTime)time; +@end diff --git a/Iris/Sources/IrisInject/FakeCaptureObjects.mm b/Iris/Sources/IrisInject/FakeCaptureObjects.mm index 2e9c290..736e7a6 100644 --- a/Iris/Sources/IrisInject/FakeCaptureObjects.mm +++ b/Iris/Sources/IrisInject/FakeCaptureObjects.mm @@ -1,6 +1,7 @@ // FakeCaptureObjects.mm #import "FakeCaptureObjects.h" #import +#import @implementation MSCFakeConnection @end @@ -87,4 +88,131 @@ - (AVCaptureDevice *)device { }); return @[fakePort]; } + +// AVFoundation's own implementation walks internal state that class_createInstance +// never set up (-[AVCaptureDeviceInput multiCamPorts] dereferences it and +// segfaults), so answer typed port lookups ourselves. Only video is synthesised: +// callers asking for metadata/audio/depth ports get an empty array, which is the +// documented "no such port" answer and one every caller already handles. +- (NSArray *)portsWithMediaType:(AVMediaType)mediaType + sourceDeviceType:(AVCaptureDeviceType)sourceDeviceType + sourceDevicePosition:(AVCaptureDevicePosition)sourceDevicePosition { + if (mediaType == nil || [mediaType isEqualToString:AVMediaTypeVideo]) { + return self.ports; + } + return @[]; +} +@end + +// MARK: - Synthesised metadata objects + +/// Synthesised metadata objects are pooled: allocated once and reused, never +/// released. +/// +/// AVMetadataObject's own -dealloc walks internals that class_createInstance +/// never set up and segfaults — reproducibly, even on an instance with nothing +/// assigned — so these must never be deallocated. Recycling a fixed set keeps +/// memory bounded instead. An object stays valid for the delegate callback it +/// is delivered in, which is the same lifetime the real API promises; it is +/// reused after MSC_METADATA_POOL_SIZE further detections. +#define MSC_METADATA_POOL_SIZE 16 + +static id MSCPooledMetadataObject(Class cls) { + static NSMutableDictionary *pools = nil; + static NSMutableDictionary *cursors = nil; + static os_unfair_lock lock = OS_UNFAIR_LOCK_INIT; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + pools = [NSMutableDictionary dictionary]; + cursors = [NSMutableDictionary dictionary]; + }); + + NSString *key = NSStringFromClass(cls); + os_unfair_lock_lock(&lock); + + NSMutableArray *pool = pools[key]; + if (!pool) { + pool = [NSMutableArray arrayWithCapacity:MSC_METADATA_POOL_SIZE]; + for (NSUInteger i = 0; i < MSC_METADATA_POOL_SIZE; i++) { + // __bridge, not __bridge_transfer: the array's retain is the only + // ownership these ever get, so nothing releases them back to zero. + id instance = (__bridge id)(__bridge void *)class_createInstance(cls, 0); + if (instance) { + [pool addObject:instance]; + } + } + pools[key] = pool; + cursors[key] = @0; + } + + id object = nil; + if (pool.count > 0) { + NSUInteger cursor = cursors[key].unsignedIntegerValue % pool.count; + object = pool[cursor]; + cursors[key] = @(cursor + 1); + } + + os_unfair_lock_unlock(&lock); + return object; +} + +@implementation MSCFakeMachineReadableCodeObject { + AVMetadataObjectType _irisType; + NSString *_irisStringValue; + CGRect _irisBounds; + NSArray *_irisCorners; + CMTime _irisTime; +} + ++ (instancetype)objectWithType:(AVMetadataObjectType)type + stringValue:(NSString *)stringValue + bounds:(CGRect)bounds + corners:(NSArray *)corners + time:(CMTime)time { + MSCFakeMachineReadableCodeObject *object = + MSCPooledMetadataObject([MSCFakeMachineReadableCodeObject class]); + if (object) { + object->_irisType = [type copy]; + object->_irisStringValue = [stringValue copy]; + object->_irisBounds = bounds; + object->_irisCorners = [corners copy]; + object->_irisTime = time; + } + return object; +} + +- (AVMetadataObjectType)type { return _irisType; } +- (NSString *)stringValue { return _irisStringValue; } +- (CGRect)bounds { return _irisBounds; } +- (NSArray *)corners { return _irisCorners ?: @[]; } +- (CMTime)time { return _irisTime; } +- (CMTime)duration { return kCMTimeZero; } +@end + +@implementation MSCFakeFaceObject { + NSInteger _irisFaceID; + CGRect _irisBounds; + CMTime _irisTime; +} + ++ (instancetype)objectWithFaceID:(NSInteger)faceID + bounds:(CGRect)bounds + time:(CMTime)time { + MSCFakeFaceObject *object = + MSCPooledMetadataObject([MSCFakeFaceObject class]); + if (object) { + object->_irisFaceID = faceID; + object->_irisBounds = bounds; + object->_irisTime = time; + } + return object; +} + +- (AVMetadataObjectType)type { return AVMetadataObjectTypeFace; } +- (NSInteger)faceID { return _irisFaceID; } +- (CGRect)bounds { return _irisBounds; } +- (CMTime)time { return _irisTime; } +- (CMTime)duration { return kCMTimeZero; } +- (BOOL)hasRollAngle { return NO; } +- (BOOL)hasYawAngle { return NO; } @end diff --git a/README.md b/README.md index 81dd67e..2bbbb1f 100644 --- a/README.md +++ b/README.md @@ -218,6 +218,16 @@ Open the camera management dashboard: sim cam ``` +Apps that scan with `AVCaptureMetadataOutput` work too: the injector runs QR +and face detection over the injected frames and delivers real +`AVMetadataMachineReadableCodeObject`s to the app's metadata delegate, so +pointing the camera at a QR code (`sim cam start --image qr.png`, or just +holding one up to your webcam) triggers the app's scanner. + +```bash +sim cam start --image qr.png # the app's QR scanner fires as if it saw a code +``` + Read the [SIM-CLI Camera Architecture](docs/SIM_CAM_ARCHITECTURE.md) for technical details on shared memory and dylib injection. ## Configuration diff --git a/docs/SIM_CAM_ARCHITECTURE.md b/docs/SIM_CAM_ARCHITECTURE.md index 4100fb6..e5ff526 100644 --- a/docs/SIM_CAM_ARCHITECTURE.md +++ b/docs/SIM_CAM_ARCHITECTURE.md @@ -212,6 +212,44 @@ flowchart LR The host app receives the delegate callbacks exactly as it would from a hardware camera. +### Metadata objects (QR codes and faces) + +Apps that scan codes rarely read pixels themselves — they attach an +`AVCaptureMetadataOutput` and wait for `AVMetadataMachineReadableCodeObject`s. +Nothing in the injected session produces those, because there is no real +capture connection behind it, so the injector runs detection itself over the +same frame it just delivered and calls the app's metadata delegate directly. + +```mermaid +flowchart LR + Frame["Injected CVPixelBuffer"] --> Detector["CIDetector
QR / face"] + Detector --> Objects["MSCFakeMachineReadableCodeObject
MSCFakeFaceObject"] + Objects --> Delegate["AVCaptureMetadataOutputObjectsDelegate"] +``` + +Three constraints shape this: + +- **Vision is unavailable in the Simulator.** Every `VNImageRequestHandler` + fails with *"Could not create inference context"*, which rules out + `VNDetectBarcodesRequest` and its full symbology set. Core Image's older CPU + detectors do work, so detection uses `CIDetector` — which covers **QR codes + and faces only**. `availableMetadataObjectTypes` advertises exactly those two, + because apps intersect their requested types against that list and would + otherwise wait forever for an EAN or PDF417 callback. +- **Detection is throttled** to `IRIS_RECOGNITION_INTERVAL` (200 ms) and + single-flight, on its own serial queue. Frames arrive at up to 120fps; + scanning every one costs milliseconds each and gains nothing. +- **Metadata objects are pooled, never released.** `AVMetadataObject`'s + `-dealloc` walks internals that `class_createInstance` never set up and + segfaults — reproducibly, even on an instance with nothing assigned. A fixed + ring of instances is allocated once and recycled, so that `dealloc` never + runs and memory stays bounded. An object is valid for the callback it arrives + in, the same lifetime the real API promises. + +`AVCaptureMetadataOutput.setMetadataObjectTypes:` is intercepted and recorded +rather than forwarded: the real setter validates against its own (empty) +available list and raises `NSInvalidArgumentException`. + --- ## 4. Camera Switching Logic From 78ce039f6ecaca05d4af65f2bcebcb96d74a49ce Mon Sep 17 00:00:00 2001 From: Roger Hughes Date: Tue, 28 Jul 2026 08:51:54 +0100 Subject: [PATCH 2/2] fix(iris): close the metadata pool race and enforce single-flight MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review feedback on #19. The pooled metadata objects were handed round a ring, so a slot could in principle be rewritten by the recognition queue while a slow delegate was still reading it. Pool size bounded the window rather than closing it, so slots are now checked out and back in: acquisition marks a slot in use and MSCReleasePooledMetadataObjects returns it once the delegate callback that carried it has returned — exactly the lifetime the real API promises. The pool grows to a cap under pressure and logs rather than silently dropping when a delegate stops returning. runRecognition read recognitionInFlight in one lock region and set it in another, so two frames arriving together could both claim a pass. Every precondition and the claim now share a single region. Also from review: - availableMetadataObjectTypes explains why it doesn't chain, as the setMetadataObjectTypes: hook next to it already did. - Drop the no-op __bridge roundtrip around class_createInstance. - kCMTimeZero (deprecated since iOS 12) -> CMTimeMake(0, 1). - IrisConstants.h said Vision where the implementation uses CIDetector. - Note that a synthesised faceID is an index within one detection pass, not a tracking id stable across frames as it is on real hardware. Re-verified end to end: 138 detections over a live QR feed with zero pool exhaustion, the app's barcode callback firing throughout, and its returning -visitor lookup answering 200. Co-Authored-By: Claude Opus 5 (1M context) --- Iris/Shared/include/IrisConstants.h | 7 +- Iris/Sources/IrisInject/CaptureHooks.mm | 41 +++--- Iris/Sources/IrisInject/FakeCaptureObjects.h | 21 ++- Iris/Sources/IrisInject/FakeCaptureObjects.mm | 122 ++++++++++++++---- docs/SIM_CAM_ARCHITECTURE.md | 30 ++++- 5 files changed, 166 insertions(+), 55 deletions(-) diff --git a/Iris/Shared/include/IrisConstants.h b/Iris/Shared/include/IrisConstants.h index bca6b98..a1875a1 100755 --- a/Iris/Shared/include/IrisConstants.h +++ b/Iris/Shared/include/IrisConstants.h @@ -44,7 +44,8 @@ #define IRIS_STALE_THRESHOLD_NS (500 * 1000 * 1000ull) // 500 ms // Recognition (barcode/face) cadence -// Vision runs over the injected frames to synthesise AVCaptureMetadataOutput -// results. Detection costs milliseconds per frame and no consumer benefits -// from scanning every one, so it is throttled to this interval (seconds). +// Core Image detectors run over the injected frames to synthesise +// AVCaptureMetadataOutput results (Vision is unavailable in the Simulator). +// Detection costs milliseconds per frame and no consumer benefits from scanning +// every one, so it is throttled to this interval (seconds). #define IRIS_RECOGNITION_INTERVAL 0.2 diff --git a/Iris/Sources/IrisInject/CaptureHooks.mm b/Iris/Sources/IrisInject/CaptureHooks.mm index ce2ad6b..65f6484 100755 --- a/Iris/Sources/IrisInject/CaptureHooks.mm +++ b/Iris/Sources/IrisInject/CaptureHooks.mm @@ -239,6 +239,11 @@ - (void)iris_setMetadataObjectsDelegate: startDelivery(); } +// Also deliberately NOT chained, and for the mirror-image reason to the setter +// below: the real getter reports what the (absent) capture hardware supports, +// which in the Simulator is an empty list — an app intersecting its wanted types +// against that gets nothing and never attaches a scanner at all. What we can +// actually synthesise is the honest answer here. - (NSArray *)iris_availableMetadataObjectTypes { return MSCSupportedMetadataTypes(); } @@ -491,35 +496,32 @@ void stopDelivery(void) { /// costs milliseconds each, and nothing downstream benefits from scanning every /// one. void runRecognition(CVPixelBufferRef pixelBuffer, CMTime time) { + CFAbsoluteTime now = CFAbsoluteTimeGetCurrent(); + + // One lock region, every precondition and the claim together: reading the + // in-flight flag and setting it have to be atomic, or two frames arriving back + // to back can both observe "not in flight" and each enqueue a pass. os_unfair_lock_lock(&gState.lock); id delegate = gState.metadataDelegate; dispatch_queue_t delegateQueue = gState.metadataDelegateQueue; AVCaptureMetadataOutput *output = gState.metadataOutput; NSArray *requested = gState.requestedMetadataTypes; - bool inFlight = gState.recognitionInFlight; - CFAbsoluteTime last = gState.lastRecognition; dispatch_queue_t queue = gState.recognitionQueue; - os_unfair_lock_unlock(&gState.lock); - - if (!delegate || !delegateQueue || !queue || requested.count == 0) - return; - if (inFlight) - return; - - CFAbsoluteTime now = CFAbsoluteTimeGetCurrent(); - if (now - last < IRIS_RECOGNITION_INTERVAL) - return; bool wantsCodes = [requested containsObject:AVMetadataObjectTypeQRCode]; bool wantsFaces = [requested containsObject:AVMetadataObjectTypeFace]; - if (!wantsCodes && !wantsFaces) - return; - - os_unfair_lock_lock(&gState.lock); - gState.recognitionInFlight = true; - gState.lastRecognition = now; + bool claimed = delegate && delegateQueue && queue && + (wantsCodes || wantsFaces) && !gState.recognitionInFlight && + (now - gState.lastRecognition >= IRIS_RECOGNITION_INTERVAL); + if (claimed) { + gState.recognitionInFlight = true; + gState.lastRecognition = now; + } os_unfair_lock_unlock(&gState.lock); + if (!claimed) + return; + CVPixelBufferRetain(pixelBuffer); dispatch_async(queue, ^{ NSMutableArray *objects = [NSMutableArray array]; @@ -600,6 +602,9 @@ void runRecognition(CVPixelBufferRef pixelBuffer, CMTime time) { didOutputMetadataObjects:objects fromConnection:connection]; } + // Back into the pool only now the callback has returned — while it was + // running, these slots were exclusively the delegate's to read. + MSCReleasePooledMetadataObjects(objects); }); }); } diff --git a/Iris/Sources/IrisInject/FakeCaptureObjects.h b/Iris/Sources/IrisInject/FakeCaptureObjects.h index db2f366..c109eed 100644 --- a/Iris/Sources/IrisInject/FakeCaptureObjects.h +++ b/Iris/Sources/IrisInject/FakeCaptureObjects.h @@ -17,12 +17,16 @@ @interface MSCFakeCaptureInput : AVCaptureDeviceInput @end -/// Metadata objects synthesised from Vision results run over the injected -/// frames. AVFoundation's metadata classes have no public initialiser, so -/// these are built with class_createInstance and answer every accessor from +/// Metadata objects synthesised from Core Image detector results run over the +/// injected frames. AVFoundation's metadata classes have no public initialiser, +/// so these are built with class_createInstance and answer every accessor from /// their own ivars — the same approach MSCFakeCaptureDevice takes. Real /// subclasses, not duck types, because apps commonly downcast with /// `as? AVMetadataMachineReadableCodeObject`. +/// +/// Instances are pooled and checked out; every object obtained from the +/// factories below must be handed back with MSCReleasePooledMetadataObjects +/// once the delegate callback that carried it has returned. @interface MSCFakeMachineReadableCodeObject : AVMetadataMachineReadableCodeObject + (instancetype)objectWithType:(AVMetadataObjectType)type stringValue:(NSString *)stringValue @@ -36,3 +40,14 @@ bounds:(CGRect)bounds time:(CMTime)time; @end + +/// Returns pooled metadata objects for reuse. Call once the delegate callback +/// they were delivered in has returned; objects not handed back are leaked from +/// the pool's point of view and reduce the slots available to later detections. +#ifdef __cplusplus +extern "C" { +#endif +void MSCReleasePooledMetadataObjects(NSArray *objects); +#ifdef __cplusplus +} +#endif diff --git a/Iris/Sources/IrisInject/FakeCaptureObjects.mm b/Iris/Sources/IrisInject/FakeCaptureObjects.mm index 736e7a6..b8759e7 100644 --- a/Iris/Sources/IrisInject/FakeCaptureObjects.mm +++ b/Iris/Sources/IrisInject/FakeCaptureObjects.mm @@ -112,50 +112,115 @@ - (AVCaptureDevice *)device { /// AVMetadataObject's own -dealloc walks internals that class_createInstance /// never set up and segfaults — reproducibly, even on an instance with nothing /// assigned — so these must never be deallocated. Recycling a fixed set keeps -/// memory bounded instead. An object stays valid for the delegate callback it -/// is delivered in, which is the same lifetime the real API promises; it is -/// reused after MSC_METADATA_POOL_SIZE further detections. +/// memory bounded instead. +/// +/// Slots are checked out and back in rather than handed round a ring, because a +/// pooled object's ivars are written by the recognition queue and read by the +/// delegate on a queue we don't control. A ring makes overlap merely unlikely +/// (bounded by pool size × cadence); check-out makes it impossible — a slot is +/// never handed out again until the callback it was delivered in has returned, +/// which is exactly the lifetime the real API promises. Callers must therefore +/// pair every acquisition with MSCReleasePooledMetadataObjects. +/// +/// If every slot is checked out the pool grows, up to a hard cap; past that, +/// acquisition returns nil and the caller drops that object. A pathological +/// delegate that never returns costs detections, not memory or correctness. #define MSC_METADATA_POOL_SIZE 16 +#define MSC_METADATA_POOL_MAX 64 + +static NSMutableDictionary *sMetadataPools = nil; +static NSMutableDictionary *sMetadataInUse = nil; +static os_unfair_lock sMetadataPoolLock = OS_UNFAIR_LOCK_INIT; -static id MSCPooledMetadataObject(Class cls) { - static NSMutableDictionary *pools = nil; - static NSMutableDictionary *cursors = nil; - static os_unfair_lock lock = OS_UNFAIR_LOCK_INIT; +static void MSCMetadataPoolInit(void) { static dispatch_once_t onceToken; dispatch_once(&onceToken, ^{ - pools = [NSMutableDictionary dictionary]; - cursors = [NSMutableDictionary dictionary]; + sMetadataPools = [NSMutableDictionary dictionary]; + sMetadataInUse = [NSMutableDictionary dictionary]; }); +} + +/// Never released — see the pool note above, and the +1 from +/// class_createInstance is deliberately abandoned. +static id MSCCreateUnreleasedInstance(Class cls) { + return class_createInstance(cls, 0); +} + +static id MSCAcquirePooledMetadataObject(Class cls) { + MSCMetadataPoolInit(); NSString *key = NSStringFromClass(cls); - os_unfair_lock_lock(&lock); + os_unfair_lock_lock(&sMetadataPoolLock); - NSMutableArray *pool = pools[key]; + NSMutableArray *pool = sMetadataPools[key]; if (!pool) { pool = [NSMutableArray arrayWithCapacity:MSC_METADATA_POOL_SIZE]; for (NSUInteger i = 0; i < MSC_METADATA_POOL_SIZE; i++) { - // __bridge, not __bridge_transfer: the array's retain is the only - // ownership these ever get, so nothing releases them back to zero. - id instance = (__bridge id)(__bridge void *)class_createInstance(cls, 0); + id instance = MSCCreateUnreleasedInstance(cls); if (instance) { [pool addObject:instance]; } } - pools[key] = pool; - cursors[key] = @0; + sMetadataPools[key] = pool; + sMetadataInUse[key] = [NSMutableIndexSet indexSet]; + } + + NSMutableIndexSet *inUse = sMetadataInUse[key]; + NSUInteger slot = NSNotFound; + for (NSUInteger i = 0; i < pool.count; i++) { + if (![inUse containsIndex:i]) { + slot = i; + break; + } + } + if (slot == NSNotFound && pool.count < MSC_METADATA_POOL_MAX) { + id instance = MSCCreateUnreleasedInstance(cls); + if (instance) { + [pool addObject:instance]; + slot = pool.count - 1; + } } id object = nil; - if (pool.count > 0) { - NSUInteger cursor = cursors[key].unsignedIntegerValue % pool.count; - object = pool[cursor]; - cursors[key] = @(cursor + 1); + if (slot != NSNotFound) { + [inUse addIndex:slot]; + object = pool[slot]; } + NSUInteger checkedOut = inUse.count; + + os_unfair_lock_unlock(&sMetadataPoolLock); - os_unfair_lock_unlock(&lock); + if (!object) { + // Only reachable if delegate callbacks are not returning: every slot is + // still checked out at the cap. Say so — the alternative is detections + // that quietly stop producing results. + NSLog(@"[IrisInject] Metadata pool exhausted (%lu/%d checked out) — " + @"dropping a %@. Is the metadata delegate blocking its queue?", + (unsigned long)checkedOut, MSC_METADATA_POOL_MAX, key); + } return object; } +void MSCReleasePooledMetadataObjects(NSArray *objects) { + if (objects.count == 0) { + return; + } + + os_unfair_lock_lock(&sMetadataPoolLock); + for (id object in objects) { + NSString *key = NSStringFromClass(object_getClass(object)); + NSMutableArray *pool = sMetadataPools[key]; + if (!pool) { + continue; + } + NSUInteger slot = [pool indexOfObjectIdenticalTo:object]; + if (slot != NSNotFound) { + [sMetadataInUse[key] removeIndex:slot]; + } + } + os_unfair_lock_unlock(&sMetadataPoolLock); +} + @implementation MSCFakeMachineReadableCodeObject { AVMetadataObjectType _irisType; NSString *_irisStringValue; @@ -170,7 +235,7 @@ + (instancetype)objectWithType:(AVMetadataObjectType)type corners:(NSArray *)corners time:(CMTime)time { MSCFakeMachineReadableCodeObject *object = - MSCPooledMetadataObject([MSCFakeMachineReadableCodeObject class]); + MSCAcquirePooledMetadataObject([MSCFakeMachineReadableCodeObject class]); if (object) { object->_irisType = [type copy]; object->_irisStringValue = [stringValue copy]; @@ -186,7 +251,9 @@ - (NSString *)stringValue { return _irisStringValue; } - (CGRect)bounds { return _irisBounds; } - (NSArray *)corners { return _irisCorners ?: @[]; } - (CMTime)time { return _irisTime; } -- (CMTime)duration { return kCMTimeZero; } +// CMTimeMake, not the deprecated kCMTimeZero. Detection is instantaneous from +// the consumer's point of view: one frame in, one result out. +- (CMTime)duration { return CMTimeMake(0, 1); } @end @implementation MSCFakeFaceObject { @@ -199,7 +266,7 @@ + (instancetype)objectWithFaceID:(NSInteger)faceID bounds:(CGRect)bounds time:(CMTime)time { MSCFakeFaceObject *object = - MSCPooledMetadataObject([MSCFakeFaceObject class]); + MSCAcquirePooledMetadataObject([MSCFakeFaceObject class]); if (object) { object->_irisFaceID = faceID; object->_irisBounds = bounds; @@ -209,10 +276,15 @@ + (instancetype)objectWithFaceID:(NSInteger)faceID } - (AVMetadataObjectType)type { return AVMetadataObjectTypeFace; } +// Not a tracking id, unlike the real one: CIDetector offers no identity across +// frames, so this is only an index within a single detection pass. Apps that +// follow a face between frames by faceID won't work here. - (NSInteger)faceID { return _irisFaceID; } - (CGRect)bounds { return _irisBounds; } - (CMTime)time { return _irisTime; } -- (CMTime)duration { return kCMTimeZero; } +// CMTimeMake, not the deprecated kCMTimeZero. Detection is instantaneous from +// the consumer's point of view: one frame in, one result out. +- (CMTime)duration { return CMTimeMake(0, 1); } - (BOOL)hasRollAngle { return NO; } - (BOOL)hasYawAngle { return NO; } @end diff --git a/docs/SIM_CAM_ARCHITECTURE.md b/docs/SIM_CAM_ARCHITECTURE.md index e5ff526..9cc83f5 100644 --- a/docs/SIM_CAM_ARCHITECTURE.md +++ b/docs/SIM_CAM_ARCHITECTURE.md @@ -238,17 +238,35 @@ Three constraints shape this: otherwise wait forever for an EAN or PDF417 callback. - **Detection is throttled** to `IRIS_RECOGNITION_INTERVAL` (200 ms) and single-flight, on its own serial queue. Frames arrive at up to 120fps; - scanning every one costs milliseconds each and gains nothing. + scanning every one costs milliseconds each and gains nothing. The flag that + enforces single-flight is read *and* claimed inside one lock region, so two + frames arriving together can't both start a pass. - **Metadata objects are pooled, never released.** `AVMetadataObject`'s `-dealloc` walks internals that `class_createInstance` never set up and - segfaults — reproducibly, even on an instance with nothing assigned. A fixed - ring of instances is allocated once and recycled, so that `dealloc` never - runs and memory stays bounded. An object is valid for the callback it arrives - in, the same lifetime the real API promises. + segfaults — reproducibly, even on an instance with nothing assigned. A pool of + instances is allocated once and recycled, so that `dealloc` never runs and + memory stays bounded. + + Slots are **checked out and back in**, not handed round a ring. A pooled + object's ivars are written on the recognition queue and read by the delegate on + a queue the injector doesn't control, so a slot is only reusable once the + callback carrying it has returned — which is exactly the lifetime the real API + promises. A ring would make overlap unlikely (bounded by pool size × cadence) + rather than impossible. The pool grows on demand to a hard cap; past that, + detection drops objects instead of consuming memory, so a delegate that never + returns costs results rather than correctness. `AVCaptureMetadataOutput.setMetadataObjectTypes:` is intercepted and recorded rather than forwarded: the real setter validates against its own (empty) -available list and raises `NSInvalidArgumentException`. +available list and raises `NSInvalidArgumentException`. Its +`availableMetadataObjectTypes` getter is likewise answered locally, for the +mirror-image reason — the real one reports what the absent hardware supports, +i.e. nothing, so an app intersecting against it would never attach a scanner. + +**Known limitation:** `AVMetadataFaceObject.faceID` is a *tracking* id on real +hardware, stable for a face across frames. `CIDetector` exposes no identity +between frames, so the synthesised `faceID` is only an index within one detection +pass — apps that follow a face by id won't behave as they would on a device. ---