diff --git a/Iris/Scripts/build.sh b/Iris/Scripts/build.sh index da5f2f3..2f39eaa 100755 --- a/Iris/Scripts/build.sh +++ b/Iris/Scripts/build.sh @@ -135,6 +135,7 @@ compile_arch() { -framework Foundation \ -framework QuartzCore \ -framework VideoToolbox \ + -framework CoreImage \ -framework CoreGraphics \ -framework IOSurface \ "${BUILD_DIR}/SharedFrameReader_${ARCH}.o" \ diff --git a/Iris/Shared/include/IrisConstants.h b/Iris/Shared/include/IrisConstants.h index 460a351..a1875a1 100755 --- a/Iris/Shared/include/IrisConstants.h +++ b/Iris/Shared/include/IrisConstants.h @@ -42,3 +42,10 @@ // If no frame is published within this many nanoseconds, the injector treats // the producer as stalled and may deliver a placeholder frame. #define IRIS_STALE_THRESHOLD_NS (500 * 1000 * 1000ull) // 500 ms + +// Recognition (barcode/face) cadence +// Core Image detectors run over the injected frames to synthesise +// AVCaptureMetadataOutput results (Vision is unavailable in the Simulator). +// Detection costs milliseconds per frame and no consumer benefits from scanning +// every one, so it is throttled to this interval (seconds). +#define IRIS_RECOGNITION_INTERVAL 0.2 diff --git a/Iris/Sources/IrisInject/CaptureHooks.mm b/Iris/Sources/IrisInject/CaptureHooks.mm index d202f29..65f6484 100755 --- a/Iris/Sources/IrisInject/CaptureHooks.mm +++ b/Iris/Sources/IrisInject/CaptureHooks.mm @@ -2,6 +2,7 @@ #import #import #import +#import #import #import #import @@ -25,6 +26,17 @@ NSHashTable *previewLayers = nil; + // Metadata (barcode/face) delivery. The app never sees a real capture + // connection, so recognition runs here over the injected frames and the + // results are handed straight to its AVCaptureMetadataOutput delegate. + id metadataDelegate = nil; + dispatch_queue_t metadataDelegateQueue = nullptr; + AVCaptureMetadataOutput *metadataOutput = nil; + NSArray *requestedMetadataTypes = nil; + dispatch_queue_t recognitionQueue = nullptr; + bool recognitionInFlight = false; + CFAbsoluteTime lastRecognition = 0; + int32_t fps = 30; bool running = false; bool avSessionStarted = false; @@ -35,6 +47,7 @@ void startDelivery(void); void stopDelivery(void); void deliverFrame(void); +void runRecognition(CVPixelBufferRef pixelBuffer, CMTime time); void MSCAutoStartSessionIfNeeded(AVCaptureSession *session) { if (!session) @@ -145,6 +158,116 @@ - (void)iris_setSampleBufferDelegate: @end +// MARK: - AVCaptureMetadataOutput (IrisHook) + +namespace { +/// What we can actually synthesise in the Simulator. +/// +/// Vision is unavailable there — every request fails with "Could not create +/// inference context" (its ML models have no backing device), which rules out +/// VNDetectBarcodesRequest and the full symbology set. Core Image's older +/// CPU detectors do work, and they cover QR codes and faces. Advertise exactly +/// those: apps intersect their requested types against this list, so claiming +/// EAN/PDF417 here would just make them wait for callbacks that never come. +NSArray *MSCSupportedMetadataTypes(void) { + static NSArray *types = nil; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + types = @[ AVMetadataObjectTypeQRCode, AVMetadataObjectTypeFace ]; + }); + return types; +} + +/// CIDetector construction is expensive; -featuresInImage: is thread-safe, so +/// one detector per type is reused for the life of the process. +CIDetector *MSCDetector(NSString *type) { + static NSMutableDictionary *detectors = nil; + static os_unfair_lock detectorLock = OS_UNFAIR_LOCK_INIT; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + detectors = [NSMutableDictionary dictionary]; + }); + + os_unfair_lock_lock(&detectorLock); + CIDetector *detector = detectors[type]; + if (!detector) { + detector = [CIDetector + detectorOfType:type + context:nil + options:@{CIDetectorAccuracy : CIDetectorAccuracyHigh}]; + if (detector) + detectors[type] = detector; + } + os_unfair_lock_unlock(&detectorLock); + return detector; +} + +/// Core Image works in pixels with a bottom-left origin; AVMetadataObject uses +/// a 0-1 space with a top-left origin. +CGRect MSCNormalisedBounds(CGRect rect, CGRect extent) { + if (CGRectIsEmpty(extent)) + return CGRectZero; + return CGRectMake(rect.origin.x / extent.size.width, + 1.0 - CGRectGetMaxY(rect) / extent.size.height, + rect.size.width / extent.size.width, + rect.size.height / extent.size.height); +} + +NSDictionary *MSCNormalisedCorner(CGPoint point, CGRect extent) { + if (CGRectIsEmpty(extent)) + return @{}; + CGPoint normalised = CGPointMake(point.x / extent.size.width, + 1.0 - point.y / extent.size.height); + return (__bridge_transfer NSDictionary *) + CGPointCreateDictionaryRepresentation(normalised); +} +} // namespace + +@implementation AVCaptureMetadataOutput (IrisHook) + +- (void)iris_setMetadataObjectsDelegate: + (id)delegate + queue:(dispatch_queue_t)queue { + os_unfair_lock_lock(&gState.lock); + gState.metadataDelegate = delegate; + gState.metadataDelegateQueue = queue ?: dispatch_get_main_queue(); + gState.metadataOutput = self; + os_unfair_lock_unlock(&gState.lock); + + NSLog(@"[IrisInject] Captured metadata delegate=%@ queue=%@", delegate, queue); + [self iris_setMetadataObjectsDelegate:delegate queue:queue]; + startDelivery(); +} + +// Also deliberately NOT chained, and for the mirror-image reason to the setter +// below: the real getter reports what the (absent) capture hardware supports, +// which in the Simulator is an empty list — an app intersecting its wanted types +// against that gets nothing and never attaches a scanner at all. What we can +// actually synthesise is the honest answer here. +- (NSArray *)iris_availableMetadataObjectTypes { + return MSCSupportedMetadataTypes(); +} + +- (void)iris_setMetadataObjectTypes:(NSArray *)types { + os_unfair_lock_lock(&gState.lock); + gState.requestedMetadataTypes = [types copy]; + os_unfair_lock_unlock(&gState.lock); + NSLog(@"[IrisInject] Metadata types requested: %lu (%@)", + (unsigned long)types.count, [types componentsJoinedByString:@", "]); + // Deliberately NOT forwarded: the real setter validates against its own + // (empty) available list and raises NSInvalidArgumentException. The getter + // below answers from what was recorded here instead. +} + +- (NSArray *)iris_metadataObjectTypes { + os_unfair_lock_lock(&gState.lock); + NSArray *types = gState.requestedMetadataTypes; + os_unfair_lock_unlock(&gState.lock); + return types ?: @[]; +} + +@end + // Fix for AVCapturePhotoOutput crash on iOS Simulator @interface AVCaptureOutput (IrisHookFix) @end @@ -366,6 +489,126 @@ void stopDelivery(void) { NSLog(@"[IrisInject] Delivery stopped"); } +/// Runs Core Image detectors over an injected frame and hands the results to +/// the app's AVCaptureMetadataOutput delegate as synthesised metadata objects. +/// +/// Throttled and single-flight: frames arrive at up to 120fps while detection +/// costs milliseconds each, and nothing downstream benefits from scanning every +/// one. +void runRecognition(CVPixelBufferRef pixelBuffer, CMTime time) { + CFAbsoluteTime now = CFAbsoluteTimeGetCurrent(); + + // One lock region, every precondition and the claim together: reading the + // in-flight flag and setting it have to be atomic, or two frames arriving back + // to back can both observe "not in flight" and each enqueue a pass. + os_unfair_lock_lock(&gState.lock); + id delegate = gState.metadataDelegate; + dispatch_queue_t delegateQueue = gState.metadataDelegateQueue; + AVCaptureMetadataOutput *output = gState.metadataOutput; + NSArray *requested = gState.requestedMetadataTypes; + dispatch_queue_t queue = gState.recognitionQueue; + + bool wantsCodes = [requested containsObject:AVMetadataObjectTypeQRCode]; + bool wantsFaces = [requested containsObject:AVMetadataObjectTypeFace]; + bool claimed = delegate && delegateQueue && queue && + (wantsCodes || wantsFaces) && !gState.recognitionInFlight && + (now - gState.lastRecognition >= IRIS_RECOGNITION_INTERVAL); + if (claimed) { + gState.recognitionInFlight = true; + gState.lastRecognition = now; + } + os_unfair_lock_unlock(&gState.lock); + + if (!claimed) + return; + + CVPixelBufferRetain(pixelBuffer); + dispatch_async(queue, ^{ + NSMutableArray *objects = [NSMutableArray array]; + + @autoreleasepool { + CIImage *image = [CIImage imageWithCVPixelBuffer:pixelBuffer]; + CGRect extent = image.extent; + + if (wantsCodes) { + CIDetector *detector = MSCDetector(CIDetectorTypeQRCode); + for (CIFeature *feature in [detector featuresInImage:image]) { + if (![feature isKindOfClass:[CIQRCodeFeature class]]) + continue; + CIQRCodeFeature *code = (CIQRCodeFeature *)feature; + if (code.messageString.length == 0) + continue; + + NSArray *corners = @[ + MSCNormalisedCorner(code.topLeft, extent), + MSCNormalisedCorner(code.topRight, extent), + MSCNormalisedCorner(code.bottomRight, extent), + MSCNormalisedCorner(code.bottomLeft, extent), + ]; + MSCFakeMachineReadableCodeObject *object = + [MSCFakeMachineReadableCodeObject + objectWithType:AVMetadataObjectTypeQRCode + stringValue:code.messageString + bounds:MSCNormalisedBounds(code.bounds, extent) + corners:corners + time:time]; + if (object) + [objects addObject:object]; + } + } + + if (wantsFaces) { + CIDetector *detector = MSCDetector(CIDetectorTypeFace); + NSInteger faceID = 1; + for (CIFeature *feature in [detector featuresInImage:image]) { + if (![feature isKindOfClass:[CIFaceFeature class]]) + continue; + MSCFakeFaceObject *object = [MSCFakeFaceObject + objectWithFaceID:faceID++ + bounds:MSCNormalisedBounds(feature.bounds, extent) + time:time]; + if (object) + [objects addObject:object]; + } + } + } + + CVPixelBufferRelease(pixelBuffer); + + os_unfair_lock_lock(&gState.lock); + gState.recognitionInFlight = false; + os_unfair_lock_unlock(&gState.lock); + + if (objects.count == 0) + return; + + NSLog(@"[IrisInject] Recognised %lu metadata object(s)", + (unsigned long)objects.count); + + dispatch_async(delegateQueue, ^{ + if ([delegate respondsToSelector:@selector(captureOutput: + didOutputMetadataObjects:fromConnection:)]) { + AVCaptureConnection *connection = output.connections.firstObject; + if (!connection) { + static MSCFakeConnection *fakeConnection = nil; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + fakeConnection = (MSCFakeConnection *)class_createInstance( + [MSCFakeConnection class], 0); + }); + connection = fakeConnection; + } + [delegate captureOutput:output + didOutputMetadataObjects:objects + fromConnection:connection]; + } + // Back into the pool only now the callback has returned — while it was + // running, these slots were exclusively the delegate's to read. + MSCReleasePooledMetadataObjects(objects); + }); + }); +} + void deliverFrame(void) { if (!gState.reader) return; @@ -386,6 +629,10 @@ void deliverFrame(void) { CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer((__bridge CMSampleBufferRef)arcSampleBuf); if (pixelBuffer) { + runRecognition(pixelBuffer, + CMSampleBufferGetPresentationTimeStamp( + (__bridge CMSampleBufferRef)arcSampleBuf)); + CGImageRef cgImage = NULL; if (VTCreateCGImageFromCVPixelBuffer(pixelBuffer, NULL, &cgImage) == noErr && @@ -442,6 +689,8 @@ void deliverFrame(void) { void MSCInstallHooks(SharedFrameReader *reader, int32_t fps) { gState.deliveryQueue = dispatch_queue_create("com.iris.delivery", DISPATCH_QUEUE_SERIAL); + gState.recognitionQueue = + dispatch_queue_create("com.iris.recognition", DISPATCH_QUEUE_SERIAL); gState.reader = reader; gState.fps = fps; gState.factory = [[MSCSampleBufferFactory alloc] initWithFPS:fps]; @@ -466,6 +715,19 @@ void MSCInstallHooks(SharedFrameReader *reader, int32_t fps) { @selector(setSampleBufferDelegate:queue:), @selector(iris_setSampleBufferDelegate:queue:)); + swizzleInstance([AVCaptureMetadataOutput class], + @selector(setMetadataObjectsDelegate:queue:), + @selector(iris_setMetadataObjectsDelegate:queue:)); + swizzleInstance([AVCaptureMetadataOutput class], + @selector(availableMetadataObjectTypes), + @selector(iris_availableMetadataObjectTypes)); + swizzleInstance([AVCaptureMetadataOutput class], + @selector(setMetadataObjectTypes:), + @selector(iris_setMetadataObjectTypes:)); + swizzleInstance([AVCaptureMetadataOutput class], + @selector(metadataObjectTypes), + @selector(iris_metadataObjectTypes)); + Class layerCls = NSClassFromString(@"AVCaptureVideoPreviewLayer"); if (layerCls) { swizzleInstance(layerCls, @selector(setSession:), diff --git a/Iris/Sources/IrisInject/FakeCaptureObjects.h b/Iris/Sources/IrisInject/FakeCaptureObjects.h index 7d69f23..c109eed 100644 --- a/Iris/Sources/IrisInject/FakeCaptureObjects.h +++ b/Iris/Sources/IrisInject/FakeCaptureObjects.h @@ -16,3 +16,38 @@ @interface MSCFakeCaptureInput : AVCaptureDeviceInput @end + +/// Metadata objects synthesised from Core Image detector results run over the +/// injected frames. AVFoundation's metadata classes have no public initialiser, +/// so these are built with class_createInstance and answer every accessor from +/// their own ivars — the same approach MSCFakeCaptureDevice takes. Real +/// subclasses, not duck types, because apps commonly downcast with +/// `as? AVMetadataMachineReadableCodeObject`. +/// +/// Instances are pooled and checked out; every object obtained from the +/// factories below must be handed back with MSCReleasePooledMetadataObjects +/// once the delegate callback that carried it has returned. +@interface MSCFakeMachineReadableCodeObject : AVMetadataMachineReadableCodeObject ++ (instancetype)objectWithType:(AVMetadataObjectType)type + stringValue:(NSString *)stringValue + bounds:(CGRect)bounds + corners:(NSArray *)corners + time:(CMTime)time; +@end + +@interface MSCFakeFaceObject : AVMetadataFaceObject ++ (instancetype)objectWithFaceID:(NSInteger)faceID + bounds:(CGRect)bounds + time:(CMTime)time; +@end + +/// Returns pooled metadata objects for reuse. Call once the delegate callback +/// they were delivered in has returned; objects not handed back are leaked from +/// the pool's point of view and reduce the slots available to later detections. +#ifdef __cplusplus +extern "C" { +#endif +void MSCReleasePooledMetadataObjects(NSArray *objects); +#ifdef __cplusplus +} +#endif diff --git a/Iris/Sources/IrisInject/FakeCaptureObjects.mm b/Iris/Sources/IrisInject/FakeCaptureObjects.mm index 2e9c290..b8759e7 100644 --- a/Iris/Sources/IrisInject/FakeCaptureObjects.mm +++ b/Iris/Sources/IrisInject/FakeCaptureObjects.mm @@ -1,6 +1,7 @@ // FakeCaptureObjects.mm #import "FakeCaptureObjects.h" #import +#import @implementation MSCFakeConnection @end @@ -87,4 +88,203 @@ - (AVCaptureDevice *)device { }); return @[fakePort]; } + +// AVFoundation's own implementation walks internal state that class_createInstance +// never set up (-[AVCaptureDeviceInput multiCamPorts] dereferences it and +// segfaults), so answer typed port lookups ourselves. Only video is synthesised: +// callers asking for metadata/audio/depth ports get an empty array, which is the +// documented "no such port" answer and one every caller already handles. +- (NSArray *)portsWithMediaType:(AVMediaType)mediaType + sourceDeviceType:(AVCaptureDeviceType)sourceDeviceType + sourceDevicePosition:(AVCaptureDevicePosition)sourceDevicePosition { + if (mediaType == nil || [mediaType isEqualToString:AVMediaTypeVideo]) { + return self.ports; + } + return @[]; +} +@end + +// MARK: - Synthesised metadata objects + +/// Synthesised metadata objects are pooled: allocated once and reused, never +/// released. +/// +/// AVMetadataObject's own -dealloc walks internals that class_createInstance +/// never set up and segfaults — reproducibly, even on an instance with nothing +/// assigned — so these must never be deallocated. Recycling a fixed set keeps +/// memory bounded instead. +/// +/// Slots are checked out and back in rather than handed round a ring, because a +/// pooled object's ivars are written by the recognition queue and read by the +/// delegate on a queue we don't control. A ring makes overlap merely unlikely +/// (bounded by pool size × cadence); check-out makes it impossible — a slot is +/// never handed out again until the callback it was delivered in has returned, +/// which is exactly the lifetime the real API promises. Callers must therefore +/// pair every acquisition with MSCReleasePooledMetadataObjects. +/// +/// If every slot is checked out the pool grows, up to a hard cap; past that, +/// acquisition returns nil and the caller drops that object. A pathological +/// delegate that never returns costs detections, not memory or correctness. +#define MSC_METADATA_POOL_SIZE 16 +#define MSC_METADATA_POOL_MAX 64 + +static NSMutableDictionary *sMetadataPools = nil; +static NSMutableDictionary *sMetadataInUse = nil; +static os_unfair_lock sMetadataPoolLock = OS_UNFAIR_LOCK_INIT; + +static void MSCMetadataPoolInit(void) { + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + sMetadataPools = [NSMutableDictionary dictionary]; + sMetadataInUse = [NSMutableDictionary dictionary]; + }); +} + +/// Never released — see the pool note above, and the +1 from +/// class_createInstance is deliberately abandoned. +static id MSCCreateUnreleasedInstance(Class cls) { + return class_createInstance(cls, 0); +} + +static id MSCAcquirePooledMetadataObject(Class cls) { + MSCMetadataPoolInit(); + + NSString *key = NSStringFromClass(cls); + os_unfair_lock_lock(&sMetadataPoolLock); + + NSMutableArray *pool = sMetadataPools[key]; + if (!pool) { + pool = [NSMutableArray arrayWithCapacity:MSC_METADATA_POOL_SIZE]; + for (NSUInteger i = 0; i < MSC_METADATA_POOL_SIZE; i++) { + id instance = MSCCreateUnreleasedInstance(cls); + if (instance) { + [pool addObject:instance]; + } + } + sMetadataPools[key] = pool; + sMetadataInUse[key] = [NSMutableIndexSet indexSet]; + } + + NSMutableIndexSet *inUse = sMetadataInUse[key]; + NSUInteger slot = NSNotFound; + for (NSUInteger i = 0; i < pool.count; i++) { + if (![inUse containsIndex:i]) { + slot = i; + break; + } + } + if (slot == NSNotFound && pool.count < MSC_METADATA_POOL_MAX) { + id instance = MSCCreateUnreleasedInstance(cls); + if (instance) { + [pool addObject:instance]; + slot = pool.count - 1; + } + } + + id object = nil; + if (slot != NSNotFound) { + [inUse addIndex:slot]; + object = pool[slot]; + } + NSUInteger checkedOut = inUse.count; + + os_unfair_lock_unlock(&sMetadataPoolLock); + + if (!object) { + // Only reachable if delegate callbacks are not returning: every slot is + // still checked out at the cap. Say so — the alternative is detections + // that quietly stop producing results. + NSLog(@"[IrisInject] Metadata pool exhausted (%lu/%d checked out) — " + @"dropping a %@. Is the metadata delegate blocking its queue?", + (unsigned long)checkedOut, MSC_METADATA_POOL_MAX, key); + } + return object; +} + +void MSCReleasePooledMetadataObjects(NSArray *objects) { + if (objects.count == 0) { + return; + } + + os_unfair_lock_lock(&sMetadataPoolLock); + for (id object in objects) { + NSString *key = NSStringFromClass(object_getClass(object)); + NSMutableArray *pool = sMetadataPools[key]; + if (!pool) { + continue; + } + NSUInteger slot = [pool indexOfObjectIdenticalTo:object]; + if (slot != NSNotFound) { + [sMetadataInUse[key] removeIndex:slot]; + } + } + os_unfair_lock_unlock(&sMetadataPoolLock); +} + +@implementation MSCFakeMachineReadableCodeObject { + AVMetadataObjectType _irisType; + NSString *_irisStringValue; + CGRect _irisBounds; + NSArray *_irisCorners; + CMTime _irisTime; +} + ++ (instancetype)objectWithType:(AVMetadataObjectType)type + stringValue:(NSString *)stringValue + bounds:(CGRect)bounds + corners:(NSArray *)corners + time:(CMTime)time { + MSCFakeMachineReadableCodeObject *object = + MSCAcquirePooledMetadataObject([MSCFakeMachineReadableCodeObject class]); + if (object) { + object->_irisType = [type copy]; + object->_irisStringValue = [stringValue copy]; + object->_irisBounds = bounds; + object->_irisCorners = [corners copy]; + object->_irisTime = time; + } + return object; +} + +- (AVMetadataObjectType)type { return _irisType; } +- (NSString *)stringValue { return _irisStringValue; } +- (CGRect)bounds { return _irisBounds; } +- (NSArray *)corners { return _irisCorners ?: @[]; } +- (CMTime)time { return _irisTime; } +// CMTimeMake, not the deprecated kCMTimeZero. Detection is instantaneous from +// the consumer's point of view: one frame in, one result out. +- (CMTime)duration { return CMTimeMake(0, 1); } +@end + +@implementation MSCFakeFaceObject { + NSInteger _irisFaceID; + CGRect _irisBounds; + CMTime _irisTime; +} + ++ (instancetype)objectWithFaceID:(NSInteger)faceID + bounds:(CGRect)bounds + time:(CMTime)time { + MSCFakeFaceObject *object = + MSCAcquirePooledMetadataObject([MSCFakeFaceObject class]); + if (object) { + object->_irisFaceID = faceID; + object->_irisBounds = bounds; + object->_irisTime = time; + } + return object; +} + +- (AVMetadataObjectType)type { return AVMetadataObjectTypeFace; } +// Not a tracking id, unlike the real one: CIDetector offers no identity across +// frames, so this is only an index within a single detection pass. Apps that +// follow a face between frames by faceID won't work here. +- (NSInteger)faceID { return _irisFaceID; } +- (CGRect)bounds { return _irisBounds; } +- (CMTime)time { return _irisTime; } +// CMTimeMake, not the deprecated kCMTimeZero. Detection is instantaneous from +// the consumer's point of view: one frame in, one result out. +- (CMTime)duration { return CMTimeMake(0, 1); } +- (BOOL)hasRollAngle { return NO; } +- (BOOL)hasYawAngle { return NO; } @end diff --git a/README.md b/README.md index 81dd67e..2bbbb1f 100644 --- a/README.md +++ b/README.md @@ -218,6 +218,16 @@ Open the camera management dashboard: sim cam ``` +Apps that scan with `AVCaptureMetadataOutput` work too: the injector runs QR +and face detection over the injected frames and delivers real +`AVMetadataMachineReadableCodeObject`s to the app's metadata delegate, so +pointing the camera at a QR code (`sim cam start --image qr.png`, or just +holding one up to your webcam) triggers the app's scanner. + +```bash +sim cam start --image qr.png # the app's QR scanner fires as if it saw a code +``` + Read the [SIM-CLI Camera Architecture](docs/SIM_CAM_ARCHITECTURE.md) for technical details on shared memory and dylib injection. ## Configuration diff --git a/docs/SIM_CAM_ARCHITECTURE.md b/docs/SIM_CAM_ARCHITECTURE.md index 4100fb6..9cc83f5 100644 --- a/docs/SIM_CAM_ARCHITECTURE.md +++ b/docs/SIM_CAM_ARCHITECTURE.md @@ -212,6 +212,62 @@ flowchart LR The host app receives the delegate callbacks exactly as it would from a hardware camera. +### Metadata objects (QR codes and faces) + +Apps that scan codes rarely read pixels themselves — they attach an +`AVCaptureMetadataOutput` and wait for `AVMetadataMachineReadableCodeObject`s. +Nothing in the injected session produces those, because there is no real +capture connection behind it, so the injector runs detection itself over the +same frame it just delivered and calls the app's metadata delegate directly. + +```mermaid +flowchart LR + Frame["Injected CVPixelBuffer"] --> Detector["CIDetector
QR / face"] + Detector --> Objects["MSCFakeMachineReadableCodeObject
MSCFakeFaceObject"] + Objects --> Delegate["AVCaptureMetadataOutputObjectsDelegate"] +``` + +Three constraints shape this: + +- **Vision is unavailable in the Simulator.** Every `VNImageRequestHandler` + fails with *"Could not create inference context"*, which rules out + `VNDetectBarcodesRequest` and its full symbology set. Core Image's older CPU + detectors do work, so detection uses `CIDetector` — which covers **QR codes + and faces only**. `availableMetadataObjectTypes` advertises exactly those two, + because apps intersect their requested types against that list and would + otherwise wait forever for an EAN or PDF417 callback. +- **Detection is throttled** to `IRIS_RECOGNITION_INTERVAL` (200 ms) and + single-flight, on its own serial queue. Frames arrive at up to 120fps; + scanning every one costs milliseconds each and gains nothing. The flag that + enforces single-flight is read *and* claimed inside one lock region, so two + frames arriving together can't both start a pass. +- **Metadata objects are pooled, never released.** `AVMetadataObject`'s + `-dealloc` walks internals that `class_createInstance` never set up and + segfaults — reproducibly, even on an instance with nothing assigned. A pool of + instances is allocated once and recycled, so that `dealloc` never runs and + memory stays bounded. + + Slots are **checked out and back in**, not handed round a ring. A pooled + object's ivars are written on the recognition queue and read by the delegate on + a queue the injector doesn't control, so a slot is only reusable once the + callback carrying it has returned — which is exactly the lifetime the real API + promises. A ring would make overlap unlikely (bounded by pool size × cadence) + rather than impossible. The pool grows on demand to a hard cap; past that, + detection drops objects instead of consuming memory, so a delegate that never + returns costs results rather than correctness. + +`AVCaptureMetadataOutput.setMetadataObjectTypes:` is intercepted and recorded +rather than forwarded: the real setter validates against its own (empty) +available list and raises `NSInvalidArgumentException`. Its +`availableMetadataObjectTypes` getter is likewise answered locally, for the +mirror-image reason — the real one reports what the absent hardware supports, +i.e. nothing, so an app intersecting against it would never attach a scanner. + +**Known limitation:** `AVMetadataFaceObject.faceID` is a *tracking* id on real +hardware, stable for a face across frames. `CIDetector` exposes no identity +between frames, so the synthesised `faceID` is only an index within one detection +pass — apps that follow a face by id won't behave as they would on a device. + --- ## 4. Camera Switching Logic