From ea8d7050080c48c63ec69548f167b5663546d2e4 Mon Sep 17 00:00:00 2001 From: Brandon McAnsh Date: Fri, 21 Aug 2026 17:53:06 -0400 Subject: [PATCH 1/2] fix(scanner): read the luminance plane at its own row stride CodeExtractor built the scanner's input from CVPixelBufferGetBytesPerRow, which reports the buffer-level stride, not plane 0's. On a planar 420YpCbCr8 buffer that is roughly 1.5x the luma row -- 2884 for a 1920-wide frame -- so the pointer handed to the scanner did not describe the bytes behind it. The scanner memcpy's height * width from that pointer, so every frame whose plane is padded (CoreVideo aligns plane 0 to 64 bytes: 1440 -> 1472, 1000 -> 1024) fed it skewed rows. Read the plane-level geometry instead (width/height/bytesPerRow of plane 0) and strip row padding when the plane is padded. When rowStride == width the plane is already what the scanner wants, so it is passed through with bytesNoCopy and no per-frame allocation. Two related fixes fall out of the same path: - The zero-copy Data used to escape the CVPixelBufferLockBaseAddress scope; the base address is only guaranteed while the buffer is locked. The scan now runs inside the lock via withLuminanceSample. - KikCodes.scan(data:width:height:) dropped the result of the quality overload it delegates to -- a missing return in Code.mm -- so the three-argument entry point always returned nil. The packing rule matches Android's, which lives in the shared :libs:codes:kikcode module (LuminancePlane). Both platforms feed the same C++ scanner -- the sources under CodeScanner/src are byte-identical to Android's vendor/kik/scanner/src/main/cpp -- so the pixel-buffer glue is the only place the two can disagree. CodeScanSweepTests is the iOS half of the harness that mirrors Android's KikCodeScanTest: it renders real codes, pushes them through real CVPixelBuffers at packed and padded widths, and asserts the packing rule agrees byte-for-byte with the Kotlin one over the same geometries. --- CodeScanner/CodeScanner/Code.mm | 2 +- .../Main/Bill/Extraction/CodeExtractor.swift | 101 ++++-- FlipcashTests/CodeScanSweepTests.swift | 308 ++++++++++++++++++ 3 files changed, 385 insertions(+), 26 deletions(-) create mode 100644 FlipcashTests/CodeScanSweepTests.swift diff --git a/CodeScanner/CodeScanner/Code.mm b/CodeScanner/CodeScanner/Code.mm index 0ec6a640b..a589cdc94 100644 --- a/CodeScanner/CodeScanner/Code.mm +++ b/CodeScanner/CodeScanner/Code.mm @@ -56,7 +56,7 @@ + (nonnull NSData *)decode:(nonnull NSData *)data { } + (nullable NSData *)scan:(nonnull NSData *)data width:(NSInteger)width height:(NSInteger)height { - [self scan:data width:width height:height quality:KikCodesScanQualityHigh]; + return [self scan:data width:width height:height quality:KikCodesScanQualityHigh]; } + (nullable NSData *)scan:(nonnull NSData *)data width:(NSInteger)width height:(NSInteger)height quality:(KikCodesScanQuality)quality { diff --git a/Flipcash/Core/Screens/Main/Bill/Extraction/CodeExtractor.swift b/Flipcash/Core/Screens/Main/Bill/Extraction/CodeExtractor.swift index 28236f968..9de166cab 100644 --- a/Flipcash/Core/Screens/Main/Bill/Extraction/CodeExtractor.swift +++ b/Flipcash/Core/Screens/Main/Bill/Extraction/CodeExtractor.swift @@ -17,19 +17,15 @@ class CodeExtractor: CameraSessionExtractor { required init() {} func extract(output: AVCaptureOutput, sampleBuffer: CMSampleBuffer, connection: AVCaptureConnection) -> ScannedCode? { - let sample = extractSample(from: sampleBuffer) - - guard let sample = sample else { - return nil + // The scan runs inside withLuminanceSample so the sample's zero-copy view of the plane + // stays valid -- the base address is only guaranteed while the pixel buffer is locked. + withLuminanceSample(from: sampleBuffer) { sample in + Self.processSample( + sample: sample, + quality: .best, + container: &container + ) } - - let payload = Self.processSample( - sample: sample, - quality: .best, - container: &container - ) - - return payload } private static func processSample(sample: Sample, quality: KikCodesScanQuality) -> (Data, ScannedCode)? { @@ -61,31 +57,86 @@ class CodeExtractor: CameraSessionExtractor { return nil } - private func extractSample(from sampleBuffer: CMSampleBuffer) -> Sample? { + /// Vends the frame's luminance (Y) plane as a `Sample`, tightly packed the way `kikCodeScan` + /// expects, and calls `body` with it. + /// + /// Internal rather than private so `CodeScanSweepTests` can drive it with synthesized frames. + /// + /// The sample is only valid for the duration of `body`: when the plane is already tightly + /// packed its `data` is a no-copy view of the locked pixel buffer, which CoreVideo only + /// guarantees between lock and unlock. + func withLuminanceSample( + from sampleBuffer: CMSampleBuffer, + _ body: (Sample) -> T? + ) -> T? { guard let buffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return nil } - - CVPixelBufferLockBaseAddress(buffer, CVPixelBufferLockFlags(rawValue: 0)) + + CVPixelBufferLockBaseAddress(buffer, .readOnly) defer { - CVPixelBufferUnlockBaseAddress(buffer, CVPixelBufferLockFlags(rawValue: 0)) + CVPixelBufferUnlockBaseAddress(buffer, .readOnly) } - + guard let base = CVPixelBufferGetBaseAddressOfPlane(buffer, 0) else { return nil } - - let bytesPerRow = CVPixelBufferGetBytesPerRow(buffer) - let width = CVPixelBufferGetWidth(buffer) - let height = CVPixelBufferGetHeight(buffer) - + + // The capture format is planar 420, so these must be read per-plane. + // CVPixelBufferGetBytesPerRow reports a whole-buffer value for planar formats -- at + // 1920x1080 it returns 2884 rather than the plane's actual 1920 -- which both over-claims + // the buffer's length and hides the row padding below. + let width = CVPixelBufferGetWidthOfPlane(buffer, 0) + let height = CVPixelBufferGetHeightOfPlane(buffer, 0) + let rowStride = CVPixelBufferGetBytesPerRowOfPlane(buffer, 0) + let sample = Sample( width: width, height: height, - data: Data(bytesNoCopy: base, count: bytesPerRow * height, deallocator: .none) + data: Self.luminanceData(base: base, width: width, height: height, rowStride: rowStride) ) - - return sample + + return body(sample) + } + + /// Produces the tightly packed `width * height` buffer `kikCodeScan` reads. + /// + /// CoreVideo aligns plane rows to 64 bytes, so a capture width that is not a multiple of 64 + /// arrives padded -- 1440 wide comes back with a 1472-byte stride. Handing that straight to the + /// scanner shears the image by a growing offset per row. Widths that are already 64-aligned + /// (1920 among them, which is why `.hd1920x1080` has always worked) need no copy at all. + /// + /// This mirrors `LuminancePlane` in the shared Kotlin module, which Android applies to the same + /// decision; there is no pixel-stride term because plane 0 of a 420 buffer is always one byte + /// per pixel. + static func luminanceData( + base: UnsafeRawPointer, + width: Int, + height: Int, + rowStride: Int + ) -> Data { + let scannedByteCount = width * height + + guard rowStride != width else { + return Data( + bytesNoCopy: UnsafeMutableRawPointer(mutating: base), + count: scannedByteCount, + deallocator: .none + ) + } + + var data = Data(count: scannedByteCount) + data.withUnsafeMutableBytes { destination in + guard let destination = destination.baseAddress else { return } + for row in 0.. tightly packed + (1440, 1080), // padded to a 1472-byte stride + (1000, 750), // padded to a 1024-byte stride + ] + + static let codeScales: [CGFloat] = [0.5, 0.7, 0.9] + + /// `kikCodeEncodeRemote` takes a 20-byte payload. + static let payload = Data((0..<20).map { UInt8(($0 &* 7 &+ 11) % 251) }) + + // MARK: - Sweep - + + @Test("rendered codes decode at every resolution and code scale") + func sweepRenderedCodesAcrossResolutionsAndScales() throws { + var decoded = 0 + var attempted = 0 + + for resolution in Self.resolutions { + for scale in Self.codeScales { + attempted += 1 + + let buffer = try Self.makeFrame( + width: resolution.width, + height: resolution.height, + codeScale: scale + ) + + if let payload = Self.scan(buffer) { + #expect(payload == Self.payload) + decoded += 1 + } else { + Issue.record( + """ + no decode at \(resolution.width)x\(resolution.height) scale=\(scale) \ + stride=\(CVPixelBufferGetBytesPerRowOfPlane(buffer, 0)) + """ + ) + } + } + } + + #expect(decoded == attempted, "sweep: \(decoded)/\(attempted) decoded") + } + + /// The regression that matters: a padded capture width must decode to the same payload as a + /// packed one. Before the stride fix the padded frames sheared by 32-64 bytes per row and + /// decoded to nothing. + @Test("padded and packed capture widths decode identically") + func paddedAndPackedWidthsDecodeIdentically() throws { + let packed = try Self.makeFrame(width: 1920, height: 1080, codeScale: 0.7) + let padded = try Self.makeFrame(width: 1440, height: 1080, codeScale: 0.7) + + #expect(CVPixelBufferGetBytesPerRowOfPlane(packed, 0) == 1920, "expected a packed plane") + #expect(CVPixelBufferGetBytesPerRowOfPlane(padded, 0) > 1440, "expected a padded plane") + + #expect(Self.scan(packed) == Self.payload) + #expect(Self.scan(padded) == Self.payload) + } + + // MARK: - Packing rule - + + /// Mirrors `LuminancePlaneTest` in `:libs:codes:kikcode` commonTest. The two implementations + /// have to agree byte for byte, so they are checked against the same geometries and the same + /// `(i % 251)` fill. + @Test("packing rule matches the shared Kotlin rule") + func packingRuleMatchesSharedRule() { + let geometries: [(width: Int, height: Int, rowStride: Int)] = [ + (640, 480, 640), + (640, 480, 768), + (1280, 720, 1280), + (1280, 720, 1408), + (1920, 1080, 1920), + (1920, 1080, 2048), + ] + + for geometry in geometries { + let plane = [UInt8]((0..<(geometry.rowStride * geometry.height)).map { UInt8($0 % 251) }) + + let packed = plane.withUnsafeBytes { raw in + CodeExtractor.luminanceData( + base: raw.baseAddress!, + width: geometry.width, + height: geometry.height, + rowStride: geometry.rowStride + ) + } + + #expect( + packed.count == geometry.width * geometry.height, + "wrong length at \(geometry.width)x\(geometry.height)/\(geometry.rowStride)" + ) + + for row in 0.. Data? { + let extractor = CodeExtractor() + return extractor.withLuminanceSample(from: makeSampleBuffer(buffer)) { sample in + guard + let scanned = KikCodes.scan( + sample.data, + width: sample.width, + height: sample.height, + quality: .best + ) + else { + return nil + } + return KikCodes.decode(scanned) + } + } + + /// Renders a code into the luminance plane of a real 420 pixel buffer, letting CoreVideo pick + /// the row stride so the padded cases are the ones the camera would actually hand us. + private static func makeFrame(width: Int, height: Int, codeScale: CGFloat) throws -> CVPixelBuffer { + let image = try renderCode(side: CGFloat(min(width, height)) * codeScale) + + var buffer: CVPixelBuffer? + let status = CVPixelBufferCreate( + kCFAllocatorDefault, + width, + height, + kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, + [kCVPixelBufferIOSurfacePropertiesKey: [:] as CFDictionary] as CFDictionary, + &buffer + ) + + guard status == kCVReturnSuccess, let buffer else { + throw ScanHarnessError.pixelBufferCreationFailed(status) + } + + CVPixelBufferLockBaseAddress(buffer, []) + defer { CVPixelBufferUnlockBaseAddress(buffer, []) } + + guard let base = CVPixelBufferGetBaseAddressOfPlane(buffer, 0) else { + throw ScanHarnessError.missingPlane + } + + let rowStride = CVPixelBufferGetBytesPerRowOfPlane(buffer, 0) + + // Drawing straight into the plane at its own stride is what makes the padded cases real: + // CoreGraphics writes the padding bytes the camera would have left there. + guard let context = CGContext( + data: base, + width: width, + height: height, + bitsPerComponent: 8, + bytesPerRow: rowStride, + space: CGColorSpaceCreateDeviceGray(), + bitmapInfo: CGImageAlphaInfo.none.rawValue + ) else { + throw ScanHarnessError.contextCreationFailed + } + + context.setFillColor(gray: 1.0, alpha: 1.0) + context.fill(CGRect(x: 0, y: 0, width: width, height: height)) + + if let cgImage = image.cgImage { + let side = image.size.width + context.draw( + cgImage, + in: CGRect( + x: (CGFloat(width) - side) / 2, + y: (CGFloat(height) - side) / 2, + width: side, + height: side + ) + ) + } + + // Neutral chroma, so the frame is a plausible greyscale image end to end. + if CVPixelBufferGetPlaneCount(buffer) > 1, let chroma = CVPixelBufferGetBaseAddressOfPlane(buffer, 1) { + memset(chroma, 128, CVPixelBufferGetBytesPerRowOfPlane(buffer, 1) * CVPixelBufferGetHeightOfPlane(buffer, 1)) + } + + return buffer + } + + /// Renders `CodeView`, which draws the code together with the centre badge. The badge matters: + /// the native detector finds a code by its centre ellipse, so a code rendered with an empty + /// well is undetectable. + private static func renderCode(side: CGFloat) throws -> UIImage { + let encoded = KikCodes.encode(payload) + + let renderer = ImageRenderer( + content: CodeView(data: encoded) + .foregroundStyle(.black) + .frame(width: side, height: side) + .background(Color.white) + ) + renderer.scale = 1.0 + + guard let image = renderer.uiImage else { + throw ScanHarnessError.renderFailed + } + return image + } + + private static func makeSampleBuffer(_ pixelBuffer: CVPixelBuffer) -> CMSampleBuffer { + var formatDescription: CMFormatDescription? + CMVideoFormatDescriptionCreateForImageBuffer( + allocator: kCFAllocatorDefault, + imageBuffer: pixelBuffer, + formatDescriptionOut: &formatDescription + ) + + var timing = CMSampleTimingInfo( + duration: CMTime(value: 1, timescale: 30), + presentationTimeStamp: .zero, + decodeTimeStamp: .invalid + ) + + var sampleBuffer: CMSampleBuffer? + CMSampleBufferCreateReadyWithImageBuffer( + allocator: kCFAllocatorDefault, + imageBuffer: pixelBuffer, + formatDescription: formatDescription!, + sampleTiming: &timing, + sampleBufferOut: &sampleBuffer + ) + + return sampleBuffer! + } + + enum ScanHarnessError: Error { + case pixelBufferCreationFailed(CVReturn) + case missingPlane + case contextCreationFailed + case renderFailed + } +} From 356766b58583cdec61277345081f81f41f0cdd1b Mon Sep 17 00:00:00 2001 From: Brandon McAnsh Date: Fri, 21 Aug 2026 22:38:16 -0400 Subject: [PATCH 2/2] test(scanner): render the sweep's codes in the polarity the detector expects The harness drew black marks on white. The detector thresholds for *bright* blobs and fits an ellipse to the centre badge before it ever looks at the ring to decide whether the marks are inverted, so a dark badge is not a candidate at all and nothing was ever detected. Draw light marks and a light badge on a dark field, which is what the bill draws and what Android's harness renders. --- FlipcashTests/CodeScanSweepTests.swift | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/FlipcashTests/CodeScanSweepTests.swift b/FlipcashTests/CodeScanSweepTests.swift index c499ee8e8..fe7a49859 100644 --- a/FlipcashTests/CodeScanSweepTests.swift +++ b/FlipcashTests/CodeScanSweepTests.swift @@ -229,7 +229,8 @@ struct CodeScanSweepTests { throw ScanHarnessError.contextCreationFailed } - context.setFillColor(gray: 1.0, alpha: 1.0) + // Dark field, matching the rendered code's polarity and Android's harness. + context.setFillColor(gray: 0.0, alpha: 1.0) context.fill(CGRect(x: 0, y: 0, width: width, height: height)) if let cgImage = image.cgImage { @@ -253,17 +254,22 @@ struct CodeScanSweepTests { return buffer } - /// Renders `CodeView`, which draws the code together with the centre badge. The badge matters: - /// the native detector finds a code by its centre ellipse, so a code rendered with an empty - /// well is undetectable. + /// Renders `CodeView`, which draws the code together with the centre badge. + /// + /// Polarity is not cosmetic. The detector finds a candidate code by thresholding for *bright* + /// blobs and fitting an ellipse to the centre badge, and only then looks at the ring around it + /// to decide whether the marks are inverted. So the badge has to be the bright part: light + /// marks and a light badge on a dark field, which is also what the bill draws and what + /// Android's harness renders. Black-on-white leaves the badge as a dark hole and nothing is + /// ever detected. private static func renderCode(side: CGFloat) throws -> UIImage { let encoded = KikCodes.encode(payload) let renderer = ImageRenderer( content: CodeView(data: encoded) - .foregroundStyle(.black) + .foregroundStyle(.white) .frame(width: side, height: side) - .background(Color.white) + .background(Color.black) ) renderer.scale = 1.0