diff --git a/android/app/build.gradle b/android/app/build.gradle index 099b5b15a..cf733f781 100644 --- a/android/app/build.gradle +++ b/android/app/build.gradle @@ -191,6 +191,10 @@ dependencies { implementation jscFlavor } + // EXIF orientation: the detector, the crop, and the on-screen photo + // must all agree on one upright pixel grid. + implementation("androidx.exifinterface:exifinterface:1.3.7") + testImplementation("junit:junit:4.13.2") testImplementation("org.robolectric:robolectric:4.13") testImplementation("org.mockito:mockito-core:5.11.0") diff --git a/android/app/src/main/java/org/ganesha/elebook/imagetensor/ImageTensorModule.kt b/android/app/src/main/java/org/ganesha/elebook/imagetensor/ImageTensorModule.kt index 206a95b5a..e143ce65b 100644 --- a/android/app/src/main/java/org/ganesha/elebook/imagetensor/ImageTensorModule.kt +++ b/android/app/src/main/java/org/ganesha/elebook/imagetensor/ImageTensorModule.kt @@ -2,7 +2,9 @@ package org.ganesha.elebook.imagetensor import android.graphics.Bitmap import android.graphics.BitmapFactory +import android.graphics.Matrix import android.net.Uri +import androidx.exifinterface.media.ExifInterface import com.facebook.react.bridge.Promise import com.facebook.react.bridge.ReactApplicationContext import com.facebook.react.bridge.ReactContextBaseJavaModule @@ -104,6 +106,47 @@ class ImageTensorModule(reactContext: ReactApplicationContext) : return output } + /** + * Re-orient a decoded bitmap into the upright grid that EXIF-aware + * viewers display. + * + * `BitmapFactory` returns the stored sensor pixels and ignores the EXIF + * orientation tag, while React Native's (Fresco, autoRotate) and + * every photo viewer apply it. Without this step the detector normalizes + * boxes against one grid and the UI draws them on another, so boxes are + * transposed on screen and crops -- the pixels MiewID actually embeds -- + * are cut from the wrong region. Applying the tag once, here, keeps the + * detector, the overlay, and the crop on a single coordinate frame. + * + * Returns the input untouched when no transform is needed; otherwise a + * new bitmap. Callers own recycling the input. + * + * Exposed for unit testing. + */ + fun applyExifOrientation(bitmap: Bitmap, orientation: Int): Bitmap { + val matrix = Matrix() + when (orientation) { + ExifInterface.ORIENTATION_ROTATE_90 -> matrix.postRotate(90f) + ExifInterface.ORIENTATION_ROTATE_180 -> matrix.postRotate(180f) + ExifInterface.ORIENTATION_ROTATE_270 -> matrix.postRotate(270f) + ExifInterface.ORIENTATION_FLIP_HORIZONTAL -> matrix.postScale(-1f, 1f) + ExifInterface.ORIENTATION_FLIP_VERTICAL -> matrix.postScale(1f, -1f) + ExifInterface.ORIENTATION_TRANSPOSE -> { + matrix.postRotate(90f) + matrix.postScale(-1f, 1f) + } + ExifInterface.ORIENTATION_TRANSVERSE -> { + matrix.postRotate(270f) + matrix.postScale(-1f, 1f) + } + // ORIENTATION_NORMAL, ORIENTATION_UNDEFINED, anything unknown. + else -> return bitmap + } + return Bitmap.createBitmap( + bitmap, 0, 0, bitmap.width, bitmap.height, matrix, true, + ) + } + /** * Crop a bitmap to pixel coordinates and save as JPEG. * Exposed for unit testing. @@ -208,7 +251,22 @@ class ImageTensorModule(reactContext: ReactApplicationContext) : }.start() } + /** + * Decode an image and return it on the upright display grid. + * + * Every consumer in this module -- the detector tensor and the saved crop -- + * goes through here, so they cannot disagree about orientation. + */ private fun loadBitmap(uri: String): Bitmap? { + val decoded = decodeBitmap(uri) ?: return null + val upright = applyExifOrientation(decoded, readExifOrientation(uri)) + if (upright !== decoded) { + decoded.recycle() + } + return upright + } + + private fun decodeBitmap(uri: String): Bitmap? { return try { val parsed = Uri.parse(uri) when (parsed.scheme) { @@ -228,4 +286,29 @@ class ImageTensorModule(reactContext: ReactApplicationContext) : null } } + + /** + * Read the EXIF orientation tag, defaulting to NORMAL when the image has no + * tag or cannot be parsed. An unreadable tag must not fail the capture. + */ + private fun readExifOrientation(uri: String): Int { + return try { + val parsed = Uri.parse(uri) + val exif = when (parsed.scheme) { + "content" -> { + reactApplicationContext.contentResolver.openInputStream(parsed)?.use { stream -> + ExifInterface(stream) + } + } + "file" -> parsed.path?.let { ExifInterface(it) } + else -> if (File(uri).exists()) ExifInterface(uri) else null + } + exif?.getAttributeInt( + ExifInterface.TAG_ORIENTATION, + ExifInterface.ORIENTATION_NORMAL, + ) ?: ExifInterface.ORIENTATION_NORMAL + } catch (_: Exception) { + ExifInterface.ORIENTATION_NORMAL + } + } } diff --git a/android/app/src/test/java/org/ganesha/elebook/imagetensor/ImageTensorOrientationTest.kt b/android/app/src/test/java/org/ganesha/elebook/imagetensor/ImageTensorOrientationTest.kt new file mode 100644 index 000000000..49550f890 --- /dev/null +++ b/android/app/src/test/java/org/ganesha/elebook/imagetensor/ImageTensorOrientationTest.kt @@ -0,0 +1,385 @@ +package org.ganesha.elebook.imagetensor + +import android.app.Application +import android.graphics.Bitmap +import android.graphics.Color +import android.graphics.BitmapFactory +import androidx.exifinterface.media.ExifInterface +import com.facebook.react.bridge.Promise +import com.facebook.react.bridge.ReactApplicationContext +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.junit.runner.RunWith +import org.mockito.kotlin.any +import org.mockito.kotlin.doAnswer +import org.mockito.kotlin.mock +import org.mockito.kotlin.whenever +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config +import org.robolectric.annotation.GraphicsMode +import java.io.File +import java.io.FileOutputStream +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit + +/** + * EXIF orientation parity between the detector grid and the display grid. + * + * Runs under [GraphicsMode.Mode.NATIVE] because the legacy Robolectric bitmap + * shadow does not transform pixels for matrix-backed `Bitmap.createBitmap`, + * so rotations would silently read back as blank. + */ +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [33], application = Application::class) +@GraphicsMode(GraphicsMode.Mode.NATIVE) +class ImageTensorOrientationTest { + + private companion object { + /** MiewID's model input size -- see onnxInferenceService. */ + const val MIEWID_INPUT = 440 + } + + @get:Rule + val tmp = TemporaryFolder() + + // ------------------------------------------------------------------------- + // applyExifOrientation — EXIF/display coordinate-frame parity + // + // The detector normalizes boxes against whatever pixel grid loadBitmap + // returns; React Native's and every EXIF-aware viewer show the + // rotated grid. When those disagree, boxes are transposed on screen and + // crops are cut from the wrong region. These tests pin both to one grid. + // ------------------------------------------------------------------------- + + /** Builds a bitmap from rows of colors so rotations are unambiguous. */ + private fun bitmapOf(rows: Array): Bitmap { + val height = rows.size + val width = rows[0].size + val bmp = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888) + for (y in 0 until height) { + for (x in 0 until width) { + bmp.setPixel(x, y, rows[y][x]) + } + } + return bmp + } + + private fun assertSamePixels(expected: Bitmap, actual: Bitmap) { + assertEquals("width", expected.width, actual.width) + assertEquals("height", expected.height, actual.height) + for (y in 0 until expected.height) { + for (x in 0 until expected.width) { + assertEquals( + "pixel ($x,$y)", + expected.getPixel(x, y), + actual.getPixel(x, y), + ) + } + } + } + + /** The photo as a viewer displays it: 4 wide, 2 tall, every cell distinct. */ + private fun uprightFixture(): Bitmap = bitmapOf( + arrayOf( + intArrayOf(Color.RED, Color.GREEN, Color.BLUE, Color.YELLOW), + intArrayOf(Color.CYAN, Color.MAGENTA, Color.WHITE, Color.BLACK), + ), + ) + + @Test + fun `applyExifOrientation ROTATE_90 restores the upright grid`() { + // What a sensor writes when the photo must be turned 90 degrees clockwise. + val stored = bitmapOf( + arrayOf( + intArrayOf(Color.YELLOW, Color.BLACK), + intArrayOf(Color.BLUE, Color.WHITE), + intArrayOf(Color.GREEN, Color.MAGENTA), + intArrayOf(Color.RED, Color.CYAN), + ), + ) + val upright = uprightFixture() + + val restored = ImageTensorModule.applyExifOrientation( + stored, + ExifInterface.ORIENTATION_ROTATE_90, + ) + + assertSamePixels(upright, restored) + } + + @Test + fun `applyExifOrientation ROTATE_180 restores the upright grid`() { + val stored = bitmapOf( + arrayOf( + intArrayOf(Color.BLACK, Color.WHITE, Color.MAGENTA, Color.CYAN), + intArrayOf(Color.YELLOW, Color.BLUE, Color.GREEN, Color.RED), + ), + ) + val upright = uprightFixture() + + val restored = ImageTensorModule.applyExifOrientation( + stored, + ExifInterface.ORIENTATION_ROTATE_180, + ) + + assertSamePixels(upright, restored) + } + + @Test + fun `applyExifOrientation ROTATE_270 restores the upright grid`() { + val stored = bitmapOf( + arrayOf( + intArrayOf(Color.CYAN, Color.RED), + intArrayOf(Color.MAGENTA, Color.GREEN), + intArrayOf(Color.WHITE, Color.BLUE), + intArrayOf(Color.BLACK, Color.YELLOW), + ), + ) + val upright = uprightFixture() + + val restored = ImageTensorModule.applyExifOrientation( + stored, + ExifInterface.ORIENTATION_ROTATE_270, + ) + + assertSamePixels(upright, restored) + } + + @Test + fun `applyExifOrientation NORMAL leaves an already-upright grid untouched`() { + val upright = uprightFixture() + + val result = ImageTensorModule.applyExifOrientation( + uprightFixture(), + ExifInterface.ORIENTATION_NORMAL, + ) + + assertSamePixels(upright, result) + } + + @Test + fun `applyExifOrientation UNDEFINED leaves an already-upright grid untouched`() { + val upright = uprightFixture() + + val result = ImageTensorModule.applyExifOrientation( + uprightFixture(), + ExifInterface.ORIENTATION_UNDEFINED, + ) + + assertSamePixels(upright, result) + } + + @Test + fun `applyExifOrientation FLIP_HORIZONTAL mirrors the stored grid`() { + val stored = bitmapOf( + arrayOf( + intArrayOf(Color.YELLOW, Color.BLUE, Color.GREEN, Color.RED), + intArrayOf(Color.BLACK, Color.WHITE, Color.MAGENTA, Color.CYAN), + ), + ) + val upright = uprightFixture() + + val restored = ImageTensorModule.applyExifOrientation( + stored, + ExifInterface.ORIENTATION_FLIP_HORIZONTAL, + ) + + assertSamePixels(upright, restored) + } + + @Test + fun `applyExifOrientation FLIP_VERTICAL mirrors the stored grid`() { + val stored = bitmapOf( + arrayOf( + intArrayOf(Color.CYAN, Color.MAGENTA, Color.WHITE, Color.BLACK), + intArrayOf(Color.RED, Color.GREEN, Color.BLUE, Color.YELLOW), + ), + ) + + val restored = ImageTensorModule.applyExifOrientation( + stored, + ExifInterface.ORIENTATION_FLIP_VERTICAL, + ) + + assertSamePixels(uprightFixture(), restored) + } + + @Test + fun `applyExifOrientation TRANSPOSE restores the upright grid`() { + val stored = bitmapOf( + arrayOf( + intArrayOf(Color.RED, Color.CYAN), + intArrayOf(Color.GREEN, Color.MAGENTA), + intArrayOf(Color.BLUE, Color.WHITE), + intArrayOf(Color.YELLOW, Color.BLACK), + ), + ) + + val restored = ImageTensorModule.applyExifOrientation( + stored, + ExifInterface.ORIENTATION_TRANSPOSE, + ) + + assertSamePixels(uprightFixture(), restored) + } + + @Test + fun `applyExifOrientation TRANSVERSE restores the upright grid`() { + val stored = bitmapOf( + arrayOf( + intArrayOf(Color.BLACK, Color.YELLOW), + intArrayOf(Color.WHITE, Color.BLUE), + intArrayOf(Color.MAGENTA, Color.GREEN), + intArrayOf(Color.CYAN, Color.RED), + ), + ) + + val restored = ImageTensorModule.applyExifOrientation( + stored, + ExifInterface.ORIENTATION_TRANSVERSE, + ) + + assertSamePixels(uprightFixture(), restored) + } + + // ------------------------------------------------------------------------- + // MiewID matchability guard + // + // MiewID embeds the cropped pixels. If the coordinate frame regresses, the + // model sees a different picture for the same animal and match scores drop + // silently — no crash, no failing screen. These assert byte-for-byte that + // storage orientation cannot change what the model receives. + // ------------------------------------------------------------------------- + + @Test + fun `MiewID tensor is identical whether the photo is stored upright or EXIF-rotated`() { + val stored = bitmapOf( + arrayOf( + intArrayOf(Color.YELLOW, Color.BLACK), + intArrayOf(Color.BLUE, Color.WHITE), + intArrayOf(Color.GREEN, Color.MAGENTA), + intArrayOf(Color.RED, Color.CYAN), + ), + ) + val fromExif = ImageTensorModule.applyExifOrientation( + stored, + ExifInterface.ORIENTATION_ROTATE_90, + ) + + val mean = doubleArrayOf(0.485, 0.456, 0.406) + val std = doubleArrayOf(0.229, 0.224, 0.225) + val expected = ImageTensorModule.bitmapToNchw( + uprightFixture(), MIEWID_INPUT, MIEWID_INPUT, mean, std, 1.0 / 255.0, false, + ) + val actual = ImageTensorModule.bitmapToNchw( + fromExif, MIEWID_INPUT, MIEWID_INPUT, mean, std, 1.0 / 255.0, false, + ) + + assertEquals(expected.size, actual.size) + for (i in expected.indices) { + assertEquals("tensor element $i", expected[i], actual[i], 0.0) + } + } + + @Test + fun `crop region is identical whether the photo is stored upright or EXIF-rotated`() { + val stored = bitmapOf( + arrayOf( + intArrayOf(Color.YELLOW, Color.BLACK), + intArrayOf(Color.BLUE, Color.WHITE), + intArrayOf(Color.GREEN, Color.MAGENTA), + intArrayOf(Color.RED, Color.CYAN), + ), + ) + val fromExif = ImageTensorModule.applyExifOrientation( + stored, + ExifInterface.ORIENTATION_ROTATE_90, + ) + val upright = uprightFixture() + + // The right half of the animal, as the detector would report it. + val expectedCrop = Bitmap.createBitmap(upright, 2, 0, 2, 2) + val actualCrop = Bitmap.createBitmap(fromExif, 2, 0, 2, 2) + + assertSamePixels(expectedCrop, actualCrop) + } + + // ------------------------------------------------------------------------- + // Call-site guard + // + // The tests above exercise applyExifOrientation directly, so they would all + // still pass if someone removed the call from loadBitmap. This one goes + // through the public cropImage bridge method with a real EXIF-tagged JPEG, + // so it fails if the wiring is dropped. + // + // cropImage is used rather than imageToTensor because the latter resolves a + // WritableNativeArray, which needs the React Native JNI that Robolectric + // does not load. + // ------------------------------------------------------------------------- + + /** Writes an asymmetric JPEG and tags it with an EXIF orientation. */ + private fun writeTaggedJpeg(orientation: Int): File { + val stored = bitmapOf( + arrayOf( + intArrayOf(Color.RED, Color.RED, Color.BLUE, Color.BLUE), + intArrayOf(Color.RED, Color.RED, Color.BLUE, Color.BLUE), + ), + ) + val file = File(tmp.root, "tagged.jpg") + FileOutputStream(file).use { out -> + stored.compress(Bitmap.CompressFormat.JPEG, 100, out) + } + stored.recycle() + val exif = ExifInterface(file.absolutePath) + exif.setAttribute(ExifInterface.TAG_ORIENTATION, orientation.toString()) + exif.saveAttributes() + return file + } + + @Test + fun `cropImage crops from the upright grid for an EXIF-rotated file`() { + // Stored 4x2, red on the left. ORIENTATION_ROTATE_90 means it displays + // as 2x4 with red on TOP, so the top half crop must be entirely red. + val source = writeTaggedJpeg(ExifInterface.ORIENTATION_ROTATE_90) + val outPath = File(tmp.root, "crop.jpg").absolutePath + + val module = ImageTensorModule(mock()) + val latch = CountDownLatch(1) + val failure = arrayOfNulls(1) + val promise = mock() + whenever(promise.resolve(any())) doAnswer { latch.countDown(); null } + whenever(promise.reject(any(), any())) doAnswer { invocation -> + failure[0] = invocation.getArgument(1) + latch.countDown() + null + } + + module.cropImage(source.absolutePath, 0.0, 0.0, 1.0, 0.5, outPath, promise) + + assertTrue("cropImage did not finish", latch.await(10, TimeUnit.SECONDS)) + assertEquals(null, failure[0]) + + val crop = BitmapFactory.decodeFile(outPath) + assertTrue("no output written", crop != null) + + // Upright is 2 wide by 4 tall; the top half is 2x2. + assertEquals("crop width", 2, crop.width) + assertEquals("crop height", 2, crop.height) + + // Every pixel red, allowing for JPEG loss. Without the orientation fix + // this crop is a 4x1 strip that is half blue. + for (y in 0 until crop.height) { + for (x in 0 until crop.width) { + val pixel = crop.getPixel(x, y) + assertTrue( + "pixel ($x,$y) should be red, was ${Integer.toHexString(pixel)}", + Color.red(pixel) > 150 && Color.blue(pixel) < 100, + ) + } + } + crop.recycle() + } +} diff --git a/docs/setup.md b/docs/setup.md index a98d704a4..17911f159 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -62,6 +62,15 @@ Tokens use OS secure storage. The storage namespace incorporates all six deploym See [deployment configuration](../src/config/deployment.ts) and [token handling](../src/services/entraAuthService.ts) for the implementation. +## Existing photo observations + +The EXIF orientation fix applies to newly processed photos: the detector tensor +and saved crop use the same upright image coordinates as the displayed photo. +Existing observations, bounding boxes, crops, embeddings, and review decisions +are not recomputed or migrated. A previously incorrect crop or match does not +become correct merely by upgrading; inspect affected records before syncing. +Do not clear application data or silently reprocess existing field records. + ## Local passphrase locks The local lock uses a versioned PBKDF2-HMAC-SHA-256 verifier with 600,000 iterations and a native-generated salt. `react-native-get-random-values` must be linked in the native app; salts use its native interface directly, without the JavaScript debugger fallback. Missing native randomness fails lock creation or migration rather than using a weaker source. diff --git a/ios/ImageTensorModule.swift b/ios/ImageTensorModule.swift index 7969af922..ba5c81637 100644 --- a/ios/ImageTensorModule.swift +++ b/ios/ImageTensorModule.swift @@ -71,7 +71,8 @@ class ImageTensorModule: NSObject { rejecter reject: @escaping RCTPromiseRejectBlock ) { DispatchQueue.global(qos: .userInitiated).async { - guard let image = Self.loadImage(from: uri), let cgImage = image.cgImage else { + guard let image = Self.loadImage(from: uri), + let cgImage = Self.uprightImage(image)?.cgImage else { reject("IMAGE_ERROR", "Could not load image: \(uri)", nil) return } @@ -135,6 +136,29 @@ class ImageTensorModule: NSObject { return nil } + /// Redraw an image onto the upright grid described by its EXIF orientation. + /// + /// `UIImage.cgImage` is the raw stored buffer and ignores `imageOrientation`, + /// while `imageToTensor` goes through `resizeImage`, whose `draw(in:)` honours + /// it. Cropping the raw buffer therefore cuts from the wrong region for any + /// rotated photo: the on-screen box looks right while the crop MiewID embeds + /// is wrong. Normalising here puts the crop on the same grid as the detector, + /// the overlay, and every EXIF-aware viewer. + /// + /// Returns the input unchanged when it is already upright. + static func uprightImage(_ image: UIImage) -> UIImage? { + if image.imageOrientation == .up { + return image + } + let format = UIGraphicsImageRendererFormat.default() + format.scale = image.scale + format.opaque = false + let renderer = UIGraphicsImageRenderer(size: image.size, format: format) + return renderer.image { _ in + image.draw(in: CGRect(origin: .zero, size: image.size)) + } + } + static func resizeImage(_ image: UIImage, to size: CGSize) -> UIImage? { let renderer = UIGraphicsImageRenderer(size: size) return renderer.image { _ in diff --git a/ios/OffgridMobile/AppDelegate.swift b/ios/OffgridMobile/AppDelegate.swift index c9940513b..160ab5f98 100644 --- a/ios/OffgridMobile/AppDelegate.swift +++ b/ios/OffgridMobile/AppDelegate.swift @@ -41,6 +41,13 @@ class AppDelegate: UIResponder, UIApplicationDelegate, RNAppAuthAuthorizationFlo _ application: UIApplication, didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]? = nil ) -> Bool { + guard Self.shouldStartReactNative( + environment: ProcessInfo.processInfo.environment, + testRuntimeLoaded: NSClassFromString("XCTestCase") != nil + ) else { + return true + } + let delegate = ReactNativeDelegate() let factory = RCTReactNativeFactory(delegate: delegate) delegate.dependencyProvider = RCTAppDependencyProvider() @@ -58,6 +65,17 @@ class AppDelegate: UIResponder, UIApplicationDelegate, RNAppAuthAuthorizationFlo return true } + + static func shouldStartReactNative( + environment: [String: String], + testRuntimeLoaded: Bool + ) -> Bool { +#if DEBUG + return environment["XCTestConfigurationFilePath"] == nil && !testRuntimeLoaded +#else + return true +#endif + } } class ReactNativeDelegate: RCTDefaultReactNativeFactoryDelegate { diff --git a/ios/OffgridMobileTests/OffgridMobileTests.swift b/ios/OffgridMobileTests/OffgridMobileTests.swift index 3737d0299..69af6c293 100644 --- a/ios/OffgridMobileTests/OffgridMobileTests.swift +++ b/ios/OffgridMobileTests/OffgridMobileTests.swift @@ -1,5 +1,7 @@ import XCTest import PDFKit +import ImageIO +import UniformTypeIdentifiers @testable import OffgridMobile @@ -612,7 +614,9 @@ final class DownloadManagerModuleTests: XCTestCase { multiFileDestDir: nil, isMultiFile: false ) - module.downloads[100] = info + module.queue.sync(flags: .barrier) { + module.downloads[100] = info + } let exp = expectation(description: "getActiveDownloads returns completed entry") module.getActiveDownloads( @@ -657,7 +661,9 @@ final class DownloadManagerModuleTests: XCTestCase { multiFileDestDir: nil, isMultiFile: false ) - module.downloads[200] = info + module.queue.sync(flags: .barrier) { + module.downloads[200] = info + } let exp = expectation(description: "moveCompletedDownload moves file") module.moveCompletedDownload( @@ -707,7 +713,9 @@ final class DownloadManagerModuleTests: XCTestCase { multiFileDestDir: nil, isMultiFile: false ) - module.downloads[300] = info + module.queue.sync(flags: .barrier) { + module.downloads[300] = info + } let exp = expectation(description: "moveCompletedDownload rejects not-completed download") module.moveCompletedDownload( @@ -734,6 +742,29 @@ final class DownloadManagerModuleTests: XCTestCase { /// the build itself would fail — making this test a compile-time guard. final class AppDelegateBackgroundSessionTests: XCTestCase { + @MainActor + func testNormalLaunchStillStartsReactNative() { + XCTAssertTrue(AppDelegate.shouldStartReactNative(environment: [:], testRuntimeLoaded: false)) + } + + @MainActor + func testNativeTestMarkersSkipReactNativeStartup() { + XCTAssertFalse(AppDelegate.shouldStartReactNative( + environment: ["XCTestConfigurationFilePath": "/tmp/native.xctestconfiguration"], + testRuntimeLoaded: false + )) + XCTAssertFalse(AppDelegate.shouldStartReactNative(environment: [:], testRuntimeLoaded: true)) + } + + @MainActor + func testNativeTestHostDoesNotCreateAReactRuntime() { + let delegate = AppDelegate() + XCTAssertTrue(delegate.application(UIApplication.shared, didFinishLaunchingWithOptions: nil)) + XCTAssertNil(delegate.reactNativeFactory) + XCTAssertNil(delegate.reactNativeDelegate) + XCTAssertNil(delegate.window) + } + func testAppDelegateRespondsToBackgroundURLSessionSelector() { let appDelegate = AppDelegate() let responds = appDelegate.responds( @@ -911,6 +942,226 @@ final class ImageTensorModuleTests: XCTestCase { // -- Helpers -- + // MARK: - EXIF orientation parity + // + // `imageToTensor` goes through `resizeImage`, whose `draw(in:)` applies + // `imageOrientation`; `cropImage` used the raw `cgImage`, which does not. + // For any rotated photo that put the detector and the crop on different + // grids, so MiewID embedded the wrong pixels while the overlay looked fine. + + /// A renderer pinned to scale 1, so the backing buffer is exactly the + /// requested pixel size. The default format uses the screen scale, which + /// would make a "4x2" image an 8x4 or 12x6 buffer and break the dimension + /// assertions below on a 2x or 3x simulator. + private func unscaledRenderer(width: Int, height: Int) -> UIGraphicsImageRenderer { + let format = UIGraphicsImageRendererFormat.default() + format.scale = 1 + format.opaque = true + return UIGraphicsImageRenderer( + size: CGSize(width: width, height: height), + format: format + ) + } + + /// Left half and right half differ, so a rotation shows up in the pixels. + private func makeSplitImage(width: Int, height: Int) -> UIImage { + return unscaledRenderer(width: width, height: height).image { ctx in + UIColor.red.setFill() + ctx.fill(CGRect(x: 0, y: 0, width: width / 2, height: height)) + UIColor.blue.setFill() + ctx.fill(CGRect(x: width / 2, y: 0, width: width - width / 2, height: height)) + } + } + + /// Raw 0-255 channel values via the module's own extractor, as NCHW. + private func channels(of image: UIImage, width: Int, height: Int) -> [Double] { + guard let cgImage = image.cgImage else { return [] } + return ImageTensorModule.extractNchw( + from: cgImage, + width: width, + height: height, + mean: [0, 0, 0], + std: [1, 1, 1], + scale: 1.0, + bgr: false + ) ?? [] + } + + private func red(_ nchw: [Double], x: Int, y: Int, width: Int, height: Int) -> Double { + return nchw[y * width + x] + } + + private func blue(_ nchw: [Double], x: Int, y: Int, width: Int, height: Int) -> Double { + return nchw[2 * height * width + y * width + x] + } + + func testUprightImageLeavesAnAlreadyUprightImageUnchanged() { + let image = makeSplitImage(width: 4, height: 2) + + let result = ImageTensorModule.uprightImage(image) + + XCTAssertEqual(result?.size.width, 4) + XCTAssertEqual(result?.size.height, 2) + } + + func testUprightImageRotatesARightOrientedImageOntoTheDisplayGrid() { + // A sensor buffer that must be turned 90 degrees clockwise to display. + let stored = makeSplitImage(width: 4, height: 2) + let tagged = UIImage(cgImage: stored.cgImage!, scale: 1, orientation: .right) + + guard let upright = ImageTensorModule.uprightImage(tagged) else { + return XCTFail("uprightImage returned nil") + } + + XCTAssertEqual(upright.size.width, 2, "width and height should swap") + XCTAssertEqual(upright.size.height, 4) + + // Turning clockwise sends the red left half to the top. + let px = channels(of: upright, width: 2, height: 4) + XCTAssertGreaterThan(red(px, x: 0, y: 0, width: 2, height: 4), 200) + XCTAssertLessThan(blue(px, x: 0, y: 0, width: 2, height: 4), 55) + XCTAssertLessThan(red(px, x: 0, y: 3, width: 2, height: 4), 55) + XCTAssertGreaterThan(blue(px, x: 0, y: 3, width: 2, height: 4), 200) + } + + /// MiewID matchability guard: the pixels the model receives must not depend + /// on how the photo happened to be stored. + func testRotatedSourceYieldsTheSameGridAsAnUprightSource() { + let stored = makeSplitImage(width: 4, height: 2) + let tagged = UIImage(cgImage: stored.cgImage!, scale: 1, orientation: .right) + + guard let fromExif = ImageTensorModule.uprightImage(tagged) else { + return XCTFail("uprightImage returned nil") + } + + // The same scene already stored upright: red on top, blue below. + let reference = unscaledRenderer(width: 2, height: 4).image { ctx in + UIColor.red.setFill() + ctx.fill(CGRect(x: 0, y: 0, width: 2, height: 2)) + UIColor.blue.setFill() + ctx.fill(CGRect(x: 0, y: 2, width: 2, height: 2)) + } + + let actual = channels(of: fromExif, width: 2, height: 4) + let expected = channels(of: reference, width: 2, height: 4) + + XCTAssertEqual(actual.count, expected.count) + for index in 0..= 5 ? 20 : 40 + let height = orientation >= 5 ? 40 : 20 + XCTAssertEqual(pixels.width, width) + XCTAssertEqual(pixels.height, height) + XCTAssertEqual(crop.imageOrientation, .up) + let tensor = channels(of: crop, width: width, height: height) + assertColor(tensor, pixel: (height / 2) * width + width / 2, planeSize: width * height, + color: uprightQuadrants[orientation - 1][0], orientation: orientation) + } + } + + func testImageToTensorUsesUprightPixelsForEveryExifOrientation() throws { + let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + + for orientation in 1...8 { + let source = directory.appendingPathComponent("tensor-\(orientation).jpg") + try writeExifFixture(to: source, orientation: orientation) + let completed = expectation(description: "tensor EXIF \(orientation)") + var tensor: [Double]? + module.imageToTensor( + source.path, width: 16, height: 16, mean: [0, 0, 0], std: [1, 1, 1], + scale: 1.0, channelOrder: "RGB", + resolver: { value in + tensor = value as? [Double] + completed.fulfill() + }, + rejecter: { _, message, _ in + XCTFail("EXIF \(orientation) tensor rejected: \(message ?? "unknown")") + completed.fulfill() + } + ) + waitForExpectations(timeout: 5) + let values = try XCTUnwrap(tensor) + for quadrant in 0..<4 { + let pixel = (4 + (quadrant / 2) * 8) * 16 + 4 + (quadrant % 2) * 8 + assertColor(values, pixel: pixel, planeSize: 256, + color: uprightQuadrants[orientation - 1][quadrant], orientation: orientation) + } + } + } + private func createTestImage(width: Int, height: Int, color: UIColor) -> UIImage { let size = CGSize(width: width, height: height) let renderer = UIGraphicsImageRenderer(size: size) diff --git a/kb/wildlife-reid-mobile/outputs/reports/2026-09-09-codex-exif-orientation-review.md b/kb/wildlife-reid-mobile/outputs/reports/2026-09-09-codex-exif-orientation-review.md new file mode 100644 index 000000000..714f05a96 --- /dev/null +++ b/kb/wildlife-reid-mobile/outputs/reports/2026-09-09-codex-exif-orientation-review.md @@ -0,0 +1,94 @@ +# Codex review — EXIF orientation coordinate frame (PR #35) + +**Date:** 2026-09-09 +**Reviewer:** Codex 5.5 (`codex exec`, read-only sandbox) +**Subject:** `fix/exif-orientation-coordinate-frame` — Android `ImageTensorModule.kt`, iOS `ImageTensorModule.swift`, and their tests +**Verdict:** *"The orientation fix is correct and converged geometrically. The tests are not fully converged."* No Critical defect. + +## Background + +Detection boxes and MiewID crops were normalized against the raw sensor pixel +buffer, while the app displays the EXIF-rotated image. For any rotated photo +those are two different pictures. + +- **Android:** `BitmapFactory` ignores the orientation tag; Fresco applies it + (`setAutoRotateEnabled(true)`). Boxes were transposed on screen and crops cut + from the wrong region. +- **iOS:** `imageToTensor` honoured orientation implicitly, because + `resizeImage` uses `UIImage.draw(in:)`. `cropImage` used the raw `cgImage`. + The overlay looked right while the crop fed to MiewID was wrong, degrading + match scores with no visible symptom. + +Discovered during the review of PR #34, which fixed a separate overlay-scaling +bug and is unrelated to this one. + +## Findings and disposition + +| # | Severity | Finding | Disposition | +|---|---|---|---| +| 1 | Major | iOS fixtures used the renderer's default screen scale while rewrapping the buffer at `scale: 1`, so a "4x2" image is 8x4 or 12x6 on a 2x/3x simulator and the dimension assertions fail despite correct production code. | **Fixed.** Both fixtures now render through a renderer pinned to scale 1. Found independently before reading the review. | +| 2 | Major | Every orientation test called `applyExifOrientation` / `uprightImage` directly, so removing the call from `loadBitmap` or `cropImage` would have left them all green. | **Android verified.** Added an end-to-end `cropImage` test with a real EXIF-tagged JPEG; unwiring `loadBitmap` made that test fail. **iOS follow-up added:** real tagged JPEGs for all eight orientations through `cropImage` and `imageToTensor`, checking upright dimensions and independently specified quadrant colors. macOS CI is the required execution gate. | +| 3 | Minor | Exceptional paths skip bitmap recycling: a transform failure leaks the decoded bitmap, a tensor failure leaks the caller's, a crop or file-write failure leaks both. Successful-path ownership is correct, with no double-recycle or use-after-recycle. | **Deferred.** GC reclaims these; deterministic release is a behaviour change that needs its own tests. Worth a follow-up. | +| 4 | Minor | Crop quantization still differs across platforms: Android truncates origin and extent independently, iOS passes fractional bounds to Core Graphics, which expands to integral bounds. For `x=1.5, width=2.5` Android selects `[1,3)` and iOS `[1,4)`. | **Deferred, pre-existing.** Not an EXIF error. Fix is a shared integer-bound calculation on both platforms. | +| 5 | Correct | All eight Android transforms and every stored fixture verified against an independently derived source-to-upright pixel mapping table, including the `FₓR90 = TRANSPOSE` and `FₓR270 = TRANSVERSE` post-multiplication order. | No action. | +| 6 | Correct | iOS `uprightImage` uses orientation-aware `image.size` and preserves `image.scale`. `UIImage(cgImage:)` at the end of `cropImage` correctly uses `.up` because orientation is already baked into those pixels. Both platforms now match the RN `` display frame. | No action. | +| 7 | Correct | `content://` decoding and EXIF parsing each open a fresh stream closed with `use`; attributes are read at construction, so reading orientation after closure is valid. Unknown values safely return the input. | No action. | + +## Verification performed locally + +Android, on WSL, all passing: + +| Gate | Result | +|---|---| +| `compileDebugKotlin` | pass | +| `lintDebug` | pass, 2 warnings, none new | +| Android unit suite | 30 tests, 0 failures | +| Orientation suite | 12 tests, 0 failures | + +Red-green was genuine: the nine pure-function tests were watched failing against +a no-op stub before implementation, and the call-site test was watched failing +against an unwired `loadBitmap`. + +## Open gaps + +- **The local review hosts have no Xcode.** However, `.github/workflows/ci.yml` + already runs `npm test` on macOS, including native iOS tests through + `scripts/run-apple-check.js`. Do not add a duplicate iOS test job. +- **The first PR35 CI run was not green.** Run `34443887005` executed and passed + all three new helper-level orientation tests, then reported overall failure + after an unexpected exit associated with + `DownloadManagerModuleTests.testCompletedDownloadEntryPersistsUntilMoved()`. + Logs show the debug test host attempting React Native startup without Metro + or a bundled script. This was not a Hermes compile failure or an EXIF assertion. +- **Native-host isolation and bridge tests need macOS verification.** The + follow-up skips React startup only in debug native XCTest hosts, with tests + for normal-launch policy and the actual delegate path. Tagged JPEG bridge + tests cover all eight orientation values; a successful new CI run is required + before acceptance. Native tests are not silently skipped on Windows. +- **A separate test-fixture race was subsequently identified.** PR34 run + `34483668391` passed the host-isolation tests, then crashed in + `persistStateLocked` during background restoration with + `-[__NSCFNumber count]: unrecognized selector`. The download tests were + writing directly into the shared dictionary while restoration read it. + All three fixture injections now use the module's existing barrier queue. + This is test-only synchronization, not a production download change; + require macOS CI for this follow-up too. +- **The Android `content://` path is untested.** Only the plain file path is + covered; the gallery path needs an instrumented test. +- **Physical camera/gallery validation is still required.** Helper and bridge + tests do not replace checking the full capture, detector, overlay, and crop + flow with tagged photos on devices. + +## Follow-ups worth filing + +1. Require a green existing macOS test job, including the tagged-image bridge + and native-host tests, before merging. +2. Deterministic bitmap release on exceptional paths (finding 3). +3. Shared integer crop-bound calculation across platforms (finding 4). +4. Downsample at decode. `applyExifOrientation` allocates a second + full-resolution bitmap, doubling peak memory while both exist. This + compounds an existing problem: capture already decodes at full resolution + with no downsampling (`quality: 1`, no `maxWidth`/`maxHeight`). +5. Observations captured on Android before this fix still carry boxes in the + old frame. They are not migrated, and nothing marks which convention + produced them.