diff --git a/ui/scanner/src/main/kotlin/com/getcode/ui/scanner/CodeScanner.kt b/ui/scanner/src/main/kotlin/com/getcode/ui/scanner/CodeScanner.kt index 8b19708d03..fdd8cb4c33 100644 --- a/ui/scanner/src/main/kotlin/com/getcode/ui/scanner/CodeScanner.kt +++ b/ui/scanner/src/main/kotlin/com/getcode/ui/scanner/CodeScanner.kt @@ -225,9 +225,19 @@ fun CodeScanner( mutableStateOf(PreviewView.StreamState.IDLE) } - LaunchedEffect(streamState) { + LaunchedEffect(streamState, gestureController) { if (streamState == PreviewView.StreamState.STREAMING) { trace("camera ready") + + // Exposure is metered on the centre of the frame rather than the whole of it, so a + // bright code in a dark room is exposed for the code and not for the room. See + // `applyBaselineMetering` -- overexposure does not degrade detection, it ends it. + // + // Deliberately keyed on the stream actually running, not on the bind returning: + // `startFocusAndMetering` is refused with "Camera is not active" until the camera has + // opened, and it refuses *silently*. Asking any earlier would leave whole-frame + // metering in place while looking, from the source, exactly like a fix. + gestureController?.applyBaselineMetering() } } diff --git a/ui/scanner/src/main/kotlin/com/getcode/ui/scanner/internal/CameraGestureController.kt b/ui/scanner/src/main/kotlin/com/getcode/ui/scanner/internal/CameraGestureController.kt index 1422b5cff1..8c7f2d5a14 100644 --- a/ui/scanner/src/main/kotlin/com/getcode/ui/scanner/internal/CameraGestureController.kt +++ b/ui/scanner/src/main/kotlin/com/getcode/ui/scanner/internal/CameraGestureController.kt @@ -10,6 +10,7 @@ import androidx.camera.core.CameraControl import androidx.camera.core.CameraInfo import androidx.camera.core.FocusMeteringAction import androidx.camera.core.MeteringPoint +import androidx.camera.core.SurfaceOrientedMeteringPointFactory import androidx.compose.ui.geometry.Offset import java.util.concurrent.TimeUnit import kotlin.math.pow @@ -88,11 +89,24 @@ internal class CameraGestureController( override fun onSingleTapUp(event: MotionEvent): Boolean { val point = onTap(Offset(event.x, event.y)) - val action = FocusMeteringAction.Builder(point, FocusMeteringAction.FLAG_AF) - .setAutoCancelDuration(5, TimeUnit.SECONDS) + // AE as well as AF: a tap means "read this", and on a washed-out code the exposure + // is the half that is actually broken. + val action = FocusMeteringAction.Builder( + point, + FocusMeteringAction.FLAG_AF or FocusMeteringAction.FLAG_AE, + ) + .setAutoCancelDuration(TAP_METERING_SECONDS, TimeUnit.SECONDS) .build() cameraControl.startFocusAndMetering(action) + + // Auto-cancel drops *all* 3A regions, not just the ones this tap set, so without + // re-arming, the first tap would cost the centre exposure region permanently. + handler.removeCallbacks(restoreBaselineMetering) + handler.postDelayed( + restoreBaselineMetering, + TimeUnit.SECONDS.toMillis(TAP_METERING_SECONDS), + ) return true } @@ -114,6 +128,35 @@ internal class CameraGestureController( } ) + private val restoreBaselineMetering = Runnable { applyBaselineMetering() } + + /** + * Meter exposure on the centre of the frame, where the code is, and keep it there. + * + * With no region set the camera meters the whole frame, which for a scanner is the wrong + * subject: a code on a phone screen is a small bright rectangle in a mostly dark scene, so + * whole-frame metering exposes for the room and drives the screen into clipping. + * + * That is fatal rather than merely degrading, because the native detector splits light from + * dark at a fixed luminance of 170 with no adaptive fallback -- measured in + * `WashoutToleranceTest`, the code's dark ink must land below 170 and its light ink above it, + * and contrast beyond that barely matters. A clipped code is not a poor input, it is a uniform + * white slab with no contours to find, and detection stops dead. iOS pins + * `exposurePointOfInterest` to the centre for the same reason. + * + * AE only, deliberately. Adding FLAG_AF would fire a one-shot autofocus and, with auto-cancel + * disabled, leave focus locked at whatever distance it happened to land on -- the opposite of + * what a scanner wants. Focus stays in the camera's own continuous mode. + */ + fun applyBaselineMetering() { + val centre = SurfaceOrientedMeteringPointFactory(1f, 1f).createPoint(0.5f, 0.5f) + val action = FocusMeteringAction.Builder(centre, FocusMeteringAction.FLAG_AE) + .disableAutoCancel() + .build() + + cameraControl.startFocusAndMetering(action) + } + fun onTouchEvent(event: MotionEvent) { if (gesturesEnabled) { if (initialZoomLevel == -1f) { @@ -156,4 +199,8 @@ internal class CameraGestureController( } }) } -} \ No newline at end of file + + private companion object { + const val TAP_METERING_SECONDS = 5L + } +} diff --git a/vendor/kik/scanner/src/androidTest/kotlin/com/kik/scan/AnalysisResolutionTest.kt b/vendor/kik/scanner/src/androidTest/kotlin/com/kik/scan/AnalysisResolutionTest.kt index f3c9445668..12ec5af9e9 100644 --- a/vendor/kik/scanner/src/androidTest/kotlin/com/kik/scan/AnalysisResolutionTest.kt +++ b/vendor/kik/scanner/src/androidTest/kotlin/com/kik/scan/AnalysisResolutionTest.kt @@ -6,7 +6,10 @@ import android.util.Size import android.view.Surface import androidx.camera.core.CameraSelector import androidx.camera.core.ImageAnalysis +import androidx.camera.core.Camera +import androidx.camera.core.FocusMeteringAction import androidx.camera.core.Preview +import androidx.camera.core.SurfaceOrientedMeteringPointFactory import androidx.camera.core.resolutionselector.AspectRatioStrategy import androidx.camera.core.resolutionselector.ResolutionSelector import androidx.camera.core.resolutionselector.ResolutionStrategy @@ -230,6 +233,72 @@ class AnalysisResolutionTest { ) } + /** + * The centre exposure region the scanner asks for is actually accepted by this camera. + * + * `startFocusAndMetering` is a request, not a command: a device that does not support AE + * regions reports zero `maxMeteringPointsAe` and the call resolves as unsuccessful, leaving + * whole-frame metering in place. That failure is completely silent at runtime -- the scanner + * keeps working, just as badly as before -- so the only way to know the fix took is to ask. + * + * Logs rather than fails on an unsupported device: not every camera offers AE regions, and + * that is a fact about the hardware rather than a defect in the scanner. + */ + @Test + fun centreExposureMeteringIsSupported() { + val context = InstrumentationRegistry.getInstrumentation().targetContext + val provider = ProcessCameraProvider.getInstance(context).get(10, TimeUnit.SECONDS) + val owner = TestLifecycleOwner() + val instrumentation = InstrumentationRegistry.getInstrumentation() + val analysis = shippingAnalysis() + val executor = Executors.newSingleThreadExecutor() + var camera: Camera? = null + + instrumentation.runOnMainSync { + provider.unbindAll() + camera = provider.bindToLifecycle(owner, selector, Preview.Builder().build(), analysis) + owner.resume() + } + + try { + val bound = requireNotNull(camera) { "camera did not bind" } + + // Wait for a real frame before asking. Binding returns long before the camera opens, + // and `startFocusAndMetering` on a camera that is not yet active fails with + // OperationCanceledException -- which would look identical to an unsupported device. + val frame = CountDownLatch(1) + analysis.setAnalyzer(executor) { image -> frame.countDown(); image.close() } + val active = frame.await(15, TimeUnit.SECONDS) + assertTrue(active, "camera never delivered a frame, so metering could not be tested") + + val centre = SurfaceOrientedMeteringPointFactory(1f, 1f).createPoint(0.5f, 0.5f) + val action = FocusMeteringAction.Builder(centre, FocusMeteringAction.FLAG_AE) + .disableAutoCancel() + .build() + + val supported = bound.cameraInfo.isFocusMeteringSupported(action) + val result = runCatching { + bound.cameraControl.startFocusAndMetering(action).get(5, TimeUnit.SECONDS) + } + + Log.i( + TAG, + // No `isFocusSuccessful` here: it reports the *focus* outcome and is + // documented to return false whenever the action carries no AF flag, so on an + // AE-only request it says nothing and reads like a failure. + "centre AE metering: supported=$supported " + + "requestOutcome=${result.exceptionOrNull()?.toString() ?: "accepted"}", + ) + } finally { + instrumentation.runOnMainSync { + analysis.clearAnalyzer() + provider.unbindAll() + owner.destroy() + } + executor.shutdown() + } + } + private companion object { const val TAG = "KikCodeRange" } diff --git a/vendor/kik/scanner/src/androidTest/kotlin/com/kik/scan/WashoutToleranceTest.kt b/vendor/kik/scanner/src/androidTest/kotlin/com/kik/scan/WashoutToleranceTest.kt new file mode 100644 index 0000000000..ac379c9306 --- /dev/null +++ b/vendor/kik/scanner/src/androidTest/kotlin/com/kik/scan/WashoutToleranceTest.kt @@ -0,0 +1,200 @@ +package com.kik.scan + +import android.graphics.Bitmap +import android.graphics.Canvas +import android.graphics.Color +import android.util.Log +import androidx.core.content.ContextCompat +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import com.getcode.codes.kikcode.LuminancePlane +import com.kik.kikx.kikcodes.ScanQuality +import com.kik.kikx.kikcodes.implementation.KikCodeScannerImpl +import com.kik.kikx.kincodes.KikCodeContentRendererImpl +import com.kik.kikx.models.ScannableKikCode +import kotlinx.coroutines.runBlocking +import org.junit.Test +import org.junit.runner.RunWith +import kotlin.test.assertTrue + +/** + * How much exposure error the detector survives. + * + * Candidate finding is built entirely on one line of `scanner.cpp`: + * + * threshold(greyscale, whitish, 170, 255, THRESH_BINARY); + * + * Every contour, every ellipse, and therefore every code the scanner ever finds comes out of that + * binary image. The cutoff is an absolute luminance value, not a local or adaptive one, and it runs + * after two unsharp passes that push highlights further up. So the detector does not ask whether the + * code has contrast -- it asks whether the code's light ink lands above 170 and its dark ink lands + * below it. A frame can be perfectly sharp, perfectly framed and perfectly in focus and still be + * undetectable simply for sitting at the wrong exposure. + * + * That makes auto-exposure part of the scanner, and the two platforms configure it differently: + * iOS pins `exposurePointOfInterest` to the centre of the frame, Android leaves metering to the + * camera's whole-frame default and never requests AE at all. Pointed at a bright phone screen in a + * dim room, whole-frame metering exposes for the dark surround and blows the screen out. + * + * This measures the window rather than deriving it, because the unsharp passes move the boundary + * and only a measurement says by how much. + * + * Frames are synthetic and otherwise ideal, so these bounds are the generous end of what a real + * camera sees. + */ +@RunWith(AndroidJUnit4::class) +class WashoutToleranceTest { + + private val renderer = KikCodeContentRendererImpl().apply { + badge = requireNotNull( + ContextCompat.getDrawable( + InstrumentationRegistry.getInstrumentation().context, + com.kik.kikx.test.R.drawable.ic_logo_round_white, + ) + ) + } + private val scanner = KikCodeScannerImpl() + + private fun encodeRemoteCode(seed: Int): Pair { + val payload = ByteArray(REMOTE_PAYLOAD_BYTES) { ((it * 7 + seed) and 0xFF).toByte() } + return payload to requireNotNull(Scanner.encode(payload)) { "native encode returned null" } + } + + /** + * Renders a code into a Y plane whose full black-to-white range has been squeezed into + * [blackLevel]..[whiteLevel]. + * + * This is what an exposure error does to a frame. Overexposure lifts the whole range towards + * white and clips it there; underexposure crushes it towards black; veiling glare off a bright + * emissive panel lifts the floor without moving the ceiling. All three are the same + * transformation with different endpoints, and all three are applied to the whole frame, + * because a camera's exposure is a property of the frame and not of the subject. + */ + private fun renderAtLevels( + encoded: ByteArray, + width: Int, + height: Int, + codePx: Int, + blackLevel: Int, + whiteLevel: Int, + ): ByteArray { + val bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888) + val canvas = Canvas(bitmap) + canvas.drawColor(Color.BLACK) + + canvas.save() + canvas.translate((width - codePx) / 2f, (height - codePx) / 2f) + renderer.render(encoded, codePx, canvas) + canvas.restore() + + val pixels = IntArray(width * height) + bitmap.getPixels(pixels, 0, width, 0, 0, width, height) + bitmap.recycle() + + // Precomputed so the remap is a table lookup rather than arithmetic per pixel; these + // sweeps render hundreds of multi-megapixel frames. + val span = whiteLevel - blackLevel + val map = ByteArray(256) { luma -> + (blackLevel + luma * span / 255).coerceIn(0, 255).toByte() + } + + val plane = ByteArray(width * height) + for (i in 0 until width * height) { + val p = pixels[i] + val luma = ( + ( + 77 * ((p shr 16) and 0xFF) + + 150 * ((p shr 8) and 0xFF) + + 29 * (p and 0xFF) + ) shr 8 + ) + plane[i] = map[luma] + } + return plane + } + + private fun decodesAtLevels( + encoded: ByteArray, + payload: ByteArray, + blackLevel: Int, + whiteLevel: Int, + ): Boolean { + val plane = renderAtLevels(encoded, WIDTH, HEIGHT, CODE_PX, blackLevel, whiteLevel) + val converted = LuminancePlane.unpad(plane, WIDTH, HEIGHT, WIDTH, 1) + val result = runBlocking { + scanner.scanKikCode(converted, WIDTH, HEIGHT, ScanQuality.Best).getOrNull() + } + return result is ScannableKikCode.RemoteKikCode && result.payloadId.contentEquals(payload) + } + + /** + * Overexposure: the highlights are already clipped at white and the floor keeps rising. + * + * This is the reported failure. A phone screen at full brightness in a dim room is the worst + * case a payment scanner has, because whole-frame metering averages in all that surrounding + * darkness and drives the exposure up until the screen is a white slab. + */ + @Test + fun overexposureFloor() { + val (payload, encoded) = encodeRemoteCode(3) + var highestDecodable = -1 + for (black in 0..250 step 10) { + val ok = decodesAtLevels(encoded, payload, black, 255) + Log.i(TAG, "overexposed black=$black white=255 decoded=$ok") + if (ok) highestDecodable = black else break + } + Log.i(TAG, "RESULT overexposure: highest decodable black level = $highestDecodable") + + assertTrue( + highestDecodable >= 0, + "the detector failed even on a correctly exposed frame -- the sweep is measuring " + + "something other than exposure", + ) + } + + /** Underexposure, for symmetry: the floor is at black and the ceiling keeps falling. */ + @Test + fun underexposureCeiling() { + val (payload, encoded) = encodeRemoteCode(3) + var lowestDecodable = -1 + for (white in 255 downTo 5 step 10) { + val ok = decodesAtLevels(encoded, payload, 0, white) + Log.i(TAG, "underexposed black=0 white=$white decoded=$ok") + if (ok) lowestDecodable = white else break + } + Log.i(TAG, "RESULT underexposure: lowest decodable white level = $lowestDecodable") + } + + /** + * Contrast held around a mid-grey, so the band closes in on 128 from both sides. + * + * If the detector adapted to the frame it would keep working here until the contrast fell into + * the noise. If it is pinned to an absolute cutoff it will instead fail the moment the band + * stops straddling that cutoff, while the code is still obviously legible. Which of those two + * happens is the whole question. + */ + @Test + fun contrastAroundMidGrey() { + val (payload, encoded) = encodeRemoteCode(3) + var lowestDecodable = -1 + for (half in 128 downTo 5 step 5) { + val ok = decodesAtLevels(encoded, payload, 128 - half, 128 + half) + Log.i(TAG, "midgrey black=${128 - half} white=${128 + half} span=${2 * half} decoded=$ok") + if (ok) lowestDecodable = 2 * half else break + } + Log.i(TAG, "RESULT mid-grey: smallest decodable contrast span = $lowestDecodable of 255") + } + + private companion object { + const val TAG = "KikCodeRange" + const val REMOTE_PAYLOAD_BYTES = 20 + const val WIDTH = 1920 + const val HEIGHT = 1080 + + /** + * A comfortably large code -- roughly 40% of the frame height, far above the size floor + * measured in `KikCodeRangeTest`. Size must not be the thing under test here. + */ + const val CODE_PX = 432 + } +}