Implement KMP Selfie Check Composable with the flow View Model. (#1062)

Test: All Unit tests are passing (bar KI).
Test: Manually tested via samples/testapp on Android and iOS.

Signed-off-by: koukarine <koukarine@google.com>

Author: koukarine

multipaz-compose/src/androidMain/kotlin/org/multipaz/compose/Util.android.kt

@@ -46,22 +46,22 @@ actual fun cropRotateScaleImage(
     cx: Double,
     cy: Double,
     angleDegrees: Double,
-    outputWidth: Int,
-    outputHeight: Int,
-    targetWidth: Int
+    outputWidthPx: Int,
+    outputHeightPx: Int,
+    targetWidthPx: Int
 ): ImageBitmap {
     val androidBitmap = frameData.cameraImage.imageProxy.toBitmap()
-    val finalScale = targetWidth.toFloat() / outputWidth.toFloat()
-    val finalOutputHeight = (outputHeight * finalScale).toInt()
+    val finalScale = targetWidthPx.toFloat() / outputWidthPx.toFloat()
+    val finalOutputHeight = (outputHeightPx * finalScale).toInt()
     val matrix = Matrix() // Use Android's Matrix
 
     matrix.postTranslate(-cx.toFloat(), -cy.toFloat())
     matrix.postRotate(angleDegrees.toFloat())
-    matrix.postTranslate((outputWidth / 2).toFloat(), (outputHeight / 2).toFloat())
+    matrix.postTranslate((outputWidthPx / 2).toFloat(), (outputHeightPx / 2).toFloat())
     matrix.postScale(finalScale, finalScale)
 
     // Create the output bitmap with the final scaled dimensions.
-    val resultBitmap = createBitmap(targetWidth, finalOutputHeight, androidBitmap.config ?: Bitmap.Config.ARGB_8888)
+    val resultBitmap = createBitmap(targetWidthPx, finalOutputHeight, androidBitmap.config ?: Bitmap.Config.ARGB_8888)
     Canvas(resultBitmap).drawBitmap(androidBitmap, matrix, paint)
 
     return resultBitmap.asImageBitmap()

multipaz-compose/src/androidMain/kotlin/org/multipaz/compose/camera/Camera.android.kt

@@ -13,6 +13,7 @@ import androidx.camera.core.resolutionselector.ResolutionStrategy
 import androidx.camera.lifecycle.ProcessCameraProvider
 import androidx.camera.view.PreviewView
 import androidx.camera.view.TransformExperimental
+import androidx.compose.foundation.layout.add
 import androidx.compose.runtime.Composable
 import androidx.compose.runtime.DisposableEffect
 import androidx.compose.runtime.getValue
@@ -31,6 +32,8 @@ import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.withContext
 import org.multipaz.util.Logger
 import java.util.concurrent.Executors
+import kotlin.collections.isNotEmpty
+import kotlin.collections.toTypedArray
 
 private const val TAG = "Camera"
 
@@ -47,7 +50,7 @@ actual fun Camera(
     val lifecycleOwner = LocalLifecycleOwner.current
     val cameraProviderFuture = remember { ProcessCameraProvider.getInstance(context) }
     val executor = remember { Executors.newSingleThreadExecutor() }
-    val previewView = remember { mutableStateOf<PreviewView?>(if (showCameraPreview) PreviewView(context) else null) }
+    var activePreviewView by remember { mutableStateOf<PreviewView?>(null) }
     var currentDisplayRotation by remember { mutableIntStateOf(Surface.ROTATION_0) }
 
     DisposableEffect(Unit) { // Keyed by Unit to run once and clean up with the composable
@@ -73,81 +76,113 @@ actual fun Camera(
         }
     }
 
-    DisposableEffect(cameraSelection) {
-        var preview: Preview? = null
-        if (showCameraPreview) {
-            preview = Preview.Builder().build()
-            // COMPATIBLE is needed b/c we're using `outputTransform.matrix`
-            previewView.value!!.implementationMode = PreviewView.ImplementationMode.COMPATIBLE
-            preview.setSurfaceProvider(previewView.value!!.surfaceProvider)
-        }
-
+    DisposableEffect(cameraSelection, showCameraPreview, activePreviewView, currentDisplayRotation) {
         val cameraProvider = cameraProviderFuture.get()
 
-        val resolutionStrategy = ResolutionStrategy(
-            captureResolution.getDimensions(),
-            ResolutionStrategy.FALLBACK_RULE_CLOSEST_LOWER
-        )
-        val resolutionSelector = ResolutionSelector.Builder()
-            .setResolutionStrategy(resolutionStrategy)
-            .build()
-
-        val imageAnalysis = ImageAnalysis.Builder()
-            .setResolutionSelector(resolutionSelector)
-            .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
-            .setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_YUV_420_888)
-            .build()
-
-        imageAnalysis.setAnalyzer(executor) { imageProxy ->
-            runBlocking {
-                val transformationProxy = if (showCameraPreview) {
-                    withContext(Dispatchers.Main) {
-                        getCorrectionMatrix(imageProxy, previewView.value!!)
+        if (!cameraProviderFuture.isDone) {
+            onDispose { /** Not bound. */ }
+        } else {
+            cameraProvider.unbindAll()
+
+            val resolutionStrategy = ResolutionStrategy(
+                captureResolution.getDimensions(),
+                ResolutionStrategy.FALLBACK_RULE_CLOSEST_LOWER
+            )
+            val resolutionSelector = ResolutionSelector.Builder()
+                .setResolutionStrategy(resolutionStrategy)
+                .build()
+
+            val imageAnalysis = ImageAnalysis.Builder()
+                .setResolutionSelector(resolutionSelector)
+                .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
+                .setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_YUV_420_888)
+                .setTargetRotation(currentDisplayRotation)
+                .build()
+
+            imageAnalysis.setAnalyzer(executor) { imageProxy ->
+                runBlocking {
+                    val transformationProxy = if (showCameraPreview && activePreviewView != null) {
+                        withContext(Dispatchers.Main) {
+                            if (activePreviewView?.surfaceProvider != null && activePreviewView?.outputTransform != null) {
+                                getCorrectionMatrix(imageProxy, activePreviewView!!)
+                            } else {
+                                Matrix() // Fallback if preview is not ready.
+                            }
+                        }
+                    } else {
+                        Matrix()
                     }
-                } else {
-                    Matrix()
+                    val frame = CameraFrame(
+                        cameraImage = CameraImage(imageProxy),
+                        width = imageProxy.width,
+                        height = imageProxy.height,
+                        rotation = calculateDetectorAngle(currentDisplayRotation, cameraSelection),
+                        previewTransformation = transformationProxy
+                    )
+                    onFrameCaptured(frame)
                 }
-                val frame = CameraFrame(
-                    cameraImage = CameraImage(imageProxy),
-                    width = imageProxy.width,
-                    height = imageProxy.height,
-                    rotation = calculateDetectorAngle(currentDisplayRotation, cameraSelection), 
-                    previewTransformation = transformationProxy
-                )
-                onFrameCaptured(frame)
+                imageProxy.close()
             }
-            imageProxy.close()
-        }
 
-        cameraProviderFuture.get().unbindAll()
-        if (showCameraPreview) {
-            cameraProviderFuture.get().bindToLifecycle(
-                lifecycleOwner,
-                cameraSelection.toAndroidCameraSelector(),
-                preview,
-                imageAnalysis
-            )
-        } else {
-            cameraProviderFuture.get().bindToLifecycle(
-                lifecycleOwner,
-                cameraSelection.toAndroidCameraSelector(),
-                imageAnalysis
-            )
-        }
+            try {
+                if (showCameraPreview && activePreviewView != null) {
+                    val preview = Preview.Builder()
+                        .build()
+                        .also {
+                            activePreviewView!!.implementationMode = PreviewView.ImplementationMode.COMPATIBLE
+                            it.setSurfaceProvider(activePreviewView!!.surfaceProvider)
+                        }
+                    cameraProvider.bindToLifecycle(
+                        lifecycleOwner,
+                        cameraSelection.toAndroidCameraSelector(),
+                        preview,
+                        imageAnalysis
+                    )
+                } else {
+                    // Image Analysis only if preview is not shown.
+                    cameraProvider.bindToLifecycle(
+                        lifecycleOwner,
+                        cameraSelection.toAndroidCameraSelector(),
+                        imageAnalysis
+                    )
+                }
+            } catch (exc: Exception) {
+                Logger.e(TAG, "Use case binding failed", exc)
+            }
 
-        onDispose {
-            cameraProvider.unbindAll()
-            executor.shutdown()
+            onDispose {
+                cameraProvider.unbindAll()
+            }
         }
     }
 
     if (showCameraPreview) {
         AndroidView(
             modifier = modifier,
-            factory = { context ->
-                previewView.value!!
+            factory = { ctx ->
+                PreviewView(ctx).apply {
+                    implementationMode = PreviewView.ImplementationMode.COMPATIBLE
+                    activePreviewView = this
+                }
+            },
+            onRelease = {
+                activePreviewView = null
             }
         )
+    } else {
+        if (activePreviewView != null) {
+            // This ensures the DisposableEffect re-runs if the view was previously shown.
+            activePreviewView = null
+        }
+    }
+
+    DisposableEffect(Unit) {
+        onDispose {
+            Logger.d(TAG, "Shutting down camera executor.")
+            if (!executor.isShutdown) {
+                executor.shutdown()
+            }
+        }
     }
 }
 
@@ -177,7 +212,7 @@ private fun CameraSelection.toAndroidCameraSelector() =
     }
 
 @OptIn(TransformExperimental::class)
-private fun getCorrectionMatrix(imageProxy: ImageProxy, previewView: PreviewView) : Matrix {
+private fun getCorrectionMatrix(imageProxy: ImageProxy, previewView: PreviewView): Matrix {
 
     // This matrix maps (-1, -1) -> (1, 1) space to preview-coordinate system. This includes
     // any scaling, rotation, or cropping that's done in the preview.
@@ -188,8 +223,8 @@ private fun getCorrectionMatrix(imageProxy: ImageProxy, previewView: PreviewView
     // By the scale and translate below, we modify the matrix so it maps
     // (0, 0) -> (width, height) of the frame to analyze to the preview
     // coordinate system
-    composeMatrix.scale(2f/imageProxy.width, 2f/imageProxy.height, 1f)
-    composeMatrix.translate(-0.5f*imageProxy.width, -0.5f*imageProxy.height, 1.0f)
+    composeMatrix.scale(2f / imageProxy.width, 2f / imageProxy.height, 1f)
+    composeMatrix.translate(-0.5f * imageProxy.width, -0.5f * imageProxy.height, 1.0f)
 
     return composeMatrix
 }

multipaz-compose/src/commonMain/kotlin/org/multipaz/compose/Util.kt

@@ -29,7 +29,7 @@ expect fun getApplicationInfo(appId: String): ApplicationInfo
 expect fun decodeImage(encodedData: ByteArray): ImageBitmap
 
 /**
- * Encodes a bitmap to PNG
+ * Encodes a bitmap to PNG.
  *
  * @param image the image to encode.
  * @return a [ByteString] with the encoded data.
@@ -82,7 +82,7 @@ expect fun cropRotateScaleImage(
     cx: Double,
     cy: Double,
     angleDegrees: Double,
-    outputWidth: Int,
-    outputHeight: Int,
-    targetWidth: Int // You might want to make targetWidth/Height nullable or handle aspect ratio
+    outputWidthPx: Int,
+    outputHeightPx: Int,
+    targetWidthPx: Int
 ): ImageBitmap

multipaz-compose/src/commonMain/kotlin/org/multipaz/compose/camera/CameraFrame.kt

@@ -36,4 +36,8 @@ data class CameraFrame(
      * If preview is disabled, this is the identity matrix.
      */
     val previewTransformation: Matrix
-)
+
+) {
+    /** Determine if the rotation angle indicates the camera was used from a landscape phone orientation mode. */
+    val isLandscape: Boolean = (rotation == 90 || rotation == 270)
+}