Skip to content

Commit

Permalink
Improve Bitmap processing efficiency (#351)
Browse files Browse the repository at this point in the history 
* Remove saveAsGrayscale

* Improve Bitmap processing efficiency

* Nits

* Update Changelog
  • Loading branch information
@vanshg
vanshg authored Apr 29, 2024
1 parent 08263f4 commit 741c2d5
Show file tree
Hide file tree
Showing 4 changed files with 65 additions and 93 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
## Unreleased

* Fixed a bug where Document Capture would occasionally fail
* Optimize Bitmap processing for less memory usage and improved quality
* Fixed a bug where Document Verification would get stuck on the processing screen

## 10.1.0
Expand Down
136 changes: 56 additions & 80 deletions lib/src/main/java/com/smileidentity/util/Util.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,8 @@ import android.database.Cursor
import android.graphics.Bitmap
import android.graphics.Bitmap.CompressFormat.JPEG
import android.graphics.BitmapFactory
import android.graphics.Canvas
import android.graphics.ColorMatrix
import android.graphics.ColorMatrixColorFilter
import android.graphics.BitmapFactory.Options
import android.graphics.Matrix
import android.graphics.Paint
import android.graphics.Rect
import android.net.Uri
import android.os.Build.VERSION.SDK_INT
Expand All @@ -23,7 +20,6 @@ import androidx.annotation.IntRange
import androidx.annotation.StringRes
import androidx.camera.core.ImageProxy
import androidx.camera.core.impl.utils.Exif
import androidx.core.graphics.scale
import com.google.mlkit.vision.common.InputImage
import com.smileidentity.R
import com.smileidentity.SmileID
Expand All @@ -36,6 +32,7 @@ import io.sentry.Breadcrumb
import io.sentry.SentryLevel
import java.io.File
import java.io.Serializable
import kotlin.math.max
import kotlinx.collections.immutable.toImmutableList
import kotlinx.coroutines.CoroutineExceptionHandler
import okio.IOException
Expand Down Expand Up @@ -88,7 +85,7 @@ internal fun isImageAtLeast(
height: Int? = 1080,
): Boolean {
if (uri == null) return false
val options = BitmapFactory.Options().apply { inJustDecodeBounds = true }
val options = Options().apply { inJustDecodeBounds = true }
context.contentResolver.openInputStream(uri).use {
BitmapFactory.decodeStream(it, null, options)
}
Expand All @@ -111,15 +108,15 @@ internal fun isValidDocumentImage(context: Context, uri: Uri?) =
isImageAtLeast(context, uri, width = 1920, height = 1080)

fun Bitmap.rotated(rotationDegrees: Int, flipX: Boolean = false, flipY: Boolean = false): Bitmap {
val matrix = Matrix()
val matrix = Matrix().apply {
// Rotate the image back to straight.
postRotate(rotationDegrees.toFloat())

// Rotate the image back to straight.
matrix.postRotate(rotationDegrees.toFloat())
// Mirror the image along the X or Y axis.
postScale(if (flipX) -1.0f else 1.0f, if (flipY) -1.0f else 1.0f)
}

// Mirror the image along the X or Y axis.
matrix.postScale(if (flipX) -1.0f else 1.0f, if (flipY) -1.0f else 1.0f)
val rotatedBitmap =
Bitmap.createBitmap(this, 0, 0, width, height, matrix, true)
val rotatedBitmap = Bitmap.createBitmap(this, 0, 0, width, height, matrix, true)

// Recycle the old bitmap if it has changed.
if (rotatedBitmap !== this) {
Expand All @@ -129,119 +126,98 @@ fun Bitmap.rotated(rotationDegrees: Int, flipX: Boolean = false, flipY: Boolean
}

/**
* Post-processes the image stored in [bitmap] and saves to [file]. The image is scaled to
* [maxOutputSize], but maintains the aspect ratio. The image can also converted to grayscale.
* Post-processes the image stored in [bitmap] and saves to [file]. The image is scaled such that
* the longer dimension equals [resizeLongerDimensionTo] while maintaining the aspect ratio.
*
* Only one of [maxOutputSize] or [desiredAspectRatio] can be set. Setting both is not supported,
* and will throw an [IllegalArgumentException].
* Only one of [resizeLongerDimensionTo] or [desiredAspectRatio] can be set. Setting both is not
* supported, and will throw an [IllegalArgumentException].
*/
@SuppressLint("RestrictedApi")
internal fun postProcessImageBitmap(
bitmap: Bitmap,
file: File,
saveAsGrayscale: Boolean = false,
processRotation: Boolean = false,
@IntRange(from = 0, to = 100) compressionQuality: Int = 100,
maxOutputSize: Size? = null,
resizeLongerDimensionTo: Int? = null,
desiredAspectRatio: Float? = null,
): File {
check(compressionQuality in 0..100) { "Compression quality must be between 0 and 100" }
if (maxOutputSize != null && desiredAspectRatio != null) {
if (resizeLongerDimensionTo != null && desiredAspectRatio != null) {
throw IllegalArgumentException("Only one of maxOutputSize or desiredAspectRatio can be set")
}
var mutableBitmap = bitmap.copy(Bitmap.Config.ARGB_8888, true)
if (saveAsGrayscale) {
val canvas = Canvas(mutableBitmap)
val colorMatrix = ColorMatrix().apply { setSaturation(0f) }
val paint = Paint().apply { colorFilter = ColorMatrixColorFilter(colorMatrix) }
canvas.drawBitmap(mutableBitmap, 0f, 0f, paint)
}

val matrix = Matrix()
val didSwapDimensions: Boolean
if (processRotation) {
val exif = Exif.createFromFile(file)
val degrees = exif.rotation.toFloat()
val degrees = exif.rotation
didSwapDimensions = degrees == 90 || degrees == 270
val scale = if (exif.isFlippedHorizontally) -1F else 1F
val matrix = Matrix().apply {
postScale(scale, 1F)
postRotate(degrees)
}
mutableBitmap = Bitmap.createBitmap(
mutableBitmap,
0,
0,
mutableBitmap.width,
mutableBitmap.height,
matrix,
true,
)
matrix.postScale(scale, 1F)
matrix.postRotate(degrees.toFloat())
} else {
didSwapDimensions = false
}

// If size is the original Bitmap size, then no scaling will be performed by the underlying call
// Aspect ratio will be maintained by retaining the larger dimension
val outputSize = maxOutputSize?.let { size ->
val aspectRatioInput = mutableBitmap.width.toFloat() / mutableBitmap.height
val aspectRatioMax = size.width.toFloat() / size.height
var outputWidth = size.width
var outputHeight = size.height
if (aspectRatioInput > aspectRatioMax) {
outputHeight = (outputWidth / aspectRatioInput).toInt()
} else {
outputWidth = (outputHeight * aspectRatioInput).toInt()
}
Size(outputWidth, outputHeight)
resizeLongerDimensionTo?.let {
val maxDimensionSize = max(bitmap.width, bitmap.height)
val scaleFactor = it.toFloat() / maxDimensionSize
matrix.postScale(scaleFactor, scaleFactor)
}

// Crop height to match desired aspect ratio. This specific behavior is because we force
// portrait mode when doing document captures, so the image should always be taller than it is
// wide. If the image is wider than it is tall, then we return as-is
// For reference, the default aspect ratio of an ID card is around ~1.6
// NB! This assumes that the portrait mode pic will be taller than it is wide
val croppedHeight = desiredAspectRatio?.let {
return@let if (mutableBitmap.width > mutableBitmap.height) {
Timber.w("Image is wider than it is tall, so not cropping the height")
mutableBitmap.height
val (x, y, newSize) = desiredAspectRatio?.let {
val width = if (didSwapDimensions) bitmap.height else bitmap.width
val height = if (didSwapDimensions) bitmap.width else bitmap.height
if (width > height) {
return@let Triple(0, 0, Size(bitmap.width, bitmap.height))
}
val newHeight = (width / it).toInt().coerceIn(0..height)
val y = (height - newHeight) / 2
return@let if (didSwapDimensions) {
Triple(y, 0, Size(newHeight, width))
} else {
(mutableBitmap.width / it).toInt().coerceIn(0..mutableBitmap.height)
Triple(0, y, Size(width, newHeight))
}
} ?: mutableBitmap.height
} ?: Triple(0, 0, Size(bitmap.width, bitmap.height))

// Center crop the bitmap to the specified croppedHeight and apply the matrix
file.outputStream().use {
outputSize?.let { outputSize ->
// Filter is set to false for improved performance at the expense of image quality
mutableBitmap = mutableBitmap.scale(outputSize.width, outputSize.height, filter = false)
}

desiredAspectRatio?.let {
// Center crop the bitmap to the specified croppedHeight
mutableBitmap = Bitmap.createBitmap(
mutableBitmap,
0,
(mutableBitmap.height - croppedHeight) / 2,
mutableBitmap.width,
croppedHeight,
)
val compressSuccess = Bitmap.createBitmap(
bitmap,
x,
y,
newSize.width,
newSize.height,
matrix,
true,
).compress(JPEG, compressionQuality, it)
if (!compressSuccess) {
SmileIDCrashReporting.hub.addBreadcrumb("Failed to compress bitmap")
throw IOException("Failed to compress bitmap")
}

mutableBitmap.compress(JPEG, compressionQuality, it)
}
return file
}

/**
* Post-processes the image stored in `file`, in-place
* Post-processes the image stored in [file] in-place
*/
internal fun postProcessImage(
file: File,
saveAsGrayscale: Boolean = false,
processRotation: Boolean = true,
compressionQuality: Int = 100,
desiredAspectRatio: Float? = null,
): File {
val bitmap = BitmapFactory.decodeFile(file.absolutePath)
val options = Options().apply { inMutable = true }
val bitmap = BitmapFactory.decodeFile(file.absolutePath, options)
return postProcessImageBitmap(
bitmap = bitmap,
file = file,
saveAsGrayscale = saveAsGrayscale,
processRotation = processRotation,
compressionQuality = compressionQuality,
desiredAspectRatio = desiredAspectRatio,
Expand Down
11 changes: 4 additions & 7 deletions lib/src/main/java/com/smileidentity/viewmodel/SelfieViewModel.kt
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package com.smileidentity.viewmodel

import android.util.Size
import androidx.annotation.OptIn
import androidx.annotation.StringRes
import androidx.annotation.VisibleForTesting
Expand Down Expand Up @@ -61,8 +60,8 @@ private val UI_DEBOUNCE_DURATION = 250.milliseconds
private const val INTRA_IMAGE_MIN_DELAY_MS = 350
private const val NUM_LIVENESS_IMAGES = 7
private const val TOTAL_STEPS = NUM_LIVENESS_IMAGES + 1 // 7 B&W Liveness + 1 Color Selfie
private val LIVENESS_IMAGE_SIZE = Size(320, 320)
private val SELFIE_IMAGE_SIZE = Size(640, 640)
private const val LIVENESS_IMAGE_SIZE = 320
private const val SELFIE_IMAGE_SIZE = 640
private const val NO_FACE_RESET_DELAY_MS = 3000
private const val FACE_ROTATION_THRESHOLD = 0.75f
private const val MIN_FACE_AREA_THRESHOLD = 0.15f
Expand Down Expand Up @@ -213,9 +212,8 @@ class SelfieViewModel(
postProcessImageBitmap(
bitmap = bitmap,
file = livenessFile,
saveAsGrayscale = false,
compressionQuality = 80,
maxOutputSize = LIVENESS_IMAGE_SIZE,
resizeLongerDimensionTo = LIVENESS_IMAGE_SIZE,
)
livenessFiles.add(livenessFile)
_uiState.update { it.copy(progress = livenessFiles.size / TOTAL_STEPS.toFloat()) }
Expand All @@ -225,9 +223,8 @@ class SelfieViewModel(
postProcessImageBitmap(
bitmap = bitmap,
file = selfieFile!!,
saveAsGrayscale = false,
compressionQuality = 80,
maxOutputSize = SELFIE_IMAGE_SIZE,
resizeLongerDimensionTo = SELFIE_IMAGE_SIZE,
)
shouldAnalyzeImages = false
_uiState.update { it.copy(progress = 1f, selfieToConfirm = selfieFile) }
Expand Down
10 changes: 4 additions & 6 deletions lib/src/main/java/com/smileidentity/viewmodel/SmartSelfieV2ViewModel.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,8 @@ import timber.log.Timber
private const val HISTORY_LENGTH = 7
private const val INTRA_IMAGE_MIN_DELAY_MS = 250
private const val NUM_LIVENESS_IMAGES = 4
private val LIVENESS_IMAGE_SIZE = android.util.Size(320, 320)
private val SELFIE_IMAGE_SIZE = android.util.Size(640, 640)
private const val LIVENESS_IMAGE_SIZE = 320
private const val SELFIE_IMAGE_SIZE = 640
private const val FACE_QUALITY_THRESHOLD = 0.5f
private const val MIN_FACE_FILL_THRESHOLD = 0.15f
private const val MAX_FACE_FILL_THRESHOLD = 0.25f
Expand Down Expand Up @@ -243,9 +243,8 @@ class SmartSelfieV2ViewModel(
postProcessImageBitmap(
bitmap = fullSelfieBmp,
file = livenessFile,
saveAsGrayscale = false,
compressionQuality = 80,
maxOutputSize = LIVENESS_IMAGE_SIZE,
resizeLongerDimensionTo = LIVENESS_IMAGE_SIZE,
)
livenessFiles.add(livenessFile)
return@addOnSuccessListener
Expand All @@ -259,9 +258,8 @@ class SmartSelfieV2ViewModel(
postProcessImageBitmap(
bitmap = fullSelfieBmp,
file = selfieFile,
saveAsGrayscale = false,
compressionQuality = 80,
maxOutputSize = SELFIE_IMAGE_SIZE,
resizeLongerDimensionTo = SELFIE_IMAGE_SIZE,
)

val proxy = { e: Throwable -> onResult(SmileIDResult.Error(e)) }
Expand Down

0 comments on commit 741c2d5

Please sign in to comment.