fix(barcode-scanning): incorrect coordinate calculation on iOS (#127)

* wip * docs * wip
capawesome-team · Feb 16, 2024 · 454bad4 · 454bad4
1 parent 0f0a47c
commit 454bad4
Show file tree

Hide file tree

Showing 6 changed files with 47 additions and 13 deletions.
diff --git a/.changeset/sour-brooms-deny.md b/.changeset/sour-brooms-deny.md
@@ -0,0 +1,5 @@
+---
+'@capacitor-mlkit/barcode-scanning': patch
+---
+
+fix(ios): incorrect coordinate calculation
diff --git a/packages/barcode-scanning/README.md b/packages/barcode-scanning/README.md
@@ -714,7 +714,6 @@ Remove all listeners for this plugin.
 | ---------------- | ------------------------------------------------- | ---------------------------------------------------------------------------------------- | ----- |
 | **`formats`**    | <code>BarcodeFormat[]</code>                      | Improve the speed of the barcode scanner by configuring the barcode formats to scan for. | 0.0.1 |
 | **`lensFacing`** | <code><a href="#lensfacing">LensFacing</a></code> | Configure the camera (front or back) to use.                                             | 0.0.1 |
-| **`zoomRatio`**  | <code>number</code>                               | The initial zoom ratio of the camera.                                                    | 5.4.0 |
 
 
 #### ReadBarcodesFromImageResult

diff --git a/...in/java/io/capawesome/capacitorjs/plugins/mlkit/barcodescanning/BarcodeScannerHelper.java b/...in/java/io/capawesome/capacitorjs/plugins/mlkit/barcodescanning/BarcodeScannerHelper.java
@@ -164,23 +164,31 @@ private static JSONArray convertByteArrayToJsonArray(byte[] bytes) {
     }
 
     private static Point[] normalizeCornerPoints(@NonNull Point[] cornerPoints, @NonNull Point imageSize, @NonNull Point screenSize) {
+        // Log corner points
+        // Logger.debug("Corner points: " + cornerPoints[0] + ", " + cornerPoints[1] + ", " + cornerPoints[2] + ", " + cornerPoints[3]);
         double screenWidth = screenSize.x;
         double screenHeight = screenSize.y;
         double imageWidth = imageSize.x;
         double imageHeight = imageSize.y;
+        // Swap the image dimensions if the image is in landscape mode
         if (screenWidth > screenHeight) {
             imageWidth = imageSize.y;
             imageHeight = imageSize.x;
         }
+        // Calculate the scale of the image
         double scale = Math.max(screenHeight / imageWidth, screenWidth / imageHeight);
+        // Calculate the invisible area of the image
         double invisibleWidth = imageHeight * scale - screenWidth;
         double invisibleHeight = imageWidth * scale - screenHeight;
         Point[] normalizedCornerPoints = new Point[cornerPoints.length];
         for (int i = 0; i < cornerPoints.length; i++) {
+            // Scale the points and move them to the center of the screen
             int x = (int) ((cornerPoints[i].x * scale) - (invisibleWidth / 2));
             int y = (int) ((cornerPoints[i].y * scale) - (invisibleHeight / 2));
             normalizedCornerPoints[i] = new Point(x, y);
         }
+        // Log normalized corner points
+        // Logger.debug("Normalized corner points: " + normalizedCornerPoints[0] + ", " + normalizedCornerPoints[1] + ", " + normalizedCornerPoints[2] + ", " + normalizedCornerPoints[3]);
         return normalizedCornerPoints;
     }
 }
diff --git a/packages/barcode-scanning/ios/Plugin/BarcodeScannerHelper.swift b/packages/barcode-scanning/ios/Plugin/BarcodeScannerHelper.swift
@@ -8,30 +8,51 @@ import MLKitBarcodeScanning
 // swiftlint:disable cyclomatic_complexity
 public class BarcodeScannerHelper {
     // swiftlint:disable identifier_name
-    public static func normalizeCornerPoints(cornerPoints: [NSValue], imageSize: CGSize, scale: CGFloat = UIScreen.main.scale) -> [NSValue] {
-        let screenSize: CGRect = UIScreen.main.bounds
-        let imageWidth = imageSize.width
-        let imageHeight = imageSize.height
+    public static func normalizeCornerPoints(cornerPoints: [NSValue], imageSize: CGSize, screenSize: CGSize) -> [NSValue] {
+        // Log corner points
+        // CAPLog.print("Corner points (\(cornerPoints[0].cgPointValue.x), \(cornerPoints[0].cgPointValue.y)), (\(cornerPoints[1].cgPointValue.x), \(cornerPoints[1].cgPointValue.y)), (\(cornerPoints[2].cgPointValue.x), \(cornerPoints[2].cgPointValue.y)), (\(cornerPoints[3].cgPointValue.x), \(cornerPoints[3].cgPointValue.y))")
+        let screenWidth = screenSize.width
+        let screenHeight = screenSize.height
+        var imageWidth = imageSize.width
+        var imageHeight = imageSize.height
+        // Swap the image dimensions if the image is in landscape mode
+        if screenWidth > screenHeight {
+            imageWidth = imageSize.height
+            imageHeight = imageSize.width
+        }
+        // Calculate the scale of the image
+        let scale = max(screenHeight / CGFloat(imageWidth), screenWidth / CGFloat(imageHeight))
+        // Calculate the invisible area of the image
+        let invisibleWidth = imageHeight * scale - screenWidth
+        let invisibleHeight = imageWidth * scale - screenHeight
         let isPortrait = UIDevice.current.orientation == .portrait || UIDevice.current.orientation == .portraitUpsideDown
         var normalizedCornerPoints = [NSValue]()
         for cornerPoint in cornerPoints {
-            var x = Int((cornerPoint.cgPointValue.x / CGFloat(imageWidth)) * screenSize.width * scale)
-            var y = Int((cornerPoint.cgPointValue.y / CGFloat(imageHeight)) * screenSize.height * scale)
+            var x = Int(((cornerPoint.cgPointValue.x * scale) - (invisibleWidth / 2)))
+            var y = Int(((cornerPoint.cgPointValue.y * scale) - (invisibleHeight / 2)))
             if isPortrait {
-                x = Int((1 - (cornerPoint.cgPointValue.y / CGFloat(imageHeight))) * screenSize.width * scale)
-                y = Int((cornerPoint.cgPointValue.x / CGFloat(imageWidth)) * screenSize.height * scale)
+                x = Int((((imageHeight - cornerPoint.cgPointValue.y) * scale) - (invisibleWidth / 2)))
+                y = Int(((cornerPoint.cgPointValue.x * scale) - (invisibleHeight / 2)))
             }
             let point = CGPoint(x: x, y: y)
             let value = NSValue(cgPoint: point)
             normalizedCornerPoints.append(value)
         }
+        // If the image is in portrait mode, the corner points need to be rotated
+        if isPortrait {
+            let lastNormalizedCornerPoints = normalizedCornerPoints.removeLast()
+            normalizedCornerPoints.insert(lastNormalizedCornerPoints, at: 0)
+        }
+        // Log normalized corner points
+        // CAPLog.print("Normalized corner points (\(normalizedCornerPoints[0].cgPointValue.x), \(normalizedCornerPoints[0].cgPointValue.y)), (\(normalizedCornerPoints[1].cgPointValue.x), \(normalizedCornerPoints[1].cgPointValue.y)), (\(normalizedCornerPoints[2].cgPointValue.x), \(normalizedCornerPoints[2].cgPointValue.y)), (\(normalizedCornerPoints[3].cgPointValue.x), \(normalizedCornerPoints[3].cgPointValue.y))")
         return normalizedCornerPoints
     }
 
-    public static func createBarcodeResultForBarcode(_ barcode: Barcode, imageSize: CGSize?, scale: CGFloat = UIScreen.main.scale) -> JSObject {
+    public static func createBarcodeResultForBarcode(_ barcode: Barcode, imageSize: CGSize?) -> JSObject {
         var cornerPointsResult = [[Int]]()
         if let cornerPoints = barcode.cornerPoints, let imageSize = imageSize {
-            let normalizedCornerPoints = normalizeCornerPoints(cornerPoints: cornerPoints, imageSize: imageSize, scale: scale)
+            let screenSize = CGSize(width: UIScreen.main.bounds.width * UIScreen.main.scale, height: UIScreen.main.bounds.height * UIScreen.main.scale)
+            let normalizedCornerPoints = normalizeCornerPoints(cornerPoints: cornerPoints, imageSize: imageSize, screenSize: screenSize)
             for cornerPoint in normalizedCornerPoints {
                 var value = [Int]()
                 value.append(Int(cornerPoint.cgPointValue.x))

diff --git a/packages/barcode-scanning/ios/Plugin/BarcodeScannerPlugin.swift b/packages/barcode-scanning/ios/Plugin/BarcodeScannerPlugin.swift
@@ -115,7 +115,7 @@ public class BarcodeScannerPlugin: CAPPlugin {
                 }
                 var barcodeResults = JSArray()
                 for barcode in barcodes ?? [] {
-                    barcodeResults.append(BarcodeScannerHelper.createBarcodeResultForBarcode(barcode, imageSize: nil, scale: 1))
+                    barcodeResults.append(BarcodeScannerHelper.createBarcodeResultForBarcode(barcode, imageSize: nil))
                 }
                 call.resolve([
                     "barcodes": barcodeResults

diff --git a/packages/barcode-scanning/ios/Plugin/BarcodeScannerView.swift b/packages/barcode-scanning/ios/Plugin/BarcodeScannerView.swift
@@ -262,8 +262,9 @@ public protocol BarcodeScannerViewDelegate {
     private func filterBarcodesOutsideTheDetectionArea(_ barcodes: [Barcode], imageSize: CGSize?, detectionArea: CGRect) -> [Barcode] {
         return barcodes.filter { barcode in
             if let cornerPoints = barcode.cornerPoints, let imageSize = imageSize {
+                let screenSize = CGSize(width: UIScreen.main.bounds.width, height: UIScreen.main.bounds.height)
                 let normalizedCornerPoints = BarcodeScannerHelper.normalizeCornerPoints(cornerPoints: cornerPoints,
-                                                                                        imageSize: imageSize, scale: 1)
+                                                                                        imageSize: imageSize, screenSize: screenSize)
 
                 let topLeft = normalizedCornerPoints[0].cgPointValue
                 let topRight = normalizedCornerPoints[1].cgPointValue