From 4448c5491a907076f05359762c3fecf94659b56f Mon Sep 17 00:00:00 2001 From: Robin Genz Date: Thu, 15 Feb 2024 20:46:17 +0100 Subject: [PATCH 1/3] wip --- packages/barcode-scanning/README.md | 1 - .../barcodescanning/BarcodeScannerHelper.java | 8 ++++ .../ios/Plugin/BarcodeScannerHelper.swift | 37 ++++++++++++++----- .../ios/Plugin/BarcodeScannerPlugin.swift | 2 +- .../ios/Plugin/BarcodeScannerView.swift | 2 +- 5 files changed, 37 insertions(+), 13 deletions(-) diff --git a/packages/barcode-scanning/README.md b/packages/barcode-scanning/README.md index d522dfd..7712d1f 100644 --- a/packages/barcode-scanning/README.md +++ b/packages/barcode-scanning/README.md @@ -714,7 +714,6 @@ Remove all listeners for this plugin. | ---------------- | ------------------------------------------------- | ---------------------------------------------------------------------------------------- | ----- | | **`formats`** | BarcodeFormat[] | Improve the speed of the barcode scanner by configuring the barcode formats to scan for. | 0.0.1 | | **`lensFacing`** | LensFacing | Configure the camera (front or back) to use. | 0.0.1 | -| **`zoomRatio`** | number | The initial zoom ratio of the camera. | 5.4.0 | #### ReadBarcodesFromImageResult diff --git a/packages/barcode-scanning/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/barcodescanning/BarcodeScannerHelper.java b/packages/barcode-scanning/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/barcodescanning/BarcodeScannerHelper.java index 413aa7c..6d06ffe 100644 --- a/packages/barcode-scanning/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/barcodescanning/BarcodeScannerHelper.java +++ b/packages/barcode-scanning/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/barcodescanning/BarcodeScannerHelper.java @@ -164,23 +164,31 @@ private static JSONArray convertByteArrayToJsonArray(byte[] bytes) { } private static Point[] normalizeCornerPoints(@NonNull Point[] cornerPoints, @NonNull Point imageSize, @NonNull Point screenSize) { + // Log corner points + Logger.debug("Corner points: " + cornerPoints[0] + ", " + cornerPoints[1] + ", " + cornerPoints[2] + ", " + cornerPoints[3]); double screenWidth = screenSize.x; double screenHeight = screenSize.y; double imageWidth = imageSize.x; double imageHeight = imageSize.y; + // Swap the image dimensions if the image is in landscape mode if (screenWidth > screenHeight) { imageWidth = imageSize.y; imageHeight = imageSize.x; } + // Calculate the scale of the image double scale = Math.max(screenHeight / imageWidth, screenWidth / imageHeight); + // Calculate the invisible area of the image double invisibleWidth = imageHeight * scale - screenWidth; double invisibleHeight = imageWidth * scale - screenHeight; Point[] normalizedCornerPoints = new Point[cornerPoints.length]; for (int i = 0; i < cornerPoints.length; i++) { + // Scale the points and move them to the center of the screen int x = (int) ((cornerPoints[i].x * scale) - (invisibleWidth / 2)); int y = (int) ((cornerPoints[i].y * scale) - (invisibleHeight / 2)); normalizedCornerPoints[i] = new Point(x, y); } + // Log normalized corner points + Logger.debug("Normalized corner points: " + normalizedCornerPoints[0] + ", " + normalizedCornerPoints[1] + ", " + normalizedCornerPoints[2] + ", " + normalizedCornerPoints[3]); return normalizedCornerPoints; } } diff --git a/packages/barcode-scanning/ios/Plugin/BarcodeScannerHelper.swift b/packages/barcode-scanning/ios/Plugin/BarcodeScannerHelper.swift index 47f38da..a927826 100644 --- a/packages/barcode-scanning/ios/Plugin/BarcodeScannerHelper.swift +++ b/packages/barcode-scanning/ios/Plugin/BarcodeScannerHelper.swift @@ -8,30 +8,47 @@ import MLKitBarcodeScanning // swiftlint:disable cyclomatic_complexity public class BarcodeScannerHelper { // swiftlint:disable identifier_name - public static func normalizeCornerPoints(cornerPoints: [NSValue], imageSize: CGSize, scale: CGFloat = UIScreen.main.scale) -> [NSValue] { - let screenSize: CGRect = UIScreen.main.bounds - let imageWidth = imageSize.width - let imageHeight = imageSize.height + public static func normalizeCornerPoints(cornerPoints: [NSValue], imageSize: CGSize) -> [NSValue] { + CAPLog.print("Corner points (\(cornerPoints[0].cgPointValue.x), \(cornerPoints[0].cgPointValue.y)), (\(cornerPoints[1].cgPointValue.x), \(cornerPoints[1].cgPointValue.y)), (\(cornerPoints[2].cgPointValue.x), \(cornerPoints[2].cgPointValue.y)), (\(cornerPoints[3].cgPointValue.x), \(cornerPoints[3].cgPointValue.y))") + let screenWidth = UIScreen.main.bounds.width + let screenHeight = UIScreen.main.bounds.height + var imageWidth = imageSize.width + var imageHeight = imageSize.height + + if screenWidth > screenHeight { + imageWidth = imageSize.height + imageHeight = imageSize.width + } + + let scale = max(screenHeight / CGFloat(imageWidth), screenWidth / CGFloat(imageHeight)) + let isPortrait = UIDevice.current.orientation == .portrait || UIDevice.current.orientation == .portraitUpsideDown + let invisibleWidth = imageHeight * scale - screenWidth + let invisibleHeight = imageWidth * scale - screenHeight var normalizedCornerPoints = [NSValue]() for cornerPoint in cornerPoints { - var x = Int((cornerPoint.cgPointValue.x / CGFloat(imageWidth)) * screenSize.width * scale) - var y = Int((cornerPoint.cgPointValue.y / CGFloat(imageHeight)) * screenSize.height * scale) + var x = Int(((cornerPoint.cgPointValue.x * scale) - (invisibleWidth / 2)) * UIScreen.main.scale) + var y = Int(((cornerPoint.cgPointValue.y * scale) - (invisibleHeight / 2)) * UIScreen.main.scale) if isPortrait { - x = Int((1 - (cornerPoint.cgPointValue.y / CGFloat(imageHeight))) * screenSize.width * scale) - y = Int((cornerPoint.cgPointValue.x / CGFloat(imageWidth)) * screenSize.height * scale) + x = Int((((imageHeight - cornerPoint.cgPointValue.y) * scale) - (invisibleWidth / 2)) * UIScreen.main.scale) + y = Int(((cornerPoint.cgPointValue.x * scale) - (invisibleHeight / 2)) * UIScreen.main.scale) } let point = CGPoint(x: x, y: y) let value = NSValue(cgPoint: point) normalizedCornerPoints.append(value) } + if isPortrait { + let lastNormalizedCornerPoints = normalizedCornerPoints.removeLast() + normalizedCornerPoints.insert(lastNormalizedCornerPoints, at: 0) + } + CAPLog.print("Normalized corner points (\(normalizedCornerPoints[0].cgPointValue.x), \(normalizedCornerPoints[0].cgPointValue.y)), (\(normalizedCornerPoints[1].cgPointValue.x), \(normalizedCornerPoints[1].cgPointValue.y)), (\(normalizedCornerPoints[2].cgPointValue.x), \(normalizedCornerPoints[2].cgPointValue.y)), (\(normalizedCornerPoints[3].cgPointValue.x), \(normalizedCornerPoints[3].cgPointValue.y))") return normalizedCornerPoints } - public static func createBarcodeResultForBarcode(_ barcode: Barcode, imageSize: CGSize?, scale: CGFloat = UIScreen.main.scale) -> JSObject { + public static func createBarcodeResultForBarcode(_ barcode: Barcode, imageSize: CGSize?, scale222: CGFloat = UIScreen.main.scale) -> JSObject { var cornerPointsResult = [[Int]]() if let cornerPoints = barcode.cornerPoints, let imageSize = imageSize { - let normalizedCornerPoints = normalizeCornerPoints(cornerPoints: cornerPoints, imageSize: imageSize, scale: scale) + let normalizedCornerPoints = normalizeCornerPoints(cornerPoints: cornerPoints, imageSize: imageSize) for cornerPoint in normalizedCornerPoints { var value = [Int]() value.append(Int(cornerPoint.cgPointValue.x)) diff --git a/packages/barcode-scanning/ios/Plugin/BarcodeScannerPlugin.swift b/packages/barcode-scanning/ios/Plugin/BarcodeScannerPlugin.swift index c5005a0..5d81472 100644 --- a/packages/barcode-scanning/ios/Plugin/BarcodeScannerPlugin.swift +++ b/packages/barcode-scanning/ios/Plugin/BarcodeScannerPlugin.swift @@ -115,7 +115,7 @@ public class BarcodeScannerPlugin: CAPPlugin { } var barcodeResults = JSArray() for barcode in barcodes ?? [] { - barcodeResults.append(BarcodeScannerHelper.createBarcodeResultForBarcode(barcode, imageSize: nil, scale: 1)) + barcodeResults.append(BarcodeScannerHelper.createBarcodeResultForBarcode(barcode, imageSize: nil)) } call.resolve([ "barcodes": barcodeResults diff --git a/packages/barcode-scanning/ios/Plugin/BarcodeScannerView.swift b/packages/barcode-scanning/ios/Plugin/BarcodeScannerView.swift index 29fd2bf..f7852fe 100644 --- a/packages/barcode-scanning/ios/Plugin/BarcodeScannerView.swift +++ b/packages/barcode-scanning/ios/Plugin/BarcodeScannerView.swift @@ -263,7 +263,7 @@ public protocol BarcodeScannerViewDelegate { return barcodes.filter { barcode in if let cornerPoints = barcode.cornerPoints, let imageSize = imageSize { let normalizedCornerPoints = BarcodeScannerHelper.normalizeCornerPoints(cornerPoints: cornerPoints, - imageSize: imageSize, scale: 1) + imageSize: imageSize) let topLeft = normalizedCornerPoints[0].cgPointValue let topRight = normalizedCornerPoints[1].cgPointValue From bf82af3b3fa8c5c34875029a234398ccd8c2d030 Mon Sep 17 00:00:00 2001 From: Robin Genz Date: Thu, 15 Feb 2024 20:49:08 +0100 Subject: [PATCH 2/3] docs --- .changeset/sour-brooms-deny.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/sour-brooms-deny.md diff --git a/.changeset/sour-brooms-deny.md b/.changeset/sour-brooms-deny.md new file mode 100644 index 0000000..35516e5 --- /dev/null +++ b/.changeset/sour-brooms-deny.md @@ -0,0 +1,5 @@ +--- +'@capacitor-mlkit/barcode-scanning': patch +--- + +fix(ios): incorrect coordinate calculation From b1cf5ce65f0cfd05e5cfa7843e35e93d22a6698c Mon Sep 17 00:00:00 2001 From: Robin Genz Date: Fri, 16 Feb 2024 09:11:06 +0100 Subject: [PATCH 3/3] wip --- .../barcodescanning/BarcodeScannerHelper.java | 4 +-- .../ios/Plugin/BarcodeScannerHelper.swift | 34 +++++++++++-------- .../ios/Plugin/BarcodeScannerView.swift | 3 +- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/packages/barcode-scanning/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/barcodescanning/BarcodeScannerHelper.java b/packages/barcode-scanning/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/barcodescanning/BarcodeScannerHelper.java index 6d06ffe..bb9402b 100644 --- a/packages/barcode-scanning/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/barcodescanning/BarcodeScannerHelper.java +++ b/packages/barcode-scanning/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/barcodescanning/BarcodeScannerHelper.java @@ -165,7 +165,7 @@ private static JSONArray convertByteArrayToJsonArray(byte[] bytes) { private static Point[] normalizeCornerPoints(@NonNull Point[] cornerPoints, @NonNull Point imageSize, @NonNull Point screenSize) { // Log corner points - Logger.debug("Corner points: " + cornerPoints[0] + ", " + cornerPoints[1] + ", " + cornerPoints[2] + ", " + cornerPoints[3]); + // Logger.debug("Corner points: " + cornerPoints[0] + ", " + cornerPoints[1] + ", " + cornerPoints[2] + ", " + cornerPoints[3]); double screenWidth = screenSize.x; double screenHeight = screenSize.y; double imageWidth = imageSize.x; @@ -188,7 +188,7 @@ private static Point[] normalizeCornerPoints(@NonNull Point[] cornerPoints, @Non normalizedCornerPoints[i] = new Point(x, y); } // Log normalized corner points - Logger.debug("Normalized corner points: " + normalizedCornerPoints[0] + ", " + normalizedCornerPoints[1] + ", " + normalizedCornerPoints[2] + ", " + normalizedCornerPoints[3]); + // Logger.debug("Normalized corner points: " + normalizedCornerPoints[0] + ", " + normalizedCornerPoints[1] + ", " + normalizedCornerPoints[2] + ", " + normalizedCornerPoints[3]); return normalizedCornerPoints; } } diff --git a/packages/barcode-scanning/ios/Plugin/BarcodeScannerHelper.swift b/packages/barcode-scanning/ios/Plugin/BarcodeScannerHelper.swift index a927826..7b3334c 100644 --- a/packages/barcode-scanning/ios/Plugin/BarcodeScannerHelper.swift +++ b/packages/barcode-scanning/ios/Plugin/BarcodeScannerHelper.swift @@ -8,47 +8,51 @@ import MLKitBarcodeScanning // swiftlint:disable cyclomatic_complexity public class BarcodeScannerHelper { // swiftlint:disable identifier_name - public static func normalizeCornerPoints(cornerPoints: [NSValue], imageSize: CGSize) -> [NSValue] { - CAPLog.print("Corner points (\(cornerPoints[0].cgPointValue.x), \(cornerPoints[0].cgPointValue.y)), (\(cornerPoints[1].cgPointValue.x), \(cornerPoints[1].cgPointValue.y)), (\(cornerPoints[2].cgPointValue.x), \(cornerPoints[2].cgPointValue.y)), (\(cornerPoints[3].cgPointValue.x), \(cornerPoints[3].cgPointValue.y))") - let screenWidth = UIScreen.main.bounds.width - let screenHeight = UIScreen.main.bounds.height + public static func normalizeCornerPoints(cornerPoints: [NSValue], imageSize: CGSize, screenSize: CGSize) -> [NSValue] { + // Log corner points + // CAPLog.print("Corner points (\(cornerPoints[0].cgPointValue.x), \(cornerPoints[0].cgPointValue.y)), (\(cornerPoints[1].cgPointValue.x), \(cornerPoints[1].cgPointValue.y)), (\(cornerPoints[2].cgPointValue.x), \(cornerPoints[2].cgPointValue.y)), (\(cornerPoints[3].cgPointValue.x), \(cornerPoints[3].cgPointValue.y))") + let screenWidth = screenSize.width + let screenHeight = screenSize.height var imageWidth = imageSize.width var imageHeight = imageSize.height - + // Swap the image dimensions if the image is in landscape mode if screenWidth > screenHeight { imageWidth = imageSize.height imageHeight = imageSize.width } - + // Calculate the scale of the image let scale = max(screenHeight / CGFloat(imageWidth), screenWidth / CGFloat(imageHeight)) - - let isPortrait = UIDevice.current.orientation == .portrait || UIDevice.current.orientation == .portraitUpsideDown + // Calculate the invisible area of the image let invisibleWidth = imageHeight * scale - screenWidth let invisibleHeight = imageWidth * scale - screenHeight + let isPortrait = UIDevice.current.orientation == .portrait || UIDevice.current.orientation == .portraitUpsideDown var normalizedCornerPoints = [NSValue]() for cornerPoint in cornerPoints { - var x = Int(((cornerPoint.cgPointValue.x * scale) - (invisibleWidth / 2)) * UIScreen.main.scale) - var y = Int(((cornerPoint.cgPointValue.y * scale) - (invisibleHeight / 2)) * UIScreen.main.scale) + var x = Int(((cornerPoint.cgPointValue.x * scale) - (invisibleWidth / 2))) + var y = Int(((cornerPoint.cgPointValue.y * scale) - (invisibleHeight / 2))) if isPortrait { - x = Int((((imageHeight - cornerPoint.cgPointValue.y) * scale) - (invisibleWidth / 2)) * UIScreen.main.scale) - y = Int(((cornerPoint.cgPointValue.x * scale) - (invisibleHeight / 2)) * UIScreen.main.scale) + x = Int((((imageHeight - cornerPoint.cgPointValue.y) * scale) - (invisibleWidth / 2))) + y = Int(((cornerPoint.cgPointValue.x * scale) - (invisibleHeight / 2))) } let point = CGPoint(x: x, y: y) let value = NSValue(cgPoint: point) normalizedCornerPoints.append(value) } + // If the image is in portrait mode, the corner points need to be rotated if isPortrait { let lastNormalizedCornerPoints = normalizedCornerPoints.removeLast() normalizedCornerPoints.insert(lastNormalizedCornerPoints, at: 0) } - CAPLog.print("Normalized corner points (\(normalizedCornerPoints[0].cgPointValue.x), \(normalizedCornerPoints[0].cgPointValue.y)), (\(normalizedCornerPoints[1].cgPointValue.x), \(normalizedCornerPoints[1].cgPointValue.y)), (\(normalizedCornerPoints[2].cgPointValue.x), \(normalizedCornerPoints[2].cgPointValue.y)), (\(normalizedCornerPoints[3].cgPointValue.x), \(normalizedCornerPoints[3].cgPointValue.y))") + // Log normalized corner points + // CAPLog.print("Normalized corner points (\(normalizedCornerPoints[0].cgPointValue.x), \(normalizedCornerPoints[0].cgPointValue.y)), (\(normalizedCornerPoints[1].cgPointValue.x), \(normalizedCornerPoints[1].cgPointValue.y)), (\(normalizedCornerPoints[2].cgPointValue.x), \(normalizedCornerPoints[2].cgPointValue.y)), (\(normalizedCornerPoints[3].cgPointValue.x), \(normalizedCornerPoints[3].cgPointValue.y))") return normalizedCornerPoints } - public static func createBarcodeResultForBarcode(_ barcode: Barcode, imageSize: CGSize?, scale222: CGFloat = UIScreen.main.scale) -> JSObject { + public static func createBarcodeResultForBarcode(_ barcode: Barcode, imageSize: CGSize?) -> JSObject { var cornerPointsResult = [[Int]]() if let cornerPoints = barcode.cornerPoints, let imageSize = imageSize { - let normalizedCornerPoints = normalizeCornerPoints(cornerPoints: cornerPoints, imageSize: imageSize) + let screenSize = CGSize(width: UIScreen.main.bounds.width * UIScreen.main.scale, height: UIScreen.main.bounds.height * UIScreen.main.scale) + let normalizedCornerPoints = normalizeCornerPoints(cornerPoints: cornerPoints, imageSize: imageSize, screenSize: screenSize) for cornerPoint in normalizedCornerPoints { var value = [Int]() value.append(Int(cornerPoint.cgPointValue.x)) diff --git a/packages/barcode-scanning/ios/Plugin/BarcodeScannerView.swift b/packages/barcode-scanning/ios/Plugin/BarcodeScannerView.swift index f7852fe..00cee83 100644 --- a/packages/barcode-scanning/ios/Plugin/BarcodeScannerView.swift +++ b/packages/barcode-scanning/ios/Plugin/BarcodeScannerView.swift @@ -262,8 +262,9 @@ public protocol BarcodeScannerViewDelegate { private func filterBarcodesOutsideTheDetectionArea(_ barcodes: [Barcode], imageSize: CGSize?, detectionArea: CGRect) -> [Barcode] { return barcodes.filter { barcode in if let cornerPoints = barcode.cornerPoints, let imageSize = imageSize { + let screenSize = CGSize(width: UIScreen.main.bounds.width, height: UIScreen.main.bounds.height) let normalizedCornerPoints = BarcodeScannerHelper.normalizeCornerPoints(cornerPoints: cornerPoints, - imageSize: imageSize) + imageSize: imageSize, screenSize: screenSize) let topLeft = normalizedCornerPoints[0].cgPointValue let topRight = normalizedCornerPoints[1].cgPointValue