From 4ca54ae5ef517db0f6794e14ab29958eb3ad6731 Mon Sep 17 00:00:00 2001 From: Robin Genz Date: Fri, 16 Feb 2024 13:16:39 +0100 Subject: [PATCH] fix(barcode-scanning): incorrect coordinate calculation in landscape mode on iOS (#128) --- .changeset/cool-beds-cheat.md | 5 +++ .../ios/Plugin/BarcodeScanner.swift | 19 ++++----- .../ios/Plugin/BarcodeScannerHelper.swift | 42 +++++++++++++------ .../ios/Plugin/BarcodeScannerPlugin.swift | 10 ++--- .../ios/Plugin/BarcodeScannerView.swift | 27 ++++++------ 5 files changed, 64 insertions(+), 39 deletions(-) create mode 100644 .changeset/cool-beds-cheat.md diff --git a/.changeset/cool-beds-cheat.md b/.changeset/cool-beds-cheat.md new file mode 100644 index 0000000..d5c071f --- /dev/null +++ b/.changeset/cool-beds-cheat.md @@ -0,0 +1,5 @@ +--- +'@capacitor-mlkit/barcode-scanning': patch +--- + +fix(ios): incorrect coordinate calculation in landscape mode diff --git a/packages/barcode-scanning/ios/Plugin/BarcodeScanner.swift b/packages/barcode-scanning/ios/Plugin/BarcodeScanner.swift index 052c7a7..9de324c 100644 --- a/packages/barcode-scanning/ios/Plugin/BarcodeScanner.swift +++ b/packages/barcode-scanning/ios/Plugin/BarcodeScanner.swift @@ -14,7 +14,7 @@ typealias MLKitBarcodeScanner = MLKitBarcodeScanning.BarcodeScanner public let plugin: BarcodeScannerPlugin private var cameraView: BarcodeScannerView? - private var scanCompletionHandler: (([Barcode]?, String?) -> Void)? + private var scanCompletionHandler: (([Barcode]?, AVCaptureVideoOrientation?, String?) -> Void)? init(plugin: BarcodeScannerPlugin) { self.plugin = plugin @@ -70,7 +70,7 @@ typealias MLKitBarcodeScanner = MLKitBarcodeScanning.BarcodeScanner } } - @objc public func scan(settings: ScanSettings, completion: @escaping (([Barcode]?, String?) -> Void)) { + public func scan(settings: ScanSettings, completion: @escaping (([Barcode]?, AVCaptureVideoOrientation?, String?) -> Void)) { self.stopScan() guard let webView = self.plugin.webView else { @@ -86,7 +86,7 @@ typealias MLKitBarcodeScanner = MLKitBarcodeScanning.BarcodeScanner self.cameraView = cameraView } catch let error { CAPLog.print(error.localizedDescription, error) - completion(nil, error.localizedDescription) + completion(nil, nil, error.localizedDescription) return } } @@ -254,27 +254,26 @@ typealias MLKitBarcodeScanner = MLKitBarcodeScanning.BarcodeScanner webView.scrollView.backgroundColor = UIColor.white } - private func handleScannedBarcode(barcode: Barcode, imageSize: CGSize) { - plugin.notifyBarcodeScannedListener(barcode: barcode, imageSize: imageSize) + private func handleScannedBarcode(barcode: Barcode, imageSize: CGSize, videoOrientation: AVCaptureVideoOrientation?) { + plugin.notifyBarcodeScannedListener(barcode: barcode, imageSize: imageSize, videoOrientation: videoOrientation) } - } extension BarcodeScanner: BarcodeScannerViewDelegate { - public func onBarcodesDetected(barcodes: [Barcode], imageSize: CGSize) { + public func onBarcodesDetected(barcodes: [Barcode], imageSize: CGSize, videoOrientation: AVCaptureVideoOrientation?) { if let scanCompletionHandler = self.scanCompletionHandler { - scanCompletionHandler(barcodes, nil) + scanCompletionHandler(barcodes, videoOrientation, nil) self.stopScan() } else { for barcode in barcodes { - self.handleScannedBarcode(barcode: barcode, imageSize: imageSize) + self.handleScannedBarcode(barcode: barcode, imageSize: imageSize, videoOrientation: videoOrientation) } } } public func onCancel() { if let scanCompletionHandler = self.scanCompletionHandler { - scanCompletionHandler(nil, plugin.errorScanCanceled) + scanCompletionHandler(nil, nil, plugin.errorScanCanceled) } self.stopScan() } diff --git a/packages/barcode-scanning/ios/Plugin/BarcodeScannerHelper.swift b/packages/barcode-scanning/ios/Plugin/BarcodeScannerHelper.swift index 7b3334c..77ac9fe 100644 --- a/packages/barcode-scanning/ios/Plugin/BarcodeScannerHelper.swift +++ b/packages/barcode-scanning/ios/Plugin/BarcodeScannerHelper.swift @@ -4,11 +4,12 @@ import Foundation import Capacitor import MLKitBarcodeScanning +import AVFoundation // swiftlint:disable cyclomatic_complexity public class BarcodeScannerHelper { // swiftlint:disable identifier_name - public static func normalizeCornerPoints(cornerPoints: [NSValue], imageSize: CGSize, screenSize: CGSize) -> [NSValue] { + public static func normalizeCornerPoints(cornerPoints: [NSValue], imageSize: CGSize, screenSize: CGSize, videoOrientation: AVCaptureVideoOrientation?) -> [NSValue] { // Log corner points // CAPLog.print("Corner points (\(cornerPoints[0].cgPointValue.x), \(cornerPoints[0].cgPointValue.y)), (\(cornerPoints[1].cgPointValue.x), \(cornerPoints[1].cgPointValue.y)), (\(cornerPoints[2].cgPointValue.x), \(cornerPoints[2].cgPointValue.y)), (\(cornerPoints[3].cgPointValue.x), \(cornerPoints[3].cgPointValue.y))") let screenWidth = screenSize.width @@ -25,34 +26,51 @@ public class BarcodeScannerHelper { // Calculate the invisible area of the image let invisibleWidth = imageHeight * scale - screenWidth let invisibleHeight = imageWidth * scale - screenHeight - let isPortrait = UIDevice.current.orientation == .portrait || UIDevice.current.orientation == .portraitUpsideDown var normalizedCornerPoints = [NSValue]() for cornerPoint in cornerPoints { - var x = Int(((cornerPoint.cgPointValue.x * scale) - (invisibleWidth / 2))) - var y = Int(((cornerPoint.cgPointValue.y * scale) - (invisibleHeight / 2))) - if isPortrait { - x = Int((((imageHeight - cornerPoint.cgPointValue.y) * scale) - (invisibleWidth / 2))) - y = Int(((cornerPoint.cgPointValue.x * scale) - (invisibleHeight / 2))) + var x: CGFloat + var y: CGFloat + switch videoOrientation { + case .portrait, .portraitUpsideDown: + x = ((imageHeight - cornerPoint.cgPointValue.y) * scale) - (invisibleWidth / 2) + y = (cornerPoint.cgPointValue.x * scale) - (invisibleHeight / 2) + break + case .landscapeLeft: + x = ((imageHeight - cornerPoint.cgPointValue.x) * scale) - (invisibleWidth / 2) + y = ((imageWidth - cornerPoint.cgPointValue.y) * scale) - (invisibleHeight / 2) + break + default: + x = (cornerPoint.cgPointValue.x * scale) - (invisibleWidth / 2) + y = (cornerPoint.cgPointValue.y * scale) - (invisibleHeight / 2) + break } - let point = CGPoint(x: x, y: y) + let point = CGPoint(x: Int(x), y: Int(y)) let value = NSValue(cgPoint: point) normalizedCornerPoints.append(value) } - // If the image is in portrait mode, the corner points need to be rotated - if isPortrait { + // If the video orientation is not in landscape right mode, the corner points need to be rotated + switch videoOrientation { + case .portrait: let lastNormalizedCornerPoints = normalizedCornerPoints.removeLast() normalizedCornerPoints.insert(lastNormalizedCornerPoints, at: 0) + case .landscapeLeft: + var lastNormalizedCornerPoints = normalizedCornerPoints.removeLast() + normalizedCornerPoints.insert(lastNormalizedCornerPoints, at: 0) + lastNormalizedCornerPoints = normalizedCornerPoints.removeLast() + normalizedCornerPoints.insert(lastNormalizedCornerPoints, at: 0) + default: + break } // Log normalized corner points // CAPLog.print("Normalized corner points (\(normalizedCornerPoints[0].cgPointValue.x), \(normalizedCornerPoints[0].cgPointValue.y)), (\(normalizedCornerPoints[1].cgPointValue.x), \(normalizedCornerPoints[1].cgPointValue.y)), (\(normalizedCornerPoints[2].cgPointValue.x), \(normalizedCornerPoints[2].cgPointValue.y)), (\(normalizedCornerPoints[3].cgPointValue.x), \(normalizedCornerPoints[3].cgPointValue.y))") return normalizedCornerPoints } - public static func createBarcodeResultForBarcode(_ barcode: Barcode, imageSize: CGSize?) -> JSObject { + public static func createBarcodeResultForBarcode(_ barcode: Barcode, imageSize: CGSize?, videoOrientation: AVCaptureVideoOrientation?) -> JSObject { var cornerPointsResult = [[Int]]() if let cornerPoints = barcode.cornerPoints, let imageSize = imageSize { let screenSize = CGSize(width: UIScreen.main.bounds.width * UIScreen.main.scale, height: UIScreen.main.bounds.height * UIScreen.main.scale) - let normalizedCornerPoints = normalizeCornerPoints(cornerPoints: cornerPoints, imageSize: imageSize, screenSize: screenSize) + let normalizedCornerPoints = normalizeCornerPoints(cornerPoints: cornerPoints, imageSize: imageSize, screenSize: screenSize, videoOrientation: videoOrientation) for cornerPoint in normalizedCornerPoints { var value = [Int]() value.append(Int(cornerPoint.cgPointValue.x)) diff --git a/packages/barcode-scanning/ios/Plugin/BarcodeScannerPlugin.swift b/packages/barcode-scanning/ios/Plugin/BarcodeScannerPlugin.swift index 5d81472..526034d 100644 --- a/packages/barcode-scanning/ios/Plugin/BarcodeScannerPlugin.swift +++ b/packages/barcode-scanning/ios/Plugin/BarcodeScannerPlugin.swift @@ -84,7 +84,7 @@ public class BarcodeScannerPlugin: CAPPlugin { } var barcodeResults = JSArray() for barcode in barcodes ?? [] { - barcodeResults.append(BarcodeScannerHelper.createBarcodeResultForBarcode(barcode, imageSize: nil)) + barcodeResults.append(BarcodeScannerHelper.createBarcodeResultForBarcode(barcode, imageSize: nil, videoOrientation: nil)) } call.resolve([ "barcodes": barcodeResults @@ -108,14 +108,14 @@ public class BarcodeScannerPlugin: CAPPlugin { call.reject(error.localizedDescription) return } - self.implementation?.scan(settings: settings, completion: { barcodes, errorMessage in + self.implementation?.scan(settings: settings, completion: { barcodes, videoOrientation, errorMessage in if let errorMessage = errorMessage { call.reject(errorMessage) return } var barcodeResults = JSArray() for barcode in barcodes ?? [] { - barcodeResults.append(BarcodeScannerHelper.createBarcodeResultForBarcode(barcode, imageSize: nil)) + barcodeResults.append(BarcodeScannerHelper.createBarcodeResultForBarcode(barcode, imageSize: nil, videoOrientation: videoOrientation)) } call.resolve([ "barcodes": barcodeResults @@ -234,9 +234,9 @@ public class BarcodeScannerPlugin: CAPPlugin { } } - @objc func notifyBarcodeScannedListener(barcode: Barcode, imageSize: CGSize) { + func notifyBarcodeScannedListener(barcode: Barcode, imageSize: CGSize, videoOrientation: AVCaptureVideoOrientation?) { var result = JSObject() - result["barcode"] = BarcodeScannerHelper.createBarcodeResultForBarcode(barcode, imageSize: imageSize) + result["barcode"] = BarcodeScannerHelper.createBarcodeResultForBarcode(barcode, imageSize: imageSize, videoOrientation: videoOrientation) notifyListeners(barcodeScannedEvent, data: result) } } diff --git a/packages/barcode-scanning/ios/Plugin/BarcodeScannerView.swift b/packages/barcode-scanning/ios/Plugin/BarcodeScannerView.swift index 00cee83..b24cef9 100644 --- a/packages/barcode-scanning/ios/Plugin/BarcodeScannerView.swift +++ b/packages/barcode-scanning/ios/Plugin/BarcodeScannerView.swift @@ -10,7 +10,7 @@ import MLKitVision // swiftlint:disable class_delegate_protocol public protocol BarcodeScannerViewDelegate { - func onBarcodesDetected(barcodes: [Barcode], imageSize: CGSize) + func onBarcodesDetected(barcodes: [Barcode], imageSize: CGSize, videoOrientation: AVCaptureVideoOrientation?) func onCancel() func onTorchToggle() } @@ -23,6 +23,7 @@ public protocol BarcodeScannerViewDelegate { private var captureSession: AVCaptureSession? private var barcodeScannerInstance: MLKitBarcodeScanner? private var videoPreviewLayer: AVCaptureVideoPreviewLayer? + private var videoOrientation: AVCaptureVideoOrientation? private var cancelButton: UIButton? private var torchButton: UIButton? private var detectionAreaView: UIView? @@ -107,7 +108,9 @@ public protocol BarcodeScannerViewDelegate { } if let interfaceOrientation = UIApplication.shared.windows.first(where: { $0.isKeyWindow })?.windowScene?.interfaceOrientation { - self.videoPreviewLayer?.connection?.videoOrientation = interfaceOrientationToVideoOrientation(interfaceOrientation) + let videoOrientation = interfaceOrientationToVideoOrientation(interfaceOrientation) + self.videoPreviewLayer?.connection?.videoOrientation = videoOrientation + self.videoOrientation = videoOrientation } } @@ -117,7 +120,7 @@ public protocol BarcodeScannerViewDelegate { } let visionImage = VisionImage(buffer: sampleBuffer) visionImage.orientation = imageOrientation( - deviceOrientation: UIDevice.current.orientation, + videoOrientation: self.videoOrientation, cameraPosition: AVCaptureDevice.Position.back) var barcodes: [Barcode] = [] do { @@ -137,12 +140,12 @@ public protocol BarcodeScannerViewDelegate { let imageSize = CGSize(width: imageWidth, height: imageHeight) if let detectionAreaViewFrame = self.detectionAreaViewFrame { barcodes = filterBarcodesOutsideTheDetectionArea(barcodes, imageSize: imageSize, - detectionArea: detectionAreaViewFrame) + detectionArea: detectionAreaViewFrame, videoOrientation: videoOrientation) if barcodes.isEmpty { return } } - onBarcodesDetected(barcodes: barcodes, imageSize: imageSize) + onBarcodesDetected(barcodes: barcodes, imageSize: imageSize, videoOrientation: videoOrientation) } private func interfaceOrientationToVideoOrientation(_ orientation: UIInterfaceOrientation) -> AVCaptureVideoOrientation { @@ -161,10 +164,10 @@ public protocol BarcodeScannerViewDelegate { } private func imageOrientation( - deviceOrientation: UIDeviceOrientation, + videoOrientation: AVCaptureVideoOrientation?, cameraPosition: AVCaptureDevice.Position ) -> UIImage.Orientation { - switch deviceOrientation { + switch videoOrientation { case .portrait: return cameraPosition == .front ? .leftMirrored : .right case .landscapeLeft: @@ -173,7 +176,7 @@ public protocol BarcodeScannerViewDelegate { return cameraPosition == .front ? .rightMirrored : .left case .landscapeRight: return cameraPosition == .front ? .upMirrored : .down - case .faceDown, .faceUp, .unknown: + case .none: return .up @unknown default: return .up @@ -259,12 +262,12 @@ public protocol BarcodeScannerViewDelegate { self.detectionAreaViewFrame = nil } - private func filterBarcodesOutsideTheDetectionArea(_ barcodes: [Barcode], imageSize: CGSize?, detectionArea: CGRect) -> [Barcode] { + private func filterBarcodesOutsideTheDetectionArea(_ barcodes: [Barcode], imageSize: CGSize?, detectionArea: CGRect, videoOrientation: AVCaptureVideoOrientation?) -> [Barcode] { return barcodes.filter { barcode in if let cornerPoints = barcode.cornerPoints, let imageSize = imageSize { let screenSize = CGSize(width: UIScreen.main.bounds.width, height: UIScreen.main.bounds.height) let normalizedCornerPoints = BarcodeScannerHelper.normalizeCornerPoints(cornerPoints: cornerPoints, - imageSize: imageSize, screenSize: screenSize) + imageSize: imageSize, screenSize: screenSize, videoOrientation: videoOrientation) let topLeft = normalizedCornerPoints[0].cgPointValue let topRight = normalizedCornerPoints[1].cgPointValue @@ -288,8 +291,8 @@ public protocol BarcodeScannerViewDelegate { } } - @objc private func onBarcodesDetected(barcodes: [Barcode], imageSize: CGSize) { - self.delegate?.onBarcodesDetected(barcodes: barcodes, imageSize: imageSize) + private func onBarcodesDetected(barcodes: [Barcode], imageSize: CGSize, videoOrientation: AVCaptureVideoOrientation?) { + self.delegate?.onBarcodesDetected(barcodes: barcodes, imageSize: imageSize, videoOrientation: videoOrientation) } @objc private func onCancel() {