Skip to content

Commit

Permalink
fix(barcode-scanning): incorrect coordinate calculation in landscape …
Browse files Browse the repository at this point in the history 
…mode on iOS
  • Loading branch information
@robingenz
robingenz committed Feb 16, 2024
1 parent c297f7e commit 4330451
Show file tree
Hide file tree
Showing 5 changed files with 64 additions and 39 deletions.
5 changes: 5 additions & 0 deletions .changeset/cool-beds-cheat.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'@capacitor-mlkit/barcode-scanning': patch
---

fix(ios): incorrect coordinate calculation in landscape mode
19 changes: 9 additions & 10 deletions packages/barcode-scanning/ios/Plugin/BarcodeScanner.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ typealias MLKitBarcodeScanner = MLKitBarcodeScanning.BarcodeScanner
public let plugin: BarcodeScannerPlugin

private var cameraView: BarcodeScannerView?
private var scanCompletionHandler: (([Barcode]?, String?) -> Void)?
private var scanCompletionHandler: (([Barcode]?, AVCaptureVideoOrientation?, String?) -> Void)?

init(plugin: BarcodeScannerPlugin) {
self.plugin = plugin
Expand Down Expand Up @@ -70,7 +70,7 @@ typealias MLKitBarcodeScanner = MLKitBarcodeScanning.BarcodeScanner
}
}

@objc public func scan(settings: ScanSettings, completion: @escaping (([Barcode]?, String?) -> Void)) {
public func scan(settings: ScanSettings, completion: @escaping (([Barcode]?, AVCaptureVideoOrientation?, String?) -> Void)) {
self.stopScan()

guard let webView = self.plugin.webView else {
Expand All @@ -86,7 +86,7 @@ typealias MLKitBarcodeScanner = MLKitBarcodeScanning.BarcodeScanner
self.cameraView = cameraView
} catch let error {
CAPLog.print(error.localizedDescription, error)
completion(nil, error.localizedDescription)
completion(nil, nil, error.localizedDescription)
return
}
}
Expand Down Expand Up @@ -254,27 +254,26 @@ typealias MLKitBarcodeScanner = MLKitBarcodeScanning.BarcodeScanner
webView.scrollView.backgroundColor = UIColor.white
}

private func handleScannedBarcode(barcode: Barcode, imageSize: CGSize) {
plugin.notifyBarcodeScannedListener(barcode: barcode, imageSize: imageSize)
private func handleScannedBarcode(barcode: Barcode, imageSize: CGSize, videoOrientation: AVCaptureVideoOrientation?) {
plugin.notifyBarcodeScannedListener(barcode: barcode, imageSize: imageSize, videoOrientation: videoOrientation)
}

}

extension BarcodeScanner: BarcodeScannerViewDelegate {
public func onBarcodesDetected(barcodes: [Barcode], imageSize: CGSize) {
public func onBarcodesDetected(barcodes: [Barcode], imageSize: CGSize, videoOrientation: AVCaptureVideoOrientation?) {
if let scanCompletionHandler = self.scanCompletionHandler {
scanCompletionHandler(barcodes, nil)
scanCompletionHandler(barcodes, videoOrientation, nil)
self.stopScan()
} else {
for barcode in barcodes {
self.handleScannedBarcode(barcode: barcode, imageSize: imageSize)
self.handleScannedBarcode(barcode: barcode, imageSize: imageSize, videoOrientation: videoOrientation)
}
}
}

public func onCancel() {
if let scanCompletionHandler = self.scanCompletionHandler {
scanCompletionHandler(nil, plugin.errorScanCanceled)
scanCompletionHandler(nil, nil, plugin.errorScanCanceled)
}
self.stopScan()
}
Expand Down
42 changes: 30 additions & 12 deletions packages/barcode-scanning/ios/Plugin/BarcodeScannerHelper.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,12 @@
import Foundation
import Capacitor
import MLKitBarcodeScanning
import AVFoundation

// swiftlint:disable cyclomatic_complexity
public class BarcodeScannerHelper {
// swiftlint:disable identifier_name
public static func normalizeCornerPoints(cornerPoints: [NSValue], imageSize: CGSize, screenSize: CGSize) -> [NSValue] {
public static func normalizeCornerPoints(cornerPoints: [NSValue], imageSize: CGSize, screenSize: CGSize, videoOrientation: AVCaptureVideoOrientation?) -> [NSValue] {
// Log corner points
// CAPLog.print("Corner points (\(cornerPoints[0].cgPointValue.x), \(cornerPoints[0].cgPointValue.y)), (\(cornerPoints[1].cgPointValue.x), \(cornerPoints[1].cgPointValue.y)), (\(cornerPoints[2].cgPointValue.x), \(cornerPoints[2].cgPointValue.y)), (\(cornerPoints[3].cgPointValue.x), \(cornerPoints[3].cgPointValue.y))")
let screenWidth = screenSize.width
Expand All @@ -25,34 +26,51 @@ public class BarcodeScannerHelper {
// Calculate the invisible area of the image
let invisibleWidth = imageHeight * scale - screenWidth
let invisibleHeight = imageWidth * scale - screenHeight
let isPortrait = UIDevice.current.orientation == .portrait || UIDevice.current.orientation == .portraitUpsideDown
var normalizedCornerPoints = [NSValue]()
for cornerPoint in cornerPoints {
var x = Int(((cornerPoint.cgPointValue.x * scale) - (invisibleWidth / 2)))
var y = Int(((cornerPoint.cgPointValue.y * scale) - (invisibleHeight / 2)))
if isPortrait {
x = Int((((imageHeight - cornerPoint.cgPointValue.y) * scale) - (invisibleWidth / 2)))
y = Int(((cornerPoint.cgPointValue.x * scale) - (invisibleHeight / 2)))
var x: CGFloat
var y: CGFloat
switch videoOrientation {
case .portrait, .portraitUpsideDown:
x = ((imageHeight - cornerPoint.cgPointValue.y) * scale) - (invisibleWidth / 2)
y = (cornerPoint.cgPointValue.x * scale) - (invisibleHeight / 2)
break
case .landscapeLeft:
x = ((imageHeight - cornerPoint.cgPointValue.x) * scale) - (invisibleWidth / 2)
y = ((imageWidth - cornerPoint.cgPointValue.y) * scale) - (invisibleHeight / 2)
break
default:
x = (cornerPoint.cgPointValue.x * scale) - (invisibleWidth / 2)
y = (cornerPoint.cgPointValue.y * scale) - (invisibleHeight / 2)
break
}
let point = CGPoint(x: x, y: y)
let point = CGPoint(x: Int(x), y: Int(y))
let value = NSValue(cgPoint: point)
normalizedCornerPoints.append(value)
}
// If the image is in portrait mode, the corner points need to be rotated
if isPortrait {
// If the video orientation is not in landscape right mode, the corner points need to be rotated
switch videoOrientation {
case .portrait:
let lastNormalizedCornerPoints = normalizedCornerPoints.removeLast()
normalizedCornerPoints.insert(lastNormalizedCornerPoints, at: 0)
case .landscapeLeft:
var lastNormalizedCornerPoints = normalizedCornerPoints.removeLast()
normalizedCornerPoints.insert(lastNormalizedCornerPoints, at: 0)
lastNormalizedCornerPoints = normalizedCornerPoints.removeLast()
normalizedCornerPoints.insert(lastNormalizedCornerPoints, at: 0)
default:
break
}
// Log normalized corner points
// CAPLog.print("Normalized corner points (\(normalizedCornerPoints[0].cgPointValue.x), \(normalizedCornerPoints[0].cgPointValue.y)), (\(normalizedCornerPoints[1].cgPointValue.x), \(normalizedCornerPoints[1].cgPointValue.y)), (\(normalizedCornerPoints[2].cgPointValue.x), \(normalizedCornerPoints[2].cgPointValue.y)), (\(normalizedCornerPoints[3].cgPointValue.x), \(normalizedCornerPoints[3].cgPointValue.y))")
return normalizedCornerPoints
}

public static func createBarcodeResultForBarcode(_ barcode: Barcode, imageSize: CGSize?) -> JSObject {
public static func createBarcodeResultForBarcode(_ barcode: Barcode, imageSize: CGSize?, videoOrientation: AVCaptureVideoOrientation?) -> JSObject {
var cornerPointsResult = [[Int]]()
if let cornerPoints = barcode.cornerPoints, let imageSize = imageSize {
let screenSize = CGSize(width: UIScreen.main.bounds.width * UIScreen.main.scale, height: UIScreen.main.bounds.height * UIScreen.main.scale)
let normalizedCornerPoints = normalizeCornerPoints(cornerPoints: cornerPoints, imageSize: imageSize, screenSize: screenSize)
let normalizedCornerPoints = normalizeCornerPoints(cornerPoints: cornerPoints, imageSize: imageSize, screenSize: screenSize, videoOrientation: videoOrientation)
for cornerPoint in normalizedCornerPoints {
var value = [Int]()
value.append(Int(cornerPoint.cgPointValue.x))
Expand Down
10 changes: 5 additions & 5 deletions packages/barcode-scanning/ios/Plugin/BarcodeScannerPlugin.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ public class BarcodeScannerPlugin: CAPPlugin {
}
var barcodeResults = JSArray()
for barcode in barcodes ?? [] {
barcodeResults.append(BarcodeScannerHelper.createBarcodeResultForBarcode(barcode, imageSize: nil))
barcodeResults.append(BarcodeScannerHelper.createBarcodeResultForBarcode(barcode, imageSize: nil, videoOrientation: nil))
}
call.resolve([
"barcodes": barcodeResults
Expand All @@ -108,14 +108,14 @@ public class BarcodeScannerPlugin: CAPPlugin {
call.reject(error.localizedDescription)
return
}
self.implementation?.scan(settings: settings, completion: { barcodes, errorMessage in
self.implementation?.scan(settings: settings, completion: { barcodes, videoOrientation, errorMessage in
if let errorMessage = errorMessage {
call.reject(errorMessage)
return
}
var barcodeResults = JSArray()
for barcode in barcodes ?? [] {
barcodeResults.append(BarcodeScannerHelper.createBarcodeResultForBarcode(barcode, imageSize: nil))
barcodeResults.append(BarcodeScannerHelper.createBarcodeResultForBarcode(barcode, imageSize: nil, videoOrientation: videoOrientation))
}
call.resolve([
"barcodes": barcodeResults
Expand Down Expand Up @@ -234,9 +234,9 @@ public class BarcodeScannerPlugin: CAPPlugin {
}
}

@objc func notifyBarcodeScannedListener(barcode: Barcode, imageSize: CGSize) {
func notifyBarcodeScannedListener(barcode: Barcode, imageSize: CGSize, videoOrientation: AVCaptureVideoOrientation?) {
var result = JSObject()
result["barcode"] = BarcodeScannerHelper.createBarcodeResultForBarcode(barcode, imageSize: imageSize)
result["barcode"] = BarcodeScannerHelper.createBarcodeResultForBarcode(barcode, imageSize: imageSize, videoOrientation: videoOrientation)
notifyListeners(barcodeScannedEvent, data: result)
}
}
Expand Down
27 changes: 15 additions & 12 deletions packages/barcode-scanning/ios/Plugin/BarcodeScannerView.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import MLKitVision

// swiftlint:disable class_delegate_protocol
public protocol BarcodeScannerViewDelegate {
func onBarcodesDetected(barcodes: [Barcode], imageSize: CGSize)
func onBarcodesDetected(barcodes: [Barcode], imageSize: CGSize, videoOrientation: AVCaptureVideoOrientation?)
func onCancel()
func onTorchToggle()
}
Expand All @@ -23,6 +23,7 @@ public protocol BarcodeScannerViewDelegate {
private var captureSession: AVCaptureSession?
private var barcodeScannerInstance: MLKitBarcodeScanner?
private var videoPreviewLayer: AVCaptureVideoPreviewLayer?
private var videoOrientation: AVCaptureVideoOrientation?
private var cancelButton: UIButton?
private var torchButton: UIButton?
private var detectionAreaView: UIView?
Expand Down Expand Up @@ -107,7 +108,9 @@ public protocol BarcodeScannerViewDelegate {
}

if let interfaceOrientation = UIApplication.shared.windows.first(where: { $0.isKeyWindow })?.windowScene?.interfaceOrientation {
self.videoPreviewLayer?.connection?.videoOrientation = interfaceOrientationToVideoOrientation(interfaceOrientation)
let videoOrientation = interfaceOrientationToVideoOrientation(interfaceOrientation)
self.videoPreviewLayer?.connection?.videoOrientation = videoOrientation
self.videoOrientation = videoOrientation
}
}

Expand All @@ -117,7 +120,7 @@ public protocol BarcodeScannerViewDelegate {
}
let visionImage = VisionImage(buffer: sampleBuffer)
visionImage.orientation = imageOrientation(
deviceOrientation: UIDevice.current.orientation,
videoOrientation: self.videoOrientation,
cameraPosition: AVCaptureDevice.Position.back)
var barcodes: [Barcode] = []
do {
Expand All @@ -137,12 +140,12 @@ public protocol BarcodeScannerViewDelegate {
let imageSize = CGSize(width: imageWidth, height: imageHeight)
if let detectionAreaViewFrame = self.detectionAreaViewFrame {
barcodes = filterBarcodesOutsideTheDetectionArea(barcodes, imageSize: imageSize,
detectionArea: detectionAreaViewFrame)
detectionArea: detectionAreaViewFrame, videoOrientation: videoOrientation)
if barcodes.isEmpty {
return
}
}
onBarcodesDetected(barcodes: barcodes, imageSize: imageSize)
onBarcodesDetected(barcodes: barcodes, imageSize: imageSize, videoOrientation: videoOrientation)
}

private func interfaceOrientationToVideoOrientation(_ orientation: UIInterfaceOrientation) -> AVCaptureVideoOrientation {
Expand All @@ -161,10 +164,10 @@ public protocol BarcodeScannerViewDelegate {
}

private func imageOrientation(
deviceOrientation: UIDeviceOrientation,
videoOrientation: AVCaptureVideoOrientation?,
cameraPosition: AVCaptureDevice.Position
) -> UIImage.Orientation {
switch deviceOrientation {
switch videoOrientation {
case .portrait:
return cameraPosition == .front ? .leftMirrored : .right
case .landscapeLeft:
Expand All @@ -173,7 +176,7 @@ public protocol BarcodeScannerViewDelegate {
return cameraPosition == .front ? .rightMirrored : .left
case .landscapeRight:
return cameraPosition == .front ? .upMirrored : .down
case .faceDown, .faceUp, .unknown:
case .none:
return .up
@unknown default:
return .up
Expand Down Expand Up @@ -259,12 +262,12 @@ public protocol BarcodeScannerViewDelegate {
self.detectionAreaViewFrame = nil
}

private func filterBarcodesOutsideTheDetectionArea(_ barcodes: [Barcode], imageSize: CGSize?, detectionArea: CGRect) -> [Barcode] {
private func filterBarcodesOutsideTheDetectionArea(_ barcodes: [Barcode], imageSize: CGSize?, detectionArea: CGRect, videoOrientation: AVCaptureVideoOrientation?) -> [Barcode] {
return barcodes.filter { barcode in
if let cornerPoints = barcode.cornerPoints, let imageSize = imageSize {
let screenSize = CGSize(width: UIScreen.main.bounds.width, height: UIScreen.main.bounds.height)
let normalizedCornerPoints = BarcodeScannerHelper.normalizeCornerPoints(cornerPoints: cornerPoints,
imageSize: imageSize, screenSize: screenSize)
imageSize: imageSize, screenSize: screenSize, videoOrientation: videoOrientation)

let topLeft = normalizedCornerPoints[0].cgPointValue
let topRight = normalizedCornerPoints[1].cgPointValue
Expand All @@ -288,8 +291,8 @@ public protocol BarcodeScannerViewDelegate {
}
}

@objc private func onBarcodesDetected(barcodes: [Barcode], imageSize: CGSize) {
self.delegate?.onBarcodesDetected(barcodes: barcodes, imageSize: imageSize)
private func onBarcodesDetected(barcodes: [Barcode], imageSize: CGSize, videoOrientation: AVCaptureVideoOrientation?) {
self.delegate?.onBarcodesDetected(barcodes: barcodes, imageSize: imageSize, videoOrientation: videoOrientation)
}

@objc private func onCancel() {
Expand Down

0 comments on commit 4330451

Please sign in to comment.