Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

#115 Device orientation for isFlat #116

Closed
wants to merge 7 commits into from
Closed

#115 Device orientation for isFlat #116

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
7885654
Add screen size to barcode result so that you can map it however you …
CmdrDats Dec 27, 2023
7884b53
commit changeset
CmdrDats Dec 27, 2023
d53681b
small changelog tweak
CmdrDats Dec 27, 2023
bafb2fb
Redact screenSize - window.devicePixelRatio is the correct method for…
CmdrDats Dec 28, 2023
8a355b5
Clean up PR a touch
CmdrDats Dec 28, 2023
2526326
spurious whitespace
CmdrDats Dec 28, 2023
0b8ef13
To get the `isPortrait`, we need to sift it through from source.
CmdrDats Dec 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/two-steaks-impress.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'@capacitor-mlkit/barcode-scanning': patch
---

Tweak the ios orientation to default 'portrait' when it reports 'flat'
18 changes: 9 additions & 9 deletions packages/barcode-scanning/ios/Plugin/BarcodeScanner.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ typealias MLKitBarcodeScanner = MLKitBarcodeScanning.BarcodeScanner
public let plugin: BarcodeScannerPlugin

private var cameraView: BarcodeScannerView?
private var scanCompletionHandler: (([Barcode]?, String?) -> Void)?
private var scanCompletionHandler: (([Barcode]?, Bool, String?) -> Void)?

init(plugin: BarcodeScannerPlugin) {
self.plugin = plugin
Expand Down Expand Up @@ -74,7 +74,7 @@ typealias MLKitBarcodeScanner = MLKitBarcodeScanning.BarcodeScanner
}
}

@objc public func scan(settings: ScanSettings, completion: @escaping (([Barcode]?, String?) -> Void)) {
@objc public func scan(settings: ScanSettings, completion: @escaping (([Barcode]?, Bool, String?) -> Void)) {
self.stopScan()

guard let webView = self.plugin.webView else {
Expand All @@ -90,7 +90,7 @@ typealias MLKitBarcodeScanner = MLKitBarcodeScanning.BarcodeScanner
self.cameraView = cameraView
} catch let error {
CAPLog.print(error.localizedDescription, error)
completion(nil, error.localizedDescription)
completion(nil, false, error.localizedDescription)
return
}
}
Expand Down Expand Up @@ -258,27 +258,27 @@ typealias MLKitBarcodeScanner = MLKitBarcodeScanning.BarcodeScanner
webView.scrollView.backgroundColor = UIColor.white
}

private func handleScannedBarcode(barcode: Barcode, imageSize: CGSize) {
plugin.notifyBarcodeScannedListener(barcode: barcode, imageSize: imageSize)
private func handleScannedBarcode(barcode: Barcode, imageSize: CGSize, isPortrait: Bool) {
plugin.notifyBarcodeScannedListener(barcode: barcode, imageSize: imageSize, isPortrait: isPortrait)
}

}

extension BarcodeScanner: BarcodeScannerViewDelegate {
public func onBarcodesDetected(barcodes: [Barcode], imageSize: CGSize) {
public func onBarcodesDetected(barcodes: [Barcode], imageSize: CGSize, isPortrait: Bool) {
if let scanCompletionHandler = self.scanCompletionHandler {
scanCompletionHandler(barcodes, nil)
scanCompletionHandler(barcodes, isPortrait, nil)
self.stopScan()
} else {
for barcode in barcodes {
self.handleScannedBarcode(barcode: barcode, imageSize: imageSize)
self.handleScannedBarcode(barcode: barcode, imageSize: imageSize, isPortrait: isPortrait)
}
}
}

public func onCancel() {
if let scanCompletionHandler = self.scanCompletionHandler {
scanCompletionHandler(nil, plugin.errorScanCanceled)
scanCompletionHandler(nil, false, plugin.errorScanCanceled)
}
self.stopScan()
}
Expand Down
15 changes: 9 additions & 6 deletions packages/barcode-scanning/ios/Plugin/BarcodeScannerHelper.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,24 +2,27 @@
* Copyright (c) 2023 Robin Genz
*/
import Foundation
import AVFoundation
import Capacitor
import MLKitBarcodeScanning

// swiftlint:disable cyclomatic_complexity
public class BarcodeScannerHelper {
// swiftlint:disable identifier_name
public static func normalizeCornerPoints(cornerPoints: [NSValue], imageSize: CGSize, scale: CGFloat = UIScreen.main.scale) -> [NSValue] {
public static func normalizeCornerPoints(cornerPoints: [NSValue], imageSize: CGSize, scale: CGFloat = UIScreen.main.scale, isPortrait: Bool) -> [NSValue] {
let screenSize: CGRect = UIScreen.main.bounds
let imageWidth = imageSize.width
let imageHeight = imageSize.height
let isPortrait = UIDevice.current.orientation == .portrait || UIDevice.current.orientation == .portraitUpsideDown

var normalizedCornerPoints = [NSValue]()
for cornerPoint in cornerPoints {
var x = Int((cornerPoint.cgPointValue.x / CGFloat(imageWidth)) * screenSize.width * scale)
var y = Int((cornerPoint.cgPointValue.y / CGFloat(imageHeight)) * screenSize.height * scale)
var x = 0, y = 0
if isPortrait {
x = Int((1 - (cornerPoint.cgPointValue.y / CGFloat(imageHeight))) * screenSize.width * scale)
y = Int((cornerPoint.cgPointValue.x / CGFloat(imageWidth)) * screenSize.height * scale)
} else {
x = Int((1 - (cornerPoint.cgPointValue.x / CGFloat(imageWidth))) * screenSize.width * scale)
y = Int((1 - (cornerPoint.cgPointValue.y / CGFloat(imageHeight))) * screenSize.height * scale)
}
let point = CGPoint(x: x, y: y)
let value = NSValue(cgPoint: point)
Expand All @@ -28,10 +31,10 @@ public class BarcodeScannerHelper {
return normalizedCornerPoints
}

public static func createBarcodeResultForBarcode(_ barcode: Barcode, imageSize: CGSize?, scale: CGFloat = UIScreen.main.scale) -> JSObject {
public static func createBarcodeResultForBarcode(_ barcode: Barcode, imageSize: CGSize?, scale: CGFloat = UIScreen.main.scale, isPortrait: Bool) -> JSObject {
var cornerPointsResult = [[Int]]()
if let cornerPoints = barcode.cornerPoints, let imageSize = imageSize {
let normalizedCornerPoints = normalizeCornerPoints(cornerPoints: cornerPoints, imageSize: imageSize, scale: scale)
let normalizedCornerPoints = normalizeCornerPoints(cornerPoints: cornerPoints, imageSize: imageSize, scale: scale, isPortrait: isPortrait)
for cornerPoint in normalizedCornerPoints {
var value = [Int]()
value.append(Int(cornerPoint.cgPointValue.x))
Expand Down
10 changes: 5 additions & 5 deletions packages/barcode-scanning/ios/Plugin/BarcodeScannerPlugin.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ public class BarcodeScannerPlugin: CAPPlugin {
}
var barcodeResults = JSArray()
for barcode in barcodes ?? [] {
barcodeResults.append(BarcodeScannerHelper.createBarcodeResultForBarcode(barcode, imageSize: nil))
barcodeResults.append(BarcodeScannerHelper.createBarcodeResultForBarcode(barcode, imageSize: nil, isPortrait: UIDevice.current.orientation.isPortrait))
}
call.resolve([
"barcodes": barcodeResults
Expand All @@ -108,14 +108,14 @@ public class BarcodeScannerPlugin: CAPPlugin {
call.reject(error.localizedDescription)
return
}
self.implementation?.scan(settings: settings, completion: { barcodes, errorMessage in
self.implementation?.scan(settings: settings, completion: { barcodes, isPortrait, errorMessage in
if let errorMessage = errorMessage {
call.reject(errorMessage)
return
}
var barcodeResults = JSArray()
for barcode in barcodes ?? [] {
barcodeResults.append(BarcodeScannerHelper.createBarcodeResultForBarcode(barcode, imageSize: nil, scale: 1))
barcodeResults.append(BarcodeScannerHelper.createBarcodeResultForBarcode(barcode, imageSize: nil, scale: 1, isPortrait: isPortrait))
}
call.resolve([
"barcodes": barcodeResults
Expand Down Expand Up @@ -234,9 +234,9 @@ public class BarcodeScannerPlugin: CAPPlugin {
}
}

@objc func notifyBarcodeScannedListener(barcode: Barcode, imageSize: CGSize) {
@objc func notifyBarcodeScannedListener(barcode: Barcode, imageSize: CGSize, isPortrait: Bool) {
var result = JSObject()
result["barcode"] = BarcodeScannerHelper.createBarcodeResultForBarcode(barcode, imageSize: imageSize)
result["barcode"] = BarcodeScannerHelper.createBarcodeResultForBarcode(barcode, imageSize: imageSize, isPortrait: isPortrait)
notifyListeners(barcodeScannedEvent, data: result)
}
}
Expand Down
27 changes: 17 additions & 10 deletions packages/barcode-scanning/ios/Plugin/BarcodeScannerView.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import MLKitVision

// swiftlint:disable class_delegate_protocol
public protocol BarcodeScannerViewDelegate {
func onBarcodesDetected(barcodes: [Barcode], imageSize: CGSize)
func onBarcodesDetected(barcodes: [Barcode], imageSize: CGSize, isPortrait: Bool)
func onCancel()
func onTorchToggle()
}
Expand All @@ -23,6 +23,7 @@ public protocol BarcodeScannerViewDelegate {
private var captureSession: AVCaptureSession?
private var barcodeScannerInstance: MLKitBarcodeScanner?
private var videoPreviewLayer: AVCaptureVideoPreviewLayer?
private var videoOrientation: AVCaptureVideoOrientation?
private var cancelButton: UIButton?
private var torchButton: UIButton?
private var detectionAreaView: UIView?
Expand Down Expand Up @@ -107,17 +108,23 @@ public protocol BarcodeScannerViewDelegate {
}

if let interfaceOrientation = UIApplication.shared.windows.first(where: { $0.isKeyWindow })?.windowScene?.interfaceOrientation {
self.videoPreviewLayer?.connection?.videoOrientation = interfaceOrientationToVideoOrientation(interfaceOrientation)
let orientation = interfaceOrientationToVideoOrientation(interfaceOrientation)
videoOrientation = orientation
self.videoPreviewLayer?.connection?.videoOrientation = orientation
}
}

public func isPortraitAV() -> Bool {
return videoOrientation == .portrait || videoOrientation == .portraitUpsideDown || (videoOrientation == nil && UIDevice.current.orientation.isPortrait)
}

public func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
guard let barcodeScannerInstance = self.barcodeScannerInstance else {
return
}
let visionImage = VisionImage(buffer: sampleBuffer)
visionImage.orientation = imageOrientation(
deviceOrientation: UIDevice.current.orientation,
videoOrientation: videoOrientation,
cameraPosition: AVCaptureDevice.Position.back)
var barcodes: [Barcode] = []
do {
Expand All @@ -142,7 +149,7 @@ public protocol BarcodeScannerViewDelegate {
return
}
}
onBarcodesDetected(barcodes: barcodes, imageSize: imageSize)
onBarcodesDetected(barcodes: barcodes, imageSize: imageSize, isPortrait: isPortraitAV())
}

private func interfaceOrientationToVideoOrientation(_ orientation: UIInterfaceOrientation) -> AVCaptureVideoOrientation {
Expand All @@ -161,10 +168,10 @@ public protocol BarcodeScannerViewDelegate {
}

private func imageOrientation(
deviceOrientation: UIDeviceOrientation,
videoOrientation: AVCaptureVideoOrientation?,
cameraPosition: AVCaptureDevice.Position
) -> UIImage.Orientation {
switch deviceOrientation {
switch videoOrientation {
case .portrait:
return cameraPosition == .front ? .leftMirrored : .right
case .landscapeLeft:
Expand All @@ -173,7 +180,7 @@ public protocol BarcodeScannerViewDelegate {
return cameraPosition == .front ? .rightMirrored : .left
case .landscapeRight:
return cameraPosition == .front ? .upMirrored : .down
case .faceDown, .faceUp, .unknown:
case .none:
return .up
@unknown default:
return .up
Expand Down Expand Up @@ -263,7 +270,7 @@ public protocol BarcodeScannerViewDelegate {
return barcodes.filter { barcode in
if let cornerPoints = barcode.cornerPoints, let imageSize = imageSize {
let normalizedCornerPoints = BarcodeScannerHelper.normalizeCornerPoints(cornerPoints: cornerPoints,
imageSize: imageSize, scale: 1)
imageSize: imageSize, scale: 1, isPortrait: isPortraitAV())

let topLeft = normalizedCornerPoints[0].cgPointValue
let topRight = normalizedCornerPoints[1].cgPointValue
Expand All @@ -287,8 +294,8 @@ public protocol BarcodeScannerViewDelegate {
}
}

@objc private func onBarcodesDetected(barcodes: [Barcode], imageSize: CGSize) {
self.delegate?.onBarcodesDetected(barcodes: barcodes, imageSize: imageSize)
@objc private func onBarcodesDetected(barcodes: [Barcode], imageSize: CGSize, isPortrait: Bool) {
self.delegate?.onBarcodesDetected(barcodes: barcodes, imageSize: imageSize, isPortrait: isPortrait)
}

@objc private func onCancel() {
Expand Down