From e372657b56b3544442757984771a0de4c1bcb0a7 Mon Sep 17 00:00:00 2001 From: Gyuhwan Park Date: Sun, 31 Jul 2022 19:55:06 +0900 Subject: [PATCH 1/8] refactor(ulalacacore/ipc): rename macro 'FIXME_MARK_AS_PACKED_STRUCT' --- ulalacacore/UlalacaCore/ipc/messages/_global.h | 6 +++--- ulalacacore/UlalacaCore/ipc/messages/broker.h | 6 +++--- ulalacacore/UlalacaCore/ipc/messages/private.h | 8 ++++---- ulalacacore/UlalacaCore/ipc/messages/projector.h | 14 ++++++++------ 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/ulalacacore/UlalacaCore/ipc/messages/_global.h b/ulalacacore/UlalacaCore/ipc/messages/_global.h index 0b6150a..270c23c 100644 --- a/ulalacacore/UlalacaCore/ipc/messages/_global.h +++ b/ulalacacore/UlalacaCore/ipc/messages/_global.h @@ -6,14 +6,14 @@ /** * FIXME: naming */ -#define FIXME_MARK_AS_PACKED_STRUCT __attribute__ ((packed)) +#define MARK_AS_PACKED_STRUCT __attribute__ ((packed)) struct ULIPCRect { short x; short y; short width; short height; -} FIXME_MARK_AS_PACKED_STRUCT; +} MARK_AS_PACKED_STRUCT; struct ULIPCHeader { uint16_t messageType; @@ -24,7 +24,7 @@ struct ULIPCHeader { uint64_t timestamp; uint64_t length; -} FIXME_MARK_AS_PACKED_STRUCT; +} MARK_AS_PACKED_STRUCT; #endif \ No newline at end of file diff --git a/ulalacacore/UlalacaCore/ipc/messages/broker.h b/ulalacacore/UlalacaCore/ipc/messages/broker.h index 131278f..a99acbd 100644 --- a/ulalacacore/UlalacaCore/ipc/messages/broker.h +++ b/ulalacacore/UlalacaCore/ipc/messages/broker.h @@ -26,16 +26,16 @@ struct ULIPCSessionRequestResolved { uint8_t isLoginSession; char path[1024]; -} FIXME_MARK_AS_PACKED_STRUCT; +} MARK_AS_PACKED_STRUCT; struct ULIPCSessionRequestRejected { uint8_t reason; -} FIXME_MARK_AS_PACKED_STRUCT; +} MARK_AS_PACKED_STRUCT; /* message definition: client -> server */ struct ULIPCSessionRequest { char username[64]; char password[256]; -} FIXME_MARK_AS_PACKED_STRUCT; +} MARK_AS_PACKED_STRUCT; #endif \ No newline at end of file diff --git a/ulalacacore/UlalacaCore/ipc/messages/private.h b/ulalacacore/UlalacaCore/ipc/messages/private.h index 3f0b940..05e0469 100644 --- a/ulalacacore/UlalacaCore/ipc/messages/private.h +++ b/ulalacacore/UlalacaCore/ipc/messages/private.h @@ -33,16 +33,16 @@ static const uint8_t ANNOUNCEMENT_FLAG_IS_BUSY = 3; struct ULIPCPrivateACK { uint8_t flags; -} FIXME_MARK_AS_PACKED_STRUCT; +} MARK_AS_PACKED_STRUCT; struct ULIPCPrivateNAK { uint8_t flags; -} FIXME_MARK_AS_PACKED_STRUCT; +} MARK_AS_PACKED_STRUCT; struct ULIPCPrivateControl { uint8_t type; uint8_t flags; -} FIXME_MARK_AS_PACKED_STRUCT; +} MARK_AS_PACKED_STRUCT; struct ULIPCPrivateAnnouncement { uint8_t type; @@ -50,6 +50,6 @@ struct ULIPCPrivateAnnouncement { char username[64]; char endpoint[1024]; uint8_t flags; -} FIXME_MARK_AS_PACKED_STRUCT; +} MARK_AS_PACKED_STRUCT; #endif diff --git a/ulalacacore/UlalacaCore/ipc/messages/projector.h b/ulalacacore/UlalacaCore/ipc/messages/projector.h index bcd0e84..882acc1 100644 --- a/ulalacacore/UlalacaCore/ipc/messages/projector.h +++ b/ulalacacore/UlalacaCore/ipc/messages/projector.h @@ -51,12 +51,12 @@ static const uint8_t MOUSE_EVENT_BUTTON_MIDDLE = 2; struct ULIPCScreenUpdateNotify { uint8_t type; struct ULIPCRect rect; -} FIXME_MARK_AS_PACKED_STRUCT; +} MARK_AS_PACKED_STRUCT; struct ULIPCScreenUpdateCommit { struct ULIPCRect screenRect; uint64_t bitmapLength; -} FIXME_MARK_AS_PACKED_STRUCT; +} MARK_AS_PACKED_STRUCT; /* message definition: client -> server */ @@ -70,27 +70,29 @@ struct ULIPCKeyboardEvent { uint32_t keyCode; uint16_t flags; -} FIXME_MARK_AS_PACKED_STRUCT; +} MARK_AS_PACKED_STRUCT; struct ULIPCMouseMoveEvent { uint16_t x; uint16_t y; uint16_t flags; -} FIXME_MARK_AS_PACKED_STRUCT; +} MARK_AS_PACKED_STRUCT; struct ULIPCMouseButtonEvent { uint8_t type; uint8_t button; uint16_t flags; -} FIXME_MARK_AS_PACKED_STRUCT; +} MARK_AS_PACKED_STRUCT; struct ULIPCMouseWheelEvent { int32_t deltaX; int32_t deltaY; uint16_t flags; -} FIXME_MARK_AS_PACKED_STRUCT; +} MARK_AS_PACKED_STRUCT; + + #endif \ No newline at end of file From d66d9fd20cf769391c26151c6f23b97b59b4f0ae Mon Sep 17 00:00:00 2001 From: Gyuhwan Park Date: Sun, 31 Jul 2022 19:56:20 +0900 Subject: [PATCH 2/8] feature(ulalacacore/ipc): add message for output supression --- .../UlalacaCore/ipc/messages/projector.h | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/ulalacacore/UlalacaCore/ipc/messages/projector.h b/ulalacacore/UlalacaCore/ipc/messages/projector.h index 882acc1..0603a89 100644 --- a/ulalacacore/UlalacaCore/ipc/messages/projector.h +++ b/ulalacacore/UlalacaCore/ipc/messages/projector.h @@ -17,6 +17,12 @@ static const uint16_t TYPE_EVENT_MOUSE_MOVE = 0x0321; static const uint16_t TYPE_EVENT_MOUSE_BUTTON = 0x0322; static const uint16_t TYPE_EVENT_MOUSE_WHEEL = 0x0323; +static const uint16_t TYPE_PROJECTION_START = 0x0401; +static const uint16_t TYPE_PROJECTION_STOP = 0x0402; + +static const uint16_t TYPE_PROJECTION_SET_VIEWPORT = 0x0421; + + /* constants: Screen update notification */ static const uint8_t SCREEN_UPDATE_NOTIFY_TYPE_ENTIRE_SCREEN = 0; static const uint8_t SCREEN_UPDATE_NOTIFY_TYPE_PARTIAL = 1; @@ -93,6 +99,20 @@ struct ULIPCMouseWheelEvent { uint16_t flags; } MARK_AS_PACKED_STRUCT; +struct ULIPCProjectionStart { + uint16_t flags; +} MARK_AS_PACKED_STRUCT; +struct ULIPCProjectionStop { + uint16_t flags; +} MARK_AS_PACKED_STRUCT; + +struct ULIPCProjectionSetViewport { + uint8_t monitorId; + uint16_t width; + uint16_t height; + + uint16_t flags; +} MARK_AS_PACKED_STRUCT; #endif \ No newline at end of file From 607e2a25a900671567b1519a89c550b3418351a0 Mon Sep 17 00:00:00 2001 From: Gyuhwan Park Date: Sun, 31 Jul 2022 19:57:10 +0900 Subject: [PATCH 3/8] feature(sessionprojector): add handler for TYPE_PROJECTION_* --- .../sessionprojector/ProjectionSession.swift | 37 +++++++++++++++++-- .../sessionprojector/ScreenRecorder.swift | 13 +++++++ 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/sessionprojector/sessionprojector/ProjectionSession.swift b/sessionprojector/sessionprojector/ProjectionSession.swift index 6c30805..cc47b08 100644 --- a/sessionprojector/sessionprojector/ProjectionSession.swift +++ b/sessionprojector/sessionprojector/ProjectionSession.swift @@ -13,19 +13,27 @@ enum ProjectionSessionError: Error { case socketReadError } + + class ProjectionSession { + private let logger: ULLogger + public let socket: MMUnixSocketConnection + public var eventInjector: EventInjector? = nil public let mainDisplayId = CGMainDisplayID() public let serialQueue = DispatchQueue(label: "ProjectionSession") public let updateLock = DispatchSemaphore(value: 1) private(set) public var messageId: UInt64 = 1; + private(set) public var suppressOutput: Bool = true - public var eventInjector: EventInjector? = nil + private(set) public var mainDisplay: ViewportInfo? init(_ socket: MMUnixSocketConnection) { self.socket = socket + + self.logger = createLogger("ProjectionSession (fd \(self.socket.fd()))") } func startSession(errorHandler: @escaping (Error) -> Void) { @@ -56,13 +64,36 @@ class ProjectionSession { eventInjector?.post(mouseWheelEvent: try socket.readCStruct(ULIPCMouseWheelEvent.self)) break + case TYPE_PROJECTION_START: + try socket.readCStruct(ULIPCProjectionStart.self) + suppressOutput = false + break + case TYPE_PROJECTION_STOP: + try socket.readCStruct(ULIPCProjectionStop.self) + suppressOutput = true + break + + case TYPE_PROJECTION_SET_VIEWPORT: + self.setViewport(with: try socket.readCStruct(ULIPCProjectionSetViewport.self)) + break + default: let buffer = UnsafeMutableRawPointer.allocate(byteCount: Int(header.length), alignment: 0) try socket.readEx(buffer, size: Int(header.length)) } } } - + + private func setViewport(with message: ULIPCProjectionSetViewport) { + if (message.monitorId != 0) { + logger.debug("multi display layout is not supported yet") + return + } + + mainDisplay = ViewportInfo(width: message.width, height: message.height) + } + + private func writeMessage(_ message: T, type: UInt16) { let messageLength = MemoryLayout.size(ofValue: message) let header = ULIPCHeader( @@ -75,7 +106,7 @@ class ProjectionSession { socket.write(withUnsafePointer(to: header) { $0 }, size: MemoryLayout.size(ofValue: header)) socket.write(withUnsafePointer(to: message) { $0 }, size: messageLength) - + messageId += 1 } diff --git a/sessionprojector/sessionprojector/ScreenRecorder.swift b/sessionprojector/sessionprojector/ScreenRecorder.swift index 742a2bc..4de2a62 100644 --- a/sessionprojector/sessionprojector/ScreenRecorder.swift +++ b/sessionprojector/sessionprojector/ScreenRecorder.swift @@ -30,11 +30,24 @@ enum ScreenRecorderError: LocalizedError { } } +struct ViewportInfo { + var width: UInt16 + var height: UInt16 + + func toCGSize() -> CGSize { + CGSize(width: Int(width), height: Int(height)) + } +} + protocol ScreenUpdateSubscriber { var identifier: Int { get } + var mainDisplay: ViewportInfo? { + get + } + func screenUpdated(where rect: CGRect) func screenReady(image: CGImage, rect: CGRect) func screenResolutionChanged(to resolution: (Int, Int)) From ebd01722ba3053ce19cd117f297abc2ab7db9dbf Mon Sep 17 00:00:00 2001 From: Gyuhwan Park Date: Sun, 31 Jul 2022 19:58:21 +0900 Subject: [PATCH 4/8] checkpoint(sessionproejctor): implement resize --- .../sessionprojector/SCScreenRecorder.swift | 91 +++++++++++++++++-- 1 file changed, 82 insertions(+), 9 deletions(-) diff --git a/sessionprojector/sessionprojector/SCScreenRecorder.swift b/sessionprojector/sessionprojector/SCScreenRecorder.swift index dfe386a..89a46cf 100644 --- a/sessionprojector/sessionprojector/SCScreenRecorder.swift +++ b/sessionprojector/sessionprojector/SCScreenRecorder.swift @@ -8,6 +8,8 @@ import CoreGraphics import VideoToolbox import ScreenCaptureKit +import UlalacaCore + fileprivate struct FrameInfo { init?(from sampleBuffer: CMSampleBuffer) { let attachmentsArray = CMSampleBufferGetSampleAttachmentsArray( @@ -44,17 +46,51 @@ fileprivate struct FrameInfo { } class SCScreenRecorder: NSObject, ScreenRecorder { + private static let staticLogger = createLogger("SCScreenRecorder::*") + private let logger = createLogger("SCScreenRecorder") + + private var ciContext: CIContext + + private var stream: SCStream? + private var subscriptions: [ScreenUpdateSubscriber] = [] + private var prevDisplayTime: UInt64 = 0 + + private var streamQueue = DispatchQueue( label: "UlalacaStreamRecorder", qos: .userInteractive ) - private var stream: SCStream? - private var subscriptions: [ScreenUpdateSubscriber] = [] - private var prevDisplayTime: UInt64 = 0 + private static func createCoreImageContext(useMetal: Bool = true) -> CIContext { + staticLogger.debug("creating CIContext") + + if let ciContext = NSGraphicsContext.current?.ciContext { + staticLogger.debug("acquired CIContext from NSGraphicsContext.current") + return ciContext + } + + if (useMetal) { + if let metalDevice = MTLCreateSystemDefaultDevice() { + staticLogger.debug("created CIContext using Metal API") + return CIContext(mtlDevice: metalDevice) + } + } + + if let cgContext = NSGraphicsContext.current?.cgContext { + staticLogger.debug("created CIContext using cgContext") + return CIContext(cgContext: cgContext) + } else { + staticLogger.error("creating CIContext using software renderer, this will impact performance (is hardware acceleration available?)") + return CIContext(options: [ + .useSoftwareRenderer: true + ]) + } + } override init() { super.init() + + self.ciContext = SCScreenRecorder.createCoreImageContext(useMetal: true) } func subscribeUpdate(_ subscriber: ScreenUpdateSubscriber) { @@ -132,11 +168,6 @@ extension SCScreenRecorder: SCStreamOutput { return } - var image: CGImage? - VTCreateCGImageFromCVPixelBuffer(sampleBuffer.imageBuffer!, options: nil, imageOut: &image) - CVPixelBufferUnlockBaseAddress(sampleBuffer.imageBuffer!, .readOnly) - - let now = CMTime(value: Int64(mach_absolute_time()), timescale: 1000000000) let frameTimestamp = sampleBuffer.presentationTimeStamp @@ -147,7 +178,49 @@ extension SCScreenRecorder: SCStreamOutput { subscriptions.forEach { $0.screenUpdated(where: rect) } } } - subscriptions.forEach { $0.screenReady(image: image!, rect: frameInfo.contentRect!) } + subscriptions.forEach { subscriber in + notifyScreenReady(which: sampleBuffer, rect: frameInfo.contentRect!, to: subscriber) + } + } + + func notifyScreenReady(which sampleBuffer: CMSampleBuffer, rect: CGRect, to subscriber: ScreenUpdateSubscriber) { + var image: CGImage? + + if let viewportInfo = subscriber.mainDisplay { + let pixelBuffer = sampleBuffer.resize(size: viewportInfo.toCGSize(), context: ciContext) + VTCreateCGImageFromCVPixelBuffer(pixelBuffer, options: nil, imageOut: &image) + subscriber.screenReady(image: image, rect: CGRect(0, 0, Int(viewportInfo.width), Int(viewportInfo.height))) + } else { + VTCreateCGImageFromCVPixelBuffer(sampleBuffer.imageBuffer!, options: nil, imageOut: &image) + subscriber.screenReady(image: image, rect: rect) + } + } } +fileprivate extension CMSampleBuffer { + func resize(size desiredSize: CGSize, context: CIContext) -> CVPixelBuffer { + let imageBuffer = self.imageBuffer! + + let outPixelBuffer: CVPixelBuffer? = nil + let result = CVPixelBufferCreate( + nil, + Int(desiredSize.width), Int(desiredSize.height), + CVPixelBufferGetPixelFormatType(imageBuffer), + nil, + &outPixelBuffer + ) + + let size = CVImageBufferGetEncodedSize(imageBuffer) + let ciImage = CIImage(cvImageBuffer: imageBuffer) + + let sx = CGFloat(desiredSize.width) / CGFloat(size.width) + let sy = CGFloat(desiredSize.height) / CGFloat(size.height) + + let scale = CGAffineTransform(scaleX: sx, y: sy) + let scaledImage = ciImage.transformed(by: scale) + context.render(scaledImage, to: outPixelBuffer) + + return outPixelBuffer! + } +} From ba68e3dacfac97574582c8877b25e8cb4b22ba8b Mon Sep 17 00:00:00 2001 From: Gyuhwan Park Date: Mon, 1 Aug 2022 17:33:09 +0900 Subject: [PATCH 5/8] feature(sessionprojector): add experimental support for resizing, display configuration change --- .../project.pbxproj | 8 +++ .../sessionprojector/CGRect+toULIPCRect.swift | 16 +++++ .../sessionprojector/ProjectionSession.swift | 39 ++++++----- .../sessionprojector/SCScreenRecorder.swift | 70 +++++++++++++++---- .../sessionprojector/ScreenRecorder.swift | 23 +++++- .../sessionprojector/ULIPCRect+scale.swift | 16 +++++ 6 files changed, 138 insertions(+), 34 deletions(-) create mode 100644 sessionprojector/sessionprojector/CGRect+toULIPCRect.swift create mode 100644 sessionprojector/sessionprojector/ULIPCRect+scale.swift diff --git a/sessionprojector/sessionprojector.xcodeproj/project.pbxproj b/sessionprojector/sessionprojector.xcodeproj/project.pbxproj index 1fd57cb..eb94204 100644 --- a/sessionprojector/sessionprojector.xcodeproj/project.pbxproj +++ b/sessionprojector/sessionprojector.xcodeproj/project.pbxproj @@ -10,6 +10,8 @@ 85721156283B607000C36D5F /* UlalacaCore.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 85721155283B607000C36D5F /* UlalacaCore.framework */; }; 85721157283B607000C36D5F /* UlalacaCore.framework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 85721155283B607000C36D5F /* UlalacaCore.framework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; F4EBD136805519C3F8D8314A /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4EBD3D9F922ED6C0BBA9607 /* AppDelegate.swift */; }; + F4EBD18EFC3BFC5565288F9C /* CGRect+toULIPCRect.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4EBD846347B06D7E7B73FC1 /* CGRect+toULIPCRect.swift */; }; + F4EBD3107D60426E249C21FD /* ULIPCRect+scale.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4EBDCBA0826CA18FB1E77CF /* ULIPCRect+scale.swift */; }; F4EBD35869E7EEC01C4CE48D /* SessionManagerClient.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4EBDFDEDFC5A96EE4B3F91C /* SessionManagerClient.swift */; }; F4EBD455F06C38A5ABA1EA20 /* ProjectionSession.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4EBDEADE7680E1D1466830F /* ProjectionSession.swift */; }; F4EBD517D25C43D7397EA0FD /* MainMenu.xib in Resources */ = {isa = PBXBuildFile; fileRef = F4EBD2D2B450575F0A669D2E /* MainMenu.xib */; }; @@ -47,8 +49,10 @@ F4EBD55AD46E2D55483DA879 /* AVFScreenRecorder.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AVFScreenRecorder.swift; sourceTree = ""; }; F4EBD5938B6F75A2117975F5 /* pl.unstabler.ulalaca.sessionprojector.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist; path = pl.unstabler.ulalaca.sessionprojector.plist; sourceTree = ""; }; F4EBD72F7A1676E55BB21493 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; + F4EBD846347B06D7E7B73FC1 /* CGRect+toULIPCRect.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "CGRect+toULIPCRect.swift"; sourceTree = ""; }; F4EBDB589E76D41675D27BF0 /* ProjectionServer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ProjectionServer.swift; sourceTree = ""; }; F4EBDB6CE8B3BF020BD4FE81 /* sessionprojector.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = sessionprojector.entitlements; sourceTree = ""; }; + F4EBDCBA0826CA18FB1E77CF /* ULIPCRect+scale.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "ULIPCRect+scale.swift"; sourceTree = ""; }; F4EBDEADE7680E1D1466830F /* ProjectionSession.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ProjectionSession.swift; sourceTree = ""; }; F4EBDEC6CB9E4322333D6C24 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.xib; name = Base; path = Base.lproj/MainMenu.xib; sourceTree = ""; }; F4EBDFDEDFC5A96EE4B3F91C /* SessionManagerClient.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SessionManagerClient.swift; sourceTree = ""; }; @@ -91,6 +95,8 @@ F4EBDFDEDFC5A96EE4B3F91C /* SessionManagerClient.swift */, F4EBD0BFDAED6400792ABDFA /* SCScreenRecorder.swift */, F4EBD55AD46E2D55483DA879 /* AVFScreenRecorder.swift */, + F4EBDCBA0826CA18FB1E77CF /* ULIPCRect+scale.swift */, + F4EBD846347B06D7E7B73FC1 /* CGRect+toULIPCRect.swift */, ); path = sessionprojector; sourceTree = ""; @@ -201,6 +207,8 @@ F4EBD35869E7EEC01C4CE48D /* SessionManagerClient.swift in Sources */, F4EBDCD57BF1530790AC739E /* SCScreenRecorder.swift in Sources */, F4EBD732593CF09DECD7B40A /* AVFScreenRecorder.swift in Sources */, + F4EBD3107D60426E249C21FD /* ULIPCRect+scale.swift in Sources */, + F4EBD18EFC3BFC5565288F9C /* CGRect+toULIPCRect.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/sessionprojector/sessionprojector/CGRect+toULIPCRect.swift b/sessionprojector/sessionprojector/CGRect+toULIPCRect.swift new file mode 100644 index 0000000..fb5549b --- /dev/null +++ b/sessionprojector/sessionprojector/CGRect+toULIPCRect.swift @@ -0,0 +1,16 @@ +// +// Created by Gyuhwan Park on 2022/08/01. +// + +import Foundation + +extension CGRect { + func toULIPCRect() -> ULIPCRect { + return ULIPCRect( + x: Int16(origin.x), + y: Int16(origin.y), + width: Int16(size.width), + height: Int16(size.height) + ) + } +} \ No newline at end of file diff --git a/sessionprojector/sessionprojector/ProjectionSession.swift b/sessionprojector/sessionprojector/ProjectionSession.swift index cc47b08..5449fdb 100644 --- a/sessionprojector/sessionprojector/ProjectionSession.swift +++ b/sessionprojector/sessionprojector/ProjectionSession.swift @@ -28,12 +28,13 @@ class ProjectionSession { private(set) public var messageId: UInt64 = 1; private(set) public var suppressOutput: Bool = true - private(set) public var mainDisplay: ViewportInfo? + private(set) public var screenResolution: CGSize = CGSize(width: 0, height: 0) + private(set) public var mainViewport: ViewportInfo? init(_ socket: MMUnixSocketConnection) { self.socket = socket - self.logger = createLogger("ProjectionSession (fd \(self.socket.fd()))") + self.logger = createLogger("ProjectionSession (fd \(self.socket.descriptor()))") } func startSession(errorHandler: @escaping (Error) -> Void) { @@ -55,7 +56,11 @@ class ProjectionSession { eventInjector?.post(keyEvent: try socket.readCStruct(ULIPCKeyboardEvent.self)) break case TYPE_EVENT_MOUSE_MOVE: - eventInjector?.post(mouseMoveEvent: try socket.readCStruct(ULIPCMouseMoveEvent.self)) + eventInjector?.post( + mouseMoveEvent: try socket.readCStruct(ULIPCMouseMoveEvent.self), + scaleX: Double(screenResolution.width) / Double(mainViewport!.width), + scaleY: Double(screenResolution.height) / Double(mainViewport!.height) + ) break case TYPE_EVENT_MOUSE_BUTTON: eventInjector?.post(mouseButtonEvent: try socket.readCStruct(ULIPCMouseButtonEvent.self)) @@ -90,7 +95,7 @@ class ProjectionSession { return } - mainDisplay = ViewportInfo(width: message.width, height: message.height) + mainViewport = ViewportInfo(width: message.width, height: message.height) } @@ -119,15 +124,13 @@ extension ProjectionSession: ScreenUpdateSubscriber { func screenUpdated(where rect: CGRect) { self.serialQueue.sync { + let sx = mainViewport?.scaleX(Int(screenResolution.width)) ?? 1.0 + let sy = mainViewport?.scaleY(Int(screenResolution.height)) ?? 1.0 + self.writeMessage( - ULIPCScreenUpdateNotify( + ULIPCScreenUpdateNotify( type: SCREEN_UPDATE_NOTIFY_TYPE_PARTIAL, - rect: ULIPCRect( - x: Int16(rect.origin.x), - y: Int16(rect.origin.y), - width: Int16(rect.size.width), - height: Int16(rect.size.height) - ) + rect: rect.toULIPCRect().scale(x: sx, y: sy) ), type: TYPE_SCREEN_UPDATE_NOTIFY ) @@ -141,13 +144,11 @@ extension ProjectionSession: ScreenUpdateSubscriber { let pointer = CFDataGetBytePtr(rawData)! let length = CFDataGetLength(rawData) + let sx = mainViewport?.scaleX(Int(screenResolution.width)) ?? 1.0 + let sy = mainViewport?.scaleY(Int(screenResolution.height)) ?? 1.0 + let message = ULIPCScreenUpdateCommit( - screenRect: ULIPCRect( - x: Int16(rect.origin.x), - y: Int16(rect.origin.y), - width: Int16(rect.size.width), - height: Int16(rect.size.height) - ), + screenRect: rect.toULIPCRect().scale(x: sx, y: sy), bitmapLength: UInt64(length) ) @@ -156,8 +157,8 @@ extension ProjectionSession: ScreenUpdateSubscriber { } } - func screenResolutionChanged(to resolution: (Int, Int)) { - + func screenResolutionChanged(to resolution: CGSize) { + self.screenResolution = resolution } } diff --git a/sessionprojector/sessionprojector/SCScreenRecorder.swift b/sessionprojector/sessionprojector/SCScreenRecorder.swift index 89a46cf..91e81f0 100644 --- a/sessionprojector/sessionprojector/SCScreenRecorder.swift +++ b/sessionprojector/sessionprojector/SCScreenRecorder.swift @@ -55,6 +55,11 @@ class SCScreenRecorder: NSObject, ScreenRecorder { private var subscriptions: [ScreenUpdateSubscriber] = [] private var prevDisplayTime: UInt64 = 0 + private var currentScreenResolution = CGSize(width: 0, height: 0) + + // HACK + private var this: SCScreenRecorder! + private var streamQueue = DispatchQueue( label: "UlalacaStreamRecorder", @@ -88,9 +93,23 @@ class SCScreenRecorder: NSObject, ScreenRecorder { } override init() { + self.ciContext = SCScreenRecorder.createCoreImageContext(useMetal: true) super.init() - self.ciContext = SCScreenRecorder.createCoreImageContext(useMetal: true) + self.this = self + self.listenToDisplayConfigurationChange() + } + + private func listenToDisplayConfigurationChange() { + CGDisplayRegisterReconfigurationCallback({ display, flags, userInfo in + let this = userInfo!.bindMemory(to: SCScreenRecorder.self, capacity: 1).pointee + + Task { + try? await this.stop() + try! await this.prepare() + try! await this.start() + } + }, &this) } func subscribeUpdate(_ subscriber: ScreenUpdateSubscriber) { @@ -100,6 +119,7 @@ class SCScreenRecorder: NSObject, ScreenRecorder { } subscriptions.append(subscriber) + subscriber.screenResolutionChanged(to: self.currentScreenResolution) } func unsubscribeUpdate(_ subscriber: ScreenUpdateSubscriber) { @@ -117,6 +137,7 @@ class SCScreenRecorder: NSObject, ScreenRecorder { configuration.queueDepth = 1 configuration.showsCursor = true + let displays = try await SCShareableContent.current.displays guard let primaryDisplay = displays.first else { throw ScreenRecorderError.initializationError @@ -143,8 +164,8 @@ class SCScreenRecorder: NSObject, ScreenRecorder { } } - func stop() throws { - + func stop() async throws { + try await stream!.stopCapture() } } @@ -168,31 +189,54 @@ extension SCScreenRecorder: SCStreamOutput { return } + if (frameInfo.contentRect!.size != self.currentScreenResolution) { + logger.debug("resolution changed to \(frameInfo.contentRect!.size)") + self.currentScreenResolution = frameInfo.contentRect!.size + + subscriptions.forEach { subscriber in + subscriber.screenResolutionChanged(to: frameInfo.contentRect!.size) + } + } + let now = CMTime(value: Int64(mach_absolute_time()), timescale: 1000000000) let frameTimestamp = sampleBuffer.presentationTimeStamp let timedelta = abs(now.seconds - frameTimestamp.seconds) if let dirtyRects = frameInfo.dirtyRects { - dirtyRects.forEach { rect in - subscriptions.forEach { $0.screenUpdated(where: rect) } + subscriptions.forEach { subscriber in + if let mainDisplay = subscriber.mainViewport { + subscriber.screenUpdated(where: frameInfo.contentRect!) + } else { + dirtyRects.forEach { rect in + subscriber.screenUpdated(where: rect) + } + } + + } + + subscriptions.forEach { subscriber in + if (!subscriber.suppressOutput) { + notifyScreenReady(which: sampleBuffer, rect: frameInfo.contentRect!, to: subscriber) + } } - } - subscriptions.forEach { subscriber in - notifyScreenReady(which: sampleBuffer, rect: frameInfo.contentRect!, to: subscriber) } } func notifyScreenReady(which sampleBuffer: CMSampleBuffer, rect: CGRect, to subscriber: ScreenUpdateSubscriber) { var image: CGImage? - if let viewportInfo = subscriber.mainDisplay { + if let viewportInfo = subscriber.mainViewport { let pixelBuffer = sampleBuffer.resize(size: viewportInfo.toCGSize(), context: ciContext) + CVPixelBufferLockBaseAddress(pixelBuffer, .readOnly) VTCreateCGImageFromCVPixelBuffer(pixelBuffer, options: nil, imageOut: &image) - subscriber.screenReady(image: image, rect: CGRect(0, 0, Int(viewportInfo.width), Int(viewportInfo.height))) + CVPixelBufferUnlockBaseAddress(pixelBuffer, .readOnly) + subscriber.screenReady(image: image!, rect: CGRect(x: 0, y: 0, width: Int(viewportInfo.width), height: Int(viewportInfo.height))) } else { + CVPixelBufferLockBaseAddress(sampleBuffer.imageBuffer!, .readOnly) VTCreateCGImageFromCVPixelBuffer(sampleBuffer.imageBuffer!, options: nil, imageOut: &image) - subscriber.screenReady(image: image, rect: rect) + CVPixelBufferUnlockBaseAddress(sampleBuffer.imageBuffer!, .readOnly) + subscriber.screenReady(image: image!, rect: rect) } } @@ -202,7 +246,7 @@ fileprivate extension CMSampleBuffer { func resize(size desiredSize: CGSize, context: CIContext) -> CVPixelBuffer { let imageBuffer = self.imageBuffer! - let outPixelBuffer: CVPixelBuffer? = nil + var outPixelBuffer: CVPixelBuffer? = nil let result = CVPixelBufferCreate( nil, Int(desiredSize.width), Int(desiredSize.height), @@ -219,7 +263,7 @@ fileprivate extension CMSampleBuffer { let scale = CGAffineTransform(scaleX: sx, y: sy) let scaledImage = ciImage.transformed(by: scale) - context.render(scaledImage, to: outPixelBuffer) + context.render(scaledImage, to: outPixelBuffer!) return outPixelBuffer! } diff --git a/sessionprojector/sessionprojector/ScreenRecorder.swift b/sessionprojector/sessionprojector/ScreenRecorder.swift index 4de2a62..cb7b1c6 100644 --- a/sessionprojector/sessionprojector/ScreenRecorder.swift +++ b/sessionprojector/sessionprojector/ScreenRecorder.swift @@ -34,6 +34,21 @@ struct ViewportInfo { var width: UInt16 var height: UInt16 + func scaleX(_ value: IntegerLiteralType) -> Double { + if (value <= 0) { + return 0.0 + } + + return Double(width) / Double(value); + } + func scaleY(_ value: IntegerLiteralType) -> Double { + if (value <= 0) { + return 0.0 + } + + return Double(height) / Double(value); + } + func toCGSize() -> CGSize { CGSize(width: Int(width), height: Int(height)) } @@ -44,13 +59,17 @@ protocol ScreenUpdateSubscriber { get } - var mainDisplay: ViewportInfo? { + var suppressOutput: Bool { + get + } + + var mainViewport: ViewportInfo? { get } func screenUpdated(where rect: CGRect) func screenReady(image: CGImage, rect: CGRect) - func screenResolutionChanged(to resolution: (Int, Int)) + func screenResolutionChanged(to resolution: CGSize) } protocol ScreenRecorder: NSObject { diff --git a/sessionprojector/sessionprojector/ULIPCRect+scale.swift b/sessionprojector/sessionprojector/ULIPCRect+scale.swift new file mode 100644 index 0000000..2819983 --- /dev/null +++ b/sessionprojector/sessionprojector/ULIPCRect+scale.swift @@ -0,0 +1,16 @@ +// +// Created by Gyuhwan Park on 2022/08/01. +// + +import Foundation + +extension ULIPCRect { + func scale(x sx: Double, y sy: Double) -> ULIPCRect { + return ULIPCRect( + x: Int16(Double(x) * sx), + y: Int16(Double(y) * sy), + width: Int16(Double(width) * sx), + height: Int16(Double(height) * sy) + ) + } +} \ No newline at end of file From 12f222cb961991bcec05560379b9a3a875721500 Mon Sep 17 00:00:00 2001 From: Gyuhwan Park Date: Mon, 1 Aug 2022 17:36:44 +0900 Subject: [PATCH 6/8] feature(sessionprojector/EventInjector): add mouse position scaling --- .../sessionprojector/EventInjector.swift | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/sessionprojector/sessionprojector/EventInjector.swift b/sessionprojector/sessionprojector/EventInjector.swift index a60f37b..cb39118 100644 --- a/sessionprojector/sessionprojector/EventInjector.swift +++ b/sessionprojector/sessionprojector/EventInjector.swift @@ -39,24 +39,13 @@ class EventInjector { } func prepare() throws { - guard - /* let eventTap = CGEvent.tapCreate( - tap: .cgSessionEventTap, - place: .headInsertEventTap, - options: .defaultTap, - eventsOfInterest: UInt64(CGEventType.null.rawValue), - callback: { (proxy, type, event, refcon) in - return Unmanaged.passUnretained(event) - }, - userInfo: nil - ), */ let eventSource = CGEventSource( + guard let eventSource = CGEventSource( stateID: .combinedSessionState ) else { throw EventInjectorError.initializationError } - // self.eventTap = eventTap self.eventSource = eventSource } @@ -87,10 +76,10 @@ class EventInjector { cgEvent.post(tap: .cgSessionEventTap) } - func post(mouseMoveEvent event: ULIPCMouseMoveEvent) { + func post(mouseMoveEvent event: ULIPCMouseMoveEvent, scaleX: Double = 1.0, scaleY: Double = 1.0) { var mouseType: CGEventType = .mouseMoved var mouseButton: CGMouseButton = .left - let position = CGPoint(x: Int(event.x), y: Int(event.y)) + let position = CGPoint(x: Int(Double(event.x) * scaleX), y: Int(Double(event.y) * scaleY)) if (mouseDownState & EventInjector.MOUSE_DOWN_STATE_LEFT > 0) { mouseType = .leftMouseDragged From 2c84c9e3133259efda1a0cc97f0fa219da63aff6 Mon Sep 17 00:00:00 2001 From: Gyuhwan Park Date: Thu, 6 Oct 2022 21:01:49 +0900 Subject: [PATCH 7/8] fix(SCScreenRecorder): exclude sessionprojector itself from capture target --- .../sessionprojector/SCScreenRecorder.swift | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/sessionprojector/sessionprojector/SCScreenRecorder.swift b/sessionprojector/sessionprojector/SCScreenRecorder.swift index 91e81f0..56cbba8 100644 --- a/sessionprojector/sessionprojector/SCScreenRecorder.swift +++ b/sessionprojector/sessionprojector/SCScreenRecorder.swift @@ -139,6 +139,11 @@ class SCScreenRecorder: NSObject, ScreenRecorder { let displays = try await SCShareableContent.current.displays + let sessionProjectorApp = try await SCShareableContent.current.applications.filter { app in + // FIXME: hard-coded bundle id + app.bundleIdentifier == "pl.unstabler.ulalaca.sessionprojector" + }.first! + guard let primaryDisplay = displays.first else { throw ScreenRecorderError.initializationError } @@ -146,7 +151,12 @@ class SCScreenRecorder: NSObject, ScreenRecorder { configuration.width = primaryDisplay.width configuration.height = primaryDisplay.height - let filter = SCContentFilter(display: primaryDisplay, excludingWindows: []) + // since macOS 12.3↑, passing empty array to excludingWindows breaks SCStream + let filter = SCContentFilter( + display: primaryDisplay, + excludingApplications: [sessionProjectorApp], + exceptingWindows: [] + ) stream = SCStream(filter: filter, configuration: configuration, delegate: self) guard let stream = stream else { From 6906011a404c9ac27ca11d7c3d19893cf1959895 Mon Sep 17 00:00:00 2001 From: Gyuhwan Park Date: Thu, 6 Oct 2022 21:06:08 +0900 Subject: [PATCH 8/8] fix(ULIPCRect): prevent visual artifacts when updating screen partially --- .../sessionprojector/SCScreenRecorder.swift | 11 +++-------- .../sessionprojector/ULIPCRect+scale.swift | 8 ++++---- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/sessionprojector/sessionprojector/SCScreenRecorder.swift b/sessionprojector/sessionprojector/SCScreenRecorder.swift index 56cbba8..5fa40bc 100644 --- a/sessionprojector/sessionprojector/SCScreenRecorder.swift +++ b/sessionprojector/sessionprojector/SCScreenRecorder.swift @@ -63,7 +63,7 @@ class SCScreenRecorder: NSObject, ScreenRecorder { private var streamQueue = DispatchQueue( label: "UlalacaStreamRecorder", - qos: .userInteractive + qos: .background ) private static func createCoreImageContext(useMetal: Bool = true) -> CIContext { @@ -215,14 +215,9 @@ extension SCScreenRecorder: SCStreamOutput { if let dirtyRects = frameInfo.dirtyRects { subscriptions.forEach { subscriber in - if let mainDisplay = subscriber.mainViewport { - subscriber.screenUpdated(where: frameInfo.contentRect!) - } else { - dirtyRects.forEach { rect in - subscriber.screenUpdated(where: rect) - } + dirtyRects.forEach { rect in + subscriber.screenUpdated(where: rect) } - } subscriptions.forEach { subscriber in diff --git a/sessionprojector/sessionprojector/ULIPCRect+scale.swift b/sessionprojector/sessionprojector/ULIPCRect+scale.swift index 2819983..a85b5b8 100644 --- a/sessionprojector/sessionprojector/ULIPCRect+scale.swift +++ b/sessionprojector/sessionprojector/ULIPCRect+scale.swift @@ -7,10 +7,10 @@ import Foundation extension ULIPCRect { func scale(x sx: Double, y sy: Double) -> ULIPCRect { return ULIPCRect( - x: Int16(Double(x) * sx), - y: Int16(Double(y) * sy), - width: Int16(Double(width) * sx), - height: Int16(Double(height) * sy) + x: Int16(floor(Double(x) * sx)), + y: Int16(floor(Double(y) * sy)), + width: Int16(ceil(Double(width) * sx)), + height: Int16(ceil(Double(height) * sy)) ) } } \ No newline at end of file