feature(sessionprojector): add experimental support for resizing, display configuration change

sessionprojector/sessionprojector.xcodeproj/project.pbxproj

@@ -10,6 +10,8 @@
 		85721156283B607000C36D5F /* UlalacaCore.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 85721155283B607000C36D5F /* UlalacaCore.framework */; };
 		85721157283B607000C36D5F /* UlalacaCore.framework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 85721155283B607000C36D5F /* UlalacaCore.framework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; };
 		F4EBD136805519C3F8D8314A /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4EBD3D9F922ED6C0BBA9607 /* AppDelegate.swift */; };
+		F4EBD18EFC3BFC5565288F9C /* CGRect+toULIPCRect.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4EBD846347B06D7E7B73FC1 /* CGRect+toULIPCRect.swift */; };
+		F4EBD3107D60426E249C21FD /* ULIPCRect+scale.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4EBDCBA0826CA18FB1E77CF /* ULIPCRect+scale.swift */; };
 		F4EBD35869E7EEC01C4CE48D /* SessionManagerClient.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4EBDFDEDFC5A96EE4B3F91C /* SessionManagerClient.swift */; };
 		F4EBD455F06C38A5ABA1EA20 /* ProjectionSession.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4EBDEADE7680E1D1466830F /* ProjectionSession.swift */; };
 		F4EBD517D25C43D7397EA0FD /* MainMenu.xib in Resources */ = {isa = PBXBuildFile; fileRef = F4EBD2D2B450575F0A669D2E /* MainMenu.xib */; };
@@ -47,8 +49,10 @@
 		F4EBD55AD46E2D55483DA879 /* AVFScreenRecorder.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AVFScreenRecorder.swift; sourceTree = "<group>"; };
 		F4EBD5938B6F75A2117975F5 /* pl.unstabler.ulalaca.sessionprojector.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist; path = pl.unstabler.ulalaca.sessionprojector.plist; sourceTree = "<group>"; };
 		F4EBD72F7A1676E55BB21493 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = "<group>"; };
+		F4EBD846347B06D7E7B73FC1 /* CGRect+toULIPCRect.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "CGRect+toULIPCRect.swift"; sourceTree = "<group>"; };
 		F4EBDB589E76D41675D27BF0 /* ProjectionServer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ProjectionServer.swift; sourceTree = "<group>"; };
 		F4EBDB6CE8B3BF020BD4FE81 /* sessionprojector.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = sessionprojector.entitlements; sourceTree = "<group>"; };
+		F4EBDCBA0826CA18FB1E77CF /* ULIPCRect+scale.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "ULIPCRect+scale.swift"; sourceTree = "<group>"; };
 		F4EBDEADE7680E1D1466830F /* ProjectionSession.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ProjectionSession.swift; sourceTree = "<group>"; };
 		F4EBDEC6CB9E4322333D6C24 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.xib; name = Base; path = Base.lproj/MainMenu.xib; sourceTree = "<group>"; };
 		F4EBDFDEDFC5A96EE4B3F91C /* SessionManagerClient.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SessionManagerClient.swift; sourceTree = "<group>"; };
@@ -91,6 +95,8 @@
 				F4EBDFDEDFC5A96EE4B3F91C /* SessionManagerClient.swift */,
 				F4EBD0BFDAED6400792ABDFA /* SCScreenRecorder.swift */,
 				F4EBD55AD46E2D55483DA879 /* AVFScreenRecorder.swift */,
+				F4EBDCBA0826CA18FB1E77CF /* ULIPCRect+scale.swift */,
+				F4EBD846347B06D7E7B73FC1 /* CGRect+toULIPCRect.swift */,
 			);
 			path = sessionprojector;
 			sourceTree = "<group>";
@@ -201,6 +207,8 @@
 				F4EBD35869E7EEC01C4CE48D /* SessionManagerClient.swift in Sources */,
 				F4EBDCD57BF1530790AC739E /* SCScreenRecorder.swift in Sources */,
 				F4EBD732593CF09DECD7B40A /* AVFScreenRecorder.swift in Sources */,
+				F4EBD3107D60426E249C21FD /* ULIPCRect+scale.swift in Sources */,
+				F4EBD18EFC3BFC5565288F9C /* CGRect+toULIPCRect.swift in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};

sessionprojector/sessionprojector/CGRect+toULIPCRect.swift

@@ -0,0 +1,16 @@
+//
+// Created by Gyuhwan Park on 2022/08/01.
+//
+
+import Foundation
+
+extension CGRect {
+    func toULIPCRect() -> ULIPCRect {
+        return ULIPCRect(
+            x: Int16(origin.x),
+            y: Int16(origin.y),
+            width: Int16(size.width),
+            height: Int16(size.height)
+        )
+    }
+}

sessionprojector/sessionprojector/EventInjector.swift

@@ -39,24 +39,13 @@ class EventInjector {
     }
 
     func prepare() throws {
-        guard
-                /* let eventTap = CGEvent.tapCreate(
-                tap: .cgSessionEventTap,
-                place: .headInsertEventTap,
-                options: .defaultTap,
-                eventsOfInterest: UInt64(CGEventType.null.rawValue),
-                callback: { (proxy, type, event, refcon) in
-                    return Unmanaged.passUnretained(event)
-                },
-                userInfo: nil
-        ), */ let eventSource = CGEventSource(
+        guard let eventSource = CGEventSource(
                 stateID: .combinedSessionState
         )
         else {
             throw EventInjectorError.initializationError
         }
 
-        // self.eventTap = eventTap
         self.eventSource = eventSource
     }
 
@@ -87,10 +76,10 @@ class EventInjector {
         cgEvent.post(tap: .cgSessionEventTap)
     }
 
-    func post(mouseMoveEvent event: ULIPCMouseMoveEvent) {
+    func post(mouseMoveEvent event: ULIPCMouseMoveEvent, scaleX: Double = 1.0, scaleY: Double = 1.0) {
         var mouseType: CGEventType = .mouseMoved
         var mouseButton: CGMouseButton = .left
-        let position = CGPoint(x: Int(event.x), y: Int(event.y))
+        let position = CGPoint(x: Int(Double(event.x) * scaleX), y: Int(Double(event.y) * scaleY))
 
         if (mouseDownState & EventInjector.MOUSE_DOWN_STATE_LEFT > 0) {
             mouseType = .leftMouseDragged

sessionprojector/sessionprojector/ProjectionSession.swift

@@ -13,19 +13,28 @@ enum ProjectionSessionError: Error {
     case socketReadError
 }
 
+
+
 class ProjectionSession {
+    private let logger: ULLogger
+
     public let socket: MMUnixSocketConnection
+    public var eventInjector: EventInjector? = nil
 
     public let mainDisplayId = CGMainDisplayID()
     public let serialQueue = DispatchQueue(label: "ProjectionSession")
     public let updateLock = DispatchSemaphore(value: 1)
 
     private(set) public var messageId: UInt64 = 1;
+    private(set) public var suppressOutput: Bool = true
 
-    public var eventInjector: EventInjector? = nil
+    private(set) public var screenResolution: CGSize = CGSize(width: 0, height: 0)
+    private(set) public var mainViewport: ViewportInfo?
 
     init(_ socket: MMUnixSocketConnection) {
         self.socket = socket
+
+        self.logger = createLogger("ProjectionSession (fd \(self.socket.descriptor()))")
     }
 
     func startSession(errorHandler: @escaping (Error) -> Void) {
@@ -47,7 +56,11 @@ class ProjectionSession {
                 eventInjector?.post(keyEvent: try socket.readCStruct(ULIPCKeyboardEvent.self))
                 break
             case TYPE_EVENT_MOUSE_MOVE:
-                eventInjector?.post(mouseMoveEvent: try socket.readCStruct(ULIPCMouseMoveEvent.self))
+                eventInjector?.post(
+                    mouseMoveEvent: try socket.readCStruct(ULIPCMouseMoveEvent.self),
+                    scaleX: Double(screenResolution.width) / Double(mainViewport!.width),
+                    scaleY: Double(screenResolution.height) / Double(mainViewport!.height)
+                )
                 break
             case TYPE_EVENT_MOUSE_BUTTON:
                 eventInjector?.post(mouseButtonEvent: try socket.readCStruct(ULIPCMouseButtonEvent.self))
@@ -56,13 +69,36 @@ class ProjectionSession {
                 eventInjector?.post(mouseWheelEvent: try socket.readCStruct(ULIPCMouseWheelEvent.self))
                 break
 
+            case TYPE_PROJECTION_START:
+                try socket.readCStruct(ULIPCProjectionStart.self)
+                suppressOutput = false
+                break
+            case TYPE_PROJECTION_STOP:
+                try socket.readCStruct(ULIPCProjectionStop.self)
+                suppressOutput = true
+                break
+
+            case TYPE_PROJECTION_SET_VIEWPORT:
+                self.setViewport(with: try socket.readCStruct(ULIPCProjectionSetViewport.self))
+                break
+
             default:
                 let buffer = UnsafeMutableRawPointer.allocate(byteCount: Int(header.length), alignment: 0)
                 try socket.readEx(buffer, size: Int(header.length))
             }
         }
     }
-
+
+    private func setViewport(with message: ULIPCProjectionSetViewport) {
+        if (message.monitorId != 0) {
+            logger.debug("multi display layout is not supported yet")
+            return
+        }
+
+        mainViewport = ViewportInfo(width: message.width, height: message.height)
+    }
+
+
     private func writeMessage<T>(_ message: T, type: UInt16) {
         let messageLength = MemoryLayout.size(ofValue: message)
         let header = ULIPCHeader(
@@ -75,7 +111,7 @@ class ProjectionSession {
 
         socket.write(withUnsafePointer(to: header) { $0 }, size: MemoryLayout.size(ofValue: header))
         socket.write(withUnsafePointer(to: message) { $0 }, size: messageLength)
-        
+
         messageId += 1
     }
 
@@ -88,15 +124,13 @@ extension ProjectionSession: ScreenUpdateSubscriber {
 
     func screenUpdated(where rect: CGRect) {
         self.serialQueue.sync {
+            let sx = mainViewport?.scaleX(Int(screenResolution.width)) ?? 1.0
+            let sy = mainViewport?.scaleY(Int(screenResolution.height)) ?? 1.0
+
             self.writeMessage(
-                ULIPCScreenUpdateNotify(
+                    ULIPCScreenUpdateNotify(
                     type: SCREEN_UPDATE_NOTIFY_TYPE_PARTIAL,
-                    rect: ULIPCRect(
-                            x: Int16(rect.origin.x),
-                            y: Int16(rect.origin.y),
-                            width: Int16(rect.size.width),
-                            height: Int16(rect.size.height)
-                    )
+                    rect: rect.toULIPCRect().scale(x: sx, y: sy)
                 ),
                 type: TYPE_SCREEN_UPDATE_NOTIFY
             )
@@ -110,13 +144,11 @@ extension ProjectionSession: ScreenUpdateSubscriber {
             let pointer = CFDataGetBytePtr(rawData)!
             let length = CFDataGetLength(rawData)
 
+            let sx = mainViewport?.scaleX(Int(screenResolution.width)) ?? 1.0
+            let sy = mainViewport?.scaleY(Int(screenResolution.height)) ?? 1.0
+
             let message = ULIPCScreenUpdateCommit(
-                screenRect: ULIPCRect(
-                        x: Int16(rect.origin.x),
-                        y: Int16(rect.origin.y),
-                        width: Int16(rect.size.width),
-                        height: Int16(rect.size.height)
-                ),
+                screenRect: rect.toULIPCRect().scale(x: sx, y: sy),
                 bitmapLength: UInt64(length)
             )
 
@@ -125,8 +157,8 @@ extension ProjectionSession: ScreenUpdateSubscriber {
         }
     }
 
-    func screenResolutionChanged(to resolution: (Int, Int)) {
-
+    func screenResolutionChanged(to resolution: CGSize) {
+        self.screenResolution = resolution
     }
 
 }