diff --git a/Scoot.xcodeproj/project.pbxproj b/Scoot.xcodeproj/project.pbxproj index 39ee89e..00cde93 100644 --- a/Scoot.xcodeproj/project.pbxproj +++ b/Scoot.xcodeproj/project.pbxproj @@ -32,6 +32,8 @@ E59C1631264185940033E2CC /* TreeTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = E59C1630264185940033E2CC /* TreeTests.swift */; }; E59C1633264AC4F60033E2CC /* CGSize+Extensions.swift in Sources */ = {isa = PBXBuildFile; fileRef = E59C1632264AC4F60033E2CC /* CGSize+Extensions.swift */; }; E59C1635264AC6FB0033E2CC /* Comparable+Extensions.swift in Sources */ = {isa = PBXBuildFile; fileRef = E59C1634264AC6FB0033E2CC /* Comparable+Extensions.swift */; }; + E59F854C278938F100A9FBF6 /* Positionable.swift in Sources */ = {isa = PBXBuildFile; fileRef = E59F854B278938F000A9FBF6 /* Positionable.swift */; }; + E59F854F27893B3A00A9FBF6 /* Positionable+Crowding.swift in Sources */ = {isa = PBXBuildFile; fileRef = E59F854E27893B3A00A9FBF6 /* Positionable+Crowding.swift */; }; E5B1DE4126686E5B00F1AA77 /* KeyboardInputWindow+UI.swift in Sources */ = {isa = PBXBuildFile; fileRef = E5B1DE4026686E5B00F1AA77 /* KeyboardInputWindow+UI.swift */; }; E5B1DE4326686E9D00F1AA77 /* KeyboardInputWindow+Actions.swift in Sources */ = {isa = PBXBuildFile; fileRef = E5B1DE4226686E9D00F1AA77 /* KeyboardInputWindow+Actions.swift */; }; E5B9214A2656A18D000A0A75 /* FeedbackView.swift in Sources */ = {isa = PBXBuildFile; fileRef = E5B921492656A18D000A0A75 /* FeedbackView.swift */; }; @@ -40,6 +42,8 @@ E5BD595C26615F5300BB5181 /* JumpWindow.swift in Sources */ = {isa = PBXBuildFile; fileRef = E5BD595B26615F5300BB5181 /* JumpWindow.swift */; }; E5BE2E9126F25D7100BB754D /* AXSwift in Frameworks */ = {isa = PBXBuildFile; productRef = E5BE2E9026F25D7100BB754D /* AXSwift */; }; E5E87FFD2630A3550094FE9B /* GridView.swift in Sources */ = {isa = PBXBuildFile; fileRef = E5E87FFC2630A3550094FE9B /* GridView.swift */; }; + E5EB845127837F380086A5D6 /* GeometryTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = E5EB845027837F380086A5D6 /* GeometryTests.swift */; }; + E5EB84532787E3F80086A5D6 /* PositionableTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = E5EB84522787E3F80086A5D6 /* PositionableTests.swift */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ @@ -90,6 +94,8 @@ E59C1630264185940033E2CC /* TreeTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TreeTests.swift; sourceTree = ""; }; E59C1632264AC4F60033E2CC /* CGSize+Extensions.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CGSize+Extensions.swift"; sourceTree = ""; }; E59C1634264AC6FB0033E2CC /* Comparable+Extensions.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Comparable+Extensions.swift"; sourceTree = ""; }; + E59F854B278938F000A9FBF6 /* Positionable.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Positionable.swift; sourceTree = ""; }; + E59F854E27893B3A00A9FBF6 /* Positionable+Crowding.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Positionable+Crowding.swift"; sourceTree = ""; }; E5B1DE4026686E5B00F1AA77 /* KeyboardInputWindow+UI.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "KeyboardInputWindow+UI.swift"; sourceTree = ""; }; E5B1DE4226686E9D00F1AA77 /* KeyboardInputWindow+Actions.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "KeyboardInputWindow+Actions.swift"; sourceTree = ""; }; E5B921492656A18D000A0A75 /* FeedbackView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FeedbackView.swift; sourceTree = ""; }; @@ -98,6 +104,8 @@ E5BD595B26615F5300BB5181 /* JumpWindow.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = JumpWindow.swift; sourceTree = ""; }; E5E87FF4263061250094FE9B /* README.md */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = net.daringfireball.markdown; name = README.md; path = ../scoot/README.md; sourceTree = ""; }; E5E87FFC2630A3550094FE9B /* GridView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; name = GridView.swift; path = ../../scoot/Scoot/GridView.swift; sourceTree = ""; }; + E5EB845027837F380086A5D6 /* GeometryTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = GeometryTests.swift; sourceTree = ""; }; + E5EB84522787E3F80086A5D6 /* PositionableTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PositionableTests.swift; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -167,6 +175,8 @@ E5776232262F04D300849D7D /* Main.storyboard */, E577622C262F04D100849D7D /* AppDelegate.swift */, E52BFD162763D5FB00BB65AF /* Accessibility.swift */, + E59F854B278938F000A9FBF6 /* Positionable.swift */, + E59F854E27893B3A00A9FBF6 /* Positionable+Crowding.swift */, E5776260262F067700849D7D /* TransparentWindow.swift */, E5BD595726615DB700BB5181 /* KeyboardInputWindow.swift */, E5BD595926615E5600BB5181 /* KeyboardInputWindow+Delegate.swift */, @@ -196,7 +206,9 @@ isa = PBXGroup; children = ( E59C1629263E52540033E2CC /* GridTests.swift */, + E5EB845027837F380086A5D6 /* GeometryTests.swift */, E59C1630264185940033E2CC /* TreeTests.swift */, + E5EB84522787E3F80086A5D6 /* PositionableTests.swift */, E5776241262F04D300849D7D /* Info.plist */, ); path = ScootTests; @@ -356,6 +368,7 @@ E511785B2636EEC800B4202F /* NSScreen+Extensions.swift in Sources */, E567D2A2263C77FC00A0A861 /* Grid.swift in Sources */, E5776261262F067700849D7D /* TransparentWindow.swift in Sources */, + E59F854C278938F100A9FBF6 /* Positionable.swift in Sources */, E59C1633264AC4F60033E2CC /* CGSize+Extensions.swift in Sources */, E5776269262F0EC700849D7D /* JumpWindowController.swift in Sources */, E577622F262F04D100849D7D /* JumpViewController.swift in Sources */, @@ -373,6 +386,7 @@ E59C1635264AC6FB0033E2CC /* Comparable+Extensions.swift in Sources */, E5B1DE4126686E5B00F1AA77 /* KeyboardInputWindow+UI.swift in Sources */, E5BD595C26615F5300BB5181 /* JumpWindow.swift in Sources */, + E59F854F27893B3A00A9FBF6 /* Positionable+Crowding.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -381,7 +395,9 @@ buildActionMask = 2147483647; files = ( E59C162A263E52540033E2CC /* GridTests.swift in Sources */, + E5EB845127837F380086A5D6 /* GeometryTests.swift in Sources */, E59C1631264185940033E2CC /* TreeTests.swift in Sources */, + E5EB84532787E3F80086A5D6 /* PositionableTests.swift in Sources */, E59C162F264184500033E2CC /* Tree.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; diff --git a/Scoot/Accessibility.swift b/Scoot/Accessibility.swift index ce8c7e0..689f6d2 100644 --- a/Scoot/Accessibility.swift +++ b/Scoot/Accessibility.swift @@ -3,7 +3,7 @@ import AppKit struct Accessibility { - struct Element { + struct Element: Positionable, Equatable { let role: Role let subrole: Subrole? diff --git a/Scoot/Extensions/CGRect+Extensions.swift b/Scoot/Extensions/CGRect+Extensions.swift index b99f10d..b1e53c8 100644 --- a/Scoot/Extensions/CGRect+Extensions.swift +++ b/Scoot/Extensions/CGRect+Extensions.swift @@ -1,5 +1,7 @@ import Cocoa +// MARK: - Coordinate System Conversions + extension CGRect { /// Convert from the Core Graphics/ Quartz/ Carbon coordinate system @@ -13,4 +15,29 @@ extension CGRect { width: self.width, height: self.height) } + +} + +// MARK: - Geometry + +extension CGRect { + + /// The area of the rectangle. + var area: CGFloat { + self.width * self.height + } + + /// Returns the percentage overlap between the two rects. + func percentageOverlapping(_ other: CGRect) -> CGFloat { + guard self.intersects(other) else { + return 0 + } + + if self.contains(other) || other.contains(self) { + return 1 + } + + return self.intersection(other).area / ((self.area + other.area) / 2) + } + } diff --git a/Scoot/KeyboardInputWindow.swift b/Scoot/KeyboardInputWindow.swift index 3e99a1c..7901a39 100644 --- a/Scoot/KeyboardInputWindow.swift +++ b/Scoot/KeyboardInputWindow.swift @@ -165,13 +165,11 @@ class KeyboardInputWindow: TransparentWindow { } let elements = Accessibility - .getAccessibleElementsForFocusedWindow(of: app) - .reduce([], { // Filter out any elements that have duplicate frames. - accumulator, element in - - accumulator.contains(where: { element.frame == $0.frame }) - ? accumulator : accumulator + [element] - }) + .getAccessibleElementsForFocusedWindow(of: app) + // Because Scoot places labels vertically, horizontal congestion is + // less of an issue in practice. For this reason, add padding in the y + // direction only (`paddingY`). + .reducingCrowding(intersectionThreshold: 0.1, paddingX: 0.0, paddingY: 10.0) var data = [(elements: [Accessibility.Element], screenRects: [CGRect])]() diff --git a/Scoot/Positionable+Crowding.swift b/Scoot/Positionable+Crowding.swift new file mode 100644 index 0000000..a223933 --- /dev/null +++ b/Scoot/Positionable+Crowding.swift @@ -0,0 +1,90 @@ +import Cocoa + +extension Array where Element: Positionable, Element: Equatable { + + /// Implements a naive algorithm to reduce "crowding", by filtering out + /// elements that are positioned too closely together. + /// + /// If two elements have duplicate frames, one of the elements is removed. + /// + /// If two elements have frames that overlap more than + /// `intersectionThreshold`%, the element with the smaller frame is + /// removed. + /// + /// If two elements have frames that do not intersect, but do intersect + /// with padding applied (see `paddingX` and `paddingY` parameters), the + /// element with the smaller frame is removed. + /// + /// - Parameter intersectionThreshold: the percentage (expressed as a float + /// ranging between 0 and 1) that two elements' frames need to overlap in + /// order for the element with the smaller frame to be removed. + /// + /// - Parameter paddingX: the number of pixels to increase the width of an + /// element's frame by, when testing to see if the padded frame intersects + /// with another element. + /// + /// - Parameter paddingY: the number of pixels to increase the height of an + /// element's frame by, when testing to see if the padded frame intersects + /// with another element. + func reducingCrowding(intersectionThreshold: CGFloat = 0.1, paddingX: CGFloat = 0, paddingY: CGFloat = 0) -> [Element] { + + var discard = [Element]() + + func nextPartialResult(accumulator: [Element], element: Element) -> [Element] { + + let predicate: (Element) -> Bool = { + + // An element that is already in the accumulator. + let accumulated = $0 + + // The element currently under consideration for inclusion in + // the accumulator. + let candidate = element + + if candidate.frame == accumulated.frame { + // Do not include the candidate in the accumulator. + return true + } + + let framesIntersect = candidate.frame.intersects(accumulated.frame) + + let percentageOverlapping = candidate.frame.percentageOverlapping(accumulated.frame) + + let paddedFramesIntersect = candidate.frame.insetBy(dx: -paddingX, dy: -paddingY).intersects(accumulated.frame) + + if (framesIntersect && percentageOverlapping >= intersectionThreshold) || (!framesIntersect && paddedFramesIntersect) { + // To reduce crowding, only one element is kept (either the + // candidate, or the accumulated). We choose the element + // with the larger area. + if candidate.frame.area < accumulated.frame.area { + // The frame with the larger area is already in the + // accumulator. Don't include the candidate. + return true + } else { + // Include the candidate in the accumulator, because + // its frame has a larger area. (However, the other + // element is already in the accumulator, and needs to + // be removed.) + discard.append(accumulated) + return false + } + } + + // No other criteria disqualified the candidate; add it to the + // accumulator. + return false + } + + if accumulator.contains(where: predicate) { + return accumulator + } else { + return accumulator + [element] + } + } + + return self.reduce([], nextPartialResult).filter { + !discard.contains($0) + } + + } +} diff --git a/Scoot/Positionable.swift b/Scoot/Positionable.swift new file mode 100644 index 0000000..5b47d22 --- /dev/null +++ b/Scoot/Positionable.swift @@ -0,0 +1,5 @@ +import Cocoa + +protocol Positionable { + var frame: CGRect { get } +} diff --git a/ScootTests/GeometryTests.swift b/ScootTests/GeometryTests.swift new file mode 100644 index 0000000..64d49b8 --- /dev/null +++ b/ScootTests/GeometryTests.swift @@ -0,0 +1,86 @@ +import XCTest +@testable import Scoot + +func makeRect(width: CGFloat, height: CGFloat) -> CGRect { + CGRect(origin: .zero, size: CGSize(width: width, height: height)) +} + +class GeometryTests: XCTestCase { + + func testRectArea() { + XCTAssertEqual(makeRect(width: 0, height: 0).area, 0) + XCTAssertEqual(makeRect(width: 0, height: 1).area, 0) + XCTAssertEqual(makeRect(width: 2, height: 4).area, 8) + XCTAssertEqual(makeRect(width: 4, height: 2).area, 8) + XCTAssertEqual(makeRect(width: -4, height: 2).area, 8) + XCTAssertEqual(makeRect(width: 4, height: -2).area, 8) + XCTAssertEqual(makeRect(width: 16, height: 4).area, 64) + } + + func testRectNoOverlap() { + let a = CGRect(x: 1, y: 1, width: 4, height: 4) + let b = CGRect(x: 9, y: 9, width: 4, height: 4) + + XCTAssertEqual(a.percentageOverlapping(b), 0) + + let c = CGRect(x: 1, y: 1, width: 0, height: 0) + let d = CGRect(x: 2, y: 2, width: 0, height: 0) + + XCTAssertEqual(c.percentageOverlapping(d), 0) + } + + func testRectFullOverlap() { + let a = CGRect(x: 0, y: 0, width: 4, height: 4) + let b = CGRect(x: 0, y: 0, width: 4, height: 4) + + XCTAssertEqual(a.percentageOverlapping(b), 1) + + let c = CGRect(x: 0, y: 0, width: 4, height: 4) + let d = CGRect(x: 1, y: 1, width: 2, height: 2) + + XCTAssertEqual(c.percentageOverlapping(d), 1) + + let e = CGRect(x: 1, y: 1, width: 2, height: 2) + let f = CGRect(x: 0, y: 0, width: 4, height: 4) + + XCTAssertEqual(e.percentageOverlapping(f), 1) + + let g = CGRect(x: 0, y: 0, width: 4, height: 4) + let h = CGRect(x: 2, y: 2, width: 2, height: 2) + + XCTAssertEqual(g.percentageOverlapping(h), 1) + } + + func testRectPartialOverlap() { + let a = CGRect(x: 0, y: 0, width: 4, height: 4) + let b = CGRect(x: 2, y: 2, width: 4, height: 4) + + XCTAssertEqual(a.percentageOverlapping(b), 0.25) + + let c = CGRect(x: 0, y: 0, width: 4, height: 4) + let d = CGRect(x: 3, y: 3, width: 4, height: 4) + + XCTAssertEqual(c.percentageOverlapping(d), 0.0625) + + let e = CGRect(x: 0, y: 0, width: 4, height: 4) + let f = CGRect(x: 1, y: 1, width: 4, height: 4) + + XCTAssertEqual(e.percentageOverlapping(f), 0.5625) + + let g = CGRect(x: 1, y: 1, width: 4, height: 4) + let h = CGRect(x: 1.2, y: 1.2, width: 4, height: 4) + + XCTAssertEqual(g.percentageOverlapping(h), 0.9025) + + let i = CGRect(x: -1, y: 0, width: 10, height: 10) + let j = CGRect(x: 1, y: 0, width: 10, height: 10) + + XCTAssertEqual(i.percentageOverlapping(j), 0.8) + + let k = CGRect(x: -1, y: 0, width: 10, height: 10) + let l = CGRect(x: 1, y: 0, width: 30, height: 10) + + XCTAssertEqual(k.percentageOverlapping(l), 0.4) + } + +} diff --git a/ScootTests/PositionableTests.swift b/ScootTests/PositionableTests.swift new file mode 100644 index 0000000..6c09fb7 --- /dev/null +++ b/ScootTests/PositionableTests.swift @@ -0,0 +1,139 @@ +import XCTest +@testable import Scoot + +struct Item: Positionable, Equatable { + var frame: CGRect +} + +func reduceCrowding(_ items: [Item]) -> [Item] { + items.reducingCrowding(intersectionThreshold: 0.1, paddingX: 0, paddingY: 10) +} + +class PositionableTests: XCTestCase { + + func testNoItemsRemovedWhenFramesAreNotCloseEnough() { + var items = [Item]() + + XCTAssertEqual(reduceCrowding(items), items) + + items = [Item(frame: .zero)] + + XCTAssertEqual(reduceCrowding(items), items) + + items = [ + Item(frame: CGRect(origin: .zero, size: CGSize(width: 10, height: 10))), + Item(frame: CGRect(origin: CGPoint(x: 20, y: 20), size: CGSize(width: 10, height: 10))), + ] + + XCTAssertEqual(reduceCrowding(items), items) + + items = [ + Item(frame: CGRect(origin: .zero, size: CGSize(width: 10, height: 10))), + Item(frame: CGRect(origin: CGPoint(x: 20, y: 20), size: CGSize(width: 10, height: 10))), + Item(frame: CGRect(origin: CGPoint(x: 40, y: 40), size: CGSize(width: 10, height: 10))), + ] + + XCTAssertEqual(reduceCrowding(items), items) + } + + func testItemsWithDuplicateFramesRemoved() { + let item = Item(frame: CGRect(origin: CGPoint(x: 2, y: 4), size: CGSize(width: 10, height: 10))) + + XCTAssertEqual(reduceCrowding([item, item, item, item, item]), [item]) + XCTAssertEqual(reduceCrowding([item, item, item, item]), [item]) + XCTAssertEqual(reduceCrowding([item, item, item]), [item]) + XCTAssertEqual(reduceCrowding([item, item, item]), [item]) + XCTAssertEqual(reduceCrowding([item, item]), [item]) + XCTAssertEqual(reduceCrowding([item]), [item]) + + let other = Item(frame: .zero) + + XCTAssertEqual(reduceCrowding([item, other, item, item, item, item]), [item, other]) + XCTAssertEqual(reduceCrowding([item, other, item, item, item]), [item, other]) + XCTAssertEqual(reduceCrowding([item, other, item, item]), [item, other]) + XCTAssertEqual(reduceCrowding([item, other, item, item]), [item, other]) + XCTAssertEqual(reduceCrowding([item, other, item]), [item, other]) + XCTAssertEqual(reduceCrowding([item, other]), [item, other]) + + XCTAssertEqual(reduceCrowding([other, item, item, item, item, item]), [other, item]) + } + + func testItemsNotRemovedWhenFramesOverlapLessThanTenPercent() { + let size = CGSize(width: 10, height: 10) + + let a = Item(frame: CGRect(origin: .zero, size: size)) + let b = Item(frame: CGRect(origin: CGPoint(x: 9, y: 9), size: size)) + + let items = [a, b] + + let percentageOverlapping = a.frame.percentageOverlapping(b.frame) + + XCTAssertGreaterThan(percentageOverlapping, 0) + XCTAssertLessThan(percentageOverlapping, 0.1) + + XCTAssertEqual(reduceCrowding(items), items) + } + + func testItemsRemovedWhenFramesOverlapAtLeastTenPercent() { + let size = CGSize(width: 10, height: 10) + + let a0 = Item(frame: CGRect(origin: .zero, size: size + CGSize(width: 1, height: 1))) + let a1 = Item(frame: CGRect(origin: .zero, size: size + CGSize(width: 0, height: 0))) + + let b0 = Item(frame: CGRect(origin: CGPoint(x: 6.5, y: 6.5), size: size + CGSize(width: 0, height: 0))) + let b1 = Item(frame: CGRect(origin: CGPoint(x: 6.5, y: 6.5), size: size + CGSize(width: 1, height: 1))) + + let overlap0 = a0.frame.percentageOverlapping(b0.frame) + + XCTAssertGreaterThan(overlap0, 0) + XCTAssertGreaterThanOrEqual(overlap0, 0.1) + + XCTAssertEqual(reduceCrowding([a0, b0]), [a0]) + XCTAssertEqual(reduceCrowding([b0, a0]), [a0]) + + let overlap1 = a1.frame.percentageOverlapping(b1.frame) + + XCTAssertGreaterThan(overlap1, 0) + XCTAssertGreaterThanOrEqual(overlap1, 0.1) + + XCTAssertEqual(reduceCrowding([a1, b1]), [b1]) + XCTAssertEqual(reduceCrowding([b1, a1]), [b1]) + } + + func testItemsRemovedWhenPaddedFrameOverlaps() { + let smaller = CGSize(width: 10, height: 10) + let bigger = CGSize(width: 10.5, height: 10.5) + + let a = Item(frame: CGRect(origin: CGPoint(x: 0, y: 0), size: smaller)) + let b = Item(frame: CGRect(origin: CGPoint(x: 0, y: 11), size: bigger)) + + XCTAssertFalse(a.frame.intersects(b.frame)) + + XCTAssertEqual(reduceCrowding([a, b]), [b]) + XCTAssertEqual(reduceCrowding([b, a]), [b]) + } + + func testComplexCrowdingScenarios() { + let a = Item(frame: CGRect(origin: .zero, size: CGSize(width: 1, height: 1))) + let b = Item(frame: CGRect(origin: .zero, size: CGSize(width: 1, height: 1))) + + let c = Item(frame: CGRect(origin: CGPoint(x: -1, y: 4), size: CGSize(width: 8, height: 8))) + let d = Item(frame: CGRect(origin: CGPoint(x: -1, y: 4), size: CGSize(width: 8, height: 8))) + + let e = Item(frame: CGRect(origin: CGPoint(x: -9, y: 9), size: CGSize(width: 2, height: 2))) + let f = Item(frame: CGRect(origin: CGPoint(x: -9, y: 9), size: CGSize(width: 2, height: 2))) + + XCTAssertEqual(reduceCrowding([a]), [a]) + + XCTAssertEqual(reduceCrowding([a, c]), [c]) + + XCTAssertEqual(reduceCrowding([a, c, e]), [c, e]) + + XCTAssertEqual(reduceCrowding([a, b, c, d, e, f]), [c, e]) + + XCTAssertEqual(reduceCrowding([a, b, a, b, c, d, c, d, e, f, e, f]), [c, e]) + + XCTAssertEqual(reduceCrowding([a, b, b, a, c, d, d, c, e, f, f, e]), [c, e]) + } + +}