Skip to content

Commit

Permalink
Improve legibility when many elements are in close proximity.
Browse files Browse the repository at this point in the history
Resolves #2.
  • Loading branch information
mjrusso committed Jan 9, 2022
1 parent 44ab726 commit 323229b
Show file tree
Hide file tree
Showing 8 changed files with 369 additions and 8 deletions.
16 changes: 16 additions & 0 deletions Scoot.xcodeproj/project.pbxproj
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@
E59C1631264185940033E2CC /* TreeTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = E59C1630264185940033E2CC /* TreeTests.swift */; };
E59C1633264AC4F60033E2CC /* CGSize+Extensions.swift in Sources */ = {isa = PBXBuildFile; fileRef = E59C1632264AC4F60033E2CC /* CGSize+Extensions.swift */; };
E59C1635264AC6FB0033E2CC /* Comparable+Extensions.swift in Sources */ = {isa = PBXBuildFile; fileRef = E59C1634264AC6FB0033E2CC /* Comparable+Extensions.swift */; };
E59F854C278938F100A9FBF6 /* Positionable.swift in Sources */ = {isa = PBXBuildFile; fileRef = E59F854B278938F000A9FBF6 /* Positionable.swift */; };
E59F854F27893B3A00A9FBF6 /* Positionable+Crowding.swift in Sources */ = {isa = PBXBuildFile; fileRef = E59F854E27893B3A00A9FBF6 /* Positionable+Crowding.swift */; };
E5B1DE4126686E5B00F1AA77 /* KeyboardInputWindow+UI.swift in Sources */ = {isa = PBXBuildFile; fileRef = E5B1DE4026686E5B00F1AA77 /* KeyboardInputWindow+UI.swift */; };
E5B1DE4326686E9D00F1AA77 /* KeyboardInputWindow+Actions.swift in Sources */ = {isa = PBXBuildFile; fileRef = E5B1DE4226686E9D00F1AA77 /* KeyboardInputWindow+Actions.swift */; };
E5B9214A2656A18D000A0A75 /* FeedbackView.swift in Sources */ = {isa = PBXBuildFile; fileRef = E5B921492656A18D000A0A75 /* FeedbackView.swift */; };
Expand All @@ -40,6 +42,8 @@
E5BD595C26615F5300BB5181 /* JumpWindow.swift in Sources */ = {isa = PBXBuildFile; fileRef = E5BD595B26615F5300BB5181 /* JumpWindow.swift */; };
E5BE2E9126F25D7100BB754D /* AXSwift in Frameworks */ = {isa = PBXBuildFile; productRef = E5BE2E9026F25D7100BB754D /* AXSwift */; };
E5E87FFD2630A3550094FE9B /* GridView.swift in Sources */ = {isa = PBXBuildFile; fileRef = E5E87FFC2630A3550094FE9B /* GridView.swift */; };
E5EB845127837F380086A5D6 /* GeometryTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = E5EB845027837F380086A5D6 /* GeometryTests.swift */; };
E5EB84532787E3F80086A5D6 /* PositionableTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = E5EB84522787E3F80086A5D6 /* PositionableTests.swift */; };
/* End PBXBuildFile section */

/* Begin PBXContainerItemProxy section */
Expand Down Expand Up @@ -90,6 +94,8 @@
E59C1630264185940033E2CC /* TreeTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TreeTests.swift; sourceTree = "<group>"; };
E59C1632264AC4F60033E2CC /* CGSize+Extensions.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CGSize+Extensions.swift"; sourceTree = "<group>"; };
E59C1634264AC6FB0033E2CC /* Comparable+Extensions.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Comparable+Extensions.swift"; sourceTree = "<group>"; };
E59F854B278938F000A9FBF6 /* Positionable.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Positionable.swift; sourceTree = "<group>"; };
E59F854E27893B3A00A9FBF6 /* Positionable+Crowding.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Positionable+Crowding.swift"; sourceTree = "<group>"; };
E5B1DE4026686E5B00F1AA77 /* KeyboardInputWindow+UI.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "KeyboardInputWindow+UI.swift"; sourceTree = "<group>"; };
E5B1DE4226686E9D00F1AA77 /* KeyboardInputWindow+Actions.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "KeyboardInputWindow+Actions.swift"; sourceTree = "<group>"; };
E5B921492656A18D000A0A75 /* FeedbackView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FeedbackView.swift; sourceTree = "<group>"; };
Expand All @@ -98,6 +104,8 @@
E5BD595B26615F5300BB5181 /* JumpWindow.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = JumpWindow.swift; sourceTree = "<group>"; };
E5E87FF4263061250094FE9B /* README.md */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = net.daringfireball.markdown; name = README.md; path = ../scoot/README.md; sourceTree = "<group>"; };
E5E87FFC2630A3550094FE9B /* GridView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; name = GridView.swift; path = ../../scoot/Scoot/GridView.swift; sourceTree = "<group>"; };
E5EB845027837F380086A5D6 /* GeometryTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = GeometryTests.swift; sourceTree = "<group>"; };
E5EB84522787E3F80086A5D6 /* PositionableTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PositionableTests.swift; sourceTree = "<group>"; };
/* End PBXFileReference section */

/* Begin PBXFrameworksBuildPhase section */
Expand Down Expand Up @@ -167,6 +175,8 @@
E5776232262F04D300849D7D /* Main.storyboard */,
E577622C262F04D100849D7D /* AppDelegate.swift */,
E52BFD162763D5FB00BB65AF /* Accessibility.swift */,
E59F854B278938F000A9FBF6 /* Positionable.swift */,
E59F854E27893B3A00A9FBF6 /* Positionable+Crowding.swift */,
E5776260262F067700849D7D /* TransparentWindow.swift */,
E5BD595726615DB700BB5181 /* KeyboardInputWindow.swift */,
E5BD595926615E5600BB5181 /* KeyboardInputWindow+Delegate.swift */,
Expand Down Expand Up @@ -196,7 +206,9 @@
isa = PBXGroup;
children = (
E59C1629263E52540033E2CC /* GridTests.swift */,
E5EB845027837F380086A5D6 /* GeometryTests.swift */,
E59C1630264185940033E2CC /* TreeTests.swift */,
E5EB84522787E3F80086A5D6 /* PositionableTests.swift */,
E5776241262F04D300849D7D /* Info.plist */,
);
path = ScootTests;
Expand Down Expand Up @@ -356,6 +368,7 @@
E511785B2636EEC800B4202F /* NSScreen+Extensions.swift in Sources */,
E567D2A2263C77FC00A0A861 /* Grid.swift in Sources */,
E5776261262F067700849D7D /* TransparentWindow.swift in Sources */,
E59F854C278938F100A9FBF6 /* Positionable.swift in Sources */,
E59C1633264AC4F60033E2CC /* CGSize+Extensions.swift in Sources */,
E5776269262F0EC700849D7D /* JumpWindowController.swift in Sources */,
E577622F262F04D100849D7D /* JumpViewController.swift in Sources */,
Expand All @@ -373,6 +386,7 @@
E59C1635264AC6FB0033E2CC /* Comparable+Extensions.swift in Sources */,
E5B1DE4126686E5B00F1AA77 /* KeyboardInputWindow+UI.swift in Sources */,
E5BD595C26615F5300BB5181 /* JumpWindow.swift in Sources */,
E59F854F27893B3A00A9FBF6 /* Positionable+Crowding.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
Expand All @@ -381,7 +395,9 @@
buildActionMask = 2147483647;
files = (
E59C162A263E52540033E2CC /* GridTests.swift in Sources */,
E5EB845127837F380086A5D6 /* GeometryTests.swift in Sources */,
E59C1631264185940033E2CC /* TreeTests.swift in Sources */,
E5EB84532787E3F80086A5D6 /* PositionableTests.swift in Sources */,
E59C162F264184500033E2CC /* Tree.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
Expand Down
2 changes: 1 addition & 1 deletion Scoot/Accessibility.swift
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import AppKit

struct Accessibility {

struct Element {
struct Element: Positionable, Equatable {
let role: Role

let subrole: Subrole?
Expand Down
27 changes: 27 additions & 0 deletions Scoot/Extensions/CGRect+Extensions.swift
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import Cocoa

// MARK: - Coordinate System Conversions

extension CGRect {

/// Convert from the Core Graphics/ Quartz/ Carbon coordinate system
Expand All @@ -13,4 +15,29 @@ extension CGRect {
width: self.width,
height: self.height)
}

}

// MARK: - Geometry

extension CGRect {

/// The area of the rectangle.
var area: CGFloat {
self.width * self.height
}

/// Returns the percentage overlap between the two rects.
func percentageOverlapping(_ other: CGRect) -> CGFloat {
guard self.intersects(other) else {
return 0
}

if self.contains(other) || other.contains(self) {
return 1
}

return self.intersection(other).area / ((self.area + other.area) / 2)
}

}
12 changes: 5 additions & 7 deletions Scoot/KeyboardInputWindow.swift
Original file line number Diff line number Diff line change
Expand Up @@ -165,13 +165,11 @@ class KeyboardInputWindow: TransparentWindow {
}

let elements = Accessibility
.getAccessibleElementsForFocusedWindow(of: app)
.reduce([], { // Filter out any elements that have duplicate frames.
accumulator, element in

accumulator.contains(where: { element.frame == $0.frame })
? accumulator : accumulator + [element]
})
.getAccessibleElementsForFocusedWindow(of: app)
// Because Scoot places labels vertically, horizontal congestion is
// less of an issue in practice. For this reason, add padding in the y
// direction only (`paddingY`).
.reducingCrowding(intersectionThreshold: 0.1, paddingX: 0.0, paddingY: 10.0)

var data = [(elements: [Accessibility.Element], screenRects: [CGRect])]()

Expand Down
90 changes: 90 additions & 0 deletions Scoot/Positionable+Crowding.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
import Cocoa

extension Array where Element: Positionable, Element: Equatable {

/// Implements a naive algorithm to reduce "crowding", by filtering out
/// elements that are positioned too closely together.
///
/// If two elements have duplicate frames, one of the elements is removed.
///
/// If two elements have frames that overlap more than
/// `intersectionThreshold`%, the element with the smaller frame is
/// removed.
///
/// If two elements have frames that do not intersect, but do intersect
/// with padding applied (see `paddingX` and `paddingY` parameters), the
/// element with the smaller frame is removed.
///
/// - Parameter intersectionThreshold: the percentage (expressed as a float
/// ranging between 0 and 1) that two elements' frames need to overlap in
/// order for the element with the smaller frame to be removed.
///
/// - Parameter paddingX: the number of pixels to increase the width of an
/// element's frame by, when testing to see if the padded frame intersects
/// with another element.
///
/// - Parameter paddingY: the number of pixels to increase the height of an
/// element's frame by, when testing to see if the padded frame intersects
/// with another element.
func reducingCrowding(intersectionThreshold: CGFloat = 0.1, paddingX: CGFloat = 0, paddingY: CGFloat = 0) -> [Element] {

var discard = [Element]()

func nextPartialResult(accumulator: [Element], element: Element) -> [Element] {

let predicate: (Element) -> Bool = {

// An element that is already in the accumulator.
let accumulated = $0

// The element currently under consideration for inclusion in
// the accumulator.
let candidate = element

if candidate.frame == accumulated.frame {
// Do not include the candidate in the accumulator.
return true
}

let framesIntersect = candidate.frame.intersects(accumulated.frame)

let percentageOverlapping = candidate.frame.percentageOverlapping(accumulated.frame)

let paddedFramesIntersect = candidate.frame.insetBy(dx: -paddingX, dy: -paddingY).intersects(accumulated.frame)

if (framesIntersect && percentageOverlapping >= intersectionThreshold) || (!framesIntersect && paddedFramesIntersect) {
// To reduce crowding, only one element is kept (either the
// candidate, or the accumulated). We choose the element
// with the larger area.
if candidate.frame.area < accumulated.frame.area {
// The frame with the larger area is already in the
// accumulator. Don't include the candidate.
return true
} else {
// Include the candidate in the accumulator, because
// its frame has a larger area. (However, the other
// element is already in the accumulator, and needs to
// be removed.)
discard.append(accumulated)
return false
}
}

// No other criteria disqualified the candidate; add it to the
// accumulator.
return false
}

if accumulator.contains(where: predicate) {
return accumulator
} else {
return accumulator + [element]
}
}

return self.reduce([], nextPartialResult).filter {
!discard.contains($0)
}

}
}
5 changes: 5 additions & 0 deletions Scoot/Positionable.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
import Cocoa

protocol Positionable {
var frame: CGRect { get }
}
86 changes: 86 additions & 0 deletions ScootTests/GeometryTests.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
import XCTest
@testable import Scoot

func makeRect(width: CGFloat, height: CGFloat) -> CGRect {
CGRect(origin: .zero, size: CGSize(width: width, height: height))
}

class GeometryTests: XCTestCase {

func testRectArea() {
XCTAssertEqual(makeRect(width: 0, height: 0).area, 0)
XCTAssertEqual(makeRect(width: 0, height: 1).area, 0)
XCTAssertEqual(makeRect(width: 2, height: 4).area, 8)
XCTAssertEqual(makeRect(width: 4, height: 2).area, 8)
XCTAssertEqual(makeRect(width: -4, height: 2).area, 8)
XCTAssertEqual(makeRect(width: 4, height: -2).area, 8)
XCTAssertEqual(makeRect(width: 16, height: 4).area, 64)
}

func testRectNoOverlap() {
let a = CGRect(x: 1, y: 1, width: 4, height: 4)
let b = CGRect(x: 9, y: 9, width: 4, height: 4)

XCTAssertEqual(a.percentageOverlapping(b), 0)

let c = CGRect(x: 1, y: 1, width: 0, height: 0)
let d = CGRect(x: 2, y: 2, width: 0, height: 0)

XCTAssertEqual(c.percentageOverlapping(d), 0)
}

func testRectFullOverlap() {
let a = CGRect(x: 0, y: 0, width: 4, height: 4)
let b = CGRect(x: 0, y: 0, width: 4, height: 4)

XCTAssertEqual(a.percentageOverlapping(b), 1)

let c = CGRect(x: 0, y: 0, width: 4, height: 4)
let d = CGRect(x: 1, y: 1, width: 2, height: 2)

XCTAssertEqual(c.percentageOverlapping(d), 1)

let e = CGRect(x: 1, y: 1, width: 2, height: 2)
let f = CGRect(x: 0, y: 0, width: 4, height: 4)

XCTAssertEqual(e.percentageOverlapping(f), 1)

let g = CGRect(x: 0, y: 0, width: 4, height: 4)
let h = CGRect(x: 2, y: 2, width: 2, height: 2)

XCTAssertEqual(g.percentageOverlapping(h), 1)
}

func testRectPartialOverlap() {
let a = CGRect(x: 0, y: 0, width: 4, height: 4)
let b = CGRect(x: 2, y: 2, width: 4, height: 4)

XCTAssertEqual(a.percentageOverlapping(b), 0.25)

let c = CGRect(x: 0, y: 0, width: 4, height: 4)
let d = CGRect(x: 3, y: 3, width: 4, height: 4)

XCTAssertEqual(c.percentageOverlapping(d), 0.0625)

let e = CGRect(x: 0, y: 0, width: 4, height: 4)
let f = CGRect(x: 1, y: 1, width: 4, height: 4)

XCTAssertEqual(e.percentageOverlapping(f), 0.5625)

let g = CGRect(x: 1, y: 1, width: 4, height: 4)
let h = CGRect(x: 1.2, y: 1.2, width: 4, height: 4)

XCTAssertEqual(g.percentageOverlapping(h), 0.9025)

let i = CGRect(x: -1, y: 0, width: 10, height: 10)
let j = CGRect(x: 1, y: 0, width: 10, height: 10)

XCTAssertEqual(i.percentageOverlapping(j), 0.8)

let k = CGRect(x: -1, y: 0, width: 10, height: 10)
let l = CGRect(x: 1, y: 0, width: 30, height: 10)

XCTAssertEqual(k.percentageOverlapping(l), 0.4)
}

}
Loading

0 comments on commit 323229b

Please sign in to comment.