Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 33 additions & 8 deletions apps/desktop/native/computer-use/ComputerUse.swift
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,9 @@ func attributes(_ element: AXUIElement, _ names: [String]) -> [String: CFTypeRef
/// Elements bucketed by CFHash, so a walk's membership and parent lookups stay linear in its size.
struct ElementIndex {
private var buckets: [CFHashCode: [(element: AXUIElement, id: String)]] = [:]
private(set) var count = 0
func id(of element: AXUIElement) -> String? { buckets[CFHash(element)]?.first { CFEqual($0.element, element) }?.id }
mutating func insert(_ element: AXUIElement, id: String) { buckets[CFHash(element), default: []].append((element, id)) }
mutating func insert(_ element: AXUIElement, id: String) { buckets[CFHash(element), default: []].append((element, id)); count += 1 }
}
let NODE_ATTRIBUTES = [kAXRoleAttribute, kAXSubroleAttribute, kAXTitleAttribute, kAXDescriptionAttribute, kAXPlaceholderValueAttribute,
kAXHelpAttribute, kAXValueAttribute, kAXFocusedAttribute, kAXEnabledAttribute, kAXPositionAttribute, kAXSizeAttribute]
Expand Down Expand Up @@ -304,7 +305,18 @@ final class Snapshot {
var selectedWindowId: String?
var nodes: [[String: Any]] = []
var truncated = false
init(app: NSRunningApplication) { pid = app.processIdentifier; launchDate = app.launchDate }
/// Every element this app session has been given an id, so the same element keeps its id across observations.
private(set) var index = ElementIndex()
private(set) var nextOrdinal: Int
/// Reuses `previous`'s ids when it observed the same process instance and new ids continue from
/// `nextOrdinal`, which the driver keeps per process so ids never repeat while it runs.
init(app: NSRunningApplication, previous: Snapshot? = nil, nextOrdinal: Int = 0) {
pid = app.processIdentifier; launchDate = app.launchDate; self.nextOrdinal = nextOrdinal
// One read adds at most 2,000 elements, so the carried index stays within 20,000; past that the next read starts fresh.
if let previous, previous.pid == pid, previous.launchDate == launchDate, previous.index.count <= 18_000 {
index = previous.index
}
}
func read(preferredWindowFrame: CGRect? = nil) async throws {
try requireAccessibility()
let root = AXUIElementCreateApplication(pid)
Expand Down Expand Up @@ -342,12 +354,12 @@ final class Snapshot {
queue.append(AXTraversalEntry(element: unsafeBitCast(raw, to: AXUIElement.self), parent: root, depth: 1, menu: false, priority: true))
}
var menuCount = 0
var index = ElementIndex()
var seen = ElementIndex()
while nodes.count < 2000 && Date().timeIntervalSince(created) < 8, let entry = queue.next() {
try checkCancellation()
let element = entry.element
AXUIElementSetMessagingTimeout(element, 0.25)
if index.id(of: element) != nil { continue }
if seen.id(of: element) != nil { continue }
let values = attributes(element, NODE_ATTRIBUTES)
let role = values[kAXRoleAttribute] as? String ?? "AXUnknown"
let isMenu = entry.menu || [kAXMenuBarRole, kAXMenuRole, kAXMenuItemRole, kAXMenuBarItemRole].contains(role)
Expand All @@ -359,9 +371,13 @@ final class Snapshot {
guard menuCount < 80 else { continue }
menuCount += 1
}
let id = "e\(nodes.count)"; elements[id] = element; index.insert(element, id: id)
let id: String
if let known = index.id(of: element) { id = known } else {
id = "e\(nextOrdinal)"; nextOrdinal += 1; index.insert(element, id: id)
}
elements[id] = element; seen.insert(element, id: id)
var node: [String: Any] = ["elementId": id, "role": role, "actions": [String]()]
if let parent = entry.parent, let parentID = index.id(of: parent) { node["parentId"] = parentID }
if let parent = entry.parent, let parentID = seen.id(of: parent) { node["parentId"] = parentID }
let isSecure = values[kAXSubroleAttribute] as? String == kAXSecureTextFieldSubrole
var labels = [kAXTitleAttribute, kAXDescriptionAttribute].compactMap { values[$0] as? String }.filter { !$0.isEmpty }
if labels.isEmpty { labels = [kAXPlaceholderValueAttribute, kAXHelpAttribute].compactMap { values[$0] as? String }.filter { !$0.isEmpty } }
Expand Down Expand Up @@ -688,6 +704,10 @@ func discoverApps() throws -> [[String: Any]] {
@MainActor
final class Driver {
var snapshots: [String: Snapshot] = [:]
/// Each app's latest observation, kept after actions consume `snapshots` so the next read reuses its element ids.
var lastObserved: [String: Snapshot] = [:]
/// Each app's next element id, kept apart from the evictable history so ids never repeat while that process runs.
var nextOrdinals: [String: (pid: pid_t, launchDate: Date?, next: Int)] = [:]
func point(_ p: Parameters, app: NSRunningApplication, snapshot: Snapshot, end: Bool = false) throws -> CGPoint {
if let elementId = p.elementId, !end {
guard let frame = elementRect(try snapshot.element(elementId)) else { throw ComputerError("element_unavailable", "Element has no usable frame.") }
Expand Down Expand Up @@ -741,7 +761,9 @@ final class Driver {
return ["kind": "action", "action": "activate_app", "bundleId": app.bundleIdentifier!, "dispatched": true, "verified": true]
case "get_app_state":
try requireAccessibility()
let app = try await appFor(p.bundleId, launch: true); let snapshot = Snapshot(app: app)
let app = try await appFor(p.bundleId, launch: true); let bundle = app.bundleIdentifier!
let ordinal = nextOrdinals[bundle].flatMap { $0.pid == app.processIdentifier && $0.launchDate == app.launchDate ? $0.next : nil } ?? 0
let snapshot = Snapshot(app: app, previous: lastObserved[bundle], nextOrdinal: ordinal)
let preferredFrame = try p.windowId.map { try windowRect($0, pid: app.processIdentifier) }
try await snapshot.read(preferredWindowFrame: preferredFrame)
let axWindows = snapshot.elements.values.filter { attribute($0, kAXRoleAttribute) as? String == kAXWindowRole }
Expand All @@ -765,7 +787,10 @@ final class Driver {
result.merge(rectJSON(rect)) { _, new in new }; return result
}
if snapshots.count >= 16 { snapshots.removeAll() }
snapshots[app.bundleIdentifier!] = snapshot
if lastObserved[bundle] == nil, lastObserved.count >= 16, let evicted = lastObserved.keys.first { lastObserved.removeValue(forKey: evicted) }
snapshots[bundle] = snapshot
lastObserved[bundle] = snapshot
nextOrdinals[bundle] = (app.processIdentifier, app.launchDate, snapshot.nextOrdinal)
let root = AXUIElementCreateApplication(app.processIdentifier)
let focusedID = attribute(root, kAXFocusedWindowAttribute).flatMap { CFGetTypeID($0) == AXUIElementGetTypeID() ? snapshot.windowId(holding: unsafeBitCast($0, to: AXUIElement.self)) : nil }
for index in snapshot.nodes.indices {
Expand Down
11 changes: 9 additions & 2 deletions apps/desktop/native/computer-use/tests/Fixture.swift
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ final class InfiniteFrameView: NSView {
}
final class FixtureDelegate: NSObject, NSApplicationDelegate {
var window: NSWindow!
var stack: NSStackView!
var count = 0
let counter = NSTextField(labelWithString: "Count: 0")
let scrollCounter = NSTextField(labelWithString: "Scroll: 0")
Expand All @@ -47,14 +48,15 @@ final class FixtureDelegate: NSObject, NSApplicationDelegate {
NSApplication.shared.mainMenu = menu
window = NSWindow(contentRect: NSRect(x: 100, y: 100, width: 600, height: 720), styleMask: [.titled, .closable, .resizable], backing: .buffered, defer: false)
window.title = "Mothership Computer Use Test Fixture"
let stack = NSStackView(); stack.orientation = .vertical; stack.spacing = 16
stack = NSStackView(); stack.orientation = .vertical; stack.spacing = 16
stack.translatesAutoresizingMaskIntoConstraints = false
let field = NSTextField(string: "Fixture text"); field.setAccessibilityIdentifier("fixture-text"); field.setAccessibilityLabel("Fixture editable text")
let secure = NSSecureTextField(string: "fixture-only-secret"); secure.setAccessibilityIdentifier("fixture-password"); secure.setAccessibilityLabel("Fixture password")
let button = NSButton(title: "Increment", target: self, action: #selector(increment)); button.setAccessibilityIdentifier("fixture-increment")
let clickField = ClickFocusField(string: ""); clickField.setAccessibilityLabel("Fixture click-focus field")
let caretField = FixedCaretField(string: String(repeating: "keep ", count: 40)); caretField.setAccessibilityLabel("Fixture fixed-caret field")
let sheetButton = NSButton(title: "Show sheet", target: self, action: #selector(showSheet)); sheetButton.setAccessibilityIdentifier("fixture-show-sheet")
let addRowButton = NSButton(title: "Add row", target: self, action: #selector(addRow))
let slider = NSSlider(value: 25, minValue: 0, maxValue: 100, target: nil, action: nil); slider.setAccessibilityLabel("Fixture slider")
let time = NSDatePicker(); time.datePickerStyle = .textField; time.datePickerElements = .hourMinute; time.setAccessibilityLabel("Fixture time")
time.dateValue = Calendar.current.date(bySettingHour: 10, minute: 15, second: 0, of: Date()) ?? Date(); time.target = self; time.action = #selector(timeChanged(_:))
Expand All @@ -68,14 +70,19 @@ final class FixtureDelegate: NSObject, NSApplicationDelegate {
scroll.documentView = document
scroll.contentView.postsBoundsChangedNotifications = true
NotificationCenter.default.addObserver(self, selector: #selector(scrolled(_:)), name: NSView.boundsDidChangeNotification, object: scroll.contentView)
for view in [field, secure, button, counter, clickField, caretField, sheetButton, slider, time, timeLabel, marker, infinite, scrollCounter, scroll] { stack.addArrangedSubview(view) }
for view in [addRowButton, field, secure, button, counter, clickField, caretField, sheetButton, slider, time, timeLabel, marker, infinite, scrollCounter, scroll] { stack.addArrangedSubview(view) }
window.contentView!.addSubview(stack)
NSLayoutConstraint.activate([stack.leadingAnchor.constraint(equalTo: window.contentView!.leadingAnchor, constant: 30), stack.trailingAnchor.constraint(equalTo: window.contentView!.trailingAnchor, constant: -30), stack.topAnchor.constraint(equalTo: window.contentView!.topAnchor, constant: 30), field.widthAnchor.constraint(equalTo: stack.widthAnchor), secure.widthAnchor.constraint(equalTo: stack.widthAnchor), clickField.widthAnchor.constraint(equalTo: stack.widthAnchor), caretField.widthAnchor.constraint(equalTo: stack.widthAnchor), slider.widthAnchor.constraint(equalTo: stack.widthAnchor), scroll.widthAnchor.constraint(equalTo: stack.widthAnchor), scroll.heightAnchor.constraint(equalToConstant: 230), marker.widthAnchor.constraint(equalToConstant: 100), marker.heightAnchor.constraint(equalToConstant: 70)])
window.makeKeyAndOrderFront(nil)
window.makeFirstResponder(field)
NSApplication.shared.activate()
print("fixture-ready"); fflush(stdout)
}
/// Adds a label ahead of every control in Accessibility order, so positional ids after it would all shift.
@objc func addRow() {
let row = NSTextField(labelWithString: "Added row"); row.frame = NSRect(x: 4, y: 4, width: 120, height: 18)
window.contentView!.addSubview(row, positioned: .below, relativeTo: stack)
}
@objc func increment() { count += 1; counter.stringValue = "Count: \(count)" }
@objc func showSheet() { let alert = NSAlert(); alert.messageText = "Fixture sheet"; alert.addButton(withTitle: "Dismiss"); alert.beginSheetModal(for: window) }
@objc func timeChanged(_ picker: NSDatePicker) {
Expand Down
8 changes: 7 additions & 1 deletion apps/desktop/native/computer-use/tests/live.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,12 @@ def coordinate(snapshot, node, fx=0.5, fy=0.5):
else:
assert 'screen_capture_permission_required' in snapshot.get('screenshotError', '')
print('BLOCKED: screenshot verification skipped because Screen Recording grant is missing', flush=True)
# Increment follows the inserted row and is never the focused editor, which traversal always lists first.
increment_id = find(snapshot, role='AXButton', label='Increment')['elementId']
mutate('click', snapshot, elementId=find(snapshot, role='AXButton', label='Add row')['elementId'])
snapshot = state()
assert any(node.get('value', node.get('label')) == 'Added row' for node in snapshot['nodes']), 'Add row did not insert its label'
assert find(snapshot, role='AXButton', label='Increment')['elementId'] == increment_id, 'An element changed id when another was added above it'
baseline = int(next(node.get('value', node.get('label', '')) for node in snapshot['nodes'] if node.get('value', node.get('label', '')).startswith('Count: ')).split(': ')[1])
button = find(snapshot, role='AXButton', label='Increment')
snapshot = mutate('click', snapshot, elementId=button['elementId'])
Expand Down Expand Up @@ -212,7 +218,7 @@ def batch(snapshot, steps):
assert error['dispatchState'] == 'not_started', error
assert find(state(), role='AXTextField', label='Fixture click-focus field').get('value') == 'replaced', 'A background app was clicked or typed into'
call('activate_app', bundleId=bundle)
print('PASS: real fixture state, a large closed menu not reported as truncation, secure suppression, an element reported at infinity read without a frame, AX click/set/action, Unicode typing, keyboard chord, coordinate drag and scrolling, text refused by a slider, typing into a date picker, a window target typing into its focused editor, a title-bar click, a batch that applies, stops at a failing step, refuses before dispatch and stops when a sheet opens, click-to-focus for an editor that ignores Accessibility focus (inserting at the end, refusing an unconfirmed caret, and none in the background); every mutation verified in fresh state; screenshot=' + ('verified' if permissions['screenRecording'] else 'BLOCKED'))
print('PASS: real fixture state, element ids stable across observations, a large closed menu not reported as truncation, secure suppression, an element reported at infinity read without a frame, AX click/set/action, Unicode typing, keyboard chord, coordinate drag and scrolling, text refused by a slider, typing into a date picker, a window target typing into its focused editor, a title-bar click, a batch that applies, stops at a failing step, refuses before dispatch and stops when a sheet opens, click-to-focus for an editor that ignores Accessibility focus (inserting at the end, refusing an unconfirmed caret, and none in the background); every mutation verified in fresh state; screenshot=' + ('verified' if permissions['screenRecording'] else 'BLOCKED'))
finally:
if fixture:
fixture.terminate()
Expand Down
Loading