diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index a6afafb0f6f..33104ba8046 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -109,8 +109,9 @@ func attributes(_ element: AXUIElement, _ names: [String]) -> [String: CFTypeRef /// Elements bucketed by CFHash, so a walk's membership and parent lookups stay linear in its size. struct ElementIndex { private var buckets: [CFHashCode: [(element: AXUIElement, id: String)]] = [:] + private(set) var count = 0 func id(of element: AXUIElement) -> String? { buckets[CFHash(element)]?.first { CFEqual($0.element, element) }?.id } - mutating func insert(_ element: AXUIElement, id: String) { buckets[CFHash(element), default: []].append((element, id)) } + mutating func insert(_ element: AXUIElement, id: String) { buckets[CFHash(element), default: []].append((element, id)); count += 1 } } let NODE_ATTRIBUTES = [kAXRoleAttribute, kAXSubroleAttribute, kAXTitleAttribute, kAXDescriptionAttribute, kAXPlaceholderValueAttribute, kAXHelpAttribute, kAXValueAttribute, kAXFocusedAttribute, kAXEnabledAttribute, kAXPositionAttribute, kAXSizeAttribute] @@ -304,7 +305,18 @@ final class Snapshot { var selectedWindowId: String? var nodes: [[String: Any]] = [] var truncated = false - init(app: NSRunningApplication) { pid = app.processIdentifier; launchDate = app.launchDate } + /// Every element this app session has been given an id, so the same element keeps its id across observations. + private(set) var index = ElementIndex() + private(set) var nextOrdinal: Int + /// Reuses `previous`'s ids when it observed the same process instance and new ids continue from + /// `nextOrdinal`, which the driver keeps per process so ids never repeat while it runs. + init(app: NSRunningApplication, previous: Snapshot? = nil, nextOrdinal: Int = 0) { + pid = app.processIdentifier; launchDate = app.launchDate; self.nextOrdinal = nextOrdinal + // One read adds at most 2,000 elements, so the carried index stays within 20,000; past that the next read starts fresh. + if let previous, previous.pid == pid, previous.launchDate == launchDate, previous.index.count <= 18_000 { + index = previous.index + } + } func read(preferredWindowFrame: CGRect? = nil) async throws { try requireAccessibility() let root = AXUIElementCreateApplication(pid) @@ -342,12 +354,12 @@ final class Snapshot { queue.append(AXTraversalEntry(element: unsafeBitCast(raw, to: AXUIElement.self), parent: root, depth: 1, menu: false, priority: true)) } var menuCount = 0 - var index = ElementIndex() + var seen = ElementIndex() while nodes.count < 2000 && Date().timeIntervalSince(created) < 8, let entry = queue.next() { try checkCancellation() let element = entry.element AXUIElementSetMessagingTimeout(element, 0.25) - if index.id(of: element) != nil { continue } + if seen.id(of: element) != nil { continue } let values = attributes(element, NODE_ATTRIBUTES) let role = values[kAXRoleAttribute] as? String ?? "AXUnknown" let isMenu = entry.menu || [kAXMenuBarRole, kAXMenuRole, kAXMenuItemRole, kAXMenuBarItemRole].contains(role) @@ -359,9 +371,13 @@ final class Snapshot { guard menuCount < 80 else { continue } menuCount += 1 } - let id = "e\(nodes.count)"; elements[id] = element; index.insert(element, id: id) + let id: String + if let known = index.id(of: element) { id = known } else { + id = "e\(nextOrdinal)"; nextOrdinal += 1; index.insert(element, id: id) + } + elements[id] = element; seen.insert(element, id: id) var node: [String: Any] = ["elementId": id, "role": role, "actions": [String]()] - if let parent = entry.parent, let parentID = index.id(of: parent) { node["parentId"] = parentID } + if let parent = entry.parent, let parentID = seen.id(of: parent) { node["parentId"] = parentID } let isSecure = values[kAXSubroleAttribute] as? String == kAXSecureTextFieldSubrole var labels = [kAXTitleAttribute, kAXDescriptionAttribute].compactMap { values[$0] as? String }.filter { !$0.isEmpty } if labels.isEmpty { labels = [kAXPlaceholderValueAttribute, kAXHelpAttribute].compactMap { values[$0] as? String }.filter { !$0.isEmpty } } @@ -688,6 +704,10 @@ func discoverApps() throws -> [[String: Any]] { @MainActor final class Driver { var snapshots: [String: Snapshot] = [:] + /// Each app's latest observation, kept after actions consume `snapshots` so the next read reuses its element ids. + var lastObserved: [String: Snapshot] = [:] + /// Each app's next element id, kept apart from the evictable history so ids never repeat while that process runs. + var nextOrdinals: [String: (pid: pid_t, launchDate: Date?, next: Int)] = [:] func point(_ p: Parameters, app: NSRunningApplication, snapshot: Snapshot, end: Bool = false) throws -> CGPoint { if let elementId = p.elementId, !end { guard let frame = elementRect(try snapshot.element(elementId)) else { throw ComputerError("element_unavailable", "Element has no usable frame.") } @@ -741,7 +761,9 @@ final class Driver { return ["kind": "action", "action": "activate_app", "bundleId": app.bundleIdentifier!, "dispatched": true, "verified": true] case "get_app_state": try requireAccessibility() - let app = try await appFor(p.bundleId, launch: true); let snapshot = Snapshot(app: app) + let app = try await appFor(p.bundleId, launch: true); let bundle = app.bundleIdentifier! + let ordinal = nextOrdinals[bundle].flatMap { $0.pid == app.processIdentifier && $0.launchDate == app.launchDate ? $0.next : nil } ?? 0 + let snapshot = Snapshot(app: app, previous: lastObserved[bundle], nextOrdinal: ordinal) let preferredFrame = try p.windowId.map { try windowRect($0, pid: app.processIdentifier) } try await snapshot.read(preferredWindowFrame: preferredFrame) let axWindows = snapshot.elements.values.filter { attribute($0, kAXRoleAttribute) as? String == kAXWindowRole } @@ -765,7 +787,10 @@ final class Driver { result.merge(rectJSON(rect)) { _, new in new }; return result } if snapshots.count >= 16 { snapshots.removeAll() } - snapshots[app.bundleIdentifier!] = snapshot + if lastObserved[bundle] == nil, lastObserved.count >= 16, let evicted = lastObserved.keys.first { lastObserved.removeValue(forKey: evicted) } + snapshots[bundle] = snapshot + lastObserved[bundle] = snapshot + nextOrdinals[bundle] = (app.processIdentifier, app.launchDate, snapshot.nextOrdinal) let root = AXUIElementCreateApplication(app.processIdentifier) let focusedID = attribute(root, kAXFocusedWindowAttribute).flatMap { CFGetTypeID($0) == AXUIElementGetTypeID() ? snapshot.windowId(holding: unsafeBitCast($0, to: AXUIElement.self)) : nil } for index in snapshot.nodes.indices { diff --git a/apps/desktop/native/computer-use/tests/Fixture.swift b/apps/desktop/native/computer-use/tests/Fixture.swift index 32d4b189127..ef46f549c04 100644 --- a/apps/desktop/native/computer-use/tests/Fixture.swift +++ b/apps/desktop/native/computer-use/tests/Fixture.swift @@ -25,6 +25,7 @@ final class InfiniteFrameView: NSView { } final class FixtureDelegate: NSObject, NSApplicationDelegate { var window: NSWindow! + var stack: NSStackView! var count = 0 let counter = NSTextField(labelWithString: "Count: 0") let scrollCounter = NSTextField(labelWithString: "Scroll: 0") @@ -47,7 +48,7 @@ final class FixtureDelegate: NSObject, NSApplicationDelegate { NSApplication.shared.mainMenu = menu window = NSWindow(contentRect: NSRect(x: 100, y: 100, width: 600, height: 720), styleMask: [.titled, .closable, .resizable], backing: .buffered, defer: false) window.title = "Mothership Computer Use Test Fixture" - let stack = NSStackView(); stack.orientation = .vertical; stack.spacing = 16 + stack = NSStackView(); stack.orientation = .vertical; stack.spacing = 16 stack.translatesAutoresizingMaskIntoConstraints = false let field = NSTextField(string: "Fixture text"); field.setAccessibilityIdentifier("fixture-text"); field.setAccessibilityLabel("Fixture editable text") let secure = NSSecureTextField(string: "fixture-only-secret"); secure.setAccessibilityIdentifier("fixture-password"); secure.setAccessibilityLabel("Fixture password") @@ -55,6 +56,7 @@ final class FixtureDelegate: NSObject, NSApplicationDelegate { let clickField = ClickFocusField(string: ""); clickField.setAccessibilityLabel("Fixture click-focus field") let caretField = FixedCaretField(string: String(repeating: "keep ", count: 40)); caretField.setAccessibilityLabel("Fixture fixed-caret field") let sheetButton = NSButton(title: "Show sheet", target: self, action: #selector(showSheet)); sheetButton.setAccessibilityIdentifier("fixture-show-sheet") + let addRowButton = NSButton(title: "Add row", target: self, action: #selector(addRow)) let slider = NSSlider(value: 25, minValue: 0, maxValue: 100, target: nil, action: nil); slider.setAccessibilityLabel("Fixture slider") let time = NSDatePicker(); time.datePickerStyle = .textField; time.datePickerElements = .hourMinute; time.setAccessibilityLabel("Fixture time") time.dateValue = Calendar.current.date(bySettingHour: 10, minute: 15, second: 0, of: Date()) ?? Date(); time.target = self; time.action = #selector(timeChanged(_:)) @@ -68,7 +70,7 @@ final class FixtureDelegate: NSObject, NSApplicationDelegate { scroll.documentView = document scroll.contentView.postsBoundsChangedNotifications = true NotificationCenter.default.addObserver(self, selector: #selector(scrolled(_:)), name: NSView.boundsDidChangeNotification, object: scroll.contentView) - for view in [field, secure, button, counter, clickField, caretField, sheetButton, slider, time, timeLabel, marker, infinite, scrollCounter, scroll] { stack.addArrangedSubview(view) } + for view in [addRowButton, field, secure, button, counter, clickField, caretField, sheetButton, slider, time, timeLabel, marker, infinite, scrollCounter, scroll] { stack.addArrangedSubview(view) } window.contentView!.addSubview(stack) NSLayoutConstraint.activate([stack.leadingAnchor.constraint(equalTo: window.contentView!.leadingAnchor, constant: 30), stack.trailingAnchor.constraint(equalTo: window.contentView!.trailingAnchor, constant: -30), stack.topAnchor.constraint(equalTo: window.contentView!.topAnchor, constant: 30), field.widthAnchor.constraint(equalTo: stack.widthAnchor), secure.widthAnchor.constraint(equalTo: stack.widthAnchor), clickField.widthAnchor.constraint(equalTo: stack.widthAnchor), caretField.widthAnchor.constraint(equalTo: stack.widthAnchor), slider.widthAnchor.constraint(equalTo: stack.widthAnchor), scroll.widthAnchor.constraint(equalTo: stack.widthAnchor), scroll.heightAnchor.constraint(equalToConstant: 230), marker.widthAnchor.constraint(equalToConstant: 100), marker.heightAnchor.constraint(equalToConstant: 70)]) window.makeKeyAndOrderFront(nil) @@ -76,6 +78,11 @@ final class FixtureDelegate: NSObject, NSApplicationDelegate { NSApplication.shared.activate() print("fixture-ready"); fflush(stdout) } + /// Adds a label ahead of every control in Accessibility order, so positional ids after it would all shift. + @objc func addRow() { + let row = NSTextField(labelWithString: "Added row"); row.frame = NSRect(x: 4, y: 4, width: 120, height: 18) + window.contentView!.addSubview(row, positioned: .below, relativeTo: stack) + } @objc func increment() { count += 1; counter.stringValue = "Count: \(count)" } @objc func showSheet() { let alert = NSAlert(); alert.messageText = "Fixture sheet"; alert.addButton(withTitle: "Dismiss"); alert.beginSheetModal(for: window) } @objc func timeChanged(_ picker: NSDatePicker) { diff --git a/apps/desktop/native/computer-use/tests/live.py b/apps/desktop/native/computer-use/tests/live.py index 34f4e9ff208..0ec3251440f 100644 --- a/apps/desktop/native/computer-use/tests/live.py +++ b/apps/desktop/native/computer-use/tests/live.py @@ -91,6 +91,12 @@ def coordinate(snapshot, node, fx=0.5, fy=0.5): else: assert 'screen_capture_permission_required' in snapshot.get('screenshotError', '') print('BLOCKED: screenshot verification skipped because Screen Recording grant is missing', flush=True) + # Increment follows the inserted row and is never the focused editor, which traversal always lists first. + increment_id = find(snapshot, role='AXButton', label='Increment')['elementId'] + mutate('click', snapshot, elementId=find(snapshot, role='AXButton', label='Add row')['elementId']) + snapshot = state() + assert any(node.get('value', node.get('label')) == 'Added row' for node in snapshot['nodes']), 'Add row did not insert its label' + assert find(snapshot, role='AXButton', label='Increment')['elementId'] == increment_id, 'An element changed id when another was added above it' baseline = int(next(node.get('value', node.get('label', '')) for node in snapshot['nodes'] if node.get('value', node.get('label', '')).startswith('Count: ')).split(': ')[1]) button = find(snapshot, role='AXButton', label='Increment') snapshot = mutate('click', snapshot, elementId=button['elementId']) @@ -212,7 +218,7 @@ def batch(snapshot, steps): assert error['dispatchState'] == 'not_started', error assert find(state(), role='AXTextField', label='Fixture click-focus field').get('value') == 'replaced', 'A background app was clicked or typed into' call('activate_app', bundleId=bundle) - print('PASS: real fixture state, a large closed menu not reported as truncation, secure suppression, an element reported at infinity read without a frame, AX click/set/action, Unicode typing, keyboard chord, coordinate drag and scrolling, text refused by a slider, typing into a date picker, a window target typing into its focused editor, a title-bar click, a batch that applies, stops at a failing step, refuses before dispatch and stops when a sheet opens, click-to-focus for an editor that ignores Accessibility focus (inserting at the end, refusing an unconfirmed caret, and none in the background); every mutation verified in fresh state; screenshot=' + ('verified' if permissions['screenRecording'] else 'BLOCKED')) + print('PASS: real fixture state, element ids stable across observations, a large closed menu not reported as truncation, secure suppression, an element reported at infinity read without a frame, AX click/set/action, Unicode typing, keyboard chord, coordinate drag and scrolling, text refused by a slider, typing into a date picker, a window target typing into its focused editor, a title-bar click, a batch that applies, stops at a failing step, refuses before dispatch and stops when a sheet opens, click-to-focus for an editor that ignores Accessibility focus (inserting at the end, refusing an unconfirmed caret, and none in the background); every mutation verified in fresh state; screenshot=' + ('verified' if permissions['screenRecording'] else 'BLOCKED')) finally: if fixture: fixture.terminate()