From d26ed9d2705094708a208bc03cdd3e748ffbc1fd Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 14 Nov 2026 05:57:11 -0800 Subject: [PATCH 1/5] improvement(computer-use): keep an element's id stable across observations --- .../native/computer-use/ComputerUse.swift | 30 ++++++++++++++----- .../native/computer-use/tests/Fixture.swift | 8 +++-- .../desktop/native/computer-use/tests/live.py | 7 ++++- 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index a6afafb0f6f..68fa8029d26 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -109,8 +109,9 @@ func attributes(_ element: AXUIElement, _ names: [String]) -> [String: CFTypeRef /// Elements bucketed by CFHash, so a walk's membership and parent lookups stay linear in its size. struct ElementIndex { private var buckets: [CFHashCode: [(element: AXUIElement, id: String)]] = [:] + private(set) var count = 0 func id(of element: AXUIElement) -> String? { buckets[CFHash(element)]?.first { CFEqual($0.element, element) }?.id } - mutating func insert(_ element: AXUIElement, id: String) { buckets[CFHash(element), default: []].append((element, id)) } + mutating func insert(_ element: AXUIElement, id: String) { buckets[CFHash(element), default: []].append((element, id)); count += 1 } } let NODE_ATTRIBUTES = [kAXRoleAttribute, kAXSubroleAttribute, kAXTitleAttribute, kAXDescriptionAttribute, kAXPlaceholderValueAttribute, kAXHelpAttribute, kAXValueAttribute, kAXFocusedAttribute, kAXEnabledAttribute, kAXPositionAttribute, kAXSizeAttribute] @@ -304,7 +305,18 @@ final class Snapshot { var selectedWindowId: String? var nodes: [[String: Any]] = [] var truncated = false - init(app: NSRunningApplication) { pid = app.processIdentifier; launchDate = app.launchDate } + /// Every element this app session has been given an id, so the same element keeps its id across observations. + private(set) var index = ElementIndex() + private(set) var nextOrdinal = 0 + /// Carries ids over from `previous` when it observed the same process instance. + init(app: NSRunningApplication, previous: Snapshot? = nil) { + pid = app.processIdentifier; launchDate = app.launchDate + if let previous, previous.pid == pid, previous.launchDate == launchDate { + // Ids never repeat within a session; past the cap the next read starts a fresh index so it stays bounded. + nextOrdinal = previous.nextOrdinal + if previous.index.count < 20_000 { index = previous.index } + } + } func read(preferredWindowFrame: CGRect? = nil) async throws { try requireAccessibility() let root = AXUIElementCreateApplication(pid) @@ -342,12 +354,12 @@ final class Snapshot { queue.append(AXTraversalEntry(element: unsafeBitCast(raw, to: AXUIElement.self), parent: root, depth: 1, menu: false, priority: true)) } var menuCount = 0 - var index = ElementIndex() + var seen = ElementIndex() while nodes.count < 2000 && Date().timeIntervalSince(created) < 8, let entry = queue.next() { try checkCancellation() let element = entry.element AXUIElementSetMessagingTimeout(element, 0.25) - if index.id(of: element) != nil { continue } + if seen.id(of: element) != nil { continue } let values = attributes(element, NODE_ATTRIBUTES) let role = values[kAXRoleAttribute] as? String ?? "AXUnknown" let isMenu = entry.menu || [kAXMenuBarRole, kAXMenuRole, kAXMenuItemRole, kAXMenuBarItemRole].contains(role) @@ -359,9 +371,13 @@ final class Snapshot { guard menuCount < 80 else { continue } menuCount += 1 } - let id = "e\(nodes.count)"; elements[id] = element; index.insert(element, id: id) + let id: String + if let known = index.id(of: element) { id = known } else { + id = "e\(nextOrdinal)"; nextOrdinal += 1; index.insert(element, id: id) + } + elements[id] = element; seen.insert(element, id: id) var node: [String: Any] = ["elementId": id, "role": role, "actions": [String]()] - if let parent = entry.parent, let parentID = index.id(of: parent) { node["parentId"] = parentID } + if let parent = entry.parent, let parentID = seen.id(of: parent) { node["parentId"] = parentID } let isSecure = values[kAXSubroleAttribute] as? String == kAXSecureTextFieldSubrole var labels = [kAXTitleAttribute, kAXDescriptionAttribute].compactMap { values[$0] as? String }.filter { !$0.isEmpty } if labels.isEmpty { labels = [kAXPlaceholderValueAttribute, kAXHelpAttribute].compactMap { values[$0] as? String }.filter { !$0.isEmpty } } @@ -741,7 +757,7 @@ final class Driver { return ["kind": "action", "action": "activate_app", "bundleId": app.bundleIdentifier!, "dispatched": true, "verified": true] case "get_app_state": try requireAccessibility() - let app = try await appFor(p.bundleId, launch: true); let snapshot = Snapshot(app: app) + let app = try await appFor(p.bundleId, launch: true); let snapshot = Snapshot(app: app, previous: snapshots[p.bundleId]) let preferredFrame = try p.windowId.map { try windowRect($0, pid: app.processIdentifier) } try await snapshot.read(preferredWindowFrame: preferredFrame) let axWindows = snapshot.elements.values.filter { attribute($0, kAXRoleAttribute) as? String == kAXWindowRole } diff --git a/apps/desktop/native/computer-use/tests/Fixture.swift b/apps/desktop/native/computer-use/tests/Fixture.swift index 32d4b189127..16dd1f945f9 100644 --- a/apps/desktop/native/computer-use/tests/Fixture.swift +++ b/apps/desktop/native/computer-use/tests/Fixture.swift @@ -25,6 +25,7 @@ final class InfiniteFrameView: NSView { } final class FixtureDelegate: NSObject, NSApplicationDelegate { var window: NSWindow! + var stack: NSStackView! var count = 0 let counter = NSTextField(labelWithString: "Count: 0") let scrollCounter = NSTextField(labelWithString: "Scroll: 0") @@ -47,7 +48,7 @@ final class FixtureDelegate: NSObject, NSApplicationDelegate { NSApplication.shared.mainMenu = menu window = NSWindow(contentRect: NSRect(x: 100, y: 100, width: 600, height: 720), styleMask: [.titled, .closable, .resizable], backing: .buffered, defer: false) window.title = "Mothership Computer Use Test Fixture" - let stack = NSStackView(); stack.orientation = .vertical; stack.spacing = 16 + stack = NSStackView(); stack.orientation = .vertical; stack.spacing = 16 stack.translatesAutoresizingMaskIntoConstraints = false let field = NSTextField(string: "Fixture text"); field.setAccessibilityIdentifier("fixture-text"); field.setAccessibilityLabel("Fixture editable text") let secure = NSSecureTextField(string: "fixture-only-secret"); secure.setAccessibilityIdentifier("fixture-password"); secure.setAccessibilityLabel("Fixture password") @@ -55,6 +56,7 @@ final class FixtureDelegate: NSObject, NSApplicationDelegate { let clickField = ClickFocusField(string: ""); clickField.setAccessibilityLabel("Fixture click-focus field") let caretField = FixedCaretField(string: String(repeating: "keep ", count: 40)); caretField.setAccessibilityLabel("Fixture fixed-caret field") let sheetButton = NSButton(title: "Show sheet", target: self, action: #selector(showSheet)); sheetButton.setAccessibilityIdentifier("fixture-show-sheet") + let addRowButton = NSButton(title: "Add row", target: self, action: #selector(addRow)) let slider = NSSlider(value: 25, minValue: 0, maxValue: 100, target: nil, action: nil); slider.setAccessibilityLabel("Fixture slider") let time = NSDatePicker(); time.datePickerStyle = .textField; time.datePickerElements = .hourMinute; time.setAccessibilityLabel("Fixture time") time.dateValue = Calendar.current.date(bySettingHour: 10, minute: 15, second: 0, of: Date()) ?? Date(); time.target = self; time.action = #selector(timeChanged(_:)) @@ -68,7 +70,7 @@ final class FixtureDelegate: NSObject, NSApplicationDelegate { scroll.documentView = document scroll.contentView.postsBoundsChangedNotifications = true NotificationCenter.default.addObserver(self, selector: #selector(scrolled(_:)), name: NSView.boundsDidChangeNotification, object: scroll.contentView) - for view in [field, secure, button, counter, clickField, caretField, sheetButton, slider, time, timeLabel, marker, infinite, scrollCounter, scroll] { stack.addArrangedSubview(view) } + for view in [addRowButton, field, secure, button, counter, clickField, caretField, sheetButton, slider, time, timeLabel, marker, infinite, scrollCounter, scroll] { stack.addArrangedSubview(view) } window.contentView!.addSubview(stack) NSLayoutConstraint.activate([stack.leadingAnchor.constraint(equalTo: window.contentView!.leadingAnchor, constant: 30), stack.trailingAnchor.constraint(equalTo: window.contentView!.trailingAnchor, constant: -30), stack.topAnchor.constraint(equalTo: window.contentView!.topAnchor, constant: 30), field.widthAnchor.constraint(equalTo: stack.widthAnchor), secure.widthAnchor.constraint(equalTo: stack.widthAnchor), clickField.widthAnchor.constraint(equalTo: stack.widthAnchor), caretField.widthAnchor.constraint(equalTo: stack.widthAnchor), slider.widthAnchor.constraint(equalTo: stack.widthAnchor), scroll.widthAnchor.constraint(equalTo: stack.widthAnchor), scroll.heightAnchor.constraint(equalToConstant: 230), marker.widthAnchor.constraint(equalToConstant: 100), marker.heightAnchor.constraint(equalToConstant: 70)]) window.makeKeyAndOrderFront(nil) @@ -76,6 +78,8 @@ final class FixtureDelegate: NSObject, NSApplicationDelegate { NSApplication.shared.activate() print("fixture-ready"); fflush(stdout) } + /// Inserts a label above every other control, so later elements move in traversal order. + @objc func addRow() { stack.insertArrangedSubview(NSTextField(labelWithString: "Added row"), at: 0) } @objc func increment() { count += 1; counter.stringValue = "Count: \(count)" } @objc func showSheet() { let alert = NSAlert(); alert.messageText = "Fixture sheet"; alert.addButton(withTitle: "Dismiss"); alert.beginSheetModal(for: window) } @objc func timeChanged(_ picker: NSDatePicker) { diff --git a/apps/desktop/native/computer-use/tests/live.py b/apps/desktop/native/computer-use/tests/live.py index 34f4e9ff208..63e3f164443 100644 --- a/apps/desktop/native/computer-use/tests/live.py +++ b/apps/desktop/native/computer-use/tests/live.py @@ -82,6 +82,11 @@ def coordinate(snapshot, node, fx=0.5, fy=0.5): snapshot = state(screenshot=True) assert 'fixture-only-secret' not in json.dumps(snapshot), 'Secure value leaked' assert snapshot['truncated'] is False, 'Closed menu items past the snapshot cap marked the state truncated' + field_id = find(snapshot, role='AXTextField', label='Fixture editable text')['elementId'] + mutate('click', snapshot, elementId=find(snapshot, role='AXButton', label='Add row')['elementId']) + snapshot = state() + assert any(node.get('value', node.get('label')) == 'Added row' for node in snapshot['nodes']), 'Add row did not insert its label' + assert find(snapshot, role='AXTextField', label='Fixture editable text')['elementId'] == field_id, 'An element changed id when another was added above it' infinite = find(snapshot, label='Fixture infinite frame') assert not {'x', 'y', 'width', 'height'} & infinite.keys(), f'An infinite frame reached the reply: {infinite}' image = snapshot.get('screenshot') @@ -212,7 +217,7 @@ def batch(snapshot, steps): assert error['dispatchState'] == 'not_started', error assert find(state(), role='AXTextField', label='Fixture click-focus field').get('value') == 'replaced', 'A background app was clicked or typed into' call('activate_app', bundleId=bundle) - print('PASS: real fixture state, a large closed menu not reported as truncation, secure suppression, an element reported at infinity read without a frame, AX click/set/action, Unicode typing, keyboard chord, coordinate drag and scrolling, text refused by a slider, typing into a date picker, a window target typing into its focused editor, a title-bar click, a batch that applies, stops at a failing step, refuses before dispatch and stops when a sheet opens, click-to-focus for an editor that ignores Accessibility focus (inserting at the end, refusing an unconfirmed caret, and none in the background); every mutation verified in fresh state; screenshot=' + ('verified' if permissions['screenRecording'] else 'BLOCKED')) + print('PASS: real fixture state, element ids stable across observations, a large closed menu not reported as truncation, secure suppression, an element reported at infinity read without a frame, AX click/set/action, Unicode typing, keyboard chord, coordinate drag and scrolling, text refused by a slider, typing into a date picker, a window target typing into its focused editor, a title-bar click, a batch that applies, stops at a failing step, refuses before dispatch and stops when a sheet opens, click-to-focus for an editor that ignores Accessibility focus (inserting at the end, refusing an unconfirmed caret, and none in the background); every mutation verified in fresh state; screenshot=' + ('verified' if permissions['screenRecording'] else 'BLOCKED')) finally: if fixture: fixture.terminate() From d8fc195501cf457fe58211cf58bfb4c1cac8eb78 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 14 Nov 2026 06:02:55 -0800 Subject: [PATCH 2/5] fix the snapshot key and make the stable-id fixture insert ahead of the field --- apps/desktop/native/computer-use/ComputerUse.swift | 2 +- apps/desktop/native/computer-use/tests/Fixture.swift | 7 +++++-- apps/desktop/native/computer-use/tests/live.py | 10 +++++----- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index 68fa8029d26..2fb8432e599 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -757,7 +757,7 @@ final class Driver { return ["kind": "action", "action": "activate_app", "bundleId": app.bundleIdentifier!, "dispatched": true, "verified": true] case "get_app_state": try requireAccessibility() - let app = try await appFor(p.bundleId, launch: true); let snapshot = Snapshot(app: app, previous: snapshots[p.bundleId]) + let app = try await appFor(p.bundleId, launch: true); let snapshot = Snapshot(app: app, previous: snapshots[app.bundleIdentifier!]) let preferredFrame = try p.windowId.map { try windowRect($0, pid: app.processIdentifier) } try await snapshot.read(preferredWindowFrame: preferredFrame) let axWindows = snapshot.elements.values.filter { attribute($0, kAXRoleAttribute) as? String == kAXWindowRole } diff --git a/apps/desktop/native/computer-use/tests/Fixture.swift b/apps/desktop/native/computer-use/tests/Fixture.swift index 16dd1f945f9..ef46f549c04 100644 --- a/apps/desktop/native/computer-use/tests/Fixture.swift +++ b/apps/desktop/native/computer-use/tests/Fixture.swift @@ -78,8 +78,11 @@ final class FixtureDelegate: NSObject, NSApplicationDelegate { NSApplication.shared.activate() print("fixture-ready"); fflush(stdout) } - /// Inserts a label above every other control, so later elements move in traversal order. - @objc func addRow() { stack.insertArrangedSubview(NSTextField(labelWithString: "Added row"), at: 0) } + /// Adds a label ahead of every control in Accessibility order, so positional ids after it would all shift. + @objc func addRow() { + let row = NSTextField(labelWithString: "Added row"); row.frame = NSRect(x: 4, y: 4, width: 120, height: 18) + window.contentView!.addSubview(row, positioned: .below, relativeTo: stack) + } @objc func increment() { count += 1; counter.stringValue = "Count: \(count)" } @objc func showSheet() { let alert = NSAlert(); alert.messageText = "Fixture sheet"; alert.addButton(withTitle: "Dismiss"); alert.beginSheetModal(for: window) } @objc func timeChanged(_ picker: NSDatePicker) { diff --git a/apps/desktop/native/computer-use/tests/live.py b/apps/desktop/native/computer-use/tests/live.py index 63e3f164443..19a2a0764f6 100644 --- a/apps/desktop/native/computer-use/tests/live.py +++ b/apps/desktop/native/computer-use/tests/live.py @@ -82,11 +82,6 @@ def coordinate(snapshot, node, fx=0.5, fy=0.5): snapshot = state(screenshot=True) assert 'fixture-only-secret' not in json.dumps(snapshot), 'Secure value leaked' assert snapshot['truncated'] is False, 'Closed menu items past the snapshot cap marked the state truncated' - field_id = find(snapshot, role='AXTextField', label='Fixture editable text')['elementId'] - mutate('click', snapshot, elementId=find(snapshot, role='AXButton', label='Add row')['elementId']) - snapshot = state() - assert any(node.get('value', node.get('label')) == 'Added row' for node in snapshot['nodes']), 'Add row did not insert its label' - assert find(snapshot, role='AXTextField', label='Fixture editable text')['elementId'] == field_id, 'An element changed id when another was added above it' infinite = find(snapshot, label='Fixture infinite frame') assert not {'x', 'y', 'width', 'height'} & infinite.keys(), f'An infinite frame reached the reply: {infinite}' image = snapshot.get('screenshot') @@ -96,6 +91,11 @@ def coordinate(snapshot, node, fx=0.5, fy=0.5): else: assert 'screen_capture_permission_required' in snapshot.get('screenshotError', '') print('BLOCKED: screenshot verification skipped because Screen Recording grant is missing', flush=True) + field_id = find(snapshot, role='AXTextField', label='Fixture editable text')['elementId'] + mutate('click', snapshot, elementId=find(snapshot, role='AXButton', label='Add row')['elementId']) + snapshot = state() + assert any(node.get('value', node.get('label')) == 'Added row' for node in snapshot['nodes']), 'Add row did not insert its label' + assert find(snapshot, role='AXTextField', label='Fixture editable text')['elementId'] == field_id, 'An element changed id when another was added above it' baseline = int(next(node.get('value', node.get('label', '')) for node in snapshot['nodes'] if node.get('value', node.get('label', '')).startswith('Count: ')).split(': ')[1]) button = find(snapshot, role='AXButton', label='Increment') snapshot = mutate('click', snapshot, elementId=button['elementId']) From 574c41e63e444a1f9720ab41dfbfbf2c50190c1c Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 14 Nov 2026 06:11:40 -0800 Subject: [PATCH 3/5] assert stable ids on an element after the inserted row --- apps/desktop/native/computer-use/tests/live.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/desktop/native/computer-use/tests/live.py b/apps/desktop/native/computer-use/tests/live.py index 19a2a0764f6..0ec3251440f 100644 --- a/apps/desktop/native/computer-use/tests/live.py +++ b/apps/desktop/native/computer-use/tests/live.py @@ -91,11 +91,12 @@ def coordinate(snapshot, node, fx=0.5, fy=0.5): else: assert 'screen_capture_permission_required' in snapshot.get('screenshotError', '') print('BLOCKED: screenshot verification skipped because Screen Recording grant is missing', flush=True) - field_id = find(snapshot, role='AXTextField', label='Fixture editable text')['elementId'] + # Increment follows the inserted row and is never the focused editor, which traversal always lists first. + increment_id = find(snapshot, role='AXButton', label='Increment')['elementId'] mutate('click', snapshot, elementId=find(snapshot, role='AXButton', label='Add row')['elementId']) snapshot = state() assert any(node.get('value', node.get('label')) == 'Added row' for node in snapshot['nodes']), 'Add row did not insert its label' - assert find(snapshot, role='AXTextField', label='Fixture editable text')['elementId'] == field_id, 'An element changed id when another was added above it' + assert find(snapshot, role='AXButton', label='Increment')['elementId'] == increment_id, 'An element changed id when another was added above it' baseline = int(next(node.get('value', node.get('label', '')) for node in snapshot['nodes'] if node.get('value', node.get('label', '')).startswith('Count: ')).split(': ')[1]) button = find(snapshot, role='AXButton', label='Increment') snapshot = mutate('click', snapshot, elementId=button['elementId']) From 90bd259e9a9f4df8530d25de258fab38f992b7d4 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 14 Nov 2026 06:18:18 -0800 Subject: [PATCH 4/5] keep element ids across actions, which consume the actionable snapshot --- apps/desktop/native/computer-use/ComputerUse.swift | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index 2fb8432e599..98b8cf13e90 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -704,6 +704,8 @@ func discoverApps() throws -> [[String: Any]] { @MainActor final class Driver { var snapshots: [String: Snapshot] = [:] + /// Each app's latest observation, kept after actions consume `snapshots` so the next read reuses its element ids. + var lastObserved: [String: Snapshot] = [:] func point(_ p: Parameters, app: NSRunningApplication, snapshot: Snapshot, end: Bool = false) throws -> CGPoint { if let elementId = p.elementId, !end { guard let frame = elementRect(try snapshot.element(elementId)) else { throw ComputerError("element_unavailable", "Element has no usable frame.") } @@ -757,7 +759,7 @@ final class Driver { return ["kind": "action", "action": "activate_app", "bundleId": app.bundleIdentifier!, "dispatched": true, "verified": true] case "get_app_state": try requireAccessibility() - let app = try await appFor(p.bundleId, launch: true); let snapshot = Snapshot(app: app, previous: snapshots[app.bundleIdentifier!]) + let app = try await appFor(p.bundleId, launch: true); let snapshot = Snapshot(app: app, previous: lastObserved[app.bundleIdentifier!]) let preferredFrame = try p.windowId.map { try windowRect($0, pid: app.processIdentifier) } try await snapshot.read(preferredWindowFrame: preferredFrame) let axWindows = snapshot.elements.values.filter { attribute($0, kAXRoleAttribute) as? String == kAXWindowRole } @@ -781,7 +783,9 @@ final class Driver { result.merge(rectJSON(rect)) { _, new in new }; return result } if snapshots.count >= 16 { snapshots.removeAll() } + if lastObserved.count >= 16 { lastObserved.removeAll() } snapshots[app.bundleIdentifier!] = snapshot + lastObserved[app.bundleIdentifier!] = snapshot let root = AXUIElementCreateApplication(app.processIdentifier) let focusedID = attribute(root, kAXFocusedWindowAttribute).flatMap { CFGetTypeID($0) == AXUIElementGetTypeID() ? snapshot.windowId(holding: unsafeBitCast($0, to: AXUIElement.self)) : nil } for index in snapshot.nodes.indices { From a3e9354211cc809c850bf520784a12ea5f24b409 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 14 Nov 2026 11:39:35 -0800 Subject: [PATCH 5/5] keep each app's id counter apart from the evictable history and carry the index only when a full read fits the cap --- .../native/computer-use/ComputerUse.swift | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index 98b8cf13e90..33104ba8046 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -307,14 +307,14 @@ final class Snapshot { var truncated = false /// Every element this app session has been given an id, so the same element keeps its id across observations. private(set) var index = ElementIndex() - private(set) var nextOrdinal = 0 - /// Carries ids over from `previous` when it observed the same process instance. - init(app: NSRunningApplication, previous: Snapshot? = nil) { - pid = app.processIdentifier; launchDate = app.launchDate - if let previous, previous.pid == pid, previous.launchDate == launchDate { - // Ids never repeat within a session; past the cap the next read starts a fresh index so it stays bounded. - nextOrdinal = previous.nextOrdinal - if previous.index.count < 20_000 { index = previous.index } + private(set) var nextOrdinal: Int + /// Reuses `previous`'s ids when it observed the same process instance and new ids continue from + /// `nextOrdinal`, which the driver keeps per process so ids never repeat while it runs. + init(app: NSRunningApplication, previous: Snapshot? = nil, nextOrdinal: Int = 0) { + pid = app.processIdentifier; launchDate = app.launchDate; self.nextOrdinal = nextOrdinal + // One read adds at most 2,000 elements, so the carried index stays within 20,000; past that the next read starts fresh. + if let previous, previous.pid == pid, previous.launchDate == launchDate, previous.index.count <= 18_000 { + index = previous.index } } func read(preferredWindowFrame: CGRect? = nil) async throws { @@ -706,6 +706,8 @@ final class Driver { var snapshots: [String: Snapshot] = [:] /// Each app's latest observation, kept after actions consume `snapshots` so the next read reuses its element ids. var lastObserved: [String: Snapshot] = [:] + /// Each app's next element id, kept apart from the evictable history so ids never repeat while that process runs. + var nextOrdinals: [String: (pid: pid_t, launchDate: Date?, next: Int)] = [:] func point(_ p: Parameters, app: NSRunningApplication, snapshot: Snapshot, end: Bool = false) throws -> CGPoint { if let elementId = p.elementId, !end { guard let frame = elementRect(try snapshot.element(elementId)) else { throw ComputerError("element_unavailable", "Element has no usable frame.") } @@ -759,7 +761,9 @@ final class Driver { return ["kind": "action", "action": "activate_app", "bundleId": app.bundleIdentifier!, "dispatched": true, "verified": true] case "get_app_state": try requireAccessibility() - let app = try await appFor(p.bundleId, launch: true); let snapshot = Snapshot(app: app, previous: lastObserved[app.bundleIdentifier!]) + let app = try await appFor(p.bundleId, launch: true); let bundle = app.bundleIdentifier! + let ordinal = nextOrdinals[bundle].flatMap { $0.pid == app.processIdentifier && $0.launchDate == app.launchDate ? $0.next : nil } ?? 0 + let snapshot = Snapshot(app: app, previous: lastObserved[bundle], nextOrdinal: ordinal) let preferredFrame = try p.windowId.map { try windowRect($0, pid: app.processIdentifier) } try await snapshot.read(preferredWindowFrame: preferredFrame) let axWindows = snapshot.elements.values.filter { attribute($0, kAXRoleAttribute) as? String == kAXWindowRole } @@ -783,9 +787,10 @@ final class Driver { result.merge(rectJSON(rect)) { _, new in new }; return result } if snapshots.count >= 16 { snapshots.removeAll() } - if lastObserved.count >= 16 { lastObserved.removeAll() } - snapshots[app.bundleIdentifier!] = snapshot - lastObserved[app.bundleIdentifier!] = snapshot + if lastObserved[bundle] == nil, lastObserved.count >= 16, let evicted = lastObserved.keys.first { lastObserved.removeValue(forKey: evicted) } + snapshots[bundle] = snapshot + lastObserved[bundle] = snapshot + nextOrdinals[bundle] = (app.processIdentifier, app.launchDate, snapshot.nextOrdinal) let root = AXUIElementCreateApplication(app.processIdentifier) let focusedID = attribute(root, kAXFocusedWindowAttribute).flatMap { CFGetTypeID($0) == AXUIElementGetTypeID() ? snapshot.windowId(holding: unsafeBitCast($0, to: AXUIElement.self)) : nil } for index in snapshot.nodes.indices {