diff --git a/apps/desktop/build/entitlements.mac.plist b/apps/desktop/build/entitlements.mac.plist index edf7bcaf4b9..d7593a8f68b 100644 --- a/apps/desktop/build/entitlements.mac.plist +++ b/apps/desktop/build/entitlements.mac.plist @@ -14,5 +14,12 @@ Hardened Runtime denies the camera no matter what the user allowed. --> com.apple.security.device.camera + + com.apple.security.personal-information.calendars + + com.apple.security.personal-information.addressbook + diff --git a/apps/desktop/electron-builder.yml b/apps/desktop/electron-builder.yml index 5b303ac360b..bad2a1939a0 100644 --- a/apps/desktop/electron-builder.yml +++ b/apps/desktop/electron-builder.yml @@ -70,6 +70,9 @@ mac: # unless the bundle declares why it wants the device. extendInfo: NSScreenCaptureUsageDescription: Mothership captures approved app windows to understand and verify computer-use actions. + NSRemindersFullAccessUsageDescription: Mothership adds reminders you confirm and lists your reminder lists so it can file them correctly. + NSCalendarsFullAccessUsageDescription: Mothership adds calendar events you confirm and lists your calendars so it can file them correctly. + NSContactsUsageDescription: Mothership adds contacts and contact groups you confirm. NSMicrophoneUsageDescription: Sim uses your microphone for voice input in Chat and for meetings you join in the built-in browser. NSCameraUsageDescription: Sim uses your camera for meetings you join in the built-in browser, such as Google Meet. # macOS 15 blocks connections to private addresses (self-hosted servers on a diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index 907084daef8..fec42c5c1a5 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -1,5 +1,7 @@ import AppKit import ApplicationServices +import Contacts +import EventKit import ScreenCaptureKit nonisolated(unsafe) var cancellationRequested: sig_atomic_t = 0 @@ -39,6 +41,219 @@ struct Parameters: Decodable { var button: String? var clickCount: Int? var accessibilityAction: String? + var command: AppCommand? +} +/// One allowlisted app command. Commands go through EventKit and Contacts, never Apple Events or +/// scripts, so they cannot reach a shell; each is a typed create or a list read, with no update, +/// delete or search. +struct AppCommand: Decodable { + var name: String + var title: String? + var notes: String? + var due: String? + var listId: String? + var priority: Int? + var start: String? + var end: String? + var allDay: Bool? + var location: String? + var calendarId: String? + var alertMinutesBefore: Int? + var givenName: String? + var familyName: String? + var organization: String? + var jobTitle: String? + var emails: [String]? + var phones: [String]? + var groupId: String? +} +func invalidCommand(_ message: String) -> ComputerError { ComputerError("invalid_arguments", message, dispatchState: "not_started") } +/// A required, trimmed text field of bounded length. +func commandText(_ value: String?, _ field: String, max: Int = 1024) throws -> String { + let text = (value ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + guard !text.isEmpty, text.count <= max else { throw invalidCommand("\(field) is required and limited to \(max) characters.") } + return text +} +func optionalText(_ value: String?, _ field: String, max: Int = 1024) throws -> String? { + guard let value, !value.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { return nil } + return try commandText(value, field, max: max) +} +/// The event store for reminders, available only once the user has granted full Reminders access. +/// Checking never prompts; the grant happens from settings, outside the tool call. +func remindersStore() throws -> EKEventStore { + guard EKEventStore.authorizationStatus(for: .reminder) == .fullAccess else { + throw ComputerError("reminders_permission_required", "Allow Reminders access for Mothership in System Settings, then retry.", dispatchState: "not_started") + } + return EKEventStore() +} +/// A due date: `YYYY-MM-DD` is date-only; a date-time must be ISO 8601 with an explicit offset or `Z`, +/// so the reminder never depends on the helper's time zone guess. +func dueComponents(_ text: String) throws -> DateComponents { + var calendar = Calendar(identifier: .gregorian) + if text.range(of: #"^\d{4}-\d{2}-\d{2}$"#, options: .regularExpression) != nil { + let parts = text.split(separator: "-").compactMap { Int($0) } + let components = DateComponents(calendar: calendar, year: parts[0], month: parts[1], day: parts[2]) + guard components.isValidDate else { throw ComputerError("invalid_arguments", "Due date is not a real date.", dispatchState: "not_started") } + return DateComponents(year: parts[0], month: parts[1], day: parts[2]) + } + let formatter = ISO8601DateFormatter(); formatter.formatOptions = [.withInternetDateTime] + guard let date = formatter.date(from: text) else { + throw ComputerError("invalid_arguments", "Due must be YYYY-MM-DD or an ISO 8601 date-time with an offset.", dispatchState: "not_started") + } + calendar.timeZone = .current + return calendar.dateComponents([.year, .month, .day, .hour, .minute, .timeZone], from: date) +} +/// Reminders or Calendar access as the status reports it, read without prompting. +func eventKitAuthorization(_ type: EKEntityType) -> String { + switch EKEventStore.authorizationStatus(for: type) { + case .fullAccess: return "granted" + case .notDetermined: return "not_determined" + default: return "denied" + } +} +func contactsAuthorization() -> String { + switch CNContactStore.authorizationStatus(for: .contacts) { + case .authorized: return "granted" + case .notDetermined: return "not_determined" + default: return "denied" + } +} +func calendarStore() throws -> EKEventStore { + guard EKEventStore.authorizationStatus(for: .event) == .fullAccess else { + throw ComputerError("calendars_permission_required", "Allow Calendar access for Mothership in System Settings, then retry.", dispatchState: "not_started") + } + return EKEventStore() +} +func contactStore() throws -> CNContactStore { + guard CNContactStore.authorizationStatus(for: .contacts) == .authorized else { + throw ComputerError("contacts_permission_required", "Allow Contacts access for Mothership in System Settings, then retry.", dispatchState: "not_started") + } + return CNContactStore() +} +/// An event boundary: an ISO 8601 date-time with an explicit offset, or `YYYY-MM-DD` for all-day events. +func eventDate(_ text: String?, _ field: String, allDay: Bool) throws -> Date { + let text = try commandText(text, field, max: 64) + if allDay { + let parts = try dueComponents(text) + guard parts.hour == nil, let date = Calendar.current.date(from: parts) else { throw invalidCommand("All-day \(field) must be YYYY-MM-DD.") } + return date + } + let formatter = ISO8601DateFormatter(); formatter.formatOptions = [.withInternetDateTime] + guard let date = formatter.date(from: text) else { throw invalidCommand("\(field) must be an ISO 8601 date-time with an offset.") } + return date +} +func runAppCommand(_ command: AppCommand) throws -> [String: Any] { + switch command.name { + case "list_reminder_lists": + let lists = try remindersStore().calendars(for: .reminder).prefix(200).map { list -> [String: Any] in + ["id": list.calendarIdentifier, "title": String(list.title.prefix(1024)), "allowsModifications": list.allowsContentModifications] + } + return ["kind": "command", "command": command.name, "items": lists] + case "create_reminder": + let title = (command.title ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + guard !title.isEmpty, title.count <= 1024 else { throw ComputerError("invalid_arguments", "A reminder needs a title of at most 1024 characters.", dispatchState: "not_started") } + guard (command.notes?.count ?? 0) <= 8000 else { throw ComputerError("invalid_arguments", "Notes are limited to 8000 characters.", dispatchState: "not_started") } + guard [0, 1, 5, 9].contains(command.priority ?? 0) else { throw ComputerError("invalid_arguments", "Priority must be 0 (none), 1 (high), 5 (medium) or 9 (low).", dispatchState: "not_started") } + let due = try command.due.map(dueComponents) + let store = try remindersStore() + let list: EKCalendar + if let listId = command.listId { + guard let found = store.calendar(withIdentifier: listId), found.allowedEntityTypes.contains(.reminder) else { + throw ComputerError("list_unavailable", "No reminder list has this id; call list_reminder_lists.", dispatchState: "not_started") + } + list = found + } else { + guard let fallback = store.defaultCalendarForNewReminders() else { throw ComputerError("list_unavailable", "Reminders has no default list; pass listId.", dispatchState: "not_started") } + list = fallback + } + guard list.allowsContentModifications else { throw ComputerError("list_read_only", "This reminder list is read-only.", dispatchState: "not_started") } + let reminder = EKReminder(eventStore: store) + reminder.calendar = list; reminder.title = title; reminder.notes = command.notes + reminder.priority = command.priority ?? 0; reminder.dueDateComponents = due + inputDispatched = true + try store.save(reminder, commit: true) + return ["kind": "command", "command": command.name, "created": ["id": reminder.calendarItemIdentifier, "title": title, "list": list.title]] + case "list_calendars": + let calendars = try calendarStore().calendars(for: .event).prefix(200).map { calendar -> [String: Any] in + ["id": calendar.calendarIdentifier, "title": String(calendar.title.prefix(1024)), "allowsModifications": calendar.allowsContentModifications] + } + return ["kind": "command", "command": command.name, "items": calendars] + case "create_event": + let title = try commandText(command.title, "Title") + let allDay = command.allDay ?? false + let start = try eventDate(command.start, "Start", allDay: allDay) + let end = try eventDate(command.end, "End", allDay: allDay) + guard allDay ? end >= start : end > start else { throw invalidCommand("End must be after start.") } + guard end.timeIntervalSince(start) <= 31 * 86_400 else { throw invalidCommand("An event can last at most 31 days.") } + let location = try optionalText(command.location, "Location") + let notes = try optionalText(command.notes, "Notes", max: 8000) + if let minutes = command.alertMinutesBefore, !(0...40_320).contains(minutes) { throw invalidCommand("alertMinutesBefore must be 0 to 40320.") } + let store = try calendarStore() + let calendar: EKCalendar + if let calendarId = command.calendarId { + guard let found = store.calendar(withIdentifier: calendarId), found.allowedEntityTypes.contains(.event) else { + throw ComputerError("list_unavailable", "No calendar has this id; call list_calendars.", dispatchState: "not_started") + } + calendar = found + } else { + guard let fallback = store.defaultCalendarForNewEvents else { throw ComputerError("list_unavailable", "Calendar has no default calendar; pass calendarId.", dispatchState: "not_started") } + calendar = fallback + } + guard calendar.allowsContentModifications else { throw ComputerError("list_read_only", "This calendar is read-only.", dispatchState: "not_started") } + let event = EKEvent(eventStore: store) + event.calendar = calendar; event.title = title; event.isAllDay = allDay + event.startDate = start; event.endDate = end; event.location = location; event.notes = notes + if let minutes = command.alertMinutesBefore { event.addAlarm(EKAlarm(relativeOffset: TimeInterval(-minutes * 60))) } + inputDispatched = true + try store.save(event, span: .thisEvent, commit: true) + return ["kind": "command", "command": command.name, "created": ["id": event.eventIdentifier ?? event.calendarItemIdentifier, "title": title, "list": calendar.title]] + case "list_contact_groups": + let groups = try contactStore().groups(matching: nil).prefix(200).map { group -> [String: Any] in + ["id": group.identifier, "title": String(group.name.prefix(1024)), "allowsModifications": true] + } + return ["kind": "command", "command": command.name, "items": groups] + case "create_contact_group": + let name = try commandText(command.title, "Group name") + let store = try contactStore() + // A group with this name already exists after a lost reply; return it rather than add a twin. + if let existing = try store.groups(matching: nil).first(where: { $0.name == name }) { + return ["kind": "command", "command": command.name, "created": ["id": existing.identifier, "title": name, "list": "Contacts"]] + } + let group = CNMutableGroup(); group.name = name + let request = CNSaveRequest(); request.add(group, toContainerWithIdentifier: nil) + inputDispatched = true + try store.execute(request) + return ["kind": "command", "command": command.name, "created": ["id": group.identifier, "title": name, "list": "Contacts"]] + case "create_contact": + let given = try optionalText(command.givenName, "Given name"), family = try optionalText(command.familyName, "Family name") + let organization = try optionalText(command.organization, "Organization"), jobTitle = try optionalText(command.jobTitle, "Job title") + guard given != nil || family != nil || organization != nil else { throw invalidCommand("A contact needs a name or an organization.") } + let emails = try (command.emails ?? []).map { try commandText($0, "Email", max: 320) } + let phones = try (command.phones ?? []).map { try commandText($0, "Phone", max: 64) } + guard emails.count <= 5, phones.count <= 5 else { throw invalidCommand("A contact takes at most 5 emails and 5 phones.") } + guard emails.allSatisfy({ $0.contains("@") }) else { throw invalidCommand("Each email needs an @.") } + let store = try contactStore() + var group: CNGroup? + if let groupId = command.groupId { + guard let found = try store.groups(matching: CNGroup.predicateForGroups(withIdentifiers: [groupId])).first else { + throw ComputerError("list_unavailable", "No contact group has this id; call list_contact_groups.", dispatchState: "not_started") + } + group = found + } + let contact = CNMutableContact() + contact.givenName = given ?? ""; contact.familyName = family ?? "" + contact.organizationName = organization ?? ""; contact.jobTitle = jobTitle ?? "" + contact.emailAddresses = emails.map { CNLabeledValue(label: CNLabelWork, value: $0 as NSString) } + contact.phoneNumbers = phones.map { CNLabeledValue(label: CNLabelPhoneNumberMain, value: CNPhoneNumber(stringValue: $0)) } + let request = CNSaveRequest(); request.add(contact, toContainerWithIdentifier: nil) + if let group { request.addMember(contact, to: group) } + inputDispatched = true + try store.execute(request) + let display = [given, family].compactMap { $0 }.joined(separator: " ") + return ["kind": "command", "command": command.name, "created": ["id": contact.identifier, "title": display.isEmpty ? organization ?? "" : display, "list": group?.name ?? "Contacts"]] + default: + throw ComputerError("unknown_command", "Unknown app command.", dispatchState: "not_started") + } } struct Request: Decodable { let id: String; let method: String; let params: Parameters } func required(_ value: T?, _ name: String) throws -> T { @@ -798,11 +1013,17 @@ final class Driver { let options = [kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String: true] as CFDictionary _ = AXIsProcessTrustedWithOptions(options) case "screenCapture": _ = CGRequestScreenCaptureAccess() + case "reminders": _ = try? await EKEventStore().requestFullAccessToReminders() + case "calendars": _ = try? await EKEventStore().requestFullAccessToEvents() + case "contacts": _ = try? await CNContactStore().requestAccess(for: .contacts) default: throw ComputerError("invalid_arguments", "Unknown permission.") } - return ["kind": "status", "platform": "darwin", "accessibility": AXIsProcessTrusted(), "screenRecording": CGPreflightScreenCaptureAccess()] - case "status": return ["kind": "status", "platform": "darwin", "accessibility": AXIsProcessTrusted(), "screenRecording": CGPreflightScreenCaptureAccess()] + return ["kind": "status", "platform": "darwin", "accessibility": AXIsProcessTrusted(), "screenRecording": CGPreflightScreenCaptureAccess(), "reminders": eventKitAuthorization(.reminder), "calendars": eventKitAuthorization(.event), "contacts": contactsAuthorization()] + case "status": return ["kind": "status", "platform": "darwin", "accessibility": AXIsProcessTrusted(), "screenRecording": CGPreflightScreenCaptureAccess(), "reminders": eventKitAuthorization(.reminder), "calendars": eventKitAuthorization(.event), "contacts": contactsAuthorization()] case "list_apps": return ["kind": "apps", "apps": try discoverApps()] + case "app_command": + inputDispatched = false + return try runAppCommand(required(p.command, "command")) case "activate_app": try requireAccessibility() let app = try await appFor(p.bundleId, launch: true) diff --git a/apps/desktop/native/computer-use/Info.plist b/apps/desktop/native/computer-use/Info.plist index 2ba6eb3d71d..1db88f73464 100644 --- a/apps/desktop/native/computer-use/Info.plist +++ b/apps/desktop/native/computer-use/Info.plist @@ -11,5 +11,8 @@ LSMinimumSystemVersion14.0 LSUIElement NSHighResolutionCapable +NSCalendarsFullAccessUsageDescriptionMothership adds calendar events you confirm and lists your calendars so it can file them correctly. +NSContactsUsageDescriptionMothership adds contacts and contact groups you confirm. +NSRemindersFullAccessUsageDescriptionMothership adds reminders you confirm and lists your reminder lists so it can file them correctly. NSScreenCaptureUsageDescriptionMothership captures the app window you select to understand and verify computer-use actions. diff --git a/apps/desktop/native/computer-use/README.md b/apps/desktop/native/computer-use/README.md index 1603675f8e1..13bb80c6ae5 100644 --- a/apps/desktop/native/computer-use/README.md +++ b/apps/desktop/native/computer-use/README.md @@ -6,7 +6,9 @@ Compile each architecture with `swiftc -parse-as-library -O -target arm64-apple- ## Protocol -Persistent newline-delimited JSON over stdin/stdout: `{id,method,params}` returns `{id,result}` or `{id,error:{code,message,dispatchState?}}`. The producer contract is `worker/packages/contracts/src/computer-use.ts` in mothership, copied into the desktop bridge by contract sync. `request_permission` is a local onboarding method, never a model tool; params are `{permission:"accessibility"|"screenCapture"}`. Status/preflight never triggers prompts. Errors go through the protocol, and stdout contains no logs. +Persistent newline-delimited JSON over stdin/stdout: `{id,method,params}` returns `{id,result}` or `{id,error:{code,message,dispatchState?}}`. The producer contract is `worker/packages/contracts/src/computer-use.ts` in mothership, copied into the desktop bridge by contract sync. `request_permission` is a local onboarding method, never a model tool; params are `{permission:"accessibility"|"screenCapture"|"reminders"|"calendars"|"contacts"}`. Status/preflight never triggers prompts; it reports `reminders`, `calendars` and `contacts` as `granted`, `denied` or `not_determined`. + +`app_command` takes `{command:{name,...}}` and runs one allowlisted command through EventKit or Contacts, never Apple Events or scripts. Reads: `list_reminder_lists`, `list_calendars` and `list_contact_groups` return `{kind:"command",items:[{id,title,allowsModifications}]}`. Creates return `{kind:"command",created:{id,title,list}}`: `create_reminder` (`title`, `notes`, `due` as `YYYY-MM-DD` or ISO 8601 with an offset, `listId`, `priority` 0/1/5/9), `create_event` (`title`, `start`/`end` as ISO 8601 with offsets or `YYYY-MM-DD` with `allDay`, at most 31 days, `location`, `notes`, `calendarId`, `alertMinutesBefore`), `create_contact_group` (`title`; an existing group with that name is returned instead of a duplicate) and `create_contact` (`givenName`, `familyName`, `organization`, `jobTitle`, up to 5 `emails` and `phones`, `groupId`). Without the matching access a command fails with `reminders_permission_required`, `calendars_permission_required` or `contacts_permission_required` before touching anything. There is no update, delete or search. The desktop service confirms each create with the user before sending it. The helper observes up to 2000 accessibility nodes (depth 64, queue 4000, traversal deadline 8 seconds, each AX call timeout 250ms). Selected-window and focused-element ancestry are prioritized. The menu bar and menus are read in place, so an open menu's items and their submenus come first; items of closed menus are read last, capped at 80 menu nodes. Electron accessibility is enabled using its documented `AXManualAccessibility` attribute. The helper observes 100 windows and 1000 apps. Requests are bounded to 128 KiB during ingestion. Text/value input is capped at 32000 UTF-16 units. State references are per-process, per-app, expire after 60 seconds, and are consumed by every action that reaches the app; a request that fails before sending anything reports `dispatchState: "not_started"` and keeps them valid. An Accessibility failure counts as sent unless Accessibility rejected the request before the app saw it (unsupported attribute or action, illegal argument). Restarts invalidate all references. Screen captures exclude window shadows, preserve the selected window’s coordinate geometry, and are capped to 1600 pixels on the longest side and 8 MiB encoded. Complete JSON replies are capped at 16 MiB: oversized observations omit screenshots first, then truncate accessibility nodes, while preserving action outcomes. Screenshots are optional and failure is reported in `screenshotError` without discarding accessibility state. macOS 26 uses the screenshot-specific capture API; the macOS 14/15 API path includes only the selected window in a display filter and crops its exact frame. That older path requires the window to fit within one display and reports `capture_geometry_unavailable` otherwise. diff --git a/apps/desktop/native/computer-use/tests/NativeTests.swift b/apps/desktop/native/computer-use/tests/NativeTests.swift index dccb74ed7ef..22d5e75e010 100644 --- a/apps/desktop/native/computer-use/tests/NativeTests.swift +++ b/apps/desktop/native/computer-use/tests/NativeTests.swift @@ -36,6 +36,24 @@ import AppKit precondition(isProtectedApp(bundle), bundle) } for bundle in ["com.apple.TextEdit", "com.apple.finder", "com.apple.Safari"] { precondition(!isProtectedApp(bundle), bundle) } + let dateOnly = try dueComponents("2027-05-04") + precondition(dateOnly.year == 2027 && dateOnly.month == 5 && dateOnly.day == 4 && dateOnly.hour == nil) + let timed = try dueComponents("2027-05-04T09:30:00Z") + precondition(timed.hour != nil && timed.minute != nil) + for due in ["2027-02-30", "2027-05-04T09:30:00", "May 4", "2027-5-4"] { + expectError("invalid_arguments") { _ = try dueComponents(due) } + } + expectError("unknown_command") { _ = try runAppCommand(AppCommand(name: "run_script")) } + expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "create_reminder", title: " ")) } + expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "create_reminder", title: "Pay rent", priority: 3)) } + expectError("invalid_arguments") { _ = try eventDate("2027-06-09T07:15:00", "Start", allDay: false) } + expectError("invalid_arguments") { _ = try eventDate("2027-06-09T07:15:00+01:00", "Start", allDay: true) } + precondition(try eventDate("2027-06-09T07:15:00+01:00", "Start", allDay: false) == Date(timeIntervalSince1970: 1_812_521_700)) + expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "create_event", title: "Flight", start: "2027-06-09T08:10:00Z", end: "2027-06-09T07:15:00Z")) } + expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "create_event", title: "Trip", start: "2027-06-01T00:00:00Z", end: "2027-08-01T00:00:00Z")) } + expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "create_contact", jobTitle: "Engineer")) } + expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "create_contact", givenName: "Ana", emails: ["not-an-email"])) } + expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "create_contact_group", title: " ")) } var queue = AXTraversalQueue(limit: 1000) queue.append(AXTraversalEntry(element: "menu", parent: nil, depth: 0, menu: true, priority: false)) queue.append(AXTraversalEntry(element: "content", parent: nil, depth: 0, menu: false, priority: false)) diff --git a/apps/desktop/src/main/computer-use/service.test.ts b/apps/desktop/src/main/computer-use/service.test.ts index c5d463b75d1..6ba819cb6ad 100644 --- a/apps/desktop/src/main/computer-use/service.test.ts +++ b/apps/desktop/src/main/computer-use/service.test.ts @@ -28,6 +28,12 @@ function setup(enabled = true, askForApps = true) { kind: 'apps', apps: [{ bundleId: 'com.example.Fixture', name: 'Fixture', pid: 42, isActive: false }], } + if (method === 'app_command') + return { + kind: 'command', + command: 'create_reminder', + created: { id: 'r1', title: 'Pay rent', list: 'Reminders' }, + } if (method === 'get_app_state') return { kind: 'state', @@ -48,6 +54,7 @@ function setup(enabled = true, askForApps = true) { }) const native = { request, stop: vi.fn() } const approveApp = vi.fn(async (): Promise<'always' | 'deny' | 'once'> => 'always') + const confirmAppCommand = vi.fn(async (_request: { appName: string; summary: string }) => true) const onActivity = vi.fn() const openPermissionSettings = vi.fn(async () => {}) const setStopShortcutActive = vi.fn((active: boolean) => active) @@ -56,6 +63,7 @@ function setup(enabled = true, askForApps = true) { native, supported: true, approveApp, + confirmAppCommand, onActivity, setStopShortcutActive, openPermissionSettings, @@ -77,6 +85,7 @@ function setup(enabled = true, askForApps = true) { config, native, approveApp, + confirmAppCommand, onActivity, setStopShortcutActive, openPermissionSettings, @@ -116,6 +125,80 @@ describe('native computer use authority and lifecycle', () => { expect(native.request.mock.calls.map(([method]) => method)).toEqual(['list_apps']) }) + it('confirms each app-command write before the helper runs it, and never observes after', async () => { + const { service, native, confirmAppCommand } = setup() + const create = { + action: 'app_command', + command: { name: 'create_reminder', title: 'Pay rent', due: '2027-05-04', priority: 1 }, + } + confirmAppCommand.mockResolvedValueOnce(false) + await expect(service.execute('declined', 'chat', create)).rejects.toMatchObject({ + details: { code: 'user_declined', dispatchState: 'not_started' }, + }) + expect(native.request.mock.calls.map(([method]) => method)).not.toContain('app_command') + expect(confirmAppCommand.mock.calls[0]?.[0].summary).toContain('Priority: High') + + await expect(service.execute('confirmed', 'chat', create)).resolves.toMatchObject({ + kind: 'command', + created: { id: 'r1' }, + }) + const methods = native.request.mock.calls.map(([method]) => method) + expect(methods.filter((method) => method === 'app_command')).toHaveLength(1) + expect(methods).not.toContain('get_app_state') + }) + + it.each([ + { + name: 'create_event', + title: 'TP 1947: LIS to OPO', + start: '2027-06-09T07:15:00+01:00', + end: '2027-06-09T08:10:00+01:00', + }, + { name: 'create_contact_group', title: 'Hires Q1 2027' }, + { name: 'create_contact', givenName: 'Ana', emails: ['ana@example.com'] }, + ])('asks the user before every $name write', async (command) => { + const { service, native, confirmAppCommand } = setup() + confirmAppCommand.mockResolvedValueOnce(false) + await expect( + service.execute(`declined-${command.name}`, 'chat', { action: 'app_command', command }) + ).rejects.toMatchObject({ details: { code: 'user_declined' } }) + expect(confirmAppCommand).toHaveBeenCalledOnce() + expect(native.request.mock.calls.map(([method]) => method)).not.toContain('app_command') + }) + + it('refuses an app command when the user denies the app, before asking to confirm', async () => { + const { service, native, approveApp, confirmAppCommand } = setup() + approveApp.mockResolvedValue('deny') + await expect( + service.execute('denied', 'chat', { + action: 'app_command', + command: { name: 'create_reminder', title: 'Pay rent' }, + }) + ).rejects.toThrow('denied') + expect(confirmAppCommand).not.toHaveBeenCalled() + expect(native.request.mock.calls.map(([method]) => method)).not.toContain('app_command') + }) + + it('lists reminder lists without a write confirmation', async () => { + const { service, native, confirmAppCommand } = setup() + native.request.mockImplementation(async (method: string) => + method === 'app_command' + ? { + kind: 'command', + command: 'list_reminder_lists', + items: [{ id: 'l1', title: 'Sprint 14', allowsModifications: true }], + } + : { kind: 'apps', apps: [] } + ) + await expect( + service.execute('lists', 'chat', { + action: 'app_command', + command: { name: 'list_reminder_lists' }, + }) + ).resolves.toMatchObject({ items: [{ title: 'Sprint 14' }] }) + expect(confirmAppCommand).not.toHaveBeenCalled() + }) + it('binds observed elements to their originating chat', async () => { const { state, click, native } = setup() await state('owner') diff --git a/apps/desktop/src/main/computer-use/service.ts b/apps/desktop/src/main/computer-use/service.ts index 397f5041589..db71a9d98cb 100644 --- a/apps/desktop/src/main/computer-use/service.ts +++ b/apps/desktop/src/main/computer-use/service.ts @@ -2,13 +2,15 @@ import { type ComputerUseActivity, type ComputerUseAppPermission, ComputerUseError, + type ComputerUsePermission, type ComputerUseStatus, } from '@sim/desktop-bridge' import { - type ComputerUseInput, + type AppCommand, type ComputerUseResult, - ComputerUseSchema, type ComputerUseSnapshot, + type ComputerUseWithAppCommandsInput, + ComputerUseWithAppCommandsSchema, } from '@sim/desktop-bridge/computer-use' import { getErrorMessage } from '@sim/utils/errors' import { omit } from '@sim/utils/object' @@ -23,15 +25,20 @@ interface ComputerUseServiceDeps { app: ComputerUseAppPermission, signal: AbortSignal ) => Promise<'once' | 'always' | 'deny'> + /** Confirms one app-command write. It shows the exact fields, since nothing happens on screen. */ + confirmAppCommand: ( + request: { appName: string; summary: string }, + signal: AbortSignal + ) => Promise onActivity: (activity: ComputerUseActivity | null) => void setStopShortcutActive?: (active: boolean) => boolean - openPermissionSettings?: (permission: 'accessibility' | 'screenCapture') => Promise + openPermissionSettings?: (permission: ComputerUsePermission) => Promise } interface ToolWork { toolCallId: string scopeId: string - input: ComputerUseInput + input: ComputerUseWithAppCommandsInput controller: AbortController } @@ -42,6 +49,67 @@ interface SnapshotOwnership { elementWindows: Map } +/** The app each command writes to; its Computer Use app permission governs the command. */ +const APP_COMMAND_APPS: Record = { + list_reminder_lists: 'com.apple.reminders', + create_reminder: 'com.apple.reminders', + list_calendars: 'com.apple.iCal', + create_event: 'com.apple.iCal', + list_contact_groups: 'com.apple.AddressBook', + create_contact_group: 'com.apple.AddressBook', + create_contact: 'com.apple.AddressBook', +} + +const PRIORITY_NAMES: Record = { 1: 'High', 5: 'Medium', 9: 'Low' } + +/** The exact fields a write will save, for the user to confirm; null for a read, which needs none. */ +function writeSummary(command: AppCommand): string | null { + const lines = (entries: Array<[string, string | number | undefined]>) => + entries + .filter(([, value]) => value !== undefined && value !== '') + .map(([label, value]) => `${label}: ${String(value).slice(0, 500)}`) + .join('\n') + switch (command.name) { + case 'create_reminder': + return lines([ + ['Reminder', command.title], + ['Due', command.due], + ['Priority', command.priority ? PRIORITY_NAMES[command.priority] : undefined], + ['List', command.listId ?? 'default'], + ['Notes', command.notes], + ]) + case 'create_event': + return lines([ + ['Event', command.title], + ['Starts', command.start], + ['Ends', command.end], + ['All day', command.allDay ? 'yes' : undefined], + ['Location', command.location], + [ + 'Alert', + command.alertMinutesBefore !== undefined + ? `${command.alertMinutesBefore} minutes before` + : undefined, + ], + ['Calendar', command.calendarId ?? 'default'], + ['Notes', command.notes], + ]) + case 'create_contact_group': + return lines([['Contact group', command.title]]) + case 'create_contact': + return lines([ + ['Contact', [command.givenName, command.familyName].filter(Boolean).join(' ') || undefined], + ['Organization', command.organization], + ['Job title', command.jobTitle], + ['Emails', command.emails?.join(', ')], + ['Phones', command.phones?.join(', ')], + ['Group', command.groupId], + ]) + default: + return null + } +} + /** Native snapshots cannot transfer authority between chats or restarted helpers. */ export class ComputerUseService { private admissions = new Map() @@ -82,6 +150,9 @@ export class ComputerUseService { status.permissions = { accessibility: native.accessibility, screenCapture: native.screenRecording, + reminders: native.reminders === 'granted', + calendars: native.calendars === 'granted', + contacts: native.contacts === 'granted', } status.enabled = this.isEnabled() status.activeAction = this.activity @@ -111,9 +182,7 @@ export class ComputerUseService { return this.getStatus() } - async requestPermission( - permission: 'accessibility' | 'screenCapture' - ): Promise { + async requestPermission(permission: ComputerUsePermission): Promise { if (!this.deps.supported) return this.getStatus() await this.deps.native.request('request_permission', { permission }) await this.deps.openPermissionSettings?.(permission) @@ -200,7 +269,7 @@ export class ComputerUseService { if (this.queued.has(toolCallId)) return Promise.reject(new Error('This Computer Use action is already running.')) // A newer server can send arguments this build predates; nothing has been queued yet. - const parsed = ComputerUseSchema.safeParse(raw) + const parsed = ComputerUseWithAppCommandsSchema.safeParse(raw) if (!parsed.success) return Promise.reject( new ComputerUseError({ @@ -240,6 +309,7 @@ export class ComputerUseService { stopShortcutAvailable: this.deps.setStopShortcutActive?.(true) ?? false, } this.deps.onActivity(this.activity) + if (input.action === 'app_command') return await this.runAppCommand(input.command, work) if ('bundleId' in input) { if (/^ai\.sim\.desktop(?:\.|$)/.test(input.bundleId)) { throw new Error('Computer Use cannot operate Sim or its own permission controls.') @@ -349,6 +419,37 @@ export class ComputerUseService { } } + /** + * A typed write through the OS's data frameworks leaves nothing on screen to watch, so each one + * needs the app's usual Computer Use permission and its own confirmation of the exact fields. + */ + private async runAppCommand(command: AppCommand, work: ToolWork): Promise { + const appName = await this.authorizeApp(APP_COMMAND_APPS[command.name], work) + if (this.activity) { + this.activity = { ...this.activity, appName } + this.deps.onActivity(this.activity) + } + const summary = writeSummary(command) + if (summary !== null) { + const confirmed = await this.deps.confirmAppCommand( + { appName, summary }, + work.controller.signal + ) + this.check(work) + if (!confirmed) + throw new ComputerUseError({ + code: 'user_declined', + message: 'The user declined to add this item. Do not retry it.', + dispatchState: 'not_started', + }) + } + this.check(work) + const result = await this.deps.native.request('app_command', { command }) + if (result.kind !== 'command' || result.command !== command.name) + throw new Error('Computer Use returned a result for a different command.') + return result + } + private rememberSnapshot(work: ToolWork, snapshot: ComputerUseSnapshot, windowId?: string): void { if (!('bundleId' in work.input) || snapshot.bundleId !== work.input.bundleId) throw new Error('Computer Use returned state for a different app.') diff --git a/apps/desktop/src/main/index.ts b/apps/desktop/src/main/index.ts index c6781c8666d..0fc6fdc6e0a 100644 --- a/apps/desktop/src/main/index.ts +++ b/apps/desktop/src/main/index.ts @@ -1,5 +1,6 @@ import { release } from 'node:os' import { join } from 'node:path' +import type { ComputerUsePermission } from '@sim/desktop-bridge' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import type { OpenDialogOptions, Session, WebContents } from 'electron' @@ -136,6 +137,15 @@ function reportHandoffFailure(error: unknown): void { /** Bounds what a hand-edited settings file can make an update relaunch open. */ const MAX_RELAUNCH_WINDOWS = 16 +/** The System Settings privacy pane for each Computer Use permission. */ +const PRIVACY_PANES: Record = { + accessibility: 'Privacy_Accessibility', + screenCapture: 'Privacy_ScreenCapture', + reminders: 'Privacy_Reminders', + calendars: 'Privacy_Calendars', + contacts: 'Privacy_Contacts', +} + const DOCK_ICON_FOR_CHANNEL = { prod: 'dock-icon.png', staging: 'dock-icon-staging.png', @@ -200,9 +210,7 @@ function main(): void { native: computerNative, openPermissionSettings: (permission) => shell.openExternal( - permission === 'accessibility' - ? 'x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility' - : 'x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture' + `x-apple.systempreferences:com.apple.preference.security?${PRIVACY_PANES[permission]}` ), setStopShortcutActive: (active) => { if (active && !computerStopShortcutRegistered) { @@ -229,6 +237,20 @@ function main(): void { }) return result.response === 0 ? 'once' : result.response === 1 ? 'always' : 'deny' }, + confirmAppCommand: async ({ appName, summary }, signal) => { + const result = await dialog.showMessageBox({ + type: 'question', + title: 'Computer Use', + message: `Let Mothership add this to ${appName}?`, + detail: summary, + buttons: ['Add', 'Cancel'], + defaultId: 1, + cancelId: 1, + noLink: true, + signal, + }) + return result.response === 0 + }, onActivity: (activity) => { for (const win of getWindows()) { if (isAppOrigin(win.webContents.getURL(), appOrigin())) { diff --git a/apps/desktop/src/main/ipc.test.ts b/apps/desktop/src/main/ipc.test.ts index c4fab9ad534..e1844913b03 100644 --- a/apps/desktop/src/main/ipc.test.ts +++ b/apps/desktop/src/main/ipc.test.ts @@ -607,6 +607,7 @@ describe('registerIpcHandlers', () => { native, supported: true, approveApp, + confirmAppCommand: vi.fn(async () => true), onActivity: vi.fn(), }) deps.computerUse = service diff --git a/apps/desktop/src/main/ipc.ts b/apps/desktop/src/main/ipc.ts index 0d60c338715..f40e1aa23d6 100644 --- a/apps/desktop/src/main/ipc.ts +++ b/apps/desktop/src/main/ipc.ts @@ -19,6 +19,7 @@ import { type DesktopUpdateState, type DesktopWindowState, type DesktopZoomPercent, + isComputerUsePermission, isDesktopAppearanceTheme, isDesktopScopeId, isDesktopZoomPercent, @@ -747,7 +748,7 @@ export function registerIpcHandlers(deps: IpcDeps): void { needsUserActivation: true, denied: null, handler: (permission) => { - if (permission !== 'accessibility' && permission !== 'screenCapture') + if (!isComputerUsePermission(permission)) throw new Error('Unknown Computer Use permission.') return deps.computerUse?.requestPermission(permission) }, diff --git a/apps/desktop/src/preload/index.ts b/apps/desktop/src/preload/index.ts index 873b880fba2..93cbc618c52 100644 --- a/apps/desktop/src/preload/index.ts +++ b/apps/desktop/src/preload/index.ts @@ -29,6 +29,7 @@ import type { BrowserSiteInfo, BrowserToolbarCommand, ComputerUseActivity, + ComputerUsePermission, DesktopAppearanceTheme, DesktopCommand, DesktopExecutorDevice, @@ -51,7 +52,7 @@ import type { TerminalShortcutCommand, TerminalThemeProfile, } from '@sim/desktop-bridge' -import type { ComputerUseInput } from '@sim/desktop-bridge/computer-use' +import type { ComputerUseWithAppCommandsInput } from '@sim/desktop-bridge/computer-use' import { type ScopedTerminalCommandEvent, type ScopedTerminalTabsState, @@ -128,15 +129,16 @@ const api: SimDesktopApi = { ? { computerUse: { stableElementIds: true, + appCommands: true, getActivity: () => ipcRenderer.invoke('computer-use:get-activity'), getStatus: () => ipcRenderer.invoke('computer-use:status'), setEnabled: (enabled: boolean) => ipcRenderer.invoke('computer-use:set-enabled', enabled), - requestPermission: (permission: 'accessibility' | 'screenCapture') => + requestPermission: (permission: ComputerUsePermission) => ipcRenderer.invoke('computer-use:request-permission', permission), setAskForApps: (ask: boolean) => ipcRenderer.invoke('computer-use:set-ask-for-apps', ask), listAppPermissions: () => ipcRenderer.invoke('computer-use:list-permissions'), revokeApp: (bundleId: string) => ipcRenderer.invoke('computer-use:revoke-app', bundleId), - executeTool: (toolCallId: string, params: ComputerUseInput) => + executeTool: (toolCallId: string, params: ComputerUseWithAppCommandsInput) => ipcRenderer.invoke('computer-use:execute-tool', toolCallId, params), cancel: (toolCallId?: string) => ipcRenderer.invoke('computer-use:cancel', toolCallId), onActivity: (callback: (activity: ComputerUseActivity | null) => void) => { diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/computer-use.tsx b/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/computer-use.tsx index 146524f724b..31fea8e7c32 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/computer-use.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/computer-use.tsx @@ -1,7 +1,11 @@ 'use client' import { useCallback, useEffect, useRef, useState } from 'react' -import type { ComputerUseAppPermission } from '@sim/desktop-bridge' +import { + COMPUTER_USE_PERMISSIONS, + type ComputerUseAppPermission, + type ComputerUsePermission, +} from '@sim/desktop-bridge' import { Chip, ChipSwitch, Label, toast } from '@sim/emcn' import { useParams } from 'next/navigation' import { ComputerUseActivity } from '@/components/computer-use/activity' @@ -16,6 +20,15 @@ const ON_OFF_OPTIONS = [ ] /** The activity stays visible so an in-flight action can be stopped; the device controls need the permission group's consent. */ +/** Shells without app commands report no app-data permissions, so their rows stay hidden. */ +const PERMISSION_LABELS: Record = { + accessibility: 'Accessibility', + screenCapture: 'Screen Recording', + reminders: 'Reminders', + calendars: 'Calendar', + contacts: 'Contacts', +} + export function ComputerUseSettings() { const { workspaceId, organizationId } = useParams<{ workspaceId?: string @@ -102,8 +115,10 @@ function ComputerUseSettingsControls() { )} {status?.enabled && ( <> - {(['accessibility', 'screenCapture'] as const).map((permission) => { - const label = permission === 'accessibility' ? 'Accessibility' : 'Screen Recording' + {COMPUTER_USE_PERMISSIONS.filter( + (permission) => status.permissions[permission] !== undefined + ).map((permission) => { + const label = PERMISSION_LABELS[permission] return (
diff --git a/apps/sim/components/computer-use/activity.tsx b/apps/sim/components/computer-use/activity.tsx index f365b9bddee..e0505576e64 100644 --- a/apps/sim/components/computer-use/activity.tsx +++ b/apps/sim/components/computer-use/activity.tsx @@ -18,6 +18,7 @@ const ACTION_LABELS: Record = { set_value: 'Editing', perform_action: 'Interacting', batch: 'Running steps', + app_command: 'Adding an item', } /** Remains visible while a native action is active, even if the permission group or device access revokes it. */ diff --git a/apps/sim/lib/computer-use/transport.ts b/apps/sim/lib/computer-use/transport.ts index 1e9346e1ede..f08dd278389 100644 --- a/apps/sim/lib/computer-use/transport.ts +++ b/apps/sim/lib/computer-use/transport.ts @@ -1,16 +1,16 @@ import { ComputerUseError } from '@sim/desktop-bridge' import { - type ComputerUseInput, ComputerUseNativeErrorSchema, type ComputerUseResult, ComputerUseResultSchema, + type ComputerUseWithAppCommandsInput, } from '@sim/desktop-bridge/computer-use' import { getDesktopBridge } from '@/lib/desktop' /** Desktop main independently authorizes the tool ID and executes only the server's canonical args. */ export async function executeComputerUseTool( toolCallId: string, - input: ComputerUseInput, + input: ComputerUseWithAppCommandsInput, signal?: AbortSignal ): Promise { signal?.throwIfAborted() diff --git a/apps/sim/lib/desktop/index.ts b/apps/sim/lib/desktop/index.ts index 307c72b9a75..113ef0ce6ee 100644 --- a/apps/sim/lib/desktop/index.ts +++ b/apps/sim/lib/desktop/index.ts @@ -160,6 +160,7 @@ export interface DesktopChatCapabilities { browser?: true terminal?: true computerUse?: true + computerUseAppCommands?: true browserSessions?: BrowserKnownSession[] terminals?: DesktopTerminalHint[] /** The desktop's background executor, offered to run this turn's desktop tools. */ @@ -238,6 +239,9 @@ export async function getDesktopChatCapabilities( ...(browser ? { browser: true as const } : {}), ...(terminal ? { terminal: true as const } : {}), ...(computerUse ? { computerUse: true as const } : {}), + ...(computerUse && bridge?.computerUse?.appCommands === true + ? { computerUseAppCommands: true as const } + : {}), ...(terminals.length > 0 ? { terminals } : {}), ...(browserSessions.length > 0 ? { browserSessions } : {}), ...(executorDevice diff --git a/apps/sim/lib/mothership/chat/payload.test.ts b/apps/sim/lib/mothership/chat/payload.test.ts index b300f46a5d3..34a8a9c5a03 100644 --- a/apps/sim/lib/mothership/chat/payload.test.ts +++ b/apps/sim/lib/mothership/chat/payload.test.ts @@ -888,6 +888,26 @@ describe('desktop request capabilities', () => { } ) + it.each([false, true])( + 'offers app commands only where computer use itself is allowed (allowed=%s)', + async (allowed) => { + mockComputerUseAvailable.mockResolvedValueOnce(allowed) + const payload = await buildCopilotRequestPayload({ + message: 'Add my sprint reminders', + workspaceId: 'workspace', + userId: 'user', + principal: { kind: 'session', userId: 'user', sessionId: 'session' }, + userMessageId: 'message', + mode: 'agent', + model: 'gpt-6-astra', + browser: true, + computerUse: true, + computerUseAppCommands: true, + }) + expect(payload.desktop?.computerUseAppCommands ?? false).toBe(allowed) + } + ) + it('preserves desktop capabilities and current session hints on the worker wire', async () => { const payload = await buildCopilotRequestPayload({ message: 'Inspect my local page', @@ -905,6 +925,7 @@ describe('desktop request capabilities', () => { }) expect(payload.desktop).toEqual({ computerUse: false, + computerUseAppCommands: false, browser: true, terminal: true, terminals: [{ id: 'terminal-1', cwd: '/work/app', active: true }], @@ -926,6 +947,7 @@ describe('desktop request capabilities', () => { }) expect(payload.desktop).toEqual({ computerUse: false, + computerUseAppCommands: false, localFiles: true, browser: false, terminal: false, diff --git a/apps/sim/lib/mothership/chat/payload.ts b/apps/sim/lib/mothership/chat/payload.ts index a6cace00771..de83de7fa29 100644 --- a/apps/sim/lib/mothership/chat/payload.ts +++ b/apps/sim/lib/mothership/chat/payload.ts @@ -99,6 +99,7 @@ interface BuildPayloadParams { browser?: boolean terminalCapable?: boolean computerUse?: boolean + computerUseAppCommands?: boolean terminals?: Array<{ id: string cwd?: string @@ -556,6 +557,7 @@ export async function buildCopilotRequestPayload(params: BuildPayloadParams): Pr browser: params.browser === true, terminal: params.terminalCapable === true, computerUse, + computerUseAppCommands: computerUse && params.computerUseAppCommands === true, terminals: params.terminalCapable ? (params.terminals ?? []).slice(0, 20) : [], browserSessions: params.browser ? (params.browserSessions ?? []).slice(0, 20) : [], }, diff --git a/apps/sim/lib/mothership/chat/post.ts b/apps/sim/lib/mothership/chat/post.ts index dc5fbb0e553..3d6018bf194 100644 --- a/apps/sim/lib/mothership/chat/post.ts +++ b/apps/sim/lib/mothership/chat/post.ts @@ -163,6 +163,7 @@ const ChatMessageSchema = z browser: z.boolean().optional(), terminal: z.boolean().optional(), computerUse: z.boolean().optional(), + computerUseAppCommands: z.boolean().optional(), terminals: z .array( z.object({ @@ -287,6 +288,7 @@ type UnifiedChatBranch = browser?: boolean terminalCapable?: boolean computerUse?: boolean + computerUseAppCommands?: boolean terminals?: Terminals browserSessions?: BrowserSessions }) => Promise @@ -329,6 +331,7 @@ type UnifiedChatBranch = browser?: boolean terminalCapable?: boolean computerUse?: boolean + computerUseAppCommands?: boolean terminals?: Terminals browserSessions?: BrowserSessions }) => Promise @@ -627,6 +630,7 @@ async function resolveBranch(params: { browser: payloadParams.browser, terminalCapable: payloadParams.terminalCapable, computerUse: payloadParams.computerUse, + computerUseAppCommands: payloadParams.computerUseAppCommands, terminals: payloadParams.terminals, browserSessions: payloadParams.browserSessions, }), @@ -691,6 +695,7 @@ async function resolveBranch(params: { browser: payloadParams.browser, terminalCapable: payloadParams.terminalCapable, computerUse: payloadParams.computerUse, + computerUseAppCommands: payloadParams.computerUseAppCommands, terminals: payloadParams.terminals, browserSessions: payloadParams.browserSessions, }), @@ -1206,6 +1211,7 @@ export async function handleUnifiedChatPost(req: NextRequest) { browser: body.desktopCapabilities?.browser === true, terminalCapable: body.desktopCapabilities?.terminal === true, computerUse: body.desktopCapabilities?.computerUse === true, + computerUseAppCommands: body.desktopCapabilities?.computerUseAppCommands === true, terminals: body.desktopCapabilities?.terminals, browserSessions: body.desktopCapabilities?.browserSessions, }) @@ -1234,6 +1240,7 @@ export async function handleUnifiedChatPost(req: NextRequest) { browser: body.desktopCapabilities?.browser === true, terminalCapable: body.desktopCapabilities?.terminal === true, computerUse: body.desktopCapabilities?.computerUse === true, + computerUseAppCommands: body.desktopCapabilities?.computerUseAppCommands === true, terminals: body.desktopCapabilities?.terminals, browserSessions: body.desktopCapabilities?.browserSessions, }) diff --git a/apps/sim/lib/mothership/generated/protocol.ts b/apps/sim/lib/mothership/generated/protocol.ts index f42d7216caa..3c84a60c102 100644 --- a/apps/sim/lib/mothership/generated/protocol.ts +++ b/apps/sim/lib/mothership/generated/protocol.ts @@ -108,6 +108,7 @@ export type ModelSelection = z.infer; export const DesktopContextSchema = z.object({ localFiles: z.boolean().optional(), computerUse: z.boolean().default(false), + computerUseAppCommands: z.boolean().default(false), browser: z.boolean().default(false), terminal: z.boolean().default(false), terminals: z diff --git a/apps/sim/lib/mothership/tools/client/computer-tool-execution.ts b/apps/sim/lib/mothership/tools/client/computer-tool-execution.ts index 4539b3e6f8d..69c674a9990 100644 --- a/apps/sim/lib/mothership/tools/client/computer-tool-execution.ts +++ b/apps/sim/lib/mothership/tools/client/computer-tool-execution.ts @@ -1,5 +1,5 @@ import { COMPUTER_USE_TOOL_TIMEOUT_MS, ComputerUseError } from '@sim/desktop-bridge' -import { ComputerUseSchema } from '@sim/desktop-bridge/computer-use' +import { ComputerUseWithAppCommandsSchema } from '@sim/desktop-bridge/computer-use' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { omit } from '@sim/utils/object' @@ -148,7 +148,7 @@ export async function executeComputerToolOnClient( 'This computer action is stale. Inspect the app again before deciding what to do.', { doNotRetry: true, outcomeUnknown: true } ) - const parsed = ComputerUseSchema.safeParse(omit(params, ['activity'])) + const parsed = ComputerUseWithAppCommandsSchema.safeParse(omit(params, ['activity'])) if (!parsed.success) return reject('Computer action arguments are invalid. Inspect the tool schema and try again.') if (claim !== 'claimed') @@ -245,7 +245,9 @@ export async function executeComputerToolOnClient( ? `Input was dispatched, but observation failed: ${result.observationError} Inspect the app without repeating the input.` : result.kind === 'action' && !result.verified ? 'Input was dispatched. Inspect the returned observation or the app to verify its effect.' - : 'Computer observation completed', + : result.kind === 'command' && result.created + ? `Created "${result.created.title}" in ${result.created.list}. It exists now: never repeat this create.` + : 'Computer observation completed', data: computerToolResultForModel(result, { baseSnapshotId: 'snapshotId' in parsed.data ? parsed.data.snapshotId : undefined, stableElementIds: getDesktopBridge()?.computerUse?.stableElementIds === true, diff --git a/packages/desktop-bridge/src/computer-use.generated.ts b/packages/desktop-bridge/src/computer-use.generated.ts index feef762b713..5dbe409d692 100644 --- a/packages/desktop-bridge/src/computer-use.generated.ts +++ b/packages/desktop-bridge/src/computer-use.generated.ts @@ -229,11 +229,96 @@ export const ComputerUseSchema = z.union([ ]); export type ComputerUseInput = z.infer; +/** Typed app commands: one allowlisted create or list read through the OS's own data frameworks, never a script. */ +const CommandTitle = z.string().min(1).max(1024); +const CommandNotes = z.string().max(8000).optional(); +const EventBoundary = z + .string() + .min(10) + .max(64) + .describe("ISO 8601 date-time with an explicit offset, or YYYY-MM-DD when allDay is true."); + +/** Typed app commands: one allowlisted create or list read through the OS's own data frameworks, never a script. */ +export const AppCommandSchema = z.discriminatedUnion("name", [ + z.strictObject({ name: z.literal("list_reminder_lists") }), + z.strictObject({ + name: z.literal("create_reminder"), + title: CommandTitle, + notes: CommandNotes, + due: z + .string() + .min(10) + .max(64) + .optional() + .describe("YYYY-MM-DD for a date-only reminder, or an ISO 8601 date-time with an explicit offset."), + listId: z + .string() + .min(1) + .max(256) + .optional() + .describe("An id from list_reminder_lists; omit for the default list."), + priority: z + .union([z.literal(0), z.literal(1), z.literal(5), z.literal(9)]) + .optional() + .describe("0 none, 1 high, 5 medium, 9 low."), + }), + z.strictObject({ name: z.literal("list_calendars") }), + z.strictObject({ + name: z.literal("create_event"), + title: CommandTitle, + start: EventBoundary, + end: EventBoundary, + allDay: z.boolean().optional(), + location: z.string().max(1024).optional(), + notes: CommandNotes, + calendarId: z + .string() + .min(1) + .max(256) + .optional() + .describe("An id from list_calendars; omit for the default calendar."), + alertMinutesBefore: z.number().int().min(0).max(40_320).optional(), + }), + z.strictObject({ name: z.literal("list_contact_groups") }), + z.strictObject({ + name: z.literal("create_contact_group"), + title: CommandTitle.describe( + "Group name; an existing group with this name is returned instead of a duplicate.", + ), + }), + z.strictObject({ + name: z.literal("create_contact"), + givenName: z.string().max(1024).optional(), + familyName: z.string().max(1024).optional(), + organization: z.string().max(1024).optional(), + jobTitle: z.string().max(1024).optional(), + emails: z.array(z.string().min(3).max(320)).max(5).optional(), + phones: z.array(z.string().min(1).max(64)).max(5).optional(), + groupId: z + .string() + .min(1) + .max(256) + .optional() + .describe("An id from list_contact_groups or create_contact_group."), + }), +]); +export type AppCommand = z.infer; + +/** The computer tool for desktops that support app commands. */ +export const ComputerUseWithAppCommandsSchema = z.union([ + ...ComputerUseSchema.options, + z.strictObject({ action: z.literal("app_command"), command: AppCommandSchema }), +]); +export type ComputerUseWithAppCommandsInput = z.infer; + export const ComputerUseStatusSchema = z.strictObject({ kind: z.literal("status"), platform: z.literal("darwin"), accessibility: z.boolean(), screenRecording: z.boolean(), + reminders: z.enum(["granted", "denied", "not_determined"]).optional(), + calendars: z.enum(["granted", "denied", "not_determined"]).optional(), + contacts: z.enum(["granted", "denied", "not_determined"]).optional(), }); export const ComputerUseAppSchema = z.strictObject({ @@ -301,8 +386,35 @@ export const ComputerUseSnapshotSchema = z.strictObject({ }); export type ComputerUseSnapshot = z.infer; +export const ComputerUseCommandResultSchema = z.strictObject({ + kind: z.literal("command"), + command: z.enum([ + "list_reminder_lists", + "create_reminder", + "list_calendars", + "create_event", + "list_contact_groups", + "create_contact_group", + "create_contact", + ]), + created: z + .strictObject({ id: z.string().min(1).max(256), title: z.string().max(1024), list: z.string().max(1024) }) + .optional(), + items: z + .array( + z.strictObject({ + id: z.string().min(1).max(256), + title: z.string().max(1024), + allowsModifications: z.boolean(), + }), + ) + .max(200) + .optional(), +}); + export const ComputerUseResultSchema = z.discriminatedUnion("kind", [ ComputerUseStatusSchema, + ComputerUseCommandResultSchema, z.strictObject({ kind: z.literal("apps"), apps: z.array(ComputerUseAppSchema).max(1000) }), ComputerUseSnapshotSchema, z diff --git a/packages/desktop-bridge/src/index.ts b/packages/desktop-bridge/src/index.ts index 4ec4b1bf27c..c6ef6ee150a 100644 --- a/packages/desktop-bridge/src/index.ts +++ b/packages/desktop-bridge/src/index.ts @@ -1,7 +1,7 @@ import type { - ComputerUseInput, ComputerUseNativeError, ComputerUseResult, + ComputerUseWithAppCommandsInput, } from './computer-use.generated' import type { DesktopLocalFileRequest, DesktopLocalFileResponse } from './local-files' @@ -58,7 +58,14 @@ export interface ComputerUseActivity { export interface ComputerUseStatus { supported: boolean enabled: boolean - permissions: { accessibility: boolean; screenCapture: boolean } + /** The app-data permissions are absent in shells without app commands. */ + permissions: { + accessibility: boolean + screenCapture: boolean + reminders?: boolean + calendars?: boolean + contacts?: boolean + } activeAction: ComputerUseActivity | null /** Whether a first use of each app needs approval; absent in older shells, which always ask. */ askForApps?: boolean @@ -83,6 +90,20 @@ export class ComputerUseError extends Error { } } +/** The macOS privacy permissions Computer Use can ask for. */ +export const COMPUTER_USE_PERMISSIONS = [ + 'accessibility', + 'screenCapture', + 'reminders', + 'calendars', + 'contacts', +] as const +export type ComputerUsePermission = (typeof COMPUTER_USE_PERMISSIONS)[number] + +export function isComputerUsePermission(value: unknown): value is ComputerUsePermission { + return COMPUTER_USE_PERMISSIONS.some((permission) => permission === value) +} + /** Optional so a new web deployment remains compatible with older desktop shells. */ interface SimDesktopComputerUseApi { /** @@ -90,18 +111,20 @@ interface SimDesktopComputerUseApi { * can list only what changed. Absent in older shells, whose ids are positional. */ stableElementIds?: true + /** The computer tool accepts typed app commands (reminders); absent in older shells. */ + appCommands?: true /** Current activity without starting the helper; absent in older shells. */ getActivity?(): Promise getStatus(): Promise setEnabled(enabled: boolean): Promise - requestPermission(permission: 'accessibility' | 'screenCapture'): Promise + requestPermission(permission: ComputerUsePermission): Promise /** Absent in older shells, which always ask before a first use of each app. */ setAskForApps?(ask: boolean): Promise listAppPermissions(): Promise revokeApp(bundleId: string): Promise executeTool( toolCallId: string, - params: ComputerUseInput + params: ComputerUseWithAppCommandsInput ): Promise cancel(toolCallId?: string): Promise onActivity(callback: (activity: ComputerUseActivity | null) => void): () => void