From c6822a502b79a0d82d1a126ac0bd0ef519007e36 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 18 Nov 2026 17:04:50 -0800 Subject: [PATCH 1/2] feat(computer-use): add typed reminder app commands behind a desktop capability --- apps/desktop/build/entitlements.mac.plist | 4 + apps/desktop/electron-builder.yml | 1 + .../native/computer-use/ComputerUse.swift | 88 ++++++++++++++++++- apps/desktop/native/computer-use/Info.plist | 1 + apps/desktop/native/computer-use/README.md | 4 +- .../computer-use/tests/NativeTests.swift | 10 +++ .../src/main/computer-use/service.test.ts | 64 ++++++++++++++ apps/desktop/src/main/computer-use/service.ts | 70 +++++++++++++-- apps/desktop/src/main/index.ts | 26 +++++- apps/desktop/src/main/ipc.test.ts | 1 + apps/desktop/src/main/ipc.ts | 6 +- apps/desktop/src/preload/index.ts | 8 +- .../components/desktop/computer-use.tsx | 57 +++++++----- apps/sim/components/computer-use/activity.tsx | 1 + apps/sim/lib/computer-use/transport.ts | 4 +- apps/sim/lib/desktop/index.ts | 4 + apps/sim/lib/mothership/chat/payload.test.ts | 22 +++++ apps/sim/lib/mothership/chat/payload.ts | 2 + apps/sim/lib/mothership/chat/post.ts | 7 ++ apps/sim/lib/mothership/generated/protocol.ts | 1 + .../tools/client/computer-tool-execution.ts | 8 +- .../src/computer-use.generated.ts | 54 ++++++++++++ packages/desktop-bridge/src/index.ts | 14 ++- 23 files changed, 407 insertions(+), 50 deletions(-) diff --git a/apps/desktop/build/entitlements.mac.plist b/apps/desktop/build/entitlements.mac.plist index edf7bcaf4b9..42715ee2373 100644 --- a/apps/desktop/build/entitlements.mac.plist +++ b/apps/desktop/build/entitlements.mac.plist @@ -14,5 +14,9 @@ Hardened Runtime denies the camera no matter what the user allowed. --> com.apple.security.device.camera + + com.apple.security.personal-information.calendars + diff --git a/apps/desktop/electron-builder.yml b/apps/desktop/electron-builder.yml index 5b303ac360b..7a8a26dae59 100644 --- a/apps/desktop/electron-builder.yml +++ b/apps/desktop/electron-builder.yml @@ -70,6 +70,7 @@ mac: # unless the bundle declares why it wants the device. extendInfo: NSScreenCaptureUsageDescription: Mothership captures approved app windows to understand and verify computer-use actions. + NSRemindersFullAccessUsageDescription: Mothership adds reminders you confirm and lists your reminder lists so it can file them correctly. NSMicrophoneUsageDescription: Sim uses your microphone for voice input in Chat and for meetings you join in the built-in browser. NSCameraUsageDescription: Sim uses your camera for meetings you join in the built-in browser, such as Google Meet. # macOS 15 blocks connections to private addresses (self-hosted servers on a diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index 907084daef8..cf59a5decbb 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -1,5 +1,6 @@ import AppKit import ApplicationServices +import EventKit import ScreenCaptureKit nonisolated(unsafe) var cancellationRequested: sig_atomic_t = 0 @@ -39,6 +40,85 @@ struct Parameters: Decodable { var button: String? var clickCount: Int? var accessibilityAction: String? + var command: AppCommand? +} +/// One allowlisted app command. Commands go through EventKit, never Apple Events or scripts, so they +/// cannot reach a shell; each is a typed create or a list read, with no update, delete or search. +struct AppCommand: Decodable { + var name: String + var title: String? + var notes: String? + var due: String? + var listId: String? + var priority: Int? +} +/// The event store for reminders, available only once the user has granted full Reminders access. +/// Checking never prompts; the grant happens from settings, outside the tool call. +func remindersStore() throws -> EKEventStore { + guard EKEventStore.authorizationStatus(for: .reminder) == .fullAccess else { + throw ComputerError("reminders_permission_required", "Allow Reminders access for Mothership in System Settings, then retry.", dispatchState: "not_started") + } + return EKEventStore() +} +/// A due date: `YYYY-MM-DD` is date-only; a date-time must be ISO 8601 with an explicit offset or `Z`, +/// so the reminder never depends on the helper's time zone guess. +func dueComponents(_ text: String) throws -> DateComponents { + var calendar = Calendar(identifier: .gregorian) + if text.range(of: #"^\d{4}-\d{2}-\d{2}$"#, options: .regularExpression) != nil { + let parts = text.split(separator: "-").compactMap { Int($0) } + let components = DateComponents(calendar: calendar, year: parts[0], month: parts[1], day: parts[2]) + guard components.isValidDate else { throw ComputerError("invalid_arguments", "Due date is not a real date.", dispatchState: "not_started") } + return DateComponents(year: parts[0], month: parts[1], day: parts[2]) + } + let formatter = ISO8601DateFormatter(); formatter.formatOptions = [.withInternetDateTime] + guard let date = formatter.date(from: text) else { + throw ComputerError("invalid_arguments", "Due must be YYYY-MM-DD or an ISO 8601 date-time with an offset.", dispatchState: "not_started") + } + calendar.timeZone = .current + return calendar.dateComponents([.year, .month, .day, .hour, .minute, .timeZone], from: date) +} +/// Reminders access as the status reports it, read without prompting. +func remindersAuthorization() -> String { + switch EKEventStore.authorizationStatus(for: .reminder) { + case .fullAccess: return "granted" + case .notDetermined: return "not_determined" + default: return "denied" + } +} +func runAppCommand(_ command: AppCommand) throws -> [String: Any] { + switch command.name { + case "list_reminder_lists": + let lists = try remindersStore().calendars(for: .reminder).prefix(200).map { list -> [String: Any] in + ["id": list.calendarIdentifier, "title": String(list.title.prefix(1024)), "allowsModifications": list.allowsContentModifications] + } + return ["kind": "command", "command": command.name, "items": lists] + case "create_reminder": + let title = (command.title ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + guard !title.isEmpty, title.count <= 1024 else { throw ComputerError("invalid_arguments", "A reminder needs a title of at most 1024 characters.", dispatchState: "not_started") } + guard (command.notes?.count ?? 0) <= 8000 else { throw ComputerError("invalid_arguments", "Notes are limited to 8000 characters.", dispatchState: "not_started") } + guard [0, 1, 5, 9].contains(command.priority ?? 0) else { throw ComputerError("invalid_arguments", "Priority must be 0 (none), 1 (high), 5 (medium) or 9 (low).", dispatchState: "not_started") } + let due = try command.due.map(dueComponents) + let store = try remindersStore() + let list: EKCalendar + if let listId = command.listId { + guard let found = store.calendar(withIdentifier: listId), found.allowedEntityTypes.contains(.reminder) else { + throw ComputerError("list_unavailable", "No reminder list has this id; call list_reminder_lists.", dispatchState: "not_started") + } + list = found + } else { + guard let fallback = store.defaultCalendarForNewReminders() else { throw ComputerError("list_unavailable", "Reminders has no default list; pass listId.", dispatchState: "not_started") } + list = fallback + } + guard list.allowsContentModifications else { throw ComputerError("list_read_only", "This reminder list is read-only.", dispatchState: "not_started") } + let reminder = EKReminder(eventStore: store) + reminder.calendar = list; reminder.title = title; reminder.notes = command.notes + reminder.priority = command.priority ?? 0; reminder.dueDateComponents = due + inputDispatched = true + try store.save(reminder, commit: true) + return ["kind": "command", "command": command.name, "created": ["id": reminder.calendarItemIdentifier, "title": title, "list": list.title]] + default: + throw ComputerError("unknown_command", "Unknown app command.", dispatchState: "not_started") + } } struct Request: Decodable { let id: String; let method: String; let params: Parameters } func required(_ value: T?, _ name: String) throws -> T { @@ -798,11 +878,15 @@ final class Driver { let options = [kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String: true] as CFDictionary _ = AXIsProcessTrustedWithOptions(options) case "screenCapture": _ = CGRequestScreenCaptureAccess() + case "reminders": _ = try? await EKEventStore().requestFullAccessToReminders() default: throw ComputerError("invalid_arguments", "Unknown permission.") } - return ["kind": "status", "platform": "darwin", "accessibility": AXIsProcessTrusted(), "screenRecording": CGPreflightScreenCaptureAccess()] - case "status": return ["kind": "status", "platform": "darwin", "accessibility": AXIsProcessTrusted(), "screenRecording": CGPreflightScreenCaptureAccess()] + return ["kind": "status", "platform": "darwin", "accessibility": AXIsProcessTrusted(), "screenRecording": CGPreflightScreenCaptureAccess(), "reminders": remindersAuthorization()] + case "status": return ["kind": "status", "platform": "darwin", "accessibility": AXIsProcessTrusted(), "screenRecording": CGPreflightScreenCaptureAccess(), "reminders": remindersAuthorization()] case "list_apps": return ["kind": "apps", "apps": try discoverApps()] + case "app_command": + inputDispatched = false + return try runAppCommand(required(p.command, "command")) case "activate_app": try requireAccessibility() let app = try await appFor(p.bundleId, launch: true) diff --git a/apps/desktop/native/computer-use/Info.plist b/apps/desktop/native/computer-use/Info.plist index 2ba6eb3d71d..3f1966337ff 100644 --- a/apps/desktop/native/computer-use/Info.plist +++ b/apps/desktop/native/computer-use/Info.plist @@ -11,5 +11,6 @@ LSMinimumSystemVersion14.0 LSUIElement NSHighResolutionCapable +NSRemindersFullAccessUsageDescriptionMothership adds reminders you confirm and lists your reminder lists so it can file them correctly. NSScreenCaptureUsageDescriptionMothership captures the app window you select to understand and verify computer-use actions. diff --git a/apps/desktop/native/computer-use/README.md b/apps/desktop/native/computer-use/README.md index 1603675f8e1..b122c3f0ed3 100644 --- a/apps/desktop/native/computer-use/README.md +++ b/apps/desktop/native/computer-use/README.md @@ -6,7 +6,9 @@ Compile each architecture with `swiftc -parse-as-library -O -target arm64-apple- ## Protocol -Persistent newline-delimited JSON over stdin/stdout: `{id,method,params}` returns `{id,result}` or `{id,error:{code,message,dispatchState?}}`. The producer contract is `worker/packages/contracts/src/computer-use.ts` in mothership, copied into the desktop bridge by contract sync. `request_permission` is a local onboarding method, never a model tool; params are `{permission:"accessibility"|"screenCapture"}`. Status/preflight never triggers prompts. Errors go through the protocol, and stdout contains no logs. +Persistent newline-delimited JSON over stdin/stdout: `{id,method,params}` returns `{id,result}` or `{id,error:{code,message,dispatchState?}}`. The producer contract is `worker/packages/contracts/src/computer-use.ts` in mothership, copied into the desktop bridge by contract sync. `request_permission` is a local onboarding method, never a model tool; params are `{permission:"accessibility"|"screenCapture"|"reminders"}`. Status/preflight never triggers prompts; it reports `reminders` as `granted`, `denied` or `not_determined`. + +`app_command` takes `{command:{name,...}}` and runs one allowlisted command through EventKit, never Apple Events or scripts: `list_reminder_lists` returns `{kind:"command",items:[{id,title,allowsModifications}]}`, and `create_reminder` (`title`, optional `notes`, `due` as `YYYY-MM-DD` or an ISO 8601 date-time with an offset, `listId`, `priority` 0/1/5/9) returns `{kind:"command",created:{id,title,list}}`. Without full Reminders access it fails with `reminders_permission_required` before touching anything. There is no update, delete or search. The desktop service confirms each create with the user before sending it. Errors go through the protocol, and stdout contains no logs. The helper observes up to 2000 accessibility nodes (depth 64, queue 4000, traversal deadline 8 seconds, each AX call timeout 250ms). Selected-window and focused-element ancestry are prioritized. The menu bar and menus are read in place, so an open menu's items and their submenus come first; items of closed menus are read last, capped at 80 menu nodes. Electron accessibility is enabled using its documented `AXManualAccessibility` attribute. The helper observes 100 windows and 1000 apps. Requests are bounded to 128 KiB during ingestion. Text/value input is capped at 32000 UTF-16 units. State references are per-process, per-app, expire after 60 seconds, and are consumed by every action that reaches the app; a request that fails before sending anything reports `dispatchState: "not_started"` and keeps them valid. An Accessibility failure counts as sent unless Accessibility rejected the request before the app saw it (unsupported attribute or action, illegal argument). Restarts invalidate all references. Screen captures exclude window shadows, preserve the selected window’s coordinate geometry, and are capped to 1600 pixels on the longest side and 8 MiB encoded. Complete JSON replies are capped at 16 MiB: oversized observations omit screenshots first, then truncate accessibility nodes, while preserving action outcomes. Screenshots are optional and failure is reported in `screenshotError` without discarding accessibility state. macOS 26 uses the screenshot-specific capture API; the macOS 14/15 API path includes only the selected window in a display filter and crops its exact frame. That older path requires the window to fit within one display and reports `capture_geometry_unavailable` otherwise. diff --git a/apps/desktop/native/computer-use/tests/NativeTests.swift b/apps/desktop/native/computer-use/tests/NativeTests.swift index dccb74ed7ef..9764e078709 100644 --- a/apps/desktop/native/computer-use/tests/NativeTests.swift +++ b/apps/desktop/native/computer-use/tests/NativeTests.swift @@ -36,6 +36,16 @@ import AppKit precondition(isProtectedApp(bundle), bundle) } for bundle in ["com.apple.TextEdit", "com.apple.finder", "com.apple.Safari"] { precondition(!isProtectedApp(bundle), bundle) } + let dateOnly = try dueComponents("2027-05-04") + precondition(dateOnly.year == 2027 && dateOnly.month == 5 && dateOnly.day == 4 && dateOnly.hour == nil) + let timed = try dueComponents("2027-05-04T09:30:00Z") + precondition(timed.hour != nil && timed.minute != nil) + for due in ["2027-02-30", "2027-05-04T09:30:00", "May 4", "2027-5-4"] { + expectError("invalid_arguments") { _ = try dueComponents(due) } + } + expectError("unknown_command") { _ = try runAppCommand(AppCommand(name: "run_script")) } + expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "create_reminder", title: " ")) } + expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "create_reminder", title: "Pay rent", priority: 3)) } var queue = AXTraversalQueue(limit: 1000) queue.append(AXTraversalEntry(element: "menu", parent: nil, depth: 0, menu: true, priority: false)) queue.append(AXTraversalEntry(element: "content", parent: nil, depth: 0, menu: false, priority: false)) diff --git a/apps/desktop/src/main/computer-use/service.test.ts b/apps/desktop/src/main/computer-use/service.test.ts index c5d463b75d1..9af996dc964 100644 --- a/apps/desktop/src/main/computer-use/service.test.ts +++ b/apps/desktop/src/main/computer-use/service.test.ts @@ -28,6 +28,12 @@ function setup(enabled = true, askForApps = true) { kind: 'apps', apps: [{ bundleId: 'com.example.Fixture', name: 'Fixture', pid: 42, isActive: false }], } + if (method === 'app_command') + return { + kind: 'command', + command: 'create_reminder', + created: { id: 'r1', title: 'Pay rent', list: 'Reminders' }, + } if (method === 'get_app_state') return { kind: 'state', @@ -48,6 +54,7 @@ function setup(enabled = true, askForApps = true) { }) const native = { request, stop: vi.fn() } const approveApp = vi.fn(async (): Promise<'always' | 'deny' | 'once'> => 'always') + const confirmAppCommand = vi.fn(async (_request: { appName: string; summary: string }) => true) const onActivity = vi.fn() const openPermissionSettings = vi.fn(async () => {}) const setStopShortcutActive = vi.fn((active: boolean) => active) @@ -56,6 +63,7 @@ function setup(enabled = true, askForApps = true) { native, supported: true, approveApp, + confirmAppCommand, onActivity, setStopShortcutActive, openPermissionSettings, @@ -77,6 +85,7 @@ function setup(enabled = true, askForApps = true) { config, native, approveApp, + confirmAppCommand, onActivity, setStopShortcutActive, openPermissionSettings, @@ -116,6 +125,61 @@ describe('native computer use authority and lifecycle', () => { expect(native.request.mock.calls.map(([method]) => method)).toEqual(['list_apps']) }) + it('confirms each app-command write before the helper runs it, and never observes after', async () => { + const { service, native, confirmAppCommand } = setup() + const create = { + action: 'app_command', + command: { name: 'create_reminder', title: 'Pay rent', due: '2027-05-04', priority: 1 }, + } + confirmAppCommand.mockResolvedValueOnce(false) + await expect(service.execute('declined', 'chat', create)).rejects.toMatchObject({ + details: { code: 'user_declined', dispatchState: 'not_started' }, + }) + expect(native.request.mock.calls.map(([method]) => method)).not.toContain('app_command') + expect(confirmAppCommand.mock.calls[0]?.[0].summary).toContain('Priority: High') + + await expect(service.execute('confirmed', 'chat', create)).resolves.toMatchObject({ + kind: 'command', + created: { id: 'r1' }, + }) + const methods = native.request.mock.calls.map(([method]) => method) + expect(methods.filter((method) => method === 'app_command')).toHaveLength(1) + expect(methods).not.toContain('get_app_state') + }) + + it('refuses an app command when the user denies the app, before asking to confirm', async () => { + const { service, native, approveApp, confirmAppCommand } = setup() + approveApp.mockResolvedValue('deny') + await expect( + service.execute('denied', 'chat', { + action: 'app_command', + command: { name: 'create_reminder', title: 'Pay rent' }, + }) + ).rejects.toThrow('denied') + expect(confirmAppCommand).not.toHaveBeenCalled() + expect(native.request.mock.calls.map(([method]) => method)).not.toContain('app_command') + }) + + it('lists reminder lists without a write confirmation', async () => { + const { service, native, confirmAppCommand } = setup() + native.request.mockImplementation(async (method: string) => + method === 'app_command' + ? { + kind: 'command', + command: 'list_reminder_lists', + items: [{ id: 'l1', title: 'Sprint 14', allowsModifications: true }], + } + : { kind: 'apps', apps: [] } + ) + await expect( + service.execute('lists', 'chat', { + action: 'app_command', + command: { name: 'list_reminder_lists' }, + }) + ).resolves.toMatchObject({ items: [{ title: 'Sprint 14' }] }) + expect(confirmAppCommand).not.toHaveBeenCalled() + }) + it('binds observed elements to their originating chat', async () => { const { state, click, native } = setup() await state('owner') diff --git a/apps/desktop/src/main/computer-use/service.ts b/apps/desktop/src/main/computer-use/service.ts index 397f5041589..d5199b65182 100644 --- a/apps/desktop/src/main/computer-use/service.ts +++ b/apps/desktop/src/main/computer-use/service.ts @@ -2,13 +2,15 @@ import { type ComputerUseActivity, type ComputerUseAppPermission, ComputerUseError, + type ComputerUsePermission, type ComputerUseStatus, } from '@sim/desktop-bridge' import { - type ComputerUseInput, + type AppCommand, type ComputerUseResult, - ComputerUseSchema, type ComputerUseSnapshot, + type ComputerUseWithAppCommandsInput, + ComputerUseWithAppCommandsSchema, } from '@sim/desktop-bridge/computer-use' import { getErrorMessage } from '@sim/utils/errors' import { omit } from '@sim/utils/object' @@ -23,15 +25,20 @@ interface ComputerUseServiceDeps { app: ComputerUseAppPermission, signal: AbortSignal ) => Promise<'once' | 'always' | 'deny'> + /** Confirms one app-command write. It shows the exact fields, since nothing happens on screen. */ + confirmAppCommand: ( + request: { appName: string; summary: string }, + signal: AbortSignal + ) => Promise onActivity: (activity: ComputerUseActivity | null) => void setStopShortcutActive?: (active: boolean) => boolean - openPermissionSettings?: (permission: 'accessibility' | 'screenCapture') => Promise + openPermissionSettings?: (permission: ComputerUsePermission) => Promise } interface ToolWork { toolCallId: string scopeId: string - input: ComputerUseInput + input: ComputerUseWithAppCommandsInput controller: AbortController } @@ -42,6 +49,14 @@ interface SnapshotOwnership { elementWindows: Map } +/** The app each command writes to; its Computer Use app permission governs the command. */ +const APP_COMMAND_APPS: Record = { + list_reminder_lists: 'com.apple.reminders', + create_reminder: 'com.apple.reminders', +} + +const PRIORITY_NAMES: Record = { 1: 'High', 5: 'Medium', 9: 'Low' } + /** Native snapshots cannot transfer authority between chats or restarted helpers. */ export class ComputerUseService { private admissions = new Map() @@ -82,6 +97,7 @@ export class ComputerUseService { status.permissions = { accessibility: native.accessibility, screenCapture: native.screenRecording, + reminders: native.reminders === 'granted', } status.enabled = this.isEnabled() status.activeAction = this.activity @@ -111,9 +127,7 @@ export class ComputerUseService { return this.getStatus() } - async requestPermission( - permission: 'accessibility' | 'screenCapture' - ): Promise { + async requestPermission(permission: ComputerUsePermission): Promise { if (!this.deps.supported) return this.getStatus() await this.deps.native.request('request_permission', { permission }) await this.deps.openPermissionSettings?.(permission) @@ -200,7 +214,7 @@ export class ComputerUseService { if (this.queued.has(toolCallId)) return Promise.reject(new Error('This Computer Use action is already running.')) // A newer server can send arguments this build predates; nothing has been queued yet. - const parsed = ComputerUseSchema.safeParse(raw) + const parsed = ComputerUseWithAppCommandsSchema.safeParse(raw) if (!parsed.success) return Promise.reject( new ComputerUseError({ @@ -240,6 +254,7 @@ export class ComputerUseService { stopShortcutAvailable: this.deps.setStopShortcutActive?.(true) ?? false, } this.deps.onActivity(this.activity) + if (input.action === 'app_command') return await this.runAppCommand(input.command, work) if ('bundleId' in input) { if (/^ai\.sim\.desktop(?:\.|$)/.test(input.bundleId)) { throw new Error('Computer Use cannot operate Sim or its own permission controls.') @@ -349,6 +364,45 @@ export class ComputerUseService { } } + /** + * A typed write through the OS's data frameworks leaves nothing on screen to watch, so each one + * needs the app's usual Computer Use permission and its own confirmation of the exact fields. + */ + private async runAppCommand(command: AppCommand, work: ToolWork): Promise { + const appName = await this.authorizeApp(APP_COMMAND_APPS[command.name], work) + if (this.activity) { + this.activity = { ...this.activity, appName } + this.deps.onActivity(this.activity) + } + if (command.name === 'create_reminder') { + const summary = [ + `Title: ${command.title}`, + command.due ? `Due: ${command.due}` : null, + command.priority ? `Priority: ${PRIORITY_NAMES[command.priority]}` : null, + command.listId ? `List: ${command.listId}` : 'List: default', + command.notes ? `Notes: ${command.notes.slice(0, 500)}` : null, + ] + .filter((line) => line !== null) + .join('\n') + const confirmed = await this.deps.confirmAppCommand( + { appName, summary }, + work.controller.signal + ) + this.check(work) + if (!confirmed) + throw new ComputerUseError({ + code: 'user_declined', + message: 'The user declined to create this reminder. Do not retry it.', + dispatchState: 'not_started', + }) + } + this.check(work) + const result = await this.deps.native.request('app_command', { command }) + if (result.kind !== 'command' || result.command !== command.name) + throw new Error('Computer Use returned a result for a different command.') + return result + } + private rememberSnapshot(work: ToolWork, snapshot: ComputerUseSnapshot, windowId?: string): void { if (!('bundleId' in work.input) || snapshot.bundleId !== work.input.bundleId) throw new Error('Computer Use returned state for a different app.') diff --git a/apps/desktop/src/main/index.ts b/apps/desktop/src/main/index.ts index c6781c8666d..83d30bc8c5c 100644 --- a/apps/desktop/src/main/index.ts +++ b/apps/desktop/src/main/index.ts @@ -1,5 +1,6 @@ import { release } from 'node:os' import { join } from 'node:path' +import type { ComputerUsePermission } from '@sim/desktop-bridge' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import type { OpenDialogOptions, Session, WebContents } from 'electron' @@ -136,6 +137,13 @@ function reportHandoffFailure(error: unknown): void { /** Bounds what a hand-edited settings file can make an update relaunch open. */ const MAX_RELAUNCH_WINDOWS = 16 +/** The System Settings privacy pane for each Computer Use permission. */ +const PRIVACY_PANES: Record = { + accessibility: 'Privacy_Accessibility', + screenCapture: 'Privacy_ScreenCapture', + reminders: 'Privacy_Reminders', +} + const DOCK_ICON_FOR_CHANNEL = { prod: 'dock-icon.png', staging: 'dock-icon-staging.png', @@ -200,9 +208,7 @@ function main(): void { native: computerNative, openPermissionSettings: (permission) => shell.openExternal( - permission === 'accessibility' - ? 'x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility' - : 'x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture' + `x-apple.systempreferences:com.apple.preference.security?${PRIVACY_PANES[permission]}` ), setStopShortcutActive: (active) => { if (active && !computerStopShortcutRegistered) { @@ -229,6 +235,20 @@ function main(): void { }) return result.response === 0 ? 'once' : result.response === 1 ? 'always' : 'deny' }, + confirmAppCommand: async ({ appName, summary }, signal) => { + const result = await dialog.showMessageBox({ + type: 'question', + title: 'Computer Use', + message: `Let Mothership add this to ${appName}?`, + detail: summary, + buttons: ['Add', 'Cancel'], + defaultId: 1, + cancelId: 1, + noLink: true, + signal, + }) + return result.response === 0 + }, onActivity: (activity) => { for (const win of getWindows()) { if (isAppOrigin(win.webContents.getURL(), appOrigin())) { diff --git a/apps/desktop/src/main/ipc.test.ts b/apps/desktop/src/main/ipc.test.ts index c4fab9ad534..e1844913b03 100644 --- a/apps/desktop/src/main/ipc.test.ts +++ b/apps/desktop/src/main/ipc.test.ts @@ -607,6 +607,7 @@ describe('registerIpcHandlers', () => { native, supported: true, approveApp, + confirmAppCommand: vi.fn(async () => true), onActivity: vi.fn(), }) deps.computerUse = service diff --git a/apps/desktop/src/main/ipc.ts b/apps/desktop/src/main/ipc.ts index 0d60c338715..31cf9d3aaf5 100644 --- a/apps/desktop/src/main/ipc.ts +++ b/apps/desktop/src/main/ipc.ts @@ -747,7 +747,11 @@ export function registerIpcHandlers(deps: IpcDeps): void { needsUserActivation: true, denied: null, handler: (permission) => { - if (permission !== 'accessibility' && permission !== 'screenCapture') + if ( + permission !== 'accessibility' && + permission !== 'screenCapture' && + permission !== 'reminders' + ) throw new Error('Unknown Computer Use permission.') return deps.computerUse?.requestPermission(permission) }, diff --git a/apps/desktop/src/preload/index.ts b/apps/desktop/src/preload/index.ts index 873b880fba2..93cbc618c52 100644 --- a/apps/desktop/src/preload/index.ts +++ b/apps/desktop/src/preload/index.ts @@ -29,6 +29,7 @@ import type { BrowserSiteInfo, BrowserToolbarCommand, ComputerUseActivity, + ComputerUsePermission, DesktopAppearanceTheme, DesktopCommand, DesktopExecutorDevice, @@ -51,7 +52,7 @@ import type { TerminalShortcutCommand, TerminalThemeProfile, } from '@sim/desktop-bridge' -import type { ComputerUseInput } from '@sim/desktop-bridge/computer-use' +import type { ComputerUseWithAppCommandsInput } from '@sim/desktop-bridge/computer-use' import { type ScopedTerminalCommandEvent, type ScopedTerminalTabsState, @@ -128,15 +129,16 @@ const api: SimDesktopApi = { ? { computerUse: { stableElementIds: true, + appCommands: true, getActivity: () => ipcRenderer.invoke('computer-use:get-activity'), getStatus: () => ipcRenderer.invoke('computer-use:status'), setEnabled: (enabled: boolean) => ipcRenderer.invoke('computer-use:set-enabled', enabled), - requestPermission: (permission: 'accessibility' | 'screenCapture') => + requestPermission: (permission: ComputerUsePermission) => ipcRenderer.invoke('computer-use:request-permission', permission), setAskForApps: (ask: boolean) => ipcRenderer.invoke('computer-use:set-ask-for-apps', ask), listAppPermissions: () => ipcRenderer.invoke('computer-use:list-permissions'), revokeApp: (bundleId: string) => ipcRenderer.invoke('computer-use:revoke-app', bundleId), - executeTool: (toolCallId: string, params: ComputerUseInput) => + executeTool: (toolCallId: string, params: ComputerUseWithAppCommandsInput) => ipcRenderer.invoke('computer-use:execute-tool', toolCallId, params), cancel: (toolCallId?: string) => ipcRenderer.invoke('computer-use:cancel', toolCallId), onActivity: (callback: (activity: ComputerUseActivity | null) => void) => { diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/computer-use.tsx b/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/computer-use.tsx index 146524f724b..517151a48cb 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/computer-use.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/computer-use.tsx @@ -16,6 +16,15 @@ const ON_OFF_OPTIONS = [ ] /** The activity stays visible so an in-flight action can be stopped; the device controls need the permission group's consent. */ +/** Reminders is listed only by shells that support app commands. */ +const PERMISSIONS = ['accessibility', 'screenCapture', 'reminders'] as const + +const PERMISSION_LABELS: Record<(typeof PERMISSIONS)[number], string> = { + accessibility: 'Accessibility', + screenCapture: 'Screen Recording', + reminders: 'Reminders', +} + export function ComputerUseSettings() { const { workspaceId, organizationId } = useParams<{ workspaceId?: string @@ -102,29 +111,31 @@ function ComputerUseSettingsControls() { )} {status?.enabled && ( <> - {(['accessibility', 'screenCapture'] as const).map((permission) => { - const label = permission === 'accessibility' ? 'Accessibility' : 'Screen Recording' - return ( -
- - {status.permissions[permission] ? ( - Allowed - ) : ( - - void update(async () => - setStatus(await bridge.requestPermission(permission)) - ) - } - > - Allow - - )} -
- ) - })} + {PERMISSIONS.filter((permission) => status.permissions[permission] !== undefined).map( + (permission) => { + const label = PERMISSION_LABELS[permission] + return ( +
+ + {status.permissions[permission] ? ( + Allowed + ) : ( + + void update(async () => + setStatus(await bridge.requestPermission(permission)) + ) + } + > + Allow + + )} +
+ ) + } + )} {setAskForApps && askForApps !== undefined && (