= {
+ accessibility: 'Accessibility',
+ screenCapture: 'Screen Recording',
+ reminders: 'Reminders',
+ calendars: 'Calendar',
+ contacts: 'Contacts',
+}
+
export function ComputerUseSettings() {
const { workspaceId, organizationId } = useParams<{
workspaceId?: string
@@ -102,8 +115,10 @@ function ComputerUseSettingsControls() {
)}
{status?.enabled && (
<>
- {(['accessibility', 'screenCapture'] as const).map((permission) => {
- const label = permission === 'accessibility' ? 'Accessibility' : 'Screen Recording'
+ {COMPUTER_USE_PERMISSIONS.filter(
+ (permission) => status.permissions[permission] !== undefined
+ ).map((permission) => {
+ const label = PERMISSION_LABELS[permission]
return (
diff --git a/apps/sim/components/computer-use/activity.tsx b/apps/sim/components/computer-use/activity.tsx
index f365b9bddee..e0505576e64 100644
--- a/apps/sim/components/computer-use/activity.tsx
+++ b/apps/sim/components/computer-use/activity.tsx
@@ -18,6 +18,7 @@ const ACTION_LABELS: Record = {
set_value: 'Editing',
perform_action: 'Interacting',
batch: 'Running steps',
+ app_command: 'Adding an item',
}
/** Remains visible while a native action is active, even if the permission group or device access revokes it. */
diff --git a/apps/sim/lib/computer-use/transport.ts b/apps/sim/lib/computer-use/transport.ts
index 1e9346e1ede..f08dd278389 100644
--- a/apps/sim/lib/computer-use/transport.ts
+++ b/apps/sim/lib/computer-use/transport.ts
@@ -1,16 +1,16 @@
import { ComputerUseError } from '@sim/desktop-bridge'
import {
- type ComputerUseInput,
ComputerUseNativeErrorSchema,
type ComputerUseResult,
ComputerUseResultSchema,
+ type ComputerUseWithAppCommandsInput,
} from '@sim/desktop-bridge/computer-use'
import { getDesktopBridge } from '@/lib/desktop'
/** Desktop main independently authorizes the tool ID and executes only the server's canonical args. */
export async function executeComputerUseTool(
toolCallId: string,
- input: ComputerUseInput,
+ input: ComputerUseWithAppCommandsInput,
signal?: AbortSignal
): Promise {
signal?.throwIfAborted()
diff --git a/apps/sim/lib/desktop/index.ts b/apps/sim/lib/desktop/index.ts
index 307c72b9a75..113ef0ce6ee 100644
--- a/apps/sim/lib/desktop/index.ts
+++ b/apps/sim/lib/desktop/index.ts
@@ -160,6 +160,7 @@ export interface DesktopChatCapabilities {
browser?: true
terminal?: true
computerUse?: true
+ computerUseAppCommands?: true
browserSessions?: BrowserKnownSession[]
terminals?: DesktopTerminalHint[]
/** The desktop's background executor, offered to run this turn's desktop tools. */
@@ -238,6 +239,9 @@ export async function getDesktopChatCapabilities(
...(browser ? { browser: true as const } : {}),
...(terminal ? { terminal: true as const } : {}),
...(computerUse ? { computerUse: true as const } : {}),
+ ...(computerUse && bridge?.computerUse?.appCommands === true
+ ? { computerUseAppCommands: true as const }
+ : {}),
...(terminals.length > 0 ? { terminals } : {}),
...(browserSessions.length > 0 ? { browserSessions } : {}),
...(executorDevice
diff --git a/apps/sim/lib/mothership/chat/payload.test.ts b/apps/sim/lib/mothership/chat/payload.test.ts
index b300f46a5d3..34a8a9c5a03 100644
--- a/apps/sim/lib/mothership/chat/payload.test.ts
+++ b/apps/sim/lib/mothership/chat/payload.test.ts
@@ -888,6 +888,26 @@ describe('desktop request capabilities', () => {
}
)
+ it.each([false, true])(
+ 'offers app commands only where computer use itself is allowed (allowed=%s)',
+ async (allowed) => {
+ mockComputerUseAvailable.mockResolvedValueOnce(allowed)
+ const payload = await buildCopilotRequestPayload({
+ message: 'Add my sprint reminders',
+ workspaceId: 'workspace',
+ userId: 'user',
+ principal: { kind: 'session', userId: 'user', sessionId: 'session' },
+ userMessageId: 'message',
+ mode: 'agent',
+ model: 'gpt-6-astra',
+ browser: true,
+ computerUse: true,
+ computerUseAppCommands: true,
+ })
+ expect(payload.desktop?.computerUseAppCommands ?? false).toBe(allowed)
+ }
+ )
+
it('preserves desktop capabilities and current session hints on the worker wire', async () => {
const payload = await buildCopilotRequestPayload({
message: 'Inspect my local page',
@@ -905,6 +925,7 @@ describe('desktop request capabilities', () => {
})
expect(payload.desktop).toEqual({
computerUse: false,
+ computerUseAppCommands: false,
browser: true,
terminal: true,
terminals: [{ id: 'terminal-1', cwd: '/work/app', active: true }],
@@ -926,6 +947,7 @@ describe('desktop request capabilities', () => {
})
expect(payload.desktop).toEqual({
computerUse: false,
+ computerUseAppCommands: false,
localFiles: true,
browser: false,
terminal: false,
diff --git a/apps/sim/lib/mothership/chat/payload.ts b/apps/sim/lib/mothership/chat/payload.ts
index a6cace00771..de83de7fa29 100644
--- a/apps/sim/lib/mothership/chat/payload.ts
+++ b/apps/sim/lib/mothership/chat/payload.ts
@@ -99,6 +99,7 @@ interface BuildPayloadParams {
browser?: boolean
terminalCapable?: boolean
computerUse?: boolean
+ computerUseAppCommands?: boolean
terminals?: Array<{
id: string
cwd?: string
@@ -556,6 +557,7 @@ export async function buildCopilotRequestPayload(params: BuildPayloadParams): Pr
browser: params.browser === true,
terminal: params.terminalCapable === true,
computerUse,
+ computerUseAppCommands: computerUse && params.computerUseAppCommands === true,
terminals: params.terminalCapable ? (params.terminals ?? []).slice(0, 20) : [],
browserSessions: params.browser ? (params.browserSessions ?? []).slice(0, 20) : [],
},
diff --git a/apps/sim/lib/mothership/chat/post.ts b/apps/sim/lib/mothership/chat/post.ts
index dc5fbb0e553..3d6018bf194 100644
--- a/apps/sim/lib/mothership/chat/post.ts
+++ b/apps/sim/lib/mothership/chat/post.ts
@@ -163,6 +163,7 @@ const ChatMessageSchema = z
browser: z.boolean().optional(),
terminal: z.boolean().optional(),
computerUse: z.boolean().optional(),
+ computerUseAppCommands: z.boolean().optional(),
terminals: z
.array(
z.object({
@@ -287,6 +288,7 @@ type UnifiedChatBranch =
browser?: boolean
terminalCapable?: boolean
computerUse?: boolean
+ computerUseAppCommands?: boolean
terminals?: Terminals
browserSessions?: BrowserSessions
}) => Promise
@@ -329,6 +331,7 @@ type UnifiedChatBranch =
browser?: boolean
terminalCapable?: boolean
computerUse?: boolean
+ computerUseAppCommands?: boolean
terminals?: Terminals
browserSessions?: BrowserSessions
}) => Promise
@@ -627,6 +630,7 @@ async function resolveBranch(params: {
browser: payloadParams.browser,
terminalCapable: payloadParams.terminalCapable,
computerUse: payloadParams.computerUse,
+ computerUseAppCommands: payloadParams.computerUseAppCommands,
terminals: payloadParams.terminals,
browserSessions: payloadParams.browserSessions,
}),
@@ -691,6 +695,7 @@ async function resolveBranch(params: {
browser: payloadParams.browser,
terminalCapable: payloadParams.terminalCapable,
computerUse: payloadParams.computerUse,
+ computerUseAppCommands: payloadParams.computerUseAppCommands,
terminals: payloadParams.terminals,
browserSessions: payloadParams.browserSessions,
}),
@@ -1206,6 +1211,7 @@ export async function handleUnifiedChatPost(req: NextRequest) {
browser: body.desktopCapabilities?.browser === true,
terminalCapable: body.desktopCapabilities?.terminal === true,
computerUse: body.desktopCapabilities?.computerUse === true,
+ computerUseAppCommands: body.desktopCapabilities?.computerUseAppCommands === true,
terminals: body.desktopCapabilities?.terminals,
browserSessions: body.desktopCapabilities?.browserSessions,
})
@@ -1234,6 +1240,7 @@ export async function handleUnifiedChatPost(req: NextRequest) {
browser: body.desktopCapabilities?.browser === true,
terminalCapable: body.desktopCapabilities?.terminal === true,
computerUse: body.desktopCapabilities?.computerUse === true,
+ computerUseAppCommands: body.desktopCapabilities?.computerUseAppCommands === true,
terminals: body.desktopCapabilities?.terminals,
browserSessions: body.desktopCapabilities?.browserSessions,
})
diff --git a/apps/sim/lib/mothership/generated/protocol.ts b/apps/sim/lib/mothership/generated/protocol.ts
index f42d7216caa..3c84a60c102 100644
--- a/apps/sim/lib/mothership/generated/protocol.ts
+++ b/apps/sim/lib/mothership/generated/protocol.ts
@@ -108,6 +108,7 @@ export type ModelSelection = z.infer;
export const DesktopContextSchema = z.object({
localFiles: z.boolean().optional(),
computerUse: z.boolean().default(false),
+ computerUseAppCommands: z.boolean().default(false),
browser: z.boolean().default(false),
terminal: z.boolean().default(false),
terminals: z
diff --git a/apps/sim/lib/mothership/tools/client/computer-tool-execution.ts b/apps/sim/lib/mothership/tools/client/computer-tool-execution.ts
index 4539b3e6f8d..69c674a9990 100644
--- a/apps/sim/lib/mothership/tools/client/computer-tool-execution.ts
+++ b/apps/sim/lib/mothership/tools/client/computer-tool-execution.ts
@@ -1,5 +1,5 @@
import { COMPUTER_USE_TOOL_TIMEOUT_MS, ComputerUseError } from '@sim/desktop-bridge'
-import { ComputerUseSchema } from '@sim/desktop-bridge/computer-use'
+import { ComputerUseWithAppCommandsSchema } from '@sim/desktop-bridge/computer-use'
import { createLogger } from '@sim/logger'
import { getErrorMessage } from '@sim/utils/errors'
import { omit } from '@sim/utils/object'
@@ -148,7 +148,7 @@ export async function executeComputerToolOnClient(
'This computer action is stale. Inspect the app again before deciding what to do.',
{ doNotRetry: true, outcomeUnknown: true }
)
- const parsed = ComputerUseSchema.safeParse(omit(params, ['activity']))
+ const parsed = ComputerUseWithAppCommandsSchema.safeParse(omit(params, ['activity']))
if (!parsed.success)
return reject('Computer action arguments are invalid. Inspect the tool schema and try again.')
if (claim !== 'claimed')
@@ -245,7 +245,9 @@ export async function executeComputerToolOnClient(
? `Input was dispatched, but observation failed: ${result.observationError} Inspect the app without repeating the input.`
: result.kind === 'action' && !result.verified
? 'Input was dispatched. Inspect the returned observation or the app to verify its effect.'
- : 'Computer observation completed',
+ : result.kind === 'command' && result.created
+ ? `Created "${result.created.title}" in ${result.created.list}. It exists now: never repeat this create.`
+ : 'Computer observation completed',
data: computerToolResultForModel(result, {
baseSnapshotId: 'snapshotId' in parsed.data ? parsed.data.snapshotId : undefined,
stableElementIds: getDesktopBridge()?.computerUse?.stableElementIds === true,
diff --git a/packages/desktop-bridge/src/computer-use.generated.ts b/packages/desktop-bridge/src/computer-use.generated.ts
index feef762b713..5dbe409d692 100644
--- a/packages/desktop-bridge/src/computer-use.generated.ts
+++ b/packages/desktop-bridge/src/computer-use.generated.ts
@@ -229,11 +229,96 @@ export const ComputerUseSchema = z.union([
]);
export type ComputerUseInput = z.infer;
+/** Typed app commands: one allowlisted create or list read through the OS's own data frameworks, never a script. */
+const CommandTitle = z.string().min(1).max(1024);
+const CommandNotes = z.string().max(8000).optional();
+const EventBoundary = z
+ .string()
+ .min(10)
+ .max(64)
+ .describe("ISO 8601 date-time with an explicit offset, or YYYY-MM-DD when allDay is true.");
+
+/** Typed app commands: one allowlisted create or list read through the OS's own data frameworks, never a script. */
+export const AppCommandSchema = z.discriminatedUnion("name", [
+ z.strictObject({ name: z.literal("list_reminder_lists") }),
+ z.strictObject({
+ name: z.literal("create_reminder"),
+ title: CommandTitle,
+ notes: CommandNotes,
+ due: z
+ .string()
+ .min(10)
+ .max(64)
+ .optional()
+ .describe("YYYY-MM-DD for a date-only reminder, or an ISO 8601 date-time with an explicit offset."),
+ listId: z
+ .string()
+ .min(1)
+ .max(256)
+ .optional()
+ .describe("An id from list_reminder_lists; omit for the default list."),
+ priority: z
+ .union([z.literal(0), z.literal(1), z.literal(5), z.literal(9)])
+ .optional()
+ .describe("0 none, 1 high, 5 medium, 9 low."),
+ }),
+ z.strictObject({ name: z.literal("list_calendars") }),
+ z.strictObject({
+ name: z.literal("create_event"),
+ title: CommandTitle,
+ start: EventBoundary,
+ end: EventBoundary,
+ allDay: z.boolean().optional(),
+ location: z.string().max(1024).optional(),
+ notes: CommandNotes,
+ calendarId: z
+ .string()
+ .min(1)
+ .max(256)
+ .optional()
+ .describe("An id from list_calendars; omit for the default calendar."),
+ alertMinutesBefore: z.number().int().min(0).max(40_320).optional(),
+ }),
+ z.strictObject({ name: z.literal("list_contact_groups") }),
+ z.strictObject({
+ name: z.literal("create_contact_group"),
+ title: CommandTitle.describe(
+ "Group name; an existing group with this name is returned instead of a duplicate.",
+ ),
+ }),
+ z.strictObject({
+ name: z.literal("create_contact"),
+ givenName: z.string().max(1024).optional(),
+ familyName: z.string().max(1024).optional(),
+ organization: z.string().max(1024).optional(),
+ jobTitle: z.string().max(1024).optional(),
+ emails: z.array(z.string().min(3).max(320)).max(5).optional(),
+ phones: z.array(z.string().min(1).max(64)).max(5).optional(),
+ groupId: z
+ .string()
+ .min(1)
+ .max(256)
+ .optional()
+ .describe("An id from list_contact_groups or create_contact_group."),
+ }),
+]);
+export type AppCommand = z.infer;
+
+/** The computer tool for desktops that support app commands. */
+export const ComputerUseWithAppCommandsSchema = z.union([
+ ...ComputerUseSchema.options,
+ z.strictObject({ action: z.literal("app_command"), command: AppCommandSchema }),
+]);
+export type ComputerUseWithAppCommandsInput = z.infer;
+
export const ComputerUseStatusSchema = z.strictObject({
kind: z.literal("status"),
platform: z.literal("darwin"),
accessibility: z.boolean(),
screenRecording: z.boolean(),
+ reminders: z.enum(["granted", "denied", "not_determined"]).optional(),
+ calendars: z.enum(["granted", "denied", "not_determined"]).optional(),
+ contacts: z.enum(["granted", "denied", "not_determined"]).optional(),
});
export const ComputerUseAppSchema = z.strictObject({
@@ -301,8 +386,35 @@ export const ComputerUseSnapshotSchema = z.strictObject({
});
export type ComputerUseSnapshot = z.infer;
+export const ComputerUseCommandResultSchema = z.strictObject({
+ kind: z.literal("command"),
+ command: z.enum([
+ "list_reminder_lists",
+ "create_reminder",
+ "list_calendars",
+ "create_event",
+ "list_contact_groups",
+ "create_contact_group",
+ "create_contact",
+ ]),
+ created: z
+ .strictObject({ id: z.string().min(1).max(256), title: z.string().max(1024), list: z.string().max(1024) })
+ .optional(),
+ items: z
+ .array(
+ z.strictObject({
+ id: z.string().min(1).max(256),
+ title: z.string().max(1024),
+ allowsModifications: z.boolean(),
+ }),
+ )
+ .max(200)
+ .optional(),
+});
+
export const ComputerUseResultSchema = z.discriminatedUnion("kind", [
ComputerUseStatusSchema,
+ ComputerUseCommandResultSchema,
z.strictObject({ kind: z.literal("apps"), apps: z.array(ComputerUseAppSchema).max(1000) }),
ComputerUseSnapshotSchema,
z
diff --git a/packages/desktop-bridge/src/index.ts b/packages/desktop-bridge/src/index.ts
index 4ec4b1bf27c..c6ef6ee150a 100644
--- a/packages/desktop-bridge/src/index.ts
+++ b/packages/desktop-bridge/src/index.ts
@@ -1,7 +1,7 @@
import type {
- ComputerUseInput,
ComputerUseNativeError,
ComputerUseResult,
+ ComputerUseWithAppCommandsInput,
} from './computer-use.generated'
import type { DesktopLocalFileRequest, DesktopLocalFileResponse } from './local-files'
@@ -58,7 +58,14 @@ export interface ComputerUseActivity {
export interface ComputerUseStatus {
supported: boolean
enabled: boolean
- permissions: { accessibility: boolean; screenCapture: boolean }
+ /** The app-data permissions are absent in shells without app commands. */
+ permissions: {
+ accessibility: boolean
+ screenCapture: boolean
+ reminders?: boolean
+ calendars?: boolean
+ contacts?: boolean
+ }
activeAction: ComputerUseActivity | null
/** Whether a first use of each app needs approval; absent in older shells, which always ask. */
askForApps?: boolean
@@ -83,6 +90,20 @@ export class ComputerUseError extends Error {
}
}
+/** The macOS privacy permissions Computer Use can ask for. */
+export const COMPUTER_USE_PERMISSIONS = [
+ 'accessibility',
+ 'screenCapture',
+ 'reminders',
+ 'calendars',
+ 'contacts',
+] as const
+export type ComputerUsePermission = (typeof COMPUTER_USE_PERMISSIONS)[number]
+
+export function isComputerUsePermission(value: unknown): value is ComputerUsePermission {
+ return COMPUTER_USE_PERMISSIONS.some((permission) => permission === value)
+}
+
/** Optional so a new web deployment remains compatible with older desktop shells. */
interface SimDesktopComputerUseApi {
/**
@@ -90,18 +111,20 @@ interface SimDesktopComputerUseApi {
* can list only what changed. Absent in older shells, whose ids are positional.
*/
stableElementIds?: true
+ /** The computer tool accepts typed app commands (reminders); absent in older shells. */
+ appCommands?: true
/** Current activity without starting the helper; absent in older shells. */
getActivity?(): Promise
getStatus(): Promise
setEnabled(enabled: boolean): Promise
- requestPermission(permission: 'accessibility' | 'screenCapture'): Promise
+ requestPermission(permission: ComputerUsePermission): Promise
/** Absent in older shells, which always ask before a first use of each app. */
setAskForApps?(ask: boolean): Promise
listAppPermissions(): Promise
revokeApp(bundleId: string): Promise
executeTool(
toolCallId: string,
- params: ComputerUseInput
+ params: ComputerUseWithAppCommandsInput
): Promise
cancel(toolCallId?: string): Promise
onActivity(callback: (activity: ComputerUseActivity | null) => void): () => void