diff --git a/apps/sim/.env.example b/apps/sim/.env.example index 1bacb5f98a9..ec281638828 100644 --- a/apps/sim/.env.example +++ b/apps/sim/.env.example @@ -31,6 +31,7 @@ NEXT_PUBLIC_APP_URL=http://localhost:3000 # NEXT_PUBLIC_CHAT_DISABLED=true # Hides the Chat module: the workspace lands on your first workflow, and the chats list, scheduled tasks, and editor Chat panel are absent. Chat is shown when unset; `bun run setup` sets this for you if you skip the chat key # MOTHERSHIP_BENCHMARK_ENABLED=true # Shows the organization Benchmark tab and enables its authenticated APIs # COPILOT_DEV_URL=http://localhost:8080 # Benchmark worker endpoint; requires the worker benchmark profile and the normal COPILOT_API_KEY +# MOTHERSHIP_REQUIRE_RUN_GRANT=true # Refuse Mothership worker calls that carry no run grant for their user and chat; set once your Mothership forwards run grants # Remote Function sandboxes (Optional) # Build a dedicated Function base before enabling JavaScript imports, Python, or Shell. diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index 5a757490398..2654d8820c7 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -162,6 +162,7 @@ export const env = createEnv({ /** Enables the organization benchmark UI and its authenticated endpoints. */ MOTHERSHIP_BENCHMARK_ENABLED: z.boolean().optional(), MOTHERSHIP_BENCHMARK_URL: z.string().url().optional(), + MOTHERSHIP_REQUIRE_RUN_GRANT: z.boolean().optional(), // Worker calls must carry a run grant naming their user and chat; unset, a call without one is still served SIM_AGENT_API_URL: z.string().url().optional(), // URL for internal sim agent API MSHIP_SYSPROMPT_OVERRIDE: z.string().min(1).optional(), // Enterprise-only highest-priority Mothership system prompt override forwarded by Sim COPILOT_SOURCE_ENV: z.enum(['dev', 'staging', 'prod']).optional(), // Source Sim environment sent to mothership for callbacks diff --git a/apps/sim/lib/mothership/auth/internal.test.ts b/apps/sim/lib/mothership/auth/internal.test.ts index a740bb9753f..ee6b8fa5ba0 100644 --- a/apps/sim/lib/mothership/auth/internal.test.ts +++ b/apps/sim/lib/mothership/auth/internal.test.ts @@ -1,7 +1,10 @@ import { createMockRequest } from '@sim/testing' import { copilotHttpMock, copilotHttpMockFns } from '@sim/testing/mocks/copilot-http.mock' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { resetEnvMock, setEnv } from '@sim/testing/mocks/env.mock' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { internalCopilotAuth } from '@/lib/mothership/auth/internal' +import { issueRunGrant } from '@/lib/mothership/auth/run-grant' +import { RUN_GRANT_HEADER } from '@/lib/mothership/generated/sim-transport' vi.mock('@/lib/mothership/request/http', () => copilotHttpMock) @@ -44,4 +47,37 @@ describe('worker control identity', () => { apiKey.mockReturnValue({ success: false }) await expect(auth.authenticate(request(headers))).rejects.toThrow('Unauthorized') }) + describe('run grants', () => { + afterEach(resetEnvMock) + const auth = internalCopilotAuth('control', { organization: true }) + const grant = () => + issueRunGrant({ + userId: 'actor', + chatId: 'chat-1', + workspaceId: null, + organizationId: 'org-1', + }) + it('admits identity headers the run grant names, and refuses forged or mismatched grants', async () => { + await expect( + auth.authenticate(request({ ...headers, [RUN_GRANT_HEADER]: await grant() })) + ).resolves.toMatchObject({ subjectUserId: 'actor', resourceScope: { chatId: 'chat-1' } }) + await expect( + auth.authenticate( + request({ + ...headers, + 'x-mothership-chat-id': 'chat-2', + [RUN_GRANT_HEADER]: await grant(), + }) + ) + ).rejects.toThrow('Unauthorized') + await expect( + auth.authenticate(request({ ...headers, [RUN_GRANT_HEADER]: 'not-a-grant' })) + ).rejects.toThrow('Unauthorized') + }) + it('refuses a call without a grant once grants are required', async () => { + await expect(auth.authenticate(request(headers))).resolves.toBeDefined() + setEnv({ MOTHERSHIP_REQUIRE_RUN_GRANT: true }) + await expect(auth.authenticate(request(headers))).rejects.toThrow('Unauthorized') + }) + }) }) diff --git a/apps/sim/lib/mothership/auth/internal.ts b/apps/sim/lib/mothership/auth/internal.ts index f6e8a49e97d..4389a9a5158 100644 --- a/apps/sim/lib/mothership/auth/internal.ts +++ b/apps/sim/lib/mothership/auth/internal.ts @@ -6,9 +6,14 @@ import { createTrustedCopilotPrincipal, createTrustedOrganizationCopilotPrincipal, } from '@/lib/mothership/auth/application-delegation' +import { acceptsRunGrant, checkRunGrant, rememberRunGrant } from '@/lib/mothership/auth/run-grant' +import { RUN_GRANT_HEADER } from '@/lib/mothership/generated/sim-transport' import { checkInternalApiKey } from '@/lib/mothership/request/http' -/** Identity headers are assertions from the authenticated worker, never browser credentials. */ +/** + * Identity headers are assertions from the authenticated worker, never browser credentials; the + * run grant Sim issued with the turn proves them (`run-grant.ts`). + */ export function internalCopilotAuth(audience: string, options: { organization?: boolean } = {}) { return { async authenticate( @@ -21,22 +26,32 @@ export function internalCopilotAuth(audience: string, options: { organization?: const chatId = request.headers.get('x-mothership-chat-id') if (!userId || Boolean(workspaceId) === Boolean(organizationId)) throw new InternalUnauthenticatedError() - if (organizationId) { - if (!options.organization || !chatId) throw new InternalUnauthenticatedError() - return createTrustedOrganizationCopilotPrincipal( - { userId, organizationId, chatId, delegationId: `worker:${generateId()}` }, - { audience, ttlMs: 60_000 } - ) - } - return createTrustedCopilotPrincipal( - { - userId, - workspaceId: workspaceId!, - ...(chatId ? { chatId } : {}), - delegationId: `worker:${generateId()}`, - }, - { audience, ttlMs: 60_000 } - ) + if (organizationId && (!options.organization || !chatId)) + throw new InternalUnauthenticatedError() + const grant = await checkRunGrant(request.headers.get(RUN_GRANT_HEADER), { + userId, + ...(chatId ? { chatId } : {}), + ...(workspaceId ? { workspaceId } : {}), + ...(organizationId ? { organizationId } : {}), + }) + if (!acceptsRunGrant(grant)) throw new InternalUnauthenticatedError() + const principal = + organizationId && chatId + ? createTrustedOrganizationCopilotPrincipal( + { userId, organizationId, chatId, delegationId: `worker:${generateId()}` }, + { audience, ttlMs: 60_000 } + ) + : createTrustedCopilotPrincipal( + { + userId, + workspaceId: workspaceId!, + ...(chatId ? { chatId } : {}), + delegationId: `worker:${generateId()}`, + }, + { audience, ttlMs: 60_000 } + ) + rememberRunGrant(principal, grant) + return principal }, } } diff --git a/apps/sim/lib/mothership/auth/run-grant.test.ts b/apps/sim/lib/mothership/auth/run-grant.test.ts new file mode 100644 index 00000000000..b90f226addd --- /dev/null +++ b/apps/sim/lib/mothership/auth/run-grant.test.ts @@ -0,0 +1,103 @@ +import { resetEnvMock, setEnv } from '@sim/testing/mocks/env.mock' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { generateInternalToken } from '@/lib/auth/internal' +import { + acceptsRunGrant, + checkRunGrant, + issueRunGrant, + rememberRunGrant, + renewedRunGrant, + runGrantHeaders, +} from '@/lib/mothership/auth/run-grant' + +const HOUR_MS = 60 * 60 * 1000 +const turn = { userId: 'actor', chatId: 'chat-1', workspaceId: 'ws-1', organizationId: null } +const claimed = { userId: 'actor', chatId: 'chat-1', workspaceId: 'ws-1' } + +describe('run grants', () => { + afterEach(() => { + resetEnvMock() + vi.useRealTimers() + }) + + it('admits a call naming exactly the identity its grant names, and no other', async () => { + const grant = issueRunGrant(turn) + expect(await checkRunGrant(grant, claimed)).toMatchObject({ ok: true, identity: turn }) + for (const other of [ + { ...claimed, userId: 'someone-else' }, + { ...claimed, chatId: 'chat-2' }, + { ...claimed, workspaceId: 'ws-2' }, + { userId: 'actor', chatId: 'chat-1', organizationId: 'org-1' }, + ]) { + expect(await checkRunGrant(grant, other)).toEqual({ ok: false, reason: 'mismatch' }) + } + }) + + it('admits a grant for a turn with no chat only on calls that name no chat', async () => { + const chatless = issueRunGrant({ ...turn, chatId: undefined }) + expect(await checkRunGrant(chatless, claimed)).toEqual({ ok: false, reason: 'mismatch' }) + const { chatId: _chatId, ...noChat } = claimed + expect(await checkRunGrant(chatless, noChat)).toMatchObject({ ok: true }) + }) + + it('issues no grants and serves grant-less calls where Sim runs without an auth secret', async () => { + const grant = issueRunGrant(turn) + setEnv({ BETTER_AUTH_SECRET: undefined }) + expect(runGrantHeaders({ userId: 'actor', chatId: 'chat-1', workspaceId: 'ws-1' })).toEqual({}) + const check = await checkRunGrant(grant, claimed) + expect(check).toEqual({ ok: false, reason: 'missing' }) + expect(acceptsRunGrant(check)).toBe(true) + }) + + it('reads the enforcement switch as the string the environment holds', async () => { + const missing = await checkRunGrant(undefined, claimed) + for (const [value, accepted] of [ + ['false', true], + ['0', true], + ['true', false], + ['1', false], + ] as const) { + setEnv({ MOTHERSHIP_REQUIRE_RUN_GRANT: value }) + expect([value, acceptsRunGrant(missing)]).toEqual([value, accepted]) + } + }) + + it('refuses a token the worker could mint itself with the secret it holds', async () => { + const forged = await generateInternalToken('actor') + const check = await checkRunGrant(forged, claimed) + expect(check).toEqual({ ok: false, reason: 'invalid' }) + expect(acceptsRunGrant(check)).toBe(false) + }) + + it('serves a call with a missing or expired grant only until grants are required', async () => { + vi.useFakeTimers({ toFake: ['Date'] }) + vi.setSystemTime(Date.now()) + const grant = issueRunGrant(turn) + vi.setSystemTime(Date.now() + 25 * HOUR_MS) + const expired = await checkRunGrant(grant, claimed) + const missing = await checkRunGrant(undefined, claimed) + expect([expired, missing]).toEqual([ + { ok: false, reason: 'expired' }, + { ok: false, reason: 'missing' }, + ]) + expect([acceptsRunGrant(expired), acceptsRunGrant(missing)]).toEqual([true, true]) + setEnv({ MOTHERSHIP_REQUIRE_RUN_GRANT: true }) + expect([acceptsRunGrant(expired), acceptsRunGrant(missing)]).toEqual([false, false]) + }) + + it('renews a verified grant once past half its life, for the same identity only', async () => { + vi.useFakeTimers({ toFake: ['Date'] }) + const issuedAt = Date.now() + vi.setSystemTime(issuedAt) + const verified = {} + rememberRunGrant(verified, await checkRunGrant(issueRunGrant(turn), claimed)) + vi.setSystemTime(issuedAt + 11 * HOUR_MS) + expect(renewedRunGrant(verified)).toBeUndefined() + vi.setSystemTime(issuedAt + 13 * HOUR_MS) + const renewed = renewedRunGrant(verified) + expect(await checkRunGrant(renewed, claimed)).toMatchObject({ ok: true, identity: turn }) + const unverified = {} + rememberRunGrant(unverified, await checkRunGrant(undefined, claimed)) + expect(renewedRunGrant(unverified)).toBeUndefined() + }) +}) diff --git a/apps/sim/lib/mothership/auth/run-grant.ts b/apps/sim/lib/mothership/auth/run-grant.ts new file mode 100644 index 00000000000..280d6313775 --- /dev/null +++ b/apps/sim/lib/mothership/auth/run-grant.ts @@ -0,0 +1,167 @@ +import { createHmac, hkdfSync } from 'node:crypto' +import { errors, jwtVerify } from 'jose' +import { env, isTruthy } from '@/lib/core/config/env' +import { RUN_GRANT_HEADER } from '@/lib/mothership/generated/sim-transport' + +/** + * Run grants: Sim's signature over the user, chat and owner of a Mothership turn, sent with every + * leg it starts. The worker returns the chat's grant on each call it makes back to Sim, so those + * calls act only for an identity Sim handed it, not for whichever user the worker names. + * + * The key is derived from Sim's own auth secret, which the worker never holds, under a label used + * for nothing else, so Sim alone can issue or check a grant. + */ +const ISSUER = 'sim' +const AUDIENCE = 'sim:mothership-run-grant' +const KEY_LABEL = 'sim/mothership-run-grant/v1' +/** Longer than any gap between a run's calls to Sim; a run-control answer renews it past half. */ +const RUN_GRANT_TTL_SECONDS = 24 * 60 * 60 + +/** Who a grant names: the user, the chat (when the turn had one) and its single owner. */ +export interface RunGrantIdentity { + userId: string + /** Absent when the turn had no chat; such a grant admits only calls that name no chat. */ + chatId?: string + workspaceId: string | null + organizationId: string | null +} + +/** The outcome of checking a call's grant: the verified grant, or why there is none. */ +export type RunGrantCheck = + | { ok: true; issuedAt: number; identity: RunGrantIdentity } + | { ok: false; reason: 'missing' | 'expired' | 'invalid' | 'mismatch' } + +/** + * The grant key, or undefined where Sim runs without an auth secret (`DISABLE_AUTH`): such a + * deployment issues no grants and serves worker calls only while grants are not required. + */ +function grantKey(): Uint8Array | undefined { + if (!env.BETTER_AUTH_SECRET) return undefined + return new Uint8Array( + hkdfSync('sha256', env.BETTER_AUTH_SECRET, new Uint8Array(0), KEY_LABEL, 32) + ) +} + +function signRunGrant(identity: RunGrantIdentity, key: Uint8Array): string { + const issuedAt = Math.floor(Date.now() / 1000) + const encode = (value: object) => Buffer.from(JSON.stringify(value)).toString('base64url') + const unsigned = `${encode({ alg: 'HS256', typ: 'JWT' })}.${encode({ + iss: ISSUER, + aud: AUDIENCE, + sub: identity.userId, + iat: issuedAt, + exp: issuedAt + RUN_GRANT_TTL_SECONDS, + ...(identity.chatId ? { chat: identity.chatId } : {}), + ws: identity.workspaceId, + org: identity.organizationId, + })}` + return `${unsigned}.${createHmac('sha256', key).update(unsigned).digest('base64url')}` +} + +/** + * A grant for `identity`: a compact HS256 JWT, signed synchronously, so issuing one on every leg + * Sim sends adds no await to the dispatch path; `checkRunGrant` verifies it as any JWT. Throws + * where Sim has no auth secret, where `runGrantHeaders` issues none. + */ +export function issueRunGrant(identity: RunGrantIdentity): string { + const key = grantKey() + if (!key) throw new Error('Run grants need BETTER_AUTH_SECRET') + return signRunGrant(identity, key) +} + +/** + * The header carrying a turn's grant, for a leg Sim sends: none for a turn with no single owner, + * or where Sim has no auth secret to sign with. + */ +export function runGrantHeaders(context: { + userId: string + chatId?: string + workspaceId?: string + organizationId?: string +}): Record { + const key = grantKey() + if (!key || Boolean(context.workspaceId) === Boolean(context.organizationId)) return {} + return { + [RUN_GRANT_HEADER]: signRunGrant( + { + userId: context.userId, + ...(context.chatId ? { chatId: context.chatId } : {}), + workspaceId: context.workspaceId ?? null, + organizationId: context.organizationId ?? null, + }, + key + ), + } +} + +/** + * Whether `token` is a grant Sim issued for exactly the identity a worker call names: the same + * user, owner and chat, where a grant with no chat matches only a call that names none. + */ +export async function checkRunGrant( + token: string | null | undefined, + claimed: { userId: string; chatId?: string; workspaceId?: string; organizationId?: string } +): Promise { + const key = grantKey() + if (!token || !key) return { ok: false, reason: 'missing' } + let payload: Awaited>['payload'] + try { + ;({ payload } = await jwtVerify(token, key, { + issuer: ISSUER, + audience: AUDIENCE, + algorithms: ['HS256'], + requiredClaims: ['sub', 'iat', 'exp'], + })) + } catch (error) { + return { ok: false, reason: error instanceof errors.JWTExpired ? 'expired' : 'invalid' } + } + const identity: RunGrantIdentity = { + userId: String(payload.sub), + ...(typeof payload.chat === 'string' ? { chatId: payload.chat } : {}), + workspaceId: typeof payload.ws === 'string' ? payload.ws : null, + organizationId: typeof payload.org === 'string' ? payload.org : null, + } + if ( + identity.userId !== claimed.userId || + identity.workspaceId !== (claimed.workspaceId ?? null) || + identity.organizationId !== (claimed.organizationId ?? null) || + identity.chatId !== claimed.chatId + ) + return { ok: false, reason: 'mismatch' } + return { ok: true, issuedAt: Number(payload.iat), identity } +} + +/** + * Whether a worker call may go on. A forged or mismatched grant never; a valid one always; a + * missing or expired one only until `MOTHERSHIP_REQUIRE_RUN_GRANT` is true, so workers that + * predate grants, and runs that outlived theirs, keep working while grants roll out. + */ +export function acceptsRunGrant(check: RunGrantCheck): boolean { + if (check.ok) return true + if (check.reason === 'invalid' || check.reason === 'mismatch') return false + return !isTruthy(env.MOTHERSHIP_REQUIRE_RUN_GRANT) +} + +/** The grant each worker principal was admitted on, keyed by the principal object itself. */ +const admittedGrants = new WeakMap>() + +/** + * Records the grant a worker principal was admitted on, so the same principal object can later + * be renewed (`renewedRunGrant`). Only a verified grant is kept; any other check records nothing. + */ +export function rememberRunGrant(principal: object, check: RunGrantCheck): void { + if (check.ok) admittedGrants.set(principal, check) +} + +/** + * A fresh grant for the same identity, once the one this call carried is past half its life. + * Only a grant Sim verified is renewed, so renewal never grants an identity Sim had not. The + * caller sends it only after rechecking the user's access. + */ +export function renewedRunGrant(principal: object): string | undefined { + const admitted = admittedGrants.get(principal) + const key = grantKey() + if (!admitted || !key || Date.now() / 1000 - admitted.issuedAt < RUN_GRANT_TTL_SECONDS / 2) + return undefined + return signRunGrant(admitted.identity, key) +} diff --git a/apps/sim/lib/mothership/generated/run-control.ts b/apps/sim/lib/mothership/generated/run-control.ts index db2b9565882..44914a06b1d 100644 --- a/apps/sim/lib/mothership/generated/run-control.ts +++ b/apps/sim/lib/mothership/generated/run-control.ts @@ -3,11 +3,22 @@ import { z } from "zod"; +/** Sim's run grant (`RUN_GRANT_HEADER` in sim-transport): a token the worker returns, never reads. */ +export const RunGrant = z.string().min(1).max(4096); + export const RunControlRequest = z.object({ chatId: z.string().uuid(), streamId: z.string().min(1).max(200), }); export type RunControlRequest = z.infer; -export const RunControlState = z.object({ stopped: z.boolean() }); +export const RunControlState = z.object({ + stopped: z.boolean(), + /** + * A fresh run grant for the same user and chat, sent when the grant the call carried is past + * half its life: a run that keeps checking its control keeps a valid grant however long it + * runs. The worker uses it from its next call on. + */ + grant: RunGrant.optional(), +}); export type RunControlState = z.infer; diff --git a/apps/sim/lib/mothership/generated/sim-transport.ts b/apps/sim/lib/mothership/generated/sim-transport.ts index c869ff1ebb8..ffce7241db0 100644 --- a/apps/sim/lib/mothership/generated/sim-transport.ts +++ b/apps/sim/lib/mothership/generated/sim-transport.ts @@ -6,7 +6,7 @@ import { RecallChatsRequest } from "./chat-recall"; import { IntegrationCatalogRequest } from "./integration-catalog"; import { MemoryNotesAccessInput } from "./memory-notes-access"; import { MemoryScopeRequest } from "./memory-scope"; -import { RunControlRequest } from "./run-control"; +import { RunControlRequest, RunGrant } from "./run-control"; import { TaskWakeRequest, WorkflowWatchRequest } from "./tasks"; const SimChannelId = z.string().regex(/^[a-f0-9]{64}$/); @@ -17,6 +17,14 @@ export const SimConnection = z.discriminatedUnion("mode", [ ]); export type SimConnection = z.infer; +/** + * Sim's run grant: its signature over the user, chat and owner of a turn, sent with every turn + * it starts. The worker cannot mint one; it returns the latest grant for the chat on each call + * it makes to Sim (this header over HTTP, `grant` on a control request), and Sim serves the call + * only for the identity the grant names. + */ +export const RUN_GRANT_HEADER = "x-sim-run-grant"; + export const SimScope = z .object({ userId: z.string().min(1), @@ -47,6 +55,8 @@ export const SimControlRequest = z.object({ scope: SimScope, operation: SimControlOperation, expiresAt: z.number().int().positive(), + /** The chat's run grant (`RUN_GRANT_HEADER`); absent from a worker that predates grants. */ + grant: RunGrant.optional(), }); export type SimControlRequest = z.infer; diff --git a/apps/sim/lib/mothership/request/application/read-control.test.ts b/apps/sim/lib/mothership/request/application/read-control.test.ts index 1cf0ed037d1..2f5a12da04c 100644 --- a/apps/sim/lib/mothership/request/application/read-control.test.ts +++ b/apps/sim/lib/mothership/request/application/read-control.test.ts @@ -11,6 +11,7 @@ import { } from '@sim/testing/mocks/permission-groups-resolve.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' import { createTrustedOrganizationCopilotPrincipal } from '@/lib/mothership/auth/application-delegation' +import { checkRunGrant, issueRunGrant, rememberRunGrant } from '@/lib/mothership/auth/run-grant' import { RUN_CONTROL_AUDIENCE, readRunControl, @@ -151,4 +152,41 @@ describe('organization run-control authorization', () => { stopped: false, }) }) + it('renews a half-spent grant for a run still going, and none for a finished one', async () => { + vi.useFakeTimers({ toFake: ['Date'] }) + const issuedAt = Date.now() + try { + mocks.stopped.mockResolvedValue(false) + for (const [status, renewed] of [ + ['running', true], + ['complete', false], + ] as const) { + vi.setSystemTime(issuedAt) + const grant = issueRunGrant({ + userId: 'actor', + chatId: 'chat-1', + workspaceId: null, + organizationId: 'org-1', + }) + vi.setSystemTime(issuedAt + 13 * 60 * 60 * 1000) + const caller = principal() + rememberRunGrant( + caller, + await checkRunGrant(grant, { userId: 'actor', chatId: 'chat-1', organizationId: 'org-1' }) + ) + mocks.run.mockResolvedValueOnce({ + chatId: 'chat-1', + userId: 'actor', + organizationId: 'org-1', + workspaceId: null, + status, + }) + queueChat() + const state = await readRunControl.execute({ principal: caller, input }) + expect([status, 'grant' in state]).toEqual([status, renewed]) + } + } finally { + vi.useRealTimers() + } + }) }) diff --git a/apps/sim/lib/mothership/request/application/read-control.ts b/apps/sim/lib/mothership/request/application/read-control.ts index 44b909bd767..2b22cf8d04e 100644 --- a/apps/sim/lib/mothership/request/application/read-control.ts +++ b/apps/sim/lib/mothership/request/application/read-control.ts @@ -13,7 +13,12 @@ import { import { OrganizationDelegationAuthorizationError } from '@/lib/core/application/organization-authorization' import { defineOrganizationOperation } from '@/lib/core/application/organization-operation' import { asOrchestrationError, OrchestrationError } from '@/lib/core/orchestration/types' -import { getLatestRunForStream, isRunStopRequested } from '@/lib/mothership/async-runs/repository' +import { + getLatestRunForStream, + isRunStopRequested, + TERMINAL_RUN_STATUSES, +} from '@/lib/mothership/async-runs/repository' +import { renewedRunGrant } from '@/lib/mothership/auth/run-grant' import { defineAuthorizedChatUseCase } from '@/lib/mothership/chat/application/authorized-chat-use-case' import { resolveOwnedChatContext } from '@/lib/mothership/chat/application/context' import type { RunControlRequest, RunControlState } from '@/lib/mothership/generated/run-control' @@ -69,7 +74,11 @@ const readLiveRunControl = defineAuthorizedChatUseCase({ ) { throw new OrchestrationError('not_found', 'Stream not found') } - return { stopped: await isRunStopRequested({ ...context, streamId: input.streamId }) } + return { + stopped: await isRunStopRequested({ ...context, streamId: input.streamId }), + // A finished run keeps no grant renewed. + finished: TERMINAL_RUN_STATUSES.includes(run.status), + } }, }) @@ -125,7 +134,12 @@ export const readRunControl: AuthorizingUseCase< async execute(args) { requireRunControlCaller(args.principal, args.input) try { - return await readLiveRunControl.execute(args) + const { stopped, finished } = await readLiveRunControl.execute(args) + if (stopped || finished) return { stopped } + // The chat and the user's access were just rechecked, so a run that keeps checking its + // control keeps a valid grant however long it runs, and only while it runs. + const grant = renewedRunGrant(args.principal) + return grant ? { stopped, grant } : { stopped } } catch (error) { const refusal = asOrchestrationError(error) if ( diff --git a/apps/sim/lib/mothership/request/lifecycle/run.test.ts b/apps/sim/lib/mothership/request/lifecycle/run.test.ts index 62d2ad4fe1c..bab96c502c8 100644 --- a/apps/sim/lib/mothership/request/lifecycle/run.test.ts +++ b/apps/sim/lib/mothership/request/lifecycle/run.test.ts @@ -73,6 +73,7 @@ const { mockCheckAttributedUsageLimits: vi.fn(), mockEnv: { INTERNAL_API_SECRET: 'transport-test-secret-000000000000000000', + BETTER_AUTH_SECRET: 'run-grant-test-auth-secret-0000000000000', COPILOT_API_KEY: undefined as string | undefined, MSHIP_SYSPROMPT_OVERRIDE: undefined as string | undefined, MOTHERSHIP_BENCHMARK_URL: 'https://benchmark.test', @@ -231,12 +232,14 @@ vi.mock('@/lib/mothership/request/enterprise-byok', () => ({ })) import { resetUsageGateCache } from '@/lib/billing/core/usage-gate-cache' +import { checkRunGrant } from '@/lib/mothership/auth/run-grant' import { buildPersistedAssistantMessage } from '@/lib/mothership/chat/persisted-message' import { CLIENT_TOOL_RESULT_TIMEOUT_MS } from '@/lib/mothership/constants' import { MothershipStreamV1CompletionStatus, MothershipStreamV1ToolOutcome, } from '@/lib/mothership/generated/mothership-stream-v1' +import { RUN_GRANT_HEADER } from '@/lib/mothership/generated/sim-transport' import { BillingLimitError, CopilotBackendError, @@ -2323,92 +2326,103 @@ describe('runCopilotLifecycle', () => { it.each([ { workspaceId: 'ws-1', organizationId: undefined }, { workspaceId: undefined, organizationId: 'org-1' }, - ])('uses immutable attribution on initial and resume legs for %j', async (owner) => { - const billingAttribution = { - actorUserId: 'user-1', - workspaceId: owner.workspaceId ?? null, - billedAccountUserId: 'owner-1', - organizationId: 'org-1', - billingEntity: { type: 'organization' as const, id: 'org-1' }, - billingPeriod: { - start: '2026-07-01T00:00:00.000Z', - end: '2026-08-01T00:00:00.000Z', - }, - payerSubscription: null, - } - setEnvFlags({ isHosted: true, isBillingEnabled: true }) - mockEnv.COPILOT_API_KEY = 'sim-agent-key' - mockRunStreamLoop.mockImplementationOnce( - async ( - _fetchUrl: string, - _fetchOptions: RequestInit, - context: StreamingContext - ): Promise => { - context.toolCalls.set('tool-1', { - id: 'tool-1', - name: 'read', - status: MothershipStreamV1ToolOutcome.success, - result: { success: true, output: { content: 'file contents' } }, - }) - context.awaitingAsyncContinuation = { - checkpointId: 'ckpt-1', - pendingToolCallIds: ['tool-1'], - } + ])( + 'sends immutable attribution and a run grant for the turn on initial and resume legs for %j', + async (owner) => { + const billingAttribution = { + actorUserId: 'user-1', + workspaceId: owner.workspaceId ?? null, + billedAccountUserId: 'owner-1', + organizationId: 'org-1', + billingEntity: { type: 'organization' as const, id: 'org-1' }, + billingPeriod: { + start: '2026-07-01T00:00:00.000Z', + end: '2026-08-01T00:00:00.000Z', + }, + payerSubscription: null, } - ) - mockRunStreamLoop.mockResolvedValueOnce(undefined) + setEnvFlags({ isHosted: true, isBillingEnabled: true }) + mockEnv.COPILOT_API_KEY = 'sim-agent-key' + mockRunStreamLoop.mockImplementationOnce( + async ( + _fetchUrl: string, + _fetchOptions: RequestInit, + context: StreamingContext + ): Promise => { + context.toolCalls.set('tool-1', { + id: 'tool-1', + name: 'read', + status: MothershipStreamV1ToolOutcome.success, + result: { success: true, output: { content: 'file contents' } }, + }) + context.awaitingAsyncContinuation = { + checkpointId: 'ckpt-1', + pendingToolCallIds: ['tool-1'], + } + } + ) + mockRunStreamLoop.mockResolvedValueOnce(undefined) - await runCopilotLifecycle( - { - message: 'hello', - ...(owner.organizationId ? { mode: 'assistant' } : {}), - messageId: 'message-1', - billingRequestId: 'caller-controlled', - }, - { - userId: 'user-1', - ...owner, - chatId: 'chat-1', - executionId: 'execution-1', - runId: 'run-1', - simRequestId: 'request-1', - billingAttribution, - } - ) + await runCopilotLifecycle( + { + message: 'hello', + ...(owner.organizationId ? { mode: 'assistant' } : {}), + messageId: 'message-1', + billingRequestId: 'caller-controlled', + }, + { + userId: 'user-1', + ...owner, + chatId: 'chat-1', + executionId: 'execution-1', + runId: 'run-1', + simRequestId: 'request-1', + billingAttribution, + } + ) - const firstHeaders = mockRunStreamLoop.mock.calls[0]?.[1].headers as Record - const billingRequestId = firstHeaders['x-sim-billing-request-id'] - expect(billingRequestId).toMatch( - /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i - ) - expect(billingRequestId).not.toBe('caller-controlled') + const firstHeaders = mockRunStreamLoop.mock.calls[0]?.[1].headers as Record + const billingRequestId = firstHeaders['x-sim-billing-request-id'] + expect(billingRequestId).toMatch( + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i + ) + expect(billingRequestId).not.toBe('caller-controlled') - expect(mockRunStreamLoop).toHaveBeenCalledTimes(2) - expect(mockLoadCopilotSearchIntegrations).not.toHaveBeenCalled() - for (const call of mockRunStreamLoop.mock.calls) { - const body = JSON.parse(String(call[1].body)) - if (call === mockRunStreamLoop.mock.calls[0]) { - expect(body).toMatchObject( - owner.organizationId - ? { organizationId: owner.organizationId } - : { workspaceId: owner.workspaceId } + expect(mockRunStreamLoop).toHaveBeenCalledTimes(2) + expect(mockLoadCopilotSearchIntegrations).not.toHaveBeenCalled() + for (const call of mockRunStreamLoop.mock.calls) { + const body = JSON.parse(String(call[1].body)) + if (call === mockRunStreamLoop.mock.calls[0]) { + expect(body).toMatchObject( + owner.organizationId + ? { organizationId: owner.organizationId } + : { workspaceId: owner.workspaceId } + ) + } else { + expect(body).not.toHaveProperty('workspaceId') + expect(body).not.toHaveProperty('organizationId') + expect(body).not.toHaveProperty('checkpointId') + } + const headers = call[1].headers as Record + expect(headers).toMatchObject({ + 'x-api-key': 'sim-agent-key', + 'x-sim-billing-protocol': 'attribution-v1', + 'x-sim-billing-request-id': billingRequestId, + }) + expect(JSON.parse(decodeURIComponent(headers['x-sim-billing-attribution']))).toEqual( + billingAttribution ) - } else { - expect(body).not.toHaveProperty('workspaceId') - expect(body).not.toHaveProperty('organizationId') - expect(body).not.toHaveProperty('checkpointId') + expect( + await checkRunGrant(headers[RUN_GRANT_HEADER], { + userId: 'user-1', + chatId: 'chat-1', + ...(owner.workspaceId ? { workspaceId: owner.workspaceId } : {}), + ...(owner.organizationId ? { organizationId: owner.organizationId } : {}), + }) + ).toMatchObject({ ok: true }) } - const headers = call[1].headers as Record - expect(headers).toMatchObject({ - 'x-api-key': 'sim-agent-key', - 'x-sim-billing-protocol': 'attribution-v1', - 'x-sim-billing-request-id': billingRequestId, - }) - expect(JSON.parse(decodeURIComponent(headers['x-sim-billing-attribution']))).toEqual( - billingAttribution - ) } - }) + ) it('cold recovery reads spend only against the original billing identity', async () => { const attribution = { diff --git a/apps/sim/lib/mothership/request/lifecycle/run.ts b/apps/sim/lib/mothership/request/lifecycle/run.ts index 135b1bc3f0c..3b805a039f5 100644 --- a/apps/sim/lib/mothership/request/lifecycle/run.ts +++ b/apps/sim/lib/mothership/request/lifecycle/run.ts @@ -26,6 +26,7 @@ import { updateRunStatus, } from '@/lib/mothership/async-runs/repository' import type { CopilotResourceAdmission } from '@/lib/mothership/auth/application-delegation' +import { runGrantHeaders } from '@/lib/mothership/auth/run-grant' import { CLIENT_TOOL_RESULT_TIMEOUT_MS, TOOL_WATCHDOG_RESUME_GRACE_MS, @@ -985,7 +986,10 @@ async function runResumeLegWithRetry( url, { method: 'POST', - headers: mothershipRequestHeaders(hostedBillingRequest, options.simRequestId), + headers: { + ...mothershipRequestHeaders(hostedBillingRequest, options.simRequestId), + ...runGrantHeaders(execContext), + }, body: JSON.stringify(body), }, leg, @@ -1330,7 +1334,10 @@ async function runCheckpointLoop( `${mothershipBaseURL}${route}`, { method: 'POST', - headers: mothershipRequestHeaders(hostedBillingRequest, mothershipRequestId), + headers: { + ...mothershipRequestHeaders(hostedBillingRequest, mothershipRequestId), + ...runGrantHeaders(execContext), + }, body: JSON.stringify({ ...payload, ...(route === '/api/mothership' || diff --git a/apps/sim/lib/mothership/transport/control.test.ts b/apps/sim/lib/mothership/transport/control.test.ts index 280c6e2fac0..a248bc4eff4 100644 --- a/apps/sim/lib/mothership/transport/control.test.ts +++ b/apps/sim/lib/mothership/transport/control.test.ts @@ -43,6 +43,7 @@ vi.mock('@/lib/mothership/tasks/application/context', () => ({ TASK_DELEGATION_A vi.mock('@/lib/mothership/tasks/wake', () => ({ runWakeTurn: handlers.wake })) import { OrchestrationError } from '@/lib/core/orchestration/types' +import { issueRunGrant } from '@/lib/mothership/auth/run-grant' import type { SimControlOperation, SimControlRequest, @@ -282,3 +283,20 @@ it("routes a chat's other chats through the same use case for both delivery scop ).status ).toBe(403) }) + +it("serves a control request whose run grant names its scope, and refuses another chat's grant", async () => { + handlers.memory.mockResolvedValue({ + userId: 'user', + organizationId: null, + workspaceId: 'workspace', + }) + const grant = (chatId: string) => + issueRunGrant({ userId: 'user', chatId, workspaceId: 'workspace', organizationId: null }) + const operation = { kind: 'memory_scope', input: { chatId: 'chat' } } as const + expect( + (await executeSimControl({ ...request(operation), grant: await grant('chat') })).status + ).toBe(200) + expect( + await executeSimControl({ ...request(operation), grant: await grant('another-chat') }) + ).toEqual({ status: 401, body: '{"error":"Run grant refused"}' }) +}) diff --git a/apps/sim/lib/mothership/transport/control.ts b/apps/sim/lib/mothership/transport/control.ts index 7c8ced95702..33cae8b2fdb 100644 --- a/apps/sim/lib/mothership/transport/control.ts +++ b/apps/sim/lib/mothership/transport/control.ts @@ -6,6 +6,7 @@ import { createTrustedCopilotPrincipal, createTrustedOrganizationCopilotPrincipal, } from '@/lib/mothership/auth/application-delegation' +import { acceptsRunGrant, checkRunGrant, rememberRunGrant } from '@/lib/mothership/auth/run-grant' import { RECALL_CHATS_AUDIENCE, readRecallChats, @@ -44,6 +45,8 @@ export async function executeSimControl(request: SimControlRequest): Promise