From 3f53b266ca332d028696347e70c0e49e2b5f0c2d Mon Sep 17 00:00:00 2001 From: Mukhamediyar <86684638+myxamediyar@users.noreply.github.com> Date: Wed, 18 Nov 2026 16:00:03 -0800 Subject: [PATCH 1/2] feat(mothership): serve worker calls only for a user and chat sim granted Sim's worker-facing routes and its checkpoint control requests took the user, chat and owner the worker named, behind a secret the worker holds. Sim now issues a run grant with every leg it sends: an HS256 JWT over the turn's user, chat and workspace or organization, keyed by HKDF from Sim's auth secret under its own label, lasting a day. The worker returns it on each call. - internalCopilotAuth and the checkpoint executor check the grant against the identity the call names: a forged or mismatched grant is refused; a missing or expired one is served until MOTHERSHIP_REQUIRE_RUN_GRANT, so workers that predate grants keep working during rollout. - Run control renews a verified grant past half its life after rechecking the chat and the user's access, so a run that keeps checking its control keeps a valid grant however long it runs. Only a grant Sim verified is renewed. - Grants are signed synchronously, so issuing one adds no await to the dispatch path. --- apps/sim/.env.example | 1 + apps/sim/lib/core/config/env.ts | 1 + apps/sim/lib/mothership/auth/internal.test.ts | 38 +++- apps/sim/lib/mothership/auth/internal.ts | 49 +++-- .../sim/lib/mothership/auth/run-grant.test.ts | 73 ++++++++ apps/sim/lib/mothership/auth/run-grant.ts | 137 ++++++++++++++ .../lib/mothership/generated/run-control.ts | 13 +- .../lib/mothership/generated/sim-transport.ts | 12 +- .../request/application/read-control.ts | 8 +- .../mothership/request/lifecycle/run.test.ts | 172 ++++++++++-------- .../lib/mothership/request/lifecycle/run.ts | 11 +- .../lib/mothership/transport/control.test.ts | 18 ++ apps/sim/lib/mothership/transport/control.ts | 4 + 13 files changed, 435 insertions(+), 102 deletions(-) create mode 100644 apps/sim/lib/mothership/auth/run-grant.test.ts create mode 100644 apps/sim/lib/mothership/auth/run-grant.ts diff --git a/apps/sim/.env.example b/apps/sim/.env.example index 1bacb5f98a9..ec281638828 100644 --- a/apps/sim/.env.example +++ b/apps/sim/.env.example @@ -31,6 +31,7 @@ NEXT_PUBLIC_APP_URL=http://localhost:3000 # NEXT_PUBLIC_CHAT_DISABLED=true # Hides the Chat module: the workspace lands on your first workflow, and the chats list, scheduled tasks, and editor Chat panel are absent. Chat is shown when unset; `bun run setup` sets this for you if you skip the chat key # MOTHERSHIP_BENCHMARK_ENABLED=true # Shows the organization Benchmark tab and enables its authenticated APIs # COPILOT_DEV_URL=http://localhost:8080 # Benchmark worker endpoint; requires the worker benchmark profile and the normal COPILOT_API_KEY +# MOTHERSHIP_REQUIRE_RUN_GRANT=true # Refuse Mothership worker calls that carry no run grant for their user and chat; set once your Mothership forwards run grants # Remote Function sandboxes (Optional) # Build a dedicated Function base before enabling JavaScript imports, Python, or Shell. diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index 5a757490398..2654d8820c7 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -162,6 +162,7 @@ export const env = createEnv({ /** Enables the organization benchmark UI and its authenticated endpoints. */ MOTHERSHIP_BENCHMARK_ENABLED: z.boolean().optional(), MOTHERSHIP_BENCHMARK_URL: z.string().url().optional(), + MOTHERSHIP_REQUIRE_RUN_GRANT: z.boolean().optional(), // Worker calls must carry a run grant naming their user and chat; unset, a call without one is still served SIM_AGENT_API_URL: z.string().url().optional(), // URL for internal sim agent API MSHIP_SYSPROMPT_OVERRIDE: z.string().min(1).optional(), // Enterprise-only highest-priority Mothership system prompt override forwarded by Sim COPILOT_SOURCE_ENV: z.enum(['dev', 'staging', 'prod']).optional(), // Source Sim environment sent to mothership for callbacks diff --git a/apps/sim/lib/mothership/auth/internal.test.ts b/apps/sim/lib/mothership/auth/internal.test.ts index a740bb9753f..ee6b8fa5ba0 100644 --- a/apps/sim/lib/mothership/auth/internal.test.ts +++ b/apps/sim/lib/mothership/auth/internal.test.ts @@ -1,7 +1,10 @@ import { createMockRequest } from '@sim/testing' import { copilotHttpMock, copilotHttpMockFns } from '@sim/testing/mocks/copilot-http.mock' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { resetEnvMock, setEnv } from '@sim/testing/mocks/env.mock' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { internalCopilotAuth } from '@/lib/mothership/auth/internal' +import { issueRunGrant } from '@/lib/mothership/auth/run-grant' +import { RUN_GRANT_HEADER } from '@/lib/mothership/generated/sim-transport' vi.mock('@/lib/mothership/request/http', () => copilotHttpMock) @@ -44,4 +47,37 @@ describe('worker control identity', () => { apiKey.mockReturnValue({ success: false }) await expect(auth.authenticate(request(headers))).rejects.toThrow('Unauthorized') }) + describe('run grants', () => { + afterEach(resetEnvMock) + const auth = internalCopilotAuth('control', { organization: true }) + const grant = () => + issueRunGrant({ + userId: 'actor', + chatId: 'chat-1', + workspaceId: null, + organizationId: 'org-1', + }) + it('admits identity headers the run grant names, and refuses forged or mismatched grants', async () => { + await expect( + auth.authenticate(request({ ...headers, [RUN_GRANT_HEADER]: await grant() })) + ).resolves.toMatchObject({ subjectUserId: 'actor', resourceScope: { chatId: 'chat-1' } }) + await expect( + auth.authenticate( + request({ + ...headers, + 'x-mothership-chat-id': 'chat-2', + [RUN_GRANT_HEADER]: await grant(), + }) + ) + ).rejects.toThrow('Unauthorized') + await expect( + auth.authenticate(request({ ...headers, [RUN_GRANT_HEADER]: 'not-a-grant' })) + ).rejects.toThrow('Unauthorized') + }) + it('refuses a call without a grant once grants are required', async () => { + await expect(auth.authenticate(request(headers))).resolves.toBeDefined() + setEnv({ MOTHERSHIP_REQUIRE_RUN_GRANT: true }) + await expect(auth.authenticate(request(headers))).rejects.toThrow('Unauthorized') + }) + }) }) diff --git a/apps/sim/lib/mothership/auth/internal.ts b/apps/sim/lib/mothership/auth/internal.ts index f6e8a49e97d..4389a9a5158 100644 --- a/apps/sim/lib/mothership/auth/internal.ts +++ b/apps/sim/lib/mothership/auth/internal.ts @@ -6,9 +6,14 @@ import { createTrustedCopilotPrincipal, createTrustedOrganizationCopilotPrincipal, } from '@/lib/mothership/auth/application-delegation' +import { acceptsRunGrant, checkRunGrant, rememberRunGrant } from '@/lib/mothership/auth/run-grant' +import { RUN_GRANT_HEADER } from '@/lib/mothership/generated/sim-transport' import { checkInternalApiKey } from '@/lib/mothership/request/http' -/** Identity headers are assertions from the authenticated worker, never browser credentials. */ +/** + * Identity headers are assertions from the authenticated worker, never browser credentials; the + * run grant Sim issued with the turn proves them (`run-grant.ts`). + */ export function internalCopilotAuth(audience: string, options: { organization?: boolean } = {}) { return { async authenticate( @@ -21,22 +26,32 @@ export function internalCopilotAuth(audience: string, options: { organization?: const chatId = request.headers.get('x-mothership-chat-id') if (!userId || Boolean(workspaceId) === Boolean(organizationId)) throw new InternalUnauthenticatedError() - if (organizationId) { - if (!options.organization || !chatId) throw new InternalUnauthenticatedError() - return createTrustedOrganizationCopilotPrincipal( - { userId, organizationId, chatId, delegationId: `worker:${generateId()}` }, - { audience, ttlMs: 60_000 } - ) - } - return createTrustedCopilotPrincipal( - { - userId, - workspaceId: workspaceId!, - ...(chatId ? { chatId } : {}), - delegationId: `worker:${generateId()}`, - }, - { audience, ttlMs: 60_000 } - ) + if (organizationId && (!options.organization || !chatId)) + throw new InternalUnauthenticatedError() + const grant = await checkRunGrant(request.headers.get(RUN_GRANT_HEADER), { + userId, + ...(chatId ? { chatId } : {}), + ...(workspaceId ? { workspaceId } : {}), + ...(organizationId ? { organizationId } : {}), + }) + if (!acceptsRunGrant(grant)) throw new InternalUnauthenticatedError() + const principal = + organizationId && chatId + ? createTrustedOrganizationCopilotPrincipal( + { userId, organizationId, chatId, delegationId: `worker:${generateId()}` }, + { audience, ttlMs: 60_000 } + ) + : createTrustedCopilotPrincipal( + { + userId, + workspaceId: workspaceId!, + ...(chatId ? { chatId } : {}), + delegationId: `worker:${generateId()}`, + }, + { audience, ttlMs: 60_000 } + ) + rememberRunGrant(principal, grant) + return principal }, } } diff --git a/apps/sim/lib/mothership/auth/run-grant.test.ts b/apps/sim/lib/mothership/auth/run-grant.test.ts new file mode 100644 index 00000000000..2fc1e6d086b --- /dev/null +++ b/apps/sim/lib/mothership/auth/run-grant.test.ts @@ -0,0 +1,73 @@ +import { resetEnvMock, setEnv } from '@sim/testing/mocks/env.mock' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { generateInternalToken } from '@/lib/auth/internal' +import { + acceptsRunGrant, + checkRunGrant, + issueRunGrant, + rememberRunGrant, + renewedRunGrant, +} from '@/lib/mothership/auth/run-grant' + +const HOUR_MS = 60 * 60 * 1000 +const turn = { userId: 'actor', chatId: 'chat-1', workspaceId: 'ws-1', organizationId: null } +const claimed = { userId: 'actor', chatId: 'chat-1', workspaceId: 'ws-1' } + +describe('run grants', () => { + afterEach(() => { + resetEnvMock() + vi.useRealTimers() + }) + + it('admits a call naming exactly the identity its grant names, and no other', async () => { + const grant = issueRunGrant(turn) + expect(await checkRunGrant(grant, claimed)).toMatchObject({ ok: true, identity: turn }) + for (const other of [ + { ...claimed, userId: 'someone-else' }, + { ...claimed, chatId: 'chat-2' }, + { ...claimed, workspaceId: 'ws-2' }, + { userId: 'actor', chatId: 'chat-1', organizationId: 'org-1' }, + ]) { + expect(await checkRunGrant(grant, other)).toEqual({ ok: false, reason: 'mismatch' }) + } + }) + + it('refuses a token the worker could mint itself with the secret it holds', async () => { + const forged = await generateInternalToken('actor') + const check = await checkRunGrant(forged, claimed) + expect(check).toEqual({ ok: false, reason: 'invalid' }) + expect(acceptsRunGrant(check)).toBe(false) + }) + + it('serves a call with a missing or expired grant only until grants are required', async () => { + vi.useFakeTimers({ toFake: ['Date'] }) + vi.setSystemTime(Date.now()) + const grant = issueRunGrant(turn) + vi.setSystemTime(Date.now() + 25 * HOUR_MS) + const expired = await checkRunGrant(grant, claimed) + const missing = await checkRunGrant(undefined, claimed) + expect([expired, missing]).toEqual([ + { ok: false, reason: 'expired' }, + { ok: false, reason: 'missing' }, + ]) + expect([acceptsRunGrant(expired), acceptsRunGrant(missing)]).toEqual([true, true]) + setEnv({ MOTHERSHIP_REQUIRE_RUN_GRANT: true }) + expect([acceptsRunGrant(expired), acceptsRunGrant(missing)]).toEqual([false, false]) + }) + + it('renews a verified grant once past half its life, for the same identity only', async () => { + vi.useFakeTimers({ toFake: ['Date'] }) + const issuedAt = Date.now() + vi.setSystemTime(issuedAt) + const verified = {} + rememberRunGrant(verified, await checkRunGrant(issueRunGrant(turn), claimed)) + vi.setSystemTime(issuedAt + 11 * HOUR_MS) + expect(renewedRunGrant(verified)).toBeUndefined() + vi.setSystemTime(issuedAt + 13 * HOUR_MS) + const renewed = renewedRunGrant(verified) + expect(await checkRunGrant(renewed, claimed)).toMatchObject({ ok: true, identity: turn }) + const unverified = {} + rememberRunGrant(unverified, await checkRunGrant(undefined, claimed)) + expect(renewedRunGrant(unverified)).toBeUndefined() + }) +}) diff --git a/apps/sim/lib/mothership/auth/run-grant.ts b/apps/sim/lib/mothership/auth/run-grant.ts new file mode 100644 index 00000000000..b488174b199 --- /dev/null +++ b/apps/sim/lib/mothership/auth/run-grant.ts @@ -0,0 +1,137 @@ +import { createHmac, hkdfSync } from 'node:crypto' +import { errors, jwtVerify } from 'jose' +import { env } from '@/lib/core/config/env' +import { RUN_GRANT_HEADER } from '@/lib/mothership/generated/sim-transport' + +/** + * Run grants: Sim's signature over the user, chat and owner of a Mothership turn, sent with every + * leg it starts. The worker returns the chat's grant on each call it makes back to Sim, so those + * calls act only for an identity Sim handed it, not for whichever user the worker names. + * + * The key is derived from Sim's own auth secret, which the worker never holds, under a label used + * for nothing else, so Sim alone can issue or check a grant. + */ +const ISSUER = 'sim' +const AUDIENCE = 'sim:mothership-run-grant' +const KEY_LABEL = 'sim/mothership-run-grant/v1' +/** Longer than any gap between a run's calls to Sim; a run-control answer renews it past half. */ +const RUN_GRANT_TTL_SECONDS = 24 * 60 * 60 + +export interface RunGrantIdentity { + userId: string + /** Absent when the turn had no chat; the grant then binds the user and owner only. */ + chatId?: string + workspaceId: string | null + organizationId: string | null +} + +export type RunGrantCheck = + | { ok: true; issuedAt: number; identity: RunGrantIdentity } + | { ok: false; reason: 'missing' | 'expired' | 'invalid' | 'mismatch' } + +function grantKey(): Uint8Array { + return new Uint8Array( + hkdfSync('sha256', env.BETTER_AUTH_SECRET, new Uint8Array(0), KEY_LABEL, 32) + ) +} + +/** + * A compact HS256 JWT, signed synchronously: issuing runs on every leg Sim sends, so it adds no + * await to the dispatch path. `checkRunGrant` verifies it as any JWT. + */ +export function issueRunGrant(identity: RunGrantIdentity): string { + const issuedAt = Math.floor(Date.now() / 1000) + const encode = (value: object) => Buffer.from(JSON.stringify(value)).toString('base64url') + const unsigned = `${encode({ alg: 'HS256', typ: 'JWT' })}.${encode({ + iss: ISSUER, + aud: AUDIENCE, + sub: identity.userId, + iat: issuedAt, + exp: issuedAt + RUN_GRANT_TTL_SECONDS, + ...(identity.chatId ? { chat: identity.chatId } : {}), + ws: identity.workspaceId, + org: identity.organizationId, + })}` + return `${unsigned}.${createHmac('sha256', grantKey()).update(unsigned).digest('base64url')}` +} + +/** The header carrying a turn's grant, for a leg Sim sends: none for a turn with no single owner. */ +export function runGrantHeaders(context: { + userId: string + chatId?: string + workspaceId?: string + organizationId?: string +}): Record { + if (Boolean(context.workspaceId) === Boolean(context.organizationId)) return {} + return { + [RUN_GRANT_HEADER]: issueRunGrant({ + userId: context.userId, + ...(context.chatId ? { chatId: context.chatId } : {}), + workspaceId: context.workspaceId ?? null, + organizationId: context.organizationId ?? null, + }), + } +} + +/** Whether `token` is a grant Sim issued for exactly the identity a worker call names. */ +export async function checkRunGrant( + token: string | null | undefined, + claimed: { userId: string; chatId?: string; workspaceId?: string; organizationId?: string } +): Promise { + if (!token) return { ok: false, reason: 'missing' } + let payload: Awaited>['payload'] + try { + ;({ payload } = await jwtVerify(token, grantKey(), { + issuer: ISSUER, + audience: AUDIENCE, + algorithms: ['HS256'], + requiredClaims: ['sub', 'iat', 'exp'], + })) + } catch (error) { + return { ok: false, reason: error instanceof errors.JWTExpired ? 'expired' : 'invalid' } + } + const identity: RunGrantIdentity = { + userId: String(payload.sub), + ...(typeof payload.chat === 'string' ? { chatId: payload.chat } : {}), + workspaceId: typeof payload.ws === 'string' ? payload.ws : null, + organizationId: typeof payload.org === 'string' ? payload.org : null, + } + if ( + identity.userId !== claimed.userId || + identity.workspaceId !== (claimed.workspaceId ?? null) || + identity.organizationId !== (claimed.organizationId ?? null) || + (identity.chatId !== undefined && identity.chatId !== claimed.chatId) + ) + return { ok: false, reason: 'mismatch' } + return { ok: true, issuedAt: Number(payload.iat), identity } +} + +/** + * Whether a worker call may go on. A forged or mismatched grant never; a valid one always; a + * missing or expired one only until `MOTHERSHIP_REQUIRE_RUN_GRANT`, so workers that predate + * grants, and runs that outlived theirs, keep working while grants roll out. + */ +export function acceptsRunGrant(check: RunGrantCheck): boolean { + if (check.ok) return true + if (check.reason === 'invalid' || check.reason === 'mismatch') return false + return !env.MOTHERSHIP_REQUIRE_RUN_GRANT +} + +/** The grant each worker principal was admitted on, keyed by the principal object itself. */ +const admittedGrants = new WeakMap>() + +export function rememberRunGrant(principal: object, check: RunGrantCheck): void { + if (check.ok) admittedGrants.set(principal, check) +} + +/** + * A fresh grant for the same identity, once the one this call carried is past half its life. + * Only a grant Sim verified is renewed, so renewal never grants an identity Sim had not. The + * caller sends it only after rechecking the user's access. + */ +export function renewedRunGrant(principal: object): string | undefined { + const admitted = admittedGrants.get(principal) + if (!admitted || Date.now() / 1000 - admitted.issuedAt < RUN_GRANT_TTL_SECONDS / 2) + return undefined + return issueRunGrant(admitted.identity) +} diff --git a/apps/sim/lib/mothership/generated/run-control.ts b/apps/sim/lib/mothership/generated/run-control.ts index db2b9565882..44914a06b1d 100644 --- a/apps/sim/lib/mothership/generated/run-control.ts +++ b/apps/sim/lib/mothership/generated/run-control.ts @@ -3,11 +3,22 @@ import { z } from "zod"; +/** Sim's run grant (`RUN_GRANT_HEADER` in sim-transport): a token the worker returns, never reads. */ +export const RunGrant = z.string().min(1).max(4096); + export const RunControlRequest = z.object({ chatId: z.string().uuid(), streamId: z.string().min(1).max(200), }); export type RunControlRequest = z.infer; -export const RunControlState = z.object({ stopped: z.boolean() }); +export const RunControlState = z.object({ + stopped: z.boolean(), + /** + * A fresh run grant for the same user and chat, sent when the grant the call carried is past + * half its life: a run that keeps checking its control keeps a valid grant however long it + * runs. The worker uses it from its next call on. + */ + grant: RunGrant.optional(), +}); export type RunControlState = z.infer; diff --git a/apps/sim/lib/mothership/generated/sim-transport.ts b/apps/sim/lib/mothership/generated/sim-transport.ts index c869ff1ebb8..ffce7241db0 100644 --- a/apps/sim/lib/mothership/generated/sim-transport.ts +++ b/apps/sim/lib/mothership/generated/sim-transport.ts @@ -6,7 +6,7 @@ import { RecallChatsRequest } from "./chat-recall"; import { IntegrationCatalogRequest } from "./integration-catalog"; import { MemoryNotesAccessInput } from "./memory-notes-access"; import { MemoryScopeRequest } from "./memory-scope"; -import { RunControlRequest } from "./run-control"; +import { RunControlRequest, RunGrant } from "./run-control"; import { TaskWakeRequest, WorkflowWatchRequest } from "./tasks"; const SimChannelId = z.string().regex(/^[a-f0-9]{64}$/); @@ -17,6 +17,14 @@ export const SimConnection = z.discriminatedUnion("mode", [ ]); export type SimConnection = z.infer; +/** + * Sim's run grant: its signature over the user, chat and owner of a turn, sent with every turn + * it starts. The worker cannot mint one; it returns the latest grant for the chat on each call + * it makes to Sim (this header over HTTP, `grant` on a control request), and Sim serves the call + * only for the identity the grant names. + */ +export const RUN_GRANT_HEADER = "x-sim-run-grant"; + export const SimScope = z .object({ userId: z.string().min(1), @@ -47,6 +55,8 @@ export const SimControlRequest = z.object({ scope: SimScope, operation: SimControlOperation, expiresAt: z.number().int().positive(), + /** The chat's run grant (`RUN_GRANT_HEADER`); absent from a worker that predates grants. */ + grant: RunGrant.optional(), }); export type SimControlRequest = z.infer; diff --git a/apps/sim/lib/mothership/request/application/read-control.ts b/apps/sim/lib/mothership/request/application/read-control.ts index 44b909bd767..edc2373544b 100644 --- a/apps/sim/lib/mothership/request/application/read-control.ts +++ b/apps/sim/lib/mothership/request/application/read-control.ts @@ -14,6 +14,7 @@ import { OrganizationDelegationAuthorizationError } from '@/lib/core/application import { defineOrganizationOperation } from '@/lib/core/application/organization-operation' import { asOrchestrationError, OrchestrationError } from '@/lib/core/orchestration/types' import { getLatestRunForStream, isRunStopRequested } from '@/lib/mothership/async-runs/repository' +import { renewedRunGrant } from '@/lib/mothership/auth/run-grant' import { defineAuthorizedChatUseCase } from '@/lib/mothership/chat/application/authorized-chat-use-case' import { resolveOwnedChatContext } from '@/lib/mothership/chat/application/context' import type { RunControlRequest, RunControlState } from '@/lib/mothership/generated/run-control' @@ -125,7 +126,12 @@ export const readRunControl: AuthorizingUseCase< async execute(args) { requireRunControlCaller(args.principal, args.input) try { - return await readLiveRunControl.execute(args) + const state = await readLiveRunControl.execute(args) + if (state.stopped) return state + // The chat and the user's access were just rechecked, so a run that keeps checking its + // control keeps a valid grant however long it runs. + const grant = renewedRunGrant(args.principal) + return grant ? { ...state, grant } : state } catch (error) { const refusal = asOrchestrationError(error) if ( diff --git a/apps/sim/lib/mothership/request/lifecycle/run.test.ts b/apps/sim/lib/mothership/request/lifecycle/run.test.ts index 62d2ad4fe1c..bab96c502c8 100644 --- a/apps/sim/lib/mothership/request/lifecycle/run.test.ts +++ b/apps/sim/lib/mothership/request/lifecycle/run.test.ts @@ -73,6 +73,7 @@ const { mockCheckAttributedUsageLimits: vi.fn(), mockEnv: { INTERNAL_API_SECRET: 'transport-test-secret-000000000000000000', + BETTER_AUTH_SECRET: 'run-grant-test-auth-secret-0000000000000', COPILOT_API_KEY: undefined as string | undefined, MSHIP_SYSPROMPT_OVERRIDE: undefined as string | undefined, MOTHERSHIP_BENCHMARK_URL: 'https://benchmark.test', @@ -231,12 +232,14 @@ vi.mock('@/lib/mothership/request/enterprise-byok', () => ({ })) import { resetUsageGateCache } from '@/lib/billing/core/usage-gate-cache' +import { checkRunGrant } from '@/lib/mothership/auth/run-grant' import { buildPersistedAssistantMessage } from '@/lib/mothership/chat/persisted-message' import { CLIENT_TOOL_RESULT_TIMEOUT_MS } from '@/lib/mothership/constants' import { MothershipStreamV1CompletionStatus, MothershipStreamV1ToolOutcome, } from '@/lib/mothership/generated/mothership-stream-v1' +import { RUN_GRANT_HEADER } from '@/lib/mothership/generated/sim-transport' import { BillingLimitError, CopilotBackendError, @@ -2323,92 +2326,103 @@ describe('runCopilotLifecycle', () => { it.each([ { workspaceId: 'ws-1', organizationId: undefined }, { workspaceId: undefined, organizationId: 'org-1' }, - ])('uses immutable attribution on initial and resume legs for %j', async (owner) => { - const billingAttribution = { - actorUserId: 'user-1', - workspaceId: owner.workspaceId ?? null, - billedAccountUserId: 'owner-1', - organizationId: 'org-1', - billingEntity: { type: 'organization' as const, id: 'org-1' }, - billingPeriod: { - start: '2026-07-01T00:00:00.000Z', - end: '2026-08-01T00:00:00.000Z', - }, - payerSubscription: null, - } - setEnvFlags({ isHosted: true, isBillingEnabled: true }) - mockEnv.COPILOT_API_KEY = 'sim-agent-key' - mockRunStreamLoop.mockImplementationOnce( - async ( - _fetchUrl: string, - _fetchOptions: RequestInit, - context: StreamingContext - ): Promise => { - context.toolCalls.set('tool-1', { - id: 'tool-1', - name: 'read', - status: MothershipStreamV1ToolOutcome.success, - result: { success: true, output: { content: 'file contents' } }, - }) - context.awaitingAsyncContinuation = { - checkpointId: 'ckpt-1', - pendingToolCallIds: ['tool-1'], - } + ])( + 'sends immutable attribution and a run grant for the turn on initial and resume legs for %j', + async (owner) => { + const billingAttribution = { + actorUserId: 'user-1', + workspaceId: owner.workspaceId ?? null, + billedAccountUserId: 'owner-1', + organizationId: 'org-1', + billingEntity: { type: 'organization' as const, id: 'org-1' }, + billingPeriod: { + start: '2026-07-01T00:00:00.000Z', + end: '2026-08-01T00:00:00.000Z', + }, + payerSubscription: null, } - ) - mockRunStreamLoop.mockResolvedValueOnce(undefined) + setEnvFlags({ isHosted: true, isBillingEnabled: true }) + mockEnv.COPILOT_API_KEY = 'sim-agent-key' + mockRunStreamLoop.mockImplementationOnce( + async ( + _fetchUrl: string, + _fetchOptions: RequestInit, + context: StreamingContext + ): Promise => { + context.toolCalls.set('tool-1', { + id: 'tool-1', + name: 'read', + status: MothershipStreamV1ToolOutcome.success, + result: { success: true, output: { content: 'file contents' } }, + }) + context.awaitingAsyncContinuation = { + checkpointId: 'ckpt-1', + pendingToolCallIds: ['tool-1'], + } + } + ) + mockRunStreamLoop.mockResolvedValueOnce(undefined) - await runCopilotLifecycle( - { - message: 'hello', - ...(owner.organizationId ? { mode: 'assistant' } : {}), - messageId: 'message-1', - billingRequestId: 'caller-controlled', - }, - { - userId: 'user-1', - ...owner, - chatId: 'chat-1', - executionId: 'execution-1', - runId: 'run-1', - simRequestId: 'request-1', - billingAttribution, - } - ) + await runCopilotLifecycle( + { + message: 'hello', + ...(owner.organizationId ? { mode: 'assistant' } : {}), + messageId: 'message-1', + billingRequestId: 'caller-controlled', + }, + { + userId: 'user-1', + ...owner, + chatId: 'chat-1', + executionId: 'execution-1', + runId: 'run-1', + simRequestId: 'request-1', + billingAttribution, + } + ) - const firstHeaders = mockRunStreamLoop.mock.calls[0]?.[1].headers as Record - const billingRequestId = firstHeaders['x-sim-billing-request-id'] - expect(billingRequestId).toMatch( - /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i - ) - expect(billingRequestId).not.toBe('caller-controlled') + const firstHeaders = mockRunStreamLoop.mock.calls[0]?.[1].headers as Record + const billingRequestId = firstHeaders['x-sim-billing-request-id'] + expect(billingRequestId).toMatch( + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i + ) + expect(billingRequestId).not.toBe('caller-controlled') - expect(mockRunStreamLoop).toHaveBeenCalledTimes(2) - expect(mockLoadCopilotSearchIntegrations).not.toHaveBeenCalled() - for (const call of mockRunStreamLoop.mock.calls) { - const body = JSON.parse(String(call[1].body)) - if (call === mockRunStreamLoop.mock.calls[0]) { - expect(body).toMatchObject( - owner.organizationId - ? { organizationId: owner.organizationId } - : { workspaceId: owner.workspaceId } + expect(mockRunStreamLoop).toHaveBeenCalledTimes(2) + expect(mockLoadCopilotSearchIntegrations).not.toHaveBeenCalled() + for (const call of mockRunStreamLoop.mock.calls) { + const body = JSON.parse(String(call[1].body)) + if (call === mockRunStreamLoop.mock.calls[0]) { + expect(body).toMatchObject( + owner.organizationId + ? { organizationId: owner.organizationId } + : { workspaceId: owner.workspaceId } + ) + } else { + expect(body).not.toHaveProperty('workspaceId') + expect(body).not.toHaveProperty('organizationId') + expect(body).not.toHaveProperty('checkpointId') + } + const headers = call[1].headers as Record + expect(headers).toMatchObject({ + 'x-api-key': 'sim-agent-key', + 'x-sim-billing-protocol': 'attribution-v1', + 'x-sim-billing-request-id': billingRequestId, + }) + expect(JSON.parse(decodeURIComponent(headers['x-sim-billing-attribution']))).toEqual( + billingAttribution ) - } else { - expect(body).not.toHaveProperty('workspaceId') - expect(body).not.toHaveProperty('organizationId') - expect(body).not.toHaveProperty('checkpointId') + expect( + await checkRunGrant(headers[RUN_GRANT_HEADER], { + userId: 'user-1', + chatId: 'chat-1', + ...(owner.workspaceId ? { workspaceId: owner.workspaceId } : {}), + ...(owner.organizationId ? { organizationId: owner.organizationId } : {}), + }) + ).toMatchObject({ ok: true }) } - const headers = call[1].headers as Record - expect(headers).toMatchObject({ - 'x-api-key': 'sim-agent-key', - 'x-sim-billing-protocol': 'attribution-v1', - 'x-sim-billing-request-id': billingRequestId, - }) - expect(JSON.parse(decodeURIComponent(headers['x-sim-billing-attribution']))).toEqual( - billingAttribution - ) } - }) + ) it('cold recovery reads spend only against the original billing identity', async () => { const attribution = { diff --git a/apps/sim/lib/mothership/request/lifecycle/run.ts b/apps/sim/lib/mothership/request/lifecycle/run.ts index 135b1bc3f0c..3b805a039f5 100644 --- a/apps/sim/lib/mothership/request/lifecycle/run.ts +++ b/apps/sim/lib/mothership/request/lifecycle/run.ts @@ -26,6 +26,7 @@ import { updateRunStatus, } from '@/lib/mothership/async-runs/repository' import type { CopilotResourceAdmission } from '@/lib/mothership/auth/application-delegation' +import { runGrantHeaders } from '@/lib/mothership/auth/run-grant' import { CLIENT_TOOL_RESULT_TIMEOUT_MS, TOOL_WATCHDOG_RESUME_GRACE_MS, @@ -985,7 +986,10 @@ async function runResumeLegWithRetry( url, { method: 'POST', - headers: mothershipRequestHeaders(hostedBillingRequest, options.simRequestId), + headers: { + ...mothershipRequestHeaders(hostedBillingRequest, options.simRequestId), + ...runGrantHeaders(execContext), + }, body: JSON.stringify(body), }, leg, @@ -1330,7 +1334,10 @@ async function runCheckpointLoop( `${mothershipBaseURL}${route}`, { method: 'POST', - headers: mothershipRequestHeaders(hostedBillingRequest, mothershipRequestId), + headers: { + ...mothershipRequestHeaders(hostedBillingRequest, mothershipRequestId), + ...runGrantHeaders(execContext), + }, body: JSON.stringify({ ...payload, ...(route === '/api/mothership' || diff --git a/apps/sim/lib/mothership/transport/control.test.ts b/apps/sim/lib/mothership/transport/control.test.ts index 280c6e2fac0..a248bc4eff4 100644 --- a/apps/sim/lib/mothership/transport/control.test.ts +++ b/apps/sim/lib/mothership/transport/control.test.ts @@ -43,6 +43,7 @@ vi.mock('@/lib/mothership/tasks/application/context', () => ({ TASK_DELEGATION_A vi.mock('@/lib/mothership/tasks/wake', () => ({ runWakeTurn: handlers.wake })) import { OrchestrationError } from '@/lib/core/orchestration/types' +import { issueRunGrant } from '@/lib/mothership/auth/run-grant' import type { SimControlOperation, SimControlRequest, @@ -282,3 +283,20 @@ it("routes a chat's other chats through the same use case for both delivery scop ).status ).toBe(403) }) + +it("serves a control request whose run grant names its scope, and refuses another chat's grant", async () => { + handlers.memory.mockResolvedValue({ + userId: 'user', + organizationId: null, + workspaceId: 'workspace', + }) + const grant = (chatId: string) => + issueRunGrant({ userId: 'user', chatId, workspaceId: 'workspace', organizationId: null }) + const operation = { kind: 'memory_scope', input: { chatId: 'chat' } } as const + expect( + (await executeSimControl({ ...request(operation), grant: await grant('chat') })).status + ).toBe(200) + expect( + await executeSimControl({ ...request(operation), grant: await grant('another-chat') }) + ).toEqual({ status: 401, body: '{"error":"Run grant refused"}' }) +}) diff --git a/apps/sim/lib/mothership/transport/control.ts b/apps/sim/lib/mothership/transport/control.ts index 7c8ced95702..33cae8b2fdb 100644 --- a/apps/sim/lib/mothership/transport/control.ts +++ b/apps/sim/lib/mothership/transport/control.ts @@ -6,6 +6,7 @@ import { createTrustedCopilotPrincipal, createTrustedOrganizationCopilotPrincipal, } from '@/lib/mothership/auth/application-delegation' +import { acceptsRunGrant, checkRunGrant, rememberRunGrant } from '@/lib/mothership/auth/run-grant' import { RECALL_CHATS_AUDIENCE, readRecallChats, @@ -44,6 +45,8 @@ export async function executeSimControl(request: SimControlRequest): Promise Date: Wed, 18 Nov 2026 16:33:29 -0800 Subject: [PATCH 2/2] fix(mothership): read the grant switch as a string, run without an auth secret, and match chatless grants exactly - MOTHERSHIP_REQUIRE_RUN_GRANT arrives as a string, since env skips validation, so "false" turned enforcement on. It is now read with isTruthy. - A Sim running with DISABLE_AUTH may have no BETTER_AUTH_SECRET. It now issues no grants and treats a call's grant as missing, instead of throwing on every leg. - A grant issued for a turn with no chat matched any chat a call named. A grant's chat must now equal the call's, and absent matches only absent. - Run control renewed grants for runs that had finished without a Stop. It now renews only while the run is still going. - TSDoc on the module's exported names. Autopsy: the tests set the switch as a boolean, always had an auth secret, issued every grant with a chat, and checked renewal in the grant module rather than in run control's answer for a finished run. Each case now has a test that fails without its fix. --- .../sim/lib/mothership/auth/run-grant.test.ts | 30 +++++++ apps/sim/lib/mothership/auth/run-grant.ts | 82 +++++++++++++------ .../request/application/read-control.test.ts | 38 +++++++++ .../request/application/read-control.ts | 20 +++-- .../src/mocks/mothership-async-runs.mock.ts | 1 + 5 files changed, 139 insertions(+), 32 deletions(-) diff --git a/apps/sim/lib/mothership/auth/run-grant.test.ts b/apps/sim/lib/mothership/auth/run-grant.test.ts index 2fc1e6d086b..b90f226addd 100644 --- a/apps/sim/lib/mothership/auth/run-grant.test.ts +++ b/apps/sim/lib/mothership/auth/run-grant.test.ts @@ -7,6 +7,7 @@ import { issueRunGrant, rememberRunGrant, renewedRunGrant, + runGrantHeaders, } from '@/lib/mothership/auth/run-grant' const HOUR_MS = 60 * 60 * 1000 @@ -32,6 +33,35 @@ describe('run grants', () => { } }) + it('admits a grant for a turn with no chat only on calls that name no chat', async () => { + const chatless = issueRunGrant({ ...turn, chatId: undefined }) + expect(await checkRunGrant(chatless, claimed)).toEqual({ ok: false, reason: 'mismatch' }) + const { chatId: _chatId, ...noChat } = claimed + expect(await checkRunGrant(chatless, noChat)).toMatchObject({ ok: true }) + }) + + it('issues no grants and serves grant-less calls where Sim runs without an auth secret', async () => { + const grant = issueRunGrant(turn) + setEnv({ BETTER_AUTH_SECRET: undefined }) + expect(runGrantHeaders({ userId: 'actor', chatId: 'chat-1', workspaceId: 'ws-1' })).toEqual({}) + const check = await checkRunGrant(grant, claimed) + expect(check).toEqual({ ok: false, reason: 'missing' }) + expect(acceptsRunGrant(check)).toBe(true) + }) + + it('reads the enforcement switch as the string the environment holds', async () => { + const missing = await checkRunGrant(undefined, claimed) + for (const [value, accepted] of [ + ['false', true], + ['0', true], + ['true', false], + ['1', false], + ] as const) { + setEnv({ MOTHERSHIP_REQUIRE_RUN_GRANT: value }) + expect([value, acceptsRunGrant(missing)]).toEqual([value, accepted]) + } + }) + it('refuses a token the worker could mint itself with the secret it holds', async () => { const forged = await generateInternalToken('actor') const check = await checkRunGrant(forged, claimed) diff --git a/apps/sim/lib/mothership/auth/run-grant.ts b/apps/sim/lib/mothership/auth/run-grant.ts index b488174b199..280d6313775 100644 --- a/apps/sim/lib/mothership/auth/run-grant.ts +++ b/apps/sim/lib/mothership/auth/run-grant.ts @@ -1,6 +1,6 @@ import { createHmac, hkdfSync } from 'node:crypto' import { errors, jwtVerify } from 'jose' -import { env } from '@/lib/core/config/env' +import { env, isTruthy } from '@/lib/core/config/env' import { RUN_GRANT_HEADER } from '@/lib/mothership/generated/sim-transport' /** @@ -17,29 +17,32 @@ const KEY_LABEL = 'sim/mothership-run-grant/v1' /** Longer than any gap between a run's calls to Sim; a run-control answer renews it past half. */ const RUN_GRANT_TTL_SECONDS = 24 * 60 * 60 +/** Who a grant names: the user, the chat (when the turn had one) and its single owner. */ export interface RunGrantIdentity { userId: string - /** Absent when the turn had no chat; the grant then binds the user and owner only. */ + /** Absent when the turn had no chat; such a grant admits only calls that name no chat. */ chatId?: string workspaceId: string | null organizationId: string | null } +/** The outcome of checking a call's grant: the verified grant, or why there is none. */ export type RunGrantCheck = | { ok: true; issuedAt: number; identity: RunGrantIdentity } | { ok: false; reason: 'missing' | 'expired' | 'invalid' | 'mismatch' } -function grantKey(): Uint8Array { +/** + * The grant key, or undefined where Sim runs without an auth secret (`DISABLE_AUTH`): such a + * deployment issues no grants and serves worker calls only while grants are not required. + */ +function grantKey(): Uint8Array | undefined { + if (!env.BETTER_AUTH_SECRET) return undefined return new Uint8Array( hkdfSync('sha256', env.BETTER_AUTH_SECRET, new Uint8Array(0), KEY_LABEL, 32) ) } -/** - * A compact HS256 JWT, signed synchronously: issuing runs on every leg Sim sends, so it adds no - * await to the dispatch path. `checkRunGrant` verifies it as any JWT. - */ -export function issueRunGrant(identity: RunGrantIdentity): string { +function signRunGrant(identity: RunGrantIdentity, key: Uint8Array): string { const issuedAt = Math.floor(Date.now() / 1000) const encode = (value: object) => Buffer.from(JSON.stringify(value)).toString('base64url') const unsigned = `${encode({ alg: 'HS256', typ: 'JWT' })}.${encode({ @@ -52,36 +55,58 @@ export function issueRunGrant(identity: RunGrantIdentity): string { ws: identity.workspaceId, org: identity.organizationId, })}` - return `${unsigned}.${createHmac('sha256', grantKey()).update(unsigned).digest('base64url')}` + return `${unsigned}.${createHmac('sha256', key).update(unsigned).digest('base64url')}` +} + +/** + * A grant for `identity`: a compact HS256 JWT, signed synchronously, so issuing one on every leg + * Sim sends adds no await to the dispatch path; `checkRunGrant` verifies it as any JWT. Throws + * where Sim has no auth secret, where `runGrantHeaders` issues none. + */ +export function issueRunGrant(identity: RunGrantIdentity): string { + const key = grantKey() + if (!key) throw new Error('Run grants need BETTER_AUTH_SECRET') + return signRunGrant(identity, key) } -/** The header carrying a turn's grant, for a leg Sim sends: none for a turn with no single owner. */ +/** + * The header carrying a turn's grant, for a leg Sim sends: none for a turn with no single owner, + * or where Sim has no auth secret to sign with. + */ export function runGrantHeaders(context: { userId: string chatId?: string workspaceId?: string organizationId?: string }): Record { - if (Boolean(context.workspaceId) === Boolean(context.organizationId)) return {} + const key = grantKey() + if (!key || Boolean(context.workspaceId) === Boolean(context.organizationId)) return {} return { - [RUN_GRANT_HEADER]: issueRunGrant({ - userId: context.userId, - ...(context.chatId ? { chatId: context.chatId } : {}), - workspaceId: context.workspaceId ?? null, - organizationId: context.organizationId ?? null, - }), + [RUN_GRANT_HEADER]: signRunGrant( + { + userId: context.userId, + ...(context.chatId ? { chatId: context.chatId } : {}), + workspaceId: context.workspaceId ?? null, + organizationId: context.organizationId ?? null, + }, + key + ), } } -/** Whether `token` is a grant Sim issued for exactly the identity a worker call names. */ +/** + * Whether `token` is a grant Sim issued for exactly the identity a worker call names: the same + * user, owner and chat, where a grant with no chat matches only a call that names none. + */ export async function checkRunGrant( token: string | null | undefined, claimed: { userId: string; chatId?: string; workspaceId?: string; organizationId?: string } ): Promise { - if (!token) return { ok: false, reason: 'missing' } + const key = grantKey() + if (!token || !key) return { ok: false, reason: 'missing' } let payload: Awaited>['payload'] try { - ;({ payload } = await jwtVerify(token, grantKey(), { + ;({ payload } = await jwtVerify(token, key, { issuer: ISSUER, audience: AUDIENCE, algorithms: ['HS256'], @@ -100,7 +125,7 @@ export async function checkRunGrant( identity.userId !== claimed.userId || identity.workspaceId !== (claimed.workspaceId ?? null) || identity.organizationId !== (claimed.organizationId ?? null) || - (identity.chatId !== undefined && identity.chatId !== claimed.chatId) + identity.chatId !== claimed.chatId ) return { ok: false, reason: 'mismatch' } return { ok: true, issuedAt: Number(payload.iat), identity } @@ -108,18 +133,22 @@ export async function checkRunGrant( /** * Whether a worker call may go on. A forged or mismatched grant never; a valid one always; a - * missing or expired one only until `MOTHERSHIP_REQUIRE_RUN_GRANT`, so workers that predate - * grants, and runs that outlived theirs, keep working while grants roll out. + * missing or expired one only until `MOTHERSHIP_REQUIRE_RUN_GRANT` is true, so workers that + * predate grants, and runs that outlived theirs, keep working while grants roll out. */ export function acceptsRunGrant(check: RunGrantCheck): boolean { if (check.ok) return true if (check.reason === 'invalid' || check.reason === 'mismatch') return false - return !env.MOTHERSHIP_REQUIRE_RUN_GRANT + return !isTruthy(env.MOTHERSHIP_REQUIRE_RUN_GRANT) } /** The grant each worker principal was admitted on, keyed by the principal object itself. */ const admittedGrants = new WeakMap>() +/** + * Records the grant a worker principal was admitted on, so the same principal object can later + * be renewed (`renewedRunGrant`). Only a verified grant is kept; any other check records nothing. + */ export function rememberRunGrant(principal: object, check: RunGrantCheck): void { if (check.ok) admittedGrants.set(principal, check) } @@ -131,7 +160,8 @@ export function rememberRunGrant(principal: object, check: RunGrantCheck): void */ export function renewedRunGrant(principal: object): string | undefined { const admitted = admittedGrants.get(principal) - if (!admitted || Date.now() / 1000 - admitted.issuedAt < RUN_GRANT_TTL_SECONDS / 2) + const key = grantKey() + if (!admitted || !key || Date.now() / 1000 - admitted.issuedAt < RUN_GRANT_TTL_SECONDS / 2) return undefined - return issueRunGrant(admitted.identity) + return signRunGrant(admitted.identity, key) } diff --git a/apps/sim/lib/mothership/request/application/read-control.test.ts b/apps/sim/lib/mothership/request/application/read-control.test.ts index 1cf0ed037d1..2f5a12da04c 100644 --- a/apps/sim/lib/mothership/request/application/read-control.test.ts +++ b/apps/sim/lib/mothership/request/application/read-control.test.ts @@ -11,6 +11,7 @@ import { } from '@sim/testing/mocks/permission-groups-resolve.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' import { createTrustedOrganizationCopilotPrincipal } from '@/lib/mothership/auth/application-delegation' +import { checkRunGrant, issueRunGrant, rememberRunGrant } from '@/lib/mothership/auth/run-grant' import { RUN_CONTROL_AUDIENCE, readRunControl, @@ -151,4 +152,41 @@ describe('organization run-control authorization', () => { stopped: false, }) }) + it('renews a half-spent grant for a run still going, and none for a finished one', async () => { + vi.useFakeTimers({ toFake: ['Date'] }) + const issuedAt = Date.now() + try { + mocks.stopped.mockResolvedValue(false) + for (const [status, renewed] of [ + ['running', true], + ['complete', false], + ] as const) { + vi.setSystemTime(issuedAt) + const grant = issueRunGrant({ + userId: 'actor', + chatId: 'chat-1', + workspaceId: null, + organizationId: 'org-1', + }) + vi.setSystemTime(issuedAt + 13 * 60 * 60 * 1000) + const caller = principal() + rememberRunGrant( + caller, + await checkRunGrant(grant, { userId: 'actor', chatId: 'chat-1', organizationId: 'org-1' }) + ) + mocks.run.mockResolvedValueOnce({ + chatId: 'chat-1', + userId: 'actor', + organizationId: 'org-1', + workspaceId: null, + status, + }) + queueChat() + const state = await readRunControl.execute({ principal: caller, input }) + expect([status, 'grant' in state]).toEqual([status, renewed]) + } + } finally { + vi.useRealTimers() + } + }) }) diff --git a/apps/sim/lib/mothership/request/application/read-control.ts b/apps/sim/lib/mothership/request/application/read-control.ts index edc2373544b..2b22cf8d04e 100644 --- a/apps/sim/lib/mothership/request/application/read-control.ts +++ b/apps/sim/lib/mothership/request/application/read-control.ts @@ -13,7 +13,11 @@ import { import { OrganizationDelegationAuthorizationError } from '@/lib/core/application/organization-authorization' import { defineOrganizationOperation } from '@/lib/core/application/organization-operation' import { asOrchestrationError, OrchestrationError } from '@/lib/core/orchestration/types' -import { getLatestRunForStream, isRunStopRequested } from '@/lib/mothership/async-runs/repository' +import { + getLatestRunForStream, + isRunStopRequested, + TERMINAL_RUN_STATUSES, +} from '@/lib/mothership/async-runs/repository' import { renewedRunGrant } from '@/lib/mothership/auth/run-grant' import { defineAuthorizedChatUseCase } from '@/lib/mothership/chat/application/authorized-chat-use-case' import { resolveOwnedChatContext } from '@/lib/mothership/chat/application/context' @@ -70,7 +74,11 @@ const readLiveRunControl = defineAuthorizedChatUseCase({ ) { throw new OrchestrationError('not_found', 'Stream not found') } - return { stopped: await isRunStopRequested({ ...context, streamId: input.streamId }) } + return { + stopped: await isRunStopRequested({ ...context, streamId: input.streamId }), + // A finished run keeps no grant renewed. + finished: TERMINAL_RUN_STATUSES.includes(run.status), + } }, }) @@ -126,12 +134,12 @@ export const readRunControl: AuthorizingUseCase< async execute(args) { requireRunControlCaller(args.principal, args.input) try { - const state = await readLiveRunControl.execute(args) - if (state.stopped) return state + const { stopped, finished } = await readLiveRunControl.execute(args) + if (stopped || finished) return { stopped } // The chat and the user's access were just rechecked, so a run that keeps checking its - // control keeps a valid grant however long it runs. + // control keeps a valid grant however long it runs, and only while it runs. const grant = renewedRunGrant(args.principal) - return grant ? { ...state, grant } : state + return grant ? { stopped, grant } : { stopped } } catch (error) { const refusal = asOrchestrationError(error) if ( diff --git a/packages/testing/src/mocks/mothership-async-runs.mock.ts b/packages/testing/src/mocks/mothership-async-runs.mock.ts index 36291d7fdf3..11374061370 100644 --- a/packages/testing/src/mocks/mothership-async-runs.mock.ts +++ b/packages/testing/src/mocks/mothership-async-runs.mock.ts @@ -96,6 +96,7 @@ export const mothershipAsyncRunsMockFns = { * ``` */ export const mothershipAsyncRunsMock = { + TERMINAL_RUN_STATUSES: ['complete', 'error', 'cancelled'], withRunAdmissionLock: mothershipAsyncRunsMockFns.mockWithRunAdmissionLock, requestRunStop: mothershipAsyncRunsMockFns.mockRequestRunStop, stopLiveRunsTx: mothershipAsyncRunsMockFns.mockStopLiveRunsTx,