diff --git a/apps/sim/lib/mothership/request/handlers/handlers.test.ts b/apps/sim/lib/mothership/request/handlers/handlers.test.ts index 1851585e9cb..044a6a26895 100644 --- a/apps/sim/lib/mothership/request/handlers/handlers.test.ts +++ b/apps/sim/lib/mothership/request/handlers/handlers.test.ts @@ -1366,7 +1366,10 @@ describe('sse-handlers tool lifecycle', () => { describe('workflow tool pickup grace', () => { /** Dispatches a workflow call whose browser wait only ends when the server takes it over. */ - async function dispatchUnpickedWorkflowCall(toolCallId: string, hasClientViewer: boolean) { + async function dispatchUnpickedWorkflowCall( + toolCallId: string, + hasClientViewer: () => Promise + ) { waitForWorkflowToolCompletion.mockImplementationOnce((params: unknown) => { const abortSignal = toRecord(params).abortSignal return new Promise((resolve) => { @@ -1391,14 +1394,14 @@ describe('sse-handlers tool lifecycle', () => { } satisfies StreamEvent, context, execContext, - { interactive: true, hasClientViewer: async () => hasClientViewer } + { interactive: true, hasClientViewer } ) } it('runs a call no chat view is reading on the server after the short grace', async () => { vi.useFakeTimers() try { - await dispatchUnpickedWorkflowCall('tool-no-viewer', false) + await dispatchUnpickedWorkflowCall('tool-no-viewer', async () => false) await vi.advanceTimersByTimeAsync(5_000) expect(claimWorkflowToolExecution).toHaveBeenCalledWith( 'tool-no-viewer', @@ -1413,10 +1416,37 @@ describe('sse-handlers tool lifecycle', () => { } }) + it('gives a view that re-attached during the short grace the rest of the full grace', async () => { + vi.useFakeTimers() + try { + let attached = false + await dispatchUnpickedWorkflowCall('tool-reattached', async () => attached) + // A reload: the send's response dropped, and the view re-attaches two seconds later. + await vi.advanceTimersByTimeAsync(2_000) + attached = true + await vi.advanceTimersByTimeAsync(27_000) + expect(claimWorkflowToolExecution).not.toHaveBeenCalledWith( + 'tool-reattached', + expect.any(String), + 'sim' + ) + + await vi.advanceTimersByTimeAsync(1_000) + await Promise.allSettled(context.pendingToolPromises.values()) + expect(claimWorkflowToolExecution).toHaveBeenCalledWith( + 'tool-reattached', + expect.any(String), + 'sim' + ) + } finally { + vi.useRealTimers() + } + }) + it('keeps waiting the full grace for a call a chat view is reading', async () => { vi.useFakeTimers() try { - await dispatchUnpickedWorkflowCall('tool-viewer', true) + await dispatchUnpickedWorkflowCall('tool-viewer', async () => true) await vi.advanceTimersByTimeAsync(29_000) expect(claimWorkflowToolExecution).not.toHaveBeenCalledWith( 'tool-viewer', diff --git a/apps/sim/lib/mothership/request/handlers/tool.ts b/apps/sim/lib/mothership/request/handlers/tool.ts index 4a0ecb6773c..038944e3420 100644 --- a/apps/sim/lib/mothership/request/handlers/tool.ts +++ b/apps/sim/lib/mothership/request/handlers/tool.ts @@ -243,14 +243,25 @@ function desktopClaimsLocalReads( /** * How long a workflow tool waits for a chat view to pick it up before the server runs it. The * caller declared its executors at turn setup (ChatRequest.clientCapabilities): no pickup - * capability means no wait. With the capability, a turn no view is reading only waits for one - * that is re-attaching. The claim arbitration is the same either way. + * capability means no wait. With the capability, a turn no view is reading waits only for one + * that is re-attaching, and a view that attached by the end of that short wait (a reload) gets + * the rest of the full grace. The claim arbitration is the same either way. */ -async function workflowToolPickupGraceMs(options: OrchestratorOptions): Promise { - if (options.clientToolPickupExpected === false) return 0 - if (!options.hasClientViewer) return COPILOT_WORKFLOW_TOOL_CLIENT_GRACE_MS - const viewer = await options.hasClientViewer().catch(() => true) - return viewer ? COPILOT_WORKFLOW_TOOL_CLIENT_GRACE_MS : COPILOT_WORKFLOW_TOOL_NO_VIEWER_GRACE_MS +async function workflowToolPickupGrace( + options: OrchestratorOptions +): Promise<{ graceMs: number; extendGrace?: () => Promise }> { + if (options.clientToolPickupExpected === false) return { graceMs: 0 } + const hasClientViewer = options.hasClientViewer + if (!hasClientViewer) return { graceMs: COPILOT_WORKFLOW_TOOL_CLIENT_GRACE_MS } + const viewing = () => hasClientViewer().catch(() => true) + if (await viewing()) return { graceMs: COPILOT_WORKFLOW_TOOL_CLIENT_GRACE_MS } + return { + graceMs: COPILOT_WORKFLOW_TOOL_NO_VIEWER_GRACE_MS, + extendGrace: async () => + (await viewing()) + ? COPILOT_WORKFLOW_TOOL_CLIENT_GRACE_MS - COPILOT_WORKFLOW_TOOL_NO_VIEWER_GRACE_MS + : 0, + } } /** @@ -1127,7 +1138,7 @@ async function dispatchToolExecution( workflowId: resolveWorkflowToolTargetId(args, execContext.workflowId), timeoutMs, deadlineAt: toolCall.deadlineAt, - graceMs: await workflowToolPickupGraceMs(options), + ...(await workflowToolPickupGrace(options)), ...(toolCall.waitTimeoutMs !== undefined ? { waitTimeoutMs: toolCall.waitTimeoutMs } : {}), diff --git a/apps/sim/lib/mothership/request/lifecycle/start.test.ts b/apps/sim/lib/mothership/request/lifecycle/start.test.ts index d547cf46957..ea3f19216e5 100644 --- a/apps/sim/lib/mothership/request/lifecycle/start.test.ts +++ b/apps/sim/lib/mothership/request/lifecycle/start.test.ts @@ -2,6 +2,7 @@ import { propagation, trace } from '@opentelemetry/api' import { W3CTraceContextPropagator } from '@opentelemetry/core' import { TracerProvider } from '@opentelemetry/sdk-trace' import { dbChainMockFns, resetDbChainMock, resetEnvFlagsMock, setEnvFlags } from '@sim/testing' +import { createDeferred } from '@sim/testing/helpers/deferred' import { billingAttributionMock, billingAttributionMockFns, @@ -42,6 +43,7 @@ const { releasePendingChatStream, unregisterActiveStream, buildChatTitleContext, + hasStreamViewer, } = vi.hoisted(() => ({ runCopilotLifecycle: vi.fn(), resetBuffer: vi.fn(), @@ -56,6 +58,7 @@ const { releasePendingChatStream: vi.fn(), unregisterActiveStream: vi.fn(), buildChatTitleContext: vi.fn().mockResolvedValue(undefined), + hasStreamViewer: vi.fn(), })) const BILLING_ATTRIBUTION = { @@ -108,6 +111,7 @@ vi.mock('@/lib/mothership/request/session', () => ({ releasePendingChatStream, registerActiveStream, unregisterActiveStream, + hasStreamViewer, startAbortPoller: vi.fn().mockReturnValue(setInterval(() => {}, 999999)), isExplicitStopReason: (reason: unknown) => reason === 'user_stop:abortActiveStream', SSE_RESPONSE_HEADERS: {}, @@ -405,6 +409,42 @@ describe('createSSEStream terminal error handling', () => { } ) + it('counts the send as viewed while its response is attached, and asks for a reader once it is not', async () => { + const lifecycleDone = createDeferred() + let hasClientViewer: (() => Promise) | undefined + runCopilotLifecycle.mockImplementationOnce( + async (_payload: unknown, options: { hasClientViewer?: () => Promise }) => { + hasClientViewer = options.hasClientViewer + await lifecycleDone.promise + return { success: true, content: '', contentBlocks: [], toolCalls: [] } + } + ) + hasStreamViewer.mockResolvedValue(false) + const stream = createSSEStream({ + requestPayload: { message: 'hello' }, + userId: 'user-1', + streamId: 'stream-viewer', + executionId: 'exec-viewer', + runId: 'run-viewer', + currentChat: null, + message: 'hello', + titleModel: 'gpt-5.4', + requestId: 'req-viewer', + orchestrateOptions: { userId: 'user-1' }, + }) + await vi.waitFor(() => expect(hasClientViewer).toBeDefined()) + + expect(await hasClientViewer?.()).toBe(true) + + await stream.cancel() + expect(await hasClientViewer?.()).toBe(false) + hasStreamViewer.mockResolvedValueOnce(true) + expect(await hasClientViewer?.()).toBe(true) + expect(hasStreamViewer).toHaveBeenCalledWith('stream-viewer') + + lifecycleDone.resolve() + }) + it('starts the agent while title metadata is pending and forwards recovered inventory', async () => { let resolveContext!: (value: string) => void buildChatTitleContext.mockImplementationOnce( diff --git a/apps/sim/lib/mothership/request/session/viewer.ts b/apps/sim/lib/mothership/request/session/viewer.ts index cba3e79b7c2..66842900e63 100644 --- a/apps/sim/lib/mothership/request/session/viewer.ts +++ b/apps/sim/lib/mothership/request/session/viewer.ts @@ -27,9 +27,12 @@ async function markStreamViewer(streamId: string, attachmentId: string): Promise const redis = getRedisClient() if (!redis) return const key = getViewersKey(streamId) + const now = Date.now() + // Drops readers that died without leaving, which a live reader's refresh keeps the key past. await redis .multi() - .zadd(key, Date.now() + STREAM_VIEWER_TTL_MS, attachmentId) + .zremrangebyscore(key, '-inf', now) + .zadd(key, now + STREAM_VIEWER_TTL_MS, attachmentId) .pexpire(key, STREAM_VIEWER_TTL_MS) .exec() } diff --git a/apps/sim/lib/mothership/request/tools/workflow-client-fallback.test.ts b/apps/sim/lib/mothership/request/tools/workflow-client-fallback.test.ts index ec29de68d56..38218983407 100644 --- a/apps/sim/lib/mothership/request/tools/workflow-client-fallback.test.ts +++ b/apps/sim/lib/mothership/request/tools/workflow-client-fallback.test.ts @@ -78,6 +78,40 @@ describe('raceWorkflowToolClientPickup', () => { expect(params.runOnServer).not.toHaveBeenCalled() }) + it('returns a browser result that lands while the grace extension is still being checked', async () => { + let finish!: (completion: unknown) => void + waitForWorkflowToolCompletion.mockImplementation( + () => new Promise((resolve) => (finish = resolve)) + ) + const params = baseParams({ graceMs: 5_000, extendGrace: () => new Promise(() => {}) }) + + const promise = raceWorkflowToolClientPickup(params as never) + await vi.advanceTimersByTimeAsync(5_000) + finish({ status: 'success', data: { ok: true } }) + const outcome = await promise + + expect(outcome.winner).toBe('client') + expect(outcome.completion).toEqual({ status: 'success', data: { ok: true } }) + expect(claimWorkflowToolExecution).not.toHaveBeenCalled() + }) + + it('counts a slow extension check toward the extension', async () => { + pendingUntilAborted() + claimWorkflowToolExecution.mockResolvedValue({ toolCallId: 'tool-1' }) + const params = baseParams({ + graceMs: 5_000, + extendGrace: () => new Promise((resolve) => setTimeout(() => resolve(25_000), 4_000)), + }) + + const promise = raceWorkflowToolClientPickup(params as never) + await vi.advanceTimersByTimeAsync(29_000) + expect(claimWorkflowToolExecution).toHaveBeenCalledTimes(0) + await vi.advanceTimersByTimeAsync(1_000) + await promise + + expect(claimWorkflowToolExecution).toHaveBeenCalledTimes(1) + }) + it('runs the tool server-side when the grace elapses and the claim is won', async () => { pendingUntilAborted() claimWorkflowToolExecution.mockResolvedValue({ toolCallId: 'tool-1' }) diff --git a/apps/sim/lib/mothership/request/tools/workflow-client-fallback.ts b/apps/sim/lib/mothership/request/tools/workflow-client-fallback.ts index 8fffd112a91..6311c488771 100644 --- a/apps/sim/lib/mothership/request/tools/workflow-client-fallback.ts +++ b/apps/sim/lib/mothership/request/tools/workflow-client-fallback.ts @@ -51,6 +51,11 @@ interface RaceWorkflowToolClientPickupParams { /** When the call's deadline passes: past it the server does not take the call over. */ deadlineAt?: number graceMs: number + /** + * Asked once when the grace ends with no result: how much longer a view that attached during + * the grace should get (0 for none). Bounded by `timeoutMs` like the grace itself. + */ + extendGrace?: () => Promise /** * How long to wait for a browser's run before settling the call still running (the frame's * `waitTimeoutMs`); the browser's run goes on. Undefined waits until it settles. @@ -191,10 +196,29 @@ export async function raceWorkflowToolClientPickup( // would outlive the wait it is supposed to bound. const effectiveGraceMs = Math.min(graceMs, timeoutMs) - const first = await Promise.race([ - clientWait.then((completion) => ({ kind: 'client' as const, completion })), - sleep(effectiveGraceMs).then(() => ({ kind: 'grace' as const })), - ]) + const clientDone = clientWait.then((completion) => ({ kind: 'client' as const, completion })) + const raceGrace = (ms: number) => + Promise.race([clientDone, sleep(ms).then(() => ({ kind: 'grace' as const }))]) + + let first = await raceGrace(effectiveGraceMs) + if (first.kind === 'grace' && params.extendGrace) { + // The check counts toward the extension, and a result that lands during it wins at once. + const checkStartedAt = Date.now() + const extension = await Promise.race([ + clientDone, + params + .extendGrace() + .catch(() => 0) + .then((ms) => ({ kind: 'extend' as const, ms })), + ]) + if (extension.kind === 'client') { + first = extension + } else { + const extraMs = + Math.min(extension.ms, timeoutMs - effectiveGraceMs) - (Date.now() - checkStartedAt) + if (extraMs > 0) first = await raceGrace(extraMs) + } + } // A non-null client result inside the grace window is the normal path. // A null one means the wait itself already expired (timeoutMs <= graceMs), so