diff --git a/apps/sim/app/api/copilot/chat/stream/route.test.ts b/apps/sim/app/api/copilot/chat/stream/route.test.ts index edcdaac695c..06f25004aa4 100644 --- a/apps/sim/app/api/copilot/chat/stream/route.test.ts +++ b/apps/sim/app/api/copilot/chat/stream/route.test.ts @@ -31,6 +31,7 @@ vi.mock('@/lib/mothership/request/session', () => ({ readEvents, readFilePreviewSessions, findReplayGap, + holdStreamViewer: async () => () => {}, readRingPosition: async () => ({ requestedAfterSeq: 0, oldestSeq: 0, latestSeq: 0 }), ringCanServe: () => true, replayGapTerminal: async () => ({ gapDetected: true, envelopes: [] }), diff --git a/apps/sim/app/api/copilot/chat/stream/route.ts b/apps/sim/app/api/copilot/chat/stream/route.ts index b557b0a0145..d214fb471a2 100644 --- a/apps/sim/app/api/copilot/chat/stream/route.ts +++ b/apps/sim/app/api/copilot/chat/stream/route.ts @@ -35,6 +35,7 @@ import { encodeSSEEnvelope, findReplayGap, forwardRunReplay, + holdStreamViewer, isTerminalStreamStatus, openRunReplay, RunReplayUnavailableError, @@ -397,6 +398,8 @@ async function handleResumeRequestBody({ controllerClosed = true } request.signal.addEventListener('abort', abortListener, { once: true }) + // An attached reader is a chat view that starts this turn's workflow tools. + const releaseViewer = await holdStreamViewer(streamId) /** Delivers the ring's events after the cursor, or returns null if it trimmed the next one. */ const flushEvents = async (): Promise => { @@ -591,6 +594,7 @@ async function handleResumeRequestBody({ } markSpanForError(rootSpan, error) } finally { + releaseViewer() request.signal.removeEventListener('abort', abortListener) // Read before closing: closing the controller here is this route ending, not the client. const clientDisconnected = controllerClosed diff --git a/apps/sim/lib/mothership/constants.ts b/apps/sim/lib/mothership/constants.ts index 63180031d15..0af77ea2793 100644 --- a/apps/sim/lib/mothership/constants.ts +++ b/apps/sim/lib/mothership/constants.ts @@ -66,6 +66,13 @@ export const CHAT_RUN_DEADLINE_MS = 3_600_000 */ export const COPILOT_WORKFLOW_TOOL_CLIENT_GRACE_MS = 30_000 +/** + * The workflow grace when no chat view is reading the turn's stream at dispatch: a tab closed, + * a switch to another chat or page. It only bridges a view that is re-attaching right then (a + * reload, a reconnect after a network drop), which reads the call within a reconnect poll. + */ +export const COPILOT_WORKFLOW_TOOL_NO_VIEWER_GRACE_MS = 5_000 + /** * How long a desktop tool call (browser, terminal, local import) waits for the desktop app to * claim it before it fails as never started. diff --git a/apps/sim/lib/mothership/request/handlers/handlers.test.ts b/apps/sim/lib/mothership/request/handlers/handlers.test.ts index 1aadd9f8aa1..1851585e9cb 100644 --- a/apps/sim/lib/mothership/request/handlers/handlers.test.ts +++ b/apps/sim/lib/mothership/request/handlers/handlers.test.ts @@ -9,6 +9,7 @@ import { mothershipClientToolWaiterMockFns, } from '@sim/testing/mocks/mothership-client-tool-waiter.mock' import { sleep } from '@sim/utils/helpers' +import { toRecord } from '@sim/utils/object' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { AsyncToolCallOwnershipError } from '@/lib/mothership/async-runs/errors' import { restoreStreamingContext } from '@/lib/mothership/request/context/restore' @@ -1363,6 +1364,79 @@ describe('sse-handlers tool lifecycle', () => { expect(workflowResults[0].payload.executor).toBe(MothershipStreamV1ToolExecutor.sim) }) + describe('workflow tool pickup grace', () => { + /** Dispatches a workflow call whose browser wait only ends when the server takes it over. */ + async function dispatchUnpickedWorkflowCall(toolCallId: string, hasClientViewer: boolean) { + waitForWorkflowToolCompletion.mockImplementationOnce((params: unknown) => { + const abortSignal = toRecord(params).abortSignal + return new Promise((resolve) => { + if (abortSignal instanceof AbortSignal) { + abortSignal.addEventListener('abort', () => resolve(null)) + } + }) + }) + claimWorkflowToolExecution.mockResolvedValueOnce({ toolCallId }) + executeTool.mockResolvedValueOnce({ success: true, output: {} }) + await sseHandlers.tool( + { + type: MothershipStreamV1EventType.tool, + payload: { + toolCallId, + toolName: 'run_workflow', + arguments: { workflowId: 'workflow-1' }, + executor: MothershipStreamV1ToolExecutor.client, + mode: MothershipStreamV1ToolMode.async, + phase: MothershipStreamV1ToolPhase.call, + }, + } satisfies StreamEvent, + context, + execContext, + { interactive: true, hasClientViewer: async () => hasClientViewer } + ) + } + + it('runs a call no chat view is reading on the server after the short grace', async () => { + vi.useFakeTimers() + try { + await dispatchUnpickedWorkflowCall('tool-no-viewer', false) + await vi.advanceTimersByTimeAsync(5_000) + expect(claimWorkflowToolExecution).toHaveBeenCalledWith( + 'tool-no-viewer', + expect.any(String), + 'sim' + ) + + await Promise.allSettled(context.pendingToolPromises.values()) + expect(executeTool.mock.calls.at(-1)?.[0]).toBe('run_workflow') + } finally { + vi.useRealTimers() + } + }) + + it('keeps waiting the full grace for a call a chat view is reading', async () => { + vi.useFakeTimers() + try { + await dispatchUnpickedWorkflowCall('tool-viewer', true) + await vi.advanceTimersByTimeAsync(29_000) + expect(claimWorkflowToolExecution).not.toHaveBeenCalledWith( + 'tool-viewer', + expect.any(String), + 'sim' + ) + + await vi.advanceTimersByTimeAsync(1_000) + await Promise.allSettled(context.pendingToolPromises.values()) + expect(claimWorkflowToolExecution).toHaveBeenCalledWith( + 'tool-viewer', + expect.any(String), + 'sim' + ) + } finally { + vi.useRealTimers() + } + }) + }) + it('claims and runs the same workflow when the call omits an explicit workflowId', async () => { // The waiter resolves the target via resolveWorkflowToolTargetId(args, ctx) // while the handler resolves it as params.workflowId || context.workflowId. diff --git a/apps/sim/lib/mothership/request/handlers/tool.ts b/apps/sim/lib/mothership/request/handlers/tool.ts index 33183671aef..4a0ecb6773c 100644 --- a/apps/sim/lib/mothership/request/handlers/tool.ts +++ b/apps/sim/lib/mothership/request/handlers/tool.ts @@ -18,6 +18,7 @@ import { getAsyncToolCall, upsertAsyncToolCall } from '@/lib/mothership/async-ru import { CLIENT_TOOL_RESULT_TIMEOUT_MS, COPILOT_WORKFLOW_TOOL_CLIENT_GRACE_MS, + COPILOT_WORKFLOW_TOOL_NO_VIEWER_GRACE_MS, DESKTOP_TOOL_PICKUP_GRACE_MS, } from '@/lib/mothership/constants' import { @@ -239,6 +240,19 @@ function desktopClaimsLocalReads( return options?.desktopClaimsLocalReads === true || desktopDeviceId !== null } +/** + * How long a workflow tool waits for a chat view to pick it up before the server runs it. The + * caller declared its executors at turn setup (ChatRequest.clientCapabilities): no pickup + * capability means no wait. With the capability, a turn no view is reading only waits for one + * that is re-attaching. The claim arbitration is the same either way. + */ +async function workflowToolPickupGraceMs(options: OrchestratorOptions): Promise { + if (options.clientToolPickupExpected === false) return 0 + if (!options.hasClientViewer) return COPILOT_WORKFLOW_TOOL_CLIENT_GRACE_MS + const viewer = await options.hasClientViewer().catch(() => true) + return viewer ? COPILOT_WORKFLOW_TOOL_CLIENT_GRACE_MS : COPILOT_WORKFLOW_TOOL_NO_VIEWER_GRACE_MS +} + /** * Upsert the durable `async_tool_calls` row before the authoritative tool-call * SSE frame is forwarded to the client, so `/api/copilot/confirm` and @@ -1113,14 +1127,7 @@ async function dispatchToolExecution( workflowId: resolveWorkflowToolTargetId(args, execContext.workflowId), timeoutMs, deadlineAt: toolCall.deadlineAt, - // The caller declared its executors at turn setup (ChatRequest.clientCapabilities): - // no pickup capability → zero grace, the race resolves straight to the server - // claim. Same claim arbitration either way — the grace only encodes how long a - // client might plausibly appear. - graceMs: - options.clientToolPickupExpected === false - ? 0 - : COPILOT_WORKFLOW_TOOL_CLIENT_GRACE_MS, + graceMs: await workflowToolPickupGraceMs(options), ...(toolCall.waitTimeoutMs !== undefined ? { waitTimeoutMs: toolCall.waitTimeoutMs } : {}), diff --git a/apps/sim/lib/mothership/request/lifecycle/start.ts b/apps/sim/lib/mothership/request/lifecycle/start.ts index 0ae55e90c2d..4af8f6fdaf3 100644 --- a/apps/sim/lib/mothership/request/lifecycle/start.ts +++ b/apps/sim/lib/mothership/request/lifecycle/start.ts @@ -46,6 +46,7 @@ import { type CopilotLifecycleOutcome, startCopilotOtelRoot } from '@/lib/mother import { cleanupAbortMarker, clearFilePreviewSessions, + hasStreamViewer, isExplicitStopReason, registerActiveStream, releasePendingChatStream, @@ -427,6 +428,8 @@ export function createSSEStream(params: StreamingOrchestrationParams): ReadableS const result = await runCopilotLifecycle(requestPayload, { ...orchestrateOptions, + hasClientViewer: async () => + !publisher.clientDisconnected || (await hasStreamViewer(streamId)), executionId, runId, trace: collector, diff --git a/apps/sim/lib/mothership/request/session/index.ts b/apps/sim/lib/mothership/request/session/index.ts index d31150c625c..36ca37c59ed 100644 --- a/apps/sim/lib/mothership/request/session/index.ts +++ b/apps/sim/lib/mothership/request/session/index.ts @@ -83,4 +83,5 @@ export { } from './run-replay' export { encodeSSEEnvelope, SSE_RESPONSE_HEADERS } from './sse' export type { StreamBatchEvent } from './types' +export { hasStreamViewer, holdStreamViewer } from './viewer' export { StreamWriter, type StreamWriterOptions } from './writer' diff --git a/apps/sim/lib/mothership/request/session/viewer.integration.ts b/apps/sim/lib/mothership/request/session/viewer.integration.ts new file mode 100644 index 00000000000..de2e43749ff --- /dev/null +++ b/apps/sim/lib/mothership/request/session/viewer.integration.ts @@ -0,0 +1,61 @@ +/** + * Chat stream reader presence against real Redis: a reader attached on one instance must be + * visible to the instance dispatching that turn's workflow tools, and must stop counting the + * moment it leaves, so a turn nobody reads no longer waits out the full pickup grace. + */ +import { afterAll, describe, expect, it, vi } from 'vitest' + +const { redisUrl } = await vi.hoisted(async () => { + const { readTestRedisUrl } = await import('@sim/db/testing/test-infrastructure') + const url = readTestRedisUrl() + if (url) process.env.REDIS_URL = url + return { redisUrl: url } +}) + +import { sleep } from '@sim/utils/helpers' +import { generateId } from '@sim/utils/id' +import { closeRedisConnection } from '@/lib/core/config/redis' +import { hasStreamViewer, holdStreamViewer } from '@/lib/mothership/request/session/viewer' + +describe.runIf(Boolean(redisUrl))('chat stream reader presence', () => { + afterAll(async () => { + await closeRedisConnection() + }) + + it('counts a reader as attached once its hold resolves', async () => { + const streamId = generateId() + const release = await holdStreamViewer(streamId) + try { + expect(await hasStreamViewer(streamId)).toBe(true) + } finally { + release() + } + }) + + it('sees no reader on a stream nobody attached to', async () => { + expect(await hasStreamViewer(generateId())).toBe(false) + }) + + it('keeps counting a stream while another reader stays attached', async () => { + const streamId = generateId() + const leaving = await holdStreamViewer(streamId) + const staying = await holdStreamViewer(streamId) + try { + leaving() + await expect.poll(() => hasStreamViewer(streamId)).toBe(true) + } finally { + staying() + } + }) + + it('stops counting a reader as soon as it leaves, and never re-marks it', async () => { + const streamId = generateId() + const release = await holdStreamViewer(streamId) + release() + await expect.poll(() => hasStreamViewer(streamId)).toBe(false) + // Longer than one refresh: a reader still refreshing would have re-marked itself by now. + await sleep(3_500) + + expect(await hasStreamViewer(streamId)).toBe(false) + }) +}) diff --git a/apps/sim/lib/mothership/request/session/viewer.ts b/apps/sim/lib/mothership/request/session/viewer.ts new file mode 100644 index 00000000000..cba3e79b7c2 --- /dev/null +++ b/apps/sim/lib/mothership/request/session/viewer.ts @@ -0,0 +1,82 @@ +import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' +import { getRedisClient } from '@/lib/core/config/redis' + +/** + * Which chat streams have a reconnected reader attached right now, on any instance. + * + * A chat view starts client-routed workflow tools from the stream it reads. The original send + * knows in-process whether its own response is still attached; a view that re-attached through + * the reconnect route (a reload, a chat switch back, another tab) may sit on another instance, + * so each attachment holds its own entry here, scored by when it lapses, for as long as it + * stays attached. Leaving removes the entry at once; a reader that dies without leaving lapses. + */ + +const logger = createLogger('SessionStreamViewer') + +/** Long enough that one late refresh never makes an attached reader look gone. */ +const STREAM_VIEWER_TTL_MS = 10_000 +const STREAM_VIEWER_REFRESH_MS = 3_000 + +function getViewersKey(streamId: string) { + return `mothership_stream:${streamId}:viewers` +} + +async function markStreamViewer(streamId: string, attachmentId: string): Promise { + const redis = getRedisClient() + if (!redis) return + const key = getViewersKey(streamId) + await redis + .multi() + .zadd(key, Date.now() + STREAM_VIEWER_TTL_MS, attachmentId) + .pexpire(key, STREAM_VIEWER_TTL_MS) + .exec() +} + +async function unmarkStreamViewer(streamId: string, attachmentId: string): Promise { + const redis = getRedisClient() + if (!redis) return + await redis.zrem(getViewersKey(streamId), attachmentId) +} + +function warnMarkFailed(streamId: string, error: unknown) { + logger.warn('Could not record a chat stream reader', { + streamId, + error: toError(error).message, + }) +} + +/** + * Records a reader attached to the stream until the returned release runs. Resolves once the + * first mark is written, so the reader counts before it reads its first event. + */ +export async function holdStreamViewer(streamId: string): Promise<() => void> { + const attachmentId = generateId() + const refresh = () => + markStreamViewer(streamId, attachmentId).catch((error) => warnMarkFailed(streamId, error)) + await refresh() + const interval = setInterval(refresh, STREAM_VIEWER_REFRESH_MS) + return () => { + clearInterval(interval) + unmarkStreamViewer(streamId, attachmentId).catch((error) => warnMarkFailed(streamId, error)) + } +} + +/** + * Whether a reconnected reader is attached to the stream. Answers true whenever it cannot + * tell (no Redis, a failed read), so an unknown never cuts a present view's pickup window. + */ +export async function hasStreamViewer(streamId: string): Promise { + const redis = getRedisClient() + if (!redis) return true + try { + return (await redis.zcount(getViewersKey(streamId), Date.now(), '+inf')) > 0 + } catch (error) { + logger.warn('Could not read whether a chat stream reader is attached', { + streamId, + error: toError(error).message, + }) + return true + } +} diff --git a/apps/sim/lib/mothership/request/types.ts b/apps/sim/lib/mothership/request/types.ts index 6792e5b560e..f37597a1942 100644 --- a/apps/sim/lib/mothership/request/types.ts +++ b/apps/sim/lib/mothership/request/types.ts @@ -290,6 +290,13 @@ export interface OrchestratorOptions { * `interactive`, which is a trust classification, not executor routing. */ clientToolPickupExpected?: boolean + /** + * Whether a chat view is reading this turn's stream right now: the send's own response, or a + * reader re-attached through the reconnect route. Answers true when it cannot tell. With no + * view, a workflow tool waits only a short grace for one to attach before running server-side. + * Absent on legs that cannot tell, which keep the full grace. + */ + hasClientViewer?: () => Promise /** * The turn's desktop claims local reads (`read_local_file`, user-local VFS reads) through * authorize before reading, so they are persisted pending and fail fast when nothing picks them