From 7c5ef5f8bf79a6fc02ba123c283e32a72fa8d28b Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 17 Nov 2026 20:48:41 -0800 Subject: [PATCH 1/2] fix(mothership): skip the 30s workflow pickup wait when no chat view is reading A client-routed workflow tool waited 30s for a chat view to pick it up before the server ran it, even when no view was reading the turn's stream at all (tab closed, user on another chat or page). Dispatch now checks whether the send's own response is still attached or a reconnected reader holds a short-lived Redis mark; with neither, it waits only 5s for a view that is re-attaching. A present view keeps the full grace, and an unknown (no Redis, failed read) counts as present. --- .../app/api/copilot/chat/stream/route.test.ts | 1 + apps/sim/app/api/copilot/chat/stream/route.ts | 4 ++ apps/sim/lib/mothership/constants.ts | 7 ++ .../request/handlers/handlers.test.ts | 69 +++++++++++++++++++ .../lib/mothership/request/handlers/tool.ts | 23 ++++--- .../lib/mothership/request/lifecycle/start.ts | 3 + .../lib/mothership/request/session/index.ts | 1 + .../request/session/viewer.integration.ts | 49 +++++++++++++ .../lib/mothership/request/session/viewer.ts | 64 +++++++++++++++++ apps/sim/lib/mothership/request/types.ts | 7 ++ 10 files changed, 220 insertions(+), 8 deletions(-) create mode 100644 apps/sim/lib/mothership/request/session/viewer.integration.ts create mode 100644 apps/sim/lib/mothership/request/session/viewer.ts diff --git a/apps/sim/app/api/copilot/chat/stream/route.test.ts b/apps/sim/app/api/copilot/chat/stream/route.test.ts index edcdaac695c..ced90c21241 100644 --- a/apps/sim/app/api/copilot/chat/stream/route.test.ts +++ b/apps/sim/app/api/copilot/chat/stream/route.test.ts @@ -31,6 +31,7 @@ vi.mock('@/lib/mothership/request/session', () => ({ readEvents, readFilePreviewSessions, findReplayGap, + holdStreamViewer: () => () => {}, readRingPosition: async () => ({ requestedAfterSeq: 0, oldestSeq: 0, latestSeq: 0 }), ringCanServe: () => true, replayGapTerminal: async () => ({ gapDetected: true, envelopes: [] }), diff --git a/apps/sim/app/api/copilot/chat/stream/route.ts b/apps/sim/app/api/copilot/chat/stream/route.ts index b557b0a0145..b2fa881dd66 100644 --- a/apps/sim/app/api/copilot/chat/stream/route.ts +++ b/apps/sim/app/api/copilot/chat/stream/route.ts @@ -35,6 +35,7 @@ import { encodeSSEEnvelope, findReplayGap, forwardRunReplay, + holdStreamViewer, isTerminalStreamStatus, openRunReplay, RunReplayUnavailableError, @@ -397,6 +398,8 @@ async function handleResumeRequestBody({ controllerClosed = true } request.signal.addEventListener('abort', abortListener, { once: true }) + // An attached reader is a chat view that starts this turn's workflow tools. + const releaseViewer = holdStreamViewer(streamId) /** Delivers the ring's events after the cursor, or returns null if it trimmed the next one. */ const flushEvents = async (): Promise => { @@ -591,6 +594,7 @@ async function handleResumeRequestBody({ } markSpanForError(rootSpan, error) } finally { + releaseViewer() request.signal.removeEventListener('abort', abortListener) // Read before closing: closing the controller here is this route ending, not the client. const clientDisconnected = controllerClosed diff --git a/apps/sim/lib/mothership/constants.ts b/apps/sim/lib/mothership/constants.ts index 63180031d15..0af77ea2793 100644 --- a/apps/sim/lib/mothership/constants.ts +++ b/apps/sim/lib/mothership/constants.ts @@ -66,6 +66,13 @@ export const CHAT_RUN_DEADLINE_MS = 3_600_000 */ export const COPILOT_WORKFLOW_TOOL_CLIENT_GRACE_MS = 30_000 +/** + * The workflow grace when no chat view is reading the turn's stream at dispatch: a tab closed, + * a switch to another chat or page. It only bridges a view that is re-attaching right then (a + * reload, a reconnect after a network drop), which reads the call within a reconnect poll. + */ +export const COPILOT_WORKFLOW_TOOL_NO_VIEWER_GRACE_MS = 5_000 + /** * How long a desktop tool call (browser, terminal, local import) waits for the desktop app to * claim it before it fails as never started. diff --git a/apps/sim/lib/mothership/request/handlers/handlers.test.ts b/apps/sim/lib/mothership/request/handlers/handlers.test.ts index 1aadd9f8aa1..4c04613a6f0 100644 --- a/apps/sim/lib/mothership/request/handlers/handlers.test.ts +++ b/apps/sim/lib/mothership/request/handlers/handlers.test.ts @@ -1363,6 +1363,75 @@ describe('sse-handlers tool lifecycle', () => { expect(workflowResults[0].payload.executor).toBe(MothershipStreamV1ToolExecutor.sim) }) + describe('workflow tool pickup grace', () => { + /** Dispatches a workflow call whose browser wait only ends when the server takes it over. */ + async function dispatchUnpickedWorkflowCall(toolCallId: string, hasClientViewer: boolean) { + waitForWorkflowToolCompletion.mockImplementationOnce( + ({ abortSignal }: { abortSignal?: AbortSignal }) => + new Promise((resolve) => abortSignal?.addEventListener('abort', () => resolve(null))) + ) + claimWorkflowToolExecution.mockResolvedValueOnce({ toolCallId }) + executeTool.mockResolvedValueOnce({ success: true, output: {} }) + await sseHandlers.tool( + { + type: MothershipStreamV1EventType.tool, + payload: { + toolCallId, + toolName: 'run_workflow', + arguments: { workflowId: 'workflow-1' }, + executor: MothershipStreamV1ToolExecutor.client, + mode: MothershipStreamV1ToolMode.async, + phase: MothershipStreamV1ToolPhase.call, + }, + } satisfies StreamEvent, + context, + execContext, + { interactive: true, hasClientViewer: async () => hasClientViewer } + ) + } + + it('runs a call no chat view is reading on the server after the short grace', async () => { + vi.useFakeTimers() + try { + await dispatchUnpickedWorkflowCall('tool-no-viewer', false) + await vi.advanceTimersByTimeAsync(5_000) + expect(claimWorkflowToolExecution).toHaveBeenCalledWith( + 'tool-no-viewer', + expect.any(String), + 'sim' + ) + + await Promise.allSettled(context.pendingToolPromises.values()) + expect(executeTool.mock.calls.at(-1)?.[0]).toBe('run_workflow') + } finally { + vi.useRealTimers() + } + }) + + it('keeps waiting the full grace for a call a chat view is reading', async () => { + vi.useFakeTimers() + try { + await dispatchUnpickedWorkflowCall('tool-viewer', true) + await vi.advanceTimersByTimeAsync(29_000) + expect(claimWorkflowToolExecution).not.toHaveBeenCalledWith( + 'tool-viewer', + expect.any(String), + 'sim' + ) + + await vi.advanceTimersByTimeAsync(1_000) + await Promise.allSettled(context.pendingToolPromises.values()) + expect(claimWorkflowToolExecution).toHaveBeenCalledWith( + 'tool-viewer', + expect.any(String), + 'sim' + ) + } finally { + vi.useRealTimers() + } + }) + }) + it('claims and runs the same workflow when the call omits an explicit workflowId', async () => { // The waiter resolves the target via resolveWorkflowToolTargetId(args, ctx) // while the handler resolves it as params.workflowId || context.workflowId. diff --git a/apps/sim/lib/mothership/request/handlers/tool.ts b/apps/sim/lib/mothership/request/handlers/tool.ts index 33183671aef..4a0ecb6773c 100644 --- a/apps/sim/lib/mothership/request/handlers/tool.ts +++ b/apps/sim/lib/mothership/request/handlers/tool.ts @@ -18,6 +18,7 @@ import { getAsyncToolCall, upsertAsyncToolCall } from '@/lib/mothership/async-ru import { CLIENT_TOOL_RESULT_TIMEOUT_MS, COPILOT_WORKFLOW_TOOL_CLIENT_GRACE_MS, + COPILOT_WORKFLOW_TOOL_NO_VIEWER_GRACE_MS, DESKTOP_TOOL_PICKUP_GRACE_MS, } from '@/lib/mothership/constants' import { @@ -239,6 +240,19 @@ function desktopClaimsLocalReads( return options?.desktopClaimsLocalReads === true || desktopDeviceId !== null } +/** + * How long a workflow tool waits for a chat view to pick it up before the server runs it. The + * caller declared its executors at turn setup (ChatRequest.clientCapabilities): no pickup + * capability means no wait. With the capability, a turn no view is reading only waits for one + * that is re-attaching. The claim arbitration is the same either way. + */ +async function workflowToolPickupGraceMs(options: OrchestratorOptions): Promise { + if (options.clientToolPickupExpected === false) return 0 + if (!options.hasClientViewer) return COPILOT_WORKFLOW_TOOL_CLIENT_GRACE_MS + const viewer = await options.hasClientViewer().catch(() => true) + return viewer ? COPILOT_WORKFLOW_TOOL_CLIENT_GRACE_MS : COPILOT_WORKFLOW_TOOL_NO_VIEWER_GRACE_MS +} + /** * Upsert the durable `async_tool_calls` row before the authoritative tool-call * SSE frame is forwarded to the client, so `/api/copilot/confirm` and @@ -1113,14 +1127,7 @@ async function dispatchToolExecution( workflowId: resolveWorkflowToolTargetId(args, execContext.workflowId), timeoutMs, deadlineAt: toolCall.deadlineAt, - // The caller declared its executors at turn setup (ChatRequest.clientCapabilities): - // no pickup capability → zero grace, the race resolves straight to the server - // claim. Same claim arbitration either way — the grace only encodes how long a - // client might plausibly appear. - graceMs: - options.clientToolPickupExpected === false - ? 0 - : COPILOT_WORKFLOW_TOOL_CLIENT_GRACE_MS, + graceMs: await workflowToolPickupGraceMs(options), ...(toolCall.waitTimeoutMs !== undefined ? { waitTimeoutMs: toolCall.waitTimeoutMs } : {}), diff --git a/apps/sim/lib/mothership/request/lifecycle/start.ts b/apps/sim/lib/mothership/request/lifecycle/start.ts index 0ae55e90c2d..4af8f6fdaf3 100644 --- a/apps/sim/lib/mothership/request/lifecycle/start.ts +++ b/apps/sim/lib/mothership/request/lifecycle/start.ts @@ -46,6 +46,7 @@ import { type CopilotLifecycleOutcome, startCopilotOtelRoot } from '@/lib/mother import { cleanupAbortMarker, clearFilePreviewSessions, + hasStreamViewer, isExplicitStopReason, registerActiveStream, releasePendingChatStream, @@ -427,6 +428,8 @@ export function createSSEStream(params: StreamingOrchestrationParams): ReadableS const result = await runCopilotLifecycle(requestPayload, { ...orchestrateOptions, + hasClientViewer: async () => + !publisher.clientDisconnected || (await hasStreamViewer(streamId)), executionId, runId, trace: collector, diff --git a/apps/sim/lib/mothership/request/session/index.ts b/apps/sim/lib/mothership/request/session/index.ts index d31150c625c..36ca37c59ed 100644 --- a/apps/sim/lib/mothership/request/session/index.ts +++ b/apps/sim/lib/mothership/request/session/index.ts @@ -83,4 +83,5 @@ export { } from './run-replay' export { encodeSSEEnvelope, SSE_RESPONSE_HEADERS } from './sse' export type { StreamBatchEvent } from './types' +export { hasStreamViewer, holdStreamViewer } from './viewer' export { StreamWriter, type StreamWriterOptions } from './writer' diff --git a/apps/sim/lib/mothership/request/session/viewer.integration.ts b/apps/sim/lib/mothership/request/session/viewer.integration.ts new file mode 100644 index 00000000000..42f4d6e6a8b --- /dev/null +++ b/apps/sim/lib/mothership/request/session/viewer.integration.ts @@ -0,0 +1,49 @@ +/** + * Chat stream reader presence against real Redis: a reader attached on one instance must be + * visible to the instance dispatching that turn's workflow tools, and must stop counting once it + * leaves, so a turn nobody reads no longer waits out the full pickup grace. + */ +import { afterAll, describe, expect, it, vi } from 'vitest' + +await vi.hoisted(async () => { + const { readTestRedisUrl } = await import('@sim/db/testing/test-infrastructure') + process.env.REDIS_URL = readTestRedisUrl() +}) + +import { sleep } from '@sim/utils/helpers' +import { generateId } from '@sim/utils/id' +import { closeRedisConnection, getRedisClient } from '@/lib/core/config/redis' +import { hasStreamViewer, holdStreamViewer } from '@/lib/mothership/request/session/viewer' + +describe('chat stream reader presence', () => { + afterAll(async () => { + await closeRedisConnection() + }) + + it('sees a reader attached to the stream', async () => { + const streamId = generateId() + const release = holdStreamViewer(streamId) + try { + await expect.poll(() => hasStreamViewer(streamId)).toBe(true) + } finally { + release() + } + }) + + it('sees no reader on a stream nobody attached to', async () => { + expect(await hasStreamViewer(generateId())).toBe(false) + }) + + it('never re-marks a reader after it leaves', async () => { + const streamId = generateId() + const release = holdStreamViewer(streamId) + await expect.poll(() => hasStreamViewer(streamId)).toBe(true) + release() + // Stands in for the TTL running out, so the test does not wait it out. + await getRedisClient()?.del(`mothership_stream:${streamId}:viewer`) + // Longer than one refresh: a reader still refreshing would have re-marked itself by now. + await sleep(3_500) + + expect(await hasStreamViewer(streamId)).toBe(false) + }) +}) diff --git a/apps/sim/lib/mothership/request/session/viewer.ts b/apps/sim/lib/mothership/request/session/viewer.ts new file mode 100644 index 00000000000..95aa919fde7 --- /dev/null +++ b/apps/sim/lib/mothership/request/session/viewer.ts @@ -0,0 +1,64 @@ +import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import { getRedisClient } from '@/lib/core/config/redis' + +/** + * Which chat streams have a reconnected reader attached right now, on any instance. + * + * A chat view starts client-routed workflow tools from the stream it reads. The original send + * knows in-process whether its own response is still attached; a view that re-attached through + * the reconnect route (a reload, a chat switch back, another tab) may sit on another instance, + * so it marks itself here for as long as it stays attached. + */ + +const logger = createLogger('SessionStreamViewer') + +/** Long enough that one late refresh never makes an attached reader look gone. */ +const STREAM_VIEWER_TTL_MS = 10_000 +const STREAM_VIEWER_REFRESH_MS = 3_000 + +function getViewerKey(streamId: string) { + return `mothership_stream:${streamId}:viewer` +} + +async function markStreamViewer(streamId: string): Promise { + const redis = getRedisClient() + if (!redis) return + await redis.set(getViewerKey(streamId), '1', 'PX', STREAM_VIEWER_TTL_MS) +} + +/** + * Marks a reader attached to the stream until the returned release runs. The mark lapses on + * its own a few seconds after release, so a crashed reader never holds it. + */ +export function holdStreamViewer(streamId: string): () => void { + const refresh = () => { + markStreamViewer(streamId).catch((error) => { + logger.warn('Could not mark a chat stream reader attached', { + streamId, + error: toError(error).message, + }) + }) + } + refresh() + const interval = setInterval(refresh, STREAM_VIEWER_REFRESH_MS) + return () => clearInterval(interval) +} + +/** + * Whether a reconnected reader is attached to the stream. Answers true whenever it cannot + * tell (no Redis, a failed read), so an unknown never cuts a present view's pickup window. + */ +export async function hasStreamViewer(streamId: string): Promise { + const redis = getRedisClient() + if (!redis) return true + try { + return (await redis.exists(getViewerKey(streamId))) > 0 + } catch (error) { + logger.warn('Could not read whether a chat stream reader is attached', { + streamId, + error: toError(error).message, + }) + return true + } +} diff --git a/apps/sim/lib/mothership/request/types.ts b/apps/sim/lib/mothership/request/types.ts index 6792e5b560e..f37597a1942 100644 --- a/apps/sim/lib/mothership/request/types.ts +++ b/apps/sim/lib/mothership/request/types.ts @@ -290,6 +290,13 @@ export interface OrchestratorOptions { * `interactive`, which is a trust classification, not executor routing. */ clientToolPickupExpected?: boolean + /** + * Whether a chat view is reading this turn's stream right now: the send's own response, or a + * reader re-attached through the reconnect route. Answers true when it cannot tell. With no + * view, a workflow tool waits only a short grace for one to attach before running server-side. + * Absent on legs that cannot tell, which keep the full grace. + */ + hasClientViewer?: () => Promise /** * The turn's desktop claims local reads (`read_local_file`, user-local VFS reads) through * authorize before reading, so they are persisted pending and fail fast when nothing picks them From 234b10d83602b6660c778fd434fe21eb37548637 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 17 Nov 2026 20:54:08 -0800 Subject: [PATCH 2/2] fix(mothership): clear a reader's mark when it leaves and count it before its first event Each reconnect attachment now holds its own entry in a per-stream sorted set, removed on release, so leaving stops counting at once instead of lingering until the TTL. The route awaits the first mark before reading, and the Redis suite skips when no test Redis is configured locally. --- .../app/api/copilot/chat/stream/route.test.ts | 2 +- apps/sim/app/api/copilot/chat/stream/route.ts | 2 +- .../request/handlers/handlers.test.ts | 13 +++-- .../request/session/viewer.integration.ts | 40 ++++++++----- .../lib/mothership/request/session/viewer.ts | 56 ++++++++++++------- 5 files changed, 74 insertions(+), 39 deletions(-) diff --git a/apps/sim/app/api/copilot/chat/stream/route.test.ts b/apps/sim/app/api/copilot/chat/stream/route.test.ts index ced90c21241..06f25004aa4 100644 --- a/apps/sim/app/api/copilot/chat/stream/route.test.ts +++ b/apps/sim/app/api/copilot/chat/stream/route.test.ts @@ -31,7 +31,7 @@ vi.mock('@/lib/mothership/request/session', () => ({ readEvents, readFilePreviewSessions, findReplayGap, - holdStreamViewer: () => () => {}, + holdStreamViewer: async () => () => {}, readRingPosition: async () => ({ requestedAfterSeq: 0, oldestSeq: 0, latestSeq: 0 }), ringCanServe: () => true, replayGapTerminal: async () => ({ gapDetected: true, envelopes: [] }), diff --git a/apps/sim/app/api/copilot/chat/stream/route.ts b/apps/sim/app/api/copilot/chat/stream/route.ts index b2fa881dd66..d214fb471a2 100644 --- a/apps/sim/app/api/copilot/chat/stream/route.ts +++ b/apps/sim/app/api/copilot/chat/stream/route.ts @@ -399,7 +399,7 @@ async function handleResumeRequestBody({ } request.signal.addEventListener('abort', abortListener, { once: true }) // An attached reader is a chat view that starts this turn's workflow tools. - const releaseViewer = holdStreamViewer(streamId) + const releaseViewer = await holdStreamViewer(streamId) /** Delivers the ring's events after the cursor, or returns null if it trimmed the next one. */ const flushEvents = async (): Promise => { diff --git a/apps/sim/lib/mothership/request/handlers/handlers.test.ts b/apps/sim/lib/mothership/request/handlers/handlers.test.ts index 4c04613a6f0..1851585e9cb 100644 --- a/apps/sim/lib/mothership/request/handlers/handlers.test.ts +++ b/apps/sim/lib/mothership/request/handlers/handlers.test.ts @@ -9,6 +9,7 @@ import { mothershipClientToolWaiterMockFns, } from '@sim/testing/mocks/mothership-client-tool-waiter.mock' import { sleep } from '@sim/utils/helpers' +import { toRecord } from '@sim/utils/object' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { AsyncToolCallOwnershipError } from '@/lib/mothership/async-runs/errors' import { restoreStreamingContext } from '@/lib/mothership/request/context/restore' @@ -1366,10 +1367,14 @@ describe('sse-handlers tool lifecycle', () => { describe('workflow tool pickup grace', () => { /** Dispatches a workflow call whose browser wait only ends when the server takes it over. */ async function dispatchUnpickedWorkflowCall(toolCallId: string, hasClientViewer: boolean) { - waitForWorkflowToolCompletion.mockImplementationOnce( - ({ abortSignal }: { abortSignal?: AbortSignal }) => - new Promise((resolve) => abortSignal?.addEventListener('abort', () => resolve(null))) - ) + waitForWorkflowToolCompletion.mockImplementationOnce((params: unknown) => { + const abortSignal = toRecord(params).abortSignal + return new Promise((resolve) => { + if (abortSignal instanceof AbortSignal) { + abortSignal.addEventListener('abort', () => resolve(null)) + } + }) + }) claimWorkflowToolExecution.mockResolvedValueOnce({ toolCallId }) executeTool.mockResolvedValueOnce({ success: true, output: {} }) await sseHandlers.tool( diff --git a/apps/sim/lib/mothership/request/session/viewer.integration.ts b/apps/sim/lib/mothership/request/session/viewer.integration.ts index 42f4d6e6a8b..de2e43749ff 100644 --- a/apps/sim/lib/mothership/request/session/viewer.integration.ts +++ b/apps/sim/lib/mothership/request/session/viewer.integration.ts @@ -1,30 +1,32 @@ /** * Chat stream reader presence against real Redis: a reader attached on one instance must be - * visible to the instance dispatching that turn's workflow tools, and must stop counting once it - * leaves, so a turn nobody reads no longer waits out the full pickup grace. + * visible to the instance dispatching that turn's workflow tools, and must stop counting the + * moment it leaves, so a turn nobody reads no longer waits out the full pickup grace. */ import { afterAll, describe, expect, it, vi } from 'vitest' -await vi.hoisted(async () => { +const { redisUrl } = await vi.hoisted(async () => { const { readTestRedisUrl } = await import('@sim/db/testing/test-infrastructure') - process.env.REDIS_URL = readTestRedisUrl() + const url = readTestRedisUrl() + if (url) process.env.REDIS_URL = url + return { redisUrl: url } }) import { sleep } from '@sim/utils/helpers' import { generateId } from '@sim/utils/id' -import { closeRedisConnection, getRedisClient } from '@/lib/core/config/redis' +import { closeRedisConnection } from '@/lib/core/config/redis' import { hasStreamViewer, holdStreamViewer } from '@/lib/mothership/request/session/viewer' -describe('chat stream reader presence', () => { +describe.runIf(Boolean(redisUrl))('chat stream reader presence', () => { afterAll(async () => { await closeRedisConnection() }) - it('sees a reader attached to the stream', async () => { + it('counts a reader as attached once its hold resolves', async () => { const streamId = generateId() - const release = holdStreamViewer(streamId) + const release = await holdStreamViewer(streamId) try { - await expect.poll(() => hasStreamViewer(streamId)).toBe(true) + expect(await hasStreamViewer(streamId)).toBe(true) } finally { release() } @@ -34,13 +36,23 @@ describe('chat stream reader presence', () => { expect(await hasStreamViewer(generateId())).toBe(false) }) - it('never re-marks a reader after it leaves', async () => { + it('keeps counting a stream while another reader stays attached', async () => { + const streamId = generateId() + const leaving = await holdStreamViewer(streamId) + const staying = await holdStreamViewer(streamId) + try { + leaving() + await expect.poll(() => hasStreamViewer(streamId)).toBe(true) + } finally { + staying() + } + }) + + it('stops counting a reader as soon as it leaves, and never re-marks it', async () => { const streamId = generateId() - const release = holdStreamViewer(streamId) - await expect.poll(() => hasStreamViewer(streamId)).toBe(true) + const release = await holdStreamViewer(streamId) release() - // Stands in for the TTL running out, so the test does not wait it out. - await getRedisClient()?.del(`mothership_stream:${streamId}:viewer`) + await expect.poll(() => hasStreamViewer(streamId)).toBe(false) // Longer than one refresh: a reader still refreshing would have re-marked itself by now. await sleep(3_500) diff --git a/apps/sim/lib/mothership/request/session/viewer.ts b/apps/sim/lib/mothership/request/session/viewer.ts index 95aa919fde7..cba3e79b7c2 100644 --- a/apps/sim/lib/mothership/request/session/viewer.ts +++ b/apps/sim/lib/mothership/request/session/viewer.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' import { getRedisClient } from '@/lib/core/config/redis' /** @@ -8,7 +9,8 @@ import { getRedisClient } from '@/lib/core/config/redis' * A chat view starts client-routed workflow tools from the stream it reads. The original send * knows in-process whether its own response is still attached; a view that re-attached through * the reconnect route (a reload, a chat switch back, another tab) may sit on another instance, - * so it marks itself here for as long as it stays attached. + * so each attachment holds its own entry here, scored by when it lapses, for as long as it + * stays attached. Leaving removes the entry at once; a reader that dies without leaving lapses. */ const logger = createLogger('SessionStreamViewer') @@ -17,32 +19,48 @@ const logger = createLogger('SessionStreamViewer') const STREAM_VIEWER_TTL_MS = 10_000 const STREAM_VIEWER_REFRESH_MS = 3_000 -function getViewerKey(streamId: string) { - return `mothership_stream:${streamId}:viewer` +function getViewersKey(streamId: string) { + return `mothership_stream:${streamId}:viewers` } -async function markStreamViewer(streamId: string): Promise { +async function markStreamViewer(streamId: string, attachmentId: string): Promise { const redis = getRedisClient() if (!redis) return - await redis.set(getViewerKey(streamId), '1', 'PX', STREAM_VIEWER_TTL_MS) + const key = getViewersKey(streamId) + await redis + .multi() + .zadd(key, Date.now() + STREAM_VIEWER_TTL_MS, attachmentId) + .pexpire(key, STREAM_VIEWER_TTL_MS) + .exec() +} + +async function unmarkStreamViewer(streamId: string, attachmentId: string): Promise { + const redis = getRedisClient() + if (!redis) return + await redis.zrem(getViewersKey(streamId), attachmentId) +} + +function warnMarkFailed(streamId: string, error: unknown) { + logger.warn('Could not record a chat stream reader', { + streamId, + error: toError(error).message, + }) } /** - * Marks a reader attached to the stream until the returned release runs. The mark lapses on - * its own a few seconds after release, so a crashed reader never holds it. + * Records a reader attached to the stream until the returned release runs. Resolves once the + * first mark is written, so the reader counts before it reads its first event. */ -export function holdStreamViewer(streamId: string): () => void { - const refresh = () => { - markStreamViewer(streamId).catch((error) => { - logger.warn('Could not mark a chat stream reader attached', { - streamId, - error: toError(error).message, - }) - }) - } - refresh() +export async function holdStreamViewer(streamId: string): Promise<() => void> { + const attachmentId = generateId() + const refresh = () => + markStreamViewer(streamId, attachmentId).catch((error) => warnMarkFailed(streamId, error)) + await refresh() const interval = setInterval(refresh, STREAM_VIEWER_REFRESH_MS) - return () => clearInterval(interval) + return () => { + clearInterval(interval) + unmarkStreamViewer(streamId, attachmentId).catch((error) => warnMarkFailed(streamId, error)) + } } /** @@ -53,7 +71,7 @@ export async function hasStreamViewer(streamId: string): Promise { const redis = getRedisClient() if (!redis) return true try { - return (await redis.exists(getViewerKey(streamId))) > 0 + return (await redis.zcount(getViewersKey(streamId), Date.now(), '+inf')) > 0 } catch (error) { logger.warn('Could not read whether a chat stream reader is attached', { streamId,