Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions apps/sim/app/api/copilot/chat/stream/route.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ vi.mock('@/lib/mothership/request/session', () => ({
readEvents,
readFilePreviewSessions,
findReplayGap,
holdStreamViewer: async () => () => {},
readRingPosition: async () => ({ requestedAfterSeq: 0, oldestSeq: 0, latestSeq: 0 }),
ringCanServe: () => true,
replayGapTerminal: async () => ({ gapDetected: true, envelopes: [] }),
Expand Down
4 changes: 4 additions & 0 deletions apps/sim/app/api/copilot/chat/stream/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import {
encodeSSEEnvelope,
findReplayGap,
forwardRunReplay,
holdStreamViewer,
isTerminalStreamStatus,
openRunReplay,
RunReplayUnavailableError,
Expand Down Expand Up @@ -397,6 +398,8 @@ async function handleResumeRequestBody({
controllerClosed = true
}
request.signal.addEventListener('abort', abortListener, { once: true })
// An attached reader is a chat view that starts this turn's workflow tools.
const releaseViewer = await holdStreamViewer(streamId)

/** Delivers the ring's events after the cursor, or returns null if it trimmed the next one. */
const flushEvents = async (): Promise<number | null> => {
Expand Down Expand Up @@ -591,6 +594,7 @@ async function handleResumeRequestBody({
}
markSpanForError(rootSpan, error)
} finally {
releaseViewer()
request.signal.removeEventListener('abort', abortListener)
// Read before closing: closing the controller here is this route ending, not the client.
const clientDisconnected = controllerClosed
Expand Down
7 changes: 7 additions & 0 deletions apps/sim/lib/mothership/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,13 @@ export const CHAT_RUN_DEADLINE_MS = 3_600_000
*/
export const COPILOT_WORKFLOW_TOOL_CLIENT_GRACE_MS = 30_000

/**
* The workflow grace when no chat view is reading the turn's stream at dispatch: a tab closed,
* a switch to another chat or page. It only bridges a view that is re-attaching right then (a
* reload, a reconnect after a network drop), which reads the call within a reconnect poll.
*/
export const COPILOT_WORKFLOW_TOOL_NO_VIEWER_GRACE_MS = 5_000

/**
* How long a desktop tool call (browser, terminal, local import) waits for the desktop app to
* claim it before it fails as never started.
Expand Down
74 changes: 74 additions & 0 deletions apps/sim/lib/mothership/request/handlers/handlers.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import {
mothershipClientToolWaiterMockFns,
} from '@sim/testing/mocks/mothership-client-tool-waiter.mock'
import { sleep } from '@sim/utils/helpers'
import { toRecord } from '@sim/utils/object'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { AsyncToolCallOwnershipError } from '@/lib/mothership/async-runs/errors'
import { restoreStreamingContext } from '@/lib/mothership/request/context/restore'
Expand Down Expand Up @@ -1363,6 +1364,79 @@ describe('sse-handlers tool lifecycle', () => {
expect(workflowResults[0].payload.executor).toBe(MothershipStreamV1ToolExecutor.sim)
})

describe('workflow tool pickup grace', () => {
/** Dispatches a workflow call whose browser wait only ends when the server takes it over. */
async function dispatchUnpickedWorkflowCall(toolCallId: string, hasClientViewer: boolean) {
waitForWorkflowToolCompletion.mockImplementationOnce((params: unknown) => {
const abortSignal = toRecord(params).abortSignal
return new Promise((resolve) => {
if (abortSignal instanceof AbortSignal) {
abortSignal.addEventListener('abort', () => resolve(null))
}
})
})
claimWorkflowToolExecution.mockResolvedValueOnce({ toolCallId })
executeTool.mockResolvedValueOnce({ success: true, output: {} })
await sseHandlers.tool(
{
type: MothershipStreamV1EventType.tool,
payload: {
toolCallId,
toolName: 'run_workflow',
arguments: { workflowId: 'workflow-1' },
executor: MothershipStreamV1ToolExecutor.client,
mode: MothershipStreamV1ToolMode.async,
phase: MothershipStreamV1ToolPhase.call,
},
} satisfies StreamEvent,
context,
execContext,
{ interactive: true, hasClientViewer: async () => hasClientViewer }
)
}

it('runs a call no chat view is reading on the server after the short grace', async () => {
vi.useFakeTimers()
try {
await dispatchUnpickedWorkflowCall('tool-no-viewer', false)
await vi.advanceTimersByTimeAsync(5_000)
expect(claimWorkflowToolExecution).toHaveBeenCalledWith(
'tool-no-viewer',
expect.any(String),
'sim'
)

await Promise.allSettled(context.pendingToolPromises.values())
expect(executeTool.mock.calls.at(-1)?.[0]).toBe('run_workflow')
Comment thread
waleedlatif1 marked this conversation as resolved.
Resolved
} finally {
vi.useRealTimers()
}
})

it('keeps waiting the full grace for a call a chat view is reading', async () => {
vi.useFakeTimers()
try {
await dispatchUnpickedWorkflowCall('tool-viewer', true)
await vi.advanceTimersByTimeAsync(29_000)
expect(claimWorkflowToolExecution).not.toHaveBeenCalledWith(
'tool-viewer',
expect.any(String),
'sim'
)

await vi.advanceTimersByTimeAsync(1_000)
await Promise.allSettled(context.pendingToolPromises.values())
expect(claimWorkflowToolExecution).toHaveBeenCalledWith(
'tool-viewer',
expect.any(String),
'sim'
)
} finally {
vi.useRealTimers()
}
})
})

it('claims and runs the same workflow when the call omits an explicit workflowId', async () => {
// The waiter resolves the target via resolveWorkflowToolTargetId(args, ctx)
// while the handler resolves it as params.workflowId || context.workflowId.
Expand Down
23 changes: 15 additions & 8 deletions apps/sim/lib/mothership/request/handlers/tool.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import { getAsyncToolCall, upsertAsyncToolCall } from '@/lib/mothership/async-ru
import {
CLIENT_TOOL_RESULT_TIMEOUT_MS,
COPILOT_WORKFLOW_TOOL_CLIENT_GRACE_MS,
COPILOT_WORKFLOW_TOOL_NO_VIEWER_GRACE_MS,
DESKTOP_TOOL_PICKUP_GRACE_MS,
} from '@/lib/mothership/constants'
import {
Expand Down Expand Up @@ -239,6 +240,19 @@ function desktopClaimsLocalReads(
return options?.desktopClaimsLocalReads === true || desktopDeviceId !== null
}

/**
* How long a workflow tool waits for a chat view to pick it up before the server runs it. The
* caller declared its executors at turn setup (ChatRequest.clientCapabilities): no pickup
* capability means no wait. With the capability, a turn no view is reading only waits for one
* that is re-attaching. The claim arbitration is the same either way.
*/
async function workflowToolPickupGraceMs(options: OrchestratorOptions): Promise<number> {
if (options.clientToolPickupExpected === false) return 0
if (!options.hasClientViewer) return COPILOT_WORKFLOW_TOOL_CLIENT_GRACE_MS
const viewer = await options.hasClientViewer().catch(() => true)
return viewer ? COPILOT_WORKFLOW_TOOL_CLIENT_GRACE_MS : COPILOT_WORKFLOW_TOOL_NO_VIEWER_GRACE_MS
}

/**
* Upsert the durable `async_tool_calls` row before the authoritative tool-call
* SSE frame is forwarded to the client, so `/api/copilot/confirm` and
Expand Down Expand Up @@ -1113,14 +1127,7 @@ async function dispatchToolExecution(
workflowId: resolveWorkflowToolTargetId(args, execContext.workflowId),
timeoutMs,
deadlineAt: toolCall.deadlineAt,
// The caller declared its executors at turn setup (ChatRequest.clientCapabilities):
// no pickup capability → zero grace, the race resolves straight to the server
// claim. Same claim arbitration either way — the grace only encodes how long a
// client might plausibly appear.
graceMs:
options.clientToolPickupExpected === false
? 0
: COPILOT_WORKFLOW_TOOL_CLIENT_GRACE_MS,
graceMs: await workflowToolPickupGraceMs(options),
...(toolCall.waitTimeoutMs !== undefined
? { waitTimeoutMs: toolCall.waitTimeoutMs }
: {}),
Expand Down
3 changes: 3 additions & 0 deletions apps/sim/lib/mothership/request/lifecycle/start.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ import { type CopilotLifecycleOutcome, startCopilotOtelRoot } from '@/lib/mother
import {
cleanupAbortMarker,
clearFilePreviewSessions,
hasStreamViewer,
isExplicitStopReason,
registerActiveStream,
releasePendingChatStream,
Expand Down Expand Up @@ -427,6 +428,8 @@ export function createSSEStream(params: StreamingOrchestrationParams): ReadableS

const result = await runCopilotLifecycle(requestPayload, {
...orchestrateOptions,
hasClientViewer: async () =>
!publisher.clientDisconnected || (await hasStreamViewer(streamId)),
executionId,
runId,
trace: collector,
Expand Down
1 change: 1 addition & 0 deletions apps/sim/lib/mothership/request/session/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -83,4 +83,5 @@ export {
} from './run-replay'
export { encodeSSEEnvelope, SSE_RESPONSE_HEADERS } from './sse'
export type { StreamBatchEvent } from './types'
export { hasStreamViewer, holdStreamViewer } from './viewer'
export { StreamWriter, type StreamWriterOptions } from './writer'
61 changes: 61 additions & 0 deletions apps/sim/lib/mothership/request/session/viewer.integration.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
/**
* Chat stream reader presence against real Redis: a reader attached on one instance must be
* visible to the instance dispatching that turn's workflow tools, and must stop counting the
* moment it leaves, so a turn nobody reads no longer waits out the full pickup grace.
*/
import { afterAll, describe, expect, it, vi } from 'vitest'

const { redisUrl } = await vi.hoisted(async () => {
const { readTestRedisUrl } = await import('@sim/db/testing/test-infrastructure')
const url = readTestRedisUrl()
if (url) process.env.REDIS_URL = url
return { redisUrl: url }
})

import { sleep } from '@sim/utils/helpers'
import { generateId } from '@sim/utils/id'
import { closeRedisConnection } from '@/lib/core/config/redis'
import { hasStreamViewer, holdStreamViewer } from '@/lib/mothership/request/session/viewer'

describe.runIf(Boolean(redisUrl))('chat stream reader presence', () => {
afterAll(async () => {
await closeRedisConnection()
})

it('counts a reader as attached once its hold resolves', async () => {
const streamId = generateId()
const release = await holdStreamViewer(streamId)
try {
expect(await hasStreamViewer(streamId)).toBe(true)
} finally {
release()
}
})

it('sees no reader on a stream nobody attached to', async () => {
expect(await hasStreamViewer(generateId())).toBe(false)
})

it('keeps counting a stream while another reader stays attached', async () => {
const streamId = generateId()
const leaving = await holdStreamViewer(streamId)
const staying = await holdStreamViewer(streamId)
try {
leaving()
await expect.poll(() => hasStreamViewer(streamId)).toBe(true)
} finally {
staying()
}
})

it('stops counting a reader as soon as it leaves, and never re-marks it', async () => {
const streamId = generateId()
const release = await holdStreamViewer(streamId)
release()
await expect.poll(() => hasStreamViewer(streamId)).toBe(false)
// Longer than one refresh: a reader still refreshing would have re-marked itself by now.
await sleep(3_500)

expect(await hasStreamViewer(streamId)).toBe(false)
})
})
82 changes: 82 additions & 0 deletions apps/sim/lib/mothership/request/session/viewer.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
import { createLogger } from '@sim/logger'
import { toError } from '@sim/utils/errors'
import { generateId } from '@sim/utils/id'
import { getRedisClient } from '@/lib/core/config/redis'

/**
* Which chat streams have a reconnected reader attached right now, on any instance.
*
* A chat view starts client-routed workflow tools from the stream it reads. The original send
* knows in-process whether its own response is still attached; a view that re-attached through
* the reconnect route (a reload, a chat switch back, another tab) may sit on another instance,
* so each attachment holds its own entry here, scored by when it lapses, for as long as it
* stays attached. Leaving removes the entry at once; a reader that dies without leaving lapses.
*/

const logger = createLogger('SessionStreamViewer')

/** Long enough that one late refresh never makes an attached reader look gone. */
const STREAM_VIEWER_TTL_MS = 10_000
const STREAM_VIEWER_REFRESH_MS = 3_000

function getViewersKey(streamId: string) {
return `mothership_stream:${streamId}:viewers`
}

async function markStreamViewer(streamId: string, attachmentId: string): Promise<void> {
const redis = getRedisClient()
if (!redis) return
const key = getViewersKey(streamId)
await redis
.multi()
.zadd(key, Date.now() + STREAM_VIEWER_TTL_MS, attachmentId)
.pexpire(key, STREAM_VIEWER_TTL_MS)
.exec()
}

async function unmarkStreamViewer(streamId: string, attachmentId: string): Promise<void> {
const redis = getRedisClient()
if (!redis) return
await redis.zrem(getViewersKey(streamId), attachmentId)
}

function warnMarkFailed(streamId: string, error: unknown) {
logger.warn('Could not record a chat stream reader', {
streamId,
error: toError(error).message,
})
}

/**
* Records a reader attached to the stream until the returned release runs. Resolves once the
* first mark is written, so the reader counts before it reads its first event.
*/
export async function holdStreamViewer(streamId: string): Promise<() => void> {
const attachmentId = generateId()
const refresh = () =>
markStreamViewer(streamId, attachmentId).catch((error) => warnMarkFailed(streamId, error))
await refresh()
const interval = setInterval(refresh, STREAM_VIEWER_REFRESH_MS)
return () => {
clearInterval(interval)
unmarkStreamViewer(streamId, attachmentId).catch((error) => warnMarkFailed(streamId, error))
}
}

/**
* Whether a reconnected reader is attached to the stream. Answers true whenever it cannot
* tell (no Redis, a failed read), so an unknown never cuts a present view's pickup window.
*/
export async function hasStreamViewer(streamId: string): Promise<boolean> {
const redis = getRedisClient()
if (!redis) return true
try {
return (await redis.zcount(getViewersKey(streamId), Date.now(), '+inf')) > 0
} catch (error) {
logger.warn('Could not read whether a chat stream reader is attached', {
streamId,
error: toError(error).message,
})
return true
}
}
7 changes: 7 additions & 0 deletions apps/sim/lib/mothership/request/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,13 @@ export interface OrchestratorOptions {
* `interactive`, which is a trust classification, not executor routing.
*/
clientToolPickupExpected?: boolean
/**
* Whether a chat view is reading this turn's stream right now: the send's own response, or a
* reader re-attached through the reconnect route. Answers true when it cannot tell. With no
* view, a workflow tool waits only a short grace for one to attach before running server-side.
* Absent on legs that cannot tell, which keep the full grace.
*/
hasClientViewer?: () => Promise<boolean>
/**
* The turn's desktop claims local reads (`read_local_file`, user-local VFS reads) through
* authorize before reading, so they are persisted pending and fail fast when nothing picks them
Expand Down
Loading