Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 34 additions & 4 deletions apps/sim/lib/mothership/request/handlers/handlers.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1366,7 +1366,10 @@ describe('sse-handlers tool lifecycle', () => {

describe('workflow tool pickup grace', () => {
/** Dispatches a workflow call whose browser wait only ends when the server takes it over. */
async function dispatchUnpickedWorkflowCall(toolCallId: string, hasClientViewer: boolean) {
async function dispatchUnpickedWorkflowCall(
toolCallId: string,
hasClientViewer: () => Promise<boolean>
) {
waitForWorkflowToolCompletion.mockImplementationOnce((params: unknown) => {
const abortSignal = toRecord(params).abortSignal
return new Promise((resolve) => {
Expand All @@ -1391,14 +1394,14 @@ describe('sse-handlers tool lifecycle', () => {
} satisfies StreamEvent,
context,
execContext,
{ interactive: true, hasClientViewer: async () => hasClientViewer }
{ interactive: true, hasClientViewer }
)
}

it('runs a call no chat view is reading on the server after the short grace', async () => {
vi.useFakeTimers()
try {
await dispatchUnpickedWorkflowCall('tool-no-viewer', false)
await dispatchUnpickedWorkflowCall('tool-no-viewer', async () => false)
await vi.advanceTimersByTimeAsync(5_000)
expect(claimWorkflowToolExecution).toHaveBeenCalledWith(
'tool-no-viewer',
Expand All @@ -1413,10 +1416,37 @@ describe('sse-handlers tool lifecycle', () => {
}
})

it('gives a view that re-attached during the short grace the rest of the full grace', async () => {
vi.useFakeTimers()
try {
let attached = false
await dispatchUnpickedWorkflowCall('tool-reattached', async () => attached)
// A reload: the send's response dropped, and the view re-attaches two seconds later.
await vi.advanceTimersByTimeAsync(2_000)
attached = true
await vi.advanceTimersByTimeAsync(27_000)
expect(claimWorkflowToolExecution).not.toHaveBeenCalledWith(
'tool-reattached',
expect.any(String),
'sim'
)

await vi.advanceTimersByTimeAsync(1_000)
await Promise.allSettled(context.pendingToolPromises.values())
expect(claimWorkflowToolExecution).toHaveBeenCalledWith(
'tool-reattached',
expect.any(String),
'sim'
)
} finally {
vi.useRealTimers()
}
})

it('keeps waiting the full grace for a call a chat view is reading', async () => {
vi.useFakeTimers()
try {
await dispatchUnpickedWorkflowCall('tool-viewer', true)
await dispatchUnpickedWorkflowCall('tool-viewer', async () => true)
await vi.advanceTimersByTimeAsync(29_000)
expect(claimWorkflowToolExecution).not.toHaveBeenCalledWith(
'tool-viewer',
Expand Down
27 changes: 19 additions & 8 deletions apps/sim/lib/mothership/request/handlers/tool.ts
Original file line number Diff line number Diff line change
Expand Up @@ -243,14 +243,25 @@ function desktopClaimsLocalReads(
/**
* How long a workflow tool waits for a chat view to pick it up before the server runs it. The
* caller declared its executors at turn setup (ChatRequest.clientCapabilities): no pickup
* capability means no wait. With the capability, a turn no view is reading only waits for one
* that is re-attaching. The claim arbitration is the same either way.
* capability means no wait. With the capability, a turn no view is reading waits only for one
* that is re-attaching, and a view that attached by the end of that short wait (a reload) gets
* the rest of the full grace. The claim arbitration is the same either way.
*/
async function workflowToolPickupGraceMs(options: OrchestratorOptions): Promise<number> {
if (options.clientToolPickupExpected === false) return 0
if (!options.hasClientViewer) return COPILOT_WORKFLOW_TOOL_CLIENT_GRACE_MS
const viewer = await options.hasClientViewer().catch(() => true)
return viewer ? COPILOT_WORKFLOW_TOOL_CLIENT_GRACE_MS : COPILOT_WORKFLOW_TOOL_NO_VIEWER_GRACE_MS
async function workflowToolPickupGrace(
options: OrchestratorOptions
): Promise<{ graceMs: number; extendGrace?: () => Promise<number> }> {
if (options.clientToolPickupExpected === false) return { graceMs: 0 }
const hasClientViewer = options.hasClientViewer
if (!hasClientViewer) return { graceMs: COPILOT_WORKFLOW_TOOL_CLIENT_GRACE_MS }
const viewing = () => hasClientViewer().catch(() => true)
if (await viewing()) return { graceMs: COPILOT_WORKFLOW_TOOL_CLIENT_GRACE_MS }
return {
graceMs: COPILOT_WORKFLOW_TOOL_NO_VIEWER_GRACE_MS,
extendGrace: async () =>
(await viewing())
? COPILOT_WORKFLOW_TOOL_CLIENT_GRACE_MS - COPILOT_WORKFLOW_TOOL_NO_VIEWER_GRACE_MS
: 0,
}
}

/**
Expand Down Expand Up @@ -1127,7 +1138,7 @@ async function dispatchToolExecution(
workflowId: resolveWorkflowToolTargetId(args, execContext.workflowId),
timeoutMs,
deadlineAt: toolCall.deadlineAt,
graceMs: await workflowToolPickupGraceMs(options),
...(await workflowToolPickupGrace(options)),
...(toolCall.waitTimeoutMs !== undefined
? { waitTimeoutMs: toolCall.waitTimeoutMs }
: {}),
Expand Down
40 changes: 40 additions & 0 deletions apps/sim/lib/mothership/request/lifecycle/start.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { propagation, trace } from '@opentelemetry/api'
import { W3CTraceContextPropagator } from '@opentelemetry/core'
import { TracerProvider } from '@opentelemetry/sdk-trace'
import { dbChainMockFns, resetDbChainMock, resetEnvFlagsMock, setEnvFlags } from '@sim/testing'
import { createDeferred } from '@sim/testing/helpers/deferred'
import {
billingAttributionMock,
billingAttributionMockFns,
Expand Down Expand Up @@ -42,6 +43,7 @@ const {
releasePendingChatStream,
unregisterActiveStream,
buildChatTitleContext,
hasStreamViewer,
} = vi.hoisted(() => ({
runCopilotLifecycle: vi.fn(),
resetBuffer: vi.fn(),
Expand All @@ -56,6 +58,7 @@ const {
releasePendingChatStream: vi.fn(),
unregisterActiveStream: vi.fn(),
buildChatTitleContext: vi.fn().mockResolvedValue(undefined),
hasStreamViewer: vi.fn(),
}))

const BILLING_ATTRIBUTION = {
Expand Down Expand Up @@ -108,6 +111,7 @@ vi.mock('@/lib/mothership/request/session', () => ({
releasePendingChatStream,
registerActiveStream,
unregisterActiveStream,
hasStreamViewer,
startAbortPoller: vi.fn().mockReturnValue(setInterval(() => {}, 999999)),
isExplicitStopReason: (reason: unknown) => reason === 'user_stop:abortActiveStream',
SSE_RESPONSE_HEADERS: {},
Expand Down Expand Up @@ -405,6 +409,42 @@ describe('createSSEStream terminal error handling', () => {
}
)

it('counts the send as viewed while its response is attached, and asks for a reader once it is not', async () => {
const lifecycleDone = createDeferred<void>()
let hasClientViewer: (() => Promise<boolean>) | undefined
runCopilotLifecycle.mockImplementationOnce(
async (_payload: unknown, options: { hasClientViewer?: () => Promise<boolean> }) => {
hasClientViewer = options.hasClientViewer
await lifecycleDone.promise
return { success: true, content: '', contentBlocks: [], toolCalls: [] }
}
)
hasStreamViewer.mockResolvedValue(false)
const stream = createSSEStream({
requestPayload: { message: 'hello' },
userId: 'user-1',
streamId: 'stream-viewer',
executionId: 'exec-viewer',
runId: 'run-viewer',
currentChat: null,
message: 'hello',
titleModel: 'gpt-5.4',
requestId: 'req-viewer',
orchestrateOptions: { userId: 'user-1' },
})
await vi.waitFor(() => expect(hasClientViewer).toBeDefined())

expect(await hasClientViewer?.()).toBe(true)

await stream.cancel()
expect(await hasClientViewer?.()).toBe(false)
hasStreamViewer.mockResolvedValueOnce(true)
expect(await hasClientViewer?.()).toBe(true)
expect(hasStreamViewer).toHaveBeenCalledWith('stream-viewer')

lifecycleDone.resolve()
})

it('starts the agent while title metadata is pending and forwards recovered inventory', async () => {
let resolveContext!: (value: string) => void
buildChatTitleContext.mockImplementationOnce(
Expand Down
5 changes: 4 additions & 1 deletion apps/sim/lib/mothership/request/session/viewer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,12 @@ async function markStreamViewer(streamId: string, attachmentId: string): Promise
const redis = getRedisClient()
if (!redis) return
const key = getViewersKey(streamId)
const now = Date.now()
// Drops readers that died without leaving, which a live reader's refresh keeps the key past.
await redis
.multi()
.zadd(key, Date.now() + STREAM_VIEWER_TTL_MS, attachmentId)
.zremrangebyscore(key, '-inf', now)
.zadd(key, now + STREAM_VIEWER_TTL_MS, attachmentId)
.pexpire(key, STREAM_VIEWER_TTL_MS)
.exec()
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,40 @@ describe('raceWorkflowToolClientPickup', () => {
expect(params.runOnServer).not.toHaveBeenCalled()
})

it('returns a browser result that lands while the grace extension is still being checked', async () => {
let finish!: (completion: unknown) => void
waitForWorkflowToolCompletion.mockImplementation(
() => new Promise((resolve) => (finish = resolve))
)
const params = baseParams({ graceMs: 5_000, extendGrace: () => new Promise(() => {}) })

const promise = raceWorkflowToolClientPickup(params as never)
await vi.advanceTimersByTimeAsync(5_000)
finish({ status: 'success', data: { ok: true } })
const outcome = await promise

expect(outcome.winner).toBe('client')
expect(outcome.completion).toEqual({ status: 'success', data: { ok: true } })
expect(claimWorkflowToolExecution).not.toHaveBeenCalled()
})

it('counts a slow extension check toward the extension', async () => {
pendingUntilAborted()
claimWorkflowToolExecution.mockResolvedValue({ toolCallId: 'tool-1' })
const params = baseParams({
graceMs: 5_000,
extendGrace: () => new Promise((resolve) => setTimeout(() => resolve(25_000), 4_000)),
})

const promise = raceWorkflowToolClientPickup(params as never)
await vi.advanceTimersByTimeAsync(29_000)
expect(claimWorkflowToolExecution).toHaveBeenCalledTimes(0)
await vi.advanceTimersByTimeAsync(1_000)
await promise

expect(claimWorkflowToolExecution).toHaveBeenCalledTimes(1)
})

it('runs the tool server-side when the grace elapses and the claim is won', async () => {
pendingUntilAborted()
claimWorkflowToolExecution.mockResolvedValue({ toolCallId: 'tool-1' })
Expand Down
32 changes: 28 additions & 4 deletions apps/sim/lib/mothership/request/tools/workflow-client-fallback.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,11 @@ interface RaceWorkflowToolClientPickupParams {
/** When the call's deadline passes: past it the server does not take the call over. */
deadlineAt?: number
graceMs: number
/**
* Asked once when the grace ends with no result: how much longer a view that attached during
* the grace should get (0 for none). Bounded by `timeoutMs` like the grace itself.
*/
extendGrace?: () => Promise<number>
/**
* How long to wait for a browser's run before settling the call still running (the frame's
* `waitTimeoutMs`); the browser's run goes on. Undefined waits until it settles.
Expand Down Expand Up @@ -191,10 +196,29 @@ export async function raceWorkflowToolClientPickup(
// would outlive the wait it is supposed to bound.
const effectiveGraceMs = Math.min(graceMs, timeoutMs)

const first = await Promise.race([
clientWait.then((completion) => ({ kind: 'client' as const, completion })),
sleep(effectiveGraceMs).then(() => ({ kind: 'grace' as const })),
])
const clientDone = clientWait.then((completion) => ({ kind: 'client' as const, completion }))
const raceGrace = (ms: number) =>
Promise.race([clientDone, sleep(ms).then(() => ({ kind: 'grace' as const }))])

let first = await raceGrace(effectiveGraceMs)
if (first.kind === 'grace' && params.extendGrace) {
// The check counts toward the extension, and a result that lands during it wins at once.
const checkStartedAt = Date.now()
const extension = await Promise.race([
clientDone,
params
.extendGrace()
.catch(() => 0)
.then((ms) => ({ kind: 'extend' as const, ms })),
])
if (extension.kind === 'client') {
first = extension
} else {
const extraMs =
Math.min(extension.ms, timeoutMs - effectiveGraceMs) - (Date.now() - checkStartedAt)
if (extraMs > 0) first = await raceGrace(extraMs)
}
}
Comment thread
waleedlatif1 marked this conversation as resolved.
Resolved

// A non-null client result inside the grace window is the normal path.
// A null one means the wait itself already expired (timeoutMs <= graceMs), so
Expand Down
Loading