Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -482,6 +482,10 @@ describe('POST /api/v2/workflows/[workflowId]/execute', () => {
blockName: 'Send Email',
blockType: 'gmail',
})
// An agent reads the head of an oversized run, so the error precedes the unbounded output.
const keys = Object.keys(body.data)
expect(keys.indexOf('error')).toBeLessThan(keys.indexOf('output'))
expect(keys.indexOf('durationMs')).toBeLessThan(keys.indexOf('output'))
})

it('queues async runs and returns a 202 receipt with the v2 runs statusUrl', async () => {
Expand Down
9 changes: 7 additions & 2 deletions apps/sim/app/api/v2/workflows/[workflowId]/execute/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -123,18 +123,23 @@ function encodeNdjson(value: unknown): Uint8Array {
return ndjsonEncoder.encode(`${JSON.stringify(value)}\n`)
}

/**
* The run resource, its bounded fields ahead of `output` and `blockOutputs`: a reader that keeps
* only the head of an oversized response (an agent's output window) still sees the status, the
* failing block and its error, and the duration.
*/
function presentRun(result: ExecuteWorkflowServiceRun) {
return {
runId: result.executionId,
workflowId: result.workflowId,
status: result.status,
output: result.output ?? null,
blockOutputs: result.blockOutputs ?? null,
error: result.error,
startedAt: result.startedAt,
endedAt: result.endedAt,
durationMs: result.durationMs,
executionTimeoutSeconds: executionTimeoutSeconds(result.executionTimeoutMs),
output: result.output ?? null,
blockOutputs: result.blockOutputs ?? null,
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -111,13 +111,13 @@ export const POST = withRouteHandler(
runId: result.executionId,
workflowId,
status: result.status as 'completed' | 'failed' | 'paused' | 'cancelled',
output: result.output ?? null,
// Resume has no request body to name selectors in, so selection never applies here.
blockOutputs: null,
error: result.structuredError,
startedAt: result.metadata?.startTime,
endedAt: result.metadata?.endTime,
durationMs: result.metadata?.duration,
output: result.output ?? null,
// Resume has no request body to name selectors in, so selection never applies here.
blockOutputs: null,
},
{ headers }
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -148,21 +148,91 @@ describe('run_workflow model-facing result budget', () => {
expect(presented.find((log) => log.blockName === 'Query 1')?.output).toEqual(logs[1].output)
})

/** The final output is what the run was for, so it is never compacted and needs the headroom. */
it('leaves room for a large final output beside the bounded logs', async () => {
const finalOutput = { rows: tableRows(4_800, 3_200_000) }
/**
* The agent stores a result past its window behind a paged read, which costs it a round to learn
* whether the run passed. Block outputs give way first, so the verdict, the failing block, and a
* final output that fits all arrive inline.
*/
it('fits a failed run with bulky block outputs inside the agent window', async () => {
const logs = [
...traceShapedLogs().slice(0, 3),
{ blockId: 'send', blockName: 'Send', success: false, error: 'SMTP refused', output: {} },
]
const finalOutput = { summary: 'partial report', rowCount: 1_882 }
mocks.executeWorkflowUseCase.mockResolvedValue({
success: true,
success: false,
output: finalOutput,
logs: traceShapedLogs(),
logs,
metadata: { executionId: EXECUTION_ID },
})

const settled = await executeRunWorkflow({ workflowId: 'wf-1' }, context)
const projection = inspectToolResultForCopilot(settled, registry, 'run_workflow')

expect(projection.safe).toBe(true)
expect((projection.result.output as Record<string, unknown>).output).toEqual(finalOutput)
expect(JSON.stringify(projection.result).length).toBeLessThanOrEqual(24_000)
expect(projection.result.error).toBe('Send: SMTP refused')
const output = projection.result.output as Record<string, unknown>
expect(output.output).toEqual(finalOutput)
const presented = output.logs as Array<Record<string, unknown>>
expect(presented.map((log) => log.blockName)).toEqual(logs.map((log) => log.blockName))
expect(presented[3]).toMatchObject({ success: false, error: 'SMTP refused' })
})

/** A short secret under a long name grows when projected, so the window is measured after it. */
it('fits the window as projected when short secrets expand into longer names', async () => {
const password = 'pw-12345'
const shortSecret = new ResolvedSecretTraceRegistry([
{ name: 'DATABASE_PASSWORD', plaintext: password, encryptedValue: 'ciphertext' },
])
shortSecret.recordResolved('DATABASE_PASSWORD', password, { propagated: true })
mocks.executeWorkflowUseCase.mockResolvedValue({
success: true,
output: { done: true },
logs: [
{
blockId: 'echo',
blockName: 'Echo',
success: true,
// About 14K characters raw, over 31K once each copy becomes {{DATABASE_PASSWORD}}.
output: { tokens: Array.from({ length: 1_300 }, () => password) },
},
],
metadata: { executionId: EXECUTION_ID },
})

const settled = await executeRunWorkflow({ workflowId: 'wf-1' }, callContext(shortSecret))
const projection = inspectToolResultForCopilot(settled, shortSecret, 'run_workflow')

expect(projection.safe).toBe(true)
expect(JSON.stringify(projection.result.output).length).toBeLessThanOrEqual(16_000)
const output = projection.result.output as { output: unknown; logs: Array<{ output: unknown }> }
expect(output.output).toEqual({ done: true })
expect(output.logs[0]?.output).toEqual(expect.stringContaining(`logs get ${EXECUTION_ID}`))
})

/** Many small block entries leave less room than a full preview, so the preview shrinks to fit. */
it('shrinks the final output preview to the room the block entries leave', async () => {
const logs = Array.from({ length: 150 }, (_, index) => ({
blockId: `step-${index}`,
blockName: `Step ${index} of the nightly import`,
success: true,
output: { ok: true },
}))
mocks.executeWorkflowUseCase.mockResolvedValue({
success: true,
output: { report: 'z'.repeat(50_000) },
logs,
metadata: { executionId: EXECUTION_ID },
})
const inactive = secretRegistry({ active: false })

const settled = await executeRunWorkflow({ workflowId: 'wf-1' }, callContext(inactive))
const output = settled.output as { output: unknown; logs: unknown[] }

expect(JSON.stringify({ output }).length).toBeLessThanOrEqual(16_000)
expect(output.logs).toEqual(logs)
expect(output.output).toEqual(expect.stringMatching(/^\{"report":"z+ …\[final output cut at/))
})

/** Narrow rows reach the traversal cap while staying far under every byte budget. */
Expand All @@ -186,7 +256,9 @@ describe('run_workflow model-facing result budget', () => {
const projection = inspectToolResultForCopilot(settled, registry, 'run_workflow')

expect(projection.safe).toBe(true)
expect((projection.result.output as Record<string, unknown>).output).toEqual(finalOutput)
expect((projection.result.output as Record<string, unknown>).output).toEqual(
expect.stringContaining(`logs get ${EXECUTION_ID} --trace`)
)
})

it('returns selected values in full, bypassing the log budget', async () => {
Expand Down Expand Up @@ -343,11 +415,10 @@ describe('run_workflow model-facing result budget', () => {
})

/**
* Without an active secret the projection passes JSON through under its byte cap alone, so
* nothing is bounded: a lifted output is the whole point of run_block and reaches the worker in
* full, which spills an oversized one to storage for the model to read.
* Without an active secret a cut cannot strand a secret fragment, so an oversized lifted output
* keeps its head inline as a preview and names where the rest is.
*/
it('returns a large lifted output and its logs in full when no secret is active', async () => {
it('previews an oversized lifted output when no secret is active', async () => {
const rows = wideRows()
mocks.executeWorkflowUseCase.mockResolvedValue({
success: true,
Expand All @@ -368,52 +439,35 @@ describe('run_workflow model-facing result budget', () => {

expect(projection.safe).toBe(true)
const output = projection.result.output as Record<string, unknown>
expect(output.output).toEqual({ rows })
expect((output.logs as Array<Record<string, unknown>>)[1]?.output).toEqual({ rows })
expect(JSON.stringify(output).length).toBeLessThanOrEqual(16_000)
expect(output.output).toEqual(expect.stringMatching(/^\{"rows":\[\{"c0":"r0"/))
expect(output.output).toEqual(expect.stringContaining(`logs get ${EXECUTION_ID} --trace`))
expect(output.outputFrom).toEqual({ blockId: 'query', blockName: 'Query' })
expect((output.logs as Array<Record<string, unknown>>)[0]?.output).toEqual({ ok: true })
})

/**
* Nothing is bounded while the whole result fits the projection's caps, so a result that crossed
* in full before still does: a lifted output and its log copy of about 27.5k values each, or one
* 4.5 MB block output.
* A preview keeps a prefix that can cut through a secret, so with one active an oversized final
* output becomes a pointer whose text does not depend on the secret.
*/
it('returns a result that fits the caps in full while a secret is active', async () => {
const rows = wideRows()
mocks.executeWorkflowUseCase.mockResolvedValue({
success: true,
output: {},
logs: [{ blockId: 'query', blockName: 'Query', success: true, output: { rows } }],
metadata: { executionId: EXECUTION_ID },
})
const lifted = inspectToolResultForCopilot(
await executeRunWorkflowUntilBlock(
{ workflowId: 'wf-1', stopAfterBlockId: 'query' },
context
),
registry,
'run_workflow'
)
expect(lifted.safe).toBe(true)
const liftedOutput = lifted.result.output as Record<string, unknown>
expect(liftedOutput.output).toEqual({ rows })
expect((liftedOutput.logs as Array<Record<string, unknown>>)[0]?.output).toEqual({ rows })
it('replaces an oversized final output with a pointer, never a preview, while a secret is active', async () => {
async function finalOutputFor(secret: string) {
mocks.executeWorkflowUseCase.mockResolvedValue({
success: true,
output: { report: `${'r'.repeat(20_000)}${secret}`, token: secret },
logs: [{ blockId: 'small', blockName: 'Small', success: true, output: { ok: true } }],
metadata: { executionId: EXECUTION_ID },
})
const settled = await executeRunWorkflow({ workflowId: 'wf-1' }, context)
expect(inspectToolResultForCopilot(settled, registry, 'run_workflow').safe).toBe(true)
return (settled.output as Record<string, unknown>).output
}

const text = { text: 'y'.repeat(4_500_000) }
mocks.executeWorkflowUseCase.mockResolvedValue({
success: true,
output: { done: true },
logs: [{ blockId: 'big', blockName: 'Big', success: true, output: text }],
metadata: { executionId: EXECUTION_ID },
})
const wide = inspectToolResultForCopilot(
await executeRunWorkflow({ workflowId: 'wf-1' }, context),
registry,
'run_workflow'
)
expect(wide.safe).toBe(true)
expect((wide.result.output as { logs: Array<{ output: unknown }> }).logs[0]?.output).toEqual(
text
const short = await finalOutputFor(SECRET)
expect(short).toEqual(
expect.stringMatching(/^…\[output omitted: \d+ values; inspect with logs get /)
)
expect(await finalOutputFor(`${SECRET}-but-considerably-longer`)).toBe(short)
})

/**
Expand Down
Loading
Loading