diff --git a/apps/sim/app/api/v2/workflows/[workflowId]/execute/route.test.ts b/apps/sim/app/api/v2/workflows/[workflowId]/execute/route.test.ts index f2cb5c0acd4..0f8ecd035de 100644 --- a/apps/sim/app/api/v2/workflows/[workflowId]/execute/route.test.ts +++ b/apps/sim/app/api/v2/workflows/[workflowId]/execute/route.test.ts @@ -482,6 +482,10 @@ describe('POST /api/v2/workflows/[workflowId]/execute', () => { blockName: 'Send Email', blockType: 'gmail', }) + // An agent reads the head of an oversized run, so the error precedes the unbounded output. + const keys = Object.keys(body.data) + expect(keys.indexOf('error')).toBeLessThan(keys.indexOf('output')) + expect(keys.indexOf('durationMs')).toBeLessThan(keys.indexOf('output')) }) it('queues async runs and returns a 202 receipt with the v2 runs statusUrl', async () => { diff --git a/apps/sim/app/api/v2/workflows/[workflowId]/execute/route.ts b/apps/sim/app/api/v2/workflows/[workflowId]/execute/route.ts index 39ec3bc305a..ab53f25617b 100644 --- a/apps/sim/app/api/v2/workflows/[workflowId]/execute/route.ts +++ b/apps/sim/app/api/v2/workflows/[workflowId]/execute/route.ts @@ -123,18 +123,23 @@ function encodeNdjson(value: unknown): Uint8Array { return ndjsonEncoder.encode(`${JSON.stringify(value)}\n`) } +/** + * The run resource, its bounded fields ahead of `output` and `blockOutputs`: a reader that keeps + * only the head of an oversized response (an agent's output window) still sees the status, the + * failing block and its error, and the duration. + */ function presentRun(result: ExecuteWorkflowServiceRun) { return { runId: result.executionId, workflowId: result.workflowId, status: result.status, - output: result.output ?? null, - blockOutputs: result.blockOutputs ?? null, error: result.error, startedAt: result.startedAt, endedAt: result.endedAt, durationMs: result.durationMs, executionTimeoutSeconds: executionTimeoutSeconds(result.executionTimeoutMs), + output: result.output ?? null, + blockOutputs: result.blockOutputs ?? null, } } diff --git a/apps/sim/app/api/v2/workflows/[workflowId]/runs/[runId]/resume/route.ts b/apps/sim/app/api/v2/workflows/[workflowId]/runs/[runId]/resume/route.ts index c3aff4232f4..e34f27d2bcb 100644 --- a/apps/sim/app/api/v2/workflows/[workflowId]/runs/[runId]/resume/route.ts +++ b/apps/sim/app/api/v2/workflows/[workflowId]/runs/[runId]/resume/route.ts @@ -111,13 +111,13 @@ export const POST = withRouteHandler( runId: result.executionId, workflowId, status: result.status as 'completed' | 'failed' | 'paused' | 'cancelled', - output: result.output ?? null, - // Resume has no request body to name selectors in, so selection never applies here. - blockOutputs: null, error: result.structuredError, startedAt: result.metadata?.startTime, endedAt: result.metadata?.endTime, durationMs: result.metadata?.duration, + output: result.output ?? null, + // Resume has no request body to name selectors in, so selection never applies here. + blockOutputs: null, }, { headers } ) diff --git a/apps/sim/lib/mothership/tools/handlers/workflow/run-workflow-result-budget.test.ts b/apps/sim/lib/mothership/tools/handlers/workflow/run-workflow-result-budget.test.ts index ab92e9d0dd0..6be80722273 100644 --- a/apps/sim/lib/mothership/tools/handlers/workflow/run-workflow-result-budget.test.ts +++ b/apps/sim/lib/mothership/tools/handlers/workflow/run-workflow-result-budget.test.ts @@ -148,13 +148,21 @@ describe('run_workflow model-facing result budget', () => { expect(presented.find((log) => log.blockName === 'Query 1')?.output).toEqual(logs[1].output) }) - /** The final output is what the run was for, so it is never compacted and needs the headroom. */ - it('leaves room for a large final output beside the bounded logs', async () => { - const finalOutput = { rows: tableRows(4_800, 3_200_000) } + /** + * The agent stores a result past its window behind a paged read, which costs it a round to learn + * whether the run passed. Block outputs give way first, so the verdict, the failing block, and a + * final output that fits all arrive inline. + */ + it('fits a failed run with bulky block outputs inside the agent window', async () => { + const logs = [ + ...traceShapedLogs().slice(0, 3), + { blockId: 'send', blockName: 'Send', success: false, error: 'SMTP refused', output: {} }, + ] + const finalOutput = { summary: 'partial report', rowCount: 1_882 } mocks.executeWorkflowUseCase.mockResolvedValue({ - success: true, + success: false, output: finalOutput, - logs: traceShapedLogs(), + logs, metadata: { executionId: EXECUTION_ID }, }) @@ -162,7 +170,69 @@ describe('run_workflow model-facing result budget', () => { const projection = inspectToolResultForCopilot(settled, registry, 'run_workflow') expect(projection.safe).toBe(true) - expect((projection.result.output as Record).output).toEqual(finalOutput) + expect(JSON.stringify(projection.result).length).toBeLessThanOrEqual(24_000) + expect(projection.result.error).toBe('Send: SMTP refused') + const output = projection.result.output as Record + expect(output.output).toEqual(finalOutput) + const presented = output.logs as Array> + expect(presented.map((log) => log.blockName)).toEqual(logs.map((log) => log.blockName)) + expect(presented[3]).toMatchObject({ success: false, error: 'SMTP refused' }) + }) + + /** A short secret under a long name grows when projected, so the window is measured after it. */ + it('fits the window as projected when short secrets expand into longer names', async () => { + const password = 'pw-12345' + const shortSecret = new ResolvedSecretTraceRegistry([ + { name: 'DATABASE_PASSWORD', plaintext: password, encryptedValue: 'ciphertext' }, + ]) + shortSecret.recordResolved('DATABASE_PASSWORD', password, { propagated: true }) + mocks.executeWorkflowUseCase.mockResolvedValue({ + success: true, + output: { done: true }, + logs: [ + { + blockId: 'echo', + blockName: 'Echo', + success: true, + // About 14K characters raw, over 31K once each copy becomes {{DATABASE_PASSWORD}}. + output: { tokens: Array.from({ length: 1_300 }, () => password) }, + }, + ], + metadata: { executionId: EXECUTION_ID }, + }) + + const settled = await executeRunWorkflow({ workflowId: 'wf-1' }, callContext(shortSecret)) + const projection = inspectToolResultForCopilot(settled, shortSecret, 'run_workflow') + + expect(projection.safe).toBe(true) + expect(JSON.stringify(projection.result.output).length).toBeLessThanOrEqual(16_000) + const output = projection.result.output as { output: unknown; logs: Array<{ output: unknown }> } + expect(output.output).toEqual({ done: true }) + expect(output.logs[0]?.output).toEqual(expect.stringContaining(`logs get ${EXECUTION_ID}`)) + }) + + /** Many small block entries leave less room than a full preview, so the preview shrinks to fit. */ + it('shrinks the final output preview to the room the block entries leave', async () => { + const logs = Array.from({ length: 150 }, (_, index) => ({ + blockId: `step-${index}`, + blockName: `Step ${index} of the nightly import`, + success: true, + output: { ok: true }, + })) + mocks.executeWorkflowUseCase.mockResolvedValue({ + success: true, + output: { report: 'z'.repeat(50_000) }, + logs, + metadata: { executionId: EXECUTION_ID }, + }) + const inactive = secretRegistry({ active: false }) + + const settled = await executeRunWorkflow({ workflowId: 'wf-1' }, callContext(inactive)) + const output = settled.output as { output: unknown; logs: unknown[] } + + expect(JSON.stringify({ output }).length).toBeLessThanOrEqual(16_000) + expect(output.logs).toEqual(logs) + expect(output.output).toEqual(expect.stringMatching(/^\{"report":"z+ …\[final output cut at/)) }) /** Narrow rows reach the traversal cap while staying far under every byte budget. */ @@ -186,7 +256,9 @@ describe('run_workflow model-facing result budget', () => { const projection = inspectToolResultForCopilot(settled, registry, 'run_workflow') expect(projection.safe).toBe(true) - expect((projection.result.output as Record).output).toEqual(finalOutput) + expect((projection.result.output as Record).output).toEqual( + expect.stringContaining(`logs get ${EXECUTION_ID} --trace`) + ) }) it('returns selected values in full, bypassing the log budget', async () => { @@ -343,11 +415,10 @@ describe('run_workflow model-facing result budget', () => { }) /** - * Without an active secret the projection passes JSON through under its byte cap alone, so - * nothing is bounded: a lifted output is the whole point of run_block and reaches the worker in - * full, which spills an oversized one to storage for the model to read. + * Without an active secret a cut cannot strand a secret fragment, so an oversized lifted output + * keeps its head inline as a preview and names where the rest is. */ - it('returns a large lifted output and its logs in full when no secret is active', async () => { + it('previews an oversized lifted output when no secret is active', async () => { const rows = wideRows() mocks.executeWorkflowUseCase.mockResolvedValue({ success: true, @@ -368,52 +439,35 @@ describe('run_workflow model-facing result budget', () => { expect(projection.safe).toBe(true) const output = projection.result.output as Record - expect(output.output).toEqual({ rows }) - expect((output.logs as Array>)[1]?.output).toEqual({ rows }) + expect(JSON.stringify(output).length).toBeLessThanOrEqual(16_000) + expect(output.output).toEqual(expect.stringMatching(/^\{"rows":\[\{"c0":"r0"/)) + expect(output.output).toEqual(expect.stringContaining(`logs get ${EXECUTION_ID} --trace`)) + expect(output.outputFrom).toEqual({ blockId: 'query', blockName: 'Query' }) + expect((output.logs as Array>)[0]?.output).toEqual({ ok: true }) }) /** - * Nothing is bounded while the whole result fits the projection's caps, so a result that crossed - * in full before still does: a lifted output and its log copy of about 27.5k values each, or one - * 4.5 MB block output. + * A preview keeps a prefix that can cut through a secret, so with one active an oversized final + * output becomes a pointer whose text does not depend on the secret. */ - it('returns a result that fits the caps in full while a secret is active', async () => { - const rows = wideRows() - mocks.executeWorkflowUseCase.mockResolvedValue({ - success: true, - output: {}, - logs: [{ blockId: 'query', blockName: 'Query', success: true, output: { rows } }], - metadata: { executionId: EXECUTION_ID }, - }) - const lifted = inspectToolResultForCopilot( - await executeRunWorkflowUntilBlock( - { workflowId: 'wf-1', stopAfterBlockId: 'query' }, - context - ), - registry, - 'run_workflow' - ) - expect(lifted.safe).toBe(true) - const liftedOutput = lifted.result.output as Record - expect(liftedOutput.output).toEqual({ rows }) - expect((liftedOutput.logs as Array>)[0]?.output).toEqual({ rows }) + it('replaces an oversized final output with a pointer, never a preview, while a secret is active', async () => { + async function finalOutputFor(secret: string) { + mocks.executeWorkflowUseCase.mockResolvedValue({ + success: true, + output: { report: `${'r'.repeat(20_000)}${secret}`, token: secret }, + logs: [{ blockId: 'small', blockName: 'Small', success: true, output: { ok: true } }], + metadata: { executionId: EXECUTION_ID }, + }) + const settled = await executeRunWorkflow({ workflowId: 'wf-1' }, context) + expect(inspectToolResultForCopilot(settled, registry, 'run_workflow').safe).toBe(true) + return (settled.output as Record).output + } - const text = { text: 'y'.repeat(4_500_000) } - mocks.executeWorkflowUseCase.mockResolvedValue({ - success: true, - output: { done: true }, - logs: [{ blockId: 'big', blockName: 'Big', success: true, output: text }], - metadata: { executionId: EXECUTION_ID }, - }) - const wide = inspectToolResultForCopilot( - await executeRunWorkflow({ workflowId: 'wf-1' }, context), - registry, - 'run_workflow' - ) - expect(wide.safe).toBe(true) - expect((wide.result.output as { logs: Array<{ output: unknown }> }).logs[0]?.output).toEqual( - text + const short = await finalOutputFor(SECRET) + expect(short).toEqual( + expect.stringMatching(/^…\[output omitted: \d+ values; inspect with logs get /) ) + expect(await finalOutputFor(`${SECRET}-but-considerably-longer`)).toBe(short) }) /** diff --git a/apps/sim/lib/mothership/tools/workflow-output.ts b/apps/sim/lib/mothership/tools/workflow-output.ts index 136a4a9f50f..7b5217ccae6 100644 --- a/apps/sim/lib/mothership/tools/workflow-output.ts +++ b/apps/sim/lib/mothership/tools/workflow-output.ts @@ -4,6 +4,7 @@ import { MAX_CONTENT_NODES, MAX_MODEL_CONTENT_BYTES, measureModelContent, + projectResolvedSecretModelContent, } from '@/executor/utils/resolved-secret-content-projection' import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' @@ -19,8 +20,8 @@ function presentWorkflowLogs(logs: unknown, select?: string[]): Record= 0xd800 && last <= 0xdbff) end -= 1 + return `${serialized.slice(0, end)} …[final output cut at ${end} of ${serialized.length} chars; inspect with logs get ${executionId ?? ''} --trace]` +} + +/** + * Holds a run result to {@link MODEL_RUN_RESULT_MAX_CHARS} so the agent reads pass/fail, the + * failing block's error, and each block's status inline. Echoed block inputs and outputs are + * replaced with pointers, largest first, until the result fits; the per-block entries keep their + * name, id, status, and error. If that is not enough, the final output is cut to a preview that + * fits what is left, or to a pointer when a secret is active. Values a caller named with `select` + * are kept. A result that still does not fit is returned as far as it was reduced. + * + * While `registry` makes the projection walk the result, its size is measured as projected: + * replacing many short secrets with their longer names can grow it. Raw sizes rank the parts and + * skip the projection while the raw result is plainly over. + */ +function fitRunResultToWindow( + data: Record, + error: string | undefined, + executionId: string | undefined, + registry: ResolvedSecretTraceRegistry | undefined, + walksContent: boolean +): Record { + const envelope = (candidate: Record) => + error === undefined ? { output: candidate } : { output: candidate, error } + const fits = (candidate: Record, rawChars: number): boolean => { + if (rawChars > MODEL_RUN_RESULT_MAX_CHARS) return false + if (!walksContent) return true + const projection = projectResolvedSecretModelContent(envelope(candidate), registry) + const projected = projection.safe ? serialize(projection.value)?.length : undefined + return (projected ?? rawChars) <= MODEL_RUN_RESULT_MAX_CHARS + } + + let raw = serialize(envelope(data))?.length + if (raw === undefined || fits(data, raw)) return data + + let fitted = data + if (Array.isArray(data.logs)) { + const parts: Array<{ index: number; field: 'input' | 'output'; size: number }> = [] + for (const [index, entry] of data.logs.entries()) { + if (!isPlainRecord(entry)) continue + for (const field of ['input', 'output'] as const) { + if (entry[field] === undefined) continue + const size = serialize(entry[field])?.length + if (size !== undefined) parts.push({ index, field, size }) + } + } + parts.sort((left, right) => right.size - left.size) + const logs = [...data.logs] + for (const part of parts) { + const entry = logs[part.index] as Record + const pointer = + part.field === 'input' + ? omittedInputMarker('', executionId ?? '') + : blockOutputPointer( + sizeBlockOutput(entry.output)?.label ?? 'output omitted', + executionId + ) + const saved = part.size - JSON.stringify(pointer).length + if (saved <= 0) continue + logs[part.index] = { ...entry, [part.field]: pointer } + raw -= saved + fitted = { ...data, logs } + if (fits(fitted, raw)) return fitted + } + } + if (!Object.hasOwn(fitted, 'output')) return fitted + + // A final output no longer than a preview is what the run was for; it stays whole. + const serialized = serialize(fitted.output) + if (serialized === undefined || serialized.length <= FINAL_OUTPUT_PREVIEW_CHARS) return fitted + const label = sizeBlockOutput(fitted.output)?.label + if (!label) return fitted + const pointed = { ...fitted, output: blockOutputPointer(label, executionId) } + if (walksContent) return pointed + + const rest = serialize(envelope(pointed))?.length ?? Number.POSITIVE_INFINITY + let chars = Math.min(FINAL_OUTPUT_PREVIEW_CHARS, MODEL_RUN_RESULT_MAX_CHARS - rest) + while (chars >= FINAL_OUTPUT_PREVIEW_MIN_CHARS && chars < serialized.length) { + const previewed = { ...fitted, output: finalOutputPreview(serialized, chars, executionId) } + const previewedChars = serialize(envelope(previewed))?.length ?? Number.POSITIVE_INFINITY + if (previewedChars <= MODEL_RUN_RESULT_MAX_CHARS) return previewed + // JSON escapes inside the preview grow it once it is embedded as a string. + chars -= previewedChars - MODEL_RUN_RESULT_MAX_CHARS + } + return pointed +} + +/** + * Bounds a run's whole model-facing result before secret projection, in two steps. First, when + * `registry` makes the projection walk the result, one past any of its caps is withheld entirely. + * So a result that would pass a cap first has its bulkiest block-log outputs replaced with + * pointers, down to their budget, and if it still would, its final output too. The result is + * measured whole, envelope and error included. A block output that run_block or + * run_workflow_until_block lifted into `output` is the run's final output here too. Then every + * result, secret or not, is held to the agent's inline window ({@link fitRunResultToWindow}). A + * result that fits both is returned untouched. */ export function boundRunResultForModel( data: Record, @@ -208,7 +332,16 @@ export function boundRunResultForModel( executionId: string | undefined, registry: ResolvedSecretTraceRegistry | undefined ): Record { - if (!copilotProjectionWalksContent(registry)) return data + const walksContent = copilotProjectionWalksContent(registry) + const capped = walksContent ? boundToProjectionCaps(data, error, executionId) : data + return fitRunResultToWindow(capped, error, executionId, registry, walksContent) +} + +function boundToProjectionCaps( + data: Record, + error: string | undefined, + executionId: string | undefined +): Record { // A result JSON cannot encode is refused whatever its size, so only one past a cap is bounded. const passesCap = (candidate: Record): boolean => measureModelContent(error === undefined ? { output: candidate } : { output: candidate, error })