diff --git a/apps/sim/lib/internal/mongodb/execute-tool.ts b/apps/sim/lib/internal/mongodb/execute-tool.ts index 252e9eb0b93..21c77a372cd 100644 --- a/apps/sim/lib/internal/mongodb/execute-tool.ts +++ b/apps/sim/lib/internal/mongodb/execute-tool.ts @@ -1,5 +1,6 @@ import { getErrorMessage } from '@sim/utils/errors' import type { z } from 'zod' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { executeMongodbAggregation, executeMongodbDelete, @@ -40,6 +41,7 @@ async function executeOperation( return Response.json(result) } catch (error) { signal?.throwIfAborted() + if (isPayloadSizeLimitError(error)) throw error if (error instanceof MongodbOperationInputError) { return Response.json({ error: error.message }, { status: 400 }) } diff --git a/apps/sim/lib/internal/mongodb/operations.ts b/apps/sim/lib/internal/mongodb/operations.ts index 6807224259d..859857ce5d8 100644 --- a/apps/sim/lib/internal/mongodb/operations.ts +++ b/apps/sim/lib/internal/mongodb/operations.ts @@ -23,9 +23,25 @@ import type { MongodbQueryInput, MongodbUpdateInput, } from '@/lib/internal/mongodb/schema' +import { createResponseRowsBudget } from '@/lib/internal/tool-operations/response-limits' export class MongodbOperationInputError extends Error {} +/** + * Reads documents one at a time, failing with the tool response size error as soon as they cannot + * fit the response. Leaving the loop early closes the server cursor, so an oversized result is + * never read into memory in full. + */ +async function readDocuments(cursor: AsyncIterable, toolId: string) { + const budget = createResponseRowsBudget(toolId) + const documents: Document[] = [] + for await (const document of cursor) { + budget.admit(document) + documents.push(document) + } + return documents +} + async function withMongodbClient( input: MongodbConnectionConfig, signal: AbortSignal | undefined, @@ -71,7 +87,7 @@ export function executeMongodbQuery(input: MongodbQueryInput, signal?: AbortSign } cursor = cursor.limit(input.limit || 100) - const documents = await cursor.toArray() + const documents = await readDocuments(cursor, 'mongodb_query') signal?.throwIfAborted() return { message: `Found ${documents.length} documents`, @@ -92,11 +108,10 @@ export function executeMongodbAggregation(input: MongodbExecuteInput, signal?: A const pipeline = JSON.parse(input.pipeline) as Document[] return withMongodbClient(input, signal, async (client) => { - const documents = await client - .db(input.database) - .collection(collectionName) - .aggregate(pipeline, { signal }) - .toArray() + const documents = await readDocuments( + client.db(input.database).collection(collectionName).aggregate(pipeline, { signal }), + 'mongodb_execute' + ) signal?.throwIfAborted() return { message: `Aggregation completed, returned ${documents.length} documents`, diff --git a/apps/sim/lib/internal/neo4j/execute-tool.ts b/apps/sim/lib/internal/neo4j/execute-tool.ts index f542d648bb3..b53ca36a0b1 100644 --- a/apps/sim/lib/internal/neo4j/execute-tool.ts +++ b/apps/sim/lib/internal/neo4j/execute-tool.ts @@ -9,6 +9,7 @@ import { neo4jQueryContract, neo4jUpdateContract, } from '@/lib/api/contracts/tools/databases/neo4j' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { executeNeo4jCreate, executeNeo4jDelete, @@ -35,6 +36,7 @@ async function executeOperation( return Response.json(await execute(parsed.data, signal)) } catch (error) { signal?.throwIfAborted() + if (isPayloadSizeLimitError(error)) throw error if (error instanceof Neo4jOperationInputError) { return Response.json({ error: `Query validation failed: ${error.message}` }, { status: 400 }) } diff --git a/apps/sim/lib/internal/neo4j/operations.test.ts b/apps/sim/lib/internal/neo4j/operations.test.ts index 73226de5479..54c7b104f06 100644 --- a/apps/sim/lib/internal/neo4j/operations.test.ts +++ b/apps/sim/lib/internal/neo4j/operations.test.ts @@ -11,6 +11,7 @@ vi.mock('@/lib/internal/neo4j/client', () => ({ createNeo4jDriver: mocks.createDriver, })) +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { executeNeo4jQuery, Neo4jOperationInputError } from '@/lib/internal/neo4j/operations' const INPUT = { @@ -24,7 +25,8 @@ const INPUT = { parameters: {}, } -function queryResult() { +/** A driver `Result` streaming `count` records, recording whether the rest was discarded. */ +function streamingResult(count: number, value = 'Ada') { const updates = { nodesCreated: 0, nodesDeleted: 0, @@ -38,19 +40,32 @@ function queryResult() { constraintsAdded: 0, constraintsRemoved: 0, } + const summary = { + resultAvailableAfter: { toNumber: () => 2 }, + resultConsumedAfter: { toNumber: () => 3 }, + counters: { updates: () => updates }, + } + const state = { pulled: 0, discarded: false } + const record = { keys: ['name'], get: () => value } return { - records: [{ keys: ['name'], get: vi.fn().mockReturnValue('Ada') }], - summary: { - resultAvailableAfter: { toNumber: () => 2 }, - resultConsumedAfter: { toNumber: () => 3 }, - counters: { updates: () => updates }, - }, + state, + [Symbol.asyncIterator]: () => ({ + next: async () => { + if (state.pulled >= count) return { done: true, value: summary } + state.pulled++ + return { done: false, value: record } + }, + return: async () => { + state.discarded = true + return { done: true, value: summary } + }, + }), } } describe('Neo4j operations', () => { beforeEach(() => { - mocks.run.mockResolvedValue(queryResult()) + mocks.run.mockImplementation(() => streamingResult(1)) mocks.sessionClose.mockResolvedValue(undefined) mocks.driverClose.mockResolvedValue(undefined) mocks.createDriver.mockResolvedValue({ @@ -59,6 +74,29 @@ describe('Neo4j operations', () => { }) }) + it('returns every record of a result that fits the response limit', async () => { + const result = streamingResult(20_000) + mocks.run.mockReturnValue(result) + + const response = await executeNeo4jQuery(INPUT) + + expect(response).toMatchObject({ recordCount: 20_000, summary: { resultAvailableAfter: 2 } }) + expect(result.state.discarded).toBe(false) + }) + + it('fails with the response size error and discards the rest once records cannot fit', async () => { + const result = streamingResult(1_000, 'x'.repeat(64 * 1024)) + mocks.run.mockReturnValue(result) + + const error = await executeNeo4jQuery(INPUT).catch((caught: unknown) => caught) + + expect(isPayloadSizeLimitError(error)).toBe(true) + expect(error).toMatchObject({ label: 'neo4j_query response body' }) + expect(result.state.discarded).toBe(true) + expect(result.state.pulled).toBeLessThan(1_000) + expect(mocks.sessionClose).toHaveBeenCalled() + }) + it('rejects invalid Cypher before opening a driver', async () => { await expect(executeNeo4jQuery({ ...INPUT, cypherQuery: ' ' })).rejects.toBeInstanceOf( Neo4jOperationInputError diff --git a/apps/sim/lib/internal/neo4j/operations.ts b/apps/sim/lib/internal/neo4j/operations.ts index 476b1151072..0e644a1e653 100644 --- a/apps/sim/lib/internal/neo4j/operations.ts +++ b/apps/sim/lib/internal/neo4j/operations.ts @@ -1,5 +1,5 @@ import { createLogger } from '@sim/logger' -import type { QueryResult, Session } from 'neo4j-driver' +import type { Record as Neo4jRecord, Result, ResultSummary, Session } from 'neo4j-driver' import type { Neo4jCreateRequest, Neo4jDeleteRequest, @@ -11,6 +11,7 @@ import type { } from '@/lib/api/contracts/tools/databases/neo4j' import { createNeo4jDriver } from '@/lib/internal/neo4j/client' import { convertNeo4jValue } from '@/lib/internal/neo4j/values' +import { createResponseRowsBudget } from '@/lib/internal/tool-operations/response-limits' import type { Neo4jNodeSchema, Neo4jRelationshipSchema } from '@/tools/neo4j/types' const logger = createLogger('Neo4jOperations') @@ -43,21 +44,44 @@ function bindSessionAbort(session: Session, signal?: AbortSignal): () => void { return () => signal.removeEventListener('abort', abort) } -function projectRecords(result: QueryResult): Array> { - return result.records.map((record) => { - const projected: Record = {} - for (const key of record.keys) { - if (typeof key === 'string') projected[key] = convertNeo4jValue(record.get(key)) +function projectRecord(record: Neo4jRecord): Record { + const projected: Record = {} + for (const key of record.keys) { + if (typeof key === 'string') projected[key] = convertNeo4jValue(record.get(key)) + } + return projected +} + +/** + * Streams the records, failing with the tool response size error as soon as they cannot fit the + * response. Returning the iterator early tells the server to discard the rest, so an oversized + * result is never buffered in full. `delete` returns no records, so it only waits for the summary. + */ +async function readResult(result: Result, kind: StatementKind) { + if (kind === 'delete') return { records: [], summary: await result.summary() } + const budget = createResponseRowsBudget(`neo4j_${kind}`) + const records: Array> = [] + const iterator = result[Symbol.asyncIterator]() + try { + let next = await iterator.next() + while (!next.done) { + const projected = projectRecord(next.value) + budget.admit(projected) + records.push(projected) + next = await iterator.next() } - return projected - }) + return { records, summary: next.value } + } catch (error) { + await iterator.return?.().catch(() => undefined) + throw error + } } -function projectSummary(result: QueryResult) { - const updates = result.summary.counters.updates() +function projectSummary(summary: ResultSummary) { + const updates = summary.counters.updates() return { - resultAvailableAfter: result.summary.resultAvailableAfter.toNumber(), - resultConsumedAfter: result.summary.resultConsumedAfter.toNumber(), + resultAvailableAfter: summary.resultAvailableAfter.toNumber(), + resultConsumedAfter: summary.resultConsumedAfter.toNumber(), counters: { nodesCreated: updates.nodesCreated, nodesDeleted: updates.nodesDeleted, @@ -74,9 +98,11 @@ function projectSummary(result: QueryResult) { } } -function statementResponse(kind: StatementKind, result: QueryResult) { - const records = projectRecords(result) - const summary = projectSummary(result) +function statementResponse( + kind: StatementKind, + { records, summary: resultSummary }: Awaited> +) { + const summary = projectSummary(resultSummary) switch (kind) { case 'query': return { @@ -128,7 +154,7 @@ async function executeStatement(input: StatementInput, kind: StatementKind, sign const session = driver.session({ database: input.database }) const unbindAbort = bindSessionAbort(session, signal) try { - const result = await session.run(input.cypherQuery, input.parameters ?? {}) + const result = await readResult(session.run(input.cypherQuery, input.parameters ?? {}), kind) signal?.throwIfAborted() return statementResponse(kind, result) } finally { diff --git a/apps/sim/lib/internal/tool-operations/response-limits.test.ts b/apps/sim/lib/internal/tool-operations/response-limits.test.ts new file mode 100644 index 00000000000..c2aa1e29570 --- /dev/null +++ b/apps/sim/lib/internal/tool-operations/response-limits.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from 'vitest' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { + createResponseRowsBudget, + MAX_TOOL_RESPONSE_BODY_BYTES, +} from '@/lib/internal/tool-operations/response-limits' + +/** Admits every row, returning the error the budget threw, if any. */ +function admitAll(rows: unknown[]): unknown { + const budget = createResponseRowsBudget('test_tool') + try { + for (const row of rows) budget.admit(row) + return undefined + } catch (error) { + return error + } +} + +describe('response rows budget', () => { + it('admits any number of rows while their array still fits the response limit', () => { + expect(admitAll(Array.from({ length: 50_000 }, (_, id) => ({ id })))).toBeUndefined() + }) + + it('fails exactly where the serialized array passes the response limit', () => { + const fitting = ['x'.repeat(MAX_TOOL_RESPONSE_BODY_BYTES - 4)] + expect(Buffer.byteLength(JSON.stringify(fitting))).toBe(MAX_TOOL_RESPONSE_BODY_BYTES) + expect(admitAll(fitting)).toBeUndefined() + + const error = admitAll(['x'.repeat(MAX_TOOL_RESPONSE_BODY_BYTES - 3)]) + expect(isPayloadSizeLimitError(error)).toBe(true) + expect(error).toMatchObject({ + label: 'test_tool response body', + maxBytes: MAX_TOOL_RESPONSE_BODY_BYTES, + }) + }) + + it('counts UTF-8 bytes, so multibyte text fails on its encoded size', () => { + const row = '漢'.repeat(MAX_TOOL_RESPONSE_BODY_BYTES / 3) + expect(row.length).toBeLessThan(MAX_TOOL_RESPONSE_BODY_BYTES) + expect(isPayloadSizeLimitError(admitAll([row]))).toBe(true) + }) +}) diff --git a/apps/sim/lib/internal/tool-operations/response-limits.ts b/apps/sim/lib/internal/tool-operations/response-limits.ts index fb8828b4ef1..2c7ccb5e1b1 100644 --- a/apps/sim/lib/internal/tool-operations/response-limits.ts +++ b/apps/sim/lib/internal/tool-operations/response-limits.ts @@ -1,2 +1,28 @@ +import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' + /** Maximum inline tool response size; binary file outputs use stored descriptors. */ export const MAX_TOOL_RESPONSE_BODY_BYTES = 10 * 1024 * 1024 + +/** + * Fails a streamed result as soon as its serialized rows alone exceed + * {@link MAX_TOOL_RESPONSE_BODY_BYTES}. Bytes are counted the way the response serializes them + * (UTF-8, plus the array's brackets and commas), and the body around the rows is larger still, so + * this fails exactly the results the executor's response limit would reject, but before the rest + * of the result is read into memory. The error is the same one that limit throws. + */ +export function createResponseRowsBudget(toolId: string) { + let bytes = 1 + return { + admit(row: unknown): void { + const serialized = JSON.stringify(row) + bytes += (serialized === undefined ? 4 : Buffer.byteLength(serialized, 'utf8')) + 1 + if (bytes > MAX_TOOL_RESPONSE_BODY_BYTES) { + throw new PayloadSizeLimitError({ + label: `${toolId} response body`, + maxBytes: MAX_TOOL_RESPONSE_BODY_BYTES, + observedBytes: bytes, + }) + } + }, + } +} diff --git a/apps/sim/tools/upstash/redis_delete.ts b/apps/sim/tools/upstash/redis_delete.ts index c78ed35f747..4a2a947d081 100644 --- a/apps/sim/tools/upstash/redis_delete.ts +++ b/apps/sim/tools/upstash/redis_delete.ts @@ -1,6 +1,7 @@ import { UPSTASH_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' import type { UpstashRedisDeleteParams, UpstashRedisDeleteResponse } from '@/tools/upstash/types' +import { upstashPathSegment } from '@/tools/upstash/utils' export const upstashRedisDeleteTool: ToolConfig< UpstashRedisDeleteParams, @@ -34,7 +35,7 @@ export const upstashRedisDeleteTool: ToolConfig< }, request: { - url: (params) => `${params.restUrl}/del/${encodeURIComponent(params.key)}`, + url: (params) => `${params.restUrl}/del/${upstashPathSegment(params.key, 'key')}`, method: 'POST', headers: (params) => ({ Authorization: `Bearer ${params.restToken}`, diff --git a/apps/sim/tools/upstash/redis_exists.ts b/apps/sim/tools/upstash/redis_exists.ts index cf50887cc48..0d065b77080 100644 --- a/apps/sim/tools/upstash/redis_exists.ts +++ b/apps/sim/tools/upstash/redis_exists.ts @@ -1,6 +1,7 @@ import { UPSTASH_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' import type { UpstashRedisExistsParams, UpstashRedisExistsResponse } from '@/tools/upstash/types' +import { upstashPathSegment } from '@/tools/upstash/utils' export const upstashRedisExistsTool: ToolConfig< UpstashRedisExistsParams, @@ -35,7 +36,7 @@ export const upstashRedisExistsTool: ToolConfig< }, request: { - url: (params) => `${params.restUrl}/exists/${encodeURIComponent(params.key)}`, + url: (params) => `${params.restUrl}/exists/${upstashPathSegment(params.key, 'key')}`, method: 'GET', headers: (params) => ({ Authorization: `Bearer ${params.restToken}`, diff --git a/apps/sim/tools/upstash/redis_expire.ts b/apps/sim/tools/upstash/redis_expire.ts index eff08b7286b..990eaf15ea0 100644 --- a/apps/sim/tools/upstash/redis_expire.ts +++ b/apps/sim/tools/upstash/redis_expire.ts @@ -1,6 +1,7 @@ import { UPSTASH_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' import type { UpstashRedisExpireParams, UpstashRedisExpireResponse } from '@/tools/upstash/types' +import { upstashInteger, upstashPathSegment } from '@/tools/upstash/utils' export const upstashRedisExpireTool: ToolConfig< UpstashRedisExpireParams, @@ -40,7 +41,8 @@ export const upstashRedisExpireTool: ToolConfig< }, request: { - url: (params) => `${params.restUrl}/expire/${encodeURIComponent(params.key)}/${params.seconds}`, + url: (params) => + `${params.restUrl}/expire/${upstashPathSegment(params.key, 'key')}/${upstashInteger(params.seconds, 'seconds')}`, method: 'GET', headers: (params) => ({ Authorization: `Bearer ${params.restToken}`, diff --git a/apps/sim/tools/upstash/redis_get.ts b/apps/sim/tools/upstash/redis_get.ts index 265da4b28d2..5475e558251 100644 --- a/apps/sim/tools/upstash/redis_get.ts +++ b/apps/sim/tools/upstash/redis_get.ts @@ -1,6 +1,7 @@ import { UPSTASH_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' import type { UpstashRedisGetParams, UpstashRedisGetResponse } from '@/tools/upstash/types' +import { upstashPathSegment } from '@/tools/upstash/utils' export const upstashRedisGetTool: ToolConfig = { id: 'upstash_redis_get', @@ -31,7 +32,7 @@ export const upstashRedisGetTool: ToolConfig `${params.restUrl}/get/${encodeURIComponent(params.key)}`, + url: (params) => `${params.restUrl}/get/${upstashPathSegment(params.key, 'key')}`, method: 'GET', headers: (params) => ({ Authorization: `Bearer ${params.restToken}`, diff --git a/apps/sim/tools/upstash/redis_hget.ts b/apps/sim/tools/upstash/redis_hget.ts index f42f938bb3d..f28ad5ec1f5 100644 --- a/apps/sim/tools/upstash/redis_hget.ts +++ b/apps/sim/tools/upstash/redis_hget.ts @@ -1,6 +1,7 @@ import { UPSTASH_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' import type { UpstashRedisHGetParams, UpstashRedisHGetResponse } from '@/tools/upstash/types' +import { upstashPathSegment } from '@/tools/upstash/utils' export const upstashRedisHGetTool: ToolConfig = { id: 'upstash_redis_hget', @@ -38,7 +39,7 @@ export const upstashRedisHGetTool: ToolConfig - `${params.restUrl}/hget/${encodeURIComponent(params.key)}/${encodeURIComponent(params.field)}`, + `${params.restUrl}/hget/${upstashPathSegment(params.key, 'key')}/${upstashPathSegment(params.field, 'field')}`, method: 'GET', headers: (params) => ({ Authorization: `Bearer ${params.restToken}`, diff --git a/apps/sim/tools/upstash/redis_hgetall.ts b/apps/sim/tools/upstash/redis_hgetall.ts index d5478dc2660..431688aa4f9 100644 --- a/apps/sim/tools/upstash/redis_hgetall.ts +++ b/apps/sim/tools/upstash/redis_hgetall.ts @@ -1,6 +1,7 @@ import { UPSTASH_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' import type { UpstashRedisHGetAllParams, UpstashRedisHGetAllResponse } from '@/tools/upstash/types' +import { upstashPathSegment } from '@/tools/upstash/utils' export const upstashRedisHGetAllTool: ToolConfig< UpstashRedisHGetAllParams, @@ -34,7 +35,7 @@ export const upstashRedisHGetAllTool: ToolConfig< }, request: { - url: (params) => `${params.restUrl}/hgetall/${encodeURIComponent(params.key)}`, + url: (params) => `${params.restUrl}/hgetall/${upstashPathSegment(params.key, 'key')}`, method: 'GET', headers: (params) => ({ Authorization: `Bearer ${params.restToken}`, diff --git a/apps/sim/tools/upstash/redis_hset.ts b/apps/sim/tools/upstash/redis_hset.ts index e41505b2c89..a906708b67f 100644 --- a/apps/sim/tools/upstash/redis_hset.ts +++ b/apps/sim/tools/upstash/redis_hset.ts @@ -1,6 +1,7 @@ import { UPSTASH_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' import type { UpstashRedisHSetParams, UpstashRedisHSetResponse } from '@/tools/upstash/types' +import { upstashPathSegment } from '@/tools/upstash/utils' export const upstashRedisHSetTool: ToolConfig = { id: 'upstash_redis_hset', @@ -44,7 +45,7 @@ export const upstashRedisHSetTool: ToolConfig - `${params.restUrl}/hset/${encodeURIComponent(params.key)}/${encodeURIComponent(params.field)}`, + `${params.restUrl}/hset/${upstashPathSegment(params.key, 'key')}/${upstashPathSegment(params.field, 'field')}`, method: 'POST', headers: (params) => ({ Authorization: `Bearer ${params.restToken}`, diff --git a/apps/sim/tools/upstash/redis_incr.ts b/apps/sim/tools/upstash/redis_incr.ts index a5eb881fb24..752c5a991c7 100644 --- a/apps/sim/tools/upstash/redis_incr.ts +++ b/apps/sim/tools/upstash/redis_incr.ts @@ -1,6 +1,7 @@ import { UPSTASH_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' import type { UpstashRedisIncrParams, UpstashRedisIncrResponse } from '@/tools/upstash/types' +import { upstashPathSegment } from '@/tools/upstash/utils' export const upstashRedisIncrTool: ToolConfig = { id: 'upstash_redis_incr', @@ -32,7 +33,7 @@ export const upstashRedisIncrTool: ToolConfig `${params.restUrl}/incr/${encodeURIComponent(params.key)}`, + url: (params) => `${params.restUrl}/incr/${upstashPathSegment(params.key, 'key')}`, method: 'GET', headers: (params) => ({ Authorization: `Bearer ${params.restToken}`, diff --git a/apps/sim/tools/upstash/redis_incrby.ts b/apps/sim/tools/upstash/redis_incrby.ts index 64ef47a3513..eb11efc4001 100644 --- a/apps/sim/tools/upstash/redis_incrby.ts +++ b/apps/sim/tools/upstash/redis_incrby.ts @@ -1,6 +1,7 @@ import { UPSTASH_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' import type { UpstashRedisIncrbyParams, UpstashRedisIncrbyResponse } from '@/tools/upstash/types' +import { upstashInteger, upstashPathSegment } from '@/tools/upstash/utils' export const upstashRedisIncrbyTool: ToolConfig< UpstashRedisIncrbyParams, @@ -42,7 +43,7 @@ export const upstashRedisIncrbyTool: ToolConfig< request: { url: (params) => - `${params.restUrl}/incrby/${encodeURIComponent(params.key)}/${params.increment}`, + `${params.restUrl}/incrby/${upstashPathSegment(params.key, 'key')}/${upstashInteger(params.increment, 'increment')}`, method: 'GET', headers: (params) => ({ Authorization: `Bearer ${params.restToken}`, diff --git a/apps/sim/tools/upstash/redis_keys.ts b/apps/sim/tools/upstash/redis_keys.ts index 4cdca491888..27d27be07e1 100644 --- a/apps/sim/tools/upstash/redis_keys.ts +++ b/apps/sim/tools/upstash/redis_keys.ts @@ -1,6 +1,7 @@ import { UPSTASH_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' import type { UpstashRedisKeysParams, UpstashRedisKeysResponse } from '@/tools/upstash/types' +import { upstashPathSegment } from '@/tools/upstash/utils' export const upstashRedisKeysTool: ToolConfig = { id: 'upstash_redis_keys', @@ -33,7 +34,7 @@ export const upstashRedisKeysTool: ToolConfig { const pattern = params.pattern || '*' - return `${params.restUrl}/keys/${encodeURIComponent(pattern)}` + return `${params.restUrl}/keys/${upstashPathSegment(pattern, 'pattern')}` }, method: 'GET', headers: (params) => ({ diff --git a/apps/sim/tools/upstash/redis_lpush.ts b/apps/sim/tools/upstash/redis_lpush.ts index 351d3d3f955..916b03223a2 100644 --- a/apps/sim/tools/upstash/redis_lpush.ts +++ b/apps/sim/tools/upstash/redis_lpush.ts @@ -1,6 +1,7 @@ import { UPSTASH_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' import type { UpstashRedisLPushParams, UpstashRedisLPushResponse } from '@/tools/upstash/types' +import { upstashPathSegment } from '@/tools/upstash/utils' export const upstashRedisLPushTool: ToolConfig = { @@ -39,7 +40,7 @@ export const upstashRedisLPushTool: ToolConfig `${params.restUrl}/lpush/${encodeURIComponent(params.key)}`, + url: (params) => `${params.restUrl}/lpush/${upstashPathSegment(params.key, 'key')}`, method: 'POST', headers: (params) => ({ Authorization: `Bearer ${params.restToken}`, diff --git a/apps/sim/tools/upstash/redis_lrange.ts b/apps/sim/tools/upstash/redis_lrange.ts index 2a7c64ad422..bb542f76130 100644 --- a/apps/sim/tools/upstash/redis_lrange.ts +++ b/apps/sim/tools/upstash/redis_lrange.ts @@ -1,6 +1,7 @@ import { UPSTASH_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' import type { UpstashRedisLRangeParams, UpstashRedisLRangeResponse } from '@/tools/upstash/types' +import { upstashInteger, upstashPathSegment } from '@/tools/upstash/utils' export const upstashRedisLRangeTool: ToolConfig< UpstashRedisLRangeParams, @@ -48,7 +49,7 @@ export const upstashRedisLRangeTool: ToolConfig< request: { url: (params) => - `${params.restUrl}/lrange/${encodeURIComponent(params.key)}/${params.start}/${params.stop}`, + `${params.restUrl}/lrange/${upstashPathSegment(params.key, 'key')}/${upstashInteger(params.start, 'start')}/${upstashInteger(params.stop, 'stop')}`, method: 'GET', headers: (params) => ({ Authorization: `Bearer ${params.restToken}`, diff --git a/apps/sim/tools/upstash/redis_set.ts b/apps/sim/tools/upstash/redis_set.ts index 8699a7fac37..be072efeba6 100644 --- a/apps/sim/tools/upstash/redis_set.ts +++ b/apps/sim/tools/upstash/redis_set.ts @@ -1,6 +1,7 @@ import { UPSTASH_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' import type { UpstashRedisSetParams, UpstashRedisSetResponse } from '@/tools/upstash/types' +import { upstashInteger, upstashPathSegment } from '@/tools/upstash/utils' export const upstashRedisSetTool: ToolConfig = { id: 'upstash_redis_set', @@ -45,9 +46,9 @@ export const upstashRedisSetTool: ToolConfig { - const base = `${params.restUrl}/set/${encodeURIComponent(params.key)}` + const base = `${params.restUrl}/set/${upstashPathSegment(params.key, 'key')}` if (params.ex) { - return `${base}?EX=${params.ex}` + return `${base}?EX=${upstashInteger(params.ex, 'ex')}` } return base }, diff --git a/apps/sim/tools/upstash/redis_setnx.ts b/apps/sim/tools/upstash/redis_setnx.ts index 019f5e7f52e..d2c4772b2d8 100644 --- a/apps/sim/tools/upstash/redis_setnx.ts +++ b/apps/sim/tools/upstash/redis_setnx.ts @@ -1,6 +1,7 @@ import { UPSTASH_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' import type { UpstashRedisSetnxParams, UpstashRedisSetnxResponse } from '@/tools/upstash/types' +import { upstashPathSegment } from '@/tools/upstash/utils' export const upstashRedisSetnxTool: ToolConfig = { @@ -39,7 +40,7 @@ export const upstashRedisSetnxTool: ToolConfig `${params.restUrl}/setnx/${encodeURIComponent(params.key)}`, + url: (params) => `${params.restUrl}/setnx/${upstashPathSegment(params.key, 'key')}`, method: 'POST', headers: (params) => ({ Authorization: `Bearer ${params.restToken}`, diff --git a/apps/sim/tools/upstash/redis_ttl.ts b/apps/sim/tools/upstash/redis_ttl.ts index d283b839f2f..62ad690dc46 100644 --- a/apps/sim/tools/upstash/redis_ttl.ts +++ b/apps/sim/tools/upstash/redis_ttl.ts @@ -1,6 +1,7 @@ import { UPSTASH_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' import type { UpstashRedisTtlParams, UpstashRedisTtlResponse } from '@/tools/upstash/types' +import { upstashPathSegment } from '@/tools/upstash/utils' export const upstashRedisTtlTool: ToolConfig = { id: 'upstash_redis_ttl', @@ -32,7 +33,7 @@ export const upstashRedisTtlTool: ToolConfig `${params.restUrl}/ttl/${encodeURIComponent(params.key)}`, + url: (params) => `${params.restUrl}/ttl/${upstashPathSegment(params.key, 'key')}`, method: 'GET', headers: (params) => ({ Authorization: `Bearer ${params.restToken}`, diff --git a/apps/sim/tools/upstash/upstash.test.ts b/apps/sim/tools/upstash/upstash.test.ts new file mode 100644 index 00000000000..7465de84df4 --- /dev/null +++ b/apps/sim/tools/upstash/upstash.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from 'vitest' +import type { ToolConfig } from '@/tools/types' +import { upstashRedisExpireTool } from '@/tools/upstash/redis_expire' +import { upstashRedisHGetTool } from '@/tools/upstash/redis_hget' +import { upstashRedisIncrbyTool } from '@/tools/upstash/redis_incrby' +import { upstashRedisLRangeTool } from '@/tools/upstash/redis_lrange' + +const connection = { restUrl: 'https://example.upstash.io', restToken: 'token' } + +/** Builds a tool's request path from untyped params, the way an agent's tool call arrives. */ +function requestPath

(tool: ToolConfig

, params: Record): string { + const { url } = tool.request + return new URL(typeof url === 'function' ? url({ ...connection, ...params } as P) : url).pathname +} + +describe('Upstash REST paths', () => { + it('refuses a dot-segment key that would turn the field into the command', () => { + expect(() => requestPath(upstashRedisHGetTool, { key: '..', field: 'flushall' })).toThrow( + 'key cannot be "." or ".."' + ) + }) + + it.each([ + ['increment', upstashRedisIncrbyTool, { key: 'k', increment: '../../flushall' }], + ['seconds', upstashRedisExpireTool, { key: 'k', seconds: '1/../../flushall' }], + ['start', upstashRedisLRangeTool, { key: 'k', start: '../../flushall', stop: 1 }], + ] as const)('refuses a non-integer %s', (name, tool, params) => { + expect(() => requestPath(tool as ToolConfig, params)).toThrow(`${name} must be an integer`) + }) + + it('keeps ordinary keys, fields, and negative integers in their own segments', () => { + expect(requestPath(upstashRedisHGetTool, { key: 'user/1', field: 'a b' })).toBe( + '/hget/user%2F1/a%20b' + ) + expect(requestPath(upstashRedisLRangeTool, { key: 'list', start: 0, stop: -1 })).toBe( + '/lrange/list/0/-1' + ) + expect(requestPath(upstashRedisIncrbyTool, { key: 'n', increment: '-5' })).toBe('/incrby/n/-5') + }) +}) diff --git a/apps/sim/tools/upstash/utils.ts b/apps/sim/tools/upstash/utils.ts new file mode 100644 index 00000000000..22946c92c14 --- /dev/null +++ b/apps/sim/tools/upstash/utils.ts @@ -0,0 +1,21 @@ +/** + * Encodes one Redis argument as an Upstash REST path segment. A bare `.` or `..` is refused + * because URL parsing resolves it (encoded or not) as a dot segment, which would shift the + * remaining arguments into the command position (for example `HGET .. flushall` becoming + * `FLUSHALL`). + */ +export function upstashPathSegment(value: string, name: string): string { + if (value === '.' || value === '..') { + throw new Error(`${name} cannot be "." or ".." in an Upstash REST request`) + } + return encodeURIComponent(value) +} + +/** Formats an integer Redis argument for an Upstash REST path or query, refusing anything else. */ +export function upstashInteger(value: unknown, name: string): string { + const parsed = typeof value === 'string' && value.trim() !== '' ? Number(value) : value + if (typeof parsed !== 'number' || !Number.isSafeInteger(parsed)) { + throw new Error(`${name} must be an integer`) + } + return String(parsed) +}