From 0647a315df7703196afc43ad41c3e76d123b2e44 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 18 Nov 2026 10:18:32 -0800 Subject: [PATCH 1/3] fix(integrations): validate postgresql, mysql, mongodb, neo4j and upstash --- .../content/docs/integrations/mongodb.mdx | 4 + apps/docs/content/docs/integrations/mysql.mdx | 4 + apps/docs/content/docs/integrations/neo4j.mdx | 4 + .../content/docs/integrations/postgresql.mdx | 10 +++ apps/sim/blocks/blocks/mongodb.ts | 8 ++ apps/sim/blocks/blocks/mysql.ts | 8 ++ apps/sim/blocks/blocks/neo4j.ts | 8 ++ apps/sim/blocks/blocks/postgresql.ts | 8 ++ apps/sim/lib/internal/mongodb/operations.ts | 59 +++++++++---- apps/sim/lib/internal/mysql/client.test.ts | 33 ++++++++ apps/sim/lib/internal/mysql/client.ts | 16 +++- apps/sim/lib/internal/mysql/operations.ts | 46 +++++----- apps/sim/lib/internal/mysql/queries.ts | 16 ++-- apps/sim/lib/internal/mysql/schema.ts | 3 +- apps/sim/lib/internal/neo4j/operations.ts | 23 ++++- .../sim/lib/internal/postgresql/operations.ts | 46 +++++----- .../lib/internal/postgresql/queries.test.ts | 16 ++++ apps/sim/lib/internal/postgresql/queries.ts | 26 +++--- .../tool-operations/response-limits.test.ts | 33 ++++++++ .../tool-operations/response-limits.ts | 83 +++++++++++++++++++ apps/sim/tools/mongodb/execute.ts | 15 ++++ apps/sim/tools/mongodb/query.ts | 15 ++++ apps/sim/tools/mongodb/types.ts | 2 + apps/sim/tools/mysql/execute.ts | 15 ++++ apps/sim/tools/mysql/query.ts | 15 ++++ apps/sim/tools/mysql/types.ts | 2 + apps/sim/tools/neo4j/execute.ts | 15 ++++ apps/sim/tools/neo4j/query.ts | 15 ++++ apps/sim/tools/neo4j/types.ts | 2 + apps/sim/tools/postgresql/delete.ts | 15 ++++ apps/sim/tools/postgresql/execute.ts | 15 ++++ apps/sim/tools/postgresql/insert.ts | 15 ++++ apps/sim/tools/postgresql/query.ts | 15 ++++ apps/sim/tools/postgresql/types.ts | 2 + apps/sim/tools/postgresql/update.ts | 15 ++++ apps/sim/tools/upstash/redis_delete.ts | 3 +- apps/sim/tools/upstash/redis_exists.ts | 3 +- apps/sim/tools/upstash/redis_expire.ts | 4 +- apps/sim/tools/upstash/redis_get.ts | 3 +- apps/sim/tools/upstash/redis_hget.ts | 3 +- apps/sim/tools/upstash/redis_hgetall.ts | 3 +- apps/sim/tools/upstash/redis_hset.ts | 3 +- apps/sim/tools/upstash/redis_incr.ts | 3 +- apps/sim/tools/upstash/redis_incrby.ts | 3 +- apps/sim/tools/upstash/redis_keys.ts | 3 +- apps/sim/tools/upstash/redis_lpush.ts | 3 +- apps/sim/tools/upstash/redis_lrange.ts | 3 +- apps/sim/tools/upstash/redis_set.ts | 5 +- apps/sim/tools/upstash/redis_setnx.ts | 3 +- apps/sim/tools/upstash/redis_ttl.ts | 3 +- apps/sim/tools/upstash/upstash.test.ts | 40 +++++++++ apps/sim/tools/upstash/utils.ts | 21 +++++ 52 files changed, 626 insertions(+), 110 deletions(-) create mode 100644 apps/sim/lib/internal/tool-operations/response-limits.test.ts create mode 100644 apps/sim/tools/upstash/upstash.test.ts create mode 100644 apps/sim/tools/upstash/utils.ts diff --git a/apps/docs/content/docs/integrations/mongodb.mdx b/apps/docs/content/docs/integrations/mongodb.mdx index 3317de7a94e..a3ec82dfb5d 100644 --- a/apps/docs/content/docs/integrations/mongodb.mdx +++ b/apps/docs/content/docs/integrations/mongodb.mdx @@ -50,6 +50,8 @@ Execute find operation on MongoDB collection | `message` | string | Operation status message | | `documents` | array | Array of documents returned from the query | | `documentCount` | number | Number of documents returned | +| `truncated` | boolean | Present and true only when documents were dropped to stay inside the response ceilings \(10,000 documents or 10 MiB\). Absent means the result is complete | +| `truncationReason` | string | Which ceiling was hit and how to read the rest | @@ -177,6 +179,8 @@ Execute MongoDB aggregation pipeline | `message` | string | Operation status message | | `documents` | array | Array of documents returned from aggregation | | `documentCount` | number | Number of documents returned | +| `truncated` | boolean | Present and true only when documents were dropped to stay inside the response ceilings \(10,000 documents or 10 MiB\). Absent means the result is complete | +| `truncationReason` | string | Which ceiling was hit and how to read the rest | diff --git a/apps/docs/content/docs/integrations/mysql.mdx b/apps/docs/content/docs/integrations/mysql.mdx index 9ec39e201cc..0a252d54092 100644 --- a/apps/docs/content/docs/integrations/mysql.mdx +++ b/apps/docs/content/docs/integrations/mysql.mdx @@ -51,6 +51,8 @@ Execute SELECT query on MySQL database | `message` | string | Operation status message | | `rows` | array | Array of rows returned from the query | | `rowCount` | number | Number of rows returned | +| `truncated` | boolean | Present and true only when rows were dropped to stay inside the response ceilings \(10,000 rows or 10 MiB\). Absent means the result is complete | +| `truncationReason` | string | Which ceiling was hit and how to read the rest | @@ -143,6 +145,8 @@ Execute raw SQL query on MySQL database | `message` | string | Operation status message | | `rows` | array | Array of rows returned from the query | | `rowCount` | number | Number of rows affected | +| `truncated` | boolean | Present and true only when rows were dropped to stay inside the response ceilings \(10,000 rows or 10 MiB\). Absent means the result is complete | +| `truncationReason` | string | Which ceiling was hit and how to read the rest | diff --git a/apps/docs/content/docs/integrations/neo4j.mdx b/apps/docs/content/docs/integrations/neo4j.mdx index 82225094fcc..12dfd462e95 100644 --- a/apps/docs/content/docs/integrations/neo4j.mdx +++ b/apps/docs/content/docs/integrations/neo4j.mdx @@ -46,6 +46,8 @@ Execute MATCH queries to read nodes and relationships from Neo4j graph database. | `records` | array | Array of records returned from the query | | `recordCount` | number | Number of records returned | | `summary` | json | Query execution summary with timing and counters | +| `truncated` | boolean | Present and true only when records were dropped to stay inside the response ceilings \(10,000 records or 10 MiB\). Absent means the result is complete | +| `truncationReason` | string | Which ceiling was hit and how to read the rest | @@ -189,6 +191,8 @@ Execute arbitrary Cypher queries on Neo4j graph database for complex operations | `records` | array | Array of records returned from the query | | `recordCount` | number | Number of records returned | | `summary` | json | Execution summary with timing and counters | +| `truncated` | boolean | Present and true only when records were dropped to stay inside the response ceilings \(10,000 records or 10 MiB\). Absent means the result is complete | +| `truncationReason` | string | Which ceiling was hit and how to read the rest | diff --git a/apps/docs/content/docs/integrations/postgresql.mdx b/apps/docs/content/docs/integrations/postgresql.mdx index c1237fda4c7..a43d6d05e32 100644 --- a/apps/docs/content/docs/integrations/postgresql.mdx +++ b/apps/docs/content/docs/integrations/postgresql.mdx @@ -48,6 +48,8 @@ Execute a SELECT query on PostgreSQL database | `message` | string | Operation status message | | `rows` | array | Array of rows returned from the query | | `rowCount` | number | Number of rows returned | +| `truncated` | boolean | Present and true only when rows were dropped to stay inside the response ceilings \(10,000 rows or 10 MiB\). Absent means the result is complete | +| `truncationReason` | string | Which ceiling was hit and how to read the rest | @@ -71,6 +73,8 @@ Insert data into PostgreSQL database | `message` | string | Operation status message | | `rows` | array | Inserted data \(if RETURNING clause used\) | | `rowCount` | number | Number of rows inserted | +| `truncated` | boolean | Present and true only when rows were dropped to stay inside the response ceilings \(10,000 rows or 10 MiB\). Absent means the result is complete | +| `truncationReason` | string | Which ceiling was hit and how to read the rest | @@ -95,6 +99,8 @@ Update data in PostgreSQL database | `message` | string | Operation status message | | `rows` | array | Updated data \(if RETURNING clause used\) | | `rowCount` | number | Number of rows updated | +| `truncated` | boolean | Present and true only when rows were dropped to stay inside the response ceilings \(10,000 rows or 10 MiB\). Absent means the result is complete | +| `truncationReason` | string | Which ceiling was hit and how to read the rest | @@ -118,6 +124,8 @@ Delete data from PostgreSQL database | `message` | string | Operation status message | | `rows` | array | Deleted data \(if RETURNING clause used\) | | `rowCount` | number | Number of rows deleted | +| `truncated` | boolean | Present and true only when rows were dropped to stay inside the response ceilings \(10,000 rows or 10 MiB\). Absent means the result is complete | +| `truncationReason` | string | Which ceiling was hit and how to read the rest | @@ -140,6 +148,8 @@ Execute raw SQL query on PostgreSQL database | `message` | string | Operation status message | | `rows` | array | Array of rows returned from the query | | `rowCount` | number | Number of rows affected | +| `truncated` | boolean | Present and true only when rows were dropped to stay inside the response ceilings \(10,000 rows or 10 MiB\). Absent means the result is complete | +| `truncationReason` | string | Which ceiling was hit and how to read the rest | diff --git a/apps/sim/blocks/blocks/mongodb.ts b/apps/sim/blocks/blocks/mongodb.ts index c0743ea5593..6af751932b5 100644 --- a/apps/sim/blocks/blocks/mongodb.ts +++ b/apps/sim/blocks/blocks/mongodb.ts @@ -959,6 +959,14 @@ Return ONLY the MongoDB query filter as valid JSON - no explanations, no markdow type: 'number', description: 'Number of documents affected by the operation', }, + truncated: { + type: 'boolean', + description: 'True when the result hit a row or byte ceiling and documents were dropped', + }, + truncationReason: { + type: 'string', + description: 'Explanation of the ceiling that truncated the result', + }, insertedId: { type: 'string', description: 'ID of the inserted document (single insert)', diff --git a/apps/sim/blocks/blocks/mysql.ts b/apps/sim/blocks/blocks/mysql.ts index 18ebb35a8e4..0a27f2a3a93 100644 --- a/apps/sim/blocks/blocks/mysql.ts +++ b/apps/sim/blocks/blocks/mysql.ts @@ -412,6 +412,14 @@ Return ONLY the SQL query - no explanations, no markdown, no extra text.`, type: 'number', description: 'Number of rows affected by the operation', }, + truncated: { + type: 'boolean', + description: 'True when the result hit a row or byte ceiling and rows were dropped', + }, + truncationReason: { + type: 'string', + description: 'Explanation of the ceiling that truncated the result', + }, }, } diff --git a/apps/sim/blocks/blocks/neo4j.ts b/apps/sim/blocks/blocks/neo4j.ts index ae525f2e75e..11030f5e89a 100644 --- a/apps/sim/blocks/blocks/neo4j.ts +++ b/apps/sim/blocks/blocks/neo4j.ts @@ -706,6 +706,14 @@ Return ONLY valid JSON.`, type: 'number', description: 'Number of records returned or affected', }, + truncated: { + type: 'boolean', + description: 'True when the result hit a row or byte ceiling and records were dropped', + }, + truncationReason: { + type: 'string', + description: 'Explanation of the ceiling that truncated the result', + }, summary: { type: 'json', description: 'Execution summary with timing and database change counters', diff --git a/apps/sim/blocks/blocks/postgresql.ts b/apps/sim/blocks/blocks/postgresql.ts index 955b69f1a66..122ccb086cd 100644 --- a/apps/sim/blocks/blocks/postgresql.ts +++ b/apps/sim/blocks/blocks/postgresql.ts @@ -427,6 +427,14 @@ Return ONLY the SQL query - no explanations, no markdown, no extra text.`, type: 'number', description: 'Number of rows affected by the operation', }, + truncated: { + type: 'boolean', + description: 'True when the result hit a row or byte ceiling and rows were dropped', + }, + truncationReason: { + type: 'string', + description: 'Explanation of the ceiling that truncated the result', + }, tables: { type: 'array', description: 'Array of table schemas with columns, keys, and indexes (introspect operation)', diff --git a/apps/sim/lib/internal/mongodb/operations.ts b/apps/sim/lib/internal/mongodb/operations.ts index 6807224259d..874e92da2b6 100644 --- a/apps/sim/lib/internal/mongodb/operations.ts +++ b/apps/sim/lib/internal/mongodb/operations.ts @@ -23,9 +23,45 @@ import type { MongodbQueryInput, MongodbUpdateInput, } from '@/lib/internal/mongodb/schema' +import { + createResultRowsBudget, + resultTruncationReason, +} from '@/lib/internal/tool-operations/response-limits' export class MongodbOperationInputError extends Error {} +/** + * Reads documents until the response budget is spent. Leaving the loop early closes the server + * cursor, so an unbounded find or aggregation never materializes past the ceiling. + */ +async function readCappedDocuments( + cursor: AsyncIterable, + describe: (count: number) => string +) { + const budget = createResultRowsBudget() + const documents: Document[] = [] + let truncated = false + for await (const document of cursor) { + if (!budget.admit(document)) { + truncated = true + break + } + documents.push(document) + } + const message = describe(documents.length) + if (!truncated) return { message, documents } + const truncationReason = resultTruncationReason( + documents.length, + 'Narrow the filter or page with $skip and $limit to read the rest.' + ) + return { + message: `${message} ${truncationReason}`, + documents, + truncated: true, + truncationReason, + } +} + async function withMongodbClient( input: MongodbConnectionConfig, signal: AbortSignal | undefined, @@ -71,13 +107,9 @@ export function executeMongodbQuery(input: MongodbQueryInput, signal?: AbortSign } cursor = cursor.limit(input.limit || 100) - const documents = await cursor.toArray() + const result = await readCappedDocuments(cursor, (count) => `Found ${count} documents`) signal?.throwIfAborted() - return { - message: `Found ${documents.length} documents`, - documents, - documentCount: documents.length, - } + return { ...result, documentCount: result.documents.length } }) } @@ -92,17 +124,12 @@ export function executeMongodbAggregation(input: MongodbExecuteInput, signal?: A const pipeline = JSON.parse(input.pipeline) as Document[] return withMongodbClient(input, signal, async (client) => { - const documents = await client - .db(input.database) - .collection(collectionName) - .aggregate(pipeline, { signal }) - .toArray() + const result = await readCappedDocuments( + client.db(input.database).collection(collectionName).aggregate(pipeline, { signal }), + (count) => `Aggregation completed, returned ${count} documents` + ) signal?.throwIfAborted() - return { - message: `Aggregation completed, returned ${documents.length} documents`, - documents, - documentCount: documents.length, - } + return { ...result, documentCount: result.documents.length } }) } diff --git a/apps/sim/lib/internal/mysql/client.test.ts b/apps/sim/lib/internal/mysql/client.test.ts index 06b4eab31a8..bfefc67135c 100644 --- a/apps/sim/lib/internal/mysql/client.test.ts +++ b/apps/sim/lib/internal/mysql/client.test.ts @@ -85,6 +85,39 @@ describe('MySQL client', () => { expect(socket.setNoDelay).toHaveBeenCalledWith(true) }) + it('connects unencrypted for ssl=preferred when the server does not offer TLS', async () => { + const connection = { end: vi.fn(), destroy: vi.fn() } + mockCreateConnection + .mockRejectedValueOnce( + Object.assign(new Error('Server does not support secure connection'), { + code: 'HANDSHAKE_NO_SSL_SUPPORT', + }) + ) + .mockResolvedValueOnce(connection) + + await expect(createMysqlConnection({ ...CONNECTION_CONFIG, ssl: 'preferred' })).resolves.toBe( + connection + ) + expect(mockCreateConnection).toHaveBeenCalledTimes(2) + expect(mockCreateConnection.mock.calls[1][0].ssl).toBeUndefined() + }) + + it.each(['required', 'disabled'] as const)( + 'does not retry ssl=%s after a handshake failure', + async (ssl) => { + mockCreateConnection.mockRejectedValueOnce( + Object.assign(new Error('Server does not support secure connection'), { + code: 'HANDSHAKE_NO_SSL_SUPPORT', + }) + ) + + await expect(createMysqlConnection({ ...CONNECTION_CONFIG, ssl })).rejects.toThrow( + 'Server does not support secure connection' + ) + expect(mockCreateConnection).toHaveBeenCalledTimes(1) + } + ) + it.each([ new Map([['key', 'value']]), new Set(['value']), diff --git a/apps/sim/lib/internal/mysql/client.ts b/apps/sim/lib/internal/mysql/client.ts index 3fd425bba5c..82deb218b5c 100644 --- a/apps/sim/lib/internal/mysql/client.ts +++ b/apps/sim/lib/internal/mysql/client.ts @@ -1,4 +1,5 @@ import net from 'node:net' +import { isRecordLike } from '@sim/utils/object' import mysql from 'mysql2/promise' import { validateDatabaseHost } from '@/lib/core/security/input-validation.server' @@ -40,6 +41,15 @@ function assertMysqlExecuteValues( } } +function isNoSslSupportError(error: unknown): boolean { + return isRecordLike(error) && error.code === 'HANDSHAKE_NO_SSL_SUPPORT' +} + +/** + * Connects with the requested TLS mode. mysql2 has no "prefer" mode (with `ssl` set it refuses a + * server without TLS), so `preferred` retries unencrypted only when the server does not offer + * TLS, matching MySQL's own `ssl-mode=PREFERRED`. + */ export async function createMysqlConnection( config: MysqlConnectionConfig, signal?: AbortSignal @@ -77,7 +87,11 @@ export async function createMysqlConnection( } try { - const connection = await mysql.createConnection(connectionConfig) + const connection = await mysql.createConnection(connectionConfig).catch((error: unknown) => { + if (config.ssl !== 'preferred' || !isNoSslSupportError(error)) throw error + signal?.throwIfAborted() + return mysql.createConnection({ ...connectionConfig, ssl: undefined }) + }) signal?.throwIfAborted() return connection } catch (error) { diff --git a/apps/sim/lib/internal/mysql/operations.ts b/apps/sim/lib/internal/mysql/operations.ts index 48f215d33c7..e4f17ade40d 100644 --- a/apps/sim/lib/internal/mysql/operations.ts +++ b/apps/sim/lib/internal/mysql/operations.ts @@ -15,6 +15,7 @@ import type { MysqlQueryInput, MysqlUpdateInput, } from '@/lib/internal/mysql/schema' +import { toRowsResponseBody } from '@/lib/internal/tool-operations/response-limits' export class MysqlOperationInputError extends Error {} @@ -45,11 +46,10 @@ export function executeMysqlQuery(input: MysqlQueryInput, signal?: AbortSignal) return withMysqlConnection(input, signal, async (connection) => { const result = await queryMysql(connection, input.query, undefined, signal) - return { - message: `Query executed successfully. ${result.rowCount} row(s) returned.`, - rows: result.rows, - rowCount: result.rowCount, - } + return toRowsResponseBody( + result, + `Query executed successfully. ${result.rowCount} row(s) returned.` + ) }) } @@ -58,11 +58,10 @@ export function executeMysqlStatement(input: MysqlExecuteInput, signal?: AbortSi return withMysqlConnection(input, signal, async (connection) => { const result = await queryMysql(connection, input.query, undefined, signal) - return { - message: `SQL executed successfully. ${result.rowCount} row(s) affected.`, - rows: result.rows, - rowCount: result.rowCount, - } + return toRowsResponseBody( + result, + `SQL executed successfully. ${result.rowCount} row(s) affected.` + ) }) } @@ -70,11 +69,10 @@ export function executeMysqlInsert(input: MysqlInsertInput, signal?: AbortSignal return withMysqlConnection(input, signal, async (connection) => { const { query, values } = buildMysqlInsertQuery(input.table, input.data) const result = await queryMysql(connection, query, values, signal) - return { - message: `Data inserted successfully. ${result.rowCount} row(s) affected.`, - rows: result.rows, - rowCount: result.rowCount, - } + return toRowsResponseBody( + result, + `Data inserted successfully. ${result.rowCount} row(s) affected.` + ) }) } @@ -82,11 +80,10 @@ export function executeMysqlUpdate(input: MysqlUpdateInput, signal?: AbortSignal return withMysqlConnection(input, signal, async (connection) => { const { query, values } = buildMysqlUpdateQuery(input.table, input.data, input.where) const result = await queryMysql(connection, query, values, signal) - return { - message: `Data updated successfully. ${result.rowCount} row(s) affected.`, - rows: result.rows, - rowCount: result.rowCount, - } + return toRowsResponseBody( + result, + `Data updated successfully. ${result.rowCount} row(s) affected.` + ) }) } @@ -94,11 +91,10 @@ export function executeMysqlDelete(input: MysqlDeleteInput, signal?: AbortSignal return withMysqlConnection(input, signal, async (connection) => { const { query, values } = buildMysqlDeleteQuery(input.table, input.where) const result = await queryMysql(connection, query, values, signal) - return { - message: `Data deleted successfully. ${result.rowCount} row(s) affected.`, - rows: result.rows, - rowCount: result.rowCount, - } + return toRowsResponseBody( + result, + `Data deleted successfully. ${result.rowCount} row(s) affected.` + ) }) } diff --git a/apps/sim/lib/internal/mysql/queries.ts b/apps/sim/lib/internal/mysql/queries.ts index d72a1c80a4e..4a1fcedd067 100644 --- a/apps/sim/lib/internal/mysql/queries.ts +++ b/apps/sim/lib/internal/mysql/queries.ts @@ -1,10 +1,9 @@ import type mysql from 'mysql2/promise' import { executeMysqlCommand } from '@/lib/internal/mysql/client' - -export interface MysqlRowsResult { - rows: unknown[] - rowCount: number -} +import { + type CappedRowsResult, + capStatementRows, +} from '@/lib/internal/tool-operations/response-limits' export interface MysqlIntrospectionResult { tables: Array<{ @@ -76,14 +75,11 @@ export async function queryMysql( query: string, values?: unknown[], signal?: AbortSignal -): Promise { +): Promise { const result = await executeMysqlCommand(connection, query, values, signal) if (Array.isArray(result)) { - return { - rows: result, - rowCount: result.length, - } + return capStatementRows(result, result.length, 'Page with LIMIT and OFFSET to read the rest.') } return { diff --git a/apps/sim/lib/internal/mysql/schema.ts b/apps/sim/lib/internal/mysql/schema.ts index 13e63f7d3db..4524c2e0b50 100644 --- a/apps/sim/lib/internal/mysql/schema.ts +++ b/apps/sim/lib/internal/mysql/schema.ts @@ -4,7 +4,8 @@ import { z } from 'zod' const sslModeSchema = z.enum(['disabled', 'required', 'preferred']).default('preferred') /** * TLS mode for a MySQL connection: `required` encrypts and verifies the server certificate, - * `preferred` encrypts without verifying it, and `disabled` connects without TLS. + * `preferred` encrypts without verifying it when the server offers TLS and connects unencrypted + * when it does not, and `disabled` connects without TLS. */ export type MysqlSslMode = z.output diff --git a/apps/sim/lib/internal/neo4j/operations.ts b/apps/sim/lib/internal/neo4j/operations.ts index 476b1151072..662db7cc3f9 100644 --- a/apps/sim/lib/internal/neo4j/operations.ts +++ b/apps/sim/lib/internal/neo4j/operations.ts @@ -11,6 +11,10 @@ import type { } from '@/lib/api/contracts/tools/databases/neo4j' import { createNeo4jDriver } from '@/lib/internal/neo4j/client' import { convertNeo4jValue } from '@/lib/internal/neo4j/values' +import { + capResultRows, + resultTruncationReason, +} from '@/lib/internal/tool-operations/response-limits' import type { Neo4jNodeSchema, Neo4jRelationshipSchema } from '@/tools/neo4j/types' const logger = createLogger('Neo4jOperations') @@ -75,22 +79,35 @@ function projectSummary(result: QueryResult) { } function statementResponse(kind: StatementKind, result: QueryResult) { - const records = projectRecords(result) + const { rows: records, truncated } = capResultRows(projectRecords(result)) const summary = projectSummary(result) + const truncation = truncated + ? { + truncated: true, + truncationReason: resultTruncationReason( + records.length, + 'Page with SKIP and LIMIT to read the rest.' + ), + } + : undefined + const withTruncation = (message: string) => + truncation ? `${message} ${truncation.truncationReason}` : message switch (kind) { case 'query': return { - message: `Found ${records.length} records`, + message: withTruncation(`Found ${records.length} records`), records, recordCount: records.length, summary, + ...truncation, } case 'execute': return { - message: `Query executed successfully, returned ${records.length} records`, + message: withTruncation(`Query executed successfully, returned ${records.length} records`), records, recordCount: records.length, summary, + ...truncation, } case 'create': return { diff --git a/apps/sim/lib/internal/postgresql/operations.ts b/apps/sim/lib/internal/postgresql/operations.ts index f07ac3f9bf0..f4976c09647 100644 --- a/apps/sim/lib/internal/postgresql/operations.ts +++ b/apps/sim/lib/internal/postgresql/operations.ts @@ -15,6 +15,7 @@ import type { PostgresqlQueryInput, PostgresqlUpdateInput, } from '@/lib/internal/postgresql/schema' +import { toRowsResponseBody } from '@/lib/internal/tool-operations/response-limits' import type { PostgresConnectionConfig } from '@/tools/postgresql/types' export class PostgresqlOperationInputError extends Error {} @@ -35,11 +36,10 @@ async function withPostgresClient( export function executePostgresqlQuery(input: PostgresqlQueryInput, signal?: AbortSignal) { return withPostgresClient(input, signal, async (client) => { const result = await queryPostgres(client, input.query, [], signal) - return { - message: `Query executed successfully. ${result.rowCount} row(s) returned.`, - rows: result.rows, - rowCount: result.rowCount, - } + return toRowsResponseBody( + result, + `Query executed successfully. ${result.rowCount} row(s) returned.` + ) }) } @@ -53,44 +53,40 @@ export function executePostgresqlStatement(input: PostgresqlExecuteInput, signal return withPostgresClient(input, signal, async (client) => { const result = await queryPostgres(client, input.query, [], signal) - return { - message: `SQL executed successfully. ${result.rowCount} row(s) affected.`, - rows: result.rows, - rowCount: result.rowCount, - } + return toRowsResponseBody( + result, + `SQL executed successfully. ${result.rowCount} row(s) affected.` + ) }) } export function executePostgresqlInsert(input: PostgresqlInsertInput, signal?: AbortSignal) { return withPostgresClient(input, signal, async (client) => { const result = await insertPostgresRows(client, input.table, input.data, signal) - return { - message: `Data inserted successfully. ${result.rowCount} row(s) affected.`, - rows: result.rows, - rowCount: result.rowCount, - } + return toRowsResponseBody( + result, + `Data inserted successfully. ${result.rowCount} row(s) affected.` + ) }) } export function executePostgresqlUpdate(input: PostgresqlUpdateInput, signal?: AbortSignal) { return withPostgresClient(input, signal, async (client) => { const result = await updatePostgresRows(client, input.table, input.data, input.where, signal) - return { - message: `Data updated successfully. ${result.rowCount} row(s) affected.`, - rows: result.rows, - rowCount: result.rowCount, - } + return toRowsResponseBody( + result, + `Data updated successfully. ${result.rowCount} row(s) affected.` + ) }) } export function executePostgresqlDelete(input: PostgresqlDeleteInput, signal?: AbortSignal) { return withPostgresClient(input, signal, async (client) => { const result = await deletePostgresRows(client, input.table, input.where, signal) - return { - message: `Data deleted successfully. ${result.rowCount} row(s) affected.`, - rows: result.rows, - rowCount: result.rowCount, - } + return toRowsResponseBody( + result, + `Data deleted successfully. ${result.rowCount} row(s) affected.` + ) }) } diff --git a/apps/sim/lib/internal/postgresql/queries.test.ts b/apps/sim/lib/internal/postgresql/queries.test.ts index 118cfea29b5..06107b16192 100644 --- a/apps/sim/lib/internal/postgresql/queries.test.ts +++ b/apps/sim/lib/internal/postgresql/queries.test.ts @@ -12,6 +12,7 @@ import type { PostgresClient } from '@/lib/internal/postgresql/client' import { deletePostgresRows, insertPostgresRows, + queryPostgres, sanitizePostgresIdentifier, updatePostgresRows, validatePostgresQuery, @@ -46,6 +47,21 @@ describe('PostgreSQL queries', () => { expect(mockExecutePostgresQuery).toHaveBeenCalledWith(pendingQuery, controller.signal) }) + it('caps a large SELECT and keeps the full row count', async () => { + const rows = Object.assign( + Array.from({ length: 10_001 }, (_, id) => ({ id })), + { count: 10_001 } + ) + mockExecutePostgresQuery.mockResolvedValue(rows) + + const result = await queryPostgres(mockClient, 'SELECT id FROM users') + + expect(result.rows).toHaveLength(10_000) + expect(result.rowCount).toBe(10_001) + expect(result.truncated).toBe(true) + expect(result.truncationReason).toContain('LIMIT and OFFSET') + }) + it('builds parameterized UPDATE values while preserving the existing WHERE syntax', async () => { await updatePostgresRows(mockClient, 'users', { active: false }, 'id = 42') diff --git a/apps/sim/lib/internal/postgresql/queries.ts b/apps/sim/lib/internal/postgresql/queries.ts index 11939fad59c..9bfc3fc5787 100644 --- a/apps/sim/lib/internal/postgresql/queries.ts +++ b/apps/sim/lib/internal/postgresql/queries.ts @@ -1,10 +1,9 @@ import type { PostgresClient } from '@/lib/internal/postgresql/client' import { executePostgresQuery } from '@/lib/internal/postgresql/client' - -export interface PostgresRowsResult { - rows: unknown[] - rowCount: number -} +import { + type CappedRowsResult, + capStatementRows, +} from '@/lib/internal/tool-operations/response-limits' export interface PostgresIntrospectionResult { tables: Array<{ @@ -75,16 +74,17 @@ export async function queryPostgres( query: string, params: unknown[] = [], signal?: AbortSignal -): Promise { +): Promise { type PostgresParameters = NonNullable[1]> const result = await executePostgresQuery( client.unsafe(query, params as PostgresParameters), signal ) - return { - rows: result, - rowCount: result.count ?? result.length ?? 0, - } + return capStatementRows( + result, + result.count ?? result.length ?? 0, + 'Page with LIMIT and OFFSET to read the rest.' + ) } export function validatePostgresQuery(query: string): { isValid: boolean; error?: string } { @@ -118,7 +118,7 @@ export async function insertPostgresRows( table: string, data: Record, signal?: AbortSignal -): Promise { +): Promise { const sanitizedTable = sanitizePostgresIdentifier(table) const columns = Object.keys(data) const sanitizedColumns = columns.map((column) => sanitizePostgresIdentifier(column)) @@ -135,7 +135,7 @@ export async function updatePostgresRows( data: Record, where: string, signal?: AbortSignal -): Promise { +): Promise { validateWhereClause(where) const sanitizedTable = sanitizePostgresIdentifier(table) @@ -153,7 +153,7 @@ export async function deletePostgresRows( table: string, where: string, signal?: AbortSignal -): Promise { +): Promise { validateWhereClause(where) const sanitizedTable = sanitizePostgresIdentifier(table) diff --git a/apps/sim/lib/internal/tool-operations/response-limits.test.ts b/apps/sim/lib/internal/tool-operations/response-limits.test.ts new file mode 100644 index 00000000000..0049f94557b --- /dev/null +++ b/apps/sim/lib/internal/tool-operations/response-limits.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from 'vitest' +import { + capResultRows, + MAX_TOOL_RESPONSE_BODY_BYTES, + toRowsResponseBody, +} from '@/lib/internal/tool-operations/response-limits' + +describe('database result caps', () => { + it('keeps every row of a result inside both ceilings', () => { + const rows = Array.from({ length: 10_000 }, (_, id) => ({ id })) + expect(capResultRows(rows)).toEqual({ rows, truncated: false }) + }) + + it('stops at the row ceiling', () => { + const rows = Array.from({ length: 10_001 }, (_, id) => ({ id })) + const capped = capResultRows(rows) + expect(capped.truncated).toBe(true) + expect(capped.rows).toHaveLength(10_000) + }) + + it('measures UTF-8 bytes, so multibyte text cannot exceed the response ceiling', () => { + const value = '漢'.repeat(100_000) + const capped = capResultRows(Array.from({ length: 50 }, () => ({ value }))) + expect(capped.truncated).toBe(true) + const body = JSON.stringify(toRowsResponseBody({ ...capped, rowCount: 50 }, 'Query done.')) + expect(Buffer.byteLength(body, 'utf8')).toBeLessThanOrEqual(MAX_TOOL_RESPONSE_BODY_BYTES) + }) + + it('drops a single row larger than the ceiling instead of admitting it', () => { + const capped = capResultRows([{ blob: 'x'.repeat(MAX_TOOL_RESPONSE_BODY_BYTES) }]) + expect(capped).toEqual({ rows: [], truncated: true }) + }) +}) diff --git a/apps/sim/lib/internal/tool-operations/response-limits.ts b/apps/sim/lib/internal/tool-operations/response-limits.ts index fb8828b4ef1..f93738615f3 100644 --- a/apps/sim/lib/internal/tool-operations/response-limits.ts +++ b/apps/sim/lib/internal/tool-operations/response-limits.ts @@ -1,2 +1,85 @@ /** Maximum inline tool response size; binary file outputs use stored descriptors. */ export const MAX_TOOL_RESPONSE_BODY_BYTES = 10 * 1024 * 1024 + +/** Most rows a single database statement returns inline. */ +const MAX_TOOL_RESULT_ROWS = 10_000 + +/** + * Bytes the serialized rows array may occupy: the response ceiling minus a reserve for the + * envelope around it (`message`, counts, `truncated`, `truncationReason`). + */ +const MAX_TOOL_RESULT_ROWS_BYTES = MAX_TOOL_RESPONSE_BODY_BYTES - 4096 + +/** + * Admits rows into a tool response until the row or UTF-8 byte ceiling is reached. Each row is + * serialized once, the way the response will serialize it, and one byte per row covers the array + * punctuation. A row that does not fit is refused even when it is the first, so one huge value + * cannot bypass the ceiling. + */ +export function createResultRowsBudget() { + let admitted = 0 + let bytes = 1 + return { + admit(row: unknown): boolean { + if (admitted >= MAX_TOOL_RESULT_ROWS) return false + const serialized = JSON.stringify(row) + const rowBytes = serialized === undefined ? 4 : Buffer.byteLength(serialized, 'utf8') + if (bytes + rowBytes + 1 > MAX_TOOL_RESULT_ROWS_BYTES) return false + bytes += rowBytes + 1 + admitted++ + return true + }, + } +} + +/** Keeps the leading rows that fit {@link createResultRowsBudget}. */ +export function capResultRows(rows: readonly TRow[]): { rows: TRow[]; truncated: boolean } { + const budget = createResultRowsBudget() + const capped: TRow[] = [] + for (const row of rows) { + if (!budget.admit(row)) break + capped.push(row) + } + return { rows: capped, truncated: capped.length < rows.length } +} + +/** Tells the caller which ceiling truncated a result and how to read the rest. */ +export function resultTruncationReason(returned: number, paginationHint: string): string { + return returned === 0 + ? 'No rows returned: the first row alone exceeds the 10 MiB response ceiling. Select fewer columns or slice large values.' + : `Result truncated to ${returned} row(s): one statement returns at most ${MAX_TOOL_RESULT_ROWS.toLocaleString('en-US')} rows or 10 MiB. ${paginationHint}` +} + +/** A capped statement result: rows as returned, plus the truncation notice when rows were dropped. */ +export interface CappedRowsResult { + rows: unknown[] + rowCount: number + truncated?: true + truncationReason?: string +} + +/** Caps a statement's rows; `rowCount` stays what the statement returned or affected. */ +export function capStatementRows( + rows: readonly unknown[], + rowCount: number, + paginationHint: string +): CappedRowsResult { + const capped = capResultRows(rows) + if (!capped.truncated) return { rows: capped.rows, rowCount } + return { + rows: capped.rows, + rowCount, + truncated: true, + truncationReason: resultTruncationReason(capped.rows.length, paginationHint), + } +} + +/** The response body for a capped statement, surfacing any truncation in the message too. */ +export function toRowsResponseBody(result: CappedRowsResult, message: string) { + return { + message: result.truncationReason ? `${message} ${result.truncationReason}` : message, + rows: result.rows, + rowCount: result.rowCount, + ...(result.truncated && { truncated: true, truncationReason: result.truncationReason }), + } +} diff --git a/apps/sim/tools/mongodb/execute.ts b/apps/sim/tools/mongodb/execute.ts index afe0f38c9dc..762b85fd074 100644 --- a/apps/sim/tools/mongodb/execute.ts +++ b/apps/sim/tools/mongodb/execute.ts @@ -91,6 +91,10 @@ export const executeTool: InternalToolConfig message: data.message || 'Query executed successfully', documents: data.documents || [], documentCount: data.documentCount || 0, + ...(data.truncated && { + truncated: true, + truncationReason: data.truncationReason, + }), }, error: undefined, } @@ -114,5 +118,16 @@ export const queryTool: InternalToolConfig message: { type: 'string', description: 'Operation status message' }, documents: { type: 'array', description: 'Array of documents returned from the query' }, documentCount: { type: 'number', description: 'Number of documents returned' }, + truncated: { + type: 'boolean', + description: + 'Present and true only when documents were dropped to stay inside the response ceilings (10,000 documents or 10 MiB). Absent means the result is complete', + optional: true, + }, + truncationReason: { + type: 'string', + description: 'Which ceiling was hit and how to read the rest', + optional: true, + }, }, } diff --git a/apps/sim/tools/mongodb/types.ts b/apps/sim/tools/mongodb/types.ts index b20a59aa9fc..be559b735b8 100644 --- a/apps/sim/tools/mongodb/types.ts +++ b/apps/sim/tools/mongodb/types.ts @@ -68,6 +68,8 @@ interface MongoDBBaseResponse extends ToolResponse { message: string documents?: unknown[] documentCount: number + truncated?: boolean + truncationReason?: string insertedId?: string insertedIds?: string[] modifiedCount?: number diff --git a/apps/sim/tools/mysql/execute.ts b/apps/sim/tools/mysql/execute.ts index 580a9c7b032..0b8ec83a6a2 100644 --- a/apps/sim/tools/mysql/execute.ts +++ b/apps/sim/tools/mysql/execute.ts @@ -80,6 +80,10 @@ export const executeTool: InternalToolConfig message: data.message || 'Query executed successfully', rows: data.rows || [], rowCount: data.rowCount || 0, + ...(data.truncated && { + truncated: true, + truncationReason: data.truncationReason, + }), }, error: undefined, } @@ -89,5 +93,16 @@ export const executeTool: InternalToolConfig message: { type: 'string', description: 'Operation status message' }, rows: { type: 'array', description: 'Array of rows returned from the query' }, rowCount: { type: 'number', description: 'Number of rows affected' }, + truncated: { + type: 'boolean', + description: + 'Present and true only when rows were dropped to stay inside the response ceilings (10,000 rows or 10 MiB). Absent means the result is complete', + optional: true, + }, + truncationReason: { + type: 'string', + description: 'Which ceiling was hit and how to read the rest', + optional: true, + }, }, } diff --git a/apps/sim/tools/mysql/query.ts b/apps/sim/tools/mysql/query.ts index 41ff5d4d06a..8bed01fcd40 100644 --- a/apps/sim/tools/mysql/query.ts +++ b/apps/sim/tools/mysql/query.ts @@ -79,6 +79,10 @@ export const queryTool: InternalToolConfig = { message: data.message || 'Query executed successfully', rows: data.rows || [], rowCount: data.rowCount || 0, + ...(data.truncated && { + truncated: true, + truncationReason: data.truncationReason, + }), }, error: undefined, } @@ -88,5 +92,16 @@ export const queryTool: InternalToolConfig = { message: { type: 'string', description: 'Operation status message' }, rows: { type: 'array', description: 'Array of rows returned from the query' }, rowCount: { type: 'number', description: 'Number of rows returned' }, + truncated: { + type: 'boolean', + description: + 'Present and true only when rows were dropped to stay inside the response ceilings (10,000 rows or 10 MiB). Absent means the result is complete', + optional: true, + }, + truncationReason: { + type: 'string', + description: 'Which ceiling was hit and how to read the rest', + optional: true, + }, }, } diff --git a/apps/sim/tools/mysql/types.ts b/apps/sim/tools/mysql/types.ts index be1100ab831..89fcdc9d50d 100644 --- a/apps/sim/tools/mysql/types.ts +++ b/apps/sim/tools/mysql/types.ts @@ -38,6 +38,8 @@ interface MySQLBaseResponse extends ToolResponse { message: string rows: unknown[] rowCount: number + truncated?: boolean + truncationReason?: string } error?: string } diff --git a/apps/sim/tools/neo4j/execute.ts b/apps/sim/tools/neo4j/execute.ts index a8ecb09444c..8ff0322b0ec 100644 --- a/apps/sim/tools/neo4j/execute.ts +++ b/apps/sim/tools/neo4j/execute.ts @@ -87,6 +87,10 @@ export const executeTool: InternalToolConfig records: data.records || [], recordCount: data.recordCount || 0, summary: data.summary, + ...(data.truncated && { + truncated: true, + truncationReason: data.truncationReason, + }), }, error: undefined, } @@ -97,5 +101,16 @@ export const executeTool: InternalToolConfig records: { type: 'array', description: 'Array of records returned from the query' }, recordCount: { type: 'number', description: 'Number of records returned' }, summary: { type: 'json', description: 'Execution summary with timing and counters' }, + truncated: { + type: 'boolean', + description: + 'Present and true only when records were dropped to stay inside the response ceilings (10,000 records or 10 MiB). Absent means the result is complete', + optional: true, + }, + truncationReason: { + type: 'string', + description: 'Which ceiling was hit and how to read the rest', + optional: true, + }, }, } diff --git a/apps/sim/tools/neo4j/query.ts b/apps/sim/tools/neo4j/query.ts index 68895e345af..277cb7a2150 100644 --- a/apps/sim/tools/neo4j/query.ts +++ b/apps/sim/tools/neo4j/query.ts @@ -88,6 +88,10 @@ export const queryTool: InternalToolConfig = { records: data.records || [], recordCount: data.recordCount || 0, summary: data.summary, + ...(data.truncated && { + truncated: true, + truncationReason: data.truncationReason, + }), }, error: undefined, } @@ -98,5 +102,16 @@ export const queryTool: InternalToolConfig = { records: { type: 'array', description: 'Array of records returned from the query' }, recordCount: { type: 'number', description: 'Number of records returned' }, summary: { type: 'json', description: 'Query execution summary with timing and counters' }, + truncated: { + type: 'boolean', + description: + 'Present and true only when records were dropped to stay inside the response ceilings (10,000 records or 10 MiB). Absent means the result is complete', + optional: true, + }, + truncationReason: { + type: 'string', + description: 'Which ceiling was hit and how to read the rest', + optional: true, + }, }, } diff --git a/apps/sim/tools/neo4j/types.ts b/apps/sim/tools/neo4j/types.ts index ce5b70f4565..5d74922616d 100644 --- a/apps/sim/tools/neo4j/types.ts +++ b/apps/sim/tools/neo4j/types.ts @@ -45,6 +45,8 @@ interface Neo4jBaseResponse extends ToolResponse { message: string records?: unknown[] recordCount?: number + truncated?: boolean + truncationReason?: string summary?: { resultAvailableAfter: number resultConsumedAfter: number diff --git a/apps/sim/tools/postgresql/delete.ts b/apps/sim/tools/postgresql/delete.ts index 0f17bd09bca..4ae980a799c 100644 --- a/apps/sim/tools/postgresql/delete.ts +++ b/apps/sim/tools/postgresql/delete.ts @@ -86,6 +86,10 @@ export const deleteTool: InternalToolConfig `${params.restUrl}/del/${encodeURIComponent(params.key)}`, + url: (params) => `${params.restUrl}/del/${upstashPathSegment(params.key, 'key')}`, method: 'POST', headers: (params) => ({ Authorization: `Bearer ${params.restToken}`, diff --git a/apps/sim/tools/upstash/redis_exists.ts b/apps/sim/tools/upstash/redis_exists.ts index cf50887cc48..0d065b77080 100644 --- a/apps/sim/tools/upstash/redis_exists.ts +++ b/apps/sim/tools/upstash/redis_exists.ts @@ -1,6 +1,7 @@ import { UPSTASH_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' import type { UpstashRedisExistsParams, UpstashRedisExistsResponse } from '@/tools/upstash/types' +import { upstashPathSegment } from '@/tools/upstash/utils' export const upstashRedisExistsTool: ToolConfig< UpstashRedisExistsParams, @@ -35,7 +36,7 @@ export const upstashRedisExistsTool: ToolConfig< }, request: { - url: (params) => `${params.restUrl}/exists/${encodeURIComponent(params.key)}`, + url: (params) => `${params.restUrl}/exists/${upstashPathSegment(params.key, 'key')}`, method: 'GET', headers: (params) => ({ Authorization: `Bearer ${params.restToken}`, diff --git a/apps/sim/tools/upstash/redis_expire.ts b/apps/sim/tools/upstash/redis_expire.ts index eff08b7286b..990eaf15ea0 100644 --- a/apps/sim/tools/upstash/redis_expire.ts +++ b/apps/sim/tools/upstash/redis_expire.ts @@ -1,6 +1,7 @@ import { UPSTASH_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' import type { UpstashRedisExpireParams, UpstashRedisExpireResponse } from '@/tools/upstash/types' +import { upstashInteger, upstashPathSegment } from '@/tools/upstash/utils' export const upstashRedisExpireTool: ToolConfig< UpstashRedisExpireParams, @@ -40,7 +41,8 @@ export const upstashRedisExpireTool: ToolConfig< }, request: { - url: (params) => `${params.restUrl}/expire/${encodeURIComponent(params.key)}/${params.seconds}`, + url: (params) => + `${params.restUrl}/expire/${upstashPathSegment(params.key, 'key')}/${upstashInteger(params.seconds, 'seconds')}`, method: 'GET', headers: (params) => ({ Authorization: `Bearer ${params.restToken}`, diff --git a/apps/sim/tools/upstash/redis_get.ts b/apps/sim/tools/upstash/redis_get.ts index 265da4b28d2..5475e558251 100644 --- a/apps/sim/tools/upstash/redis_get.ts +++ b/apps/sim/tools/upstash/redis_get.ts @@ -1,6 +1,7 @@ import { UPSTASH_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' import type { UpstashRedisGetParams, UpstashRedisGetResponse } from '@/tools/upstash/types' +import { upstashPathSegment } from '@/tools/upstash/utils' export const upstashRedisGetTool: ToolConfig = { id: 'upstash_redis_get', @@ -31,7 +32,7 @@ export const upstashRedisGetTool: ToolConfig `${params.restUrl}/get/${encodeURIComponent(params.key)}`, + url: (params) => `${params.restUrl}/get/${upstashPathSegment(params.key, 'key')}`, method: 'GET', headers: (params) => ({ Authorization: `Bearer ${params.restToken}`, diff --git a/apps/sim/tools/upstash/redis_hget.ts b/apps/sim/tools/upstash/redis_hget.ts index f42f938bb3d..f28ad5ec1f5 100644 --- a/apps/sim/tools/upstash/redis_hget.ts +++ b/apps/sim/tools/upstash/redis_hget.ts @@ -1,6 +1,7 @@ import { UPSTASH_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' import type { UpstashRedisHGetParams, UpstashRedisHGetResponse } from '@/tools/upstash/types' +import { upstashPathSegment } from '@/tools/upstash/utils' export const upstashRedisHGetTool: ToolConfig = { id: 'upstash_redis_hget', @@ -38,7 +39,7 @@ export const upstashRedisHGetTool: ToolConfig - `${params.restUrl}/hget/${encodeURIComponent(params.key)}/${encodeURIComponent(params.field)}`, + `${params.restUrl}/hget/${upstashPathSegment(params.key, 'key')}/${upstashPathSegment(params.field, 'field')}`, method: 'GET', headers: (params) => ({ Authorization: `Bearer ${params.restToken}`, diff --git a/apps/sim/tools/upstash/redis_hgetall.ts b/apps/sim/tools/upstash/redis_hgetall.ts index d5478dc2660..431688aa4f9 100644 --- a/apps/sim/tools/upstash/redis_hgetall.ts +++ b/apps/sim/tools/upstash/redis_hgetall.ts @@ -1,6 +1,7 @@ import { UPSTASH_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' import type { UpstashRedisHGetAllParams, UpstashRedisHGetAllResponse } from '@/tools/upstash/types' +import { upstashPathSegment } from '@/tools/upstash/utils' export const upstashRedisHGetAllTool: ToolConfig< UpstashRedisHGetAllParams, @@ -34,7 +35,7 @@ export const upstashRedisHGetAllTool: ToolConfig< }, request: { - url: (params) => `${params.restUrl}/hgetall/${encodeURIComponent(params.key)}`, + url: (params) => `${params.restUrl}/hgetall/${upstashPathSegment(params.key, 'key')}`, method: 'GET', headers: (params) => ({ Authorization: `Bearer ${params.restToken}`, diff --git a/apps/sim/tools/upstash/redis_hset.ts b/apps/sim/tools/upstash/redis_hset.ts index e41505b2c89..a906708b67f 100644 --- a/apps/sim/tools/upstash/redis_hset.ts +++ b/apps/sim/tools/upstash/redis_hset.ts @@ -1,6 +1,7 @@ import { UPSTASH_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' import type { UpstashRedisHSetParams, UpstashRedisHSetResponse } from '@/tools/upstash/types' +import { upstashPathSegment } from '@/tools/upstash/utils' export const upstashRedisHSetTool: ToolConfig = { id: 'upstash_redis_hset', @@ -44,7 +45,7 @@ export const upstashRedisHSetTool: ToolConfig - `${params.restUrl}/hset/${encodeURIComponent(params.key)}/${encodeURIComponent(params.field)}`, + `${params.restUrl}/hset/${upstashPathSegment(params.key, 'key')}/${upstashPathSegment(params.field, 'field')}`, method: 'POST', headers: (params) => ({ Authorization: `Bearer ${params.restToken}`, diff --git a/apps/sim/tools/upstash/redis_incr.ts b/apps/sim/tools/upstash/redis_incr.ts index a5eb881fb24..752c5a991c7 100644 --- a/apps/sim/tools/upstash/redis_incr.ts +++ b/apps/sim/tools/upstash/redis_incr.ts @@ -1,6 +1,7 @@ import { UPSTASH_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' import type { UpstashRedisIncrParams, UpstashRedisIncrResponse } from '@/tools/upstash/types' +import { upstashPathSegment } from '@/tools/upstash/utils' export const upstashRedisIncrTool: ToolConfig = { id: 'upstash_redis_incr', @@ -32,7 +33,7 @@ export const upstashRedisIncrTool: ToolConfig `${params.restUrl}/incr/${encodeURIComponent(params.key)}`, + url: (params) => `${params.restUrl}/incr/${upstashPathSegment(params.key, 'key')}`, method: 'GET', headers: (params) => ({ Authorization: `Bearer ${params.restToken}`, diff --git a/apps/sim/tools/upstash/redis_incrby.ts b/apps/sim/tools/upstash/redis_incrby.ts index 64ef47a3513..eb11efc4001 100644 --- a/apps/sim/tools/upstash/redis_incrby.ts +++ b/apps/sim/tools/upstash/redis_incrby.ts @@ -1,6 +1,7 @@ import { UPSTASH_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' import type { UpstashRedisIncrbyParams, UpstashRedisIncrbyResponse } from '@/tools/upstash/types' +import { upstashInteger, upstashPathSegment } from '@/tools/upstash/utils' export const upstashRedisIncrbyTool: ToolConfig< UpstashRedisIncrbyParams, @@ -42,7 +43,7 @@ export const upstashRedisIncrbyTool: ToolConfig< request: { url: (params) => - `${params.restUrl}/incrby/${encodeURIComponent(params.key)}/${params.increment}`, + `${params.restUrl}/incrby/${upstashPathSegment(params.key, 'key')}/${upstashInteger(params.increment, 'increment')}`, method: 'GET', headers: (params) => ({ Authorization: `Bearer ${params.restToken}`, diff --git a/apps/sim/tools/upstash/redis_keys.ts b/apps/sim/tools/upstash/redis_keys.ts index 4cdca491888..27d27be07e1 100644 --- a/apps/sim/tools/upstash/redis_keys.ts +++ b/apps/sim/tools/upstash/redis_keys.ts @@ -1,6 +1,7 @@ import { UPSTASH_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' import type { UpstashRedisKeysParams, UpstashRedisKeysResponse } from '@/tools/upstash/types' +import { upstashPathSegment } from '@/tools/upstash/utils' export const upstashRedisKeysTool: ToolConfig = { id: 'upstash_redis_keys', @@ -33,7 +34,7 @@ export const upstashRedisKeysTool: ToolConfig { const pattern = params.pattern || '*' - return `${params.restUrl}/keys/${encodeURIComponent(pattern)}` + return `${params.restUrl}/keys/${upstashPathSegment(pattern, 'pattern')}` }, method: 'GET', headers: (params) => ({ diff --git a/apps/sim/tools/upstash/redis_lpush.ts b/apps/sim/tools/upstash/redis_lpush.ts index 351d3d3f955..916b03223a2 100644 --- a/apps/sim/tools/upstash/redis_lpush.ts +++ b/apps/sim/tools/upstash/redis_lpush.ts @@ -1,6 +1,7 @@ import { UPSTASH_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' import type { UpstashRedisLPushParams, UpstashRedisLPushResponse } from '@/tools/upstash/types' +import { upstashPathSegment } from '@/tools/upstash/utils' export const upstashRedisLPushTool: ToolConfig = { @@ -39,7 +40,7 @@ export const upstashRedisLPushTool: ToolConfig `${params.restUrl}/lpush/${encodeURIComponent(params.key)}`, + url: (params) => `${params.restUrl}/lpush/${upstashPathSegment(params.key, 'key')}`, method: 'POST', headers: (params) => ({ Authorization: `Bearer ${params.restToken}`, diff --git a/apps/sim/tools/upstash/redis_lrange.ts b/apps/sim/tools/upstash/redis_lrange.ts index 2a7c64ad422..bb542f76130 100644 --- a/apps/sim/tools/upstash/redis_lrange.ts +++ b/apps/sim/tools/upstash/redis_lrange.ts @@ -1,6 +1,7 @@ import { UPSTASH_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' import type { UpstashRedisLRangeParams, UpstashRedisLRangeResponse } from '@/tools/upstash/types' +import { upstashInteger, upstashPathSegment } from '@/tools/upstash/utils' export const upstashRedisLRangeTool: ToolConfig< UpstashRedisLRangeParams, @@ -48,7 +49,7 @@ export const upstashRedisLRangeTool: ToolConfig< request: { url: (params) => - `${params.restUrl}/lrange/${encodeURIComponent(params.key)}/${params.start}/${params.stop}`, + `${params.restUrl}/lrange/${upstashPathSegment(params.key, 'key')}/${upstashInteger(params.start, 'start')}/${upstashInteger(params.stop, 'stop')}`, method: 'GET', headers: (params) => ({ Authorization: `Bearer ${params.restToken}`, diff --git a/apps/sim/tools/upstash/redis_set.ts b/apps/sim/tools/upstash/redis_set.ts index 8699a7fac37..be072efeba6 100644 --- a/apps/sim/tools/upstash/redis_set.ts +++ b/apps/sim/tools/upstash/redis_set.ts @@ -1,6 +1,7 @@ import { UPSTASH_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' import type { UpstashRedisSetParams, UpstashRedisSetResponse } from '@/tools/upstash/types' +import { upstashInteger, upstashPathSegment } from '@/tools/upstash/utils' export const upstashRedisSetTool: ToolConfig = { id: 'upstash_redis_set', @@ -45,9 +46,9 @@ export const upstashRedisSetTool: ToolConfig { - const base = `${params.restUrl}/set/${encodeURIComponent(params.key)}` + const base = `${params.restUrl}/set/${upstashPathSegment(params.key, 'key')}` if (params.ex) { - return `${base}?EX=${params.ex}` + return `${base}?EX=${upstashInteger(params.ex, 'ex')}` } return base }, diff --git a/apps/sim/tools/upstash/redis_setnx.ts b/apps/sim/tools/upstash/redis_setnx.ts index 019f5e7f52e..d2c4772b2d8 100644 --- a/apps/sim/tools/upstash/redis_setnx.ts +++ b/apps/sim/tools/upstash/redis_setnx.ts @@ -1,6 +1,7 @@ import { UPSTASH_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' import type { UpstashRedisSetnxParams, UpstashRedisSetnxResponse } from '@/tools/upstash/types' +import { upstashPathSegment } from '@/tools/upstash/utils' export const upstashRedisSetnxTool: ToolConfig = { @@ -39,7 +40,7 @@ export const upstashRedisSetnxTool: ToolConfig `${params.restUrl}/setnx/${encodeURIComponent(params.key)}`, + url: (params) => `${params.restUrl}/setnx/${upstashPathSegment(params.key, 'key')}`, method: 'POST', headers: (params) => ({ Authorization: `Bearer ${params.restToken}`, diff --git a/apps/sim/tools/upstash/redis_ttl.ts b/apps/sim/tools/upstash/redis_ttl.ts index d283b839f2f..62ad690dc46 100644 --- a/apps/sim/tools/upstash/redis_ttl.ts +++ b/apps/sim/tools/upstash/redis_ttl.ts @@ -1,6 +1,7 @@ import { UPSTASH_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' import type { UpstashRedisTtlParams, UpstashRedisTtlResponse } from '@/tools/upstash/types' +import { upstashPathSegment } from '@/tools/upstash/utils' export const upstashRedisTtlTool: ToolConfig = { id: 'upstash_redis_ttl', @@ -32,7 +33,7 @@ export const upstashRedisTtlTool: ToolConfig `${params.restUrl}/ttl/${encodeURIComponent(params.key)}`, + url: (params) => `${params.restUrl}/ttl/${upstashPathSegment(params.key, 'key')}`, method: 'GET', headers: (params) => ({ Authorization: `Bearer ${params.restToken}`, diff --git a/apps/sim/tools/upstash/upstash.test.ts b/apps/sim/tools/upstash/upstash.test.ts new file mode 100644 index 00000000000..7465de84df4 --- /dev/null +++ b/apps/sim/tools/upstash/upstash.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from 'vitest' +import type { ToolConfig } from '@/tools/types' +import { upstashRedisExpireTool } from '@/tools/upstash/redis_expire' +import { upstashRedisHGetTool } from '@/tools/upstash/redis_hget' +import { upstashRedisIncrbyTool } from '@/tools/upstash/redis_incrby' +import { upstashRedisLRangeTool } from '@/tools/upstash/redis_lrange' + +const connection = { restUrl: 'https://example.upstash.io', restToken: 'token' } + +/** Builds a tool's request path from untyped params, the way an agent's tool call arrives. */ +function requestPath

(tool: ToolConfig

, params: Record): string { + const { url } = tool.request + return new URL(typeof url === 'function' ? url({ ...connection, ...params } as P) : url).pathname +} + +describe('Upstash REST paths', () => { + it('refuses a dot-segment key that would turn the field into the command', () => { + expect(() => requestPath(upstashRedisHGetTool, { key: '..', field: 'flushall' })).toThrow( + 'key cannot be "." or ".."' + ) + }) + + it.each([ + ['increment', upstashRedisIncrbyTool, { key: 'k', increment: '../../flushall' }], + ['seconds', upstashRedisExpireTool, { key: 'k', seconds: '1/../../flushall' }], + ['start', upstashRedisLRangeTool, { key: 'k', start: '../../flushall', stop: 1 }], + ] as const)('refuses a non-integer %s', (name, tool, params) => { + expect(() => requestPath(tool as ToolConfig, params)).toThrow(`${name} must be an integer`) + }) + + it('keeps ordinary keys, fields, and negative integers in their own segments', () => { + expect(requestPath(upstashRedisHGetTool, { key: 'user/1', field: 'a b' })).toBe( + '/hget/user%2F1/a%20b' + ) + expect(requestPath(upstashRedisLRangeTool, { key: 'list', start: 0, stop: -1 })).toBe( + '/lrange/list/0/-1' + ) + expect(requestPath(upstashRedisIncrbyTool, { key: 'n', increment: '-5' })).toBe('/incrby/n/-5') + }) +}) diff --git a/apps/sim/tools/upstash/utils.ts b/apps/sim/tools/upstash/utils.ts new file mode 100644 index 00000000000..22946c92c14 --- /dev/null +++ b/apps/sim/tools/upstash/utils.ts @@ -0,0 +1,21 @@ +/** + * Encodes one Redis argument as an Upstash REST path segment. A bare `.` or `..` is refused + * because URL parsing resolves it (encoded or not) as a dot segment, which would shift the + * remaining arguments into the command position (for example `HGET .. flushall` becoming + * `FLUSHALL`). + */ +export function upstashPathSegment(value: string, name: string): string { + if (value === '.' || value === '..') { + throw new Error(`${name} cannot be "." or ".." in an Upstash REST request`) + } + return encodeURIComponent(value) +} + +/** Formats an integer Redis argument for an Upstash REST path or query, refusing anything else. */ +export function upstashInteger(value: unknown, name: string): string { + const parsed = typeof value === 'string' && value.trim() !== '' ? Number(value) : value + if (typeof parsed !== 'number' || !Number.isSafeInteger(parsed)) { + throw new Error(`${name} must be an integer`) + } + return String(parsed) +} From a852401bbc0f1d059cb114b3d5c0bc3755eaa8c2 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 18 Nov 2026 10:34:37 -0800 Subject: [PATCH 2/3] fix(integrations): stream neo4j records under the cap, keep mysql preferred tls as documented, tune truncation hints --- apps/sim/lib/internal/mongodb/operations.ts | 17 ++- apps/sim/lib/internal/mysql/client.test.ts | 33 ----- apps/sim/lib/internal/mysql/client.ts | 16 +-- apps/sim/lib/internal/mysql/schema.ts | 3 +- .../sim/lib/internal/neo4j/operations.test.ts | 45 +++++-- apps/sim/lib/internal/neo4j/operations.ts | 125 +++++++++--------- apps/sim/lib/internal/postgresql/queries.ts | 2 +- .../tool-operations/response-limits.test.ts | 9 +- 8 files changed, 122 insertions(+), 128 deletions(-) diff --git a/apps/sim/lib/internal/mongodb/operations.ts b/apps/sim/lib/internal/mongodb/operations.ts index 874e92da2b6..b99c590a00a 100644 --- a/apps/sim/lib/internal/mongodb/operations.ts +++ b/apps/sim/lib/internal/mongodb/operations.ts @@ -36,7 +36,8 @@ export class MongodbOperationInputError extends Error {} */ async function readCappedDocuments( cursor: AsyncIterable, - describe: (count: number) => string + describe: (count: number) => string, + paginationHint: string ) { const budget = createResultRowsBudget() const documents: Document[] = [] @@ -50,10 +51,7 @@ async function readCappedDocuments( } const message = describe(documents.length) if (!truncated) return { message, documents } - const truncationReason = resultTruncationReason( - documents.length, - 'Narrow the filter or page with $skip and $limit to read the rest.' - ) + const truncationReason = resultTruncationReason(documents.length, paginationHint) return { message: `${message} ${truncationReason}`, documents, @@ -107,7 +105,11 @@ export function executeMongodbQuery(input: MongodbQueryInput, signal?: AbortSign } cursor = cursor.limit(input.limit || 100) - const result = await readCappedDocuments(cursor, (count) => `Found ${count} documents`) + const result = await readCappedDocuments( + cursor, + (count) => `Found ${count} documents`, + 'Narrow the filter, or page with $skip and $limit in an aggregation pipeline.' + ) signal?.throwIfAborted() return { ...result, documentCount: result.documents.length } }) @@ -126,7 +128,8 @@ export function executeMongodbAggregation(input: MongodbExecuteInput, signal?: A return withMongodbClient(input, signal, async (client) => { const result = await readCappedDocuments( client.db(input.database).collection(collectionName).aggregate(pipeline, { signal }), - (count) => `Aggregation completed, returned ${count} documents` + (count) => `Aggregation completed, returned ${count} documents`, + 'Page with $skip and $limit stages to read the rest.' ) signal?.throwIfAborted() return { ...result, documentCount: result.documents.length } diff --git a/apps/sim/lib/internal/mysql/client.test.ts b/apps/sim/lib/internal/mysql/client.test.ts index bfefc67135c..06b4eab31a8 100644 --- a/apps/sim/lib/internal/mysql/client.test.ts +++ b/apps/sim/lib/internal/mysql/client.test.ts @@ -85,39 +85,6 @@ describe('MySQL client', () => { expect(socket.setNoDelay).toHaveBeenCalledWith(true) }) - it('connects unencrypted for ssl=preferred when the server does not offer TLS', async () => { - const connection = { end: vi.fn(), destroy: vi.fn() } - mockCreateConnection - .mockRejectedValueOnce( - Object.assign(new Error('Server does not support secure connection'), { - code: 'HANDSHAKE_NO_SSL_SUPPORT', - }) - ) - .mockResolvedValueOnce(connection) - - await expect(createMysqlConnection({ ...CONNECTION_CONFIG, ssl: 'preferred' })).resolves.toBe( - connection - ) - expect(mockCreateConnection).toHaveBeenCalledTimes(2) - expect(mockCreateConnection.mock.calls[1][0].ssl).toBeUndefined() - }) - - it.each(['required', 'disabled'] as const)( - 'does not retry ssl=%s after a handshake failure', - async (ssl) => { - mockCreateConnection.mockRejectedValueOnce( - Object.assign(new Error('Server does not support secure connection'), { - code: 'HANDSHAKE_NO_SSL_SUPPORT', - }) - ) - - await expect(createMysqlConnection({ ...CONNECTION_CONFIG, ssl })).rejects.toThrow( - 'Server does not support secure connection' - ) - expect(mockCreateConnection).toHaveBeenCalledTimes(1) - } - ) - it.each([ new Map([['key', 'value']]), new Set(['value']), diff --git a/apps/sim/lib/internal/mysql/client.ts b/apps/sim/lib/internal/mysql/client.ts index 82deb218b5c..3fd425bba5c 100644 --- a/apps/sim/lib/internal/mysql/client.ts +++ b/apps/sim/lib/internal/mysql/client.ts @@ -1,5 +1,4 @@ import net from 'node:net' -import { isRecordLike } from '@sim/utils/object' import mysql from 'mysql2/promise' import { validateDatabaseHost } from '@/lib/core/security/input-validation.server' @@ -41,15 +40,6 @@ function assertMysqlExecuteValues( } } -function isNoSslSupportError(error: unknown): boolean { - return isRecordLike(error) && error.code === 'HANDSHAKE_NO_SSL_SUPPORT' -} - -/** - * Connects with the requested TLS mode. mysql2 has no "prefer" mode (with `ssl` set it refuses a - * server without TLS), so `preferred` retries unencrypted only when the server does not offer - * TLS, matching MySQL's own `ssl-mode=PREFERRED`. - */ export async function createMysqlConnection( config: MysqlConnectionConfig, signal?: AbortSignal @@ -87,11 +77,7 @@ export async function createMysqlConnection( } try { - const connection = await mysql.createConnection(connectionConfig).catch((error: unknown) => { - if (config.ssl !== 'preferred' || !isNoSslSupportError(error)) throw error - signal?.throwIfAborted() - return mysql.createConnection({ ...connectionConfig, ssl: undefined }) - }) + const connection = await mysql.createConnection(connectionConfig) signal?.throwIfAborted() return connection } catch (error) { diff --git a/apps/sim/lib/internal/mysql/schema.ts b/apps/sim/lib/internal/mysql/schema.ts index 4524c2e0b50..13e63f7d3db 100644 --- a/apps/sim/lib/internal/mysql/schema.ts +++ b/apps/sim/lib/internal/mysql/schema.ts @@ -4,8 +4,7 @@ import { z } from 'zod' const sslModeSchema = z.enum(['disabled', 'required', 'preferred']).default('preferred') /** * TLS mode for a MySQL connection: `required` encrypts and verifies the server certificate, - * `preferred` encrypts without verifying it when the server offers TLS and connects unencrypted - * when it does not, and `disabled` connects without TLS. + * `preferred` encrypts without verifying it, and `disabled` connects without TLS. */ export type MysqlSslMode = z.output diff --git a/apps/sim/lib/internal/neo4j/operations.test.ts b/apps/sim/lib/internal/neo4j/operations.test.ts index 73226de5479..4c64ccec51b 100644 --- a/apps/sim/lib/internal/neo4j/operations.test.ts +++ b/apps/sim/lib/internal/neo4j/operations.test.ts @@ -24,7 +24,8 @@ const INPUT = { parameters: {}, } -function queryResult() { +/** A driver `Result` streaming `count` records, recording whether the rest was discarded. */ +function streamingResult(count: number) { const updates = { nodesCreated: 0, nodesDeleted: 0, @@ -38,19 +39,32 @@ function queryResult() { constraintsAdded: 0, constraintsRemoved: 0, } + const summary = { + resultAvailableAfter: { toNumber: () => 2 }, + resultConsumedAfter: { toNumber: () => 3 }, + counters: { updates: () => updates }, + } + const state = { pulled: 0, discarded: false } + const record = { keys: ['name'], get: () => 'Ada' } return { - records: [{ keys: ['name'], get: vi.fn().mockReturnValue('Ada') }], - summary: { - resultAvailableAfter: { toNumber: () => 2 }, - resultConsumedAfter: { toNumber: () => 3 }, - counters: { updates: () => updates }, - }, + state, + [Symbol.asyncIterator]: () => ({ + next: async () => { + if (state.pulled >= count) return { done: true, value: summary } + state.pulled++ + return { done: false, value: record } + }, + return: async () => { + state.discarded = true + return { done: true, value: summary } + }, + }), } } describe('Neo4j operations', () => { beforeEach(() => { - mocks.run.mockResolvedValue(queryResult()) + mocks.run.mockImplementation(() => streamingResult(1)) mocks.sessionClose.mockResolvedValue(undefined) mocks.driverClose.mockResolvedValue(undefined) mocks.createDriver.mockResolvedValue({ @@ -59,6 +73,21 @@ describe('Neo4j operations', () => { }) }) + it('stops pulling records at the ceiling and keeps the summary', async () => { + const result = streamingResult(20_000) + mocks.run.mockReturnValue(result) + + const response = await executeNeo4jQuery(INPUT) + + expect(result.state).toEqual({ pulled: 10_001, discarded: true }) + expect(response).toMatchObject({ + recordCount: 10_000, + truncated: true, + summary: { resultAvailableAfter: 2 }, + }) + expect(mocks.sessionClose).toHaveBeenCalled() + }) + it('rejects invalid Cypher before opening a driver', async () => { await expect(executeNeo4jQuery({ ...INPUT, cypherQuery: ' ' })).rejects.toBeInstanceOf( Neo4jOperationInputError diff --git a/apps/sim/lib/internal/neo4j/operations.ts b/apps/sim/lib/internal/neo4j/operations.ts index 662db7cc3f9..ee3f3e0e2c2 100644 --- a/apps/sim/lib/internal/neo4j/operations.ts +++ b/apps/sim/lib/internal/neo4j/operations.ts @@ -1,5 +1,5 @@ import { createLogger } from '@sim/logger' -import type { QueryResult, Session } from 'neo4j-driver' +import type { Record as Neo4jRecord, Result, ResultSummary, Session } from 'neo4j-driver' import type { Neo4jCreateRequest, Neo4jDeleteRequest, @@ -12,7 +12,7 @@ import type { import { createNeo4jDriver } from '@/lib/internal/neo4j/client' import { convertNeo4jValue } from '@/lib/internal/neo4j/values' import { - capResultRows, + createResultRowsBudget, resultTruncationReason, } from '@/lib/internal/tool-operations/response-limits' import type { Neo4jNodeSchema, Neo4jRelationshipSchema } from '@/tools/neo4j/types' @@ -47,21 +47,42 @@ function bindSessionAbort(session: Session, signal?: AbortSignal): () => void { return () => signal.removeEventListener('abort', abort) } -function projectRecords(result: QueryResult): Array> { - return result.records.map((record) => { - const projected: Record = {} - for (const key of record.keys) { - if (typeof key === 'string') projected[key] = convertNeo4jValue(record.get(key)) +function projectRecord(record: Neo4jRecord): Record { + const projected: Record = {} + for (const key of record.keys) { + if (typeof key === 'string') projected[key] = convertNeo4jValue(record.get(key)) + } + return projected +} + +/** + * Streams records until the response budget is spent. Returning the iterator early discards the + * rest on the server (the statement still completes, so the summary counters stay accurate) + * instead of buffering every record in memory. + */ +async function readCappedRecords(result: Result) { + const budget = createResultRowsBudget() + const records: Array> = [] + const iterator = result[Symbol.asyncIterator]() + let next = await iterator.next() + while (!next.done) { + const projected = projectRecord(next.value) + if (!budget.admit(projected)) { + const rest = await iterator.return?.() + if (!rest?.done) throw new Error('Neo4j result ended without a summary') + return { records, summary: rest.value, truncated: true } } - return projected - }) + records.push(projected) + next = await iterator.next() + } + return { records, summary: next.value, truncated: false } } -function projectSummary(result: QueryResult) { - const updates = result.summary.counters.updates() +function projectSummary(summary: ResultSummary) { + const updates = summary.counters.updates() return { - resultAvailableAfter: result.summary.resultAvailableAfter.toNumber(), - resultConsumedAfter: result.summary.resultConsumedAfter.toNumber(), + resultAvailableAfter: summary.resultAvailableAfter.toNumber(), + resultConsumedAfter: summary.resultConsumedAfter.toNumber(), counters: { nodesCreated: updates.nodesCreated, nodesDeleted: updates.nodesDeleted, @@ -78,58 +99,42 @@ function projectSummary(result: QueryResult) { } } -function statementResponse(kind: StatementKind, result: QueryResult) { - const { rows: records, truncated } = capResultRows(projectRecords(result)) - const summary = projectSummary(result) - const truncation = truncated - ? { - truncated: true, - truncationReason: resultTruncationReason( - records.length, - 'Page with SKIP and LIMIT to read the rest.' - ), - } +function statementResponse( + kind: StatementKind, + read: Awaited> +) { + const { records } = read + const summary = projectSummary(read.summary) + const truncationReason = read.truncated + ? resultTruncationReason( + records.length, + kind === 'query' + ? 'Page with SKIP and LIMIT to read the rest.' + : 'A write statement still ran in full; page a read with SKIP and LIMIT to read the rest.' + ) : undefined - const withTruncation = (message: string) => - truncation ? `${message} ${truncation.truncationReason}` : message + const withRecords = (message: string) => ({ + message: truncationReason ? `${message} ${truncationReason}` : message, + records, + recordCount: records.length, + summary, + ...(truncationReason && { truncated: true, truncationReason }), + }) switch (kind) { case 'query': - return { - message: withTruncation(`Found ${records.length} records`), - records, - recordCount: records.length, - summary, - ...truncation, - } + return withRecords(`Found ${records.length} records`) case 'execute': - return { - message: withTruncation(`Query executed successfully, returned ${records.length} records`), - records, - recordCount: records.length, - summary, - ...truncation, - } + return withRecords(`Query executed successfully, returned ${records.length} records`) case 'create': - return { - message: `Created ${summary.counters.nodesCreated} nodes and ${summary.counters.relationshipsCreated} relationships`, - records, - recordCount: records.length, - summary, - } + return withRecords( + `Created ${summary.counters.nodesCreated} nodes and ${summary.counters.relationshipsCreated} relationships` + ) case 'update': - return { - message: `Updated ${summary.counters.propertiesSet} properties`, - records, - recordCount: records.length, - summary, - } + return withRecords(`Updated ${summary.counters.propertiesSet} properties`) case 'merge': - return { - message: `Merge completed: ${summary.counters.nodesCreated} nodes created, ${summary.counters.relationshipsCreated} relationships created`, - records, - recordCount: records.length, - summary, - } + return withRecords( + `Merge completed: ${summary.counters.nodesCreated} nodes created, ${summary.counters.relationshipsCreated} relationships created` + ) case 'delete': return { message: `Deleted ${summary.counters.nodesDeleted} nodes and ${summary.counters.relationshipsDeleted} relationships`, @@ -145,9 +150,9 @@ async function executeStatement(input: StatementInput, kind: StatementKind, sign const session = driver.session({ database: input.database }) const unbindAbort = bindSessionAbort(session, signal) try { - const result = await session.run(input.cypherQuery, input.parameters ?? {}) + const read = await readCappedRecords(session.run(input.cypherQuery, input.parameters ?? {})) signal?.throwIfAborted() - return statementResponse(kind, result) + return statementResponse(kind, read) } finally { unbindAbort() await session.close().catch(() => undefined) diff --git a/apps/sim/lib/internal/postgresql/queries.ts b/apps/sim/lib/internal/postgresql/queries.ts index 9bfc3fc5787..13f64000fae 100644 --- a/apps/sim/lib/internal/postgresql/queries.ts +++ b/apps/sim/lib/internal/postgresql/queries.ts @@ -83,7 +83,7 @@ export async function queryPostgres( return capStatementRows( result, result.count ?? result.length ?? 0, - 'Page with LIMIT and OFFSET to read the rest.' + 'A write statement still ran in full; page a SELECT with LIMIT and OFFSET to read the rest.' ) } diff --git a/apps/sim/lib/internal/tool-operations/response-limits.test.ts b/apps/sim/lib/internal/tool-operations/response-limits.test.ts index 0049f94557b..a0dc548c79c 100644 --- a/apps/sim/lib/internal/tool-operations/response-limits.test.ts +++ b/apps/sim/lib/internal/tool-operations/response-limits.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest' import { capResultRows, + capStatementRows, MAX_TOOL_RESPONSE_BODY_BYTES, toRowsResponseBody, } from '@/lib/internal/tool-operations/response-limits' @@ -20,9 +21,13 @@ describe('database result caps', () => { it('measures UTF-8 bytes, so multibyte text cannot exceed the response ceiling', () => { const value = '漢'.repeat(100_000) - const capped = capResultRows(Array.from({ length: 50 }, () => ({ value }))) + const capped = capStatementRows( + Array.from({ length: 50 }, () => ({ value })), + 50, + 'Page the rest.' + ) expect(capped.truncated).toBe(true) - const body = JSON.stringify(toRowsResponseBody({ ...capped, rowCount: 50 }, 'Query done.')) + const body = JSON.stringify(toRowsResponseBody(capped, 'Query done.')) expect(Buffer.byteLength(body, 'utf8')).toBeLessThanOrEqual(MAX_TOOL_RESPONSE_BODY_BYTES) }) From 08d75c68ca37924915bb29ba439d93dd2d122f2f Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 18 Nov 2026 12:05:40 -0800 Subject: [PATCH 3/3] fix(integrations): drop the row cap; stream mongodb and neo4j reads and fail at the response size limit --- .../content/docs/integrations/mongodb.mdx | 4 - apps/docs/content/docs/integrations/mysql.mdx | 4 - apps/docs/content/docs/integrations/neo4j.mdx | 4 - .../content/docs/integrations/postgresql.mdx | 10 -- apps/sim/blocks/blocks/mongodb.ts | 8 -- apps/sim/blocks/blocks/mysql.ts | 8 -- apps/sim/blocks/blocks/neo4j.ts | 8 -- apps/sim/blocks/blocks/postgresql.ts | 8 -- apps/sim/lib/internal/mongodb/execute-tool.ts | 2 + apps/sim/lib/internal/mongodb/operations.ts | 57 ++++------ apps/sim/lib/internal/mysql/operations.ts | 46 ++++---- apps/sim/lib/internal/mysql/queries.ts | 16 +-- apps/sim/lib/internal/neo4j/execute-tool.ts | 2 + .../sim/lib/internal/neo4j/operations.test.ts | 27 +++-- apps/sim/lib/internal/neo4j/operations.ts | 100 +++++++++--------- .../sim/lib/internal/postgresql/operations.ts | 46 ++++---- .../lib/internal/postgresql/queries.test.ts | 16 --- apps/sim/lib/internal/postgresql/queries.ts | 26 ++--- .../tool-operations/response-limits.test.ts | 56 +++++----- .../tool-operations/response-limits.ts | 91 +++------------- apps/sim/tools/mongodb/execute.ts | 15 --- apps/sim/tools/mongodb/query.ts | 15 --- apps/sim/tools/mongodb/types.ts | 2 - apps/sim/tools/mysql/execute.ts | 15 --- apps/sim/tools/mysql/query.ts | 15 --- apps/sim/tools/mysql/types.ts | 2 - apps/sim/tools/neo4j/execute.ts | 15 --- apps/sim/tools/neo4j/query.ts | 15 --- apps/sim/tools/neo4j/types.ts | 2 - apps/sim/tools/postgresql/delete.ts | 15 --- apps/sim/tools/postgresql/execute.ts | 15 --- apps/sim/tools/postgresql/insert.ts | 15 --- apps/sim/tools/postgresql/query.ts | 15 --- apps/sim/tools/postgresql/types.ts | 2 - apps/sim/tools/postgresql/update.ts | 15 --- 35 files changed, 215 insertions(+), 497 deletions(-) diff --git a/apps/docs/content/docs/integrations/mongodb.mdx b/apps/docs/content/docs/integrations/mongodb.mdx index a3ec82dfb5d..3317de7a94e 100644 --- a/apps/docs/content/docs/integrations/mongodb.mdx +++ b/apps/docs/content/docs/integrations/mongodb.mdx @@ -50,8 +50,6 @@ Execute find operation on MongoDB collection | `message` | string | Operation status message | | `documents` | array | Array of documents returned from the query | | `documentCount` | number | Number of documents returned | -| `truncated` | boolean | Present and true only when documents were dropped to stay inside the response ceilings \(10,000 documents or 10 MiB\). Absent means the result is complete | -| `truncationReason` | string | Which ceiling was hit and how to read the rest | @@ -179,8 +177,6 @@ Execute MongoDB aggregation pipeline | `message` | string | Operation status message | | `documents` | array | Array of documents returned from aggregation | | `documentCount` | number | Number of documents returned | -| `truncated` | boolean | Present and true only when documents were dropped to stay inside the response ceilings \(10,000 documents or 10 MiB\). Absent means the result is complete | -| `truncationReason` | string | Which ceiling was hit and how to read the rest | diff --git a/apps/docs/content/docs/integrations/mysql.mdx b/apps/docs/content/docs/integrations/mysql.mdx index 0a252d54092..9ec39e201cc 100644 --- a/apps/docs/content/docs/integrations/mysql.mdx +++ b/apps/docs/content/docs/integrations/mysql.mdx @@ -51,8 +51,6 @@ Execute SELECT query on MySQL database | `message` | string | Operation status message | | `rows` | array | Array of rows returned from the query | | `rowCount` | number | Number of rows returned | -| `truncated` | boolean | Present and true only when rows were dropped to stay inside the response ceilings \(10,000 rows or 10 MiB\). Absent means the result is complete | -| `truncationReason` | string | Which ceiling was hit and how to read the rest | @@ -145,8 +143,6 @@ Execute raw SQL query on MySQL database | `message` | string | Operation status message | | `rows` | array | Array of rows returned from the query | | `rowCount` | number | Number of rows affected | -| `truncated` | boolean | Present and true only when rows were dropped to stay inside the response ceilings \(10,000 rows or 10 MiB\). Absent means the result is complete | -| `truncationReason` | string | Which ceiling was hit and how to read the rest | diff --git a/apps/docs/content/docs/integrations/neo4j.mdx b/apps/docs/content/docs/integrations/neo4j.mdx index 12dfd462e95..82225094fcc 100644 --- a/apps/docs/content/docs/integrations/neo4j.mdx +++ b/apps/docs/content/docs/integrations/neo4j.mdx @@ -46,8 +46,6 @@ Execute MATCH queries to read nodes and relationships from Neo4j graph database. | `records` | array | Array of records returned from the query | | `recordCount` | number | Number of records returned | | `summary` | json | Query execution summary with timing and counters | -| `truncated` | boolean | Present and true only when records were dropped to stay inside the response ceilings \(10,000 records or 10 MiB\). Absent means the result is complete | -| `truncationReason` | string | Which ceiling was hit and how to read the rest | @@ -191,8 +189,6 @@ Execute arbitrary Cypher queries on Neo4j graph database for complex operations | `records` | array | Array of records returned from the query | | `recordCount` | number | Number of records returned | | `summary` | json | Execution summary with timing and counters | -| `truncated` | boolean | Present and true only when records were dropped to stay inside the response ceilings \(10,000 records or 10 MiB\). Absent means the result is complete | -| `truncationReason` | string | Which ceiling was hit and how to read the rest | diff --git a/apps/docs/content/docs/integrations/postgresql.mdx b/apps/docs/content/docs/integrations/postgresql.mdx index a43d6d05e32..c1237fda4c7 100644 --- a/apps/docs/content/docs/integrations/postgresql.mdx +++ b/apps/docs/content/docs/integrations/postgresql.mdx @@ -48,8 +48,6 @@ Execute a SELECT query on PostgreSQL database | `message` | string | Operation status message | | `rows` | array | Array of rows returned from the query | | `rowCount` | number | Number of rows returned | -| `truncated` | boolean | Present and true only when rows were dropped to stay inside the response ceilings \(10,000 rows or 10 MiB\). Absent means the result is complete | -| `truncationReason` | string | Which ceiling was hit and how to read the rest | @@ -73,8 +71,6 @@ Insert data into PostgreSQL database | `message` | string | Operation status message | | `rows` | array | Inserted data \(if RETURNING clause used\) | | `rowCount` | number | Number of rows inserted | -| `truncated` | boolean | Present and true only when rows were dropped to stay inside the response ceilings \(10,000 rows or 10 MiB\). Absent means the result is complete | -| `truncationReason` | string | Which ceiling was hit and how to read the rest | @@ -99,8 +95,6 @@ Update data in PostgreSQL database | `message` | string | Operation status message | | `rows` | array | Updated data \(if RETURNING clause used\) | | `rowCount` | number | Number of rows updated | -| `truncated` | boolean | Present and true only when rows were dropped to stay inside the response ceilings \(10,000 rows or 10 MiB\). Absent means the result is complete | -| `truncationReason` | string | Which ceiling was hit and how to read the rest | @@ -124,8 +118,6 @@ Delete data from PostgreSQL database | `message` | string | Operation status message | | `rows` | array | Deleted data \(if RETURNING clause used\) | | `rowCount` | number | Number of rows deleted | -| `truncated` | boolean | Present and true only when rows were dropped to stay inside the response ceilings \(10,000 rows or 10 MiB\). Absent means the result is complete | -| `truncationReason` | string | Which ceiling was hit and how to read the rest | @@ -148,8 +140,6 @@ Execute raw SQL query on PostgreSQL database | `message` | string | Operation status message | | `rows` | array | Array of rows returned from the query | | `rowCount` | number | Number of rows affected | -| `truncated` | boolean | Present and true only when rows were dropped to stay inside the response ceilings \(10,000 rows or 10 MiB\). Absent means the result is complete | -| `truncationReason` | string | Which ceiling was hit and how to read the rest | diff --git a/apps/sim/blocks/blocks/mongodb.ts b/apps/sim/blocks/blocks/mongodb.ts index 6af751932b5..c0743ea5593 100644 --- a/apps/sim/blocks/blocks/mongodb.ts +++ b/apps/sim/blocks/blocks/mongodb.ts @@ -959,14 +959,6 @@ Return ONLY the MongoDB query filter as valid JSON - no explanations, no markdow type: 'number', description: 'Number of documents affected by the operation', }, - truncated: { - type: 'boolean', - description: 'True when the result hit a row or byte ceiling and documents were dropped', - }, - truncationReason: { - type: 'string', - description: 'Explanation of the ceiling that truncated the result', - }, insertedId: { type: 'string', description: 'ID of the inserted document (single insert)', diff --git a/apps/sim/blocks/blocks/mysql.ts b/apps/sim/blocks/blocks/mysql.ts index 0a27f2a3a93..18ebb35a8e4 100644 --- a/apps/sim/blocks/blocks/mysql.ts +++ b/apps/sim/blocks/blocks/mysql.ts @@ -412,14 +412,6 @@ Return ONLY the SQL query - no explanations, no markdown, no extra text.`, type: 'number', description: 'Number of rows affected by the operation', }, - truncated: { - type: 'boolean', - description: 'True when the result hit a row or byte ceiling and rows were dropped', - }, - truncationReason: { - type: 'string', - description: 'Explanation of the ceiling that truncated the result', - }, }, } diff --git a/apps/sim/blocks/blocks/neo4j.ts b/apps/sim/blocks/blocks/neo4j.ts index 11030f5e89a..ae525f2e75e 100644 --- a/apps/sim/blocks/blocks/neo4j.ts +++ b/apps/sim/blocks/blocks/neo4j.ts @@ -706,14 +706,6 @@ Return ONLY valid JSON.`, type: 'number', description: 'Number of records returned or affected', }, - truncated: { - type: 'boolean', - description: 'True when the result hit a row or byte ceiling and records were dropped', - }, - truncationReason: { - type: 'string', - description: 'Explanation of the ceiling that truncated the result', - }, summary: { type: 'json', description: 'Execution summary with timing and database change counters', diff --git a/apps/sim/blocks/blocks/postgresql.ts b/apps/sim/blocks/blocks/postgresql.ts index 122ccb086cd..955b69f1a66 100644 --- a/apps/sim/blocks/blocks/postgresql.ts +++ b/apps/sim/blocks/blocks/postgresql.ts @@ -427,14 +427,6 @@ Return ONLY the SQL query - no explanations, no markdown, no extra text.`, type: 'number', description: 'Number of rows affected by the operation', }, - truncated: { - type: 'boolean', - description: 'True when the result hit a row or byte ceiling and rows were dropped', - }, - truncationReason: { - type: 'string', - description: 'Explanation of the ceiling that truncated the result', - }, tables: { type: 'array', description: 'Array of table schemas with columns, keys, and indexes (introspect operation)', diff --git a/apps/sim/lib/internal/mongodb/execute-tool.ts b/apps/sim/lib/internal/mongodb/execute-tool.ts index 252e9eb0b93..21c77a372cd 100644 --- a/apps/sim/lib/internal/mongodb/execute-tool.ts +++ b/apps/sim/lib/internal/mongodb/execute-tool.ts @@ -1,5 +1,6 @@ import { getErrorMessage } from '@sim/utils/errors' import type { z } from 'zod' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { executeMongodbAggregation, executeMongodbDelete, @@ -40,6 +41,7 @@ async function executeOperation( return Response.json(result) } catch (error) { signal?.throwIfAborted() + if (isPayloadSizeLimitError(error)) throw error if (error instanceof MongodbOperationInputError) { return Response.json({ error: error.message }, { status: 400 }) } diff --git a/apps/sim/lib/internal/mongodb/operations.ts b/apps/sim/lib/internal/mongodb/operations.ts index b99c590a00a..859857ce5d8 100644 --- a/apps/sim/lib/internal/mongodb/operations.ts +++ b/apps/sim/lib/internal/mongodb/operations.ts @@ -23,41 +23,23 @@ import type { MongodbQueryInput, MongodbUpdateInput, } from '@/lib/internal/mongodb/schema' -import { - createResultRowsBudget, - resultTruncationReason, -} from '@/lib/internal/tool-operations/response-limits' +import { createResponseRowsBudget } from '@/lib/internal/tool-operations/response-limits' export class MongodbOperationInputError extends Error {} /** - * Reads documents until the response budget is spent. Leaving the loop early closes the server - * cursor, so an unbounded find or aggregation never materializes past the ceiling. + * Reads documents one at a time, failing with the tool response size error as soon as they cannot + * fit the response. Leaving the loop early closes the server cursor, so an oversized result is + * never read into memory in full. */ -async function readCappedDocuments( - cursor: AsyncIterable, - describe: (count: number) => string, - paginationHint: string -) { - const budget = createResultRowsBudget() +async function readDocuments(cursor: AsyncIterable, toolId: string) { + const budget = createResponseRowsBudget(toolId) const documents: Document[] = [] - let truncated = false for await (const document of cursor) { - if (!budget.admit(document)) { - truncated = true - break - } + budget.admit(document) documents.push(document) } - const message = describe(documents.length) - if (!truncated) return { message, documents } - const truncationReason = resultTruncationReason(documents.length, paginationHint) - return { - message: `${message} ${truncationReason}`, - documents, - truncated: true, - truncationReason, - } + return documents } async function withMongodbClient( @@ -105,13 +87,13 @@ export function executeMongodbQuery(input: MongodbQueryInput, signal?: AbortSign } cursor = cursor.limit(input.limit || 100) - const result = await readCappedDocuments( - cursor, - (count) => `Found ${count} documents`, - 'Narrow the filter, or page with $skip and $limit in an aggregation pipeline.' - ) + const documents = await readDocuments(cursor, 'mongodb_query') signal?.throwIfAborted() - return { ...result, documentCount: result.documents.length } + return { + message: `Found ${documents.length} documents`, + documents, + documentCount: documents.length, + } }) } @@ -126,13 +108,16 @@ export function executeMongodbAggregation(input: MongodbExecuteInput, signal?: A const pipeline = JSON.parse(input.pipeline) as Document[] return withMongodbClient(input, signal, async (client) => { - const result = await readCappedDocuments( + const documents = await readDocuments( client.db(input.database).collection(collectionName).aggregate(pipeline, { signal }), - (count) => `Aggregation completed, returned ${count} documents`, - 'Page with $skip and $limit stages to read the rest.' + 'mongodb_execute' ) signal?.throwIfAborted() - return { ...result, documentCount: result.documents.length } + return { + message: `Aggregation completed, returned ${documents.length} documents`, + documents, + documentCount: documents.length, + } }) } diff --git a/apps/sim/lib/internal/mysql/operations.ts b/apps/sim/lib/internal/mysql/operations.ts index e4f17ade40d..48f215d33c7 100644 --- a/apps/sim/lib/internal/mysql/operations.ts +++ b/apps/sim/lib/internal/mysql/operations.ts @@ -15,7 +15,6 @@ import type { MysqlQueryInput, MysqlUpdateInput, } from '@/lib/internal/mysql/schema' -import { toRowsResponseBody } from '@/lib/internal/tool-operations/response-limits' export class MysqlOperationInputError extends Error {} @@ -46,10 +45,11 @@ export function executeMysqlQuery(input: MysqlQueryInput, signal?: AbortSignal) return withMysqlConnection(input, signal, async (connection) => { const result = await queryMysql(connection, input.query, undefined, signal) - return toRowsResponseBody( - result, - `Query executed successfully. ${result.rowCount} row(s) returned.` - ) + return { + message: `Query executed successfully. ${result.rowCount} row(s) returned.`, + rows: result.rows, + rowCount: result.rowCount, + } }) } @@ -58,10 +58,11 @@ export function executeMysqlStatement(input: MysqlExecuteInput, signal?: AbortSi return withMysqlConnection(input, signal, async (connection) => { const result = await queryMysql(connection, input.query, undefined, signal) - return toRowsResponseBody( - result, - `SQL executed successfully. ${result.rowCount} row(s) affected.` - ) + return { + message: `SQL executed successfully. ${result.rowCount} row(s) affected.`, + rows: result.rows, + rowCount: result.rowCount, + } }) } @@ -69,10 +70,11 @@ export function executeMysqlInsert(input: MysqlInsertInput, signal?: AbortSignal return withMysqlConnection(input, signal, async (connection) => { const { query, values } = buildMysqlInsertQuery(input.table, input.data) const result = await queryMysql(connection, query, values, signal) - return toRowsResponseBody( - result, - `Data inserted successfully. ${result.rowCount} row(s) affected.` - ) + return { + message: `Data inserted successfully. ${result.rowCount} row(s) affected.`, + rows: result.rows, + rowCount: result.rowCount, + } }) } @@ -80,10 +82,11 @@ export function executeMysqlUpdate(input: MysqlUpdateInput, signal?: AbortSignal return withMysqlConnection(input, signal, async (connection) => { const { query, values } = buildMysqlUpdateQuery(input.table, input.data, input.where) const result = await queryMysql(connection, query, values, signal) - return toRowsResponseBody( - result, - `Data updated successfully. ${result.rowCount} row(s) affected.` - ) + return { + message: `Data updated successfully. ${result.rowCount} row(s) affected.`, + rows: result.rows, + rowCount: result.rowCount, + } }) } @@ -91,10 +94,11 @@ export function executeMysqlDelete(input: MysqlDeleteInput, signal?: AbortSignal return withMysqlConnection(input, signal, async (connection) => { const { query, values } = buildMysqlDeleteQuery(input.table, input.where) const result = await queryMysql(connection, query, values, signal) - return toRowsResponseBody( - result, - `Data deleted successfully. ${result.rowCount} row(s) affected.` - ) + return { + message: `Data deleted successfully. ${result.rowCount} row(s) affected.`, + rows: result.rows, + rowCount: result.rowCount, + } }) } diff --git a/apps/sim/lib/internal/mysql/queries.ts b/apps/sim/lib/internal/mysql/queries.ts index 4a1fcedd067..d72a1c80a4e 100644 --- a/apps/sim/lib/internal/mysql/queries.ts +++ b/apps/sim/lib/internal/mysql/queries.ts @@ -1,9 +1,10 @@ import type mysql from 'mysql2/promise' import { executeMysqlCommand } from '@/lib/internal/mysql/client' -import { - type CappedRowsResult, - capStatementRows, -} from '@/lib/internal/tool-operations/response-limits' + +export interface MysqlRowsResult { + rows: unknown[] + rowCount: number +} export interface MysqlIntrospectionResult { tables: Array<{ @@ -75,11 +76,14 @@ export async function queryMysql( query: string, values?: unknown[], signal?: AbortSignal -): Promise { +): Promise { const result = await executeMysqlCommand(connection, query, values, signal) if (Array.isArray(result)) { - return capStatementRows(result, result.length, 'Page with LIMIT and OFFSET to read the rest.') + return { + rows: result, + rowCount: result.length, + } } return { diff --git a/apps/sim/lib/internal/neo4j/execute-tool.ts b/apps/sim/lib/internal/neo4j/execute-tool.ts index f542d648bb3..b53ca36a0b1 100644 --- a/apps/sim/lib/internal/neo4j/execute-tool.ts +++ b/apps/sim/lib/internal/neo4j/execute-tool.ts @@ -9,6 +9,7 @@ import { neo4jQueryContract, neo4jUpdateContract, } from '@/lib/api/contracts/tools/databases/neo4j' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { executeNeo4jCreate, executeNeo4jDelete, @@ -35,6 +36,7 @@ async function executeOperation( return Response.json(await execute(parsed.data, signal)) } catch (error) { signal?.throwIfAborted() + if (isPayloadSizeLimitError(error)) throw error if (error instanceof Neo4jOperationInputError) { return Response.json({ error: `Query validation failed: ${error.message}` }, { status: 400 }) } diff --git a/apps/sim/lib/internal/neo4j/operations.test.ts b/apps/sim/lib/internal/neo4j/operations.test.ts index 4c64ccec51b..54c7b104f06 100644 --- a/apps/sim/lib/internal/neo4j/operations.test.ts +++ b/apps/sim/lib/internal/neo4j/operations.test.ts @@ -11,6 +11,7 @@ vi.mock('@/lib/internal/neo4j/client', () => ({ createNeo4jDriver: mocks.createDriver, })) +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { executeNeo4jQuery, Neo4jOperationInputError } from '@/lib/internal/neo4j/operations' const INPUT = { @@ -25,7 +26,7 @@ const INPUT = { } /** A driver `Result` streaming `count` records, recording whether the rest was discarded. */ -function streamingResult(count: number) { +function streamingResult(count: number, value = 'Ada') { const updates = { nodesCreated: 0, nodesDeleted: 0, @@ -45,7 +46,7 @@ function streamingResult(count: number) { counters: { updates: () => updates }, } const state = { pulled: 0, discarded: false } - const record = { keys: ['name'], get: () => 'Ada' } + const record = { keys: ['name'], get: () => value } return { state, [Symbol.asyncIterator]: () => ({ @@ -73,18 +74,26 @@ describe('Neo4j operations', () => { }) }) - it('stops pulling records at the ceiling and keeps the summary', async () => { + it('returns every record of a result that fits the response limit', async () => { const result = streamingResult(20_000) mocks.run.mockReturnValue(result) const response = await executeNeo4jQuery(INPUT) - expect(result.state).toEqual({ pulled: 10_001, discarded: true }) - expect(response).toMatchObject({ - recordCount: 10_000, - truncated: true, - summary: { resultAvailableAfter: 2 }, - }) + expect(response).toMatchObject({ recordCount: 20_000, summary: { resultAvailableAfter: 2 } }) + expect(result.state.discarded).toBe(false) + }) + + it('fails with the response size error and discards the rest once records cannot fit', async () => { + const result = streamingResult(1_000, 'x'.repeat(64 * 1024)) + mocks.run.mockReturnValue(result) + + const error = await executeNeo4jQuery(INPUT).catch((caught: unknown) => caught) + + expect(isPayloadSizeLimitError(error)).toBe(true) + expect(error).toMatchObject({ label: 'neo4j_query response body' }) + expect(result.state.discarded).toBe(true) + expect(result.state.pulled).toBeLessThan(1_000) expect(mocks.sessionClose).toHaveBeenCalled() }) diff --git a/apps/sim/lib/internal/neo4j/operations.ts b/apps/sim/lib/internal/neo4j/operations.ts index ee3f3e0e2c2..0e644a1e653 100644 --- a/apps/sim/lib/internal/neo4j/operations.ts +++ b/apps/sim/lib/internal/neo4j/operations.ts @@ -11,10 +11,7 @@ import type { } from '@/lib/api/contracts/tools/databases/neo4j' import { createNeo4jDriver } from '@/lib/internal/neo4j/client' import { convertNeo4jValue } from '@/lib/internal/neo4j/values' -import { - createResultRowsBudget, - resultTruncationReason, -} from '@/lib/internal/tool-operations/response-limits' +import { createResponseRowsBudget } from '@/lib/internal/tool-operations/response-limits' import type { Neo4jNodeSchema, Neo4jRelationshipSchema } from '@/tools/neo4j/types' const logger = createLogger('Neo4jOperations') @@ -56,26 +53,28 @@ function projectRecord(record: Neo4jRecord): Record { } /** - * Streams records until the response budget is spent. Returning the iterator early discards the - * rest on the server (the statement still completes, so the summary counters stay accurate) - * instead of buffering every record in memory. + * Streams the records, failing with the tool response size error as soon as they cannot fit the + * response. Returning the iterator early tells the server to discard the rest, so an oversized + * result is never buffered in full. `delete` returns no records, so it only waits for the summary. */ -async function readCappedRecords(result: Result) { - const budget = createResultRowsBudget() +async function readResult(result: Result, kind: StatementKind) { + if (kind === 'delete') return { records: [], summary: await result.summary() } + const budget = createResponseRowsBudget(`neo4j_${kind}`) const records: Array> = [] const iterator = result[Symbol.asyncIterator]() - let next = await iterator.next() - while (!next.done) { - const projected = projectRecord(next.value) - if (!budget.admit(projected)) { - const rest = await iterator.return?.() - if (!rest?.done) throw new Error('Neo4j result ended without a summary') - return { records, summary: rest.value, truncated: true } + try { + let next = await iterator.next() + while (!next.done) { + const projected = projectRecord(next.value) + budget.admit(projected) + records.push(projected) + next = await iterator.next() } - records.push(projected) - next = await iterator.next() + return { records, summary: next.value } + } catch (error) { + await iterator.return?.().catch(() => undefined) + throw error } - return { records, summary: next.value, truncated: false } } function projectSummary(summary: ResultSummary) { @@ -101,40 +100,45 @@ function projectSummary(summary: ResultSummary) { function statementResponse( kind: StatementKind, - read: Awaited> + { records, summary: resultSummary }: Awaited> ) { - const { records } = read - const summary = projectSummary(read.summary) - const truncationReason = read.truncated - ? resultTruncationReason( - records.length, - kind === 'query' - ? 'Page with SKIP and LIMIT to read the rest.' - : 'A write statement still ran in full; page a read with SKIP and LIMIT to read the rest.' - ) - : undefined - const withRecords = (message: string) => ({ - message: truncationReason ? `${message} ${truncationReason}` : message, - records, - recordCount: records.length, - summary, - ...(truncationReason && { truncated: true, truncationReason }), - }) + const summary = projectSummary(resultSummary) switch (kind) { case 'query': - return withRecords(`Found ${records.length} records`) + return { + message: `Found ${records.length} records`, + records, + recordCount: records.length, + summary, + } case 'execute': - return withRecords(`Query executed successfully, returned ${records.length} records`) + return { + message: `Query executed successfully, returned ${records.length} records`, + records, + recordCount: records.length, + summary, + } case 'create': - return withRecords( - `Created ${summary.counters.nodesCreated} nodes and ${summary.counters.relationshipsCreated} relationships` - ) + return { + message: `Created ${summary.counters.nodesCreated} nodes and ${summary.counters.relationshipsCreated} relationships`, + records, + recordCount: records.length, + summary, + } case 'update': - return withRecords(`Updated ${summary.counters.propertiesSet} properties`) + return { + message: `Updated ${summary.counters.propertiesSet} properties`, + records, + recordCount: records.length, + summary, + } case 'merge': - return withRecords( - `Merge completed: ${summary.counters.nodesCreated} nodes created, ${summary.counters.relationshipsCreated} relationships created` - ) + return { + message: `Merge completed: ${summary.counters.nodesCreated} nodes created, ${summary.counters.relationshipsCreated} relationships created`, + records, + recordCount: records.length, + summary, + } case 'delete': return { message: `Deleted ${summary.counters.nodesDeleted} nodes and ${summary.counters.relationshipsDeleted} relationships`, @@ -150,9 +154,9 @@ async function executeStatement(input: StatementInput, kind: StatementKind, sign const session = driver.session({ database: input.database }) const unbindAbort = bindSessionAbort(session, signal) try { - const read = await readCappedRecords(session.run(input.cypherQuery, input.parameters ?? {})) + const result = await readResult(session.run(input.cypherQuery, input.parameters ?? {}), kind) signal?.throwIfAborted() - return statementResponse(kind, read) + return statementResponse(kind, result) } finally { unbindAbort() await session.close().catch(() => undefined) diff --git a/apps/sim/lib/internal/postgresql/operations.ts b/apps/sim/lib/internal/postgresql/operations.ts index f4976c09647..f07ac3f9bf0 100644 --- a/apps/sim/lib/internal/postgresql/operations.ts +++ b/apps/sim/lib/internal/postgresql/operations.ts @@ -15,7 +15,6 @@ import type { PostgresqlQueryInput, PostgresqlUpdateInput, } from '@/lib/internal/postgresql/schema' -import { toRowsResponseBody } from '@/lib/internal/tool-operations/response-limits' import type { PostgresConnectionConfig } from '@/tools/postgresql/types' export class PostgresqlOperationInputError extends Error {} @@ -36,10 +35,11 @@ async function withPostgresClient( export function executePostgresqlQuery(input: PostgresqlQueryInput, signal?: AbortSignal) { return withPostgresClient(input, signal, async (client) => { const result = await queryPostgres(client, input.query, [], signal) - return toRowsResponseBody( - result, - `Query executed successfully. ${result.rowCount} row(s) returned.` - ) + return { + message: `Query executed successfully. ${result.rowCount} row(s) returned.`, + rows: result.rows, + rowCount: result.rowCount, + } }) } @@ -53,40 +53,44 @@ export function executePostgresqlStatement(input: PostgresqlExecuteInput, signal return withPostgresClient(input, signal, async (client) => { const result = await queryPostgres(client, input.query, [], signal) - return toRowsResponseBody( - result, - `SQL executed successfully. ${result.rowCount} row(s) affected.` - ) + return { + message: `SQL executed successfully. ${result.rowCount} row(s) affected.`, + rows: result.rows, + rowCount: result.rowCount, + } }) } export function executePostgresqlInsert(input: PostgresqlInsertInput, signal?: AbortSignal) { return withPostgresClient(input, signal, async (client) => { const result = await insertPostgresRows(client, input.table, input.data, signal) - return toRowsResponseBody( - result, - `Data inserted successfully. ${result.rowCount} row(s) affected.` - ) + return { + message: `Data inserted successfully. ${result.rowCount} row(s) affected.`, + rows: result.rows, + rowCount: result.rowCount, + } }) } export function executePostgresqlUpdate(input: PostgresqlUpdateInput, signal?: AbortSignal) { return withPostgresClient(input, signal, async (client) => { const result = await updatePostgresRows(client, input.table, input.data, input.where, signal) - return toRowsResponseBody( - result, - `Data updated successfully. ${result.rowCount} row(s) affected.` - ) + return { + message: `Data updated successfully. ${result.rowCount} row(s) affected.`, + rows: result.rows, + rowCount: result.rowCount, + } }) } export function executePostgresqlDelete(input: PostgresqlDeleteInput, signal?: AbortSignal) { return withPostgresClient(input, signal, async (client) => { const result = await deletePostgresRows(client, input.table, input.where, signal) - return toRowsResponseBody( - result, - `Data deleted successfully. ${result.rowCount} row(s) affected.` - ) + return { + message: `Data deleted successfully. ${result.rowCount} row(s) affected.`, + rows: result.rows, + rowCount: result.rowCount, + } }) } diff --git a/apps/sim/lib/internal/postgresql/queries.test.ts b/apps/sim/lib/internal/postgresql/queries.test.ts index 06107b16192..118cfea29b5 100644 --- a/apps/sim/lib/internal/postgresql/queries.test.ts +++ b/apps/sim/lib/internal/postgresql/queries.test.ts @@ -12,7 +12,6 @@ import type { PostgresClient } from '@/lib/internal/postgresql/client' import { deletePostgresRows, insertPostgresRows, - queryPostgres, sanitizePostgresIdentifier, updatePostgresRows, validatePostgresQuery, @@ -47,21 +46,6 @@ describe('PostgreSQL queries', () => { expect(mockExecutePostgresQuery).toHaveBeenCalledWith(pendingQuery, controller.signal) }) - it('caps a large SELECT and keeps the full row count', async () => { - const rows = Object.assign( - Array.from({ length: 10_001 }, (_, id) => ({ id })), - { count: 10_001 } - ) - mockExecutePostgresQuery.mockResolvedValue(rows) - - const result = await queryPostgres(mockClient, 'SELECT id FROM users') - - expect(result.rows).toHaveLength(10_000) - expect(result.rowCount).toBe(10_001) - expect(result.truncated).toBe(true) - expect(result.truncationReason).toContain('LIMIT and OFFSET') - }) - it('builds parameterized UPDATE values while preserving the existing WHERE syntax', async () => { await updatePostgresRows(mockClient, 'users', { active: false }, 'id = 42') diff --git a/apps/sim/lib/internal/postgresql/queries.ts b/apps/sim/lib/internal/postgresql/queries.ts index 13f64000fae..11939fad59c 100644 --- a/apps/sim/lib/internal/postgresql/queries.ts +++ b/apps/sim/lib/internal/postgresql/queries.ts @@ -1,9 +1,10 @@ import type { PostgresClient } from '@/lib/internal/postgresql/client' import { executePostgresQuery } from '@/lib/internal/postgresql/client' -import { - type CappedRowsResult, - capStatementRows, -} from '@/lib/internal/tool-operations/response-limits' + +export interface PostgresRowsResult { + rows: unknown[] + rowCount: number +} export interface PostgresIntrospectionResult { tables: Array<{ @@ -74,17 +75,16 @@ export async function queryPostgres( query: string, params: unknown[] = [], signal?: AbortSignal -): Promise { +): Promise { type PostgresParameters = NonNullable[1]> const result = await executePostgresQuery( client.unsafe(query, params as PostgresParameters), signal ) - return capStatementRows( - result, - result.count ?? result.length ?? 0, - 'A write statement still ran in full; page a SELECT with LIMIT and OFFSET to read the rest.' - ) + return { + rows: result, + rowCount: result.count ?? result.length ?? 0, + } } export function validatePostgresQuery(query: string): { isValid: boolean; error?: string } { @@ -118,7 +118,7 @@ export async function insertPostgresRows( table: string, data: Record, signal?: AbortSignal -): Promise { +): Promise { const sanitizedTable = sanitizePostgresIdentifier(table) const columns = Object.keys(data) const sanitizedColumns = columns.map((column) => sanitizePostgresIdentifier(column)) @@ -135,7 +135,7 @@ export async function updatePostgresRows( data: Record, where: string, signal?: AbortSignal -): Promise { +): Promise { validateWhereClause(where) const sanitizedTable = sanitizePostgresIdentifier(table) @@ -153,7 +153,7 @@ export async function deletePostgresRows( table: string, where: string, signal?: AbortSignal -): Promise { +): Promise { validateWhereClause(where) const sanitizedTable = sanitizePostgresIdentifier(table) diff --git a/apps/sim/lib/internal/tool-operations/response-limits.test.ts b/apps/sim/lib/internal/tool-operations/response-limits.test.ts index a0dc548c79c..c2aa1e29570 100644 --- a/apps/sim/lib/internal/tool-operations/response-limits.test.ts +++ b/apps/sim/lib/internal/tool-operations/response-limits.test.ts @@ -1,38 +1,42 @@ import { describe, expect, it } from 'vitest' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { - capResultRows, - capStatementRows, + createResponseRowsBudget, MAX_TOOL_RESPONSE_BODY_BYTES, - toRowsResponseBody, } from '@/lib/internal/tool-operations/response-limits' -describe('database result caps', () => { - it('keeps every row of a result inside both ceilings', () => { - const rows = Array.from({ length: 10_000 }, (_, id) => ({ id })) - expect(capResultRows(rows)).toEqual({ rows, truncated: false }) - }) +/** Admits every row, returning the error the budget threw, if any. */ +function admitAll(rows: unknown[]): unknown { + const budget = createResponseRowsBudget('test_tool') + try { + for (const row of rows) budget.admit(row) + return undefined + } catch (error) { + return error + } +} - it('stops at the row ceiling', () => { - const rows = Array.from({ length: 10_001 }, (_, id) => ({ id })) - const capped = capResultRows(rows) - expect(capped.truncated).toBe(true) - expect(capped.rows).toHaveLength(10_000) +describe('response rows budget', () => { + it('admits any number of rows while their array still fits the response limit', () => { + expect(admitAll(Array.from({ length: 50_000 }, (_, id) => ({ id })))).toBeUndefined() }) - it('measures UTF-8 bytes, so multibyte text cannot exceed the response ceiling', () => { - const value = '漢'.repeat(100_000) - const capped = capStatementRows( - Array.from({ length: 50 }, () => ({ value })), - 50, - 'Page the rest.' - ) - expect(capped.truncated).toBe(true) - const body = JSON.stringify(toRowsResponseBody(capped, 'Query done.')) - expect(Buffer.byteLength(body, 'utf8')).toBeLessThanOrEqual(MAX_TOOL_RESPONSE_BODY_BYTES) + it('fails exactly where the serialized array passes the response limit', () => { + const fitting = ['x'.repeat(MAX_TOOL_RESPONSE_BODY_BYTES - 4)] + expect(Buffer.byteLength(JSON.stringify(fitting))).toBe(MAX_TOOL_RESPONSE_BODY_BYTES) + expect(admitAll(fitting)).toBeUndefined() + + const error = admitAll(['x'.repeat(MAX_TOOL_RESPONSE_BODY_BYTES - 3)]) + expect(isPayloadSizeLimitError(error)).toBe(true) + expect(error).toMatchObject({ + label: 'test_tool response body', + maxBytes: MAX_TOOL_RESPONSE_BODY_BYTES, + }) }) - it('drops a single row larger than the ceiling instead of admitting it', () => { - const capped = capResultRows([{ blob: 'x'.repeat(MAX_TOOL_RESPONSE_BODY_BYTES) }]) - expect(capped).toEqual({ rows: [], truncated: true }) + it('counts UTF-8 bytes, so multibyte text fails on its encoded size', () => { + const row = '漢'.repeat(MAX_TOOL_RESPONSE_BODY_BYTES / 3) + expect(row.length).toBeLessThan(MAX_TOOL_RESPONSE_BODY_BYTES) + expect(isPayloadSizeLimitError(admitAll([row]))).toBe(true) }) }) diff --git a/apps/sim/lib/internal/tool-operations/response-limits.ts b/apps/sim/lib/internal/tool-operations/response-limits.ts index f93738615f3..2c7ccb5e1b1 100644 --- a/apps/sim/lib/internal/tool-operations/response-limits.ts +++ b/apps/sim/lib/internal/tool-operations/response-limits.ts @@ -1,85 +1,28 @@ +import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' + /** Maximum inline tool response size; binary file outputs use stored descriptors. */ export const MAX_TOOL_RESPONSE_BODY_BYTES = 10 * 1024 * 1024 -/** Most rows a single database statement returns inline. */ -const MAX_TOOL_RESULT_ROWS = 10_000 - -/** - * Bytes the serialized rows array may occupy: the response ceiling minus a reserve for the - * envelope around it (`message`, counts, `truncated`, `truncationReason`). - */ -const MAX_TOOL_RESULT_ROWS_BYTES = MAX_TOOL_RESPONSE_BODY_BYTES - 4096 - /** - * Admits rows into a tool response until the row or UTF-8 byte ceiling is reached. Each row is - * serialized once, the way the response will serialize it, and one byte per row covers the array - * punctuation. A row that does not fit is refused even when it is the first, so one huge value - * cannot bypass the ceiling. + * Fails a streamed result as soon as its serialized rows alone exceed + * {@link MAX_TOOL_RESPONSE_BODY_BYTES}. Bytes are counted the way the response serializes them + * (UTF-8, plus the array's brackets and commas), and the body around the rows is larger still, so + * this fails exactly the results the executor's response limit would reject, but before the rest + * of the result is read into memory. The error is the same one that limit throws. */ -export function createResultRowsBudget() { - let admitted = 0 +export function createResponseRowsBudget(toolId: string) { let bytes = 1 return { - admit(row: unknown): boolean { - if (admitted >= MAX_TOOL_RESULT_ROWS) return false + admit(row: unknown): void { const serialized = JSON.stringify(row) - const rowBytes = serialized === undefined ? 4 : Buffer.byteLength(serialized, 'utf8') - if (bytes + rowBytes + 1 > MAX_TOOL_RESULT_ROWS_BYTES) return false - bytes += rowBytes + 1 - admitted++ - return true + bytes += (serialized === undefined ? 4 : Buffer.byteLength(serialized, 'utf8')) + 1 + if (bytes > MAX_TOOL_RESPONSE_BODY_BYTES) { + throw new PayloadSizeLimitError({ + label: `${toolId} response body`, + maxBytes: MAX_TOOL_RESPONSE_BODY_BYTES, + observedBytes: bytes, + }) + } }, } } - -/** Keeps the leading rows that fit {@link createResultRowsBudget}. */ -export function capResultRows(rows: readonly TRow[]): { rows: TRow[]; truncated: boolean } { - const budget = createResultRowsBudget() - const capped: TRow[] = [] - for (const row of rows) { - if (!budget.admit(row)) break - capped.push(row) - } - return { rows: capped, truncated: capped.length < rows.length } -} - -/** Tells the caller which ceiling truncated a result and how to read the rest. */ -export function resultTruncationReason(returned: number, paginationHint: string): string { - return returned === 0 - ? 'No rows returned: the first row alone exceeds the 10 MiB response ceiling. Select fewer columns or slice large values.' - : `Result truncated to ${returned} row(s): one statement returns at most ${MAX_TOOL_RESULT_ROWS.toLocaleString('en-US')} rows or 10 MiB. ${paginationHint}` -} - -/** A capped statement result: rows as returned, plus the truncation notice when rows were dropped. */ -export interface CappedRowsResult { - rows: unknown[] - rowCount: number - truncated?: true - truncationReason?: string -} - -/** Caps a statement's rows; `rowCount` stays what the statement returned or affected. */ -export function capStatementRows( - rows: readonly unknown[], - rowCount: number, - paginationHint: string -): CappedRowsResult { - const capped = capResultRows(rows) - if (!capped.truncated) return { rows: capped.rows, rowCount } - return { - rows: capped.rows, - rowCount, - truncated: true, - truncationReason: resultTruncationReason(capped.rows.length, paginationHint), - } -} - -/** The response body for a capped statement, surfacing any truncation in the message too. */ -export function toRowsResponseBody(result: CappedRowsResult, message: string) { - return { - message: result.truncationReason ? `${message} ${result.truncationReason}` : message, - rows: result.rows, - rowCount: result.rowCount, - ...(result.truncated && { truncated: true, truncationReason: result.truncationReason }), - } -} diff --git a/apps/sim/tools/mongodb/execute.ts b/apps/sim/tools/mongodb/execute.ts index 762b85fd074..afe0f38c9dc 100644 --- a/apps/sim/tools/mongodb/execute.ts +++ b/apps/sim/tools/mongodb/execute.ts @@ -91,10 +91,6 @@ export const executeTool: InternalToolConfig message: data.message || 'Query executed successfully', documents: data.documents || [], documentCount: data.documentCount || 0, - ...(data.truncated && { - truncated: true, - truncationReason: data.truncationReason, - }), }, error: undefined, } @@ -118,16 +114,5 @@ export const queryTool: InternalToolConfig message: { type: 'string', description: 'Operation status message' }, documents: { type: 'array', description: 'Array of documents returned from the query' }, documentCount: { type: 'number', description: 'Number of documents returned' }, - truncated: { - type: 'boolean', - description: - 'Present and true only when documents were dropped to stay inside the response ceilings (10,000 documents or 10 MiB). Absent means the result is complete', - optional: true, - }, - truncationReason: { - type: 'string', - description: 'Which ceiling was hit and how to read the rest', - optional: true, - }, }, } diff --git a/apps/sim/tools/mongodb/types.ts b/apps/sim/tools/mongodb/types.ts index be559b735b8..b20a59aa9fc 100644 --- a/apps/sim/tools/mongodb/types.ts +++ b/apps/sim/tools/mongodb/types.ts @@ -68,8 +68,6 @@ interface MongoDBBaseResponse extends ToolResponse { message: string documents?: unknown[] documentCount: number - truncated?: boolean - truncationReason?: string insertedId?: string insertedIds?: string[] modifiedCount?: number diff --git a/apps/sim/tools/mysql/execute.ts b/apps/sim/tools/mysql/execute.ts index 0b8ec83a6a2..580a9c7b032 100644 --- a/apps/sim/tools/mysql/execute.ts +++ b/apps/sim/tools/mysql/execute.ts @@ -80,10 +80,6 @@ export const executeTool: InternalToolConfig message: data.message || 'Query executed successfully', rows: data.rows || [], rowCount: data.rowCount || 0, - ...(data.truncated && { - truncated: true, - truncationReason: data.truncationReason, - }), }, error: undefined, } @@ -93,16 +89,5 @@ export const executeTool: InternalToolConfig message: { type: 'string', description: 'Operation status message' }, rows: { type: 'array', description: 'Array of rows returned from the query' }, rowCount: { type: 'number', description: 'Number of rows affected' }, - truncated: { - type: 'boolean', - description: - 'Present and true only when rows were dropped to stay inside the response ceilings (10,000 rows or 10 MiB). Absent means the result is complete', - optional: true, - }, - truncationReason: { - type: 'string', - description: 'Which ceiling was hit and how to read the rest', - optional: true, - }, }, } diff --git a/apps/sim/tools/mysql/query.ts b/apps/sim/tools/mysql/query.ts index 8bed01fcd40..41ff5d4d06a 100644 --- a/apps/sim/tools/mysql/query.ts +++ b/apps/sim/tools/mysql/query.ts @@ -79,10 +79,6 @@ export const queryTool: InternalToolConfig = { message: data.message || 'Query executed successfully', rows: data.rows || [], rowCount: data.rowCount || 0, - ...(data.truncated && { - truncated: true, - truncationReason: data.truncationReason, - }), }, error: undefined, } @@ -92,16 +88,5 @@ export const queryTool: InternalToolConfig = { message: { type: 'string', description: 'Operation status message' }, rows: { type: 'array', description: 'Array of rows returned from the query' }, rowCount: { type: 'number', description: 'Number of rows returned' }, - truncated: { - type: 'boolean', - description: - 'Present and true only when rows were dropped to stay inside the response ceilings (10,000 rows or 10 MiB). Absent means the result is complete', - optional: true, - }, - truncationReason: { - type: 'string', - description: 'Which ceiling was hit and how to read the rest', - optional: true, - }, }, } diff --git a/apps/sim/tools/mysql/types.ts b/apps/sim/tools/mysql/types.ts index 89fcdc9d50d..be1100ab831 100644 --- a/apps/sim/tools/mysql/types.ts +++ b/apps/sim/tools/mysql/types.ts @@ -38,8 +38,6 @@ interface MySQLBaseResponse extends ToolResponse { message: string rows: unknown[] rowCount: number - truncated?: boolean - truncationReason?: string } error?: string } diff --git a/apps/sim/tools/neo4j/execute.ts b/apps/sim/tools/neo4j/execute.ts index 8ff0322b0ec..a8ecb09444c 100644 --- a/apps/sim/tools/neo4j/execute.ts +++ b/apps/sim/tools/neo4j/execute.ts @@ -87,10 +87,6 @@ export const executeTool: InternalToolConfig records: data.records || [], recordCount: data.recordCount || 0, summary: data.summary, - ...(data.truncated && { - truncated: true, - truncationReason: data.truncationReason, - }), }, error: undefined, } @@ -101,16 +97,5 @@ export const executeTool: InternalToolConfig records: { type: 'array', description: 'Array of records returned from the query' }, recordCount: { type: 'number', description: 'Number of records returned' }, summary: { type: 'json', description: 'Execution summary with timing and counters' }, - truncated: { - type: 'boolean', - description: - 'Present and true only when records were dropped to stay inside the response ceilings (10,000 records or 10 MiB). Absent means the result is complete', - optional: true, - }, - truncationReason: { - type: 'string', - description: 'Which ceiling was hit and how to read the rest', - optional: true, - }, }, } diff --git a/apps/sim/tools/neo4j/query.ts b/apps/sim/tools/neo4j/query.ts index 277cb7a2150..68895e345af 100644 --- a/apps/sim/tools/neo4j/query.ts +++ b/apps/sim/tools/neo4j/query.ts @@ -88,10 +88,6 @@ export const queryTool: InternalToolConfig = { records: data.records || [], recordCount: data.recordCount || 0, summary: data.summary, - ...(data.truncated && { - truncated: true, - truncationReason: data.truncationReason, - }), }, error: undefined, } @@ -102,16 +98,5 @@ export const queryTool: InternalToolConfig = { records: { type: 'array', description: 'Array of records returned from the query' }, recordCount: { type: 'number', description: 'Number of records returned' }, summary: { type: 'json', description: 'Query execution summary with timing and counters' }, - truncated: { - type: 'boolean', - description: - 'Present and true only when records were dropped to stay inside the response ceilings (10,000 records or 10 MiB). Absent means the result is complete', - optional: true, - }, - truncationReason: { - type: 'string', - description: 'Which ceiling was hit and how to read the rest', - optional: true, - }, }, } diff --git a/apps/sim/tools/neo4j/types.ts b/apps/sim/tools/neo4j/types.ts index 5d74922616d..ce5b70f4565 100644 --- a/apps/sim/tools/neo4j/types.ts +++ b/apps/sim/tools/neo4j/types.ts @@ -45,8 +45,6 @@ interface Neo4jBaseResponse extends ToolResponse { message: string records?: unknown[] recordCount?: number - truncated?: boolean - truncationReason?: string summary?: { resultAvailableAfter: number resultConsumedAfter: number diff --git a/apps/sim/tools/postgresql/delete.ts b/apps/sim/tools/postgresql/delete.ts index 4ae980a799c..0f17bd09bca 100644 --- a/apps/sim/tools/postgresql/delete.ts +++ b/apps/sim/tools/postgresql/delete.ts @@ -86,10 +86,6 @@ export const deleteTool: InternalToolConfig