Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 116 additions & 5 deletions apps/sim/lib/workflows/application/read-workflow-lint-tables.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -307,11 +307,42 @@ describe('standalone table diagnostics against the actual Table block', () => {
).toEqual([expect.objectContaining({ field: 'data', value: ['missing'] })])
})

it('defers a filter with more mixed references than the placeholder search covers', async () => {
const conditions = [
...['a', 'b', 'c', 'd'].map((name) => `{"field":"name","op":"eq","value":<start.${name}>}`),
...['e', 'f', 'g'].map((name) => `{"field":"name","op":"in","value":<start.${name}>}`),
]
const manyReferenceData = `{${['a', 'b', 'c', 'd', 'e', 'f', 'g']
.map((name) => `"name": <start.${name}>`)
.join(',')},"missing":1}`

it.each([
['insert_row', manyReferenceData, undefined],
[
'update_rows_by_filter',
manyReferenceData,
'{"all":[{"field":"name","op":"eq","value":<start.n>},{"field":"name","op":"in","value":<start.names>}]}',
],
])(
'checks literal row keys beside many bare references (%s)',
async (operation, data, filter) => {
setGraph(block('write', { operation, bulkFilterMode: 'json', data, filter }))
const result = await lint()
expect(result.notes.filter((note) => note.includes('could not complete'))).toEqual([])
expect(
result.unresolvedReferences.filter((reference) => reference.kind === 'resource')
).toEqual([expect.objectContaining({ field: 'data', value: ['missing'] })])
}
)

it('checks the literal conditions beside a whole-condition reference', async () => {
setGraph(
block('query', {
filter: '{"all":[<start.condition>,{"field":"other","op":"eq","value":1}]}',
})
)
const result = await lint()
expect(result.notes.filter((note) => note.includes('could not complete'))).toEqual([])
expect(result.tableFieldIssues.map(({ field }) => field)).toEqual(['other'])
})

it('defers a filter group made only of references and still checks row data', async () => {
const conditions = ['a', 'b', 'c', 'd', 'e', 'f', 'g'].map((name) => `<start.${name}>`)
setGraph(
block('write', {
operation: 'update_rows_by_filter',
Expand All @@ -330,6 +361,86 @@ describe('standalone table diagnostics against the actual Table block', () => {
).toEqual([expect.objectContaining({ field: 'data', value: ['missing'] })])
})

it('checks row data whose keys resemble a reference-only filter group', async () => {
setGraph(
block('write', {
operation: 'update_rows_by_filter',
bulkFilterMode: 'json',
filter: '<start.filter>',
data: '{"all":[<start.values>],"missing":1}',
})
)
const result = await lint()
expect(
result.unresolvedReferences.filter((reference) => reference.kind === 'resource')
).toEqual([expect.objectContaining({ field: 'data', value: ['all', 'missing'] })])
})

it('reports a quoted in-list value that matches a bare reference elsewhere', async () => {
setGraph(
block('query', {
filter:
'{"all":[{"field":"name","op":"eq","value":<start.names>},{"field":"name","op":"in","value":"<start.names>"}]}',
})
)
const result = await lint()
expect(result.notes).toEqual(
expect.arrayContaining([
expect.stringContaining(
'Table input checks in block "query" could not complete before runtime:'
),
])
)
})

it('keeps a __proto__ row key beside a bare reference', async () => {
setGraph(
block('write', {
operation: 'insert_row',
data: '{"name":<start.name>,"__proto__":"extra"}',
})
)
const result = await lint()
expect(
result.unresolvedReferences.filter((reference) => reference.kind === 'resource')
).toEqual([expect.objectContaining({ field: 'data', value: ['__proto__'] })])
})

it('blames malformed row data, not a whole-field filter reference', async () => {
setGraph(
block('write', {
operation: 'update_rows_by_filter',
bulkFilterMode: 'json',
filter: '<start.filter>',
data: '{"name":"Ada",}',
})
)
const result = await lint()
expect(result.notes).toEqual(
expect.arrayContaining([
expect.stringContaining(
'Table input checks in block "write" could not complete before runtime: Invalid JSON in Row Data'
),
])
)
})

it('does not call deferred whole-field row data a non-static object', async () => {
setGraph(
block('write', {
operation: 'update_rows_by_filter',
bulkFilterMode: 'json',
filter: '<start.filter>',
data: '<start.row>',
})
)
const result = await lint()
expect(result.notes).toContain(
'Table inputs in block "write" depend on references resolved at runtime and were not checked: data, filterInput.'
)
expect(result.notes.filter((note) => note.includes('not a static object'))).toEqual([])
})

it('still reports an invalid literal condition beside a reference', async () => {
setGraph(
block('query', {
Expand Down
142 changes: 81 additions & 61 deletions apps/sim/lib/workflows/application/read-workflow-lint.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { requirePrincipalSubjectUserId } from '@sim/auth/principal'
import { getErrorMessage } from '@sim/utils/errors'
import { generateShortId } from '@sim/utils/id'
import { isPlainRecord } from '@sim/utils/object'
import { findWorkflowReferenceTokens } from '@sim/utils/workflow-references'
import type { CursorKey } from '@/lib/api/list-query'
Expand Down Expand Up @@ -258,74 +259,89 @@ function hasRuntimeReference(value: string): boolean {
/**
* Outside Function code a reference resolves to JSON text (objects, arrays, numbers, booleans), so
* `{"name": <start.name>}` or a whole-field `<start.row>` is valid JSON at runtime but not before.
* Each reference written outside a JSON string is replaced by a placeholder so the Table block's
* own transform reads the literal parts: a quoted string, or a one-item list for slots such as a
* filter `in`, trying every combination. A field that is a single whole reference has no literal
* part, so if no combination fits it is left out and returned in `runtimeFields`. When the search
* is exhaustive and still fails, the error comes from the literal parts (malformed JSON, an
* invalid literal condition) and is rethrown. Past the search bound that cannot be told apart, so
* every reference-bearing field is left out instead and the other inputs are still checked. Only
* this lint-local copy is rewritten; references to string outputs left unquoted are reported by
* `collectUnquotedJsonStringReferences`.
* Each reference written outside a JSON string becomes a quoted placeholder so the Table block's
* own transform reads the literal parts (see {@link withReferencePlaceholders}). If the transform
* still fails and a field has no literal part, that field is left out, returned in
* `runtimeFields`, and the transform runs once more; an error from that final attempt comes from
* the literal parts and is rethrown. Only this lint-local copy is rewritten; references to string
* outputs left unquoted are reported by `collectUnquotedJsonStringReferences`.
*/
function normalizeTableInputs(
transform: (params: Record<string, unknown>) => Record<string, unknown>,
params: Record<string, unknown>
): { normalized: Record<string, unknown>; runtimeFields: string[] } {
const referenceFields: string[] = []
const wholeReferenceFields: string[] = []
const prepared: Record<string, unknown> = {}
const runtimeFields: string[] = []
for (const [key, value] of Object.entries(params)) {
if (typeof value !== 'string') continue
const trimmed = value.trim()
const tokens = findWorkflowReferenceTokens(trimmed)
if (tokens.length === 0) continue
referenceFields.push(key)
if (tokens.length === 1 && tokens[0].start === 0 && tokens[0].end === trimmed.length) {
wholeReferenceFields.push(key)
}
}
if (referenceFields.length === 0) return { normalized: transform(params), runtimeFields: [] }
const tiers = wholeReferenceFields.length > 0 ? [[], wholeReferenceFields] : [[]]
let searchBounded = false
let firstError: unknown
for (const runtimeFields of tiers) {
const withPlaceholders = (asList: (ordinal: number) => boolean) => {
let ordinal = 0
const candidate: Record<string, unknown> = {}
for (const [key, value] of Object.entries(params)) {
if (runtimeFields.includes(key)) continue
candidate[key] =
typeof value === 'string'
? replaceBareReferences(value, (reference) =>
asList(ordinal++) ? `[${JSON.stringify(reference)}]` : JSON.stringify(reference)
)
: value
}
return { candidate, count: ordinal }
}
const { count } = withPlaceholders(() => false)
if (count > MAX_PLACEHOLDER_SEARCH_REFERENCES) {
searchBounded = true
if (typeof value !== 'string') {
prepared[key] = value
continue
}
for (let mask = 0; mask < 2 ** count; mask++) {
try {
const { candidate } = withPlaceholders((ordinal) => ((mask >> ordinal) & 1) === 1)
return { normalized: transform(candidate), runtimeFields }
} catch (error) {
firstError ??= error
}
}
const input = withReferencePlaceholders(value)
prepared[key] = input.value
if (input.runtimeOnly) runtimeFields.push(key)
}
try {
return { normalized: transform(prepared), runtimeFields: [] }
} catch (error) {
if (runtimeFields.length === 0) throw error
}
if (!searchBounded) throw firstError
const literal = Object.fromEntries(
Object.entries(params).filter(([key]) => !referenceFields.includes(key))
)
return { normalized: transform(literal), runtimeFields: referenceFields }
for (const key of runtimeFields) delete prepared[key]
return { normalized: transform(prepared), runtimeFields }
}

/** Bounds the placeholder search to 2^6 pure transform calls per Table block. */
const MAX_PLACEHOLDER_SEARCH_REFERENCES = 6
/**
* Quotes each bare reference occurrence under its own placeholder, so an already-quoted string
* with the same text is never mistaken for one. A quoted reference fits row values and most
* filter values; the value of an `in`/`nin` condition must be a list, so it becomes a one-item
* list, and a condition or group made only of references has no literal part, so it is left out
* of an `all`/`any` group that keeps a literal member. `runtimeOnly` marks a field whose whole
* value is such a reference or group.
*/
function withReferencePlaceholders(json: string): { value: string; runtimeOnly: boolean } {
const nonce = generateShortId()
const references = new Map<string, string>()
const quoted = replaceBareReferences(json, (reference) => {
const placeholder = `${nonce}:${references.size}`
references.set(placeholder, reference)
return JSON.stringify(placeholder)
})
if (references.size === 0) return { value: json, runtimeOnly: false }
let parsed: unknown
try {
parsed = JSON.parse(quoted)
} catch {
const readable = replaceBareReferences(json, (reference) => JSON.stringify(reference))
return { value: readable, runtimeOnly: false }
}
const isPlaceholder = (node: unknown): node is string =>
typeof node === 'string' && references.has(node)
const isDynamic = (node: unknown): boolean => {
if (isPlaceholder(node)) return true
if (!isPlainRecord(node) || Object.keys(node).length !== 1) return false
const members = node.all ?? node.any
return Array.isArray(members) && members.length > 0 && members.every(isDynamic)
Comment thread
waleedlatif1 marked this conversation as resolved.
Resolved
}
const adapt = (node: unknown): unknown => {
if (isPlaceholder(node)) return references.get(node)
if (Array.isArray(node)) return node.map(adapt)
if (!isPlainRecord(node)) return node
return Object.fromEntries(
Object.entries(node).map(([key, value]) => {
const name = references.get(key) ?? key
if ((key === 'all' || key === 'any') && Array.isArray(value)) {
const literal = value.filter((member) => !isDynamic(member))
return [name, adapt(literal.length > 0 ? literal : value)]
}
if (key === 'value' && (node.op === 'in' || node.op === 'nin') && isPlaceholder(value)) {
return [name, [adapt(value)]]
}
return [name, adapt(value)]
})
)
}
return { value: JSON.stringify(adapt(parsed)), runtimeOnly: isDynamic(parsed) }
}

function replaceBareReferences(json: string, placeholder: (reference: string) => string): string {
const tokens = findWorkflowReferenceTokens(json)
Expand Down Expand Up @@ -411,12 +427,14 @@ async function readTableDiagnostics(
const transform = getBlock(block.type)?.tools.config?.params
if (!transform) throw new Error('Table input normalization is unavailable')
let normalized: Record<string, unknown>
let runtimeFields: string[]
try {
const inputs = normalizeTableInputs(transform, params)
normalized = inputs.normalized
if (inputs.runtimeFields.length > 0) {
runtimeFields = inputs.runtimeFields
if (runtimeFields.length > 0) {
result.notes.push(
`Table inputs in block "${block.name || blockId}" depend on references resolved at runtime and were not checked: ${inputs.runtimeFields.join(', ')}.`
`Table inputs in block "${block.name || blockId}" depend on references resolved at runtime and were not checked: ${runtimeFields.join(', ')}.`
)
}
} catch (error) {
Expand Down Expand Up @@ -464,8 +482,10 @@ async function readTableDiagnostics(
})
}
}
if (Object.hasOwn(normalized, 'data')) inspectRow(normalized.data, 'data')
if (Object.hasOwn(normalized, 'rows')) {
if (Object.hasOwn(normalized, 'data') && !runtimeFields.includes('data')) {
inspectRow(normalized.data, 'data')
}
if (Object.hasOwn(normalized, 'rows') && !runtimeFields.includes('rows')) {
if (Array.isArray(normalized.rows)) {
normalized.rows.forEach((row, index) => inspectRow(row, `rows[${index}]`))
} else {
Expand Down
Loading