Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion apps/sim/tools/elasticsearch/bulk.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import type {
} from '@/tools/elasticsearch/types'
import { buildAuthHeaders, buildBaseUrl } from '@/tools/elasticsearch/utils'
import type { ToolConfig } from '@/tools/types'
import { safeUrlPathSegment } from '@/tools/url-path'

export const bulkTool: ToolConfig<ElasticsearchBulkParams, ElasticsearchBulkResponse> = {
id: 'elasticsearch_bulk',
Expand Down Expand Up @@ -77,7 +78,7 @@ export const bulkTool: ToolConfig<ElasticsearchBulkParams, ElasticsearchBulkResp
url: (params) => {
const baseUrl = buildBaseUrl(params)
let url = params.index
? `${baseUrl}/${encodeURIComponent(params.index)}/_bulk`
? `${baseUrl}/${safeUrlPathSegment(params.index, 'index')}/_bulk`
: `${baseUrl}/_bulk`

if (params.refresh) {
Expand Down
3 changes: 2 additions & 1 deletion apps/sim/tools/elasticsearch/count.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import type {
} from '@/tools/elasticsearch/types'
import { buildAuthHeaders, buildBaseUrl } from '@/tools/elasticsearch/utils'
import type { ToolConfig } from '@/tools/types'
import { safeUrlPathSegment } from '@/tools/url-path'

export const countTool: ToolConfig<ElasticsearchCountParams, ElasticsearchCountResponse> = {
id: 'elasticsearch_count',
Expand Down Expand Up @@ -70,7 +71,7 @@ export const countTool: ToolConfig<ElasticsearchCountParams, ElasticsearchCountR
request: {
url: (params) => {
const baseUrl = buildBaseUrl(params)
return `${baseUrl}/${encodeURIComponent(params.index)}/_count`
return `${baseUrl}/${safeUrlPathSegment(params.index, 'index')}/_count`
},
method: 'POST',
headers: (params) => buildAuthHeaders(params),
Expand Down
3 changes: 2 additions & 1 deletion apps/sim/tools/elasticsearch/create_index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import type {
} from '@/tools/elasticsearch/types'
import { buildAuthHeaders, buildBaseUrl } from '@/tools/elasticsearch/utils'
import type { ToolConfig } from '@/tools/types'
import { safeUrlPathSegment } from '@/tools/url-path'

export const createIndexTool: ToolConfig<
ElasticsearchCreateIndexParams,
Expand Down Expand Up @@ -76,7 +77,7 @@ export const createIndexTool: ToolConfig<
request: {
url: (params) => {
const baseUrl = buildBaseUrl(params)
return `${baseUrl}/${encodeURIComponent(params.index)}`
return `${baseUrl}/${safeUrlPathSegment(params.index, 'index')}`
},
method: 'PUT',
headers: (params) => buildAuthHeaders(params),
Expand Down
10 changes: 9 additions & 1 deletion apps/sim/tools/elasticsearch/delete_document.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import type {
} from '@/tools/elasticsearch/types'
import { buildAuthHeaders, buildBaseUrl } from '@/tools/elasticsearch/utils'
import type { ToolConfig } from '@/tools/types'
import { safeUrlPathSegment } from '@/tools/url-path'

export const deleteDocumentTool: ToolConfig<
ElasticsearchDeleteDocumentParams,
Expand Down Expand Up @@ -77,7 +78,14 @@ export const deleteDocumentTool: ToolConfig<
request: {
url: (params) => {
const baseUrl = buildBaseUrl(params)
let url = `${baseUrl}/${encodeURIComponent(params.index)}/_doc/${encodeURIComponent(params.documentId)}`
let url = `${baseUrl}/${safeUrlPathSegment(params.index, 'index')}/_doc/${safeUrlPathSegment(
Comment thread
waleedlatif1 marked this conversation as resolved.
Resolved
params.documentId,
'documentId',
{
allowSeparators: true,
preserveWhitespace: true,
}
)}`

if (params.refresh) {
url += `?refresh=${params.refresh}`
Expand Down
3 changes: 2 additions & 1 deletion apps/sim/tools/elasticsearch/delete_index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import type {
} from '@/tools/elasticsearch/types'
import { buildAuthHeaders, buildBaseUrl } from '@/tools/elasticsearch/utils'
import type { ToolConfig } from '@/tools/types'
import { safeUrlPathSegment } from '@/tools/url-path'

export const deleteIndexTool: ToolConfig<
ElasticsearchDeleteIndexParams,
Expand Down Expand Up @@ -66,7 +67,7 @@ export const deleteIndexTool: ToolConfig<
request: {
url: (params) => {
const baseUrl = buildBaseUrl(params)
return `${baseUrl}/${encodeURIComponent(params.index)}`
return `${baseUrl}/${safeUrlPathSegment(params.index, 'index')}`
},
method: 'DELETE',
headers: (params) => buildAuthHeaders(params),
Expand Down
10 changes: 9 additions & 1 deletion apps/sim/tools/elasticsearch/get_document.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import type {
} from '@/tools/elasticsearch/types'
import { buildAuthHeaders, buildBaseUrl } from '@/tools/elasticsearch/utils'
import type { ToolConfig } from '@/tools/types'
import { safeUrlPathSegment } from '@/tools/url-path'

export const getDocumentTool: ToolConfig<
ElasticsearchGetDocumentParams,
Expand Down Expand Up @@ -82,7 +83,14 @@ export const getDocumentTool: ToolConfig<
request: {
url: (params) => {
const baseUrl = buildBaseUrl(params)
let url = `${baseUrl}/${encodeURIComponent(params.index)}/_doc/${encodeURIComponent(params.documentId)}`
let url = `${baseUrl}/${safeUrlPathSegment(params.index, 'index')}/_doc/${safeUrlPathSegment(
params.documentId,
'documentId',
{
allowSeparators: true,
preserveWhitespace: true,
}
)}`

const queryParams: string[] = []
if (params.sourceIncludes) {
Expand Down
3 changes: 2 additions & 1 deletion apps/sim/tools/elasticsearch/get_index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import type {
} from '@/tools/elasticsearch/types'
import { buildAuthHeaders, buildBaseUrl } from '@/tools/elasticsearch/utils'
import type { ToolConfig } from '@/tools/types'
import { safeUrlPathSegment } from '@/tools/url-path'

export const getIndexTool: ToolConfig<ElasticsearchGetIndexParams, ElasticsearchIndexInfoResponse> =
{
Expand Down Expand Up @@ -64,7 +65,7 @@ export const getIndexTool: ToolConfig<ElasticsearchGetIndexParams, Elasticsearch
request: {
url: (params) => {
const baseUrl = buildBaseUrl(params)
return `${baseUrl}/${encodeURIComponent(params.index)}`
return `${baseUrl}/${safeUrlPathSegment(params.index, 'index')}`
},
method: 'GET',
headers: (params) => buildAuthHeaders(params),
Expand Down
8 changes: 6 additions & 2 deletions apps/sim/tools/elasticsearch/index_document.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import type {
} from '@/tools/elasticsearch/types'
import { buildAuthHeaders, buildBaseUrl } from '@/tools/elasticsearch/utils'
import type { ToolConfig } from '@/tools/types'
import { safeUrlPathSegment } from '@/tools/url-path'

export const indexDocumentTool: ToolConfig<
ElasticsearchIndexDocumentParams,
Expand Down Expand Up @@ -82,9 +83,12 @@ export const indexDocumentTool: ToolConfig<
request: {
url: (params) => {
const baseUrl = buildBaseUrl(params)
let url = `${baseUrl}/${encodeURIComponent(params.index)}/_doc`
let url = `${baseUrl}/${safeUrlPathSegment(params.index, 'index')}/_doc`
if (params.documentId) {
url += `/${encodeURIComponent(params.documentId)}`
url += `/${safeUrlPathSegment(params.documentId, 'documentId', {
allowSeparators: true,
preserveWhitespace: true,
})}`
}
if (params.refresh) {
url += `?refresh=${params.refresh}`
Expand Down
3 changes: 2 additions & 1 deletion apps/sim/tools/elasticsearch/search.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import type {
} from '@/tools/elasticsearch/types'
import { buildAuthHeaders, buildBaseUrl } from '@/tools/elasticsearch/utils'
import type { ToolConfig } from '@/tools/types'
import { safeUrlPathSegment } from '@/tools/url-path'

export const searchTool: ToolConfig<ElasticsearchSearchParams, ElasticsearchSearchResponse> = {
id: 'elasticsearch_search',
Expand Down Expand Up @@ -105,7 +106,7 @@ export const searchTool: ToolConfig<ElasticsearchSearchParams, ElasticsearchSear
request: {
url: (params) => {
const baseUrl = buildBaseUrl(params)
return `${baseUrl}/${encodeURIComponent(params.index)}/_search`
return `${baseUrl}/${safeUrlPathSegment(params.index, 'index')}/_search`
},
method: 'POST',
headers: (params) => buildAuthHeaders(params),
Expand Down
10 changes: 9 additions & 1 deletion apps/sim/tools/elasticsearch/update_document.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import type {
} from '@/tools/elasticsearch/types'
import { buildAuthHeaders, buildBaseUrl } from '@/tools/elasticsearch/utils'
import type { ToolConfig } from '@/tools/types'
import { safeUrlPathSegment } from '@/tools/url-path'

export const updateDocumentTool: ToolConfig<
ElasticsearchUpdateDocumentParams,
Expand Down Expand Up @@ -82,7 +83,14 @@ export const updateDocumentTool: ToolConfig<
request: {
url: (params) => {
const baseUrl = buildBaseUrl(params)
let url = `${baseUrl}/${encodeURIComponent(params.index)}/_update/${encodeURIComponent(params.documentId)}`
let url = `${baseUrl}/${safeUrlPathSegment(params.index, 'index')}/_update/${safeUrlPathSegment(
params.documentId,
'documentId',
{
allowSeparators: true,
preserveWhitespace: true,
}
)}`

if (params.retryOnConflict !== undefined) {
url += `?retry_on_conflict=${params.retryOnConflict}`
Expand Down
5 changes: 3 additions & 2 deletions apps/sim/tools/nested-model-input-adapters.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -374,8 +374,9 @@ describe('nested model-input adapters', () => {
apiKey: 'key',
rerank,
searchQuery: patch.searchQuery as string,
}) as { rerank: { query: { text: string } } }
expect(body.rerank.query.text).toBe('{{SEARCH_QUERY}}')
}) as { query: { inputs: { text: string } }; rerank: { query?: string } }
expect(body.query.inputs.text).toBe('{{SEARCH_QUERY}}')
expect(body.rerank.query).toBeUndefined()
})

it('preserves Pinecone singleton and legacy NDJSON input forms', () => {
Expand Down
6 changes: 5 additions & 1 deletion apps/sim/tools/pinecone/fetch.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import type { PineconeFetchParams, PineconeResponse, PineconeVector } from '@/tools/pinecone/types'
import { parseJsonParam } from '@/tools/pinecone/utils'
import type { ToolConfig } from '@/tools/types'

export const fetchTool: ToolConfig<PineconeFetchParams, PineconeResponse> = {
Expand Down Expand Up @@ -39,7 +40,10 @@ export const fetchTool: ToolConfig<PineconeFetchParams, PineconeResponse> = {
url: (params) => {
const baseUrl = `${params.indexHost}/vectors/fetch`
const queryParams = new URLSearchParams()
queryParams.append('ids', params.ids.join(','))
const ids = parseJsonParam<unknown>(params.ids, 'ids')
for (const id of Array.isArray(ids) ? ids : [ids]) {
queryParams.append('ids', String(id))
}
if (params.namespace) {
queryParams.append('namespace', params.namespace)
}
Expand Down
62 changes: 62 additions & 0 deletions apps/sim/tools/pinecone/requests.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
import { describe, expect, it } from 'vitest'
import { fetchTool } from '@/tools/pinecone/fetch'
import { searchTextTool } from '@/tools/pinecone/search_text'
import type { PineconeFetchParams, PineconeSearchTextParams } from '@/tools/pinecone/types'

const INDEX_HOST = 'https://example-index.svc.example.pinecone.io'

function buildUrl<P>(tool: { request: { url: unknown } }, params: P): URL {
const url = tool.request.url as (params: P) => string
return new URL(url(params))
}

describe('pinecone_fetch request', () => {
it('repeats the ids query parameter once per vector id', () => {
const url = buildUrl<PineconeFetchParams>(fetchTool, {
indexHost: INDEX_HOST,
ids: ['vec-1', 'vec-2'],
namespace: 'docs',
apiKey: 'key',
})

expect(url.searchParams.getAll('ids')).toEqual(['vec-1', 'vec-2'])
})
})

describe('pinecone_search_text request', () => {
const baseParams: PineconeSearchTextParams = {
indexHost: INDEX_HOST,
searchQuery: 'capital of France',
apiKey: 'key',
}

it('rejects a missing namespace instead of searching a namespace named "undefined"', () => {
expect(() => buildUrl<PineconeSearchTextParams>(searchTextTool, baseParams)).toThrow(
/namespace is required/
)
})

it('sends the rerank query as a string', () => {
const body = searchTextTool.request.body as (
params: PineconeSearchTextParams
) => Record<string, Record<string, unknown>>

const withoutQuery = body({
...baseParams,
rerank: { model: 'bge-reranker-v2-m3', rank_fields: ['chunk_text'] },
})
expect(withoutQuery.rerank.query).toBeUndefined()

const withObjectQuery = body({
...baseParams,
rerank: { model: 'bge-reranker-v2-m3', query: { text: 'Paris' } },
})
expect(withObjectQuery.rerank.query).toBe('Paris')

const withBlankQuery = body({
...baseParams,
rerank: { model: 'bge-reranker-v2-m3', query: ' ' },
})
expect(withBlankQuery.rerank.query).toBeUndefined()
})
})
20 changes: 14 additions & 6 deletions apps/sim/tools/pinecone/search_text.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import type {
PineconeSearchTextParams,
} from '@/tools/pinecone/types'
import type { ToolConfig } from '@/tools/types'
import { safeUrlPathSegment } from '@/tools/url-path'

function parseRerank(
value: PineconeSearchTextParams['rerank']
Expand All @@ -22,6 +23,7 @@ function selectRerankModelInput(
const rerank = parseRerank(value)
if (!rerank) return undefined
const query = rerank.query
if (typeof query === 'string') return { query }
if (
query === null ||
typeof query !== 'object' ||
Expand Down Expand Up @@ -132,7 +134,8 @@ export const searchTextTool: ToolConfig<PineconeSearchTextParams, PineconeRespon
},
},
method: 'POST',
url: (params) => `${params.indexHost}/records/namespaces/${params.namespace}/search`,
url: (params) =>
`${params.indexHost}/records/namespaces/${safeUrlPathSegment(params.namespace ?? '', 'namespace', { allowSeparators: true, preserveWhitespace: true })}/search`,
headers: (params) => ({
'Api-Key': params.apiKey,
'Content-Type': 'application/json',
Expand Down Expand Up @@ -162,11 +165,16 @@ export const searchTextTool: ToolConfig<PineconeSearchTextParams, PineconeRespon
}

if (params.rerank) {
body.rerank = typeof params.rerank === 'string' ? JSON.parse(params.rerank) : params.rerank
// If rerank query is not specified, use the search query
if (!body.rerank.query) {
body.rerank.query = { text: params.searchQuery }
}
const { query: rerankQuery, ...rerank } = parseRerank(params.rerank) ?? {}
const queryText =
rerankQuery !== null && typeof rerankQuery === 'object' && !Array.isArray(rerankQuery)
? (rerankQuery as Record<string, unknown>).text
: rerankQuery
// Pinecone takes `rerank.query` as a string and reranks against the search query when unset.
body.rerank =
typeof queryText === 'string' && queryText.trim()
? { ...rerank, query: queryText }
: rerank
Comment thread
waleedlatif1 marked this conversation as resolved.
Resolved
}

return body
Expand Down
7 changes: 4 additions & 3 deletions apps/sim/tools/pinecone/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -81,13 +81,14 @@ export interface PineconeUpsertTextParams extends PineconeBaseParams {

interface PineconeRerank {
model: string
rank_fields: string[]
rank_fields?: string[]
top_n?: number
parameters?: Record<string, any>
query?: { text: string }
query?: string | { text: string }
}

export interface PineconeSearchTextParams extends PineconeBaseParams {
export interface PineconeSearchTextParams extends Omit<PineconeBaseParams, 'namespace'> {
namespace?: string
searchQuery: string
topK?: string
fields?: string[] | string
Expand Down
4 changes: 3 additions & 1 deletion apps/sim/tools/pinecone/upsert_text.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import type {
PineconeUpsertTextRecord,
} from '@/tools/pinecone/types'
import type { ToolConfig } from '@/tools/types'
import { safeUrlPathSegment } from '@/tools/url-path'

function parseUpsertTextRecords(
input: PineconeUpsertTextParams['records']
Expand Down Expand Up @@ -90,7 +91,8 @@ export const upsertTextTool: ToolConfig<PineconeUpsertTextParams, PineconeRespon
},
},
method: 'POST',
url: (params) => `${params.indexHost}/records/namespaces/${params.namespace}/upsert`,
url: (params) =>
`${params.indexHost}/records/namespaces/${safeUrlPathSegment(params.namespace, 'namespace', { allowSeparators: true, preserveWhitespace: true })}/upsert`,
headers: (params) => ({
'Api-Key': params.apiKey,
'Content-Type': 'application/x-ndjson',
Expand Down
4 changes: 3 additions & 1 deletion apps/sim/tools/qdrant/fetch_points.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import type { QdrantFetchParams, QdrantResponse } from '@/tools/qdrant/types'
import { POINT_OUTPUT_PROPERTIES, QDRANT_RESPONSE_OUTPUT_PROPERTIES } from '@/tools/qdrant/types'
import { QDRANT_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials'
import type { ToolConfig } from '@/tools/types'
import { safeUrlPathSegment } from '@/tools/url-path'

export const fetchPointsTool: ToolConfig<QdrantFetchParams, QdrantResponse> = {
id: 'qdrant_fetch_points',
Expand Down Expand Up @@ -57,7 +58,8 @@ export const fetchPointsTool: ToolConfig<QdrantFetchParams, QdrantResponse> = {

request: {
method: 'POST',
url: (params) => `${params.url.replace(/\/$/, '')}/collections/${params.collection}/points`,
url: (params) =>
`${params.url.replace(/\/$/, '')}/collections/${safeUrlPathSegment(params.collection, 'collection', { preserveWhitespace: true })}/points`,
headers: (params) => ({
'Content-Type': 'application/json',
...(params.apiKey ? { 'api-key': params.apiKey } : {}),
Expand Down
Loading
Loading