diff --git a/apps/sim/connectors/confluence/app-macros.ts b/apps/sim/connectors/confluence/app-macros.ts new file mode 100644 index 00000000000..2b30ead9d3a --- /dev/null +++ b/apps/sim/connectors/confluence/app-macros.ts @@ -0,0 +1,171 @@ +import * as cheerio from 'cheerio' +import { htmlToPlainText } from '@/connectors/utils' + +/** + * Caelor Optics stores every module under one Connect macro and names the module + * in its `macro` parameter. The HTML module keeps the page author's markup in + * `contentByMode` (escaped JSON), and Confluence renders it only as an iframe + * bootstrap script, so neither body format carries its text as page prose. + */ +const OPTICS_MACRO_NAME = 'caelor-optics-entry' + +/** Every Connect app macro renders into this iframe container. */ +const CONNECT_APP_CONTAINER = 'ap-container' + +/** Markup whose text is never prose. Scripts are read for their data first. */ +const NON_PROSE_SELECTOR = 'script, style, noscript, template' + +/** + * A string or number property of an object literal, introduced by `{` or `,` so + * ternaries (`a ? 'x' : 'y'`) and `case 'x':` labels are not mistaken for data. + * JSON objects match too, so both `{name: 'Zoom'}` and `{"name": "Zoom"}` read. + */ +const OBJECT_PROPERTY_PATTERN = + /([{,])\s*(?:([A-Za-z_$][\w$]*)|"((?:[^"\\\r\n]|\\.)*)"|'((?:[^'\\\r\n]|\\.)*)')\s*:\s*(?:"((?:[^"\\\r\n]|\\.)*)"|'((?:[^'\\\r\n]|\\.)*)'|(-?\d+(?:\.\d+)?)(?![\w.]))/g + +/** Values that are encoded payloads (inline images, fonts), not readable data. */ +const MAX_UNBROKEN_VALUE_LENGTH = 200 +const BASE64_ALPHABET_PATTERN = /^[A-Za-z0-9+/]{24,}={0,2}$/ + +/** + * A string literal (kept, so a `//` inside `'https://…'` survives), a regex + * literal (kept, so the `//` ending `/https?:\/\//` survives; recognized only + * where an expression can start, so division is not mistaken for one), or a + * comment (dropped, so commented-out records never read as page data). + */ +const STRING_OR_COMMENT_PATTERN = + /("(?:[^"\\\r\n]|\\.)*"|'(?:[^'\\\r\n]|\\.)*'|`(?:[^`\\]|\\.)*`|\/(?<=(?:^|[=(,:;!&|?{}[\]+\-*%~^<>]|\breturn)\s*\/)(?![*/])(?:[^/\\\r\n[]|\\.|\[(?:[^\]\\\r\n]|\\.)*\])+\/[a-z]*)|\/\/[^\r\n]*|\/\*[\s\S]*?\*\//g + +/** + * Whether a rendered `view` body may hold an Optics HTML macro, whose text only + * the storage body carries. Rendered Connect macros name themselves in + * `data-macro-name`; a page left with no text but an app container is checked + * too, so a rendering without that attribute still recovers. Other app pages + * (draw.io, Lucidchart) that have prose skip the extra request. + */ +export function mayHoldOpticsHtmlMacro(viewHtml: string, viewText: string): boolean { + return ( + viewHtml.includes(OPTICS_MACRO_NAME) || + (!viewText.trim() && viewHtml.includes(CONNECT_APP_CONTAINER)) + ) +} + +/** Parses a storage-format body, whose `ac:`/`ri:` elements and CDATA need lenient XML handling. */ +export function loadStorageBody(storage: string): cheerio.CheerioAPI { + return cheerio.load( + storage, + { xml: { xmlMode: false, recognizeCDATA: true, recognizeSelfClosing: true } }, + false + ) +} + +/** + * Text of a storage-format Optics HTML macro, or null when the element is any + * other macro. The HTML is authored on the page itself, so it is the same for + * every reader who can open the page. + */ +export function readOpticsHtmlMacroText( + $: cheerio.CheerioAPI, + element: Parameters[0] +): string | null { + const macro = $(element) + if (macro.attr('ac:name') !== OPTICS_MACRO_NAME) return null + const parameter = (name: string) => macro.children(`ac\\:parameter[ac\\:name="${name}"]`) + if (parameter('macro').text().trim() !== 'html') return null + return readMarkupSources(parameter('contentByMode').text()) + .map(markupToText) + .filter(Boolean) + .join(' ') +} + +/** Text of every Optics HTML macro in a storage-format body. */ +export function extractOpticsHtmlMacroText(storage: string): string { + if (!storage.includes(OPTICS_MACRO_NAME)) return '' + const $ = loadStorageBody(storage) + return $('ac\\:structured-macro') + .toArray() + .map((element) => readOpticsHtmlMacroText($, element)) + .filter(Boolean) + .join(' ') +} + +/** + * `contentByMode` is JSON keyed by editor mode; its markup is whichever string + * values contain tags. A value that is not JSON is read as markup directly. + */ +function readMarkupSources(content: string): string[] { + let parsed: unknown + try { + parsed = JSON.parse(content) + } catch { + return [content] + } + const sources: string[] = [] + const collect = (value: unknown) => { + if (typeof value === 'string') { + if (/<[a-z!]/i.test(value)) sources.push(value) + } else if (Array.isArray(value)) { + value.forEach(collect) + } else if (value && typeof value === 'object') { + Object.values(value).forEach(collect) + } + } + collect(parsed) + return sources +} + +/** + * Visible text of authored markup followed by the data its scripts declare. A + * page that renders a list from a JS object keeps nothing visible but a mount + * point, so the object's properties are what a reader actually sees. + */ +function markupToText(markup: string): string { + const $ = cheerio.load(markup) + const scriptData = $('script') + .toArray() + .map((script) => readScriptData($(script).text())) + .filter(Boolean) + $(NON_PROSE_SELECTOR).replaceWith(' ') + return [htmlToPlainText($.html()), ...scriptData].filter(Boolean).join(' ') +} + +/** Object-literal properties as `key: value` lines, one line per object. */ +function readScriptData(script: string): string { + const records: string[][] = [[]] + const code = script.replace(STRING_OR_COMMENT_PATTERN, (_, literal?: string) => literal ?? ' ') + for (const match of code.matchAll(OBJECT_PROPERTY_PATTERN)) { + const [, opener, bareKey, doubleKey, singleKey, doubleValue, singleValue, numberValue] = match + const key = bareKey ?? unescapeJsString(doubleKey ?? singleKey ?? '') + const value = numberValue ?? unescapeJsString(doubleValue ?? singleValue ?? '').trim() + if (opener === '{') records.push([]) + if (key && isReadableValue(value)) records[records.length - 1].push(`${key}: ${value}`) + } + return records + .filter((record) => record.length > 0) + .map((record) => `${record.join(', ')}.`) + .join(' ') +} + +function isReadableValue(value: string): boolean { + if (!value || value.startsWith('data:') || isBase64Payload(value)) return false + return value.length <= MAX_UNBROKEN_VALUE_LENGTH || /\s/.test(value) +} + +/** + * Base64-alphabet runs that also carry the evidence of encoding: padding or + * `+`/`/`, or the upper, lower, and digit mix random bytes produce. A long name + * (`MicrosoftAuthenticationLibrary`) or numeric id has the alphabet but not this. + */ +function isBase64Payload(value: string): boolean { + if (!BASE64_ALPHABET_PATTERN.test(value)) return false + return /[+/=]/.test(value) || (/[a-z]/.test(value) && /[A-Z]/.test(value) && /\d/.test(value)) +} + +const JS_ESCAPES: Record = { n: ' ', r: ' ', t: ' ', b: '', f: '', v: '' } + +function unescapeJsString(value: string): string { + return value.replace(/\\(u[0-9a-fA-F]{4}|x[0-9a-fA-F]{2}|[\s\S])/g, (_, sequence: string) => { + if (sequence.length > 1) return String.fromCharCode(Number.parseInt(sequence.slice(1), 16)) + return JS_ESCAPES[sequence] ?? sequence + }) +} diff --git a/apps/sim/connectors/confluence/confluence.test.ts b/apps/sim/connectors/confluence/confluence.test.ts index f8338f2b726..3b35181f0ab 100644 --- a/apps/sim/connectors/confluence/confluence.test.ts +++ b/apps/sim/connectors/confluence/confluence.test.ts @@ -653,3 +653,203 @@ describe('confluence mirrored permissions', () => { }) }) }) + +describe('Confluence Caelor Optics HTML macros', () => { + const config = { domain: 'example.atlassian.net', spaceKey: 'DOCS' } + const escapeXml = (value: string) => + value.replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"') + const opticsHtml = + '

Find your app

' + + '
' + + "' + const opticsMacro = (contentByMode: string, macro = 'html') => + '' + + `${macro}` + + `${escapeXml(contentByMode)}` + + '' + const opticsView = + '
' + + '
' + + '' + + '
' + + function stubPage(bodies: { view: string; storage: string }) { + const requestedFormats: string[] = [] + stubFetchWithoutAttachments( + vi.fn(async (input: string | URL | Request) => { + const format = new URL(String(input)).searchParams.get('body-format') ?? 'view' + requestedFormats.push(format) + return Response.json({ + id: 'optics-page', + title: 'Find your app', + status: 'current', + spaceId: 'space-1', + version: { number: 4 }, + body: { [format]: { value: format === 'storage' ? bodies.storage : bodies.view } }, + }) + }) + ) + return requestedFormats + } + + function expectOpticsText(content: string | undefined) { + expect(content).toContain('Find your app') + expect(content).toContain('name: Zoom, wave: 3, date: 2026-12-04') + expect(content).toContain('name: Slack, wave: 1') + expect(content).not.toContain('color:red') + expect(content).not.toContain('base64') + expect(content).not.toContain('getElementById') + } + + it('indexes the authored HTML and script data of an Optics-only rendered page', async () => { + stubPage({ view: opticsView, storage: opticsMacro(JSON.stringify({ html: opticsHtml })) }) + + const document = await confluenceConnector.getDocument('token', config, 'optics-page', { + cloudId: 'cloud-1', + }) + + expect(document?.skippedReason).toBeUndefined() + expectOpticsText(document?.content) + }) + + it('keeps rendered prose alongside the Optics macro text', async () => { + stubPage({ + view: `

Read this first

${opticsView}`, + storage: `

Read this first

${opticsMacro(JSON.stringify({ html: opticsHtml }))}`, + }) + + const document = await confluenceConnector.getDocument('token', config, 'optics-page', { + cloudId: 'cloud-1', + }) + + expect(document?.content).toContain('Read this first') + expectOpticsText(document?.content) + }) + + it('keeps script data around regex literals and long names, without comments or base64', async () => { + const markup = + '

Find your app

" + stubPage({ view: opticsView, storage: opticsMacro(JSON.stringify({ html: markup })) }) + + const document = await confluenceConnector.getDocument('token', config, 'optics-page', { + cloudId: 'cloud-1', + }) + + expect(document?.content).toContain('name: Zoom, url: https://zoom.example/a') + expect(document?.content).toContain( + 'name: MicrosoftAuthenticationLibrary, id: 100200300400500600700' + ) + expect(document?.content).not.toContain('Retired app') + expect(document?.content).not.toContain('Draft app') + expect(document?.content).not.toContain('iVBORw0KGgo') + }) + + it('fails hydration when the storage response omits its body so the page retries', async () => { + stubFetchWithoutAttachments( + vi.fn(async (input: string | URL | Request) => { + const format = new URL(String(input)).searchParams.get('body-format') + return Response.json({ + id: 'optics-page', + status: 'current', + version: { number: 4 }, + body: format === 'storage' ? {} : { view: { value: `

Prose

${opticsView}` } }, + }) + }) + ) + + await expect( + confluenceConnector.getDocument('token', config, 'optics-page', { cloudId: 'cloud-1' }) + ).rejects.toThrow('missing its storage body') + }) + + it('reads a contentByMode that holds raw HTML instead of JSON', async () => { + stubPage({ view: opticsView, storage: opticsMacro(opticsHtml) }) + + const document = await confluenceConnector.getDocument('token', config, 'optics-page', { + cloudId: 'cloud-1', + }) + + expectOpticsText(document?.content) + }) + + it('does not fetch the storage body for a rendered page without app macros', async () => { + const requestedFormats = stubPage({ view: '

Plain page

', storage: '

Plain page

' }) + + const document = await confluenceConnector.getDocument('token', config, 'optics-page', { + cloudId: 'cloud-1', + }) + + expect(document?.content).toBe('Plain page') + expect(requestedFormats).toEqual(['view']) + }) + + it('skips the storage request for another app macro on a page with prose', async () => { + const requestedFormats = stubPage({ + view: + '

Diagram notes

' + + '
', + storage: '

Diagram notes

', + }) + + const document = await confluenceConnector.getDocument('token', config, 'optics-page', { + cloudId: 'cloud-1', + }) + + expect(document?.content).toBe('Diagram notes') + expect(requestedFormats).toEqual(['view']) + }) + + it('reads storage for an app-only page whose rendering omits the macro name', async () => { + stubPage({ + view: opticsView.replace(' data-macro-name="caelor-optics-entry"', ''), + storage: opticsMacro(JSON.stringify({ html: opticsHtml })), + }) + + const document = await confluenceConnector.getDocument('token', config, 'optics-page', { + cloudId: 'cloud-1', + }) + + expectOpticsText(document?.content) + }) + + it('indexes an Optics HTML macro nested in a layout on the permission-scoped path', async () => { + stubPage({ + view: opticsView, + storage: + '' + + '' + + opticsMacro(JSON.stringify({ html: opticsHtml })) + + '', + }) + + const document = await confluenceConnector.getDocument('token', config, 'optics-page', { + cloudId: 'cloud-1', + mirrorsSourceAcls: true, + }) + + expect(document?.skippedReason).toBeUndefined() + expectOpticsText(document?.content) + }) + + it('still treats other Optics modules as dynamic content on the permission-scoped path', async () => { + stubPage({ + view: opticsView, + storage: opticsMacro(JSON.stringify({ html: opticsHtml }), 'cards'), + }) + + const document = await confluenceConnector.getDocument('token', config, 'optics-page', { + cloudId: 'cloud-1', + mirrorsSourceAcls: true, + }) + + expect(document?.skippedReason).toBe(DYNAMIC_CONTENT_SKIP_REASON) + }) +}) diff --git a/apps/sim/connectors/confluence/confluence.ts b/apps/sim/connectors/confluence/confluence.ts index e7943ea3da4..41fb406007b 100644 --- a/apps/sim/connectors/confluence/confluence.ts +++ b/apps/sim/connectors/confluence/confluence.ts @@ -19,6 +19,12 @@ import { type RetryOptions, VALIDATE_RETRY_OPTIONS, } from '@/lib/knowledge/documents/utils' +import { + extractOpticsHtmlMacroText, + loadStorageBody, + mayHoldOpticsHtmlMacro, + readOpticsHtmlMacroText, +} from '@/connectors/confluence/app-macros' import { getConfluenceAttachment, isConfluenceAttachment, @@ -348,11 +354,7 @@ export interface ConfluenceStorageText { * inclusion and third-party macros may render differently for each reader. */ export function extractConfluenceStorageText(storage: string): ConfluenceStorageText { - const $ = cheerio.load( - storage, - { xml: { xmlMode: false, recognizeCDATA: true, recognizeSelfClosing: true } }, - false - ) + const $ = loadStorageBody(storage) let droppedDynamicContent = false for (const element of $(ADF_NODE_SELECTOR).toArray().reverse()) { @@ -374,9 +376,13 @@ export function extractConfluenceStorageText(storage: string): ConfluenceStorage } $(STORAGE_MACRO_SELECTOR).each((_, element) => { - if (!LOCAL_STORAGE_MACROS.has($(element).attr('ac:name') ?? '')) { + if (LOCAL_STORAGE_MACROS.has($(element).attr('ac:name') ?? '')) return + const opticsText = readOpticsHtmlMacroText($, element) + if (opticsText === null) { droppedDynamicContent = true $(element).remove() + } else { + $(element).replaceWith($('

').text(opticsText)) } }) @@ -890,15 +896,20 @@ export const confluenceConnector: ConnectorConfig = { const bodyFormat = scopedContent ? 'storage' : 'view' let page: Record | null = null let contentType: 'page' | 'blogpost' = 'page' - for (const endpoint of ['pages', 'blogposts'] as const) { - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/${endpoint}/${encodeURIComponent(externalId)}?body-format=${bodyFormat}&include-labels=true` - const response = await fetchWithRetry(url, { + const contentUrl = (endpoint: 'pages' | 'blogposts') => + `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/${endpoint}/${encodeURIComponent(externalId)}` + const fetchContent = (url: string) => + fetchWithRetry(url, { method: 'GET', headers: { Accept: 'application/json', Authorization: `Bearer ${accessToken}`, }, }) + for (const endpoint of ['pages', 'blogposts'] as const) { + const response = await fetchContent( + `${contentUrl(endpoint)}?body-format=${bodyFormat}&include-labels=true` + ) if (response.ok) { page = await response.json() @@ -919,7 +930,21 @@ export const confluenceConnector: ConnectorConfig = { } const rawContent = representation.value const scoped = scopedContent ? extractConfluenceStorageText(rawContent) : null - const plainText = scoped ? scoped.text : confluenceViewToPlainText(rawContent) + let plainText = scoped ? scoped.text : confluenceViewToPlainText(rawContent) + if (!scoped && mayHoldOpticsHtmlMacro(rawContent, plainText)) { + const endpoint = contentType === 'page' ? 'pages' : 'blogposts' + const response = await fetchContent(`${contentUrl(endpoint)}?body-format=storage`) + if (response.status === 401) throw await createRetryableHttpError(response) + if (!response.ok) { + throw new Error(`Failed to get Confluence storage body: ${response.status}`) + } + const storagePage = (await response.json()) as { body?: { storage?: { value?: unknown } } } + const storage = storagePage.body?.storage?.value + if (typeof storage !== 'string') { + throw new Error('Confluence content is missing its storage body') + } + plainText = [plainText, extractOpticsHtmlMacroText(storage)].filter(Boolean).join(' ') + } const links = page._links as Record | undefined const stub = pageToStub(