Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
171 changes: 171 additions & 0 deletions apps/sim/connectors/confluence/app-macros.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,171 @@
import * as cheerio from 'cheerio'
import { htmlToPlainText } from '@/connectors/utils'

/**
* Caelor Optics stores every module under one Connect macro and names the module
* in its `macro` parameter. The HTML module keeps the page author's markup in
* `contentByMode` (escaped JSON), and Confluence renders it only as an iframe
* bootstrap script, so neither body format carries its text as page prose.
*/
const OPTICS_MACRO_NAME = 'caelor-optics-entry'

/** Every Connect app macro renders into this iframe container. */
const CONNECT_APP_CONTAINER = 'ap-container'

/** Markup whose text is never prose. Scripts are read for their data first. */
const NON_PROSE_SELECTOR = 'script, style, noscript, template'

/**
* A string or number property of an object literal, introduced by `{` or `,` so
* ternaries (`a ? 'x' : 'y'`) and `case 'x':` labels are not mistaken for data.
* JSON objects match too, so both `{name: 'Zoom'}` and `{"name": "Zoom"}` read.
*/
const OBJECT_PROPERTY_PATTERN =
/([{,])\s*(?:([A-Za-z_$][\w$]*)|"((?:[^"\\\r\n]|\\.)*)"|'((?:[^'\\\r\n]|\\.)*)')\s*:\s*(?:"((?:[^"\\\r\n]|\\.)*)"|'((?:[^'\\\r\n]|\\.)*)'|(-?\d+(?:\.\d+)?)(?![\w.]))/g

/** Values that are encoded payloads (inline images, fonts), not readable data. */
const MAX_UNBROKEN_VALUE_LENGTH = 200
const BASE64_ALPHABET_PATTERN = /^[A-Za-z0-9+/]{24,}={0,2}$/

/**
* A string literal (kept, so a `//` inside `'https://…'` survives), a regex
* literal (kept, so the `//` ending `/https?:\/\//` survives; recognized only
* where an expression can start, so division is not mistaken for one), or a
* comment (dropped, so commented-out records never read as page data).
*/
const STRING_OR_COMMENT_PATTERN =
/("(?:[^"\\\r\n]|\\.)*"|'(?:[^'\\\r\n]|\\.)*'|`(?:[^`\\]|\\.)*`|\/(?<=(?:^|[=(,:;!&|?{}[\]+\-*%~^<>]|\breturn)\s*\/)(?![*/])(?:[^/\\\r\n[]|\\.|\[(?:[^\]\\\r\n]|\\.)*\])+\/[a-z]*)|\/\/[^\r\n]*|\/\*[\s\S]*?\*\//g

/**
* Whether a rendered `view` body may hold an Optics HTML macro, whose text only
* the storage body carries. Rendered Connect macros name themselves in
* `data-macro-name`; a page left with no text but an app container is checked
* too, so a rendering without that attribute still recovers. Other app pages
* (draw.io, Lucidchart) that have prose skip the extra request.
*/
export function mayHoldOpticsHtmlMacro(viewHtml: string, viewText: string): boolean {
return (
viewHtml.includes(OPTICS_MACRO_NAME) ||
(!viewText.trim() && viewHtml.includes(CONNECT_APP_CONTAINER))
)
}

/** Parses a storage-format body, whose `ac:`/`ri:` elements and CDATA need lenient XML handling. */
export function loadStorageBody(storage: string): cheerio.CheerioAPI {
return cheerio.load(
storage,
{ xml: { xmlMode: false, recognizeCDATA: true, recognizeSelfClosing: true } },
false
)
}

/**
* Text of a storage-format Optics HTML macro, or null when the element is any
* other macro. The HTML is authored on the page itself, so it is the same for
* every reader who can open the page.
*/
export function readOpticsHtmlMacroText(
$: cheerio.CheerioAPI,
element: Parameters<cheerio.CheerioAPI>[0]
): string | null {
const macro = $(element)
if (macro.attr('ac:name') !== OPTICS_MACRO_NAME) return null
const parameter = (name: string) => macro.children(`ac\\:parameter[ac\\:name="${name}"]`)
if (parameter('macro').text().trim() !== 'html') return null
return readMarkupSources(parameter('contentByMode').text())
.map(markupToText)
.filter(Boolean)
.join(' ')
}

/** Text of every Optics HTML macro in a storage-format body. */
export function extractOpticsHtmlMacroText(storage: string): string {
if (!storage.includes(OPTICS_MACRO_NAME)) return ''
const $ = loadStorageBody(storage)
return $('ac\\:structured-macro')
.toArray()
.map((element) => readOpticsHtmlMacroText($, element))
.filter(Boolean)
.join(' ')
}

/**
* `contentByMode` is JSON keyed by editor mode; its markup is whichever string
* values contain tags. A value that is not JSON is read as markup directly.
*/
function readMarkupSources(content: string): string[] {
let parsed: unknown
try {
parsed = JSON.parse(content)
} catch {
return [content]
}
const sources: string[] = []
const collect = (value: unknown) => {
if (typeof value === 'string') {
if (/<[a-z!]/i.test(value)) sources.push(value)
} else if (Array.isArray(value)) {
value.forEach(collect)
} else if (value && typeof value === 'object') {
Object.values(value).forEach(collect)
}
}
collect(parsed)
return sources
}

/**
* Visible text of authored markup followed by the data its scripts declare. A
* page that renders a list from a JS object keeps nothing visible but a mount
* point, so the object's properties are what a reader actually sees.
*/
function markupToText(markup: string): string {
const $ = cheerio.load(markup)
const scriptData = $('script')
.toArray()
.map((script) => readScriptData($(script).text()))
.filter(Boolean)
$(NON_PROSE_SELECTOR).replaceWith(' ')
return [htmlToPlainText($.html()), ...scriptData].filter(Boolean).join(' ')
}

/** Object-literal properties as `key: value` lines, one line per object. */
function readScriptData(script: string): string {
const records: string[][] = [[]]
const code = script.replace(STRING_OR_COMMENT_PATTERN, (_, literal?: string) => literal ?? ' ')
Comment thread
waleedlatif1 marked this conversation as resolved.
Resolved
for (const match of code.matchAll(OBJECT_PROPERTY_PATTERN)) {
const [, opener, bareKey, doubleKey, singleKey, doubleValue, singleValue, numberValue] = match
const key = bareKey ?? unescapeJsString(doubleKey ?? singleKey ?? '')
const value = numberValue ?? unescapeJsString(doubleValue ?? singleValue ?? '').trim()
if (opener === '{') records.push([])
if (key && isReadableValue(value)) records[records.length - 1].push(`${key}: ${value}`)
}
return records
.filter((record) => record.length > 0)
.map((record) => `${record.join(', ')}.`)
.join(' ')
}

function isReadableValue(value: string): boolean {
if (!value || value.startsWith('data:') || isBase64Payload(value)) return false
return value.length <= MAX_UNBROKEN_VALUE_LENGTH || /\s/.test(value)
Comment thread
waleedlatif1 marked this conversation as resolved.
Resolved
}

/**
* Base64-alphabet runs that also carry the evidence of encoding: padding or
* `+`/`/`, or the upper, lower, and digit mix random bytes produce. A long name
* (`MicrosoftAuthenticationLibrary`) or numeric id has the alphabet but not this.
*/
function isBase64Payload(value: string): boolean {
if (!BASE64_ALPHABET_PATTERN.test(value)) return false
return /[+/=]/.test(value) || (/[a-z]/.test(value) && /[A-Z]/.test(value) && /\d/.test(value))
}

const JS_ESCAPES: Record<string, string> = { n: ' ', r: ' ', t: ' ', b: '', f: '', v: '' }

function unescapeJsString(value: string): string {
return value.replace(/\\(u[0-9a-fA-F]{4}|x[0-9a-fA-F]{2}|[\s\S])/g, (_, sequence: string) => {
if (sequence.length > 1) return String.fromCharCode(Number.parseInt(sequence.slice(1), 16))
return JS_ESCAPES[sequence] ?? sequence
})
}
200 changes: 200 additions & 0 deletions apps/sim/connectors/confluence/confluence.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -653,3 +653,203 @@ describe('confluence mirrored permissions', () => {
})
})
})

describe('Confluence Caelor Optics HTML macros', () => {
const config = { domain: 'example.atlassian.net', spaceKey: 'DOCS' }
const escapeXml = (value: string) =>
value.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;')
const opticsHtml =
'<style>.card{color:red}</style><h2>Find your app</h2>' +
'<img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUg"><div id="apps"></div>' +
"<script>const apps = [{name: 'Zoom', wave: 3, date: '2026-12-04'}, " +
'{"name": "Slack", "wave": 1}];' +
" const logo = {src: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUg'};" +
" function render() { document.getElementById('apps').innerHTML = apps.map((a) => a.name).join(', ') }" +
'</script>'
const opticsMacro = (contentByMode: string, macro = 'html') =>
'<ac:structured-macro ac:name="caelor-optics-entry" ac:schema-version="1">' +
`<ac:parameter ac:name="macro">${macro}</ac:parameter>` +
`<ac:parameter ac:name="contentByMode">${escapeXml(contentByMode)}</ac:parameter>` +
'</ac:structured-macro>'
const opticsView =
'<div class="conf-macro output-block" data-macro-name="caelor-optics-entry">' +
'<div class="ap-container" id="ap-optics"><div class="ap-content"></div>' +
'<script class="ap-iframe-body-script">(function(){ var data = {"addon_key":"optics"}; }());</script>' +
'</div></div>'

function stubPage(bodies: { view: string; storage: string }) {
const requestedFormats: string[] = []
stubFetchWithoutAttachments(
vi.fn(async (input: string | URL | Request) => {
const format = new URL(String(input)).searchParams.get('body-format') ?? 'view'
requestedFormats.push(format)
return Response.json({
id: 'optics-page',
title: 'Find your app',
status: 'current',
spaceId: 'space-1',
version: { number: 4 },
body: { [format]: { value: format === 'storage' ? bodies.storage : bodies.view } },
})
})
)
return requestedFormats
}

function expectOpticsText(content: string | undefined) {
expect(content).toContain('Find your app')
expect(content).toContain('name: Zoom, wave: 3, date: 2026-12-04')
expect(content).toContain('name: Slack, wave: 1')
expect(content).not.toContain('color:red')
expect(content).not.toContain('base64')
expect(content).not.toContain('getElementById')
}

it('indexes the authored HTML and script data of an Optics-only rendered page', async () => {
stubPage({ view: opticsView, storage: opticsMacro(JSON.stringify({ html: opticsHtml })) })

const document = await confluenceConnector.getDocument('token', config, 'optics-page', {
cloudId: 'cloud-1',
})

expect(document?.skippedReason).toBeUndefined()
expectOpticsText(document?.content)
})

it('keeps rendered prose alongside the Optics macro text', async () => {
stubPage({
view: `<p>Read this first</p>${opticsView}`,
storage: `<p>Read this first</p>${opticsMacro(JSON.stringify({ html: opticsHtml }))}`,
})

const document = await confluenceConnector.getDocument('token', config, 'optics-page', {
cloudId: 'cloud-1',
})

expect(document?.content).toContain('Read this first')
expectOpticsText(document?.content)
})

it('keeps script data around regex literals and long names, without comments or base64', async () => {
const markup =
'<h2>Find your app</h2><script>' +
"// const retired = {name: 'Retired app'};\n" +
"/* {name: 'Draft app'} */ const re = /https?:\\/\\//; const apps = [{name: 'Zoom', url: 'https://zoom.example/a'}," +
" {name: 'MicrosoftAuthenticationLibrary', id: '100200300400500600700'}];" +
" const icon = {png: 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB'};</script>"
stubPage({ view: opticsView, storage: opticsMacro(JSON.stringify({ html: markup })) })

const document = await confluenceConnector.getDocument('token', config, 'optics-page', {
cloudId: 'cloud-1',
})

expect(document?.content).toContain('name: Zoom, url: https://zoom.example/a')
expect(document?.content).toContain(
'name: MicrosoftAuthenticationLibrary, id: 100200300400500600700'
)
expect(document?.content).not.toContain('Retired app')
expect(document?.content).not.toContain('Draft app')
expect(document?.content).not.toContain('iVBORw0KGgo')
})

it('fails hydration when the storage response omits its body so the page retries', async () => {
stubFetchWithoutAttachments(
vi.fn(async (input: string | URL | Request) => {
const format = new URL(String(input)).searchParams.get('body-format')
return Response.json({
id: 'optics-page',
status: 'current',
version: { number: 4 },
body: format === 'storage' ? {} : { view: { value: `<p>Prose</p>${opticsView}` } },
})
})
)

await expect(
confluenceConnector.getDocument('token', config, 'optics-page', { cloudId: 'cloud-1' })
).rejects.toThrow('missing its storage body')
})

it('reads a contentByMode that holds raw HTML instead of JSON', async () => {
stubPage({ view: opticsView, storage: opticsMacro(opticsHtml) })

const document = await confluenceConnector.getDocument('token', config, 'optics-page', {
cloudId: 'cloud-1',
})

expectOpticsText(document?.content)
})

it('does not fetch the storage body for a rendered page without app macros', async () => {
const requestedFormats = stubPage({ view: '<p>Plain page</p>', storage: '<p>Plain page</p>' })

const document = await confluenceConnector.getDocument('token', config, 'optics-page', {
cloudId: 'cloud-1',
})

expect(document?.content).toBe('Plain page')
expect(requestedFormats).toEqual(['view'])
})

it('skips the storage request for another app macro on a page with prose', async () => {
const requestedFormats = stubPage({
view:
'<p>Diagram notes</p><div class="ap-container" id="ap-drawio">' +
'<script class="ap-iframe-body-script">(function(){ var data = {"addon_key":"drawio"}; }());</script></div>',
storage: '<p>Diagram notes</p>',
})

const document = await confluenceConnector.getDocument('token', config, 'optics-page', {
cloudId: 'cloud-1',
})

expect(document?.content).toBe('Diagram notes')
expect(requestedFormats).toEqual(['view'])
})

it('reads storage for an app-only page whose rendering omits the macro name', async () => {
stubPage({
view: opticsView.replace(' data-macro-name="caelor-optics-entry"', ''),
storage: opticsMacro(JSON.stringify({ html: opticsHtml })),
})

const document = await confluenceConnector.getDocument('token', config, 'optics-page', {
cloudId: 'cloud-1',
})

expectOpticsText(document?.content)
})

it('indexes an Optics HTML macro nested in a layout on the permission-scoped path', async () => {
stubPage({
view: opticsView,
storage:
'<ac:structured-macro ac:name="section"><ac:rich-text-body>' +
'<ac:structured-macro ac:name="column"><ac:rich-text-body>' +
opticsMacro(JSON.stringify({ html: opticsHtml })) +
'</ac:rich-text-body></ac:structured-macro></ac:rich-text-body></ac:structured-macro>',
})

const document = await confluenceConnector.getDocument('token', config, 'optics-page', {
cloudId: 'cloud-1',
mirrorsSourceAcls: true,
})

expect(document?.skippedReason).toBeUndefined()
expectOpticsText(document?.content)
})

it('still treats other Optics modules as dynamic content on the permission-scoped path', async () => {
stubPage({
view: opticsView,
storage: opticsMacro(JSON.stringify({ html: opticsHtml }), 'cards'),
})

const document = await confluenceConnector.getDocument('token', config, 'optics-page', {
cloudId: 'cloud-1',
mirrorsSourceAcls: true,
})

expect(document?.skippedReason).toBe(DYNAMIC_CONTENT_SKIP_REASON)
})
})
Loading
Loading