diff --git a/apps/pii/requirements.txt b/apps/pii/requirements.txt index bd120fd57c7..d1425258bce 100644 --- a/apps/pii/requirements.txt +++ b/apps/pii/requirements.txt @@ -1,6 +1,6 @@ # Pinned for reproducible image builds. Bump deliberately. -presidio-analyzer==2.2.362 -presidio-anonymizer==2.2.362 +presidio-analyzer==2.2.366 +presidio-anonymizer==2.2.366 spacy==3.8.14 fastapi==0.138.0 uvicorn[standard]==0.49.0 diff --git a/apps/pii/server.py b/apps/pii/server.py index 972061807c2..09f5f4f0c26 100644 --- a/apps/pii/server.py +++ b/apps/pii/server.py @@ -893,9 +893,10 @@ def redact_batch(req: RedactBatchRequest) -> dict[str, list[str]]: ).text ) logger.info( - "redact_batch lang=%s texts=%d entities=%s nlp=%s spans=%d duration_ms=%.1f", + "redact_batch lang=%s texts=%d chars=%d entities=%s nlp=%s spans=%d duration_ms=%.1f", req.language, len(req.texts), + sum(map(len, req.texts)), len(entities) if entities else "all", "skip" if _regex_only(entities, req.score_threshold) else "full", total_spans, diff --git a/apps/sim/lib/guardrails/mask-client.test.ts b/apps/sim/lib/guardrails/mask-client.test.ts index 9da95d11d06..6c7b6ddad97 100644 --- a/apps/sim/lib/guardrails/mask-client.test.ts +++ b/apps/sim/lib/guardrails/mask-client.test.ts @@ -68,14 +68,6 @@ describe('maskPIIBatchViaHttp', () => { it('gives up after the retry budget is exhausted on a persistent 5xx', async () => { fetchMock.mockImplementation(async () => new Response('down', { status: 503 })) - await expect(maskPIIBatchViaHttp(['a'], [])).rejects.toThrow(/mask-batch request failed/) - expect(fetchMock).toHaveBeenCalledTimes(8) - expect(mockSleep).toHaveBeenCalledTimes(7) - }) - - it('stops after three attempts on a persistent 500, which can be a deterministic input failure', async () => { - fetchMock.mockImplementation(async () => new Response('failed', { status: 500 })) - await expect(maskPIIBatchViaHttp(['a'], [])).rejects.toThrow(/mask-batch request failed/) expect(fetchMock).toHaveBeenCalledTimes(3) expect(mockSleep).toHaveBeenCalledTimes(2) diff --git a/apps/sim/lib/guardrails/mask-client.ts b/apps/sim/lib/guardrails/mask-client.ts index b1de4f2093f..a2c2acb03a3 100644 --- a/apps/sim/lib/guardrails/mask-client.ts +++ b/apps/sim/lib/guardrails/mask-client.ts @@ -26,24 +26,17 @@ const logger = createLogger('PiiMaskClient') const CHUNK_CONCURRENCY = env.PII_MASK_CHUNK_CONCURRENCY ?? 64 /** - * Per-chunk retry budget for transient failures (network errors, 408/429/5xx). - * A large payload fans out into many chunk requests, so a single blip — an ALB - * 502 during a deploy, a Presidio pod restart — must not fail the whole - * redaction (and, on the execution-altering stages, abort the run). With the - * default 500ms→30s jittered backoff this rides out ~2 minutes of outage per - * chunk before giving up. Deterministic failures (4xx, shape mismatches) throw - * immediately. + * Attempts per chunk for transient failures (network errors, 408/429/5xx), the + * standard retry budget of the AWS and model-provider SDKs. A large payload fans + * out into many chunk requests, so one blip (a dropped connection, a Presidio + * worker restart) must not fail the whole redaction, but every retry re-runs the + * chunk's full NER pass: a chunk that timed out or failed on its own input fails + * the same way again, so more attempts only multiply the work and delay the + * fail-safe. With the default jittered backoff this covers about 1.5s of failures; + * a longer outage falls to the caller's fail-safe. Deterministic failures (4xx, + * shape mismatches) throw immediately. */ -const MAX_CHUNK_ATTEMPTS = 8 - -/** - * Attempts for a chunk the route answers with a 500. The route keeps 503 for a - * Presidio outage and 422 for input Presidio refused, so a 500 is an unclassified - * failure that may be deterministic for this input: two retries cover a one-off - * crash without spending the full outage budget resending input that fails the - * same way every time. - */ -const MAX_ATTEMPTS_AFTER_SERVER_ERROR = 3 +const MAX_CHUNK_ATTEMPTS = 3 const RETRYABLE_STATUSES = new Set([408, 429, 500, 502, 503, 504]) @@ -206,11 +199,7 @@ async function postChunk( try { return await postChunkOnce(url, texts, entityTypes, language, customPatterns) } catch (error) { - const maxAttempts = - error instanceof MaskChunkHttpError && error.status === 500 - ? MAX_ATTEMPTS_AFTER_SERVER_ERROR - : MAX_CHUNK_ATTEMPTS - if (attempt >= maxAttempts || !isRetryableChunkError(error)) { + if (attempt >= MAX_CHUNK_ATTEMPTS || !isRetryableChunkError(error)) { throw error } const retryAfterMs = error instanceof MaskChunkHttpError ? error.retryAfterMs : null @@ -227,8 +216,7 @@ async function postChunkOnce( customPatterns: CustomPiiPattern[] | undefined ): Promise { // Mint per attempt: a single token (5min TTL) can expire mid-batch when a - // large execution fans out into many sequential chunk requests or a chunk - // spends its retry budget waiting out an outage. + // large execution fans out into many sequential chunk requests. const token = await generateInternalToken() // boundary-raw-fetch: internal server-to-server call to the app container (internal JWT auth, configurable base URL) diff --git a/apps/sim/scripts/test-pii-redaction-e2e.ts b/apps/sim/scripts/test-pii-redaction-e2e.ts index 34066b273e9..4df3a842e91 100644 --- a/apps/sim/scripts/test-pii-redaction-e2e.ts +++ b/apps/sim/scripts/test-pii-redaction-e2e.ts @@ -18,6 +18,9 @@ import { getErrorMessage } from '@sim/utils/errors' * - Presidio masks names and an email on both sides of a long unbroken run of * brackets in place, within seconds; spaCy's own tokenizer needs time quadratic * in such a run's length. + * - Presidio masks every one of thousands of emails in one text within seconds; + * Presidio before 2.2.365 compared every match with every other one when removing + * duplicates and resolving overlaps. * - A persisted log holding a 5M-character base64 PDF and a 3M-character text that * names a person at the end: the PDF becomes the binary marker without reaching * Presidio, the name is masked by real NER, and no request fails. @@ -35,6 +38,8 @@ if (!reportPath) throw new Error('PII_E2E_REPORT_PATH must be explicitly provide const SPACY_MAX_LENGTH = 1_000_000 /** Where Presidio's first NER window ends (`NER_WINDOW_CHARS` in apps/pii/server.py). */ const FIRST_NER_WINDOW_END = 100_000 +/** Matches in one text: past the pairwise-comparison cost of Presidio before 2.2.365. */ +const MANY_MATCHES = 15_000 const FILLER = 'The quarterly report covers revenue, costs, and the hiring plan.\n' const NAME_SENTENCE = 'Please send questions to Ada Lovelace before Friday.' @@ -142,6 +147,31 @@ async function main(): Promise { const pdfBase64 = pdfBytes.toString('base64') const longText = `${FILLER.repeat(Math.ceil(3_000_000 / FILLER.length))}${NAME_SENTENCE}` + /** + * Masks one text with a `/redact_batch` request sent straight to Presidio. A + * `timeoutMs` makes a request that runs longer fail its check. + */ + const redactWithPresidio = async ( + request: { text: string } & Record, + timeoutMs?: number + ): Promise => { + const { text, ...options } = request + const response = await realFetch(new URL('/redact_batch', presidioUrl), { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ texts: [text], language: 'en', ...options }), + signal: timeoutMs === undefined ? undefined : AbortSignal.timeout(timeoutMs), + }) + const body = await response.text() + assert.equal( + response.status, + 200, + `Presidio answered ${response.status}: ${body.slice(0, 200)}` + ) + const [masked] = (JSON.parse(body) as { texts: string[] }).texts + return masked + } + await check('Presidio masks a text longer than one NER pass allows', async () => { const prose = (length: number) => FILLER.repeat(Math.ceil(length / FILLER.length)).slice(0, length) @@ -153,23 +183,11 @@ async function main(): Promise { 'badge EMP 123456 issued\n' + prose(SPACY_MAX_LENGTH) + NAME_SENTENCE - const response = await realFetch(new URL('/redact_batch', presidioUrl), { - method: 'POST', - headers: { 'content-type': 'application/json' }, - body: JSON.stringify({ - texts: [text], - language: 'en', - entities: ['PERSON'], - patterns: [{ regex: 'EMP\\s+\\d{6}', replacement: 'EMPLOYEE_ID', name: 'Employee ID' }], - }), + const masked = await redactWithPresidio({ + text, + entities: ['PERSON'], + patterns: [{ regex: 'EMP\\s+\\d{6}', replacement: 'EMPLOYEE_ID', name: 'Employee ID' }], }) - const body = await response.text() - assert.equal( - response.status, - 200, - `Presidio answered ${response.status}: ${body.slice(0, 200)}` - ) - const [masked] = (JSON.parse(body) as { texts: string[] }).texts assert( masked.includes(`${lead} before Friday.`), 'The name across the first NER window edge was not masked as one span' @@ -184,23 +202,7 @@ async function main(): Promise { const run = `${'['.repeat(20_000)}${']'.repeat(20_000)}` const after = '   Write to ada@example.com today.\n' const text = `${NAME_SENTENCE} ${run}${after}${NAME_SENTENCE}` - const response = await realFetch(new URL('/redact_batch', presidioUrl), { - method: 'POST', - headers: { 'content-type': 'application/json' }, - body: JSON.stringify({ - texts: [text], - language: 'en', - entities: ['PERSON', 'EMAIL_ADDRESS'], - }), - signal: AbortSignal.timeout(30_000), - }) - const body = await response.text() - assert.equal( - response.status, - 200, - `Presidio answered ${response.status}: ${body.slice(0, 200)}` - ) - const [masked] = (JSON.parse(body) as { texts: string[] }).texts + const masked = await redactWithPresidio({ text, entities: ['PERSON', 'EMAIL_ADDRESS'] }, 30_000) const maskedSentence = 'Please send questions to before Friday.' assert.equal( masked, @@ -210,6 +212,18 @@ async function main(): Promise { return { textChars: text.length } }) + await check('Presidio masks a text with thousands of matches within seconds', async () => { + const rows = Array.from({ length: MANY_MATCHES }, (_, i) => `row ${i}: ada${i}@example.com\n`) + const text = rows.join('') + const masked = await redactWithPresidio({ text, entities: ['EMAIL_ADDRESS'] }, 30_000) + assert.equal( + masked, + rows.map((_, i) => `row ${i}: \n`).join(''), + 'An email was not masked in place' + ) + return { textChars: text.length, matches: MANY_MATCHES } + }) + await check('A persisted log replaces the PDF and masks the long text', async () => { const since = maskRequests.length const result = await redactPIIFromExecution(