From 5bc7312d6e035c5ae98e4344e6812cc7a375fc28 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 17 Nov 2026 21:16:40 -0800 Subject: [PATCH] improvement(pii): mask texts with many matches in linear time and retry failed chunks three times Presidio before 2.2.365 compared every match with every other one when removing duplicates and resolving overlaps, so a large text with thousands of matches took minutes. 2.2.366 replaces both with sorted sweeps. The mask client now makes three attempts per chunk instead of eight, since a retried chunk re-runs its whole NER pass. --- apps/pii/requirements.txt | 4 +- apps/pii/server.py | 3 +- apps/sim/lib/guardrails/mask-client.test.ts | 8 --- apps/sim/lib/guardrails/mask-client.ts | 36 ++++------ apps/sim/scripts/test-pii-redaction-e2e.ts | 80 ++++++++++++--------- 5 files changed, 63 insertions(+), 68 deletions(-) diff --git a/apps/pii/requirements.txt b/apps/pii/requirements.txt index bd120fd57c7..d1425258bce 100644 --- a/apps/pii/requirements.txt +++ b/apps/pii/requirements.txt @@ -1,6 +1,6 @@ # Pinned for reproducible image builds. Bump deliberately. -presidio-analyzer==2.2.362 -presidio-anonymizer==2.2.362 +presidio-analyzer==2.2.366 +presidio-anonymizer==2.2.366 spacy==3.8.14 fastapi==0.138.0 uvicorn[standard]==0.49.0 diff --git a/apps/pii/server.py b/apps/pii/server.py index 972061807c2..09f5f4f0c26 100644 --- a/apps/pii/server.py +++ b/apps/pii/server.py @@ -893,9 +893,10 @@ def redact_batch(req: RedactBatchRequest) -> dict[str, list[str]]: ).text ) logger.info( - "redact_batch lang=%s texts=%d entities=%s nlp=%s spans=%d duration_ms=%.1f", + "redact_batch lang=%s texts=%d chars=%d entities=%s nlp=%s spans=%d duration_ms=%.1f", req.language, len(req.texts), + sum(map(len, req.texts)), len(entities) if entities else "all", "skip" if _regex_only(entities, req.score_threshold) else "full", total_spans, diff --git a/apps/sim/lib/guardrails/mask-client.test.ts b/apps/sim/lib/guardrails/mask-client.test.ts index 9da95d11d06..6c7b6ddad97 100644 --- a/apps/sim/lib/guardrails/mask-client.test.ts +++ b/apps/sim/lib/guardrails/mask-client.test.ts @@ -68,14 +68,6 @@ describe('maskPIIBatchViaHttp', () => { it('gives up after the retry budget is exhausted on a persistent 5xx', async () => { fetchMock.mockImplementation(async () => new Response('down', { status: 503 })) - await expect(maskPIIBatchViaHttp(['a'], [])).rejects.toThrow(/mask-batch request failed/) - expect(fetchMock).toHaveBeenCalledTimes(8) - expect(mockSleep).toHaveBeenCalledTimes(7) - }) - - it('stops after three attempts on a persistent 500, which can be a deterministic input failure', async () => { - fetchMock.mockImplementation(async () => new Response('failed', { status: 500 })) - await expect(maskPIIBatchViaHttp(['a'], [])).rejects.toThrow(/mask-batch request failed/) expect(fetchMock).toHaveBeenCalledTimes(3) expect(mockSleep).toHaveBeenCalledTimes(2) diff --git a/apps/sim/lib/guardrails/mask-client.ts b/apps/sim/lib/guardrails/mask-client.ts index b1de4f2093f..a2c2acb03a3 100644 --- a/apps/sim/lib/guardrails/mask-client.ts +++ b/apps/sim/lib/guardrails/mask-client.ts @@ -26,24 +26,17 @@ const logger = createLogger('PiiMaskClient') const CHUNK_CONCURRENCY = env.PII_MASK_CHUNK_CONCURRENCY ?? 64 /** - * Per-chunk retry budget for transient failures (network errors, 408/429/5xx). - * A large payload fans out into many chunk requests, so a single blip — an ALB - * 502 during a deploy, a Presidio pod restart — must not fail the whole - * redaction (and, on the execution-altering stages, abort the run). With the - * default 500ms→30s jittered backoff this rides out ~2 minutes of outage per - * chunk before giving up. Deterministic failures (4xx, shape mismatches) throw - * immediately. + * Attempts per chunk for transient failures (network errors, 408/429/5xx), the + * standard retry budget of the AWS and model-provider SDKs. A large payload fans + * out into many chunk requests, so one blip (a dropped connection, a Presidio + * worker restart) must not fail the whole redaction, but every retry re-runs the + * chunk's full NER pass: a chunk that timed out or failed on its own input fails + * the same way again, so more attempts only multiply the work and delay the + * fail-safe. With the default jittered backoff this covers about 1.5s of failures; + * a longer outage falls to the caller's fail-safe. Deterministic failures (4xx, + * shape mismatches) throw immediately. */ -const MAX_CHUNK_ATTEMPTS = 8 - -/** - * Attempts for a chunk the route answers with a 500. The route keeps 503 for a - * Presidio outage and 422 for input Presidio refused, so a 500 is an unclassified - * failure that may be deterministic for this input: two retries cover a one-off - * crash without spending the full outage budget resending input that fails the - * same way every time. - */ -const MAX_ATTEMPTS_AFTER_SERVER_ERROR = 3 +const MAX_CHUNK_ATTEMPTS = 3 const RETRYABLE_STATUSES = new Set([408, 429, 500, 502, 503, 504]) @@ -206,11 +199,7 @@ async function postChunk( try { return await postChunkOnce(url, texts, entityTypes, language, customPatterns) } catch (error) { - const maxAttempts = - error instanceof MaskChunkHttpError && error.status === 500 - ? MAX_ATTEMPTS_AFTER_SERVER_ERROR - : MAX_CHUNK_ATTEMPTS - if (attempt >= maxAttempts || !isRetryableChunkError(error)) { + if (attempt >= MAX_CHUNK_ATTEMPTS || !isRetryableChunkError(error)) { throw error } const retryAfterMs = error instanceof MaskChunkHttpError ? error.retryAfterMs : null @@ -227,8 +216,7 @@ async function postChunkOnce( customPatterns: CustomPiiPattern[] | undefined ): Promise { // Mint per attempt: a single token (5min TTL) can expire mid-batch when a - // large execution fans out into many sequential chunk requests or a chunk - // spends its retry budget waiting out an outage. + // large execution fans out into many sequential chunk requests. const token = await generateInternalToken() // boundary-raw-fetch: internal server-to-server call to the app container (internal JWT auth, configurable base URL) diff --git a/apps/sim/scripts/test-pii-redaction-e2e.ts b/apps/sim/scripts/test-pii-redaction-e2e.ts index 34066b273e9..4df3a842e91 100644 --- a/apps/sim/scripts/test-pii-redaction-e2e.ts +++ b/apps/sim/scripts/test-pii-redaction-e2e.ts @@ -18,6 +18,9 @@ import { getErrorMessage } from '@sim/utils/errors' * - Presidio masks names and an email on both sides of a long unbroken run of * brackets in place, within seconds; spaCy's own tokenizer needs time quadratic * in such a run's length. + * - Presidio masks every one of thousands of emails in one text within seconds; + * Presidio before 2.2.365 compared every match with every other one when removing + * duplicates and resolving overlaps. * - A persisted log holding a 5M-character base64 PDF and a 3M-character text that * names a person at the end: the PDF becomes the binary marker without reaching * Presidio, the name is masked by real NER, and no request fails. @@ -35,6 +38,8 @@ if (!reportPath) throw new Error('PII_E2E_REPORT_PATH must be explicitly provide const SPACY_MAX_LENGTH = 1_000_000 /** Where Presidio's first NER window ends (`NER_WINDOW_CHARS` in apps/pii/server.py). */ const FIRST_NER_WINDOW_END = 100_000 +/** Matches in one text: past the pairwise-comparison cost of Presidio before 2.2.365. */ +const MANY_MATCHES = 15_000 const FILLER = 'The quarterly report covers revenue, costs, and the hiring plan.\n' const NAME_SENTENCE = 'Please send questions to Ada Lovelace before Friday.' @@ -142,6 +147,31 @@ async function main(): Promise { const pdfBase64 = pdfBytes.toString('base64') const longText = `${FILLER.repeat(Math.ceil(3_000_000 / FILLER.length))}${NAME_SENTENCE}` + /** + * Masks one text with a `/redact_batch` request sent straight to Presidio. A + * `timeoutMs` makes a request that runs longer fail its check. + */ + const redactWithPresidio = async ( + request: { text: string } & Record, + timeoutMs?: number + ): Promise => { + const { text, ...options } = request + const response = await realFetch(new URL('/redact_batch', presidioUrl), { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ texts: [text], language: 'en', ...options }), + signal: timeoutMs === undefined ? undefined : AbortSignal.timeout(timeoutMs), + }) + const body = await response.text() + assert.equal( + response.status, + 200, + `Presidio answered ${response.status}: ${body.slice(0, 200)}` + ) + const [masked] = (JSON.parse(body) as { texts: string[] }).texts + return masked + } + await check('Presidio masks a text longer than one NER pass allows', async () => { const prose = (length: number) => FILLER.repeat(Math.ceil(length / FILLER.length)).slice(0, length) @@ -153,23 +183,11 @@ async function main(): Promise { 'badge EMP 123456 issued\n' + prose(SPACY_MAX_LENGTH) + NAME_SENTENCE - const response = await realFetch(new URL('/redact_batch', presidioUrl), { - method: 'POST', - headers: { 'content-type': 'application/json' }, - body: JSON.stringify({ - texts: [text], - language: 'en', - entities: ['PERSON'], - patterns: [{ regex: 'EMP\\s+\\d{6}', replacement: 'EMPLOYEE_ID', name: 'Employee ID' }], - }), + const masked = await redactWithPresidio({ + text, + entities: ['PERSON'], + patterns: [{ regex: 'EMP\\s+\\d{6}', replacement: 'EMPLOYEE_ID', name: 'Employee ID' }], }) - const body = await response.text() - assert.equal( - response.status, - 200, - `Presidio answered ${response.status}: ${body.slice(0, 200)}` - ) - const [masked] = (JSON.parse(body) as { texts: string[] }).texts assert( masked.includes(`${lead} before Friday.`), 'The name across the first NER window edge was not masked as one span' @@ -184,23 +202,7 @@ async function main(): Promise { const run = `${'['.repeat(20_000)}${']'.repeat(20_000)}` const after = '   Write to ada@example.com today.\n' const text = `${NAME_SENTENCE} ${run}${after}${NAME_SENTENCE}` - const response = await realFetch(new URL('/redact_batch', presidioUrl), { - method: 'POST', - headers: { 'content-type': 'application/json' }, - body: JSON.stringify({ - texts: [text], - language: 'en', - entities: ['PERSON', 'EMAIL_ADDRESS'], - }), - signal: AbortSignal.timeout(30_000), - }) - const body = await response.text() - assert.equal( - response.status, - 200, - `Presidio answered ${response.status}: ${body.slice(0, 200)}` - ) - const [masked] = (JSON.parse(body) as { texts: string[] }).texts + const masked = await redactWithPresidio({ text, entities: ['PERSON', 'EMAIL_ADDRESS'] }, 30_000) const maskedSentence = 'Please send questions to before Friday.' assert.equal( masked, @@ -210,6 +212,18 @@ async function main(): Promise { return { textChars: text.length } }) + await check('Presidio masks a text with thousands of matches within seconds', async () => { + const rows = Array.from({ length: MANY_MATCHES }, (_, i) => `row ${i}: ada${i}@example.com\n`) + const text = rows.join('') + const masked = await redactWithPresidio({ text, entities: ['EMAIL_ADDRESS'] }, 30_000) + assert.equal( + masked, + rows.map((_, i) => `row ${i}: \n`).join(''), + 'An email was not masked in place' + ) + return { textChars: text.length, matches: MANY_MATCHES } + }) + await check('A persisted log replaces the PDF and masks the long text', async () => { const since = maskRequests.length const result = await redactPIIFromExecution(