Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions apps/pii/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Pinned for reproducible image builds. Bump deliberately.
presidio-analyzer==2.2.362
presidio-anonymizer==2.2.362
presidio-analyzer==2.2.366
presidio-anonymizer==2.2.366
spacy==3.8.14
fastapi==0.138.0
uvicorn[standard]==0.49.0
Expand Down
3 changes: 2 additions & 1 deletion apps/pii/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -893,9 +893,10 @@ def redact_batch(req: RedactBatchRequest) -> dict[str, list[str]]:
).text
)
logger.info(
"redact_batch lang=%s texts=%d entities=%s nlp=%s spans=%d duration_ms=%.1f",
"redact_batch lang=%s texts=%d chars=%d entities=%s nlp=%s spans=%d duration_ms=%.1f",
req.language,
len(req.texts),
sum(map(len, req.texts)),
len(entities) if entities else "all",
"skip" if _regex_only(entities, req.score_threshold) else "full",
total_spans,
Expand Down
8 changes: 0 additions & 8 deletions apps/sim/lib/guardrails/mask-client.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -68,14 +68,6 @@ describe('maskPIIBatchViaHttp', () => {
it('gives up after the retry budget is exhausted on a persistent 5xx', async () => {
fetchMock.mockImplementation(async () => new Response('down', { status: 503 }))

await expect(maskPIIBatchViaHttp(['a'], [])).rejects.toThrow(/mask-batch request failed/)
expect(fetchMock).toHaveBeenCalledTimes(8)
expect(mockSleep).toHaveBeenCalledTimes(7)
})

it('stops after three attempts on a persistent 500, which can be a deterministic input failure', async () => {
fetchMock.mockImplementation(async () => new Response('failed', { status: 500 }))

await expect(maskPIIBatchViaHttp(['a'], [])).rejects.toThrow(/mask-batch request failed/)
expect(fetchMock).toHaveBeenCalledTimes(3)
expect(mockSleep).toHaveBeenCalledTimes(2)
Expand Down
36 changes: 12 additions & 24 deletions apps/sim/lib/guardrails/mask-client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,24 +26,17 @@ const logger = createLogger('PiiMaskClient')
const CHUNK_CONCURRENCY = env.PII_MASK_CHUNK_CONCURRENCY ?? 64

/**
* Per-chunk retry budget for transient failures (network errors, 408/429/5xx).
* A large payload fans out into many chunk requests, so a single blip — an ALB
* 502 during a deploy, a Presidio pod restart — must not fail the whole
* redaction (and, on the execution-altering stages, abort the run). With the
* default 500ms→30s jittered backoff this rides out ~2 minutes of outage per
* chunk before giving up. Deterministic failures (4xx, shape mismatches) throw
* immediately.
* Attempts per chunk for transient failures (network errors, 408/429/5xx), the
* standard retry budget of the AWS and model-provider SDKs. A large payload fans
* out into many chunk requests, so one blip (a dropped connection, a Presidio
* worker restart) must not fail the whole redaction, but every retry re-runs the
* chunk's full NER pass: a chunk that timed out or failed on its own input fails
* the same way again, so more attempts only multiply the work and delay the
* fail-safe. With the default jittered backoff this covers about 1.5s of failures;
* a longer outage falls to the caller's fail-safe. Deterministic failures (4xx,
* shape mismatches) throw immediately.
*/
const MAX_CHUNK_ATTEMPTS = 8

/**
* Attempts for a chunk the route answers with a 500. The route keeps 503 for a
* Presidio outage and 422 for input Presidio refused, so a 500 is an unclassified
* failure that may be deterministic for this input: two retries cover a one-off
* crash without spending the full outage budget resending input that fails the
* same way every time.
*/
const MAX_ATTEMPTS_AFTER_SERVER_ERROR = 3
const MAX_CHUNK_ATTEMPTS = 3
Comment thread
waleedlatif1 marked this conversation as resolved.
Resolved

const RETRYABLE_STATUSES = new Set([408, 429, 500, 502, 503, 504])

Expand Down Expand Up @@ -206,11 +199,7 @@ async function postChunk(
try {
return await postChunkOnce(url, texts, entityTypes, language, customPatterns)
} catch (error) {
const maxAttempts =
error instanceof MaskChunkHttpError && error.status === 500
? MAX_ATTEMPTS_AFTER_SERVER_ERROR
: MAX_CHUNK_ATTEMPTS
if (attempt >= maxAttempts || !isRetryableChunkError(error)) {
if (attempt >= MAX_CHUNK_ATTEMPTS || !isRetryableChunkError(error)) {
throw error
}
const retryAfterMs = error instanceof MaskChunkHttpError ? error.retryAfterMs : null
Expand All @@ -227,8 +216,7 @@ async function postChunkOnce(
customPatterns: CustomPiiPattern[] | undefined
): Promise<string[]> {
// Mint per attempt: a single token (5min TTL) can expire mid-batch when a
// large execution fans out into many sequential chunk requests or a chunk
// spends its retry budget waiting out an outage.
// large execution fans out into many sequential chunk requests.
const token = await generateInternalToken()

// boundary-raw-fetch: internal server-to-server call to the app container (internal JWT auth, configurable base URL)
Expand Down
80 changes: 47 additions & 33 deletions apps/sim/scripts/test-pii-redaction-e2e.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@ import { getErrorMessage } from '@sim/utils/errors'
* - Presidio masks names and an email on both sides of a long unbroken run of
* brackets in place, within seconds; spaCy's own tokenizer needs time quadratic
* in such a run's length.
* - Presidio masks every one of thousands of emails in one text within seconds;
* Presidio before 2.2.365 compared every match with every other one when removing
* duplicates and resolving overlaps.
* - A persisted log holding a 5M-character base64 PDF and a 3M-character text that
* names a person at the end: the PDF becomes the binary marker without reaching
* Presidio, the name is masked by real NER, and no request fails.
Expand All @@ -35,6 +38,8 @@ if (!reportPath) throw new Error('PII_E2E_REPORT_PATH must be explicitly provide
const SPACY_MAX_LENGTH = 1_000_000
/** Where Presidio's first NER window ends (`NER_WINDOW_CHARS` in apps/pii/server.py). */
const FIRST_NER_WINDOW_END = 100_000
/** Matches in one text: past the pairwise-comparison cost of Presidio before 2.2.365. */
const MANY_MATCHES = 15_000

const FILLER = 'The quarterly report covers revenue, costs, and the hiring plan.\n'
const NAME_SENTENCE = 'Please send questions to Ada Lovelace before Friday.'
Expand Down Expand Up @@ -142,6 +147,31 @@ async function main(): Promise<void> {
const pdfBase64 = pdfBytes.toString('base64')
const longText = `${FILLER.repeat(Math.ceil(3_000_000 / FILLER.length))}${NAME_SENTENCE}`

/**
* Masks one text with a `/redact_batch` request sent straight to Presidio. A
* `timeoutMs` makes a request that runs longer fail its check.
*/
const redactWithPresidio = async (
request: { text: string } & Record<string, unknown>,
timeoutMs?: number
): Promise<string> => {
const { text, ...options } = request
const response = await realFetch(new URL('/redact_batch', presidioUrl), {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({ texts: [text], language: 'en', ...options }),
signal: timeoutMs === undefined ? undefined : AbortSignal.timeout(timeoutMs),
})
const body = await response.text()
assert.equal(
response.status,
200,
`Presidio answered ${response.status}: ${body.slice(0, 200)}`
)
const [masked] = (JSON.parse(body) as { texts: string[] }).texts
return masked
}

await check('Presidio masks a text longer than one NER pass allows', async () => {
const prose = (length: number) =>
FILLER.repeat(Math.ceil(length / FILLER.length)).slice(0, length)
Expand All @@ -153,23 +183,11 @@ async function main(): Promise<void> {
'badge EMP 123456 issued\n' +
prose(SPACY_MAX_LENGTH) +
NAME_SENTENCE
const response = await realFetch(new URL('/redact_batch', presidioUrl), {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({
texts: [text],
language: 'en',
entities: ['PERSON'],
patterns: [{ regex: 'EMP\\s+\\d{6}', replacement: 'EMPLOYEE_ID', name: 'Employee ID' }],
}),
const masked = await redactWithPresidio({
text,
entities: ['PERSON'],
patterns: [{ regex: 'EMP\\s+\\d{6}', replacement: 'EMPLOYEE_ID', name: 'Employee ID' }],
})
const body = await response.text()
assert.equal(
response.status,
200,
`Presidio answered ${response.status}: ${body.slice(0, 200)}`
)
const [masked] = (JSON.parse(body) as { texts: string[] }).texts
assert(
masked.includes(`${lead}<PERSON> before Friday.`),
'The name across the first NER window edge was not masked as one <PERSON> span'
Expand All @@ -184,23 +202,7 @@ async function main(): Promise<void> {
const run = `${'['.repeat(20_000)}${']'.repeat(20_000)}`
const after = '   Write to ada@example.com today.\n'
const text = `${NAME_SENTENCE} ${run}${after}${NAME_SENTENCE}`
const response = await realFetch(new URL('/redact_batch', presidioUrl), {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({
texts: [text],
language: 'en',
entities: ['PERSON', 'EMAIL_ADDRESS'],
}),
signal: AbortSignal.timeout(30_000),
})
const body = await response.text()
assert.equal(
response.status,
200,
`Presidio answered ${response.status}: ${body.slice(0, 200)}`
)
const [masked] = (JSON.parse(body) as { texts: string[] }).texts
const masked = await redactWithPresidio({ text, entities: ['PERSON', 'EMAIL_ADDRESS'] }, 30_000)
const maskedSentence = 'Please send questions to <PERSON> before Friday.'
assert.equal(
masked,
Expand All @@ -210,6 +212,18 @@ async function main(): Promise<void> {
return { textChars: text.length }
})

await check('Presidio masks a text with thousands of matches within seconds', async () => {
const rows = Array.from({ length: MANY_MATCHES }, (_, i) => `row ${i}: ada${i}@example.com\n`)
const text = rows.join('')
const masked = await redactWithPresidio({ text, entities: ['EMAIL_ADDRESS'] }, 30_000)
assert.equal(
masked,
rows.map((_, i) => `row ${i}: <EMAIL_ADDRESS>\n`).join(''),
'An email was not masked in place'
)
return { textChars: text.length, matches: MANY_MATCHES }
})

await check('A persisted log replaces the PDF and masks the long text', async () => {
const since = maskRequests.length
const result = await redactPIIFromExecution(
Expand Down
Loading