diff --git a/README.md b/README.md index aa73f51..6c271cc 100644 --- a/README.md +++ b/README.md @@ -82,9 +82,11 @@ const anonymizer = createAnonymizer({ ### Schema-aware entity mode For epilot entities, when the entity schema is available (or resolvable). Classification uses, -in order: the attribute's `data_classification` (`'public'` opt-out / `'pii'` opt-in), your -`overrides`, the curated defaults, then attribute-type defaults (`email`, `phone`, `address`, -`payment`). +in order: `data_classification: 'public'` (opt-out), your `overrides`, the curated defaults, +attribute-type defaults (`email`, `phone`, `address`, `payment`, user relations, …), then +`data_classification: 'pii'` (opt-in). A `pii` attribute is never kept, and it still gets the +specific strategy a curated default or its type gives it (a `pii` `first_name` stays a `person` +pseudonym). ```ts // single entity (or a partial entity, e.g. an activity diff) @@ -153,12 +155,18 @@ Being upfront about what this package does **not** do is part of its job: destroys utility.) 2. **Free text is kept by default, but identifiers in it are scrubbed.** Unclassified free-text fields are not fully masked in schema-aware mode (masking them all would destroy utility), - but a catch-all pass scrubs any email / phone / IBAN embedded in kept string values (custom - fields, computed titles, attribute types not special-cased). Well-known risky fields + but a catch-all pass scrubs identifiers embedded in kept string values (custom fields, + computed titles, attribute types not special-cased): emails, phones, IBANs (mod-97 checked), + German street addresses ("Hauptstraße 5", "Schiltacher Str. 33" — postal code and city stay), + and PII inside URLs (components are decoded first; map links lose the place name, place ids + and the precise position, coordinates are rounded to 2 decimals ≈ 1 km). Only the matched + span is replaced. Well-known risky fields (note/message content, opportunity descriptions, …) are fully redacted via the curated defaults; organizations opt further fields in with `data_classification: 'pii'` or out with `'public'` (which also exempts them from the catch-all). Names or other non-pattern PII typed - into unclassified fields can still slip through. + into unclassified fields can still slip through. The street detector is pattern based: a + capitalized word ending in a street suffix followed by a 1-3 digit number also matches + non-addresses ("Arbeitsplatz 2"), and streets without a suffix ("An der Kirche 3") are missed. 3. **Heuristics are best-effort.** The non-schema mode catches common field names and email/phone/IBAN patterns. Phones match in international (`+49 …`, `0049 …`) and German/Austrian national format with a separator after the area code (`030 1234567`, `0221/123456`, @@ -187,7 +195,7 @@ counts as PII". | Addresses (`address`, `relation_address`) | keep postal code / city / country; pseudonymize street; drop the rest | | Birthdates, PII-flagged dates | truncate to year | | Payment items | pseudonymize IBAN/holder; keep bank name / BIC | -| User-relation attributes (`relation_user`, e.g. `contact_owner`) | pseudonymize name/email, **redact credentials** (`token`, `password`, …), keep ids/status/settings | +| User-relation attributes (`relation_user`, `internal_user`, e.g. `contact_owner`) | pseudonymize name/email, **redact credentials** (`token`, `password`, …), keep ids/status/settings | | Consent attributes (`consent`, e.g. `consent_email_marketing`) | pseudonymize the contact identifier (email/phone), keep consent status and events | | Message parties (`from`, `to`, `cc`, `bcc`, `reply_to`) | pseudonymize names, scrub addresses | | Activity `caller` (incl. nested `EpilotAuth` claims) | same rules as user-relation attributes | @@ -195,7 +203,9 @@ counts as PII". | File names (`file` `filename` / `_title`) | `value` pseudonym | | Entity fields not in the schema (e.g. `journey_data`) | field-name heuristics of `anonymizeUnknown` | | `data_classification: 'pii'` fields | pseudonym (single-line) / `[REDACTED]` (multiline) | -| Everything else | emails / phones / IBANs embedded in strings are scrubbed, the rest is unchanged | +| Computed titles built from personal data (`billing_account`, `calendar_event`, `contract`, `lead`, `request`, `ticket` `_title`) | `text` pseudonym | +| Map links (Google, Apple, Bing, OpenStreetMap, short links) in unclassified fields | place → `street` pseudonym + postal code / city, place ids pseudonymized, coordinates rounded to ~1 km | +| Everything else | emails / phones / IBANs / street addresses embedded in strings (and URL components) are scrubbed, the rest is unchanged | ## Development diff --git a/src/defaults.ts b/src/defaults.ts index ec8d7f4..995f815 100644 --- a/src/defaults.ts +++ b/src/defaults.ts @@ -26,6 +26,8 @@ export const STRATEGY_BY_ATTRIBUTE_TYPE: Record = // user-relation attributes embed full epilot user objects (name, email, and often an auth // `token`) — never pass them through raw relation_user: 'user', + // `internal_user` attributes (e.g. template `created_by`) embed the same user objects + internal_user: 'user', // address relations (e.g. opportunity `delivery_address`) carry the referenced address // values alongside the `$relation_ref` relation_address: 'address', @@ -79,6 +81,13 @@ export const DEFAULT_PII_FIELDS: Record = { // computed titles of messages/threads are the subject/topic 'message:_title': 'redact', 'thread:_title': 'redact', + // computed titles built from personal data (title templates in the entity blueprints) + 'billing_account:_title': 'text', // "{{billing_contact.0._title}} - …" = the contact's name + 'calendar_event:_title': 'text', // the event subject + 'contract:_title': 'text', // contract_name + 'lead:_title': 'text', // lead_title + 'request:_title': 'text', // request_title + 'ticket:_title': 'text', // ticket_title // uploaded file names regularly contain customer names (e.g. "Rechnung_Mustermann.pdf"); // `_title` is the file name too, so both map to the same pseudonym 'file:filename': 'value', diff --git a/src/entity.ts b/src/entity.ts index ae02114..b5d95bb 100644 --- a/src/entity.ts +++ b/src/entity.ts @@ -17,11 +17,14 @@ import { isPlainObject, mapArrayShared, mapEntries, mapEntriesShared } from './u * * Precedence (first match wins): * 1. `data_classification: 'public'` on the schema attribute -> keep (org opt-out) - * 2. `data_classification: 'pii'` on the schema attribute -> mask (org opt-in, e.g. free text) - * 3. consumer `overrides` (same key format as the curated map) - * 4. curated built-in PII fields (`DEFAULT_PII_FIELDS`) - * 5. defaults by attribute type (email, phone, address, payment) + * 2. consumer `overrides` (same key format as the curated map) + * 3. curated built-in PII fields (`DEFAULT_PII_FIELDS`) + * 4. defaults by attribute type (email, phone, address, payment, user relations, …) + * 5. `data_classification: 'pii'` on the schema attribute -> mask (org opt-in, e.g. free text): + * dates keep the year, multiline text is redacted, single-line values are pseudonymized * 6. keep + * + * A `pii` attribute is never kept: an override or curated default to `keep` does not apply to it. */ export const getAttributeStrategy = (params: { slug: string; @@ -35,31 +38,31 @@ export const getAttributeStrategy = (params: { return 'keep'; } - if (attribute?.data_classification === 'pii') { - const byType = attribute.type ? STRATEGY_BY_ATTRIBUTE_TYPE[attribute.type] : undefined; - if (byType) { - return byType; - } - if (attribute.type === 'date' || attribute.type === 'datetime') { - return 'date_year'; - } - - // free text: multiline content is redacted, single-line values stay joinable - return attribute.multiline ? 'redact' : 'value'; - } - + const isPii = attribute?.data_classification === 'pii'; const configured = overrides?.[`${slug}:${attributeName}`] ?? overrides?.[`*:${attributeName}`] ?? DEFAULT_PII_FIELDS[`${slug}:${attributeName}`] ?? DEFAULT_PII_FIELDS[`*:${attributeName}`]; - if (configured) { + if (configured && !(isPii && configured === 'keep')) { return configured; } const byType = attribute?.type ? STRATEGY_BY_ATTRIBUTE_TYPE[attribute.type] : undefined; + if (byType) { + return byType; + } + + if (isPii) { + if (attribute.type === 'date' || attribute.type === 'datetime') { + return 'date_year'; + } + + // free text: multiline content is redacted, single-line values stay joinable + return attribute.multiline ? 'redact' : 'value'; + } - return byType ?? 'keep'; + return 'keep'; }; type KeyPlan = { attribute?: Attribute; strategy: AnonymizationStrategy; explicitlyKept: boolean }; diff --git a/src/performance.test.ts b/src/performance.test.ts index 8159153..43ecfa6 100644 --- a/src/performance.test.ts +++ b/src/performance.test.ts @@ -107,3 +107,37 @@ describe('anonymizeResponse on large responses', () => { expect(second.results[0].first_name).toBe(first.results[0].first_name); }); }); + +describe('value-level detection cost', () => { + const anonymizer = createAnonymizer({ orgId: '123', secret: 'test-secret' }); + + it('scrubs a large mixed response quickly (URLs, streets, ids, numbers, labels)', async () => { + const strings = [ + 'https://www.google.de/maps/place/Hauptstra%C3%9Fe+5,+10115+Berlin/@52.5200066,13.404954,17z', + 'https://docs.example.com/v1/entities/73557d72-ff64-46ef-886d-93d221e45ea4?size=100', + 'Termin Hauptstraße 5, 10115 Berlin', + 'A-WP_PV-129-2504252_2505234_V2', + '2026-10-24T05:30:00.000Z', + 'Einzelversorgungsgebiet', + '1-0:1.8.0', + '73557d72-ff64-46ef-886d-93d221e45ea4', + ]; + const body = { + results: Array.from({ length: 2_000 }, (_, i) => ({ + _id: `o${i}`, + _org: '123', + _schema: 'opportunity', + ...Object.fromEntries(strings.map((value, j) => [`custom_${j}`, value])), + })), + }; + + const started = performance.now(); + const result = (await anonymizer.anonymizeResponse(body, { schemaResolver: async () => null })) as any; + const elapsed = performance.now() - started; + + expect(result.results[0].custom_0).not.toContain('Hauptstra'); + expect(result.results[0].custom_1).toBe(strings[1]); + // 16k strings; generous bound so slow CI runners do not flake, catches pathological regexes + expect(elapsed).toBeLessThan(2_000); + }); +}); diff --git a/src/pseudonymizer.ts b/src/pseudonymizer.ts index 648ebb1..f07b741 100644 --- a/src/pseudonymizer.ts +++ b/src/pseudonymizer.ts @@ -2,6 +2,7 @@ import { createHmac } from 'crypto'; import { PSEUDONYM_EMAIL_DOMAIN, REDACTED_PLACEHOLDER } from './defaults'; import type { AnonymizerLogger } from './types'; +import { replaceIban, scrubStreets, scrubWithUrls, type ValueReplacers } from './value-detection'; /** * Deterministic, one-way pseudonymization of PII values. @@ -27,7 +28,10 @@ export type Pseudonymizer = { street: (value: string) => string; dateToYear: (value: string) => string; redact: (value?: unknown) => string; - /** replaces emails / phone numbers / IBANs embedded in free text; idempotent */ + /** + * replaces emails / phone numbers / IBANs / street addresses embedded in free text, and the + * place + precise position of map links (URL components are decoded first); idempotent + */ scrubText: (value: string) => string; }; @@ -67,7 +71,7 @@ export const createPseudonymizer = (params: { redact, // still scrub identifiers OUT of free text (replacing only the matched substrings), // rather than blanking the whole string — so the catch-all pass keeps non-PII text intact - scrubText: makeScrubText(redact, redact, redact), + scrubText: makeScrubText({ email: redact, iban: redact, phone: redact, street: redact, value: redact }), }; } @@ -82,6 +86,8 @@ export const createPseudonymizer = (params: { const phone = (value: string) => `+00 000 ${parseInt(hash('phone', value), 16).toString().slice(0, 8).padStart(8, '0')}`; const iban = pseudonymize('iban', 'iban'); + const street = pseudonymize('street', 'street'); + const value = pseudonymize('value', 'value'); return { person: pseudonymize('person', 'person'), @@ -89,46 +95,61 @@ export const createPseudonymizer = (params: { email, phone, iban, - value: pseudonymize('value', 'value'), + value, text: pseudonymize('text', 'title'), - street: pseudonymize('street', 'street'), + street, dateToYear: (value: string) => { const year = new Date(value).getFullYear(); return Number.isFinite(year) ? `${year}-01-01` : REDACTED_PLACEHOLDER; }, redact: () => REDACTED_PLACEHOLDER, - scrubText: makeScrubText(email, iban, phone), + scrubText: makeScrubText({ email, iban, phone, street, value }), }; }; +type ScrubReplacers = ValueReplacers & { + email: (value: string) => string; + iban: (value: string) => string; + phone: (value: string) => string; +}; + +/** Every detector needs an `@` (emails), a `/` (URLs) or a digit (IBANs, phones, house numbers) */ +const SCRUB_CANDIDATE = /[@/\d]/; + /** - * Build a text scrubber that replaces email / IBAN / phone substrings embedded in free text, - * leaving the rest of the string intact. `email`/`iban`/`phone` are the pseudonymizer's own - * replacers (or the redactor when no secret is configured). Idempotent: pseudonymized emails - * are skipped, and phone/IBAN pseudonyms do not re-match the patterns. + * Build a text scrubber that replaces email / IBAN / phone / street-address substrings embedded + * in free text, leaving the rest of the string intact. Embedded URLs are decoded and scrubbed + * per component; map links lose the place and precise position (see value-detection.ts). + * The replacers are the pseudonymizer's own (or the redactor when no secret is configured). + * Idempotent: pseudonymized emails are skipped, and the other pseudonyms do not re-match. */ -const makeScrubText = - (email: (v: string) => string, iban: (v: string) => string, phone: (v: string) => string) => (value: string) => { +const makeScrubText = (replacers: ScrubReplacers) => { + const { email, iban, phone } = replacers; + + const scrubPlain = (value: string, context?: 'path'): string => { // Cheap prefilters, so the bulk of strings in a large response (enums, labels, numbers, // timestamps) skip the full patterns: emails need an `@`, and every IBAN / phone match // contains a NUMBER_CANDIDATE match (see below). const hasAt = value.includes('@'); const hasNumber = NUMBER_CANDIDATE.test(value); - if (!hasAt && !hasNumber) { - return value; - } let result = hasAt ? value.replace(EMAIL_PATTERN, (match) => (match.endsWith(`@${PSEUDONYM_EMAIL_DOMAIN}`) ? match : email(match))) : value; if (hasNumber) { - result = result.replace(IBAN_PATTERN, iban).replace(PHONE_PATTERN, phone).replace(NATIONAL_PHONE_PATTERN, phone); + result = result.replace(IBAN_PATTERN, (match) => replaceIban(match, iban)); + if (context !== 'path') { + result = result.replace(PHONE_PATTERN, phone).replace(NATIONAL_PHONE_PATTERN, phone); + } } - return result; + return scrubStreets(result, replacers); }; + return (value: string): string => (SCRUB_CANDIDATE.test(value) ? scrubWithUrls(value, scrubPlain, replacers) : value); +}; + /** * Necessary condition for IBAN_PATTERN / PHONE_PATTERN / NATIONAL_PHONE_PATTERN: the prefix * each of them must start with (same lookbehinds). A superset by construction — it may let diff --git a/src/value-detection.test.ts b/src/value-detection.test.ts new file mode 100644 index 0000000..6c4522a --- /dev/null +++ b/src/value-detection.test.ts @@ -0,0 +1,206 @@ +import { describe, expect, it } from 'vitest'; + +import { createAnonymizer, getAttributeStrategy } from './index'; +import type { Schema } from './types'; +import { isValidIban } from './value-detection'; + +const anonymizer = createAnonymizer({ orgId: '16581838', secret: 'test-secret' }); + +const MAPS_LINK = + 'https://www.google.de/maps/place/Schiltacher+Str.+33,+70569+Stuttgart/@48.7413596,9.1362755,725m/data=!3m2!1e3!4b1!4m6!3m5!1s0x4799dc937d6a341f:0x6f4012ba497d9f53!8m2!3d48.7413596!4d9.1388504!16s%2Fg%2F11c1jq8f84'; + +const opportunitySchema: Schema = { + slug: 'opportunity', + attributes: [ + { name: 'link_gebaeudeansicht', type: 'string', label: 'Link Gebäudeansicht' }, + { name: 'public_link', type: 'string', data_classification: 'public' }, + ], +}; + +const expectNoAddress = (value: string) => { + for (const leaked of ['Schiltacher', 'Str.+33', '48.7413596', '9.1362755', '9.1388504', '0x4799dc937d6a341f', '11c1jq8f84']) { + expect(value).not.toContain(leaked); + } +}; + +describe('map links (STABLE360-13580: link_gebaeudeansicht)', () => { + it('masks the place and rounds the position of a Google Maps link in an unclassified attribute', async () => { + const body = { + _id: '73557d72', + _org: '16581838', + _schema: 'opportunity', + link_gebaeudeansicht: MAPS_LINK, + }; + const result = await anonymizer.anonymizeResponse(body, { schemaResolver: async () => opportunitySchema }); + + expectNoAddress(result.link_gebaeudeansicht); + expect(result.link_gebaeudeansicht).toMatch( + /^https:\/\/www\.google\.de\/maps\/place\/street_[0-9a-f]{8},\+70569\+Stuttgart\/@48\.74,9\.14,725m$/, + ); + }); + + it('masks the link inside activity feed payloads and operations', async () => { + const body = { + results: [ + { + type: 'EntityUpdated', + payload: { entity: { _id: 'o1', _org: '16581838', _schema: 'opportunity', link_gebaeudeansicht: MAPS_LINK } }, + operations: [{ operation: 'updateEntity', payload: { link_gebaeudeansicht: MAPS_LINK } }], + }, + ], + }; + const result = (await anonymizer.anonymizeResponse(body, { schemaResolver: async () => opportunitySchema })) as any; + + expectNoAddress(JSON.stringify(result)); + }); + + it('keeps an attribute classified public untouched', async () => { + const body = { _id: 'o1', _org: '16581838', _schema: 'opportunity', public_link: MAPS_LINK }; + const result = await anonymizer.anonymizeResponse(body, { schemaResolver: async () => opportunitySchema }); + + expect(result.public_link).toBe(MAPS_LINK); + }); + + it('is deterministic and idempotent', () => { + const once = anonymizer.scrub(MAPS_LINK) as string; + + expect(anonymizer.scrub(MAPS_LINK)).toBe(once); + expect(anonymizer.scrub(once)).toBe(once); + }); + + it.each([ + ['google query', 'https://maps.google.com/?q=Hauptstra%C3%9Fe+5,+10115+Berlin', ['Hauptstr', 'Hauptstraße']], + ['google coordinates query', 'https://www.google.com/maps/search/?api=1&query=48.7413596,9.1362755', ['7413596']], + ['google place id', 'https://www.google.com/maps/place/?q=place_id:ChIJ123&cid=1234567890', ['ChIJ123', '1234567890']], + ['apple', 'https://maps.apple.com/?address=Berliner+Stra%C3%9Fe+12,+10115+Berlin&ll=52.5200066,13.404954', ['Berliner', '5200066']], + ['bing', 'https://www.bing.com/maps?where1=Am+Ring+3+50667+K%C3%B6ln&cp=50.9375123~6.9602786', ['Ring', '9375123']], + ['openstreetmap', 'https://www.openstreetmap.org/?mlat=48.7413596&mlon=9.1362755#map=19/48.7413596/9.1362755', ['7413596', '1362755']], + ['short link', 'https://maps.app.goo.gl/AbCdEf123456', ['AbCdEf123456']], + ])('anonymizes %s links', (_, link, leaks) => { + const result = anonymizer.scrub(`Gebäude: ${link}`) as string; + + expect(result.startsWith('Gebäude: https://')).toBe(true); + for (const leaked of leaks) { + expect(result).not.toContain(leaked); + } + expect(anonymizer.scrub(result)).toBe(result); + }); + + it('keeps postal code and city of a map query', () => { + const result = anonymizer.scrub('https://maps.google.com/?q=Hauptstra%C3%9Fe+5,+10115+Berlin') as string; + + expect(new URL(result).searchParams.get('q')).toMatch(/^street_[0-9a-f]{8}, 10115 Berlin$/); + }); +}); + +describe('PII embedded in other URLs', () => { + it('scrubs URL-encoded emails, phones in query values and streets in paths', () => { + const url = 'https://portal.example.com/u/max.mustermann%40example.com/haus/Hauptstra%C3%9Fe%205?tel=%2B49%20170%201234567&tab=1'; + const result = anonymizer.scrub(url) as string; + + for (const leaked of ['mustermann', 'Hauptstra', '1234567']) { + expect(result).not.toContain(leaked); + } + expect(result).toContain('anonymized.invalid'); + expect(result).toContain('tab=1'); + expect(anonymizer.scrub(result)).toBe(result); + }); + + it('removes credentials from URL userinfo', () => { + const result = anonymizer.scrub('https://max:geheim@files.example.com/a.pdf') as string; + + expect(result).not.toContain('max'); + expect(result).not.toContain('geheim'); + }); +}); + +describe('German street addresses in free text', () => { + it.each([ + 'Hauptstraße 5', + 'Hauptstrasse 5', + 'Hauptstr. 5', + 'Schiltacher Str. 33', + 'Berliner Straße 12a', + 'Max-Planck-Str. 12-14', + 'Am Ring 3', + 'Lindenallee 7', + 'Kirchgasse 2 b', + 'Mühlenweg 10/1', + 'Kurfürstendamm 21', + ])('replaces "%s" and keeps postal code + city', (street) => { + const result = anonymizer.scrub(`Vor-Ort-Termin: ${street}, 70569 Stuttgart.`) as string; + + expect(result).toMatch(/^Vor-Ort-Termin: street_[0-9a-f]{8}, 70569 Stuttgart\.$/); + expect(anonymizer.scrub(result)).toBe(result); + }); +}); + +describe('IBAN checksum', () => { + it('validates the mod-97 checksum', () => { + expect(isValidIban('DE89370400440532013000')).toBe(true); + expect(isValidIban('AT611904300234573201')).toBe(true); + expect(isValidIban('DE89370400440532013001')).toBe(false); + }); + + it('masks a valid IBAN followed by uppercase text, but not IBAN-shaped codes', () => { + const result = anonymizer.scrub('DE89 3704 0044 0532 0130 00 BIC COBADEFFXXX') as string; + + expect(result).toMatch(/^iban_[0-9a-f]{8} BIC COBADEFFXXX$/); + expect(anonymizer.scrub('Code DE12ABCDEFGHIJKLMN')).toBe('Code DE12ABCDEFGHIJKLMN'); + }); +}); + +describe('false positives stay unchanged', () => { + it.each([ + 'A-WP_PV-129-2504252_2505234_V2', + 'OV60WPE004009', + '1-0:1.8.0', + '2026-10-24', + '2026-10-24T05:30:00.000Z', + '1.299,00 €', + '19,99 EUR', + '73557d72-ff64-46ef-886d-93d221e45ea4', + 'Wärmepumpe Wärmebedarf', + 'Spring 2026', + 'Abnahme PV: 03.03.26 14 Uhr', + 'https://www.epilot.cloud/de/blog/energy-xrm-2026', + 'https://portal.example.com/030-1234567', + 'https://docs.example.com/v1/entities/73557d72-ff64-46ef-886d-93d221e45ea4?size=100#section-2', + 'https://www.google.de/search?q=w%C3%A4rmepumpe', + 'Einzelversorgungsgebiet', + 'Möhringen', + ])('%s', (value) => { + expect(anonymizer.scrub(value)).toBe(value); + }); +}); + +describe('classification fixes', () => { + it('maps internal_user attributes to the user strategy', () => { + expect(getAttributeStrategy({ slug: 'email_template', attributeName: 'created_by', attribute: { name: 'created_by', type: 'internal_user' } })).toBe('user'); + }); + + it.each(['billing_account', 'calendar_event', 'contract', 'lead', 'request', 'ticket'])('masks %s _title', (slug) => { + expect(getAttributeStrategy({ slug, attributeName: '_title' })).toBe('text'); + }); + + it('lets pii honor curated strategies (names stay person pseudonyms)', () => { + const pii = (name: string, type = 'string') => + getAttributeStrategy({ slug: 'contact', attributeName: name, attribute: { name, type, data_classification: 'pii' } }); + + expect(pii('first_name')).toBe('person'); + expect(pii('salutation')).toBe('redact'); + expect(pii('email', 'email')).toBe('email'); + expect(pii('customer_number')).toBe('value'); + }); + + it('never keeps a pii attribute, even with a keep override', () => { + expect( + getAttributeStrategy({ + slug: 'contact', + attributeName: 'notes', + attribute: { name: 'notes', type: 'string', multiline: true, data_classification: 'pii' }, + overrides: { 'contact:notes': 'keep' }, + }), + ).toBe('redact'); + }); +}); diff --git a/src/value-detection.ts b/src/value-detection.ts new file mode 100644 index 0000000..18845f3 --- /dev/null +++ b/src/value-detection.ts @@ -0,0 +1,323 @@ +/** + * Value-level PII detection for strings no field name or type classified. + * + * Used by the catch-all scrub (`scrubText`): only the matched span is replaced, the rest of the + * value stays intact. Covers what field-name rules cannot see, e.g. a custom + * `link_gebaeudeansicht` attribute holding a Google Maps link to the customer's house. + */ + +/** Replacers for detected spans (the pseudonymizer's own, or the redactor without a secret) */ +export type ValueReplacers = { + street: (value: string) => string; + value: (value: string) => string; +}; + +/** + * Text scrubber for URL components (emails / IBANs / phones / streets). `path` skips phone + * numbers: digit-and-dash path segments are ids and slugs far more often ("/030-1234567"). + */ +type PlainScrub = (value: string, context?: 'path') => string; + +// --- German street addresses ------------------------------------------------------------- + +const SUFFIX_LOWER = '(?:stra(?:ß|ss)e|str\\.|weg|platz|allee|gasse|ring|damm)'; +const SUFFIX_CAP = '(?:Stra(?:ß|ss)e|Str\\.|Weg|Platz|Allee|Gasse|Ring|Damm)'; + +/** + * A street name followed by a house number: "Hauptstraße 5", "Schiltacher Str. 33", + * "Max-Planck-Str. 12a", "Am Ring 3", "Lindenallee 12-14". + * + * - the street word is capitalized (German nouns): a compound ending in a lowercase suffix + * ("Hauptstraße", "Mühlenweg") or a standalone capitalized suffix after up to two + * capitalized words ("Berliner Straße", "Am Alten Weg") + * - house numbers have 1-3 digits, so years ("Spring 2026") do not match + * - postal code and city are NOT part of the match: they stay visible, like on address items + */ +export const STREET_PATTERN = new RegExp( + `(? + STREET_CANDIDATE.test(value) ? value.replace(STREET_PATTERN, (match) => replacers.street(match)) : value; + +// --- IBAN checksum ----------------------------------------------------------------------- + +/** ISO 13616 mod-97 check of a compact IBAN */ +export const isValidIban = (compact: string): boolean => { + if (!/^[A-Z]{2}\d{2}[A-Z0-9]{11,30}$/.test(compact)) { + return false; + } + + const rearranged = compact.slice(4) + compact.slice(0, 4); + let remainder = 0; + for (const char of rearranged) { + const digits = char >= 'A' ? String(char.charCodeAt(0) - 55) : char; + for (const digit of digits) { + remainder = (remainder * 10 + Number(digit)) % 97; + } + } + + return remainder === 1; +}; + +/** + * Pseudonymize an IBAN_PATTERN match only when it is a valid IBAN. The pattern is greedy over + * space-separated groups ("DE89 3704 … 00 BIC"), so shorter prefixes ending at a space are + * tried too; the unmatched tail is kept. + */ +export const replaceIban = (match: string, iban: (value: string) => string): string => { + const ends = [match.length]; + for (let i = match.length - 1; i > 0; i--) { + if (match[i] === ' ') { + ends.push(i); + } + } + + for (const end of ends) { + const candidate = match.slice(0, end).trimEnd(); + if (isValidIban(candidate.replace(/ /g, ''))) { + return iban(candidate) + match.slice(candidate.length); + } + } + + return match; +}; + +// --- URLs --------------------------------------------------------------------------------- + +/** URLs embedded in a string value */ +export const URL_TOKEN = /\bhttps?:\/\/[^\s<>"'`]+/g; + +const PSEUDONYM_PATTERN = /^(?:street|value)_[0-9a-f]{8}\b|^\[REDACTED\]/; + +/** lat/lng with more precision than 2 decimals (~1 km) */ +const PRECISE_COORDINATE = /-?\d{1,3}\.\d{3,}/g; +const COORDINATE_PAIR = /^\s*-?\d{1,3}(?:\.\d+)?\s*[,~ ]\s*-?\d{1,3}(?:\.\d+)?\s*$/; + +const roundCoordinates = (value: string): string => + value.replace(PRECISE_COORDINATE, (number) => (Math.round(Number(number) * 100) / 100).toFixed(2)); + +const safeDecode = (value: string): string => { + try { + return decodeURIComponent(value); + } catch { + return value; + } +}; + +/** Map-link location text (place name / address / search query) -> pseudonym, keeping zip + city */ +const anonymizePlace = (text: string, replacers: ValueReplacers): string => { + if (!text.trim() || PSEUDONYM_PATTERN.test(text.trim())) { + return text; + } + if (COORDINATE_PAIR.test(text)) { + return roundCoordinates(text); + } + + const zipCity = text.match(ZIP_CITY_PATTERN)?.[0]; + + return zipCity ? `${replacers.street(text)}, ${zipCity}` : replacers.street(text); +}; + +type MapProvider = 'google' | 'apple' | 'bing' | 'osm' | 'shortlink'; + +const getMapProvider = (url: URL): MapProvider | undefined => { + const host = url.hostname.toLowerCase(); + const path = url.pathname.toLowerCase(); + + if (host === 'maps.app.goo.gl' || (host === 'goo.gl' && path.startsWith('/maps'))) { + return 'shortlink'; + } + if (/^maps\.google\./.test(host) || (/(^|\.)google\.[a-z]{2,3}(\.[a-z]{2})?$/.test(host) && path.startsWith('/maps'))) { + return 'google'; + } + if (host === 'maps.apple.com') { + return 'apple'; + } + if (/(^|\.)bing\.com$/.test(host) && path.startsWith('/maps')) { + return 'bing'; + } + if (/(^|\.)(openstreetmap\.org|osm\.org)$/.test(host)) { + return 'osm'; + } + + return undefined; +}; + +/** Google Maps path segments that are routing keywords, not locations */ +const GOOGLE_PATH_KEYWORDS = new Set(['maps', 'place', 'search', 'dir', 'embed', 'v1', 'preview', 'contrib', '']); + +/** Query params of map links carrying a location (address text or coordinates) */ +const MAP_LOCATION_PARAMS = new Set([ + 'q', + 'query', + 'address', + 'daddr', + 'saddr', + 'destination', + 'origin', + 'll', + 'sll', + 'center', + 'near', + 'name', + 'where1', + 'cp', + 'sp', + 'rtp', + 'mlat', + 'mlon', + 'lat', + 'lon', + 'lng', + 'route', +]); + +/** Query params of map links identifying a concrete place (resolve to the address) */ +const MAP_PLACE_ID_PARAMS = new Set(['cid', 'ftid', 'place_id', 'query_place_id', 'auid', 'place-id', 'ss']); + +const encodeGoogleSegment = (value: string) => encodeURIComponent(value).replace(/%20/g, '+').replace(/%2C/gi, ','); + +const anonymizeMapUrl = (url: URL, provider: MapProvider, replacers: ValueReplacers): string => { + if (provider === 'shortlink') { + // the short id resolves to the shared place + const id = url.pathname.split('/').filter(Boolean).pop(); + if (id && !PSEUDONYM_PATTERN.test(id)) { + url.pathname = url.pathname.replace(id, replacers.value(id)); + } + + return url.toString(); + } + + if (provider === 'google') { + url.pathname = url.pathname + .split('/') + // `data=` holds place feature / knowledge-graph ids and precise coordinates + .filter((segment) => !segment.startsWith('data=')) + .map((segment) => { + if (GOOGLE_PATH_KEYWORDS.has(segment.toLowerCase())) { + return segment; + } + if (segment.startsWith('@')) { + return roundCoordinates(segment); + } + + return encodeGoogleSegment(anonymizePlace(safeDecode(segment.replace(/\+/g, ' ')), replacers)); + }) + .join('/'); + } + + for (const [key, val] of [...url.searchParams.entries()]) { + const lowerKey = key.toLowerCase(); + if (MAP_PLACE_ID_PARAMS.has(lowerKey)) { + url.searchParams.set(key, PSEUDONYM_PATTERN.test(val) ? val : replacers.value(val)); + } else if (lowerKey === 'mlat' || lowerKey === 'mlon' || lowerKey === 'lat' || lowerKey === 'lon' || lowerKey === 'lng') { + url.searchParams.set(key, roundCoordinates(val)); + } else if (MAP_LOCATION_PARAMS.has(lowerKey)) { + url.searchParams.set(key, anonymizePlace(val, replacers)); + } + } + + // OpenStreetMap: "#map=19/48.7413596/9.1362755" + if (url.hash) { + url.hash = roundCoordinates(url.hash); + } + + return url.toString(); +}; + +/** Scrub a generic URL's components; returns the input unchanged when nothing was found */ +const anonymizeGenericUrl = (raw: string, url: URL, scrub: PlainScrub, replacers: ValueReplacers): string => { + let changed = false; + + if (url.username || url.password) { + url.username = url.username ? replacers.value(safeDecode(url.username)) : ''; + url.password = ''; + changed = true; + } + + const segments = url.pathname.split('/').map((segment) => { + const decoded = safeDecode(segment); + const scrubbed = scrub(decoded, 'path'); + if (scrubbed === decoded) { + return segment; + } + changed = true; + + return encodeURIComponent(scrubbed); + }); + + for (const [key, val] of [...url.searchParams.entries()]) { + const scrubbed = scrub(val); + if (scrubbed !== val) { + url.searchParams.set(key, scrubbed); + changed = true; + } + } + + let hash = url.hash; + if (hash) { + const decoded = safeDecode(hash); + const scrubbed = scrub(decoded); + if (scrubbed !== decoded) { + hash = scrubbed; + changed = true; + } + } + + if (!changed) { + return raw; + } + + url.pathname = segments.join('/'); + url.hash = hash; + + return url.toString(); +}; + +/** Anonymize one URL: map links lose place + precise position, other URLs their embedded PII */ +export const anonymizeUrl = (raw: string, scrub: PlainScrub, replacers: ValueReplacers): string => { + let url: URL; + try { + url = new URL(raw); + } catch { + return scrub(raw); + } + + const provider = getMapProvider(url); + if (provider) { + const anonymized = anonymizeMapUrl(url, provider, replacers); + // keep an already-anonymized link byte-identical (idempotency) + return anonymized === new URL(raw).toString() ? raw : anonymized; + } + + return anonymizeGenericUrl(raw, url, scrub, replacers); +}; + +/** Apply `scrub` to the non-URL parts of a string and `anonymizeUrl` to each embedded URL */ +export const scrubWithUrls = (value: string, scrub: PlainScrub, replacers: ValueReplacers): string => { + if (!value.includes('://')) { + return scrub(value); + } + + let result = ''; + let last = 0; + for (const match of value.matchAll(URL_TOKEN)) { + const index = match.index ?? 0; + result += scrub(value.slice(last, index)) + anonymizeUrl(match[0], scrub, replacers); + last = index + match[0].length; + } + + return result + scrub(value.slice(last)); +};