feat: value-level PII detection for unclassified fields (STABLE360-13580)

README.md

@@ -82,9 +82,11 @@ const anonymizer = createAnonymizer({
 ### Schema-aware entity mode
 
 For epilot entities, when the entity schema is available (or resolvable). Classification uses,
-in order: the attribute's `data_classification` (`'public'` opt-out / `'pii'` opt-in), your
-`overrides`, the curated defaults, then attribute-type defaults (`email`, `phone`, `address`,
-`payment`).
+in order: `data_classification: 'public'` (opt-out), your `overrides`, the curated defaults,
+attribute-type defaults (`email`, `phone`, `address`, `payment`, user relations, …), then
+`data_classification: 'pii'` (opt-in). A `pii` attribute is never kept, and it still gets the
+specific strategy a curated default or its type gives it (a `pii` `first_name` stays a `person`
+pseudonym).
 
 ```ts
 // single entity (or a partial entity, e.g. an activity diff)
@@ -153,12 +155,18 @@ Being upfront about what this package does **not** do is part of its job:
    destroys utility.)
 2. **Free text is kept by default, but identifiers in it are scrubbed.** Unclassified free-text
    fields are not fully masked in schema-aware mode (masking them all would destroy utility),
-   but a catch-all pass scrubs any email / phone / IBAN embedded in kept string values (custom
-   fields, computed titles, attribute types not special-cased). Well-known risky fields
+   but a catch-all pass scrubs identifiers embedded in kept string values (custom fields,
+   computed titles, attribute types not special-cased): emails, phones, IBANs (mod-97 checked),
+   German street addresses ("Hauptstraße 5", "Schiltacher Str. 33" — postal code and city stay),
+   and PII inside URLs (components are decoded first; map links lose the place name, place ids
+   and the precise position, coordinates are rounded to 2 decimals ≈ 1 km). Only the matched
+   span is replaced. Well-known risky fields
    (note/message content, opportunity descriptions, …) are fully redacted via the curated
    defaults; organizations opt further fields in with `data_classification: 'pii'` or out with
    `'public'` (which also exempts them from the catch-all). Names or other non-pattern PII typed
-   into unclassified fields can still slip through.
+   into unclassified fields can still slip through. The street detector is pattern based: a
+   capitalized word ending in a street suffix followed by a 1-3 digit number also matches
+   non-addresses ("Arbeitsplatz 2"), and streets without a suffix ("An der Kirche 3") are missed.
 3. **Heuristics are best-effort.** The non-schema mode catches common field names and
    email/phone/IBAN patterns. Phones match in international (`+49 …`, `0049 …`) and German/Austrian
    national format with a separator after the area code (`030 1234567`, `0221/123456`,
@@ -187,15 +195,17 @@ counts as PII".
 | Addresses (`address`, `relation_address`) | keep postal code / city / country; pseudonymize street; drop the rest |
 | Birthdates, PII-flagged dates | truncate to year |
 | Payment items | pseudonymize IBAN/holder; keep bank name / BIC |
-| User-relation attributes (`relation_user`, e.g. `contact_owner`) | pseudonymize name/email, **redact credentials** (`token`, `password`, …), keep ids/status/settings |
+| User-relation attributes (`relation_user`, `internal_user`, e.g. `contact_owner`) | pseudonymize name/email, **redact credentials** (`token`, `password`, …), keep ids/status/settings |
 | Consent attributes (`consent`, e.g. `consent_email_marketing`) | pseudonymize the contact identifier (email/phone), keep consent status and events |
 | Message parties (`from`, `to`, `cc`, `bcc`, `reply_to`) | pseudonymize names, scrub addresses |
 | Activity `caller` (incl. nested `EpilotAuth` claims) | same rules as user-relation attributes |
 | Curated free-text (note/message content and titles, thread topic/title, opportunity description, …) | `[REDACTED]` |
 | File names (`file` `filename` / `_title`) | `value` pseudonym |
 | Entity fields not in the schema (e.g. `journey_data`) | field-name heuristics of `anonymizeUnknown` |
 | `data_classification: 'pii'` fields | pseudonym (single-line) / `[REDACTED]` (multiline) |
-| Everything else | emails / phones / IBANs embedded in strings are scrubbed, the rest is unchanged |
+| Computed titles built from personal data (`billing_account`, `calendar_event`, `contract`, `lead`, `request`, `ticket` `_title`) | `text` pseudonym |
+| Map links (Google, Apple, Bing, OpenStreetMap, short links) in unclassified fields | place → `street` pseudonym + postal code / city, place ids pseudonymized, coordinates rounded to ~1 km |
+| Everything else | emails / phones / IBANs / street addresses embedded in strings (and URL components) are scrubbed, the rest is unchanged |
 
 ## Development
 

src/defaults.ts

@@ -26,6 +26,8 @@ export const STRATEGY_BY_ATTRIBUTE_TYPE: Record<string, AnonymizationStrategy> =
   // user-relation attributes embed full epilot user objects (name, email, and often an auth
   // `token`) — never pass them through raw
   relation_user: 'user',
+  // `internal_user` attributes (e.g. template `created_by`) embed the same user objects
+  internal_user: 'user',
   // address relations (e.g. opportunity `delivery_address`) carry the referenced address
   // values alongside the `$relation_ref`
   relation_address: 'address',
@@ -79,6 +81,13 @@ export const DEFAULT_PII_FIELDS: Record<string, AnonymizationStrategy> = {
   // computed titles of messages/threads are the subject/topic
   'message:_title': 'redact',
   'thread:_title': 'redact',
+  // computed titles built from personal data (title templates in the entity blueprints)
+  'billing_account:_title': 'text', // "{{billing_contact.0._title}} - …" = the contact's name
+  'calendar_event:_title': 'text', // the event subject
+  'contract:_title': 'text', // contract_name
+  'lead:_title': 'text', // lead_title
+  'request:_title': 'text', // request_title
+  'ticket:_title': 'text', // ticket_title
   // uploaded file names regularly contain customer names (e.g. "Rechnung_Mustermann.pdf");
   // `_title` is the file name too, so both map to the same pseudonym
   'file:filename': 'value',

src/entity.ts

@@ -17,11 +17,14 @@ import { isPlainObject, mapArrayShared, mapEntries, mapEntriesShared } from './u
  *
  * Precedence (first match wins):
  * 1. `data_classification: 'public'` on the schema attribute -> keep (org opt-out)
- * 2. `data_classification: 'pii'` on the schema attribute -> mask (org opt-in, e.g. free text)
- * 3. consumer `overrides` (same key format as the curated map)
- * 4. curated built-in PII fields (`DEFAULT_PII_FIELDS`)
- * 5. defaults by attribute type (email, phone, address, payment)
+ * 2. consumer `overrides` (same key format as the curated map)
+ * 3. curated built-in PII fields (`DEFAULT_PII_FIELDS`)
+ * 4. defaults by attribute type (email, phone, address, payment, user relations, …)
+ * 5. `data_classification: 'pii'` on the schema attribute -> mask (org opt-in, e.g. free text):
+ *    dates keep the year, multiline text is redacted, single-line values are pseudonymized
  * 6. keep
+ *
+ * A `pii` attribute is never kept: an override or curated default to `keep` does not apply to it.
  */
 export const getAttributeStrategy = (params: {
   slug: string;
@@ -35,31 +38,31 @@ export const getAttributeStrategy = (params: {
     return 'keep';
   }
 
-  if (attribute?.data_classification === 'pii') {
-    const byType = attribute.type ? STRATEGY_BY_ATTRIBUTE_TYPE[attribute.type] : undefined;
-    if (byType) {
-      return byType;
-    }
-    if (attribute.type === 'date' || attribute.type === 'datetime') {
-      return 'date_year';
-    }
-
-    // free text: multiline content is redacted, single-line values stay joinable
-    return attribute.multiline ? 'redact' : 'value';
-  }
-
+  const isPii = attribute?.data_classification === 'pii';
   const configured =
     overrides?.[`${slug}:${attributeName}`] ??
     overrides?.[`*:${attributeName}`] ??
     DEFAULT_PII_FIELDS[`${slug}:${attributeName}`] ??
     DEFAULT_PII_FIELDS[`*:${attributeName}`];
-  if (configured) {
+  if (configured && !(isPii && configured === 'keep')) {
     return configured;
   }
 
   const byType = attribute?.type ? STRATEGY_BY_ATTRIBUTE_TYPE[attribute.type] : undefined;
+  if (byType) {
+    return byType;
+  }
+
+  if (isPii) {
+    if (attribute.type === 'date' || attribute.type === 'datetime') {
+      return 'date_year';
+    }
+
+    // free text: multiline content is redacted, single-line values stay joinable
+    return attribute.multiline ? 'redact' : 'value';
+  }
 
-  return byType ?? 'keep';
+  return 'keep';
 };
 
 type KeyPlan = { attribute?: Attribute; strategy: AnonymizationStrategy; explicitlyKept: boolean };

src/performance.test.ts

@@ -107,3 +107,37 @@ describe('anonymizeResponse on large responses', () => {
     expect(second.results[0].first_name).toBe(first.results[0].first_name);
   });
 });
+
+describe('value-level detection cost', () => {
+  const anonymizer = createAnonymizer({ orgId: '123', secret: 'test-secret' });
+
+  it('scrubs a large mixed response quickly (URLs, streets, ids, numbers, labels)', async () => {
+    const strings = [
+      'https://www.google.de/maps/place/Hauptstra%C3%9Fe+5,+10115+Berlin/@52.5200066,13.404954,17z',
+      'https://docs.example.com/v1/entities/73557d72-ff64-46ef-886d-93d221e45ea4?size=100',
+      'Termin Hauptstraße 5, 10115 Berlin',
+      'A-WP_PV-129-2504252_2505234_V2',
+      '2026-10-24T05:30:00.000Z',
+      'Einzelversorgungsgebiet',
+      '1-0:1.8.0',
+      '73557d72-ff64-46ef-886d-93d221e45ea4',
+    ];
+    const body = {
+      results: Array.from({ length: 2_000 }, (_, i) => ({
+        _id: `o${i}`,
+        _org: '123',
+        _schema: 'opportunity',
+        ...Object.fromEntries(strings.map((value, j) => [`custom_${j}`, value])),
+      })),
+    };
+
+    const started = performance.now();
+    const result = (await anonymizer.anonymizeResponse(body, { schemaResolver: async () => null })) as any;
+    const elapsed = performance.now() - started;
+
+    expect(result.results[0].custom_0).not.toContain('Hauptstra');
+    expect(result.results[0].custom_1).toBe(strings[1]);
+    // 16k strings; generous bound so slow CI runners do not flake, catches pathological regexes
+    expect(elapsed).toBeLessThan(2_000);
+  });
+});

src/pseudonymizer.ts

@@ -2,6 +2,7 @@ import { createHmac } from 'crypto';
 
 import { PSEUDONYM_EMAIL_DOMAIN, REDACTED_PLACEHOLDER } from './defaults';
 import type { AnonymizerLogger } from './types';
+import { replaceIban, scrubStreets, scrubWithUrls, type ValueReplacers } from './value-detection';
 
 /**
  * Deterministic, one-way pseudonymization of PII values.
@@ -27,7 +28,10 @@ export type Pseudonymizer = {
   street: (value: string) => string;
   dateToYear: (value: string) => string;
   redact: (value?: unknown) => string;
-  /** replaces emails / phone numbers / IBANs embedded in free text; idempotent */
+  /**
+   * replaces emails / phone numbers / IBANs / street addresses embedded in free text, and the
+   * place + precise position of map links (URL components are decoded first); idempotent
+   */
   scrubText: (value: string) => string;
 };
 
@@ -67,7 +71,7 @@ export const createPseudonymizer = (params: {
       redact,
       // still scrub identifiers OUT of free text (replacing only the matched substrings),
       // rather than blanking the whole string — so the catch-all pass keeps non-PII text intact
-      scrubText: makeScrubText(redact, redact, redact),
+      scrubText: makeScrubText({ email: redact, iban: redact, phone: redact, street: redact, value: redact }),
     };
   }
 
@@ -82,53 +86,70 @@ export const createPseudonymizer = (params: {
   const phone = (value: string) =>
     `+00 000 ${parseInt(hash('phone', value), 16).toString().slice(0, 8).padStart(8, '0')}`;
   const iban = pseudonymize('iban', 'iban');
+  const street = pseudonymize('street', 'street');
+  const value = pseudonymize('value', 'value');
 
   return {
     person: pseudonymize('person', 'person'),
     company: pseudonymize('company', 'company'),
     email,
     phone,
     iban,
-    value: pseudonymize('value', 'value'),
+    value,
     text: pseudonymize('text', 'title'),
-    street: pseudonymize('street', 'street'),
+    street,
     dateToYear: (value: string) => {
       const year = new Date(value).getFullYear();
 
       return Number.isFinite(year) ? `${year}-01-01` : REDACTED_PLACEHOLDER;
     },
     redact: () => REDACTED_PLACEHOLDER,
-    scrubText: makeScrubText(email, iban, phone),
+    scrubText: makeScrubText({ email, iban, phone, street, value }),
   };
 };
 
+type ScrubReplacers = ValueReplacers & {
+  email: (value: string) => string;
+  iban: (value: string) => string;
+  phone: (value: string) => string;
+};
+
+/** Every detector needs an `@` (emails), a `/` (URLs) or a digit (IBANs, phones, house numbers) */
+const SCRUB_CANDIDATE = /[@/\d]/;
+
 /**
- * Build a text scrubber that replaces email / IBAN / phone substrings embedded in free text,
- * leaving the rest of the string intact. `email`/`iban`/`phone` are the pseudonymizer's own
- * replacers (or the redactor when no secret is configured). Idempotent: pseudonymized emails
- * are skipped, and phone/IBAN pseudonyms do not re-match the patterns.
+ * Build a text scrubber that replaces email / IBAN / phone / street-address substrings embedded
+ * in free text, leaving the rest of the string intact. Embedded URLs are decoded and scrubbed
+ * per component; map links lose the place and precise position (see value-detection.ts).
+ * The replacers are the pseudonymizer's own (or the redactor when no secret is configured).
+ * Idempotent: pseudonymized emails are skipped, and the other pseudonyms do not re-match.
  */
-const makeScrubText =
-  (email: (v: string) => string, iban: (v: string) => string, phone: (v: string) => string) => (value: string) => {
+const makeScrubText = (replacers: ScrubReplacers) => {
+  const { email, iban, phone } = replacers;
+
+  const scrubPlain = (value: string, context?: 'path'): string => {
     // Cheap prefilters, so the bulk of strings in a large response (enums, labels, numbers,
     // timestamps) skip the full patterns: emails need an `@`, and every IBAN / phone match
     // contains a NUMBER_CANDIDATE match (see below).
     const hasAt = value.includes('@');
     const hasNumber = NUMBER_CANDIDATE.test(value);
-    if (!hasAt && !hasNumber) {
-      return value;
-    }
 
     let result = hasAt
       ? value.replace(EMAIL_PATTERN, (match) => (match.endsWith(`@${PSEUDONYM_EMAIL_DOMAIN}`) ? match : email(match)))
       : value;
     if (hasNumber) {
-      result = result.replace(IBAN_PATTERN, iban).replace(PHONE_PATTERN, phone).replace(NATIONAL_PHONE_PATTERN, phone);
+      result = result.replace(IBAN_PATTERN, (match) => replaceIban(match, iban));
+      if (context !== 'path') {
+        result = result.replace(PHONE_PATTERN, phone).replace(NATIONAL_PHONE_PATTERN, phone);
+      }
     }
 
-    return result;
+    return scrubStreets(result, replacers);
   };
 
+  return (value: string): string => (SCRUB_CANDIDATE.test(value) ? scrubWithUrls(value, scrubPlain, replacers) : value);
+};
+
 /**
  * Necessary condition for IBAN_PATTERN / PHONE_PATTERN / NATIONAL_PHONE_PATTERN: the prefix
  * each of them must start with (same lookbehinds). A superset by construction — it may let