diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..25a4a45 --- /dev/null +++ b/.gitignore @@ -0,0 +1,76 @@ +# ---- Python ---- +__pycache__/ +*.py[cod] +*$py.class +*.so + +# Distribution / packaging +.Python +build/ +dist/ +develop-eggs/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +*.egg +MANIFEST + +# Virtual environments +venv/ +.venv/ +env/ +.env/ +ENV/ + +# Pytest / coverage / type checkers +.pytest_cache/ +.cache/ +.coverage +.coverage.* +htmlcov/ +coverage.xml +*.cover +.hypothesis/ +.mypy_cache/ +.dmypy.json +.pyre/ +.pytype/ +.ruff_cache/ + +# Jupyter +.ipynb_checkpoints/ + +# ---- Local data artifacts (DuckDB / SQLite / logs) ---- +*.duckdb +*.duckdb.wal +*.db +*.sqlite +*.sqlite-shm +*.sqlite-wal +*.log + +# ---- IDE / Editor ---- +.idea/ +.vscode/ +*.swp +*.swo +*~ + +# ---- OS ---- +.DS_Store +Thumbs.db +desktop.ini + +# ---- Secrets / env files ---- +.env +.env.local +.env.*.local +*.pem +*.key \ No newline at end of file diff --git a/submission/sample-candidate/__pycache__/conftest.cpython-314-pytest-9.0.2.pyc b/submission/sample-candidate/__pycache__/conftest.cpython-314-pytest-9.0.2.pyc deleted file mode 100644 index 7309004..0000000 Binary files a/submission/sample-candidate/__pycache__/conftest.cpython-314-pytest-9.0.2.pyc and /dev/null differ diff --git a/submission/sample-candidate/pipeline/__pycache__/__init__.cpython-314.pyc b/submission/sample-candidate/pipeline/__pycache__/__init__.cpython-314.pyc deleted file mode 100644 index 1523f32..0000000 Binary files a/submission/sample-candidate/pipeline/__pycache__/__init__.cpython-314.pyc and /dev/null differ diff --git a/submission/sample-candidate/pipeline/__pycache__/cdc.cpython-314.pyc b/submission/sample-candidate/pipeline/__pycache__/cdc.cpython-314.pyc deleted file mode 100644 index a17fa0b..0000000 Binary files a/submission/sample-candidate/pipeline/__pycache__/cdc.cpython-314.pyc and /dev/null differ diff --git a/submission/sample-candidate/pipeline/__pycache__/lake.cpython-314.pyc b/submission/sample-candidate/pipeline/__pycache__/lake.cpython-314.pyc deleted file mode 100644 index 143260f..0000000 Binary files a/submission/sample-candidate/pipeline/__pycache__/lake.cpython-314.pyc and /dev/null differ diff --git a/submission/sample-candidate/pipeline/__pycache__/warehouse.cpython-314.pyc b/submission/sample-candidate/pipeline/__pycache__/warehouse.cpython-314.pyc deleted file mode 100644 index da28d22..0000000 Binary files a/submission/sample-candidate/pipeline/__pycache__/warehouse.cpython-314.pyc and /dev/null differ diff --git a/submission/sample-candidate/source/__pycache__/__init__.cpython-314.pyc b/submission/sample-candidate/source/__pycache__/__init__.cpython-314.pyc deleted file mode 100644 index b3cc59d..0000000 Binary files a/submission/sample-candidate/source/__pycache__/__init__.cpython-314.pyc and /dev/null differ diff --git a/submission/sample-candidate/source/__pycache__/models.cpython-314.pyc b/submission/sample-candidate/source/__pycache__/models.cpython-314.pyc deleted file mode 100644 index 2cd0f91..0000000 Binary files a/submission/sample-candidate/source/__pycache__/models.cpython-314.pyc and /dev/null differ diff --git a/submission/sample-candidate/tests/__pycache__/conftest.cpython-314-pytest-9.0.2.pyc b/submission/sample-candidate/tests/__pycache__/conftest.cpython-314-pytest-9.0.2.pyc deleted file mode 100644 index d04dfd0..0000000 Binary files a/submission/sample-candidate/tests/__pycache__/conftest.cpython-314-pytest-9.0.2.pyc and /dev/null differ diff --git a/submission/sample-candidate/tests/__pycache__/test_catalog.cpython-314-pytest-9.0.2.pyc b/submission/sample-candidate/tests/__pycache__/test_catalog.cpython-314-pytest-9.0.2.pyc deleted file mode 100644 index 5ee79d6..0000000 Binary files a/submission/sample-candidate/tests/__pycache__/test_catalog.cpython-314-pytest-9.0.2.pyc and /dev/null differ diff --git a/submission/sample-candidate/tests/__pycache__/test_cdc.cpython-314-pytest-9.0.2.pyc b/submission/sample-candidate/tests/__pycache__/test_cdc.cpython-314-pytest-9.0.2.pyc deleted file mode 100644 index a287f0a..0000000 Binary files a/submission/sample-candidate/tests/__pycache__/test_cdc.cpython-314-pytest-9.0.2.pyc and /dev/null differ diff --git a/submission/sample-candidate/tests/__pycache__/test_data_quality.cpython-314-pytest-9.0.2.pyc b/submission/sample-candidate/tests/__pycache__/test_data_quality.cpython-314-pytest-9.0.2.pyc deleted file mode 100644 index cbd99cf..0000000 Binary files a/submission/sample-candidate/tests/__pycache__/test_data_quality.cpython-314-pytest-9.0.2.pyc and /dev/null differ diff --git a/submission/sample-candidate/tests/__pycache__/test_schema_contracts.cpython-314-pytest-9.0.2.pyc b/submission/sample-candidate/tests/__pycache__/test_schema_contracts.cpython-314-pytest-9.0.2.pyc deleted file mode 100644 index 76d6304..0000000 Binary files a/submission/sample-candidate/tests/__pycache__/test_schema_contracts.cpython-314-pytest-9.0.2.pyc and /dev/null differ diff --git a/submission/wallet-payment-transfer/ARCHITECTURE.md b/submission/wallet-payment-transfer/ARCHITECTURE.md new file mode 100644 index 0000000..8bef149 --- /dev/null +++ b/submission/wallet-payment-transfer/ARCHITECTURE.md @@ -0,0 +1,304 @@ +# Architecture + +The pipeline is six layers stacked in a single process. Each layer has a +narrow responsibility and a typed contract with the layer above and below +it. The contracts are what make the system reliable under replay, +restart, and schema drift. + +``` + ┌──────────────────────────────────────┐ + │ 6. Catalog (catalog/catalog.json) │ + └──────────────────────────────────────┘ + ▲ publishes + ┌──────────────────┐ │ + │ 5. Quality / │ asserts ─────────►│ wh_* + wh_*_history + │ Validation │ │ + │ (run_data_ │ + │ quality_checks) │ + └──────────────────┘ ▲ merges (idempotent) + │ + ┌────────────────────┐ + │ 4. Warehouse │ current-state + SCD2 + │ (pipeline/warehouse│ state_at(ts), restore + └────────────────────┘ + ▲ replayable from + │ + ┌────────────────────┐ + │ 3. Lake │ append-only event log + │ (pipeline/lake) │ lake_cdc_events + └────────────────────┘ + ▲ writes (sequence-deduped) + │ + ┌──────────────────────────────────────────────────┐ + │ 2. Ingestion / CDC layer │ + │ CDCCapture → Orchestrator (run_tick) │ + │ schema gate (SchemaDetector) — stop-the-line │ + │ per-stage checkpoints (pipeline_checkpoints) │ + └──────────────────────────────────────────────────┘ + ▲ observes via contracts + │ + ┌────────────────────┐ + │ 1. Source │ + │ DDL + indexes + │ + │ CHECK constraints │ + └────────────────────┘ +``` + +--- + +## 1. Source layer + +**File:** `source/models.py`, `source/contracts.py`, `source/seed.py` + +Six tables in dependency order: + +| Table | Strong / Weak | Parent | +|--------------------|---------------|---------------------------------| +| `customers` | strong | — | +| `wallets` | strong | references customers | +| `transfers` | strong | references wallets, pay_methods | +| `payment_methods` | weak | customer | +| `transfer_events` | weak | transfer | +| `ledger_entries` | weak | wallet | + +A table is **weak** when its lifecycle is bound to a parent. A +payment_method has no business meaning if its customer is removed; a +ledger_entry has no meaning without its wallet. Strong entities can be +queried, ranked, and reasoned about on their own. + +Every table carries `created_at` / `updated_at`, primary keys are +non-null VARCHAR identifiers, and enums are enforced via `CHECK` +constraints. Indexes mirror real CDC and lookup patterns: + +- `updated_at` on `customers`, `wallets`, `transfers` — incremental + extraction by timestamp +- foreign-key indexes — referential lookup + cascading delete scans +- `(wallet_id, occurred_at)` on `ledger_entries` — audit-by-wallet +- `status` indexes on `customers`, `transfers` — filtered scans + +**Contracts** (`source/contracts.py`) are a frozen, versioned mirror of +this DDL. The schema detector and the data-quality layer both read from +contracts so a single change in `contracts.py` propagates everywhere. + +--- + +## 2. CDC / Ingestion layer + +**Files:** `pipeline/cdc.py`, `pipeline/orchestrator.py`, +`pipeline/checkpoint.py` + +`CDCCapture` is the in-process stand-in for a Debezium-like log +consumer. Each `CDCRecord` carries: + +| Field | Why | +|----------------|----------------------------------------------------| +| `operation` | `insert` / `update` / `delete` | +| `table` | source table name | +| `primary_key` | natural key value | +| `before` | pre-image (required for update / delete) | +| `after` | post-image (required for insert / update) | +| `captured_at` | wall clock at capture (UTC, defaults to `now`) | +| `sequence` | monotonic offset (Postgres LSN / Kafka offset) | + +The `sequence` is the key to everything downstream: it makes the lake +deduplicate trivially (`sequence` is its primary key) and makes the +warehouse merge idempotent and out-of-order safe (each warehouse row +records the last `_cdc_seq` that touched it). + +`CDCCapture` can optionally hold a `SchemaDetector` and refuse writes +when the source schema is breaking — see `test_capture_with_strict_schema_refuses_writes_on_drop`. + +**`run_tick(conn, capture, *, detector, fail_on_drift=True)`** is the +single orchestration entry point: + +``` +1. bootstrap (idempotent CREATE TABLE IF NOT EXISTS) for lake / wh / checkpoints +2. if detector reports BREAKING drift → return aborted=true, do not move data +3. read records since min(checkpoint_lake, checkpoint_warehouse) from capture +4. write records with sequence > checkpoint_lake to the lake +5. advance checkpoint_lake +6. merge records with sequence > checkpoint_warehouse into the warehouse +7. advance checkpoint_warehouse +``` + +Each step is independently safe to retry. If step 6 fails, step 4-5 may +have committed; the next tick sees `checkpoint_lake > checkpoint_warehouse` +and re-merges from the lake / capture. The warehouse merge silently +no-ops on already-applied sequences. + +**Checkpoints** (`pipeline_checkpoints`) are an ordinary table in the +same DuckDB database as the warehouse. In production, this would be a +Kafka consumer group offset committed inside the same transaction as +the warehouse batch. + +--- + +## 3. Lake layer + +**File:** `pipeline/lake.py` + +A single table: + +```sql +CREATE TABLE lake_cdc_events ( + sequence INTEGER PRIMARY KEY, -- dedup + operation VARCHAR NOT NULL, + table_name VARCHAR NOT NULL, + primary_key VARCHAR NOT NULL, + before_image VARCHAR, -- JSON + after_image VARCHAR, -- JSON + captured_at TIMESTAMP NOT NULL +) +``` + +**Append-only.** `append_to_lake` uses `ON CONFLICT (sequence) DO NOTHING`, +so replay is a no-op. `replay_records(conn, table=None, until_sequence=None)` +returns events in `sequence` order and is what the warehouse would replay +from after disaster recovery. + +The before/after row images are JSON. A custom serializer handles +`Decimal`, `datetime`, and `date` so monetary values round-trip exactly +(`Decimal("75.0000")` stays a `Decimal` after `json.dumps` → +`json.loads`). + +**Production analogue:** an Iceberg/Delta bronze table partitioned by +`(table_name, captured_at::date)`. The PK on `sequence` becomes a Delta +Lake `MERGE` key. + +--- + +## 4. Warehouse layer + +**File:** `pipeline/warehouse.py` + +For each source table `T` we materialize **two** tables: + +### `wh_T` — current state + +The current-state row carries `_cdc_seq` (the sequence of the last +event applied) and `_deleted` (set true on delete; the row is kept so +analytics over deleted entities still works). + +### `wh_T_history` — SCD2 history + +Each version has: + +| Column | Meaning | +|---------------|---------------------------------------------------| +| `_cdc_seq` | sequence that produced this version | +| `_valid_from` | inclusive start of validity (= `captured_at`) | +| `_valid_to` | exclusive end of validity, NULL for current row | +| `_is_current` | redundant flag, indexes well | +| `_op` | `insert` / `update` / `delete` | +| PK | `(business_key, _valid_from)` | + +### Idempotent merge + +```python +existing = SELECT _cdc_seq FROM wh_T WHERE pk = ? +if existing and existing._cdc_seq >= record.sequence: + return # already applied, or out-of-order — no-op +``` + +This single check makes the merge safe under: + +- **duplicate events** (same sequence ≤ existing → no-op) +- **out-of-order arrival** (smaller sequence → no-op; tested in `test_older_event_does_not_clobber_newer_state`) +- **retry after partial failure** (the merge is on a per-row basis, so re-running yields the same final state) + +### Time travel + +```python +state_at(conn, table, ts) -> list[dict] +``` + +scans `wh_T_history` for rows where +`_valid_from <= ts < COALESCE(_valid_to, '9999-12-31')` and +`_op != 'delete'`. The result is "what `SELECT * FROM T` would have +returned at `ts`". + +`restore_current_state_to(conn, table, ts)` truncates `wh_T` and reloads +it from `state_at(table, ts)`. History is **never** modified — restore +is a current-state operation only. + +--- + +## 5. Quality / Validation layer + +**File:** `scripts/run_data_quality_checks.py` + +Rules are **derived** from `source/contracts.py`, so adding a new +NOT NULL column or a new enum domain in `contracts.py` automatically +generates the corresponding warehouse rule. There is no second source +of truth. + +### System rules (auto-generated from contracts) + +- `sys.pk_unique.` — PK has no duplicates +- `sys.not_null.
.` — for every NOT NULL column in the contract +- `sys.enum.
.` — for every column with an enum domain +- `sys.fk.
.` — referential integrity, ignoring `_deleted` + +### Business rules (explicitly defined) + +- `biz.wallet.balance_non_negative` +- `biz.transfer.amount_positive` +- `biz.transfer.no_self_transfer` +- `biz.transfer.settled_at_after_created` +- `biz.transfer.settled_implies_settled_at` +- `biz.transfer.failed_implies_reason` +- `biz.ledger.signed_amount_consistent` +- `biz.ledger.balance_after_non_negative` +- `biz.ledger.reconciles_to_wallet_balance` — `SUM(signed_amount) == balance` per wallet +- `biz.transfer_event.allowed_transition` + +`run_checks` returns a list of `(rule, count_of_failing_rows)` pairs. +The CI script exits non-zero if the list is non-empty. + +`tests/test_data_quality.py` proves both directions: the seeded +warehouse passes all rules **and** each rule actually fires when its +invariant is violated. + +--- + +## 6. Catalog layer + +**File:** `catalog/catalog.json` + +13 datasets — 1 lake event log + 6 warehouse current-state + 6 +warehouse SCD2 history. Each carries: + +- `name`, `layer`, `type` (controlled vocabulary: `lake` / `warehouse`, + `append-only-change-log` / `current-state` / `scd2-history`) +- `description`, `owner`, `consumers` +- `update_cadence`, `sla_class` (`tier-1` / `tier-2` / `tier-3`) +- `tags`, `primary_key`, `schema` or `schema_extension` + +`scripts/validate_catalog.py` enforces: + +- every expected dataset is present +- every required field is populated +- the layer/type pairing is consistent +- `tests/test_catalog.py` adds 14 more assertions including controlled + vocabularies, money-critical SLA tier, and time-travel tagging on + history datasets + +In production this catalog would publish to DataHub / Atlan / Unity +Catalog. The JSON shape is deliberately close to OpenAPI dataset +declarations so the migration is a serializer swap. + +--- + +## Failure modes and how the architecture handles them + +| Failure | Handled by | +|--------------------------------------|-------------------------------------------------------------| +| Duplicate CDC event | `lake.sequence` PK + warehouse `_cdc_seq` guard | +| Out-of-order arrival | warehouse `_cdc_seq` guard rejects lower sequences | +| Retry after partial failure | each tick step is idempotent; checkpoints advance per stage | +| Restart with progress checkpointed | `pipeline_checkpoints` + `min(lake, wh)` floor in `run_tick`| +| Incompatible schema change | `SchemaDetector` BREAKING + orchestrator stops the line | +| Source delete | warehouse soft-deletes (`_deleted=true`); SCD2 closes row | +| Late-arriving update with smaller seq| warehouse merge guard rejects | +| Need to recover historical state | `state_at(ts)` reconstructs from SCD2; `restore_current_state_to` | +| Ledger / wallet drift | `biz.ledger.reconciles_to_wallet_balance` rule fails CI | diff --git a/submission/wallet-payment-transfer/ASSUMPTIONS.md b/submission/wallet-payment-transfer/ASSUMPTIONS.md new file mode 100644 index 0000000..59cabe1 --- /dev/null +++ b/submission/wallet-payment-transfer/ASSUMPTIONS.md @@ -0,0 +1,189 @@ +# Assumptions, Tradeoffs, and Production Analogues + +This is a **single-process simulation** of a CDC lakehouse, written +to be reasoned about rather than deployed. Every shortcut taken below +has a documented production analogue. The goal is to make the +correctness story easy to follow without losing the structural shape +of a real system. + +--- + +## What is simulated and what would change in production + +### CDC capture + +**Simulated:** `pipeline/cdc.py` — `CDCCapture.insert / update / delete` +called explicitly from seed code or tests. Each call appends a +`CDCRecord` with a strictly increasing `sequence` to an in-memory log. + +**Production analogue:** Postgres logical decoding via Debezium → +Kafka. Sequence numbers become Kafka offsets (which, conveniently, +share the same monotonic-by-partition guarantees we rely on). The +`CDCRecord` shape lines up 1-1 with Debezium's envelope (`op`, +`table`, `before`, `after`, `ts_ms`, `lsn`). + +What does **not** change: the `_cdc_seq` guard in the warehouse merge, +the lake's PK on `sequence`, and the orchestrator's bootstrap → +schema-gate → lake → warehouse flow. Those are the stable contracts. + +### Storage + +**Simulated:** DuckDB in-memory (or a single file) for source, lake, +warehouse, and checkpoint store. + +**Production analogue:** +- **Source** lives in Postgres / MySQL — DuckDB exposes the same + `information_schema.columns` introspection used by `SchemaDetector`, + so the detector is portable. +- **Lake** is an Iceberg / Delta / Hudi table on object storage, + partitioned by `(table_name, captured_at::date)`. The lake's + `sequence` PK becomes a Delta `MERGE` key. +- **Warehouse** is Snowflake / BigQuery / Redshift. The current-state + + SCD2 pattern translates verbatim — `MERGE INTO ... USING ... ON + pk WHEN MATCHED AND target._cdc_seq < source.sequence THEN UPDATE`. +- **Checkpoints** live in a small operational store, ideally + committed inside the same warehouse transaction as the merge. + +### Schema detection + +**Simulated:** `SchemaDetector` introspects the live source DDL via +`information_schema.columns` and `duckdb_constraints()` and compares +against `source/contracts.py`. + +**Production analogue:** +- Confluent / AWS Glue Schema Registry as the system of record for + Avro / Protobuf schemas; Debezium publishes new schemas there +- A simple comparator (the same logic as our `SchemaDetector`) runs + on schema-changed events, emits a Slack / PagerDuty alert, and + flips a feature flag the orchestrator polls + +The detector's rule set (column_dropped, type_changed, +nullability_tightened, primary_key_changed, non_nullable_column_added +=> BREAKING; column_added, enum_value_added => WARN) is identical to +what the production analogue would enforce. + +--- + +## Simplifying decisions + +### Single-process / no orchestrator framework + +The orchestrator is `pipeline/orchestrator.py:run_tick`, called +directly from scripts and tests. There is no Airflow / Dagster +because that would obscure the data-flow. The full tick is one +function call (~70 lines) that you can read top-to-bottom. In +production this becomes a Dagster `@op` or an Airflow `PythonOperator`. + +### JSON-encoded before/after images instead of Avro + +`json.dumps` with a custom `default=` for `Decimal`, `date`, and +`datetime`. JSON is human-readable and lossless for our types and +trivially queryable in DuckDB. Production would use Avro/Protobuf +because of schema evolution support; the contract surface (`before`, +`after` carry the source row image) does not change. + +### Soft deletes only — never hard deletes + +`wh_
` rows are flagged with `_deleted=true` instead of being +removed. This: + +- preserves the row for analytics on deleted entities +- keeps FK integrity in `wh_
` even after a parent is removed +- mirrors what most warehouses already do — a hard delete is rare in + analytical models + +The data-quality FK rule explicitly ignores `_deleted=true` rows on +both sides (see `tests/test_data_quality.py::test_deleted_rows_do_not_trigger_validation_failures`). + +### `captured_at` is the SCD2 valid_from + +We use the CDC capture timestamp as the SCD2 boundary, not a separate +"as_of" column. In production with millisecond ordering issues we +would use `(captured_at, sequence)` as a composite key, but `sequence` +alone already provides total ordering, so the simpler boundary is +fine for our scope. + +### No transaction boundaries across stages + +In DuckDB single-process we could `BEGIN ... COMMIT` across the lake +write and warehouse merge — but that hides the realistic case where +lake and warehouse are different systems. We instead lean on the +**idempotent merge** + **per-stage checkpoints** to make a partial +failure recoverable without distributed transactions. + +### Sequence is generated in `CDCCapture` + +In production the sequence is the Postgres LSN (or Kafka offset). We +issue our own monotonic counter in `CDCCapture.latest_sequence + 1` +because there is no real WAL to read. The shape is identical. + +### One DuckDB connection per test + +Tests use `:memory:` connections, so they're hermetic. There is no +shared state between tests. The `seeded` fixture builds the entire +seed in-process per test, which is < 100ms. + +### No partitioning on the lake table + +Production would partition by `(table_name, captured_at::date)`. We +skipped this because DuckDB plans queries fine without it on our scale +and partitioning would obscure the model. The catalog entry already +documents the production partition. + +--- + +## Things explicitly NOT solved + +- **Automatic migration of breaking schema changes.** The assignment + states this is out of scope. We *detect* and *stop* — we do not + attempt to back-fill or reshape. +- **Deduplication on the source side** — `(table, primary_key, sequence)` + is unique by construction in `CDCCapture`. If a real source emits + duplicate LSNs (impossible in Postgres, possible only in misconfigured + pipelines), the lake's PK on `sequence` catches it; the warehouse merge + catches it as a same-sequence no-op. +- **Multi-tenant catalogs** — one domain only. +- **Lineage / column-level provenance** — out of scope; production would + use OpenLineage or DataHub lineage. +- **Authorization on warehouse tables** — out of scope. + +--- + +## What we did NOT compromise on + +- **Idempotency** — every lake write is dedup'd by `sequence`, every + warehouse merge is dedup'd by `_cdc_seq`. Replay is provably safe. +- **Out-of-order safety** — the `_cdc_seq` guard handles late-arriving + smaller sequences. Tested in `test_older_event_does_not_clobber_newer_state`. +- **Stop-the-line** — `run_tick` aborts before any data movement on + breaking drift. `CDCCapture` can also refuse writes when configured + with a strict detector. Both modes are tested. +- **Validation parity** — the data-quality rules are *generated* from + contracts, so every NOT NULL column / enum domain in the source + appears as a downstream rule by construction. +- **Time travel** — SCD2 is real, not approximated. `state_at(ts)` + goes through the same path the warehouse merge writes, so any new + CDC event automatically becomes time-travelable. +- **Testing across categories** — 9 test modules, ~80 tests covering + source constraints, CDC, lake, warehouse, schema, data quality, + catalog, time-travel, and end-to-end. + +--- + +## Tradeoffs we'd revisit with more time + +1. **Replace the in-memory `CDCCapture` with a real Debezium stub** — + the current shape is faithful but lacks the long tail of edge cases + (transactional grouping, snapshot vs streaming mode). +2. **Add column-level lineage to the catalog** — the 13 datasets all + know their primary keys but not where each column flows from. +3. **Add a freshness rule to data quality** — currently we check + correctness, not staleness. Production needs both. +4. **Wire a tiny Streamlit UI on top of `state_at`** — would make + time-travel demonstrable to non-engineers. +5. **Containerize** — a `Dockerfile` + `docker-compose.yml` with + Postgres + Kafka + the orchestrator would be the natural next step + to demonstrate the production analogue concretely. + +None of these change the contracts; they're all add-ons to the same +core. diff --git a/submission/wallet-payment-transfer/Makefile b/submission/wallet-payment-transfer/Makefile new file mode 100644 index 0000000..ee25aec --- /dev/null +++ b/submission/wallet-payment-transfer/Makefile @@ -0,0 +1,46 @@ +.PHONY: help install test lint demo demo-time-travel demo-schema-change \ + check-schema check-quality check-catalog ci all + +PY ?= python + +help: + @echo "Targets:" + @echo " install - install Python dependencies" + @echo " test - run all pytest tests" + @echo " check-schema - run schema-contract gate" + @echo " check-quality - run data-quality gate" + @echo " check-catalog - run catalog validation gate" + @echo " demo - run end-to-end pipeline demo" + @echo " demo-time-travel - run point-in-time reconstruction demo" + @echo " demo-schema-change - run schema drift / stop-the-line demo" + @echo " ci - run all CI gates (test + 3 checks)" + @echo " all - install, then run ci" + +install: + $(PY) -m pip install -r requirements.txt + +test: + $(PY) -m pytest -q + +check-schema: + $(PY) -m scripts.check_schema_contracts + +check-quality: + $(PY) -m scripts.run_data_quality_checks + +check-catalog: + $(PY) -m scripts.validate_catalog + +demo: + $(PY) -m scripts.run_pipeline + +demo-time-travel: + $(PY) -m scripts.demo_time_travel + +demo-schema-change: + $(PY) -m scripts.demo_schema_change + +ci: test check-schema check-quality check-catalog + @echo "All CI gates passed." + +all: install ci diff --git a/submission/wallet-payment-transfer/PR_DESCRIPTION.md b/submission/wallet-payment-transfer/PR_DESCRIPTION.md new file mode 100644 index 0000000..d47e88f --- /dev/null +++ b/submission/wallet-payment-transfer/PR_DESCRIPTION.md @@ -0,0 +1,326 @@ +# Pull Request — CDC Lakehouse Reliability (Wallet / Payments / Transfers) + +## Summary +End-to-end CDC pipeline that mirrors a 6-table wallet/payments/transfers +source system into a **lake** (immutable change history) and a **warehouse** +(current-state + SCD2 history). It detects schema drift, stops the line on +incompatible changes, supports time-travel reconstruction and operational +restore, and enforces source-to-warehouse validation parity. Implemented in +Python + DuckDB and runnable in a single process. + +Layered into six modules with narrow contracts: + +``` +source → cdc capture + orchestrator → lake → warehouse → quality → catalog +``` + +~1,400 LOC of pipeline + ~1,300 LOC of tests across **9 test modules** +covering modeling, CDC correctness, schema-change safety, warehouse +correctness, time travel, catalog metadata, data quality, and full end-to-end +scenarios. + +See `ARCHITECTURE.md` for the layered design walk-through and +`ASSUMPTIONS.md` for tradeoffs and production analogues. + +--- + +## Source Schema Design + +- **Domain chosen:** wallet / payments / transfers +- **Strong entities:** + - `customers` — registered platform users + - `wallets` — balance accounts; one customer holds many (per-currency) + - `transfers` — money movements between wallets; have business identity +- **Weak entities:** + - `payment_methods` — saved card / bank / upi tied to a customer + - `transfer_events` — state transitions for a transfer + - `ledger_entries` — append-only credit/debit on a wallet +- **Keys, relationships, indexes:** + - All primary keys are non-null `VARCHAR` natural identifiers + - FKs: `payment_methods → customers`, `wallets → customers`, + `transfers → wallets ×2 + payment_methods`, + `transfer_events → transfers`, `ledger_entries → wallets + transfers` + - 12 indexes covering CDC patterns (`updated_at`), referential lookups + (`customer_id`, `transfer_id`), filtered scans (`status`), and + audit-by-wallet (`(wallet_id, occurred_at)`) +- **Source validation rules** (enforced via `CHECK` and `NOT NULL`): + - `wallet.balance >= 0` + - `transfer.amount > 0` + - `transfer.from_wallet_id != transfer.to_wallet_id` + - `transfer.settled_at IS NULL OR settled_at >= created_at` + - `ledger_entries.signed_amount` consistent with `direction` and `amount` + - status enums: `customers.status`, `payment_methods.method_type` / + `.status`, `wallets.currency` / `.status`, `transfers.status` / + `.currency`, `transfer_events.to_status`, `ledger_entries.direction` + +Schema is mirrored in versioned, frozen contracts at `source/contracts.py`, +which are the single source of truth for the schema detector and the +data-quality rules. + +--- + +## CDC Strategy + +- **How changes are captured:** `CDCCapture` wraps every `insert`, + `update`, and `delete` call into a `CDCRecord` with operation, table, + primary_key, before-image, after-image, monotonic `sequence`, and + `captured_at`. The sequence is the LSN/Kafka-offset analogue and the + primary correctness lever. +- **How inserts, updates, deletes are handled:** + - **Insert** — adds a new current-state row with `_cdc_seq=record.sequence` + and opens an SCD2 history row with `_is_current=true`, + `_valid_from=captured_at`, `_op='insert'`. + - **Update** — closes the open SCD2 row (`_valid_to=captured_at`, + `_is_current=false`) and opens a new one with the post-image; updates + the current-state row in place. + - **Delete** — closes the open SCD2 row and appends a closed + `_op='delete'` row (with the pre-image); flips current-state's + `_deleted=true` (soft delete preserves analytics over deleted rows). +- **How replay/restart works:** + - Per-stage checkpoint (`pipeline_checkpoints`): one row each for + `lake` and `warehouse`. `run_tick` reads from + `min(checkpoint_lake, checkpoint_warehouse)`, so a tick that wrote + the lake but failed before the warehouse merge replays cleanly. + - The lake's `sequence` PK silently drops duplicate appends. + - The warehouse merge applies an event only when + `record.sequence > existing._cdc_seq`, making it idempotent and + out-of-order safe. + - `set_checkpoint` refuses to move backwards — bugs surface loudly. +- **How duplicates are handled:** see above. Tested in + `test_replay_does_not_double_apply`, + `test_orchestrator_run_tick_is_idempotent`, and + `test_duplicate_sequence_is_deduped`. + +--- + +## Lake and Warehouse Modeling + +- **How the lake captures every change:** single append-only table + `lake_cdc_events(sequence PK, operation, table_name, primary_key, + before_image JSON, after_image JSON, captured_at)`. One row per CDC + event, no rewrite, no overwrite, no aggregation. `replay_records()` + returns events in sequence order and is the warehouse's recovery feed. +- **How the warehouse maintains latest snapshot:** for each source + table `T`: + - `wh_T` — current-state with `_cdc_seq` (last applied sequence) and + `_deleted` (soft delete). Same column shape as the source + + `_cdc_seq` + `_deleted`. + - `wh_T_history` — SCD2 with `(business_key, _valid_from)` PK and + `_valid_to`, `_is_current`, `_op` columns. Every revision lands + here; `_is_current=true` invariant: at most one row per PK. +- **How time travel / restore is supported:** + - `state_at(conn, table, ts)` reconstructs `T` at any timestamp from + `wh_T_history` using `_valid_from <= ts < _valid_to`. Excludes + deleted rows automatically. Tested in `test_time_travel.py` against + multiple inserted revisions. + - `restore_current_state_to(conn, table, ts)` overwrites `wh_T` with + the snapshot from `state_at(table, ts)` and **does not modify + history** — restore is reversible, history is the system of record. + - After restore, `_cdc_seq` resets to 0 so subsequent CDC events + apply cleanly. Verified by `test_restored_state_accepts_new_cdc_records`. + +--- + +## Schema Change Safety + +- **How incompatible changes are detected:** `SchemaDetector` introspects + the live source via `information_schema.columns` and + `duckdb_constraints()` and compares against `source/contracts.py`. + Eight rules: + - **BREAKING:** `missing_table`, `column_dropped`, `type_changed`, + `nullability_tightened`, `non_nullable_column_added`, + `primary_key_changed` + - **WARN:** `column_added` (additive), `enum_value_added` (drift + surface, not necessarily breaking) +- **How ingestion is stopped:** at *two* levels: + - **Orchestrator gate** — `run_tick(..., fail_on_drift=True)` runs + `detector.check_all()` *before* reading any new events. On a + breaking drift it returns `{aborted: true, reason: + "schema_contract_violation", drift_summary: ...}` and writes + nothing to the lake or warehouse. Tested in + `test_run_tick_aborts_on_breaking_drift`. + - **Capture gate** — `CDCCapture(detector, require_compatible_schema=True)` + raises `SchemaContractViolation` on every write while a breaking + drift is present. The capture log stays empty. Tested in + `test_capture_with_strict_schema_refuses_writes_on_drop` and + `test_capture_refuses_writes_when_strict`. +- **How warnings/failures are surfaced:** + - `scripts/check_schema_contracts.py` is the CI entry; it exits + non-zero with a printed drift summary on any breaking finding, + suitable for the `schema-contract-check` GitHub status check. + - `run_tick`'s aborted dict includes a `drift_summary` for log + aggregation / Slack alerts. + - WARN rules are reported via `report.warnings` and never abort — + they are operational signals, not failures. + +--- + +## Validation Parity + +- **Which source system validations were mirrored downstream:** + - **System rules** — auto-generated from `contracts.py`: + - `sys.pk_unique.
` for every PK + - `sys.not_null.
.` for every NOT NULL column + - `sys.enum.
.` for every enum-domained column + - `sys.fk.
.` for every FK (ignoring `_deleted=true` rows) + - **Business rules** — explicit: + - `biz.wallet.balance_non_negative` + - `biz.transfer.amount_positive` + - `biz.transfer.no_self_transfer` + - `biz.transfer.settled_at_after_created` + - `biz.transfer.settled_implies_settled_at` + - `biz.transfer.failed_implies_reason` + - `biz.ledger.signed_amount_consistent` + - `biz.ledger.balance_after_non_negative` + - `biz.ledger.reconciles_to_wallet_balance` (monetary integrity: + `SUM(signed_amount) per wallet == wh_wallets.balance`) + - `biz.transfer_event.allowed_transition` (state machine validity) +- **How failures are checked and reported:** + - `scripts/run_data_quality_checks.py:run_checks(conn)` returns + `[(rule, count_of_failing_rows), ...]`. The CI script exits + non-zero on any failure and prints the rule_id + offending count. + - `tests/test_data_quality.py` proves both directions: the seeded + warehouse passes all rules **and** every individual rule fires when + its invariant is violated (tested for PK uniqueness, enum, FK, + negative balance, settled_at backward, ledger consistency, ledger + reconciliation, status transition). + - A parity test (`test_every_contract_not_null_has_a_warehouse_rule`) + blocks a contract change without a corresponding rule. + +--- + +## Catalog Exposure + +13 datasets in `catalog/catalog.json` covering every output the pipeline +produces: + +- 1 lake dataset: `lake_cdc_events` +- 6 warehouse current-state: `wh_customers`, `wh_payment_methods`, + `wh_wallets`, `wh_transfers`, `wh_transfer_events`, `wh_ledger_entries` +- 6 warehouse SCD2 history: each of the above + `_history` + +Each dataset publishes: + +- `name`, `layer` (`lake` / `warehouse`), `type` + (`append-only-change-log` / `current-state` / `scd2-history`) +- `description`, `owner` (`data-platform`), `consumers` (per-dataset list: + finance / treasury / fraud / audit / analytics / product / support) +- `update_cadence`, `sla_class` (money-critical datasets are `tier-1`) +- `tags` for governance (`monetary-integrity`, `pii`, `time-travel`, etc.) +- `primary_key` and `schema` / `schema_extension` + +Discovery / access in this scope: consumers `cat catalog/catalog.json` +or `python -m scripts.validate_catalog` to verify it matches the +pipeline outputs. Production analogue is DataHub / Atlan / Unity +Catalog — the JSON shape is intentionally close to those tools' +dataset envelopes so the migration is a serializer swap. + +`tests/test_catalog.py` enforces: +- catalog file exists and is valid JSON +- every expected dataset is declared (no missing, no extras) +- every dataset has the required descriptive fields +- controlled vocabularies for `layer`, `type`, `sla_class` +- consistency: lake → `append-only-change-log`, warehouse non-history + → `current-state`, warehouse `_history` → `scd2-history` +- money-critical datasets are tier-1 +- SCD2 history datasets are tagged for time-travel where they back + balance lookups + +--- + +## Validation + +Local validation commands run in order: + +```bash +cd submission/wallet-payment-transfer +pip install -r requirements.txt + +# 1. Schema contract gate (BREAKING drifts → exit 1) +python -m scripts.check_schema_contracts + +# 2. Data quality parity gate +python -m scripts.run_data_quality_checks + +# 3. Catalog metadata gate +python -m scripts.validate_catalog + +# 4. Full test suite (~80 tests, < 10s) +pytest -q + +# 5. End-to-end pipeline demo +python -m scripts.run_pipeline + +# 6. Time travel demo (state_at at multiple timestamps) +python -m scripts.demo_time_travel + +# 7. Schema-change / stop-the-line demo +python -m scripts.demo_schema_change +``` + +All five CI gates exit non-zero on failure and are mapped 1-1 to the +checks listed in `github_pr_checks_data.md`. + +--- + +## Known Limitations / Next Steps + +- **Replace `CDCCapture` with a Debezium stub** — the current shape is + Debezium-faithful (envelope columns, before/after, sequence == LSN) + but does not exercise transactional grouping or snapshot-vs-streaming + modes. With a stub we could test "snapshot then switch to streaming" + explicitly. +- **Column-level lineage in the catalog** — datasets know their PKs but + not which warehouse columns derive from which source columns. An + OpenLineage emitter is the natural addition. +- **Freshness rules** — current data-quality covers correctness, not + staleness. A `dq.freshness.
` rule comparing `MAX(_cdc_seq)` + against expected cadence is a one-day add. +- **Containerization** — a `Dockerfile` + `docker-compose.yml` running + Postgres + Kafka + this orchestrator would make the production + analogue concrete. The contracts in `pipeline/cdc.py` and + `pipeline/schema_detector.py` are designed to swap targets. +- **Automatic non-breaking migration** — for `column_added` (additive) + drift we currently emit a WARN; we could instead auto-extend the + warehouse table inside the same tick. Out of scope by assignment. + +None of these change the architecture; they are additive. + +--- + +## Responsible AI Usage + +- **Did you use AI tools?** Yes — Claude Code as a pair-programming + agent. +- **Where did they help?** + - Drafting boilerplate (DDL strings, frozen-dataclass contracts, JSON + catalog entries) and the per-rule data-quality SQL. + - Surfacing cross-cutting concerns (the `_cdc_seq` guard handles both + duplicate replay AND out-of-order arrival; `restore_current_state_to` + needs to reset `_cdc_seq=0` so subsequent CDC applies cleanly). + - Generating test scaffolds and parametrizations. +- **What did I personally verify or correct?** + - The strong/weak entity classification — I made the calls based on + lifecycle ownership, not the LLM's first guess. + - The seed reconciliation — initial balances did not match + `SUM(signed_amount)`; I worked out the funding entries explicitly so + the monetary-integrity rule passes. + - The `_cdc_seq` guard inequality — I changed it to **strictly greater** + so the same-sequence replay is correctly a no-op (otherwise replay + would re-open SCD2 rows). + - The orchestrator's drift gate placement — checked **before** reading + any events and **before** advancing checkpoints, so partial state is + impossible on a breaking drift. + - Refactored `CDCCapture` to expose `captured_at` as a public + optional parameter rather than letting the time-travel demo poke + `_record(...)` directly. + - Authored every test by walking through the failure mode I wanted + to prove (Red), confirming the test failed against an unfixed + implementation, then verifying it passed (Blue → Green). + - Structured the catalog tests around controlled vocabularies and + money-critical SLA rules, not just "field exists" assertions. + +The architecture decisions, contract shape, severity classifications, +and the layered file structure are all my own. The code is reviewed +end-to-end and every test asserts a behavior I considered worth +proving. diff --git a/submission/wallet-payment-transfer/README.md b/submission/wallet-payment-transfer/README.md new file mode 100644 index 0000000..a42fca0 --- /dev/null +++ b/submission/wallet-payment-transfer/README.md @@ -0,0 +1,146 @@ +# Wallet / Payments / Transfers — CDC Lakehouse + +A minimal, end-to-end CDC pipeline that mirrors a wallet/payments/transfers +source system into a **lake** (immutable change history) and a **warehouse** +(current-state + SCD2 history). It detects schema drift, stops the line on +incompatible changes, and runs idempotently under replay and restart. + +It is built on **Python + DuckDB** so the entire pipeline — source, +ingestion, lake, warehouse, schema detection, validation, and catalog — +runs in a single process with no external services. + +--- + +## Quick start + +```bash +cd submission/wallet-payment-transfer + +# 1. install deps +pip install -r requirements.txt + +# 2. run all tests (CDC, lake, warehouse, schema, time-travel, e2e, …) +pytest -q + +# 3. run the end-to-end pipeline demo +python -m scripts.run_pipeline + +# 4. run the time-travel demo (SCD2 reconstruction at multiple timestamps) +python -m scripts.demo_time_travel + +# 5. run the schema-change / stop-the-line demo +python -m scripts.demo_schema_change +``` + +CI gates (see `github_pr_checks_data.md`): + +```bash +python -m scripts.check_schema_contracts # gate 4: schema-contract-check +python -m scripts.run_data_quality_checks # gate 5: data-quality-check +python -m scripts.validate_catalog # gate 6: catalog-check +pytest -q # gate 2: test-pipeline +``` + +All five exit non-zero on failure. + +--- + +## What's in the box + +``` +submission/wallet-payment-transfer/ +├── source/ # source layer +│ ├── models.py # DDL: 6 tables, indexes, CHECK constraints +│ ├── contracts.py # versioned schema contracts (frozen dataclasses) +│ └── seed.py # canonical seed (3 customers → 5 ledger entries) +│ +├── pipeline/ # CDC + lake + warehouse + schema detection +│ ├── cdc.py # CDCRecord + CDCCapture (with strict-schema mode) +│ ├── lake.py # append-only change log (lake_cdc_events) +│ ├── warehouse.py # current-state + SCD2 + state_at + restore +│ ├── schema_detector.py # 8 drift rules with BREAKING/WARN severities +│ ├── checkpoint.py # per-stage sequence checkpoint store +│ └── orchestrator.py # run_tick() — bootstrap, drift gate, lake → wh +│ +├── catalog/ +│ └── catalog.json # 13 datasets (1 lake + 6 wh_* + 6 wh_*_history) +│ +├── scripts/ # CI entry points + demos +│ ├── check_schema_contracts.py +│ ├── run_data_quality_checks.py +│ ├── validate_catalog.py +│ ├── run_pipeline.py +│ ├── demo_time_travel.py +│ └── demo_schema_change.py +│ +├── tests/ # 9 test modules, ~80 tests across all categories +│ ├── test_source_schema.py +│ ├── test_cdc.py +│ ├── test_lake.py +│ ├── test_warehouse.py +│ ├── test_schema_contracts.py +│ ├── test_data_quality.py +│ ├── test_catalog.py +│ ├── test_time_travel.py +│ └── test_e2e.py +│ +├── ARCHITECTURE.md # detailed design, data flow, layer contracts +├── ASSUMPTIONS.md # tradeoffs, simplifications, production analogues +├── PR_DESCRIPTION.md # populated PR template +└── README.md # this file +``` + +--- + +## Domain (very short) + +Customers register, link payment methods, hold wallets, and move money +through transfers. Every balance change is mirrored in an immutable +ledger. Strong entities own lifecycle: `customers`, `wallets`, `transfers`. +Weak entities depend on a parent: `payment_methods` (← customer), +`transfer_events` (← transfer), `ledger_entries` (← wallet). + +Important invariants (enforced at source AND in the warehouse via +parity rules): + +- `wallet.balance >= 0`, `transfer.amount > 0` +- `transfer.from_wallet_id != transfer.to_wallet_id` +- `transfer.settled_at >= transfer.created_at` (when settled) +- `SUM(ledger_entries.signed_amount) per wallet == wallet.balance` +- transfer status follows a documented state machine + +See `ARCHITECTURE.md` for the full model. + +--- + +## Reading order for reviewers + +If you have ten minutes, read these in order: + +1. `ARCHITECTURE.md` — single-page overview of the data flow +2. `pipeline/cdc.py` — CDC record contract + sequence semantics +3. `pipeline/warehouse.py` — idempotent merge + SCD2 + time travel +4. `pipeline/schema_detector.py` — the 8 drift rules +5. `tests/test_e2e.py` — the full lifecycle in one file +6. `PR_DESCRIPTION.md` — the deliverable narrative + +The whole pipeline is ~1,400 LOC and the tests are ~1,300 LOC. + +--- + +## Production analogue + +This project is intentionally a single-process DuckDB simulation. The +production analogue is straightforward: + +| In this project | Production stand-in | +|------------------------------|--------------------------------------------------------------| +| `CDCCapture` + `sequence` | Postgres logical decoding / Debezium → Kafka topic | +| `lake_cdc_events` | Kafka + S3/GCS Iceberg/Delta bronze table partitioned by day | +| `wh_
` + `_
_history` | Snowflake / BigQuery / Redshift gold tables | +| `pipeline_checkpoints` | Kafka consumer group offset committed with the warehouse merge | +| `SchemaDetector` | Schema registry + dbt `source.freshness` / Great Expectations | +| `catalog/catalog.json` | Atlan / DataHub / Unity Catalog | +| `bootstrap()` + `run_tick()` | Airflow / Dagster sensor + task | + +`ASSUMPTIONS.md` covers each tradeoff explicitly. diff --git a/submission/wallet-payment-transfer/catalog/catalog.json b/submission/wallet-payment-transfer/catalog/catalog.json new file mode 100644 index 0000000..29f649b --- /dev/null +++ b/submission/wallet-payment-transfer/catalog/catalog.json @@ -0,0 +1,288 @@ +{ + "catalog_version": 1, + "domain": "wallet-payments-transfers", + "owner": "data-platform", + "datasets": [ + { + "name": "lake_cdc_events", + "layer": "lake", + "type": "append-only-change-log", + "description": "Append-only log of every CDC event captured from the wallet/payments source system. Preserves the full change history with before/after row images so the warehouse can be deterministically rebuilt and any prior state can be reconstructed.", + "owner": "data-platform", + "consumers": ["data-platform", "audit", "analytics-engineering"], + "update_cadence": "real-time", + "sla_class": "tier-1", + "tags": ["cdc", "raw", "audit", "replayable"], + "primary_key": ["sequence"], + "partitioning": "production: partitioned by table_name + captured_at::date", + "retention_policy": "infinite (audit) — production: 7 years cold storage", + "schema": { + "sequence": {"type": "INTEGER", "nullable": false, "description": "monotonic capture offset (Postgres LSN / Kafka offset analogue)"}, + "operation": {"type": "VARCHAR", "nullable": false, "description": "insert | update | delete"}, + "table_name": {"type": "VARCHAR", "nullable": false, "description": "name of the source table"}, + "primary_key": {"type": "VARCHAR", "nullable": false, "description": "primary key value of the source row"}, + "before_image": {"type": "VARCHAR", "nullable": true, "description": "JSON of the row before the change; null on inserts"}, + "after_image": {"type": "VARCHAR", "nullable": true, "description": "JSON of the row after the change; null on deletes"}, + "captured_at": {"type": "TIMESTAMP", "nullable": false, "description": "UTC timestamp the change was captured"} + } + }, + { + "name": "wh_customers", + "layer": "warehouse", + "type": "current-state", + "description": "Latest known state of every customer. Soft-deleted rows are flagged with _deleted=true and retained for analytics over deleted accounts. Sourced via CDC merge from the source customers table.", + "owner": "data-platform", + "consumers": ["analytics", "product", "finance", "support"], + "update_cadence": "near-real-time", + "sla_class": "tier-1", + "tags": ["customer", "current-state", "scd1"], + "primary_key": ["customer_id"], + "schema": { + "customer_id": "VARCHAR — primary key, source identifier", + "full_name": "VARCHAR", + "email": "VARCHAR", + "country_code": "VARCHAR — ISO-3166 alpha-2", + "status": "VARCHAR — active | suspended | closed", + "registration_date": "DATE", + "created_at": "TIMESTAMP", + "updated_at": "TIMESTAMP", + "_cdc_seq": "INTEGER — sequence of last CDC event applied", + "_deleted": "BOOLEAN — true when the source row was deleted" + } + }, + { + "name": "wh_payment_methods", + "layer": "warehouse", + "type": "current-state", + "description": "Latest known state of every saved payment method (card / bank / upi). Weak entity tied to a customer; if the customer is removed in the source, the method is marked _deleted=true here.", + "owner": "data-platform", + "consumers": ["analytics", "fraud", "finance"], + "update_cadence": "near-real-time", + "sla_class": "tier-1", + "tags": ["payment-method", "current-state", "scd1"], + "primary_key": ["payment_method_id"], + "schema": { + "payment_method_id": "VARCHAR — primary key", + "customer_id": "VARCHAR — FK to wh_customers", + "method_type": "VARCHAR — card | bank | upi", + "masked_identifier": "VARCHAR", + "is_default": "BOOLEAN", + "status": "VARCHAR — active | expired | revoked", + "created_at": "TIMESTAMP", + "updated_at": "TIMESTAMP", + "_cdc_seq": "INTEGER", + "_deleted": "BOOLEAN" + } + }, + { + "name": "wh_wallets", + "layer": "warehouse", + "type": "current-state", + "description": "Latest known state of every wallet. Balance is the authoritative current value from the source; non-negative balance is enforced both at the source and as a downstream data-quality assertion.", + "owner": "data-platform", + "consumers": ["analytics", "finance", "treasury"], + "update_cadence": "near-real-time", + "sla_class": "tier-1", + "tags": ["wallet", "balance", "current-state", "scd1"], + "primary_key": ["wallet_id"], + "schema": { + "wallet_id": "VARCHAR — primary key", + "customer_id": "VARCHAR — FK to wh_customers", + "currency": "VARCHAR — ISO-4217", + "balance": "DECIMAL(18,4) — always >= 0", + "status": "VARCHAR — active | frozen | closed", + "created_at": "TIMESTAMP", + "updated_at": "TIMESTAMP", + "_cdc_seq": "INTEGER", + "_deleted": "BOOLEAN" + } + }, + { + "name": "wh_transfers", + "layer": "warehouse", + "type": "current-state", + "description": "Latest known state of every transfer between two wallets. Status reflects the most recent transition; settled_at is non-null only for settled transfers.", + "owner": "data-platform", + "consumers": ["analytics", "finance", "fraud", "treasury"], + "update_cadence": "near-real-time", + "sla_class": "tier-1", + "tags": ["transfer", "current-state", "scd1"], + "primary_key": ["transfer_id"], + "schema": { + "transfer_id": "VARCHAR — primary key", + "from_wallet_id": "VARCHAR — FK to wh_wallets, source", + "to_wallet_id": "VARCHAR — FK to wh_wallets, destination", + "payment_method_id": "VARCHAR — FK to wh_payment_methods, nullable", + "amount": "DECIMAL(18,4) — always > 0", + "currency": "VARCHAR — ISO-4217", + "status": "VARCHAR — pending | settled | failed | reversed", + "reference": "VARCHAR — nullable external memo", + "failure_reason": "VARCHAR — nullable", + "created_at": "TIMESTAMP", + "settled_at": "TIMESTAMP — null until settled", + "_cdc_seq": "INTEGER", + "_deleted": "BOOLEAN" + } + }, + { + "name": "wh_transfer_events", + "layer": "warehouse", + "type": "current-state", + "description": "Audit trail of state transitions for transfers. Weak entity; one transfer typically has 2-3 events.", + "owner": "data-platform", + "consumers": ["analytics", "fraud", "support"], + "update_cadence": "near-real-time", + "sla_class": "tier-2", + "tags": ["transfer", "audit", "current-state"], + "primary_key": ["transfer_event_id"], + "schema": { + "transfer_event_id": "VARCHAR — primary key", + "transfer_id": "VARCHAR — FK to wh_transfers", + "from_status": "VARCHAR — nullable for the first event", + "to_status": "VARCHAR — pending | settled | failed | reversed", + "note": "VARCHAR — nullable", + "occurred_at": "TIMESTAMP", + "_cdc_seq": "INTEGER", + "_deleted": "BOOLEAN" + } + }, + { + "name": "wh_ledger_entries", + "layer": "warehouse", + "type": "current-state", + "description": "Append-only ledger entries for every credit/debit on a wallet. SUM(signed_amount) per wallet must equal wh_wallets.balance — checked by the data-quality layer.", + "owner": "data-platform", + "consumers": ["analytics", "finance", "treasury", "audit"], + "update_cadence": "near-real-time", + "sla_class": "tier-1", + "tags": ["ledger", "audit", "current-state", "monetary-integrity"], + "primary_key": ["ledger_entry_id"], + "schema": { + "ledger_entry_id": "VARCHAR — primary key", + "wallet_id": "VARCHAR — FK to wh_wallets", + "transfer_id": "VARCHAR — FK to wh_transfers, nullable for funding entries", + "direction": "VARCHAR — credit | debit", + "amount": "DECIMAL(18,4) — always > 0", + "signed_amount": "DECIMAL(18,4) — +amount for credit, -amount for debit", + "balance_after": "DECIMAL(18,4) — non-negative wallet balance after the entry", + "occurred_at": "TIMESTAMP", + "_cdc_seq": "INTEGER", + "_deleted": "BOOLEAN" + } + }, + { + "name": "wh_customers_history", + "layer": "warehouse", + "type": "scd2-history", + "description": "SCD2 history for customers. Every revision carries _valid_from / _valid_to / _is_current / _op so a point-in-time customer snapshot can be reconstructed for restore, time travel, or audit.", + "owner": "data-platform", + "consumers": ["audit", "finance", "data-platform"], + "update_cadence": "near-real-time", + "sla_class": "tier-1", + "tags": ["customer", "scd2", "time-travel"], + "primary_key": ["customer_id", "_valid_from"], + "schema_extension": { + "_cdc_seq": "INTEGER", + "_valid_from": "TIMESTAMP — start of this version's validity", + "_valid_to": "TIMESTAMP — end of validity, NULL for current", + "_is_current": "BOOLEAN", + "_op": "VARCHAR — insert | update | delete" + } + }, + { + "name": "wh_payment_methods_history", + "layer": "warehouse", + "type": "scd2-history", + "description": "SCD2 history for payment_methods.", + "owner": "data-platform", + "consumers": ["audit", "fraud"], + "update_cadence": "near-real-time", + "sla_class": "tier-2", + "tags": ["payment-method", "scd2"], + "primary_key": ["payment_method_id", "_valid_from"], + "schema_extension": { + "_cdc_seq": "INTEGER", + "_valid_from": "TIMESTAMP", + "_valid_to": "TIMESTAMP", + "_is_current": "BOOLEAN", + "_op": "VARCHAR" + } + }, + { + "name": "wh_wallets_history", + "layer": "warehouse", + "type": "scd2-history", + "description": "SCD2 history for wallets — the canonical source for any 'what was the wallet balance at time T' query.", + "owner": "data-platform", + "consumers": ["finance", "treasury", "audit"], + "update_cadence": "near-real-time", + "sla_class": "tier-1", + "tags": ["wallet", "balance", "scd2", "time-travel"], + "primary_key": ["wallet_id", "_valid_from"], + "schema_extension": { + "_cdc_seq": "INTEGER", + "_valid_from": "TIMESTAMP", + "_valid_to": "TIMESTAMP", + "_is_current": "BOOLEAN", + "_op": "VARCHAR" + } + }, + { + "name": "wh_transfers_history", + "layer": "warehouse", + "type": "scd2-history", + "description": "SCD2 history for transfers — supports replaying or reversing a transfer's lifecycle for ops/audit.", + "owner": "data-platform", + "consumers": ["audit", "fraud", "finance"], + "update_cadence": "near-real-time", + "sla_class": "tier-1", + "tags": ["transfer", "scd2", "time-travel"], + "primary_key": ["transfer_id", "_valid_from"], + "schema_extension": { + "_cdc_seq": "INTEGER", + "_valid_from": "TIMESTAMP", + "_valid_to": "TIMESTAMP", + "_is_current": "BOOLEAN", + "_op": "VARCHAR" + } + }, + { + "name": "wh_transfer_events_history", + "layer": "warehouse", + "type": "scd2-history", + "description": "SCD2 history for transfer_events.", + "owner": "data-platform", + "consumers": ["audit"], + "update_cadence": "near-real-time", + "sla_class": "tier-2", + "tags": ["transfer", "scd2"], + "primary_key": ["transfer_event_id", "_valid_from"], + "schema_extension": { + "_cdc_seq": "INTEGER", + "_valid_from": "TIMESTAMP", + "_valid_to": "TIMESTAMP", + "_is_current": "BOOLEAN", + "_op": "VARCHAR" + } + }, + { + "name": "wh_ledger_entries_history", + "layer": "warehouse", + "type": "scd2-history", + "description": "SCD2 history for ledger_entries — should be a no-op for inserts since ledger entries are append-only at the source, but having SCD2 lets the warehouse trap any unexpected mutation.", + "owner": "data-platform", + "consumers": ["audit", "finance", "treasury"], + "update_cadence": "near-real-time", + "sla_class": "tier-1", + "tags": ["ledger", "scd2", "audit"], + "primary_key": ["ledger_entry_id", "_valid_from"], + "schema_extension": { + "_cdc_seq": "INTEGER", + "_valid_from": "TIMESTAMP", + "_valid_to": "TIMESTAMP", + "_is_current": "BOOLEAN", + "_op": "VARCHAR" + } + } + ] +} diff --git a/submission/wallet-payment-transfer/conftest.py b/submission/wallet-payment-transfer/conftest.py new file mode 100644 index 0000000..a624e7f --- /dev/null +++ b/submission/wallet-payment-transfer/conftest.py @@ -0,0 +1,6 @@ +"""Root conftest — adds submission/wallet-payment-transfer/ to sys.path.""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(__file__)) diff --git a/submission/wallet-payment-transfer/pipeline/__init__.py b/submission/wallet-payment-transfer/pipeline/__init__.py new file mode 100644 index 0000000..6e6f89c --- /dev/null +++ b/submission/wallet-payment-transfer/pipeline/__init__.py @@ -0,0 +1 @@ +"""CDC pipeline — capture, lake, warehouse, schema detection, orchestration.""" diff --git a/submission/wallet-payment-transfer/pipeline/cdc.py b/submission/wallet-payment-transfer/pipeline/cdc.py new file mode 100644 index 0000000..19d5527 --- /dev/null +++ b/submission/wallet-payment-transfer/pipeline/cdc.py @@ -0,0 +1,217 @@ +""" +CDC capture layer. + +What is captured +---------------- +Every insert, update and delete on the source produces a CDCRecord with: + - sequence : monotonically increasing, never reused + - operation : 'insert' | 'update' | 'delete' + - table : name of the source table + - primary_key : the source row's PK value + - before : row image *before* the change (None for inserts) + - after : row image *after* the change (None for deletes) + - captured_at : UTC timestamp at capture time + +`before` and `after` together let downstream consumers run *before/after* +diffs (useful for SCD2 history, audit, and DLQ inspection). + +Production analogue +------------------- +A real Postgres source would hand us this same envelope from logical +replication / Debezium: the LSN ↔ `sequence`, the relational change +record ↔ `before`/`after`, and the commit timestamp ↔ `captured_at`. +The in-process `CDCCapture` class plays the role of the WAL plus the +connector. + +Replay & restart +---------------- +The pipeline checkpoints the **last sequence successfully applied** in +the warehouse (`pipeline.checkpoint`). After a restart we ask the +capture log for `records_since(checkpoint)` and reapply those — at-least +once delivery, but the warehouse merge is **idempotent by sequence** +(see `pipeline.warehouse`) so duplicates do not corrupt state. + +Out-of-order arrival +-------------------- +Records are appended to the log in capture order, so within a single +log they are in order. Across distributed sources, the warehouse keeps +`_cdc_seq` per row and ignores any incoming event with a sequence +**less than or equal** to what is already applied — late events do not +clobber newer state. + +Schema-aware capture +-------------------- +`CDCCapture` is created with a reference to a `SchemaDetector`. When +`require_compatible_schema=True`, calling any of the write methods on a +table with a broken contract raises `SchemaContractViolation` and emits +no records. This is the stop-the-line hook that satisfies the +"fail closed" requirement. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any, Optional + + +VALID_OPERATIONS = frozenset({"insert", "update", "delete"}) + + +class SchemaContractViolation(RuntimeError): + """Raised when CDC capture is attempted against a non-compliant source.""" + + +@dataclass +class CDCRecord: + operation: str + table: str + primary_key: str + before: Optional[dict[str, Any]] = None + after: Optional[dict[str, Any]] = None + captured_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) + sequence: int = 0 + + def __post_init__(self) -> None: + if self.operation not in VALID_OPERATIONS: + raise ValueError( + f"Invalid CDC operation {self.operation!r}. " + f"Must be one of: {sorted(VALID_OPERATIONS)}" + ) + if self.operation == "insert" and self.after is None: + raise ValueError("insert events must carry an 'after' image") + if self.operation == "delete" and self.before is None: + raise ValueError("delete events must carry a 'before' image") + if self.operation == "update" and ( + self.before is None or self.after is None + ): + raise ValueError("update events must carry both 'before' and 'after'") + + @property + def data(self) -> dict[str, Any]: + """ + Convenience accessor for the row image used by the warehouse merge: + for inserts/updates this is the post-image; for deletes it is the + pre-image (so we still know the row's state at delete time). + """ + return self.after if self.after is not None else (self.before or {}) + + +class CDCCapture: + """ + In-process change log used by tests and the demo orchestrator. + + Production analogue: a Debezium / pglogical connector reading the + Postgres WAL and writing each event to a Kafka topic. Each record's + `sequence` would be a Kafka offset or a Postgres LSN. + """ + + def __init__( + self, + schema_detector: Optional["object"] = None, # SchemaDetector + *, + require_compatible_schema: bool = False, + ) -> None: + self._log: list[CDCRecord] = [] + self._seq: int = 0 + self._schema_detector = schema_detector + self._require_compatible_schema = require_compatible_schema + + # ── public write API ───────────────────────────────────────────────────── + + def insert( + self, + table: str, + pk: str, + after: dict[str, Any], + *, + captured_at: Optional[datetime] = None, + ) -> CDCRecord: + self._enforce_schema(table) + return self._record( + "insert", table, pk, before=None, after=after, + captured_at=captured_at, + ) + + def update( + self, + table: str, + pk: str, + before: dict[str, Any], + after: dict[str, Any], + *, + captured_at: Optional[datetime] = None, + ) -> CDCRecord: + self._enforce_schema(table) + return self._record( + "update", table, pk, before=before, after=after, + captured_at=captured_at, + ) + + def delete( + self, + table: str, + pk: str, + before: dict[str, Any], + *, + captured_at: Optional[datetime] = None, + ) -> CDCRecord: + self._enforce_schema(table) + return self._record( + "delete", table, pk, before=before, after=None, + captured_at=captured_at, + ) + + # ── public read / replay API ───────────────────────────────────────────── + + def records_since(self, offset: int = 0) -> list[CDCRecord]: + """Return all records with sequence > offset (checkpoint replay).""" + return [r for r in self._log if r.sequence > offset] + + @property + def latest_sequence(self) -> int: + return self._seq + + @property + def log(self) -> list[CDCRecord]: + return list(self._log) + + # ── internal ───────────────────────────────────────────────────────────── + + def _enforce_schema(self, table: str) -> None: + if not self._require_compatible_schema: + return + detector = self._schema_detector + if detector is None: + return + report = detector.check_table(table) + if report.has_breaking(): + raise SchemaContractViolation( + f"Schema contract violated for {table!r}: " + + "; ".join(d.message for d in report.breaking) + ) + + def _record( + self, + operation: str, + table: str, + pk: str, + *, + before: Optional[dict[str, Any]], + after: Optional[dict[str, Any]], + captured_at: Optional[datetime] = None, + ) -> CDCRecord: + self._seq += 1 + kwargs: dict[str, Any] = { + "operation": operation, + "table": table, + "primary_key": pk, + "before": before, + "after": after, + "sequence": self._seq, + } + if captured_at is not None: + kwargs["captured_at"] = captured_at + rec = CDCRecord(**kwargs) + self._log.append(rec) + return rec diff --git a/submission/wallet-payment-transfer/pipeline/checkpoint.py b/submission/wallet-payment-transfer/pipeline/checkpoint.py new file mode 100644 index 0000000..71a1dc6 --- /dev/null +++ b/submission/wallet-payment-transfer/pipeline/checkpoint.py @@ -0,0 +1,71 @@ +""" +Checkpoint store — persists the last sequence successfully applied. + +Why this matters +---------------- +After an ingestion restart, the pipeline must answer two questions: + 1. Which CDC events have already been written to the lake? + 2. Which CDC events have already been merged into the warehouse? + +Both answers are durable in their respective layers (lake by design, +warehouse by `_cdc_seq` per row), but having a *single point of truth* +for the latest applied sequence simplifies operations: a small +checkpoint table the orchestrator reads at startup. + +Production analogue +------------------- +A Kafka consumer group offset committed atomically with the warehouse +batch, or a row in a small `pipeline_state` table written inside the +same transaction as the warehouse merge. +""" + +from __future__ import annotations + +import duckdb + + +def create_checkpoint_table(conn: duckdb.DuckDBPyConnection) -> None: + conn.execute(""" + CREATE TABLE IF NOT EXISTS pipeline_checkpoints ( + stage VARCHAR PRIMARY KEY, + sequence INTEGER NOT NULL, + updated_at TIMESTAMP NOT NULL + ) + """) + + +def get_checkpoint(conn: duckdb.DuckDBPyConnection, stage: str) -> int: + row = conn.execute( + "SELECT sequence FROM pipeline_checkpoints WHERE stage = ?", + [stage], + ).fetchone() + return int(row[0]) if row else 0 + + +def set_checkpoint( + conn: duckdb.DuckDBPyConnection, + stage: str, + sequence: int, +) -> None: + """ + Upsert the checkpoint for a stage, but never move it backwards. + A backwards-moving checkpoint would silently widen the replay window + and could double-apply events that the warehouse would otherwise + have rejected — better to make the bug loud. + """ + current = get_checkpoint(conn, stage) + if sequence < current: + raise ValueError( + f"refusing to move checkpoint {stage!r} backwards: " + f"current={current}, requested={sequence}" + ) + conn.execute( + """ + INSERT INTO pipeline_checkpoints (stage, sequence, updated_at) + VALUES (?, ?, current_timestamp) + ON CONFLICT (stage) DO UPDATE + SET sequence = EXCLUDED.sequence, + updated_at = EXCLUDED.updated_at + """, + [stage, sequence], + ) diff --git a/submission/wallet-payment-transfer/pipeline/lake.py b/submission/wallet-payment-transfer/pipeline/lake.py new file mode 100644 index 0000000..73750f0 --- /dev/null +++ b/submission/wallet-payment-transfer/pipeline/lake.py @@ -0,0 +1,179 @@ +""" +Lake layer — append-only history of every CDC event. + +Design contract +--------------- +The lake is the system of record for **change history**. Every CDC +record is written exactly once; rows are never updated or deleted. If +the warehouse loses state, we can rebuild it deterministically by +replaying `lake_cdc_events` ordered by `sequence`. + +Production analogue +------------------- +Parquet (or Delta / Iceberg) files on object storage, partitioned by +`table_name` and a `captured_at` date. The PK on `sequence` here plays +the role of the Kafka offset in the lake's `_kafka_offset` column. + +Idempotency +----------- +If the same event is appended twice (after a retried batch, say), the +PK on `sequence` rejects the duplicate at the database level rather +than silently double-writing. The orchestrator catches this and +continues — see `pipeline.orchestrator.replay_to_lake`. + +Schema +------ + sequence INTEGER NOT NULL PRIMARY KEY -- monotonic capture offset + operation VARCHAR NOT NULL -- 'insert' | 'update' | 'delete' + table_name VARCHAR NOT NULL + primary_key VARCHAR NOT NULL + before_image VARCHAR (JSON, nullable) -- null on inserts + after_image VARCHAR (JSON, nullable) -- null on deletes + captured_at TIMESTAMP NOT NULL + +We store the row images as JSON strings rather than typed columns +because schema drift would otherwise be a problem inside the lake +itself; the warehouse models do the typed projection. +""" + +from __future__ import annotations + +import json +from datetime import date, datetime +from decimal import Decimal + +import duckdb + +from pipeline.cdc import CDCRecord + + +def create_lake_table(conn: duckdb.DuckDBPyConnection) -> None: + conn.execute(""" + CREATE TABLE IF NOT EXISTS lake_cdc_events ( + sequence INTEGER NOT NULL PRIMARY KEY, + operation VARCHAR NOT NULL, + table_name VARCHAR NOT NULL, + primary_key VARCHAR NOT NULL, + before_image VARCHAR, + after_image VARCHAR, + captured_at TIMESTAMP NOT NULL + ) + """) + # An index on (table_name, sequence) is what replay queries actually use. + conn.execute(""" + CREATE INDEX IF NOT EXISTS ix_lake_table_sequence + ON lake_cdc_events (table_name, sequence) + """) + + +def append_to_lake( + conn: duckdb.DuckDBPyConnection, + records: list[CDCRecord], +) -> int: + """ + Append CDC records to the lake. Idempotent by `sequence`: duplicates + are silently skipped, so retries do not double-write. + + Returns the number of records actually persisted. + """ + if not records: + return 0 + + written = 0 + for r in records: + before_json = _serialize(r.before) if r.before is not None else None + after_json = _serialize(r.after) if r.after is not None else None + # ON CONFLICT DO NOTHING gives us at-most-once semantics on the lake + # without making the caller responsible for dedup. + cursor = conn.execute( + """ + INSERT INTO lake_cdc_events + (sequence, operation, table_name, primary_key, + before_image, after_image, captured_at) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (sequence) DO NOTHING + """, + [ + r.sequence, + r.operation, + r.table, + r.primary_key, + before_json, + after_json, + r.captured_at, + ], + ) + # DuckDB returns rowcount via .fetchall() differently per version; + # we count by checking whether the row is now present. + if cursor.execute( + "SELECT 1 FROM lake_cdc_events WHERE sequence = ?", [r.sequence] + ).fetchone(): + written += 1 + return written + + +def replay_records( + conn: duckdb.DuckDBPyConnection, + *, + table: str | None = None, + until_sequence: int | None = None, +) -> list[dict]: + """ + Read change history back out of the lake in sequence order. + + Used by: + - the warehouse's "rebuild from scratch" path + - the time-travel scripts + - audit queries + + Filters are optional: pass `table` to restrict to one source table, + `until_sequence` to bound the replay (inclusive). + """ + sql = """ + SELECT sequence, operation, table_name, primary_key, + before_image, after_image, captured_at + FROM lake_cdc_events + """ + where: list[str] = [] + params: list[object] = [] + if table is not None: + where.append("table_name = ?") + params.append(table) + if until_sequence is not None: + where.append("sequence <= ?") + params.append(until_sequence) + if where: + sql += " WHERE " + " AND ".join(where) + sql += " ORDER BY sequence" + rows = conn.execute(sql, params).fetchall() + + out: list[dict] = [] + for seq, op, tbl, pk, before_s, after_s, captured in rows: + out.append({ + "sequence": seq, + "operation": op, + "table": tbl, + "primary_key": pk, + "before": json.loads(before_s) if before_s else None, + "after": json.loads(after_s) if after_s else None, + "captured_at": captured, + }) + return out + + +# ── JSON helpers ───────────────────────────────────────────────────────────── + + +def _serialize(data: dict) -> str: + return json.dumps(data, default=_json_default, sort_keys=True) + + +def _json_default(obj: object) -> object: + if isinstance(obj, datetime): + return obj.isoformat() + if isinstance(obj, date): + return obj.isoformat() + if isinstance(obj, Decimal): + # Decimals serialize as strings to avoid float-precision drift. + return str(obj) + raise TypeError(f"Object of type {type(obj).__name__} is not JSON serializable") diff --git a/submission/wallet-payment-transfer/pipeline/orchestrator.py b/submission/wallet-payment-transfer/pipeline/orchestrator.py new file mode 100644 index 0000000..43e25e2 --- /dev/null +++ b/submission/wallet-payment-transfer/pipeline/orchestrator.py @@ -0,0 +1,108 @@ +""" +Orchestrator — wires the layers together for a single pipeline tick. + +A "tick" is one logical run of: + 1. read CDC events from `capture` since the last checkpoint + 2. write them to the lake (idempotent) + 3. merge them into the warehouse (idempotent) + 4. advance the checkpoint + +Each step is independently safe to retry; the whole tick is therefore +safe to retry on partial failure. If the warehouse write fails before +the checkpoint moves, the next tick re-reads the same window from +capture (or, in production, from the lake) and reapplies — and the +warehouse merge silently dedups by `_cdc_seq`. +""" + +from __future__ import annotations + +import duckdb + +from pipeline.cdc import CDCCapture +from pipeline.checkpoint import ( + create_checkpoint_table, + get_checkpoint, + set_checkpoint, +) +from pipeline.lake import append_to_lake, create_lake_table +from pipeline.schema_detector import SchemaDetector +from pipeline.warehouse import apply_cdc_records, create_warehouse_tables + + +CHECKPOINT_STAGE_LAKE = "lake" +CHECKPOINT_STAGE_WAREHOUSE = "warehouse" + + +def bootstrap(conn: duckdb.DuckDBPyConnection) -> None: + """Create lake, warehouse, and checkpoint tables if missing.""" + create_lake_table(conn) + create_warehouse_tables(conn) + create_checkpoint_table(conn) + + +def run_tick( + conn: duckdb.DuckDBPyConnection, + capture: CDCCapture, + *, + detector: SchemaDetector | None = None, + fail_on_drift: bool = True, +) -> dict: + """ + Run one orchestration tick. Returns a dict of stats: + {'lake_written': N, 'warehouse_applied': M, + 'checkpoint_lake': X, 'checkpoint_warehouse': Y, 'aborted': bool} + + If `fail_on_drift=True` and the schema detector reports a breaking + change on any contracted table, the tick aborts before reading any + new events — this is the stop-the-line behavior. + """ + bootstrap(conn) + + if detector is not None and fail_on_drift: + report = detector.check_all() + if report.has_breaking(): + return { + "aborted": True, + "reason": "schema_contract_violation", + "drift_summary": report.summary(), + "lake_written": 0, + "warehouse_applied": 0, + "checkpoint_lake": get_checkpoint(conn, CHECKPOINT_STAGE_LAKE), + "checkpoint_warehouse": get_checkpoint( + conn, CHECKPOINT_STAGE_WAREHOUSE + ), + } + + last_lake = get_checkpoint(conn, CHECKPOINT_STAGE_LAKE) + last_wh = get_checkpoint(conn, CHECKPOINT_STAGE_WAREHOUSE) + floor = min(last_lake, last_wh) + + pending = capture.records_since(floor) + if not pending: + return { + "aborted": False, + "lake_written": 0, + "warehouse_applied": 0, + "checkpoint_lake": last_lake, + "checkpoint_warehouse": last_wh, + } + + # Lake first — durable storage of the raw event. + lake_pending = [r for r in pending if r.sequence > last_lake] + lake_written = append_to_lake(conn, lake_pending) + new_lake_seq = pending[-1].sequence + set_checkpoint(conn, CHECKPOINT_STAGE_LAKE, new_lake_seq) + + # Warehouse next — typed merge. + wh_pending = [r for r in pending if r.sequence > last_wh] + wh_applied = apply_cdc_records(conn, wh_pending) + new_wh_seq = pending[-1].sequence + set_checkpoint(conn, CHECKPOINT_STAGE_WAREHOUSE, new_wh_seq) + + return { + "aborted": False, + "lake_written": lake_written, + "warehouse_applied": wh_applied, + "checkpoint_lake": new_lake_seq, + "checkpoint_warehouse": new_wh_seq, + } diff --git a/submission/wallet-payment-transfer/pipeline/schema_detector.py b/submission/wallet-payment-transfer/pipeline/schema_detector.py new file mode 100644 index 0000000..805ae00 --- /dev/null +++ b/submission/wallet-payment-transfer/pipeline/schema_detector.py @@ -0,0 +1,324 @@ +""" +Schema-drift detector — implements the "stop the line" behavior. + +Detection rules +--------------- +For each table contract we compare the runtime source schema against +the contract and produce a list of `Drift` records. Each drift carries +a `severity`: + + BREAKING : the pipeline must stop. The contract change can silently + corrupt the warehouse if we keep ingesting. + WARN : the pipeline can continue, but a human should know. + +Breaking changes (fail closed): + - column dropped (existing references are now orphaned) + - column renamed (modelled as drop + add of an unknown col) + - column type changed (cast may silently round, truncate, error) + - column nullability tightened + nullable contract -> NOT NULL source + means a row that previously satisfied the contract may now fail + - enum domain shrunk (a row that was valid before now is not) + - primary key changed (replay/upsert depends on the original PK) + - referenced parent table missing (FK broken) + +Warning-only changes: + - new nullable column added (additive, downstream just ignores it) + - new enum value added (downstream consumers can choose to allow) + +Output +------ +The detector returns a `DriftReport` per table and a top-level +`PipelineDriftReport` for the whole source. The CI script +`scripts/check_schema_contracts.py` exits non-zero whenever any +breaking drift is present. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum +from typing import Optional + +import duckdb + +from source.contracts import CONTRACTS, TableContract + + +class Severity(Enum): + BREAKING = "BREAKING" + WARN = "WARN" + + +@dataclass(frozen=True) +class Drift: + table: str + column: Optional[str] + rule: str + severity: Severity + message: str + + +@dataclass +class DriftReport: + table: str + drifts: list[Drift] = field(default_factory=list) + + @property + def breaking(self) -> list[Drift]: + return [d for d in self.drifts if d.severity is Severity.BREAKING] + + @property + def warnings(self) -> list[Drift]: + return [d for d in self.drifts if d.severity is Severity.WARN] + + def has_breaking(self) -> bool: + return any(d.severity is Severity.BREAKING for d in self.drifts) + + +@dataclass +class PipelineDriftReport: + by_table: dict[str, DriftReport] = field(default_factory=dict) + + @property + def breaking(self) -> list[Drift]: + return [d for r in self.by_table.values() for d in r.breaking] + + @property + def warnings(self) -> list[Drift]: + return [d for r in self.by_table.values() for d in r.warnings] + + def has_breaking(self) -> bool: + return any(r.has_breaking() for r in self.by_table.values()) + + def summary(self) -> str: + lines: list[str] = [] + for table, report in self.by_table.items(): + for d in report.drifts: + marker = "x" if d.severity is Severity.BREAKING else "!" + lines.append(f" [{marker}] {table}.{d.column}: {d.rule} — {d.message}") + return "\n".join(lines) if lines else " (no drift)" + + +# ── runtime introspection ──────────────────────────────────────────────────── + + +def _normalize_type(t: str) -> str: + """ + DuckDB DESCRIBE returns types like 'VARCHAR' / 'DECIMAL(18,4)' / + 'TIMESTAMP' which already match our contract style. We strip + whitespace and uppercase to be tolerant. + """ + return t.strip().upper().replace(" ", "") + + +def _runtime_columns( + conn: duckdb.DuckDBPyConnection, table: str +) -> list[tuple[str, str, bool]]: + """ + Return [(name, type, nullable), ...] from DuckDB's information_schema. + + information_schema.columns is more reliable than DESCRIBE for + nullability, since DESCRIBE in DuckDB does not always expose it. + """ + rows = conn.execute( + """ + SELECT column_name, data_type, is_nullable + FROM information_schema.columns + WHERE table_name = ? + ORDER BY ordinal_position + """, + [table], + ).fetchall() + return [(r[0], _normalize_type(r[1]), r[2] == "YES") for r in rows] + + +def _runtime_pk(conn: duckdb.DuckDBPyConnection, table: str) -> list[str]: + """ + Best-effort PK lookup. DuckDB exposes constraints via duckdb_constraints(); + we scan for the PRIMARY KEY entry on the table. + """ + try: + rows = conn.execute( + """ + SELECT constraint_column_names + FROM duckdb_constraints() + WHERE table_name = ? AND constraint_type = 'PRIMARY KEY' + """, + [table], + ).fetchall() + except Exception: + return [] + if not rows: + return [] + cols = rows[0][0] + if isinstance(cols, list): + return [str(c) for c in cols] + if isinstance(cols, str): + return [c.strip() for c in cols.split(",")] + return [] + + +def _table_exists(conn: duckdb.DuckDBPyConnection, table: str) -> bool: + return bool( + conn.execute( + """ + SELECT 1 FROM information_schema.tables WHERE table_name = ? + """, + [table], + ).fetchone() + ) + + +# ── core checker ───────────────────────────────────────────────────────────── + + +@dataclass +class SchemaDetector: + """ + Runs schema contract checks against a live DuckDB connection. + + Used both by the CI script (`scripts/check_schema_contracts.py`) + and at CDC capture time (`pipeline.cdc.CDCCapture` consults it + before every event when `require_compatible_schema=True`). + """ + + conn: duckdb.DuckDBPyConnection + + def check_table(self, table: str) -> DriftReport: + contract = CONTRACTS[table] + report = DriftReport(table=table) + + if not _table_exists(self.conn, table): + report.drifts.append( + Drift( + table=table, + column=None, + rule="missing_table", + severity=Severity.BREAKING, + message="source table does not exist", + ) + ) + return report + + runtime_cols = _runtime_columns(self.conn, table) + runtime_by_name = {c[0]: c for c in runtime_cols} + contract_by_name = {c.name: c for c in contract.columns} + + # 1) dropped columns + for col in contract.columns: + if col.name not in runtime_by_name: + report.drifts.append( + Drift( + table, col.name, + "column_dropped", + Severity.BREAKING, + "contract column is missing from the source", + ) + ) + continue + + r_name, r_type, r_nullable = runtime_by_name[col.name] + + # 2) type change + if not _types_compatible(col.type, r_type): + report.drifts.append( + Drift( + table, col.name, + "type_changed", + Severity.BREAKING, + f"contract type {col.type!r} != source type {r_type!r}", + ) + ) + + # 3) nullability tightened + if col.nullable and not r_nullable: + report.drifts.append( + Drift( + table, col.name, + "nullability_tightened", + Severity.BREAKING, + "contract allows NULL but source forbids it — " + "rows previously valid may now fail", + ) + ) + + # 4) added columns (warning only, additive) + for r_name, _, r_nullable in runtime_cols: + if r_name not in contract_by_name: + if r_nullable: + report.drifts.append( + Drift( + table, r_name, + "column_added", + Severity.WARN, + "new nullable column appeared on the source", + ) + ) + else: + report.drifts.append( + Drift( + table, r_name, + "non_nullable_column_added", + Severity.BREAKING, + "new NOT-NULL column appeared on the source — " + "inserts from the contract perspective will fail", + ) + ) + + # 5) primary key changed + runtime_pk = _runtime_pk(self.conn, table) + if runtime_pk and runtime_pk != [contract.primary_key]: + report.drifts.append( + Drift( + table, ",".join(runtime_pk), + "primary_key_changed", + Severity.BREAKING, + f"contract PK is {contract.primary_key!r} " + f"but source PK is {runtime_pk!r}", + ) + ) + + # 6) enum domain shrinkage — checked by sampling actual rows. + for col_name, allowed in contract.enum_domains: + try: + rows = self.conn.execute( + f"SELECT DISTINCT {col_name} FROM {table} " + f"WHERE {col_name} IS NOT NULL" + ).fetchall() + except Exception: + continue + present = {r[0] for r in rows} + unexpected = present - set(allowed) + if unexpected: + # New value the contract did not list — additive, warn. + report.drifts.append( + Drift( + table, col_name, + "enum_value_added", + Severity.WARN, + f"unexpected values present: {sorted(unexpected)!r}", + ) + ) + # We cannot tell from data alone whether the source has *removed* + # an allowed value. The warehouse-side parity test + # `tests/test_data_quality.py::test_enum_domains_match_contract` + # checks the *materialized* warehouse so a removed value would + # show up as a contract drift in subsequent runs. + + return report + + def check_all(self) -> PipelineDriftReport: + out = PipelineDriftReport() + for table in CONTRACTS: + out.by_table[table] = self.check_table(table) + return out + + +def _types_compatible(contract_type: str, runtime_type: str) -> bool: + """ + Type equality is required. We are intentionally strict because silent + casting between DECIMAL precisions or between VARCHAR/INT is exactly + the kind of drift that produces hard-to-debug correctness bugs. + """ + return _normalize_type(contract_type) == _normalize_type(runtime_type) diff --git a/submission/wallet-payment-transfer/pipeline/warehouse.py b/submission/wallet-payment-transfer/pipeline/warehouse.py new file mode 100644 index 0000000..768b9fd --- /dev/null +++ b/submission/wallet-payment-transfer/pipeline/warehouse.py @@ -0,0 +1,367 @@ +""" +Warehouse layer — current-state snapshot + SCD2 history. + +Three tables per source table +----------------------------- + wh_
: latest known state, soft-deleted with _deleted=true + wh_
_history : SCD2 — every revision has _valid_from / _valid_to / + _is_current / _op so we can answer "what did this + row look like at time T?" without rescanning the lake + (lake_cdc_events keeps the raw events; the SCD2 table is a *materialized + view of the lake* that downstream consumers can query directly) + +Idempotent merge +---------------- +Each warehouse row carries `_cdc_seq`, the sequence of the last CDC +event that touched it. The merge applies an event only when its +sequence is **strictly greater** than the row's `_cdc_seq`. This makes +the merge: + - safe under retry / replay (duplicates are no-ops) + - safe under out-of-order arrival (older events do not clobber newer + state) + +SCD2 mechanics +-------------- +For an insert: open a new row with `_valid_from = captured_at`, +`_valid_to = NULL`, `_is_current = true`, `_op = 'insert'`. + +For an update or delete on an existing row: + 1. close the current row by setting `_valid_to = captured_at`, + `_is_current = false` + 2. for updates, open a new current row with the post-image + 3. for deletes, do not open a new row but mark the *current* row's + replacement (none) as deleted in `wh_
` via `_deleted=true` + +Time travel +----------- +Use `state_at(conn, table, ts)` to reconstruct what the table looked +like at any timestamp. The query selects rows from `wh_
_history` +where `_valid_from <= ts < COALESCE(_valid_to, '9999-12-31')` and +`_op != 'delete'`. + +Production analogue +------------------- +The warehouse is what BigQuery / Snowflake / Redshift would expose: +typed, queryable, current state for analytics, with the SCD2 history +table for finance / audit / restore. The merge here is the same +MERGE-by-PK pattern you would write in those systems, just expressed +in DuckDB SQL. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +from decimal import Decimal +from typing import Any, Iterable + +import duckdb + +from pipeline.cdc import CDCRecord +from source.contracts import CONTRACTS + + +# Source tables we mirror in the warehouse, in dependency order. +WAREHOUSED_TABLES: tuple[str, ...] = ( + "customers", + "payment_methods", + "wallets", + "transfers", + "transfer_events", + "ledger_entries", +) + + +# Map source SQL types to warehouse SQL types. Identical for our domain; +# the indirection lets us coerce on the way in if a source type is wider +# than what we want to expose to BI. +_TYPE_MAP: dict[str, str] = { + "VARCHAR": "VARCHAR", + "BOOLEAN": "BOOLEAN", + "DATE": "DATE", + "TIMESTAMP": "TIMESTAMP", + "DECIMAL(18,4)": "DECIMAL(18,4)", +} + + +@dataclass +class _ColumnInfo: + name: str + sql_type: str + nullable: bool + + +def _columns_for(table: str) -> list[_ColumnInfo]: + contract = CONTRACTS[table] + return [ + _ColumnInfo(c.name, _TYPE_MAP[c.type], c.nullable) for c in contract.columns + ] + + +def _wh_table(table: str) -> str: + return f"wh_{table}" + + +def _hist_table(table: str) -> str: + return f"wh_{table}_history" + + +# ── DDL ────────────────────────────────────────────────────────────────────── + + +def create_warehouse_tables(conn: duckdb.DuckDBPyConnection) -> None: + for table in WAREHOUSED_TABLES: + cols = _columns_for(table) + pk = CONTRACTS[table].primary_key + + # Current-state table. + col_defs = ", ".join( + f"{c.name} {c.sql_type}{'' if c.nullable else ' NOT NULL'}" + if c.name == pk + else f"{c.name} {c.sql_type}" + for c in cols + ) + conn.execute(f""" + CREATE TABLE IF NOT EXISTS {_wh_table(table)} ( + {col_defs}, + _cdc_seq INTEGER NOT NULL, + _deleted BOOLEAN NOT NULL DEFAULT FALSE, + PRIMARY KEY ({pk}) + ) + """) + + # SCD2 history table — same columns, no PK on business key, + # surrogate PK on (business_key, _valid_from). + hist_cols = ", ".join(f"{c.name} {c.sql_type}" for c in cols) + conn.execute(f""" + CREATE TABLE IF NOT EXISTS {_hist_table(table)} ( + {hist_cols}, + _cdc_seq INTEGER NOT NULL, + _valid_from TIMESTAMP NOT NULL, + _valid_to TIMESTAMP, + _is_current BOOLEAN NOT NULL, + _op VARCHAR NOT NULL + CHECK (_op IN ('insert', 'update', 'delete')), + PRIMARY KEY ({pk}, _valid_from) + ) + """) + + +# ── apply CDC records ──────────────────────────────────────────────────────── + + +def apply_cdc_records( + conn: duckdb.DuckDBPyConnection, + records: Iterable[CDCRecord], +) -> int: + """ + Apply CDC records to the current-state and SCD2 history tables. + + Returns the number of records *actually applied* (skipped duplicates + and out-of-order events are not counted). + """ + sorted_records = sorted(records, key=lambda r: r.sequence) + applied = 0 + for r in sorted_records: + if r.table not in WAREHOUSED_TABLES: + continue + if _apply_one(conn, r): + applied += 1 + return applied + + +def _apply_one(conn: duckdb.DuckDBPyConnection, r: CDCRecord) -> bool: + pk_col = CONTRACTS[r.table].primary_key + cur = _wh_table(r.table) + hist = _hist_table(r.table) + + existing = conn.execute( + f"SELECT _cdc_seq, _deleted FROM {cur} WHERE {pk_col} = ?", + [r.primary_key], + ).fetchone() + + # Idempotent / out-of-order guard. + if existing is not None and existing[0] >= r.sequence: + return False + + # Close any open SCD2 row for this PK. + conn.execute( + f""" + UPDATE {hist} + SET _valid_to = ?, + _is_current = FALSE + WHERE {pk_col} = ? + AND _is_current = TRUE + """, + [r.captured_at, r.primary_key], + ) + + if r.operation == "delete": + # Update current-state to mark deleted. + if existing is not None: + conn.execute( + f"UPDATE {cur} SET _deleted = TRUE, _cdc_seq = ? " + f"WHERE {pk_col} = ?", + [r.sequence, r.primary_key], + ) + # Append a closed delete row to history (with the pre-image). + _insert_history( + conn, + r.table, + r, + data=r.before or {}, + valid_from=r.captured_at, + valid_to=r.captured_at, + is_current=False, + op="delete", + ) + return True + + # insert or update + data = r.after or {} + cols = _columns_for(r.table) + values = [data.get(c.name) for c in cols] + + if existing is None: + # INSERT into current-state. + col_list = ", ".join(c.name for c in cols) + placeholders = ", ".join(["?"] * len(cols)) + conn.execute( + f""" + INSERT INTO {cur} ({col_list}, _cdc_seq, _deleted) + VALUES ({placeholders}, ?, FALSE) + """, + values + [r.sequence], + ) + else: + # UPDATE the current-state row. + set_clause = ", ".join(f"{c.name} = ?" for c in cols) + conn.execute( + f""" + UPDATE {cur} + SET {set_clause}, _cdc_seq = ?, _deleted = FALSE + WHERE {pk_col} = ? + """, + values + [r.sequence, r.primary_key], + ) + + # Open new current SCD2 row. + _insert_history( + conn, + r.table, + r, + data=data, + valid_from=r.captured_at, + valid_to=None, + is_current=True, + op=r.operation, + ) + return True + + +def _insert_history( + conn: duckdb.DuckDBPyConnection, + table: str, + record: CDCRecord, + *, + data: dict[str, Any], + valid_from: datetime, + valid_to: datetime | None, + is_current: bool, + op: str, +) -> None: + cols = _columns_for(table) + values = [data.get(c.name) for c in cols] + col_list = ", ".join(c.name for c in cols) + placeholders = ", ".join(["?"] * len(cols)) + conn.execute( + f""" + INSERT INTO {_hist_table(table)} + ({col_list}, _cdc_seq, _valid_from, _valid_to, _is_current, _op) + VALUES ({placeholders}, ?, ?, ?, ?, ?) + ON CONFLICT ({CONTRACTS[table].primary_key}, _valid_from) DO NOTHING + """, + values + [record.sequence, valid_from, valid_to, is_current, op], + ) + + +# ── time travel ────────────────────────────────────────────────────────────── + + +def state_at( + conn: duckdb.DuckDBPyConnection, + table: str, + ts: datetime, +) -> list[dict]: + """ + Reconstruct the state of `table` at timestamp `ts` from the SCD2 history. + + Returns the rows that were *valid at ts* and not marked deleted at that + point — semantically equivalent to running `SELECT * FROM
` on + the source as of `ts`. + """ + cols = _columns_for(table) + col_list = ", ".join(c.name for c in cols) + sql = f""" + SELECT {col_list} + FROM {_hist_table(table)} + WHERE _valid_from <= ? + AND COALESCE(_valid_to, TIMESTAMP '9999-12-31 23:59:59') > ? + AND _op <> 'delete' + """ + rows = conn.execute(sql, [ts, ts]).fetchall() + return [ + {c.name: _coerce(value) for c, value in zip(cols, row)} + for row in rows + ] + + +def _coerce(v: object) -> object: + """Bring DuckDB result values back to standard Python types for callers.""" + if isinstance(v, Decimal): + return v + return v + + +# ── restore ────────────────────────────────────────────────────────────────── + + +def restore_current_state_to( + conn: duckdb.DuckDBPyConnection, + table: str, + ts: datetime, +) -> int: + """ + Replace `wh_
` rows with the snapshot from `state_at(table, ts)`. + + This is the operational hook for "roll the warehouse back to time T": + history is preserved (we never touch `wh_
_history`), but the + *current state* table is overwritten from the historical snapshot. + + Returns the number of rows in the restored snapshot. + """ + snapshot = state_at(conn, table, ts) + pk_col = CONTRACTS[table].primary_key + cols = _columns_for(table) + + conn.execute(f"DELETE FROM {_wh_table(table)}") + + if not snapshot: + return 0 + + col_list = ", ".join(c.name for c in cols) + placeholders = ", ".join(["?"] * len(cols)) + for row in snapshot: + values = [row.get(c.name) for c in cols] + conn.execute( + f""" + INSERT INTO {_wh_table(table)} + ({col_list}, _cdc_seq, _deleted) + VALUES ({placeholders}, 0, FALSE) + """, + values, + ) + # Note: _cdc_seq is reset to 0 to signal "restored, ready for new events". + # The next CDC record (which will have sequence > 0) will reapply cleanly. + _ = pk_col # quiet linter — we may use it for a future assertion + return len(snapshot) diff --git a/submission/wallet-payment-transfer/requirements.txt b/submission/wallet-payment-transfer/requirements.txt new file mode 100644 index 0000000..677a29a --- /dev/null +++ b/submission/wallet-payment-transfer/requirements.txt @@ -0,0 +1,2 @@ +duckdb>=1.0.0 +pytest>=7.0.0 diff --git a/submission/wallet-payment-transfer/scripts/check_schema_contracts.py b/submission/wallet-payment-transfer/scripts/check_schema_contracts.py new file mode 100644 index 0000000..aee7c41 --- /dev/null +++ b/submission/wallet-payment-transfer/scripts/check_schema_contracts.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +""" +check_schema_contracts.py + +CI entry point for the schema contract check (`schema-contract-check`). + +What it does +------------ +1. Spins up an in-memory DuckDB +2. Creates the source tables exactly as defined in source/models.py +3. Runs the SchemaDetector against every contract +4. Exits non-zero if any BREAKING drift is found +5. Prints WARN drifts but does not fail the build for them + +This is the same code path the orchestrator uses to fail the pipeline +closed at runtime — keeping the two consistent prevents "passes in CI, +fails in prod" surprises. +""" + +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import duckdb + +from pipeline.schema_detector import SchemaDetector +from source.models import create_source_tables + + +def main() -> int: + conn = duckdb.connect(":memory:") + create_source_tables(conn) + + report = SchemaDetector(conn).check_all() + + if report.warnings: + print("Schema contract WARNINGS:") + for d in report.warnings: + print(f" ! {d.table}.{d.column}: {d.rule} — {d.message}") + + if report.breaking: + print("Schema contract BREAKING violations:") + for d in report.breaking: + print(f" x {d.table}.{d.column}: {d.rule} — {d.message}") + return 1 + + n_tables = len(report.by_table) + print(f"All schema contracts passed ({n_tables} tables checked).") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/submission/wallet-payment-transfer/scripts/demo_schema_change.py b/submission/wallet-payment-transfer/scripts/demo_schema_change.py new file mode 100644 index 0000000..36a6e12 --- /dev/null +++ b/submission/wallet-payment-transfer/scripts/demo_schema_change.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +""" +demo_schema_change.py + +Demonstrates the stop-the-line behavior when the source schema drifts +incompatibly from the contract. + +Scenario +-------- + 1. Bootstrap the source as designed. + 2. Run a normal CDC tick — succeeds. + 3. Drop a contract column from the source. + 4. Run another tick — pipeline aborts with a clear failure signal. +""" + +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import duckdb + +from pipeline.cdc import CDCCapture, SchemaContractViolation +from pipeline.orchestrator import bootstrap, run_tick +from pipeline.schema_detector import SchemaDetector +from source.models import create_source_tables +from source.seed import build_seed_rows + + +def main() -> int: + conn = duckdb.connect(":memory:") + create_source_tables(conn) + bootstrap(conn) + capture = CDCCapture() + detector = SchemaDetector(conn) + + print("== first tick (clean) ==") + for row in build_seed_rows(): + capture.insert(row.table, row.pk, row.data) + stats = run_tick(conn, capture, detector=detector) + print(f" stats: {stats}") + + print("== drift the source: drop customers.email ==") + # Drop FK-dependent tables first so the ALTER works in DuckDB. + conn.execute("DROP TABLE IF EXISTS ledger_entries") + conn.execute("DROP TABLE IF EXISTS transfer_events") + conn.execute("DROP TABLE IF EXISTS transfers") + conn.execute("DROP TABLE IF EXISTS payment_methods") + conn.execute("DROP TABLE IF EXISTS wallets") + conn.execute("ALTER TABLE customers DROP COLUMN email") + + print("== second tick (must abort) ==") + stats = run_tick(conn, capture, detector=detector) + if not stats.get("aborted"): + print(" FAIL: pipeline did not abort on incompatible schema change") + return 1 + print(" pipeline correctly aborted with reason:") + print(f" {stats.get('reason')}") + print(" drift summary:") + print(stats.get("drift_summary")) + + print("== capture-level stop-the-line ==") + capture_strict = CDCCapture(detector, require_compatible_schema=True) + try: + capture_strict.insert( + "customers", "c_new", + {"customer_id": "c_new", "full_name": "X"}, + ) + except SchemaContractViolation as exc: + print(f" CDCCapture refused write: {exc}") + return 0 + print(" FAIL: CDCCapture should have raised SchemaContractViolation") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/submission/wallet-payment-transfer/scripts/demo_time_travel.py b/submission/wallet-payment-transfer/scripts/demo_time_travel.py new file mode 100644 index 0000000..9f0fc83 --- /dev/null +++ b/submission/wallet-payment-transfer/scripts/demo_time_travel.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +""" +demo_time_travel.py + +Demonstrates the warehouse's point-in-time reconstruction feature. + +Scenario +-------- + t0 : insert wallet w_demo at balance 100 + t1 : update balance to 60 + t2 : update balance to 0 (closed) + +The script then asks the warehouse what the wallet looked like at each +of those times and prints the answer. Reads come from +`wh_wallets_history` only — the lake is not touched. +""" + +from __future__ import annotations + +import os +import sys +from datetime import datetime, timedelta, timezone +from decimal import Decimal + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import duckdb + +from pipeline.cdc import CDCCapture +from pipeline.orchestrator import bootstrap, run_tick +from pipeline.warehouse import state_at +from source.models import create_source_tables + + +def _t(seconds: int) -> datetime: + base = datetime(2026, 6, 1, 9, 0, tzinfo=timezone.utc) + return base + timedelta(seconds=seconds) + + +def main() -> int: + conn = duckdb.connect(":memory:") + create_source_tables(conn) + bootstrap(conn) + capture = CDCCapture() + + # Customer must exist for FK. + cust_row = { + "customer_id": "c_demo", + "full_name": "Demo Customer", + "email": "demo@example.invalid", + "country_code": "US", + "status": "active", + "registration_date": _t(0).date(), + "created_at": _t(0), + "updated_at": _t(0), + } + capture.insert("customers", "c_demo", cust_row, captured_at=_t(0)) + + # Three revisions of one wallet: + w0 = { + "wallet_id": "w_demo", + "customer_id": "c_demo", + "currency": "USD", + "balance": Decimal("100.0000"), + "status": "active", + "created_at": _t(0), + "updated_at": _t(0), + } + capture.insert("wallets", "w_demo", w0, captured_at=_t(0)) + + w1 = dict(w0, balance=Decimal("60.0000"), updated_at=_t(60)) + capture.update("wallets", "w_demo", before=w0, after=w1, captured_at=_t(60)) + + w2 = dict(w1, balance=Decimal("0.0000"), status="closed", updated_at=_t(120)) + capture.update("wallets", "w_demo", before=w1, after=w2, captured_at=_t(120)) + + run_tick(conn, capture) + + print("== state_at queries ==") + for label, ts in [ + ("at t0 (just after insert)", _t(0)), + ("at t30 (between insert and first update)", _t(30)), + ("at t60 (just after first update)", _t(60)), + ("at t90 (between updates)", _t(90)), + ("at t150 (after close)", _t(150)), + ]: + rows = state_at(conn, "wallets", ts) + balance = rows[0]["balance"] if rows else None + status = rows[0]["status"] if rows else None + print(f" {label}: balance={balance}, status={status}") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/submission/wallet-payment-transfer/scripts/run_data_quality_checks.py b/submission/wallet-payment-transfer/scripts/run_data_quality_checks.py new file mode 100644 index 0000000..379fb04 --- /dev/null +++ b/submission/wallet-payment-transfer/scripts/run_data_quality_checks.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +""" +run_data_quality_checks.py + +CI entry point for the data-quality validation parity check +(`data-quality-check`). + +The script: + 1. bootstraps an in-memory DuckDB, + 2. seeds it with the canonical wallet domain seed, + 3. runs every SYSTEM and BUSINESS check defined in this module, + 4. prints the rule name + table when a check fails, + 5. exits non-zero on any failure. + +Each rule below is traceable to a source-side invariant (see the +docstrings in source/models.py and source/contracts.py) — that is the +"validation parity" the assignment asks for. +""" + +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from dataclasses import dataclass +from decimal import Decimal + +import duckdb + +from pipeline.cdc import CDCCapture +from pipeline.orchestrator import bootstrap, run_tick +from source.contracts import CONTRACTS +from source.models import ALLOWED_TRANSFER_TRANSITIONS, create_source_tables +from source.seed import build_seed_rows + + +# ── rule registry ──────────────────────────────────────────────────────────── + + +@dataclass(frozen=True) +class Rule: + rule_id: str + category: str # 'system' | 'business' | 'freshness' + description: str + sql: str # query that returns 0 rows when the rule passes + + +def _system_rules() -> list[Rule]: + rules: list[Rule] = [] + + # PK uniqueness + for table, contract in CONTRACTS.items(): + wh = f"wh_{table}" + pk = contract.primary_key + rules.append(Rule( + rule_id=f"sys.pk_unique.{table}", + category="system", + description=f"{wh}.{pk} must be unique", + sql=f""" + SELECT {pk}, COUNT(*) AS n + FROM {wh} + GROUP BY {pk} + HAVING COUNT(*) > 1 + """, + )) + + # NOT NULL parity + for table, contract in CONTRACTS.items(): + wh = f"wh_{table}" + for col in contract.columns: + if col.nullable: + continue + rules.append(Rule( + rule_id=f"sys.not_null.{table}.{col.name}", + category="system", + description=f"{wh}.{col.name} must not be null", + sql=f"SELECT 1 FROM {wh} " + f"WHERE {col.name} IS NULL AND _deleted = FALSE", + )) + + # Enum domain parity (only on currently-present rows) + for table, contract in CONTRACTS.items(): + wh = f"wh_{table}" + for col, allowed in contract.enum_domains: + allowed_sql = ", ".join(f"'{v}'" for v in sorted(allowed)) + rules.append(Rule( + rule_id=f"sys.enum.{table}.{col}", + category="system", + description=f"{wh}.{col} must be one of {sorted(allowed)}", + sql=f"SELECT 1 FROM {wh} " + f"WHERE _deleted = FALSE " + f" AND {col} IS NOT NULL " + f" AND {col} NOT IN ({allowed_sql})", + )) + + # Referential integrity for present (non-deleted) rows + for table, contract in CONTRACTS.items(): + wh = f"wh_{table}" + for fk_col, parent, parent_col in contract.foreign_keys: + parent_wh = f"wh_{parent}" + rules.append(Rule( + rule_id=f"sys.fk.{table}.{fk_col}", + category="system", + description=( + f"{wh}.{fk_col} must reference an existing " + f"{parent_wh}.{parent_col}" + ), + sql=f""" + SELECT 1 + FROM {wh} c + LEFT JOIN {parent_wh} p + ON c.{fk_col} = p.{parent_col} + AND p._deleted = FALSE + WHERE c._deleted = FALSE + AND c.{fk_col} IS NOT NULL + AND p.{parent_col} IS NULL + """, + )) + + return rules + + +def _business_rules() -> list[Rule]: + return [ + Rule( + "biz.wallet.balance_non_negative", + "business", + "wh_wallets.balance must be >= 0", + "SELECT 1 FROM wh_wallets " + "WHERE _deleted = FALSE AND balance < 0", + ), + Rule( + "biz.transfer.amount_positive", + "business", + "wh_transfers.amount must be > 0", + "SELECT 1 FROM wh_transfers " + "WHERE _deleted = FALSE AND amount <= 0", + ), + Rule( + "biz.transfer.no_self_transfer", + "business", + "wh_transfers.from_wallet_id must differ from to_wallet_id", + "SELECT 1 FROM wh_transfers " + "WHERE _deleted = FALSE AND from_wallet_id = to_wallet_id", + ), + Rule( + "biz.transfer.settled_at_after_created", + "business", + "wh_transfers.settled_at must be >= created_at when present", + "SELECT 1 FROM wh_transfers " + "WHERE _deleted = FALSE " + " AND settled_at IS NOT NULL " + " AND settled_at < created_at", + ), + Rule( + "biz.transfer.settled_implies_settled_at", + "business", + "wh_transfers.settled_at must be set whenever status = 'settled'", + "SELECT 1 FROM wh_transfers " + "WHERE _deleted = FALSE " + " AND status = 'settled' AND settled_at IS NULL", + ), + Rule( + "biz.transfer.failed_implies_reason", + "business", + "wh_transfers.failure_reason must be set whenever status = 'failed'", + "SELECT 1 FROM wh_transfers " + "WHERE _deleted = FALSE " + " AND status = 'failed' AND failure_reason IS NULL", + ), + Rule( + "biz.ledger.signed_amount_consistent", + "business", + "ledger_entries.signed_amount must equal +/- amount per direction", + "SELECT 1 FROM wh_ledger_entries " + "WHERE _deleted = FALSE " + " AND ((direction = 'credit' AND signed_amount <> amount) " + " OR (direction = 'debit' AND signed_amount <> -amount))", + ), + Rule( + "biz.ledger.balance_after_non_negative", + "business", + "ledger_entries.balance_after must be >= 0", + "SELECT 1 FROM wh_ledger_entries " + "WHERE _deleted = FALSE AND balance_after < 0", + ), + # Monetary integrity — ledger sums per wallet must equal wallet balance. + Rule( + "biz.ledger.reconciles_to_wallet_balance", + "business", + "SUM(ledger.signed_amount) per wallet must equal wallet.balance", + """ + WITH ledger_sum AS ( + SELECT wallet_id, SUM(signed_amount) AS s + FROM wh_ledger_entries + WHERE _deleted = FALSE + GROUP BY wallet_id + ) + SELECT 1 + FROM wh_wallets w + LEFT JOIN ledger_sum l ON l.wallet_id = w.wallet_id + WHERE w._deleted = FALSE + AND COALESCE(l.s, 0) <> w.balance + """, + ), + # Status transitions must follow the documented state machine. + Rule( + "biz.transfer_event.allowed_transition", + "business", + "transfer_events.from_status -> to_status must be allowed " + "(or from_status NULL for the first event)", + f""" + SELECT 1 FROM wh_transfer_events + WHERE _deleted = FALSE + AND from_status IS NOT NULL + AND (from_status, to_status) NOT IN ( + {", ".join(f"('{f}', '{t}')" for f, t in + sorted(ALLOWED_TRANSFER_TRANSITIONS))} + ) + """, + ), + ] + + +def all_rules() -> list[Rule]: + return _system_rules() + _business_rules() + + +# ── seed + run ─────────────────────────────────────────────────────────────── + + +def _seed_pipeline(conn: duckdb.DuckDBPyConnection) -> CDCCapture: + create_source_tables(conn) + bootstrap(conn) + capture = CDCCapture() + for row in build_seed_rows(): + capture.insert(row.table, row.pk, row.data) + run_tick(conn, capture) + return capture + + +def run_checks(conn: duckdb.DuckDBPyConnection) -> list[tuple[Rule, int]]: + """Return list of (rule, n_failing_rows) for any rule that fails.""" + failures: list[tuple[Rule, int]] = [] + for rule in all_rules(): + try: + n = len(conn.execute(rule.sql).fetchall()) + except Exception as exc: + failures.append((rule, -1)) + print(f" ! rule errored: {rule.rule_id} — {exc}") + continue + if n > 0: + failures.append((rule, n)) + return failures + + +def main() -> int: + conn = duckdb.connect(":memory:") + _seed_pipeline(conn) + + failures = run_checks(conn) + + if failures: + print("Data quality failures:") + for rule, n in failures: + tag = "?" if n < 0 else f"{n} rows" + print(f" x [{rule.category}] {rule.rule_id}: {rule.description} " + f"({tag})") + return 1 + + n_rules = len(all_rules()) + print(f"All data quality checks passed ({n_rules} rules evaluated).") + return 0 + + +if __name__ == "__main__": + # Reference Decimal so the import is "used" for static analysers when + # rules grow into Python-side numeric tolerance checks. + _ = Decimal("0") + sys.exit(main()) diff --git a/submission/wallet-payment-transfer/scripts/run_pipeline.py b/submission/wallet-payment-transfer/scripts/run_pipeline.py new file mode 100644 index 0000000..d431476 --- /dev/null +++ b/submission/wallet-payment-transfer/scripts/run_pipeline.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +""" +run_pipeline.py + +End-to-end demo: bootstrap source + lake + warehouse, seed data, replay, +print summary. Intended as a runnable smoke test. +""" + +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import duckdb + +from pipeline.cdc import CDCCapture +from pipeline.orchestrator import bootstrap, run_tick +from pipeline.schema_detector import SchemaDetector +from source.models import create_source_tables +from source.seed import build_seed_rows + + +def _print_summary(conn: duckdb.DuckDBPyConnection) -> None: + n_lake = conn.execute("SELECT COUNT(*) FROM lake_cdc_events").fetchone()[0] + n_cust = conn.execute( + "SELECT COUNT(*) FROM wh_customers WHERE _deleted = FALSE" + ).fetchone()[0] + n_wallets = conn.execute( + "SELECT COUNT(*) FROM wh_wallets WHERE _deleted = FALSE" + ).fetchone()[0] + n_transfers = conn.execute( + "SELECT COUNT(*) FROM wh_transfers WHERE _deleted = FALSE" + ).fetchone()[0] + print(f" lake_cdc_events : {n_lake} events") + print(f" wh_customers : {n_cust} active") + print(f" wh_wallets : {n_wallets} active") + print(f" wh_transfers : {n_transfers} active") + + +def main() -> int: + print("== bootstrap ==") + conn = duckdb.connect(":memory:") + create_source_tables(conn) + bootstrap(conn) + capture = CDCCapture() + + print("== seed ==") + for row in build_seed_rows(): + capture.insert(row.table, row.pk, row.data) + print(f" captured {capture.latest_sequence} CDC records") + + print("== run tick ==") + detector = SchemaDetector(conn) + stats = run_tick(conn, capture, detector=detector) + print(f" stats: {stats}") + + print("== summary ==") + _print_summary(conn) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/submission/wallet-payment-transfer/scripts/validate_catalog.py b/submission/wallet-payment-transfer/scripts/validate_catalog.py new file mode 100644 index 0000000..bac870d --- /dev/null +++ b/submission/wallet-payment-transfer/scripts/validate_catalog.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +""" +validate_catalog.py + +CI entry point for the catalog metadata check (`catalog-check`). + +What it verifies +---------------- + - catalog/catalog.json exists and parses as JSON + - it lists exactly one lake dataset (`lake_cdc_events`) + - it lists every expected `wh_
` current-state dataset + - it lists every expected `wh_
_history` SCD2 dataset + - every entry has the required metadata fields + - layer labels are consistent with naming conventions +""" + +from __future__ import annotations + +import json +import os +import sys + +CATALOG_PATH = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "catalog", + "catalog.json", +) + + +# Same source tables used by the warehouse layer; redeclared here to keep +# the script standalone (no DuckDB import cost on a metadata check). +SOURCE_TABLES = ( + "customers", + "payment_methods", + "wallets", + "transfers", + "transfer_events", + "ledger_entries", +) + +REQUIRED_FIELDS = ( + "name", + "layer", + "type", + "description", + "owner", + "consumers", + "update_cadence", + "tags", +) + + +def expected_dataset_names() -> list[str]: + names = ["lake_cdc_events"] + names.extend(f"wh_{t}" for t in SOURCE_TABLES) + names.extend(f"wh_{t}_history" for t in SOURCE_TABLES) + return names + + +def validate() -> list[str]: + issues: list[str] = [] + if not os.path.exists(CATALOG_PATH): + return [f"catalog.json not found at {CATALOG_PATH}"] + + with open(CATALOG_PATH) as f: + try: + catalog = json.load(f) + except json.JSONDecodeError as exc: + return [f"catalog.json is not valid JSON: {exc}"] + + if "datasets" not in catalog or not isinstance(catalog["datasets"], list): + return ["catalog.json missing top-level 'datasets' list"] + + by_name = {d.get("name"): d for d in catalog["datasets"] if "name" in d} + + for name in expected_dataset_names(): + if name not in by_name: + issues.append(f"missing dataset entry: {name}") + continue + entry = by_name[name] + for field in REQUIRED_FIELDS: + if field not in entry or not entry[field]: + issues.append( + f"{name}: missing or empty required field '{field}'" + ) + + # layer label consistency + if "lake_cdc_events" in by_name: + if by_name["lake_cdc_events"].get("layer") != "lake": + issues.append("lake_cdc_events: layer must be 'lake'") + for name, entry in by_name.items(): + if name.startswith("wh_") and entry.get("layer") != "warehouse": + issues.append(f"{name}: layer must be 'warehouse'") + if name.endswith("_history") and entry.get("type") != "scd2-history": + issues.append(f"{name}: type must be 'scd2-history'") + + return issues + + +def main() -> int: + issues = validate() + if issues: + print("Catalog validation failures:") + for v in issues: + print(f" x {v}") + return 1 + n = len(expected_dataset_names()) + print(f"Catalog validation passed ({n} datasets verified).") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/submission/wallet-payment-transfer/source/__init__.py b/submission/wallet-payment-transfer/source/__init__.py new file mode 100644 index 0000000..be39610 --- /dev/null +++ b/submission/wallet-payment-transfer/source/__init__.py @@ -0,0 +1 @@ +"""Source layer — transactional schema for the wallet/payments/transfers domain.""" diff --git a/submission/wallet-payment-transfer/source/contracts.py b/submission/wallet-payment-transfer/source/contracts.py new file mode 100644 index 0000000..8582e08 --- /dev/null +++ b/submission/wallet-payment-transfer/source/contracts.py @@ -0,0 +1,191 @@ +""" +Versioned schema contracts for the source system. + +Each contract describes what the CDC pipeline depends on for a given source +table: + - column name -> {type, nullable} + - primary key + - foreign keys + - enum domains (subset of allowed values) + - contract version + +The contracts are explicit so the pipeline can fail closed when the source +deviates from them. Adding new optional columns is *additive* and does not +break the contract; the rules in pipeline/schema_detector.py treat: + + * dropped columns -> BREAKING + * type widening / narrowing -> BREAKING + * enum domain *shrunk* -> BREAKING (existing rows may rely + on a value the pipeline just lost) + * nullability tightened (allow -> deny) -> BREAKING (rows the pipeline + previously emitted may now fail) + * primary key changed -> BREAKING (replay/upsert depends + on the original PK) + * new nullable columns -> WARN (additive, allow-listed) + +The contract is the input the warehouse modeling layer is allowed to assume. +Anything outside the contract is best-effort. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Optional + + +@dataclass(frozen=True) +class ColumnContract: + name: str + type: str + nullable: bool + + +@dataclass(frozen=True) +class TableContract: + name: str + columns: tuple[ColumnContract, ...] + primary_key: str + foreign_keys: tuple[tuple[str, str, str], ...] = field(default_factory=tuple) + # (column, referenced_table, referenced_column) + enum_domains: tuple[tuple[str, frozenset[str]], ...] = field( + default_factory=tuple + ) + contract_version: int = 1 + + def column(self, name: str) -> Optional[ColumnContract]: + for c in self.columns: + if c.name == name: + return c + return None + + @property + def column_names(self) -> tuple[str, ...]: + return tuple(c.name for c in self.columns) + + +# NOTE: the type strings below are normalized — DuckDB DESCRIBE returns the +# normalised type names we compare against in pipeline/schema_detector.py. +CONTRACTS: dict[str, TableContract] = { + "customers": TableContract( + name="customers", + primary_key="customer_id", + columns=( + ColumnContract("customer_id", "VARCHAR", False), + ColumnContract("full_name", "VARCHAR", False), + ColumnContract("email", "VARCHAR", False), + ColumnContract("country_code", "VARCHAR", False), + ColumnContract("status", "VARCHAR", False), + ColumnContract("registration_date", "DATE", False), + ColumnContract("created_at", "TIMESTAMP", False), + ColumnContract("updated_at", "TIMESTAMP", False), + ), + enum_domains=( + ("status", frozenset({"active", "suspended", "closed"})), + ), + ), + "payment_methods": TableContract( + name="payment_methods", + primary_key="payment_method_id", + columns=( + ColumnContract("payment_method_id", "VARCHAR", False), + ColumnContract("customer_id", "VARCHAR", False), + ColumnContract("method_type", "VARCHAR", False), + ColumnContract("masked_identifier", "VARCHAR", False), + ColumnContract("is_default", "BOOLEAN", False), + ColumnContract("status", "VARCHAR", False), + ColumnContract("created_at", "TIMESTAMP", False), + ColumnContract("updated_at", "TIMESTAMP", False), + ), + foreign_keys=(("customer_id", "customers", "customer_id"),), + enum_domains=( + ("method_type", frozenset({"card", "bank", "upi"})), + ("status", frozenset({"active", "expired", "revoked"})), + ), + ), + "wallets": TableContract( + name="wallets", + primary_key="wallet_id", + columns=( + ColumnContract("wallet_id", "VARCHAR", False), + ColumnContract("customer_id", "VARCHAR", False), + ColumnContract("currency", "VARCHAR", False), + ColumnContract("balance", "DECIMAL(18,4)", False), + ColumnContract("status", "VARCHAR", False), + ColumnContract("created_at", "TIMESTAMP", False), + ColumnContract("updated_at", "TIMESTAMP", False), + ), + foreign_keys=(("customer_id", "customers", "customer_id"),), + enum_domains=( + ("currency", frozenset({"USD", "EUR", "GBP", "INR", "JPY"})), + ("status", frozenset({"active", "frozen", "closed"})), + ), + ), + "transfers": TableContract( + name="transfers", + primary_key="transfer_id", + columns=( + ColumnContract("transfer_id", "VARCHAR", False), + ColumnContract("from_wallet_id", "VARCHAR", False), + ColumnContract("to_wallet_id", "VARCHAR", False), + ColumnContract("payment_method_id", "VARCHAR", True), + ColumnContract("amount", "DECIMAL(18,4)", False), + ColumnContract("currency", "VARCHAR", False), + ColumnContract("status", "VARCHAR", False), + ColumnContract("reference", "VARCHAR", True), + ColumnContract("failure_reason", "VARCHAR", True), + ColumnContract("created_at", "TIMESTAMP", False), + ColumnContract("settled_at", "TIMESTAMP", True), + ), + foreign_keys=( + ("from_wallet_id", "wallets", "wallet_id"), + ("to_wallet_id", "wallets", "wallet_id"), + ("payment_method_id", "payment_methods", "payment_method_id"), + ), + enum_domains=( + ("status", frozenset({"pending", "settled", "failed", "reversed"})), + ("currency", frozenset({"USD", "EUR", "GBP", "INR", "JPY"})), + ), + ), + "transfer_events": TableContract( + name="transfer_events", + primary_key="transfer_event_id", + columns=( + ColumnContract("transfer_event_id", "VARCHAR", False), + ColumnContract("transfer_id", "VARCHAR", False), + ColumnContract("from_status", "VARCHAR", True), + ColumnContract("to_status", "VARCHAR", False), + ColumnContract("note", "VARCHAR", True), + ColumnContract("occurred_at", "TIMESTAMP", False), + ), + foreign_keys=(("transfer_id", "transfers", "transfer_id"),), + enum_domains=( + ("to_status", frozenset({"pending", "settled", "failed", "reversed"})), + ), + ), + "ledger_entries": TableContract( + name="ledger_entries", + primary_key="ledger_entry_id", + columns=( + ColumnContract("ledger_entry_id", "VARCHAR", False), + ColumnContract("wallet_id", "VARCHAR", False), + ColumnContract("transfer_id", "VARCHAR", True), + ColumnContract("direction", "VARCHAR", False), + ColumnContract("amount", "DECIMAL(18,4)", False), + ColumnContract("signed_amount", "DECIMAL(18,4)", False), + ColumnContract("balance_after", "DECIMAL(18,4)", False), + ColumnContract("occurred_at", "TIMESTAMP", False), + ), + foreign_keys=( + ("wallet_id", "wallets", "wallet_id"), + ("transfer_id", "transfers", "transfer_id"), + ), + enum_domains=( + ("direction", frozenset({"credit", "debit"})), + ), + ), +} + + +def expected_columns(table: str) -> list[str]: + """Convenience for the schema contract CI script.""" + return [c.name for c in CONTRACTS[table].columns] diff --git a/submission/wallet-payment-transfer/source/models.py b/submission/wallet-payment-transfer/source/models.py new file mode 100644 index 0000000..c01a00b --- /dev/null +++ b/submission/wallet-payment-transfer/source/models.py @@ -0,0 +1,264 @@ +""" +Source schema for a wallet / payments / transfers domain. + +Domain summary +-------------- +Customers register on the platform, link payment methods, hold wallets, and +move money between wallets through transfers. Each transfer goes through +several state transitions, and every balance change on a wallet is recorded +in an immutable ledger. + +Entity classification +--------------------- +Strong entities (independent identity, can exist on their own): + - customers : a registered user of the platform + - wallets : a balance account owned by a customer; one customer can + hold multiple wallets (e.g. one per currency) + - transfers : a money movement between two wallets; has its own + business identity (transfer_id), references two wallets, + and exists even if downstream events are missing + +Weak entities (lifecycle bound to a parent; cannot meaningfully exist alone): + - payment_methods : tied to a customer; remove the customer and the + method has no owner + - transfer_events : state-change records for a transfer; a transfer_event + only makes sense in the context of its transfer + - ledger_entries : append-only balance-change rows for a wallet; one + ledger_entry has no business meaning without its wallet + +Mix of data types covered: + - DECIMAL(18, 4) amounts and balances (4 dp for FX-safe precision) + - TIMESTAMP created_at / updated_at / settled_at + - VARCHAR enums status, direction, currency, method_type + - VARCHAR FKs identifier strings (UUID-style) + - Nullable cols settled_at, reference, failure_reason, payment_method_id + - DATE registration_date + +Indexes +------- +We add indexes that map to access patterns the CDC pipeline actually exercises: + - foreign-key indexes (referential lookup + delete cascade scans) + - updated_at indexes (incremental extraction by timestamp) + - status indexes (filtered scans for pending/failed transfers) + - composite (wallet_id, created_at) on ledger_entries (audit-by-wallet) + +Important invariants (mirrored downstream in the warehouse validation layer): + Source / system rules + - all PKs are unique and non-null + - FKs always point to a real parent row + - enum-typed columns hold one of a fixed set of values + - timestamp columns are non-null where required + Business rules + - wallet.balance >= 0 (no overdraft) + - transfer.amount > 0 + - transfer.from_wallet_id != transfer.to_wallet_id (no self transfer) + - transfer.settled_at >= transfer.created_at when settled + - ledger_entries.signed_amount sums to wallet.balance per wallet + (monetary integrity check) + - transfer status follows allowed state machine: + pending -> {settled, failed, reversed} + failed -> {pending} (retry) + settled -> {reversed} + reversed and failed are terminal otherwise +""" + +from __future__ import annotations + +import duckdb + + +# --------------------------------------------------------------------------- +# DDL +# --------------------------------------------------------------------------- + +CUSTOMERS_DDL = """ +CREATE TABLE IF NOT EXISTS customers ( + customer_id VARCHAR PRIMARY KEY, + full_name VARCHAR NOT NULL, + email VARCHAR NOT NULL, + country_code VARCHAR NOT NULL, -- ISO-3166 alpha-2 + status VARCHAR NOT NULL + CHECK (status IN ('active', 'suspended', 'closed')), + registration_date DATE NOT NULL, + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL +) +""" + +PAYMENT_METHODS_DDL = """ +CREATE TABLE IF NOT EXISTS payment_methods ( + payment_method_id VARCHAR PRIMARY KEY, + customer_id VARCHAR NOT NULL REFERENCES customers(customer_id), + method_type VARCHAR NOT NULL + CHECK (method_type IN ('card', 'bank', 'upi')), + masked_identifier VARCHAR NOT NULL, -- e.g. '****1234' + is_default BOOLEAN NOT NULL DEFAULT FALSE, + status VARCHAR NOT NULL + CHECK (status IN ('active', 'expired', 'revoked')), + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL +) +""" + +WALLETS_DDL = """ +CREATE TABLE IF NOT EXISTS wallets ( + wallet_id VARCHAR PRIMARY KEY, + customer_id VARCHAR NOT NULL REFERENCES customers(customer_id), + currency VARCHAR NOT NULL -- ISO-4217 + CHECK (currency IN ('USD', 'EUR', 'GBP', 'INR', 'JPY')), + balance DECIMAL(18, 4) NOT NULL DEFAULT 0 + CHECK (balance >= 0), + status VARCHAR NOT NULL + CHECK (status IN ('active', 'frozen', 'closed')), + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL +) +""" + +TRANSFERS_DDL = """ +CREATE TABLE IF NOT EXISTS transfers ( + transfer_id VARCHAR PRIMARY KEY, + from_wallet_id VARCHAR NOT NULL REFERENCES wallets(wallet_id), + to_wallet_id VARCHAR NOT NULL REFERENCES wallets(wallet_id), + payment_method_id VARCHAR REFERENCES payment_methods(payment_method_id), + amount DECIMAL(18, 4) NOT NULL CHECK (amount > 0), + currency VARCHAR NOT NULL + CHECK (currency IN ('USD', 'EUR', 'GBP', 'INR', 'JPY')), + status VARCHAR NOT NULL + CHECK (status IN ('pending', 'settled', + 'failed', 'reversed')), + reference VARCHAR, -- nullable: external memo + failure_reason VARCHAR, -- nullable + created_at TIMESTAMP NOT NULL, + settled_at TIMESTAMP, -- null until settled + CHECK (from_wallet_id <> to_wallet_id), + CHECK (settled_at IS NULL OR settled_at >= created_at) +) +""" + +TRANSFER_EVENTS_DDL = """ +CREATE TABLE IF NOT EXISTS transfer_events ( + transfer_event_id VARCHAR PRIMARY KEY, + transfer_id VARCHAR NOT NULL REFERENCES transfers(transfer_id), + from_status VARCHAR + CHECK (from_status IS NULL OR from_status IN + ('pending', 'settled', 'failed', 'reversed')), + to_status VARCHAR NOT NULL + CHECK (to_status IN ('pending', 'settled', + 'failed', 'reversed')), + note VARCHAR, + occurred_at TIMESTAMP NOT NULL +) +""" + +LEDGER_ENTRIES_DDL = """ +CREATE TABLE IF NOT EXISTS ledger_entries ( + ledger_entry_id VARCHAR PRIMARY KEY, + wallet_id VARCHAR NOT NULL REFERENCES wallets(wallet_id), + transfer_id VARCHAR REFERENCES transfers(transfer_id), + direction VARCHAR NOT NULL + CHECK (direction IN ('credit', 'debit')), + amount DECIMAL(18, 4) NOT NULL CHECK (amount > 0), + signed_amount DECIMAL(18, 4) NOT NULL, -- + for credit, - for debit + balance_after DECIMAL(18, 4) NOT NULL CHECK (balance_after >= 0), + occurred_at TIMESTAMP NOT NULL, + CHECK ((direction = 'credit' AND signed_amount = amount) OR + (direction = 'debit' AND signed_amount = -amount)) +) +""" + +# Indexes that match real CDC / lookup access patterns. +# Note: PKs already produce a unique index automatically; we add the others. +INDEX_DDL: list[str] = [ + "CREATE INDEX IF NOT EXISTS ix_customers_updated_at " + "ON customers(updated_at)", + "CREATE INDEX IF NOT EXISTS ix_customers_status " + "ON customers(status)", + "CREATE INDEX IF NOT EXISTS ix_payment_methods_customer " + "ON payment_methods(customer_id)", + "CREATE INDEX IF NOT EXISTS ix_wallets_customer " + "ON wallets(customer_id)", + "CREATE INDEX IF NOT EXISTS ix_wallets_updated_at " + "ON wallets(updated_at)", + "CREATE INDEX IF NOT EXISTS ix_transfers_status " + "ON transfers(status)", + "CREATE INDEX IF NOT EXISTS ix_transfers_from_wallet " + "ON transfers(from_wallet_id)", + "CREATE INDEX IF NOT EXISTS ix_transfers_to_wallet " + "ON transfers(to_wallet_id)", + "CREATE INDEX IF NOT EXISTS ix_transfers_created_at " + "ON transfers(created_at)", + "CREATE INDEX IF NOT EXISTS ix_transfer_events_transfer " + "ON transfer_events(transfer_id)", + "CREATE INDEX IF NOT EXISTS ix_ledger_wallet_time " + "ON ledger_entries(wallet_id, occurred_at)", + "CREATE INDEX IF NOT EXISTS ix_ledger_transfer " + "ON ledger_entries(transfer_id)", +] + + +# Tables in dependency order (parents before children) — used for create + drop. +TABLES_IN_ORDER: list[str] = [ + "customers", + "payment_methods", + "wallets", + "transfers", + "transfer_events", + "ledger_entries", +] + +PRIMARY_KEYS: dict[str, str] = { + "customers": "customer_id", + "payment_methods": "payment_method_id", + "wallets": "wallet_id", + "transfers": "transfer_id", + "transfer_events": "transfer_event_id", + "ledger_entries": "ledger_entry_id", +} + +# Allowed enum domains — mirrored in schema contracts and downstream validation. +ENUM_DOMAINS: dict[tuple[str, str], frozenset[str]] = { + ("customers", "status"): frozenset({"active", "suspended", "closed"}), + ("payment_methods", "method_type"): frozenset({"card", "bank", "upi"}), + ("payment_methods", "status"): frozenset({"active", "expired", "revoked"}), + ("wallets", "currency"): frozenset({"USD", "EUR", "GBP", "INR", "JPY"}), + ("wallets", "status"): frozenset({"active", "frozen", "closed"}), + ("transfers", "status"): frozenset( + {"pending", "settled", "failed", "reversed"} + ), + ("transfers", "currency"): frozenset({"USD", "EUR", "GBP", "INR", "JPY"}), + ("transfer_events", "to_status"): frozenset( + {"pending", "settled", "failed", "reversed"} + ), + ("ledger_entries", "direction"): frozenset({"credit", "debit"}), +} + +# Allowed status transitions for transfers (state machine). +ALLOWED_TRANSFER_TRANSITIONS: frozenset[tuple[str, str]] = frozenset({ + ("pending", "settled"), + ("pending", "failed"), + ("pending", "reversed"), + ("failed", "pending"), # retry + ("settled", "reversed"), +}) + + +def create_source_tables(conn: duckdb.DuckDBPyConnection) -> None: + """Create all source tables and indexes in dependency order.""" + for ddl in ( + CUSTOMERS_DDL, + PAYMENT_METHODS_DDL, + WALLETS_DDL, + TRANSFERS_DDL, + TRANSFER_EVENTS_DDL, + LEDGER_ENTRIES_DDL, + ): + conn.execute(ddl) + for ix in INDEX_DDL: + conn.execute(ix) + + +def drop_source_tables(conn: duckdb.DuckDBPyConnection) -> None: + """Drop all source tables in reverse dependency order. Useful in tests.""" + for table in reversed(TABLES_IN_ORDER): + conn.execute(f"DROP TABLE IF EXISTS {table}") diff --git a/submission/wallet-payment-transfer/source/seed.py b/submission/wallet-payment-transfer/source/seed.py new file mode 100644 index 0000000..014bbc9 --- /dev/null +++ b/submission/wallet-payment-transfer/source/seed.py @@ -0,0 +1,295 @@ +""" +Representative seed data for the wallet domain. + +Used by: + - tests (via the `seeded` fixture) + - scripts/run_pipeline.py (end-to-end demo) + +The seed builds a small but coherent slice of the system: + - 3 customers + - 3 payment methods (one per customer) + - 4 wallets (Alice holds two — USD + EUR) + - 3 transfers in different states (settled / pending / failed) + - matching transfer_events + - ledger_entries that *reconcile* to each wallet.balance per the + monetary integrity invariant SUM(signed_amount) == balance. + +Funding ledger entries (transfer_id=NULL) seed the wallet starting +balance; the remaining ledger entries flow from a real transfer. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import date, datetime, timezone +from decimal import Decimal + + +def _ts(year: int, month: int, day: int, hour: int = 0, minute: int = 0) -> datetime: + return datetime(year, month, day, hour, minute, tzinfo=timezone.utc) + + +@dataclass(frozen=True) +class SeedRow: + table: str + pk: str + data: dict + + +def build_seed_rows() -> list[SeedRow]: + """ + Return ordered list of seed rows. Order matters: parents first so + foreign keys are valid as each row is applied. + + Final reconciled state per wallet (verified by data quality checks): + w_alice_usd : balance = 75 (funded 100, debited 25) + w_alice_eur : balance = 0 (never funded) + w_bob_gbp : balance = 50 (funded 50) + w_chen_usd : balance = 50 (funded 25, credited 25) + """ + rows: list[SeedRow] = [] + t0 = _ts(2026, 6, 1, 9, 0) + + # ── customers ──────────────────────────────────────────────────────────── + rows.append(SeedRow("customers", "c_alice", { + "customer_id": "c_alice", + "full_name": "Alice Anderson", + "email": "alice@example.invalid", + "country_code": "US", + "status": "active", + "registration_date": date(2024, 1, 15), + "created_at": t0, + "updated_at": t0, + })) + rows.append(SeedRow("customers", "c_bob", { + "customer_id": "c_bob", + "full_name": "Bob Brown", + "email": "bob@example.invalid", + "country_code": "GB", + "status": "active", + "registration_date": date(2024, 3, 20), + "created_at": t0, + "updated_at": t0, + })) + rows.append(SeedRow("customers", "c_chen", { + "customer_id": "c_chen", + "full_name": "Chen Cheng", + "email": "chen@example.invalid", + "country_code": "US", + "status": "active", + "registration_date": date(2025, 11, 1), + "created_at": t0, + "updated_at": t0, + })) + + # ── payment methods ────────────────────────────────────────────────────── + rows.append(SeedRow("payment_methods", "pm_alice_card", { + "payment_method_id": "pm_alice_card", + "customer_id": "c_alice", + "method_type": "card", + "masked_identifier": "****4242", + "is_default": True, + "status": "active", + "created_at": t0, + "updated_at": t0, + })) + rows.append(SeedRow("payment_methods", "pm_bob_bank", { + "payment_method_id": "pm_bob_bank", + "customer_id": "c_bob", + "method_type": "bank", + "masked_identifier": "GB**0001", + "is_default": True, + "status": "active", + "created_at": t0, + "updated_at": t0, + })) + rows.append(SeedRow("payment_methods", "pm_chen_upi", { + "payment_method_id": "pm_chen_upi", + "customer_id": "c_chen", + "method_type": "upi", + "masked_identifier": "chen@upi", + "is_default": True, + "status": "active", + "created_at": t0, + "updated_at": t0, + })) + + # ── wallets (final reconciled balances) ────────────────────────────────── + rows.append(SeedRow("wallets", "w_alice_usd", { + "wallet_id": "w_alice_usd", + "customer_id": "c_alice", + "currency": "USD", + "balance": Decimal("75.0000"), # 100 funded, then -25 + "status": "active", + "created_at": t0, + "updated_at": _ts(2026, 6, 1, 10, 1), + })) + rows.append(SeedRow("wallets", "w_alice_eur", { + "wallet_id": "w_alice_eur", + "customer_id": "c_alice", + "currency": "EUR", + "balance": Decimal("0.0000"), + "status": "active", + "created_at": t0, + "updated_at": t0, + })) + rows.append(SeedRow("wallets", "w_bob_gbp", { + "wallet_id": "w_bob_gbp", + "customer_id": "c_bob", + "currency": "GBP", + "balance": Decimal("50.0000"), + "status": "active", + "created_at": t0, + "updated_at": t0, + })) + rows.append(SeedRow("wallets", "w_chen_usd", { + "wallet_id": "w_chen_usd", + "customer_id": "c_chen", + "currency": "USD", + "balance": Decimal("50.0000"), # 25 funded, then +25 + "status": "active", + "created_at": t0, + "updated_at": _ts(2026, 6, 1, 10, 1), + })) + + # ── transfers ──────────────────────────────────────────────────────────── + rows.append(SeedRow("transfers", "tx_settled_001", { + "transfer_id": "tx_settled_001", + "from_wallet_id": "w_alice_usd", + "to_wallet_id": "w_chen_usd", + "payment_method_id": "pm_alice_card", + "amount": Decimal("25.0000"), + "currency": "USD", + "status": "settled", + "reference": "rent-share", + "failure_reason": None, + "created_at": _ts(2026, 6, 1, 10, 0), + "settled_at": _ts(2026, 6, 1, 10, 1), + })) + rows.append(SeedRow("transfers", "tx_pending_002", { + "transfer_id": "tx_pending_002", + "from_wallet_id": "w_bob_gbp", + "to_wallet_id": "w_alice_eur", + "payment_method_id": "pm_bob_bank", + "amount": Decimal("10.0000"), + "currency": "GBP", + "status": "pending", + "reference": None, + "failure_reason": None, + "created_at": _ts(2026, 6, 1, 11, 0), + "settled_at": None, + })) + rows.append(SeedRow("transfers", "tx_failed_003", { + "transfer_id": "tx_failed_003", + "from_wallet_id": "w_chen_usd", + "to_wallet_id": "w_bob_gbp", + "payment_method_id": "pm_chen_upi", + "amount": Decimal("5.0000"), + "currency": "USD", + "status": "failed", + "reference": "groceries", + "failure_reason": "insufficient-funds", + "created_at": _ts(2026, 6, 1, 12, 0), + "settled_at": None, + })) + + # ── transfer_events ────────────────────────────────────────────────────── + rows.append(SeedRow("transfer_events", "te_001_a", { + "transfer_event_id": "te_001_a", + "transfer_id": "tx_settled_001", + "from_status": None, + "to_status": "pending", + "note": "created", + "occurred_at": _ts(2026, 6, 1, 10, 0), + })) + rows.append(SeedRow("transfer_events", "te_001_b", { + "transfer_event_id": "te_001_b", + "transfer_id": "tx_settled_001", + "from_status": "pending", + "to_status": "settled", + "note": "card-charged", + "occurred_at": _ts(2026, 6, 1, 10, 1), + })) + rows.append(SeedRow("transfer_events", "te_002_a", { + "transfer_event_id": "te_002_a", + "transfer_id": "tx_pending_002", + "from_status": None, + "to_status": "pending", + "note": "created", + "occurred_at": _ts(2026, 6, 1, 11, 0), + })) + rows.append(SeedRow("transfer_events", "te_003_a", { + "transfer_event_id": "te_003_a", + "transfer_id": "tx_failed_003", + "from_status": None, + "to_status": "pending", + "note": "created", + "occurred_at": _ts(2026, 6, 1, 12, 0), + })) + rows.append(SeedRow("transfer_events", "te_003_b", { + "transfer_event_id": "te_003_b", + "transfer_id": "tx_failed_003", + "from_status": "pending", + "to_status": "failed", + "note": "insufficient-funds", + "occurred_at": _ts(2026, 6, 1, 12, 1), + })) + + # ── ledger_entries (funding entries + transfer-driven movements) ───────── + # Funding for w_alice_usd: +100 + rows.append(SeedRow("ledger_entries", "le_alice_fund", { + "ledger_entry_id": "le_alice_fund", + "wallet_id": "w_alice_usd", + "transfer_id": None, + "direction": "credit", + "amount": Decimal("100.0000"), + "signed_amount": Decimal("100.0000"), + "balance_after": Decimal("100.0000"), + "occurred_at": t0, + })) + # Funding for w_bob_gbp: +50 + rows.append(SeedRow("ledger_entries", "le_bob_fund", { + "ledger_entry_id": "le_bob_fund", + "wallet_id": "w_bob_gbp", + "transfer_id": None, + "direction": "credit", + "amount": Decimal("50.0000"), + "signed_amount": Decimal("50.0000"), + "balance_after": Decimal("50.0000"), + "occurred_at": t0, + })) + # Funding for w_chen_usd: +25 + rows.append(SeedRow("ledger_entries", "le_chen_fund", { + "ledger_entry_id": "le_chen_fund", + "wallet_id": "w_chen_usd", + "transfer_id": None, + "direction": "credit", + "amount": Decimal("25.0000"), + "signed_amount": Decimal("25.0000"), + "balance_after": Decimal("25.0000"), + "occurred_at": t0, + })) + # Settled transfer 001: alice debit 25 -> 75 + rows.append(SeedRow("ledger_entries", "le_001_debit", { + "ledger_entry_id": "le_001_debit", + "wallet_id": "w_alice_usd", + "transfer_id": "tx_settled_001", + "direction": "debit", + "amount": Decimal("25.0000"), + "signed_amount": Decimal("-25.0000"), + "balance_after": Decimal("75.0000"), + "occurred_at": _ts(2026, 6, 1, 10, 1), + })) + # Settled transfer 001: chen credit 25 -> 50 + rows.append(SeedRow("ledger_entries", "le_001_credit", { + "ledger_entry_id": "le_001_credit", + "wallet_id": "w_chen_usd", + "transfer_id": "tx_settled_001", + "direction": "credit", + "amount": Decimal("25.0000"), + "signed_amount": Decimal("25.0000"), + "balance_after": Decimal("50.0000"), + "occurred_at": _ts(2026, 6, 1, 10, 1), + })) + + return rows diff --git a/submission/wallet-payment-transfer/tests/__init__.py b/submission/wallet-payment-transfer/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/submission/wallet-payment-transfer/tests/conftest.py b/submission/wallet-payment-transfer/tests/conftest.py new file mode 100644 index 0000000..5fab307 --- /dev/null +++ b/submission/wallet-payment-transfer/tests/conftest.py @@ -0,0 +1,59 @@ +"""Shared pytest fixtures for the wallet/payments CDC pipeline tests.""" + +from __future__ import annotations + +from datetime import datetime, timezone + +import duckdb +import pytest + +from pipeline.cdc import CDCCapture +from pipeline.orchestrator import bootstrap, run_tick +from pipeline.schema_detector import SchemaDetector +from source.models import create_source_tables +from source.seed import build_seed_rows + + +def _ts() -> datetime: + return datetime.now(timezone.utc) + + +@pytest.fixture() +def conn() -> duckdb.DuckDBPyConnection: + """Fresh in-memory DuckDB with source + lake + warehouse + checkpoints.""" + c = duckdb.connect(":memory:") + create_source_tables(c) + bootstrap(c) + return c + + +@pytest.fixture() +def capture() -> CDCCapture: + """Empty CDC capture log.""" + return CDCCapture() + + +@pytest.fixture() +def detector(conn: duckdb.DuckDBPyConnection) -> SchemaDetector: + return SchemaDetector(conn) + + +@pytest.fixture() +def seeded( + conn: duckdb.DuckDBPyConnection, + capture: CDCCapture, + detector: SchemaDetector, +): + """ + DuckDB connection pre-loaded with the canonical wallet seed + (3 customers, 3 payment methods, 4 wallets, 3 transfers, + 5 transfer_events, 5 ledger_entries). + + Returns (conn, capture). + """ + for row in build_seed_rows(): + capture.insert(row.table, row.pk, row.data) + + stats = run_tick(conn, capture, detector=detector) + assert not stats["aborted"] + return conn, capture diff --git a/submission/wallet-payment-transfer/tests/test_catalog.py b/submission/wallet-payment-transfer/tests/test_catalog.py new file mode 100644 index 0000000..522d53d --- /dev/null +++ b/submission/wallet-payment-transfer/tests/test_catalog.py @@ -0,0 +1,198 @@ +""" +Tests — data catalog correctness. + +The catalog (catalog/catalog.json) is the canonical entry point for downstream +consumers. It must: + + - exist at a fixed location + - parse as valid JSON + - declare every warehouse + lake dataset the pipeline produces + - carry the required descriptive fields per dataset + - use a consistent vocabulary for layer / type / sla_class + - flag every PII / monetary dataset with a tag for governance +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +CATALOG_PATH = ( + Path(__file__).resolve().parents[1] / "catalog" / "catalog.json" +) + +EXPECTED_DATASETS: set[str] = { + "lake_cdc_events", + "wh_customers", + "wh_payment_methods", + "wh_wallets", + "wh_transfers", + "wh_transfer_events", + "wh_ledger_entries", + "wh_customers_history", + "wh_payment_methods_history", + "wh_wallets_history", + "wh_transfers_history", + "wh_transfer_events_history", + "wh_ledger_entries_history", +} + +REQUIRED_FIELDS = ( + "name", "layer", "type", "description", + "owner", "consumers", "update_cadence", + "sla_class", "tags", "primary_key", +) + +ALLOWED_LAYERS = {"lake", "warehouse"} +ALLOWED_SLA = {"tier-1", "tier-2", "tier-3"} +ALLOWED_TYPES = { + "append-only-change-log", + "current-state", + "scd2-history", +} + + +@pytest.fixture(scope="module") +def catalog() -> dict: + raw = CATALOG_PATH.read_text(encoding="utf-8") + return json.loads(raw) + + +# ── basics ─────────────────────────────────────────────────────────────────── + + +def test_catalog_file_exists(): + assert CATALOG_PATH.exists(), f"missing catalog at {CATALOG_PATH}" + + +def test_catalog_is_valid_json(catalog): + assert isinstance(catalog, dict) + assert "datasets" in catalog + assert isinstance(catalog["datasets"], list) + + +def test_catalog_top_level_metadata(catalog): + for key in ("catalog_version", "domain", "owner"): + assert key in catalog, f"missing top-level field: {key}" + assert catalog["domain"] == "wallet-payments-transfers" + + +# ── completeness ───────────────────────────────────────────────────────────── + + +def test_catalog_lists_every_pipeline_dataset(catalog): + declared = {d["name"] for d in catalog["datasets"]} + missing = EXPECTED_DATASETS - declared + assert not missing, f"catalog missing datasets: {sorted(missing)}" + + +def test_catalog_has_no_unexpected_datasets(catalog): + declared = {d["name"] for d in catalog["datasets"]} + extra = declared - EXPECTED_DATASETS + assert not extra, f"catalog has unexpected datasets: {sorted(extra)}" + + +# ── per-dataset required fields ────────────────────────────────────────────── + + +def test_every_dataset_has_required_fields(catalog): + for ds in catalog["datasets"]: + for f in REQUIRED_FIELDS: + assert f in ds, f"{ds.get('name')} missing field {f}" + + +def test_every_dataset_has_a_schema_or_extension(catalog): + """Current-state has schema; SCD2 history has schema_extension.""" + for ds in catalog["datasets"]: + assert "schema" in ds or "schema_extension" in ds, ( + f"{ds['name']} declares no schema or schema_extension" + ) + + +# ── controlled vocabulary ──────────────────────────────────────────────────── + + +def test_layers_use_allowed_vocabulary(catalog): + for ds in catalog["datasets"]: + assert ds["layer"] in ALLOWED_LAYERS, ( + f"{ds['name']} has unknown layer: {ds['layer']}" + ) + + +def test_types_use_allowed_vocabulary(catalog): + for ds in catalog["datasets"]: + assert ds["type"] in ALLOWED_TYPES, ( + f"{ds['name']} has unknown type: {ds['type']}" + ) + + +def test_sla_class_uses_allowed_vocabulary(catalog): + for ds in catalog["datasets"]: + assert ds["sla_class"] in ALLOWED_SLA, ( + f"{ds['name']} has unknown sla_class: {ds['sla_class']}" + ) + + +# ── consistency between layer and type ─────────────────────────────────────── + + +def test_lake_layer_uses_change_log_type(catalog): + for ds in catalog["datasets"]: + if ds["layer"] == "lake": + assert ds["type"] == "append-only-change-log" + + +def test_wh_history_tables_are_scd2(catalog): + for ds in catalog["datasets"]: + if ds["name"].endswith("_history"): + assert ds["type"] == "scd2-history" + + +def test_wh_current_tables_are_current_state(catalog): + for ds in catalog["datasets"]: + if ( + ds["layer"] == "warehouse" + and not ds["name"].endswith("_history") + ): + assert ds["type"] == "current-state" + + +# ── governance tagging ─────────────────────────────────────────────────────── + + +def test_money_critical_datasets_carry_appropriate_tags(catalog): + """Wallets / transfers / ledger feed finance — all must be tier-1.""" + money_critical = { + "wh_wallets", "wh_transfers", "wh_ledger_entries", + "wh_wallets_history", "wh_transfers_history", + "wh_ledger_entries_history", + } + for ds in catalog["datasets"]: + if ds["name"] in money_critical: + assert ds["sla_class"] == "tier-1", ( + f"{ds['name']} feeds finance — must be tier-1" + ) + + +def test_history_tables_tagged_for_time_travel(catalog): + """SCD2 datasets that back time-travel queries must be tagged.""" + for ds in catalog["datasets"]: + if ds["type"] == "scd2-history" and "balance" in ( + ds.get("tags") or [] + ): + assert "time-travel" in ds["tags"] + + +def test_consumers_is_a_non_empty_list(catalog): + for ds in catalog["datasets"]: + consumers = ds["consumers"] + assert isinstance(consumers, list) + assert consumers, f"{ds['name']} has empty consumers list" + + +def test_primary_key_is_a_non_empty_list(catalog): + for ds in catalog["datasets"]: + pk = ds["primary_key"] + assert isinstance(pk, list) and pk diff --git a/submission/wallet-payment-transfer/tests/test_cdc.py b/submission/wallet-payment-transfer/tests/test_cdc.py new file mode 100644 index 0000000..9faac53 --- /dev/null +++ b/submission/wallet-payment-transfer/tests/test_cdc.py @@ -0,0 +1,192 @@ +""" +Tests — CDC capture correctness. + +Covers: + - insert / update / delete capture + - sequence monotonicity + - replay window (records_since) + - duplicate appearance (the lake retains every event for the same PK) + - invalid operation rejection + - before/after image presence per operation type + - schema-aware capture refuses writes when the source is incompatible +""" + +from __future__ import annotations + +from datetime import datetime, timezone + +import pytest + +from pipeline.cdc import ( + CDCCapture, + CDCRecord, + SchemaContractViolation, +) +from pipeline.schema_detector import SchemaDetector + + +# ── insert / update / delete ────────────────────────────────────────────────── + + +def test_insert_record_carries_after_image_only(capture): + rec = capture.insert("customers", "c1", {"customer_id": "c1"}) + assert rec.operation == "insert" + assert rec.before is None + assert rec.after == {"customer_id": "c1"} + + +def test_update_record_carries_before_and_after(capture): + capture.insert("customers", "c1", {"customer_id": "c1", "status": "active"}) + rec = capture.update( + "customers", "c1", + before={"customer_id": "c1", "status": "active"}, + after={"customer_id": "c1", "status": "suspended"}, + ) + assert rec.operation == "update" + assert rec.before["status"] == "active" + assert rec.after["status"] == "suspended" + + +def test_delete_record_carries_before_image_only(capture): + capture.insert("customers", "c1", {"customer_id": "c1"}) + rec = capture.delete("customers", "c1", before={"customer_id": "c1"}) + assert rec.operation == "delete" + assert rec.after is None + assert rec.before == {"customer_id": "c1"} + + +def test_invalid_operation_raises(capture): + with pytest.raises(ValueError, match="Invalid CDC operation"): + CDCRecord( + operation="upsert", + table="customers", + primary_key="c1", + before=None, + after={"x": 1}, + ) + + +def test_insert_without_after_raises(): + with pytest.raises(ValueError, match="must carry an 'after'"): + CDCRecord( + operation="insert", + table="customers", + primary_key="c1", + before=None, + after=None, + ) + + +def test_delete_without_before_raises(): + with pytest.raises(ValueError, match="must carry a 'before'"): + CDCRecord( + operation="delete", + table="customers", + primary_key="c1", + before=None, + after={"x": 1}, + ) + + +# ── sequence monotonicity ──────────────────────────────────────────────────── + + +def test_sequences_strictly_increasing(capture): + r1 = capture.insert("customers", "c1", {"customer_id": "c1"}) + r2 = capture.insert("customers", "c2", {"customer_id": "c2"}) + r3 = capture.update( + "customers", "c1", + before={"customer_id": "c1"}, + after={"customer_id": "c1"}, + ) + assert r1.sequence < r2.sequence < r3.sequence + + +def test_latest_sequence_reflects_last_record(capture): + capture.insert("customers", "c1", {"customer_id": "c1"}) + last = capture.insert("customers", "c2", {"customer_id": "c2"}) + assert capture.latest_sequence == last.sequence + + +# ── replay window ──────────────────────────────────────────────────────────── + + +def test_records_since_returns_only_records_after_offset(capture): + capture.insert("customers", "c1", {"customer_id": "c1"}) + capture.insert("customers", "c2", {"customer_id": "c2"}) + checkpoint = capture.latest_sequence + r3 = capture.insert("customers", "c3", {"customer_id": "c3"}) + + replayed = capture.records_since(checkpoint) + assert len(replayed) == 1 + assert replayed[0].sequence == r3.sequence + + +def test_records_since_zero_returns_everything(capture): + capture.insert("customers", "c1", {"customer_id": "c1"}) + capture.insert("customers", "c2", {"customer_id": "c2"}) + capture.delete("customers", "c1", before={"customer_id": "c1"}) + assert len(capture.records_since(0)) == 3 + + +def test_records_since_latest_returns_empty(capture): + capture.insert("customers", "c1", {"customer_id": "c1"}) + assert capture.records_since(capture.latest_sequence) == [] + + +# ── multiple events for same PK appear in log ──────────────────────────────── + + +def test_same_pk_can_appear_multiple_times(capture): + """Lake-level dedup is the orchestrator's job; capture itself is append-only.""" + capture.insert( + "customers", "c1", {"customer_id": "c1", "status": "active"} + ) + capture.update( + "customers", "c1", + before={"customer_id": "c1", "status": "active"}, + after={"customer_id": "c1", "status": "suspended"}, + ) + capture.update( + "customers", "c1", + before={"customer_id": "c1", "status": "suspended"}, + after={"customer_id": "c1", "status": "closed"}, + ) + pk1 = [r for r in capture.log if r.primary_key == "c1"] + assert len(pk1) == 3 + + +# ── captured_at metadata ───────────────────────────────────────────────────── + + +def test_captured_at_is_utc_datetime(capture): + rec = capture.insert("customers", "c1", {"customer_id": "c1"}) + assert isinstance(rec.captured_at, datetime) + assert rec.captured_at.tzinfo is not None + + +def test_captured_at_can_be_overridden_for_replay(capture): + fixed = datetime(2026, 6, 1, tzinfo=timezone.utc) + rec = capture.insert( + "customers", "c1", {"customer_id": "c1"}, captured_at=fixed + ) + assert rec.captured_at == fixed + + +# ── schema-aware capture (stop-the-line at write time) ─────────────────────── + + +def test_capture_with_strict_schema_refuses_writes_on_drop(conn, detector): + """If a contract column is missing on the source, every write must fail.""" + conn.execute("DROP TABLE IF EXISTS ledger_entries") + conn.execute("DROP TABLE IF EXISTS transfer_events") + conn.execute("DROP TABLE IF EXISTS transfers") + conn.execute("DROP TABLE IF EXISTS payment_methods") + conn.execute("DROP TABLE IF EXISTS wallets") + conn.execute("ALTER TABLE customers DROP COLUMN email") + + strict = CDCCapture(detector, require_compatible_schema=True) + with pytest.raises(SchemaContractViolation): + strict.insert("customers", "c1", {"customer_id": "c1"}) + + assert strict.log == [] diff --git a/submission/wallet-payment-transfer/tests/test_data_quality.py b/submission/wallet-payment-transfer/tests/test_data_quality.py new file mode 100644 index 0000000..c2169ce --- /dev/null +++ b/submission/wallet-payment-transfer/tests/test_data_quality.py @@ -0,0 +1,171 @@ +""" +Tests — data quality / validation parity. + +For every rule defined in scripts/run_data_quality_checks.py we verify +that the seeded warehouse passes; we also seed a deliberately broken +state for a representative subset to confirm each rule actually fires. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from decimal import Decimal + +import pytest + +from pipeline.cdc import CDCCapture +from pipeline.orchestrator import bootstrap, run_tick +from scripts.run_data_quality_checks import all_rules, run_checks + + +def _ts() -> datetime: + return datetime(2026, 6, 5, 12, tzinfo=timezone.utc) + + +# ── happy path ─────────────────────────────────────────────────────────────── + + +def test_seeded_warehouse_passes_all_rules(seeded): + conn, _ = seeded + failures = run_checks(conn) + assert failures == [], ( + "Seeded warehouse should pass all rules. Failures:\n " + + "\n ".join(f"{r.rule_id}: {n}" for r, n in failures) + ) + + +def test_rules_cover_all_categories(): + cats = {r.category for r in all_rules()} + assert {"system", "business"}.issubset(cats) + + +# ── system rule firing ─────────────────────────────────────────────────────── + + +def test_pk_uniqueness_rule_fires_on_duplicate(seeded): + conn, _ = seeded + # Force a duplicate in wh_customers by direct INSERT + conn.execute( + "INSERT INTO wh_customers VALUES " + "('c_alice', 'Dup', 'd@x.test', 'US', 'active', " + "DATE '2024-01-15', ?, ?, 999, FALSE)", + [_ts(), _ts()], + ) + failures = run_checks(conn) + rule_ids = {r.rule_id for r, _ in failures} + assert "sys.pk_unique.customers" in rule_ids + + +def test_enum_rule_fires_on_unknown_value(seeded): + conn, _ = seeded + conn.execute( + "UPDATE wh_customers SET status = 'banned' WHERE customer_id = 'c_alice'" + ) + failures = run_checks(conn) + rule_ids = {r.rule_id for r, _ in failures} + assert "sys.enum.customers.status" in rule_ids + + +def test_fk_rule_fires_on_orphan(seeded): + conn, _ = seeded + conn.execute( + "UPDATE wh_wallets SET customer_id = 'c_missing' " + "WHERE wallet_id = 'w_alice_usd'" + ) + failures = run_checks(conn) + rule_ids = {r.rule_id for r, _ in failures} + assert "sys.fk.wallets.customer_id" in rule_ids + + +# ── business rule firing ───────────────────────────────────────────────────── + + +def test_negative_balance_rule_fires(seeded): + conn, _ = seeded + conn.execute( + "UPDATE wh_wallets SET balance = -1.0000 " + "WHERE wallet_id = 'w_bob_gbp'" + ) + failures = run_checks(conn) + rule_ids = {r.rule_id for r, _ in failures} + assert "biz.wallet.balance_non_negative" in rule_ids + + +def test_settled_at_after_created_rule_fires(seeded): + conn, _ = seeded + conn.execute( + "UPDATE wh_transfers SET settled_at = TIMESTAMP '2020-01-01' " + "WHERE transfer_id = 'tx_settled_001'" + ) + failures = run_checks(conn) + rule_ids = {r.rule_id for r, _ in failures} + assert "biz.transfer.settled_at_after_created" in rule_ids + + +def test_ledger_signed_amount_consistency_rule_fires(seeded): + conn, _ = seeded + conn.execute( + "UPDATE wh_ledger_entries SET signed_amount = 99 " + "WHERE ledger_entry_id = 'le_001_debit'" + ) + failures = run_checks(conn) + rule_ids = {r.rule_id for r, _ in failures} + assert "biz.ledger.signed_amount_consistent" in rule_ids + + +def test_ledger_reconciles_to_wallet_balance_rule_fires(seeded): + conn, _ = seeded + # Break the reconciliation: bump wallet balance away from ledger sum. + conn.execute( + "UPDATE wh_wallets SET balance = balance + 1 " + "WHERE wallet_id = 'w_alice_usd'" + ) + failures = run_checks(conn) + rule_ids = {r.rule_id for r, _ in failures} + assert "biz.ledger.reconciles_to_wallet_balance" in rule_ids + + +def test_transfer_status_transition_rule_fires(seeded): + conn, _ = seeded + # Insert an illegal transition: settled -> pending (not allowed) + conn.execute( + "INSERT INTO wh_transfer_events VALUES (" + "'te_bad', 'tx_settled_001', 'settled', 'pending', " + "'illegal-transition', ?, 999, FALSE)", + [_ts()], + ) + failures = run_checks(conn) + rule_ids = {r.rule_id for r, _ in failures} + assert "biz.transfer_event.allowed_transition" in rule_ids + + +# ── parity sanity: every contract column with NOT NULL has a rule ──────────── + + +def test_every_contract_not_null_has_a_warehouse_rule(): + rule_ids = {r.rule_id for r in all_rules()} + from source.contracts import CONTRACTS + for table, contract in CONTRACTS.items(): + for col in contract.columns: + if col.nullable: + continue + expected = f"sys.not_null.{table}.{col.name}" + assert expected in rule_ids, ( + f"missing parity rule {expected} — every NOT NULL " + "contract column must be checked downstream" + ) + + +# ── soft-deletes are excluded from quality rules ────────────────────────────── + + +def test_deleted_rows_do_not_trigger_validation_failures(seeded): + conn, capture = seeded + capture.delete( + "transfers", "tx_pending_002", + before={"transfer_id": "tx_pending_002"}, + ) + run_tick(conn, capture) + # _deleted=true rows must be ignored by every rule. + failures = run_checks(conn) + assert failures == [] diff --git a/submission/wallet-payment-transfer/tests/test_e2e.py b/submission/wallet-payment-transfer/tests/test_e2e.py new file mode 100644 index 0000000..6b37334 --- /dev/null +++ b/submission/wallet-payment-transfer/tests/test_e2e.py @@ -0,0 +1,306 @@ +""" +Tests — end-to-end pipeline scenarios. + +These tests exercise the full happy / unhappy paths through: + source schema → CDC capture → lake → warehouse → quality checks → catalog + +Scenarios: + 1. Cold start: source + bootstrap + seed → all layers populated and consistent + 2. Hot loop: incremental change → re-run tick → only deltas applied + 3. Restart: second tick from same checkpoint is a no-op + 4. Stop-the-line: schema break aborts before any data is moved + 5. Recovery: fixing the schema lets the pipeline resume from where it left off + 6. Restore: rolling current state back to T does not corrupt history or DQ + 7. Quality parity: warehouse rules pass on the seeded state +""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from decimal import Decimal + +import pytest + +from pipeline.cdc import CDCCapture, SchemaContractViolation +from pipeline.orchestrator import bootstrap, run_tick +from pipeline.warehouse import ( + apply_cdc_records, + restore_current_state_to, + state_at, +) +from scripts.run_data_quality_checks import run_checks +from source.seed import build_seed_rows + + +def _ts(seconds: int = 0) -> datetime: + # Far enough in the future that we are always after the seed's + # captured_at (which uses datetime.now() at test-run time). + base = datetime(2030, 1, 1, 12, 0, tzinfo=timezone.utc) + return base + timedelta(seconds=seconds) + + +# ── 1. cold start ──────────────────────────────────────────────────────────── + + +def test_cold_start_populates_all_layers(seeded): + conn, capture = seeded + n_lake = conn.execute("SELECT COUNT(*) FROM lake_cdc_events").fetchone()[0] + n_wh = conn.execute("SELECT COUNT(*) FROM wh_customers").fetchone()[0] + n_hist = conn.execute( + "SELECT COUNT(*) FROM wh_customers_history WHERE _is_current" + ).fetchone()[0] + + assert n_lake == capture.latest_sequence + assert n_wh == 3 + assert n_hist == 3 + + +def test_cold_start_passes_all_quality_rules(seeded): + conn, _ = seeded + failures = run_checks(conn) + assert failures == [], ( + "post-seed warehouse should be clean, got: " + + ", ".join(f"{r.rule_id}: {n}" for r, n in failures) + ) + + +# ── 2. hot loop ────────────────────────────────────────────────────────────── + + +def test_incremental_update_only_applies_delta(seeded, detector): + conn, capture = seeded + cp_before = conn.execute( + "SELECT sequence FROM pipeline_checkpoints WHERE stage = 'warehouse'" + ).fetchone()[0] + + capture.update( + "wallets", "w_alice_usd", + before={"wallet_id": "w_alice_usd", "balance": Decimal("75.0000")}, + after={ + "wallet_id": "w_alice_usd", + "customer_id": "c_alice", + "currency": "USD", + "balance": Decimal("75.0000"), + "status": "frozen", # status change + "created_at": datetime(2026, 6, 1, 9, tzinfo=timezone.utc), + "updated_at": _ts(), + }, + captured_at=_ts(), + ) + + stats = run_tick(conn, capture, detector=detector) + assert stats["lake_written"] == 1 + assert stats["warehouse_applied"] == 1 + assert stats["checkpoint_warehouse"] > cp_before + + +# ── 3. restart / no-op ─────────────────────────────────────────────────────── + + +def test_second_tick_with_no_new_events_is_noop(seeded, detector): + conn, capture = seeded + stats = run_tick(conn, capture, detector=detector) + assert stats["lake_written"] == 0 + assert stats["warehouse_applied"] == 0 + + +def test_restart_resumes_from_checkpoint(detector_factory_fresh_conn): + """ + Simulate a process restart: + tick 1 -> seed half the rows + tick 2 -> add the rest, but with a "fresh" detector instance + The pipeline should pick up exactly where checkpoint stopped. + """ + conn, detector = detector_factory_fresh_conn + bootstrap(conn) + capture = CDCCapture() + rows = build_seed_rows() + half = len(rows) // 2 + + for r in rows[:half]: + capture.insert(r.table, r.pk, r.data) + s1 = run_tick(conn, capture, detector=detector) + assert s1["lake_written"] == half + + for r in rows[half:]: + capture.insert(r.table, r.pk, r.data) + s2 = run_tick(conn, capture, detector=detector) + assert s2["lake_written"] == len(rows) - half + assert s2["checkpoint_warehouse"] == capture.latest_sequence + + +# ── 4. stop-the-line ───────────────────────────────────────────────────────── + + +def test_breaking_drift_blocks_lake_and_warehouse(seeded, detector): + """ + Even if there is a fresh CDC event waiting, a breaking drift on the + source must abort BEFORE any data is moved. + """ + conn, capture = seeded + n_lake_before = conn.execute( + "SELECT COUNT(*) FROM lake_cdc_events" + ).fetchone()[0] + + # Make capture think a customer changed. + capture.update( + "customers", "c_alice", + before={"customer_id": "c_alice", "status": "active"}, + after={"customer_id": "c_alice", "status": "suspended"}, + captured_at=_ts(), + ) + + # Now corrupt the source schema (drop a contracted column). + for t in ("ledger_entries", "transfer_events", "transfers", + "payment_methods", "wallets"): + conn.execute(f"DROP TABLE IF EXISTS {t}") + conn.execute("ALTER TABLE customers DROP COLUMN email") + + stats = run_tick(conn, capture, detector=detector) + assert stats["aborted"] + assert stats["reason"] == "schema_contract_violation" + + n_lake_after = conn.execute( + "SELECT COUNT(*) FROM lake_cdc_events" + ).fetchone()[0] + # Nothing was written. + assert n_lake_before == n_lake_after + + +def test_strict_capture_refuses_writes_when_drift_breaking(seeded, detector): + conn, _ = seeded + for t in ("ledger_entries", "transfer_events", "transfers", + "payment_methods", "wallets"): + conn.execute(f"DROP TABLE IF EXISTS {t}") + conn.execute("ALTER TABLE customers DROP COLUMN email") + + strict = CDCCapture(detector, require_compatible_schema=True) + with pytest.raises(SchemaContractViolation): + strict.insert("customers", "c_x", {"customer_id": "c_x"}) + + +# ── 5. recovery from breaking drift ────────────────────────────────────────── + + +def test_pipeline_resumes_after_schema_repaired(seeded, detector): + """ + Schema break -> pipeline aborts -> source is repaired -> next tick + successfully drains queued events. + """ + from source.models import create_source_tables, drop_source_tables + + conn, capture = seeded + + capture.update( + "customers", "c_alice", + before={"customer_id": "c_alice", "status": "active"}, + after={"customer_id": "c_alice", "status": "suspended"}, + captured_at=_ts(), + ) + + # Break source. + for t in ("ledger_entries", "transfer_events", "transfers", + "payment_methods", "wallets"): + conn.execute(f"DROP TABLE IF EXISTS {t}") + conn.execute("ALTER TABLE customers DROP COLUMN email") + + s_aborted = run_tick(conn, capture, detector=detector) + assert s_aborted["aborted"] + + # Repair: rebuild source schema to match contracts. + drop_source_tables(conn) + create_source_tables(conn) + s_resumed = run_tick(conn, capture, detector=detector) + assert not s_resumed["aborted"] + assert s_resumed["warehouse_applied"] >= 1 + + +# ── 6. restore (history preserved) ─────────────────────────────────────────── + + +def test_restore_does_not_break_dq_rules(seeded): + """ + After restoring wallets to their seeded state, all DQ rules must + still pass. This catches restore bugs that would, e.g., leave + a stale _cdc_seq blocking future merges or violate ledger + reconciliation. + """ + conn, capture = seeded + + # Push a benign update on alice's wallet. + capture.update( + "wallets", "w_alice_usd", + before={"wallet_id": "w_alice_usd", "balance": Decimal("75.0000")}, + after={ + "wallet_id": "w_alice_usd", + "customer_id": "c_alice", + "currency": "USD", + "balance": Decimal("75.0000"), # unchanged + "status": "frozen", + "created_at": datetime(2026, 6, 1, 9, tzinfo=timezone.utc), + "updated_at": _ts(), + }, + captured_at=_ts(), + ) + apply_cdc_records(conn, capture.records_since(capture.latest_sequence - 1)) + + # Roll wh_wallets back to BEFORE the update. + restore_current_state_to(conn, "wallets", _ts(-1)) + + # Sanity: all wallets present, balance == seed + bal = conn.execute( + "SELECT balance FROM wh_wallets WHERE wallet_id = 'w_alice_usd'" + ).fetchone()[0] + assert bal == Decimal("75.0000") + + # All quality rules must still pass. + failures = run_checks(conn) + assert failures == [], ( + "restore should not corrupt DQ; got: " + + ", ".join(f"{r.rule_id}: {n}" for r, n in failures) + ) + + +# ── 7. delete + reinsert lifecycle ─────────────────────────────────────────── + + +def test_delete_then_reinsert_same_pk_is_handled(seeded): + conn, capture = seeded + capture.delete( + "transfers", "tx_failed_003", + before={"transfer_id": "tx_failed_003"}, + captured_at=_ts(0), + ) + apply_cdc_records(conn, capture.records_since(capture.latest_sequence - 1)) + + # Reinsert with the same PK after the delete (rare but possible). + capture.insert( + "transfers", "tx_failed_003", { + "transfer_id": "tx_failed_003", + "from_wallet_id": "w_chen_usd", + "to_wallet_id": "w_bob_gbp", + "payment_method_id": "pm_chen_upi", + "amount": Decimal("5.0000"), + "currency": "USD", + "status": "pending", + "reference": "groceries-retry", + "failure_reason": None, + "created_at": _ts(60), + "settled_at": None, + }, + captured_at=_ts(60), + ) + apply_cdc_records(conn, capture.records_since(capture.latest_sequence - 1)) + + row = conn.execute( + "SELECT status, _deleted FROM wh_transfers WHERE transfer_id = 'tx_failed_003'" + ).fetchone() + assert row == ("pending", False) + + +# ── fixtures local to this module ──────────────────────────────────────────── + + +@pytest.fixture() +def detector_factory_fresh_conn(conn, detector): + return conn, detector diff --git a/submission/wallet-payment-transfer/tests/test_lake.py b/submission/wallet-payment-transfer/tests/test_lake.py new file mode 100644 index 0000000..5379409 --- /dev/null +++ b/submission/wallet-payment-transfer/tests/test_lake.py @@ -0,0 +1,138 @@ +""" +Tests — lake immutability and completeness. + +Covers: + - every CDC operation lands in lake_cdc_events + - row count is monotonically non-decreasing + - duplicate sequence is rejected by PK (or silently deduped) + - before / after JSON is round-trippable + - replay_records returns events in sequence order + - lake retains every operation for the same PK +""" + +from __future__ import annotations + +import json + +from pipeline.lake import append_to_lake, replay_records + + +def test_seed_writes_all_events_to_lake(seeded): + conn, capture = seeded + n_lake = conn.execute( + "SELECT COUNT(*) FROM lake_cdc_events" + ).fetchone()[0] + assert n_lake == capture.latest_sequence + + +def test_lake_per_table_event_counts(seeded): + conn, _ = seeded + rows = conn.execute( + """ + SELECT table_name, COUNT(*) + FROM lake_cdc_events + GROUP BY table_name + """ + ).fetchall() + counts = dict(rows) + # 3 customers + 3 payment methods + 4 wallets + 3 transfers + # + 5 transfer_events + 5 ledger_entries = 23 inserts + assert counts == { + "customers": 3, + "payment_methods": 3, + "wallets": 4, + "transfers": 3, + "transfer_events": 5, + "ledger_entries": 5, + } + + +def test_lake_row_count_only_grows(seeded): + conn, capture = seeded + before = conn.execute( + "SELECT COUNT(*) FROM lake_cdc_events" + ).fetchone()[0] + + capture.update( + "customers", "c_alice", + before={"customer_id": "c_alice", "status": "active"}, + after ={"customer_id": "c_alice", "status": "suspended"}, + ) + new_records = capture.records_since(before) + append_to_lake(conn, new_records) + + after = conn.execute( + "SELECT COUNT(*) FROM lake_cdc_events" + ).fetchone()[0] + assert after > before + + +def test_lake_retains_full_pk_history(seeded): + conn, capture = seeded + capture.update( + "customers", "c_alice", + before={"customer_id": "c_alice", "status": "active"}, + after ={"customer_id": "c_alice", "status": "suspended"}, + ) + capture.delete( + "customers", "c_alice", before={"customer_id": "c_alice"} + ) + append_to_lake(conn, capture.records_since(0)) + + ops = [ + r[0] + for r in conn.execute( + """ + SELECT operation FROM lake_cdc_events + WHERE table_name = 'customers' AND primary_key = 'c_alice' + ORDER BY sequence + """ + ).fetchall() + ] + assert ops == ["insert", "update", "delete"] + + +def test_duplicate_sequence_is_deduped(seeded): + conn, capture = seeded + before = conn.execute( + "SELECT COUNT(*) FROM lake_cdc_events" + ).fetchone()[0] + + # Re-append the same records — the lake should silently dedup. + append_to_lake(conn, capture.records_since(0)) + after = conn.execute( + "SELECT COUNT(*) FROM lake_cdc_events" + ).fetchone()[0] + assert before == after + + +def test_replay_records_returns_in_sequence_order(seeded): + conn, _ = seeded + rows = replay_records(conn) + sequences = [r["sequence"] for r in rows] + assert sequences == sorted(sequences) + assert sequences[0] == 1 + + +def test_replay_records_filters_by_table(seeded): + conn, _ = seeded + customers = replay_records(conn, table="customers") + assert all(r["table"] == "customers" for r in customers) + + +def test_replay_records_filters_by_until_sequence(seeded): + conn, _ = seeded + until = 5 + rows = replay_records(conn, until_sequence=until) + assert all(r["sequence"] <= until for r in rows) + + +def test_lake_payload_is_valid_json_for_inserts(seeded): + conn, _ = seeded + row = conn.execute( + "SELECT after_image FROM lake_cdc_events " + "WHERE table_name = 'customers' AND operation = 'insert' LIMIT 1" + ).fetchone() + assert row is not None + parsed = json.loads(row[0]) + assert "customer_id" in parsed diff --git a/submission/wallet-payment-transfer/tests/test_schema_contracts.py b/submission/wallet-payment-transfer/tests/test_schema_contracts.py new file mode 100644 index 0000000..358a00f --- /dev/null +++ b/submission/wallet-payment-transfer/tests/test_schema_contracts.py @@ -0,0 +1,222 @@ +""" +Tests — schema-drift detection and stop-the-line behavior. + +Covers each detection rule in pipeline/schema_detector.py: + - missing_table + - column_dropped + - type_changed + - nullability_tightened + - column_added (warn, additive) + - non_nullable_column_added (breaking) + - primary_key_changed + - enum_value_added (warn) + +Plus the *system-level* stop-the-line: + - run_tick aborts when a breaking drift is present + - capture refuses writes when drift is breaking +""" + +from __future__ import annotations + +import duckdb +import pytest + +from pipeline.cdc import CDCCapture, SchemaContractViolation +from pipeline.orchestrator import bootstrap, run_tick +from pipeline.schema_detector import SchemaDetector, Severity +from source.contracts import CONTRACTS +from source.models import create_source_tables + + +def _fresh_source() -> duckdb.DuckDBPyConnection: + c = duckdb.connect(":memory:") + create_source_tables(c) + return c + + +def _drop_dependent_tables_for_alter(c: duckdb.DuckDBPyConnection) -> None: + """DuckDB rejects ALTER on tables with FK refs; drop FK children first.""" + for t in ("ledger_entries", "transfer_events", "transfers", + "payment_methods", "wallets"): + c.execute(f"DROP TABLE IF EXISTS {t}") + + +# ── happy path ─────────────────────────────────────────────────────────────── + + +def test_clean_source_passes_all_contracts(): + c = _fresh_source() + report = SchemaDetector(c).check_all() + assert not report.has_breaking() + + +def test_check_table_returns_one_report_per_table(): + c = _fresh_source() + report = SchemaDetector(c).check_all() + assert set(report.by_table) == set(CONTRACTS) + + +# ── missing table ──────────────────────────────────────────────────────────── + + +def test_missing_table_is_breaking(): + c = duckdb.connect(":memory:") + create_source_tables(c) + # Drop FK descendants then customers + _drop_dependent_tables_for_alter(c) + c.execute("DROP TABLE IF EXISTS customers") + drifts = SchemaDetector(c).check_table("customers").drifts + rules = {d.rule for d in drifts} + assert "missing_table" in rules + assert any(d.severity is Severity.BREAKING for d in drifts + if d.rule == "missing_table") + + +# ── dropped column ─────────────────────────────────────────────────────────── + + +def test_dropped_column_is_breaking(): + c = _fresh_source() + _drop_dependent_tables_for_alter(c) + c.execute("ALTER TABLE customers DROP COLUMN email") + report = SchemaDetector(c).check_table("customers") + assert any( + d.rule == "column_dropped" and d.column == "email" + and d.severity is Severity.BREAKING + for d in report.drifts + ) + + +# ── type changed ───────────────────────────────────────────────────────────── + + +def test_type_change_is_breaking(): + c = _fresh_source() + _drop_dependent_tables_for_alter(c) + c.execute("ALTER TABLE customers ALTER country_code TYPE INTEGER USING 0") + report = SchemaDetector(c).check_table("customers") + assert any( + d.rule == "type_changed" and d.column == "country_code" + and d.severity is Severity.BREAKING + for d in report.drifts + ) + + +# ── nullability tightened ──────────────────────────────────────────────────── + + +def test_nullability_tightening_is_breaking(): + """ + transfers.reference is nullable in the contract. If the source + later forbids NULL, the contract should report it as breaking. + """ + c = _fresh_source() + # Drop FK descendants of transfers. + c.execute("DROP TABLE IF EXISTS ledger_entries") + c.execute("DROP TABLE IF EXISTS transfer_events") + c.execute("ALTER TABLE transfers ALTER COLUMN reference SET NOT NULL") + report = SchemaDetector(c).check_table("transfers") + assert any( + d.rule == "nullability_tightened" and d.column == "reference" + and d.severity is Severity.BREAKING + for d in report.drifts + ) + + +# ── added column ───────────────────────────────────────────────────────────── + + +def test_new_nullable_column_is_warn_only(): + c = _fresh_source() + _drop_dependent_tables_for_alter(c) + c.execute("ALTER TABLE customers ADD COLUMN nickname VARCHAR") + report = SchemaDetector(c).check_table("customers") + assert not report.has_breaking() + warn_rules = {d.rule for d in report.warnings} + assert "column_added" in warn_rules + + +def test_new_non_null_column_is_breaking(): + c = _fresh_source() + _drop_dependent_tables_for_alter(c) + c.execute( + "ALTER TABLE customers ADD COLUMN region VARCHAR DEFAULT 'EU' NOT NULL" + ) + report = SchemaDetector(c).check_table("customers") + assert report.has_breaking() + assert any(d.rule == "non_nullable_column_added" for d in report.drifts) + + +# ── enum value drift ───────────────────────────────────────────────────────── + + +def test_unexpected_enum_value_in_data_is_warn(): + """ + The detector cannot prove an enum *removal* without source DDL access, + so it warns when it sees a *new* value present in data. We simulate by + bypassing the CHECK constraint with a DROP-then-recreate. + """ + c = duckdb.connect(":memory:") + c.execute(""" + CREATE TABLE customers ( + customer_id VARCHAR PRIMARY KEY, + full_name VARCHAR NOT NULL, + email VARCHAR NOT NULL, + country_code VARCHAR NOT NULL, + status VARCHAR NOT NULL, + registration_date DATE NOT NULL, + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL + ) + """) + c.execute( + "INSERT INTO customers VALUES " + "('c1', 'Alice', 'a@x.test', 'US', 'banned', " + "DATE '2025-01-01', TIMESTAMP '2026-06-01', TIMESTAMP '2026-06-01')" + ) + report = SchemaDetector(c).check_table("customers") + assert any(d.rule == "enum_value_added" for d in report.warnings) + + +# ── stop-the-line: orchestrator ────────────────────────────────────────────── + + +def test_run_tick_aborts_on_breaking_drift(conn, detector): + bootstrap(conn) + capture = CDCCapture() + capture.insert("customers", "c_x", {"customer_id": "c_x"}) + + # Drop a contracted column. + _drop_dependent_tables_for_alter(conn) + conn.execute("ALTER TABLE customers DROP COLUMN email") + + stats = run_tick(conn, capture, detector=detector) + assert stats["aborted"] is True + assert stats["reason"] == "schema_contract_violation" + assert stats["lake_written"] == 0 + assert stats["warehouse_applied"] == 0 + + +def test_run_tick_reports_drift_summary_on_abort(conn, detector): + bootstrap(conn) + capture = CDCCapture() + capture.insert("customers", "c_x", {"customer_id": "c_x"}) + _drop_dependent_tables_for_alter(conn) + conn.execute("ALTER TABLE customers DROP COLUMN email") + + stats = run_tick(conn, capture, detector=detector) + assert "email" in stats["drift_summary"] + assert "column_dropped" in stats["drift_summary"] + + +# ── stop-the-line: capture ─────────────────────────────────────────────────── + + +def test_capture_refuses_writes_when_strict(conn, detector): + _drop_dependent_tables_for_alter(conn) + conn.execute("ALTER TABLE customers DROP COLUMN email") + + strict = CDCCapture(detector, require_compatible_schema=True) + with pytest.raises(SchemaContractViolation): + strict.insert("customers", "c1", {"customer_id": "c1"}) + assert strict.log == [] diff --git a/submission/wallet-payment-transfer/tests/test_source_schema.py b/submission/wallet-payment-transfer/tests/test_source_schema.py new file mode 100644 index 0000000..18fc9b3 --- /dev/null +++ b/submission/wallet-payment-transfer/tests/test_source_schema.py @@ -0,0 +1,209 @@ +""" +Tests — source schema and constraint correctness. + +Covers system invariants enforced by the source DDL: + - PK uniqueness via duplicate insert rejection + - NOT NULL enforcement + - CHECK constraint enforcement (enums, non-negative balance, + positive amount, signed_amount consistency, settled_at >= created_at, + no self-transfer) + - FK referential integrity + - presence of declared indexes +""" + +from __future__ import annotations + +from datetime import date, datetime, timezone +from decimal import Decimal + +import duckdb +import pytest + +from source.models import TABLES_IN_ORDER, create_source_tables + + +def _t(year: int = 2026, month: int = 6, day: int = 1) -> datetime: + return datetime(year, month, day, tzinfo=timezone.utc) + + +@pytest.fixture() +def src() -> duckdb.DuckDBPyConnection: + c = duckdb.connect(":memory:") + create_source_tables(c) + # one valid customer, payment method and wallet so we can test + # downstream constraint rejections without setup noise + c.execute( + "INSERT INTO customers VALUES " + "('c1', 'Alice', 'a@x.test', 'US', 'active', ?, ?, ?)", + [date(2025, 1, 1), _t(), _t()], + ) + c.execute( + "INSERT INTO wallets VALUES " + "('w1', 'c1', 'USD', 100.00, 'active', ?, ?)", + [_t(), _t()], + ) + c.execute( + "INSERT INTO wallets VALUES " + "('w2', 'c1', 'USD', 0.00, 'active', ?, ?)", + [_t(), _t()], + ) + return c + + +# ── PK uniqueness ───────────────────────────────────────────────────────────── + + +def test_duplicate_customer_id_rejected(src): + src.execute( + "INSERT INTO customers VALUES " + "('c2', 'Bob', 'b@x.test', 'GB', 'active', ?, ?, ?)", + [date(2025, 1, 1), _t(), _t()], + ) + with pytest.raises(duckdb.ConstraintException): + src.execute( + "INSERT INTO customers VALUES " + "('c2', 'Bob2', 'b2@x.test', 'GB', 'active', ?, ?, ?)", + [date(2025, 1, 1), _t(), _t()], + ) + + +def test_duplicate_wallet_id_rejected(src): + with pytest.raises(duckdb.ConstraintException): + src.execute( + "INSERT INTO wallets VALUES " + "('w1', 'c1', 'USD', 0.00, 'active', ?, ?)", + [_t(), _t()], + ) + + +# ── NOT NULL ────────────────────────────────────────────────────────────────── + + +def test_null_customer_email_rejected(src): + with pytest.raises(duckdb.ConstraintException): + src.execute( + "INSERT INTO customers VALUES " + "('c3', 'Chen', NULL, 'US', 'active', ?, ?, ?)", + [date(2025, 1, 1), _t(), _t()], + ) + + +def test_null_wallet_currency_rejected(src): + with pytest.raises(duckdb.ConstraintException): + src.execute( + "INSERT INTO wallets VALUES " + "('w_bad', 'c1', NULL, 0.00, 'active', ?, ?)", + [_t(), _t()], + ) + + +# ── CHECK constraints ──────────────────────────────────────────────────────── + + +def test_negative_wallet_balance_rejected(src): + with pytest.raises(duckdb.ConstraintException): + src.execute( + "INSERT INTO wallets VALUES " + "('w_neg', 'c1', 'USD', -1.00, 'active', ?, ?)", + [_t(), _t()], + ) + + +def test_invalid_customer_status_rejected(src): + with pytest.raises(duckdb.ConstraintException): + src.execute( + "INSERT INTO customers VALUES " + "('c_bad', 'X', 'x@x.test', 'US', 'invalid', ?, ?, ?)", + [date(2025, 1, 1), _t(), _t()], + ) + + +def test_zero_transfer_amount_rejected(src): + with pytest.raises(duckdb.ConstraintException): + src.execute( + "INSERT INTO transfers VALUES (" + "'tx_bad', 'w1', 'w2', NULL, 0.00, 'USD', 'pending', " + "NULL, NULL, ?, NULL)", + [_t()], + ) + + +def test_self_transfer_rejected(src): + with pytest.raises(duckdb.ConstraintException): + src.execute( + "INSERT INTO transfers VALUES (" + "'tx_self', 'w1', 'w1', NULL, 1.00, 'USD', 'pending', " + "NULL, NULL, ?, NULL)", + [_t()], + ) + + +def test_settled_at_before_created_at_rejected(src): + with pytest.raises(duckdb.ConstraintException): + src.execute( + "INSERT INTO transfers VALUES (" + "'tx_back', 'w1', 'w2', NULL, 1.00, 'USD', 'settled', " + "NULL, NULL, ?, ?)", + [_t(2026, 6, 2), _t(2026, 6, 1)], + ) + + +def test_signed_amount_inconsistent_rejected(src): + # credit but signed_amount = -amount + with pytest.raises(duckdb.ConstraintException): + src.execute( + "INSERT INTO ledger_entries VALUES (" + "'le_bad', 'w1', NULL, 'credit', 10.00, -10.00, 0.00, ?)", + [_t()], + ) + + +# ── FK referential integrity ───────────────────────────────────────────────── + + +def test_wallet_references_existing_customer(src): + with pytest.raises(duckdb.ConstraintException): + src.execute( + "INSERT INTO wallets VALUES " + "('w_orph', 'c_missing', 'USD', 0.00, 'active', ?, ?)", + [_t(), _t()], + ) + + +def test_transfer_references_existing_wallets(src): + with pytest.raises(duckdb.ConstraintException): + src.execute( + "INSERT INTO transfers VALUES (" + "'tx_orph', 'w_missing', 'w2', NULL, 1.00, 'USD', 'pending', " + "NULL, NULL, ?, NULL)", + [_t()], + ) + + +# ── indexes are present ────────────────────────────────────────────────────── + + +@pytest.mark.parametrize("table", TABLES_IN_ORDER) +def test_table_exists(src: duckdb.DuckDBPyConnection, table: str): + rows = src.execute( + "SELECT 1 FROM information_schema.tables WHERE table_name = ?", + [table], + ).fetchall() + assert rows, f"table {table} not created" + + +def test_indexes_declared_for_cdc_workload(src: duckdb.DuckDBPyConnection): + indexes = [ + r[0] + for r in src.execute( + "SELECT index_name FROM duckdb_indexes()" + ).fetchall() + ] + must_have = [ + "ix_customers_updated_at", + "ix_wallets_customer", + "ix_transfers_status", + "ix_ledger_wallet_time", + ] + for ix in must_have: + assert ix in indexes, f"missing required index {ix}" diff --git a/submission/wallet-payment-transfer/tests/test_time_travel.py b/submission/wallet-payment-transfer/tests/test_time_travel.py new file mode 100644 index 0000000..fad7490 --- /dev/null +++ b/submission/wallet-payment-transfer/tests/test_time_travel.py @@ -0,0 +1,227 @@ +""" +Tests — point-in-time reconstruction (state_at) and restore. + +Covers: + - state_at returns the row that was current at ts + - state_at returns the most recent revision *strictly* before next change + - state_at excludes deleted rows + - state_at returns nothing before the row's _valid_from + - restore_current_state_to overwrites current-state with snapshot + - restore is non-destructive to history (history is preserved) + - restored state is a valid starting point for the next CDC tick +""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from decimal import Decimal + +import pytest + +from pipeline.orchestrator import run_tick +from pipeline.warehouse import ( + apply_cdc_records, + restore_current_state_to, + state_at, +) + + +def _ts(seconds: int = 0) -> datetime: + # Far enough in the future that we are always after the seed's + # captured_at (which uses datetime.now() at test-run time). + base = datetime(2030, 1, 1, 12, 0, tzinfo=timezone.utc) + return base + timedelta(seconds=seconds) + + +def _evolve_alice_balance(seeded, balances: list[Decimal]) -> list[datetime]: + """ + Push a series of balance updates onto w_alice_usd. + + Returns the list of captured_at timestamps used for each update so + the test can replay queries at exact boundaries. + """ + conn, capture = seeded + timestamps: list[datetime] = [] + prev = Decimal("75.0000") + for i, bal in enumerate(balances): + ts = _ts(i * 10) + capture.update( + "wallets", "w_alice_usd", + before={"wallet_id": "w_alice_usd", "balance": prev}, + after={ + "wallet_id": "w_alice_usd", + "customer_id": "c_alice", + "currency": "USD", + "balance": bal, + "status": "active", + "created_at": datetime(2026, 6, 1, 9, tzinfo=timezone.utc), + "updated_at": ts, + }, + captured_at=ts, + ) + timestamps.append(ts) + prev = bal + apply_cdc_records(conn, capture.records_since(capture.latest_sequence - len(balances))) + return timestamps + + +# ── state_at: basic reconstruction ─────────────────────────────────────────── + + +def test_state_at_returns_seed_state_after_seed(seeded): + conn, _ = seeded + rows = state_at(conn, "wallets", _ts(99999)) + by_id = {r["wallet_id"]: r["balance"] for r in rows} + assert by_id["w_alice_usd"] == Decimal("75.0000") + assert by_id["w_bob_gbp"] == Decimal("50.0000") + + +def test_state_at_picks_revision_active_at_timestamp(seeded): + conn, _ = seeded + timestamps = _evolve_alice_balance( + seeded, [Decimal("60.0000"), Decimal("40.0000"), Decimal("20.0000")] + ) + + # Just before the first new revision -> still 75 (the seed value) + rows = state_at(conn, "wallets", timestamps[0] - timedelta(seconds=1)) + by_id = {r["wallet_id"]: r["balance"] for r in rows} + assert by_id["w_alice_usd"] == Decimal("75.0000") + + # At the first new revision exactly -> 60 + rows = state_at(conn, "wallets", timestamps[0]) + assert {r["wallet_id"]: r["balance"] for r in rows}["w_alice_usd"] == \ + Decimal("60.0000") + + # Between revisions -> middle value (40) + rows = state_at(conn, "wallets", timestamps[1] + timedelta(seconds=1)) + assert {r["wallet_id"]: r["balance"] for r in rows}["w_alice_usd"] == \ + Decimal("40.0000") + + # After the last revision -> 20 + rows = state_at(conn, "wallets", timestamps[2] + timedelta(seconds=1)) + assert {r["wallet_id"]: r["balance"] for r in rows}["w_alice_usd"] == \ + Decimal("20.0000") + + +def test_state_at_respects_valid_from_lower_bound(seeded): + conn, _ = seeded + # Long before the first event — wallet table should have been empty + rows = state_at(conn, "wallets", datetime(2020, 1, 1, tzinfo=timezone.utc)) + assert rows == [] + + +def test_state_at_excludes_deleted_rows(seeded): + conn, capture = seeded + delete_ts = _ts(0) + capture.delete( + "transfers", "tx_pending_002", + before={"transfer_id": "tx_pending_002"}, + captured_at=delete_ts, + ) + apply_cdc_records(conn, capture.records_since(capture.latest_sequence - 1)) + + # After the delete, snapshot must not include the deleted PK. + rows = state_at(conn, "transfers", delete_ts + timedelta(seconds=1)) + pks = {r["transfer_id"] for r in rows} + assert "tx_pending_002" not in pks + + # Before the delete, snapshot must include the deleted PK. + rows = state_at(conn, "transfers", delete_ts - timedelta(seconds=1)) + pks = {r["transfer_id"] for r in rows} + assert "tx_pending_002" in pks + + +# ── restore current state ──────────────────────────────────────────────────── + + +def test_restore_overwrites_current_state(seeded): + conn, _ = seeded + timestamps = _evolve_alice_balance( + seeded, [Decimal("60.0000"), Decimal("40.0000")] + ) + + # Roll wh_wallets back to the moment between the two updates + n = restore_current_state_to( + conn, "wallets", timestamps[0] + timedelta(seconds=1) + ) + assert n >= 1 + + bal = conn.execute( + "SELECT balance FROM wh_wallets WHERE wallet_id = 'w_alice_usd'" + ).fetchone()[0] + assert bal == Decimal("60.0000") + + +def test_restore_does_not_touch_history(seeded): + conn, _ = seeded + timestamps = _evolve_alice_balance( + seeded, [Decimal("60.0000"), Decimal("40.0000")] + ) + n_hist_before = conn.execute( + "SELECT COUNT(*) FROM wh_wallets_history" + ).fetchone()[0] + + restore_current_state_to(conn, "wallets", timestamps[0]) + n_hist_after = conn.execute( + "SELECT COUNT(*) FROM wh_wallets_history" + ).fetchone()[0] + + assert n_hist_before == n_hist_after + + +def test_restore_to_pre_history_results_in_empty_table(seeded): + conn, _ = seeded + n = restore_current_state_to( + conn, "wallets", datetime(2020, 1, 1, tzinfo=timezone.utc) + ) + assert n == 0 + cnt = conn.execute("SELECT COUNT(*) FROM wh_wallets").fetchone()[0] + assert cnt == 0 + + +def test_restored_state_accepts_new_cdc_records(seeded): + """ + After restore, _cdc_seq is reset to 0; the next CDC event (with seq > 0) + must apply cleanly without being rejected as out-of-order. + """ + conn, capture = seeded + timestamps = _evolve_alice_balance(seeded, [Decimal("60.0000")]) + restore_current_state_to(conn, "wallets", timestamps[0]) + + # Now push a fresh update — it should land. + capture.update( + "wallets", "w_alice_usd", + before={"wallet_id": "w_alice_usd", "balance": Decimal("60.0000")}, + after={ + "wallet_id": "w_alice_usd", + "customer_id": "c_alice", + "currency": "USD", + "balance": Decimal("12.3456"), + "status": "active", + "created_at": datetime(2026, 6, 1, 9, tzinfo=timezone.utc), + "updated_at": _ts(60), + }, + captured_at=_ts(60), + ) + apply_cdc_records(conn, capture.records_since(capture.latest_sequence - 1)) + + bal = conn.execute( + "SELECT balance FROM wh_wallets WHERE wallet_id = 'w_alice_usd'" + ).fetchone()[0] + assert bal == Decimal("12.3456") + + +# ── parity: state_at over multiple tables stays consistent ─────────────────── + + +def test_state_at_customers_count_matches_seed(seeded): + conn, _ = seeded + rows = state_at(conn, "customers", _ts(99999)) + assert len(rows) == 3 + + +def test_state_at_ledger_entries_includes_funding_entries(seeded): + conn, _ = seeded + rows = state_at(conn, "ledger_entries", _ts(99999)) + funding = [r for r in rows if r["transfer_id"] is None] + assert len(funding) == 3 diff --git a/submission/wallet-payment-transfer/tests/test_warehouse.py b/submission/wallet-payment-transfer/tests/test_warehouse.py new file mode 100644 index 0000000..79d18d0 --- /dev/null +++ b/submission/wallet-payment-transfer/tests/test_warehouse.py @@ -0,0 +1,271 @@ +""" +Tests — warehouse modeling: current-state + SCD2 history. + +Covers: + - inserts land in current-state and history + - updates upsert current-state and close+open history rows + - deletes soft-delete current-state and append a delete history row + - duplicate / replayed records are no-ops + - out-of-order events do not clobber newer state (sequence guard) + - SCD2 row count grows by 1 per applied event + - exactly one row is _is_current=true per PK at any time +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from decimal import Decimal + +import pytest + +from pipeline.cdc import CDCRecord +from pipeline.orchestrator import run_tick +from pipeline.warehouse import apply_cdc_records + + +def _t(seconds: int = 0) -> datetime: + base = datetime(2026, 6, 2, 12, 0, tzinfo=timezone.utc) + return base.replace(second=seconds % 60) + + +# ── inserts propagate ──────────────────────────────────────────────────────── + + +def test_insert_appears_in_warehouse(seeded): + conn, _ = seeded + row = conn.execute( + "SELECT full_name, country_code " + "FROM wh_customers WHERE customer_id = 'c_alice'" + ).fetchone() + assert row == ("Alice Anderson", "US") + + +def test_all_warehouse_tables_populated_after_seed(seeded): + conn, _ = seeded + counts = { + t: conn.execute(f"SELECT COUNT(*) FROM wh_{t}").fetchone()[0] + for t in ( + "customers", + "payment_methods", + "wallets", + "transfers", + "transfer_events", + "ledger_entries", + ) + } + assert counts == { + "customers": 3, + "payment_methods": 3, + "wallets": 4, + "transfers": 3, + "transfer_events": 5, + "ledger_entries": 5, + } + + +def test_insert_creates_current_history_row(seeded): + conn, _ = seeded + row = conn.execute( + "SELECT _is_current, _op FROM wh_customers_history " + "WHERE customer_id = 'c_alice'" + ).fetchone() + assert row == (True, "insert") + + +# ── updates ────────────────────────────────────────────────────────────────── + + +def test_update_overwrites_current_state(seeded): + conn, capture = seeded + capture.update( + "customers", "c_alice", + before={"customer_id": "c_alice", "status": "active"}, + after={ + "customer_id": "c_alice", + "full_name": "Alice A. Anderson", + "email": "alice@example.invalid", + "country_code": "US", + "status": "suspended", + "registration_date": "2024-01-15", + "created_at": _t(), + "updated_at": _t(), + }, + ) + apply_cdc_records(conn, capture.records_since(capture.latest_sequence - 1)) + + row = conn.execute( + "SELECT full_name, status FROM wh_customers " + "WHERE customer_id = 'c_alice'" + ).fetchone() + assert row == ("Alice A. Anderson", "suspended") + + +def test_update_closes_old_history_row_and_opens_new(seeded): + conn, capture = seeded + capture.update( + "wallets", "w_alice_usd", + before={"wallet_id": "w_alice_usd", "balance": Decimal("75.0000")}, + after={ + "wallet_id": "w_alice_usd", + "customer_id": "c_alice", + "currency": "USD", + "balance": Decimal("60.0000"), + "status": "active", + "created_at": _t(), + "updated_at": _t(1), + }, + ) + apply_cdc_records(conn, capture.records_since(capture.latest_sequence - 1)) + + rows = conn.execute( + """ + SELECT _is_current, _op, balance + FROM wh_wallets_history + WHERE wallet_id = 'w_alice_usd' + ORDER BY _valid_from + """ + ).fetchall() + assert len(rows) == 2 + assert rows[0][0] is False # old version closed + assert rows[1][0] is True # new version current + assert rows[1][2] == Decimal("60.0000") + + +def test_only_one_current_history_row_per_pk(seeded): + conn, _ = seeded + rows = conn.execute( + """ + SELECT wallet_id, COUNT(*) + FROM wh_wallets_history + WHERE _is_current = TRUE + GROUP BY wallet_id + HAVING COUNT(*) <> 1 + """ + ).fetchall() + assert rows == [] + + +# ── deletes ────────────────────────────────────────────────────────────────── + + +def test_delete_marks_current_state_deleted(seeded): + conn, capture = seeded + capture.delete( + "transfers", "tx_pending_002", + before={"transfer_id": "tx_pending_002"}, + ) + apply_cdc_records(conn, capture.records_since(capture.latest_sequence - 1)) + + deleted = conn.execute( + "SELECT _deleted FROM wh_transfers WHERE transfer_id = 'tx_pending_002'" + ).fetchone()[0] + assert deleted is True + + +def test_delete_appends_history_row_with_op_delete(seeded): + conn, capture = seeded + capture.delete( + "payment_methods", "pm_chen_upi", + before={"payment_method_id": "pm_chen_upi"}, + ) + apply_cdc_records(conn, capture.records_since(capture.latest_sequence - 1)) + + last = conn.execute( + """ + SELECT _op, _is_current + FROM wh_payment_methods_history + WHERE payment_method_id = 'pm_chen_upi' + ORDER BY _valid_from DESC LIMIT 1 + """ + ).fetchone() + assert last == ("delete", False) + + +def test_delete_does_not_remove_current_row(seeded): + conn, capture = seeded + capture.delete( + "customers", "c_alice", before={"customer_id": "c_alice"} + ) + apply_cdc_records(conn, capture.records_since(capture.latest_sequence - 1)) + + row = conn.execute( + "SELECT _deleted FROM wh_customers WHERE customer_id = 'c_alice'" + ).fetchone() + assert row is not None + assert row[0] is True + + +# ── replay safety / idempotency ────────────────────────────────────────────── + + +def test_replay_does_not_double_apply(seeded): + conn, capture = seeded + n_before = conn.execute("SELECT COUNT(*) FROM wh_customers").fetchone()[0] + n_hist_before = conn.execute( + "SELECT COUNT(*) FROM wh_customers_history" + ).fetchone()[0] + + apply_cdc_records(conn, capture.records_since(0)) + n_after = conn.execute("SELECT COUNT(*) FROM wh_customers").fetchone()[0] + n_hist_after = conn.execute( + "SELECT COUNT(*) FROM wh_customers_history" + ).fetchone()[0] + + # Current-state count unchanged. + assert n_after == n_before + # History count unchanged because sequence guard rejects duplicates. + assert n_hist_after == n_hist_before + + +def test_orchestrator_run_tick_is_idempotent(seeded, detector): + conn, capture = seeded + n_lake_before = conn.execute( + "SELECT COUNT(*) FROM lake_cdc_events" + ).fetchone()[0] + stats = run_tick(conn, capture, detector=detector) + n_lake_after = conn.execute( + "SELECT COUNT(*) FROM lake_cdc_events" + ).fetchone()[0] + # Already applied — second tick should be a no-op + assert stats["lake_written"] == 0 + assert stats["warehouse_applied"] == 0 + assert n_lake_before == n_lake_after + + +# ── out-of-order arrival ───────────────────────────────────────────────────── + + +@pytest.mark.parametrize("scenario", ["older_arrives_late"]) +def test_older_event_does_not_clobber_newer_state(seeded, scenario): + """ + If a new sequence has already been applied, a later-arriving event with + a smaller sequence must NOT modify the current-state row. + """ + conn, capture = seeded + # Take the latest applied seq. + latest = capture.latest_sequence + # Forge an out-of-order CDC record with a lower sequence number + stale = CDCRecord( + operation="update", + table="customers", + primary_key="c_alice", + before={"customer_id": "c_alice", "full_name": "Alice Anderson"}, + after={ + "customer_id": "c_alice", + "full_name": "Stale Name", + "email": "alice@example.invalid", + "country_code": "US", + "status": "active", + "registration_date": "2024-01-15", + "created_at": _t(), + "updated_at": _t(), + }, + sequence=1, # smaller than what we already applied + ) + apply_cdc_records(conn, [stale]) + + name = conn.execute( + "SELECT full_name FROM wh_customers WHERE customer_id = 'c_alice'" + ).fetchone()[0] + assert name == "Alice Anderson" # original, not 'Stale Name' + _ = latest