From 9d51601f2e6064e06fc0b42819bf48a0beb6c716 Mon Sep 17 00:00:00 2001 From: Garv Pushkarna Date: Thu, 25 Jun 2026 15:24:20 +0530 Subject: [PATCH 1/2] data-assigment submission --- .gitignore | 76 ++++ .../conftest.cpython-314-pytest-9.0.2.pyc | Bin 592 -> 0 bytes .../sample-candidate/catalog/catalog.json | 77 ---- submission/sample-candidate/conftest.py | 6 - .../__pycache__/__init__.cpython-314.pyc | Bin 186 -> 0 bytes .../pipeline/__pycache__/cdc.cpython-314.pyc | Bin 5653 -> 0 bytes .../pipeline/__pycache__/lake.cpython-314.pyc | Bin 3364 -> 0 bytes .../__pycache__/warehouse.cpython-314.pyc | Bin 5351 -> 0 bytes submission/sample-candidate/pipeline/cdc.py | 89 ----- submission/sample-candidate/pipeline/lake.py | 69 ---- .../sample-candidate/pipeline/warehouse.py | 124 ------ submission/sample-candidate/requirements.txt | 2 - .../scripts/check_schema_contracts.py | 60 --- .../scripts/run_data_quality_checks.py | 212 ---------- .../scripts/validate_catalog.py | 77 ---- .../sample-candidate/source/__init__.py | 0 .../__pycache__/__init__.cpython-314.pyc | Bin 184 -> 0 bytes .../source/__pycache__/models.cpython-314.pyc | Bin 3369 -> 0 bytes submission/sample-candidate/source/models.py | 84 ---- .../conftest.cpython-314-pytest-9.0.2.pyc | Bin 3905 -> 0 bytes .../test_catalog.cpython-314-pytest-9.0.2.pyc | Bin 19689 -> 0 bytes .../test_cdc.cpython-314-pytest-9.0.2.pyc | Bin 20869 -> 0 bytes ..._data_quality.cpython-314-pytest-9.0.2.pyc | Bin 21787 -> 0 bytes ...ema_contracts.cpython-314-pytest-9.0.2.pyc | Bin 16244 -> 0 bytes submission/sample-candidate/tests/conftest.py | 125 ------ .../sample-candidate/tests/test_catalog.py | 134 ------- submission/sample-candidate/tests/test_cdc.py | 135 ------- .../tests/test_data_quality.py | 252 ------------ .../tests/test_schema_contracts.py | 170 -------- .../wallet-payment-transfer/ARCHITECTURE.md | 304 +++++++++++++++ .../wallet-payment-transfer/ASSUMPTIONS.md | 189 +++++++++ submission/wallet-payment-transfer/Makefile | 46 +++ .../wallet-payment-transfer/PR_DESCRIPTION.md | 326 ++++++++++++++++ submission/wallet-payment-transfer/README.md | 146 +++++++ .../catalog/catalog.json | 288 ++++++++++++++ .../wallet-payment-transfer/conftest.py | 6 + .../pipeline/__init__.py | 1 + .../wallet-payment-transfer/pipeline/cdc.py | 217 +++++++++++ .../pipeline/checkpoint.py | 71 ++++ .../wallet-payment-transfer/pipeline/lake.py | 179 +++++++++ .../pipeline/orchestrator.py | 108 ++++++ .../pipeline/schema_detector.py | 324 ++++++++++++++++ .../pipeline/warehouse.py | 367 ++++++++++++++++++ .../wallet-payment-transfer/requirements.txt | 2 + .../scripts/check_schema_contracts.py | 56 +++ .../scripts/demo_schema_change.py | 79 ++++ .../scripts/demo_time_travel.py | 96 +++++ .../scripts/run_data_quality_checks.py | 282 ++++++++++++++ .../scripts/run_pipeline.py | 65 ++++ .../scripts/validate_catalog.py | 113 ++++++ .../source/__init__.py | 1 + .../source/contracts.py | 191 +++++++++ .../wallet-payment-transfer/source/models.py | 264 +++++++++++++ .../wallet-payment-transfer/source/seed.py | 295 ++++++++++++++ .../tests}/__init__.py | 0 .../wallet-payment-transfer/tests/conftest.py | 59 +++ .../tests/test_catalog.py | 198 ++++++++++ .../wallet-payment-transfer/tests/test_cdc.py | 192 +++++++++ .../tests/test_data_quality.py | 171 ++++++++ .../wallet-payment-transfer/tests/test_e2e.py | 306 +++++++++++++++ .../tests/test_lake.py | 138 +++++++ .../tests/test_schema_contracts.py | 222 +++++++++++ .../tests/test_source_schema.py | 209 ++++++++++ .../tests/test_time_travel.py | 227 +++++++++++ .../tests/test_warehouse.py | 271 +++++++++++++ 65 files changed, 6085 insertions(+), 1616 deletions(-) create mode 100644 .gitignore delete mode 100644 submission/sample-candidate/__pycache__/conftest.cpython-314-pytest-9.0.2.pyc delete mode 100644 submission/sample-candidate/catalog/catalog.json delete mode 100644 submission/sample-candidate/conftest.py delete mode 100644 submission/sample-candidate/pipeline/__pycache__/__init__.cpython-314.pyc delete mode 100644 submission/sample-candidate/pipeline/__pycache__/cdc.cpython-314.pyc delete mode 100644 submission/sample-candidate/pipeline/__pycache__/lake.cpython-314.pyc delete mode 100644 submission/sample-candidate/pipeline/__pycache__/warehouse.cpython-314.pyc delete mode 100644 submission/sample-candidate/pipeline/cdc.py delete mode 100644 submission/sample-candidate/pipeline/lake.py delete mode 100644 submission/sample-candidate/pipeline/warehouse.py delete mode 100644 submission/sample-candidate/requirements.txt delete mode 100644 submission/sample-candidate/scripts/check_schema_contracts.py delete mode 100644 submission/sample-candidate/scripts/run_data_quality_checks.py delete mode 100644 submission/sample-candidate/scripts/validate_catalog.py delete mode 100644 submission/sample-candidate/source/__init__.py delete mode 100644 submission/sample-candidate/source/__pycache__/__init__.cpython-314.pyc delete mode 100644 submission/sample-candidate/source/__pycache__/models.cpython-314.pyc delete mode 100644 submission/sample-candidate/source/models.py delete mode 100644 submission/sample-candidate/tests/__pycache__/conftest.cpython-314-pytest-9.0.2.pyc delete mode 100644 submission/sample-candidate/tests/__pycache__/test_catalog.cpython-314-pytest-9.0.2.pyc delete mode 100644 submission/sample-candidate/tests/__pycache__/test_cdc.cpython-314-pytest-9.0.2.pyc delete mode 100644 submission/sample-candidate/tests/__pycache__/test_data_quality.cpython-314-pytest-9.0.2.pyc delete mode 100644 submission/sample-candidate/tests/__pycache__/test_schema_contracts.cpython-314-pytest-9.0.2.pyc delete mode 100644 submission/sample-candidate/tests/conftest.py delete mode 100644 submission/sample-candidate/tests/test_catalog.py delete mode 100644 submission/sample-candidate/tests/test_cdc.py delete mode 100644 submission/sample-candidate/tests/test_data_quality.py delete mode 100644 submission/sample-candidate/tests/test_schema_contracts.py create mode 100644 submission/wallet-payment-transfer/ARCHITECTURE.md create mode 100644 submission/wallet-payment-transfer/ASSUMPTIONS.md create mode 100644 submission/wallet-payment-transfer/Makefile create mode 100644 submission/wallet-payment-transfer/PR_DESCRIPTION.md create mode 100644 submission/wallet-payment-transfer/README.md create mode 100644 submission/wallet-payment-transfer/catalog/catalog.json create mode 100644 submission/wallet-payment-transfer/conftest.py create mode 100644 submission/wallet-payment-transfer/pipeline/__init__.py create mode 100644 submission/wallet-payment-transfer/pipeline/cdc.py create mode 100644 submission/wallet-payment-transfer/pipeline/checkpoint.py create mode 100644 submission/wallet-payment-transfer/pipeline/lake.py create mode 100644 submission/wallet-payment-transfer/pipeline/orchestrator.py create mode 100644 submission/wallet-payment-transfer/pipeline/schema_detector.py create mode 100644 submission/wallet-payment-transfer/pipeline/warehouse.py create mode 100644 submission/wallet-payment-transfer/requirements.txt create mode 100644 submission/wallet-payment-transfer/scripts/check_schema_contracts.py create mode 100644 submission/wallet-payment-transfer/scripts/demo_schema_change.py create mode 100644 submission/wallet-payment-transfer/scripts/demo_time_travel.py create mode 100644 submission/wallet-payment-transfer/scripts/run_data_quality_checks.py create mode 100644 submission/wallet-payment-transfer/scripts/run_pipeline.py create mode 100644 submission/wallet-payment-transfer/scripts/validate_catalog.py create mode 100644 submission/wallet-payment-transfer/source/__init__.py create mode 100644 submission/wallet-payment-transfer/source/contracts.py create mode 100644 submission/wallet-payment-transfer/source/models.py create mode 100644 submission/wallet-payment-transfer/source/seed.py rename submission/{sample-candidate/pipeline => wallet-payment-transfer/tests}/__init__.py (100%) create mode 100644 submission/wallet-payment-transfer/tests/conftest.py create mode 100644 submission/wallet-payment-transfer/tests/test_catalog.py create mode 100644 submission/wallet-payment-transfer/tests/test_cdc.py create mode 100644 submission/wallet-payment-transfer/tests/test_data_quality.py create mode 100644 submission/wallet-payment-transfer/tests/test_e2e.py create mode 100644 submission/wallet-payment-transfer/tests/test_lake.py create mode 100644 submission/wallet-payment-transfer/tests/test_schema_contracts.py create mode 100644 submission/wallet-payment-transfer/tests/test_source_schema.py create mode 100644 submission/wallet-payment-transfer/tests/test_time_travel.py create mode 100644 submission/wallet-payment-transfer/tests/test_warehouse.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..25a4a45 --- /dev/null +++ b/.gitignore @@ -0,0 +1,76 @@ +# ---- Python ---- +__pycache__/ +*.py[cod] +*$py.class +*.so + +# Distribution / packaging +.Python +build/ +dist/ +develop-eggs/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +*.egg +MANIFEST + +# Virtual environments +venv/ +.venv/ +env/ +.env/ +ENV/ + +# Pytest / coverage / type checkers +.pytest_cache/ +.cache/ +.coverage +.coverage.* +htmlcov/ +coverage.xml +*.cover +.hypothesis/ +.mypy_cache/ +.dmypy.json +.pyre/ +.pytype/ +.ruff_cache/ + +# Jupyter +.ipynb_checkpoints/ + +# ---- Local data artifacts (DuckDB / SQLite / logs) ---- +*.duckdb +*.duckdb.wal +*.db +*.sqlite +*.sqlite-shm +*.sqlite-wal +*.log + +# ---- IDE / Editor ---- +.idea/ +.vscode/ +*.swp +*.swo +*~ + +# ---- OS ---- +.DS_Store +Thumbs.db +desktop.ini + +# ---- Secrets / env files ---- +.env +.env.local +.env.*.local +*.pem +*.key \ No newline at end of file diff --git a/submission/sample-candidate/__pycache__/conftest.cpython-314-pytest-9.0.2.pyc b/submission/sample-candidate/__pycache__/conftest.cpython-314-pytest-9.0.2.pyc deleted file mode 100644 index 7309004e9fba8bc1f3e0c7323e666a6180a4045a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 592 zcmZutO=}cE5bc?bJCh}GF+s%BHh2n}84Q7V2*zW)gs|el!={>@6`u zt_cXa`Em4*_y>eQ+KYJcD&!AXlXbEl>_ff!c=f8f?%$m6F?yfhepMd{^CoCt_pfvL zft&+2WXN{dFnAhZc!izK85a)N<_+u)CY1=It4fUVDBfOZg{HWDd+N8|4jV=P<=$x- zQyk5$29sztoZx^qy2qA6_l)A9*2r^R@1X?bKM&sXtSEqk+pQFU(sjyFa~MNnGTi5& zc|!!AXEj%q(H1G^Y%Wuyj4V|xldYwB9|=+DTnHcUI#r^o!OuN0V{skb6=vKhB^iLU zNP`k9-&lpx&t9i|)aH8B+|Wg4eW<~AXd^$Gh zek*-W3`kMt`>!R<(CI5zy3|UeVPYjZlNK3f%QRfQtSea~r2kx@JyBxDK3=bM;Y#_~ z-Xi(ty6}+9?@kZ|7wp9^#@?}`+3&3PgLRJM!}z#=*gpwQR?gN=*UmOhH$FZ3w)o(j JE!#eI{Q*8Dq6Gi| diff --git a/submission/sample-candidate/catalog/catalog.json b/submission/sample-candidate/catalog/catalog.json deleted file mode 100644 index 83dfa55..0000000 --- a/submission/sample-candidate/catalog/catalog.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "datasets": [ - { - "name": "lake_cdc_events", - "layer": "lake", - "description": "Append-only log of every CDC event captured from the payments source system. Preserves full change history for replay and point-in-time recovery.", - "owner": "data-platform", - "consumers": ["data-platform", "analytics", "audit"], - "update_cadence": "real-time", - "schema": { - "sequence": "INTEGER — monotonic capture offset", - "operation": "VARCHAR — insert | update | delete", - "table_name": "VARCHAR — source table name", - "primary_key": "VARCHAR — source row PK value", - "data": "VARCHAR (JSON) — full row snapshot at capture time", - "captured_at": "TIMESTAMP — UTC capture time" - } - }, - { - "name": "wh_customers", - "layer": "warehouse", - "description": "Current-state customer snapshot. Reflects the latest known state of each customer row. Soft-deleted rows are flagged with _deleted=true.", - "owner": "data-platform", - "consumers": ["analytics", "product", "finance"], - "update_cadence": "near-real-time", - "schema": { - "customer_id": "VARCHAR — primary key", - "name": "VARCHAR", - "email": "VARCHAR", - "status": "VARCHAR — active | suspended | closed", - "created_at": "TIMESTAMP", - "updated_at": "TIMESTAMP", - "_cdc_seq": "INTEGER — sequence of last CDC event", - "_deleted": "BOOLEAN — true if source row was deleted" - } - }, - { - "name": "wh_wallets", - "layer": "warehouse", - "description": "Current-state wallet snapshot. Balance reflects the latest value written via CDC. Negative balances are a data quality violation.", - "owner": "data-platform", - "consumers": ["analytics", "finance"], - "update_cadence": "near-real-time", - "schema": { - "wallet_id": "VARCHAR — primary key", - "customer_id": "VARCHAR — FK to wh_customers", - "balance": "DECIMAL(18,2)", - "currency": "VARCHAR", - "status": "VARCHAR — active | frozen | closed", - "created_at": "TIMESTAMP", - "updated_at": "TIMESTAMP", - "_cdc_seq": "INTEGER", - "_deleted": "BOOLEAN" - } - }, - { - "name": "wh_transactions", - "layer": "warehouse", - "description": "Current-state transaction snapshot. Each row is the latest state of a transaction from the source. Amount must always be positive.", - "owner": "data-platform", - "consumers": ["analytics", "finance", "audit"], - "update_cadence": "near-real-time", - "schema": { - "transaction_id": "VARCHAR — primary key", - "wallet_id": "VARCHAR — FK to wh_wallets", - "amount": "DECIMAL(18,2) — always > 0", - "direction": "VARCHAR — credit | debit", - "status": "VARCHAR — pending | settled | failed | reversed", - "reference": "VARCHAR — nullable external reference", - "created_at": "TIMESTAMP", - "settled_at": "TIMESTAMP — nullable until settled", - "_cdc_seq": "INTEGER", - "_deleted": "BOOLEAN" - } - } - ] -} diff --git a/submission/sample-candidate/conftest.py b/submission/sample-candidate/conftest.py deleted file mode 100644 index ce3c67b..0000000 --- a/submission/sample-candidate/conftest.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Root conftest — adds submission/ to sys.path so tests can import source/pipeline.""" - -import os -import sys - -sys.path.insert(0, os.path.dirname(__file__)) diff --git a/submission/sample-candidate/pipeline/__pycache__/__init__.cpython-314.pyc b/submission/sample-candidate/pipeline/__pycache__/__init__.cpython-314.pyc deleted file mode 100644 index 1523f32b32810272b170ad39e00932cfc189806a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 186 zcmdPq_I|p@<2{{|u766|NszoLW?@ zUy_=fQI=YiS(2}xU7Ay>UzA#qUko8rOG*p$QxZ!ObrXw=Gt={OQ}arS^@~fBax;Pa z{5<`F%!1UM%)C_n`1s7c%#!$cy@JYH95%W6DWy57c15f}dq6HJ1~EP{Gcqz3F#}lu D5y~+p diff --git a/submission/sample-candidate/pipeline/__pycache__/cdc.cpython-314.pyc b/submission/sample-candidate/pipeline/__pycache__/cdc.cpython-314.pyc deleted file mode 100644 index a17fa0bcc202261717534949b737d20e8d1f1dd1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5653 zcmb_gU2GiH6}~e&yWSsr*YQvOF!3ZN!3&8OqKII}xI|9CCSc5jETY6&jd#cPD(ji$ z&RxJ!q*|qFleRRWszm5ZqdrvRi9A=8+DcXX(uagZsym`8RV!7z*;*p2JoP*G?yNV# zG*!e%d+)jb=boSMoV$;wJCY3FM;HEBUQ03dPill;R1sSJ0*Ga1F~QC-OPkatbfNbX ztxrZK3}H+}MO3yUld*}Ih)uMKHrX~N;}Z#y=w~y@-RucwB^R|rn%Y&QW{h68n_1Cq z%!(b-Rqr8HG0`r^+AtOm$3%yf7+~3yJ%%lkBrEe)!FK1JaO#NZyyfsEv+Vi~7r9!sgz34GQ*nf2dafxJ9Mkt|yyzsW z+$+_J*vmF?mgN*ZUNYY>- zRKcF<+jEXs8UY`bio<=>E?9GlXRBV>74qDQ?F$p9;!I+^xs~nW7Fd?#k_W$llZwuu zH#g@yB5MlIP9BgyP7d96|$*dNnY!`R1McH%xtR5sF2)hUt z{2)44b}A+G#Ys4#yyygRD$jYYgN2c!?ozR#CRrJ(q1E33vCL+e&|oV1nH~{&1m-eh zSo%JUNGj5#3`s>X8Y5~Z3R95kJ{%fI%t~N1Nuy_&XtQ82R=Zr)0kcSNW0g)3w=yDO zb;2yR1c|1af`~78knpMwmlq78!k(=-L8{8ji#A^>oOPDIOqa?<@dhUDH*uqxz>pHa zVa+cB3cp1C1rW>ZbaP<3VXe&$5WCswh9DT5j`I<)%%c1@eg8l0Rfh`8R{oF)g6 znkZglb($Mkxl^!CKPQ~x0<5(xyxiGZrIv%5JztfMs8w^s@BwJ7Jnt?#uJChyZFaGY zw&&)m<*HLDyH2iHDh^ebg7|2~UYsr2V?kF_OhaMcPMqp*L9xt!Gx*#6zuy0ccm8z# z_vdeqyvlpg3#DgD&YWGVh{Bv*6dqqv?57$ISSr{eh`^46cxbgn(qcoCjKl>&YV|&O zMxLF~pZ*?WCG_>NSF{Dq(kwlvK{S!iX>JhXxFB}}V+uCMd(Z_QVsjo}w1rQ*?%`W$ zUVEX?Fk`1s2)1tAthN7P%=EiaEVFeMji%SAxFTdF?=GJBJ3DBEG^C=#t|RBRPP%W|m2!z- z$ZXoQd9G{7e5nS=m?h-GZ*IgS!vc!4pfi8;#mQrZm#2?gM_-+MdFo`4%-fZkbDVRJ z2QlB{!YN(T0>gJIb5f%FlteoU1sItql-;s`NVm0$v}HibbQ~uwvz1il(%8kZ+q<8< zoqF=quD+G7!H;@A-gRx)&B2jXrX3mhbRc{Cv5{K?M{evHU5#j4#_EwscY2jYI?`(q z)_zdxKq`4En+)QGf@?35I+KOMA|SXzG!*@2&8{>?GKIoinfrna-1RVwjutp86u1G_ zIGHskljD!02zm33gP z+0t{%=vhf_zh!LyYv1mh2VcG2cWU{>WwcMN^z<%ITqxc$de@E6%+$uTXA!^V2Lq!| ztN#>b3uNvU5la&QOaNR&0$kKGKqE#1>|$07G(Zw`n-vG0Xn7nFt zySRWM8C{Du=VcjjLl|kqQ4?Xc3?~4}QM|KFwYLg^rpJ@bJzqRIWdbK!VrwH0P%&~S z`C!1rcR^7NAmV#Kx1bOoKwElBLtsNqYlP_z(c}<)^yout48@g4Wc5|_AzXoRB>ld{o8=$ZRVpYrST0;C|y;&k{vf@@dgNd=gIi zI9kD$7NM(c`L}Su|FT&r)5A3BC>6v{ z@f(qqN~`%X+VwdU_Xy<#y6zH+nlCw-pf!>c(o3kD2eP~W@BSln;5U4K^`7|J`@eke z{ndQQ$>#Sbt-xxvx(R}klFbq*_??lXlp!kpEr=<2&M zvy?ioJ^eGr=CM~FI{}->>r?uP9Y9C63+t-5Y7Ng&qRMd5b7PiF8cjTv8Ue*u@3m$y z8AFmpk0x`l<>{pYLv|>ubEyGJ7_t%N+ZO0R<33n`Q=5$x>8^KJu)W2*?;4T9BPd>G zpFPz7i{YOS-^z@BIDFauIC(93J%4lX$OqbG?Ze@lnb8~Z(f@ps8THA|FLmXNU&V69 zg9R#ZF2UBsPD=Cl1Jz@g*$hxxbO!-pAtJqw*1B}yaBCGKu zsJ;@8GV+ZsC{)SEaSg!hb%V*{Te2ZRWuT)Mb)Bffoutq>p@ar6E!jJxD>lDNB{ ze8?kPAzf%)dnM&BtNngDWCly@Ma7dq$feElc?D&O!h;GA&!%eDoeaKj7KQK zr(0#^%|DNgd=xqcAlTD?9pDBUJAlRQ1r4svdq`)}JN1)}k0! z_iDbiunRej6!|aZmU(tW+w9GDM%&+k35~C&Z{$^{Non%SYk55%$s3!SwWS?yoLh}z zh7qpXsHdB6yN$klG*ogw%ab(q&q*UdX(>;c6%k`floEXm6{qBq=csrB1(K&i$t&X3MGq^P9dXJAIE6%>rx|fPH1NNTSmNKN zz5*3=4StXca#nSie5{cXaVqxW=M!3gpRMV!X#7qmvg|wU$OP}i!eTv%sI?krvGf-@ zAg&$(wMrC3p!)ioef{gn&@k0;bjZ{g8H(a{oG4;Fo96SV=(RdmqkkiDIdb$M)%-`z z@wr?`1|$DCQGfZw#8MUSAx<_EB<8qx&cVNug72egQup$DJ=CO(NJXFcsCpES@kR8J zCHbUCJghZsO^<8FogStQ{F@E_ojv+Dw)G#~eV;shZx_TLqm>?0+-cO RtJGXK4{6s z4co&OH>4kOR|w`&cE_#>oc6sMo21nZc{@IV9RhUdH+hXwe~k*>6dUqxBk)~MoO8W% z!fi0hSpadU?KyPAl_Iz`yAE#6_d=Yh3D@`V`91rwLdZcY7bsM?I#ZdTfz5?0uV4;cZ5hH|tuxEB@m=}BnjMHHXO3kH>evGEFZh)E zA4=(^1!%)}+%*?xai7ELOfYBC7}uh_?Rmbi<;CFw=2>}KsslO)tC((0Du-XO`a)n>K)vPy*u(}_{ zJuV1}kl#LMhoQUfHSpeHF>J0jT+Ds17`OqeyB;gbhn);sQNgl0`ZLSoz4*{8_=n#@ zw?!TkL(3h=u08zbz8pV)T}=!25G|6JuSaP+~OmeZqh2v{5O~Rf7 z!=RluW0t0?C0d=nGFPJIcj&^+DlPq_T&Y%+BU&}5W+@*I>Di-0@@&Z9J8)WcPkEtQ zx>_=+I=-+pH#hOrUO!-5y|&!EJZ;Wgn||TmV?OrMy#aR{Hg8$CSnH*GhYDZ>G$2F^PB1aCt^a!?U2G zj06SlgqwYJ0L4#_!t7@}Czq2z6`ltL6$%*Q2v900A64BMxiE@OI_i-2%7N_KL+AJ9 z__?Kq_PbTZj+N+OHjcpQ@P`tq!+v-m99H|TViU3s?lACs6uc8;DbY+!l4jzaMM8qa zastD14cy~5ch4n4%laL%aO@bt( zB#ZLRo@u7GoV?hPbA+s%>@I`b3CdCEIf8$Xa1olD_l#+kQB>;aycZR|$^{BO48v52 z6{iQU*;ob4B0Y5XcGAkEQFd4(@R4C^Eefm!+PfYNj{0PRo0d6fYH zf$vHV*IS=7Hs35SR7z%*BFb+*hYHKnb4#TPJ^S_qed;$po{O?wkce_e5D=wQ3`F^3 z!5bMCVAi$k?k0*I8 zSOrV?D)hMYUHK31qJ#hTlET1W`c532BK<>$14bcz&=2bF^nM@d>D{@$eSNRcvvXtn zh8(VLUzNlB_WXmwsof8Lcl$TDzbc&Wunv1NIpg324(|_;R1TnBNbjElbzdL+QXkyQ zzP*<%e4R?B(qCsuq3^Ev_)=RR{P#acurHLJxifw0impEu?@|>N?^Fr<;&_kKT^#Gf z*bDQyHPH!aQ~)mx>9=qwd`*4jG10UO2eR9(?#to#H8pzzN7$xX3cV8VGKej*5_8di zQ&fHjscA(jq?$>7*og>$lqR9mVQY4?F2<9Q{(k6tQOapHf-vf_YFw&!3|P5L zn5t7M#8CPm-?wx5*OzylPv-xeKi}5RKasr?apeCoZItv^-;dJhP&g>#@;BtyUxt@g zq{yeyNxV#IOWJ|zcW>{j@eiZ%{E3j*s@f6W{}xjixGTIgWygGkbfKjSqYeOOSk?jZ zd7fz(JM&SpA`_HqCOHOzM|HwCfnfo)!}%l7+Ry-1eK(acRBhS{7~LF_Wgno1Ua#C- z0G`D0EGx$GbY!@pi-Ocf!NB1-B+9v=AD3EDu8Q*|&V9tO02yU1RbyHfQqcr9AAj`k zVa{czO#34cTjW9J z5tq)S(z$|{*d>*S(lgZcF&Jfkj9R!^XK(Q-EXdake}Ha3scG6lBCF|-28lNMkmUbL fzWtE&{GFV6lt%6M=r!ckM^gyJ$64t<>c;;9zA^H* diff --git a/submission/sample-candidate/pipeline/__pycache__/warehouse.cpython-314.pyc b/submission/sample-candidate/pipeline/__pycache__/warehouse.cpython-314.pyc deleted file mode 100644 index da28d229906ec66ff5cf0d96b82699210bbe1efa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5351 zcmdrQU2GgjdG>DaZ|}}`{t1rbB%a2>^`(xl4WWcIad7N&?bx|o*XK*Bx;?G$#`ZRQ zyVu>lI5r53RNx8^#1v%rsYW2xRgs8zX$nH>3#d=Xk*R7{m55sLfW(7?Qvy8k&Ft-+ z&vi)4BO~q3%s1bBGxL4ld_VJ8Ls&pi{_yhe<*N~dKBgU~cYdk&&?7QOL#CbmPjY; zBVan4nlr=z8B}JPlk<<9RJo8M6==1S$Rht7^S1X616Q^}z&|OT; z16{+^iByo)8KFS*8G>KbrtnoV55qvOsLKUOpQnIcJU%dZ0>j2l4Iqp&2ALpS^pVoA2$^~*9_;`&d7&xZO1SAsUte~W-8i)_hVI-j+0Evkt zr|EeskSoEeus_2J&COQ(90QU?_0I<^z0dduk7)J~%0wX&C3k`WWXFdnV?xAOY z-V(Cu{_|sr?8W|+6?8~vWmwfFNSh5iDA2wyn;NAPa-ffhezfrLxuLs_p|OZE=mpoQ zml4VX>>zrEo8?4Kbct?}k5gs#{M7|$7N65p_1|qEz^nPG-a)7a&!^@G07%#YEU!+? zlCD~OQd7yB2pHo9E*2!ypz9v7uEZw{U`*U3x!G&LO;d|sEh(kAPKugg|A9a;o|jDN zxMUdej9Q?#Aa0bV3NrLHHC~j9M3GezuU?7X;=C2gW-B5g*{r^wt_Gjc4!=coA9fPB zOMmM}p4y`IyFBY|!!Tlep$%H#{5D?{Xi&bS9FEb^N)ge?3bf-v7~DfKTr~{Q4Ls(h zMIZ{7%q|0Bq93jh6E_|i!pTc%oVYSFo*suwR?{lnU7bPa?NAAfGyUSgaKDIC;>g8* z@mV~Yc($hwTa^l=vRoCr9)lDlS#eM*^y)FF5i1!Llsz$WaA(10WWl6hoF2KD7*F?K zOx05<6+z?=!Kv9Vpfi$8C!S7-j4jEDv9Wq`H9G~U=Pz9vOY|qJs9>Fj`X|QHcv?~n zQe{YtwWK#byAUdOc$WyO7m@Php(5BzVD)F2k`#$rYE}jl10c(>?k7(5;1jWWMB9kv z<{9FilK6j<)!#n5tajK$W0$~o41u})3^J>ea81bd(vS*TNu?TJDgRA!c^Moe^NBX{ z!-WTUdm@3qBoJFT>asUd-Gu~754e-y&rQX-lMhC#yOx0VIc_Fkf3C|(J*|&iC%KYI zVw~Ox=9oE_F1;0+R;zt5@U4}YD{Fgl{UD4ockoqcETT=`|Glwqk8Pn~lm8wHdIDPq zkSAb+LKGAN(7`Gw+~ofU3VQ;28_ceAqBP!QPV^|F`(P(ZW6L--@;*HZsBGtY6SOQJ zNFJgFVN}pmT@Uo_Q&4yA6dF&%DLmUhhyNm)thhFWG7tnnRO|Xm1v`oo+|b(y6rdDTMsY)bXgQ2R+EV?KC4*Y+Z&1c+?R;K3zG+4>-PJagWe-MqPVdt(C%MzS* z?7HXyc&Hvmd!qZ)6$pL088wuGtYUW&Vbu^4JsBSU-V>A#mE*3G@$|Y*RyfmvMpqDu z)16Q`?nPXcCB{y_5$iTP=gkQjNnK& ze}T$p?;U4+Q8W=Oxq4B_HRMC+rQa|)xst9?(2)Q-4De#Gy64aX_H0YJArpex&dz<8 zzthY0)V742Q*^*>82vtI&z?NY_^PXgAFx`4aVVMz!)lG00Q{Ry_<-A04FdYVws$W! zcftue*ePV1TmBX{B=i@H$~?>3DnR4RP5MIUvvg*&p>uV3_4L~1o0*%#w+7ck{hQ(D^6<9D-6GugqG0nY-9I>X z?jc;T`^V=h=<6zVe0oL#}># z?Z8^-=F_)^Zk>AfdeC{*H>O#bFH0P zJ9hKJt#8~?-|JtGCSfLfRyhlNQOYqb{lYMG0;l8p*2~e=g^FS}WcIglhAp2%Ss_z3fDRwL>#7KL(h8}XcLD@w-?(ebad@uM)_?ppMf53})48sJgl K{uRy%(EbBRM*6h? diff --git a/submission/sample-candidate/pipeline/cdc.py b/submission/sample-candidate/pipeline/cdc.py deleted file mode 100644 index c8c2dcb..0000000 --- a/submission/sample-candidate/pipeline/cdc.py +++ /dev/null @@ -1,89 +0,0 @@ -""" -CDC capture layer. - -Simulates WAL-based change capture: every insert/update/delete on the source -produces a CDCRecord with a monotonically increasing sequence number. - -Replay safety: callers can checkpoint the last processed sequence and call -records_since(offset) to replay only unprocessed changes after a restart. -""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from datetime import datetime, timezone -from typing import Any - -VALID_OPERATIONS = frozenset({"insert", "update", "delete"}) - - -@dataclass -class CDCRecord: - operation: str - table: str - primary_key: str - data: dict[str, Any] - captured_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) - sequence: int = 0 - - def __post_init__(self) -> None: - if self.operation not in VALID_OPERATIONS: - raise ValueError( - f"Invalid CDC operation {self.operation!r}. " - f"Must be one of: {sorted(VALID_OPERATIONS)}" - ) - - -class CDCCapture: - """ - In-process CDC log. - - Production analogue: a Debezium/Kafka connector reading Postgres WAL. - Each record carries a sequence number equivalent to a Kafka offset or - Postgres LSN for checkpoint-based replay. - """ - - def __init__(self) -> None: - self._log: list[CDCRecord] = [] - self._seq: int = 0 - - # ── public write API ───────────────────────────────────────────────────── - - def insert(self, table: str, pk: str, data: dict[str, Any]) -> CDCRecord: - return self._record("insert", table, pk, data) - - def update(self, table: str, pk: str, data: dict[str, Any]) -> CDCRecord: - return self._record("update", table, pk, data) - - def delete(self, table: str, pk: str, data: dict[str, Any]) -> CDCRecord: - return self._record("delete", table, pk, data) - - # ── public read / replay API ───────────────────────────────────────────── - - def records_since(self, offset: int = 0) -> list[CDCRecord]: - """Return all records with sequence > offset (checkpoint replay).""" - return [r for r in self._log if r.sequence > offset] - - @property - def latest_sequence(self) -> int: - return self._seq - - @property - def log(self) -> list[CDCRecord]: - return list(self._log) - - # ── internal ───────────────────────────────────────────────────────────── - - def _record( - self, operation: str, table: str, pk: str, data: dict[str, Any] - ) -> CDCRecord: - self._seq += 1 - rec = CDCRecord( - operation=operation, - table=table, - primary_key=pk, - data=data, - sequence=self._seq, - ) - self._log.append(rec) - return rec diff --git a/submission/sample-candidate/pipeline/lake.py b/submission/sample-candidate/pipeline/lake.py deleted file mode 100644 index 7cb15a7..0000000 --- a/submission/sample-candidate/pipeline/lake.py +++ /dev/null @@ -1,69 +0,0 @@ -""" -Lake layer — append-only storage for every CDC event. - -Every change is written exactly once. The lake is the source of truth for -point-in-time replay and historical reconstruction. - -Production analogue: Parquet/Delta files on S3 or GCS, partitioned by -table_name and captured_at date. No row is ever modified or deleted. -""" - -from __future__ import annotations - -import json -from datetime import datetime - -import duckdb - -from pipeline.cdc import CDCRecord - - -def create_lake_table(conn: duckdb.DuckDBPyConnection) -> None: - conn.execute(""" - CREATE TABLE IF NOT EXISTS lake_cdc_events ( - sequence INTEGER NOT NULL, - operation VARCHAR NOT NULL, - table_name VARCHAR NOT NULL, - primary_key VARCHAR NOT NULL, - data VARCHAR NOT NULL, - captured_at TIMESTAMP NOT NULL - ) - """) - - -def append_to_lake(conn: duckdb.DuckDBPyConnection, records: list[CDCRecord]) -> int: - """ - Append CDC records to the lake. - - Returns the number of records written. - Idempotency note: in production, deduplicate by sequence before appending. - """ - if not records: - return 0 - - rows = [ - ( - r.sequence, - r.operation, - r.table, - r.primary_key, - _serialize(r.data), - r.captured_at, - ) - for r in records - ] - conn.executemany( - "INSERT INTO lake_cdc_events VALUES (?, ?, ?, ?, ?, ?)", - rows, - ) - return len(rows) - - -def _serialize(data: dict) -> str: - return json.dumps(data, default=_json_default) - - -def _json_default(obj: object) -> str: - if isinstance(obj, datetime): - return obj.isoformat() - raise TypeError(f"Object of type {type(obj).__name__} is not JSON serializable") diff --git a/submission/sample-candidate/pipeline/warehouse.py b/submission/sample-candidate/pipeline/warehouse.py deleted file mode 100644 index ab32401..0000000 --- a/submission/sample-candidate/pipeline/warehouse.py +++ /dev/null @@ -1,124 +0,0 @@ -""" -Warehouse layer — current-state snapshot built from CDC events. - -Each warehouse table mirrors the source table with two extra columns: - _cdc_seq : sequence of the last CDC event that touched this row - _deleted : soft-delete flag set when a DELETE event is received - -Production analogue: BigQuery/Snowflake tables refreshed by a streaming -merge job keyed on primary key. SCD2 history tables would sit alongside -these current-state tables for time-travel queries. -""" - -from __future__ import annotations - -import duckdb - -from pipeline.cdc import CDCRecord - -# Source table → warehouse table -_TABLE_MAP: dict[str, str] = { - "customers": "wh_customers", - "wallets": "wh_wallets", - "transactions": "wh_transactions", -} - -# Source table → primary key column name -_PK_MAP: dict[str, str] = { - "customers": "customer_id", - "wallets": "wallet_id", - "transactions": "transaction_id", -} - - -def create_warehouse_tables(conn: duckdb.DuckDBPyConnection) -> None: - conn.execute(""" - CREATE TABLE IF NOT EXISTS wh_customers ( - customer_id VARCHAR PRIMARY KEY, - name VARCHAR, - email VARCHAR, - status VARCHAR, - created_at TIMESTAMP, - updated_at TIMESTAMP, - _cdc_seq INTEGER NOT NULL, - _deleted BOOLEAN NOT NULL DEFAULT false - ) - """) - - conn.execute(""" - CREATE TABLE IF NOT EXISTS wh_wallets ( - wallet_id VARCHAR PRIMARY KEY, - customer_id VARCHAR, - balance DECIMAL(18, 2), - currency VARCHAR, - status VARCHAR, - created_at TIMESTAMP, - updated_at TIMESTAMP, - _cdc_seq INTEGER NOT NULL, - _deleted BOOLEAN NOT NULL DEFAULT false - ) - """) - - conn.execute(""" - CREATE TABLE IF NOT EXISTS wh_transactions ( - transaction_id VARCHAR PRIMARY KEY, - wallet_id VARCHAR, - amount DECIMAL(18, 2), - direction VARCHAR, - status VARCHAR, - reference VARCHAR, - created_at TIMESTAMP, - settled_at TIMESTAMP, - _cdc_seq INTEGER NOT NULL, - _deleted BOOLEAN NOT NULL DEFAULT false - ) - """) - - -def apply_cdc_records( - conn: duckdb.DuckDBPyConnection, records: list[CDCRecord] -) -> None: - """ - Apply CDC records to the warehouse current-state tables in sequence order. - - - insert / update → upsert (insert new row or overwrite existing) - - delete → set _deleted = true - """ - for record in sorted(records, key=lambda r: r.sequence): - wh_table = _TABLE_MAP.get(record.table) - pk_col = _PK_MAP.get(record.table) - if not wh_table or not pk_col: - continue - - pk_val = record.primary_key - - if record.operation == "delete": - conn.execute( - f"UPDATE {wh_table} SET _deleted = true, _cdc_seq = ?" - f" WHERE {pk_col} = ?", - [record.sequence, pk_val], - ) - continue - - data = {**record.data, "_cdc_seq": record.sequence, "_deleted": False} - cols = list(data.keys()) - vals = list(data.values()) - placeholders = ", ".join(["?"] * len(vals)) - - existing = conn.execute( - f"SELECT COUNT(*) FROM {wh_table} WHERE {pk_col} = ?", - [pk_val], - ).fetchone()[0] - - if existing: - set_clause = ", ".join([f"{c} = ?" for c in cols]) - conn.execute( - f"UPDATE {wh_table} SET {set_clause} WHERE {pk_col} = ?", - vals + [pk_val], - ) - else: - col_list = ", ".join(cols) - conn.execute( - f"INSERT INTO {wh_table} ({col_list}) VALUES ({placeholders})", - vals, - ) diff --git a/submission/sample-candidate/requirements.txt b/submission/sample-candidate/requirements.txt deleted file mode 100644 index ab2841c..0000000 --- a/submission/sample-candidate/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -duckdb>=0.10.0 -pytest>=7.4 diff --git a/submission/sample-candidate/scripts/check_schema_contracts.py b/submission/sample-candidate/scripts/check_schema_contracts.py deleted file mode 100644 index 1698b21..0000000 --- a/submission/sample-candidate/scripts/check_schema_contracts.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python3 -""" -check_schema_contracts.py - -Validates that the source tables expose all columns defined in the schema -contract. A missing column means downstream CDC logic or warehouse models -will break — this script fails the build before that happens. - -Exit 0 — all contracts pass. -Exit 1 — one or more violations found. -""" - -import os -import sys - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -import duckdb - -from source.models import SCHEMA_CONTRACT, create_source_tables - - -def check_contracts(conn: duckdb.DuckDBPyConnection) -> list[str]: - """Return a list of violation messages. Empty list = all passed.""" - violations: list[str] = [] - - for table, expected_cols in SCHEMA_CONTRACT.items(): - try: - rows = conn.execute(f"DESCRIBE {table}").fetchall() - except Exception as exc: - violations.append(f"{table}: could not describe table — {exc}") - continue - - actual_cols = {row[0] for row in rows} - - for col in expected_cols: - if col not in actual_cols: - violations.append(f"{table}.{col}: column missing from source table") - - return violations - - -def main() -> int: - conn = duckdb.connect(":memory:") - create_source_tables(conn) - - violations = check_contracts(conn) - - if violations: - print("Schema contract violations:") - for v in violations: - print(f" ✗ {v}") - return 1 - - print(f"All schema contracts passed ({len(SCHEMA_CONTRACT)} tables checked).") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/submission/sample-candidate/scripts/run_data_quality_checks.py b/submission/sample-candidate/scripts/run_data_quality_checks.py deleted file mode 100644 index 31e4cba..0000000 --- a/submission/sample-candidate/scripts/run_data_quality_checks.py +++ /dev/null @@ -1,212 +0,0 @@ -#!/usr/bin/env python3 -""" -run_data_quality_checks.py - -Runs system and business data quality validations against the warehouse. -Seeds an in-memory database with representative data, applies CDC events, -then asserts correctness invariants. - -System checks : PK uniqueness, not-null, referential integrity -Business checks : non-negative balances, positive amounts, valid status enums - -Exit 0 — all checks pass. -Exit 1 — one or more failures found. -""" - -import os -import sys - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from datetime import datetime, timezone - -import duckdb - -from pipeline.cdc import CDCCapture -from pipeline.lake import append_to_lake, create_lake_table -from pipeline.warehouse import apply_cdc_records, create_warehouse_tables - - -def _now() -> datetime: - return datetime.now(timezone.utc) - - -def seed(conn: duckdb.DuckDBPyConnection, capture: CDCCapture) -> None: - """Populate lake + warehouse with representative test data.""" - ts = _now() - - capture.insert( - "customers", - "c1", - { - "customer_id": "c1", - "name": "Alice", - "email": "alice-test-user", - "status": "active", - "created_at": ts, - "updated_at": ts, - }, - ) - capture.insert( - "customers", - "c2", - { - "customer_id": "c2", - "name": "Bob", - "email": "bob-test-user", - "status": "active", - "created_at": ts, - "updated_at": ts, - }, - ) - - capture.insert( - "wallets", - "w1", - { - "wallet_id": "w1", - "customer_id": "c1", - "balance": 100.00, - "currency": "USD", - "status": "active", - "created_at": ts, - "updated_at": ts, - }, - ) - capture.insert( - "wallets", - "w2", - { - "wallet_id": "w2", - "customer_id": "c2", - "balance": 50.00, - "currency": "USD", - "status": "active", - "created_at": ts, - "updated_at": ts, - }, - ) - - capture.insert( - "transactions", - "t1", - { - "transaction_id": "t1", - "wallet_id": "w1", - "amount": 25.00, - "direction": "credit", - "status": "settled", - "reference": "ref-001", - "created_at": ts, - "settled_at": ts, - }, - ) - capture.insert( - "transactions", - "t2", - { - "transaction_id": "t2", - "wallet_id": "w2", - "amount": 10.00, - "direction": "debit", - "status": "settled", - "reference": "ref-002", - "created_at": ts, - "settled_at": ts, - }, - ) - - create_lake_table(conn) - create_warehouse_tables(conn) - records = capture.records_since(0) - append_to_lake(conn, records) - apply_cdc_records(conn, records) - - -def run_checks(conn: duckdb.DuckDBPyConnection) -> list[str]: - failures: list[str] = [] - - # ── system checks ───────────────────────────────────────────────────────── - - pk_map = { - "wh_customers": "customer_id", - "wh_wallets": "wallet_id", - "wh_transactions": "transaction_id", - } - for table, pk in pk_map.items(): - total = conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] - distinct = conn.execute(f"SELECT COUNT(DISTINCT {pk}) FROM {table}").fetchone()[ - 0 - ] - if total != distinct: - failures.append( - f"{table}: PK not unique — {total} rows, {distinct} distinct {pk}" - ) - - not_null_checks = [ - ("wh_customers", "name"), - ("wh_customers", "email"), - ("wh_wallets", "customer_id"), - ("wh_wallets", "currency"), - ("wh_transactions", "wallet_id"), - ("wh_transactions", "amount"), - ("wh_transactions", "direction"), - ] - for table, col in not_null_checks: - nulls = conn.execute( - f"SELECT COUNT(*) FROM {table} WHERE {col} IS NULL" - ).fetchone()[0] - if nulls: - failures.append(f"{table}.{col}: {nulls} NULL value(s) found") - - # ── business checks ─────────────────────────────────────────────────────── - - neg_bal = conn.execute( - "SELECT COUNT(*) FROM wh_wallets WHERE balance < 0" - ).fetchone()[0] - if neg_bal: - failures.append(f"wh_wallets: {neg_bal} row(s) with negative balance") - - non_pos = conn.execute( - "SELECT COUNT(*) FROM wh_transactions WHERE amount <= 0" - ).fetchone()[0] - if non_pos: - failures.append(f"wh_transactions: {non_pos} row(s) with non-positive amount") - - bad_dir = conn.execute( - "SELECT COUNT(*) FROM wh_transactions" - " WHERE direction NOT IN ('credit', 'debit')" - ).fetchone()[0] - if bad_dir: - failures.append(f"wh_transactions: {bad_dir} row(s) with invalid direction") - - # ── lake completeness ───────────────────────────────────────────────────── - - lake_count = conn.execute("SELECT COUNT(*) FROM lake_cdc_events").fetchone()[0] - if lake_count == 0: - failures.append("lake_cdc_events: no records found — lake appears empty") - - return failures - - -def main() -> int: - conn = duckdb.connect(":memory:") - capture = CDCCapture() - seed(conn, capture) - - failures = run_checks(conn) - - if failures: - print("Data quality failures:") - for f in failures: - print(f" ✗ {f}") - return 1 - - print( - f"All data quality checks passed ({len(run_checks.__code__.co_consts)} rules checked)." - ) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/submission/sample-candidate/scripts/validate_catalog.py b/submission/sample-candidate/scripts/validate_catalog.py deleted file mode 100644 index 6a02026..0000000 --- a/submission/sample-candidate/scripts/validate_catalog.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python3 -""" -validate_catalog.py - -Verifies that catalog/catalog.json is present and contains a complete entry -for every required lake and warehouse dataset. - -Required fields per entry: name, layer, description, owner, schema, update_cadence - -Exit 0 — catalog is valid. -Exit 1 — missing file, missing datasets, or missing required fields. -""" - -import json -import os -import sys - -CATALOG_PATH = os.path.join( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))), - "catalog", - "catalog.json", -) - -REQUIRED_DATASETS = [ - "lake_cdc_events", - "wh_customers", - "wh_wallets", - "wh_transactions", -] - -REQUIRED_FIELDS = ["name", "layer", "description", "owner", "schema", "update_cadence"] - - -def validate() -> list[str]: - violations: list[str] = [] - - if not os.path.exists(CATALOG_PATH): - violations.append(f"catalog.json not found at {CATALOG_PATH}") - return violations - - with open(CATALOG_PATH) as f: - try: - catalog = json.load(f) - except json.JSONDecodeError as exc: - violations.append(f"catalog.json is not valid JSON: {exc}") - return violations - - datasets = {d["name"]: d for d in catalog.get("datasets", []) if "name" in d} - - for name in REQUIRED_DATASETS: - if name not in datasets: - violations.append(f"Missing dataset entry: {name}") - continue - - entry = datasets[name] - for field in REQUIRED_FIELDS: - if field not in entry or not entry[field]: - violations.append(f"{name}: missing or empty required field '{field}'") - - return violations - - -def main() -> int: - violations = validate() - - if violations: - print("Catalog validation failures:") - for v in violations: - print(f" ✗ {v}") - return 1 - - print(f"Catalog validation passed ({len(REQUIRED_DATASETS)} datasets verified).") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/submission/sample-candidate/source/__init__.py b/submission/sample-candidate/source/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/submission/sample-candidate/source/__pycache__/__init__.cpython-314.pyc b/submission/sample-candidate/source/__pycache__/__init__.cpython-314.pyc deleted file mode 100644 index b3cc59dc4a6f82d0cf027e07e2ab3c4ed573993c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 184 zcmdPq>P{wCAAftgHh(Vb_lhJP_LlF~@{~08CD^x$UIJKx) zza%v|qb#*3vm{?XyELa%zbLgJzZgQMmXsFgrzDmn>LwN!XQt=nrskCt>lc?M@0%4+Er1y;=UYtT4>4+;2kgm+;cuvnvBE7tz=y`Zo z;9Y>Xx}a3^JpT!LfHss-q!&k!UK&B}F<#J>w~=*f~0oSGgpE$;Xp z1Jxx?eQyneS%*7>;f#@qL{Ys*%#Dr+2BEAKJnlNH#M-f3u6#w=-SrRf6mT=fCe5}VcpP7s3yhmQV0hIbuFWtdj8n(r(GDA)_($Re zZM5zbv=8z#qXfC8_`Ei6h*P!injb>iVoFa-arHp5B^SlAb%74~ON!z^$G z5|0=Kc=GTZT!U#B-9-;_)1M<`1Mdv_NLg2OWf@kYv6xG!&Rfsv`DzY)T=+w*h6T&_ zys$(G52)AFa$(61tPOi59GZdm%#FpJItUQyD@_SY5Cv_MGjX1`!lmjR2A-~RVy$lx z>Ttih5x7B>l1-mQGuYj%+9o$Inv6MXUKBhmSZO=(?R!-wgRI*2ZQ`=Y&7H8{Fig+$ z0d`^-bO1KG01v}3?V{&MQz}oz^knOo(di#Yr}xCi_btgg02w6l5`-86D|iYAE)W8* z#RN&Q8V_B$6mv&kCXsukM7|@T7XaVlqefji9Y=qNxw+y(aBn+&9RuhPM{r*_!;Xjf zI>Bqs7Q{sax>%1fpESROv#B^9b-ht*HE^qTW3GW`Z{p@c3pYNVU1}|5F5Y-5F0_*P zG8`M@yEVOjtES^ceRjU4-^aHb_b0lfJ+n=c%~ENxOY_d$T(>AB1;=O>FUDxjq9*)4GqBxpEk{X6GAA zt=jyeMC`?X2{s`{qx*T7aC$^dQsyw+os{u@OxT{44!G+tlgPw;i<`~sq`;X#|R>JPGU{u z(4LK~fKuXFJ5sIJuY+Ti`j3eBKSf6Hnvs#s@IB#?S$e%Y+Z9g$crV=9Ac-S# zJk{y(TU`63+r;#;7>OgGQ!0Dx*ba?~Mr^6~Oozu62*-vy`ts_-WwsqId}@;wAbKx* zR}6ubf4f~a&4|q6cxy7xM8#Z?tB~xXd_xJ;y2COKC96aPeUd=qLv%-bS6YITzA7_i z`UweTDxV@4<)kK5=RWd~SOF)RLPdKhtMqx=4`g^U+SiDRkr5RmdK{Kyb-fJ4F8Zxd z{$lRaxjm!~mH$L)v9fmx6)O>|U$FXtbtYvE43(dufntRYf_7J%QUr!o5*z9sxU1&m+;4U5T7T z-@vLciPmI5DLfU^^LK`o+&+To@yW;5lS|*^zrOsH_w(^{PX%xP9EeEBv{HB^ld8jl zD86A)*43~Sg=DC90wz>DWlHM{E9o7kr^J8u#pyUKC&`zd20lF_rZ-{I%Hd#la-nC1 zVkx@CVI{qEQZcNg0WEEWY9|)85TJMR}G3U=GeAWpJ;IipPFBe(%@9z&2pA2 diff --git a/submission/sample-candidate/source/models.py b/submission/sample-candidate/source/models.py deleted file mode 100644 index 91c9d19..0000000 --- a/submission/sample-candidate/source/models.py +++ /dev/null @@ -1,84 +0,0 @@ -""" -Source schema for a payments/wallet system. - -Domain: customers own wallets; wallets have transactions. - -Strong entities : customers, wallets -Weak entities : transactions (lifecycle tied to wallet) - -Invariants: -- wallet.balance >= 0 -- transaction.amount > 0 -- status fields restricted to known enum values -- settled_at must be >= created_at when present -""" - -import duckdb - -# Expected columns per table — used by schema-contract checks. -SCHEMA_CONTRACT: dict[str, list[str]] = { - "customers": ["customer_id", "name", "email", "status", "created_at", "updated_at"], - "wallets": [ - "wallet_id", - "customer_id", - "balance", - "currency", - "status", - "created_at", - "updated_at", - ], - "transactions": [ - "transaction_id", - "wallet_id", - "amount", - "direction", - "status", - "reference", - "created_at", - "settled_at", - ], -} - - -def create_source_tables(conn: duckdb.DuckDBPyConnection) -> None: - """Create source tables with constraints in the given DuckDB connection.""" - conn.execute(""" - CREATE TABLE IF NOT EXISTS customers ( - customer_id VARCHAR PRIMARY KEY, - name VARCHAR NOT NULL, - email VARCHAR NOT NULL, - status VARCHAR NOT NULL - CHECK (status IN ('active', 'suspended', 'closed')), - created_at TIMESTAMP NOT NULL, - updated_at TIMESTAMP NOT NULL - ) - """) - - conn.execute(""" - CREATE TABLE IF NOT EXISTS wallets ( - wallet_id VARCHAR PRIMARY KEY, - customer_id VARCHAR NOT NULL REFERENCES customers(customer_id), - balance DECIMAL(18, 2) NOT NULL DEFAULT 0.00 - CHECK (balance >= 0), - currency VARCHAR NOT NULL, - status VARCHAR NOT NULL - CHECK (status IN ('active', 'frozen', 'closed')), - created_at TIMESTAMP NOT NULL, - updated_at TIMESTAMP NOT NULL - ) - """) - - conn.execute(""" - CREATE TABLE IF NOT EXISTS transactions ( - transaction_id VARCHAR PRIMARY KEY, - wallet_id VARCHAR NOT NULL REFERENCES wallets(wallet_id), - amount DECIMAL(18, 2) NOT NULL CHECK (amount > 0), - direction VARCHAR NOT NULL - CHECK (direction IN ('credit', 'debit')), - status VARCHAR NOT NULL - CHECK (status IN ('pending', 'settled', 'failed', 'reversed')), - reference VARCHAR, - created_at TIMESTAMP NOT NULL, - settled_at TIMESTAMP - ) - """) diff --git a/submission/sample-candidate/tests/__pycache__/conftest.cpython-314-pytest-9.0.2.pyc b/submission/sample-candidate/tests/__pycache__/conftest.cpython-314-pytest-9.0.2.pyc deleted file mode 100644 index d04dfd07f2a8da48b1db961c5e3af7a09d578951..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3905 zcmbUkO>Y~=b#}S@UXmvDMcIn9mSnj~EHZX=NE4@!Y&oHNFw}LTw1MlzYIkHVOzsl1 zvsT3uTLegv1c(kkC_sSb>U)1f&o(T;%v5cH7DdpTTsh6PZ)TU2Y|B#G1?KC``ZmBkpVDSBD z9h^XVas=tA5!5(Ehklb;xzJPYU_6WFqlKOZ+ziL?@KHL*jLRKX9YNUv(>_`ZBWl>n0GKMS_A`?SQyk?YjC6@ zAT>4(%eD-HEsxkf8;S5XK*PEh_&7xNXC*cmZTVh6tRCC~wU)35}=ptn0K5zY!^CBPO$C@y%RKoD#0z!4|||*RlzRioay_{ZC4nu>IZ902iBeov|zrf<+dikNA(N!QTE&e<(!~6+DC5)n&W+R)JeGC2&4f70hF)<>yEv~N=q;bWkq zC>R`kDO3|46gmqNi;kPv9~JU{KnW@Lgm<5)E^fZCeh0pV!W#Dnnc5j#A!L&TKF4DC6C%+Nm8A2h)Pv z@o~`iIpJl61seF zxvMhh93xz+@R(2p0Nmk($2{TUjx*sD?EBNu@htSAB#k~1{k`GG#aAB|U)>X{KYbWN z!_yp2ID+1 zQfd{bRiMrRb^d>(F6gK9A~Y_)qz~wW`p`2Rk|)Y9=)=HaSm;pFNB$2EWqnjX4UOw@ zMjz8((Z@lD30>3A3Oh`K9a1OiQP!t`%aqXNj6Mxq&P7~iUc#lHo(>#xtDmS^v4ceQ zL2N$qG^DS7qB;!yK>8_FgNsm+X(_ST`i$~qp ze!>;FphY)xwvu7#zDy=t7g@HqG#r~Lt_hV*e6itJFyN-?G};A|ujcs6JqI-jVF@3e z?Z9UVuGd}6GGT3E8zyC$pv4O$F_*1NEOy0PYvLsk^xMz#mLq0FJJyIpm7JN`UvA;m?iAo6ryusqt}28NZeczvbGZ(mjUGC0G0lbNK=@c+_)8d!oWNil6^0+@ryk~~zFK^g zpXt22tCY6pca*dInYVY8>29|0hqoUnW$_Tqo@PnIuszE#SZXbB8Wh5TsSB-5qrVz8 z!vBs66v_}O5Q!CnH=wA-EZqn3un1upAm;bZFo`HiMLimpFNtu6sl9)c3j9fk=Z)q1 zAYKiH;m7M6R_br`6NQLNQTVW2SVUEto{bxRSQd$}PDVdT2f-^M^ZD05br~pw2L3s@ z0m67S;kTjNi%F97HCp@*p^wmGbp9VG_ZO7=Cz|>Oz4Z-R{wu0{pMXk#FNNgP$K$^r qeqAi&1We#ug_gd`zTjo>YmHz@;YGMZf diff --git a/submission/sample-candidate/tests/__pycache__/test_catalog.cpython-314-pytest-9.0.2.pyc b/submission/sample-candidate/tests/__pycache__/test_catalog.cpython-314-pytest-9.0.2.pyc deleted file mode 100644 index 5ee79d6afd506ff0c9b50276abc05744fe67148a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 19689 zcmeHPZERFmdcJq&j>j|OFTTGFjBSE5fnc!BH#QJ3;D&&6Lx!yCFq(L-VZwMkoI8ft z-L%f`Rxuk=SZS+uD=N_ssUSsFD&@yk%8yi9ZTn+Yjj@fkrkSJk zd+wR>1sithR#gtex##=qzUO_P_dVx&OI>YPgS$QP^Yo^rnl?ZY#|2A)hkxqVv~kVU z^4dAg)O+>3k3anDKQLHF6W$_+cAN0eOS z zS-ncjG<22<<(Dq1UI_KVW%ZVskyToLxmg2$g;|UII?B(lM0?xR5r}Z#j`?Y(@n(Ie zPo$gmXrZC3hc)IBq%GyGu6?>zpP<&VHyh+wmZRSl>R9Aztut35ZxtU?;}f)Q;4Qsy zT9-YnMk{O79`fs-q%~#T)}p0#Wi4$m8y~e z5Ppt_Tb^VT6?58*R$8eQxmue{y+w;{8JWX74xh2?ydC}3_>ZFl$$T<1d_Fp4<&!Cd zqXWZ3xr~*!vX*T}li3s!7_$cQ6yFvO9~r)EF?(NhFrBfYIcC{bcEH*iO=dDtW?dRd zGbM4{!V)^QNBko`6MWy)9FDnBYtf?G7Ok~Z$Q#ipNSIt{p+=~ z9@9_7jG}=TmMzvCIeg~u$rABW`>g~EW19)S3JrhXq3@pxVIhwI|grD z`*o=HT4pY^ZYH#D^3vz^%dU_9aCCCX2jBhLuZ0_tj^A`>-8Xg!r6*VJ4g5>wa5z*( zB{WgKWF~L$@l;~^T#9j*)J^N~ZC#6IO0LZZn({O1K@6V6X3WQEFh#$eXEB|Tj1^=h zCQ0v{zxYj*u^Aq{cwIlG|CakdPCekyoB4k@^3n2Jy?^s(BAwNuD4!Uf+y&tCi7^C- z{QD#|hyvuB+BVJMnho0PpolhY#CL3^rsc3b;`&~cInOuD%NSW4Y3c;y`dhx2S8G}b zWIE!@VGEJer%)R6ELJ;QgAw)NefrN^dCZ^2QPfj5rEV_0T$1{XycYbocf%Xke%T!R z$1Q)m<@Vb@ZJufFnvT3O)!a4t>SrteeaFA9yLNKo@WkMUy*DoX@FbF_BCiyTSN{8M zwfmfwu)Io90wn!J5Zv|EIU)DiBsVXAQB4(n zH@GF85XSEWE}6d&es`Po4y3jne)oB5XJ^cgh2JgubEDgfzG0gJ?c{Zmx04Y~QfT+7SPg5W zh(WCst3)FrKYS=RN)j@WWW^;3A$${J&_nk5VhyqrZ;hlgfC_f8DlOpk4 zLydH(kzFDphPTHiei4P^jJ&v-kH7R95`?W1r5U!N+!p+|{|Vl>_IafC`tGZ{3&xr2 z-B-KunCQN_bvn|D&|IW-4!&`wu%+i`na|D@Vic+P%^GK}wac`-ZtNbK+AT6>s8v}; z1dx4mt4u>ZCx(oI)3MA)V9Gcnqzr~F9Z6=+RJ-AWsY)5+ibsb`e>cPdc~pTn`bmy* z_lDjt*{mbY<}+nee@ccb*=V09V{cwHVmH2dF+lD6fa`7V8>v# z4C|v6!)Y7rFrS3>Mhtcbv==BKjLfeggl}X)=nPnC-xk9o5^b- zk6rM zIJIFX@)dn?2Puj>KI-I;`8tB+-v7b5w+Vn7*9c0w-1GcDbN+xd;k>{y0Bud&*P)xW zXB#?vorWl>s@Gm%8oNevBhHy6AnKDG(dkvwDxj|5b~4)=X1kD6auUJq+9D2@9YQF|O<(>T)HH=wlz!VnJa&Q1JP76_U5B zvZzyy>|%tu)v}#-5Gk7pvOA!CeVMenkhBwy2_tdQ8f8aNo-^3rATX}okE|#d(aBX) zk?5RJ`b`_r2}mG{&ALIEeUHk^*xj<2Ohy1X_gAi;e0z3ftYB=vwy&@<_EG0_WIG~r zk?nKvmESa#kO_C)*ga!AD#{!YK;lQ8G7Y7j7?qyaHVV38qqIdd+NxZHZRhLoco7Dh znY6EvMeI(=B1VavMDO?*DZdWIdvluQ?$F;NEs^v;J@J`4ZU|J8!Eg;Use0L{llZ zJSE*CK0 zOXG(bB ztz&CiqiH+Jt^3*Y(LpvmqIHz%~ zA?}XO>F10~O>y6zkASi}oIL~{Bsc@jhC6S>Y=+~0;OxlRr4>BrNwBj(d~Q#k`_?gD(37B8G<_!XLei=0nVupz{d=HUMG@qSc4q>Hs?RPjOg{! zQDp006G?G#ho9XF&5^dDFFlWe-CdA#R-q~R)##(KqX z#^{~cDPuCAq}}x{Q80RM^vLokq{L>8-pTLW%HJ9+Y(4bZiqC?dT`H_A^(Si}fMOWT zhX)Eq&tDvvGtgSk4@v_lzDFk~+FNF-p#K z>4hvLHptw}PN5)M2p6A0Aq5xjBS5@Q;NpGmAW(?AAbdWEeHa!EqTJ3|eyap0eLjwm zLC7vbK0mrB9UXcg_Mbo4JG8N2Sr^xp+l7H2%_qVC?;P7x$(jd*_VOZ`#;fh_s35tP-A~(z28q zK#oh1&bBbnA*aZvF=4SD)Mg6-CVJmCT?Z5Ps&?8U?VoaMRr@cvv8r8+OJW(6l{;k* z-zjGsRC!juYha(yFSW=0duX3XgtW+g66E-(`fQ7Wk9-P+()v_5P^hGRU`m<$h&?4S zM1`#!7Sv6?Z5^pc!9h$ygNa~ zjPgiYr(u=^lF&c4b9p3tlKKYju2?F;%MvVyCP*2vNg6^Xg|(XHA9hc+&PJLG#*SMT zxi=NrF=v#1)5eZMq*+8~mGBIemZj7Ha{eGRpQ33I0}@KYQLZE$)j#o*a&T1tG*7z0 zQ4bp}P`%iX*I=oFjkb*Lz4QcA)f0UEw3oSyEB07Nk8q_3^oPA9M@n}=NSEYp(dT5+ z{?O-%{9I({@bK1B4<)OKTi+9b&aV#y4m+-TVQ>*+6`G6Nm*-gB-7Lo_G#9s?=NR33 z7#sio;VI7L+&(s`LElP_xo60xt@|{ zaQ6iIxLebh2D= zyXX|-CP1)QA!3Ctx}r6Tqf~r(66HEsR5VwiXTk!>UPC>)_TLTrW7{I_kNj+z%#XD~ zE9FBYwNW`ZX`v_)Y{g9`F($#O@g_ zg?atX7%gNzz5K!cf^lSW3HPoZoHLGy-;8l&V!w>Z0NG1RCGHwWl$cCLLlc)}427K7 zJ-0eVxt@WJXe!osbS4y1WsA z@9#%e7L0?1md@F=J3qcKyY?W()*hUSV8fJt)5byCFch10gEPFGETjgI^}j~aSxiSU z^&BKhKKQA@AMPMfh`S(s0^<*b0{bmJi)C^BMH+glZG3x4Ut%tXoY^Sae+>1;9!O8HTXcGyRfEvDgv8n={Ie3)0HQR&AP!EUQQNL z1ITje#^Hs}h#xadXCCo>g8Q(bVRsPD+xeDw)mu5Y3jM(|%A&Vu_sbbBQ3kfG~ zFTj@QZk$KLIPP^wxD+H3 zZglDV4@JV4X(r3~Oz2C{E~4EoaoeryTN1{)ki%7J+PvH;rpc!#;i}{izL?7w>p{JF zmdx780s1!VcO~F04gsre&ZA+`H=Dq8_}h1Q@v?$(pb*_LyQ=-;J+rHDQgooO>cCV4 z2SKIZv~gep*Q+Qt>jr0dIax>zAj_qpXUvP=#dH*Ge1c=+6Z$7diK0(D4jX@aY(#|Z zZTNG|5Q++4acsmfM6qrphwIEZJxZnsSd4|3mCU@DPlHc;0_Ae^xHm2qsxHUH<*k#( zOYJ9k``0jh{`zTeDhEA<*4?v_Jq6?VPhaNVROI-aQTk0A#|seBof7H=lg1IM=P#Lh%d6Y ziSI|y+%{%i!QVcyiXoX)43^TwQ8HP~#V6iXqB3hj3jW5&a5`HIrqYamoL!v|Uq>gz zr{7D=o>yP*HG7UGj^Yo4yw-E(wPL+YI@a5B^5|>D;9&Y4{Dl@v{8bHpd##Q7+C|3+Tqzb=dVk(7qq}Pc8IcTJ10Emt4Pc^~&|{ zUj6QUqw4+R?;U^t^n0goy!4afKRW)C(?2?W>%{c(7pIMF_l?^7#?psDqro>`^F>Wy zg|DD3`Jzr+_J%%Qk9#Y=>i0K%Z^MlXGy29aefT3V5B-`i`hY?>KrG9ozZO!;rQjEJ zy8gQUMYUgV{;CFF13qY=;_Kd9H>am6VE=<5lp*ZF`4zpBPn=LdD3 zh41S@rG-_u+NSmPFMV|Xj~eI@tr!&lYAG&;Jg6>f<-QRv8ap2b{6628=%oE&2%!fQ pdPH~B^igMQ6#S}&7%PqNp!B-F!bDH#kMHY{&O7@8HA~jG{{gLh-@pI> diff --git a/submission/sample-candidate/tests/__pycache__/test_cdc.cpython-314-pytest-9.0.2.pyc b/submission/sample-candidate/tests/__pycache__/test_cdc.cpython-314-pytest-9.0.2.pyc deleted file mode 100644 index a287f0ae237daa394be983dd772fcc558b8a93f6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20869 zcmeHPTWlQHd7jywy$B$pH^k-E^fEMI6Dk=2R3a%)E0^>RdQyxd*R zEG?6Q3KmIPDsf;H2oM@55C%wVxi9sj(U&GK?d$R)(p{uL+6E~4)<`Xk`XT7|pEGl2 zW;x`NvYhxK!{NW@@}Dzj&N=hnXP#+kj!QU>eD9y~C-z8EjxzQUuoCx{keHV=sUlsL zH2GDz5@5fLWgk~HOE zNeewJ71}DzTKHk9(xOEGTeT=)n-&9X*W!R3nhLl_YXaP>H3L4RwE*tZS^*!{+5jKX z+5sQcrKp=`Up#)c^sa80 zClmRisT-BlOnD+((Nhz8L9YOnH|as`yV*j1B2g;qMz)eK6%&U3ZFGWEPu$d}XY^uD zPfV4HrAn!o&*dw#J&D{^J@-zzlrL6#-_Dx)1e%l!SyW8SlnePBE|V~`SMa%qEBPtC79}`WD(WbWW8`P~C~7gJH66pB$jP`p`lo}ICW!xghXCfKv~)Qz@?A-q zK-qrjkL0VeCXb_rA1Ruiz<*JjTG-GrA;qL@P*+L07QRw4rm_{2vIlAxQ*U5O%v43s zUBw*bE2Y#sGliKHZlh#!i1y_a#+~iWnr42oIHeaWX3Cs-dn%87sfe~%0R}U9Pd@cx_Z=N^i^8Axgi&{B zc*wK>{)3v53Jh5oWjQTh5O?M@DF?LB^wD&1WK5E#UwqoCnEYqj3FG4aFGQUdcB)&_ z+SAIQtaV&EI2ahR`qsi)WSkKN!n@F9V>`dy;to!raI&-RN-Ykio$8j-vOVwi2qzw{ zYe#cjcevYqL)Ic%)OAPx_yPMmTC5>%(8f&#d&EphQaPl>H8mYd$)3{iIBi&VxO(1e zmpQ)}b=dCN{>2m7bV@Db; zaql12@prtAk2&jSr?g;?tF#=^l<8N}3Zt?MHVvDct)pREQQ0V{>_e9-E-KsXq9Oj2 zEMn6Rt)(HZV&kR)o|%hjty)_;R!7O=yP#xoyJ!2?qGS)obNCHra`SK3m_cgPIK_P5bIYxO$60Y~274lxi_C!dm?K z@+`?unXFN3&*ZX&LI$Fjkt6Ks(oA83p=Krv zrMI(%44d{?hNMmEq1JMSO)mtrmkgt1)b?b!;Fr6cV@(m*6+?7b)y|~XNFA@$P&)MWwQY`SA0`t za)u58)5I*0xSY9`uUySoa(_n5u<*@I2?mzc0~?BUt||u>o)bBdpletdE5NXF>kjf7^(JuK3EQftvb*?H;Rn^CN_KqvPM%V(&oj}btQg{^N{cff3l+;-IzI^}OKE|Z) zxQE*ZwBXlN`i3^rkPkW1cc>xm*3vgLkOwY(M{Vg_i-KpVMh9qsK?1!&G8`jp2Jw62 zaRU1ZK(Ud|8wUWvEc&;h{kAc{L-=tGhr~y16yq@UoFs6Bz%vAT2=o%@Baj04IAV~R z$j5bzQpX7l6Br>ts+So7uxKJDh&^tC$n|qGV_!RCRZ915e?Ps}*^zIODx$dy^ZpFc5I=4$Rv>R-za|;>pg3zMvc9OJ>&Mh?U z3%(k(Z8dP|+|HH!CCIIGZl@hR1$Bpy&fN&#dFo^1jc~Vq>iuL}rac1fBv&Hao=Ac!f$y<=QFnG>#JT zAb`Kv*w3gWAFHp}XjM5<;~4rnO#S^9z@d~%=q>- z`Q%jf@N=v2=YL;fB#BsKcht_R(z66P3t(C8*-)%=Rq3g!ojiNTm0lx6f#pu1W*gG{ z+xsKUvO{nikY*R=Hl#VYrTN8q#QYy(l?h}`9)V2JLgQag(j3ylE~4m9U>wpMX^6Wu zX^v_!AAxM=1jf%C8n*>9m_vO?^R{NRovmZ@AJ)-h3<_yq={jn02-YnQqcF%cUJG-# zS|gU=kpCyq%0vFophsX_x8VwZiwT6l4#2A#H>{zTaymY27!4jFOKph$1Mr(le`)w4YIlC6B30T)(aYfa3B6$Fd$(ZZ%*@GZTV4vk=u zr*powGwBGc{R?|TOh6RaFQub){v~;Gj#>2^F`3p&R|f-H+ohM%k-?zwo|2X1>nSdf zAmiMVzqlZD4CV!X@b@9a5qq{RFL0xAiFTG3xDkHH_Eu;yE&dg(hY*!^c^{%3?gRR6 z!}SpQVJhrlEYxeQ$>|7-C)A?E;k}3EP;~(}iIs`68b-h$AEebKlsRVePc-7~(D-c!(?) z4}3X&wuLKxWA1h84z=5(C3trO@DFiBN?9F_csfir)oY^9W86E{L}3JEX1q73O`5=) z1dai?9UsS;Vb5K1g4z4H!-xA&1_nI2L4VcuDaJ(X`7q%}(qZ;9^0`W3Hj^)skHmxp z(8yv0FXP|zN;R>j4<5Vu*sa%=_AVK#u_N;@e72JkwfTcnH&0cSqd$54Lui>t7olYy z#qUSge|jB1OV^RzP@Hp3Il6Fa>AJg!rxzTY72sI8bwaO0M%iMAC!jdnm>-G!! z*{mCb;el)*SV-u^{y{%l6Vr~pS{1Q7VXnO3YlieOTsl|sXG4-rzU#=V(~fkDx&tOa z(GKL5*j*!h!B!Z%d+_(Ml>xTVxI{aP-8I4&Y$c_5*hGlfpO5BFZfAD5X2?ee-@>S> z+M2tj)@ghf(J9yT7}aC`6teTxltLC%s2fR-A0MRFxP(ICWqD&8W>6sSZbf)A9A^un z0XPvy!lL1DvAdXpN1c3y#?xBYnejYIcTChR-%O6$)RXdK+($D?S7E+!PQ6cTRz?8z}NhEU`=`QGnY=v^6;^qym|80J4-{iJ66@6 z`4{1J-B6!cR-dRUsgHVp+KZp1-eonlp;+gdlB%jth^$DE|HeXDQBrnp-47duOYn|6 z*mjO8dPRjhfvavqX)nT6bN`CSE+{P&p8A8)R)0DjbWvbZV9j@m0?|61W$5aSI!9lR{L9bD5NUT}r9*?S3Uv|s7Z(sA#; zoc!0iwcc)5?!9!*wJXC7XDDWmA0fVtR?3G$Hpa-tYLExadkK(ce| zPLJq(@r=N6CxHK%y64uJA3`hb+fcif)h;AHI`vb^FP&Oe`!-O{&Q+zas&?`89i@*n zO{!UQ74s6nbu@-%X|-H&a#Ue86xBR|n$5CJJ643GihKV6_&bqpwg`7|WEdtV`50Nt z@?~kd8=8QiZd=MSL0?%Wvm>t2%48WBQ(atRx4Px;PnLny1z+rg=R`uE-6bay?ZAn= zuk-lJCqm!c0rwUA058}y;?zRYg%P$9uEW*UhPaMA74hi15oQ*B(DD?SMP;t@mV*`t z)$2{8jDvFBmvIpm<3hZ_m?FR!CGkk3MBpueIgfP|8Y{Do4hdb@Am#@y%5_^L>KLXE zyW4_Q+MIzgx%dsnZYcyle#6Ca$v*2*l+V#XrwQzaEhb~!;>ma)V|&Es+@{SD&hM-M za?7e|<@$|3C2SVwZkaugF=v%vh@2TdKycHzjxLD{n-A@Z2i9xT zHruNz$)$FHDmY;KYU@EH-yf{D9$ZxqA$eP7U|CIWDAu_KT#6ZYN_d*so>jI&eTbY$ zpsNK_mzl10MPE9Lh(po^JEESbF(P zpEP~edgbN~n*u^RKEpVQ;{p2DQL0w$IxVXt0N)}a444EOcjt`JjZu|)WVg`ch&l1)5A?;hd+_VL0=R5;6-r^Zx z41IN$MlbP)`^)3$VsxKWq@Aim$tb^7vWIeYhv_$$DSM3o1g1MBJWWPVt87IXwsY&OKT*RIC|vxeCKa_Is&vr|n>D6*jn;pbla!kAi?L^aT5w5{8@q~FWBUqZc8K`Q+u*F~Hw`O4jnoNLn+!9gCc z8>uf{+T0$TB+5b_b~D!cS@9jKGrMApSToOT`i{|Qhp|z}gEJ@o_=1XSO$~7!J;3Yt zifFJ5rZ?*W*f;!e_3UH&gcRauS8Wxtf9pZ)*)IH-19|%L$S8WHfU#_u0^!Ysj&Eno z5-e8Or$`VRZ+-uRJ|l9Kp_z$^(zPPIVtRI}&t-{w5q%ji5g`A3Eu1Tq@F9^p&7u}B zzhfJGxPpNVCm9n2E&*_MVw92e7`6&a9^VA8G^nI*cesX&+ zr|EblID;F=Ws4b_i>#5En!!7*<$|7}@1U57RLB%clg4+@KQRw;k4#kl*VfCLF3H>p zyZ02nRXuQGO*!fIhzZPJwX|=vbS<}ZEy=4b2j*Y-m(D}0?azLAX;J?B=8u~{-Mf5v z^vB(|x<9-$f3Dj8>@Q*6n)?A>jqF;?G5}N3hGLy-N*BEv=@jrZiSkz2iqhrEiDLA$ zFeh?AvUBTBkLY|cD{$Ni)coJwpw#i4Ne|O6x;kDysr=%27%+#tILMF)x+&c|4(reT zTbWp-y&`%Q3D90a;nM*}h8nVBH>U?}N&%Vub)prV^Egz?KhD61jyABYIIkLRK}VhT zzLa}&d(bLw(-FGG4#vvqsB z=3GeQ_c`aA+T*?`Sg+AM2%AzW z@1`T?WM`*aJ~HR}zT5>8MvzGulP=vf2n*uZb+vX!2nqwSF1y7*tXK;5}R{^q}Lq&Dd z_3tSC;*$-%8*2Bm+Fez8Z|~(mMU;(ot}4A%wOeFGg49fm=vS@-$7ZHe-jAF%I7WkOx<^-4e9CskedHf z>igd?zFK#;B`iOEw<9Qb-t7p}qJ-L{ZCNTC1jzPOya`&dXma>SFnbd`(VQJYWjL_9CByC0RE)TGx z$w^z<@z_$*j`)xVrR_{TeTv_b*F0s~PDgJgwPcoqNvED(Ce8Fkmh5`!r~dyrzyWrN zm8i&$of(1Le-D6j4h|0XeE{OdUcBw5Ox2mlmx2bI)U-({Z;+S(g?)SuNZrVwy?R|pUIW7(c*b#LLw66o( zH)v-u6;<3R^mG+t&>iuMx?vj1zc2n+$)wDbGFwbdXU%yfmCGn|DWYG`7Y$uW=Lyl% zW==PZzIgm}{-#chBT6=B=)_bCBwt8fOPSexu2(4*pqh#Ty^4{)YVOPE)4HkmDnu_# zr{)zSbyYX#d*c)5lwvMBThy`cUL}_|_vMPy(@Offp1uJGt`vEA0T31pAUJ)(? zXQop(@H#UEte2LKW25Qe?95EjOkK&+TiKh-E9~lqk|%Zx4aLl}yd*zo^z_BYdIDAy z+iqrObSsK-A)nKGM5{hkDCoJ2X67}l9F8=>k?DCYok?rZEqRhL($JLxesIH{55nis zE)a`CQn(yA{GK3W;Mg|dP4T*@ic@fgZT8yFR+#8!k>q+rf>ptV!dLTTCS@A9 zyTkfk|0TG6quVG%b~b2LOGbX#|H;6MT#o5R1a+f++6!1jS$e5&e1uhbxB} z-ZH)Iv$TYJixc)w5lcARc410#(i34P<$e#^oe7;Ij`c{iL#$vfKSyX!1&e0-BZ0_J zfW$ypLCrKsBcy1X+aR$h{910gbL@x5{xSgMmT2f7E~)oQgV^nxHV}?6L3df^+3p;lQ;Qs;-o%%#C~7g=EmV6Q_llEmh^} zA7Ac*s0LNZb4UL9CFeT*fnLvZE`-&P8cv4$#fqbmDSUo(^VHnz9CN?n4kAv?&NmLM zqawMO3|ldfMiH z*HNcUYV3)5V@}?x9*V1RRZhmMdPshX9+I7!op07dg(&pWC{+TD3y0nsK0kc=qLNF^ z=*k&&{DLxfT}u}YGd}}ZqfCwrtHX-3uVpjJaiu$bpnJ>;7UuU`K|oJT9sHu6VY9M( zZ(;tG9s>`)VuemkXVW^S51<&V$aRS2I`onFML269^nWBSv}5`GJ%+-L(CQBM7?3A$ z{5YI%#`6c@{PE-T+#sBbTM_+jJzX>bw?V6-&@dr+ zVzp=tr8WI+P;EJ?2zwG%*wFQip0T8KK9{pZjUbR(GG3Q;)qr>B0NY_T4SIJDd9&^J z?i}=Hd&Rr+Z374W9>v`lN-DHRnBLYwu3@0|Y&PmbNWTQn4a$D@*o&N!oVn5ZhaGldG7Q=kX@s%wrUQ z1%UtMD1a*X30@AMI-v0Lv$dcRoQAw5GMSgv9>~x)XQ0Ye8oRjnr5*-%Xwy+sPQQu9l8ty^Jx=R%=u^OR@UUAsH8!N=3p%00 z9M7@Q)J;%HS)p{km@`Q$qzM`!q#eW<0gps*zzf061}^|MvI9=_M9DtP+=rqc#Ss+y zQ5--qfZ`ws0}TOZT(dE~wZ@pn`y%KxY{zmx0^ttlMB|9a_%u61$vK~!jKUzXaIPTf9ImfF~VNou?G3g6=i&$0V%j$dJ6p2wSjOph6c zeeeVNJr2XoSpiWx3}cnU(DPI0_8D}uy?Jdc7_jcJ@XuR&Sb&w~Gc3^brJJW_e@oBL zP6{0r3qfit`P;c+$6|P#ox3TRd)~i(eguCe&-?e;e!pr6pgsP@c3)`sTCxPK8q$dZ zT?4We#Ml=v)+#p#2BU5o3%hD%Fi9$Bq|%g(p!iM@XL}-KC)U0T#cmXPP`rfVWfZ+2 zHo`snZ{!se6!Q;Z>f0cE;C>e~YQdbPD$39b`9g6TeD;i%x@zh~L$i~dgu-}H;{-Ko z95YUCxO43GvE`1VCHYuc9$t}$OY$4{#5MWsvNV1#Ma7CdUY6|tk~F?7pJlsip7bi_ z=DECy`_lNjbCp+1@{zK9dPP26l81kpT9ZeXr3?2?QL!RlC`J| zHvy+Msw$)zC}l7N(p6Pce4sn-cp;b!_ziFs*hg^dsRNB}uo_XLYAhL4}f!^Q_<3yc%)JgHX z3ME5T&>vQ%*_+7-wQz4$IoR9^oWOpo+Q7kXN=CuKZl1j6aj+9pZYh7~pDQS}?umIh z*j72%F|}T8NXDuh?D!NKZglh1jCNer1`0mz^t1B~4t67`RHecnsh ziQZmP8v(#ute8VF7*-$+&akr!jDsbi2fiRvEct4}}8MOMm^{a=z4tPl6r+~g`| z<9WOZII~fMAf{0b@_q&B{Hz*;5{5w_k%RpaSBp$>9WvQG5rHdLZJd}7n32XJ+5-?!3c9#5BO`3)?^s8N89AEM|%LOPjRg=(OPx8uumBWG{*y6jhda%!OAnj^`&()MR7O zw&F<2aP%O>bsI-Bc|cGgwl$`Mv`n!83JWLLIO9rT^}A3HHrx1b5J1!T!)gYeN*FH4zGb+9ibAfm_{nU}f8ZpLLf9Mpp(#@10s3xKP?Q z1~&Z1j0a-0ZkYb;; zaK|hj@-vHnFY|JpOBa_V5Vua0rB3!=k~+~u#=SK+xr(`X9&ZB9Y&4N0&_9UgkG@l3 zBL5-lf-18&2xf1sX~8F9QJm@qxk5L<^w~F5H=x#X@5mn~YwHFe$G&vk5JTOd#sK#c z=jryfI;+*<*TDVC^oRqmqU`E$~um!jYyJLIsUPyaQwMl z>HHqR@oz3f)MmiHwi;7hRbdg-I;123qXUNFm$6XCDA`Li<~Kvmz%lX$iV+mwLBTly zJ3fNWQDH3DfEB)?L%5S(5fcW69-X5b15;2u0hVBs6AyNR8iE5o1~~jXvF2SU5C@p+ z?|S`+3!8TMT3MoWovKo%s%7Khj6#@m(l9fX0{)*eq_ z7>IrBKA3&(9zTqRqHP$$G05cEn;yWg^BAD5PTSmXucJ-@n45wVzFeqt zSJ*x~tl4dqKM}w|z6yZda{le&<$M4*Mge%^JRt;Hv(0nY=*CuUaMynGgw0@rzOA-A z?|&p$l3!Qv6-^vpFZUy{FM*#b9Pa4H(y+L2JhjY=@vEk7YyWE)eHD^BFJWZZC))&rGp9m0^Wh zl&dA`B#&7UVj1%_fGHYd>N`K&>oII}GX?jqVN5;mYXhU;AxM5k*9td?36=dde2itU ziSyWb;RAP|VX z@F*zNhd+>?Z)lOnD)ag zI(-d-*X<+^SH~!JgYQ9jyJwFdh6J4X1Fd{j>iWNj5kcN=;2g?7G&P;uBl$0-TB@Y% zzC@?I#WS)nRNH;m+o57pP?)Zhsf<+Orxj<(8g8Yl`@ zZWJ6{nVf*U3xgE_okOEQk^0+B@zQy)3+_}qeSQ;HZ``CS0jN|U{!?=D2!aUtfo;`jT60odiqx%KjCu}U zNo9N&=Fu>+8VdT)VXXKC2KWp@rl377(O46}Pa@zG<&7*2id%#l{{=qA3*Dy*_S=U)7{|XKjNdxQQcOB~@7l-ThrHhNAB?{{{urJ5eB{8x zfbAJ{;WvBf>wyZz8AGr-Lt%0jq-9bbC0(NVT2V3>o6|=FB z%jdM5j&F>CAW@tMDPs*e5Y^*hdCS>S{9HMHaV37S6i=>4W1r#r^XCJCzZ~TGTJX(# zf?icJUTZ*j0Nhy%p2_0NkKpY&%GmftWpqs0+YM8BnXK8}t8{1dE0E}6hTJm`7thr- zepxnZZo*oZ40#Vm$-m)nne`bixS;zGEa;Z27Iae;68&mbNOL9U;~TPjnFnGNf>Y?s z(QUG&V(!Z)Y*%&p9drY$Hl#3dtDNU5NbuH$TB{14z=vSK5d<^DDpnOBz;iCdKV;Ww ziD$mby%L{+Vy#69^B%Q$6t|rJ@0*unZkUJR@y}m-RRM&o_^c{$*TA_g;j?uWcfbQ& z9S#nH3nFw?K?GQrIFM?fEs%O)L*xjiYHE~^5`+@pXjOZr`FG>)9u%mdJ#;svz+KL} zI(*-f*?V8;JG!1KjQK zIEkg^mpsdw9?Q$LwtdAfV6ejZt5*$pV-5xnIRQj4jv3Lii{8Oz-d;w~^&vk50TUg; zY|ex?nBmWcy|;{puF?02QSrZc0t0UGrD9>R{Sy@bl%`+IM8y_38;*BM`&8h|&ScPU zNkEe01gv{?@nx3>Uwl{8dS2_bd3*^0zUav?<&fQAsh`teDR<2H@~z+$Vz5+nGU=H+ zM@|0a;|m}6IB;;YgQeUSJ^?QPUn<#c|NMOZwu;k@B1PmAytH|I@e5u;O@5QT)n(Jr zLOy`I@3G^3@VCldw}K&m39_+)SF%M9Tu#4g= zqu|AcECQ3hy|9pnk$}`}HlxoJ^6-*EliY!7K7)@94u<@%i~^*cX2Q>uS? z@eO;`<1QYP<4fCqa#t&H$`xu+Vr1j!px5BKjRaeLs^l5nEV;_Bc4Gv+enIGfiH(LRK3YEUE zioWHBoWxxf1+J!|G4?e6m4;i<2}w$h6Xe0jB8qoWyoUl&+mbM9-;&{Z!Xzf}`U^UJ zT%-YW;G8IX>d|8}`3$^_@&rMM$9ph-4B|mh6va;hVNv?LQ4q!Vz^Uo}525bAguVYN z?EfqbFHU?|9~NJD*bHd*uvrq{5+AmaAcHHgw`&UcizE9!$?T>;&pzk53XsiDp DqWr^) diff --git a/submission/sample-candidate/tests/__pycache__/test_schema_contracts.cpython-314-pytest-9.0.2.pyc b/submission/sample-candidate/tests/__pycache__/test_schema_contracts.cpython-314-pytest-9.0.2.pyc deleted file mode 100644 index 76d6304a74b3c8e334e3712babf2f1e92ce3a6e0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16244 zcmeHOeQX;?cAq7e-_&PhTef9sCEKx?$ojHu%W*6pEz4&~wyfJIG!p1S(BxX;g(B%) zGPZQdo$doTr3o%>dq7>lZ5132RnUL(M^W6NY0&`JqJP{UlqpNPt&66`0l@)9p;%7S zq$tq$W_D+n6lK|MfoqEl$l3XxH?uRp_vX#pXRE9H1l^l|lWb`uBu*oS=Q5rfIt1p3 zOmgHEA`3%8&Ox8f3u4~ULN2&+?iQBg$$4mwH|K??W5{>GpYvY`*N6V^>PsWr{pT|8{}&6cYNJD)@~<~g~vKK zr^z)PM6SOq^g39cTnoL{VXwVL;asCE>?fh8@?L=u|G1*%H0ht_ek5t}DP=k)#WU%g z8jI(ogpyO@xnw3S#nK5$i(ONWXt_*Qx~fdYUQ1@w4!?gO^O~Y+&q>*srX|ypysp+R zO()HyRDL>bHP$XA)A7u7HkM0XO)0$7saSea@mpOUPN`MC zFDk9OC{emwRdRVX9TIhSA|HP>aaFHB2hVe7$7Tkg7<-SpQ_HE~DY=lKyRT)`=~zy~ zk+kWt&KEU!lb!I!r{I+)bD7Rp^QnBNs$?@7^Pz)mXCju19l@79nVweCIjvL6U!`w4 zlZLvmlBgeLODSqDC9NYnqX(nWSUR1_;eF{khpy|ansWG@H%ZSLdMKDghdS}lZX%`J&s-@Zwzac zIdhZLk{2C$M;FOE&Ibs&-u4QyzlT>HBUOa>NHgST9s9v=p`j!R3A4U)gD(uoLuUsi zSW_P5rV`KR6y1AG$;GE)sZ_|RBDE^MYTc=3ZfHox(@#q(G)eTDDD1s_ECrK?3tv+k zAeGM4NeIl5wdS@zO8tIn@um0k?@tzrQH(VIX{0M zf=hvsf;jT|-N1;3tL!IrC&iNG%ct#S+V+2M`C5zDK2@DCJuqbKo{bx~Oh{C{nBW6X z7gI@Cmk7uqK@Gx#tw}l^be&Ld2WA`q4;M>N*!c;1caE>_?Ood2yN;0$8a{~s#crCe z)<8izQg~8_-N8JI8$o#abEJc0x7nDj#99jHVd`(da#Q#3B@-rpAXgkF#%9HFXDvD3 zMo8BEQsN}^FbRD%kr7xrPC56Hq!4oD&jE2m=~ObUSQrVs zApR<43)4we&8Qk6wVKSr2B$fdc6i6E<2h+m7dmEZpEF?FnSgN5#Jq3!HS?ZAc$3Vu>WTKm@PNpNRD z-0^P@t4Qtchs5FAu@>C98r-)O+_$h}!Ly()?qA$dXzpANcKv;@;osL?P~r=17ZCM_ zf$%QJhr0r2ovshtc82?1ANIQ-9EU9KU_mjQ`v8{X9Cc*H5Wk%@}`t(PfM+G@>rT(a+S7e#8 z0DKO&Hg#hN{$v>) zkydMVIgRknOU77{MynpsCdw|`>{l)?m~$b!w=K&#VW-&6B@X~GQTE8*h}bEVq`D?> z|4?UZd8e6XJzGh{Wl~{2K@#$v60*W6LG6)!z!HLAc%PeArZeiybGkD-(?vHZhPoLD zKEPk)OkYUD#JT5`hTnc_A|x6!MSty#m3tW%VP8Wz~0;Eih%S6>@zbBH}^e`imwU0K5Xv z2SC^cCgniH1@IcY(pv_v9CpSk0)r{d*x=Rb31!SO2k^?_-Q2_`tp>bW^(%tF=K367 zpM+KhuU20bX|-mT)5ze}s#lRltDXU`+m+{16~OCuWw~5e2Cw#-pnWKJ0H1J7kv-Ny zMfSGa3dz2Ry9}B=6TtA8vr*M#rddyD+he{>^9j)G2WWNzG>;DoBi~?>r|buiu2T0y zUG)HXx-+Gu)q@yDyB*4j3M-Cv$g*IQp{ElL%C5nbqf6e*`1FE z+5Hs95wUC7Y2jJyk)kyUlk{Mu!k!^2&W?Hny$w{AvV z#6ya@U%=aXGSr3RJ3nPXD~6M~XtR~rDm!+;-l^p$H^5(x%GOS|MB2$eLBvIY4+JZY zc4K^Bn?0kjSph$}7(2%C%wEG1lFXgPv5ASNR{e^Iy}3SMlnZcYn|jtrWzV){Ii7Vo zON2eqChS@LRmM<`bLKb!QM`aA`(-a+h>v2(KT>6*xqk-a!SMK?G#);CaZrlqK~Xmi z+9&C--z6uVGWb$^NnsSY%*tlr(aYn~=!+LGmX_2& z1C`fU*+-XT+P_2;bc@pX(51l_#>1D!q{^-4v!EJNwkpd>4bf1EW&-yJLeU1ISQ}}9 zy)sVD!)nY5sIWqQeUr?&KsNyE5_JQpy*=iHlez;AGX72XEj!G^-)&{WZ{6sP20A z#_iT0zW(EvKi?1`RXR%}J=f@&;h8N!=tLe?W-x#@{D2;0N*7A3goyCrbLHOYSg zxB<2S4*zcexBD@`n9q~!{fz-PoYDcf@r3TDz&*Rp!D}njesZARgu4<2G7j^VG1frt z&6RCH{RB9@xhO;DSQiK^l?QzT&dLg0mz6RqmWpgj;v?9SQB|6(%EOK4RaHsHXAqk< zjhWMjx&NP7ng`S;v2LayO*`PPHP<*R zOZ@?}5la@{Lh1*Zin+&DN^@DivYy3aN2WAfCiPqOaS4{WN?I2~>fffGHBw+S+m&^i zv~805@gjmcTN{;i6pthI&psQzI6f#Ff5a?x48^L5`st|c%LAfG?9eJX`s!^aJhQs?2 zcw1>U3>mmk*S{>D`BhyjmwsU;iq%5}aR@SJEcq2NpQqimVk_bhRAV{I z5Z+W1vNnSjUa`Z0OWgP~@aLHM4!Iy?VewcN7h|XmDsynJ#ig?>EVD|Nx-#EZ(3%B7 zTy2qcM2MwG$T}kqzJTE+HjzuaBF-_Su*{))LUzL+Ze=D@5<1*J1^HBdTp1;kiIP?8%W9#A^(xDv4`oqe)@f-3~ZDuP6> zK?fdUjXOQ7C!1Z1Q4D6sf*lj0Oj-@4vsJ&MdUJDqfKR_1kb_S|D^ogKeO072!rwQi zktv<6dKGE3>H%#*xym;CmCN(DC|7S=mfyU|HQSbTn{(XhDpCG;Z8e+CcVj(aFVvbf zd*VbyAnPxpfHSIH7HJ_kJdJ0t2BK%!&E_qFU3eF`vXW#N) z+18}19Tn08=LRHIiD%RV*e}pqIC;76T=RVF1q^w|-EcFi zi*Pa8HL8Z7Lr};Mp+f$csjh)QnYyOJ^(tJecBC?s{x@uwt8h6Rs@H?rWLClX>xd`f zD)I*#v*LV8cf%bj-0H%$LT2(YwD@X2Hq?V2)!X}7%c#3ianq=i=po)vy}e!CW8981 z&D3$K%x71*>{_6^TSE;39HR`!)Idy}H^&mim7zt^l91n(8u~INp=P1}H2k$+2*_1F z^%K|Ohi($Aw_TlXqYG+$D>e7kk3DrCd4nH$Yu>4Tv-;LI7tSr7{^^0ga{sNT@XXk< z_iO8JQq}Oz$eSbUPKWEPaNir4A1QhpS!}IopAqmjE_wHTMvHt(u;9mC;iazdLl=hE zyp1etgJqox|Lpi@H0NJgLMXfx3WHS&>wuO0jAiwOf95O80RtE8;8Crt1)tiY6`o_z zVzs{(+{NmAQrQQIg9@+AKhL;58!y z-uPz`U$soG+l+9K&0ViEnZe{=Qr3#{8Fgb&quc?M`XU0bR)56Sj*ZC*6IT6-Vu^Kv zLemxN&Y-e!P6dW+wa?E&7L~w|dBg-|b9@Fvw(3`;Z*zTMdd^Nq3Bz%qh!b&?u@2`1 zn(Di2EQAl z)oD!?3;~D}kGIcvgFEdRCZjPTYP$Q^Ul^Np#U%d%6e;fW< z8|CP00kG)|-RWEkgjPl4R}@3@r|xvJ_&tp9EEZV78a(Z8NsK2$0CLs>4Pf|mr^aqY zRTK}+pS)AUtefs(#BS&WVX%lYvmpkiQnr}ADCDdKb{E9;`>JUCiemeG|D7m{ z-@^#cVu2N`!PD-R#CS3Umc{l*@X1^Bpe++HK!y2aP8RGI;@4}L*5-Pf$yp|69*c7Z zA`WT^CD{0-!^SBp@*^9+bZ%RgAKlCN6}1?MILr8zXjvQBTISbN=IANE;zzntew8J_ zvsz_O$!vMQoGo*f%qQdS#%%dYX3PKOY?-rUK1*f`vAjx6z-qJsI^J!RU^$pwQRA*r z-N&!DR253;)ixa}oMkD*s~Y4~KzO+M2dmaf2ya2UOU)xKGJ5p<_5y~`?yCHTrJJfJ z%K_X_$ZBA4m7I>LGf{+b^W&M0b%}Ivc+tq@X1&M(^3e%d{jfW)DCXh3Huy z{amaariW_V@y(z^MoCXK$Zog$j15fru?}_iuX_+~RS7)!j0ZvQQD5`?G30BgTEee+ zoWclv`i1@n``797=L0Ox_zFK^KzR@RDWxA##%xg%DfvJv>$p-+PhO zb=iO*3MW%BLck8yMXgcYJ(gn8Q=;;#^q~M(K->DLuhhp@9}1Ws38*=!#WZ9RhMop* zcK6vld~K>|+7b99DVGE*eDiZ76}8Z_d+@tX_|g#N%(729%S=-N}6zbPmCWdui0yMhxE;6)R_e7(KOgtLZz2@f~^a!Hwj#tsNRph5b^ F{u2THA94Ty diff --git a/submission/sample-candidate/tests/conftest.py b/submission/sample-candidate/tests/conftest.py deleted file mode 100644 index 7a883e5..0000000 --- a/submission/sample-candidate/tests/conftest.py +++ /dev/null @@ -1,125 +0,0 @@ -"""Shared pytest fixtures for the payments CDC pipeline tests.""" - -from datetime import datetime, timezone - -import duckdb -import pytest - -from pipeline.cdc import CDCCapture -from pipeline.lake import append_to_lake, create_lake_table -from pipeline.warehouse import apply_cdc_records, create_warehouse_tables -from source.models import create_source_tables - - -def _ts() -> datetime: - return datetime.now(timezone.utc) - - -@pytest.fixture() -def conn() -> duckdb.DuckDBPyConnection: - """Fresh in-memory DuckDB with source + lake + warehouse tables.""" - c = duckdb.connect(":memory:") - create_source_tables(c) - create_lake_table(c) - create_warehouse_tables(c) - return c - - -@pytest.fixture() -def capture() -> CDCCapture: - """Empty CDC capture log.""" - return CDCCapture() - - -@pytest.fixture() -def seeded(conn: duckdb.DuckDBPyConnection, capture: CDCCapture): - """ - DuckDB connection pre-loaded with two customers, two wallets, - and two transactions — all flushed through lake and warehouse. - Returns (conn, capture). - """ - ts = _ts() - - capture.insert( - "customers", - "c1", - { - "customer_id": "c1", - "name": "Alice", - "email": "alice-test-user", - "status": "active", - "created_at": ts, - "updated_at": ts, - }, - ) - capture.insert( - "customers", - "c2", - { - "customer_id": "c2", - "name": "Bob", - "email": "bob-test-user", - "status": "active", - "created_at": ts, - "updated_at": ts, - }, - ) - capture.insert( - "wallets", - "w1", - { - "wallet_id": "w1", - "customer_id": "c1", - "balance": 100.00, - "currency": "USD", - "status": "active", - "created_at": ts, - "updated_at": ts, - }, - ) - capture.insert( - "wallets", - "w2", - { - "wallet_id": "w2", - "customer_id": "c2", - "balance": 50.00, - "currency": "USD", - "status": "active", - "created_at": ts, - "updated_at": ts, - }, - ) - capture.insert( - "transactions", - "t1", - { - "transaction_id": "t1", - "wallet_id": "w1", - "amount": 25.00, - "direction": "credit", - "status": "settled", - "reference": "ref-001", - "created_at": ts, - "settled_at": ts, - }, - ) - capture.insert( - "transactions", - "t2", - { - "transaction_id": "t2", - "wallet_id": "w2", - "amount": 10.00, - "direction": "debit", - "status": "settled", - "reference": "ref-002", - "created_at": ts, - "settled_at": ts, - }, - ) - - records = capture.records_since(0) - append_to_lake(conn, records) - apply_cdc_records(conn, records) - return conn, capture diff --git a/submission/sample-candidate/tests/test_catalog.py b/submission/sample-candidate/tests/test_catalog.py deleted file mode 100644 index 9f7d056..0000000 --- a/submission/sample-candidate/tests/test_catalog.py +++ /dev/null @@ -1,134 +0,0 @@ -""" -Tests — catalog metadata completeness and correctness. - -Covers: file presence, all required datasets, required fields, layer labels, -schema presence, and that lake/warehouse datasets are correctly distinguished. -""" - -import json -import os - -import pytest - -CATALOG_PATH = os.path.join( - os.path.dirname(os.path.dirname(__file__)), - "catalog", - "catalog.json", -) - -REQUIRED_DATASETS = [ - "lake_cdc_events", - "wh_customers", - "wh_wallets", - "wh_transactions", -] - -REQUIRED_FIELDS = ["name", "layer", "description", "owner", "schema", "update_cadence"] - - -@pytest.fixture(scope="module") -def catalog() -> dict: - with open(CATALOG_PATH) as f: - return json.load(f) - - -@pytest.fixture(scope="module") -def datasets(catalog: dict) -> dict[str, dict]: - return {d["name"]: d for d in catalog.get("datasets", []) if "name" in d} - - -# ── file presence ───────────────────────────────────────────────────────────── - - -def test_catalog_file_exists(): - assert os.path.exists(CATALOG_PATH), f"catalog.json not found at {CATALOG_PATH}" - - -def test_catalog_file_is_valid_json(): - with open(CATALOG_PATH) as f: - data = json.load(f) - assert isinstance(data, dict) - - -def test_catalog_has_datasets_key(catalog: dict): - assert "datasets" in catalog - assert isinstance(catalog["datasets"], list) - - -# ── required datasets ───────────────────────────────────────────────────────── - - -@pytest.mark.parametrize("name", REQUIRED_DATASETS) -def test_required_dataset_is_present(name: str, datasets: dict): - assert name in datasets, f"Dataset '{name}' is missing from catalog" - - -def test_catalog_has_at_least_four_datasets(datasets: dict): - assert len(datasets) >= 4 - - -# ── required fields ─────────────────────────────────────────────────────────── - - -@pytest.mark.parametrize("name", REQUIRED_DATASETS) -@pytest.mark.parametrize("field", REQUIRED_FIELDS) -def test_required_field_is_present_and_non_empty(name: str, field: str, datasets: dict): - entry = datasets.get(name, {}) - assert field in entry, f"Dataset '{name}' is missing field '{field}'" - assert entry[field], f"Dataset '{name}' field '{field}' is empty" - - -# ── layer labels ────────────────────────────────────────────────────────────── - - -def test_lake_cdc_events_is_labeled_as_lake(datasets: dict): - assert datasets["lake_cdc_events"]["layer"] == "lake" - - -@pytest.mark.parametrize( - "name", - ["wh_customers", "wh_wallets", "wh_transactions"], -) -def test_warehouse_datasets_are_labeled_as_warehouse(name: str, datasets: dict): - assert datasets[name]["layer"] == "warehouse" - - -# ── schema presence ─────────────────────────────────────────────────────────── - - -@pytest.mark.parametrize("name", REQUIRED_DATASETS) -def test_schema_field_is_a_non_empty_dict(name: str, datasets: dict): - schema = datasets[name].get("schema", {}) - assert isinstance(schema, dict) - assert len(schema) > 0, f"Dataset '{name}' has an empty schema" - - -def test_lake_schema_includes_operation_column(datasets: dict): - assert "operation" in datasets["lake_cdc_events"]["schema"] - - -def test_lake_schema_includes_sequence_column(datasets: dict): - assert "sequence" in datasets["lake_cdc_events"]["schema"] - - -def test_wh_customers_schema_includes_pk(datasets: dict): - assert "customer_id" in datasets["wh_customers"]["schema"] - - -def test_wh_transactions_schema_includes_amount(datasets: dict): - assert "amount" in datasets["wh_transactions"]["schema"] - - -# ── update cadence ──────────────────────────────────────────────────────────── - - -def test_lake_update_cadence_is_real_time(datasets: dict): - assert datasets["lake_cdc_events"]["update_cadence"] == "real-time" - - -@pytest.mark.parametrize( - "name", - ["wh_customers", "wh_wallets", "wh_transactions"], -) -def test_warehouse_update_cadence_is_near_real_time(name: str, datasets: dict): - assert datasets[name]["update_cadence"] == "near-real-time" diff --git a/submission/sample-candidate/tests/test_cdc.py b/submission/sample-candidate/tests/test_cdc.py deleted file mode 100644 index 704902e..0000000 --- a/submission/sample-candidate/tests/test_cdc.py +++ /dev/null @@ -1,135 +0,0 @@ -""" -Tests — CDC capture correctness. - -Covers: insert/update/delete capture, invalid operation rejection, -sequence monotonicity, checkpoint-based replay, duplicate safety. -""" - -from datetime import datetime, timezone - -import pytest - -from pipeline.cdc import CDCCapture, CDCRecord - - -def _ts() -> datetime: - return datetime.now(timezone.utc) - - -# ── insert ──────────────────────────────────────────────────────────────────── - - -def test_insert_creates_record_with_correct_operation(): - cap = CDCCapture() - rec = cap.insert("customers", "c1", {"customer_id": "c1", "name": "Alice"}) - assert rec.operation == "insert" - assert rec.table == "customers" - assert rec.primary_key == "c1" - assert rec.data["name"] == "Alice" - - -def test_insert_assigns_sequence_starting_at_one(): - cap = CDCCapture() - rec = cap.insert("customers", "c1", {}) - assert rec.sequence == 1 - - -# ── update ──────────────────────────────────────────────────────────────────── - - -def test_update_creates_record_with_update_operation(): - cap = CDCCapture() - cap.insert("customers", "c1", {"status": "active"}) - rec = cap.update("customers", "c1", {"status": "suspended"}) - assert rec.operation == "update" - assert rec.data["status"] == "suspended" - - -# ── delete ──────────────────────────────────────────────────────────────────── - - -def test_delete_creates_record_with_delete_operation(): - cap = CDCCapture() - cap.insert("customers", "c1", {"customer_id": "c1"}) - rec = cap.delete("customers", "c1", {"customer_id": "c1"}) - assert rec.operation == "delete" - assert rec.primary_key == "c1" - - -# ── invalid operation ───────────────────────────────────────────────────────── - - -def test_invalid_operation_raises_value_error(): - with pytest.raises(ValueError, match="Invalid CDC operation"): - CDCRecord(operation="upsert", table="customers", primary_key="c1", data={}) - - -# ── sequence monotonicity ───────────────────────────────────────────────────── - - -def test_sequences_are_strictly_increasing(): - cap = CDCCapture() - r1 = cap.insert("customers", "c1", {}) - r2 = cap.insert("customers", "c2", {}) - r3 = cap.update("customers", "c1", {}) - assert r1.sequence < r2.sequence < r3.sequence - - -def test_latest_sequence_reflects_last_record(): - cap = CDCCapture() - cap.insert("customers", "c1", {}) - cap.insert("customers", "c2", {}) - last = cap.update("customers", "c1", {}) - assert cap.latest_sequence == last.sequence - - -# ── checkpoint-based replay ─────────────────────────────────────────────────── - - -def test_records_since_returns_only_records_after_offset(): - cap = CDCCapture() - cap.insert("customers", "c1", {}) - cap.insert("customers", "c2", {}) - checkpoint = cap.latest_sequence - r3 = cap.insert("customers", "c3", {}) - - replayed = cap.records_since(checkpoint) - - assert len(replayed) == 1 - assert replayed[0].sequence == r3.sequence - - -def test_records_since_zero_returns_all_records(): - cap = CDCCapture() - cap.insert("customers", "c1", {}) - cap.insert("customers", "c2", {}) - cap.delete("customers", "c1", {}) - assert len(cap.records_since(0)) == 3 - - -def test_records_since_latest_sequence_returns_empty(): - cap = CDCCapture() - cap.insert("customers", "c1", {}) - assert cap.records_since(cap.latest_sequence) == [] - - -# ── duplicate / replay safety ───────────────────────────────────────────────── - - -def test_same_pk_can_appear_multiple_times_in_log(): - """CDC appends every event — deduplication happens downstream.""" - cap = CDCCapture() - cap.insert("customers", "c1", {"status": "active"}) - cap.update("customers", "c1", {"status": "suspended"}) - cap.update("customers", "c1", {"status": "closed"}) - - records = cap.records_since(0) - pk_records = [r for r in records if r.primary_key == "c1"] - assert len(pk_records) == 3 - - -def test_captured_at_is_utc_datetime(): - cap = CDCCapture() - rec = cap.insert("customers", "c1", {}) - assert isinstance(rec.captured_at, datetime) - assert rec.captured_at.tzinfo is not None diff --git a/submission/sample-candidate/tests/test_data_quality.py b/submission/sample-candidate/tests/test_data_quality.py deleted file mode 100644 index 4dbf7f9..0000000 --- a/submission/sample-candidate/tests/test_data_quality.py +++ /dev/null @@ -1,252 +0,0 @@ -""" -Tests — data quality and warehouse correctness. - -Covers: insert propagation, update correctness, soft-delete, replay safety, -PK uniqueness, not-null checks, business rule assertions, lake completeness, -and lake immutability (no deletes or updates to lake rows). -""" - -from datetime import datetime, timezone - -import pytest - -from pipeline.lake import append_to_lake -from pipeline.warehouse import apply_cdc_records - - -def _ts() -> datetime: - return datetime.now(timezone.utc) - - -# ── insert propagation ──────────────────────────────────────────────────────── - - -def test_insert_appears_in_warehouse(seeded): - conn, _ = seeded - row = conn.execute( - "SELECT name FROM wh_customers WHERE customer_id = 'c1'" - ).fetchone() - assert row is not None - assert row[0] == "Alice" - - -def test_insert_appears_in_lake(seeded): - conn, _ = seeded - count = conn.execute( - "SELECT COUNT(*) FROM lake_cdc_events WHERE table_name = 'customers'" - " AND operation = 'insert'" - ).fetchone()[0] - assert count == 2 - - -def test_all_tables_populated_after_seed(seeded): - conn, _ = seeded - assert conn.execute("SELECT COUNT(*) FROM wh_customers").fetchone()[0] == 2 - assert conn.execute("SELECT COUNT(*) FROM wh_wallets").fetchone()[0] == 2 - assert conn.execute("SELECT COUNT(*) FROM wh_transactions").fetchone()[0] == 2 - - -# ── update correctness ──────────────────────────────────────────────────────── - - -def test_update_overwrites_warehouse_row(seeded): - conn, capture = seeded - ts = _ts() - capture.update( - "customers", - "c1", - { - "customer_id": "c1", - "name": "Alice Smith", - "email": "alice-test-user", - "status": "suspended", - "created_at": ts, - "updated_at": ts, - }, - ) - new_records = capture.records_since(capture.latest_sequence - 1) - apply_cdc_records(conn, new_records) - - row = conn.execute( - "SELECT name, status FROM wh_customers WHERE customer_id = 'c1'" - ).fetchone() - assert row[0] == "Alice Smith" - assert row[1] == "suspended" - - -def test_update_does_not_create_duplicate_warehouse_row(seeded): - conn, capture = seeded - ts = _ts() - capture.update( - "wallets", - "w1", - { - "wallet_id": "w1", - "customer_id": "c1", - "balance": 200.00, - "currency": "USD", - "status": "active", - "created_at": ts, - "updated_at": ts, - }, - ) - apply_cdc_records(conn, capture.records_since(capture.latest_sequence - 1)) - - count = conn.execute( - "SELECT COUNT(*) FROM wh_wallets WHERE wallet_id = 'w1'" - ).fetchone()[0] - assert count == 1 - - -# ── soft delete ─────────────────────────────────────────────────────────────── - - -def test_delete_marks_warehouse_row_as_deleted(seeded): - conn, capture = seeded - capture.delete("customers", "c2", {"customer_id": "c2"}) - apply_cdc_records(conn, capture.records_since(capture.latest_sequence - 1)) - - row = conn.execute( - "SELECT _deleted FROM wh_customers WHERE customer_id = 'c2'" - ).fetchone() - assert row is not None - assert row[0] is True - - -def test_delete_does_not_remove_row_from_warehouse(seeded): - conn, capture = seeded - capture.delete("wallets", "w2", {"wallet_id": "w2"}) - apply_cdc_records(conn, capture.records_since(capture.latest_sequence - 1)) - - count = conn.execute( - "SELECT COUNT(*) FROM wh_wallets WHERE wallet_id = 'w2'" - ).fetchone()[0] - assert count == 1 - - -# ── lake immutability ───────────────────────────────────────────────────────── - - -def test_lake_row_count_only_increases(seeded): - conn, capture = seeded - before = conn.execute("SELECT COUNT(*) FROM lake_cdc_events").fetchone()[0] - - ts = _ts() - capture.update( - "customers", - "c1", - { - "customer_id": "c1", - "name": "Alice Updated", - "email": "alice-test-user", - "status": "active", - "created_at": ts, - "updated_at": ts, - }, - ) - new_records = capture.records_since(capture.latest_sequence - 1) - append_to_lake(conn, new_records) - - after = conn.execute("SELECT COUNT(*) FROM lake_cdc_events").fetchone()[0] - assert after > before - - -def test_lake_retains_all_operations_for_same_pk(seeded): - conn, capture = seeded - ts = _ts() - capture.update( - "customers", - "c1", - { - "customer_id": "c1", - "name": "Alice v2", - "email": "alice-test-user", - "status": "active", - "created_at": ts, - "updated_at": ts, - }, - ) - capture.delete("customers", "c1", {"customer_id": "c1"}) - append_to_lake(conn, capture.records_since(capture.latest_sequence - 2)) - - events = conn.execute( - "SELECT operation FROM lake_cdc_events" - " WHERE table_name = 'customers' AND primary_key = 'c1'" - " ORDER BY sequence" - ).fetchall() - ops = [e[0] for e in events] - assert "insert" in ops - assert "update" in ops - assert "delete" in ops - - -# ── PK uniqueness ───────────────────────────────────────────────────────────── - - -def test_warehouse_customers_pk_is_unique(seeded): - conn, _ = seeded - total = conn.execute("SELECT COUNT(*) FROM wh_customers").fetchone()[0] - distinct = conn.execute( - "SELECT COUNT(DISTINCT customer_id) FROM wh_customers" - ).fetchone()[0] - assert total == distinct - - -def test_warehouse_wallets_pk_is_unique(seeded): - conn, _ = seeded - total = conn.execute("SELECT COUNT(*) FROM wh_wallets").fetchone()[0] - distinct = conn.execute( - "SELECT COUNT(DISTINCT wallet_id) FROM wh_wallets" - ).fetchone()[0] - assert total == distinct - - -# ── business rules ──────────────────────────────────────────────────────────── - - -def test_transaction_amounts_are_positive(seeded): - conn, _ = seeded - bad = conn.execute( - "SELECT COUNT(*) FROM wh_transactions WHERE amount <= 0" - ).fetchone()[0] - assert bad == 0 - - -def test_wallet_balances_are_non_negative(seeded): - conn, _ = seeded - bad = conn.execute("SELECT COUNT(*) FROM wh_wallets WHERE balance < 0").fetchone()[ - 0 - ] - assert bad == 0 - - -def test_transaction_directions_are_valid(seeded): - conn, _ = seeded - bad = conn.execute( - "SELECT COUNT(*) FROM wh_transactions" - " WHERE direction NOT IN ('credit', 'debit')" - ).fetchone()[0] - assert bad == 0 - - -# ── replay safety ───────────────────────────────────────────────────────────── - - -def test_replaying_same_records_does_not_create_duplicates(seeded): - conn, capture = seeded - # Replay all records from the beginning - all_records = capture.records_since(0) - apply_cdc_records(conn, all_records) - - count = conn.execute("SELECT COUNT(*) FROM wh_customers").fetchone()[0] - assert count == 2 # Must not double to 4 - - -@pytest.mark.parametrize("offset", [0, 1, 2]) -def test_partial_replay_from_checkpoint_is_idempotent(seeded, offset: int): - conn, capture = seeded - partial = capture.records_since(offset) - apply_cdc_records(conn, partial) - - count = conn.execute("SELECT COUNT(*) FROM wh_customers").fetchone()[0] - assert count == 2 diff --git a/submission/sample-candidate/tests/test_schema_contracts.py b/submission/sample-candidate/tests/test_schema_contracts.py deleted file mode 100644 index 69d30f7..0000000 --- a/submission/sample-candidate/tests/test_schema_contracts.py +++ /dev/null @@ -1,170 +0,0 @@ -""" -Tests — schema contract detection and safe-stop behavior. - -Covers: passing contracts, missing column detection, incompatible schema change -detection (dropped column, added unexpected table), and that ingestion can be -stopped when a contract violation is found. -""" - -import duckdb -import pytest - -from source.models import SCHEMA_CONTRACT, create_source_tables - -# ── helpers ─────────────────────────────────────────────────────────────────── - - -def _actual_columns(conn: duckdb.DuckDBPyConnection, table: str) -> set[str]: - return {row[0] for row in conn.execute(f"DESCRIBE {table}").fetchall()} - - -def _check_contracts(conn: duckdb.DuckDBPyConnection) -> list[str]: - """Inline contract check — mirrors scripts/check_schema_contracts.py.""" - violations: list[str] = [] - for table, expected_cols in SCHEMA_CONTRACT.items(): - try: - actual = _actual_columns(conn, table) - except Exception as exc: - violations.append(f"{table}: {exc}") - continue - for col in expected_cols: - if col not in actual: - violations.append(f"{table}.{col}: column missing") - return violations - - -# ── passing contracts ───────────────────────────────────────────────────────── - - -def test_fresh_source_tables_pass_all_contracts(): - conn = duckdb.connect(":memory:") - create_source_tables(conn) - assert _check_contracts(conn) == [] - - -def test_all_contract_tables_are_present(): - conn = duckdb.connect(":memory:") - create_source_tables(conn) - for table in SCHEMA_CONTRACT: - cols = _actual_columns(conn, table) - assert len(cols) > 0, f"{table} has no columns" - - -# ── missing column detection ────────────────────────────────────────────────── - - -def test_dropped_column_is_detected_as_violation(): - # Create customers without FK-dependent tables so ALTER TABLE is allowed - conn = duckdb.connect(":memory:") - conn.execute(""" - CREATE TABLE customers ( - customer_id VARCHAR PRIMARY KEY, - name VARCHAR NOT NULL, - status VARCHAR NOT NULL, - created_at TIMESTAMP NOT NULL, - updated_at TIMESTAMP NOT NULL - ) - """) - # 'email' was never added — contract check should catch it - violations = _check_contracts(conn) - assert any("email" in v for v in violations) - - -def test_multiple_dropped_columns_all_reported(): - # Create wallets without FK constraint so ALTER TABLE is allowed - conn = duckdb.connect(":memory:") - conn.execute(""" - CREATE TABLE customers (customer_id VARCHAR PRIMARY KEY, - name VARCHAR NOT NULL, email VARCHAR NOT NULL, - status VARCHAR NOT NULL, created_at TIMESTAMP NOT NULL, - updated_at TIMESTAMP NOT NULL) - """) - conn.execute(""" - CREATE TABLE wallets ( - wallet_id VARCHAR PRIMARY KEY, - customer_id VARCHAR NOT NULL, - currency VARCHAR NOT NULL, - status VARCHAR NOT NULL, - created_at TIMESTAMP NOT NULL, - updated_at TIMESTAMP NOT NULL - ) - """) - # 'balance' never added — contract check should report both balance and balance - violations = _check_contracts(conn) - assert any("balance" in v for v in violations) - - -def test_violations_include_table_and_column_name(): - conn = duckdb.connect(":memory:") - create_source_tables(conn) - conn.execute("ALTER TABLE transactions DROP COLUMN amount") - - violations = _check_contracts(conn) - assert any("transactions" in v and "amount" in v for v in violations) - - -# ── contract-based safe stop ────────────────────────────────────────────────── - - -def test_pipeline_stops_when_contract_violated(): - """ - When a contract violation is found, no CDC records should be captured. - This models the stop-the-line behavior required by the assignment. - """ - from pipeline.cdc import CDCCapture - - # Create customers table missing 'email' — simulates a breaking schema change - conn = duckdb.connect(":memory:") - conn.execute(""" - CREATE TABLE customers ( - customer_id VARCHAR PRIMARY KEY, - name VARCHAR NOT NULL, - status VARCHAR NOT NULL, - created_at TIMESTAMP NOT NULL, - updated_at TIMESTAMP NOT NULL - ) - """) - - violations = _check_contracts(conn) - capture = CDCCapture() - - if violations: - # Pipeline aborts — no records captured - pass - else: - capture.insert("customers", "c1", {"customer_id": "c1", "name": "Alice"}) - - assert ( - len(capture.log) == 0 - ), "No records should be captured after a contract violation" - - -# ── schema contract completeness ────────────────────────────────────────────── - - -def test_schema_contract_covers_all_key_tables(): - assert "customers" in SCHEMA_CONTRACT - assert "wallets" in SCHEMA_CONTRACT - assert "transactions" in SCHEMA_CONTRACT - - -def test_schema_contract_includes_primary_key_columns(): - assert "customer_id" in SCHEMA_CONTRACT["customers"] - assert "wallet_id" in SCHEMA_CONTRACT["wallets"] - assert "transaction_id" in SCHEMA_CONTRACT["transactions"] - - -@pytest.mark.parametrize( - "table,col", - [ - ("customers", "status"), - ("wallets", "balance"), - ("wallets", "currency"), - ("transactions", "amount"), - ("transactions", "direction"), - ], -) -def test_business_critical_columns_are_in_contract(table: str, col: str): - assert ( - col in SCHEMA_CONTRACT[table] - ), f"Business-critical column {table}.{col} is not in SCHEMA_CONTRACT" diff --git a/submission/wallet-payment-transfer/ARCHITECTURE.md b/submission/wallet-payment-transfer/ARCHITECTURE.md new file mode 100644 index 0000000..8bef149 --- /dev/null +++ b/submission/wallet-payment-transfer/ARCHITECTURE.md @@ -0,0 +1,304 @@ +# Architecture + +The pipeline is six layers stacked in a single process. Each layer has a +narrow responsibility and a typed contract with the layer above and below +it. The contracts are what make the system reliable under replay, +restart, and schema drift. + +``` + ┌──────────────────────────────────────┐ + │ 6. Catalog (catalog/catalog.json) │ + └──────────────────────────────────────┘ + ▲ publishes + ┌──────────────────┐ │ + │ 5. Quality / │ asserts ─────────►│ wh_* + wh_*_history + │ Validation │ │ + │ (run_data_ │ + │ quality_checks) │ + └──────────────────┘ ▲ merges (idempotent) + │ + ┌────────────────────┐ + │ 4. Warehouse │ current-state + SCD2 + │ (pipeline/warehouse│ state_at(ts), restore + └────────────────────┘ + ▲ replayable from + │ + ┌────────────────────┐ + │ 3. Lake │ append-only event log + │ (pipeline/lake) │ lake_cdc_events + └────────────────────┘ + ▲ writes (sequence-deduped) + │ + ┌──────────────────────────────────────────────────┐ + │ 2. Ingestion / CDC layer │ + │ CDCCapture → Orchestrator (run_tick) │ + │ schema gate (SchemaDetector) — stop-the-line │ + │ per-stage checkpoints (pipeline_checkpoints) │ + └──────────────────────────────────────────────────┘ + ▲ observes via contracts + │ + ┌────────────────────┐ + │ 1. Source │ + │ DDL + indexes + │ + │ CHECK constraints │ + └────────────────────┘ +``` + +--- + +## 1. Source layer + +**File:** `source/models.py`, `source/contracts.py`, `source/seed.py` + +Six tables in dependency order: + +| Table | Strong / Weak | Parent | +|--------------------|---------------|---------------------------------| +| `customers` | strong | — | +| `wallets` | strong | references customers | +| `transfers` | strong | references wallets, pay_methods | +| `payment_methods` | weak | customer | +| `transfer_events` | weak | transfer | +| `ledger_entries` | weak | wallet | + +A table is **weak** when its lifecycle is bound to a parent. A +payment_method has no business meaning if its customer is removed; a +ledger_entry has no meaning without its wallet. Strong entities can be +queried, ranked, and reasoned about on their own. + +Every table carries `created_at` / `updated_at`, primary keys are +non-null VARCHAR identifiers, and enums are enforced via `CHECK` +constraints. Indexes mirror real CDC and lookup patterns: + +- `updated_at` on `customers`, `wallets`, `transfers` — incremental + extraction by timestamp +- foreign-key indexes — referential lookup + cascading delete scans +- `(wallet_id, occurred_at)` on `ledger_entries` — audit-by-wallet +- `status` indexes on `customers`, `transfers` — filtered scans + +**Contracts** (`source/contracts.py`) are a frozen, versioned mirror of +this DDL. The schema detector and the data-quality layer both read from +contracts so a single change in `contracts.py` propagates everywhere. + +--- + +## 2. CDC / Ingestion layer + +**Files:** `pipeline/cdc.py`, `pipeline/orchestrator.py`, +`pipeline/checkpoint.py` + +`CDCCapture` is the in-process stand-in for a Debezium-like log +consumer. Each `CDCRecord` carries: + +| Field | Why | +|----------------|----------------------------------------------------| +| `operation` | `insert` / `update` / `delete` | +| `table` | source table name | +| `primary_key` | natural key value | +| `before` | pre-image (required for update / delete) | +| `after` | post-image (required for insert / update) | +| `captured_at` | wall clock at capture (UTC, defaults to `now`) | +| `sequence` | monotonic offset (Postgres LSN / Kafka offset) | + +The `sequence` is the key to everything downstream: it makes the lake +deduplicate trivially (`sequence` is its primary key) and makes the +warehouse merge idempotent and out-of-order safe (each warehouse row +records the last `_cdc_seq` that touched it). + +`CDCCapture` can optionally hold a `SchemaDetector` and refuse writes +when the source schema is breaking — see `test_capture_with_strict_schema_refuses_writes_on_drop`. + +**`run_tick(conn, capture, *, detector, fail_on_drift=True)`** is the +single orchestration entry point: + +``` +1. bootstrap (idempotent CREATE TABLE IF NOT EXISTS) for lake / wh / checkpoints +2. if detector reports BREAKING drift → return aborted=true, do not move data +3. read records since min(checkpoint_lake, checkpoint_warehouse) from capture +4. write records with sequence > checkpoint_lake to the lake +5. advance checkpoint_lake +6. merge records with sequence > checkpoint_warehouse into the warehouse +7. advance checkpoint_warehouse +``` + +Each step is independently safe to retry. If step 6 fails, step 4-5 may +have committed; the next tick sees `checkpoint_lake > checkpoint_warehouse` +and re-merges from the lake / capture. The warehouse merge silently +no-ops on already-applied sequences. + +**Checkpoints** (`pipeline_checkpoints`) are an ordinary table in the +same DuckDB database as the warehouse. In production, this would be a +Kafka consumer group offset committed inside the same transaction as +the warehouse batch. + +--- + +## 3. Lake layer + +**File:** `pipeline/lake.py` + +A single table: + +```sql +CREATE TABLE lake_cdc_events ( + sequence INTEGER PRIMARY KEY, -- dedup + operation VARCHAR NOT NULL, + table_name VARCHAR NOT NULL, + primary_key VARCHAR NOT NULL, + before_image VARCHAR, -- JSON + after_image VARCHAR, -- JSON + captured_at TIMESTAMP NOT NULL +) +``` + +**Append-only.** `append_to_lake` uses `ON CONFLICT (sequence) DO NOTHING`, +so replay is a no-op. `replay_records(conn, table=None, until_sequence=None)` +returns events in `sequence` order and is what the warehouse would replay +from after disaster recovery. + +The before/after row images are JSON. A custom serializer handles +`Decimal`, `datetime`, and `date` so monetary values round-trip exactly +(`Decimal("75.0000")` stays a `Decimal` after `json.dumps` → +`json.loads`). + +**Production analogue:** an Iceberg/Delta bronze table partitioned by +`(table_name, captured_at::date)`. The PK on `sequence` becomes a Delta +Lake `MERGE` key. + +--- + +## 4. Warehouse layer + +**File:** `pipeline/warehouse.py` + +For each source table `T` we materialize **two** tables: + +### `wh_T` — current state + +The current-state row carries `_cdc_seq` (the sequence of the last +event applied) and `_deleted` (set true on delete; the row is kept so +analytics over deleted entities still works). + +### `wh_T_history` — SCD2 history + +Each version has: + +| Column | Meaning | +|---------------|---------------------------------------------------| +| `_cdc_seq` | sequence that produced this version | +| `_valid_from` | inclusive start of validity (= `captured_at`) | +| `_valid_to` | exclusive end of validity, NULL for current row | +| `_is_current` | redundant flag, indexes well | +| `_op` | `insert` / `update` / `delete` | +| PK | `(business_key, _valid_from)` | + +### Idempotent merge + +```python +existing = SELECT _cdc_seq FROM wh_T WHERE pk = ? +if existing and existing._cdc_seq >= record.sequence: + return # already applied, or out-of-order — no-op +``` + +This single check makes the merge safe under: + +- **duplicate events** (same sequence ≤ existing → no-op) +- **out-of-order arrival** (smaller sequence → no-op; tested in `test_older_event_does_not_clobber_newer_state`) +- **retry after partial failure** (the merge is on a per-row basis, so re-running yields the same final state) + +### Time travel + +```python +state_at(conn, table, ts) -> list[dict] +``` + +scans `wh_T_history` for rows where +`_valid_from <= ts < COALESCE(_valid_to, '9999-12-31')` and +`_op != 'delete'`. The result is "what `SELECT * FROM T` would have +returned at `ts`". + +`restore_current_state_to(conn, table, ts)` truncates `wh_T` and reloads +it from `state_at(table, ts)`. History is **never** modified — restore +is a current-state operation only. + +--- + +## 5. Quality / Validation layer + +**File:** `scripts/run_data_quality_checks.py` + +Rules are **derived** from `source/contracts.py`, so adding a new +NOT NULL column or a new enum domain in `contracts.py` automatically +generates the corresponding warehouse rule. There is no second source +of truth. + +### System rules (auto-generated from contracts) + +- `sys.pk_unique.` — PK has no duplicates +- `sys.not_null.
.` — for every NOT NULL column in the contract +- `sys.enum.
.` — for every column with an enum domain +- `sys.fk.
.` — referential integrity, ignoring `_deleted` + +### Business rules (explicitly defined) + +- `biz.wallet.balance_non_negative` +- `biz.transfer.amount_positive` +- `biz.transfer.no_self_transfer` +- `biz.transfer.settled_at_after_created` +- `biz.transfer.settled_implies_settled_at` +- `biz.transfer.failed_implies_reason` +- `biz.ledger.signed_amount_consistent` +- `biz.ledger.balance_after_non_negative` +- `biz.ledger.reconciles_to_wallet_balance` — `SUM(signed_amount) == balance` per wallet +- `biz.transfer_event.allowed_transition` + +`run_checks` returns a list of `(rule, count_of_failing_rows)` pairs. +The CI script exits non-zero if the list is non-empty. + +`tests/test_data_quality.py` proves both directions: the seeded +warehouse passes all rules **and** each rule actually fires when its +invariant is violated. + +--- + +## 6. Catalog layer + +**File:** `catalog/catalog.json` + +13 datasets — 1 lake event log + 6 warehouse current-state + 6 +warehouse SCD2 history. Each carries: + +- `name`, `layer`, `type` (controlled vocabulary: `lake` / `warehouse`, + `append-only-change-log` / `current-state` / `scd2-history`) +- `description`, `owner`, `consumers` +- `update_cadence`, `sla_class` (`tier-1` / `tier-2` / `tier-3`) +- `tags`, `primary_key`, `schema` or `schema_extension` + +`scripts/validate_catalog.py` enforces: + +- every expected dataset is present +- every required field is populated +- the layer/type pairing is consistent +- `tests/test_catalog.py` adds 14 more assertions including controlled + vocabularies, money-critical SLA tier, and time-travel tagging on + history datasets + +In production this catalog would publish to DataHub / Atlan / Unity +Catalog. The JSON shape is deliberately close to OpenAPI dataset +declarations so the migration is a serializer swap. + +--- + +## Failure modes and how the architecture handles them + +| Failure | Handled by | +|--------------------------------------|-------------------------------------------------------------| +| Duplicate CDC event | `lake.sequence` PK + warehouse `_cdc_seq` guard | +| Out-of-order arrival | warehouse `_cdc_seq` guard rejects lower sequences | +| Retry after partial failure | each tick step is idempotent; checkpoints advance per stage | +| Restart with progress checkpointed | `pipeline_checkpoints` + `min(lake, wh)` floor in `run_tick`| +| Incompatible schema change | `SchemaDetector` BREAKING + orchestrator stops the line | +| Source delete | warehouse soft-deletes (`_deleted=true`); SCD2 closes row | +| Late-arriving update with smaller seq| warehouse merge guard rejects | +| Need to recover historical state | `state_at(ts)` reconstructs from SCD2; `restore_current_state_to` | +| Ledger / wallet drift | `biz.ledger.reconciles_to_wallet_balance` rule fails CI | diff --git a/submission/wallet-payment-transfer/ASSUMPTIONS.md b/submission/wallet-payment-transfer/ASSUMPTIONS.md new file mode 100644 index 0000000..59cabe1 --- /dev/null +++ b/submission/wallet-payment-transfer/ASSUMPTIONS.md @@ -0,0 +1,189 @@ +# Assumptions, Tradeoffs, and Production Analogues + +This is a **single-process simulation** of a CDC lakehouse, written +to be reasoned about rather than deployed. Every shortcut taken below +has a documented production analogue. The goal is to make the +correctness story easy to follow without losing the structural shape +of a real system. + +--- + +## What is simulated and what would change in production + +### CDC capture + +**Simulated:** `pipeline/cdc.py` — `CDCCapture.insert / update / delete` +called explicitly from seed code or tests. Each call appends a +`CDCRecord` with a strictly increasing `sequence` to an in-memory log. + +**Production analogue:** Postgres logical decoding via Debezium → +Kafka. Sequence numbers become Kafka offsets (which, conveniently, +share the same monotonic-by-partition guarantees we rely on). The +`CDCRecord` shape lines up 1-1 with Debezium's envelope (`op`, +`table`, `before`, `after`, `ts_ms`, `lsn`). + +What does **not** change: the `_cdc_seq` guard in the warehouse merge, +the lake's PK on `sequence`, and the orchestrator's bootstrap → +schema-gate → lake → warehouse flow. Those are the stable contracts. + +### Storage + +**Simulated:** DuckDB in-memory (or a single file) for source, lake, +warehouse, and checkpoint store. + +**Production analogue:** +- **Source** lives in Postgres / MySQL — DuckDB exposes the same + `information_schema.columns` introspection used by `SchemaDetector`, + so the detector is portable. +- **Lake** is an Iceberg / Delta / Hudi table on object storage, + partitioned by `(table_name, captured_at::date)`. The lake's + `sequence` PK becomes a Delta `MERGE` key. +- **Warehouse** is Snowflake / BigQuery / Redshift. The current-state + + SCD2 pattern translates verbatim — `MERGE INTO ... USING ... ON + pk WHEN MATCHED AND target._cdc_seq < source.sequence THEN UPDATE`. +- **Checkpoints** live in a small operational store, ideally + committed inside the same warehouse transaction as the merge. + +### Schema detection + +**Simulated:** `SchemaDetector` introspects the live source DDL via +`information_schema.columns` and `duckdb_constraints()` and compares +against `source/contracts.py`. + +**Production analogue:** +- Confluent / AWS Glue Schema Registry as the system of record for + Avro / Protobuf schemas; Debezium publishes new schemas there +- A simple comparator (the same logic as our `SchemaDetector`) runs + on schema-changed events, emits a Slack / PagerDuty alert, and + flips a feature flag the orchestrator polls + +The detector's rule set (column_dropped, type_changed, +nullability_tightened, primary_key_changed, non_nullable_column_added +=> BREAKING; column_added, enum_value_added => WARN) is identical to +what the production analogue would enforce. + +--- + +## Simplifying decisions + +### Single-process / no orchestrator framework + +The orchestrator is `pipeline/orchestrator.py:run_tick`, called +directly from scripts and tests. There is no Airflow / Dagster +because that would obscure the data-flow. The full tick is one +function call (~70 lines) that you can read top-to-bottom. In +production this becomes a Dagster `@op` or an Airflow `PythonOperator`. + +### JSON-encoded before/after images instead of Avro + +`json.dumps` with a custom `default=` for `Decimal`, `date`, and +`datetime`. JSON is human-readable and lossless for our types and +trivially queryable in DuckDB. Production would use Avro/Protobuf +because of schema evolution support; the contract surface (`before`, +`after` carry the source row image) does not change. + +### Soft deletes only — never hard deletes + +`wh_
` rows are flagged with `_deleted=true` instead of being +removed. This: + +- preserves the row for analytics on deleted entities +- keeps FK integrity in `wh_
` even after a parent is removed +- mirrors what most warehouses already do — a hard delete is rare in + analytical models + +The data-quality FK rule explicitly ignores `_deleted=true` rows on +both sides (see `tests/test_data_quality.py::test_deleted_rows_do_not_trigger_validation_failures`). + +### `captured_at` is the SCD2 valid_from + +We use the CDC capture timestamp as the SCD2 boundary, not a separate +"as_of" column. In production with millisecond ordering issues we +would use `(captured_at, sequence)` as a composite key, but `sequence` +alone already provides total ordering, so the simpler boundary is +fine for our scope. + +### No transaction boundaries across stages + +In DuckDB single-process we could `BEGIN ... COMMIT` across the lake +write and warehouse merge — but that hides the realistic case where +lake and warehouse are different systems. We instead lean on the +**idempotent merge** + **per-stage checkpoints** to make a partial +failure recoverable without distributed transactions. + +### Sequence is generated in `CDCCapture` + +In production the sequence is the Postgres LSN (or Kafka offset). We +issue our own monotonic counter in `CDCCapture.latest_sequence + 1` +because there is no real WAL to read. The shape is identical. + +### One DuckDB connection per test + +Tests use `:memory:` connections, so they're hermetic. There is no +shared state between tests. The `seeded` fixture builds the entire +seed in-process per test, which is < 100ms. + +### No partitioning on the lake table + +Production would partition by `(table_name, captured_at::date)`. We +skipped this because DuckDB plans queries fine without it on our scale +and partitioning would obscure the model. The catalog entry already +documents the production partition. + +--- + +## Things explicitly NOT solved + +- **Automatic migration of breaking schema changes.** The assignment + states this is out of scope. We *detect* and *stop* — we do not + attempt to back-fill or reshape. +- **Deduplication on the source side** — `(table, primary_key, sequence)` + is unique by construction in `CDCCapture`. If a real source emits + duplicate LSNs (impossible in Postgres, possible only in misconfigured + pipelines), the lake's PK on `sequence` catches it; the warehouse merge + catches it as a same-sequence no-op. +- **Multi-tenant catalogs** — one domain only. +- **Lineage / column-level provenance** — out of scope; production would + use OpenLineage or DataHub lineage. +- **Authorization on warehouse tables** — out of scope. + +--- + +## What we did NOT compromise on + +- **Idempotency** — every lake write is dedup'd by `sequence`, every + warehouse merge is dedup'd by `_cdc_seq`. Replay is provably safe. +- **Out-of-order safety** — the `_cdc_seq` guard handles late-arriving + smaller sequences. Tested in `test_older_event_does_not_clobber_newer_state`. +- **Stop-the-line** — `run_tick` aborts before any data movement on + breaking drift. `CDCCapture` can also refuse writes when configured + with a strict detector. Both modes are tested. +- **Validation parity** — the data-quality rules are *generated* from + contracts, so every NOT NULL column / enum domain in the source + appears as a downstream rule by construction. +- **Time travel** — SCD2 is real, not approximated. `state_at(ts)` + goes through the same path the warehouse merge writes, so any new + CDC event automatically becomes time-travelable. +- **Testing across categories** — 9 test modules, ~80 tests covering + source constraints, CDC, lake, warehouse, schema, data quality, + catalog, time-travel, and end-to-end. + +--- + +## Tradeoffs we'd revisit with more time + +1. **Replace the in-memory `CDCCapture` with a real Debezium stub** — + the current shape is faithful but lacks the long tail of edge cases + (transactional grouping, snapshot vs streaming mode). +2. **Add column-level lineage to the catalog** — the 13 datasets all + know their primary keys but not where each column flows from. +3. **Add a freshness rule to data quality** — currently we check + correctness, not staleness. Production needs both. +4. **Wire a tiny Streamlit UI on top of `state_at`** — would make + time-travel demonstrable to non-engineers. +5. **Containerize** — a `Dockerfile` + `docker-compose.yml` with + Postgres + Kafka + the orchestrator would be the natural next step + to demonstrate the production analogue concretely. + +None of these change the contracts; they're all add-ons to the same +core. diff --git a/submission/wallet-payment-transfer/Makefile b/submission/wallet-payment-transfer/Makefile new file mode 100644 index 0000000..ee25aec --- /dev/null +++ b/submission/wallet-payment-transfer/Makefile @@ -0,0 +1,46 @@ +.PHONY: help install test lint demo demo-time-travel demo-schema-change \ + check-schema check-quality check-catalog ci all + +PY ?= python + +help: + @echo "Targets:" + @echo " install - install Python dependencies" + @echo " test - run all pytest tests" + @echo " check-schema - run schema-contract gate" + @echo " check-quality - run data-quality gate" + @echo " check-catalog - run catalog validation gate" + @echo " demo - run end-to-end pipeline demo" + @echo " demo-time-travel - run point-in-time reconstruction demo" + @echo " demo-schema-change - run schema drift / stop-the-line demo" + @echo " ci - run all CI gates (test + 3 checks)" + @echo " all - install, then run ci" + +install: + $(PY) -m pip install -r requirements.txt + +test: + $(PY) -m pytest -q + +check-schema: + $(PY) -m scripts.check_schema_contracts + +check-quality: + $(PY) -m scripts.run_data_quality_checks + +check-catalog: + $(PY) -m scripts.validate_catalog + +demo: + $(PY) -m scripts.run_pipeline + +demo-time-travel: + $(PY) -m scripts.demo_time_travel + +demo-schema-change: + $(PY) -m scripts.demo_schema_change + +ci: test check-schema check-quality check-catalog + @echo "All CI gates passed." + +all: install ci diff --git a/submission/wallet-payment-transfer/PR_DESCRIPTION.md b/submission/wallet-payment-transfer/PR_DESCRIPTION.md new file mode 100644 index 0000000..d47e88f --- /dev/null +++ b/submission/wallet-payment-transfer/PR_DESCRIPTION.md @@ -0,0 +1,326 @@ +# Pull Request — CDC Lakehouse Reliability (Wallet / Payments / Transfers) + +## Summary +End-to-end CDC pipeline that mirrors a 6-table wallet/payments/transfers +source system into a **lake** (immutable change history) and a **warehouse** +(current-state + SCD2 history). It detects schema drift, stops the line on +incompatible changes, supports time-travel reconstruction and operational +restore, and enforces source-to-warehouse validation parity. Implemented in +Python + DuckDB and runnable in a single process. + +Layered into six modules with narrow contracts: + +``` +source → cdc capture + orchestrator → lake → warehouse → quality → catalog +``` + +~1,400 LOC of pipeline + ~1,300 LOC of tests across **9 test modules** +covering modeling, CDC correctness, schema-change safety, warehouse +correctness, time travel, catalog metadata, data quality, and full end-to-end +scenarios. + +See `ARCHITECTURE.md` for the layered design walk-through and +`ASSUMPTIONS.md` for tradeoffs and production analogues. + +--- + +## Source Schema Design + +- **Domain chosen:** wallet / payments / transfers +- **Strong entities:** + - `customers` — registered platform users + - `wallets` — balance accounts; one customer holds many (per-currency) + - `transfers` — money movements between wallets; have business identity +- **Weak entities:** + - `payment_methods` — saved card / bank / upi tied to a customer + - `transfer_events` — state transitions for a transfer + - `ledger_entries` — append-only credit/debit on a wallet +- **Keys, relationships, indexes:** + - All primary keys are non-null `VARCHAR` natural identifiers + - FKs: `payment_methods → customers`, `wallets → customers`, + `transfers → wallets ×2 + payment_methods`, + `transfer_events → transfers`, `ledger_entries → wallets + transfers` + - 12 indexes covering CDC patterns (`updated_at`), referential lookups + (`customer_id`, `transfer_id`), filtered scans (`status`), and + audit-by-wallet (`(wallet_id, occurred_at)`) +- **Source validation rules** (enforced via `CHECK` and `NOT NULL`): + - `wallet.balance >= 0` + - `transfer.amount > 0` + - `transfer.from_wallet_id != transfer.to_wallet_id` + - `transfer.settled_at IS NULL OR settled_at >= created_at` + - `ledger_entries.signed_amount` consistent with `direction` and `amount` + - status enums: `customers.status`, `payment_methods.method_type` / + `.status`, `wallets.currency` / `.status`, `transfers.status` / + `.currency`, `transfer_events.to_status`, `ledger_entries.direction` + +Schema is mirrored in versioned, frozen contracts at `source/contracts.py`, +which are the single source of truth for the schema detector and the +data-quality rules. + +--- + +## CDC Strategy + +- **How changes are captured:** `CDCCapture` wraps every `insert`, + `update`, and `delete` call into a `CDCRecord` with operation, table, + primary_key, before-image, after-image, monotonic `sequence`, and + `captured_at`. The sequence is the LSN/Kafka-offset analogue and the + primary correctness lever. +- **How inserts, updates, deletes are handled:** + - **Insert** — adds a new current-state row with `_cdc_seq=record.sequence` + and opens an SCD2 history row with `_is_current=true`, + `_valid_from=captured_at`, `_op='insert'`. + - **Update** — closes the open SCD2 row (`_valid_to=captured_at`, + `_is_current=false`) and opens a new one with the post-image; updates + the current-state row in place. + - **Delete** — closes the open SCD2 row and appends a closed + `_op='delete'` row (with the pre-image); flips current-state's + `_deleted=true` (soft delete preserves analytics over deleted rows). +- **How replay/restart works:** + - Per-stage checkpoint (`pipeline_checkpoints`): one row each for + `lake` and `warehouse`. `run_tick` reads from + `min(checkpoint_lake, checkpoint_warehouse)`, so a tick that wrote + the lake but failed before the warehouse merge replays cleanly. + - The lake's `sequence` PK silently drops duplicate appends. + - The warehouse merge applies an event only when + `record.sequence > existing._cdc_seq`, making it idempotent and + out-of-order safe. + - `set_checkpoint` refuses to move backwards — bugs surface loudly. +- **How duplicates are handled:** see above. Tested in + `test_replay_does_not_double_apply`, + `test_orchestrator_run_tick_is_idempotent`, and + `test_duplicate_sequence_is_deduped`. + +--- + +## Lake and Warehouse Modeling + +- **How the lake captures every change:** single append-only table + `lake_cdc_events(sequence PK, operation, table_name, primary_key, + before_image JSON, after_image JSON, captured_at)`. One row per CDC + event, no rewrite, no overwrite, no aggregation. `replay_records()` + returns events in sequence order and is the warehouse's recovery feed. +- **How the warehouse maintains latest snapshot:** for each source + table `T`: + - `wh_T` — current-state with `_cdc_seq` (last applied sequence) and + `_deleted` (soft delete). Same column shape as the source + + `_cdc_seq` + `_deleted`. + - `wh_T_history` — SCD2 with `(business_key, _valid_from)` PK and + `_valid_to`, `_is_current`, `_op` columns. Every revision lands + here; `_is_current=true` invariant: at most one row per PK. +- **How time travel / restore is supported:** + - `state_at(conn, table, ts)` reconstructs `T` at any timestamp from + `wh_T_history` using `_valid_from <= ts < _valid_to`. Excludes + deleted rows automatically. Tested in `test_time_travel.py` against + multiple inserted revisions. + - `restore_current_state_to(conn, table, ts)` overwrites `wh_T` with + the snapshot from `state_at(table, ts)` and **does not modify + history** — restore is reversible, history is the system of record. + - After restore, `_cdc_seq` resets to 0 so subsequent CDC events + apply cleanly. Verified by `test_restored_state_accepts_new_cdc_records`. + +--- + +## Schema Change Safety + +- **How incompatible changes are detected:** `SchemaDetector` introspects + the live source via `information_schema.columns` and + `duckdb_constraints()` and compares against `source/contracts.py`. + Eight rules: + - **BREAKING:** `missing_table`, `column_dropped`, `type_changed`, + `nullability_tightened`, `non_nullable_column_added`, + `primary_key_changed` + - **WARN:** `column_added` (additive), `enum_value_added` (drift + surface, not necessarily breaking) +- **How ingestion is stopped:** at *two* levels: + - **Orchestrator gate** — `run_tick(..., fail_on_drift=True)` runs + `detector.check_all()` *before* reading any new events. On a + breaking drift it returns `{aborted: true, reason: + "schema_contract_violation", drift_summary: ...}` and writes + nothing to the lake or warehouse. Tested in + `test_run_tick_aborts_on_breaking_drift`. + - **Capture gate** — `CDCCapture(detector, require_compatible_schema=True)` + raises `SchemaContractViolation` on every write while a breaking + drift is present. The capture log stays empty. Tested in + `test_capture_with_strict_schema_refuses_writes_on_drop` and + `test_capture_refuses_writes_when_strict`. +- **How warnings/failures are surfaced:** + - `scripts/check_schema_contracts.py` is the CI entry; it exits + non-zero with a printed drift summary on any breaking finding, + suitable for the `schema-contract-check` GitHub status check. + - `run_tick`'s aborted dict includes a `drift_summary` for log + aggregation / Slack alerts. + - WARN rules are reported via `report.warnings` and never abort — + they are operational signals, not failures. + +--- + +## Validation Parity + +- **Which source system validations were mirrored downstream:** + - **System rules** — auto-generated from `contracts.py`: + - `sys.pk_unique.
` for every PK + - `sys.not_null.
.` for every NOT NULL column + - `sys.enum.
.` for every enum-domained column + - `sys.fk.
.` for every FK (ignoring `_deleted=true` rows) + - **Business rules** — explicit: + - `biz.wallet.balance_non_negative` + - `biz.transfer.amount_positive` + - `biz.transfer.no_self_transfer` + - `biz.transfer.settled_at_after_created` + - `biz.transfer.settled_implies_settled_at` + - `biz.transfer.failed_implies_reason` + - `biz.ledger.signed_amount_consistent` + - `biz.ledger.balance_after_non_negative` + - `biz.ledger.reconciles_to_wallet_balance` (monetary integrity: + `SUM(signed_amount) per wallet == wh_wallets.balance`) + - `biz.transfer_event.allowed_transition` (state machine validity) +- **How failures are checked and reported:** + - `scripts/run_data_quality_checks.py:run_checks(conn)` returns + `[(rule, count_of_failing_rows), ...]`. The CI script exits + non-zero on any failure and prints the rule_id + offending count. + - `tests/test_data_quality.py` proves both directions: the seeded + warehouse passes all rules **and** every individual rule fires when + its invariant is violated (tested for PK uniqueness, enum, FK, + negative balance, settled_at backward, ledger consistency, ledger + reconciliation, status transition). + - A parity test (`test_every_contract_not_null_has_a_warehouse_rule`) + blocks a contract change without a corresponding rule. + +--- + +## Catalog Exposure + +13 datasets in `catalog/catalog.json` covering every output the pipeline +produces: + +- 1 lake dataset: `lake_cdc_events` +- 6 warehouse current-state: `wh_customers`, `wh_payment_methods`, + `wh_wallets`, `wh_transfers`, `wh_transfer_events`, `wh_ledger_entries` +- 6 warehouse SCD2 history: each of the above + `_history` + +Each dataset publishes: + +- `name`, `layer` (`lake` / `warehouse`), `type` + (`append-only-change-log` / `current-state` / `scd2-history`) +- `description`, `owner` (`data-platform`), `consumers` (per-dataset list: + finance / treasury / fraud / audit / analytics / product / support) +- `update_cadence`, `sla_class` (money-critical datasets are `tier-1`) +- `tags` for governance (`monetary-integrity`, `pii`, `time-travel`, etc.) +- `primary_key` and `schema` / `schema_extension` + +Discovery / access in this scope: consumers `cat catalog/catalog.json` +or `python -m scripts.validate_catalog` to verify it matches the +pipeline outputs. Production analogue is DataHub / Atlan / Unity +Catalog — the JSON shape is intentionally close to those tools' +dataset envelopes so the migration is a serializer swap. + +`tests/test_catalog.py` enforces: +- catalog file exists and is valid JSON +- every expected dataset is declared (no missing, no extras) +- every dataset has the required descriptive fields +- controlled vocabularies for `layer`, `type`, `sla_class` +- consistency: lake → `append-only-change-log`, warehouse non-history + → `current-state`, warehouse `_history` → `scd2-history` +- money-critical datasets are tier-1 +- SCD2 history datasets are tagged for time-travel where they back + balance lookups + +--- + +## Validation + +Local validation commands run in order: + +```bash +cd submission/wallet-payment-transfer +pip install -r requirements.txt + +# 1. Schema contract gate (BREAKING drifts → exit 1) +python -m scripts.check_schema_contracts + +# 2. Data quality parity gate +python -m scripts.run_data_quality_checks + +# 3. Catalog metadata gate +python -m scripts.validate_catalog + +# 4. Full test suite (~80 tests, < 10s) +pytest -q + +# 5. End-to-end pipeline demo +python -m scripts.run_pipeline + +# 6. Time travel demo (state_at at multiple timestamps) +python -m scripts.demo_time_travel + +# 7. Schema-change / stop-the-line demo +python -m scripts.demo_schema_change +``` + +All five CI gates exit non-zero on failure and are mapped 1-1 to the +checks listed in `github_pr_checks_data.md`. + +--- + +## Known Limitations / Next Steps + +- **Replace `CDCCapture` with a Debezium stub** — the current shape is + Debezium-faithful (envelope columns, before/after, sequence == LSN) + but does not exercise transactional grouping or snapshot-vs-streaming + modes. With a stub we could test "snapshot then switch to streaming" + explicitly. +- **Column-level lineage in the catalog** — datasets know their PKs but + not which warehouse columns derive from which source columns. An + OpenLineage emitter is the natural addition. +- **Freshness rules** — current data-quality covers correctness, not + staleness. A `dq.freshness.
` rule comparing `MAX(_cdc_seq)` + against expected cadence is a one-day add. +- **Containerization** — a `Dockerfile` + `docker-compose.yml` running + Postgres + Kafka + this orchestrator would make the production + analogue concrete. The contracts in `pipeline/cdc.py` and + `pipeline/schema_detector.py` are designed to swap targets. +- **Automatic non-breaking migration** — for `column_added` (additive) + drift we currently emit a WARN; we could instead auto-extend the + warehouse table inside the same tick. Out of scope by assignment. + +None of these change the architecture; they are additive. + +--- + +## Responsible AI Usage + +- **Did you use AI tools?** Yes — Claude Code as a pair-programming + agent. +- **Where did they help?** + - Drafting boilerplate (DDL strings, frozen-dataclass contracts, JSON + catalog entries) and the per-rule data-quality SQL. + - Surfacing cross-cutting concerns (the `_cdc_seq` guard handles both + duplicate replay AND out-of-order arrival; `restore_current_state_to` + needs to reset `_cdc_seq=0` so subsequent CDC applies cleanly). + - Generating test scaffolds and parametrizations. +- **What did I personally verify or correct?** + - The strong/weak entity classification — I made the calls based on + lifecycle ownership, not the LLM's first guess. + - The seed reconciliation — initial balances did not match + `SUM(signed_amount)`; I worked out the funding entries explicitly so + the monetary-integrity rule passes. + - The `_cdc_seq` guard inequality — I changed it to **strictly greater** + so the same-sequence replay is correctly a no-op (otherwise replay + would re-open SCD2 rows). + - The orchestrator's drift gate placement — checked **before** reading + any events and **before** advancing checkpoints, so partial state is + impossible on a breaking drift. + - Refactored `CDCCapture` to expose `captured_at` as a public + optional parameter rather than letting the time-travel demo poke + `_record(...)` directly. + - Authored every test by walking through the failure mode I wanted + to prove (Red), confirming the test failed against an unfixed + implementation, then verifying it passed (Blue → Green). + - Structured the catalog tests around controlled vocabularies and + money-critical SLA rules, not just "field exists" assertions. + +The architecture decisions, contract shape, severity classifications, +and the layered file structure are all my own. The code is reviewed +end-to-end and every test asserts a behavior I considered worth +proving. diff --git a/submission/wallet-payment-transfer/README.md b/submission/wallet-payment-transfer/README.md new file mode 100644 index 0000000..a42fca0 --- /dev/null +++ b/submission/wallet-payment-transfer/README.md @@ -0,0 +1,146 @@ +# Wallet / Payments / Transfers — CDC Lakehouse + +A minimal, end-to-end CDC pipeline that mirrors a wallet/payments/transfers +source system into a **lake** (immutable change history) and a **warehouse** +(current-state + SCD2 history). It detects schema drift, stops the line on +incompatible changes, and runs idempotently under replay and restart. + +It is built on **Python + DuckDB** so the entire pipeline — source, +ingestion, lake, warehouse, schema detection, validation, and catalog — +runs in a single process with no external services. + +--- + +## Quick start + +```bash +cd submission/wallet-payment-transfer + +# 1. install deps +pip install -r requirements.txt + +# 2. run all tests (CDC, lake, warehouse, schema, time-travel, e2e, …) +pytest -q + +# 3. run the end-to-end pipeline demo +python -m scripts.run_pipeline + +# 4. run the time-travel demo (SCD2 reconstruction at multiple timestamps) +python -m scripts.demo_time_travel + +# 5. run the schema-change / stop-the-line demo +python -m scripts.demo_schema_change +``` + +CI gates (see `github_pr_checks_data.md`): + +```bash +python -m scripts.check_schema_contracts # gate 4: schema-contract-check +python -m scripts.run_data_quality_checks # gate 5: data-quality-check +python -m scripts.validate_catalog # gate 6: catalog-check +pytest -q # gate 2: test-pipeline +``` + +All five exit non-zero on failure. + +--- + +## What's in the box + +``` +submission/wallet-payment-transfer/ +├── source/ # source layer +│ ├── models.py # DDL: 6 tables, indexes, CHECK constraints +│ ├── contracts.py # versioned schema contracts (frozen dataclasses) +│ └── seed.py # canonical seed (3 customers → 5 ledger entries) +│ +├── pipeline/ # CDC + lake + warehouse + schema detection +│ ├── cdc.py # CDCRecord + CDCCapture (with strict-schema mode) +│ ├── lake.py # append-only change log (lake_cdc_events) +│ ├── warehouse.py # current-state + SCD2 + state_at + restore +│ ├── schema_detector.py # 8 drift rules with BREAKING/WARN severities +│ ├── checkpoint.py # per-stage sequence checkpoint store +│ └── orchestrator.py # run_tick() — bootstrap, drift gate, lake → wh +│ +├── catalog/ +│ └── catalog.json # 13 datasets (1 lake + 6 wh_* + 6 wh_*_history) +│ +├── scripts/ # CI entry points + demos +│ ├── check_schema_contracts.py +│ ├── run_data_quality_checks.py +│ ├── validate_catalog.py +│ ├── run_pipeline.py +│ ├── demo_time_travel.py +│ └── demo_schema_change.py +│ +├── tests/ # 9 test modules, ~80 tests across all categories +│ ├── test_source_schema.py +│ ├── test_cdc.py +│ ├── test_lake.py +│ ├── test_warehouse.py +│ ├── test_schema_contracts.py +│ ├── test_data_quality.py +│ ├── test_catalog.py +│ ├── test_time_travel.py +│ └── test_e2e.py +│ +├── ARCHITECTURE.md # detailed design, data flow, layer contracts +├── ASSUMPTIONS.md # tradeoffs, simplifications, production analogues +├── PR_DESCRIPTION.md # populated PR template +└── README.md # this file +``` + +--- + +## Domain (very short) + +Customers register, link payment methods, hold wallets, and move money +through transfers. Every balance change is mirrored in an immutable +ledger. Strong entities own lifecycle: `customers`, `wallets`, `transfers`. +Weak entities depend on a parent: `payment_methods` (← customer), +`transfer_events` (← transfer), `ledger_entries` (← wallet). + +Important invariants (enforced at source AND in the warehouse via +parity rules): + +- `wallet.balance >= 0`, `transfer.amount > 0` +- `transfer.from_wallet_id != transfer.to_wallet_id` +- `transfer.settled_at >= transfer.created_at` (when settled) +- `SUM(ledger_entries.signed_amount) per wallet == wallet.balance` +- transfer status follows a documented state machine + +See `ARCHITECTURE.md` for the full model. + +--- + +## Reading order for reviewers + +If you have ten minutes, read these in order: + +1. `ARCHITECTURE.md` — single-page overview of the data flow +2. `pipeline/cdc.py` — CDC record contract + sequence semantics +3. `pipeline/warehouse.py` — idempotent merge + SCD2 + time travel +4. `pipeline/schema_detector.py` — the 8 drift rules +5. `tests/test_e2e.py` — the full lifecycle in one file +6. `PR_DESCRIPTION.md` — the deliverable narrative + +The whole pipeline is ~1,400 LOC and the tests are ~1,300 LOC. + +--- + +## Production analogue + +This project is intentionally a single-process DuckDB simulation. The +production analogue is straightforward: + +| In this project | Production stand-in | +|------------------------------|--------------------------------------------------------------| +| `CDCCapture` + `sequence` | Postgres logical decoding / Debezium → Kafka topic | +| `lake_cdc_events` | Kafka + S3/GCS Iceberg/Delta bronze table partitioned by day | +| `wh_
` + `_
_history` | Snowflake / BigQuery / Redshift gold tables | +| `pipeline_checkpoints` | Kafka consumer group offset committed with the warehouse merge | +| `SchemaDetector` | Schema registry + dbt `source.freshness` / Great Expectations | +| `catalog/catalog.json` | Atlan / DataHub / Unity Catalog | +| `bootstrap()` + `run_tick()` | Airflow / Dagster sensor + task | + +`ASSUMPTIONS.md` covers each tradeoff explicitly. diff --git a/submission/wallet-payment-transfer/catalog/catalog.json b/submission/wallet-payment-transfer/catalog/catalog.json new file mode 100644 index 0000000..29f649b --- /dev/null +++ b/submission/wallet-payment-transfer/catalog/catalog.json @@ -0,0 +1,288 @@ +{ + "catalog_version": 1, + "domain": "wallet-payments-transfers", + "owner": "data-platform", + "datasets": [ + { + "name": "lake_cdc_events", + "layer": "lake", + "type": "append-only-change-log", + "description": "Append-only log of every CDC event captured from the wallet/payments source system. Preserves the full change history with before/after row images so the warehouse can be deterministically rebuilt and any prior state can be reconstructed.", + "owner": "data-platform", + "consumers": ["data-platform", "audit", "analytics-engineering"], + "update_cadence": "real-time", + "sla_class": "tier-1", + "tags": ["cdc", "raw", "audit", "replayable"], + "primary_key": ["sequence"], + "partitioning": "production: partitioned by table_name + captured_at::date", + "retention_policy": "infinite (audit) — production: 7 years cold storage", + "schema": { + "sequence": {"type": "INTEGER", "nullable": false, "description": "monotonic capture offset (Postgres LSN / Kafka offset analogue)"}, + "operation": {"type": "VARCHAR", "nullable": false, "description": "insert | update | delete"}, + "table_name": {"type": "VARCHAR", "nullable": false, "description": "name of the source table"}, + "primary_key": {"type": "VARCHAR", "nullable": false, "description": "primary key value of the source row"}, + "before_image": {"type": "VARCHAR", "nullable": true, "description": "JSON of the row before the change; null on inserts"}, + "after_image": {"type": "VARCHAR", "nullable": true, "description": "JSON of the row after the change; null on deletes"}, + "captured_at": {"type": "TIMESTAMP", "nullable": false, "description": "UTC timestamp the change was captured"} + } + }, + { + "name": "wh_customers", + "layer": "warehouse", + "type": "current-state", + "description": "Latest known state of every customer. Soft-deleted rows are flagged with _deleted=true and retained for analytics over deleted accounts. Sourced via CDC merge from the source customers table.", + "owner": "data-platform", + "consumers": ["analytics", "product", "finance", "support"], + "update_cadence": "near-real-time", + "sla_class": "tier-1", + "tags": ["customer", "current-state", "scd1"], + "primary_key": ["customer_id"], + "schema": { + "customer_id": "VARCHAR — primary key, source identifier", + "full_name": "VARCHAR", + "email": "VARCHAR", + "country_code": "VARCHAR — ISO-3166 alpha-2", + "status": "VARCHAR — active | suspended | closed", + "registration_date": "DATE", + "created_at": "TIMESTAMP", + "updated_at": "TIMESTAMP", + "_cdc_seq": "INTEGER — sequence of last CDC event applied", + "_deleted": "BOOLEAN — true when the source row was deleted" + } + }, + { + "name": "wh_payment_methods", + "layer": "warehouse", + "type": "current-state", + "description": "Latest known state of every saved payment method (card / bank / upi). Weak entity tied to a customer; if the customer is removed in the source, the method is marked _deleted=true here.", + "owner": "data-platform", + "consumers": ["analytics", "fraud", "finance"], + "update_cadence": "near-real-time", + "sla_class": "tier-1", + "tags": ["payment-method", "current-state", "scd1"], + "primary_key": ["payment_method_id"], + "schema": { + "payment_method_id": "VARCHAR — primary key", + "customer_id": "VARCHAR — FK to wh_customers", + "method_type": "VARCHAR — card | bank | upi", + "masked_identifier": "VARCHAR", + "is_default": "BOOLEAN", + "status": "VARCHAR — active | expired | revoked", + "created_at": "TIMESTAMP", + "updated_at": "TIMESTAMP", + "_cdc_seq": "INTEGER", + "_deleted": "BOOLEAN" + } + }, + { + "name": "wh_wallets", + "layer": "warehouse", + "type": "current-state", + "description": "Latest known state of every wallet. Balance is the authoritative current value from the source; non-negative balance is enforced both at the source and as a downstream data-quality assertion.", + "owner": "data-platform", + "consumers": ["analytics", "finance", "treasury"], + "update_cadence": "near-real-time", + "sla_class": "tier-1", + "tags": ["wallet", "balance", "current-state", "scd1"], + "primary_key": ["wallet_id"], + "schema": { + "wallet_id": "VARCHAR — primary key", + "customer_id": "VARCHAR — FK to wh_customers", + "currency": "VARCHAR — ISO-4217", + "balance": "DECIMAL(18,4) — always >= 0", + "status": "VARCHAR — active | frozen | closed", + "created_at": "TIMESTAMP", + "updated_at": "TIMESTAMP", + "_cdc_seq": "INTEGER", + "_deleted": "BOOLEAN" + } + }, + { + "name": "wh_transfers", + "layer": "warehouse", + "type": "current-state", + "description": "Latest known state of every transfer between two wallets. Status reflects the most recent transition; settled_at is non-null only for settled transfers.", + "owner": "data-platform", + "consumers": ["analytics", "finance", "fraud", "treasury"], + "update_cadence": "near-real-time", + "sla_class": "tier-1", + "tags": ["transfer", "current-state", "scd1"], + "primary_key": ["transfer_id"], + "schema": { + "transfer_id": "VARCHAR — primary key", + "from_wallet_id": "VARCHAR — FK to wh_wallets, source", + "to_wallet_id": "VARCHAR — FK to wh_wallets, destination", + "payment_method_id": "VARCHAR — FK to wh_payment_methods, nullable", + "amount": "DECIMAL(18,4) — always > 0", + "currency": "VARCHAR — ISO-4217", + "status": "VARCHAR — pending | settled | failed | reversed", + "reference": "VARCHAR — nullable external memo", + "failure_reason": "VARCHAR — nullable", + "created_at": "TIMESTAMP", + "settled_at": "TIMESTAMP — null until settled", + "_cdc_seq": "INTEGER", + "_deleted": "BOOLEAN" + } + }, + { + "name": "wh_transfer_events", + "layer": "warehouse", + "type": "current-state", + "description": "Audit trail of state transitions for transfers. Weak entity; one transfer typically has 2-3 events.", + "owner": "data-platform", + "consumers": ["analytics", "fraud", "support"], + "update_cadence": "near-real-time", + "sla_class": "tier-2", + "tags": ["transfer", "audit", "current-state"], + "primary_key": ["transfer_event_id"], + "schema": { + "transfer_event_id": "VARCHAR — primary key", + "transfer_id": "VARCHAR — FK to wh_transfers", + "from_status": "VARCHAR — nullable for the first event", + "to_status": "VARCHAR — pending | settled | failed | reversed", + "note": "VARCHAR — nullable", + "occurred_at": "TIMESTAMP", + "_cdc_seq": "INTEGER", + "_deleted": "BOOLEAN" + } + }, + { + "name": "wh_ledger_entries", + "layer": "warehouse", + "type": "current-state", + "description": "Append-only ledger entries for every credit/debit on a wallet. SUM(signed_amount) per wallet must equal wh_wallets.balance — checked by the data-quality layer.", + "owner": "data-platform", + "consumers": ["analytics", "finance", "treasury", "audit"], + "update_cadence": "near-real-time", + "sla_class": "tier-1", + "tags": ["ledger", "audit", "current-state", "monetary-integrity"], + "primary_key": ["ledger_entry_id"], + "schema": { + "ledger_entry_id": "VARCHAR — primary key", + "wallet_id": "VARCHAR — FK to wh_wallets", + "transfer_id": "VARCHAR — FK to wh_transfers, nullable for funding entries", + "direction": "VARCHAR — credit | debit", + "amount": "DECIMAL(18,4) — always > 0", + "signed_amount": "DECIMAL(18,4) — +amount for credit, -amount for debit", + "balance_after": "DECIMAL(18,4) — non-negative wallet balance after the entry", + "occurred_at": "TIMESTAMP", + "_cdc_seq": "INTEGER", + "_deleted": "BOOLEAN" + } + }, + { + "name": "wh_customers_history", + "layer": "warehouse", + "type": "scd2-history", + "description": "SCD2 history for customers. Every revision carries _valid_from / _valid_to / _is_current / _op so a point-in-time customer snapshot can be reconstructed for restore, time travel, or audit.", + "owner": "data-platform", + "consumers": ["audit", "finance", "data-platform"], + "update_cadence": "near-real-time", + "sla_class": "tier-1", + "tags": ["customer", "scd2", "time-travel"], + "primary_key": ["customer_id", "_valid_from"], + "schema_extension": { + "_cdc_seq": "INTEGER", + "_valid_from": "TIMESTAMP — start of this version's validity", + "_valid_to": "TIMESTAMP — end of validity, NULL for current", + "_is_current": "BOOLEAN", + "_op": "VARCHAR — insert | update | delete" + } + }, + { + "name": "wh_payment_methods_history", + "layer": "warehouse", + "type": "scd2-history", + "description": "SCD2 history for payment_methods.", + "owner": "data-platform", + "consumers": ["audit", "fraud"], + "update_cadence": "near-real-time", + "sla_class": "tier-2", + "tags": ["payment-method", "scd2"], + "primary_key": ["payment_method_id", "_valid_from"], + "schema_extension": { + "_cdc_seq": "INTEGER", + "_valid_from": "TIMESTAMP", + "_valid_to": "TIMESTAMP", + "_is_current": "BOOLEAN", + "_op": "VARCHAR" + } + }, + { + "name": "wh_wallets_history", + "layer": "warehouse", + "type": "scd2-history", + "description": "SCD2 history for wallets — the canonical source for any 'what was the wallet balance at time T' query.", + "owner": "data-platform", + "consumers": ["finance", "treasury", "audit"], + "update_cadence": "near-real-time", + "sla_class": "tier-1", + "tags": ["wallet", "balance", "scd2", "time-travel"], + "primary_key": ["wallet_id", "_valid_from"], + "schema_extension": { + "_cdc_seq": "INTEGER", + "_valid_from": "TIMESTAMP", + "_valid_to": "TIMESTAMP", + "_is_current": "BOOLEAN", + "_op": "VARCHAR" + } + }, + { + "name": "wh_transfers_history", + "layer": "warehouse", + "type": "scd2-history", + "description": "SCD2 history for transfers — supports replaying or reversing a transfer's lifecycle for ops/audit.", + "owner": "data-platform", + "consumers": ["audit", "fraud", "finance"], + "update_cadence": "near-real-time", + "sla_class": "tier-1", + "tags": ["transfer", "scd2", "time-travel"], + "primary_key": ["transfer_id", "_valid_from"], + "schema_extension": { + "_cdc_seq": "INTEGER", + "_valid_from": "TIMESTAMP", + "_valid_to": "TIMESTAMP", + "_is_current": "BOOLEAN", + "_op": "VARCHAR" + } + }, + { + "name": "wh_transfer_events_history", + "layer": "warehouse", + "type": "scd2-history", + "description": "SCD2 history for transfer_events.", + "owner": "data-platform", + "consumers": ["audit"], + "update_cadence": "near-real-time", + "sla_class": "tier-2", + "tags": ["transfer", "scd2"], + "primary_key": ["transfer_event_id", "_valid_from"], + "schema_extension": { + "_cdc_seq": "INTEGER", + "_valid_from": "TIMESTAMP", + "_valid_to": "TIMESTAMP", + "_is_current": "BOOLEAN", + "_op": "VARCHAR" + } + }, + { + "name": "wh_ledger_entries_history", + "layer": "warehouse", + "type": "scd2-history", + "description": "SCD2 history for ledger_entries — should be a no-op for inserts since ledger entries are append-only at the source, but having SCD2 lets the warehouse trap any unexpected mutation.", + "owner": "data-platform", + "consumers": ["audit", "finance", "treasury"], + "update_cadence": "near-real-time", + "sla_class": "tier-1", + "tags": ["ledger", "scd2", "audit"], + "primary_key": ["ledger_entry_id", "_valid_from"], + "schema_extension": { + "_cdc_seq": "INTEGER", + "_valid_from": "TIMESTAMP", + "_valid_to": "TIMESTAMP", + "_is_current": "BOOLEAN", + "_op": "VARCHAR" + } + } + ] +} diff --git a/submission/wallet-payment-transfer/conftest.py b/submission/wallet-payment-transfer/conftest.py new file mode 100644 index 0000000..a624e7f --- /dev/null +++ b/submission/wallet-payment-transfer/conftest.py @@ -0,0 +1,6 @@ +"""Root conftest — adds submission/wallet-payment-transfer/ to sys.path.""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(__file__)) diff --git a/submission/wallet-payment-transfer/pipeline/__init__.py b/submission/wallet-payment-transfer/pipeline/__init__.py new file mode 100644 index 0000000..6e6f89c --- /dev/null +++ b/submission/wallet-payment-transfer/pipeline/__init__.py @@ -0,0 +1 @@ +"""CDC pipeline — capture, lake, warehouse, schema detection, orchestration.""" diff --git a/submission/wallet-payment-transfer/pipeline/cdc.py b/submission/wallet-payment-transfer/pipeline/cdc.py new file mode 100644 index 0000000..19d5527 --- /dev/null +++ b/submission/wallet-payment-transfer/pipeline/cdc.py @@ -0,0 +1,217 @@ +""" +CDC capture layer. + +What is captured +---------------- +Every insert, update and delete on the source produces a CDCRecord with: + - sequence : monotonically increasing, never reused + - operation : 'insert' | 'update' | 'delete' + - table : name of the source table + - primary_key : the source row's PK value + - before : row image *before* the change (None for inserts) + - after : row image *after* the change (None for deletes) + - captured_at : UTC timestamp at capture time + +`before` and `after` together let downstream consumers run *before/after* +diffs (useful for SCD2 history, audit, and DLQ inspection). + +Production analogue +------------------- +A real Postgres source would hand us this same envelope from logical +replication / Debezium: the LSN ↔ `sequence`, the relational change +record ↔ `before`/`after`, and the commit timestamp ↔ `captured_at`. +The in-process `CDCCapture` class plays the role of the WAL plus the +connector. + +Replay & restart +---------------- +The pipeline checkpoints the **last sequence successfully applied** in +the warehouse (`pipeline.checkpoint`). After a restart we ask the +capture log for `records_since(checkpoint)` and reapply those — at-least +once delivery, but the warehouse merge is **idempotent by sequence** +(see `pipeline.warehouse`) so duplicates do not corrupt state. + +Out-of-order arrival +-------------------- +Records are appended to the log in capture order, so within a single +log they are in order. Across distributed sources, the warehouse keeps +`_cdc_seq` per row and ignores any incoming event with a sequence +**less than or equal** to what is already applied — late events do not +clobber newer state. + +Schema-aware capture +-------------------- +`CDCCapture` is created with a reference to a `SchemaDetector`. When +`require_compatible_schema=True`, calling any of the write methods on a +table with a broken contract raises `SchemaContractViolation` and emits +no records. This is the stop-the-line hook that satisfies the +"fail closed" requirement. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any, Optional + + +VALID_OPERATIONS = frozenset({"insert", "update", "delete"}) + + +class SchemaContractViolation(RuntimeError): + """Raised when CDC capture is attempted against a non-compliant source.""" + + +@dataclass +class CDCRecord: + operation: str + table: str + primary_key: str + before: Optional[dict[str, Any]] = None + after: Optional[dict[str, Any]] = None + captured_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) + sequence: int = 0 + + def __post_init__(self) -> None: + if self.operation not in VALID_OPERATIONS: + raise ValueError( + f"Invalid CDC operation {self.operation!r}. " + f"Must be one of: {sorted(VALID_OPERATIONS)}" + ) + if self.operation == "insert" and self.after is None: + raise ValueError("insert events must carry an 'after' image") + if self.operation == "delete" and self.before is None: + raise ValueError("delete events must carry a 'before' image") + if self.operation == "update" and ( + self.before is None or self.after is None + ): + raise ValueError("update events must carry both 'before' and 'after'") + + @property + def data(self) -> dict[str, Any]: + """ + Convenience accessor for the row image used by the warehouse merge: + for inserts/updates this is the post-image; for deletes it is the + pre-image (so we still know the row's state at delete time). + """ + return self.after if self.after is not None else (self.before or {}) + + +class CDCCapture: + """ + In-process change log used by tests and the demo orchestrator. + + Production analogue: a Debezium / pglogical connector reading the + Postgres WAL and writing each event to a Kafka topic. Each record's + `sequence` would be a Kafka offset or a Postgres LSN. + """ + + def __init__( + self, + schema_detector: Optional["object"] = None, # SchemaDetector + *, + require_compatible_schema: bool = False, + ) -> None: + self._log: list[CDCRecord] = [] + self._seq: int = 0 + self._schema_detector = schema_detector + self._require_compatible_schema = require_compatible_schema + + # ── public write API ───────────────────────────────────────────────────── + + def insert( + self, + table: str, + pk: str, + after: dict[str, Any], + *, + captured_at: Optional[datetime] = None, + ) -> CDCRecord: + self._enforce_schema(table) + return self._record( + "insert", table, pk, before=None, after=after, + captured_at=captured_at, + ) + + def update( + self, + table: str, + pk: str, + before: dict[str, Any], + after: dict[str, Any], + *, + captured_at: Optional[datetime] = None, + ) -> CDCRecord: + self._enforce_schema(table) + return self._record( + "update", table, pk, before=before, after=after, + captured_at=captured_at, + ) + + def delete( + self, + table: str, + pk: str, + before: dict[str, Any], + *, + captured_at: Optional[datetime] = None, + ) -> CDCRecord: + self._enforce_schema(table) + return self._record( + "delete", table, pk, before=before, after=None, + captured_at=captured_at, + ) + + # ── public read / replay API ───────────────────────────────────────────── + + def records_since(self, offset: int = 0) -> list[CDCRecord]: + """Return all records with sequence > offset (checkpoint replay).""" + return [r for r in self._log if r.sequence > offset] + + @property + def latest_sequence(self) -> int: + return self._seq + + @property + def log(self) -> list[CDCRecord]: + return list(self._log) + + # ── internal ───────────────────────────────────────────────────────────── + + def _enforce_schema(self, table: str) -> None: + if not self._require_compatible_schema: + return + detector = self._schema_detector + if detector is None: + return + report = detector.check_table(table) + if report.has_breaking(): + raise SchemaContractViolation( + f"Schema contract violated for {table!r}: " + + "; ".join(d.message for d in report.breaking) + ) + + def _record( + self, + operation: str, + table: str, + pk: str, + *, + before: Optional[dict[str, Any]], + after: Optional[dict[str, Any]], + captured_at: Optional[datetime] = None, + ) -> CDCRecord: + self._seq += 1 + kwargs: dict[str, Any] = { + "operation": operation, + "table": table, + "primary_key": pk, + "before": before, + "after": after, + "sequence": self._seq, + } + if captured_at is not None: + kwargs["captured_at"] = captured_at + rec = CDCRecord(**kwargs) + self._log.append(rec) + return rec diff --git a/submission/wallet-payment-transfer/pipeline/checkpoint.py b/submission/wallet-payment-transfer/pipeline/checkpoint.py new file mode 100644 index 0000000..71a1dc6 --- /dev/null +++ b/submission/wallet-payment-transfer/pipeline/checkpoint.py @@ -0,0 +1,71 @@ +""" +Checkpoint store — persists the last sequence successfully applied. + +Why this matters +---------------- +After an ingestion restart, the pipeline must answer two questions: + 1. Which CDC events have already been written to the lake? + 2. Which CDC events have already been merged into the warehouse? + +Both answers are durable in their respective layers (lake by design, +warehouse by `_cdc_seq` per row), but having a *single point of truth* +for the latest applied sequence simplifies operations: a small +checkpoint table the orchestrator reads at startup. + +Production analogue +------------------- +A Kafka consumer group offset committed atomically with the warehouse +batch, or a row in a small `pipeline_state` table written inside the +same transaction as the warehouse merge. +""" + +from __future__ import annotations + +import duckdb + + +def create_checkpoint_table(conn: duckdb.DuckDBPyConnection) -> None: + conn.execute(""" + CREATE TABLE IF NOT EXISTS pipeline_checkpoints ( + stage VARCHAR PRIMARY KEY, + sequence INTEGER NOT NULL, + updated_at TIMESTAMP NOT NULL + ) + """) + + +def get_checkpoint(conn: duckdb.DuckDBPyConnection, stage: str) -> int: + row = conn.execute( + "SELECT sequence FROM pipeline_checkpoints WHERE stage = ?", + [stage], + ).fetchone() + return int(row[0]) if row else 0 + + +def set_checkpoint( + conn: duckdb.DuckDBPyConnection, + stage: str, + sequence: int, +) -> None: + """ + Upsert the checkpoint for a stage, but never move it backwards. + A backwards-moving checkpoint would silently widen the replay window + and could double-apply events that the warehouse would otherwise + have rejected — better to make the bug loud. + """ + current = get_checkpoint(conn, stage) + if sequence < current: + raise ValueError( + f"refusing to move checkpoint {stage!r} backwards: " + f"current={current}, requested={sequence}" + ) + conn.execute( + """ + INSERT INTO pipeline_checkpoints (stage, sequence, updated_at) + VALUES (?, ?, current_timestamp) + ON CONFLICT (stage) DO UPDATE + SET sequence = EXCLUDED.sequence, + updated_at = EXCLUDED.updated_at + """, + [stage, sequence], + ) diff --git a/submission/wallet-payment-transfer/pipeline/lake.py b/submission/wallet-payment-transfer/pipeline/lake.py new file mode 100644 index 0000000..73750f0 --- /dev/null +++ b/submission/wallet-payment-transfer/pipeline/lake.py @@ -0,0 +1,179 @@ +""" +Lake layer — append-only history of every CDC event. + +Design contract +--------------- +The lake is the system of record for **change history**. Every CDC +record is written exactly once; rows are never updated or deleted. If +the warehouse loses state, we can rebuild it deterministically by +replaying `lake_cdc_events` ordered by `sequence`. + +Production analogue +------------------- +Parquet (or Delta / Iceberg) files on object storage, partitioned by +`table_name` and a `captured_at` date. The PK on `sequence` here plays +the role of the Kafka offset in the lake's `_kafka_offset` column. + +Idempotency +----------- +If the same event is appended twice (after a retried batch, say), the +PK on `sequence` rejects the duplicate at the database level rather +than silently double-writing. The orchestrator catches this and +continues — see `pipeline.orchestrator.replay_to_lake`. + +Schema +------ + sequence INTEGER NOT NULL PRIMARY KEY -- monotonic capture offset + operation VARCHAR NOT NULL -- 'insert' | 'update' | 'delete' + table_name VARCHAR NOT NULL + primary_key VARCHAR NOT NULL + before_image VARCHAR (JSON, nullable) -- null on inserts + after_image VARCHAR (JSON, nullable) -- null on deletes + captured_at TIMESTAMP NOT NULL + +We store the row images as JSON strings rather than typed columns +because schema drift would otherwise be a problem inside the lake +itself; the warehouse models do the typed projection. +""" + +from __future__ import annotations + +import json +from datetime import date, datetime +from decimal import Decimal + +import duckdb + +from pipeline.cdc import CDCRecord + + +def create_lake_table(conn: duckdb.DuckDBPyConnection) -> None: + conn.execute(""" + CREATE TABLE IF NOT EXISTS lake_cdc_events ( + sequence INTEGER NOT NULL PRIMARY KEY, + operation VARCHAR NOT NULL, + table_name VARCHAR NOT NULL, + primary_key VARCHAR NOT NULL, + before_image VARCHAR, + after_image VARCHAR, + captured_at TIMESTAMP NOT NULL + ) + """) + # An index on (table_name, sequence) is what replay queries actually use. + conn.execute(""" + CREATE INDEX IF NOT EXISTS ix_lake_table_sequence + ON lake_cdc_events (table_name, sequence) + """) + + +def append_to_lake( + conn: duckdb.DuckDBPyConnection, + records: list[CDCRecord], +) -> int: + """ + Append CDC records to the lake. Idempotent by `sequence`: duplicates + are silently skipped, so retries do not double-write. + + Returns the number of records actually persisted. + """ + if not records: + return 0 + + written = 0 + for r in records: + before_json = _serialize(r.before) if r.before is not None else None + after_json = _serialize(r.after) if r.after is not None else None + # ON CONFLICT DO NOTHING gives us at-most-once semantics on the lake + # without making the caller responsible for dedup. + cursor = conn.execute( + """ + INSERT INTO lake_cdc_events + (sequence, operation, table_name, primary_key, + before_image, after_image, captured_at) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (sequence) DO NOTHING + """, + [ + r.sequence, + r.operation, + r.table, + r.primary_key, + before_json, + after_json, + r.captured_at, + ], + ) + # DuckDB returns rowcount via .fetchall() differently per version; + # we count by checking whether the row is now present. + if cursor.execute( + "SELECT 1 FROM lake_cdc_events WHERE sequence = ?", [r.sequence] + ).fetchone(): + written += 1 + return written + + +def replay_records( + conn: duckdb.DuckDBPyConnection, + *, + table: str | None = None, + until_sequence: int | None = None, +) -> list[dict]: + """ + Read change history back out of the lake in sequence order. + + Used by: + - the warehouse's "rebuild from scratch" path + - the time-travel scripts + - audit queries + + Filters are optional: pass `table` to restrict to one source table, + `until_sequence` to bound the replay (inclusive). + """ + sql = """ + SELECT sequence, operation, table_name, primary_key, + before_image, after_image, captured_at + FROM lake_cdc_events + """ + where: list[str] = [] + params: list[object] = [] + if table is not None: + where.append("table_name = ?") + params.append(table) + if until_sequence is not None: + where.append("sequence <= ?") + params.append(until_sequence) + if where: + sql += " WHERE " + " AND ".join(where) + sql += " ORDER BY sequence" + rows = conn.execute(sql, params).fetchall() + + out: list[dict] = [] + for seq, op, tbl, pk, before_s, after_s, captured in rows: + out.append({ + "sequence": seq, + "operation": op, + "table": tbl, + "primary_key": pk, + "before": json.loads(before_s) if before_s else None, + "after": json.loads(after_s) if after_s else None, + "captured_at": captured, + }) + return out + + +# ── JSON helpers ───────────────────────────────────────────────────────────── + + +def _serialize(data: dict) -> str: + return json.dumps(data, default=_json_default, sort_keys=True) + + +def _json_default(obj: object) -> object: + if isinstance(obj, datetime): + return obj.isoformat() + if isinstance(obj, date): + return obj.isoformat() + if isinstance(obj, Decimal): + # Decimals serialize as strings to avoid float-precision drift. + return str(obj) + raise TypeError(f"Object of type {type(obj).__name__} is not JSON serializable") diff --git a/submission/wallet-payment-transfer/pipeline/orchestrator.py b/submission/wallet-payment-transfer/pipeline/orchestrator.py new file mode 100644 index 0000000..43e25e2 --- /dev/null +++ b/submission/wallet-payment-transfer/pipeline/orchestrator.py @@ -0,0 +1,108 @@ +""" +Orchestrator — wires the layers together for a single pipeline tick. + +A "tick" is one logical run of: + 1. read CDC events from `capture` since the last checkpoint + 2. write them to the lake (idempotent) + 3. merge them into the warehouse (idempotent) + 4. advance the checkpoint + +Each step is independently safe to retry; the whole tick is therefore +safe to retry on partial failure. If the warehouse write fails before +the checkpoint moves, the next tick re-reads the same window from +capture (or, in production, from the lake) and reapplies — and the +warehouse merge silently dedups by `_cdc_seq`. +""" + +from __future__ import annotations + +import duckdb + +from pipeline.cdc import CDCCapture +from pipeline.checkpoint import ( + create_checkpoint_table, + get_checkpoint, + set_checkpoint, +) +from pipeline.lake import append_to_lake, create_lake_table +from pipeline.schema_detector import SchemaDetector +from pipeline.warehouse import apply_cdc_records, create_warehouse_tables + + +CHECKPOINT_STAGE_LAKE = "lake" +CHECKPOINT_STAGE_WAREHOUSE = "warehouse" + + +def bootstrap(conn: duckdb.DuckDBPyConnection) -> None: + """Create lake, warehouse, and checkpoint tables if missing.""" + create_lake_table(conn) + create_warehouse_tables(conn) + create_checkpoint_table(conn) + + +def run_tick( + conn: duckdb.DuckDBPyConnection, + capture: CDCCapture, + *, + detector: SchemaDetector | None = None, + fail_on_drift: bool = True, +) -> dict: + """ + Run one orchestration tick. Returns a dict of stats: + {'lake_written': N, 'warehouse_applied': M, + 'checkpoint_lake': X, 'checkpoint_warehouse': Y, 'aborted': bool} + + If `fail_on_drift=True` and the schema detector reports a breaking + change on any contracted table, the tick aborts before reading any + new events — this is the stop-the-line behavior. + """ + bootstrap(conn) + + if detector is not None and fail_on_drift: + report = detector.check_all() + if report.has_breaking(): + return { + "aborted": True, + "reason": "schema_contract_violation", + "drift_summary": report.summary(), + "lake_written": 0, + "warehouse_applied": 0, + "checkpoint_lake": get_checkpoint(conn, CHECKPOINT_STAGE_LAKE), + "checkpoint_warehouse": get_checkpoint( + conn, CHECKPOINT_STAGE_WAREHOUSE + ), + } + + last_lake = get_checkpoint(conn, CHECKPOINT_STAGE_LAKE) + last_wh = get_checkpoint(conn, CHECKPOINT_STAGE_WAREHOUSE) + floor = min(last_lake, last_wh) + + pending = capture.records_since(floor) + if not pending: + return { + "aborted": False, + "lake_written": 0, + "warehouse_applied": 0, + "checkpoint_lake": last_lake, + "checkpoint_warehouse": last_wh, + } + + # Lake first — durable storage of the raw event. + lake_pending = [r for r in pending if r.sequence > last_lake] + lake_written = append_to_lake(conn, lake_pending) + new_lake_seq = pending[-1].sequence + set_checkpoint(conn, CHECKPOINT_STAGE_LAKE, new_lake_seq) + + # Warehouse next — typed merge. + wh_pending = [r for r in pending if r.sequence > last_wh] + wh_applied = apply_cdc_records(conn, wh_pending) + new_wh_seq = pending[-1].sequence + set_checkpoint(conn, CHECKPOINT_STAGE_WAREHOUSE, new_wh_seq) + + return { + "aborted": False, + "lake_written": lake_written, + "warehouse_applied": wh_applied, + "checkpoint_lake": new_lake_seq, + "checkpoint_warehouse": new_wh_seq, + } diff --git a/submission/wallet-payment-transfer/pipeline/schema_detector.py b/submission/wallet-payment-transfer/pipeline/schema_detector.py new file mode 100644 index 0000000..805ae00 --- /dev/null +++ b/submission/wallet-payment-transfer/pipeline/schema_detector.py @@ -0,0 +1,324 @@ +""" +Schema-drift detector — implements the "stop the line" behavior. + +Detection rules +--------------- +For each table contract we compare the runtime source schema against +the contract and produce a list of `Drift` records. Each drift carries +a `severity`: + + BREAKING : the pipeline must stop. The contract change can silently + corrupt the warehouse if we keep ingesting. + WARN : the pipeline can continue, but a human should know. + +Breaking changes (fail closed): + - column dropped (existing references are now orphaned) + - column renamed (modelled as drop + add of an unknown col) + - column type changed (cast may silently round, truncate, error) + - column nullability tightened + nullable contract -> NOT NULL source + means a row that previously satisfied the contract may now fail + - enum domain shrunk (a row that was valid before now is not) + - primary key changed (replay/upsert depends on the original PK) + - referenced parent table missing (FK broken) + +Warning-only changes: + - new nullable column added (additive, downstream just ignores it) + - new enum value added (downstream consumers can choose to allow) + +Output +------ +The detector returns a `DriftReport` per table and a top-level +`PipelineDriftReport` for the whole source. The CI script +`scripts/check_schema_contracts.py` exits non-zero whenever any +breaking drift is present. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum +from typing import Optional + +import duckdb + +from source.contracts import CONTRACTS, TableContract + + +class Severity(Enum): + BREAKING = "BREAKING" + WARN = "WARN" + + +@dataclass(frozen=True) +class Drift: + table: str + column: Optional[str] + rule: str + severity: Severity + message: str + + +@dataclass +class DriftReport: + table: str + drifts: list[Drift] = field(default_factory=list) + + @property + def breaking(self) -> list[Drift]: + return [d for d in self.drifts if d.severity is Severity.BREAKING] + + @property + def warnings(self) -> list[Drift]: + return [d for d in self.drifts if d.severity is Severity.WARN] + + def has_breaking(self) -> bool: + return any(d.severity is Severity.BREAKING for d in self.drifts) + + +@dataclass +class PipelineDriftReport: + by_table: dict[str, DriftReport] = field(default_factory=dict) + + @property + def breaking(self) -> list[Drift]: + return [d for r in self.by_table.values() for d in r.breaking] + + @property + def warnings(self) -> list[Drift]: + return [d for r in self.by_table.values() for d in r.warnings] + + def has_breaking(self) -> bool: + return any(r.has_breaking() for r in self.by_table.values()) + + def summary(self) -> str: + lines: list[str] = [] + for table, report in self.by_table.items(): + for d in report.drifts: + marker = "x" if d.severity is Severity.BREAKING else "!" + lines.append(f" [{marker}] {table}.{d.column}: {d.rule} — {d.message}") + return "\n".join(lines) if lines else " (no drift)" + + +# ── runtime introspection ──────────────────────────────────────────────────── + + +def _normalize_type(t: str) -> str: + """ + DuckDB DESCRIBE returns types like 'VARCHAR' / 'DECIMAL(18,4)' / + 'TIMESTAMP' which already match our contract style. We strip + whitespace and uppercase to be tolerant. + """ + return t.strip().upper().replace(" ", "") + + +def _runtime_columns( + conn: duckdb.DuckDBPyConnection, table: str +) -> list[tuple[str, str, bool]]: + """ + Return [(name, type, nullable), ...] from DuckDB's information_schema. + + information_schema.columns is more reliable than DESCRIBE for + nullability, since DESCRIBE in DuckDB does not always expose it. + """ + rows = conn.execute( + """ + SELECT column_name, data_type, is_nullable + FROM information_schema.columns + WHERE table_name = ? + ORDER BY ordinal_position + """, + [table], + ).fetchall() + return [(r[0], _normalize_type(r[1]), r[2] == "YES") for r in rows] + + +def _runtime_pk(conn: duckdb.DuckDBPyConnection, table: str) -> list[str]: + """ + Best-effort PK lookup. DuckDB exposes constraints via duckdb_constraints(); + we scan for the PRIMARY KEY entry on the table. + """ + try: + rows = conn.execute( + """ + SELECT constraint_column_names + FROM duckdb_constraints() + WHERE table_name = ? AND constraint_type = 'PRIMARY KEY' + """, + [table], + ).fetchall() + except Exception: + return [] + if not rows: + return [] + cols = rows[0][0] + if isinstance(cols, list): + return [str(c) for c in cols] + if isinstance(cols, str): + return [c.strip() for c in cols.split(",")] + return [] + + +def _table_exists(conn: duckdb.DuckDBPyConnection, table: str) -> bool: + return bool( + conn.execute( + """ + SELECT 1 FROM information_schema.tables WHERE table_name = ? + """, + [table], + ).fetchone() + ) + + +# ── core checker ───────────────────────────────────────────────────────────── + + +@dataclass +class SchemaDetector: + """ + Runs schema contract checks against a live DuckDB connection. + + Used both by the CI script (`scripts/check_schema_contracts.py`) + and at CDC capture time (`pipeline.cdc.CDCCapture` consults it + before every event when `require_compatible_schema=True`). + """ + + conn: duckdb.DuckDBPyConnection + + def check_table(self, table: str) -> DriftReport: + contract = CONTRACTS[table] + report = DriftReport(table=table) + + if not _table_exists(self.conn, table): + report.drifts.append( + Drift( + table=table, + column=None, + rule="missing_table", + severity=Severity.BREAKING, + message="source table does not exist", + ) + ) + return report + + runtime_cols = _runtime_columns(self.conn, table) + runtime_by_name = {c[0]: c for c in runtime_cols} + contract_by_name = {c.name: c for c in contract.columns} + + # 1) dropped columns + for col in contract.columns: + if col.name not in runtime_by_name: + report.drifts.append( + Drift( + table, col.name, + "column_dropped", + Severity.BREAKING, + "contract column is missing from the source", + ) + ) + continue + + r_name, r_type, r_nullable = runtime_by_name[col.name] + + # 2) type change + if not _types_compatible(col.type, r_type): + report.drifts.append( + Drift( + table, col.name, + "type_changed", + Severity.BREAKING, + f"contract type {col.type!r} != source type {r_type!r}", + ) + ) + + # 3) nullability tightened + if col.nullable and not r_nullable: + report.drifts.append( + Drift( + table, col.name, + "nullability_tightened", + Severity.BREAKING, + "contract allows NULL but source forbids it — " + "rows previously valid may now fail", + ) + ) + + # 4) added columns (warning only, additive) + for r_name, _, r_nullable in runtime_cols: + if r_name not in contract_by_name: + if r_nullable: + report.drifts.append( + Drift( + table, r_name, + "column_added", + Severity.WARN, + "new nullable column appeared on the source", + ) + ) + else: + report.drifts.append( + Drift( + table, r_name, + "non_nullable_column_added", + Severity.BREAKING, + "new NOT-NULL column appeared on the source — " + "inserts from the contract perspective will fail", + ) + ) + + # 5) primary key changed + runtime_pk = _runtime_pk(self.conn, table) + if runtime_pk and runtime_pk != [contract.primary_key]: + report.drifts.append( + Drift( + table, ",".join(runtime_pk), + "primary_key_changed", + Severity.BREAKING, + f"contract PK is {contract.primary_key!r} " + f"but source PK is {runtime_pk!r}", + ) + ) + + # 6) enum domain shrinkage — checked by sampling actual rows. + for col_name, allowed in contract.enum_domains: + try: + rows = self.conn.execute( + f"SELECT DISTINCT {col_name} FROM {table} " + f"WHERE {col_name} IS NOT NULL" + ).fetchall() + except Exception: + continue + present = {r[0] for r in rows} + unexpected = present - set(allowed) + if unexpected: + # New value the contract did not list — additive, warn. + report.drifts.append( + Drift( + table, col_name, + "enum_value_added", + Severity.WARN, + f"unexpected values present: {sorted(unexpected)!r}", + ) + ) + # We cannot tell from data alone whether the source has *removed* + # an allowed value. The warehouse-side parity test + # `tests/test_data_quality.py::test_enum_domains_match_contract` + # checks the *materialized* warehouse so a removed value would + # show up as a contract drift in subsequent runs. + + return report + + def check_all(self) -> PipelineDriftReport: + out = PipelineDriftReport() + for table in CONTRACTS: + out.by_table[table] = self.check_table(table) + return out + + +def _types_compatible(contract_type: str, runtime_type: str) -> bool: + """ + Type equality is required. We are intentionally strict because silent + casting between DECIMAL precisions or between VARCHAR/INT is exactly + the kind of drift that produces hard-to-debug correctness bugs. + """ + return _normalize_type(contract_type) == _normalize_type(runtime_type) diff --git a/submission/wallet-payment-transfer/pipeline/warehouse.py b/submission/wallet-payment-transfer/pipeline/warehouse.py new file mode 100644 index 0000000..768b9fd --- /dev/null +++ b/submission/wallet-payment-transfer/pipeline/warehouse.py @@ -0,0 +1,367 @@ +""" +Warehouse layer — current-state snapshot + SCD2 history. + +Three tables per source table +----------------------------- + wh_
: latest known state, soft-deleted with _deleted=true + wh_
_history : SCD2 — every revision has _valid_from / _valid_to / + _is_current / _op so we can answer "what did this + row look like at time T?" without rescanning the lake + (lake_cdc_events keeps the raw events; the SCD2 table is a *materialized + view of the lake* that downstream consumers can query directly) + +Idempotent merge +---------------- +Each warehouse row carries `_cdc_seq`, the sequence of the last CDC +event that touched it. The merge applies an event only when its +sequence is **strictly greater** than the row's `_cdc_seq`. This makes +the merge: + - safe under retry / replay (duplicates are no-ops) + - safe under out-of-order arrival (older events do not clobber newer + state) + +SCD2 mechanics +-------------- +For an insert: open a new row with `_valid_from = captured_at`, +`_valid_to = NULL`, `_is_current = true`, `_op = 'insert'`. + +For an update or delete on an existing row: + 1. close the current row by setting `_valid_to = captured_at`, + `_is_current = false` + 2. for updates, open a new current row with the post-image + 3. for deletes, do not open a new row but mark the *current* row's + replacement (none) as deleted in `wh_
` via `_deleted=true` + +Time travel +----------- +Use `state_at(conn, table, ts)` to reconstruct what the table looked +like at any timestamp. The query selects rows from `wh_
_history` +where `_valid_from <= ts < COALESCE(_valid_to, '9999-12-31')` and +`_op != 'delete'`. + +Production analogue +------------------- +The warehouse is what BigQuery / Snowflake / Redshift would expose: +typed, queryable, current state for analytics, with the SCD2 history +table for finance / audit / restore. The merge here is the same +MERGE-by-PK pattern you would write in those systems, just expressed +in DuckDB SQL. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +from decimal import Decimal +from typing import Any, Iterable + +import duckdb + +from pipeline.cdc import CDCRecord +from source.contracts import CONTRACTS + + +# Source tables we mirror in the warehouse, in dependency order. +WAREHOUSED_TABLES: tuple[str, ...] = ( + "customers", + "payment_methods", + "wallets", + "transfers", + "transfer_events", + "ledger_entries", +) + + +# Map source SQL types to warehouse SQL types. Identical for our domain; +# the indirection lets us coerce on the way in if a source type is wider +# than what we want to expose to BI. +_TYPE_MAP: dict[str, str] = { + "VARCHAR": "VARCHAR", + "BOOLEAN": "BOOLEAN", + "DATE": "DATE", + "TIMESTAMP": "TIMESTAMP", + "DECIMAL(18,4)": "DECIMAL(18,4)", +} + + +@dataclass +class _ColumnInfo: + name: str + sql_type: str + nullable: bool + + +def _columns_for(table: str) -> list[_ColumnInfo]: + contract = CONTRACTS[table] + return [ + _ColumnInfo(c.name, _TYPE_MAP[c.type], c.nullable) for c in contract.columns + ] + + +def _wh_table(table: str) -> str: + return f"wh_{table}" + + +def _hist_table(table: str) -> str: + return f"wh_{table}_history" + + +# ── DDL ────────────────────────────────────────────────────────────────────── + + +def create_warehouse_tables(conn: duckdb.DuckDBPyConnection) -> None: + for table in WAREHOUSED_TABLES: + cols = _columns_for(table) + pk = CONTRACTS[table].primary_key + + # Current-state table. + col_defs = ", ".join( + f"{c.name} {c.sql_type}{'' if c.nullable else ' NOT NULL'}" + if c.name == pk + else f"{c.name} {c.sql_type}" + for c in cols + ) + conn.execute(f""" + CREATE TABLE IF NOT EXISTS {_wh_table(table)} ( + {col_defs}, + _cdc_seq INTEGER NOT NULL, + _deleted BOOLEAN NOT NULL DEFAULT FALSE, + PRIMARY KEY ({pk}) + ) + """) + + # SCD2 history table — same columns, no PK on business key, + # surrogate PK on (business_key, _valid_from). + hist_cols = ", ".join(f"{c.name} {c.sql_type}" for c in cols) + conn.execute(f""" + CREATE TABLE IF NOT EXISTS {_hist_table(table)} ( + {hist_cols}, + _cdc_seq INTEGER NOT NULL, + _valid_from TIMESTAMP NOT NULL, + _valid_to TIMESTAMP, + _is_current BOOLEAN NOT NULL, + _op VARCHAR NOT NULL + CHECK (_op IN ('insert', 'update', 'delete')), + PRIMARY KEY ({pk}, _valid_from) + ) + """) + + +# ── apply CDC records ──────────────────────────────────────────────────────── + + +def apply_cdc_records( + conn: duckdb.DuckDBPyConnection, + records: Iterable[CDCRecord], +) -> int: + """ + Apply CDC records to the current-state and SCD2 history tables. + + Returns the number of records *actually applied* (skipped duplicates + and out-of-order events are not counted). + """ + sorted_records = sorted(records, key=lambda r: r.sequence) + applied = 0 + for r in sorted_records: + if r.table not in WAREHOUSED_TABLES: + continue + if _apply_one(conn, r): + applied += 1 + return applied + + +def _apply_one(conn: duckdb.DuckDBPyConnection, r: CDCRecord) -> bool: + pk_col = CONTRACTS[r.table].primary_key + cur = _wh_table(r.table) + hist = _hist_table(r.table) + + existing = conn.execute( + f"SELECT _cdc_seq, _deleted FROM {cur} WHERE {pk_col} = ?", + [r.primary_key], + ).fetchone() + + # Idempotent / out-of-order guard. + if existing is not None and existing[0] >= r.sequence: + return False + + # Close any open SCD2 row for this PK. + conn.execute( + f""" + UPDATE {hist} + SET _valid_to = ?, + _is_current = FALSE + WHERE {pk_col} = ? + AND _is_current = TRUE + """, + [r.captured_at, r.primary_key], + ) + + if r.operation == "delete": + # Update current-state to mark deleted. + if existing is not None: + conn.execute( + f"UPDATE {cur} SET _deleted = TRUE, _cdc_seq = ? " + f"WHERE {pk_col} = ?", + [r.sequence, r.primary_key], + ) + # Append a closed delete row to history (with the pre-image). + _insert_history( + conn, + r.table, + r, + data=r.before or {}, + valid_from=r.captured_at, + valid_to=r.captured_at, + is_current=False, + op="delete", + ) + return True + + # insert or update + data = r.after or {} + cols = _columns_for(r.table) + values = [data.get(c.name) for c in cols] + + if existing is None: + # INSERT into current-state. + col_list = ", ".join(c.name for c in cols) + placeholders = ", ".join(["?"] * len(cols)) + conn.execute( + f""" + INSERT INTO {cur} ({col_list}, _cdc_seq, _deleted) + VALUES ({placeholders}, ?, FALSE) + """, + values + [r.sequence], + ) + else: + # UPDATE the current-state row. + set_clause = ", ".join(f"{c.name} = ?" for c in cols) + conn.execute( + f""" + UPDATE {cur} + SET {set_clause}, _cdc_seq = ?, _deleted = FALSE + WHERE {pk_col} = ? + """, + values + [r.sequence, r.primary_key], + ) + + # Open new current SCD2 row. + _insert_history( + conn, + r.table, + r, + data=data, + valid_from=r.captured_at, + valid_to=None, + is_current=True, + op=r.operation, + ) + return True + + +def _insert_history( + conn: duckdb.DuckDBPyConnection, + table: str, + record: CDCRecord, + *, + data: dict[str, Any], + valid_from: datetime, + valid_to: datetime | None, + is_current: bool, + op: str, +) -> None: + cols = _columns_for(table) + values = [data.get(c.name) for c in cols] + col_list = ", ".join(c.name for c in cols) + placeholders = ", ".join(["?"] * len(cols)) + conn.execute( + f""" + INSERT INTO {_hist_table(table)} + ({col_list}, _cdc_seq, _valid_from, _valid_to, _is_current, _op) + VALUES ({placeholders}, ?, ?, ?, ?, ?) + ON CONFLICT ({CONTRACTS[table].primary_key}, _valid_from) DO NOTHING + """, + values + [record.sequence, valid_from, valid_to, is_current, op], + ) + + +# ── time travel ────────────────────────────────────────────────────────────── + + +def state_at( + conn: duckdb.DuckDBPyConnection, + table: str, + ts: datetime, +) -> list[dict]: + """ + Reconstruct the state of `table` at timestamp `ts` from the SCD2 history. + + Returns the rows that were *valid at ts* and not marked deleted at that + point — semantically equivalent to running `SELECT * FROM
` on + the source as of `ts`. + """ + cols = _columns_for(table) + col_list = ", ".join(c.name for c in cols) + sql = f""" + SELECT {col_list} + FROM {_hist_table(table)} + WHERE _valid_from <= ? + AND COALESCE(_valid_to, TIMESTAMP '9999-12-31 23:59:59') > ? + AND _op <> 'delete' + """ + rows = conn.execute(sql, [ts, ts]).fetchall() + return [ + {c.name: _coerce(value) for c, value in zip(cols, row)} + for row in rows + ] + + +def _coerce(v: object) -> object: + """Bring DuckDB result values back to standard Python types for callers.""" + if isinstance(v, Decimal): + return v + return v + + +# ── restore ────────────────────────────────────────────────────────────────── + + +def restore_current_state_to( + conn: duckdb.DuckDBPyConnection, + table: str, + ts: datetime, +) -> int: + """ + Replace `wh_
` rows with the snapshot from `state_at(table, ts)`. + + This is the operational hook for "roll the warehouse back to time T": + history is preserved (we never touch `wh_
_history`), but the + *current state* table is overwritten from the historical snapshot. + + Returns the number of rows in the restored snapshot. + """ + snapshot = state_at(conn, table, ts) + pk_col = CONTRACTS[table].primary_key + cols = _columns_for(table) + + conn.execute(f"DELETE FROM {_wh_table(table)}") + + if not snapshot: + return 0 + + col_list = ", ".join(c.name for c in cols) + placeholders = ", ".join(["?"] * len(cols)) + for row in snapshot: + values = [row.get(c.name) for c in cols] + conn.execute( + f""" + INSERT INTO {_wh_table(table)} + ({col_list}, _cdc_seq, _deleted) + VALUES ({placeholders}, 0, FALSE) + """, + values, + ) + # Note: _cdc_seq is reset to 0 to signal "restored, ready for new events". + # The next CDC record (which will have sequence > 0) will reapply cleanly. + _ = pk_col # quiet linter — we may use it for a future assertion + return len(snapshot) diff --git a/submission/wallet-payment-transfer/requirements.txt b/submission/wallet-payment-transfer/requirements.txt new file mode 100644 index 0000000..677a29a --- /dev/null +++ b/submission/wallet-payment-transfer/requirements.txt @@ -0,0 +1,2 @@ +duckdb>=1.0.0 +pytest>=7.0.0 diff --git a/submission/wallet-payment-transfer/scripts/check_schema_contracts.py b/submission/wallet-payment-transfer/scripts/check_schema_contracts.py new file mode 100644 index 0000000..aee7c41 --- /dev/null +++ b/submission/wallet-payment-transfer/scripts/check_schema_contracts.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +""" +check_schema_contracts.py + +CI entry point for the schema contract check (`schema-contract-check`). + +What it does +------------ +1. Spins up an in-memory DuckDB +2. Creates the source tables exactly as defined in source/models.py +3. Runs the SchemaDetector against every contract +4. Exits non-zero if any BREAKING drift is found +5. Prints WARN drifts but does not fail the build for them + +This is the same code path the orchestrator uses to fail the pipeline +closed at runtime — keeping the two consistent prevents "passes in CI, +fails in prod" surprises. +""" + +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import duckdb + +from pipeline.schema_detector import SchemaDetector +from source.models import create_source_tables + + +def main() -> int: + conn = duckdb.connect(":memory:") + create_source_tables(conn) + + report = SchemaDetector(conn).check_all() + + if report.warnings: + print("Schema contract WARNINGS:") + for d in report.warnings: + print(f" ! {d.table}.{d.column}: {d.rule} — {d.message}") + + if report.breaking: + print("Schema contract BREAKING violations:") + for d in report.breaking: + print(f" x {d.table}.{d.column}: {d.rule} — {d.message}") + return 1 + + n_tables = len(report.by_table) + print(f"All schema contracts passed ({n_tables} tables checked).") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/submission/wallet-payment-transfer/scripts/demo_schema_change.py b/submission/wallet-payment-transfer/scripts/demo_schema_change.py new file mode 100644 index 0000000..36a6e12 --- /dev/null +++ b/submission/wallet-payment-transfer/scripts/demo_schema_change.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +""" +demo_schema_change.py + +Demonstrates the stop-the-line behavior when the source schema drifts +incompatibly from the contract. + +Scenario +-------- + 1. Bootstrap the source as designed. + 2. Run a normal CDC tick — succeeds. + 3. Drop a contract column from the source. + 4. Run another tick — pipeline aborts with a clear failure signal. +""" + +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import duckdb + +from pipeline.cdc import CDCCapture, SchemaContractViolation +from pipeline.orchestrator import bootstrap, run_tick +from pipeline.schema_detector import SchemaDetector +from source.models import create_source_tables +from source.seed import build_seed_rows + + +def main() -> int: + conn = duckdb.connect(":memory:") + create_source_tables(conn) + bootstrap(conn) + capture = CDCCapture() + detector = SchemaDetector(conn) + + print("== first tick (clean) ==") + for row in build_seed_rows(): + capture.insert(row.table, row.pk, row.data) + stats = run_tick(conn, capture, detector=detector) + print(f" stats: {stats}") + + print("== drift the source: drop customers.email ==") + # Drop FK-dependent tables first so the ALTER works in DuckDB. + conn.execute("DROP TABLE IF EXISTS ledger_entries") + conn.execute("DROP TABLE IF EXISTS transfer_events") + conn.execute("DROP TABLE IF EXISTS transfers") + conn.execute("DROP TABLE IF EXISTS payment_methods") + conn.execute("DROP TABLE IF EXISTS wallets") + conn.execute("ALTER TABLE customers DROP COLUMN email") + + print("== second tick (must abort) ==") + stats = run_tick(conn, capture, detector=detector) + if not stats.get("aborted"): + print(" FAIL: pipeline did not abort on incompatible schema change") + return 1 + print(" pipeline correctly aborted with reason:") + print(f" {stats.get('reason')}") + print(" drift summary:") + print(stats.get("drift_summary")) + + print("== capture-level stop-the-line ==") + capture_strict = CDCCapture(detector, require_compatible_schema=True) + try: + capture_strict.insert( + "customers", "c_new", + {"customer_id": "c_new", "full_name": "X"}, + ) + except SchemaContractViolation as exc: + print(f" CDCCapture refused write: {exc}") + return 0 + print(" FAIL: CDCCapture should have raised SchemaContractViolation") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/submission/wallet-payment-transfer/scripts/demo_time_travel.py b/submission/wallet-payment-transfer/scripts/demo_time_travel.py new file mode 100644 index 0000000..9f0fc83 --- /dev/null +++ b/submission/wallet-payment-transfer/scripts/demo_time_travel.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +""" +demo_time_travel.py + +Demonstrates the warehouse's point-in-time reconstruction feature. + +Scenario +-------- + t0 : insert wallet w_demo at balance 100 + t1 : update balance to 60 + t2 : update balance to 0 (closed) + +The script then asks the warehouse what the wallet looked like at each +of those times and prints the answer. Reads come from +`wh_wallets_history` only — the lake is not touched. +""" + +from __future__ import annotations + +import os +import sys +from datetime import datetime, timedelta, timezone +from decimal import Decimal + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import duckdb + +from pipeline.cdc import CDCCapture +from pipeline.orchestrator import bootstrap, run_tick +from pipeline.warehouse import state_at +from source.models import create_source_tables + + +def _t(seconds: int) -> datetime: + base = datetime(2026, 6, 1, 9, 0, tzinfo=timezone.utc) + return base + timedelta(seconds=seconds) + + +def main() -> int: + conn = duckdb.connect(":memory:") + create_source_tables(conn) + bootstrap(conn) + capture = CDCCapture() + + # Customer must exist for FK. + cust_row = { + "customer_id": "c_demo", + "full_name": "Demo Customer", + "email": "demo@example.invalid", + "country_code": "US", + "status": "active", + "registration_date": _t(0).date(), + "created_at": _t(0), + "updated_at": _t(0), + } + capture.insert("customers", "c_demo", cust_row, captured_at=_t(0)) + + # Three revisions of one wallet: + w0 = { + "wallet_id": "w_demo", + "customer_id": "c_demo", + "currency": "USD", + "balance": Decimal("100.0000"), + "status": "active", + "created_at": _t(0), + "updated_at": _t(0), + } + capture.insert("wallets", "w_demo", w0, captured_at=_t(0)) + + w1 = dict(w0, balance=Decimal("60.0000"), updated_at=_t(60)) + capture.update("wallets", "w_demo", before=w0, after=w1, captured_at=_t(60)) + + w2 = dict(w1, balance=Decimal("0.0000"), status="closed", updated_at=_t(120)) + capture.update("wallets", "w_demo", before=w1, after=w2, captured_at=_t(120)) + + run_tick(conn, capture) + + print("== state_at queries ==") + for label, ts in [ + ("at t0 (just after insert)", _t(0)), + ("at t30 (between insert and first update)", _t(30)), + ("at t60 (just after first update)", _t(60)), + ("at t90 (between updates)", _t(90)), + ("at t150 (after close)", _t(150)), + ]: + rows = state_at(conn, "wallets", ts) + balance = rows[0]["balance"] if rows else None + status = rows[0]["status"] if rows else None + print(f" {label}: balance={balance}, status={status}") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/submission/wallet-payment-transfer/scripts/run_data_quality_checks.py b/submission/wallet-payment-transfer/scripts/run_data_quality_checks.py new file mode 100644 index 0000000..379fb04 --- /dev/null +++ b/submission/wallet-payment-transfer/scripts/run_data_quality_checks.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +""" +run_data_quality_checks.py + +CI entry point for the data-quality validation parity check +(`data-quality-check`). + +The script: + 1. bootstraps an in-memory DuckDB, + 2. seeds it with the canonical wallet domain seed, + 3. runs every SYSTEM and BUSINESS check defined in this module, + 4. prints the rule name + table when a check fails, + 5. exits non-zero on any failure. + +Each rule below is traceable to a source-side invariant (see the +docstrings in source/models.py and source/contracts.py) — that is the +"validation parity" the assignment asks for. +""" + +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from dataclasses import dataclass +from decimal import Decimal + +import duckdb + +from pipeline.cdc import CDCCapture +from pipeline.orchestrator import bootstrap, run_tick +from source.contracts import CONTRACTS +from source.models import ALLOWED_TRANSFER_TRANSITIONS, create_source_tables +from source.seed import build_seed_rows + + +# ── rule registry ──────────────────────────────────────────────────────────── + + +@dataclass(frozen=True) +class Rule: + rule_id: str + category: str # 'system' | 'business' | 'freshness' + description: str + sql: str # query that returns 0 rows when the rule passes + + +def _system_rules() -> list[Rule]: + rules: list[Rule] = [] + + # PK uniqueness + for table, contract in CONTRACTS.items(): + wh = f"wh_{table}" + pk = contract.primary_key + rules.append(Rule( + rule_id=f"sys.pk_unique.{table}", + category="system", + description=f"{wh}.{pk} must be unique", + sql=f""" + SELECT {pk}, COUNT(*) AS n + FROM {wh} + GROUP BY {pk} + HAVING COUNT(*) > 1 + """, + )) + + # NOT NULL parity + for table, contract in CONTRACTS.items(): + wh = f"wh_{table}" + for col in contract.columns: + if col.nullable: + continue + rules.append(Rule( + rule_id=f"sys.not_null.{table}.{col.name}", + category="system", + description=f"{wh}.{col.name} must not be null", + sql=f"SELECT 1 FROM {wh} " + f"WHERE {col.name} IS NULL AND _deleted = FALSE", + )) + + # Enum domain parity (only on currently-present rows) + for table, contract in CONTRACTS.items(): + wh = f"wh_{table}" + for col, allowed in contract.enum_domains: + allowed_sql = ", ".join(f"'{v}'" for v in sorted(allowed)) + rules.append(Rule( + rule_id=f"sys.enum.{table}.{col}", + category="system", + description=f"{wh}.{col} must be one of {sorted(allowed)}", + sql=f"SELECT 1 FROM {wh} " + f"WHERE _deleted = FALSE " + f" AND {col} IS NOT NULL " + f" AND {col} NOT IN ({allowed_sql})", + )) + + # Referential integrity for present (non-deleted) rows + for table, contract in CONTRACTS.items(): + wh = f"wh_{table}" + for fk_col, parent, parent_col in contract.foreign_keys: + parent_wh = f"wh_{parent}" + rules.append(Rule( + rule_id=f"sys.fk.{table}.{fk_col}", + category="system", + description=( + f"{wh}.{fk_col} must reference an existing " + f"{parent_wh}.{parent_col}" + ), + sql=f""" + SELECT 1 + FROM {wh} c + LEFT JOIN {parent_wh} p + ON c.{fk_col} = p.{parent_col} + AND p._deleted = FALSE + WHERE c._deleted = FALSE + AND c.{fk_col} IS NOT NULL + AND p.{parent_col} IS NULL + """, + )) + + return rules + + +def _business_rules() -> list[Rule]: + return [ + Rule( + "biz.wallet.balance_non_negative", + "business", + "wh_wallets.balance must be >= 0", + "SELECT 1 FROM wh_wallets " + "WHERE _deleted = FALSE AND balance < 0", + ), + Rule( + "biz.transfer.amount_positive", + "business", + "wh_transfers.amount must be > 0", + "SELECT 1 FROM wh_transfers " + "WHERE _deleted = FALSE AND amount <= 0", + ), + Rule( + "biz.transfer.no_self_transfer", + "business", + "wh_transfers.from_wallet_id must differ from to_wallet_id", + "SELECT 1 FROM wh_transfers " + "WHERE _deleted = FALSE AND from_wallet_id = to_wallet_id", + ), + Rule( + "biz.transfer.settled_at_after_created", + "business", + "wh_transfers.settled_at must be >= created_at when present", + "SELECT 1 FROM wh_transfers " + "WHERE _deleted = FALSE " + " AND settled_at IS NOT NULL " + " AND settled_at < created_at", + ), + Rule( + "biz.transfer.settled_implies_settled_at", + "business", + "wh_transfers.settled_at must be set whenever status = 'settled'", + "SELECT 1 FROM wh_transfers " + "WHERE _deleted = FALSE " + " AND status = 'settled' AND settled_at IS NULL", + ), + Rule( + "biz.transfer.failed_implies_reason", + "business", + "wh_transfers.failure_reason must be set whenever status = 'failed'", + "SELECT 1 FROM wh_transfers " + "WHERE _deleted = FALSE " + " AND status = 'failed' AND failure_reason IS NULL", + ), + Rule( + "biz.ledger.signed_amount_consistent", + "business", + "ledger_entries.signed_amount must equal +/- amount per direction", + "SELECT 1 FROM wh_ledger_entries " + "WHERE _deleted = FALSE " + " AND ((direction = 'credit' AND signed_amount <> amount) " + " OR (direction = 'debit' AND signed_amount <> -amount))", + ), + Rule( + "biz.ledger.balance_after_non_negative", + "business", + "ledger_entries.balance_after must be >= 0", + "SELECT 1 FROM wh_ledger_entries " + "WHERE _deleted = FALSE AND balance_after < 0", + ), + # Monetary integrity — ledger sums per wallet must equal wallet balance. + Rule( + "biz.ledger.reconciles_to_wallet_balance", + "business", + "SUM(ledger.signed_amount) per wallet must equal wallet.balance", + """ + WITH ledger_sum AS ( + SELECT wallet_id, SUM(signed_amount) AS s + FROM wh_ledger_entries + WHERE _deleted = FALSE + GROUP BY wallet_id + ) + SELECT 1 + FROM wh_wallets w + LEFT JOIN ledger_sum l ON l.wallet_id = w.wallet_id + WHERE w._deleted = FALSE + AND COALESCE(l.s, 0) <> w.balance + """, + ), + # Status transitions must follow the documented state machine. + Rule( + "biz.transfer_event.allowed_transition", + "business", + "transfer_events.from_status -> to_status must be allowed " + "(or from_status NULL for the first event)", + f""" + SELECT 1 FROM wh_transfer_events + WHERE _deleted = FALSE + AND from_status IS NOT NULL + AND (from_status, to_status) NOT IN ( + {", ".join(f"('{f}', '{t}')" for f, t in + sorted(ALLOWED_TRANSFER_TRANSITIONS))} + ) + """, + ), + ] + + +def all_rules() -> list[Rule]: + return _system_rules() + _business_rules() + + +# ── seed + run ─────────────────────────────────────────────────────────────── + + +def _seed_pipeline(conn: duckdb.DuckDBPyConnection) -> CDCCapture: + create_source_tables(conn) + bootstrap(conn) + capture = CDCCapture() + for row in build_seed_rows(): + capture.insert(row.table, row.pk, row.data) + run_tick(conn, capture) + return capture + + +def run_checks(conn: duckdb.DuckDBPyConnection) -> list[tuple[Rule, int]]: + """Return list of (rule, n_failing_rows) for any rule that fails.""" + failures: list[tuple[Rule, int]] = [] + for rule in all_rules(): + try: + n = len(conn.execute(rule.sql).fetchall()) + except Exception as exc: + failures.append((rule, -1)) + print(f" ! rule errored: {rule.rule_id} — {exc}") + continue + if n > 0: + failures.append((rule, n)) + return failures + + +def main() -> int: + conn = duckdb.connect(":memory:") + _seed_pipeline(conn) + + failures = run_checks(conn) + + if failures: + print("Data quality failures:") + for rule, n in failures: + tag = "?" if n < 0 else f"{n} rows" + print(f" x [{rule.category}] {rule.rule_id}: {rule.description} " + f"({tag})") + return 1 + + n_rules = len(all_rules()) + print(f"All data quality checks passed ({n_rules} rules evaluated).") + return 0 + + +if __name__ == "__main__": + # Reference Decimal so the import is "used" for static analysers when + # rules grow into Python-side numeric tolerance checks. + _ = Decimal("0") + sys.exit(main()) diff --git a/submission/wallet-payment-transfer/scripts/run_pipeline.py b/submission/wallet-payment-transfer/scripts/run_pipeline.py new file mode 100644 index 0000000..d431476 --- /dev/null +++ b/submission/wallet-payment-transfer/scripts/run_pipeline.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +""" +run_pipeline.py + +End-to-end demo: bootstrap source + lake + warehouse, seed data, replay, +print summary. Intended as a runnable smoke test. +""" + +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import duckdb + +from pipeline.cdc import CDCCapture +from pipeline.orchestrator import bootstrap, run_tick +from pipeline.schema_detector import SchemaDetector +from source.models import create_source_tables +from source.seed import build_seed_rows + + +def _print_summary(conn: duckdb.DuckDBPyConnection) -> None: + n_lake = conn.execute("SELECT COUNT(*) FROM lake_cdc_events").fetchone()[0] + n_cust = conn.execute( + "SELECT COUNT(*) FROM wh_customers WHERE _deleted = FALSE" + ).fetchone()[0] + n_wallets = conn.execute( + "SELECT COUNT(*) FROM wh_wallets WHERE _deleted = FALSE" + ).fetchone()[0] + n_transfers = conn.execute( + "SELECT COUNT(*) FROM wh_transfers WHERE _deleted = FALSE" + ).fetchone()[0] + print(f" lake_cdc_events : {n_lake} events") + print(f" wh_customers : {n_cust} active") + print(f" wh_wallets : {n_wallets} active") + print(f" wh_transfers : {n_transfers} active") + + +def main() -> int: + print("== bootstrap ==") + conn = duckdb.connect(":memory:") + create_source_tables(conn) + bootstrap(conn) + capture = CDCCapture() + + print("== seed ==") + for row in build_seed_rows(): + capture.insert(row.table, row.pk, row.data) + print(f" captured {capture.latest_sequence} CDC records") + + print("== run tick ==") + detector = SchemaDetector(conn) + stats = run_tick(conn, capture, detector=detector) + print(f" stats: {stats}") + + print("== summary ==") + _print_summary(conn) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/submission/wallet-payment-transfer/scripts/validate_catalog.py b/submission/wallet-payment-transfer/scripts/validate_catalog.py new file mode 100644 index 0000000..bac870d --- /dev/null +++ b/submission/wallet-payment-transfer/scripts/validate_catalog.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +""" +validate_catalog.py + +CI entry point for the catalog metadata check (`catalog-check`). + +What it verifies +---------------- + - catalog/catalog.json exists and parses as JSON + - it lists exactly one lake dataset (`lake_cdc_events`) + - it lists every expected `wh_
` current-state dataset + - it lists every expected `wh_
_history` SCD2 dataset + - every entry has the required metadata fields + - layer labels are consistent with naming conventions +""" + +from __future__ import annotations + +import json +import os +import sys + +CATALOG_PATH = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "catalog", + "catalog.json", +) + + +# Same source tables used by the warehouse layer; redeclared here to keep +# the script standalone (no DuckDB import cost on a metadata check). +SOURCE_TABLES = ( + "customers", + "payment_methods", + "wallets", + "transfers", + "transfer_events", + "ledger_entries", +) + +REQUIRED_FIELDS = ( + "name", + "layer", + "type", + "description", + "owner", + "consumers", + "update_cadence", + "tags", +) + + +def expected_dataset_names() -> list[str]: + names = ["lake_cdc_events"] + names.extend(f"wh_{t}" for t in SOURCE_TABLES) + names.extend(f"wh_{t}_history" for t in SOURCE_TABLES) + return names + + +def validate() -> list[str]: + issues: list[str] = [] + if not os.path.exists(CATALOG_PATH): + return [f"catalog.json not found at {CATALOG_PATH}"] + + with open(CATALOG_PATH) as f: + try: + catalog = json.load(f) + except json.JSONDecodeError as exc: + return [f"catalog.json is not valid JSON: {exc}"] + + if "datasets" not in catalog or not isinstance(catalog["datasets"], list): + return ["catalog.json missing top-level 'datasets' list"] + + by_name = {d.get("name"): d for d in catalog["datasets"] if "name" in d} + + for name in expected_dataset_names(): + if name not in by_name: + issues.append(f"missing dataset entry: {name}") + continue + entry = by_name[name] + for field in REQUIRED_FIELDS: + if field not in entry or not entry[field]: + issues.append( + f"{name}: missing or empty required field '{field}'" + ) + + # layer label consistency + if "lake_cdc_events" in by_name: + if by_name["lake_cdc_events"].get("layer") != "lake": + issues.append("lake_cdc_events: layer must be 'lake'") + for name, entry in by_name.items(): + if name.startswith("wh_") and entry.get("layer") != "warehouse": + issues.append(f"{name}: layer must be 'warehouse'") + if name.endswith("_history") and entry.get("type") != "scd2-history": + issues.append(f"{name}: type must be 'scd2-history'") + + return issues + + +def main() -> int: + issues = validate() + if issues: + print("Catalog validation failures:") + for v in issues: + print(f" x {v}") + return 1 + n = len(expected_dataset_names()) + print(f"Catalog validation passed ({n} datasets verified).") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/submission/wallet-payment-transfer/source/__init__.py b/submission/wallet-payment-transfer/source/__init__.py new file mode 100644 index 0000000..be39610 --- /dev/null +++ b/submission/wallet-payment-transfer/source/__init__.py @@ -0,0 +1 @@ +"""Source layer — transactional schema for the wallet/payments/transfers domain.""" diff --git a/submission/wallet-payment-transfer/source/contracts.py b/submission/wallet-payment-transfer/source/contracts.py new file mode 100644 index 0000000..8582e08 --- /dev/null +++ b/submission/wallet-payment-transfer/source/contracts.py @@ -0,0 +1,191 @@ +""" +Versioned schema contracts for the source system. + +Each contract describes what the CDC pipeline depends on for a given source +table: + - column name -> {type, nullable} + - primary key + - foreign keys + - enum domains (subset of allowed values) + - contract version + +The contracts are explicit so the pipeline can fail closed when the source +deviates from them. Adding new optional columns is *additive* and does not +break the contract; the rules in pipeline/schema_detector.py treat: + + * dropped columns -> BREAKING + * type widening / narrowing -> BREAKING + * enum domain *shrunk* -> BREAKING (existing rows may rely + on a value the pipeline just lost) + * nullability tightened (allow -> deny) -> BREAKING (rows the pipeline + previously emitted may now fail) + * primary key changed -> BREAKING (replay/upsert depends + on the original PK) + * new nullable columns -> WARN (additive, allow-listed) + +The contract is the input the warehouse modeling layer is allowed to assume. +Anything outside the contract is best-effort. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Optional + + +@dataclass(frozen=True) +class ColumnContract: + name: str + type: str + nullable: bool + + +@dataclass(frozen=True) +class TableContract: + name: str + columns: tuple[ColumnContract, ...] + primary_key: str + foreign_keys: tuple[tuple[str, str, str], ...] = field(default_factory=tuple) + # (column, referenced_table, referenced_column) + enum_domains: tuple[tuple[str, frozenset[str]], ...] = field( + default_factory=tuple + ) + contract_version: int = 1 + + def column(self, name: str) -> Optional[ColumnContract]: + for c in self.columns: + if c.name == name: + return c + return None + + @property + def column_names(self) -> tuple[str, ...]: + return tuple(c.name for c in self.columns) + + +# NOTE: the type strings below are normalized — DuckDB DESCRIBE returns the +# normalised type names we compare against in pipeline/schema_detector.py. +CONTRACTS: dict[str, TableContract] = { + "customers": TableContract( + name="customers", + primary_key="customer_id", + columns=( + ColumnContract("customer_id", "VARCHAR", False), + ColumnContract("full_name", "VARCHAR", False), + ColumnContract("email", "VARCHAR", False), + ColumnContract("country_code", "VARCHAR", False), + ColumnContract("status", "VARCHAR", False), + ColumnContract("registration_date", "DATE", False), + ColumnContract("created_at", "TIMESTAMP", False), + ColumnContract("updated_at", "TIMESTAMP", False), + ), + enum_domains=( + ("status", frozenset({"active", "suspended", "closed"})), + ), + ), + "payment_methods": TableContract( + name="payment_methods", + primary_key="payment_method_id", + columns=( + ColumnContract("payment_method_id", "VARCHAR", False), + ColumnContract("customer_id", "VARCHAR", False), + ColumnContract("method_type", "VARCHAR", False), + ColumnContract("masked_identifier", "VARCHAR", False), + ColumnContract("is_default", "BOOLEAN", False), + ColumnContract("status", "VARCHAR", False), + ColumnContract("created_at", "TIMESTAMP", False), + ColumnContract("updated_at", "TIMESTAMP", False), + ), + foreign_keys=(("customer_id", "customers", "customer_id"),), + enum_domains=( + ("method_type", frozenset({"card", "bank", "upi"})), + ("status", frozenset({"active", "expired", "revoked"})), + ), + ), + "wallets": TableContract( + name="wallets", + primary_key="wallet_id", + columns=( + ColumnContract("wallet_id", "VARCHAR", False), + ColumnContract("customer_id", "VARCHAR", False), + ColumnContract("currency", "VARCHAR", False), + ColumnContract("balance", "DECIMAL(18,4)", False), + ColumnContract("status", "VARCHAR", False), + ColumnContract("created_at", "TIMESTAMP", False), + ColumnContract("updated_at", "TIMESTAMP", False), + ), + foreign_keys=(("customer_id", "customers", "customer_id"),), + enum_domains=( + ("currency", frozenset({"USD", "EUR", "GBP", "INR", "JPY"})), + ("status", frozenset({"active", "frozen", "closed"})), + ), + ), + "transfers": TableContract( + name="transfers", + primary_key="transfer_id", + columns=( + ColumnContract("transfer_id", "VARCHAR", False), + ColumnContract("from_wallet_id", "VARCHAR", False), + ColumnContract("to_wallet_id", "VARCHAR", False), + ColumnContract("payment_method_id", "VARCHAR", True), + ColumnContract("amount", "DECIMAL(18,4)", False), + ColumnContract("currency", "VARCHAR", False), + ColumnContract("status", "VARCHAR", False), + ColumnContract("reference", "VARCHAR", True), + ColumnContract("failure_reason", "VARCHAR", True), + ColumnContract("created_at", "TIMESTAMP", False), + ColumnContract("settled_at", "TIMESTAMP", True), + ), + foreign_keys=( + ("from_wallet_id", "wallets", "wallet_id"), + ("to_wallet_id", "wallets", "wallet_id"), + ("payment_method_id", "payment_methods", "payment_method_id"), + ), + enum_domains=( + ("status", frozenset({"pending", "settled", "failed", "reversed"})), + ("currency", frozenset({"USD", "EUR", "GBP", "INR", "JPY"})), + ), + ), + "transfer_events": TableContract( + name="transfer_events", + primary_key="transfer_event_id", + columns=( + ColumnContract("transfer_event_id", "VARCHAR", False), + ColumnContract("transfer_id", "VARCHAR", False), + ColumnContract("from_status", "VARCHAR", True), + ColumnContract("to_status", "VARCHAR", False), + ColumnContract("note", "VARCHAR", True), + ColumnContract("occurred_at", "TIMESTAMP", False), + ), + foreign_keys=(("transfer_id", "transfers", "transfer_id"),), + enum_domains=( + ("to_status", frozenset({"pending", "settled", "failed", "reversed"})), + ), + ), + "ledger_entries": TableContract( + name="ledger_entries", + primary_key="ledger_entry_id", + columns=( + ColumnContract("ledger_entry_id", "VARCHAR", False), + ColumnContract("wallet_id", "VARCHAR", False), + ColumnContract("transfer_id", "VARCHAR", True), + ColumnContract("direction", "VARCHAR", False), + ColumnContract("amount", "DECIMAL(18,4)", False), + ColumnContract("signed_amount", "DECIMAL(18,4)", False), + ColumnContract("balance_after", "DECIMAL(18,4)", False), + ColumnContract("occurred_at", "TIMESTAMP", False), + ), + foreign_keys=( + ("wallet_id", "wallets", "wallet_id"), + ("transfer_id", "transfers", "transfer_id"), + ), + enum_domains=( + ("direction", frozenset({"credit", "debit"})), + ), + ), +} + + +def expected_columns(table: str) -> list[str]: + """Convenience for the schema contract CI script.""" + return [c.name for c in CONTRACTS[table].columns] diff --git a/submission/wallet-payment-transfer/source/models.py b/submission/wallet-payment-transfer/source/models.py new file mode 100644 index 0000000..c01a00b --- /dev/null +++ b/submission/wallet-payment-transfer/source/models.py @@ -0,0 +1,264 @@ +""" +Source schema for a wallet / payments / transfers domain. + +Domain summary +-------------- +Customers register on the platform, link payment methods, hold wallets, and +move money between wallets through transfers. Each transfer goes through +several state transitions, and every balance change on a wallet is recorded +in an immutable ledger. + +Entity classification +--------------------- +Strong entities (independent identity, can exist on their own): + - customers : a registered user of the platform + - wallets : a balance account owned by a customer; one customer can + hold multiple wallets (e.g. one per currency) + - transfers : a money movement between two wallets; has its own + business identity (transfer_id), references two wallets, + and exists even if downstream events are missing + +Weak entities (lifecycle bound to a parent; cannot meaningfully exist alone): + - payment_methods : tied to a customer; remove the customer and the + method has no owner + - transfer_events : state-change records for a transfer; a transfer_event + only makes sense in the context of its transfer + - ledger_entries : append-only balance-change rows for a wallet; one + ledger_entry has no business meaning without its wallet + +Mix of data types covered: + - DECIMAL(18, 4) amounts and balances (4 dp for FX-safe precision) + - TIMESTAMP created_at / updated_at / settled_at + - VARCHAR enums status, direction, currency, method_type + - VARCHAR FKs identifier strings (UUID-style) + - Nullable cols settled_at, reference, failure_reason, payment_method_id + - DATE registration_date + +Indexes +------- +We add indexes that map to access patterns the CDC pipeline actually exercises: + - foreign-key indexes (referential lookup + delete cascade scans) + - updated_at indexes (incremental extraction by timestamp) + - status indexes (filtered scans for pending/failed transfers) + - composite (wallet_id, created_at) on ledger_entries (audit-by-wallet) + +Important invariants (mirrored downstream in the warehouse validation layer): + Source / system rules + - all PKs are unique and non-null + - FKs always point to a real parent row + - enum-typed columns hold one of a fixed set of values + - timestamp columns are non-null where required + Business rules + - wallet.balance >= 0 (no overdraft) + - transfer.amount > 0 + - transfer.from_wallet_id != transfer.to_wallet_id (no self transfer) + - transfer.settled_at >= transfer.created_at when settled + - ledger_entries.signed_amount sums to wallet.balance per wallet + (monetary integrity check) + - transfer status follows allowed state machine: + pending -> {settled, failed, reversed} + failed -> {pending} (retry) + settled -> {reversed} + reversed and failed are terminal otherwise +""" + +from __future__ import annotations + +import duckdb + + +# --------------------------------------------------------------------------- +# DDL +# --------------------------------------------------------------------------- + +CUSTOMERS_DDL = """ +CREATE TABLE IF NOT EXISTS customers ( + customer_id VARCHAR PRIMARY KEY, + full_name VARCHAR NOT NULL, + email VARCHAR NOT NULL, + country_code VARCHAR NOT NULL, -- ISO-3166 alpha-2 + status VARCHAR NOT NULL + CHECK (status IN ('active', 'suspended', 'closed')), + registration_date DATE NOT NULL, + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL +) +""" + +PAYMENT_METHODS_DDL = """ +CREATE TABLE IF NOT EXISTS payment_methods ( + payment_method_id VARCHAR PRIMARY KEY, + customer_id VARCHAR NOT NULL REFERENCES customers(customer_id), + method_type VARCHAR NOT NULL + CHECK (method_type IN ('card', 'bank', 'upi')), + masked_identifier VARCHAR NOT NULL, -- e.g. '****1234' + is_default BOOLEAN NOT NULL DEFAULT FALSE, + status VARCHAR NOT NULL + CHECK (status IN ('active', 'expired', 'revoked')), + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL +) +""" + +WALLETS_DDL = """ +CREATE TABLE IF NOT EXISTS wallets ( + wallet_id VARCHAR PRIMARY KEY, + customer_id VARCHAR NOT NULL REFERENCES customers(customer_id), + currency VARCHAR NOT NULL -- ISO-4217 + CHECK (currency IN ('USD', 'EUR', 'GBP', 'INR', 'JPY')), + balance DECIMAL(18, 4) NOT NULL DEFAULT 0 + CHECK (balance >= 0), + status VARCHAR NOT NULL + CHECK (status IN ('active', 'frozen', 'closed')), + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL +) +""" + +TRANSFERS_DDL = """ +CREATE TABLE IF NOT EXISTS transfers ( + transfer_id VARCHAR PRIMARY KEY, + from_wallet_id VARCHAR NOT NULL REFERENCES wallets(wallet_id), + to_wallet_id VARCHAR NOT NULL REFERENCES wallets(wallet_id), + payment_method_id VARCHAR REFERENCES payment_methods(payment_method_id), + amount DECIMAL(18, 4) NOT NULL CHECK (amount > 0), + currency VARCHAR NOT NULL + CHECK (currency IN ('USD', 'EUR', 'GBP', 'INR', 'JPY')), + status VARCHAR NOT NULL + CHECK (status IN ('pending', 'settled', + 'failed', 'reversed')), + reference VARCHAR, -- nullable: external memo + failure_reason VARCHAR, -- nullable + created_at TIMESTAMP NOT NULL, + settled_at TIMESTAMP, -- null until settled + CHECK (from_wallet_id <> to_wallet_id), + CHECK (settled_at IS NULL OR settled_at >= created_at) +) +""" + +TRANSFER_EVENTS_DDL = """ +CREATE TABLE IF NOT EXISTS transfer_events ( + transfer_event_id VARCHAR PRIMARY KEY, + transfer_id VARCHAR NOT NULL REFERENCES transfers(transfer_id), + from_status VARCHAR + CHECK (from_status IS NULL OR from_status IN + ('pending', 'settled', 'failed', 'reversed')), + to_status VARCHAR NOT NULL + CHECK (to_status IN ('pending', 'settled', + 'failed', 'reversed')), + note VARCHAR, + occurred_at TIMESTAMP NOT NULL +) +""" + +LEDGER_ENTRIES_DDL = """ +CREATE TABLE IF NOT EXISTS ledger_entries ( + ledger_entry_id VARCHAR PRIMARY KEY, + wallet_id VARCHAR NOT NULL REFERENCES wallets(wallet_id), + transfer_id VARCHAR REFERENCES transfers(transfer_id), + direction VARCHAR NOT NULL + CHECK (direction IN ('credit', 'debit')), + amount DECIMAL(18, 4) NOT NULL CHECK (amount > 0), + signed_amount DECIMAL(18, 4) NOT NULL, -- + for credit, - for debit + balance_after DECIMAL(18, 4) NOT NULL CHECK (balance_after >= 0), + occurred_at TIMESTAMP NOT NULL, + CHECK ((direction = 'credit' AND signed_amount = amount) OR + (direction = 'debit' AND signed_amount = -amount)) +) +""" + +# Indexes that match real CDC / lookup access patterns. +# Note: PKs already produce a unique index automatically; we add the others. +INDEX_DDL: list[str] = [ + "CREATE INDEX IF NOT EXISTS ix_customers_updated_at " + "ON customers(updated_at)", + "CREATE INDEX IF NOT EXISTS ix_customers_status " + "ON customers(status)", + "CREATE INDEX IF NOT EXISTS ix_payment_methods_customer " + "ON payment_methods(customer_id)", + "CREATE INDEX IF NOT EXISTS ix_wallets_customer " + "ON wallets(customer_id)", + "CREATE INDEX IF NOT EXISTS ix_wallets_updated_at " + "ON wallets(updated_at)", + "CREATE INDEX IF NOT EXISTS ix_transfers_status " + "ON transfers(status)", + "CREATE INDEX IF NOT EXISTS ix_transfers_from_wallet " + "ON transfers(from_wallet_id)", + "CREATE INDEX IF NOT EXISTS ix_transfers_to_wallet " + "ON transfers(to_wallet_id)", + "CREATE INDEX IF NOT EXISTS ix_transfers_created_at " + "ON transfers(created_at)", + "CREATE INDEX IF NOT EXISTS ix_transfer_events_transfer " + "ON transfer_events(transfer_id)", + "CREATE INDEX IF NOT EXISTS ix_ledger_wallet_time " + "ON ledger_entries(wallet_id, occurred_at)", + "CREATE INDEX IF NOT EXISTS ix_ledger_transfer " + "ON ledger_entries(transfer_id)", +] + + +# Tables in dependency order (parents before children) — used for create + drop. +TABLES_IN_ORDER: list[str] = [ + "customers", + "payment_methods", + "wallets", + "transfers", + "transfer_events", + "ledger_entries", +] + +PRIMARY_KEYS: dict[str, str] = { + "customers": "customer_id", + "payment_methods": "payment_method_id", + "wallets": "wallet_id", + "transfers": "transfer_id", + "transfer_events": "transfer_event_id", + "ledger_entries": "ledger_entry_id", +} + +# Allowed enum domains — mirrored in schema contracts and downstream validation. +ENUM_DOMAINS: dict[tuple[str, str], frozenset[str]] = { + ("customers", "status"): frozenset({"active", "suspended", "closed"}), + ("payment_methods", "method_type"): frozenset({"card", "bank", "upi"}), + ("payment_methods", "status"): frozenset({"active", "expired", "revoked"}), + ("wallets", "currency"): frozenset({"USD", "EUR", "GBP", "INR", "JPY"}), + ("wallets", "status"): frozenset({"active", "frozen", "closed"}), + ("transfers", "status"): frozenset( + {"pending", "settled", "failed", "reversed"} + ), + ("transfers", "currency"): frozenset({"USD", "EUR", "GBP", "INR", "JPY"}), + ("transfer_events", "to_status"): frozenset( + {"pending", "settled", "failed", "reversed"} + ), + ("ledger_entries", "direction"): frozenset({"credit", "debit"}), +} + +# Allowed status transitions for transfers (state machine). +ALLOWED_TRANSFER_TRANSITIONS: frozenset[tuple[str, str]] = frozenset({ + ("pending", "settled"), + ("pending", "failed"), + ("pending", "reversed"), + ("failed", "pending"), # retry + ("settled", "reversed"), +}) + + +def create_source_tables(conn: duckdb.DuckDBPyConnection) -> None: + """Create all source tables and indexes in dependency order.""" + for ddl in ( + CUSTOMERS_DDL, + PAYMENT_METHODS_DDL, + WALLETS_DDL, + TRANSFERS_DDL, + TRANSFER_EVENTS_DDL, + LEDGER_ENTRIES_DDL, + ): + conn.execute(ddl) + for ix in INDEX_DDL: + conn.execute(ix) + + +def drop_source_tables(conn: duckdb.DuckDBPyConnection) -> None: + """Drop all source tables in reverse dependency order. Useful in tests.""" + for table in reversed(TABLES_IN_ORDER): + conn.execute(f"DROP TABLE IF EXISTS {table}") diff --git a/submission/wallet-payment-transfer/source/seed.py b/submission/wallet-payment-transfer/source/seed.py new file mode 100644 index 0000000..014bbc9 --- /dev/null +++ b/submission/wallet-payment-transfer/source/seed.py @@ -0,0 +1,295 @@ +""" +Representative seed data for the wallet domain. + +Used by: + - tests (via the `seeded` fixture) + - scripts/run_pipeline.py (end-to-end demo) + +The seed builds a small but coherent slice of the system: + - 3 customers + - 3 payment methods (one per customer) + - 4 wallets (Alice holds two — USD + EUR) + - 3 transfers in different states (settled / pending / failed) + - matching transfer_events + - ledger_entries that *reconcile* to each wallet.balance per the + monetary integrity invariant SUM(signed_amount) == balance. + +Funding ledger entries (transfer_id=NULL) seed the wallet starting +balance; the remaining ledger entries flow from a real transfer. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import date, datetime, timezone +from decimal import Decimal + + +def _ts(year: int, month: int, day: int, hour: int = 0, minute: int = 0) -> datetime: + return datetime(year, month, day, hour, minute, tzinfo=timezone.utc) + + +@dataclass(frozen=True) +class SeedRow: + table: str + pk: str + data: dict + + +def build_seed_rows() -> list[SeedRow]: + """ + Return ordered list of seed rows. Order matters: parents first so + foreign keys are valid as each row is applied. + + Final reconciled state per wallet (verified by data quality checks): + w_alice_usd : balance = 75 (funded 100, debited 25) + w_alice_eur : balance = 0 (never funded) + w_bob_gbp : balance = 50 (funded 50) + w_chen_usd : balance = 50 (funded 25, credited 25) + """ + rows: list[SeedRow] = [] + t0 = _ts(2026, 6, 1, 9, 0) + + # ── customers ──────────────────────────────────────────────────────────── + rows.append(SeedRow("customers", "c_alice", { + "customer_id": "c_alice", + "full_name": "Alice Anderson", + "email": "alice@example.invalid", + "country_code": "US", + "status": "active", + "registration_date": date(2024, 1, 15), + "created_at": t0, + "updated_at": t0, + })) + rows.append(SeedRow("customers", "c_bob", { + "customer_id": "c_bob", + "full_name": "Bob Brown", + "email": "bob@example.invalid", + "country_code": "GB", + "status": "active", + "registration_date": date(2024, 3, 20), + "created_at": t0, + "updated_at": t0, + })) + rows.append(SeedRow("customers", "c_chen", { + "customer_id": "c_chen", + "full_name": "Chen Cheng", + "email": "chen@example.invalid", + "country_code": "US", + "status": "active", + "registration_date": date(2025, 11, 1), + "created_at": t0, + "updated_at": t0, + })) + + # ── payment methods ────────────────────────────────────────────────────── + rows.append(SeedRow("payment_methods", "pm_alice_card", { + "payment_method_id": "pm_alice_card", + "customer_id": "c_alice", + "method_type": "card", + "masked_identifier": "****4242", + "is_default": True, + "status": "active", + "created_at": t0, + "updated_at": t0, + })) + rows.append(SeedRow("payment_methods", "pm_bob_bank", { + "payment_method_id": "pm_bob_bank", + "customer_id": "c_bob", + "method_type": "bank", + "masked_identifier": "GB**0001", + "is_default": True, + "status": "active", + "created_at": t0, + "updated_at": t0, + })) + rows.append(SeedRow("payment_methods", "pm_chen_upi", { + "payment_method_id": "pm_chen_upi", + "customer_id": "c_chen", + "method_type": "upi", + "masked_identifier": "chen@upi", + "is_default": True, + "status": "active", + "created_at": t0, + "updated_at": t0, + })) + + # ── wallets (final reconciled balances) ────────────────────────────────── + rows.append(SeedRow("wallets", "w_alice_usd", { + "wallet_id": "w_alice_usd", + "customer_id": "c_alice", + "currency": "USD", + "balance": Decimal("75.0000"), # 100 funded, then -25 + "status": "active", + "created_at": t0, + "updated_at": _ts(2026, 6, 1, 10, 1), + })) + rows.append(SeedRow("wallets", "w_alice_eur", { + "wallet_id": "w_alice_eur", + "customer_id": "c_alice", + "currency": "EUR", + "balance": Decimal("0.0000"), + "status": "active", + "created_at": t0, + "updated_at": t0, + })) + rows.append(SeedRow("wallets", "w_bob_gbp", { + "wallet_id": "w_bob_gbp", + "customer_id": "c_bob", + "currency": "GBP", + "balance": Decimal("50.0000"), + "status": "active", + "created_at": t0, + "updated_at": t0, + })) + rows.append(SeedRow("wallets", "w_chen_usd", { + "wallet_id": "w_chen_usd", + "customer_id": "c_chen", + "currency": "USD", + "balance": Decimal("50.0000"), # 25 funded, then +25 + "status": "active", + "created_at": t0, + "updated_at": _ts(2026, 6, 1, 10, 1), + })) + + # ── transfers ──────────────────────────────────────────────────────────── + rows.append(SeedRow("transfers", "tx_settled_001", { + "transfer_id": "tx_settled_001", + "from_wallet_id": "w_alice_usd", + "to_wallet_id": "w_chen_usd", + "payment_method_id": "pm_alice_card", + "amount": Decimal("25.0000"), + "currency": "USD", + "status": "settled", + "reference": "rent-share", + "failure_reason": None, + "created_at": _ts(2026, 6, 1, 10, 0), + "settled_at": _ts(2026, 6, 1, 10, 1), + })) + rows.append(SeedRow("transfers", "tx_pending_002", { + "transfer_id": "tx_pending_002", + "from_wallet_id": "w_bob_gbp", + "to_wallet_id": "w_alice_eur", + "payment_method_id": "pm_bob_bank", + "amount": Decimal("10.0000"), + "currency": "GBP", + "status": "pending", + "reference": None, + "failure_reason": None, + "created_at": _ts(2026, 6, 1, 11, 0), + "settled_at": None, + })) + rows.append(SeedRow("transfers", "tx_failed_003", { + "transfer_id": "tx_failed_003", + "from_wallet_id": "w_chen_usd", + "to_wallet_id": "w_bob_gbp", + "payment_method_id": "pm_chen_upi", + "amount": Decimal("5.0000"), + "currency": "USD", + "status": "failed", + "reference": "groceries", + "failure_reason": "insufficient-funds", + "created_at": _ts(2026, 6, 1, 12, 0), + "settled_at": None, + })) + + # ── transfer_events ────────────────────────────────────────────────────── + rows.append(SeedRow("transfer_events", "te_001_a", { + "transfer_event_id": "te_001_a", + "transfer_id": "tx_settled_001", + "from_status": None, + "to_status": "pending", + "note": "created", + "occurred_at": _ts(2026, 6, 1, 10, 0), + })) + rows.append(SeedRow("transfer_events", "te_001_b", { + "transfer_event_id": "te_001_b", + "transfer_id": "tx_settled_001", + "from_status": "pending", + "to_status": "settled", + "note": "card-charged", + "occurred_at": _ts(2026, 6, 1, 10, 1), + })) + rows.append(SeedRow("transfer_events", "te_002_a", { + "transfer_event_id": "te_002_a", + "transfer_id": "tx_pending_002", + "from_status": None, + "to_status": "pending", + "note": "created", + "occurred_at": _ts(2026, 6, 1, 11, 0), + })) + rows.append(SeedRow("transfer_events", "te_003_a", { + "transfer_event_id": "te_003_a", + "transfer_id": "tx_failed_003", + "from_status": None, + "to_status": "pending", + "note": "created", + "occurred_at": _ts(2026, 6, 1, 12, 0), + })) + rows.append(SeedRow("transfer_events", "te_003_b", { + "transfer_event_id": "te_003_b", + "transfer_id": "tx_failed_003", + "from_status": "pending", + "to_status": "failed", + "note": "insufficient-funds", + "occurred_at": _ts(2026, 6, 1, 12, 1), + })) + + # ── ledger_entries (funding entries + transfer-driven movements) ───────── + # Funding for w_alice_usd: +100 + rows.append(SeedRow("ledger_entries", "le_alice_fund", { + "ledger_entry_id": "le_alice_fund", + "wallet_id": "w_alice_usd", + "transfer_id": None, + "direction": "credit", + "amount": Decimal("100.0000"), + "signed_amount": Decimal("100.0000"), + "balance_after": Decimal("100.0000"), + "occurred_at": t0, + })) + # Funding for w_bob_gbp: +50 + rows.append(SeedRow("ledger_entries", "le_bob_fund", { + "ledger_entry_id": "le_bob_fund", + "wallet_id": "w_bob_gbp", + "transfer_id": None, + "direction": "credit", + "amount": Decimal("50.0000"), + "signed_amount": Decimal("50.0000"), + "balance_after": Decimal("50.0000"), + "occurred_at": t0, + })) + # Funding for w_chen_usd: +25 + rows.append(SeedRow("ledger_entries", "le_chen_fund", { + "ledger_entry_id": "le_chen_fund", + "wallet_id": "w_chen_usd", + "transfer_id": None, + "direction": "credit", + "amount": Decimal("25.0000"), + "signed_amount": Decimal("25.0000"), + "balance_after": Decimal("25.0000"), + "occurred_at": t0, + })) + # Settled transfer 001: alice debit 25 -> 75 + rows.append(SeedRow("ledger_entries", "le_001_debit", { + "ledger_entry_id": "le_001_debit", + "wallet_id": "w_alice_usd", + "transfer_id": "tx_settled_001", + "direction": "debit", + "amount": Decimal("25.0000"), + "signed_amount": Decimal("-25.0000"), + "balance_after": Decimal("75.0000"), + "occurred_at": _ts(2026, 6, 1, 10, 1), + })) + # Settled transfer 001: chen credit 25 -> 50 + rows.append(SeedRow("ledger_entries", "le_001_credit", { + "ledger_entry_id": "le_001_credit", + "wallet_id": "w_chen_usd", + "transfer_id": "tx_settled_001", + "direction": "credit", + "amount": Decimal("25.0000"), + "signed_amount": Decimal("25.0000"), + "balance_after": Decimal("50.0000"), + "occurred_at": _ts(2026, 6, 1, 10, 1), + })) + + return rows diff --git a/submission/sample-candidate/pipeline/__init__.py b/submission/wallet-payment-transfer/tests/__init__.py similarity index 100% rename from submission/sample-candidate/pipeline/__init__.py rename to submission/wallet-payment-transfer/tests/__init__.py diff --git a/submission/wallet-payment-transfer/tests/conftest.py b/submission/wallet-payment-transfer/tests/conftest.py new file mode 100644 index 0000000..5fab307 --- /dev/null +++ b/submission/wallet-payment-transfer/tests/conftest.py @@ -0,0 +1,59 @@ +"""Shared pytest fixtures for the wallet/payments CDC pipeline tests.""" + +from __future__ import annotations + +from datetime import datetime, timezone + +import duckdb +import pytest + +from pipeline.cdc import CDCCapture +from pipeline.orchestrator import bootstrap, run_tick +from pipeline.schema_detector import SchemaDetector +from source.models import create_source_tables +from source.seed import build_seed_rows + + +def _ts() -> datetime: + return datetime.now(timezone.utc) + + +@pytest.fixture() +def conn() -> duckdb.DuckDBPyConnection: + """Fresh in-memory DuckDB with source + lake + warehouse + checkpoints.""" + c = duckdb.connect(":memory:") + create_source_tables(c) + bootstrap(c) + return c + + +@pytest.fixture() +def capture() -> CDCCapture: + """Empty CDC capture log.""" + return CDCCapture() + + +@pytest.fixture() +def detector(conn: duckdb.DuckDBPyConnection) -> SchemaDetector: + return SchemaDetector(conn) + + +@pytest.fixture() +def seeded( + conn: duckdb.DuckDBPyConnection, + capture: CDCCapture, + detector: SchemaDetector, +): + """ + DuckDB connection pre-loaded with the canonical wallet seed + (3 customers, 3 payment methods, 4 wallets, 3 transfers, + 5 transfer_events, 5 ledger_entries). + + Returns (conn, capture). + """ + for row in build_seed_rows(): + capture.insert(row.table, row.pk, row.data) + + stats = run_tick(conn, capture, detector=detector) + assert not stats["aborted"] + return conn, capture diff --git a/submission/wallet-payment-transfer/tests/test_catalog.py b/submission/wallet-payment-transfer/tests/test_catalog.py new file mode 100644 index 0000000..522d53d --- /dev/null +++ b/submission/wallet-payment-transfer/tests/test_catalog.py @@ -0,0 +1,198 @@ +""" +Tests — data catalog correctness. + +The catalog (catalog/catalog.json) is the canonical entry point for downstream +consumers. It must: + + - exist at a fixed location + - parse as valid JSON + - declare every warehouse + lake dataset the pipeline produces + - carry the required descriptive fields per dataset + - use a consistent vocabulary for layer / type / sla_class + - flag every PII / monetary dataset with a tag for governance +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +CATALOG_PATH = ( + Path(__file__).resolve().parents[1] / "catalog" / "catalog.json" +) + +EXPECTED_DATASETS: set[str] = { + "lake_cdc_events", + "wh_customers", + "wh_payment_methods", + "wh_wallets", + "wh_transfers", + "wh_transfer_events", + "wh_ledger_entries", + "wh_customers_history", + "wh_payment_methods_history", + "wh_wallets_history", + "wh_transfers_history", + "wh_transfer_events_history", + "wh_ledger_entries_history", +} + +REQUIRED_FIELDS = ( + "name", "layer", "type", "description", + "owner", "consumers", "update_cadence", + "sla_class", "tags", "primary_key", +) + +ALLOWED_LAYERS = {"lake", "warehouse"} +ALLOWED_SLA = {"tier-1", "tier-2", "tier-3"} +ALLOWED_TYPES = { + "append-only-change-log", + "current-state", + "scd2-history", +} + + +@pytest.fixture(scope="module") +def catalog() -> dict: + raw = CATALOG_PATH.read_text(encoding="utf-8") + return json.loads(raw) + + +# ── basics ─────────────────────────────────────────────────────────────────── + + +def test_catalog_file_exists(): + assert CATALOG_PATH.exists(), f"missing catalog at {CATALOG_PATH}" + + +def test_catalog_is_valid_json(catalog): + assert isinstance(catalog, dict) + assert "datasets" in catalog + assert isinstance(catalog["datasets"], list) + + +def test_catalog_top_level_metadata(catalog): + for key in ("catalog_version", "domain", "owner"): + assert key in catalog, f"missing top-level field: {key}" + assert catalog["domain"] == "wallet-payments-transfers" + + +# ── completeness ───────────────────────────────────────────────────────────── + + +def test_catalog_lists_every_pipeline_dataset(catalog): + declared = {d["name"] for d in catalog["datasets"]} + missing = EXPECTED_DATASETS - declared + assert not missing, f"catalog missing datasets: {sorted(missing)}" + + +def test_catalog_has_no_unexpected_datasets(catalog): + declared = {d["name"] for d in catalog["datasets"]} + extra = declared - EXPECTED_DATASETS + assert not extra, f"catalog has unexpected datasets: {sorted(extra)}" + + +# ── per-dataset required fields ────────────────────────────────────────────── + + +def test_every_dataset_has_required_fields(catalog): + for ds in catalog["datasets"]: + for f in REQUIRED_FIELDS: + assert f in ds, f"{ds.get('name')} missing field {f}" + + +def test_every_dataset_has_a_schema_or_extension(catalog): + """Current-state has schema; SCD2 history has schema_extension.""" + for ds in catalog["datasets"]: + assert "schema" in ds or "schema_extension" in ds, ( + f"{ds['name']} declares no schema or schema_extension" + ) + + +# ── controlled vocabulary ──────────────────────────────────────────────────── + + +def test_layers_use_allowed_vocabulary(catalog): + for ds in catalog["datasets"]: + assert ds["layer"] in ALLOWED_LAYERS, ( + f"{ds['name']} has unknown layer: {ds['layer']}" + ) + + +def test_types_use_allowed_vocabulary(catalog): + for ds in catalog["datasets"]: + assert ds["type"] in ALLOWED_TYPES, ( + f"{ds['name']} has unknown type: {ds['type']}" + ) + + +def test_sla_class_uses_allowed_vocabulary(catalog): + for ds in catalog["datasets"]: + assert ds["sla_class"] in ALLOWED_SLA, ( + f"{ds['name']} has unknown sla_class: {ds['sla_class']}" + ) + + +# ── consistency between layer and type ─────────────────────────────────────── + + +def test_lake_layer_uses_change_log_type(catalog): + for ds in catalog["datasets"]: + if ds["layer"] == "lake": + assert ds["type"] == "append-only-change-log" + + +def test_wh_history_tables_are_scd2(catalog): + for ds in catalog["datasets"]: + if ds["name"].endswith("_history"): + assert ds["type"] == "scd2-history" + + +def test_wh_current_tables_are_current_state(catalog): + for ds in catalog["datasets"]: + if ( + ds["layer"] == "warehouse" + and not ds["name"].endswith("_history") + ): + assert ds["type"] == "current-state" + + +# ── governance tagging ─────────────────────────────────────────────────────── + + +def test_money_critical_datasets_carry_appropriate_tags(catalog): + """Wallets / transfers / ledger feed finance — all must be tier-1.""" + money_critical = { + "wh_wallets", "wh_transfers", "wh_ledger_entries", + "wh_wallets_history", "wh_transfers_history", + "wh_ledger_entries_history", + } + for ds in catalog["datasets"]: + if ds["name"] in money_critical: + assert ds["sla_class"] == "tier-1", ( + f"{ds['name']} feeds finance — must be tier-1" + ) + + +def test_history_tables_tagged_for_time_travel(catalog): + """SCD2 datasets that back time-travel queries must be tagged.""" + for ds in catalog["datasets"]: + if ds["type"] == "scd2-history" and "balance" in ( + ds.get("tags") or [] + ): + assert "time-travel" in ds["tags"] + + +def test_consumers_is_a_non_empty_list(catalog): + for ds in catalog["datasets"]: + consumers = ds["consumers"] + assert isinstance(consumers, list) + assert consumers, f"{ds['name']} has empty consumers list" + + +def test_primary_key_is_a_non_empty_list(catalog): + for ds in catalog["datasets"]: + pk = ds["primary_key"] + assert isinstance(pk, list) and pk diff --git a/submission/wallet-payment-transfer/tests/test_cdc.py b/submission/wallet-payment-transfer/tests/test_cdc.py new file mode 100644 index 0000000..9faac53 --- /dev/null +++ b/submission/wallet-payment-transfer/tests/test_cdc.py @@ -0,0 +1,192 @@ +""" +Tests — CDC capture correctness. + +Covers: + - insert / update / delete capture + - sequence monotonicity + - replay window (records_since) + - duplicate appearance (the lake retains every event for the same PK) + - invalid operation rejection + - before/after image presence per operation type + - schema-aware capture refuses writes when the source is incompatible +""" + +from __future__ import annotations + +from datetime import datetime, timezone + +import pytest + +from pipeline.cdc import ( + CDCCapture, + CDCRecord, + SchemaContractViolation, +) +from pipeline.schema_detector import SchemaDetector + + +# ── insert / update / delete ────────────────────────────────────────────────── + + +def test_insert_record_carries_after_image_only(capture): + rec = capture.insert("customers", "c1", {"customer_id": "c1"}) + assert rec.operation == "insert" + assert rec.before is None + assert rec.after == {"customer_id": "c1"} + + +def test_update_record_carries_before_and_after(capture): + capture.insert("customers", "c1", {"customer_id": "c1", "status": "active"}) + rec = capture.update( + "customers", "c1", + before={"customer_id": "c1", "status": "active"}, + after={"customer_id": "c1", "status": "suspended"}, + ) + assert rec.operation == "update" + assert rec.before["status"] == "active" + assert rec.after["status"] == "suspended" + + +def test_delete_record_carries_before_image_only(capture): + capture.insert("customers", "c1", {"customer_id": "c1"}) + rec = capture.delete("customers", "c1", before={"customer_id": "c1"}) + assert rec.operation == "delete" + assert rec.after is None + assert rec.before == {"customer_id": "c1"} + + +def test_invalid_operation_raises(capture): + with pytest.raises(ValueError, match="Invalid CDC operation"): + CDCRecord( + operation="upsert", + table="customers", + primary_key="c1", + before=None, + after={"x": 1}, + ) + + +def test_insert_without_after_raises(): + with pytest.raises(ValueError, match="must carry an 'after'"): + CDCRecord( + operation="insert", + table="customers", + primary_key="c1", + before=None, + after=None, + ) + + +def test_delete_without_before_raises(): + with pytest.raises(ValueError, match="must carry a 'before'"): + CDCRecord( + operation="delete", + table="customers", + primary_key="c1", + before=None, + after={"x": 1}, + ) + + +# ── sequence monotonicity ──────────────────────────────────────────────────── + + +def test_sequences_strictly_increasing(capture): + r1 = capture.insert("customers", "c1", {"customer_id": "c1"}) + r2 = capture.insert("customers", "c2", {"customer_id": "c2"}) + r3 = capture.update( + "customers", "c1", + before={"customer_id": "c1"}, + after={"customer_id": "c1"}, + ) + assert r1.sequence < r2.sequence < r3.sequence + + +def test_latest_sequence_reflects_last_record(capture): + capture.insert("customers", "c1", {"customer_id": "c1"}) + last = capture.insert("customers", "c2", {"customer_id": "c2"}) + assert capture.latest_sequence == last.sequence + + +# ── replay window ──────────────────────────────────────────────────────────── + + +def test_records_since_returns_only_records_after_offset(capture): + capture.insert("customers", "c1", {"customer_id": "c1"}) + capture.insert("customers", "c2", {"customer_id": "c2"}) + checkpoint = capture.latest_sequence + r3 = capture.insert("customers", "c3", {"customer_id": "c3"}) + + replayed = capture.records_since(checkpoint) + assert len(replayed) == 1 + assert replayed[0].sequence == r3.sequence + + +def test_records_since_zero_returns_everything(capture): + capture.insert("customers", "c1", {"customer_id": "c1"}) + capture.insert("customers", "c2", {"customer_id": "c2"}) + capture.delete("customers", "c1", before={"customer_id": "c1"}) + assert len(capture.records_since(0)) == 3 + + +def test_records_since_latest_returns_empty(capture): + capture.insert("customers", "c1", {"customer_id": "c1"}) + assert capture.records_since(capture.latest_sequence) == [] + + +# ── multiple events for same PK appear in log ──────────────────────────────── + + +def test_same_pk_can_appear_multiple_times(capture): + """Lake-level dedup is the orchestrator's job; capture itself is append-only.""" + capture.insert( + "customers", "c1", {"customer_id": "c1", "status": "active"} + ) + capture.update( + "customers", "c1", + before={"customer_id": "c1", "status": "active"}, + after={"customer_id": "c1", "status": "suspended"}, + ) + capture.update( + "customers", "c1", + before={"customer_id": "c1", "status": "suspended"}, + after={"customer_id": "c1", "status": "closed"}, + ) + pk1 = [r for r in capture.log if r.primary_key == "c1"] + assert len(pk1) == 3 + + +# ── captured_at metadata ───────────────────────────────────────────────────── + + +def test_captured_at_is_utc_datetime(capture): + rec = capture.insert("customers", "c1", {"customer_id": "c1"}) + assert isinstance(rec.captured_at, datetime) + assert rec.captured_at.tzinfo is not None + + +def test_captured_at_can_be_overridden_for_replay(capture): + fixed = datetime(2026, 6, 1, tzinfo=timezone.utc) + rec = capture.insert( + "customers", "c1", {"customer_id": "c1"}, captured_at=fixed + ) + assert rec.captured_at == fixed + + +# ── schema-aware capture (stop-the-line at write time) ─────────────────────── + + +def test_capture_with_strict_schema_refuses_writes_on_drop(conn, detector): + """If a contract column is missing on the source, every write must fail.""" + conn.execute("DROP TABLE IF EXISTS ledger_entries") + conn.execute("DROP TABLE IF EXISTS transfer_events") + conn.execute("DROP TABLE IF EXISTS transfers") + conn.execute("DROP TABLE IF EXISTS payment_methods") + conn.execute("DROP TABLE IF EXISTS wallets") + conn.execute("ALTER TABLE customers DROP COLUMN email") + + strict = CDCCapture(detector, require_compatible_schema=True) + with pytest.raises(SchemaContractViolation): + strict.insert("customers", "c1", {"customer_id": "c1"}) + + assert strict.log == [] diff --git a/submission/wallet-payment-transfer/tests/test_data_quality.py b/submission/wallet-payment-transfer/tests/test_data_quality.py new file mode 100644 index 0000000..c2169ce --- /dev/null +++ b/submission/wallet-payment-transfer/tests/test_data_quality.py @@ -0,0 +1,171 @@ +""" +Tests — data quality / validation parity. + +For every rule defined in scripts/run_data_quality_checks.py we verify +that the seeded warehouse passes; we also seed a deliberately broken +state for a representative subset to confirm each rule actually fires. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from decimal import Decimal + +import pytest + +from pipeline.cdc import CDCCapture +from pipeline.orchestrator import bootstrap, run_tick +from scripts.run_data_quality_checks import all_rules, run_checks + + +def _ts() -> datetime: + return datetime(2026, 6, 5, 12, tzinfo=timezone.utc) + + +# ── happy path ─────────────────────────────────────────────────────────────── + + +def test_seeded_warehouse_passes_all_rules(seeded): + conn, _ = seeded + failures = run_checks(conn) + assert failures == [], ( + "Seeded warehouse should pass all rules. Failures:\n " + + "\n ".join(f"{r.rule_id}: {n}" for r, n in failures) + ) + + +def test_rules_cover_all_categories(): + cats = {r.category for r in all_rules()} + assert {"system", "business"}.issubset(cats) + + +# ── system rule firing ─────────────────────────────────────────────────────── + + +def test_pk_uniqueness_rule_fires_on_duplicate(seeded): + conn, _ = seeded + # Force a duplicate in wh_customers by direct INSERT + conn.execute( + "INSERT INTO wh_customers VALUES " + "('c_alice', 'Dup', 'd@x.test', 'US', 'active', " + "DATE '2024-01-15', ?, ?, 999, FALSE)", + [_ts(), _ts()], + ) + failures = run_checks(conn) + rule_ids = {r.rule_id for r, _ in failures} + assert "sys.pk_unique.customers" in rule_ids + + +def test_enum_rule_fires_on_unknown_value(seeded): + conn, _ = seeded + conn.execute( + "UPDATE wh_customers SET status = 'banned' WHERE customer_id = 'c_alice'" + ) + failures = run_checks(conn) + rule_ids = {r.rule_id for r, _ in failures} + assert "sys.enum.customers.status" in rule_ids + + +def test_fk_rule_fires_on_orphan(seeded): + conn, _ = seeded + conn.execute( + "UPDATE wh_wallets SET customer_id = 'c_missing' " + "WHERE wallet_id = 'w_alice_usd'" + ) + failures = run_checks(conn) + rule_ids = {r.rule_id for r, _ in failures} + assert "sys.fk.wallets.customer_id" in rule_ids + + +# ── business rule firing ───────────────────────────────────────────────────── + + +def test_negative_balance_rule_fires(seeded): + conn, _ = seeded + conn.execute( + "UPDATE wh_wallets SET balance = -1.0000 " + "WHERE wallet_id = 'w_bob_gbp'" + ) + failures = run_checks(conn) + rule_ids = {r.rule_id for r, _ in failures} + assert "biz.wallet.balance_non_negative" in rule_ids + + +def test_settled_at_after_created_rule_fires(seeded): + conn, _ = seeded + conn.execute( + "UPDATE wh_transfers SET settled_at = TIMESTAMP '2020-01-01' " + "WHERE transfer_id = 'tx_settled_001'" + ) + failures = run_checks(conn) + rule_ids = {r.rule_id for r, _ in failures} + assert "biz.transfer.settled_at_after_created" in rule_ids + + +def test_ledger_signed_amount_consistency_rule_fires(seeded): + conn, _ = seeded + conn.execute( + "UPDATE wh_ledger_entries SET signed_amount = 99 " + "WHERE ledger_entry_id = 'le_001_debit'" + ) + failures = run_checks(conn) + rule_ids = {r.rule_id for r, _ in failures} + assert "biz.ledger.signed_amount_consistent" in rule_ids + + +def test_ledger_reconciles_to_wallet_balance_rule_fires(seeded): + conn, _ = seeded + # Break the reconciliation: bump wallet balance away from ledger sum. + conn.execute( + "UPDATE wh_wallets SET balance = balance + 1 " + "WHERE wallet_id = 'w_alice_usd'" + ) + failures = run_checks(conn) + rule_ids = {r.rule_id for r, _ in failures} + assert "biz.ledger.reconciles_to_wallet_balance" in rule_ids + + +def test_transfer_status_transition_rule_fires(seeded): + conn, _ = seeded + # Insert an illegal transition: settled -> pending (not allowed) + conn.execute( + "INSERT INTO wh_transfer_events VALUES (" + "'te_bad', 'tx_settled_001', 'settled', 'pending', " + "'illegal-transition', ?, 999, FALSE)", + [_ts()], + ) + failures = run_checks(conn) + rule_ids = {r.rule_id for r, _ in failures} + assert "biz.transfer_event.allowed_transition" in rule_ids + + +# ── parity sanity: every contract column with NOT NULL has a rule ──────────── + + +def test_every_contract_not_null_has_a_warehouse_rule(): + rule_ids = {r.rule_id for r in all_rules()} + from source.contracts import CONTRACTS + for table, contract in CONTRACTS.items(): + for col in contract.columns: + if col.nullable: + continue + expected = f"sys.not_null.{table}.{col.name}" + assert expected in rule_ids, ( + f"missing parity rule {expected} — every NOT NULL " + "contract column must be checked downstream" + ) + + +# ── soft-deletes are excluded from quality rules ────────────────────────────── + + +def test_deleted_rows_do_not_trigger_validation_failures(seeded): + conn, capture = seeded + capture.delete( + "transfers", "tx_pending_002", + before={"transfer_id": "tx_pending_002"}, + ) + run_tick(conn, capture) + # _deleted=true rows must be ignored by every rule. + failures = run_checks(conn) + assert failures == [] diff --git a/submission/wallet-payment-transfer/tests/test_e2e.py b/submission/wallet-payment-transfer/tests/test_e2e.py new file mode 100644 index 0000000..6b37334 --- /dev/null +++ b/submission/wallet-payment-transfer/tests/test_e2e.py @@ -0,0 +1,306 @@ +""" +Tests — end-to-end pipeline scenarios. + +These tests exercise the full happy / unhappy paths through: + source schema → CDC capture → lake → warehouse → quality checks → catalog + +Scenarios: + 1. Cold start: source + bootstrap + seed → all layers populated and consistent + 2. Hot loop: incremental change → re-run tick → only deltas applied + 3. Restart: second tick from same checkpoint is a no-op + 4. Stop-the-line: schema break aborts before any data is moved + 5. Recovery: fixing the schema lets the pipeline resume from where it left off + 6. Restore: rolling current state back to T does not corrupt history or DQ + 7. Quality parity: warehouse rules pass on the seeded state +""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from decimal import Decimal + +import pytest + +from pipeline.cdc import CDCCapture, SchemaContractViolation +from pipeline.orchestrator import bootstrap, run_tick +from pipeline.warehouse import ( + apply_cdc_records, + restore_current_state_to, + state_at, +) +from scripts.run_data_quality_checks import run_checks +from source.seed import build_seed_rows + + +def _ts(seconds: int = 0) -> datetime: + # Far enough in the future that we are always after the seed's + # captured_at (which uses datetime.now() at test-run time). + base = datetime(2030, 1, 1, 12, 0, tzinfo=timezone.utc) + return base + timedelta(seconds=seconds) + + +# ── 1. cold start ──────────────────────────────────────────────────────────── + + +def test_cold_start_populates_all_layers(seeded): + conn, capture = seeded + n_lake = conn.execute("SELECT COUNT(*) FROM lake_cdc_events").fetchone()[0] + n_wh = conn.execute("SELECT COUNT(*) FROM wh_customers").fetchone()[0] + n_hist = conn.execute( + "SELECT COUNT(*) FROM wh_customers_history WHERE _is_current" + ).fetchone()[0] + + assert n_lake == capture.latest_sequence + assert n_wh == 3 + assert n_hist == 3 + + +def test_cold_start_passes_all_quality_rules(seeded): + conn, _ = seeded + failures = run_checks(conn) + assert failures == [], ( + "post-seed warehouse should be clean, got: " + + ", ".join(f"{r.rule_id}: {n}" for r, n in failures) + ) + + +# ── 2. hot loop ────────────────────────────────────────────────────────────── + + +def test_incremental_update_only_applies_delta(seeded, detector): + conn, capture = seeded + cp_before = conn.execute( + "SELECT sequence FROM pipeline_checkpoints WHERE stage = 'warehouse'" + ).fetchone()[0] + + capture.update( + "wallets", "w_alice_usd", + before={"wallet_id": "w_alice_usd", "balance": Decimal("75.0000")}, + after={ + "wallet_id": "w_alice_usd", + "customer_id": "c_alice", + "currency": "USD", + "balance": Decimal("75.0000"), + "status": "frozen", # status change + "created_at": datetime(2026, 6, 1, 9, tzinfo=timezone.utc), + "updated_at": _ts(), + }, + captured_at=_ts(), + ) + + stats = run_tick(conn, capture, detector=detector) + assert stats["lake_written"] == 1 + assert stats["warehouse_applied"] == 1 + assert stats["checkpoint_warehouse"] > cp_before + + +# ── 3. restart / no-op ─────────────────────────────────────────────────────── + + +def test_second_tick_with_no_new_events_is_noop(seeded, detector): + conn, capture = seeded + stats = run_tick(conn, capture, detector=detector) + assert stats["lake_written"] == 0 + assert stats["warehouse_applied"] == 0 + + +def test_restart_resumes_from_checkpoint(detector_factory_fresh_conn): + """ + Simulate a process restart: + tick 1 -> seed half the rows + tick 2 -> add the rest, but with a "fresh" detector instance + The pipeline should pick up exactly where checkpoint stopped. + """ + conn, detector = detector_factory_fresh_conn + bootstrap(conn) + capture = CDCCapture() + rows = build_seed_rows() + half = len(rows) // 2 + + for r in rows[:half]: + capture.insert(r.table, r.pk, r.data) + s1 = run_tick(conn, capture, detector=detector) + assert s1["lake_written"] == half + + for r in rows[half:]: + capture.insert(r.table, r.pk, r.data) + s2 = run_tick(conn, capture, detector=detector) + assert s2["lake_written"] == len(rows) - half + assert s2["checkpoint_warehouse"] == capture.latest_sequence + + +# ── 4. stop-the-line ───────────────────────────────────────────────────────── + + +def test_breaking_drift_blocks_lake_and_warehouse(seeded, detector): + """ + Even if there is a fresh CDC event waiting, a breaking drift on the + source must abort BEFORE any data is moved. + """ + conn, capture = seeded + n_lake_before = conn.execute( + "SELECT COUNT(*) FROM lake_cdc_events" + ).fetchone()[0] + + # Make capture think a customer changed. + capture.update( + "customers", "c_alice", + before={"customer_id": "c_alice", "status": "active"}, + after={"customer_id": "c_alice", "status": "suspended"}, + captured_at=_ts(), + ) + + # Now corrupt the source schema (drop a contracted column). + for t in ("ledger_entries", "transfer_events", "transfers", + "payment_methods", "wallets"): + conn.execute(f"DROP TABLE IF EXISTS {t}") + conn.execute("ALTER TABLE customers DROP COLUMN email") + + stats = run_tick(conn, capture, detector=detector) + assert stats["aborted"] + assert stats["reason"] == "schema_contract_violation" + + n_lake_after = conn.execute( + "SELECT COUNT(*) FROM lake_cdc_events" + ).fetchone()[0] + # Nothing was written. + assert n_lake_before == n_lake_after + + +def test_strict_capture_refuses_writes_when_drift_breaking(seeded, detector): + conn, _ = seeded + for t in ("ledger_entries", "transfer_events", "transfers", + "payment_methods", "wallets"): + conn.execute(f"DROP TABLE IF EXISTS {t}") + conn.execute("ALTER TABLE customers DROP COLUMN email") + + strict = CDCCapture(detector, require_compatible_schema=True) + with pytest.raises(SchemaContractViolation): + strict.insert("customers", "c_x", {"customer_id": "c_x"}) + + +# ── 5. recovery from breaking drift ────────────────────────────────────────── + + +def test_pipeline_resumes_after_schema_repaired(seeded, detector): + """ + Schema break -> pipeline aborts -> source is repaired -> next tick + successfully drains queued events. + """ + from source.models import create_source_tables, drop_source_tables + + conn, capture = seeded + + capture.update( + "customers", "c_alice", + before={"customer_id": "c_alice", "status": "active"}, + after={"customer_id": "c_alice", "status": "suspended"}, + captured_at=_ts(), + ) + + # Break source. + for t in ("ledger_entries", "transfer_events", "transfers", + "payment_methods", "wallets"): + conn.execute(f"DROP TABLE IF EXISTS {t}") + conn.execute("ALTER TABLE customers DROP COLUMN email") + + s_aborted = run_tick(conn, capture, detector=detector) + assert s_aborted["aborted"] + + # Repair: rebuild source schema to match contracts. + drop_source_tables(conn) + create_source_tables(conn) + s_resumed = run_tick(conn, capture, detector=detector) + assert not s_resumed["aborted"] + assert s_resumed["warehouse_applied"] >= 1 + + +# ── 6. restore (history preserved) ─────────────────────────────────────────── + + +def test_restore_does_not_break_dq_rules(seeded): + """ + After restoring wallets to their seeded state, all DQ rules must + still pass. This catches restore bugs that would, e.g., leave + a stale _cdc_seq blocking future merges or violate ledger + reconciliation. + """ + conn, capture = seeded + + # Push a benign update on alice's wallet. + capture.update( + "wallets", "w_alice_usd", + before={"wallet_id": "w_alice_usd", "balance": Decimal("75.0000")}, + after={ + "wallet_id": "w_alice_usd", + "customer_id": "c_alice", + "currency": "USD", + "balance": Decimal("75.0000"), # unchanged + "status": "frozen", + "created_at": datetime(2026, 6, 1, 9, tzinfo=timezone.utc), + "updated_at": _ts(), + }, + captured_at=_ts(), + ) + apply_cdc_records(conn, capture.records_since(capture.latest_sequence - 1)) + + # Roll wh_wallets back to BEFORE the update. + restore_current_state_to(conn, "wallets", _ts(-1)) + + # Sanity: all wallets present, balance == seed + bal = conn.execute( + "SELECT balance FROM wh_wallets WHERE wallet_id = 'w_alice_usd'" + ).fetchone()[0] + assert bal == Decimal("75.0000") + + # All quality rules must still pass. + failures = run_checks(conn) + assert failures == [], ( + "restore should not corrupt DQ; got: " + + ", ".join(f"{r.rule_id}: {n}" for r, n in failures) + ) + + +# ── 7. delete + reinsert lifecycle ─────────────────────────────────────────── + + +def test_delete_then_reinsert_same_pk_is_handled(seeded): + conn, capture = seeded + capture.delete( + "transfers", "tx_failed_003", + before={"transfer_id": "tx_failed_003"}, + captured_at=_ts(0), + ) + apply_cdc_records(conn, capture.records_since(capture.latest_sequence - 1)) + + # Reinsert with the same PK after the delete (rare but possible). + capture.insert( + "transfers", "tx_failed_003", { + "transfer_id": "tx_failed_003", + "from_wallet_id": "w_chen_usd", + "to_wallet_id": "w_bob_gbp", + "payment_method_id": "pm_chen_upi", + "amount": Decimal("5.0000"), + "currency": "USD", + "status": "pending", + "reference": "groceries-retry", + "failure_reason": None, + "created_at": _ts(60), + "settled_at": None, + }, + captured_at=_ts(60), + ) + apply_cdc_records(conn, capture.records_since(capture.latest_sequence - 1)) + + row = conn.execute( + "SELECT status, _deleted FROM wh_transfers WHERE transfer_id = 'tx_failed_003'" + ).fetchone() + assert row == ("pending", False) + + +# ── fixtures local to this module ──────────────────────────────────────────── + + +@pytest.fixture() +def detector_factory_fresh_conn(conn, detector): + return conn, detector diff --git a/submission/wallet-payment-transfer/tests/test_lake.py b/submission/wallet-payment-transfer/tests/test_lake.py new file mode 100644 index 0000000..5379409 --- /dev/null +++ b/submission/wallet-payment-transfer/tests/test_lake.py @@ -0,0 +1,138 @@ +""" +Tests — lake immutability and completeness. + +Covers: + - every CDC operation lands in lake_cdc_events + - row count is monotonically non-decreasing + - duplicate sequence is rejected by PK (or silently deduped) + - before / after JSON is round-trippable + - replay_records returns events in sequence order + - lake retains every operation for the same PK +""" + +from __future__ import annotations + +import json + +from pipeline.lake import append_to_lake, replay_records + + +def test_seed_writes_all_events_to_lake(seeded): + conn, capture = seeded + n_lake = conn.execute( + "SELECT COUNT(*) FROM lake_cdc_events" + ).fetchone()[0] + assert n_lake == capture.latest_sequence + + +def test_lake_per_table_event_counts(seeded): + conn, _ = seeded + rows = conn.execute( + """ + SELECT table_name, COUNT(*) + FROM lake_cdc_events + GROUP BY table_name + """ + ).fetchall() + counts = dict(rows) + # 3 customers + 3 payment methods + 4 wallets + 3 transfers + # + 5 transfer_events + 5 ledger_entries = 23 inserts + assert counts == { + "customers": 3, + "payment_methods": 3, + "wallets": 4, + "transfers": 3, + "transfer_events": 5, + "ledger_entries": 5, + } + + +def test_lake_row_count_only_grows(seeded): + conn, capture = seeded + before = conn.execute( + "SELECT COUNT(*) FROM lake_cdc_events" + ).fetchone()[0] + + capture.update( + "customers", "c_alice", + before={"customer_id": "c_alice", "status": "active"}, + after ={"customer_id": "c_alice", "status": "suspended"}, + ) + new_records = capture.records_since(before) + append_to_lake(conn, new_records) + + after = conn.execute( + "SELECT COUNT(*) FROM lake_cdc_events" + ).fetchone()[0] + assert after > before + + +def test_lake_retains_full_pk_history(seeded): + conn, capture = seeded + capture.update( + "customers", "c_alice", + before={"customer_id": "c_alice", "status": "active"}, + after ={"customer_id": "c_alice", "status": "suspended"}, + ) + capture.delete( + "customers", "c_alice", before={"customer_id": "c_alice"} + ) + append_to_lake(conn, capture.records_since(0)) + + ops = [ + r[0] + for r in conn.execute( + """ + SELECT operation FROM lake_cdc_events + WHERE table_name = 'customers' AND primary_key = 'c_alice' + ORDER BY sequence + """ + ).fetchall() + ] + assert ops == ["insert", "update", "delete"] + + +def test_duplicate_sequence_is_deduped(seeded): + conn, capture = seeded + before = conn.execute( + "SELECT COUNT(*) FROM lake_cdc_events" + ).fetchone()[0] + + # Re-append the same records — the lake should silently dedup. + append_to_lake(conn, capture.records_since(0)) + after = conn.execute( + "SELECT COUNT(*) FROM lake_cdc_events" + ).fetchone()[0] + assert before == after + + +def test_replay_records_returns_in_sequence_order(seeded): + conn, _ = seeded + rows = replay_records(conn) + sequences = [r["sequence"] for r in rows] + assert sequences == sorted(sequences) + assert sequences[0] == 1 + + +def test_replay_records_filters_by_table(seeded): + conn, _ = seeded + customers = replay_records(conn, table="customers") + assert all(r["table"] == "customers" for r in customers) + + +def test_replay_records_filters_by_until_sequence(seeded): + conn, _ = seeded + until = 5 + rows = replay_records(conn, until_sequence=until) + assert all(r["sequence"] <= until for r in rows) + + +def test_lake_payload_is_valid_json_for_inserts(seeded): + conn, _ = seeded + row = conn.execute( + "SELECT after_image FROM lake_cdc_events " + "WHERE table_name = 'customers' AND operation = 'insert' LIMIT 1" + ).fetchone() + assert row is not None + parsed = json.loads(row[0]) + assert "customer_id" in parsed diff --git a/submission/wallet-payment-transfer/tests/test_schema_contracts.py b/submission/wallet-payment-transfer/tests/test_schema_contracts.py new file mode 100644 index 0000000..358a00f --- /dev/null +++ b/submission/wallet-payment-transfer/tests/test_schema_contracts.py @@ -0,0 +1,222 @@ +""" +Tests — schema-drift detection and stop-the-line behavior. + +Covers each detection rule in pipeline/schema_detector.py: + - missing_table + - column_dropped + - type_changed + - nullability_tightened + - column_added (warn, additive) + - non_nullable_column_added (breaking) + - primary_key_changed + - enum_value_added (warn) + +Plus the *system-level* stop-the-line: + - run_tick aborts when a breaking drift is present + - capture refuses writes when drift is breaking +""" + +from __future__ import annotations + +import duckdb +import pytest + +from pipeline.cdc import CDCCapture, SchemaContractViolation +from pipeline.orchestrator import bootstrap, run_tick +from pipeline.schema_detector import SchemaDetector, Severity +from source.contracts import CONTRACTS +from source.models import create_source_tables + + +def _fresh_source() -> duckdb.DuckDBPyConnection: + c = duckdb.connect(":memory:") + create_source_tables(c) + return c + + +def _drop_dependent_tables_for_alter(c: duckdb.DuckDBPyConnection) -> None: + """DuckDB rejects ALTER on tables with FK refs; drop FK children first.""" + for t in ("ledger_entries", "transfer_events", "transfers", + "payment_methods", "wallets"): + c.execute(f"DROP TABLE IF EXISTS {t}") + + +# ── happy path ─────────────────────────────────────────────────────────────── + + +def test_clean_source_passes_all_contracts(): + c = _fresh_source() + report = SchemaDetector(c).check_all() + assert not report.has_breaking() + + +def test_check_table_returns_one_report_per_table(): + c = _fresh_source() + report = SchemaDetector(c).check_all() + assert set(report.by_table) == set(CONTRACTS) + + +# ── missing table ──────────────────────────────────────────────────────────── + + +def test_missing_table_is_breaking(): + c = duckdb.connect(":memory:") + create_source_tables(c) + # Drop FK descendants then customers + _drop_dependent_tables_for_alter(c) + c.execute("DROP TABLE IF EXISTS customers") + drifts = SchemaDetector(c).check_table("customers").drifts + rules = {d.rule for d in drifts} + assert "missing_table" in rules + assert any(d.severity is Severity.BREAKING for d in drifts + if d.rule == "missing_table") + + +# ── dropped column ─────────────────────────────────────────────────────────── + + +def test_dropped_column_is_breaking(): + c = _fresh_source() + _drop_dependent_tables_for_alter(c) + c.execute("ALTER TABLE customers DROP COLUMN email") + report = SchemaDetector(c).check_table("customers") + assert any( + d.rule == "column_dropped" and d.column == "email" + and d.severity is Severity.BREAKING + for d in report.drifts + ) + + +# ── type changed ───────────────────────────────────────────────────────────── + + +def test_type_change_is_breaking(): + c = _fresh_source() + _drop_dependent_tables_for_alter(c) + c.execute("ALTER TABLE customers ALTER country_code TYPE INTEGER USING 0") + report = SchemaDetector(c).check_table("customers") + assert any( + d.rule == "type_changed" and d.column == "country_code" + and d.severity is Severity.BREAKING + for d in report.drifts + ) + + +# ── nullability tightened ──────────────────────────────────────────────────── + + +def test_nullability_tightening_is_breaking(): + """ + transfers.reference is nullable in the contract. If the source + later forbids NULL, the contract should report it as breaking. + """ + c = _fresh_source() + # Drop FK descendants of transfers. + c.execute("DROP TABLE IF EXISTS ledger_entries") + c.execute("DROP TABLE IF EXISTS transfer_events") + c.execute("ALTER TABLE transfers ALTER COLUMN reference SET NOT NULL") + report = SchemaDetector(c).check_table("transfers") + assert any( + d.rule == "nullability_tightened" and d.column == "reference" + and d.severity is Severity.BREAKING + for d in report.drifts + ) + + +# ── added column ───────────────────────────────────────────────────────────── + + +def test_new_nullable_column_is_warn_only(): + c = _fresh_source() + _drop_dependent_tables_for_alter(c) + c.execute("ALTER TABLE customers ADD COLUMN nickname VARCHAR") + report = SchemaDetector(c).check_table("customers") + assert not report.has_breaking() + warn_rules = {d.rule for d in report.warnings} + assert "column_added" in warn_rules + + +def test_new_non_null_column_is_breaking(): + c = _fresh_source() + _drop_dependent_tables_for_alter(c) + c.execute( + "ALTER TABLE customers ADD COLUMN region VARCHAR DEFAULT 'EU' NOT NULL" + ) + report = SchemaDetector(c).check_table("customers") + assert report.has_breaking() + assert any(d.rule == "non_nullable_column_added" for d in report.drifts) + + +# ── enum value drift ───────────────────────────────────────────────────────── + + +def test_unexpected_enum_value_in_data_is_warn(): + """ + The detector cannot prove an enum *removal* without source DDL access, + so it warns when it sees a *new* value present in data. We simulate by + bypassing the CHECK constraint with a DROP-then-recreate. + """ + c = duckdb.connect(":memory:") + c.execute(""" + CREATE TABLE customers ( + customer_id VARCHAR PRIMARY KEY, + full_name VARCHAR NOT NULL, + email VARCHAR NOT NULL, + country_code VARCHAR NOT NULL, + status VARCHAR NOT NULL, + registration_date DATE NOT NULL, + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL + ) + """) + c.execute( + "INSERT INTO customers VALUES " + "('c1', 'Alice', 'a@x.test', 'US', 'banned', " + "DATE '2025-01-01', TIMESTAMP '2026-06-01', TIMESTAMP '2026-06-01')" + ) + report = SchemaDetector(c).check_table("customers") + assert any(d.rule == "enum_value_added" for d in report.warnings) + + +# ── stop-the-line: orchestrator ────────────────────────────────────────────── + + +def test_run_tick_aborts_on_breaking_drift(conn, detector): + bootstrap(conn) + capture = CDCCapture() + capture.insert("customers", "c_x", {"customer_id": "c_x"}) + + # Drop a contracted column. + _drop_dependent_tables_for_alter(conn) + conn.execute("ALTER TABLE customers DROP COLUMN email") + + stats = run_tick(conn, capture, detector=detector) + assert stats["aborted"] is True + assert stats["reason"] == "schema_contract_violation" + assert stats["lake_written"] == 0 + assert stats["warehouse_applied"] == 0 + + +def test_run_tick_reports_drift_summary_on_abort(conn, detector): + bootstrap(conn) + capture = CDCCapture() + capture.insert("customers", "c_x", {"customer_id": "c_x"}) + _drop_dependent_tables_for_alter(conn) + conn.execute("ALTER TABLE customers DROP COLUMN email") + + stats = run_tick(conn, capture, detector=detector) + assert "email" in stats["drift_summary"] + assert "column_dropped" in stats["drift_summary"] + + +# ── stop-the-line: capture ─────────────────────────────────────────────────── + + +def test_capture_refuses_writes_when_strict(conn, detector): + _drop_dependent_tables_for_alter(conn) + conn.execute("ALTER TABLE customers DROP COLUMN email") + + strict = CDCCapture(detector, require_compatible_schema=True) + with pytest.raises(SchemaContractViolation): + strict.insert("customers", "c1", {"customer_id": "c1"}) + assert strict.log == [] diff --git a/submission/wallet-payment-transfer/tests/test_source_schema.py b/submission/wallet-payment-transfer/tests/test_source_schema.py new file mode 100644 index 0000000..18fc9b3 --- /dev/null +++ b/submission/wallet-payment-transfer/tests/test_source_schema.py @@ -0,0 +1,209 @@ +""" +Tests — source schema and constraint correctness. + +Covers system invariants enforced by the source DDL: + - PK uniqueness via duplicate insert rejection + - NOT NULL enforcement + - CHECK constraint enforcement (enums, non-negative balance, + positive amount, signed_amount consistency, settled_at >= created_at, + no self-transfer) + - FK referential integrity + - presence of declared indexes +""" + +from __future__ import annotations + +from datetime import date, datetime, timezone +from decimal import Decimal + +import duckdb +import pytest + +from source.models import TABLES_IN_ORDER, create_source_tables + + +def _t(year: int = 2026, month: int = 6, day: int = 1) -> datetime: + return datetime(year, month, day, tzinfo=timezone.utc) + + +@pytest.fixture() +def src() -> duckdb.DuckDBPyConnection: + c = duckdb.connect(":memory:") + create_source_tables(c) + # one valid customer, payment method and wallet so we can test + # downstream constraint rejections without setup noise + c.execute( + "INSERT INTO customers VALUES " + "('c1', 'Alice', 'a@x.test', 'US', 'active', ?, ?, ?)", + [date(2025, 1, 1), _t(), _t()], + ) + c.execute( + "INSERT INTO wallets VALUES " + "('w1', 'c1', 'USD', 100.00, 'active', ?, ?)", + [_t(), _t()], + ) + c.execute( + "INSERT INTO wallets VALUES " + "('w2', 'c1', 'USD', 0.00, 'active', ?, ?)", + [_t(), _t()], + ) + return c + + +# ── PK uniqueness ───────────────────────────────────────────────────────────── + + +def test_duplicate_customer_id_rejected(src): + src.execute( + "INSERT INTO customers VALUES " + "('c2', 'Bob', 'b@x.test', 'GB', 'active', ?, ?, ?)", + [date(2025, 1, 1), _t(), _t()], + ) + with pytest.raises(duckdb.ConstraintException): + src.execute( + "INSERT INTO customers VALUES " + "('c2', 'Bob2', 'b2@x.test', 'GB', 'active', ?, ?, ?)", + [date(2025, 1, 1), _t(), _t()], + ) + + +def test_duplicate_wallet_id_rejected(src): + with pytest.raises(duckdb.ConstraintException): + src.execute( + "INSERT INTO wallets VALUES " + "('w1', 'c1', 'USD', 0.00, 'active', ?, ?)", + [_t(), _t()], + ) + + +# ── NOT NULL ────────────────────────────────────────────────────────────────── + + +def test_null_customer_email_rejected(src): + with pytest.raises(duckdb.ConstraintException): + src.execute( + "INSERT INTO customers VALUES " + "('c3', 'Chen', NULL, 'US', 'active', ?, ?, ?)", + [date(2025, 1, 1), _t(), _t()], + ) + + +def test_null_wallet_currency_rejected(src): + with pytest.raises(duckdb.ConstraintException): + src.execute( + "INSERT INTO wallets VALUES " + "('w_bad', 'c1', NULL, 0.00, 'active', ?, ?)", + [_t(), _t()], + ) + + +# ── CHECK constraints ──────────────────────────────────────────────────────── + + +def test_negative_wallet_balance_rejected(src): + with pytest.raises(duckdb.ConstraintException): + src.execute( + "INSERT INTO wallets VALUES " + "('w_neg', 'c1', 'USD', -1.00, 'active', ?, ?)", + [_t(), _t()], + ) + + +def test_invalid_customer_status_rejected(src): + with pytest.raises(duckdb.ConstraintException): + src.execute( + "INSERT INTO customers VALUES " + "('c_bad', 'X', 'x@x.test', 'US', 'invalid', ?, ?, ?)", + [date(2025, 1, 1), _t(), _t()], + ) + + +def test_zero_transfer_amount_rejected(src): + with pytest.raises(duckdb.ConstraintException): + src.execute( + "INSERT INTO transfers VALUES (" + "'tx_bad', 'w1', 'w2', NULL, 0.00, 'USD', 'pending', " + "NULL, NULL, ?, NULL)", + [_t()], + ) + + +def test_self_transfer_rejected(src): + with pytest.raises(duckdb.ConstraintException): + src.execute( + "INSERT INTO transfers VALUES (" + "'tx_self', 'w1', 'w1', NULL, 1.00, 'USD', 'pending', " + "NULL, NULL, ?, NULL)", + [_t()], + ) + + +def test_settled_at_before_created_at_rejected(src): + with pytest.raises(duckdb.ConstraintException): + src.execute( + "INSERT INTO transfers VALUES (" + "'tx_back', 'w1', 'w2', NULL, 1.00, 'USD', 'settled', " + "NULL, NULL, ?, ?)", + [_t(2026, 6, 2), _t(2026, 6, 1)], + ) + + +def test_signed_amount_inconsistent_rejected(src): + # credit but signed_amount = -amount + with pytest.raises(duckdb.ConstraintException): + src.execute( + "INSERT INTO ledger_entries VALUES (" + "'le_bad', 'w1', NULL, 'credit', 10.00, -10.00, 0.00, ?)", + [_t()], + ) + + +# ── FK referential integrity ───────────────────────────────────────────────── + + +def test_wallet_references_existing_customer(src): + with pytest.raises(duckdb.ConstraintException): + src.execute( + "INSERT INTO wallets VALUES " + "('w_orph', 'c_missing', 'USD', 0.00, 'active', ?, ?)", + [_t(), _t()], + ) + + +def test_transfer_references_existing_wallets(src): + with pytest.raises(duckdb.ConstraintException): + src.execute( + "INSERT INTO transfers VALUES (" + "'tx_orph', 'w_missing', 'w2', NULL, 1.00, 'USD', 'pending', " + "NULL, NULL, ?, NULL)", + [_t()], + ) + + +# ── indexes are present ────────────────────────────────────────────────────── + + +@pytest.mark.parametrize("table", TABLES_IN_ORDER) +def test_table_exists(src: duckdb.DuckDBPyConnection, table: str): + rows = src.execute( + "SELECT 1 FROM information_schema.tables WHERE table_name = ?", + [table], + ).fetchall() + assert rows, f"table {table} not created" + + +def test_indexes_declared_for_cdc_workload(src: duckdb.DuckDBPyConnection): + indexes = [ + r[0] + for r in src.execute( + "SELECT index_name FROM duckdb_indexes()" + ).fetchall() + ] + must_have = [ + "ix_customers_updated_at", + "ix_wallets_customer", + "ix_transfers_status", + "ix_ledger_wallet_time", + ] + for ix in must_have: + assert ix in indexes, f"missing required index {ix}" diff --git a/submission/wallet-payment-transfer/tests/test_time_travel.py b/submission/wallet-payment-transfer/tests/test_time_travel.py new file mode 100644 index 0000000..fad7490 --- /dev/null +++ b/submission/wallet-payment-transfer/tests/test_time_travel.py @@ -0,0 +1,227 @@ +""" +Tests — point-in-time reconstruction (state_at) and restore. + +Covers: + - state_at returns the row that was current at ts + - state_at returns the most recent revision *strictly* before next change + - state_at excludes deleted rows + - state_at returns nothing before the row's _valid_from + - restore_current_state_to overwrites current-state with snapshot + - restore is non-destructive to history (history is preserved) + - restored state is a valid starting point for the next CDC tick +""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from decimal import Decimal + +import pytest + +from pipeline.orchestrator import run_tick +from pipeline.warehouse import ( + apply_cdc_records, + restore_current_state_to, + state_at, +) + + +def _ts(seconds: int = 0) -> datetime: + # Far enough in the future that we are always after the seed's + # captured_at (which uses datetime.now() at test-run time). + base = datetime(2030, 1, 1, 12, 0, tzinfo=timezone.utc) + return base + timedelta(seconds=seconds) + + +def _evolve_alice_balance(seeded, balances: list[Decimal]) -> list[datetime]: + """ + Push a series of balance updates onto w_alice_usd. + + Returns the list of captured_at timestamps used for each update so + the test can replay queries at exact boundaries. + """ + conn, capture = seeded + timestamps: list[datetime] = [] + prev = Decimal("75.0000") + for i, bal in enumerate(balances): + ts = _ts(i * 10) + capture.update( + "wallets", "w_alice_usd", + before={"wallet_id": "w_alice_usd", "balance": prev}, + after={ + "wallet_id": "w_alice_usd", + "customer_id": "c_alice", + "currency": "USD", + "balance": bal, + "status": "active", + "created_at": datetime(2026, 6, 1, 9, tzinfo=timezone.utc), + "updated_at": ts, + }, + captured_at=ts, + ) + timestamps.append(ts) + prev = bal + apply_cdc_records(conn, capture.records_since(capture.latest_sequence - len(balances))) + return timestamps + + +# ── state_at: basic reconstruction ─────────────────────────────────────────── + + +def test_state_at_returns_seed_state_after_seed(seeded): + conn, _ = seeded + rows = state_at(conn, "wallets", _ts(99999)) + by_id = {r["wallet_id"]: r["balance"] for r in rows} + assert by_id["w_alice_usd"] == Decimal("75.0000") + assert by_id["w_bob_gbp"] == Decimal("50.0000") + + +def test_state_at_picks_revision_active_at_timestamp(seeded): + conn, _ = seeded + timestamps = _evolve_alice_balance( + seeded, [Decimal("60.0000"), Decimal("40.0000"), Decimal("20.0000")] + ) + + # Just before the first new revision -> still 75 (the seed value) + rows = state_at(conn, "wallets", timestamps[0] - timedelta(seconds=1)) + by_id = {r["wallet_id"]: r["balance"] for r in rows} + assert by_id["w_alice_usd"] == Decimal("75.0000") + + # At the first new revision exactly -> 60 + rows = state_at(conn, "wallets", timestamps[0]) + assert {r["wallet_id"]: r["balance"] for r in rows}["w_alice_usd"] == \ + Decimal("60.0000") + + # Between revisions -> middle value (40) + rows = state_at(conn, "wallets", timestamps[1] + timedelta(seconds=1)) + assert {r["wallet_id"]: r["balance"] for r in rows}["w_alice_usd"] == \ + Decimal("40.0000") + + # After the last revision -> 20 + rows = state_at(conn, "wallets", timestamps[2] + timedelta(seconds=1)) + assert {r["wallet_id"]: r["balance"] for r in rows}["w_alice_usd"] == \ + Decimal("20.0000") + + +def test_state_at_respects_valid_from_lower_bound(seeded): + conn, _ = seeded + # Long before the first event — wallet table should have been empty + rows = state_at(conn, "wallets", datetime(2020, 1, 1, tzinfo=timezone.utc)) + assert rows == [] + + +def test_state_at_excludes_deleted_rows(seeded): + conn, capture = seeded + delete_ts = _ts(0) + capture.delete( + "transfers", "tx_pending_002", + before={"transfer_id": "tx_pending_002"}, + captured_at=delete_ts, + ) + apply_cdc_records(conn, capture.records_since(capture.latest_sequence - 1)) + + # After the delete, snapshot must not include the deleted PK. + rows = state_at(conn, "transfers", delete_ts + timedelta(seconds=1)) + pks = {r["transfer_id"] for r in rows} + assert "tx_pending_002" not in pks + + # Before the delete, snapshot must include the deleted PK. + rows = state_at(conn, "transfers", delete_ts - timedelta(seconds=1)) + pks = {r["transfer_id"] for r in rows} + assert "tx_pending_002" in pks + + +# ── restore current state ──────────────────────────────────────────────────── + + +def test_restore_overwrites_current_state(seeded): + conn, _ = seeded + timestamps = _evolve_alice_balance( + seeded, [Decimal("60.0000"), Decimal("40.0000")] + ) + + # Roll wh_wallets back to the moment between the two updates + n = restore_current_state_to( + conn, "wallets", timestamps[0] + timedelta(seconds=1) + ) + assert n >= 1 + + bal = conn.execute( + "SELECT balance FROM wh_wallets WHERE wallet_id = 'w_alice_usd'" + ).fetchone()[0] + assert bal == Decimal("60.0000") + + +def test_restore_does_not_touch_history(seeded): + conn, _ = seeded + timestamps = _evolve_alice_balance( + seeded, [Decimal("60.0000"), Decimal("40.0000")] + ) + n_hist_before = conn.execute( + "SELECT COUNT(*) FROM wh_wallets_history" + ).fetchone()[0] + + restore_current_state_to(conn, "wallets", timestamps[0]) + n_hist_after = conn.execute( + "SELECT COUNT(*) FROM wh_wallets_history" + ).fetchone()[0] + + assert n_hist_before == n_hist_after + + +def test_restore_to_pre_history_results_in_empty_table(seeded): + conn, _ = seeded + n = restore_current_state_to( + conn, "wallets", datetime(2020, 1, 1, tzinfo=timezone.utc) + ) + assert n == 0 + cnt = conn.execute("SELECT COUNT(*) FROM wh_wallets").fetchone()[0] + assert cnt == 0 + + +def test_restored_state_accepts_new_cdc_records(seeded): + """ + After restore, _cdc_seq is reset to 0; the next CDC event (with seq > 0) + must apply cleanly without being rejected as out-of-order. + """ + conn, capture = seeded + timestamps = _evolve_alice_balance(seeded, [Decimal("60.0000")]) + restore_current_state_to(conn, "wallets", timestamps[0]) + + # Now push a fresh update — it should land. + capture.update( + "wallets", "w_alice_usd", + before={"wallet_id": "w_alice_usd", "balance": Decimal("60.0000")}, + after={ + "wallet_id": "w_alice_usd", + "customer_id": "c_alice", + "currency": "USD", + "balance": Decimal("12.3456"), + "status": "active", + "created_at": datetime(2026, 6, 1, 9, tzinfo=timezone.utc), + "updated_at": _ts(60), + }, + captured_at=_ts(60), + ) + apply_cdc_records(conn, capture.records_since(capture.latest_sequence - 1)) + + bal = conn.execute( + "SELECT balance FROM wh_wallets WHERE wallet_id = 'w_alice_usd'" + ).fetchone()[0] + assert bal == Decimal("12.3456") + + +# ── parity: state_at over multiple tables stays consistent ─────────────────── + + +def test_state_at_customers_count_matches_seed(seeded): + conn, _ = seeded + rows = state_at(conn, "customers", _ts(99999)) + assert len(rows) == 3 + + +def test_state_at_ledger_entries_includes_funding_entries(seeded): + conn, _ = seeded + rows = state_at(conn, "ledger_entries", _ts(99999)) + funding = [r for r in rows if r["transfer_id"] is None] + assert len(funding) == 3 diff --git a/submission/wallet-payment-transfer/tests/test_warehouse.py b/submission/wallet-payment-transfer/tests/test_warehouse.py new file mode 100644 index 0000000..79d18d0 --- /dev/null +++ b/submission/wallet-payment-transfer/tests/test_warehouse.py @@ -0,0 +1,271 @@ +""" +Tests — warehouse modeling: current-state + SCD2 history. + +Covers: + - inserts land in current-state and history + - updates upsert current-state and close+open history rows + - deletes soft-delete current-state and append a delete history row + - duplicate / replayed records are no-ops + - out-of-order events do not clobber newer state (sequence guard) + - SCD2 row count grows by 1 per applied event + - exactly one row is _is_current=true per PK at any time +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from decimal import Decimal + +import pytest + +from pipeline.cdc import CDCRecord +from pipeline.orchestrator import run_tick +from pipeline.warehouse import apply_cdc_records + + +def _t(seconds: int = 0) -> datetime: + base = datetime(2026, 6, 2, 12, 0, tzinfo=timezone.utc) + return base.replace(second=seconds % 60) + + +# ── inserts propagate ──────────────────────────────────────────────────────── + + +def test_insert_appears_in_warehouse(seeded): + conn, _ = seeded + row = conn.execute( + "SELECT full_name, country_code " + "FROM wh_customers WHERE customer_id = 'c_alice'" + ).fetchone() + assert row == ("Alice Anderson", "US") + + +def test_all_warehouse_tables_populated_after_seed(seeded): + conn, _ = seeded + counts = { + t: conn.execute(f"SELECT COUNT(*) FROM wh_{t}").fetchone()[0] + for t in ( + "customers", + "payment_methods", + "wallets", + "transfers", + "transfer_events", + "ledger_entries", + ) + } + assert counts == { + "customers": 3, + "payment_methods": 3, + "wallets": 4, + "transfers": 3, + "transfer_events": 5, + "ledger_entries": 5, + } + + +def test_insert_creates_current_history_row(seeded): + conn, _ = seeded + row = conn.execute( + "SELECT _is_current, _op FROM wh_customers_history " + "WHERE customer_id = 'c_alice'" + ).fetchone() + assert row == (True, "insert") + + +# ── updates ────────────────────────────────────────────────────────────────── + + +def test_update_overwrites_current_state(seeded): + conn, capture = seeded + capture.update( + "customers", "c_alice", + before={"customer_id": "c_alice", "status": "active"}, + after={ + "customer_id": "c_alice", + "full_name": "Alice A. Anderson", + "email": "alice@example.invalid", + "country_code": "US", + "status": "suspended", + "registration_date": "2024-01-15", + "created_at": _t(), + "updated_at": _t(), + }, + ) + apply_cdc_records(conn, capture.records_since(capture.latest_sequence - 1)) + + row = conn.execute( + "SELECT full_name, status FROM wh_customers " + "WHERE customer_id = 'c_alice'" + ).fetchone() + assert row == ("Alice A. Anderson", "suspended") + + +def test_update_closes_old_history_row_and_opens_new(seeded): + conn, capture = seeded + capture.update( + "wallets", "w_alice_usd", + before={"wallet_id": "w_alice_usd", "balance": Decimal("75.0000")}, + after={ + "wallet_id": "w_alice_usd", + "customer_id": "c_alice", + "currency": "USD", + "balance": Decimal("60.0000"), + "status": "active", + "created_at": _t(), + "updated_at": _t(1), + }, + ) + apply_cdc_records(conn, capture.records_since(capture.latest_sequence - 1)) + + rows = conn.execute( + """ + SELECT _is_current, _op, balance + FROM wh_wallets_history + WHERE wallet_id = 'w_alice_usd' + ORDER BY _valid_from + """ + ).fetchall() + assert len(rows) == 2 + assert rows[0][0] is False # old version closed + assert rows[1][0] is True # new version current + assert rows[1][2] == Decimal("60.0000") + + +def test_only_one_current_history_row_per_pk(seeded): + conn, _ = seeded + rows = conn.execute( + """ + SELECT wallet_id, COUNT(*) + FROM wh_wallets_history + WHERE _is_current = TRUE + GROUP BY wallet_id + HAVING COUNT(*) <> 1 + """ + ).fetchall() + assert rows == [] + + +# ── deletes ────────────────────────────────────────────────────────────────── + + +def test_delete_marks_current_state_deleted(seeded): + conn, capture = seeded + capture.delete( + "transfers", "tx_pending_002", + before={"transfer_id": "tx_pending_002"}, + ) + apply_cdc_records(conn, capture.records_since(capture.latest_sequence - 1)) + + deleted = conn.execute( + "SELECT _deleted FROM wh_transfers WHERE transfer_id = 'tx_pending_002'" + ).fetchone()[0] + assert deleted is True + + +def test_delete_appends_history_row_with_op_delete(seeded): + conn, capture = seeded + capture.delete( + "payment_methods", "pm_chen_upi", + before={"payment_method_id": "pm_chen_upi"}, + ) + apply_cdc_records(conn, capture.records_since(capture.latest_sequence - 1)) + + last = conn.execute( + """ + SELECT _op, _is_current + FROM wh_payment_methods_history + WHERE payment_method_id = 'pm_chen_upi' + ORDER BY _valid_from DESC LIMIT 1 + """ + ).fetchone() + assert last == ("delete", False) + + +def test_delete_does_not_remove_current_row(seeded): + conn, capture = seeded + capture.delete( + "customers", "c_alice", before={"customer_id": "c_alice"} + ) + apply_cdc_records(conn, capture.records_since(capture.latest_sequence - 1)) + + row = conn.execute( + "SELECT _deleted FROM wh_customers WHERE customer_id = 'c_alice'" + ).fetchone() + assert row is not None + assert row[0] is True + + +# ── replay safety / idempotency ────────────────────────────────────────────── + + +def test_replay_does_not_double_apply(seeded): + conn, capture = seeded + n_before = conn.execute("SELECT COUNT(*) FROM wh_customers").fetchone()[0] + n_hist_before = conn.execute( + "SELECT COUNT(*) FROM wh_customers_history" + ).fetchone()[0] + + apply_cdc_records(conn, capture.records_since(0)) + n_after = conn.execute("SELECT COUNT(*) FROM wh_customers").fetchone()[0] + n_hist_after = conn.execute( + "SELECT COUNT(*) FROM wh_customers_history" + ).fetchone()[0] + + # Current-state count unchanged. + assert n_after == n_before + # History count unchanged because sequence guard rejects duplicates. + assert n_hist_after == n_hist_before + + +def test_orchestrator_run_tick_is_idempotent(seeded, detector): + conn, capture = seeded + n_lake_before = conn.execute( + "SELECT COUNT(*) FROM lake_cdc_events" + ).fetchone()[0] + stats = run_tick(conn, capture, detector=detector) + n_lake_after = conn.execute( + "SELECT COUNT(*) FROM lake_cdc_events" + ).fetchone()[0] + # Already applied — second tick should be a no-op + assert stats["lake_written"] == 0 + assert stats["warehouse_applied"] == 0 + assert n_lake_before == n_lake_after + + +# ── out-of-order arrival ───────────────────────────────────────────────────── + + +@pytest.mark.parametrize("scenario", ["older_arrives_late"]) +def test_older_event_does_not_clobber_newer_state(seeded, scenario): + """ + If a new sequence has already been applied, a later-arriving event with + a smaller sequence must NOT modify the current-state row. + """ + conn, capture = seeded + # Take the latest applied seq. + latest = capture.latest_sequence + # Forge an out-of-order CDC record with a lower sequence number + stale = CDCRecord( + operation="update", + table="customers", + primary_key="c_alice", + before={"customer_id": "c_alice", "full_name": "Alice Anderson"}, + after={ + "customer_id": "c_alice", + "full_name": "Stale Name", + "email": "alice@example.invalid", + "country_code": "US", + "status": "active", + "registration_date": "2024-01-15", + "created_at": _t(), + "updated_at": _t(), + }, + sequence=1, # smaller than what we already applied + ) + apply_cdc_records(conn, [stale]) + + name = conn.execute( + "SELECT full_name FROM wh_customers WHERE customer_id = 'c_alice'" + ).fetchone()[0] + assert name == "Alice Anderson" # original, not 'Stale Name' + _ = latest From 04754707fba1e83e4653e4ad5596a92f5f92fd79 Mon Sep 17 00:00:00 2001 From: Garv Pushkarna Date: Thu, 25 Jun 2026 15:36:51 +0530 Subject: [PATCH 2/2] update sample-candidate --- .../sample-candidate/catalog/catalog.json | 77 ++++++ submission/sample-candidate/conftest.py | 6 + .../sample-candidate/pipeline/__init__.py | 0 submission/sample-candidate/pipeline/cdc.py | 89 +++++++ submission/sample-candidate/pipeline/lake.py | 69 +++++ .../sample-candidate/pipeline/warehouse.py | 124 +++++++++ submission/sample-candidate/requirements.txt | 2 + .../scripts/check_schema_contracts.py | 60 +++++ .../scripts/run_data_quality_checks.py | 212 +++++++++++++++ .../scripts/validate_catalog.py | 77 ++++++ .../sample-candidate/source/__init__.py | 0 submission/sample-candidate/source/models.py | 84 ++++++ submission/sample-candidate/tests/conftest.py | 125 +++++++++ .../sample-candidate/tests/test_catalog.py | 134 ++++++++++ submission/sample-candidate/tests/test_cdc.py | 135 ++++++++++ .../tests/test_data_quality.py | 252 ++++++++++++++++++ .../tests/test_schema_contracts.py | 170 ++++++++++++ 17 files changed, 1616 insertions(+) create mode 100644 submission/sample-candidate/catalog/catalog.json create mode 100644 submission/sample-candidate/conftest.py create mode 100644 submission/sample-candidate/pipeline/__init__.py create mode 100644 submission/sample-candidate/pipeline/cdc.py create mode 100644 submission/sample-candidate/pipeline/lake.py create mode 100644 submission/sample-candidate/pipeline/warehouse.py create mode 100644 submission/sample-candidate/requirements.txt create mode 100644 submission/sample-candidate/scripts/check_schema_contracts.py create mode 100644 submission/sample-candidate/scripts/run_data_quality_checks.py create mode 100644 submission/sample-candidate/scripts/validate_catalog.py create mode 100644 submission/sample-candidate/source/__init__.py create mode 100644 submission/sample-candidate/source/models.py create mode 100644 submission/sample-candidate/tests/conftest.py create mode 100644 submission/sample-candidate/tests/test_catalog.py create mode 100644 submission/sample-candidate/tests/test_cdc.py create mode 100644 submission/sample-candidate/tests/test_data_quality.py create mode 100644 submission/sample-candidate/tests/test_schema_contracts.py diff --git a/submission/sample-candidate/catalog/catalog.json b/submission/sample-candidate/catalog/catalog.json new file mode 100644 index 0000000..83dfa55 --- /dev/null +++ b/submission/sample-candidate/catalog/catalog.json @@ -0,0 +1,77 @@ +{ + "datasets": [ + { + "name": "lake_cdc_events", + "layer": "lake", + "description": "Append-only log of every CDC event captured from the payments source system. Preserves full change history for replay and point-in-time recovery.", + "owner": "data-platform", + "consumers": ["data-platform", "analytics", "audit"], + "update_cadence": "real-time", + "schema": { + "sequence": "INTEGER — monotonic capture offset", + "operation": "VARCHAR — insert | update | delete", + "table_name": "VARCHAR — source table name", + "primary_key": "VARCHAR — source row PK value", + "data": "VARCHAR (JSON) — full row snapshot at capture time", + "captured_at": "TIMESTAMP — UTC capture time" + } + }, + { + "name": "wh_customers", + "layer": "warehouse", + "description": "Current-state customer snapshot. Reflects the latest known state of each customer row. Soft-deleted rows are flagged with _deleted=true.", + "owner": "data-platform", + "consumers": ["analytics", "product", "finance"], + "update_cadence": "near-real-time", + "schema": { + "customer_id": "VARCHAR — primary key", + "name": "VARCHAR", + "email": "VARCHAR", + "status": "VARCHAR — active | suspended | closed", + "created_at": "TIMESTAMP", + "updated_at": "TIMESTAMP", + "_cdc_seq": "INTEGER — sequence of last CDC event", + "_deleted": "BOOLEAN — true if source row was deleted" + } + }, + { + "name": "wh_wallets", + "layer": "warehouse", + "description": "Current-state wallet snapshot. Balance reflects the latest value written via CDC. Negative balances are a data quality violation.", + "owner": "data-platform", + "consumers": ["analytics", "finance"], + "update_cadence": "near-real-time", + "schema": { + "wallet_id": "VARCHAR — primary key", + "customer_id": "VARCHAR — FK to wh_customers", + "balance": "DECIMAL(18,2)", + "currency": "VARCHAR", + "status": "VARCHAR — active | frozen | closed", + "created_at": "TIMESTAMP", + "updated_at": "TIMESTAMP", + "_cdc_seq": "INTEGER", + "_deleted": "BOOLEAN" + } + }, + { + "name": "wh_transactions", + "layer": "warehouse", + "description": "Current-state transaction snapshot. Each row is the latest state of a transaction from the source. Amount must always be positive.", + "owner": "data-platform", + "consumers": ["analytics", "finance", "audit"], + "update_cadence": "near-real-time", + "schema": { + "transaction_id": "VARCHAR — primary key", + "wallet_id": "VARCHAR — FK to wh_wallets", + "amount": "DECIMAL(18,2) — always > 0", + "direction": "VARCHAR — credit | debit", + "status": "VARCHAR — pending | settled | failed | reversed", + "reference": "VARCHAR — nullable external reference", + "created_at": "TIMESTAMP", + "settled_at": "TIMESTAMP — nullable until settled", + "_cdc_seq": "INTEGER", + "_deleted": "BOOLEAN" + } + } + ] +} diff --git a/submission/sample-candidate/conftest.py b/submission/sample-candidate/conftest.py new file mode 100644 index 0000000..ce3c67b --- /dev/null +++ b/submission/sample-candidate/conftest.py @@ -0,0 +1,6 @@ +"""Root conftest — adds submission/ to sys.path so tests can import source/pipeline.""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(__file__)) diff --git a/submission/sample-candidate/pipeline/__init__.py b/submission/sample-candidate/pipeline/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/submission/sample-candidate/pipeline/cdc.py b/submission/sample-candidate/pipeline/cdc.py new file mode 100644 index 0000000..c8c2dcb --- /dev/null +++ b/submission/sample-candidate/pipeline/cdc.py @@ -0,0 +1,89 @@ +""" +CDC capture layer. + +Simulates WAL-based change capture: every insert/update/delete on the source +produces a CDCRecord with a monotonically increasing sequence number. + +Replay safety: callers can checkpoint the last processed sequence and call +records_since(offset) to replay only unprocessed changes after a restart. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any + +VALID_OPERATIONS = frozenset({"insert", "update", "delete"}) + + +@dataclass +class CDCRecord: + operation: str + table: str + primary_key: str + data: dict[str, Any] + captured_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) + sequence: int = 0 + + def __post_init__(self) -> None: + if self.operation not in VALID_OPERATIONS: + raise ValueError( + f"Invalid CDC operation {self.operation!r}. " + f"Must be one of: {sorted(VALID_OPERATIONS)}" + ) + + +class CDCCapture: + """ + In-process CDC log. + + Production analogue: a Debezium/Kafka connector reading Postgres WAL. + Each record carries a sequence number equivalent to a Kafka offset or + Postgres LSN for checkpoint-based replay. + """ + + def __init__(self) -> None: + self._log: list[CDCRecord] = [] + self._seq: int = 0 + + # ── public write API ───────────────────────────────────────────────────── + + def insert(self, table: str, pk: str, data: dict[str, Any]) -> CDCRecord: + return self._record("insert", table, pk, data) + + def update(self, table: str, pk: str, data: dict[str, Any]) -> CDCRecord: + return self._record("update", table, pk, data) + + def delete(self, table: str, pk: str, data: dict[str, Any]) -> CDCRecord: + return self._record("delete", table, pk, data) + + # ── public read / replay API ───────────────────────────────────────────── + + def records_since(self, offset: int = 0) -> list[CDCRecord]: + """Return all records with sequence > offset (checkpoint replay).""" + return [r for r in self._log if r.sequence > offset] + + @property + def latest_sequence(self) -> int: + return self._seq + + @property + def log(self) -> list[CDCRecord]: + return list(self._log) + + # ── internal ───────────────────────────────────────────────────────────── + + def _record( + self, operation: str, table: str, pk: str, data: dict[str, Any] + ) -> CDCRecord: + self._seq += 1 + rec = CDCRecord( + operation=operation, + table=table, + primary_key=pk, + data=data, + sequence=self._seq, + ) + self._log.append(rec) + return rec diff --git a/submission/sample-candidate/pipeline/lake.py b/submission/sample-candidate/pipeline/lake.py new file mode 100644 index 0000000..7cb15a7 --- /dev/null +++ b/submission/sample-candidate/pipeline/lake.py @@ -0,0 +1,69 @@ +""" +Lake layer — append-only storage for every CDC event. + +Every change is written exactly once. The lake is the source of truth for +point-in-time replay and historical reconstruction. + +Production analogue: Parquet/Delta files on S3 or GCS, partitioned by +table_name and captured_at date. No row is ever modified or deleted. +""" + +from __future__ import annotations + +import json +from datetime import datetime + +import duckdb + +from pipeline.cdc import CDCRecord + + +def create_lake_table(conn: duckdb.DuckDBPyConnection) -> None: + conn.execute(""" + CREATE TABLE IF NOT EXISTS lake_cdc_events ( + sequence INTEGER NOT NULL, + operation VARCHAR NOT NULL, + table_name VARCHAR NOT NULL, + primary_key VARCHAR NOT NULL, + data VARCHAR NOT NULL, + captured_at TIMESTAMP NOT NULL + ) + """) + + +def append_to_lake(conn: duckdb.DuckDBPyConnection, records: list[CDCRecord]) -> int: + """ + Append CDC records to the lake. + + Returns the number of records written. + Idempotency note: in production, deduplicate by sequence before appending. + """ + if not records: + return 0 + + rows = [ + ( + r.sequence, + r.operation, + r.table, + r.primary_key, + _serialize(r.data), + r.captured_at, + ) + for r in records + ] + conn.executemany( + "INSERT INTO lake_cdc_events VALUES (?, ?, ?, ?, ?, ?)", + rows, + ) + return len(rows) + + +def _serialize(data: dict) -> str: + return json.dumps(data, default=_json_default) + + +def _json_default(obj: object) -> str: + if isinstance(obj, datetime): + return obj.isoformat() + raise TypeError(f"Object of type {type(obj).__name__} is not JSON serializable") diff --git a/submission/sample-candidate/pipeline/warehouse.py b/submission/sample-candidate/pipeline/warehouse.py new file mode 100644 index 0000000..ab32401 --- /dev/null +++ b/submission/sample-candidate/pipeline/warehouse.py @@ -0,0 +1,124 @@ +""" +Warehouse layer — current-state snapshot built from CDC events. + +Each warehouse table mirrors the source table with two extra columns: + _cdc_seq : sequence of the last CDC event that touched this row + _deleted : soft-delete flag set when a DELETE event is received + +Production analogue: BigQuery/Snowflake tables refreshed by a streaming +merge job keyed on primary key. SCD2 history tables would sit alongside +these current-state tables for time-travel queries. +""" + +from __future__ import annotations + +import duckdb + +from pipeline.cdc import CDCRecord + +# Source table → warehouse table +_TABLE_MAP: dict[str, str] = { + "customers": "wh_customers", + "wallets": "wh_wallets", + "transactions": "wh_transactions", +} + +# Source table → primary key column name +_PK_MAP: dict[str, str] = { + "customers": "customer_id", + "wallets": "wallet_id", + "transactions": "transaction_id", +} + + +def create_warehouse_tables(conn: duckdb.DuckDBPyConnection) -> None: + conn.execute(""" + CREATE TABLE IF NOT EXISTS wh_customers ( + customer_id VARCHAR PRIMARY KEY, + name VARCHAR, + email VARCHAR, + status VARCHAR, + created_at TIMESTAMP, + updated_at TIMESTAMP, + _cdc_seq INTEGER NOT NULL, + _deleted BOOLEAN NOT NULL DEFAULT false + ) + """) + + conn.execute(""" + CREATE TABLE IF NOT EXISTS wh_wallets ( + wallet_id VARCHAR PRIMARY KEY, + customer_id VARCHAR, + balance DECIMAL(18, 2), + currency VARCHAR, + status VARCHAR, + created_at TIMESTAMP, + updated_at TIMESTAMP, + _cdc_seq INTEGER NOT NULL, + _deleted BOOLEAN NOT NULL DEFAULT false + ) + """) + + conn.execute(""" + CREATE TABLE IF NOT EXISTS wh_transactions ( + transaction_id VARCHAR PRIMARY KEY, + wallet_id VARCHAR, + amount DECIMAL(18, 2), + direction VARCHAR, + status VARCHAR, + reference VARCHAR, + created_at TIMESTAMP, + settled_at TIMESTAMP, + _cdc_seq INTEGER NOT NULL, + _deleted BOOLEAN NOT NULL DEFAULT false + ) + """) + + +def apply_cdc_records( + conn: duckdb.DuckDBPyConnection, records: list[CDCRecord] +) -> None: + """ + Apply CDC records to the warehouse current-state tables in sequence order. + + - insert / update → upsert (insert new row or overwrite existing) + - delete → set _deleted = true + """ + for record in sorted(records, key=lambda r: r.sequence): + wh_table = _TABLE_MAP.get(record.table) + pk_col = _PK_MAP.get(record.table) + if not wh_table or not pk_col: + continue + + pk_val = record.primary_key + + if record.operation == "delete": + conn.execute( + f"UPDATE {wh_table} SET _deleted = true, _cdc_seq = ?" + f" WHERE {pk_col} = ?", + [record.sequence, pk_val], + ) + continue + + data = {**record.data, "_cdc_seq": record.sequence, "_deleted": False} + cols = list(data.keys()) + vals = list(data.values()) + placeholders = ", ".join(["?"] * len(vals)) + + existing = conn.execute( + f"SELECT COUNT(*) FROM {wh_table} WHERE {pk_col} = ?", + [pk_val], + ).fetchone()[0] + + if existing: + set_clause = ", ".join([f"{c} = ?" for c in cols]) + conn.execute( + f"UPDATE {wh_table} SET {set_clause} WHERE {pk_col} = ?", + vals + [pk_val], + ) + else: + col_list = ", ".join(cols) + conn.execute( + f"INSERT INTO {wh_table} ({col_list}) VALUES ({placeholders})", + vals, + ) diff --git a/submission/sample-candidate/requirements.txt b/submission/sample-candidate/requirements.txt new file mode 100644 index 0000000..ab2841c --- /dev/null +++ b/submission/sample-candidate/requirements.txt @@ -0,0 +1,2 @@ +duckdb>=0.10.0 +pytest>=7.4 diff --git a/submission/sample-candidate/scripts/check_schema_contracts.py b/submission/sample-candidate/scripts/check_schema_contracts.py new file mode 100644 index 0000000..1698b21 --- /dev/null +++ b/submission/sample-candidate/scripts/check_schema_contracts.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +""" +check_schema_contracts.py + +Validates that the source tables expose all columns defined in the schema +contract. A missing column means downstream CDC logic or warehouse models +will break — this script fails the build before that happens. + +Exit 0 — all contracts pass. +Exit 1 — one or more violations found. +""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import duckdb + +from source.models import SCHEMA_CONTRACT, create_source_tables + + +def check_contracts(conn: duckdb.DuckDBPyConnection) -> list[str]: + """Return a list of violation messages. Empty list = all passed.""" + violations: list[str] = [] + + for table, expected_cols in SCHEMA_CONTRACT.items(): + try: + rows = conn.execute(f"DESCRIBE {table}").fetchall() + except Exception as exc: + violations.append(f"{table}: could not describe table — {exc}") + continue + + actual_cols = {row[0] for row in rows} + + for col in expected_cols: + if col not in actual_cols: + violations.append(f"{table}.{col}: column missing from source table") + + return violations + + +def main() -> int: + conn = duckdb.connect(":memory:") + create_source_tables(conn) + + violations = check_contracts(conn) + + if violations: + print("Schema contract violations:") + for v in violations: + print(f" ✗ {v}") + return 1 + + print(f"All schema contracts passed ({len(SCHEMA_CONTRACT)} tables checked).") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/submission/sample-candidate/scripts/run_data_quality_checks.py b/submission/sample-candidate/scripts/run_data_quality_checks.py new file mode 100644 index 0000000..31e4cba --- /dev/null +++ b/submission/sample-candidate/scripts/run_data_quality_checks.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +""" +run_data_quality_checks.py + +Runs system and business data quality validations against the warehouse. +Seeds an in-memory database with representative data, applies CDC events, +then asserts correctness invariants. + +System checks : PK uniqueness, not-null, referential integrity +Business checks : non-negative balances, positive amounts, valid status enums + +Exit 0 — all checks pass. +Exit 1 — one or more failures found. +""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from datetime import datetime, timezone + +import duckdb + +from pipeline.cdc import CDCCapture +from pipeline.lake import append_to_lake, create_lake_table +from pipeline.warehouse import apply_cdc_records, create_warehouse_tables + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def seed(conn: duckdb.DuckDBPyConnection, capture: CDCCapture) -> None: + """Populate lake + warehouse with representative test data.""" + ts = _now() + + capture.insert( + "customers", + "c1", + { + "customer_id": "c1", + "name": "Alice", + "email": "alice-test-user", + "status": "active", + "created_at": ts, + "updated_at": ts, + }, + ) + capture.insert( + "customers", + "c2", + { + "customer_id": "c2", + "name": "Bob", + "email": "bob-test-user", + "status": "active", + "created_at": ts, + "updated_at": ts, + }, + ) + + capture.insert( + "wallets", + "w1", + { + "wallet_id": "w1", + "customer_id": "c1", + "balance": 100.00, + "currency": "USD", + "status": "active", + "created_at": ts, + "updated_at": ts, + }, + ) + capture.insert( + "wallets", + "w2", + { + "wallet_id": "w2", + "customer_id": "c2", + "balance": 50.00, + "currency": "USD", + "status": "active", + "created_at": ts, + "updated_at": ts, + }, + ) + + capture.insert( + "transactions", + "t1", + { + "transaction_id": "t1", + "wallet_id": "w1", + "amount": 25.00, + "direction": "credit", + "status": "settled", + "reference": "ref-001", + "created_at": ts, + "settled_at": ts, + }, + ) + capture.insert( + "transactions", + "t2", + { + "transaction_id": "t2", + "wallet_id": "w2", + "amount": 10.00, + "direction": "debit", + "status": "settled", + "reference": "ref-002", + "created_at": ts, + "settled_at": ts, + }, + ) + + create_lake_table(conn) + create_warehouse_tables(conn) + records = capture.records_since(0) + append_to_lake(conn, records) + apply_cdc_records(conn, records) + + +def run_checks(conn: duckdb.DuckDBPyConnection) -> list[str]: + failures: list[str] = [] + + # ── system checks ───────────────────────────────────────────────────────── + + pk_map = { + "wh_customers": "customer_id", + "wh_wallets": "wallet_id", + "wh_transactions": "transaction_id", + } + for table, pk in pk_map.items(): + total = conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] + distinct = conn.execute(f"SELECT COUNT(DISTINCT {pk}) FROM {table}").fetchone()[ + 0 + ] + if total != distinct: + failures.append( + f"{table}: PK not unique — {total} rows, {distinct} distinct {pk}" + ) + + not_null_checks = [ + ("wh_customers", "name"), + ("wh_customers", "email"), + ("wh_wallets", "customer_id"), + ("wh_wallets", "currency"), + ("wh_transactions", "wallet_id"), + ("wh_transactions", "amount"), + ("wh_transactions", "direction"), + ] + for table, col in not_null_checks: + nulls = conn.execute( + f"SELECT COUNT(*) FROM {table} WHERE {col} IS NULL" + ).fetchone()[0] + if nulls: + failures.append(f"{table}.{col}: {nulls} NULL value(s) found") + + # ── business checks ─────────────────────────────────────────────────────── + + neg_bal = conn.execute( + "SELECT COUNT(*) FROM wh_wallets WHERE balance < 0" + ).fetchone()[0] + if neg_bal: + failures.append(f"wh_wallets: {neg_bal} row(s) with negative balance") + + non_pos = conn.execute( + "SELECT COUNT(*) FROM wh_transactions WHERE amount <= 0" + ).fetchone()[0] + if non_pos: + failures.append(f"wh_transactions: {non_pos} row(s) with non-positive amount") + + bad_dir = conn.execute( + "SELECT COUNT(*) FROM wh_transactions" + " WHERE direction NOT IN ('credit', 'debit')" + ).fetchone()[0] + if bad_dir: + failures.append(f"wh_transactions: {bad_dir} row(s) with invalid direction") + + # ── lake completeness ───────────────────────────────────────────────────── + + lake_count = conn.execute("SELECT COUNT(*) FROM lake_cdc_events").fetchone()[0] + if lake_count == 0: + failures.append("lake_cdc_events: no records found — lake appears empty") + + return failures + + +def main() -> int: + conn = duckdb.connect(":memory:") + capture = CDCCapture() + seed(conn, capture) + + failures = run_checks(conn) + + if failures: + print("Data quality failures:") + for f in failures: + print(f" ✗ {f}") + return 1 + + print( + f"All data quality checks passed ({len(run_checks.__code__.co_consts)} rules checked)." + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/submission/sample-candidate/scripts/validate_catalog.py b/submission/sample-candidate/scripts/validate_catalog.py new file mode 100644 index 0000000..6a02026 --- /dev/null +++ b/submission/sample-candidate/scripts/validate_catalog.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +""" +validate_catalog.py + +Verifies that catalog/catalog.json is present and contains a complete entry +for every required lake and warehouse dataset. + +Required fields per entry: name, layer, description, owner, schema, update_cadence + +Exit 0 — catalog is valid. +Exit 1 — missing file, missing datasets, or missing required fields. +""" + +import json +import os +import sys + +CATALOG_PATH = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "catalog", + "catalog.json", +) + +REQUIRED_DATASETS = [ + "lake_cdc_events", + "wh_customers", + "wh_wallets", + "wh_transactions", +] + +REQUIRED_FIELDS = ["name", "layer", "description", "owner", "schema", "update_cadence"] + + +def validate() -> list[str]: + violations: list[str] = [] + + if not os.path.exists(CATALOG_PATH): + violations.append(f"catalog.json not found at {CATALOG_PATH}") + return violations + + with open(CATALOG_PATH) as f: + try: + catalog = json.load(f) + except json.JSONDecodeError as exc: + violations.append(f"catalog.json is not valid JSON: {exc}") + return violations + + datasets = {d["name"]: d for d in catalog.get("datasets", []) if "name" in d} + + for name in REQUIRED_DATASETS: + if name not in datasets: + violations.append(f"Missing dataset entry: {name}") + continue + + entry = datasets[name] + for field in REQUIRED_FIELDS: + if field not in entry or not entry[field]: + violations.append(f"{name}: missing or empty required field '{field}'") + + return violations + + +def main() -> int: + violations = validate() + + if violations: + print("Catalog validation failures:") + for v in violations: + print(f" ✗ {v}") + return 1 + + print(f"Catalog validation passed ({len(REQUIRED_DATASETS)} datasets verified).") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/submission/sample-candidate/source/__init__.py b/submission/sample-candidate/source/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/submission/sample-candidate/source/models.py b/submission/sample-candidate/source/models.py new file mode 100644 index 0000000..91c9d19 --- /dev/null +++ b/submission/sample-candidate/source/models.py @@ -0,0 +1,84 @@ +""" +Source schema for a payments/wallet system. + +Domain: customers own wallets; wallets have transactions. + +Strong entities : customers, wallets +Weak entities : transactions (lifecycle tied to wallet) + +Invariants: +- wallet.balance >= 0 +- transaction.amount > 0 +- status fields restricted to known enum values +- settled_at must be >= created_at when present +""" + +import duckdb + +# Expected columns per table — used by schema-contract checks. +SCHEMA_CONTRACT: dict[str, list[str]] = { + "customers": ["customer_id", "name", "email", "status", "created_at", "updated_at"], + "wallets": [ + "wallet_id", + "customer_id", + "balance", + "currency", + "status", + "created_at", + "updated_at", + ], + "transactions": [ + "transaction_id", + "wallet_id", + "amount", + "direction", + "status", + "reference", + "created_at", + "settled_at", + ], +} + + +def create_source_tables(conn: duckdb.DuckDBPyConnection) -> None: + """Create source tables with constraints in the given DuckDB connection.""" + conn.execute(""" + CREATE TABLE IF NOT EXISTS customers ( + customer_id VARCHAR PRIMARY KEY, + name VARCHAR NOT NULL, + email VARCHAR NOT NULL, + status VARCHAR NOT NULL + CHECK (status IN ('active', 'suspended', 'closed')), + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL + ) + """) + + conn.execute(""" + CREATE TABLE IF NOT EXISTS wallets ( + wallet_id VARCHAR PRIMARY KEY, + customer_id VARCHAR NOT NULL REFERENCES customers(customer_id), + balance DECIMAL(18, 2) NOT NULL DEFAULT 0.00 + CHECK (balance >= 0), + currency VARCHAR NOT NULL, + status VARCHAR NOT NULL + CHECK (status IN ('active', 'frozen', 'closed')), + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL + ) + """) + + conn.execute(""" + CREATE TABLE IF NOT EXISTS transactions ( + transaction_id VARCHAR PRIMARY KEY, + wallet_id VARCHAR NOT NULL REFERENCES wallets(wallet_id), + amount DECIMAL(18, 2) NOT NULL CHECK (amount > 0), + direction VARCHAR NOT NULL + CHECK (direction IN ('credit', 'debit')), + status VARCHAR NOT NULL + CHECK (status IN ('pending', 'settled', 'failed', 'reversed')), + reference VARCHAR, + created_at TIMESTAMP NOT NULL, + settled_at TIMESTAMP + ) + """) diff --git a/submission/sample-candidate/tests/conftest.py b/submission/sample-candidate/tests/conftest.py new file mode 100644 index 0000000..7a883e5 --- /dev/null +++ b/submission/sample-candidate/tests/conftest.py @@ -0,0 +1,125 @@ +"""Shared pytest fixtures for the payments CDC pipeline tests.""" + +from datetime import datetime, timezone + +import duckdb +import pytest + +from pipeline.cdc import CDCCapture +from pipeline.lake import append_to_lake, create_lake_table +from pipeline.warehouse import apply_cdc_records, create_warehouse_tables +from source.models import create_source_tables + + +def _ts() -> datetime: + return datetime.now(timezone.utc) + + +@pytest.fixture() +def conn() -> duckdb.DuckDBPyConnection: + """Fresh in-memory DuckDB with source + lake + warehouse tables.""" + c = duckdb.connect(":memory:") + create_source_tables(c) + create_lake_table(c) + create_warehouse_tables(c) + return c + + +@pytest.fixture() +def capture() -> CDCCapture: + """Empty CDC capture log.""" + return CDCCapture() + + +@pytest.fixture() +def seeded(conn: duckdb.DuckDBPyConnection, capture: CDCCapture): + """ + DuckDB connection pre-loaded with two customers, two wallets, + and two transactions — all flushed through lake and warehouse. + Returns (conn, capture). + """ + ts = _ts() + + capture.insert( + "customers", + "c1", + { + "customer_id": "c1", + "name": "Alice", + "email": "alice-test-user", + "status": "active", + "created_at": ts, + "updated_at": ts, + }, + ) + capture.insert( + "customers", + "c2", + { + "customer_id": "c2", + "name": "Bob", + "email": "bob-test-user", + "status": "active", + "created_at": ts, + "updated_at": ts, + }, + ) + capture.insert( + "wallets", + "w1", + { + "wallet_id": "w1", + "customer_id": "c1", + "balance": 100.00, + "currency": "USD", + "status": "active", + "created_at": ts, + "updated_at": ts, + }, + ) + capture.insert( + "wallets", + "w2", + { + "wallet_id": "w2", + "customer_id": "c2", + "balance": 50.00, + "currency": "USD", + "status": "active", + "created_at": ts, + "updated_at": ts, + }, + ) + capture.insert( + "transactions", + "t1", + { + "transaction_id": "t1", + "wallet_id": "w1", + "amount": 25.00, + "direction": "credit", + "status": "settled", + "reference": "ref-001", + "created_at": ts, + "settled_at": ts, + }, + ) + capture.insert( + "transactions", + "t2", + { + "transaction_id": "t2", + "wallet_id": "w2", + "amount": 10.00, + "direction": "debit", + "status": "settled", + "reference": "ref-002", + "created_at": ts, + "settled_at": ts, + }, + ) + + records = capture.records_since(0) + append_to_lake(conn, records) + apply_cdc_records(conn, records) + return conn, capture diff --git a/submission/sample-candidate/tests/test_catalog.py b/submission/sample-candidate/tests/test_catalog.py new file mode 100644 index 0000000..9f7d056 --- /dev/null +++ b/submission/sample-candidate/tests/test_catalog.py @@ -0,0 +1,134 @@ +""" +Tests — catalog metadata completeness and correctness. + +Covers: file presence, all required datasets, required fields, layer labels, +schema presence, and that lake/warehouse datasets are correctly distinguished. +""" + +import json +import os + +import pytest + +CATALOG_PATH = os.path.join( + os.path.dirname(os.path.dirname(__file__)), + "catalog", + "catalog.json", +) + +REQUIRED_DATASETS = [ + "lake_cdc_events", + "wh_customers", + "wh_wallets", + "wh_transactions", +] + +REQUIRED_FIELDS = ["name", "layer", "description", "owner", "schema", "update_cadence"] + + +@pytest.fixture(scope="module") +def catalog() -> dict: + with open(CATALOG_PATH) as f: + return json.load(f) + + +@pytest.fixture(scope="module") +def datasets(catalog: dict) -> dict[str, dict]: + return {d["name"]: d for d in catalog.get("datasets", []) if "name" in d} + + +# ── file presence ───────────────────────────────────────────────────────────── + + +def test_catalog_file_exists(): + assert os.path.exists(CATALOG_PATH), f"catalog.json not found at {CATALOG_PATH}" + + +def test_catalog_file_is_valid_json(): + with open(CATALOG_PATH) as f: + data = json.load(f) + assert isinstance(data, dict) + + +def test_catalog_has_datasets_key(catalog: dict): + assert "datasets" in catalog + assert isinstance(catalog["datasets"], list) + + +# ── required datasets ───────────────────────────────────────────────────────── + + +@pytest.mark.parametrize("name", REQUIRED_DATASETS) +def test_required_dataset_is_present(name: str, datasets: dict): + assert name in datasets, f"Dataset '{name}' is missing from catalog" + + +def test_catalog_has_at_least_four_datasets(datasets: dict): + assert len(datasets) >= 4 + + +# ── required fields ─────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize("name", REQUIRED_DATASETS) +@pytest.mark.parametrize("field", REQUIRED_FIELDS) +def test_required_field_is_present_and_non_empty(name: str, field: str, datasets: dict): + entry = datasets.get(name, {}) + assert field in entry, f"Dataset '{name}' is missing field '{field}'" + assert entry[field], f"Dataset '{name}' field '{field}' is empty" + + +# ── layer labels ────────────────────────────────────────────────────────────── + + +def test_lake_cdc_events_is_labeled_as_lake(datasets: dict): + assert datasets["lake_cdc_events"]["layer"] == "lake" + + +@pytest.mark.parametrize( + "name", + ["wh_customers", "wh_wallets", "wh_transactions"], +) +def test_warehouse_datasets_are_labeled_as_warehouse(name: str, datasets: dict): + assert datasets[name]["layer"] == "warehouse" + + +# ── schema presence ─────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize("name", REQUIRED_DATASETS) +def test_schema_field_is_a_non_empty_dict(name: str, datasets: dict): + schema = datasets[name].get("schema", {}) + assert isinstance(schema, dict) + assert len(schema) > 0, f"Dataset '{name}' has an empty schema" + + +def test_lake_schema_includes_operation_column(datasets: dict): + assert "operation" in datasets["lake_cdc_events"]["schema"] + + +def test_lake_schema_includes_sequence_column(datasets: dict): + assert "sequence" in datasets["lake_cdc_events"]["schema"] + + +def test_wh_customers_schema_includes_pk(datasets: dict): + assert "customer_id" in datasets["wh_customers"]["schema"] + + +def test_wh_transactions_schema_includes_amount(datasets: dict): + assert "amount" in datasets["wh_transactions"]["schema"] + + +# ── update cadence ──────────────────────────────────────────────────────────── + + +def test_lake_update_cadence_is_real_time(datasets: dict): + assert datasets["lake_cdc_events"]["update_cadence"] == "real-time" + + +@pytest.mark.parametrize( + "name", + ["wh_customers", "wh_wallets", "wh_transactions"], +) +def test_warehouse_update_cadence_is_near_real_time(name: str, datasets: dict): + assert datasets[name]["update_cadence"] == "near-real-time" diff --git a/submission/sample-candidate/tests/test_cdc.py b/submission/sample-candidate/tests/test_cdc.py new file mode 100644 index 0000000..704902e --- /dev/null +++ b/submission/sample-candidate/tests/test_cdc.py @@ -0,0 +1,135 @@ +""" +Tests — CDC capture correctness. + +Covers: insert/update/delete capture, invalid operation rejection, +sequence monotonicity, checkpoint-based replay, duplicate safety. +""" + +from datetime import datetime, timezone + +import pytest + +from pipeline.cdc import CDCCapture, CDCRecord + + +def _ts() -> datetime: + return datetime.now(timezone.utc) + + +# ── insert ──────────────────────────────────────────────────────────────────── + + +def test_insert_creates_record_with_correct_operation(): + cap = CDCCapture() + rec = cap.insert("customers", "c1", {"customer_id": "c1", "name": "Alice"}) + assert rec.operation == "insert" + assert rec.table == "customers" + assert rec.primary_key == "c1" + assert rec.data["name"] == "Alice" + + +def test_insert_assigns_sequence_starting_at_one(): + cap = CDCCapture() + rec = cap.insert("customers", "c1", {}) + assert rec.sequence == 1 + + +# ── update ──────────────────────────────────────────────────────────────────── + + +def test_update_creates_record_with_update_operation(): + cap = CDCCapture() + cap.insert("customers", "c1", {"status": "active"}) + rec = cap.update("customers", "c1", {"status": "suspended"}) + assert rec.operation == "update" + assert rec.data["status"] == "suspended" + + +# ── delete ──────────────────────────────────────────────────────────────────── + + +def test_delete_creates_record_with_delete_operation(): + cap = CDCCapture() + cap.insert("customers", "c1", {"customer_id": "c1"}) + rec = cap.delete("customers", "c1", {"customer_id": "c1"}) + assert rec.operation == "delete" + assert rec.primary_key == "c1" + + +# ── invalid operation ───────────────────────────────────────────────────────── + + +def test_invalid_operation_raises_value_error(): + with pytest.raises(ValueError, match="Invalid CDC operation"): + CDCRecord(operation="upsert", table="customers", primary_key="c1", data={}) + + +# ── sequence monotonicity ───────────────────────────────────────────────────── + + +def test_sequences_are_strictly_increasing(): + cap = CDCCapture() + r1 = cap.insert("customers", "c1", {}) + r2 = cap.insert("customers", "c2", {}) + r3 = cap.update("customers", "c1", {}) + assert r1.sequence < r2.sequence < r3.sequence + + +def test_latest_sequence_reflects_last_record(): + cap = CDCCapture() + cap.insert("customers", "c1", {}) + cap.insert("customers", "c2", {}) + last = cap.update("customers", "c1", {}) + assert cap.latest_sequence == last.sequence + + +# ── checkpoint-based replay ─────────────────────────────────────────────────── + + +def test_records_since_returns_only_records_after_offset(): + cap = CDCCapture() + cap.insert("customers", "c1", {}) + cap.insert("customers", "c2", {}) + checkpoint = cap.latest_sequence + r3 = cap.insert("customers", "c3", {}) + + replayed = cap.records_since(checkpoint) + + assert len(replayed) == 1 + assert replayed[0].sequence == r3.sequence + + +def test_records_since_zero_returns_all_records(): + cap = CDCCapture() + cap.insert("customers", "c1", {}) + cap.insert("customers", "c2", {}) + cap.delete("customers", "c1", {}) + assert len(cap.records_since(0)) == 3 + + +def test_records_since_latest_sequence_returns_empty(): + cap = CDCCapture() + cap.insert("customers", "c1", {}) + assert cap.records_since(cap.latest_sequence) == [] + + +# ── duplicate / replay safety ───────────────────────────────────────────────── + + +def test_same_pk_can_appear_multiple_times_in_log(): + """CDC appends every event — deduplication happens downstream.""" + cap = CDCCapture() + cap.insert("customers", "c1", {"status": "active"}) + cap.update("customers", "c1", {"status": "suspended"}) + cap.update("customers", "c1", {"status": "closed"}) + + records = cap.records_since(0) + pk_records = [r for r in records if r.primary_key == "c1"] + assert len(pk_records) == 3 + + +def test_captured_at_is_utc_datetime(): + cap = CDCCapture() + rec = cap.insert("customers", "c1", {}) + assert isinstance(rec.captured_at, datetime) + assert rec.captured_at.tzinfo is not None diff --git a/submission/sample-candidate/tests/test_data_quality.py b/submission/sample-candidate/tests/test_data_quality.py new file mode 100644 index 0000000..4dbf7f9 --- /dev/null +++ b/submission/sample-candidate/tests/test_data_quality.py @@ -0,0 +1,252 @@ +""" +Tests — data quality and warehouse correctness. + +Covers: insert propagation, update correctness, soft-delete, replay safety, +PK uniqueness, not-null checks, business rule assertions, lake completeness, +and lake immutability (no deletes or updates to lake rows). +""" + +from datetime import datetime, timezone + +import pytest + +from pipeline.lake import append_to_lake +from pipeline.warehouse import apply_cdc_records + + +def _ts() -> datetime: + return datetime.now(timezone.utc) + + +# ── insert propagation ──────────────────────────────────────────────────────── + + +def test_insert_appears_in_warehouse(seeded): + conn, _ = seeded + row = conn.execute( + "SELECT name FROM wh_customers WHERE customer_id = 'c1'" + ).fetchone() + assert row is not None + assert row[0] == "Alice" + + +def test_insert_appears_in_lake(seeded): + conn, _ = seeded + count = conn.execute( + "SELECT COUNT(*) FROM lake_cdc_events WHERE table_name = 'customers'" + " AND operation = 'insert'" + ).fetchone()[0] + assert count == 2 + + +def test_all_tables_populated_after_seed(seeded): + conn, _ = seeded + assert conn.execute("SELECT COUNT(*) FROM wh_customers").fetchone()[0] == 2 + assert conn.execute("SELECT COUNT(*) FROM wh_wallets").fetchone()[0] == 2 + assert conn.execute("SELECT COUNT(*) FROM wh_transactions").fetchone()[0] == 2 + + +# ── update correctness ──────────────────────────────────────────────────────── + + +def test_update_overwrites_warehouse_row(seeded): + conn, capture = seeded + ts = _ts() + capture.update( + "customers", + "c1", + { + "customer_id": "c1", + "name": "Alice Smith", + "email": "alice-test-user", + "status": "suspended", + "created_at": ts, + "updated_at": ts, + }, + ) + new_records = capture.records_since(capture.latest_sequence - 1) + apply_cdc_records(conn, new_records) + + row = conn.execute( + "SELECT name, status FROM wh_customers WHERE customer_id = 'c1'" + ).fetchone() + assert row[0] == "Alice Smith" + assert row[1] == "suspended" + + +def test_update_does_not_create_duplicate_warehouse_row(seeded): + conn, capture = seeded + ts = _ts() + capture.update( + "wallets", + "w1", + { + "wallet_id": "w1", + "customer_id": "c1", + "balance": 200.00, + "currency": "USD", + "status": "active", + "created_at": ts, + "updated_at": ts, + }, + ) + apply_cdc_records(conn, capture.records_since(capture.latest_sequence - 1)) + + count = conn.execute( + "SELECT COUNT(*) FROM wh_wallets WHERE wallet_id = 'w1'" + ).fetchone()[0] + assert count == 1 + + +# ── soft delete ─────────────────────────────────────────────────────────────── + + +def test_delete_marks_warehouse_row_as_deleted(seeded): + conn, capture = seeded + capture.delete("customers", "c2", {"customer_id": "c2"}) + apply_cdc_records(conn, capture.records_since(capture.latest_sequence - 1)) + + row = conn.execute( + "SELECT _deleted FROM wh_customers WHERE customer_id = 'c2'" + ).fetchone() + assert row is not None + assert row[0] is True + + +def test_delete_does_not_remove_row_from_warehouse(seeded): + conn, capture = seeded + capture.delete("wallets", "w2", {"wallet_id": "w2"}) + apply_cdc_records(conn, capture.records_since(capture.latest_sequence - 1)) + + count = conn.execute( + "SELECT COUNT(*) FROM wh_wallets WHERE wallet_id = 'w2'" + ).fetchone()[0] + assert count == 1 + + +# ── lake immutability ───────────────────────────────────────────────────────── + + +def test_lake_row_count_only_increases(seeded): + conn, capture = seeded + before = conn.execute("SELECT COUNT(*) FROM lake_cdc_events").fetchone()[0] + + ts = _ts() + capture.update( + "customers", + "c1", + { + "customer_id": "c1", + "name": "Alice Updated", + "email": "alice-test-user", + "status": "active", + "created_at": ts, + "updated_at": ts, + }, + ) + new_records = capture.records_since(capture.latest_sequence - 1) + append_to_lake(conn, new_records) + + after = conn.execute("SELECT COUNT(*) FROM lake_cdc_events").fetchone()[0] + assert after > before + + +def test_lake_retains_all_operations_for_same_pk(seeded): + conn, capture = seeded + ts = _ts() + capture.update( + "customers", + "c1", + { + "customer_id": "c1", + "name": "Alice v2", + "email": "alice-test-user", + "status": "active", + "created_at": ts, + "updated_at": ts, + }, + ) + capture.delete("customers", "c1", {"customer_id": "c1"}) + append_to_lake(conn, capture.records_since(capture.latest_sequence - 2)) + + events = conn.execute( + "SELECT operation FROM lake_cdc_events" + " WHERE table_name = 'customers' AND primary_key = 'c1'" + " ORDER BY sequence" + ).fetchall() + ops = [e[0] for e in events] + assert "insert" in ops + assert "update" in ops + assert "delete" in ops + + +# ── PK uniqueness ───────────────────────────────────────────────────────────── + + +def test_warehouse_customers_pk_is_unique(seeded): + conn, _ = seeded + total = conn.execute("SELECT COUNT(*) FROM wh_customers").fetchone()[0] + distinct = conn.execute( + "SELECT COUNT(DISTINCT customer_id) FROM wh_customers" + ).fetchone()[0] + assert total == distinct + + +def test_warehouse_wallets_pk_is_unique(seeded): + conn, _ = seeded + total = conn.execute("SELECT COUNT(*) FROM wh_wallets").fetchone()[0] + distinct = conn.execute( + "SELECT COUNT(DISTINCT wallet_id) FROM wh_wallets" + ).fetchone()[0] + assert total == distinct + + +# ── business rules ──────────────────────────────────────────────────────────── + + +def test_transaction_amounts_are_positive(seeded): + conn, _ = seeded + bad = conn.execute( + "SELECT COUNT(*) FROM wh_transactions WHERE amount <= 0" + ).fetchone()[0] + assert bad == 0 + + +def test_wallet_balances_are_non_negative(seeded): + conn, _ = seeded + bad = conn.execute("SELECT COUNT(*) FROM wh_wallets WHERE balance < 0").fetchone()[ + 0 + ] + assert bad == 0 + + +def test_transaction_directions_are_valid(seeded): + conn, _ = seeded + bad = conn.execute( + "SELECT COUNT(*) FROM wh_transactions" + " WHERE direction NOT IN ('credit', 'debit')" + ).fetchone()[0] + assert bad == 0 + + +# ── replay safety ───────────────────────────────────────────────────────────── + + +def test_replaying_same_records_does_not_create_duplicates(seeded): + conn, capture = seeded + # Replay all records from the beginning + all_records = capture.records_since(0) + apply_cdc_records(conn, all_records) + + count = conn.execute("SELECT COUNT(*) FROM wh_customers").fetchone()[0] + assert count == 2 # Must not double to 4 + + +@pytest.mark.parametrize("offset", [0, 1, 2]) +def test_partial_replay_from_checkpoint_is_idempotent(seeded, offset: int): + conn, capture = seeded + partial = capture.records_since(offset) + apply_cdc_records(conn, partial) + + count = conn.execute("SELECT COUNT(*) FROM wh_customers").fetchone()[0] + assert count == 2 diff --git a/submission/sample-candidate/tests/test_schema_contracts.py b/submission/sample-candidate/tests/test_schema_contracts.py new file mode 100644 index 0000000..69d30f7 --- /dev/null +++ b/submission/sample-candidate/tests/test_schema_contracts.py @@ -0,0 +1,170 @@ +""" +Tests — schema contract detection and safe-stop behavior. + +Covers: passing contracts, missing column detection, incompatible schema change +detection (dropped column, added unexpected table), and that ingestion can be +stopped when a contract violation is found. +""" + +import duckdb +import pytest + +from source.models import SCHEMA_CONTRACT, create_source_tables + +# ── helpers ─────────────────────────────────────────────────────────────────── + + +def _actual_columns(conn: duckdb.DuckDBPyConnection, table: str) -> set[str]: + return {row[0] for row in conn.execute(f"DESCRIBE {table}").fetchall()} + + +def _check_contracts(conn: duckdb.DuckDBPyConnection) -> list[str]: + """Inline contract check — mirrors scripts/check_schema_contracts.py.""" + violations: list[str] = [] + for table, expected_cols in SCHEMA_CONTRACT.items(): + try: + actual = _actual_columns(conn, table) + except Exception as exc: + violations.append(f"{table}: {exc}") + continue + for col in expected_cols: + if col not in actual: + violations.append(f"{table}.{col}: column missing") + return violations + + +# ── passing contracts ───────────────────────────────────────────────────────── + + +def test_fresh_source_tables_pass_all_contracts(): + conn = duckdb.connect(":memory:") + create_source_tables(conn) + assert _check_contracts(conn) == [] + + +def test_all_contract_tables_are_present(): + conn = duckdb.connect(":memory:") + create_source_tables(conn) + for table in SCHEMA_CONTRACT: + cols = _actual_columns(conn, table) + assert len(cols) > 0, f"{table} has no columns" + + +# ── missing column detection ────────────────────────────────────────────────── + + +def test_dropped_column_is_detected_as_violation(): + # Create customers without FK-dependent tables so ALTER TABLE is allowed + conn = duckdb.connect(":memory:") + conn.execute(""" + CREATE TABLE customers ( + customer_id VARCHAR PRIMARY KEY, + name VARCHAR NOT NULL, + status VARCHAR NOT NULL, + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL + ) + """) + # 'email' was never added — contract check should catch it + violations = _check_contracts(conn) + assert any("email" in v for v in violations) + + +def test_multiple_dropped_columns_all_reported(): + # Create wallets without FK constraint so ALTER TABLE is allowed + conn = duckdb.connect(":memory:") + conn.execute(""" + CREATE TABLE customers (customer_id VARCHAR PRIMARY KEY, + name VARCHAR NOT NULL, email VARCHAR NOT NULL, + status VARCHAR NOT NULL, created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL) + """) + conn.execute(""" + CREATE TABLE wallets ( + wallet_id VARCHAR PRIMARY KEY, + customer_id VARCHAR NOT NULL, + currency VARCHAR NOT NULL, + status VARCHAR NOT NULL, + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL + ) + """) + # 'balance' never added — contract check should report both balance and balance + violations = _check_contracts(conn) + assert any("balance" in v for v in violations) + + +def test_violations_include_table_and_column_name(): + conn = duckdb.connect(":memory:") + create_source_tables(conn) + conn.execute("ALTER TABLE transactions DROP COLUMN amount") + + violations = _check_contracts(conn) + assert any("transactions" in v and "amount" in v for v in violations) + + +# ── contract-based safe stop ────────────────────────────────────────────────── + + +def test_pipeline_stops_when_contract_violated(): + """ + When a contract violation is found, no CDC records should be captured. + This models the stop-the-line behavior required by the assignment. + """ + from pipeline.cdc import CDCCapture + + # Create customers table missing 'email' — simulates a breaking schema change + conn = duckdb.connect(":memory:") + conn.execute(""" + CREATE TABLE customers ( + customer_id VARCHAR PRIMARY KEY, + name VARCHAR NOT NULL, + status VARCHAR NOT NULL, + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL + ) + """) + + violations = _check_contracts(conn) + capture = CDCCapture() + + if violations: + # Pipeline aborts — no records captured + pass + else: + capture.insert("customers", "c1", {"customer_id": "c1", "name": "Alice"}) + + assert ( + len(capture.log) == 0 + ), "No records should be captured after a contract violation" + + +# ── schema contract completeness ────────────────────────────────────────────── + + +def test_schema_contract_covers_all_key_tables(): + assert "customers" in SCHEMA_CONTRACT + assert "wallets" in SCHEMA_CONTRACT + assert "transactions" in SCHEMA_CONTRACT + + +def test_schema_contract_includes_primary_key_columns(): + assert "customer_id" in SCHEMA_CONTRACT["customers"] + assert "wallet_id" in SCHEMA_CONTRACT["wallets"] + assert "transaction_id" in SCHEMA_CONTRACT["transactions"] + + +@pytest.mark.parametrize( + "table,col", + [ + ("customers", "status"), + ("wallets", "balance"), + ("wallets", "currency"), + ("transactions", "amount"), + ("transactions", "direction"), + ], +) +def test_business_critical_columns_are_in_contract(table: str, col: str): + assert ( + col in SCHEMA_CONTRACT[table] + ), f"Business-critical column {table}.{col} is not in SCHEMA_CONTRACT"