From 1948904df1120e1c95d068845871c0beaf856b1b Mon Sep 17 00:00:00 2001 From: Poorva28 Date: Thu, 2 Jul 2026 11:27:50 +0530 Subject: [PATCH] Submit assignment for ticket reservation system --- .../conftest.cpython-314-pytest-9.0.2.pyc | Bin 592 -> 0 bytes .../sample-candidate/catalog/catalog.json | 77 ------ submission/sample-candidate/conftest.py | 6 - .../sample-candidate/pipeline/__init__.py | 0 .../__pycache__/__init__.cpython-314.pyc | Bin 186 -> 0 bytes .../pipeline/__pycache__/cdc.cpython-314.pyc | Bin 5653 -> 0 bytes .../pipeline/__pycache__/lake.cpython-314.pyc | Bin 3364 -> 0 bytes .../__pycache__/warehouse.cpython-314.pyc | Bin 5351 -> 0 bytes submission/sample-candidate/pipeline/cdc.py | 89 ------- submission/sample-candidate/pipeline/lake.py | 69 ----- .../sample-candidate/pipeline/warehouse.py | 124 --------- submission/sample-candidate/requirements.txt | 2 - .../scripts/check_schema_contracts.py | 60 ----- .../scripts/run_data_quality_checks.py | 212 --------------- .../scripts/validate_catalog.py | 77 ------ .../sample-candidate/source/__init__.py | 0 .../__pycache__/__init__.cpython-314.pyc | Bin 184 -> 0 bytes .../source/__pycache__/models.cpython-314.pyc | Bin 3369 -> 0 bytes submission/sample-candidate/source/models.py | 84 ------ .../conftest.cpython-314-pytest-9.0.2.pyc | Bin 3905 -> 0 bytes .../test_catalog.cpython-314-pytest-9.0.2.pyc | Bin 19689 -> 0 bytes .../test_cdc.cpython-314-pytest-9.0.2.pyc | Bin 20869 -> 0 bytes ..._data_quality.cpython-314-pytest-9.0.2.pyc | Bin 21787 -> 0 bytes ...ema_contracts.cpython-314-pytest-9.0.2.pyc | Bin 16244 -> 0 bytes submission/sample-candidate/tests/conftest.py | 125 --------- .../sample-candidate/tests/test_catalog.py | 134 ---------- submission/sample-candidate/tests/test_cdc.py | 135 ---------- .../tests/test_data_quality.py | 252 ------------------ .../tests/test_schema_contracts.py | 170 ------------ .../ticket-reservations/LOCAL_VALIDATION.md | 185 +++++++++++++ submission/ticket-reservations/README.md | 73 +++++ submission/ticket-reservations/SOLUTION.md | 142 ++++++++++ .../conftest.cpython-313-pytest-8.4.2.pyc | Bin 0 -> 766 bytes .../ticket-reservations/catalog/catalog.json | 123 +++++++++ submission/ticket-reservations/conftest.py | 10 + .../ticket-reservations/pipeline/__init__.py | 2 + .../ticket-reservations/pipeline/cdc.py | 114 ++++++++ .../ticket-reservations/pipeline/lake.py | 79 ++++++ .../pipeline/schema_contracts.py | 52 ++++ .../ticket-reservations/pipeline/warehouse.py | 151 +++++++++++ .../ticket-reservations/requirements.txt | 2 + .../scripts/check_schema_contracts.py | 35 +++ .../scripts/run_data_quality_checks.py | 51 ++++ .../scripts/validate_catalog.py | 37 +++ .../ticket-reservations/source/__init__.py | 2 + .../ticket-reservations/source/models.py | 242 +++++++++++++++++ .../sql/snowflake/merge_reservations.sql | 28 ++ .../sql/snowflake/quality_checks.sql | 34 +++ .../sql/snowflake/refresh_from_iceberg.sql | 71 +++++ .../sql/snowflake/warehouse_tables.sql | 90 +++++++ .../sql/source/debezium_connector.json | 23 ++ .../sql/source/postgres_schema.sql | 109 ++++++++ .../sql/spark/iceberg_tables.sql | 117 ++++++++ .../sql/spark/raw_s3_event_contract.sql | 25 ++ .../spark/structured_streaming_pseudocode.py | 102 +++++++ .../conftest.cpython-313-pytest-8.4.2.pyc | Bin 0 -> 1481 bytes .../ticket-reservations/tests/conftest.py | 36 +++ .../ticket-reservations/tests/test_catalog.py | 28 ++ .../ticket-reservations/tests/test_cdc.py | 72 +++++ .../tests/test_data_quality.py | 26 ++ .../tests/test_lake_warehouse.py | 94 +++++++ .../tests/test_schema_contracts.py | 52 ++++ 62 files changed, 2207 insertions(+), 1616 deletions(-) delete mode 100644 submission/sample-candidate/__pycache__/conftest.cpython-314-pytest-9.0.2.pyc delete mode 100644 submission/sample-candidate/catalog/catalog.json delete mode 100644 submission/sample-candidate/conftest.py delete mode 100644 submission/sample-candidate/pipeline/__init__.py delete mode 100644 submission/sample-candidate/pipeline/__pycache__/__init__.cpython-314.pyc delete mode 100644 submission/sample-candidate/pipeline/__pycache__/cdc.cpython-314.pyc delete mode 100644 submission/sample-candidate/pipeline/__pycache__/lake.cpython-314.pyc delete mode 100644 submission/sample-candidate/pipeline/__pycache__/warehouse.cpython-314.pyc delete mode 100644 submission/sample-candidate/pipeline/cdc.py delete mode 100644 submission/sample-candidate/pipeline/lake.py delete mode 100644 submission/sample-candidate/pipeline/warehouse.py delete mode 100644 submission/sample-candidate/requirements.txt delete mode 100644 submission/sample-candidate/scripts/check_schema_contracts.py delete mode 100644 submission/sample-candidate/scripts/run_data_quality_checks.py delete mode 100644 submission/sample-candidate/scripts/validate_catalog.py delete mode 100644 submission/sample-candidate/source/__init__.py delete mode 100644 submission/sample-candidate/source/__pycache__/__init__.cpython-314.pyc delete mode 100644 submission/sample-candidate/source/__pycache__/models.cpython-314.pyc delete mode 100644 submission/sample-candidate/source/models.py delete mode 100644 submission/sample-candidate/tests/__pycache__/conftest.cpython-314-pytest-9.0.2.pyc delete mode 100644 submission/sample-candidate/tests/__pycache__/test_catalog.cpython-314-pytest-9.0.2.pyc delete mode 100644 submission/sample-candidate/tests/__pycache__/test_cdc.cpython-314-pytest-9.0.2.pyc delete mode 100644 submission/sample-candidate/tests/__pycache__/test_data_quality.cpython-314-pytest-9.0.2.pyc delete mode 100644 submission/sample-candidate/tests/__pycache__/test_schema_contracts.cpython-314-pytest-9.0.2.pyc delete mode 100644 submission/sample-candidate/tests/conftest.py delete mode 100644 submission/sample-candidate/tests/test_catalog.py delete mode 100644 submission/sample-candidate/tests/test_cdc.py delete mode 100644 submission/sample-candidate/tests/test_data_quality.py delete mode 100644 submission/sample-candidate/tests/test_schema_contracts.py create mode 100644 submission/ticket-reservations/LOCAL_VALIDATION.md create mode 100644 submission/ticket-reservations/README.md create mode 100644 submission/ticket-reservations/SOLUTION.md create mode 100644 submission/ticket-reservations/__pycache__/conftest.cpython-313-pytest-8.4.2.pyc create mode 100644 submission/ticket-reservations/catalog/catalog.json create mode 100644 submission/ticket-reservations/conftest.py create mode 100644 submission/ticket-reservations/pipeline/__init__.py create mode 100644 submission/ticket-reservations/pipeline/cdc.py create mode 100644 submission/ticket-reservations/pipeline/lake.py create mode 100644 submission/ticket-reservations/pipeline/schema_contracts.py create mode 100644 submission/ticket-reservations/pipeline/warehouse.py create mode 100644 submission/ticket-reservations/requirements.txt create mode 100644 submission/ticket-reservations/scripts/check_schema_contracts.py create mode 100644 submission/ticket-reservations/scripts/run_data_quality_checks.py create mode 100644 submission/ticket-reservations/scripts/validate_catalog.py create mode 100644 submission/ticket-reservations/source/__init__.py create mode 100644 submission/ticket-reservations/source/models.py create mode 100644 submission/ticket-reservations/sql/snowflake/merge_reservations.sql create mode 100644 submission/ticket-reservations/sql/snowflake/quality_checks.sql create mode 100644 submission/ticket-reservations/sql/snowflake/refresh_from_iceberg.sql create mode 100644 submission/ticket-reservations/sql/snowflake/warehouse_tables.sql create mode 100644 submission/ticket-reservations/sql/source/debezium_connector.json create mode 100644 submission/ticket-reservations/sql/source/postgres_schema.sql create mode 100644 submission/ticket-reservations/sql/spark/iceberg_tables.sql create mode 100644 submission/ticket-reservations/sql/spark/raw_s3_event_contract.sql create mode 100644 submission/ticket-reservations/sql/spark/structured_streaming_pseudocode.py create mode 100644 submission/ticket-reservations/tests/__pycache__/conftest.cpython-313-pytest-8.4.2.pyc create mode 100644 submission/ticket-reservations/tests/conftest.py create mode 100644 submission/ticket-reservations/tests/test_catalog.py create mode 100644 submission/ticket-reservations/tests/test_cdc.py create mode 100644 submission/ticket-reservations/tests/test_data_quality.py create mode 100644 submission/ticket-reservations/tests/test_lake_warehouse.py create mode 100644 submission/ticket-reservations/tests/test_schema_contracts.py diff --git a/submission/sample-candidate/__pycache__/conftest.cpython-314-pytest-9.0.2.pyc b/submission/sample-candidate/__pycache__/conftest.cpython-314-pytest-9.0.2.pyc deleted file mode 100644 index 7309004e9fba8bc1f3e0c7323e666a6180a4045a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 592 zcmZutO=}cE5bc?bJCh}GF+s%BHh2n}84Q7V2*zW)gs|el!={>@6`u zt_cXa`Em4*_y>eQ+KYJcD&!AXlXbEl>_ff!c=f8f?%$m6F?yfhepMd{^CoCt_pfvL zft&+2WXN{dFnAhZc!izK85a)N<_+u)CY1=It4fUVDBfOZg{HWDd+N8|4jV=P<=$x- zQyk5$29sztoZx^qy2qA6_l)A9*2r^R@1X?bKM&sXtSEqk+pQFU(sjyFa~MNnGTi5& zc|!!AXEj%q(H1G^Y%Wuyj4V|xldYwB9|=+DTnHcUI#r^o!OuN0V{skb6=vKhB^iLU zNP`k9-&lpx&t9i|)aH8B+|Wg4eW<~AXd^$Gh zek*-W3`kMt`>!R<(CI5zy3|UeVPYjZlNK3f%QRfQtSea~r2kx@JyBxDK3=bM;Y#_~ z-Xi(ty6}+9?@kZ|7wp9^#@?}`+3&3PgLRJM!}z#=*gpwQR?gN=*UmOhH$FZ3w)o(j JE!#eI{Q*8Dq6Gi| diff --git a/submission/sample-candidate/catalog/catalog.json b/submission/sample-candidate/catalog/catalog.json deleted file mode 100644 index 83dfa55..0000000 --- a/submission/sample-candidate/catalog/catalog.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "datasets": [ - { - "name": "lake_cdc_events", - "layer": "lake", - "description": "Append-only log of every CDC event captured from the payments source system. Preserves full change history for replay and point-in-time recovery.", - "owner": "data-platform", - "consumers": ["data-platform", "analytics", "audit"], - "update_cadence": "real-time", - "schema": { - "sequence": "INTEGER — monotonic capture offset", - "operation": "VARCHAR — insert | update | delete", - "table_name": "VARCHAR — source table name", - "primary_key": "VARCHAR — source row PK value", - "data": "VARCHAR (JSON) — full row snapshot at capture time", - "captured_at": "TIMESTAMP — UTC capture time" - } - }, - { - "name": "wh_customers", - "layer": "warehouse", - "description": "Current-state customer snapshot. Reflects the latest known state of each customer row. Soft-deleted rows are flagged with _deleted=true.", - "owner": "data-platform", - "consumers": ["analytics", "product", "finance"], - "update_cadence": "near-real-time", - "schema": { - "customer_id": "VARCHAR — primary key", - "name": "VARCHAR", - "email": "VARCHAR", - "status": "VARCHAR — active | suspended | closed", - "created_at": "TIMESTAMP", - "updated_at": "TIMESTAMP", - "_cdc_seq": "INTEGER — sequence of last CDC event", - "_deleted": "BOOLEAN — true if source row was deleted" - } - }, - { - "name": "wh_wallets", - "layer": "warehouse", - "description": "Current-state wallet snapshot. Balance reflects the latest value written via CDC. Negative balances are a data quality violation.", - "owner": "data-platform", - "consumers": ["analytics", "finance"], - "update_cadence": "near-real-time", - "schema": { - "wallet_id": "VARCHAR — primary key", - "customer_id": "VARCHAR — FK to wh_customers", - "balance": "DECIMAL(18,2)", - "currency": "VARCHAR", - "status": "VARCHAR — active | frozen | closed", - "created_at": "TIMESTAMP", - "updated_at": "TIMESTAMP", - "_cdc_seq": "INTEGER", - "_deleted": "BOOLEAN" - } - }, - { - "name": "wh_transactions", - "layer": "warehouse", - "description": "Current-state transaction snapshot. Each row is the latest state of a transaction from the source. Amount must always be positive.", - "owner": "data-platform", - "consumers": ["analytics", "finance", "audit"], - "update_cadence": "near-real-time", - "schema": { - "transaction_id": "VARCHAR — primary key", - "wallet_id": "VARCHAR — FK to wh_wallets", - "amount": "DECIMAL(18,2) — always > 0", - "direction": "VARCHAR — credit | debit", - "status": "VARCHAR — pending | settled | failed | reversed", - "reference": "VARCHAR — nullable external reference", - "created_at": "TIMESTAMP", - "settled_at": "TIMESTAMP — nullable until settled", - "_cdc_seq": "INTEGER", - "_deleted": "BOOLEAN" - } - } - ] -} diff --git a/submission/sample-candidate/conftest.py b/submission/sample-candidate/conftest.py deleted file mode 100644 index ce3c67b..0000000 --- a/submission/sample-candidate/conftest.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Root conftest — adds submission/ to sys.path so tests can import source/pipeline.""" - -import os -import sys - -sys.path.insert(0, os.path.dirname(__file__)) diff --git a/submission/sample-candidate/pipeline/__init__.py b/submission/sample-candidate/pipeline/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/submission/sample-candidate/pipeline/__pycache__/__init__.cpython-314.pyc b/submission/sample-candidate/pipeline/__pycache__/__init__.cpython-314.pyc deleted file mode 100644 index 1523f32b32810272b170ad39e00932cfc189806a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 186 zcmdPq_I|p@<2{{|u766|NszoLW?@ zUy_=fQI=YiS(2}xU7Ay>UzA#qUko8rOG*p$QxZ!ObrXw=Gt={OQ}arS^@~fBax;Pa z{5<`F%!1UM%)C_n`1s7c%#!$cy@JYH95%W6DWy57c15f}dq6HJ1~EP{Gcqz3F#}lu D5y~+p diff --git a/submission/sample-candidate/pipeline/__pycache__/cdc.cpython-314.pyc b/submission/sample-candidate/pipeline/__pycache__/cdc.cpython-314.pyc deleted file mode 100644 index a17fa0bcc202261717534949b737d20e8d1f1dd1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5653 zcmb_gU2GiH6}~e&yWSsr*YQvOF!3ZN!3&8OqKII}xI|9CCSc5jETY6&jd#cPD(ji$ z&RxJ!q*|qFleRRWszm5ZqdrvRi9A=8+DcXX(uagZsym`8RV!7z*;*p2JoP*G?yNV# zG*!e%d+)jb=boSMoV$;wJCY3FM;HEBUQ03dPill;R1sSJ0*Ga1F~QC-OPkatbfNbX ztxrZK3}H+}MO3yUld*}Ih)uMKHrX~N;}Z#y=w~y@-RucwB^R|rn%Y&QW{h68n_1Cq z%!(b-Rqr8HG0`r^+AtOm$3%yf7+~3yJ%%lkBrEe)!FK1JaO#NZyyfsEv+Vi~7r9!sgz34GQ*nf2dafxJ9Mkt|yyzsW z+$+_J*vmF?mgN*ZUNYY>- zRKcF<+jEXs8UY`bio<=>E?9GlXRBV>74qDQ?F$p9;!I+^xs~nW7Fd?#k_W$llZwuu zH#g@yB5MlIP9BgyP7d96|$*dNnY!`R1McH%xtR5sF2)hUt z{2)44b}A+G#Ys4#yyygRD$jYYgN2c!?ozR#CRrJ(q1E33vCL+e&|oV1nH~{&1m-eh zSo%JUNGj5#3`s>X8Y5~Z3R95kJ{%fI%t~N1Nuy_&XtQ82R=Zr)0kcSNW0g)3w=yDO zb;2yR1c|1af`~78knpMwmlq78!k(=-L8{8ji#A^>oOPDIOqa?<@dhUDH*uqxz>pHa zVa+cB3cp1C1rW>ZbaP<3VXe&$5WCswh9DT5j`I<)%%c1@eg8l0Rfh`8R{oF)g6 znkZglb($Mkxl^!CKPQ~x0<5(xyxiGZrIv%5JztfMs8w^s@BwJ7Jnt?#uJChyZFaGY zw&&)m<*HLDyH2iHDh^ebg7|2~UYsr2V?kF_OhaMcPMqp*L9xt!Gx*#6zuy0ccm8z# z_vdeqyvlpg3#DgD&YWGVh{Bv*6dqqv?57$ISSr{eh`^46cxbgn(qcoCjKl>&YV|&O zMxLF~pZ*?WCG_>NSF{Dq(kwlvK{S!iX>JhXxFB}}V+uCMd(Z_QVsjo}w1rQ*?%`W$ zUVEX?Fk`1s2)1tAthN7P%=EiaEVFeMji%SAxFTdF?=GJBJ3DBEG^C=#t|RBRPP%W|m2!z- z$ZXoQd9G{7e5nS=m?h-GZ*IgS!vc!4pfi8;#mQrZm#2?gM_-+MdFo`4%-fZkbDVRJ z2QlB{!YN(T0>gJIb5f%FlteoU1sItql-;s`NVm0$v}HibbQ~uwvz1il(%8kZ+q<8< zoqF=quD+G7!H;@A-gRx)&B2jXrX3mhbRc{Cv5{K?M{evHU5#j4#_EwscY2jYI?`(q z)_zdxKq`4En+)QGf@?35I+KOMA|SXzG!*@2&8{>?GKIoinfrna-1RVwjutp86u1G_ zIGHskljD!02zm33gP z+0t{%=vhf_zh!LyYv1mh2VcG2cWU{>WwcMN^z<%ITqxc$de@E6%+$uTXA!^V2Lq!| ztN#>b3uNvU5la&QOaNR&0$kKGKqE#1>|$07G(Zw`n-vG0Xn7nFt zySRWM8C{Du=VcjjLl|kqQ4?Xc3?~4}QM|KFwYLg^rpJ@bJzqRIWdbK!VrwH0P%&~S z`C!1rcR^7NAmV#Kx1bOoKwElBLtsNqYlP_z(c}<)^yout48@g4Wc5|_AzXoRB>ld{o8=$ZRVpYrST0;C|y;&k{vf@@dgNd=gIi zI9kD$7NM(c`L}Su|FT&r)5A3BC>6v{ z@f(qqN~`%X+VwdU_Xy<#y6zH+nlCw-pf!>c(o3kD2eP~W@BSln;5U4K^`7|J`@eke z{ndQQ$>#Sbt-xxvx(R}klFbq*_??lXlp!kpEr=<2&M zvy?ioJ^eGr=CM~FI{}->>r?uP9Y9C63+t-5Y7Ng&qRMd5b7PiF8cjTv8Ue*u@3m$y z8AFmpk0x`l<>{pYLv|>ubEyGJ7_t%N+ZO0R<33n`Q=5$x>8^KJu)W2*?;4T9BPd>G zpFPz7i{YOS-^z@BIDFauIC(93J%4lX$OqbG?Ze@lnb8~Z(f@ps8THA|FLmXNU&V69 zg9R#ZF2UBsPD=Cl1Jz@g*$hxxbO!-pAtJqw*1B}yaBCGKu zsJ;@8GV+ZsC{)SEaSg!hb%V*{Te2ZRWuT)Mb)Bffoutq>p@ar6E!jJxD>lDNB{ ze8?kPAzf%)dnM&BtNngDWCly@Ma7dq$feElc?D&O!h;GA&!%eDoeaKj7KQK zr(0#^%|DNgd=xqcAlTD?9pDBUJAlRQ1r4svdq`)}JN1)}k0! z_iDbiunRej6!|aZmU(tW+w9GDM%&+k35~C&Z{$^{Non%SYk55%$s3!SwWS?yoLh}z zh7qpXsHdB6yN$klG*ogw%ab(q&q*UdX(>;c6%k`floEXm6{qBq=csrB1(K&i$t&X3MGq^P9dXJAIE6%>rx|fPH1NNTSmNKN zz5*3=4StXca#nSie5{cXaVqxW=M!3gpRMV!X#7qmvg|wU$OP}i!eTv%sI?krvGf-@ zAg&$(wMrC3p!)ioef{gn&@k0;bjZ{g8H(a{oG4;Fo96SV=(RdmqkkiDIdb$M)%-`z z@wr?`1|$DCQGfZw#8MUSAx<_EB<8qx&cVNug72egQup$DJ=CO(NJXFcsCpES@kR8J zCHbUCJghZsO^<8FogStQ{F@E_ojv+Dw)G#~eV;shZx_TLqm>?0+-cO RtJGXK4{6s z4co&OH>4kOR|w`&cE_#>oc6sMo21nZc{@IV9RhUdH+hXwe~k*>6dUqxBk)~MoO8W% z!fi0hSpadU?KyPAl_Iz`yAE#6_d=Yh3D@`V`91rwLdZcY7bsM?I#ZdTfz5?0uV4;cZ5hH|tuxEB@m=}BnjMHHXO3kH>evGEFZh)E zA4=(^1!%)}+%*?xai7ELOfYBC7}uh_?Rmbi<;CFw=2>}KsslO)tC((0Du-XO`a)n>K)vPy*u(}_{ zJuV1}kl#LMhoQUfHSpeHF>J0jT+Ds17`OqeyB;gbhn);sQNgl0`ZLSoz4*{8_=n#@ zw?!TkL(3h=u08zbz8pV)T}=!25G|6JuSaP+~OmeZqh2v{5O~Rf7 z!=RluW0t0?C0d=nGFPJIcj&^+DlPq_T&Y%+BU&}5W+@*I>Di-0@@&Z9J8)WcPkEtQ zx>_=+I=-+pH#hOrUO!-5y|&!EJZ;Wgn||TmV?OrMy#aR{Hg8$CSnH*GhYDZ>G$2F^PB1aCt^a!?U2G zj06SlgqwYJ0L4#_!t7@}Czq2z6`ltL6$%*Q2v900A64BMxiE@OI_i-2%7N_KL+AJ9 z__?Kq_PbTZj+N+OHjcpQ@P`tq!+v-m99H|TViU3s?lACs6uc8;DbY+!l4jzaMM8qa zastD14cy~5ch4n4%laL%aO@bt( zB#ZLRo@u7GoV?hPbA+s%>@I`b3CdCEIf8$Xa1olD_l#+kQB>;aycZR|$^{BO48v52 z6{iQU*;ob4B0Y5XcGAkEQFd4(@R4C^Eefm!+PfYNj{0PRo0d6fYH zf$vHV*IS=7Hs35SR7z%*BFb+*hYHKnb4#TPJ^S_qed;$po{O?wkce_e5D=wQ3`F^3 z!5bMCVAi$k?k0*I8 zSOrV?D)hMYUHK31qJ#hTlET1W`c532BK<>$14bcz&=2bF^nM@d>D{@$eSNRcvvXtn zh8(VLUzNlB_WXmwsof8Lcl$TDzbc&Wunv1NIpg324(|_;R1TnBNbjElbzdL+QXkyQ zzP*<%e4R?B(qCsuq3^Ev_)=RR{P#acurHLJxifw0impEu?@|>N?^Fr<;&_kKT^#Gf z*bDQyHPH!aQ~)mx>9=qwd`*4jG10UO2eR9(?#to#H8pzzN7$xX3cV8VGKej*5_8di zQ&fHjscA(jq?$>7*og>$lqR9mVQY4?F2<9Q{(k6tQOapHf-vf_YFw&!3|P5L zn5t7M#8CPm-?wx5*OzylPv-xeKi}5RKasr?apeCoZItv^-;dJhP&g>#@;BtyUxt@g zq{yeyNxV#IOWJ|zcW>{j@eiZ%{E3j*s@f6W{}xjixGTIgWygGkbfKjSqYeOOSk?jZ zd7fz(JM&SpA`_HqCOHOzM|HwCfnfo)!}%l7+Ry-1eK(acRBhS{7~LF_Wgno1Ua#C- z0G`D0EGx$GbY!@pi-Ocf!NB1-B+9v=AD3EDu8Q*|&V9tO02yU1RbyHfQqcr9AAj`k zVa{czO#34cTjW9J z5tq)S(z$|{*d>*S(lgZcF&Jfkj9R!^XK(Q-EXdake}Ha3scG6lBCF|-28lNMkmUbL fzWtE&{GFV6lt%6M=r!ckM^gyJ$64t<>c;;9zA^H* diff --git a/submission/sample-candidate/pipeline/__pycache__/warehouse.cpython-314.pyc b/submission/sample-candidate/pipeline/__pycache__/warehouse.cpython-314.pyc deleted file mode 100644 index da28d229906ec66ff5cf0d96b82699210bbe1efa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5351 zcmdrQU2GgjdG>DaZ|}}`{t1rbB%a2>^`(xl4WWcIad7N&?bx|o*XK*Bx;?G$#`ZRQ zyVu>lI5r53RNx8^#1v%rsYW2xRgs8zX$nH>3#d=Xk*R7{m55sLfW(7?Qvy8k&Ft-+ z&vi)4BO~q3%s1bBGxL4ld_VJ8Ls&pi{_yhe<*N~dKBgU~cYdk&&?7QOL#CbmPjY; zBVan4nlr=z8B}JPlk<<9RJo8M6==1S$Rht7^S1X616Q^}z&|OT; z16{+^iByo)8KFS*8G>KbrtnoV55qvOsLKUOpQnIcJU%dZ0>j2l4Iqp&2ALpS^pVoA2$^~*9_;`&d7&xZO1SAsUte~W-8i)_hVI-j+0Evkt zr|EeskSoEeus_2J&COQ(90QU?_0I<^z0dduk7)J~%0wX&C3k`WWXFdnV?xAOY z-V(Cu{_|sr?8W|+6?8~vWmwfFNSh5iDA2wyn;NAPa-ffhezfrLxuLs_p|OZE=mpoQ zml4VX>>zrEo8?4Kbct?}k5gs#{M7|$7N65p_1|qEz^nPG-a)7a&!^@G07%#YEU!+? zlCD~OQd7yB2pHo9E*2!ypz9v7uEZw{U`*U3x!G&LO;d|sEh(kAPKugg|A9a;o|jDN zxMUdej9Q?#Aa0bV3NrLHHC~j9M3GezuU?7X;=C2gW-B5g*{r^wt_Gjc4!=coA9fPB zOMmM}p4y`IyFBY|!!Tlep$%H#{5D?{Xi&bS9FEb^N)ge?3bf-v7~DfKTr~{Q4Ls(h zMIZ{7%q|0Bq93jh6E_|i!pTc%oVYSFo*suwR?{lnU7bPa?NAAfGyUSgaKDIC;>g8* z@mV~Yc($hwTa^l=vRoCr9)lDlS#eM*^y)FF5i1!Llsz$WaA(10WWl6hoF2KD7*F?K zOx05<6+z?=!Kv9Vpfi$8C!S7-j4jEDv9Wq`H9G~U=Pz9vOY|qJs9>Fj`X|QHcv?~n zQe{YtwWK#byAUdOc$WyO7m@Php(5BzVD)F2k`#$rYE}jl10c(>?k7(5;1jWWMB9kv z<{9FilK6j<)!#n5tajK$W0$~o41u})3^J>ea81bd(vS*TNu?TJDgRA!c^Moe^NBX{ z!-WTUdm@3qBoJFT>asUd-Gu~754e-y&rQX-lMhC#yOx0VIc_Fkf3C|(J*|&iC%KYI zVw~Ox=9oE_F1;0+R;zt5@U4}YD{Fgl{UD4ockoqcETT=`|Glwqk8Pn~lm8wHdIDPq zkSAb+LKGAN(7`Gw+~ofU3VQ;28_ceAqBP!QPV^|F`(P(ZW6L--@;*HZsBGtY6SOQJ zNFJgFVN}pmT@Uo_Q&4yA6dF&%DLmUhhyNm)thhFWG7tnnRO|Xm1v`oo+|b(y6rdDTMsY)bXgQ2R+EV?KC4*Y+Z&1c+?R;K3zG+4>-PJagWe-MqPVdt(C%MzS* z?7HXyc&Hvmd!qZ)6$pL088wuGtYUW&Vbu^4JsBSU-V>A#mE*3G@$|Y*RyfmvMpqDu z)16Q`?nPXcCB{y_5$iTP=gkQjNnK& ze}T$p?;U4+Q8W=Oxq4B_HRMC+rQa|)xst9?(2)Q-4De#Gy64aX_H0YJArpex&dz<8 zzthY0)V742Q*^*>82vtI&z?NY_^PXgAFx`4aVVMz!)lG00Q{Ry_<-A04FdYVws$W! zcftue*ePV1TmBX{B=i@H$~?>3DnR4RP5MIUvvg*&p>uV3_4L~1o0*%#w+7ck{hQ(D^6<9D-6GugqG0nY-9I>X z?jc;T`^V=h=<6zVe0oL#}># z?Z8^-=F_)^Zk>AfdeC{*H>O#bFH0P zJ9hKJt#8~?-|JtGCSfLfRyhlNQOYqb{lYMG0;l8p*2~e=g^FS}WcIglhAp2%Ss_z3fDRwL>#7KL(h8}XcLD@w-?(ebad@uM)_?ppMf53})48sJgl K{uRy%(EbBRM*6h? diff --git a/submission/sample-candidate/pipeline/cdc.py b/submission/sample-candidate/pipeline/cdc.py deleted file mode 100644 index c8c2dcb..0000000 --- a/submission/sample-candidate/pipeline/cdc.py +++ /dev/null @@ -1,89 +0,0 @@ -""" -CDC capture layer. - -Simulates WAL-based change capture: every insert/update/delete on the source -produces a CDCRecord with a monotonically increasing sequence number. - -Replay safety: callers can checkpoint the last processed sequence and call -records_since(offset) to replay only unprocessed changes after a restart. -""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from datetime import datetime, timezone -from typing import Any - -VALID_OPERATIONS = frozenset({"insert", "update", "delete"}) - - -@dataclass -class CDCRecord: - operation: str - table: str - primary_key: str - data: dict[str, Any] - captured_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) - sequence: int = 0 - - def __post_init__(self) -> None: - if self.operation not in VALID_OPERATIONS: - raise ValueError( - f"Invalid CDC operation {self.operation!r}. " - f"Must be one of: {sorted(VALID_OPERATIONS)}" - ) - - -class CDCCapture: - """ - In-process CDC log. - - Production analogue: a Debezium/Kafka connector reading Postgres WAL. - Each record carries a sequence number equivalent to a Kafka offset or - Postgres LSN for checkpoint-based replay. - """ - - def __init__(self) -> None: - self._log: list[CDCRecord] = [] - self._seq: int = 0 - - # ── public write API ───────────────────────────────────────────────────── - - def insert(self, table: str, pk: str, data: dict[str, Any]) -> CDCRecord: - return self._record("insert", table, pk, data) - - def update(self, table: str, pk: str, data: dict[str, Any]) -> CDCRecord: - return self._record("update", table, pk, data) - - def delete(self, table: str, pk: str, data: dict[str, Any]) -> CDCRecord: - return self._record("delete", table, pk, data) - - # ── public read / replay API ───────────────────────────────────────────── - - def records_since(self, offset: int = 0) -> list[CDCRecord]: - """Return all records with sequence > offset (checkpoint replay).""" - return [r for r in self._log if r.sequence > offset] - - @property - def latest_sequence(self) -> int: - return self._seq - - @property - def log(self) -> list[CDCRecord]: - return list(self._log) - - # ── internal ───────────────────────────────────────────────────────────── - - def _record( - self, operation: str, table: str, pk: str, data: dict[str, Any] - ) -> CDCRecord: - self._seq += 1 - rec = CDCRecord( - operation=operation, - table=table, - primary_key=pk, - data=data, - sequence=self._seq, - ) - self._log.append(rec) - return rec diff --git a/submission/sample-candidate/pipeline/lake.py b/submission/sample-candidate/pipeline/lake.py deleted file mode 100644 index 7cb15a7..0000000 --- a/submission/sample-candidate/pipeline/lake.py +++ /dev/null @@ -1,69 +0,0 @@ -""" -Lake layer — append-only storage for every CDC event. - -Every change is written exactly once. The lake is the source of truth for -point-in-time replay and historical reconstruction. - -Production analogue: Parquet/Delta files on S3 or GCS, partitioned by -table_name and captured_at date. No row is ever modified or deleted. -""" - -from __future__ import annotations - -import json -from datetime import datetime - -import duckdb - -from pipeline.cdc import CDCRecord - - -def create_lake_table(conn: duckdb.DuckDBPyConnection) -> None: - conn.execute(""" - CREATE TABLE IF NOT EXISTS lake_cdc_events ( - sequence INTEGER NOT NULL, - operation VARCHAR NOT NULL, - table_name VARCHAR NOT NULL, - primary_key VARCHAR NOT NULL, - data VARCHAR NOT NULL, - captured_at TIMESTAMP NOT NULL - ) - """) - - -def append_to_lake(conn: duckdb.DuckDBPyConnection, records: list[CDCRecord]) -> int: - """ - Append CDC records to the lake. - - Returns the number of records written. - Idempotency note: in production, deduplicate by sequence before appending. - """ - if not records: - return 0 - - rows = [ - ( - r.sequence, - r.operation, - r.table, - r.primary_key, - _serialize(r.data), - r.captured_at, - ) - for r in records - ] - conn.executemany( - "INSERT INTO lake_cdc_events VALUES (?, ?, ?, ?, ?, ?)", - rows, - ) - return len(rows) - - -def _serialize(data: dict) -> str: - return json.dumps(data, default=_json_default) - - -def _json_default(obj: object) -> str: - if isinstance(obj, datetime): - return obj.isoformat() - raise TypeError(f"Object of type {type(obj).__name__} is not JSON serializable") diff --git a/submission/sample-candidate/pipeline/warehouse.py b/submission/sample-candidate/pipeline/warehouse.py deleted file mode 100644 index ab32401..0000000 --- a/submission/sample-candidate/pipeline/warehouse.py +++ /dev/null @@ -1,124 +0,0 @@ -""" -Warehouse layer — current-state snapshot built from CDC events. - -Each warehouse table mirrors the source table with two extra columns: - _cdc_seq : sequence of the last CDC event that touched this row - _deleted : soft-delete flag set when a DELETE event is received - -Production analogue: BigQuery/Snowflake tables refreshed by a streaming -merge job keyed on primary key. SCD2 history tables would sit alongside -these current-state tables for time-travel queries. -""" - -from __future__ import annotations - -import duckdb - -from pipeline.cdc import CDCRecord - -# Source table → warehouse table -_TABLE_MAP: dict[str, str] = { - "customers": "wh_customers", - "wallets": "wh_wallets", - "transactions": "wh_transactions", -} - -# Source table → primary key column name -_PK_MAP: dict[str, str] = { - "customers": "customer_id", - "wallets": "wallet_id", - "transactions": "transaction_id", -} - - -def create_warehouse_tables(conn: duckdb.DuckDBPyConnection) -> None: - conn.execute(""" - CREATE TABLE IF NOT EXISTS wh_customers ( - customer_id VARCHAR PRIMARY KEY, - name VARCHAR, - email VARCHAR, - status VARCHAR, - created_at TIMESTAMP, - updated_at TIMESTAMP, - _cdc_seq INTEGER NOT NULL, - _deleted BOOLEAN NOT NULL DEFAULT false - ) - """) - - conn.execute(""" - CREATE TABLE IF NOT EXISTS wh_wallets ( - wallet_id VARCHAR PRIMARY KEY, - customer_id VARCHAR, - balance DECIMAL(18, 2), - currency VARCHAR, - status VARCHAR, - created_at TIMESTAMP, - updated_at TIMESTAMP, - _cdc_seq INTEGER NOT NULL, - _deleted BOOLEAN NOT NULL DEFAULT false - ) - """) - - conn.execute(""" - CREATE TABLE IF NOT EXISTS wh_transactions ( - transaction_id VARCHAR PRIMARY KEY, - wallet_id VARCHAR, - amount DECIMAL(18, 2), - direction VARCHAR, - status VARCHAR, - reference VARCHAR, - created_at TIMESTAMP, - settled_at TIMESTAMP, - _cdc_seq INTEGER NOT NULL, - _deleted BOOLEAN NOT NULL DEFAULT false - ) - """) - - -def apply_cdc_records( - conn: duckdb.DuckDBPyConnection, records: list[CDCRecord] -) -> None: - """ - Apply CDC records to the warehouse current-state tables in sequence order. - - - insert / update → upsert (insert new row or overwrite existing) - - delete → set _deleted = true - """ - for record in sorted(records, key=lambda r: r.sequence): - wh_table = _TABLE_MAP.get(record.table) - pk_col = _PK_MAP.get(record.table) - if not wh_table or not pk_col: - continue - - pk_val = record.primary_key - - if record.operation == "delete": - conn.execute( - f"UPDATE {wh_table} SET _deleted = true, _cdc_seq = ?" - f" WHERE {pk_col} = ?", - [record.sequence, pk_val], - ) - continue - - data = {**record.data, "_cdc_seq": record.sequence, "_deleted": False} - cols = list(data.keys()) - vals = list(data.values()) - placeholders = ", ".join(["?"] * len(vals)) - - existing = conn.execute( - f"SELECT COUNT(*) FROM {wh_table} WHERE {pk_col} = ?", - [pk_val], - ).fetchone()[0] - - if existing: - set_clause = ", ".join([f"{c} = ?" for c in cols]) - conn.execute( - f"UPDATE {wh_table} SET {set_clause} WHERE {pk_col} = ?", - vals + [pk_val], - ) - else: - col_list = ", ".join(cols) - conn.execute( - f"INSERT INTO {wh_table} ({col_list}) VALUES ({placeholders})", - vals, - ) diff --git a/submission/sample-candidate/requirements.txt b/submission/sample-candidate/requirements.txt deleted file mode 100644 index ab2841c..0000000 --- a/submission/sample-candidate/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -duckdb>=0.10.0 -pytest>=7.4 diff --git a/submission/sample-candidate/scripts/check_schema_contracts.py b/submission/sample-candidate/scripts/check_schema_contracts.py deleted file mode 100644 index 1698b21..0000000 --- a/submission/sample-candidate/scripts/check_schema_contracts.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python3 -""" -check_schema_contracts.py - -Validates that the source tables expose all columns defined in the schema -contract. A missing column means downstream CDC logic or warehouse models -will break — this script fails the build before that happens. - -Exit 0 — all contracts pass. -Exit 1 — one or more violations found. -""" - -import os -import sys - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -import duckdb - -from source.models import SCHEMA_CONTRACT, create_source_tables - - -def check_contracts(conn: duckdb.DuckDBPyConnection) -> list[str]: - """Return a list of violation messages. Empty list = all passed.""" - violations: list[str] = [] - - for table, expected_cols in SCHEMA_CONTRACT.items(): - try: - rows = conn.execute(f"DESCRIBE {table}").fetchall() - except Exception as exc: - violations.append(f"{table}: could not describe table — {exc}") - continue - - actual_cols = {row[0] for row in rows} - - for col in expected_cols: - if col not in actual_cols: - violations.append(f"{table}.{col}: column missing from source table") - - return violations - - -def main() -> int: - conn = duckdb.connect(":memory:") - create_source_tables(conn) - - violations = check_contracts(conn) - - if violations: - print("Schema contract violations:") - for v in violations: - print(f" ✗ {v}") - return 1 - - print(f"All schema contracts passed ({len(SCHEMA_CONTRACT)} tables checked).") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/submission/sample-candidate/scripts/run_data_quality_checks.py b/submission/sample-candidate/scripts/run_data_quality_checks.py deleted file mode 100644 index 31e4cba..0000000 --- a/submission/sample-candidate/scripts/run_data_quality_checks.py +++ /dev/null @@ -1,212 +0,0 @@ -#!/usr/bin/env python3 -""" -run_data_quality_checks.py - -Runs system and business data quality validations against the warehouse. -Seeds an in-memory database with representative data, applies CDC events, -then asserts correctness invariants. - -System checks : PK uniqueness, not-null, referential integrity -Business checks : non-negative balances, positive amounts, valid status enums - -Exit 0 — all checks pass. -Exit 1 — one or more failures found. -""" - -import os -import sys - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from datetime import datetime, timezone - -import duckdb - -from pipeline.cdc import CDCCapture -from pipeline.lake import append_to_lake, create_lake_table -from pipeline.warehouse import apply_cdc_records, create_warehouse_tables - - -def _now() -> datetime: - return datetime.now(timezone.utc) - - -def seed(conn: duckdb.DuckDBPyConnection, capture: CDCCapture) -> None: - """Populate lake + warehouse with representative test data.""" - ts = _now() - - capture.insert( - "customers", - "c1", - { - "customer_id": "c1", - "name": "Alice", - "email": "alice-test-user", - "status": "active", - "created_at": ts, - "updated_at": ts, - }, - ) - capture.insert( - "customers", - "c2", - { - "customer_id": "c2", - "name": "Bob", - "email": "bob-test-user", - "status": "active", - "created_at": ts, - "updated_at": ts, - }, - ) - - capture.insert( - "wallets", - "w1", - { - "wallet_id": "w1", - "customer_id": "c1", - "balance": 100.00, - "currency": "USD", - "status": "active", - "created_at": ts, - "updated_at": ts, - }, - ) - capture.insert( - "wallets", - "w2", - { - "wallet_id": "w2", - "customer_id": "c2", - "balance": 50.00, - "currency": "USD", - "status": "active", - "created_at": ts, - "updated_at": ts, - }, - ) - - capture.insert( - "transactions", - "t1", - { - "transaction_id": "t1", - "wallet_id": "w1", - "amount": 25.00, - "direction": "credit", - "status": "settled", - "reference": "ref-001", - "created_at": ts, - "settled_at": ts, - }, - ) - capture.insert( - "transactions", - "t2", - { - "transaction_id": "t2", - "wallet_id": "w2", - "amount": 10.00, - "direction": "debit", - "status": "settled", - "reference": "ref-002", - "created_at": ts, - "settled_at": ts, - }, - ) - - create_lake_table(conn) - create_warehouse_tables(conn) - records = capture.records_since(0) - append_to_lake(conn, records) - apply_cdc_records(conn, records) - - -def run_checks(conn: duckdb.DuckDBPyConnection) -> list[str]: - failures: list[str] = [] - - # ── system checks ───────────────────────────────────────────────────────── - - pk_map = { - "wh_customers": "customer_id", - "wh_wallets": "wallet_id", - "wh_transactions": "transaction_id", - } - for table, pk in pk_map.items(): - total = conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] - distinct = conn.execute(f"SELECT COUNT(DISTINCT {pk}) FROM {table}").fetchone()[ - 0 - ] - if total != distinct: - failures.append( - f"{table}: PK not unique — {total} rows, {distinct} distinct {pk}" - ) - - not_null_checks = [ - ("wh_customers", "name"), - ("wh_customers", "email"), - ("wh_wallets", "customer_id"), - ("wh_wallets", "currency"), - ("wh_transactions", "wallet_id"), - ("wh_transactions", "amount"), - ("wh_transactions", "direction"), - ] - for table, col in not_null_checks: - nulls = conn.execute( - f"SELECT COUNT(*) FROM {table} WHERE {col} IS NULL" - ).fetchone()[0] - if nulls: - failures.append(f"{table}.{col}: {nulls} NULL value(s) found") - - # ── business checks ─────────────────────────────────────────────────────── - - neg_bal = conn.execute( - "SELECT COUNT(*) FROM wh_wallets WHERE balance < 0" - ).fetchone()[0] - if neg_bal: - failures.append(f"wh_wallets: {neg_bal} row(s) with negative balance") - - non_pos = conn.execute( - "SELECT COUNT(*) FROM wh_transactions WHERE amount <= 0" - ).fetchone()[0] - if non_pos: - failures.append(f"wh_transactions: {non_pos} row(s) with non-positive amount") - - bad_dir = conn.execute( - "SELECT COUNT(*) FROM wh_transactions" - " WHERE direction NOT IN ('credit', 'debit')" - ).fetchone()[0] - if bad_dir: - failures.append(f"wh_transactions: {bad_dir} row(s) with invalid direction") - - # ── lake completeness ───────────────────────────────────────────────────── - - lake_count = conn.execute("SELECT COUNT(*) FROM lake_cdc_events").fetchone()[0] - if lake_count == 0: - failures.append("lake_cdc_events: no records found — lake appears empty") - - return failures - - -def main() -> int: - conn = duckdb.connect(":memory:") - capture = CDCCapture() - seed(conn, capture) - - failures = run_checks(conn) - - if failures: - print("Data quality failures:") - for f in failures: - print(f" ✗ {f}") - return 1 - - print( - f"All data quality checks passed ({len(run_checks.__code__.co_consts)} rules checked)." - ) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/submission/sample-candidate/scripts/validate_catalog.py b/submission/sample-candidate/scripts/validate_catalog.py deleted file mode 100644 index 6a02026..0000000 --- a/submission/sample-candidate/scripts/validate_catalog.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python3 -""" -validate_catalog.py - -Verifies that catalog/catalog.json is present and contains a complete entry -for every required lake and warehouse dataset. - -Required fields per entry: name, layer, description, owner, schema, update_cadence - -Exit 0 — catalog is valid. -Exit 1 — missing file, missing datasets, or missing required fields. -""" - -import json -import os -import sys - -CATALOG_PATH = os.path.join( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))), - "catalog", - "catalog.json", -) - -REQUIRED_DATASETS = [ - "lake_cdc_events", - "wh_customers", - "wh_wallets", - "wh_transactions", -] - -REQUIRED_FIELDS = ["name", "layer", "description", "owner", "schema", "update_cadence"] - - -def validate() -> list[str]: - violations: list[str] = [] - - if not os.path.exists(CATALOG_PATH): - violations.append(f"catalog.json not found at {CATALOG_PATH}") - return violations - - with open(CATALOG_PATH) as f: - try: - catalog = json.load(f) - except json.JSONDecodeError as exc: - violations.append(f"catalog.json is not valid JSON: {exc}") - return violations - - datasets = {d["name"]: d for d in catalog.get("datasets", []) if "name" in d} - - for name in REQUIRED_DATASETS: - if name not in datasets: - violations.append(f"Missing dataset entry: {name}") - continue - - entry = datasets[name] - for field in REQUIRED_FIELDS: - if field not in entry or not entry[field]: - violations.append(f"{name}: missing or empty required field '{field}'") - - return violations - - -def main() -> int: - violations = validate() - - if violations: - print("Catalog validation failures:") - for v in violations: - print(f" ✗ {v}") - return 1 - - print(f"Catalog validation passed ({len(REQUIRED_DATASETS)} datasets verified).") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/submission/sample-candidate/source/__init__.py b/submission/sample-candidate/source/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/submission/sample-candidate/source/__pycache__/__init__.cpython-314.pyc b/submission/sample-candidate/source/__pycache__/__init__.cpython-314.pyc deleted file mode 100644 index b3cc59dc4a6f82d0cf027e07e2ab3c4ed573993c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 184 zcmdPq>P{wCAAftgHh(Vb_lhJP_LlF~@{~08CD^x$UIJKx) zza%v|qb#*3vm{?XyELa%zbLgJzZgQMmXsFgrzDmn>LwN!XQt=nrskCt>lc?M@0%4+Er1y;=UYtT4>4+;2kgm+;cuvnvBE7tz=y`Zo z;9Y>Xx}a3^JpT!LfHss-q!&k!UK&B}F<#J>w~=*f~0oSGgpE$;Xp z1Jxx?eQyneS%*7>;f#@qL{Ys*%#Dr+2BEAKJnlNH#M-f3u6#w=-SrRf6mT=fCe5}VcpP7s3yhmQV0hIbuFWtdj8n(r(GDA)_($Re zZM5zbv=8z#qXfC8_`Ei6h*P!injb>iVoFa-arHp5B^SlAb%74~ON!z^$G z5|0=Kc=GTZT!U#B-9-;_)1M<`1Mdv_NLg2OWf@kYv6xG!&Rfsv`DzY)T=+w*h6T&_ zys$(G52)AFa$(61tPOi59GZdm%#FpJItUQyD@_SY5Cv_MGjX1`!lmjR2A-~RVy$lx z>Ttih5x7B>l1-mQGuYj%+9o$Inv6MXUKBhmSZO=(?R!-wgRI*2ZQ`=Y&7H8{Fig+$ z0d`^-bO1KG01v}3?V{&MQz}oz^knOo(di#Yr}xCi_btgg02w6l5`-86D|iYAE)W8* z#RN&Q8V_B$6mv&kCXsukM7|@T7XaVlqefji9Y=qNxw+y(aBn+&9RuhPM{r*_!;Xjf zI>Bqs7Q{sax>%1fpESROv#B^9b-ht*HE^qTW3GW`Z{p@c3pYNVU1}|5F5Y-5F0_*P zG8`M@yEVOjtES^ceRjU4-^aHb_b0lfJ+n=c%~ENxOY_d$T(>AB1;=O>FUDxjq9*)4GqBxpEk{X6GAA zt=jyeMC`?X2{s`{qx*T7aC$^dQsyw+os{u@OxT{44!G+tlgPw;i<`~sq`;X#|R>JPGU{u z(4LK~fKuXFJ5sIJuY+Ti`j3eBKSf6Hnvs#s@IB#?S$e%Y+Z9g$crV=9Ac-S# zJk{y(TU`63+r;#;7>OgGQ!0Dx*ba?~Mr^6~Oozu62*-vy`ts_-WwsqId}@;wAbKx* zR}6ubf4f~a&4|q6cxy7xM8#Z?tB~xXd_xJ;y2COKC96aPeUd=qLv%-bS6YITzA7_i z`UweTDxV@4<)kK5=RWd~SOF)RLPdKhtMqx=4`g^U+SiDRkr5RmdK{Kyb-fJ4F8Zxd z{$lRaxjm!~mH$L)v9fmx6)O>|U$FXtbtYvE43(dufntRYf_7J%QUr!o5*z9sxU1&m+;4U5T7T z-@vLciPmI5DLfU^^LK`o+&+To@yW;5lS|*^zrOsH_w(^{PX%xP9EeEBv{HB^ld8jl zD86A)*43~Sg=DC90wz>DWlHM{E9o7kr^J8u#pyUKC&`zd20lF_rZ-{I%Hd#la-nC1 zVkx@CVI{qEQZcNg0WEEWY9|)85TJMR}G3U=GeAWpJ;IipPFBe(%@9z&2pA2 diff --git a/submission/sample-candidate/source/models.py b/submission/sample-candidate/source/models.py deleted file mode 100644 index 91c9d19..0000000 --- a/submission/sample-candidate/source/models.py +++ /dev/null @@ -1,84 +0,0 @@ -""" -Source schema for a payments/wallet system. - -Domain: customers own wallets; wallets have transactions. - -Strong entities : customers, wallets -Weak entities : transactions (lifecycle tied to wallet) - -Invariants: -- wallet.balance >= 0 -- transaction.amount > 0 -- status fields restricted to known enum values -- settled_at must be >= created_at when present -""" - -import duckdb - -# Expected columns per table — used by schema-contract checks. -SCHEMA_CONTRACT: dict[str, list[str]] = { - "customers": ["customer_id", "name", "email", "status", "created_at", "updated_at"], - "wallets": [ - "wallet_id", - "customer_id", - "balance", - "currency", - "status", - "created_at", - "updated_at", - ], - "transactions": [ - "transaction_id", - "wallet_id", - "amount", - "direction", - "status", - "reference", - "created_at", - "settled_at", - ], -} - - -def create_source_tables(conn: duckdb.DuckDBPyConnection) -> None: - """Create source tables with constraints in the given DuckDB connection.""" - conn.execute(""" - CREATE TABLE IF NOT EXISTS customers ( - customer_id VARCHAR PRIMARY KEY, - name VARCHAR NOT NULL, - email VARCHAR NOT NULL, - status VARCHAR NOT NULL - CHECK (status IN ('active', 'suspended', 'closed')), - created_at TIMESTAMP NOT NULL, - updated_at TIMESTAMP NOT NULL - ) - """) - - conn.execute(""" - CREATE TABLE IF NOT EXISTS wallets ( - wallet_id VARCHAR PRIMARY KEY, - customer_id VARCHAR NOT NULL REFERENCES customers(customer_id), - balance DECIMAL(18, 2) NOT NULL DEFAULT 0.00 - CHECK (balance >= 0), - currency VARCHAR NOT NULL, - status VARCHAR NOT NULL - CHECK (status IN ('active', 'frozen', 'closed')), - created_at TIMESTAMP NOT NULL, - updated_at TIMESTAMP NOT NULL - ) - """) - - conn.execute(""" - CREATE TABLE IF NOT EXISTS transactions ( - transaction_id VARCHAR PRIMARY KEY, - wallet_id VARCHAR NOT NULL REFERENCES wallets(wallet_id), - amount DECIMAL(18, 2) NOT NULL CHECK (amount > 0), - direction VARCHAR NOT NULL - CHECK (direction IN ('credit', 'debit')), - status VARCHAR NOT NULL - CHECK (status IN ('pending', 'settled', 'failed', 'reversed')), - reference VARCHAR, - created_at TIMESTAMP NOT NULL, - settled_at TIMESTAMP - ) - """) diff --git a/submission/sample-candidate/tests/__pycache__/conftest.cpython-314-pytest-9.0.2.pyc b/submission/sample-candidate/tests/__pycache__/conftest.cpython-314-pytest-9.0.2.pyc deleted file mode 100644 index d04dfd07f2a8da48b1db961c5e3af7a09d578951..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3905 zcmbUkO>Y~=b#}S@UXmvDMcIn9mSnj~EHZX=NE4@!Y&oHNFw}LTw1MlzYIkHVOzsl1 zvsT3uTLegv1c(kkC_sSb>U)1f&o(T;%v5cH7DdpTTsh6PZ)TU2Y|B#G1?KC``ZmBkpVDSBD z9h^XVas=tA5!5(Ehklb;xzJPYU_6WFqlKOZ+ziL?@KHL*jLRKX9YNUv(>_`ZBWl>n0GKMS_A`?SQyk?YjC6@ zAT>4(%eD-HEsxkf8;S5XK*PEh_&7xNXC*cmZTVh6tRCC~wU)35}=ptn0K5zY!^CBPO$C@y%RKoD#0z!4|||*RlzRioay_{ZC4nu>IZ902iBeov|zrf<+dikNA(N!QTE&e<(!~6+DC5)n&W+R)JeGC2&4f70hF)<>yEv~N=q;bWkq zC>R`kDO3|46gmqNi;kPv9~JU{KnW@Lgm<5)E^fZCeh0pV!W#Dnnc5j#A!L&TKF4DC6C%+Nm8A2h)Pv z@o~`iIpJl61seF zxvMhh93xz+@R(2p0Nmk($2{TUjx*sD?EBNu@htSAB#k~1{k`GG#aAB|U)>X{KYbWN z!_yp2ID+1 zQfd{bRiMrRb^d>(F6gK9A~Y_)qz~wW`p`2Rk|)Y9=)=HaSm;pFNB$2EWqnjX4UOw@ zMjz8((Z@lD30>3A3Oh`K9a1OiQP!t`%aqXNj6Mxq&P7~iUc#lHo(>#xtDmS^v4ceQ zL2N$qG^DS7qB;!yK>8_FgNsm+X(_ST`i$~qp ze!>;FphY)xwvu7#zDy=t7g@HqG#r~Lt_hV*e6itJFyN-?G};A|ujcs6JqI-jVF@3e z?Z9UVuGd}6GGT3E8zyC$pv4O$F_*1NEOy0PYvLsk^xMz#mLq0FJJyIpm7JN`UvA;m?iAo6ryusqt}28NZeczvbGZ(mjUGC0G0lbNK=@c+_)8d!oWNil6^0+@ryk~~zFK^g zpXt22tCY6pca*dInYVY8>29|0hqoUnW$_Tqo@PnIuszE#SZXbB8Wh5TsSB-5qrVz8 z!vBs66v_}O5Q!CnH=wA-EZqn3un1upAm;bZFo`HiMLimpFNtu6sl9)c3j9fk=Z)q1 zAYKiH;m7M6R_br`6NQLNQTVW2SVUEto{bxRSQd$}PDVdT2f-^M^ZD05br~pw2L3s@ z0m67S;kTjNi%F97HCp@*p^wmGbp9VG_ZO7=Cz|>Oz4Z-R{wu0{pMXk#FNNgP$K$^r qeqAi&1We#ug_gd`zTjo>YmHz@;YGMZf diff --git a/submission/sample-candidate/tests/__pycache__/test_catalog.cpython-314-pytest-9.0.2.pyc b/submission/sample-candidate/tests/__pycache__/test_catalog.cpython-314-pytest-9.0.2.pyc deleted file mode 100644 index 5ee79d6afd506ff0c9b50276abc05744fe67148a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 19689 zcmeHPZERFmdcJq&j>j|OFTTGFjBSE5fnc!BH#QJ3;D&&6Lx!yCFq(L-VZwMkoI8ft z-L%f`Rxuk=SZS+uD=N_ssUSsFD&@yk%8yi9ZTn+Yjj@fkrkSJk zd+wR>1sithR#gtex##=qzUO_P_dVx&OI>YPgS$QP^Yo^rnl?ZY#|2A)hkxqVv~kVU z^4dAg)O+>3k3anDKQLHF6W$_+cAN0eOS z zS-ncjG<22<<(Dq1UI_KVW%ZVskyToLxmg2$g;|UII?B(lM0?xR5r}Z#j`?Y(@n(Ie zPo$gmXrZC3hc)IBq%GyGu6?>zpP<&VHyh+wmZRSl>R9Aztut35ZxtU?;}f)Q;4Qsy zT9-YnMk{O79`fs-q%~#T)}p0#Wi4$m8y~e z5Ppt_Tb^VT6?58*R$8eQxmue{y+w;{8JWX74xh2?ydC}3_>ZFl$$T<1d_Fp4<&!Cd zqXWZ3xr~*!vX*T}li3s!7_$cQ6yFvO9~r)EF?(NhFrBfYIcC{bcEH*iO=dDtW?dRd zGbM4{!V)^QNBko`6MWy)9FDnBYtf?G7Ok~Z$Q#ipNSIt{p+=~ z9@9_7jG}=TmMzvCIeg~u$rABW`>g~EW19)S3JrhXq3@pxVIhwI|grD z`*o=HT4pY^ZYH#D^3vz^%dU_9aCCCX2jBhLuZ0_tj^A`>-8Xg!r6*VJ4g5>wa5z*( zB{WgKWF~L$@l;~^T#9j*)J^N~ZC#6IO0LZZn({O1K@6V6X3WQEFh#$eXEB|Tj1^=h zCQ0v{zxYj*u^Aq{cwIlG|CakdPCekyoB4k@^3n2Jy?^s(BAwNuD4!Uf+y&tCi7^C- z{QD#|hyvuB+BVJMnho0PpolhY#CL3^rsc3b;`&~cInOuD%NSW4Y3c;y`dhx2S8G}b zWIE!@VGEJer%)R6ELJ;QgAw)NefrN^dCZ^2QPfj5rEV_0T$1{XycYbocf%Xke%T!R z$1Q)m<@Vb@ZJufFnvT3O)!a4t>SrteeaFA9yLNKo@WkMUy*DoX@FbF_BCiyTSN{8M zwfmfwu)Io90wn!J5Zv|EIU)DiBsVXAQB4(n zH@GF85XSEWE}6d&es`Po4y3jne)oB5XJ^cgh2JgubEDgfzG0gJ?c{Zmx04Y~QfT+7SPg5W zh(WCst3)FrKYS=RN)j@WWW^;3A$${J&_nk5VhyqrZ;hlgfC_f8DlOpk4 zLydH(kzFDphPTHiei4P^jJ&v-kH7R95`?W1r5U!N+!p+|{|Vl>_IafC`tGZ{3&xr2 z-B-KunCQN_bvn|D&|IW-4!&`wu%+i`na|D@Vic+P%^GK}wac`-ZtNbK+AT6>s8v}; z1dx4mt4u>ZCx(oI)3MA)V9Gcnqzr~F9Z6=+RJ-AWsY)5+ibsb`e>cPdc~pTn`bmy* z_lDjt*{mbY<}+nee@ccb*=V09V{cwHVmH2dF+lD6fa`7V8>v# z4C|v6!)Y7rFrS3>Mhtcbv==BKjLfeggl}X)=nPnC-xk9o5^b- zk6rM zIJIFX@)dn?2Puj>KI-I;`8tB+-v7b5w+Vn7*9c0w-1GcDbN+xd;k>{y0Bud&*P)xW zXB#?vorWl>s@Gm%8oNevBhHy6AnKDG(dkvwDxj|5b~4)=X1kD6auUJq+9D2@9YQF|O<(>T)HH=wlz!VnJa&Q1JP76_U5B zvZzyy>|%tu)v}#-5Gk7pvOA!CeVMenkhBwy2_tdQ8f8aNo-^3rATX}okE|#d(aBX) zk?5RJ`b`_r2}mG{&ALIEeUHk^*xj<2Ohy1X_gAi;e0z3ftYB=vwy&@<_EG0_WIG~r zk?nKvmESa#kO_C)*ga!AD#{!YK;lQ8G7Y7j7?qyaHVV38qqIdd+NxZHZRhLoco7Dh znY6EvMeI(=B1VavMDO?*DZdWIdvluQ?$F;NEs^v;J@J`4ZU|J8!Eg;Use0L{llZ zJSE*CK0 zOXG(bB ztz&CiqiH+Jt^3*Y(LpvmqIHz%~ zA?}XO>F10~O>y6zkASi}oIL~{Bsc@jhC6S>Y=+~0;OxlRr4>BrNwBj(d~Q#k`_?gD(37B8G<_!XLei=0nVupz{d=HUMG@qSc4q>Hs?RPjOg{! zQDp006G?G#ho9XF&5^dDFFlWe-CdA#R-q~R)##(KqX z#^{~cDPuCAq}}x{Q80RM^vLokq{L>8-pTLW%HJ9+Y(4bZiqC?dT`H_A^(Si}fMOWT zhX)Eq&tDvvGtgSk4@v_lzDFk~+FNF-p#K z>4hvLHptw}PN5)M2p6A0Aq5xjBS5@Q;NpGmAW(?AAbdWEeHa!EqTJ3|eyap0eLjwm zLC7vbK0mrB9UXcg_Mbo4JG8N2Sr^xp+l7H2%_qVC?;P7x$(jd*_VOZ`#;fh_s35tP-A~(z28q zK#oh1&bBbnA*aZvF=4SD)Mg6-CVJmCT?Z5Ps&?8U?VoaMRr@cvv8r8+OJW(6l{;k* z-zjGsRC!juYha(yFSW=0duX3XgtW+g66E-(`fQ7Wk9-P+()v_5P^hGRU`m<$h&?4S zM1`#!7Sv6?Z5^pc!9h$ygNa~ zjPgiYr(u=^lF&c4b9p3tlKKYju2?F;%MvVyCP*2vNg6^Xg|(XHA9hc+&PJLG#*SMT zxi=NrF=v#1)5eZMq*+8~mGBIemZj7Ha{eGRpQ33I0}@KYQLZE$)j#o*a&T1tG*7z0 zQ4bp}P`%iX*I=oFjkb*Lz4QcA)f0UEw3oSyEB07Nk8q_3^oPA9M@n}=NSEYp(dT5+ z{?O-%{9I({@bK1B4<)OKTi+9b&aV#y4m+-TVQ>*+6`G6Nm*-gB-7Lo_G#9s?=NR33 z7#sio;VI7L+&(s`LElP_xo60xt@|{ zaQ6iIxLebh2D= zyXX|-CP1)QA!3Ctx}r6Tqf~r(66HEsR5VwiXTk!>UPC>)_TLTrW7{I_kNj+z%#XD~ zE9FBYwNW`ZX`v_)Y{g9`F($#O@g_ zg?atX7%gNzz5K!cf^lSW3HPoZoHLGy-;8l&V!w>Z0NG1RCGHwWl$cCLLlc)}427K7 zJ-0eVxt@WJXe!osbS4y1WsA z@9#%e7L0?1md@F=J3qcKyY?W()*hUSV8fJt)5byCFch10gEPFGETjgI^}j~aSxiSU z^&BKhKKQA@AMPMfh`S(s0^<*b0{bmJi)C^BMH+glZG3x4Ut%tXoY^Sae+>1;9!O8HTXcGyRfEvDgv8n={Ie3)0HQR&AP!EUQQNL z1ITje#^Hs}h#xadXCCo>g8Q(bVRsPD+xeDw)mu5Y3jM(|%A&Vu_sbbBQ3kfG~ zFTj@QZk$KLIPP^wxD+H3 zZglDV4@JV4X(r3~Oz2C{E~4EoaoeryTN1{)ki%7J+PvH;rpc!#;i}{izL?7w>p{JF zmdx780s1!VcO~F04gsre&ZA+`H=Dq8_}h1Q@v?$(pb*_LyQ=-;J+rHDQgooO>cCV4 z2SKIZv~gep*Q+Qt>jr0dIax>zAj_qpXUvP=#dH*Ge1c=+6Z$7diK0(D4jX@aY(#|Z zZTNG|5Q++4acsmfM6qrphwIEZJxZnsSd4|3mCU@DPlHc;0_Ae^xHm2qsxHUH<*k#( zOYJ9k``0jh{`zTeDhEA<*4?v_Jq6?VPhaNVROI-aQTk0A#|seBof7H=lg1IM=P#Lh%d6Y ziSI|y+%{%i!QVcyiXoX)43^TwQ8HP~#V6iXqB3hj3jW5&a5`HIrqYamoL!v|Uq>gz zr{7D=o>yP*HG7UGj^Yo4yw-E(wPL+YI@a5B^5|>D;9&Y4{Dl@v{8bHpd##Q7+C|3+Tqzb=dVk(7qq}Pc8IcTJ10Emt4Pc^~&|{ zUj6QUqw4+R?;U^t^n0goy!4afKRW)C(?2?W>%{c(7pIMF_l?^7#?psDqro>`^F>Wy zg|DD3`Jzr+_J%%Qk9#Y=>i0K%Z^MlXGy29aefT3V5B-`i`hY?>KrG9ozZO!;rQjEJ zy8gQUMYUgV{;CFF13qY=;_Kd9H>am6VE=<5lp*ZF`4zpBPn=LdD3 zh41S@rG-_u+NSmPFMV|Xj~eI@tr!&lYAG&;Jg6>f<-QRv8ap2b{6628=%oE&2%!fQ pdPH~B^igMQ6#S}&7%PqNp!B-F!bDH#kMHY{&O7@8HA~jG{{gLh-@pI> diff --git a/submission/sample-candidate/tests/__pycache__/test_cdc.cpython-314-pytest-9.0.2.pyc b/submission/sample-candidate/tests/__pycache__/test_cdc.cpython-314-pytest-9.0.2.pyc deleted file mode 100644 index a287f0ae237daa394be983dd772fcc558b8a93f6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20869 zcmeHPTWlQHd7jywy$B$pH^k-E^fEMI6Dk=2R3a%)E0^>RdQyxd*R zEG?6Q3KmIPDsf;H2oM@55C%wVxi9sj(U&GK?d$R)(p{uL+6E~4)<`Xk`XT7|pEGl2 zW;x`NvYhxK!{NW@@}Dzj&N=hnXP#+kj!QU>eD9y~C-z8EjxzQUuoCx{keHV=sUlsL zH2GDz5@5fLWgk~HOE zNeewJ71}DzTKHk9(xOEGTeT=)n-&9X*W!R3nhLl_YXaP>H3L4RwE*tZS^*!{+5jKX z+5sQcrKp=`Up#)c^sa80 zClmRisT-BlOnD+((Nhz8L9YOnH|as`yV*j1B2g;qMz)eK6%&U3ZFGWEPu$d}XY^uD zPfV4HrAn!o&*dw#J&D{^J@-zzlrL6#-_Dx)1e%l!SyW8SlnePBE|V~`SMa%qEBPtC79}`WD(WbWW8`P~C~7gJH66pB$jP`p`lo}ICW!xghXCfKv~)Qz@?A-q zK-qrjkL0VeCXb_rA1Ruiz<*JjTG-GrA;qL@P*+L07QRw4rm_{2vIlAxQ*U5O%v43s zUBw*bE2Y#sGliKHZlh#!i1y_a#+~iWnr42oIHeaWX3Cs-dn%87sfe~%0R}U9Pd@cx_Z=N^i^8Axgi&{B zc*wK>{)3v53Jh5oWjQTh5O?M@DF?LB^wD&1WK5E#UwqoCnEYqj3FG4aFGQUdcB)&_ z+SAIQtaV&EI2ahR`qsi)WSkKN!n@F9V>`dy;to!raI&-RN-Ykio$8j-vOVwi2qzw{ zYe#cjcevYqL)Ic%)OAPx_yPMmTC5>%(8f&#d&EphQaPl>H8mYd$)3{iIBi&VxO(1e zmpQ)}b=dCN{>2m7bV@Db; zaql12@prtAk2&jSr?g;?tF#=^l<8N}3Zt?MHVvDct)pREQQ0V{>_e9-E-KsXq9Oj2 zEMn6Rt)(HZV&kR)o|%hjty)_;R!7O=yP#xoyJ!2?qGS)obNCHra`SK3m_cgPIK_P5bIYxO$60Y~274lxi_C!dm?K z@+`?unXFN3&*ZX&LI$Fjkt6Ks(oA83p=Krv zrMI(%44d{?hNMmEq1JMSO)mtrmkgt1)b?b!;Fr6cV@(m*6+?7b)y|~XNFA@$P&)MWwQY`SA0`t za)u58)5I*0xSY9`uUySoa(_n5u<*@I2?mzc0~?BUt||u>o)bBdpletdE5NXF>kjf7^(JuK3EQftvb*?H;Rn^CN_KqvPM%V(&oj}btQg{^N{cff3l+;-IzI^}OKE|Z) zxQE*ZwBXlN`i3^rkPkW1cc>xm*3vgLkOwY(M{Vg_i-KpVMh9qsK?1!&G8`jp2Jw62 zaRU1ZK(Ud|8wUWvEc&;h{kAc{L-=tGhr~y16yq@UoFs6Bz%vAT2=o%@Baj04IAV~R z$j5bzQpX7l6Br>ts+So7uxKJDh&^tC$n|qGV_!RCRZ915e?Ps}*^zIODx$dy^ZpFc5I=4$Rv>R-za|;>pg3zMvc9OJ>&Mh?U z3%(k(Z8dP|+|HH!CCIIGZl@hR1$Bpy&fN&#dFo^1jc~Vq>iuL}rac1fBv&Hao=Ac!f$y<=QFnG>#JT zAb`Kv*w3gWAFHp}XjM5<;~4rnO#S^9z@d~%=q>- z`Q%jf@N=v2=YL;fB#BsKcht_R(z66P3t(C8*-)%=Rq3g!ojiNTm0lx6f#pu1W*gG{ z+xsKUvO{nikY*R=Hl#VYrTN8q#QYy(l?h}`9)V2JLgQag(j3ylE~4m9U>wpMX^6Wu zX^v_!AAxM=1jf%C8n*>9m_vO?^R{NRovmZ@AJ)-h3<_yq={jn02-YnQqcF%cUJG-# zS|gU=kpCyq%0vFophsX_x8VwZiwT6l4#2A#H>{zTaymY27!4jFOKph$1Mr(le`)w4YIlC6B30T)(aYfa3B6$Fd$(ZZ%*@GZTV4vk=u zr*powGwBGc{R?|TOh6RaFQub){v~;Gj#>2^F`3p&R|f-H+ohM%k-?zwo|2X1>nSdf zAmiMVzqlZD4CV!X@b@9a5qq{RFL0xAiFTG3xDkHH_Eu;yE&dg(hY*!^c^{%3?gRR6 z!}SpQVJhrlEYxeQ$>|7-C)A?E;k}3EP;~(}iIs`68b-h$AEebKlsRVePc-7~(D-c!(?) z4}3X&wuLKxWA1h84z=5(C3trO@DFiBN?9F_csfir)oY^9W86E{L}3JEX1q73O`5=) z1dai?9UsS;Vb5K1g4z4H!-xA&1_nI2L4VcuDaJ(X`7q%}(qZ;9^0`W3Hj^)skHmxp z(8yv0FXP|zN;R>j4<5Vu*sa%=_AVK#u_N;@e72JkwfTcnH&0cSqd$54Lui>t7olYy z#qUSge|jB1OV^RzP@Hp3Il6Fa>AJg!rxzTY72sI8bwaO0M%iMAC!jdnm>-G!! z*{mCb;el)*SV-u^{y{%l6Vr~pS{1Q7VXnO3YlieOTsl|sXG4-rzU#=V(~fkDx&tOa z(GKL5*j*!h!B!Z%d+_(Ml>xTVxI{aP-8I4&Y$c_5*hGlfpO5BFZfAD5X2?ee-@>S> z+M2tj)@ghf(J9yT7}aC`6teTxltLC%s2fR-A0MRFxP(ICWqD&8W>6sSZbf)A9A^un z0XPvy!lL1DvAdXpN1c3y#?xBYnejYIcTChR-%O6$)RXdK+($D?S7E+!PQ6cTRz?8z}NhEU`=`QGnY=v^6;^qym|80J4-{iJ66@6 z`4{1J-B6!cR-dRUsgHVp+KZp1-eonlp;+gdlB%jth^$DE|HeXDQBrnp-47duOYn|6 z*mjO8dPRjhfvavqX)nT6bN`CSE+{P&p8A8)R)0DjbWvbZV9j@m0?|61W$5aSI!9lR{L9bD5NUT}r9*?S3Uv|s7Z(sA#; zoc!0iwcc)5?!9!*wJXC7XDDWmA0fVtR?3G$Hpa-tYLExadkK(ce| zPLJq(@r=N6CxHK%y64uJA3`hb+fcif)h;AHI`vb^FP&Oe`!-O{&Q+zas&?`89i@*n zO{!UQ74s6nbu@-%X|-H&a#Ue86xBR|n$5CJJ643GihKV6_&bqpwg`7|WEdtV`50Nt z@?~kd8=8QiZd=MSL0?%Wvm>t2%48WBQ(atRx4Px;PnLny1z+rg=R`uE-6bay?ZAn= zuk-lJCqm!c0rwUA058}y;?zRYg%P$9uEW*UhPaMA74hi15oQ*B(DD?SMP;t@mV*`t z)$2{8jDvFBmvIpm<3hZ_m?FR!CGkk3MBpueIgfP|8Y{Do4hdb@Am#@y%5_^L>KLXE zyW4_Q+MIzgx%dsnZYcyle#6Ca$v*2*l+V#XrwQzaEhb~!;>ma)V|&Es+@{SD&hM-M za?7e|<@$|3C2SVwZkaugF=v%vh@2TdKycHzjxLD{n-A@Z2i9xT zHruNz$)$FHDmY;KYU@EH-yf{D9$ZxqA$eP7U|CIWDAu_KT#6ZYN_d*so>jI&eTbY$ zpsNK_mzl10MPE9Lh(po^JEESbF(P zpEP~edgbN~n*u^RKEpVQ;{p2DQL0w$IxVXt0N)}a444EOcjt`JjZu|)WVg`ch&l1)5A?;hd+_VL0=R5;6-r^Zx z41IN$MlbP)`^)3$VsxKWq@Aim$tb^7vWIeYhv_$$DSM3o1g1MBJWWPVt87IXwsY&OKT*RIC|vxeCKa_Is&vr|n>D6*jn;pbla!kAi?L^aT5w5{8@q~FWBUqZc8K`Q+u*F~Hw`O4jnoNLn+!9gCc z8>uf{+T0$TB+5b_b~D!cS@9jKGrMApSToOT`i{|Qhp|z}gEJ@o_=1XSO$~7!J;3Yt zifFJ5rZ?*W*f;!e_3UH&gcRauS8Wxtf9pZ)*)IH-19|%L$S8WHfU#_u0^!Ysj&Eno z5-e8Or$`VRZ+-uRJ|l9Kp_z$^(zPPIVtRI}&t-{w5q%ji5g`A3Eu1Tq@F9^p&7u}B zzhfJGxPpNVCm9n2E&*_MVw92e7`6&a9^VA8G^nI*cesX&+ zr|EblID;F=Ws4b_i>#5En!!7*<$|7}@1U57RLB%clg4+@KQRw;k4#kl*VfCLF3H>p zyZ02nRXuQGO*!fIhzZPJwX|=vbS<}ZEy=4b2j*Y-m(D}0?azLAX;J?B=8u~{-Mf5v z^vB(|x<9-$f3Dj8>@Q*6n)?A>jqF;?G5}N3hGLy-N*BEv=@jrZiSkz2iqhrEiDLA$ zFeh?AvUBTBkLY|cD{$Ni)coJwpw#i4Ne|O6x;kDysr=%27%+#tILMF)x+&c|4(reT zTbWp-y&`%Q3D90a;nM*}h8nVBH>U?}N&%Vub)prV^Egz?KhD61jyABYIIkLRK}VhT zzLa}&d(bLw(-FGG4#vvqsB z=3GeQ_c`aA+T*?`Sg+AM2%AzW z@1`T?WM`*aJ~HR}zT5>8MvzGulP=vf2n*uZb+vX!2nqwSF1y7*tXK;5}R{^q}Lq&Dd z_3tSC;*$-%8*2Bm+Fez8Z|~(mMU;(ot}4A%wOeFGg49fm=vS@-$7ZHe-jAF%I7WkOx<^-4e9CskedHf z>igd?zFK#;B`iOEw<9Qb-t7p}qJ-L{ZCNTC1jzPOya`&dXma>SFnbd`(VQJYWjL_9CByC0RE)TGx z$w^z<@z_$*j`)xVrR_{TeTv_b*F0s~PDgJgwPcoqNvED(Ce8Fkmh5`!r~dyrzyWrN zm8i&$of(1Le-D6j4h|0XeE{OdUcBw5Ox2mlmx2bI)U-({Z;+S(g?)SuNZrVwy?R|pUIW7(c*b#LLw66o( zH)v-u6;<3R^mG+t&>iuMx?vj1zc2n+$)wDbGFwbdXU%yfmCGn|DWYG`7Y$uW=Lyl% zW==PZzIgm}{-#chBT6=B=)_bCBwt8fOPSexu2(4*pqh#Ty^4{)YVOPE)4HkmDnu_# zr{)zSbyYX#d*c)5lwvMBThy`cUL}_|_vMPy(@Offp1uJGt`vEA0T31pAUJ)(? zXQop(@H#UEte2LKW25Qe?95EjOkK&+TiKh-E9~lqk|%Zx4aLl}yd*zo^z_BYdIDAy z+iqrObSsK-A)nKGM5{hkDCoJ2X67}l9F8=>k?DCYok?rZEqRhL($JLxesIH{55nis zE)a`CQn(yA{GK3W;Mg|dP4T*@ic@fgZT8yFR+#8!k>q+rf>ptV!dLTTCS@A9 zyTkfk|0TG6quVG%b~b2LOGbX#|H;6MT#o5R1a+f++6!1jS$e5&e1uhbxB} z-ZH)Iv$TYJixc)w5lcARc410#(i34P<$e#^oe7;Ij`c{iL#$vfKSyX!1&e0-BZ0_J zfW$ypLCrKsBcy1X+aR$h{910gbL@x5{xSgMmT2f7E~)oQgV^nxHV}?6L3df^+3p;lQ;Qs;-o%%#C~7g=EmV6Q_llEmh^} zA7Ac*s0LNZb4UL9CFeT*fnLvZE`-&P8cv4$#fqbmDSUo(^VHnz9CN?n4kAv?&NmLM zqawMO3|ldfMiH z*HNcUYV3)5V@}?x9*V1RRZhmMdPshX9+I7!op07dg(&pWC{+TD3y0nsK0kc=qLNF^ z=*k&&{DLxfT}u}YGd}}ZqfCwrtHX-3uVpjJaiu$bpnJ>;7UuU`K|oJT9sHu6VY9M( zZ(;tG9s>`)VuemkXVW^S51<&V$aRS2I`onFML269^nWBSv}5`GJ%+-L(CQBM7?3A$ z{5YI%#`6c@{PE-T+#sBbTM_+jJzX>bw?V6-&@dr+ zVzp=tr8WI+P;EJ?2zwG%*wFQip0T8KK9{pZjUbR(GG3Q;)qr>B0NY_T4SIJDd9&^J z?i}=Hd&Rr+Z374W9>v`lN-DHRnBLYwu3@0|Y&PmbNWTQn4a$D@*o&N!oVn5ZhaGldG7Q=kX@s%wrUQ z1%UtMD1a*X30@AMI-v0Lv$dcRoQAw5GMSgv9>~x)XQ0Ye8oRjnr5*-%Xwy+sPQQu9l8ty^Jx=R%=u^OR@UUAsH8!N=3p%00 z9M7@Q)J;%HS)p{km@`Q$qzM`!q#eW<0gps*zzf061}^|MvI9=_M9DtP+=rqc#Ss+y zQ5--qfZ`ws0}TOZT(dE~wZ@pn`y%KxY{zmx0^ttlMB|9a_%u61$vK~!jKUzXaIPTf9ImfF~VNou?G3g6=i&$0V%j$dJ6p2wSjOph6c zeeeVNJr2XoSpiWx3}cnU(DPI0_8D}uy?Jdc7_jcJ@XuR&Sb&w~Gc3^brJJW_e@oBL zP6{0r3qfit`P;c+$6|P#ox3TRd)~i(eguCe&-?e;e!pr6pgsP@c3)`sTCxPK8q$dZ zT?4We#Ml=v)+#p#2BU5o3%hD%Fi9$Bq|%g(p!iM@XL}-KC)U0T#cmXPP`rfVWfZ+2 zHo`snZ{!se6!Q;Z>f0cE;C>e~YQdbPD$39b`9g6TeD;i%x@zh~L$i~dgu-}H;{-Ko z95YUCxO43GvE`1VCHYuc9$t}$OY$4{#5MWsvNV1#Ma7CdUY6|tk~F?7pJlsip7bi_ z=DECy`_lNjbCp+1@{zK9dPP26l81kpT9ZeXr3?2?QL!RlC`J| zHvy+Msw$)zC}l7N(p6Pce4sn-cp;b!_ziFs*hg^dsRNB}uo_XLYAhL4}f!^Q_<3yc%)JgHX z3ME5T&>vQ%*_+7-wQz4$IoR9^oWOpo+Q7kXN=CuKZl1j6aj+9pZYh7~pDQS}?umIh z*j72%F|}T8NXDuh?D!NKZglh1jCNer1`0mz^t1B~4t67`RHecnsh ziQZmP8v(#ute8VF7*-$+&akr!jDsbi2fiRvEct4}}8MOMm^{a=z4tPl6r+~g`| z<9WOZII~fMAf{0b@_q&B{Hz*;5{5w_k%RpaSBp$>9WvQG5rHdLZJd}7n32XJ+5-?!3c9#5BO`3)?^s8N89AEM|%LOPjRg=(OPx8uumBWG{*y6jhda%!OAnj^`&()MR7O zw&F<2aP%O>bsI-Bc|cGgwl$`Mv`n!83JWLLIO9rT^}A3HHrx1b5J1!T!)gYeN*FH4zGb+9ibAfm_{nU}f8ZpLLf9Mpp(#@10s3xKP?Q z1~&Z1j0a-0ZkYb;; zaK|hj@-vHnFY|JpOBa_V5Vua0rB3!=k~+~u#=SK+xr(`X9&ZB9Y&4N0&_9UgkG@l3 zBL5-lf-18&2xf1sX~8F9QJm@qxk5L<^w~F5H=x#X@5mn~YwHFe$G&vk5JTOd#sK#c z=jryfI;+*<*TDVC^oRqmqU`E$~um!jYyJLIsUPyaQwMl z>HHqR@oz3f)MmiHwi;7hRbdg-I;123qXUNFm$6XCDA`Li<~Kvmz%lX$iV+mwLBTly zJ3fNWQDH3DfEB)?L%5S(5fcW69-X5b15;2u0hVBs6AyNR8iE5o1~~jXvF2SU5C@p+ z?|S`+3!8TMT3MoWovKo%s%7Khj6#@m(l9fX0{)*eq_ z7>IrBKA3&(9zTqRqHP$$G05cEn;yWg^BAD5PTSmXucJ-@n45wVzFeqt zSJ*x~tl4dqKM}w|z6yZda{le&<$M4*Mge%^JRt;Hv(0nY=*CuUaMynGgw0@rzOA-A z?|&p$l3!Qv6-^vpFZUy{FM*#b9Pa4H(y+L2JhjY=@vEk7YyWE)eHD^BFJWZZC))&rGp9m0^Wh zl&dA`B#&7UVj1%_fGHYd>N`K&>oII}GX?jqVN5;mYXhU;AxM5k*9td?36=dde2itU ziSyWb;RAP|VX z@F*zNhd+>?Z)lOnD)ag zI(-d-*X<+^SH~!JgYQ9jyJwFdh6J4X1Fd{j>iWNj5kcN=;2g?7G&P;uBl$0-TB@Y% zzC@?I#WS)nRNH;m+o57pP?)Zhsf<+Orxj<(8g8Yl`@ zZWJ6{nVf*U3xgE_okOEQk^0+B@zQy)3+_}qeSQ;HZ``CS0jN|U{!?=D2!aUtfo;`jT60odiqx%KjCu}U zNo9N&=Fu>+8VdT)VXXKC2KWp@rl377(O46}Pa@zG<&7*2id%#l{{=qA3*Dy*_S=U)7{|XKjNdxQQcOB~@7l-ThrHhNAB?{{{urJ5eB{8x zfbAJ{;WvBf>wyZz8AGr-Lt%0jq-9bbC0(NVT2V3>o6|=FB z%jdM5j&F>CAW@tMDPs*e5Y^*hdCS>S{9HMHaV37S6i=>4W1r#r^XCJCzZ~TGTJX(# zf?icJUTZ*j0Nhy%p2_0NkKpY&%GmftWpqs0+YM8BnXK8}t8{1dE0E}6hTJm`7thr- zepxnZZo*oZ40#Vm$-m)nne`bixS;zGEa;Z27Iae;68&mbNOL9U;~TPjnFnGNf>Y?s z(QUG&V(!Z)Y*%&p9drY$Hl#3dtDNU5NbuH$TB{14z=vSK5d<^DDpnOBz;iCdKV;Ww ziD$mby%L{+Vy#69^B%Q$6t|rJ@0*unZkUJR@y}m-RRM&o_^c{$*TA_g;j?uWcfbQ& z9S#nH3nFw?K?GQrIFM?fEs%O)L*xjiYHE~^5`+@pXjOZr`FG>)9u%mdJ#;svz+KL} zI(*-f*?V8;JG!1KjQK zIEkg^mpsdw9?Q$LwtdAfV6ejZt5*$pV-5xnIRQj4jv3Lii{8Oz-d;w~^&vk50TUg; zY|ex?nBmWcy|;{puF?02QSrZc0t0UGrD9>R{Sy@bl%`+IM8y_38;*BM`&8h|&ScPU zNkEe01gv{?@nx3>Uwl{8dS2_bd3*^0zUav?<&fQAsh`teDR<2H@~z+$Vz5+nGU=H+ zM@|0a;|m}6IB;;YgQeUSJ^?QPUn<#c|NMOZwu;k@B1PmAytH|I@e5u;O@5QT)n(Jr zLOy`I@3G^3@VCldw}K&m39_+)SF%M9Tu#4g= zqu|AcECQ3hy|9pnk$}`}HlxoJ^6-*EliY!7K7)@94u<@%i~^*cX2Q>uS? z@eO;`<1QYP<4fCqa#t&H$`xu+Vr1j!px5BKjRaeLs^l5nEV;_Bc4Gv+enIGfiH(LRK3YEUE zioWHBoWxxf1+J!|G4?e6m4;i<2}w$h6Xe0jB8qoWyoUl&+mbM9-;&{Z!Xzf}`U^UJ zT%-YW;G8IX>d|8}`3$^_@&rMM$9ph-4B|mh6va;hVNv?LQ4q!Vz^Uo}525bAguVYN z?EfqbFHU?|9~NJD*bHd*uvrq{5+AmaAcHHgw`&UcizE9!$?T>;&pzk53XsiDp DqWr^) diff --git a/submission/sample-candidate/tests/__pycache__/test_schema_contracts.cpython-314-pytest-9.0.2.pyc b/submission/sample-candidate/tests/__pycache__/test_schema_contracts.cpython-314-pytest-9.0.2.pyc deleted file mode 100644 index 76d6304a74b3c8e334e3712babf2f1e92ce3a6e0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16244 zcmeHOeQX;?cAq7e-_&PhTef9sCEKx?$ojHu%W*6pEz4&~wyfJIG!p1S(BxX;g(B%) zGPZQdo$doTr3o%>dq7>lZ5132RnUL(M^W6NY0&`JqJP{UlqpNPt&66`0l@)9p;%7S zq$tq$W_D+n6lK|MfoqEl$l3XxH?uRp_vX#pXRE9H1l^l|lWb`uBu*oS=Q5rfIt1p3 zOmgHEA`3%8&Ox8f3u4~ULN2&+?iQBg$$4mwH|K??W5{>GpYvY`*N6V^>PsWr{pT|8{}&6cYNJD)@~<~g~vKK zr^z)PM6SOq^g39cTnoL{VXwVL;asCE>?fh8@?L=u|G1*%H0ht_ek5t}DP=k)#WU%g z8jI(ogpyO@xnw3S#nK5$i(ONWXt_*Qx~fdYUQ1@w4!?gO^O~Y+&q>*srX|ypysp+R zO()HyRDL>bHP$XA)A7u7HkM0XO)0$7saSea@mpOUPN`MC zFDk9OC{emwRdRVX9TIhSA|HP>aaFHB2hVe7$7Tkg7<-SpQ_HE~DY=lKyRT)`=~zy~ zk+kWt&KEU!lb!I!r{I+)bD7Rp^QnBNs$?@7^Pz)mXCju19l@79nVweCIjvL6U!`w4 zlZLvmlBgeLODSqDC9NYnqX(nWSUR1_;eF{khpy|ansWG@H%ZSLdMKDghdS}lZX%`J&s-@Zwzac zIdhZLk{2C$M;FOE&Ibs&-u4QyzlT>HBUOa>NHgST9s9v=p`j!R3A4U)gD(uoLuUsi zSW_P5rV`KR6y1AG$;GE)sZ_|RBDE^MYTc=3ZfHox(@#q(G)eTDDD1s_ECrK?3tv+k zAeGM4NeIl5wdS@zO8tIn@um0k?@tzrQH(VIX{0M zf=hvsf;jT|-N1;3tL!IrC&iNG%ct#S+V+2M`C5zDK2@DCJuqbKo{bx~Oh{C{nBW6X z7gI@Cmk7uqK@Gx#tw}l^be&Ld2WA`q4;M>N*!c;1caE>_?Ood2yN;0$8a{~s#crCe z)<8izQg~8_-N8JI8$o#abEJc0x7nDj#99jHVd`(da#Q#3B@-rpAXgkF#%9HFXDvD3 zMo8BEQsN}^FbRD%kr7xrPC56Hq!4oD&jE2m=~ObUSQrVs zApR<43)4we&8Qk6wVKSr2B$fdc6i6E<2h+m7dmEZpEF?FnSgN5#Jq3!HS?ZAc$3Vu>WTKm@PNpNRD z-0^P@t4Qtchs5FAu@>C98r-)O+_$h}!Ly()?qA$dXzpANcKv;@;osL?P~r=17ZCM_ zf$%QJhr0r2ovshtc82?1ANIQ-9EU9KU_mjQ`v8{X9Cc*H5Wk%@}`t(PfM+G@>rT(a+S7e#8 z0DKO&Hg#hN{$v>) zkydMVIgRknOU77{MynpsCdw|`>{l)?m~$b!w=K&#VW-&6B@X~GQTE8*h}bEVq`D?> z|4?UZd8e6XJzGh{Wl~{2K@#$v60*W6LG6)!z!HLAc%PeArZeiybGkD-(?vHZhPoLD zKEPk)OkYUD#JT5`hTnc_A|x6!MSty#m3tW%VP8Wz~0;Eih%S6>@zbBH}^e`imwU0K5Xv z2SC^cCgniH1@IcY(pv_v9CpSk0)r{d*x=Rb31!SO2k^?_-Q2_`tp>bW^(%tF=K367 zpM+KhuU20bX|-mT)5ze}s#lRltDXU`+m+{16~OCuWw~5e2Cw#-pnWKJ0H1J7kv-Ny zMfSGa3dz2Ry9}B=6TtA8vr*M#rddyD+he{>^9j)G2WWNzG>;DoBi~?>r|buiu2T0y zUG)HXx-+Gu)q@yDyB*4j3M-Cv$g*IQp{ElL%C5nbqf6e*`1FE z+5Hs95wUC7Y2jJyk)kyUlk{Mu!k!^2&W?Hny$w{AvV z#6ya@U%=aXGSr3RJ3nPXD~6M~XtR~rDm!+;-l^p$H^5(x%GOS|MB2$eLBvIY4+JZY zc4K^Bn?0kjSph$}7(2%C%wEG1lFXgPv5ASNR{e^Iy}3SMlnZcYn|jtrWzV){Ii7Vo zON2eqChS@LRmM<`bLKb!QM`aA`(-a+h>v2(KT>6*xqk-a!SMK?G#);CaZrlqK~Xmi z+9&C--z6uVGWb$^NnsSY%*tlr(aYn~=!+LGmX_2& z1C`fU*+-XT+P_2;bc@pX(51l_#>1D!q{^-4v!EJNwkpd>4bf1EW&-yJLeU1ISQ}}9 zy)sVD!)nY5sIWqQeUr?&KsNyE5_JQpy*=iHlez;AGX72XEj!G^-)&{WZ{6sP20A z#_iT0zW(EvKi?1`RXR%}J=f@&;h8N!=tLe?W-x#@{D2;0N*7A3goyCrbLHOYSg zxB<2S4*zcexBD@`n9q~!{fz-PoYDcf@r3TDz&*Rp!D}njesZARgu4<2G7j^VG1frt z&6RCH{RB9@xhO;DSQiK^l?QzT&dLg0mz6RqmWpgj;v?9SQB|6(%EOK4RaHsHXAqk< zjhWMjx&NP7ng`S;v2LayO*`PPHP<*R zOZ@?}5la@{Lh1*Zin+&DN^@DivYy3aN2WAfCiPqOaS4{WN?I2~>fffGHBw+S+m&^i zv~805@gjmcTN{;i6pthI&psQzI6f#Ff5a?x48^L5`st|c%LAfG?9eJX`s!^aJhQs?2 zcw1>U3>mmk*S{>D`BhyjmwsU;iq%5}aR@SJEcq2NpQqimVk_bhRAV{I z5Z+W1vNnSjUa`Z0OWgP~@aLHM4!Iy?VewcN7h|XmDsynJ#ig?>EVD|Nx-#EZ(3%B7 zTy2qcM2MwG$T}kqzJTE+HjzuaBF-_Su*{))LUzL+Ze=D@5<1*J1^HBdTp1;kiIP?8%W9#A^(xDv4`oqe)@f-3~ZDuP6> zK?fdUjXOQ7C!1Z1Q4D6sf*lj0Oj-@4vsJ&MdUJDqfKR_1kb_S|D^ogKeO072!rwQi zktv<6dKGE3>H%#*xym;CmCN(DC|7S=mfyU|HQSbTn{(XhDpCG;Z8e+CcVj(aFVvbf zd*VbyAnPxpfHSIH7HJ_kJdJ0t2BK%!&E_qFU3eF`vXW#N) z+18}19Tn08=LRHIiD%RV*e}pqIC;76T=RVF1q^w|-EcFi zi*Pa8HL8Z7Lr};Mp+f$csjh)QnYyOJ^(tJecBC?s{x@uwt8h6Rs@H?rWLClX>xd`f zD)I*#v*LV8cf%bj-0H%$LT2(YwD@X2Hq?V2)!X}7%c#3ianq=i=po)vy}e!CW8981 z&D3$K%x71*>{_6^TSE;39HR`!)Idy}H^&mim7zt^l91n(8u~INp=P1}H2k$+2*_1F z^%K|Ohi($Aw_TlXqYG+$D>e7kk3DrCd4nH$Yu>4Tv-;LI7tSr7{^^0ga{sNT@XXk< z_iO8JQq}Oz$eSbUPKWEPaNir4A1QhpS!}IopAqmjE_wHTMvHt(u;9mC;iazdLl=hE zyp1etgJqox|Lpi@H0NJgLMXfx3WHS&>wuO0jAiwOf95O80RtE8;8Crt1)tiY6`o_z zVzs{(+{NmAQrQQIg9@+AKhL;58!y z-uPz`U$soG+l+9K&0ViEnZe{=Qr3#{8Fgb&quc?M`XU0bR)56Sj*ZC*6IT6-Vu^Kv zLemxN&Y-e!P6dW+wa?E&7L~w|dBg-|b9@Fvw(3`;Z*zTMdd^Nq3Bz%qh!b&?u@2`1 zn(Di2EQAl z)oD!?3;~D}kGIcvgFEdRCZjPTYP$Q^Ul^Np#U%d%6e;fW< z8|CP00kG)|-RWEkgjPl4R}@3@r|xvJ_&tp9EEZV78a(Z8NsK2$0CLs>4Pf|mr^aqY zRTK}+pS)AUtefs(#BS&WVX%lYvmpkiQnr}ADCDdKb{E9;`>JUCiemeG|D7m{ z-@^#cVu2N`!PD-R#CS3Umc{l*@X1^Bpe++HK!y2aP8RGI;@4}L*5-Pf$yp|69*c7Z zA`WT^CD{0-!^SBp@*^9+bZ%RgAKlCN6}1?MILr8zXjvQBTISbN=IANE;zzntew8J_ zvsz_O$!vMQoGo*f%qQdS#%%dYX3PKOY?-rUK1*f`vAjx6z-qJsI^J!RU^$pwQRA*r z-N&!DR253;)ixa}oMkD*s~Y4~KzO+M2dmaf2ya2UOU)xKGJ5p<_5y~`?yCHTrJJfJ z%K_X_$ZBA4m7I>LGf{+b^W&M0b%}Ivc+tq@X1&M(^3e%d{jfW)DCXh3Huy z{amaariW_V@y(z^MoCXK$Zog$j15fru?}_iuX_+~RS7)!j0ZvQQD5`?G30BgTEee+ zoWclv`i1@n``797=L0Ox_zFK^KzR@RDWxA##%xg%DfvJv>$p-+PhO zb=iO*3MW%BLck8yMXgcYJ(gn8Q=;;#^q~M(K->DLuhhp@9}1Ws38*=!#WZ9RhMop* zcK6vld~K>|+7b99DVGE*eDiZ76}8Z_d+@tX_|g#N%(729%S=-N}6zbPmCWdui0yMhxE;6)R_e7(KOgtLZz2@f~^a!Hwj#tsNRph5b^ F{u2THA94Ty diff --git a/submission/sample-candidate/tests/conftest.py b/submission/sample-candidate/tests/conftest.py deleted file mode 100644 index 7a883e5..0000000 --- a/submission/sample-candidate/tests/conftest.py +++ /dev/null @@ -1,125 +0,0 @@ -"""Shared pytest fixtures for the payments CDC pipeline tests.""" - -from datetime import datetime, timezone - -import duckdb -import pytest - -from pipeline.cdc import CDCCapture -from pipeline.lake import append_to_lake, create_lake_table -from pipeline.warehouse import apply_cdc_records, create_warehouse_tables -from source.models import create_source_tables - - -def _ts() -> datetime: - return datetime.now(timezone.utc) - - -@pytest.fixture() -def conn() -> duckdb.DuckDBPyConnection: - """Fresh in-memory DuckDB with source + lake + warehouse tables.""" - c = duckdb.connect(":memory:") - create_source_tables(c) - create_lake_table(c) - create_warehouse_tables(c) - return c - - -@pytest.fixture() -def capture() -> CDCCapture: - """Empty CDC capture log.""" - return CDCCapture() - - -@pytest.fixture() -def seeded(conn: duckdb.DuckDBPyConnection, capture: CDCCapture): - """ - DuckDB connection pre-loaded with two customers, two wallets, - and two transactions — all flushed through lake and warehouse. - Returns (conn, capture). - """ - ts = _ts() - - capture.insert( - "customers", - "c1", - { - "customer_id": "c1", - "name": "Alice", - "email": "alice-test-user", - "status": "active", - "created_at": ts, - "updated_at": ts, - }, - ) - capture.insert( - "customers", - "c2", - { - "customer_id": "c2", - "name": "Bob", - "email": "bob-test-user", - "status": "active", - "created_at": ts, - "updated_at": ts, - }, - ) - capture.insert( - "wallets", - "w1", - { - "wallet_id": "w1", - "customer_id": "c1", - "balance": 100.00, - "currency": "USD", - "status": "active", - "created_at": ts, - "updated_at": ts, - }, - ) - capture.insert( - "wallets", - "w2", - { - "wallet_id": "w2", - "customer_id": "c2", - "balance": 50.00, - "currency": "USD", - "status": "active", - "created_at": ts, - "updated_at": ts, - }, - ) - capture.insert( - "transactions", - "t1", - { - "transaction_id": "t1", - "wallet_id": "w1", - "amount": 25.00, - "direction": "credit", - "status": "settled", - "reference": "ref-001", - "created_at": ts, - "settled_at": ts, - }, - ) - capture.insert( - "transactions", - "t2", - { - "transaction_id": "t2", - "wallet_id": "w2", - "amount": 10.00, - "direction": "debit", - "status": "settled", - "reference": "ref-002", - "created_at": ts, - "settled_at": ts, - }, - ) - - records = capture.records_since(0) - append_to_lake(conn, records) - apply_cdc_records(conn, records) - return conn, capture diff --git a/submission/sample-candidate/tests/test_catalog.py b/submission/sample-candidate/tests/test_catalog.py deleted file mode 100644 index 9f7d056..0000000 --- a/submission/sample-candidate/tests/test_catalog.py +++ /dev/null @@ -1,134 +0,0 @@ -""" -Tests — catalog metadata completeness and correctness. - -Covers: file presence, all required datasets, required fields, layer labels, -schema presence, and that lake/warehouse datasets are correctly distinguished. -""" - -import json -import os - -import pytest - -CATALOG_PATH = os.path.join( - os.path.dirname(os.path.dirname(__file__)), - "catalog", - "catalog.json", -) - -REQUIRED_DATASETS = [ - "lake_cdc_events", - "wh_customers", - "wh_wallets", - "wh_transactions", -] - -REQUIRED_FIELDS = ["name", "layer", "description", "owner", "schema", "update_cadence"] - - -@pytest.fixture(scope="module") -def catalog() -> dict: - with open(CATALOG_PATH) as f: - return json.load(f) - - -@pytest.fixture(scope="module") -def datasets(catalog: dict) -> dict[str, dict]: - return {d["name"]: d for d in catalog.get("datasets", []) if "name" in d} - - -# ── file presence ───────────────────────────────────────────────────────────── - - -def test_catalog_file_exists(): - assert os.path.exists(CATALOG_PATH), f"catalog.json not found at {CATALOG_PATH}" - - -def test_catalog_file_is_valid_json(): - with open(CATALOG_PATH) as f: - data = json.load(f) - assert isinstance(data, dict) - - -def test_catalog_has_datasets_key(catalog: dict): - assert "datasets" in catalog - assert isinstance(catalog["datasets"], list) - - -# ── required datasets ───────────────────────────────────────────────────────── - - -@pytest.mark.parametrize("name", REQUIRED_DATASETS) -def test_required_dataset_is_present(name: str, datasets: dict): - assert name in datasets, f"Dataset '{name}' is missing from catalog" - - -def test_catalog_has_at_least_four_datasets(datasets: dict): - assert len(datasets) >= 4 - - -# ── required fields ─────────────────────────────────────────────────────────── - - -@pytest.mark.parametrize("name", REQUIRED_DATASETS) -@pytest.mark.parametrize("field", REQUIRED_FIELDS) -def test_required_field_is_present_and_non_empty(name: str, field: str, datasets: dict): - entry = datasets.get(name, {}) - assert field in entry, f"Dataset '{name}' is missing field '{field}'" - assert entry[field], f"Dataset '{name}' field '{field}' is empty" - - -# ── layer labels ────────────────────────────────────────────────────────────── - - -def test_lake_cdc_events_is_labeled_as_lake(datasets: dict): - assert datasets["lake_cdc_events"]["layer"] == "lake" - - -@pytest.mark.parametrize( - "name", - ["wh_customers", "wh_wallets", "wh_transactions"], -) -def test_warehouse_datasets_are_labeled_as_warehouse(name: str, datasets: dict): - assert datasets[name]["layer"] == "warehouse" - - -# ── schema presence ─────────────────────────────────────────────────────────── - - -@pytest.mark.parametrize("name", REQUIRED_DATASETS) -def test_schema_field_is_a_non_empty_dict(name: str, datasets: dict): - schema = datasets[name].get("schema", {}) - assert isinstance(schema, dict) - assert len(schema) > 0, f"Dataset '{name}' has an empty schema" - - -def test_lake_schema_includes_operation_column(datasets: dict): - assert "operation" in datasets["lake_cdc_events"]["schema"] - - -def test_lake_schema_includes_sequence_column(datasets: dict): - assert "sequence" in datasets["lake_cdc_events"]["schema"] - - -def test_wh_customers_schema_includes_pk(datasets: dict): - assert "customer_id" in datasets["wh_customers"]["schema"] - - -def test_wh_transactions_schema_includes_amount(datasets: dict): - assert "amount" in datasets["wh_transactions"]["schema"] - - -# ── update cadence ──────────────────────────────────────────────────────────── - - -def test_lake_update_cadence_is_real_time(datasets: dict): - assert datasets["lake_cdc_events"]["update_cadence"] == "real-time" - - -@pytest.mark.parametrize( - "name", - ["wh_customers", "wh_wallets", "wh_transactions"], -) -def test_warehouse_update_cadence_is_near_real_time(name: str, datasets: dict): - assert datasets[name]["update_cadence"] == "near-real-time" diff --git a/submission/sample-candidate/tests/test_cdc.py b/submission/sample-candidate/tests/test_cdc.py deleted file mode 100644 index 704902e..0000000 --- a/submission/sample-candidate/tests/test_cdc.py +++ /dev/null @@ -1,135 +0,0 @@ -""" -Tests — CDC capture correctness. - -Covers: insert/update/delete capture, invalid operation rejection, -sequence monotonicity, checkpoint-based replay, duplicate safety. -""" - -from datetime import datetime, timezone - -import pytest - -from pipeline.cdc import CDCCapture, CDCRecord - - -def _ts() -> datetime: - return datetime.now(timezone.utc) - - -# ── insert ──────────────────────────────────────────────────────────────────── - - -def test_insert_creates_record_with_correct_operation(): - cap = CDCCapture() - rec = cap.insert("customers", "c1", {"customer_id": "c1", "name": "Alice"}) - assert rec.operation == "insert" - assert rec.table == "customers" - assert rec.primary_key == "c1" - assert rec.data["name"] == "Alice" - - -def test_insert_assigns_sequence_starting_at_one(): - cap = CDCCapture() - rec = cap.insert("customers", "c1", {}) - assert rec.sequence == 1 - - -# ── update ──────────────────────────────────────────────────────────────────── - - -def test_update_creates_record_with_update_operation(): - cap = CDCCapture() - cap.insert("customers", "c1", {"status": "active"}) - rec = cap.update("customers", "c1", {"status": "suspended"}) - assert rec.operation == "update" - assert rec.data["status"] == "suspended" - - -# ── delete ──────────────────────────────────────────────────────────────────── - - -def test_delete_creates_record_with_delete_operation(): - cap = CDCCapture() - cap.insert("customers", "c1", {"customer_id": "c1"}) - rec = cap.delete("customers", "c1", {"customer_id": "c1"}) - assert rec.operation == "delete" - assert rec.primary_key == "c1" - - -# ── invalid operation ───────────────────────────────────────────────────────── - - -def test_invalid_operation_raises_value_error(): - with pytest.raises(ValueError, match="Invalid CDC operation"): - CDCRecord(operation="upsert", table="customers", primary_key="c1", data={}) - - -# ── sequence monotonicity ───────────────────────────────────────────────────── - - -def test_sequences_are_strictly_increasing(): - cap = CDCCapture() - r1 = cap.insert("customers", "c1", {}) - r2 = cap.insert("customers", "c2", {}) - r3 = cap.update("customers", "c1", {}) - assert r1.sequence < r2.sequence < r3.sequence - - -def test_latest_sequence_reflects_last_record(): - cap = CDCCapture() - cap.insert("customers", "c1", {}) - cap.insert("customers", "c2", {}) - last = cap.update("customers", "c1", {}) - assert cap.latest_sequence == last.sequence - - -# ── checkpoint-based replay ─────────────────────────────────────────────────── - - -def test_records_since_returns_only_records_after_offset(): - cap = CDCCapture() - cap.insert("customers", "c1", {}) - cap.insert("customers", "c2", {}) - checkpoint = cap.latest_sequence - r3 = cap.insert("customers", "c3", {}) - - replayed = cap.records_since(checkpoint) - - assert len(replayed) == 1 - assert replayed[0].sequence == r3.sequence - - -def test_records_since_zero_returns_all_records(): - cap = CDCCapture() - cap.insert("customers", "c1", {}) - cap.insert("customers", "c2", {}) - cap.delete("customers", "c1", {}) - assert len(cap.records_since(0)) == 3 - - -def test_records_since_latest_sequence_returns_empty(): - cap = CDCCapture() - cap.insert("customers", "c1", {}) - assert cap.records_since(cap.latest_sequence) == [] - - -# ── duplicate / replay safety ───────────────────────────────────────────────── - - -def test_same_pk_can_appear_multiple_times_in_log(): - """CDC appends every event — deduplication happens downstream.""" - cap = CDCCapture() - cap.insert("customers", "c1", {"status": "active"}) - cap.update("customers", "c1", {"status": "suspended"}) - cap.update("customers", "c1", {"status": "closed"}) - - records = cap.records_since(0) - pk_records = [r for r in records if r.primary_key == "c1"] - assert len(pk_records) == 3 - - -def test_captured_at_is_utc_datetime(): - cap = CDCCapture() - rec = cap.insert("customers", "c1", {}) - assert isinstance(rec.captured_at, datetime) - assert rec.captured_at.tzinfo is not None diff --git a/submission/sample-candidate/tests/test_data_quality.py b/submission/sample-candidate/tests/test_data_quality.py deleted file mode 100644 index 4dbf7f9..0000000 --- a/submission/sample-candidate/tests/test_data_quality.py +++ /dev/null @@ -1,252 +0,0 @@ -""" -Tests — data quality and warehouse correctness. - -Covers: insert propagation, update correctness, soft-delete, replay safety, -PK uniqueness, not-null checks, business rule assertions, lake completeness, -and lake immutability (no deletes or updates to lake rows). -""" - -from datetime import datetime, timezone - -import pytest - -from pipeline.lake import append_to_lake -from pipeline.warehouse import apply_cdc_records - - -def _ts() -> datetime: - return datetime.now(timezone.utc) - - -# ── insert propagation ──────────────────────────────────────────────────────── - - -def test_insert_appears_in_warehouse(seeded): - conn, _ = seeded - row = conn.execute( - "SELECT name FROM wh_customers WHERE customer_id = 'c1'" - ).fetchone() - assert row is not None - assert row[0] == "Alice" - - -def test_insert_appears_in_lake(seeded): - conn, _ = seeded - count = conn.execute( - "SELECT COUNT(*) FROM lake_cdc_events WHERE table_name = 'customers'" - " AND operation = 'insert'" - ).fetchone()[0] - assert count == 2 - - -def test_all_tables_populated_after_seed(seeded): - conn, _ = seeded - assert conn.execute("SELECT COUNT(*) FROM wh_customers").fetchone()[0] == 2 - assert conn.execute("SELECT COUNT(*) FROM wh_wallets").fetchone()[0] == 2 - assert conn.execute("SELECT COUNT(*) FROM wh_transactions").fetchone()[0] == 2 - - -# ── update correctness ──────────────────────────────────────────────────────── - - -def test_update_overwrites_warehouse_row(seeded): - conn, capture = seeded - ts = _ts() - capture.update( - "customers", - "c1", - { - "customer_id": "c1", - "name": "Alice Smith", - "email": "alice-test-user", - "status": "suspended", - "created_at": ts, - "updated_at": ts, - }, - ) - new_records = capture.records_since(capture.latest_sequence - 1) - apply_cdc_records(conn, new_records) - - row = conn.execute( - "SELECT name, status FROM wh_customers WHERE customer_id = 'c1'" - ).fetchone() - assert row[0] == "Alice Smith" - assert row[1] == "suspended" - - -def test_update_does_not_create_duplicate_warehouse_row(seeded): - conn, capture = seeded - ts = _ts() - capture.update( - "wallets", - "w1", - { - "wallet_id": "w1", - "customer_id": "c1", - "balance": 200.00, - "currency": "USD", - "status": "active", - "created_at": ts, - "updated_at": ts, - }, - ) - apply_cdc_records(conn, capture.records_since(capture.latest_sequence - 1)) - - count = conn.execute( - "SELECT COUNT(*) FROM wh_wallets WHERE wallet_id = 'w1'" - ).fetchone()[0] - assert count == 1 - - -# ── soft delete ─────────────────────────────────────────────────────────────── - - -def test_delete_marks_warehouse_row_as_deleted(seeded): - conn, capture = seeded - capture.delete("customers", "c2", {"customer_id": "c2"}) - apply_cdc_records(conn, capture.records_since(capture.latest_sequence - 1)) - - row = conn.execute( - "SELECT _deleted FROM wh_customers WHERE customer_id = 'c2'" - ).fetchone() - assert row is not None - assert row[0] is True - - -def test_delete_does_not_remove_row_from_warehouse(seeded): - conn, capture = seeded - capture.delete("wallets", "w2", {"wallet_id": "w2"}) - apply_cdc_records(conn, capture.records_since(capture.latest_sequence - 1)) - - count = conn.execute( - "SELECT COUNT(*) FROM wh_wallets WHERE wallet_id = 'w2'" - ).fetchone()[0] - assert count == 1 - - -# ── lake immutability ───────────────────────────────────────────────────────── - - -def test_lake_row_count_only_increases(seeded): - conn, capture = seeded - before = conn.execute("SELECT COUNT(*) FROM lake_cdc_events").fetchone()[0] - - ts = _ts() - capture.update( - "customers", - "c1", - { - "customer_id": "c1", - "name": "Alice Updated", - "email": "alice-test-user", - "status": "active", - "created_at": ts, - "updated_at": ts, - }, - ) - new_records = capture.records_since(capture.latest_sequence - 1) - append_to_lake(conn, new_records) - - after = conn.execute("SELECT COUNT(*) FROM lake_cdc_events").fetchone()[0] - assert after > before - - -def test_lake_retains_all_operations_for_same_pk(seeded): - conn, capture = seeded - ts = _ts() - capture.update( - "customers", - "c1", - { - "customer_id": "c1", - "name": "Alice v2", - "email": "alice-test-user", - "status": "active", - "created_at": ts, - "updated_at": ts, - }, - ) - capture.delete("customers", "c1", {"customer_id": "c1"}) - append_to_lake(conn, capture.records_since(capture.latest_sequence - 2)) - - events = conn.execute( - "SELECT operation FROM lake_cdc_events" - " WHERE table_name = 'customers' AND primary_key = 'c1'" - " ORDER BY sequence" - ).fetchall() - ops = [e[0] for e in events] - assert "insert" in ops - assert "update" in ops - assert "delete" in ops - - -# ── PK uniqueness ───────────────────────────────────────────────────────────── - - -def test_warehouse_customers_pk_is_unique(seeded): - conn, _ = seeded - total = conn.execute("SELECT COUNT(*) FROM wh_customers").fetchone()[0] - distinct = conn.execute( - "SELECT COUNT(DISTINCT customer_id) FROM wh_customers" - ).fetchone()[0] - assert total == distinct - - -def test_warehouse_wallets_pk_is_unique(seeded): - conn, _ = seeded - total = conn.execute("SELECT COUNT(*) FROM wh_wallets").fetchone()[0] - distinct = conn.execute( - "SELECT COUNT(DISTINCT wallet_id) FROM wh_wallets" - ).fetchone()[0] - assert total == distinct - - -# ── business rules ──────────────────────────────────────────────────────────── - - -def test_transaction_amounts_are_positive(seeded): - conn, _ = seeded - bad = conn.execute( - "SELECT COUNT(*) FROM wh_transactions WHERE amount <= 0" - ).fetchone()[0] - assert bad == 0 - - -def test_wallet_balances_are_non_negative(seeded): - conn, _ = seeded - bad = conn.execute("SELECT COUNT(*) FROM wh_wallets WHERE balance < 0").fetchone()[ - 0 - ] - assert bad == 0 - - -def test_transaction_directions_are_valid(seeded): - conn, _ = seeded - bad = conn.execute( - "SELECT COUNT(*) FROM wh_transactions" - " WHERE direction NOT IN ('credit', 'debit')" - ).fetchone()[0] - assert bad == 0 - - -# ── replay safety ───────────────────────────────────────────────────────────── - - -def test_replaying_same_records_does_not_create_duplicates(seeded): - conn, capture = seeded - # Replay all records from the beginning - all_records = capture.records_since(0) - apply_cdc_records(conn, all_records) - - count = conn.execute("SELECT COUNT(*) FROM wh_customers").fetchone()[0] - assert count == 2 # Must not double to 4 - - -@pytest.mark.parametrize("offset", [0, 1, 2]) -def test_partial_replay_from_checkpoint_is_idempotent(seeded, offset: int): - conn, capture = seeded - partial = capture.records_since(offset) - apply_cdc_records(conn, partial) - - count = conn.execute("SELECT COUNT(*) FROM wh_customers").fetchone()[0] - assert count == 2 diff --git a/submission/sample-candidate/tests/test_schema_contracts.py b/submission/sample-candidate/tests/test_schema_contracts.py deleted file mode 100644 index 69d30f7..0000000 --- a/submission/sample-candidate/tests/test_schema_contracts.py +++ /dev/null @@ -1,170 +0,0 @@ -""" -Tests — schema contract detection and safe-stop behavior. - -Covers: passing contracts, missing column detection, incompatible schema change -detection (dropped column, added unexpected table), and that ingestion can be -stopped when a contract violation is found. -""" - -import duckdb -import pytest - -from source.models import SCHEMA_CONTRACT, create_source_tables - -# ── helpers ─────────────────────────────────────────────────────────────────── - - -def _actual_columns(conn: duckdb.DuckDBPyConnection, table: str) -> set[str]: - return {row[0] for row in conn.execute(f"DESCRIBE {table}").fetchall()} - - -def _check_contracts(conn: duckdb.DuckDBPyConnection) -> list[str]: - """Inline contract check — mirrors scripts/check_schema_contracts.py.""" - violations: list[str] = [] - for table, expected_cols in SCHEMA_CONTRACT.items(): - try: - actual = _actual_columns(conn, table) - except Exception as exc: - violations.append(f"{table}: {exc}") - continue - for col in expected_cols: - if col not in actual: - violations.append(f"{table}.{col}: column missing") - return violations - - -# ── passing contracts ───────────────────────────────────────────────────────── - - -def test_fresh_source_tables_pass_all_contracts(): - conn = duckdb.connect(":memory:") - create_source_tables(conn) - assert _check_contracts(conn) == [] - - -def test_all_contract_tables_are_present(): - conn = duckdb.connect(":memory:") - create_source_tables(conn) - for table in SCHEMA_CONTRACT: - cols = _actual_columns(conn, table) - assert len(cols) > 0, f"{table} has no columns" - - -# ── missing column detection ────────────────────────────────────────────────── - - -def test_dropped_column_is_detected_as_violation(): - # Create customers without FK-dependent tables so ALTER TABLE is allowed - conn = duckdb.connect(":memory:") - conn.execute(""" - CREATE TABLE customers ( - customer_id VARCHAR PRIMARY KEY, - name VARCHAR NOT NULL, - status VARCHAR NOT NULL, - created_at TIMESTAMP NOT NULL, - updated_at TIMESTAMP NOT NULL - ) - """) - # 'email' was never added — contract check should catch it - violations = _check_contracts(conn) - assert any("email" in v for v in violations) - - -def test_multiple_dropped_columns_all_reported(): - # Create wallets without FK constraint so ALTER TABLE is allowed - conn = duckdb.connect(":memory:") - conn.execute(""" - CREATE TABLE customers (customer_id VARCHAR PRIMARY KEY, - name VARCHAR NOT NULL, email VARCHAR NOT NULL, - status VARCHAR NOT NULL, created_at TIMESTAMP NOT NULL, - updated_at TIMESTAMP NOT NULL) - """) - conn.execute(""" - CREATE TABLE wallets ( - wallet_id VARCHAR PRIMARY KEY, - customer_id VARCHAR NOT NULL, - currency VARCHAR NOT NULL, - status VARCHAR NOT NULL, - created_at TIMESTAMP NOT NULL, - updated_at TIMESTAMP NOT NULL - ) - """) - # 'balance' never added — contract check should report both balance and balance - violations = _check_contracts(conn) - assert any("balance" in v for v in violations) - - -def test_violations_include_table_and_column_name(): - conn = duckdb.connect(":memory:") - create_source_tables(conn) - conn.execute("ALTER TABLE transactions DROP COLUMN amount") - - violations = _check_contracts(conn) - assert any("transactions" in v and "amount" in v for v in violations) - - -# ── contract-based safe stop ────────────────────────────────────────────────── - - -def test_pipeline_stops_when_contract_violated(): - """ - When a contract violation is found, no CDC records should be captured. - This models the stop-the-line behavior required by the assignment. - """ - from pipeline.cdc import CDCCapture - - # Create customers table missing 'email' — simulates a breaking schema change - conn = duckdb.connect(":memory:") - conn.execute(""" - CREATE TABLE customers ( - customer_id VARCHAR PRIMARY KEY, - name VARCHAR NOT NULL, - status VARCHAR NOT NULL, - created_at TIMESTAMP NOT NULL, - updated_at TIMESTAMP NOT NULL - ) - """) - - violations = _check_contracts(conn) - capture = CDCCapture() - - if violations: - # Pipeline aborts — no records captured - pass - else: - capture.insert("customers", "c1", {"customer_id": "c1", "name": "Alice"}) - - assert ( - len(capture.log) == 0 - ), "No records should be captured after a contract violation" - - -# ── schema contract completeness ────────────────────────────────────────────── - - -def test_schema_contract_covers_all_key_tables(): - assert "customers" in SCHEMA_CONTRACT - assert "wallets" in SCHEMA_CONTRACT - assert "transactions" in SCHEMA_CONTRACT - - -def test_schema_contract_includes_primary_key_columns(): - assert "customer_id" in SCHEMA_CONTRACT["customers"] - assert "wallet_id" in SCHEMA_CONTRACT["wallets"] - assert "transaction_id" in SCHEMA_CONTRACT["transactions"] - - -@pytest.mark.parametrize( - "table,col", - [ - ("customers", "status"), - ("wallets", "balance"), - ("wallets", "currency"), - ("transactions", "amount"), - ("transactions", "direction"), - ], -) -def test_business_critical_columns_are_in_contract(table: str, col: str): - assert ( - col in SCHEMA_CONTRACT[table] - ), f"Business-critical column {table}.{col} is not in SCHEMA_CONTRACT" diff --git a/submission/ticket-reservations/LOCAL_VALIDATION.md b/submission/ticket-reservations/LOCAL_VALIDATION.md new file mode 100644 index 0000000..551ffa4 --- /dev/null +++ b/submission/ticket-reservations/LOCAL_VALIDATION.md @@ -0,0 +1,185 @@ +# Local Validation Guide + +This guide explains how to test the ticket reservations CDC pipeline locally. The goal is to prove platform behavior, including replay safety, schema failure handling, delete handling, historical reconstruction, and validation parity. + +The local test harness uses DuckDB and Python to simulate the behavior of the production design: + +- PostgreSQL source schema and schema contract +- Debezium-style CDC events +- append-only silver CDC event storage +- warehouse current and history tables +- catalog and quality validation scripts + +## 1. Set Up the Local Environment + +From the repository root: + +```bash +cd submission/ticket-reservations +python -m pip install -r requirements.txt +``` + +Expected result: + +- dependencies install successfully +- `pytest`, `duckdb`, and local pipeline modules are available + +## 2. Run the Full Test Suite + +```bash +pytest +``` + +This is the broadest local validation. It should pass all tests in `tests/`. + +What this proves: + +- CDC events follow the expected Debezium shape +- offsets and source LSNs are monotonic +- checkpoint replay boundaries work +- duplicate delivery is deduplicated by stable `event_id` +- invalid CDC operations are rejected +- incompatible schema changes fail closed +- additive nullable columns are allowed by contract policy +- append-only lake behavior preserves changes +- warehouse current state uses latest `source_lsn` +- deletes become soft deletes in the warehouse +- history supports point-in-time restore +- catalog entries are discoverable and documented +- quality checks detect reservation total mismatches + +## 3. Validate CDC Behavior Directly + +```bash +pytest tests/test_cdc.py +``` + +Important cases covered: + +- create, update, and delete events have Debezium-style `before`, `after`, and `op` fields +- `kafka_offset` and `source_lsn` increase over time +- `events_after(checkpoint)` returns only events after the saved checkpoint +- duplicate event delivery produces the same deterministic `event_id` +- unsupported operations such as `upsert` are rejected + +Why this matters: + +The pipeline should be restartable and replayable. If a micro-batch fails, the same CDC files can be processed again without creating duplicate business changes. + +## 4. Validate Schema Contract Safety + +```bash +pytest tests/test_schema_contracts.py +python scripts/check_schema_contracts.py +``` + +Important cases covered: + +- the source schema matches the registered contract +- dropping a required column fails ingestion +- changing a column type is detected as incompatible +- unknown tables are rejected +- nullable additive columns are accepted + +Why this matters: + +The pipeline fails closed on breaking schema changes instead of silently writing corrupted or ambiguous data. + +## 5. Validate Lake and Warehouse Semantics + +```bash +pytest tests/test_lake_warehouse.py +``` + +Important cases covered: + +- the silver CDC table keeps every unique change event +- replaying the same events does not duplicate lake records +- warehouse current records use the latest `source_lsn`, even when events arrive out of order +- deletes are represented as soft deletes in current-state tables +- warehouse history can reconstruct prior state at a requested timestamp + +Why this matters: + +The lake is a durable event history, while the warehouse exposes current and point-in-time views derived from that history. + +## 6. Validate Business Data Quality + +```bash +pytest tests/test_data_quality.py +python scripts/run_data_quality_checks.py +``` + +Important cases covered: + +- reservation totals must equal the sum of item unit prices and service fees +- mismatches are detected and returned as failures +- matching totals pass cleanly + +Why this matters: + +Warehouse validation mirrors source business rules instead of relying only on row counts or happy-path execution. + +## 7. Validate Catalog Discoverability + +```bash +pytest tests/test_catalog.py +python scripts/validate_catalog.py +``` + +Important cases covered: + +- raw, silver, warehouse current, and warehouse history datasets are listed +- each dataset has owner, layer, cadence, keys, and description metadata + +Why this matters: + +The platform output is discoverable and reviewable, not just technically written somewhere. + +## 8. Expected Negative-Test Behavior + +These behaviors are intentionally tested because they prove the pipeline handles unsafe cases: + +- dropped source column: rejected as an incompatible schema change +- source type change: reported as a schema violation +- unknown source table: rejected before ingestion +- duplicate CDC delivery: ignored by stable `event_id` +- stale out-of-order event: does not overwrite newer warehouse state +- delete event: retained in history and represented as deleted in current state +- reservation total mismatch: reported by data quality checks + +Run these targeted checks to prove each negative or unsafe case: + +```bash +pytest tests/test_schema_contracts.py::test_dropped_column_is_incompatible +pytest tests/test_schema_contracts.py::test_type_change_is_incompatible +pytest tests/test_schema_contracts.py::test_unknown_table_stops_ingestion +pytest tests/test_cdc.py::test_event_id_is_stable_for_duplicate_delivery +pytest tests/test_lake_warehouse.py::test_warehouse_current_uses_latest_lsn +pytest tests/test_lake_warehouse.py::test_delete_becomes_soft_delete_in_warehouse +pytest tests/test_data_quality.py::test_reservation_total_mismatch_is_detected +``` + +You can also run them together with: + +```bash +pytest \ + tests/test_schema_contracts.py::test_dropped_column_is_incompatible \ + tests/test_schema_contracts.py::test_type_change_is_incompatible \ + tests/test_schema_contracts.py::test_unknown_table_stops_ingestion \ + tests/test_cdc.py::test_event_id_is_stable_for_duplicate_delivery \ + tests/test_lake_warehouse.py::test_warehouse_current_uses_latest_lsn \ + tests/test_lake_warehouse.py::test_delete_becomes_soft_delete_in_warehouse \ + tests/test_data_quality.py::test_reservation_total_mismatch_is_detected +``` + +## 9. Reviewer Checklist + +Use this checklist when evaluating the local run: + +- `pytest` passes +- schema contract script passes +- catalog validation script passes +- quality check script completes and reports the expected validation result +- tests include at least one failure/negative scenario for schema safety, duplicate replay, delete handling, and business-rule validation +- warehouse behavior is derived from CDC history rather than from a blind snapshot refresh diff --git a/submission/ticket-reservations/README.md b/submission/ticket-reservations/README.md new file mode 100644 index 0000000..52aa7c2 --- /dev/null +++ b/submission/ticket-reservations/README.md @@ -0,0 +1,73 @@ +# Ticketing Reservations CDC Lakehouse + +This submission shows a small CDC pipeline for a ticketing and reservations system. + +Stack: + +- PostgreSQL source schema +- Debezium CDC into Kafka +- raw Kafka events landed in S3 +- Spark Structured Streaming on a 5-minute trigger +- Apache Iceberg tables in S3 for the silver layer +- Snowflake refresh from silver Iceberg tables into current/history tables + +## Data Flow + +```text +PostgreSQL + -> Debezium WAL CDC + -> Kafka topics + -> raw Kafka event files in S3 + -> Spark Structured Streaming every 5 minutes + -> Iceberg silver tables in S3 + silver.ticketing_cdc_events + silver.reservations_current + silver.reservation_items_current + silver.payments_current + silver.reservations_history + -> Snowflake refresh + WH_TICKETING_CURRENT.RESERVATIONS_CURRENT + WH_TICKETING_CURRENT.RESERVATION_ITEMS_CURRENT + WH_TICKETING_CURRENT.PAYMENTS_CURRENT + WH_TICKETING_HISTORY.RESERVATIONS_HISTORY +``` + +Checkpoint path: + +```text +s3://ticketing-lakehouse/checkpoints/raw-s3-cdc-to-iceberg-5min +``` + +Offsets and recovery: + +- Spark advances offsets only after a successful micro-batch. +- If the job fails, the same raw S3 files are read again from the last checkpoint. +- Duplicate events are ignored by deterministic `event_id`. +- Late events are handled by `source_lsn` ordering in the current tables. + +## Files + +- `source/models.py`: source schema and contract +- `pipeline/lake.py`: local silver-table writer used for tests +- `pipeline/warehouse.py`: local warehouse current/history writer +- `pipeline/schema_contracts.py`: incompatible schema detection +- `sql/source/postgres_schema.sql`: source DDL +- `sql/source/debezium_connector.json`: CDC connector config +- `sql/spark/raw_s3_event_contract.sql`: raw S3 event file shape +- `sql/spark/iceberg_tables.sql`: silver Iceberg tables +- `sql/spark/structured_streaming_pseudocode.py`: 5-minute Spark flow +- `sql/snowflake/*.sql`: Snowflake refresh and quality SQL +- `catalog/catalog.json`: dataset metadata + +## Run + +```bash +cd submission/ticket-reservations +python -m pip install -r requirements.txt +pytest +python scripts/check_schema_contracts.py +python scripts/validate_catalog.py +python scripts/run_data_quality_checks.py +``` + +For behavior-focused local testing steps, see `LOCAL_VALIDATION.md`. diff --git a/submission/ticket-reservations/SOLUTION.md b/submission/ticket-reservations/SOLUTION.md new file mode 100644 index 0000000..669c251 --- /dev/null +++ b/submission/ticket-reservations/SOLUTION.md @@ -0,0 +1,142 @@ +# Ticketing Reservations CDC Lakehouse Solution + +## Summary + +This is a small CDC design for a ticketing and reservations platform. Raw Kafka CDC events are already landed in S3. Spark Structured Streaming reads those files every 5 minutes, validates the schema, writes the data to Iceberg silver tables in S3 and then refreshes Snowflake warehouse current/history tables. + +## Source Model + +The source model is intentionally centered on a reservation transaction rather than on analytics facts. The tables were chosen because they represent the durable business objects and lifecycle records needed to answer: what was offered, who reserved it, which seats were included, how it was paid for and which tickets were ultimately issued. + +Strong entities: + +- `venues`: stable place where events occur. +- `events`: independently scheduled inventory container for sellable seats. +- `customers`: independent buyer identity used across reservations. +- `reservations`: the main transaction root tying customer, event, status, money and hold/confirmation timing together. + +Weak entities: + +- `seats`: dependent on a venue for real-world identity; the natural key is venue + section + row + seat number. +- `reservation_items`: line items that depend on a reservation and seat. +- `payments`: payment attempts tied to a reservation lifecycle. +- `tickets`: issued fulfillment records tied one-to-one to reservation items. + +Entity relationships: + +- A venue has many events and many seats. +- An event belongs to one venue and has many reservations. +- A customer has many reservations. +- A reservation belongs to one customer and one event and has many reservation items and payments. +- A reservation item belongs to one reservation and one seat. +- A ticket belongs to one reservation item, event, and customer. + +What makes a table strong vs weak in this model: + +- Strong tables have independent lifecycle ownership and can be reasoned about without another row as their parent. For example, a customer or venue remains meaningful even if there are no reservations. +- Weak tables refine or fulfill a parent workflow and carry business meaning through that parent. For example, a reservation item is not useful without its reservation and a ticket is fulfillment of a specific purchased item. + +Important rules: + +- event end time must be after start time +- held reservations require `hold_expires_at` +- amounts and fees must be non-negative +- payment capture cannot predate authorization +- ticket use cannot predate issuance +- reservation item seats are unique within a reservation +- ticket codes and provider references are unique to prevent duplicate fulfillment or payment application +- reservation, payment, item and ticket currencies must agree for a single reservation workflow + +Expected change patterns: + +- `venues`, `customers` and `seats` are relatively stable, with occasional profile, contact or seat-status updates. +- `events` change during planning and sales, then become mostly immutable after closure or cancellation. +- `reservations` are highly mutable while held and settle into confirmed, expired, or cancelled terminal states. +- `reservation_items` are usually inserted during hold creation and rarely updated except for pricing corrections before confirmation. +- `payments` move through authorization, capture, failure and refund states, so updates are expected and ordering matters. +- `tickets` are created after confirmation and then change mainly for voiding or usage scan events. + +## Flow + +```text +raw Kafka event files in S3 + -> Spark Structured Streaming every 5 minutes + -> Iceberg silver tables + -> Snowflake current/history refresh +``` + +## Silver Layer + +Silver tables in Iceberg: + +- `silver.ticketing_cdc_events` +- `silver.reservations_current` +- `silver.reservation_items_current` +- `silver.payments_current` +- `silver.reservations_history` + +`silver.ticketing_cdc_events` keeps every insert, update, and delete event. The current tables keep the latest row per business key. The history table stores SCD2 versions for point-in-time restore. + +## Warehouse Layer + +Snowflake tables: + +- `WH_TICKETING_CURRENT.RESERVATIONS_CURRENT` +- `WH_TICKETING_CURRENT.RESERVATION_ITEMS_CURRENT` +- `WH_TICKETING_CURRENT.PAYMENTS_CURRENT` +- `WH_TICKETING_HISTORY.RESERVATIONS_HISTORY` + +Snowflake refresh is idempotent and only applies newer `source_lsn` values. That keeps replay safe and prevents stale data from overwriting newer rows. + +## Checkpointing + +The Spark job uses a 5-minute trigger and stores offsets in: + +```text +s3://ticketing-lakehouse/checkpoints/raw-s3-cdc-to-iceberg-5min +``` + +Why it helps: + +- restart resumes from the last successful checkpoint +- duplicate delivery is ignored by `event_id` +- late events are ordered by `source_lsn` +- partial failure does not advance the checkpoint + +## Schema Safety + +Incoming rows are checked against the source contract before write. + +Breaking changes stop ingestion: + +- dropped columns +- type changes +- unknown tables +- key changes +- enum/domain contraction + +On failure, the pipeline writes a record to `ops.schema_failures` and does not continue silently. + +## Validation Parity + +Warehouse checks mirror source rules: + +- primary key uniqueness +- referential integrity +- domain checks +- reservation total checks +- payment timing checks +- SCD2 overlap checks + +## Catalog + +The catalog exposes: + +- raw Kafka event files in S3 +- silver Iceberg tables +- Snowflake current/history tables +- schema failure records + +## Responsible AI Usage + +AI helped draft the first version and organize the structure. The final design, naming and failure behavior were reviewed and simplified to match the requested pipeline shape. diff --git a/submission/ticket-reservations/__pycache__/conftest.cpython-313-pytest-8.4.2.pyc b/submission/ticket-reservations/__pycache__/conftest.cpython-313-pytest-8.4.2.pyc new file mode 100644 index 0000000000000000000000000000000000000000..723d4d953c0fd034d7222831443cbf6f4f1ca3fc GIT binary patch literal 766 zcmZuv&ubGw6rRm)Gx^cdf(I=&xJE4G(Coz=#2+X{2njSLQSlHMHpwJi-Rvwgvz7HA zEfMOmMMVFF{yC*c?SKan5pRX)-8V_MLFgRzeeb>bzVE$Z#|sNN1n2Pilb3%X{?JKs z(oh3LsF7GyP&u%8DW&E#ONliO8 z8khsKH23DgW-O_YcIe72yTh0ilDnbZXWW)s)RtcN6_sn83d*-#>9IiABI*o1Apn59UH^jMb=l_R7d$%s=zxCtDU=|r9{y+Ek>M`27ZvSkuZ z1}eJ(EVW^o)7RXSRAnzp3MaK3koEk;<;oPXP*^YGdmU~~T)@`z!BjH@^S7x=hc2gq zR93UmcnTcJm8Cf<4a0!rAsXwY)Q0m6QsIP*!RBzo^PQe6-8GoP8w5kp5YFr=oJ(HH z>9U}&rz?lC%G@8aUgXnQ2RZz5j6=gc$=;}os`M8#hSy+fniH+1bg&cCmY-k#vX z5iT6#8z*@62(Ru~r)GLbj_&TSf2tl-kIj{@7T&AwR`(jajs3?5;`5yk@fYjnnTf0d IzY6020910y*Z=?k literal 0 HcmV?d00001 diff --git a/submission/ticket-reservations/catalog/catalog.json b/submission/ticket-reservations/catalog/catalog.json new file mode 100644 index 0000000..4e0ac6f --- /dev/null +++ b/submission/ticket-reservations/catalog/catalog.json @@ -0,0 +1,123 @@ +{ + "datasets": [ + { + "name": "raw_contract.kafka_cdc_event_shape", + "layer": "raw", + "platform": "s3_file_stream", + "owner": "data-platform", + "description": "Raw Kafka CDC events landed in S3 before Spark processing.", + "cadence": "continuous_landing", + "retention": "kafka_retention_window", + "consumers": ["spark_streaming", "replay"] + }, + { + "name": "silver.ticketing_cdc_events", + "layer": "silver", + "platform": "iceberg_s3", + "owner": "data-platform", + "description": "Immutable CDC event log for ticketing changes.", + "cadence": "5_minute_microbatch", + "retention": "indefinite", + "consumers": ["data_engineering", "audit", "recovery"], + "primary_key": ["event_id"] + }, + { + "name": "silver.reservations_current", + "layer": "silver", + "platform": "iceberg_s3", + "owner": "data-platform", + "description": "Current reservation state from CDC.", + "cadence": "5_minute_microbatch", + "retention": "current_plus_history", + "consumers": ["analytics_engineering", "data_quality"], + "primary_key": ["reservation_id"] + }, + { + "name": "silver.reservation_items_current", + "layer": "silver", + "platform": "iceberg_s3", + "owner": "data-platform", + "description": "Current reservation item state from CDC.", + "cadence": "5_minute_microbatch", + "retention": "current_plus_history", + "consumers": ["analytics_engineering", "data_quality"], + "primary_key": ["reservation_item_id"] + }, + { + "name": "silver.payments_current", + "layer": "silver", + "platform": "iceberg_s3", + "owner": "data-platform", + "description": "Current payment state from CDC.", + "cadence": "5_minute_microbatch", + "retention": "current_plus_history", + "consumers": ["analytics_engineering", "finance"], + "primary_key": ["payment_id"] + }, + { + "name": "silver.reservations_history", + "layer": "silver", + "platform": "iceberg_s3", + "owner": "data-platform", + "description": "SCD2 reservation history for restore and audit.", + "cadence": "5_minute_microbatch", + "retention": "indefinite", + "consumers": ["audit", "recovery", "data_engineering"], + "primary_key": ["cdc_event_id"] + }, + { + "name": "WH_TICKETING_CURRENT.RESERVATIONS_CURRENT", + "layer": "warehouse", + "platform": "snowflake", + "owner": "analytics-engineering", + "description": "Snowflake current reservation table refreshed from silver.", + "cadence": "5_minute_refresh", + "retention": "business_defined", + "consumers": ["business_intelligence", "operations"], + "primary_key": ["reservation_id"] + }, + { + "name": "WH_TICKETING_CURRENT.RESERVATION_ITEMS_CURRENT", + "layer": "warehouse", + "platform": "snowflake", + "owner": "analytics-engineering", + "description": "Snowflake current reservation item table refreshed from silver.", + "cadence": "5_minute_refresh", + "retention": "business_defined", + "consumers": ["business_intelligence", "operations"], + "primary_key": ["reservation_item_id"] + }, + { + "name": "WH_TICKETING_CURRENT.PAYMENTS_CURRENT", + "layer": "warehouse", + "platform": "snowflake", + "owner": "analytics-engineering", + "description": "Snowflake current payment table refreshed from silver.", + "cadence": "5_minute_refresh", + "retention": "business_defined", + "consumers": ["business_intelligence", "finance"], + "primary_key": ["payment_id"] + }, + { + "name": "WH_TICKETING_HISTORY.RESERVATIONS_HISTORY", + "layer": "warehouse", + "platform": "snowflake", + "owner": "analytics-engineering", + "description": "Snowflake reservation history for point-in-time recovery.", + "cadence": "5_minute_refresh", + "retention": "business_defined", + "consumers": ["audit", "recovery"], + "primary_key": ["event_id_cdc"] + }, + { + "name": "ops.schema_failures", + "layer": "ops", + "platform": "iceberg_s3", + "owner": "data-platform", + "description": "Schema compatibility failures and stop-the-line alerts.", + "cadence": "on_failure", + "retention": "365_days", + "consumers": ["data_platform_oncall"] + } + ] +} diff --git a/submission/ticket-reservations/conftest.py b/submission/ticket-reservations/conftest.py new file mode 100644 index 0000000..e4e7afe --- /dev/null +++ b/submission/ticket-reservations/conftest.py @@ -0,0 +1,10 @@ +"""Pytest path bootstrap for the ticket-reservations submission.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) diff --git a/submission/ticket-reservations/pipeline/__init__.py b/submission/ticket-reservations/pipeline/__init__.py new file mode 100644 index 0000000..88a2e56 --- /dev/null +++ b/submission/ticket-reservations/pipeline/__init__.py @@ -0,0 +1,2 @@ +"""CDC pipeline components for ticketing reservations.""" + diff --git a/submission/ticket-reservations/pipeline/cdc.py b/submission/ticket-reservations/pipeline/cdc.py new file mode 100644 index 0000000..81bb9c3 --- /dev/null +++ b/submission/ticket-reservations/pipeline/cdc.py @@ -0,0 +1,114 @@ +""" +Debezium/Kafka-style CDC event model. + +The production pipeline uses Debezium on PostgreSQL WAL and Kafka topics. This +module is a deterministic local analogue used by tests and scripts: it carries +Kafka offsets, source LSNs, operation codes, before/after images, and stable +event ids for replay-safe downstream writes. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any + +VALID_OPS = frozenset({"c", "u", "d", "r"}) + + +@dataclass(frozen=True) +class CDCEvent: + topic: str + table: str + primary_key: str + op: str + after: dict[str, Any] | None + before: dict[str, Any] | None = None + source_lsn: int = 0 + tx_id: str = "local-tx" + kafka_partition: int = 0 + kafka_offset: int = 0 + schema_id: str = "local-v1" + source_event_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) + + def __post_init__(self) -> None: + if self.op not in VALID_OPS: + raise ValueError(f"Invalid Debezium operation {self.op!r}") + if self.op != "d" and self.after is None: + raise ValueError("Create, update, and snapshot events require an after image") + if self.op == "d" and self.before is None: + raise ValueError("Delete events require a before image") + + @property + def event_id(self) -> str: + identity = { + "table": self.table, + "pk": self.primary_key, + "lsn": self.source_lsn, + "tx_id": self.tx_id, + "op": self.op, + } + return hashlib.sha256( + json.dumps(identity, sort_keys=True).encode("utf-8") + ).hexdigest() + + @property + def current_image(self) -> dict[str, Any]: + if self.op == "d": + return self.before or {} + return self.after or {} + + +class KafkaCDCLog: + """Small in-memory Kafka topic analogue with monotonically increasing offsets.""" + + def __init__(self, topic_prefix: str = "ticketing.public") -> None: + self.topic_prefix = topic_prefix + self._events: list[CDCEvent] = [] + self._offset = 0 + self._lsn = 10_000 + + def create(self, table: str, pk: str, after: dict[str, Any]) -> CDCEvent: + return self._append(table, pk, "c", after=after, before=None) + + def update( + self, table: str, pk: str, before: dict[str, Any], after: dict[str, Any] + ) -> CDCEvent: + return self._append(table, pk, "u", after=after, before=before) + + def delete(self, table: str, pk: str, before: dict[str, Any]) -> CDCEvent: + return self._append(table, pk, "d", after=None, before=before) + + def events_after(self, offset: int = -1) -> list[CDCEvent]: + return [event for event in self._events if event.kafka_offset > offset] + + @property + def latest_offset(self) -> int: + return self._offset - 1 + + def _append( + self, + table: str, + pk: str, + op: str, + after: dict[str, Any] | None, + before: dict[str, Any] | None, + ) -> CDCEvent: + event = CDCEvent( + topic=f"{self.topic_prefix}.{table}", + table=table, + primary_key=pk, + op=op, + after=after, + before=before, + source_lsn=self._lsn, + tx_id=f"tx-{self._lsn}", + kafka_offset=self._offset, + ) + self._events.append(event) + self._offset += 1 + self._lsn += 1 + return event + diff --git a/submission/ticket-reservations/pipeline/lake.py b/submission/ticket-reservations/pipeline/lake.py new file mode 100644 index 0000000..84c80fe --- /dev/null +++ b/submission/ticket-reservations/pipeline/lake.py @@ -0,0 +1,79 @@ +"""Silver Iceberg-like append-only storage for every CDC event.""" + +from __future__ import annotations + +import json +from datetime import datetime +from typing import Any + +import duckdb + +from pipeline.cdc import CDCEvent + + +def create_lake_tables(conn: duckdb.DuckDBPyConnection) -> None: + conn.execute(""" + CREATE TABLE IF NOT EXISTS silver_ticketing_cdc_events ( + event_id VARCHAR PRIMARY KEY, + source_table VARCHAR NOT NULL, + primary_key VARCHAR NOT NULL, + op VARCHAR NOT NULL, + before_json VARCHAR, + after_json VARCHAR, + schema_id VARCHAR NOT NULL, + source_lsn BIGINT NOT NULL, + tx_id VARCHAR NOT NULL, + kafka_topic VARCHAR NOT NULL, + kafka_partition INTEGER NOT NULL, + kafka_offset BIGINT NOT NULL, + source_event_at TIMESTAMP NOT NULL, + ingested_at TIMESTAMP NOT NULL + ) + """) + + +def append_events(conn: duckdb.DuckDBPyConnection, events: list[CDCEvent]) -> int: + """Append events idempotently, as an Iceberg MERGE would do in production.""" + if not events: + return 0 + + written = 0 + for event in events: + exists = conn.execute( + "SELECT COUNT(*) FROM silver_ticketing_cdc_events WHERE event_id = ?", + [event.event_id], + ).fetchone()[0] + if exists: + continue + + conn.execute( + """ + INSERT INTO silver_ticketing_cdc_events + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + [ + event.event_id, + event.table, + event.primary_key, + event.op, + _json(event.before), + _json(event.after), + event.schema_id, + event.source_lsn, + event.tx_id, + event.topic, + event.kafka_partition, + event.kafka_offset, + event.source_event_at, + datetime.utcnow(), + ], + ) + written += 1 + + return written + + +def _json(value: dict[str, Any] | None) -> str | None: + if value is None: + return None + return json.dumps(value, sort_keys=True, default=str) diff --git a/submission/ticket-reservations/pipeline/schema_contracts.py b/submission/ticket-reservations/pipeline/schema_contracts.py new file mode 100644 index 0000000..3e2bd27 --- /dev/null +++ b/submission/ticket-reservations/pipeline/schema_contracts.py @@ -0,0 +1,52 @@ +"""Schema compatibility checks used before CDC events are written downstream.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from source.models import SCHEMA_CONTRACT + + +class IncompatibleSchemaChange(Exception): + """Raised when a source schema breaks the downstream data contract.""" + + +@dataclass(frozen=True) +class ContractViolation: + table: str + message: str + + +def validate_event_schema(table: str, columns: dict[str, str]) -> list[ContractViolation]: + if table not in SCHEMA_CONTRACT: + return [ContractViolation(table, "table is not registered in the schema contract")] + + expected = SCHEMA_CONTRACT[table]["columns"] # type: ignore[index] + violations: list[ContractViolation] = [] + + for name, expected_type in expected.items(): # type: ignore[union-attr] + if name not in columns: + violations.append(ContractViolation(table, f"{name}: missing column")) + continue + actual_type = _normalize(columns[name]) + if actual_type != _normalize(expected_type): + violations.append( + ContractViolation( + table, + f"{name}: expected {expected_type}, received {columns[name]}", + ) + ) + + return violations + + +def assert_compatible(table: str, columns: dict[str, str]) -> None: + violations = validate_event_schema(table, columns) + if violations: + details = "; ".join(f"{v.table}: {v.message}" for v in violations) + raise IncompatibleSchemaChange(details) + + +def _normalize(type_name: object) -> str: + return str(type_name).upper().replace(" ", "") + diff --git a/submission/ticket-reservations/pipeline/warehouse.py b/submission/ticket-reservations/pipeline/warehouse.py new file mode 100644 index 0000000..ca3e9cb --- /dev/null +++ b/submission/ticket-reservations/pipeline/warehouse.py @@ -0,0 +1,151 @@ +"""Snowflake-like current and history warehouse models.""" + +from __future__ import annotations + +import json + +import duckdb + +from pipeline.cdc import CDCEvent + + +def create_warehouse_tables(conn: duckdb.DuckDBPyConnection) -> None: + conn.execute(""" + CREATE TABLE IF NOT EXISTS wh_current_records ( + source_table VARCHAR NOT NULL, + primary_key VARCHAR NOT NULL, + row_json VARCHAR NOT NULL, + is_deleted BOOLEAN NOT NULL, + source_lsn BIGINT NOT NULL, + source_event_at TIMESTAMP NOT NULL, + warehouse_updated_at TIMESTAMP NOT NULL, + PRIMARY KEY (source_table, primary_key) + ) + """) + + conn.execute(""" + CREATE TABLE IF NOT EXISTS wh_history_records ( + event_id VARCHAR PRIMARY KEY, + source_table VARCHAR NOT NULL, + primary_key VARCHAR NOT NULL, + row_json VARCHAR NOT NULL, + valid_from TIMESTAMP NOT NULL, + valid_to TIMESTAMP, + is_current BOOLEAN NOT NULL, + is_deleted BOOLEAN NOT NULL, + source_lsn BIGINT NOT NULL + ) + """) + + +def apply_events(conn: duckdb.DuckDBPyConnection, events: list[CDCEvent]) -> None: + for event in sorted(events, key=lambda item: (item.source_lsn, item.kafka_offset)): + _upsert_current(conn, event) + _append_history(conn, event) + + +def point_in_time_state( + conn: duckdb.DuckDBPyConnection, table: str, as_of: str +) -> list[tuple[str, str]]: + return conn.execute( + """ + SELECT primary_key, row_json + FROM wh_history_records + WHERE source_table = ? + AND valid_from <= CAST(? AS TIMESTAMP) + AND (valid_to IS NULL OR valid_to > CAST(? AS TIMESTAMP)) + AND is_deleted = false + ORDER BY primary_key + """, + [table, as_of, as_of], + ).fetchall() + + +def _upsert_current(conn: duckdb.DuckDBPyConnection, event: CDCEvent) -> None: + current = conn.execute( + """ + SELECT source_lsn + FROM wh_current_records + WHERE source_table = ? AND primary_key = ? + """, + [event.table, event.primary_key], + ).fetchone() + + if current and current[0] >= event.source_lsn: + return + + row_json = json.dumps(event.current_image, sort_keys=True, default=str) + is_deleted = event.op == "d" + + if current: + conn.execute( + """ + UPDATE wh_current_records + SET row_json = ?, + is_deleted = ?, + source_lsn = ?, + source_event_at = ?, + warehouse_updated_at = CURRENT_TIMESTAMP + WHERE source_table = ? AND primary_key = ? + """, + [ + row_json, + is_deleted, + event.source_lsn, + event.source_event_at, + event.table, + event.primary_key, + ], + ) + else: + conn.execute( + """ + INSERT INTO wh_current_records + VALUES (?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) + """, + [ + event.table, + event.primary_key, + row_json, + is_deleted, + event.source_lsn, + event.source_event_at, + ], + ) + + +def _append_history(conn: duckdb.DuckDBPyConnection, event: CDCEvent) -> None: + exists = conn.execute( + "SELECT COUNT(*) FROM wh_history_records WHERE event_id = ?", + [event.event_id], + ).fetchone()[0] + if exists: + return + + conn.execute( + """ + UPDATE wh_history_records + SET valid_to = ?, is_current = false + WHERE source_table = ? + AND primary_key = ? + AND is_current = true + AND source_lsn < ? + """, + [event.source_event_at, event.table, event.primary_key, event.source_lsn], + ) + + conn.execute( + """ + INSERT INTO wh_history_records + VALUES (?, ?, ?, ?, ?, NULL, true, ?, ?) + """, + [ + event.event_id, + event.table, + event.primary_key, + json.dumps(event.current_image, sort_keys=True, default=str), + event.source_event_at, + event.op == "d", + event.source_lsn, + ], + ) diff --git a/submission/ticket-reservations/requirements.txt b/submission/ticket-reservations/requirements.txt new file mode 100644 index 0000000..7547b71 --- /dev/null +++ b/submission/ticket-reservations/requirements.txt @@ -0,0 +1,2 @@ +duckdb>=1.0.0 +pytest>=8.0.0 diff --git a/submission/ticket-reservations/scripts/check_schema_contracts.py b/submission/ticket-reservations/scripts/check_schema_contracts.py new file mode 100644 index 0000000..8901d97 --- /dev/null +++ b/submission/ticket-reservations/scripts/check_schema_contracts.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import duckdb + +from pipeline.schema_contracts import validate_event_schema +from source.models import SCHEMA_CONTRACT, create_source_tables + + +def main() -> int: + conn = duckdb.connect(":memory:") + create_source_tables(conn) + + failures: list[str] = [] + for table in SCHEMA_CONTRACT: + columns = { + row[0]: row[1] + for row in conn.execute(f"DESCRIBE {table}").fetchall() + } + failures.extend( + f"{violation.table}: {violation.message}" + for violation in validate_event_schema(table, columns) + ) + + if failures: + for failure in failures: + print(failure) + return 1 + + print("Schema contracts passed") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) + diff --git a/submission/ticket-reservations/scripts/run_data_quality_checks.py b/submission/ticket-reservations/scripts/run_data_quality_checks.py new file mode 100644 index 0000000..36a37fb --- /dev/null +++ b/submission/ticket-reservations/scripts/run_data_quality_checks.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import duckdb + + +def create_quality_demo(conn: duckdb.DuckDBPyConnection) -> None: + conn.execute(""" + CREATE TABLE reservations_current ( + reservation_id VARCHAR, + status VARCHAR, + total_amount DECIMAL(12, 2), + is_deleted BOOLEAN + ) + """) + conn.execute(""" + CREATE TABLE reservation_items_current ( + reservation_id VARCHAR, + unit_price DECIMAL(12, 2), + service_fee DECIMAL(12, 2), + is_deleted BOOLEAN + ) + """) + + +def reservation_total_mismatches(conn: duckdb.DuckDBPyConnection) -> list[tuple]: + return conn.execute(""" + SELECT r.reservation_id + FROM reservations_current r + JOIN reservation_items_current i + ON r.reservation_id = i.reservation_id + WHERE r.is_deleted = false + AND i.is_deleted = false + GROUP BY r.reservation_id, r.total_amount + HAVING r.total_amount <> SUM(i.unit_price + i.service_fee) + """).fetchall() + + +def main() -> int: + conn = duckdb.connect(":memory:") + create_quality_demo(conn) + failures = reservation_total_mismatches(conn) + if failures: + print(f"Reservation total mismatches: {failures}") + return 1 + print("Data quality checks passed") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) + diff --git a/submission/ticket-reservations/scripts/validate_catalog.py b/submission/ticket-reservations/scripts/validate_catalog.py new file mode 100644 index 0000000..9a2778a --- /dev/null +++ b/submission/ticket-reservations/scripts/validate_catalog.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import json +from pathlib import Path + + +REQUIRED_DATASETS = { + "raw_contract.kafka_cdc_event_shape", + "silver.ticketing_cdc_events", + "silver.reservations_current", + "silver.reservation_items_current", + "silver.payments_current", + "silver.reservations_history", + "WH_TICKETING_CURRENT.RESERVATIONS_CURRENT", + "WH_TICKETING_CURRENT.RESERVATION_ITEMS_CURRENT", + "WH_TICKETING_CURRENT.PAYMENTS_CURRENT", + "WH_TICKETING_HISTORY.RESERVATIONS_HISTORY", + "ops.schema_failures", +} + + +def main() -> int: + path = Path(__file__).resolve().parents[1] / "catalog" / "catalog.json" + catalog = json.loads(path.read_text()) + names = {dataset["name"] for dataset in catalog["datasets"]} + missing = sorted(REQUIRED_DATASETS - names) + + if missing: + print(f"Missing catalog datasets: {missing}") + return 1 + + print("Catalog metadata passed") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/submission/ticket-reservations/source/__init__.py b/submission/ticket-reservations/source/__init__.py new file mode 100644 index 0000000..57f55b6 --- /dev/null +++ b/submission/ticket-reservations/source/__init__.py @@ -0,0 +1,2 @@ +"""Ticketing source model package.""" + diff --git a/submission/ticket-reservations/source/models.py b/submission/ticket-reservations/source/models.py new file mode 100644 index 0000000..5b4b756 --- /dev/null +++ b/submission/ticket-reservations/source/models.py @@ -0,0 +1,242 @@ +""" +Source schema for a ticketing and reservations system. + +Strong entities: venues, events, customers, reservations. +Weak entities: seats, reservation_items, payments, tickets. + +The local tests use DuckDB as a lightweight relational stand-in. Production DDL +for PostgreSQL is in sql/source/postgres_schema.sql. +""" + +from __future__ import annotations + +import duckdb + + +SCHEMA_CONTRACT: dict[str, dict[str, object]] = { + "venues": { + "pk": ["venue_id"], + "columns": { + "venue_id": "VARCHAR", + "name": "VARCHAR", + "city": "VARCHAR", + "timezone": "VARCHAR", + "created_at": "TIMESTAMP", + }, + }, + "events": { + "pk": ["event_id"], + "columns": { + "event_id": "VARCHAR", + "venue_id": "VARCHAR", + "name": "VARCHAR", + "starts_at": "TIMESTAMP", + "ends_at": "TIMESTAMP", + "status": "VARCHAR", + "created_at": "TIMESTAMP", + "updated_at": "TIMESTAMP", + }, + "enum_domains": {"status": ["draft", "scheduled", "onsale", "closed", "cancelled"]}, + }, + "customers": { + "pk": ["customer_id"], + "columns": { + "customer_id": "VARCHAR", + "email": "VARCHAR", + "full_name": "VARCHAR", + "phone": "VARCHAR", + "created_at": "TIMESTAMP", + }, + }, + "seats": { + "pk": ["seat_id"], + "columns": { + "seat_id": "VARCHAR", + "venue_id": "VARCHAR", + "section": "VARCHAR", + "row_label": "VARCHAR", + "seat_number": "VARCHAR", + "status": "VARCHAR", + "updated_at": "TIMESTAMP", + }, + "enum_domains": {"status": ["available", "held", "booked", "blocked"]}, + }, + "reservations": { + "pk": ["reservation_id"], + "columns": { + "reservation_id": "VARCHAR", + "customer_id": "VARCHAR", + "event_id": "VARCHAR", + "status": "VARCHAR", + "hold_expires_at": "TIMESTAMP", + "confirmed_at": "TIMESTAMP", + "total_amount": "DECIMAL(12,2)", + "currency": "VARCHAR", + "created_at": "TIMESTAMP", + "updated_at": "TIMESTAMP", + }, + "enum_domains": {"status": ["held", "confirmed", "expired", "cancelled"]}, + }, + "reservation_items": { + "pk": ["reservation_item_id"], + "columns": { + "reservation_item_id": "VARCHAR", + "reservation_id": "VARCHAR", + "seat_id": "VARCHAR", + "unit_price": "DECIMAL(12,2)", + "service_fee": "DECIMAL(12,2)", + "currency": "VARCHAR", + "created_at": "TIMESTAMP", + }, + }, + "payments": { + "pk": ["payment_id"], + "columns": { + "payment_id": "VARCHAR", + "reservation_id": "VARCHAR", + "provider": "VARCHAR", + "provider_reference": "VARCHAR", + "status": "VARCHAR", + "amount": "DECIMAL(12,2)", + "currency": "VARCHAR", + "authorized_at": "TIMESTAMP", + "captured_at": "TIMESTAMP", + "refunded_at": "TIMESTAMP", + "created_at": "TIMESTAMP", + }, + "enum_domains": {"status": ["authorized", "captured", "failed", "refunded"]}, + }, + "tickets": { + "pk": ["ticket_id"], + "columns": { + "ticket_id": "VARCHAR", + "reservation_item_id": "VARCHAR", + "event_id": "VARCHAR", + "customer_id": "VARCHAR", + "ticket_code": "VARCHAR", + "status": "VARCHAR", + "issued_at": "TIMESTAMP", + "used_at": "TIMESTAMP", + }, + "enum_domains": {"status": ["issued", "voided", "used"]}, + }, +} + + +def contract_columns(table: str) -> dict[str, str]: + return SCHEMA_CONTRACT[table]["columns"] # type: ignore[return-value] + + +def create_source_tables(conn: duckdb.DuckDBPyConnection) -> None: + conn.execute(""" + CREATE TABLE IF NOT EXISTS venues ( + venue_id VARCHAR PRIMARY KEY, + name VARCHAR NOT NULL, + city VARCHAR NOT NULL, + timezone VARCHAR NOT NULL, + created_at TIMESTAMP NOT NULL + ) + """) + + conn.execute(""" + CREATE TABLE IF NOT EXISTS events ( + event_id VARCHAR PRIMARY KEY, + venue_id VARCHAR NOT NULL REFERENCES venues(venue_id), + name VARCHAR NOT NULL, + starts_at TIMESTAMP NOT NULL, + ends_at TIMESTAMP NOT NULL, + status VARCHAR NOT NULL + CHECK (status IN ('draft', 'scheduled', 'onsale', 'closed', 'cancelled')), + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL, + CHECK (ends_at > starts_at) + ) + """) + + conn.execute(""" + CREATE TABLE IF NOT EXISTS customers ( + customer_id VARCHAR PRIMARY KEY, + email VARCHAR NOT NULL UNIQUE, + full_name VARCHAR NOT NULL, + phone VARCHAR, + created_at TIMESTAMP NOT NULL + ) + """) + + conn.execute(""" + CREATE TABLE IF NOT EXISTS seats ( + seat_id VARCHAR PRIMARY KEY, + venue_id VARCHAR NOT NULL REFERENCES venues(venue_id), + section VARCHAR NOT NULL, + row_label VARCHAR NOT NULL, + seat_number VARCHAR NOT NULL, + status VARCHAR NOT NULL + CHECK (status IN ('available', 'held', 'booked', 'blocked')), + updated_at TIMESTAMP NOT NULL, + UNIQUE (venue_id, section, row_label, seat_number) + ) + """) + + conn.execute(""" + CREATE TABLE IF NOT EXISTS reservations ( + reservation_id VARCHAR PRIMARY KEY, + customer_id VARCHAR NOT NULL REFERENCES customers(customer_id), + event_id VARCHAR NOT NULL REFERENCES events(event_id), + status VARCHAR NOT NULL + CHECK (status IN ('held', 'confirmed', 'expired', 'cancelled')), + hold_expires_at TIMESTAMP, + confirmed_at TIMESTAMP, + total_amount DECIMAL(12, 2) NOT NULL CHECK (total_amount >= 0), + currency VARCHAR NOT NULL, + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL, + CHECK ((status = 'held' AND hold_expires_at IS NOT NULL) + OR status IN ('confirmed', 'expired', 'cancelled')) + ) + """) + + conn.execute(""" + CREATE TABLE IF NOT EXISTS reservation_items ( + reservation_item_id VARCHAR PRIMARY KEY, + reservation_id VARCHAR NOT NULL REFERENCES reservations(reservation_id), + seat_id VARCHAR NOT NULL REFERENCES seats(seat_id), + unit_price DECIMAL(12, 2) NOT NULL CHECK (unit_price >= 0), + service_fee DECIMAL(12, 2) NOT NULL DEFAULT 0 CHECK (service_fee >= 0), + currency VARCHAR NOT NULL, + created_at TIMESTAMP NOT NULL, + UNIQUE (reservation_id, seat_id) + ) + """) + + conn.execute(""" + CREATE TABLE IF NOT EXISTS payments ( + payment_id VARCHAR PRIMARY KEY, + reservation_id VARCHAR NOT NULL REFERENCES reservations(reservation_id), + provider VARCHAR NOT NULL, + provider_reference VARCHAR NOT NULL, + status VARCHAR NOT NULL + CHECK (status IN ('authorized', 'captured', 'failed', 'refunded')), + amount DECIMAL(12, 2) NOT NULL CHECK (amount >= 0), + currency VARCHAR NOT NULL, + authorized_at TIMESTAMP, + captured_at TIMESTAMP, + refunded_at TIMESTAMP, + created_at TIMESTAMP NOT NULL, + UNIQUE (provider, provider_reference), + CHECK (captured_at IS NULL OR authorized_at IS NULL OR captured_at >= authorized_at) + ) + """) + + conn.execute(""" + CREATE TABLE IF NOT EXISTS tickets ( + ticket_id VARCHAR PRIMARY KEY, + reservation_item_id VARCHAR NOT NULL UNIQUE REFERENCES reservation_items(reservation_item_id), + event_id VARCHAR NOT NULL REFERENCES events(event_id), + customer_id VARCHAR NOT NULL REFERENCES customers(customer_id), + ticket_code VARCHAR NOT NULL UNIQUE, + status VARCHAR NOT NULL CHECK (status IN ('issued', 'voided', 'used')), + issued_at TIMESTAMP NOT NULL, + used_at TIMESTAMP, + CHECK (used_at IS NULL OR used_at >= issued_at) + ) + """) diff --git a/submission/ticket-reservations/sql/snowflake/merge_reservations.sql b/submission/ticket-reservations/sql/snowflake/merge_reservations.sql new file mode 100644 index 0000000..a3f7ddb --- /dev/null +++ b/submission/ticket-reservations/sql/snowflake/merge_reservations.sql @@ -0,0 +1,28 @@ +MERGE INTO WH_TICKETING_CURRENT.RESERVATIONS_CURRENT AS target +USING STAGE_RESERVATIONS_CDC AS source +ON target.reservation_id = source.reservation_id +WHEN MATCHED AND source.source_lsn > target.source_lsn THEN UPDATE SET + customer_id = source.customer_id, + event_id = source.event_id, + status = source.status, + hold_expires_at = source.hold_expires_at, + confirmed_at = source.confirmed_at, + total_amount = source.total_amount, + currency = source.currency, + created_at = source.created_at, + updated_at = source.updated_at, + is_deleted = source.is_deleted, + source_lsn = source.source_lsn, + source_event_at = source.source_event_at, + warehouse_updated_at = CURRENT_TIMESTAMP() +WHEN NOT MATCHED THEN INSERT ( + reservation_id, customer_id, event_id, status, hold_expires_at, confirmed_at, + total_amount, currency, created_at, updated_at, is_deleted, source_lsn, + source_event_at, warehouse_updated_at +) VALUES ( + source.reservation_id, source.customer_id, source.event_id, source.status, + source.hold_expires_at, source.confirmed_at, source.total_amount, + source.currency, source.created_at, source.updated_at, source.is_deleted, + source.source_lsn, source.source_event_at, CURRENT_TIMESTAMP() +); + diff --git a/submission/ticket-reservations/sql/snowflake/quality_checks.sql b/submission/ticket-reservations/sql/snowflake/quality_checks.sql new file mode 100644 index 0000000..ab63ec0 --- /dev/null +++ b/submission/ticket-reservations/sql/snowflake/quality_checks.sql @@ -0,0 +1,34 @@ +CREATE OR REPLACE VIEW QA.RESERVATION_TOTAL_MISMATCHES AS +SELECT + r.reservation_id, + r.total_amount, + SUM(i.unit_price + i.service_fee) AS item_total +FROM WH_TICKETING_CURRENT.RESERVATIONS_CURRENT r +JOIN WH_TICKETING_CURRENT.RESERVATION_ITEMS_CURRENT i + ON r.reservation_id = i.reservation_id +WHERE r.is_deleted = false + AND i.is_deleted = false +GROUP BY r.reservation_id, r.total_amount +HAVING r.total_amount <> SUM(i.unit_price + i.service_fee); + +CREATE OR REPLACE VIEW QA.CONFIRMED_RESERVATIONS_WITHOUT_CAPTURED_PAYMENT AS +SELECT r.reservation_id +FROM WH_TICKETING_CURRENT.RESERVATIONS_CURRENT r +LEFT JOIN WH_TICKETING_CURRENT.PAYMENTS_CURRENT p + ON r.reservation_id = p.reservation_id + AND p.status = 'captured' + AND p.is_deleted = false +WHERE r.status = 'confirmed' + AND r.is_deleted = false + AND p.payment_id IS NULL; + +CREATE OR REPLACE VIEW QA.SCD2_OVERLAPS AS +SELECT reservation_id, COUNT(*) AS overlapping_rows +FROM WH_TICKETING_HISTORY.RESERVATIONS_HISTORY h1 +JOIN WH_TICKETING_HISTORY.RESERVATIONS_HISTORY h2 + ON h1.reservation_id = h2.reservation_id + AND h1.event_id_cdc <> h2.event_id_cdc + AND h1.valid_from < COALESCE(h2.valid_to, '9999-12-31'::TIMESTAMP_TZ) + AND h2.valid_from < COALESCE(h1.valid_to, '9999-12-31'::TIMESTAMP_TZ) +GROUP BY reservation_id +HAVING COUNT(*) > 0; diff --git a/submission/ticket-reservations/sql/snowflake/refresh_from_iceberg.sql b/submission/ticket-reservations/sql/snowflake/refresh_from_iceberg.sql new file mode 100644 index 0000000..8467fb0 --- /dev/null +++ b/submission/ticket-reservations/sql/snowflake/refresh_from_iceberg.sql @@ -0,0 +1,71 @@ +-- Snowflake refresh after each successful 5-minute Spark micro-batch. +-- In production these SELECTs point at Snowflake-managed Iceberg tables or +-- external Iceberg tables registered against the S3 lakehouse catalog. + +MERGE INTO WH_TICKETING_CURRENT.RESERVATIONS_CURRENT AS t +USING EXT_SILVER.RESERVATIONS_CURRENT AS s +ON t.reservation_id = s.reservation_id +WHEN MATCHED AND s.source_lsn > t.source_lsn THEN UPDATE SET + customer_id = s.customer_id, + event_id = s.event_id, + status = s.status, + hold_expires_at = s.hold_expires_at, + confirmed_at = s.confirmed_at, + total_amount = s.total_amount, + currency = s.currency, + created_at = s.created_at, + updated_at = s.updated_at, + is_deleted = s.is_deleted, + source_lsn = s.source_lsn, + source_event_at = s.source_event_at, + warehouse_updated_at = CURRENT_TIMESTAMP() +WHEN NOT MATCHED THEN INSERT VALUES ( + s.reservation_id, s.customer_id, s.event_id, s.status, s.hold_expires_at, + s.confirmed_at, s.total_amount, s.currency, s.created_at, s.updated_at, + s.is_deleted, s.source_lsn, s.source_event_at, CURRENT_TIMESTAMP() +); + +MERGE INTO WH_TICKETING_CURRENT.RESERVATION_ITEMS_CURRENT AS t +USING EXT_SILVER.RESERVATION_ITEMS_CURRENT AS s +ON t.reservation_item_id = s.reservation_item_id +WHEN MATCHED AND s.source_lsn > t.source_lsn THEN UPDATE SET + reservation_id = s.reservation_id, + seat_id = s.seat_id, + unit_price = s.unit_price, + service_fee = s.service_fee, + currency = s.currency, + created_at = s.created_at, + is_deleted = s.is_deleted, + source_lsn = s.source_lsn, + source_event_at = s.source_event_at, + warehouse_updated_at = CURRENT_TIMESTAMP() +WHEN NOT MATCHED THEN INSERT VALUES ( + s.reservation_item_id, s.reservation_id, s.seat_id, s.unit_price, + s.service_fee, s.currency, s.created_at, s.is_deleted, s.source_lsn, + s.source_event_at, CURRENT_TIMESTAMP() +); + +MERGE INTO WH_TICKETING_CURRENT.PAYMENTS_CURRENT AS t +USING EXT_SILVER.PAYMENTS_CURRENT AS s +ON t.payment_id = s.payment_id +WHEN MATCHED AND s.source_lsn > t.source_lsn THEN UPDATE SET + reservation_id = s.reservation_id, + provider = s.provider, + provider_reference = s.provider_reference, + status = s.status, + amount = s.amount, + currency = s.currency, + authorized_at = s.authorized_at, + captured_at = s.captured_at, + refunded_at = s.refunded_at, + created_at = s.created_at, + is_deleted = s.is_deleted, + source_lsn = s.source_lsn, + source_event_at = s.source_event_at, + warehouse_updated_at = CURRENT_TIMESTAMP() +WHEN NOT MATCHED THEN INSERT VALUES ( + s.payment_id, s.reservation_id, s.provider, s.provider_reference, s.status, + s.amount, s.currency, s.authorized_at, s.captured_at, s.refunded_at, + s.created_at, s.is_deleted, s.source_lsn, s.source_event_at, + CURRENT_TIMESTAMP() +); diff --git a/submission/ticket-reservations/sql/snowflake/warehouse_tables.sql b/submission/ticket-reservations/sql/snowflake/warehouse_tables.sql new file mode 100644 index 0000000..d61101a --- /dev/null +++ b/submission/ticket-reservations/sql/snowflake/warehouse_tables.sql @@ -0,0 +1,90 @@ +CREATE SCHEMA IF NOT EXISTS WH_TICKETING_CURRENT; +CREATE SCHEMA IF NOT EXISTS WH_TICKETING_HISTORY; +CREATE SCHEMA IF NOT EXISTS QA; +CREATE SCHEMA IF NOT EXISTS OPS; + +CREATE TABLE IF NOT EXISTS WH_TICKETING_CURRENT.RESERVATIONS_CURRENT ( + reservation_id STRING PRIMARY KEY, + customer_id STRING, + event_id STRING, + status STRING, + hold_expires_at TIMESTAMP_TZ, + confirmed_at TIMESTAMP_TZ, + total_amount NUMBER(12,2), + currency STRING, + created_at TIMESTAMP_TZ, + updated_at TIMESTAMP_TZ, + is_deleted BOOLEAN, + source_lsn NUMBER, + source_event_at TIMESTAMP_TZ, + warehouse_updated_at TIMESTAMP_TZ +); + +CREATE TABLE IF NOT EXISTS WH_TICKETING_CURRENT.RESERVATION_ITEMS_CURRENT ( + reservation_item_id STRING PRIMARY KEY, + reservation_id STRING, + seat_id STRING, + unit_price NUMBER(12,2), + service_fee NUMBER(12,2), + currency STRING, + created_at TIMESTAMP_TZ, + is_deleted BOOLEAN, + source_lsn NUMBER, + source_event_at TIMESTAMP_TZ, + warehouse_updated_at TIMESTAMP_TZ +); + +CREATE TABLE IF NOT EXISTS WH_TICKETING_CURRENT.PAYMENTS_CURRENT ( + payment_id STRING PRIMARY KEY, + reservation_id STRING, + provider STRING, + provider_reference STRING, + status STRING, + amount NUMBER(12,2), + currency STRING, + authorized_at TIMESTAMP_TZ, + captured_at TIMESTAMP_TZ, + refunded_at TIMESTAMP_TZ, + created_at TIMESTAMP_TZ, + is_deleted BOOLEAN, + source_lsn NUMBER, + source_event_at TIMESTAMP_TZ, + warehouse_updated_at TIMESTAMP_TZ +); + +CREATE TABLE IF NOT EXISTS WH_TICKETING_HISTORY.RESERVATIONS_HISTORY ( + reservation_id STRING, + customer_id STRING, + event_id STRING, + status STRING, + hold_expires_at TIMESTAMP_TZ, + confirmed_at TIMESTAMP_TZ, + total_amount NUMBER(12,2), + currency STRING, + created_at TIMESTAMP_TZ, + updated_at TIMESTAMP_TZ, + valid_from TIMESTAMP_TZ, + valid_to TIMESTAMP_TZ, + is_current BOOLEAN, + is_deleted BOOLEAN, + source_lsn NUMBER, + event_id_cdc STRING PRIMARY KEY +); + +CREATE TABLE IF NOT EXISTS OPS.SCHEMA_FAILURES ( + failure_id STRING PRIMARY KEY, + source_table STRING, + expected_schema VARIANT, + received_schema VARIANT, + kafka_topic STRING, + kafka_partition NUMBER, + kafka_offset NUMBER, + failed_at TIMESTAMP_TZ, + message STRING +); + +COMMENT ON TABLE WH_TICKETING_CURRENT.RESERVATIONS_CURRENT IS + 'Latest reservation state from PostgreSQL CDC. Deletes are soft-deleted by is_deleted.'; + +COMMENT ON TABLE WH_TICKETING_HISTORY.RESERVATIONS_HISTORY IS + 'SCD2 reservation history rebuilt from immutable Iceberg CDC events.'; diff --git a/submission/ticket-reservations/sql/source/debezium_connector.json b/submission/ticket-reservations/sql/source/debezium_connector.json new file mode 100644 index 0000000..5ae7dd6 --- /dev/null +++ b/submission/ticket-reservations/sql/source/debezium_connector.json @@ -0,0 +1,23 @@ +{ + "name": "ticketing-postgres-cdc", + "config": { + "connector.class": "io.debezium.connector.postgresql.PostgresConnector", + "database.hostname": "postgres", + "database.port": "5432", + "database.user": "debezium", + "database.password": "${file:/secrets/postgres.properties:password}", + "database.dbname": "ticketing", + "topic.prefix": "ticketing", + "plugin.name": "pgoutput", + "slot.name": "ticketing_cdc_slot", + "publication.name": "ticketing_publication", + "table.include.list": "public.venues,public.events,public.customers,public.seats,public.reservations,public.reservation_items,public.payments,public.tickets", + "tombstones.on.delete": "false", + "include.schema.changes": "true", + "key.converter": "io.confluent.connect.avro.AvroConverter", + "value.converter": "io.confluent.connect.avro.AvroConverter", + "key.converter.schema.registry.url": "http://schema-registry:8081", + "value.converter.schema.registry.url": "http://schema-registry:8081" + } +} + diff --git a/submission/ticket-reservations/sql/source/postgres_schema.sql b/submission/ticket-reservations/sql/source/postgres_schema.sql new file mode 100644 index 0000000..dd99666 --- /dev/null +++ b/submission/ticket-reservations/sql/source/postgres_schema.sql @@ -0,0 +1,109 @@ +-- PostgreSQL source schema for the ticketing reservation system. + +CREATE TYPE event_status AS ENUM ('draft', 'scheduled', 'onsale', 'closed', 'cancelled'); +CREATE TYPE seat_status AS ENUM ('available', 'held', 'booked', 'blocked'); +CREATE TYPE reservation_status AS ENUM ('held', 'confirmed', 'expired', 'cancelled'); +CREATE TYPE payment_status AS ENUM ('authorized', 'captured', 'failed', 'refunded'); +CREATE TYPE ticket_status AS ENUM ('issued', 'voided', 'used'); + +CREATE TABLE venues ( + venue_id UUID PRIMARY KEY, + name TEXT NOT NULL, + city TEXT NOT NULL, + timezone TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE events ( + event_id UUID PRIMARY KEY, + venue_id UUID NOT NULL REFERENCES venues(venue_id), + name TEXT NOT NULL, + starts_at TIMESTAMPTZ NOT NULL, + ends_at TIMESTAMPTZ NOT NULL, + status event_status NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CHECK (ends_at > starts_at) +); + +CREATE TABLE customers ( + customer_id UUID PRIMARY KEY, + email TEXT NOT NULL UNIQUE, + full_name TEXT NOT NULL, + phone TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE seats ( + seat_id UUID PRIMARY KEY, + venue_id UUID NOT NULL REFERENCES venues(venue_id), + section TEXT NOT NULL, + row_label TEXT NOT NULL, + seat_number TEXT NOT NULL, + status seat_status NOT NULL DEFAULT 'available', + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (venue_id, section, row_label, seat_number) +); + +CREATE TABLE reservations ( + reservation_id UUID PRIMARY KEY, + customer_id UUID NOT NULL REFERENCES customers(customer_id), + event_id UUID NOT NULL REFERENCES events(event_id), + status reservation_status NOT NULL, + hold_expires_at TIMESTAMPTZ, + confirmed_at TIMESTAMPTZ, + total_amount NUMERIC(12,2) NOT NULL CHECK (total_amount >= 0), + currency CHAR(3) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CHECK ((status = 'held' AND hold_expires_at IS NOT NULL) + OR status IN ('confirmed', 'expired', 'cancelled')) +); + +CREATE TABLE reservation_items ( + reservation_item_id UUID PRIMARY KEY, + reservation_id UUID NOT NULL REFERENCES reservations(reservation_id), + seat_id UUID NOT NULL REFERENCES seats(seat_id), + unit_price NUMERIC(12,2) NOT NULL CHECK (unit_price >= 0), + service_fee NUMERIC(12,2) NOT NULL DEFAULT 0 CHECK (service_fee >= 0), + currency CHAR(3) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (reservation_id, seat_id) +); + +CREATE TABLE payments ( + payment_id UUID PRIMARY KEY, + reservation_id UUID NOT NULL REFERENCES reservations(reservation_id), + provider TEXT NOT NULL, + provider_reference TEXT NOT NULL, + status payment_status NOT NULL, + amount NUMERIC(12,2) NOT NULL CHECK (amount >= 0), + currency CHAR(3) NOT NULL, + authorized_at TIMESTAMPTZ, + captured_at TIMESTAMPTZ, + refunded_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (provider, provider_reference), + CHECK (captured_at IS NULL OR authorized_at IS NULL OR captured_at >= authorized_at) +); + +CREATE TABLE tickets ( + ticket_id UUID PRIMARY KEY, + reservation_item_id UUID NOT NULL UNIQUE REFERENCES reservation_items(reservation_item_id), + event_id UUID NOT NULL REFERENCES events(event_id), + customer_id UUID NOT NULL REFERENCES customers(customer_id), + ticket_code TEXT NOT NULL UNIQUE, + status ticket_status NOT NULL, + issued_at TIMESTAMPTZ NOT NULL, + used_at TIMESTAMPTZ, + CHECK (used_at IS NULL OR used_at >= issued_at) +); + +CREATE INDEX idx_events_venue_start ON events (venue_id, starts_at); +CREATE INDEX idx_seats_venue_status ON seats (venue_id, status); +CREATE INDEX idx_reservations_customer_created ON reservations (customer_id, created_at DESC); +CREATE INDEX idx_reservations_event_status ON reservations (event_id, status); +CREATE INDEX idx_reservation_items_reservation ON reservation_items (reservation_id); +CREATE INDEX idx_payments_reservation_status ON payments (reservation_id, status); +CREATE INDEX idx_tickets_event_status ON tickets (event_id, status); + diff --git a/submission/ticket-reservations/sql/spark/iceberg_tables.sql b/submission/ticket-reservations/sql/spark/iceberg_tables.sql new file mode 100644 index 0000000..155bf18 --- /dev/null +++ b/submission/ticket-reservations/sql/spark/iceberg_tables.sql @@ -0,0 +1,117 @@ +-- Iceberg lakehouse tables written by Spark Structured Streaming to S3. +-- Catalog examples: +-- spark.sql.catalog.ticketing=org.apache.iceberg.spark.SparkCatalog +-- spark.sql.catalog.ticketing.type=hive +-- spark.sql.catalog.ticketing.warehouse=s3://ticketing-lakehouse/warehouse + +CREATE DATABASE IF NOT EXISTS silver; +CREATE DATABASE IF NOT EXISTS ops; + +-- Silver: immutable CDC event history and typed current/history models. +CREATE TABLE IF NOT EXISTS silver.ticketing_cdc_events ( + event_id STRING, + source_system STRING, + source_database STRING, + source_schema STRING, + source_table STRING, + primary_key_json STRING, + op STRING, + before_json STRING, + after_json STRING, + schema_id STRING, + source_lsn BIGINT, + source_tx_id STRING, + kafka_topic STRING, + kafka_partition INT, + kafka_offset BIGINT, + source_event_at TIMESTAMP, + ingested_at TIMESTAMP +) +USING iceberg +PARTITIONED BY (days(source_event_at), source_table); + +CREATE TABLE IF NOT EXISTS silver.reservations_current ( + reservation_id STRING, + customer_id STRING, + event_id STRING, + status STRING, + hold_expires_at TIMESTAMP, + confirmed_at TIMESTAMP, + total_amount DECIMAL(12,2), + currency STRING, + created_at TIMESTAMP, + updated_at TIMESTAMP, + is_deleted BOOLEAN, + source_lsn BIGINT, + source_event_at TIMESTAMP +) +USING iceberg +PARTITIONED BY (bucket(16, reservation_id)); + +CREATE TABLE IF NOT EXISTS silver.reservation_items_current ( + reservation_item_id STRING, + reservation_id STRING, + seat_id STRING, + unit_price DECIMAL(12,2), + service_fee DECIMAL(12,2), + currency STRING, + created_at TIMESTAMP, + is_deleted BOOLEAN, + source_lsn BIGINT, + source_event_at TIMESTAMP +) +USING iceberg +PARTITIONED BY (bucket(16, reservation_id)); + +CREATE TABLE IF NOT EXISTS silver.payments_current ( + payment_id STRING, + reservation_id STRING, + provider STRING, + provider_reference STRING, + status STRING, + amount DECIMAL(12,2), + currency STRING, + authorized_at TIMESTAMP, + captured_at TIMESTAMP, + refunded_at TIMESTAMP, + created_at TIMESTAMP, + is_deleted BOOLEAN, + source_lsn BIGINT, + source_event_at TIMESTAMP +) +USING iceberg +PARTITIONED BY (bucket(16, reservation_id)); + +-- Silver history: SCD2 row versions used for restore and point-in-time queries. +CREATE TABLE IF NOT EXISTS silver.reservations_history ( + reservation_id STRING, + customer_id STRING, + event_id STRING, + status STRING, + hold_expires_at TIMESTAMP, + confirmed_at TIMESTAMP, + total_amount DECIMAL(12,2), + currency STRING, + valid_from TIMESTAMP, + valid_to TIMESTAMP, + is_current BOOLEAN, + is_deleted BOOLEAN, + source_lsn BIGINT, + cdc_event_id STRING +) +USING iceberg +PARTITIONED BY (days(valid_from), bucket(16, event_id)); + +CREATE TABLE IF NOT EXISTS ops.schema_failures ( + failure_id STRING, + source_table STRING, + expected_schema_json STRING, + received_schema_json STRING, + kafka_topic STRING, + kafka_partition INT, + kafka_offset BIGINT, + failed_at TIMESTAMP, + message STRING +) +USING iceberg +PARTITIONED BY (days(failed_at), source_table); diff --git a/submission/ticket-reservations/sql/spark/raw_s3_event_contract.sql b/submission/ticket-reservations/sql/spark/raw_s3_event_contract.sql new file mode 100644 index 0000000..ff4a373 --- /dev/null +++ b/submission/ticket-reservations/sql/spark/raw_s3_event_contract.sql @@ -0,0 +1,25 @@ +-- Raw Kafka CDC events are already landed in S3 by the time this pipeline starts. +-- Example path layout: +-- s3://ticketing-raw-cdc/events/source_table=reservations/ingest_date=2026-07-02/*.json +-- +-- Each file row is one Kafka CDC message. The pipeline reads these files as a +-- Spark Structured Streaming file source and writes durable Iceberg tables. + +CREATE DATABASE IF NOT EXISTS raw_contract; + +CREATE TABLE IF NOT EXISTS raw_contract.kafka_cdc_event_shape ( + event_id STRING, + source_table STRING, + primary_key_json STRING, + op STRING, + before_json STRING, + after_json STRING, + schema_id STRING, + source_lsn BIGINT, + source_tx_id STRING, + kafka_topic STRING, + kafka_partition INT, + kafka_offset BIGINT, + source_event_at TIMESTAMP, + landed_at TIMESTAMP +); diff --git a/submission/ticket-reservations/sql/spark/structured_streaming_pseudocode.py b/submission/ticket-reservations/sql/spark/structured_streaming_pseudocode.py new file mode 100644 index 0000000..ebb3acf --- /dev/null +++ b/submission/ticket-reservations/sql/spark/structured_streaming_pseudocode.py @@ -0,0 +1,102 @@ +"""Spark Structured Streaming job for raw Kafka CDC files in S3. + +The pipeline starts after Kafka events are landed as JSON files in S3: + +raw S3 CDC files -> Spark every 5 minutes -> Iceberg silver +-> Snowflake refresh. + +This is intentionally simple. The helper functions represent small SQL/model +steps that would live beside this job in production. +""" + +from pyspark.sql import DataFrame +from pyspark.sql import functions as F +from pyspark.sql.types import LongType, StringType, StructField, StructType, TimestampType + + +RAW_CDC_PATH = "s3://ticketing-raw-cdc/events/" +CHECKPOINT_PATH = "s3://ticketing-lakehouse/checkpoints/raw-s3-cdc-to-iceberg-5min" + +RAW_CDC_SCHEMA = StructType( + [ + StructField("event_id", StringType(), False), + StructField("source_table", StringType(), False), + StructField("primary_key_json", StringType(), False), + StructField("op", StringType(), False), + StructField("before_json", StringType(), True), + StructField("after_json", StringType(), True), + StructField("schema_id", StringType(), False), + StructField("source_lsn", LongType(), False), + StructField("source_tx_id", StringType(), False), + StructField("kafka_topic", StringType(), False), + StructField("kafka_partition", LongType(), False), + StructField("kafka_offset", LongType(), False), + StructField("source_event_at", TimestampType(), False), + StructField("landed_at", TimestampType(), False), + ] +) + + +def read_raw_s3_cdc_events() -> DataFrame: + return ( + spark.readStream.format("json") + .schema(RAW_CDC_SCHEMA) + .option("maxFilesPerTrigger", 200) + .option("latestFirst", "false") + .load(RAW_CDC_PATH) + ) + + +def normalize_raw_events(raw_events: DataFrame) -> DataFrame: + return raw_events.select( + "event_id", + F.lit("ticketing").alias("source_system"), + F.lit("ticketing").alias("source_database"), + F.lit("public").alias("source_schema"), + "source_table", + "primary_key_json", + "op", + "before_json", + "after_json", + "schema_id", + "source_lsn", + "source_tx_id", + "kafka_topic", + F.col("kafka_partition").cast("int").alias("kafka_partition"), + "kafka_offset", + "source_event_at", + F.current_timestamp().alias("ingested_at"), + ) + + +def process_microbatch(raw_events: DataFrame, batch_id: int) -> None: + silver_events = normalize_raw_events(raw_events) + + # Stop before any writes when an incompatible source change appears. + assert_compatible_or_raise(silver_events) + + # Silver keeps every insert/update/delete event, idempotently by event_id. + silver_events.createOrReplaceTempView("batch_cdc_events") + spark.sql(""" + MERGE INTO silver.ticketing_cdc_events t + USING batch_cdc_events s + ON t.event_id = s.event_id + WHEN NOT MATCHED THEN INSERT * + """) + + # Silver tables include the current state and history needed for restore. + merge_silver_current_tables(silver_events) + merge_silver_history_tables(silver_events) + + # Snowflake refresh reads the just-committed silver Iceberg snapshot. + call_snowflake_refresh(batch_id=batch_id) + + +query = ( + read_raw_s3_cdc_events() + .writeStream.foreachBatch(process_microbatch) + .option("checkpointLocation", CHECKPOINT_PATH) + .trigger(processingTime="5 minutes") + .queryName("ticketing_raw_s3_cdc_to_silver_5min") + .start() +) diff --git a/submission/ticket-reservations/tests/__pycache__/conftest.cpython-313-pytest-8.4.2.pyc b/submission/ticket-reservations/tests/__pycache__/conftest.cpython-313-pytest-8.4.2.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f4bb4c72072e5100e95b66b23540859e4761f2fb GIT binary patch literal 1481 zcmaJ=-)q}e6uy#WS(bmq%@W70H_yv5is@=MfoxF57^8z>Lq@9x3MmLhzG>9Rl6S6@ z)OpHau-7pt%VQqhx3a%vkPOV7^VE&uJvsOv*tznLWF?#6UVY~~=ja}N_nc8CqZ53- ze)84xy%-^HgfV^OF_1>TAb3d(Vn_{Amq02DEH`9`MQ^1ZXL6a;RY*jhSVOC8piQEZ zkc^^q(DAJ_QuQ>XBVW9csb?V@VYM+=&q1zCp6ZX5rv+TTjl?aoJ~3hbKVq~iG4DK+ zE6IV#s7O3zd7d9wf#Z8@YhV0UWJ+Of8`XRgG zT9~+)AA&vQ?aNXLK^?`CMgfACWNYSoG*NU$AXr9K*QGJzk*)aDw@IdKnQSGlWs}#j zn`DJlpQwv{s2izkBGS`{|MMA{ zs*W0GH@FO|T()u1Rj%w(x9R+idJcr919AL7%+_EIw?lPxZM}N$9*=Dr4?wg*Md#To zVwp~pr|gggev5)IG`f!x3#2k^-H`FQUEgh*^hMjjJkttzS}5y4i#8)Gz!rB+tL2AY zz%@Grpq|~qtBo2wiqXTiXnlb}!6IG}5Y>UW3_f?Y9Pkg|BV_Zqn1S078LFiCes8JR zTPXDA3;p7UCl9(GbRWK!Uw?U8{Ipkiw^u0l7H;jbarKd>!!IA1|d+>1Ti~Y+YTK!a6HD-U$i^s%(`i|J5eWL4^j~0Tm^anPC$8b zM)Aay(=l0un-+k$c@WpZ)iKF4ZKq9L$D>s-`q4D-`58AG{AfaWW<2fHmfxfZqL~v0z1?B$#3MnHze~r$^1z^ oJ|~}?lPc;gYd@5}D_zKODL;x4IX4t&n49rwNsagm8BuiLKZwqBJ^%m! literal 0 HcmV?d00001 diff --git a/submission/ticket-reservations/tests/conftest.py b/submission/ticket-reservations/tests/conftest.py new file mode 100644 index 0000000..b75c4e6 --- /dev/null +++ b/submission/ticket-reservations/tests/conftest.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from datetime import datetime + +import duckdb +import pytest + +from pipeline.lake import create_lake_tables +from pipeline.warehouse import create_warehouse_tables +from source.models import create_source_tables + + +@pytest.fixture +def conn(): + connection = duckdb.connect(":memory:") + create_source_tables(connection) + create_lake_tables(connection) + create_warehouse_tables(connection) + return connection + + +@pytest.fixture +def reservation_row(): + return { + "reservation_id": "r1", + "customer_id": "c1", + "event_id": "e1", + "status": "held", + "hold_expires_at": datetime(2026, 7, 1, 10, 15), + "confirmed_at": None, + "total_amount": "125.00", + "currency": "USD", + "created_at": datetime(2026, 7, 1, 10, 0), + "updated_at": datetime(2026, 7, 1, 10, 0), + } + diff --git a/submission/ticket-reservations/tests/test_catalog.py b/submission/ticket-reservations/tests/test_catalog.py new file mode 100644 index 0000000..9abd341 --- /dev/null +++ b/submission/ticket-reservations/tests/test_catalog.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +import json +from pathlib import Path + + +def test_catalog_contains_lake_and_warehouse_datasets(): + path = Path(__file__).resolve().parents[1] / "catalog" / "catalog.json" + catalog = json.loads(path.read_text()) + names = {dataset["name"] for dataset in catalog["datasets"]} + + assert "raw_contract.kafka_cdc_event_shape" in names + assert "silver.ticketing_cdc_events" in names + assert "silver.reservations_history" in names + assert "WH_TICKETING_CURRENT.RESERVATIONS_CURRENT" in names + assert "WH_TICKETING_CURRENT.RESERVATION_ITEMS_CURRENT" in names + assert "WH_TICKETING_CURRENT.PAYMENTS_CURRENT" in names + assert "WH_TICKETING_HISTORY.RESERVATIONS_HISTORY" in names + + +def test_catalog_entries_have_owner_layer_and_cadence(): + path = Path(__file__).resolve().parents[1] / "catalog" / "catalog.json" + catalog = json.loads(path.read_text()) + + for dataset in catalog["datasets"]: + assert dataset["owner"] + assert dataset["layer"] + assert dataset["cadence"] diff --git a/submission/ticket-reservations/tests/test_cdc.py b/submission/ticket-reservations/tests/test_cdc.py new file mode 100644 index 0000000..3785b92 --- /dev/null +++ b/submission/ticket-reservations/tests/test_cdc.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +import pytest + +from pipeline.cdc import CDCEvent, KafkaCDCLog + + +def test_create_update_delete_events_have_debezium_shape(reservation_row): + log = KafkaCDCLog() + created = log.create("reservations", "r1", reservation_row) + updated = {**reservation_row, "status": "confirmed"} + changed = log.update("reservations", "r1", reservation_row, updated) + deleted = log.delete("reservations", "r1", updated) + + assert created.op == "c" + assert changed.before["status"] == "held" + assert changed.after["status"] == "confirmed" + assert deleted.op == "d" + assert deleted.before["reservation_id"] == "r1" + + +def test_offsets_and_lsns_are_monotonic(reservation_row): + log = KafkaCDCLog() + first = log.create("reservations", "r1", reservation_row) + second = log.update("reservations", "r1", reservation_row, reservation_row) + + assert first.kafka_offset < second.kafka_offset + assert first.source_lsn < second.source_lsn + + +def test_events_after_models_checkpoint_replay(reservation_row): + log = KafkaCDCLog() + log.create("reservations", "r1", reservation_row) + checkpoint = log.latest_offset + event = log.create("reservations", "r2", {**reservation_row, "reservation_id": "r2"}) + + assert log.events_after(checkpoint) == [event] + + +def test_event_id_is_stable_for_duplicate_delivery(reservation_row): + event = CDCEvent( + topic="ticketing.public.reservations", + table="reservations", + primary_key="r1", + op="c", + after=reservation_row, + source_lsn=100, + tx_id="tx-100", + ) + duplicate = CDCEvent( + topic="ticketing.public.reservations", + table="reservations", + primary_key="r1", + op="c", + after=reservation_row, + source_lsn=100, + tx_id="tx-100", + ) + + assert event.event_id == duplicate.event_id + + +def test_invalid_operation_is_rejected(reservation_row): + with pytest.raises(ValueError, match="Invalid Debezium operation"): + CDCEvent( + topic="ticketing.public.reservations", + table="reservations", + primary_key="r1", + op="upsert", + after=reservation_row, + ) + diff --git a/submission/ticket-reservations/tests/test_data_quality.py b/submission/ticket-reservations/tests/test_data_quality.py new file mode 100644 index 0000000..1e5b239 --- /dev/null +++ b/submission/ticket-reservations/tests/test_data_quality.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from scripts.run_data_quality_checks import create_quality_demo, reservation_total_mismatches + + +def test_reservation_total_mismatch_is_detected(): + import duckdb + + conn = duckdb.connect(":memory:") + create_quality_demo(conn) + conn.execute("INSERT INTO reservations_current VALUES ('r1', 'held', 100.00, false)") + conn.execute("INSERT INTO reservation_items_current VALUES ('r1', 80.00, 5.00, false)") + + assert reservation_total_mismatches(conn) == [("r1",)] + + +def test_matching_reservation_total_passes(): + import duckdb + + conn = duckdb.connect(":memory:") + create_quality_demo(conn) + conn.execute("INSERT INTO reservations_current VALUES ('r1', 'held', 85.00, false)") + conn.execute("INSERT INTO reservation_items_current VALUES ('r1', 80.00, 5.00, false)") + + assert reservation_total_mismatches(conn) == [] + diff --git a/submission/ticket-reservations/tests/test_lake_warehouse.py b/submission/ticket-reservations/tests/test_lake_warehouse.py new file mode 100644 index 0000000..bcd6483 --- /dev/null +++ b/submission/ticket-reservations/tests/test_lake_warehouse.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import json +from datetime import datetime, timezone + +from pipeline.cdc import CDCEvent, KafkaCDCLog +from pipeline.lake import append_events +from pipeline.warehouse import apply_events, point_in_time_state + + +def test_lake_keeps_every_change_and_deduplicates_replay(conn, reservation_row): + log = KafkaCDCLog() + created = log.create("reservations", "r1", reservation_row) + updated = log.update( + "reservations", + "r1", + reservation_row, + {**reservation_row, "status": "confirmed"}, + ) + + assert append_events(conn, [created, updated]) == 2 + assert append_events(conn, [created, updated]) == 0 + + count = conn.execute("SELECT COUNT(*) FROM silver_ticketing_cdc_events").fetchone()[0] + assert count == 2 + + +def test_warehouse_current_uses_latest_lsn(conn, reservation_row): + log = KafkaCDCLog() + created = log.create("reservations", "r1", reservation_row) + updated = log.update( + "reservations", + "r1", + reservation_row, + {**reservation_row, "status": "confirmed"}, + ) + + apply_events(conn, [updated, created]) + + row_json, is_deleted, source_lsn = conn.execute( + """ + SELECT row_json, is_deleted, source_lsn + FROM wh_current_records + WHERE source_table = 'reservations' AND primary_key = 'r1' + """ + ).fetchone() + + assert json.loads(row_json)["status"] == "confirmed" + assert is_deleted is False + assert source_lsn == updated.source_lsn + + +def test_delete_becomes_soft_delete_in_warehouse(conn, reservation_row): + log = KafkaCDCLog() + created = log.create("reservations", "r1", reservation_row) + deleted = log.delete("reservations", "r1", reservation_row) + + apply_events(conn, [created, deleted]) + + assert conn.execute( + "SELECT is_deleted FROM wh_current_records WHERE primary_key = 'r1'" + ).fetchone()[0] is True + + +def test_history_supports_point_in_time_restore(conn, reservation_row): + held = CDCEvent( + topic="ticketing.public.reservations", + table="reservations", + primary_key="r1", + op="c", + after=reservation_row, + source_lsn=1, + tx_id="tx-1", + kafka_offset=1, + source_event_at=datetime(2026, 7, 1, 10, 0, tzinfo=timezone.utc), + ) + confirmed = CDCEvent( + topic="ticketing.public.reservations", + table="reservations", + primary_key="r1", + op="u", + before=reservation_row, + after={**reservation_row, "status": "confirmed"}, + source_lsn=2, + tx_id="tx-2", + kafka_offset=2, + source_event_at=datetime(2026, 7, 1, 10, 5, tzinfo=timezone.utc), + ) + + apply_events(conn, [held, confirmed]) + + restored = point_in_time_state(conn, "reservations", "2026-07-01 10:02:00") + assert len(restored) == 1 + assert json.loads(restored[0][1])["status"] == "held" diff --git a/submission/ticket-reservations/tests/test_schema_contracts.py b/submission/ticket-reservations/tests/test_schema_contracts.py new file mode 100644 index 0000000..10d3ff7 --- /dev/null +++ b/submission/ticket-reservations/tests/test_schema_contracts.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import duckdb +import pytest + +from pipeline.schema_contracts import IncompatibleSchemaChange, assert_compatible, validate_event_schema +from source.models import SCHEMA_CONTRACT, create_source_tables + + +def _columns(conn: duckdb.DuckDBPyConnection, table: str) -> dict[str, str]: + return {row[0]: row[1] for row in conn.execute(f"DESCRIBE {table}").fetchall()} + + +def test_fresh_source_schema_matches_contract(): + conn = duckdb.connect(":memory:") + create_source_tables(conn) + + violations = [] + for table in SCHEMA_CONTRACT: + violations.extend(validate_event_schema(table, _columns(conn, table))) + + assert violations == [] + + +def test_dropped_column_is_incompatible(): + columns = dict(SCHEMA_CONTRACT["reservations"]["columns"]) + columns.pop("total_amount") + + with pytest.raises(IncompatibleSchemaChange, match="total_amount"): + assert_compatible("reservations", columns) + + +def test_type_change_is_incompatible(): + columns = dict(SCHEMA_CONTRACT["payments"]["columns"]) + columns["amount"] = "VARCHAR" + + violations = validate_event_schema("payments", columns) + + assert any("amount" in violation.message for violation in violations) + + +def test_nullable_additive_column_is_safe_by_contract_policy(): + columns = dict(SCHEMA_CONTRACT["customers"]["columns"]) + columns["marketing_opt_in"] = "BOOLEAN" + + assert validate_event_schema("customers", columns) == [] + + +def test_unknown_table_stops_ingestion(): + with pytest.raises(IncompatibleSchemaChange, match="not registered"): + assert_compatible("promo_codes", {"promo_code": "VARCHAR"}) +