diff --git a/Cargo.lock b/Cargo.lock index 52d027e..e919019 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -114,18 +114,54 @@ dependencies = [ "serde_json", "snap", "strum", - "strum_macros", + "strum_macros 0.26.4", "thiserror 1.0.69", "typed-builder", "uuid", ] +[[package]] +name = "ar_archive_writer" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eb93bbb63b9c227414f6eb3a0adfddca591a8ce1e9b60661bb08969b87e340b" +dependencies = [ + "object", +] + [[package]] name = "arc-swap" version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" +[[package]] +name = "argminmax" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70f13d10a41ac8d2ec79ee34178d61e6f47a29c2edfe7ef1721c7383b0359e65" +dependencies = [ + "num-traits", +] + +[[package]] +name = "array-init-cursor" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed51fe0f224d1d4ea768be38c51f9f831dee9d05c163c11fba0b8c44387b1fc3" + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "async-channel" version = "2.5.0" @@ -138,6 +174,28 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "async-trait" version = "0.1.89" @@ -149,6 +207,15 @@ dependencies = [ "syn", ] +[[package]] +name = "atoi_simd" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a49e05797ca52e312a0c658938b7d00693ef037799ef7187678f212d7684cf" +dependencies = [ + "debug_unsafe", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -224,11 +291,48 @@ dependencies = [ "serde", ] +[[package]] +name = "bincode" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" +dependencies = [ + "bincode_derive", + "serde", + "unty", +] + +[[package]] +name = "bincode_derive" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" +dependencies = [ + "virtue", +] + [[package]] name = "bitflags" version = "2.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394" +dependencies = [ + "serde", +] + +[[package]] +name = "blake3" +version = "1.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d2d5991425dfd0785aed03aedcf0b321d61975c9b5b3689c774a2610ae0b51e" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.0", +] [[package]] name = "block-buffer" @@ -239,12 +343,38 @@ dependencies = [ "generic-array", ] +[[package]] +name = "boxcar" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36f64beae40a84da1b4b26ff2761a5b895c12adc41dc25aaee1c4f2bbfe97a6e" + [[package]] name = "bumpalo" version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" +[[package]] +name = "bytemuck" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "bytes" version = "1.10.1" @@ -254,6 +384,15 @@ dependencies = [ "serde", ] +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + [[package]] name = "cc" version = "1.2.41" @@ -261,6 +400,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac9fe6cdbb24b6ade63616c0a0688e45bb56732262c158df3c0c4bea4ca47cb7" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] @@ -276,6 +417,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chacha20" version = "0.10.0" @@ -283,7 +430,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.3.0", "rand_core 0.10.0", ] @@ -301,6 +448,16 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf", +] + [[package]] name = "colorchoice" version = "1.0.4" @@ -317,6 +474,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "compact_str" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "serde", + "static_assertions", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -488,6 +660,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + [[package]] name = "core-foundation" version = "0.9.4" @@ -523,6 +701,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "cpufeatures" version = "0.3.0" @@ -550,6 +737,34 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -578,6 +793,12 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" +[[package]] +name = "debug_unsafe" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eed2c4702fa172d1ce21078faa7c5203e69f5394d48cc436d25928394a867a2" + [[package]] name = "deranged" version = "0.5.4" @@ -629,6 +850,12 @@ dependencies = [ "syn", ] +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + [[package]] name = "educe" version = "0.5.11" @@ -724,6 +951,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "ethnum" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40404c3f5f511ec4da6fe866ddf6a717c309fdbb69fbbad7b0f3edab8f2e835f" + [[package]] name = "event-listener" version = "5.4.1" @@ -755,6 +988,18 @@ dependencies = [ "rand 0.9.2", ] +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "fast-float2" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55" + [[package]] name = "fastrand" version = "2.3.0" @@ -819,6 +1064,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs4" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4" +dependencies = [ + "rustix", + "windows-sys 0.59.0", +] + [[package]] name = "futures" version = "0.3.31" @@ -925,8 +1180,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi 0.11.1+wasi-snapshot-preview1", + "wasm-bindgen", ] [[package]] @@ -936,9 +1193,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 5.3.0", "wasi 0.14.7+wasi-0.2.4", + "wasm-bindgen", ] [[package]] @@ -955,6 +1214,12 @@ dependencies = [ "wasip3", ] +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + [[package]] name = "h2" version = "0.4.12" @@ -980,6 +1245,8 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ + "allocator-api2", + "equivalent", "foldhash 0.1.5", ] @@ -992,8 +1259,16 @@ dependencies = [ "allocator-api2", "equivalent", "foldhash 0.2.0", + "rayon", + "serde", ] +[[package]] +name = "hashbrown" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" + [[package]] name = "heck" version = "0.4.1" @@ -1006,6 +1281,15 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "http" version = "1.3.1" @@ -1112,6 +1396,7 @@ dependencies = [ "hyper-util", "log", "rustls", + "rustls-native-certs", "rustls-pki-types", "rustls-platform-verifier", "tokio", @@ -1313,12 +1598,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.11.4" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.16.0", + "hashbrown 0.17.0", "serde", "serde_core", ] @@ -1388,6 +1673,16 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.3", + "libc", +] + [[package]] name = "js-sys" version = "0.3.81" @@ -1485,6 +1780,31 @@ version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "lz4" +version = "1.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a20b523e860d03443e98350ceaac5e71c6ba89aea7d960769ec3ce37f4de5af4" +dependencies = [ + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "matchers" version = "0.2.0" @@ -1512,6 +1832,15 @@ version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +[[package]] +name = "memmap2" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +dependencies = [ + "libc", +] + [[package]] name = "memoffset" version = "0.9.1" @@ -1595,6 +1924,7 @@ dependencies = [ "futures", "nominal-api", "parking_lot", + "polars", "prost", "reqwest", "serde_json", @@ -1608,6 +1938,15 @@ dependencies = [ "url", ] +[[package]] +name = "now" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d89e9874397a1f0a52fc1f197a8effd9735223cb2390e9dcc83ac6cd02923d0" +dependencies = [ + "chrono", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -1650,40 +1989,85 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] -name = "once_cell" -version = "1.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" - -[[package]] -name = "once_cell_polyfill" -version = "1.70.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" - -[[package]] -name = "openssl" -version = "0.10.78" +name = "object" +version = "0.37.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f38c4372413cdaaf3cc79dd92d29d7d9f5ab09b51b10dded508fb90bb70b9222" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" dependencies = [ - "bitflags", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", + "memchr", ] [[package]] -name = "openssl-macros" -version = "0.1.1" +name = "object_store" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +checksum = "fbfbfff40aeccab00ec8a910b57ca8ecf4319b335c542f2edcd19dd25a1e2a00" +dependencies = [ + "async-trait", + "base64", + "bytes", + "chrono", + "form_urlencoded", + "futures", + "http", + "http-body-util", + "humantime", + "hyper", + "itertools", + "parking_lot", + "percent-encoding", + "quick-xml", + "rand 0.9.2", + "reqwest", + "ring", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror 2.0.18", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" + +[[package]] +name = "openssl" +version = "0.10.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38c4372413cdaaf3cc79dd92d29d7d9f5ab09b51b10dded508fb90bb70b9222" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", @@ -1779,6 +2163,24 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "phf" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_shared" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "1.1.10" @@ -1817,6 +2219,481 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "planus" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3daf8e3d4b712abe1d690838f6e29fb76b76ea19589c4afa39ec30e12f62af71" +dependencies = [ + "array-init-cursor", + "hashbrown 0.15.5", +] + +[[package]] +name = "polars" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bc9ea901050c1bb8747ee411bc7fbb390f3b399931e7484719512965132a248" +dependencies = [ + "getrandom 0.2.16", + "getrandom 0.3.3", + "polars-arrow", + "polars-compute", + "polars-core", + "polars-error", + "polars-io", + "polars-lazy", + "polars-ops", + "polars-plan", + "polars-utils", + "version_check", +] + +[[package]] +name = "polars-arrow" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33d3fe43f8702cf7899ff3d516c2e5f7dc84ee6f6a3007e1a831a0ff87940704" +dependencies = [ + "bitflags", + "bytemuck", + "chrono", + "chrono-tz", + "dyn-clone", + "either", + "ethnum", + "getrandom 0.2.16", + "getrandom 0.3.3", + "hashbrown 0.16.0", + "lz4", + "num-traits", + "polars-arrow-format", + "polars-error", + "polars-schema", + "polars-utils", + "serde", + "simdutf8", + "streaming-iterator", + "strum_macros 0.27.2", + "version_check", + "zstd", +] + +[[package]] +name = "polars-arrow-format" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a556ac0ee744e61e167f34c1eb0013ce740e0ee6cd8c158b2ec0b518f10e6675" +dependencies = [ + "planus", + "serde", +] + +[[package]] +name = "polars-compute" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29cc7497378dee3a002f117e0b4e16b7cbe6c8ed3da16a0229c89294af7c3bf" +dependencies = [ + "atoi_simd", + "bytemuck", + "chrono", + "either", + "fast-float2", + "hashbrown 0.16.0", + "itoa", + "num-traits", + "polars-arrow", + "polars-error", + "polars-utils", + "rand 0.9.2", + "ryu", + "serde", + "strength_reduce", + "strum_macros 0.27.2", + "version_check", +] + +[[package]] +name = "polars-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48409b7440cb1a4aa84953fe3a4189dfbfb300a3298266a92a37363476641e40" +dependencies = [ + "bitflags", + "boxcar", + "bytemuck", + "chrono", + "either", + "hashbrown 0.16.0", + "indexmap", + "itoa", + "num-traits", + "polars-arrow", + "polars-compute", + "polars-dtype", + "polars-error", + "polars-row", + "polars-schema", + "polars-utils", + "rand 0.9.2", + "rand_distr", + "rayon", + "regex", + "serde", + "serde_json", + "strum_macros 0.27.2", + "uuid", + "version_check", + "xxhash-rust", +] + +[[package]] +name = "polars-dtype" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7007e9e8b7b657cbd339b65246af7e87f5756ee9a860119b9424ddffd2aaf133" +dependencies = [ + "boxcar", + "hashbrown 0.16.0", + "polars-arrow", + "polars-error", + "polars-utils", + "serde", + "uuid", +] + +[[package]] +name = "polars-error" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9a6be22566c89f6405f553bfdb7c8a6cb20ec51b35f3172de9a25fa3e252d85" +dependencies = [ + "object_store", + "parking_lot", + "polars-arrow-format", + "regex", + "signal-hook", + "simdutf8", +] + +[[package]] +name = "polars-expr" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6199a50d3e1afd0674fb009e340cbfb0010682b2387187a36328c00f3f2ca87b" +dependencies = [ + "bitflags", + "hashbrown 0.16.0", + "num-traits", + "polars-arrow", + "polars-compute", + "polars-core", + "polars-io", + "polars-ops", + "polars-plan", + "polars-row", + "polars-utils", + "rand 0.9.2", + "rayon", + "recursive", + "version_check", +] + +[[package]] +name = "polars-io" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be3714acdff87170141880a07f5d9233490d3bd5531c41898f6969d440feee11" +dependencies = [ + "async-trait", + "atoi_simd", + "blake3", + "bytes", + "fast-float2", + "fs4", + "futures", + "glob", + "hashbrown 0.16.0", + "home", + "itoa", + "memchr", + "memmap2", + "num-traits", + "object_store", + "percent-encoding", + "polars-arrow", + "polars-compute", + "polars-core", + "polars-error", + "polars-parquet", + "polars-schema", + "polars-utils", + "rayon", + "regex", + "reqwest", + "ryu", + "serde", + "serde_json", + "simdutf8", + "tokio", +] + +[[package]] +name = "polars-lazy" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea136c360d03aafe56e0233495e30044ce43639b8b0360a4a38e840233f048a1" +dependencies = [ + "bitflags", + "chrono", + "either", + "memchr", + "polars-arrow", + "polars-compute", + "polars-core", + "polars-expr", + "polars-io", + "polars-mem-engine", + "polars-ops", + "polars-plan", + "polars-stream", + "polars-utils", + "rayon", + "version_check", +] + +[[package]] +name = "polars-mem-engine" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f6e455ceb6e5aee7ed7d5c8944104e66992173e03a9c42f9670226318672249" +dependencies = [ + "memmap2", + "polars-arrow", + "polars-core", + "polars-error", + "polars-expr", + "polars-io", + "polars-ops", + "polars-plan", + "polars-utils", + "rayon", + "recursive", +] + +[[package]] +name = "polars-ops" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b59c80a019ef0e6f09b4416d2647076a52839305c9eb11919e8298ec667f853" +dependencies = [ + "argminmax", + "bytemuck", + "either", + "hashbrown 0.16.0", + "indexmap", + "libm", + "memchr", + "num-traits", + "polars-arrow", + "polars-compute", + "polars-core", + "polars-error", + "polars-schema", + "polars-utils", + "rayon", + "regex", + "regex-syntax", + "strum_macros 0.27.2", + "version_check", +] + +[[package]] +name = "polars-parquet" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93c2439d127c59e6bfc9d698419bdb45210068a6f501d44e6096429ad72c2eaa" +dependencies = [ + "async-stream", + "base64", + "bytemuck", + "ethnum", + "futures", + "hashbrown 0.16.0", + "num-traits", + "polars-arrow", + "polars-compute", + "polars-error", + "polars-parquet-format", + "polars-utils", + "serde", + "simdutf8", + "streaming-decompression", +] + +[[package]] +name = "polars-parquet-format" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c025243dcfe8dbc57e94d9f82eb3bef10b565ab180d5b99bed87fd8aea319ce1" +dependencies = [ + "async-trait", + "futures", +] + +[[package]] +name = "polars-plan" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65b4619f5c7e9b91f18611c9ed82ebeee4b10052160825c1316ecf4dbd4d97e6" +dependencies = [ + "bitflags", + "bytemuck", + "bytes", + "either", + "hashbrown 0.16.0", + "memmap2", + "num-traits", + "percent-encoding", + "polars-arrow", + "polars-compute", + "polars-core", + "polars-error", + "polars-io", + "polars-ops", + "polars-utils", + "rayon", + "recursive", + "sha2", + "slotmap", + "strum_macros 0.27.2", + "version_check", +] + +[[package]] +name = "polars-row" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a18d232f25b83032e280a279a1f40beb8a6f8fc43907b13dc07b1c56f3b11eea" +dependencies = [ + "bitflags", + "bytemuck", + "polars-arrow", + "polars-compute", + "polars-dtype", + "polars-error", + "polars-utils", +] + +[[package]] +name = "polars-schema" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f73e21d429ae1c23f442b0220ccfe773a9734a44e997b5062a741842909d9441" +dependencies = [ + "indexmap", + "polars-error", + "polars-utils", + "serde", + "version_check", +] + +[[package]] +name = "polars-stream" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ff19612074640a9d65e5928b7223db76ffee63e55b276f1e466d06719eb7362" +dependencies = [ + "async-channel", + "async-trait", + "atomic-waker", + "bitflags", + "chrono-tz", + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-queue", + "crossbeam-utils", + "futures", + "memmap2", + "parking_lot", + "percent-encoding", + "pin-project-lite", + "polars-arrow", + "polars-compute", + "polars-core", + "polars-error", + "polars-expr", + "polars-io", + "polars-mem-engine", + "polars-ops", + "polars-parquet", + "polars-plan", + "polars-time", + "polars-utils", + "rand 0.9.2", + "rayon", + "recursive", + "slotmap", + "tokio", + "version_check", +] + +[[package]] +name = "polars-time" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddce7a9f81d5f47d981bcee4a8db004f9596bb51f0f4d9d93667a1a00d88166c" +dependencies = [ + "atoi_simd", + "bytemuck", + "chrono", + "now", + "num-traits", + "polars-arrow", + "polars-compute", + "polars-core", + "polars-error", + "polars-ops", + "polars-utils", + "rayon", + "regex", + "strum_macros 0.27.2", +] + +[[package]] +name = "polars-utils" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "667c1bc2d2313f934d711f6e3b58d8d9f80351d14ea60af936a26b7dfb06e309" +dependencies = [ + "bincode", + "bytemuck", + "bytes", + "compact_str", + "either", + "flate2", + "foldhash 0.2.0", + "hashbrown 0.16.0", + "indexmap", + "libc", + "memmap2", + "num-traits", + "polars-error", + "rand 0.9.2", + "raw-cpuid", + "rayon", + "regex", + "rmp-serde", + "serde", + "serde_json", + "serde_stacker", + "slotmap", + "stacker", + "uuid", + "version_check", +] + [[package]] name = "portable-atomic" version = "1.11.1" @@ -1889,17 +2766,35 @@ dependencies = [ "syn", ] +[[package]] +name = "psm" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645dbe486e346d9b5de3ef16ede18c26e6c70ad97418f4874b8b1889d6e761ea" +dependencies = [ + "ar_archive_writer", + "cc", +] + [[package]] name = "py-nominal-streaming" version = "0.8.2" dependencies = [ "anyhow", + "apache-avro", "crossbeam-channel", "nominal-api", "nominal-streaming", "once_cell", "openssl", + "parking_lot", + "polars", + "polars-arrow", "pyo3", + "serde_json", + "tempfile", + "test-log", + "thiserror 2.0.18", "tokio", "tokio-util", "tracing", @@ -1974,6 +2869,71 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" +[[package]] +name = "quick-xml" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2 0.6.1", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "bytes", + "getrandom 0.3.3", + "lru-slab", + "rand 0.9.2", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2 0.6.1", + "tracing", + "windows-sys 0.60.2", +] + [[package]] name = "quote" version = "1.0.41" @@ -2073,6 +3033,65 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba" +[[package]] +name = "rand_distr" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" +dependencies = [ + "num-traits", + "rand 0.9.2", +] + +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "recursive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e" +dependencies = [ + "recursive-proc-macro-impl", + "stacker", +] + +[[package]] +name = "recursive-proc-macro-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" +dependencies = [ + "quote", + "syn", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -2137,6 +3156,7 @@ dependencies = [ "bytes", "encoding_rs", "futures-core", + "futures-util", "h2", "http", "http-body", @@ -2151,6 +3171,9 @@ dependencies = [ "native-tls", "percent-encoding", "pin-project-lite", + "quinn", + "rustls", + "rustls-native-certs", "rustls-pki-types", "serde", "serde_json", @@ -2158,12 +3181,15 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-native-tls", + "tokio-rustls", + "tokio-util", "tower", "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", ] @@ -2187,6 +3213,31 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" +[[package]] +name = "rmp" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ba8be72d372b2c9b35542551678538b562e7cf86c3315773cae48dfbfe7790c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "rmp-serde" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f81bee8c8ef9b577d1681a70ebbc962c232461e397b22c208c43c04b67a155" +dependencies = [ + "rmp", + "serde", +] + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + [[package]] name = "rustc_version" version = "0.4.1" @@ -2251,6 +3302,7 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" dependencies = [ + "web-time", "zeroize", ] @@ -2455,6 +3507,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_stacker" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4936375d50c4be7eff22293a9344f8e46f323ed2b3c243e52f89138d9bb0f4a" +dependencies = [ + "serde", + "serde_core", + "stacker", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -2467,6 +3530,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -2482,6 +3556,16 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + [[package]] name = "signal-hook-registry" version = "1.4.6" @@ -2497,12 +3581,33 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "siphasher" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" + [[package]] name = "slab" version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" +[[package]] +name = "slotmap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdd58c3c93c3d278ca835519292445cb4b0d4dc59ccfdf7ceadaab3f8aeb4038" +dependencies = [ + "version_check", +] + [[package]] name = "smallvec" version = "1.15.1" @@ -2541,6 +3646,19 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "stacker" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "640c8cdd92b6b12f5bcb1803ca3bbf5ab96e5e6b6b96b9ab77dabe9e880b3190" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.61.2", +] + [[package]] name = "staged-builder" version = "0.2.0" @@ -2563,6 +3681,33 @@ dependencies = [ "syn", ] +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "streaming-decompression" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf6cc3b19bfb128a8ad11026086e31d3ce9ad23f8ea37354b31383a187c44cf3" +dependencies = [ + "fallible-streaming-iterator", +] + +[[package]] +name = "streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" + +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + [[package]] name = "structmeta" version = "0.3.0" @@ -2605,6 +3750,18 @@ dependencies = [ "syn", ] +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "subtle" version = "2.6.1" @@ -2791,6 +3948,21 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.50.0" @@ -3114,6 +4286,12 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "unty" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" + [[package]] name = "url" version = "2.5.8" @@ -3169,6 +4347,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "virtue" +version = "0.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" + [[package]] name = "walkdir" version = "2.5.0" @@ -3315,6 +4499,19 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmparser" version = "0.244.0" @@ -3337,6 +4534,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki-root-certs" version = "1.0.6" @@ -3822,6 +5029,12 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" +[[package]] +name = "xxhash-rust" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" + [[package]] name = "yoke" version = "0.8.0" @@ -3953,3 +5166,31 @@ name = "zmij" version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fc5a66a20078bf1251bde995aa2fdcc4b800c70b5d92dd2c62abc5c60f679f8" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/nominal-streaming/Cargo.toml b/nominal-streaming/Cargo.toml index deb82e0..2216691 100644 --- a/nominal-streaming/Cargo.toml +++ b/nominal-streaming/Cargo.toml @@ -10,6 +10,7 @@ repository = "https://github.com/nominal-io/nominal-streaming" [features] default = ["logging"] logging = ["tracing-subscriber"] +polars = ["dep:polars"] [dependencies] @@ -25,6 +26,7 @@ derive_more = { workspace = true } futures = { workspace = true } nominal-api = { workspace = true } parking_lot = { workspace = true } +polars = { version = "0.52", default-features = false, features = ["dtype-struct", "dtype-array"], optional = true } prost = { workspace = true } reqwest = { workspace = true } serde_json = { workspace = true } diff --git a/nominal-streaming/src/avro_writer/consumer.rs b/nominal-streaming/src/avro_writer/consumer.rs new file mode 100644 index 0000000..d0740ed --- /dev/null +++ b/nominal-streaming/src/avro_writer/consumer.rs @@ -0,0 +1,232 @@ +use std::path::Path; +use std::sync::atomic::Ordering; +use std::sync::Arc; +use std::sync::OnceLock; + +use nominal_api::tonic::io::nominal::scout::api::proto; +use nominal_api::tonic::io::nominal::scout::api::proto::WriteRequestNominal; +use tracing::error; + +use super::error::avro_error_from_consumer_ref; +use super::error::AvroWriterError; +use super::stats::PipelineStats; +use crate::consumer::ConsumerError; +use crate::consumer::ConsumerResult; +use crate::consumer::WriteRequestConsumer; + +/// Avro file consumer that parallelizes per-series `Record` building across +/// scoped threads, then feeds the resulting records to a single +/// `apache_avro::Writer` under a mutex (identical on-disk output to +/// [`crate::consumer::AvroFileConsumer`]). Reuses +/// [`crate::consumer::points_to_avro`] for the per-dtype `Value`-tree +/// construction — only the parallel wrapper is new. +/// +/// Lives alongside [`super::AvroWriter`] because the parallelism only +/// benefits the single-file write path — the shared `AvroFileConsumer` +/// used by network-streaming callers stays untouched. +pub(super) struct ParallelAvroFileConsumer { + writer: Arc>>, +} + +impl std::fmt::Debug for ParallelAvroFileConsumer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ParallelAvroFileConsumer") + .finish_non_exhaustive() + } +} + +impl ParallelAvroFileConsumer { + pub(super) fn new_with_full_path(path: &Path) -> std::io::Result { + std::fs::create_dir_all(path.parent().unwrap_or(path))?; + let file = std::fs::OpenOptions::new() + .create(true) + .truncate(false) + .write(true) + .open(path)?; + let writer = apache_avro::Writer::builder() + .schema(&crate::consumer::CORE_AVRO_SCHEMA) + .writer(file) + .codec(apache_avro::Codec::Snappy) + .build(); + Ok(Self { + writer: Arc::new(parking_lot::Mutex::new(writer)), + }) + } +} + +impl WriteRequestConsumer for ParallelAvroFileConsumer { + fn consume(&self, request: &WriteRequestNominal) -> ConsumerResult<()> { + let records = build_records_parallel(&request.series); + self.writer + .lock() + .extend(records) + .map_err(|e| ConsumerError::AvroError(Box::new(e)))?; + Ok(()) + } +} + +/// Build avro `Record`s from a slice of `Series` in parallel using scoped +/// threads. Each series is independent (its own timestamp/value vectors and +/// tags); the expensive part — `Value::Union(0, Box::new(Value::Double(x)))` +/// allocation per point — is embarrassingly parallel. +/// +/// We use `std::thread::scope` rather than a persistent worker pool so the +/// dependency graph stays minimal; one OS-thread spawn per chunk is +/// ~20-50µs, well under 1% of wall at observed call rates. +fn build_records_parallel(series: &[proto::Series]) -> Vec> { + if series.is_empty() { + return Vec::new(); + } + + let num_threads = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(4) + .min(series.len()) + .max(1); + let chunk_size = series.len().div_ceil(num_threads); + + std::thread::scope(|s| { + let handles: Vec<_> = series + .chunks(chunk_size) + .map(|chunk| s.spawn(move || build_records_for_chunk(chunk))) + .collect(); + + let mut out: Vec> = Vec::with_capacity(series.len()); + for h in handles { + out.extend(h.join().expect("worker panicked")); + } + out + }) +} + +fn build_records_for_chunk(chunk: &[proto::Series]) -> Vec> { + use apache_avro::types::Record; + use apache_avro::types::Value; + chunk + .iter() + .map(|series| { + let (timestamps, values) = crate::consumer::points_to_avro(series.points.as_ref()); + + let mut record = Record::new(&crate::consumer::CORE_AVRO_SCHEMA) + .expect("Failed to create Avro record"); + record.put( + "channel", + series + .channel + .as_ref() + .map(|c| c.name.clone()) + .unwrap_or_else(|| "values".to_string()), + ); + record.put("timestamps", Value::Array(timestamps)); + record.put("values", Value::Array(values)); + record.put("tags", series.tags.clone()); + record + }) + .collect() +} + +/// Wraps another [`WriteRequestConsumer`] and latches the first error seen, +/// so [`super::AvroWriter`] can surface disk / avro failures from subsequent +/// `write` / `flush` / `sync` / `close` calls. +/// +/// Also records per-call wall time into `stats.consumer_consume_ns` / +/// `consumer_consume_calls` — the consumer runs on the stream's dispatcher +/// thread, so its accumulated wall time attributes downstream work to the +/// pipeline. +pub(super) struct ErrorLatchingConsumer { + pub(super) inner: C, + pub(super) first_error: Arc>>, + pub(super) stats: Arc, +} + +impl std::fmt::Debug for ErrorLatchingConsumer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ErrorLatchingConsumer") + .finish_non_exhaustive() + } +} + +impl WriteRequestConsumer for ErrorLatchingConsumer { + fn consume(&self, request: &WriteRequestNominal) -> ConsumerResult<()> { + let t0 = std::time::Instant::now(); + let result = self.inner.consume(request); + self.stats + .consumer_consume_ns + .fetch_add(t0.elapsed().as_nanos() as u64, Ordering::Relaxed); + self.stats + .consumer_consume_calls + .fetch_add(1, Ordering::Relaxed); + match result { + Ok(()) => Ok(()), + Err(e) => { + // Latch on first occurrence (first-wins via OnceLock::set). + // We call `avro_error_from_consumer_ref` rather than consuming + // `e` because we must also return `e` to the stream's + // dispatcher — `ConsumerError` isn't `Clone`. The helper + // reconstructs `Io` from `io::Error::new(kind, msg)` so the + // `Io` variant is preserved; avro errors can't be cloned so + // they flatten to `Consumer` in the latched copy. + if self.first_error.get().is_none() { + let latched = Arc::new(avro_error_from_consumer_ref(&e)); + error!("AvroWriter: encoder error latched: {:?}", latched); + let _ = self.first_error.set(latched); + } + // Return the error to the stream's dispatcher so it logs / backs off. + Err(e) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A consumer that always fails with `PermissionDenied` IoError. + /// Used by `error_latch_preserves_io_variant` (unit-struct, fixed kind). + #[derive(Debug)] + struct PermissionDeniedConsumer; + + impl WriteRequestConsumer for PermissionDeniedConsumer { + fn consume(&self, _request: &WriteRequestNominal) -> ConsumerResult<()> { + Err(ConsumerError::IoError(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "denied", + ))) + } + } + + #[test_log::test] + fn error_latch_preserves_io_variant() { + let first_error: Arc>> = Arc::new(OnceLock::new()); + let consumer = ErrorLatchingConsumer { + inner: PermissionDeniedConsumer, + first_error: first_error.clone(), + stats: Arc::new(PipelineStats::default()), + }; + + // Construct a minimal WriteRequestNominal to satisfy the trait bound. + let req = WriteRequestNominal { + series: vec![], + session_name: None, + }; + + // The consume call should fail (pass the error through) and latch. + let result = consumer.consume(&req); + assert!(result.is_err(), "consumer should have propagated the error"); + + // Inspect the latched error. + let latched = first_error.get().expect("error should have been latched"); + match latched.as_ref() { + AvroWriterError::Io(io_err) => { + assert_eq!( + io_err.kind(), + std::io::ErrorKind::PermissionDenied, + "expected PermissionDenied, got {:?}", + io_err.kind() + ); + } + other => panic!("expected AvroWriterError::Io, got {other:?}"), + } + } +} diff --git a/nominal-streaming/src/avro_writer/error.rs b/nominal-streaming/src/avro_writer/error.rs new file mode 100644 index 0000000..b8e103a --- /dev/null +++ b/nominal-streaming/src/avro_writer/error.rs @@ -0,0 +1,59 @@ +use crate::consumer::ConsumerError; + +/// Errors produced by [`super::AvroWriter`]. +/// +/// Latched errors surface via `Consumer` for any variant that couldn't be +/// preserved across the `&ConsumerError` → `AvroWriterError` boundary (avro +/// errors specifically, because `apache_avro::Error` isn't `Clone`). +/// +/// These errors are wrapped in `Arc` for idiomatic sharing of a sticky, +/// cached error across producer threads and repeated calls — once a writer +/// latches a failure, every subsequent method call returns the same +/// `Arc` without re-walking the failure path. +#[derive(Debug, thiserror::Error)] +pub enum AvroWriterError { + /// I/O error from the underlying file (open, write, sync, etc.). + /// Latched I/O errors are reconstructed from `io::ErrorKind` + message + /// so the `Io` variant is preserved through the error-latching consumer layer. + #[error("I/O error: {0}")] + Io(#[from] std::io::Error), + + /// Error from the avro writer (schema mismatch, serialization failure). + /// Avro errors latch as `Consumer` because `apache_avro::Error` isn't `Clone`. + #[error("avro error: {0}")] + Avro(#[from] Box), + + /// Error from the underlying [`crate::consumer::AvroFileConsumer`] sink, + /// or a latched avro error (avro errors flatten here because + /// `apache_avro::Error` isn't `Clone`). + #[error("consumer error: {0}")] + Consumer(String), + + /// `write` was called after `close()` completed. Always means caller + /// misuse, not encoder failure. + #[error("write attempted after close")] + SendAfterClose, +} + +impl From for AvroWriterError { + fn from(e: ConsumerError) -> Self { + match e { + ConsumerError::IoError(io) => AvroWriterError::Io(io), + ConsumerError::AvroError(avro) => AvroWriterError::Avro(avro), + other => AvroWriterError::Consumer(other.to_string()), + } + } +} + +/// Construct an `AvroWriterError` from a `ConsumerError` that the caller +/// needs to retain (so we can't consume it). Preserves the `Io` variant +/// via `io::Error::new(kind, msg)`; avro errors flatten to `Consumer` +/// because `apache_avro::Error` isn't Clone. +pub(super) fn avro_error_from_consumer_ref(e: &ConsumerError) -> AvroWriterError { + match e { + ConsumerError::IoError(io) => { + AvroWriterError::Io(std::io::Error::new(io.kind(), io.to_string())) + } + other => AvroWriterError::Consumer(other.to_string()), + } +} diff --git a/nominal-streaming/src/avro_writer/helpers.rs b/nominal-streaming/src/avro_writer/helpers.rs new file mode 100644 index 0000000..f1384b1 --- /dev/null +++ b/nominal-streaming/src/avro_writer/helpers.rs @@ -0,0 +1,71 @@ +use std::io; +use std::path::Path; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::OnceLock; +use std::time::Duration; + +use super::consumer::ErrorLatchingConsumer; +use super::consumer::ParallelAvroFileConsumer; +use super::error::AvroWriterError; +use super::stats::PipelineStats; +use crate::consumer::WriteRequestConsumer; +use crate::stream::NominalDatasetStream; +use crate::stream::NominalStreamOpts; + +/// Derive the numbered path for rotation index `index`. +/// `out.avro` at index 0 → `out_000.avro`. +pub(super) fn path_for_index(base: &Path, index: usize) -> PathBuf { + let stem = base.file_stem().unwrap_or_default().to_string_lossy(); + let suffix = base + .extension() + .map(|e| format!(".{}", e.to_string_lossy())) + .unwrap_or_default(); + base.with_file_name(format!("{stem}_{index:03}{suffix}")) +} + +/// Ensure the parent directory exists and truncate the file at `path`. +/// +/// Returns `io::Error` on failure so callers can surface the error to +/// their own error type. (Historically this panicked; library-grade APIs +/// should not.) +pub(super) fn ensure_parent_and_truncate(path: &Path) -> io::Result<()> { + if let Some(parent) = path.parent() { + if !parent.as_os_str().is_empty() { + std::fs::create_dir_all(parent)?; + } + } + std::fs::File::create(path)?; + Ok(()) +} + +/// Build a fresh `NominalDatasetStream` with the given consumer and writer opts. +pub(super) fn open_stream( + consumer: C, + max_points_per_batch: usize, + max_batch_delay: Duration, +) -> NominalDatasetStream { + let stream_opts = NominalStreamOpts { + max_points_per_record: max_points_per_batch, + max_request_delay: max_batch_delay, + max_buffered_requests: 4, + request_dispatcher_tasks: 1, + base_api_url: String::new(), + }; + NominalDatasetStream::new_with_consumer(consumer, stream_opts) +} + +/// Open a new `ParallelAvroFileConsumer` + `ErrorLatchingConsumer` for the +/// given path, sharing the `first_error` latch and `stats` counters. +pub(super) fn open_error_latching_consumer( + path: &Path, + first_error: Arc>>, + stats: Arc, +) -> io::Result { + let avro_consumer = ParallelAvroFileConsumer::new_with_full_path(path)?; + Ok(ErrorLatchingConsumer { + inner: avro_consumer, + first_error, + stats, + }) +} diff --git a/nominal-streaming/src/avro_writer/mod.rs b/nominal-streaming/src/avro_writer/mod.rs new file mode 100644 index 0000000..064a216 --- /dev/null +++ b/nominal-streaming/src/avro_writer/mod.rs @@ -0,0 +1,43 @@ +//! Purpose-built avro file writer for the Nominal schema. +//! +//! This module provides the pure-Rust file-writer primitive. It writes +//! snappy-compressed avro in the Nominal `AvroStream` schema (same schema as +//! [`crate::consumer::AvroFileConsumer`]). +//! +//! For PyO3 bindings, see the `py-nominal-streaming` crate, which wraps +//! [`AvroWriter`] / [`AvroWriterOpts`] with `#[pyclass]`. +//! +//! # When to use this vs. [`crate::stream::NominalDatasetStream`] +//! +//! - Use [`AvroWriter`] when you only need a file: no network, no fallback, +//! no retries. Fewer moving parts, simpler opts surface. +//! - Use [`crate::stream::NominalDatasetStream`] when you need to upload to +//! Nominal Core, with or without a file fallback. +//! +//! # Example +//! +//! ```rust,ignore +//! use nominal_streaming::avro_writer::{AvroWriter, AvroWriterOpts}; +//! use nominal_streaming::prelude::*; +//! use std::path::PathBuf; +//! +//! let writer = AvroWriter::new(PathBuf::from("out.avro"), AvroWriterOpts::default())?; +//! let desc = ChannelDescriptor::new("speed"); +//! writer.write(&desc, vec![DoublePoint { timestamp: None, value: 3.14 }])?; +//! writer.close()?; +//! ``` + +mod consumer; +mod error; +mod helpers; +mod opts; +#[cfg(feature = "polars")] +mod polars; +mod state; +mod stats; +mod writer; + +pub use error::AvroWriterError; +pub use opts::AvroWriterOpts; +pub use stats::PipelineStats; +pub use writer::AvroWriter; diff --git a/nominal-streaming/src/avro_writer/opts.rs b/nominal-streaming/src/avro_writer/opts.rs new file mode 100644 index 0000000..ad81060 --- /dev/null +++ b/nominal-streaming/src/avro_writer/opts.rs @@ -0,0 +1,72 @@ +use std::time::Duration; + +/// Configuration for [`super::AvroWriter`]. +/// +/// All fields have sensible defaults via [`Self::default()`]. Use +/// [`Self::new`] for the default and the `with_*` builders to tune +/// individual fields, or construct by field directly. +#[derive(Debug, Clone)] +pub struct AvroWriterOpts { + /// Maximum number of points packed into a single avro record before the + /// underlying stream's batch-processor emits it. Larger values produce fewer, + /// bigger records; smaller values produce more frequent, smaller records. + /// Default: 250,000. + pub max_points_per_batch: usize, + + /// Maximum time the underlying stream buffers a partial batch before forcing + /// a flush. Tune up for throughput-biased workloads, down for + /// latency-biased workloads. Default: 100ms. + pub max_batch_delay: Duration, + + /// Whether [`super::AvroWriter::close`] and `Drop` call `File::sync_all()` + /// before returning. Default: `true`. + pub fsync_on_close: bool, + + /// Maximum number of points to write to a single file before rotating to a + /// new numbered file. `0` means no rotation (write everything to the path + /// given to the constructor). Default: `0`. + /// + /// When > 0, filenames follow `_` starting at + /// `_000`. E.g. `out.avro` → `out_000.avro`, `out_001.avro`, ... + pub max_points_per_file: usize, +} + +impl AvroWriterOpts { + /// Construct a new opts struct with default values. Equivalent to + /// [`Self::default`]; provided for discoverability alongside the + /// `with_*` builder methods. + pub fn new() -> Self { + Self::default() + } + + pub fn with_max_points_per_batch(mut self, n: usize) -> Self { + self.max_points_per_batch = n; + self + } + + pub fn with_max_batch_delay(mut self, d: Duration) -> Self { + self.max_batch_delay = d; + self + } + + pub fn with_fsync_on_close(mut self, b: bool) -> Self { + self.fsync_on_close = b; + self + } + + pub fn with_max_points_per_file(mut self, n: usize) -> Self { + self.max_points_per_file = n; + self + } +} + +impl Default for AvroWriterOpts { + fn default() -> Self { + Self { + max_points_per_batch: 250_000, + max_batch_delay: Duration::from_millis(100), + fsync_on_close: true, + max_points_per_file: 0, + } + } +} diff --git a/nominal-streaming/src/avro_writer/polars.rs b/nominal-streaming/src/avro_writer/polars.rs new file mode 100644 index 0000000..0c57f30 --- /dev/null +++ b/nominal-streaming/src/avro_writer/polars.rs @@ -0,0 +1,462 @@ +//! Polars DataFrame → [`AvroWriter`] integration. +//! +//! Feature-gated behind the `polars` Cargo feature. Provides the pure-Rust +//! [`AvroWriter::write_dataframe`] method — column-loop, dtype dispatch, +//! row-axis rotation splitting, and null / NaN / Inf handling. +//! +//! Consumes a Rust-native `polars::DataFrame`. If you're binding from Python +//! you'll want to hand the frame across via the Arrow C Data Interface first +//! (the `polars_ffi` module in the `py-nominal-streaming` crate does that, +//! then delegates here). + +use std::collections::HashMap; +use std::sync::atomic::Ordering; +use std::sync::Arc; + +use nominal_api::tonic::google::protobuf::Timestamp; +use nominal_api::tonic::io::nominal::scout::api::proto::points::PointsType; +use nominal_api::tonic::io::nominal::scout::api::proto::DoubleArrayPoint; +use nominal_api::tonic::io::nominal::scout::api::proto::DoublePoint; +use nominal_api::tonic::io::nominal::scout::api::proto::IntegerPoint; +use nominal_api::tonic::io::nominal::scout::api::proto::StringArrayPoint; +use nominal_api::tonic::io::nominal::scout::api::proto::StringPoint; +use nominal_api::tonic::io::nominal::scout::api::proto::StructPoint; +use polars::prelude::*; + +use super::error::AvroWriterError; +use super::writer::AvroWriter; +use crate::types::ChannelDescriptor; +use crate::types::IntoPoints; + +/// Recursively convert a polars `AnyValue` into a `serde_json::Value`. +/// +/// Returns `None` **only** when the input is a true polars null +/// (`AnyValue::Null`). Callers use this `None` as a signal to skip the +/// enclosing row (at the top level) or the enclosing field (inside a +/// struct / list). +/// +/// # Non-finite float handling — read this before editing the Float arms +/// +/// Spec JSON (RFC 8259) has no representation for `NaN`, `+Infinity`, or +/// `-Infinity`. We map all three to JSON `null` rather than to the +/// non-standard literal tokens that `serde_json` refuses to emit anyway. +/// This is **intentional and load-bearing**: +/// +/// - The Nominal Core ingest backend stores the payload in a ClickHouse +/// `JSON(max_dynamic_paths=0)` column. ClickHouse's JSON parser +/// *silently accepts* non-RFC-8259 tokens at insert time but later +/// fails or returns NULL from JSON-extraction functions at query time. +/// Emitting a literal `NaN` token here would therefore produce a +/// silent-data-loss time-bomb: the avro file would ingest cleanly and +/// the row would land in the table, but the downstream chart / query +/// path would break on read with no indication of what went wrong. +/// - Dropping the field entirely (what the previous implementation did +/// via `filter_map`) changes the shape of the JSON object the reader +/// sees and is equally bad. +/// +/// Mapping to JSON `null` is lossy — on read you can't tell a polars +/// null field from a NaN/Inf field — but it is correct end-to-end, the +/// container's shape is preserved, and it round-trips through strict +/// JSON parsers. Do not change this without re-verifying the scout +/// backend's JSON-column behavior in `DirectClickHouseFileWriterV2.java`. +pub fn anyvalue_to_json_value(av: AnyValue<'_>) -> Option { + use serde_json::Value; + + // Normalize the `Struct(idx, arr, fields)` / `String(&str)` / etc. borrowed + // variants to their owned equivalents so one set of match arms covers both. + // `into_static()` is cheap for scalars (just rebinds the variant) and + // materializes owned copies only for structs/strings/binaries where it's + // genuinely needed. + let av = av.into_static(); + + match av { + AnyValue::Null => None, + AnyValue::Boolean(b) => Some(Value::Bool(b)), + AnyValue::Int8(v) => Some(Value::Number(v.into())), + AnyValue::Int16(v) => Some(Value::Number(v.into())), + AnyValue::Int32(v) => Some(Value::Number(v.into())), + AnyValue::Int64(v) => Some(Value::Number(v.into())), + AnyValue::UInt8(v) => Some(Value::Number(v.into())), + AnyValue::UInt16(v) => Some(Value::Number(v.into())), + AnyValue::UInt32(v) => Some(Value::Number(v.into())), + AnyValue::UInt64(v) => Some(Value::Number(v.into())), + // `serde_json::Number::from_f64` returns `None` for NaN / ±Inf. + // We funnel those to JSON `null` to dodge the backend time-bomb — + // see the function-level doc comment above. + AnyValue::Float32(v) => Some( + serde_json::Number::from_f64(v as f64) + .map(Value::Number) + .unwrap_or(Value::Null), + ), + AnyValue::Float64(v) => Some( + serde_json::Number::from_f64(v) + .map(Value::Number) + .unwrap_or(Value::Null), + ), + AnyValue::StringOwned(s) => Some(Value::String(s.as_str().to_string())), + AnyValue::List(series) => { + // Preserve array length: polars-null inner elements become JSON null. + let arr: Vec = series + .iter() + .map(|v| anyvalue_to_json_value(v).unwrap_or(Value::Null)) + .collect(); + Some(Value::Array(arr)) + } + AnyValue::StructOwned(boxed) => { + // StructOwned holds (Vec, Vec). We keep every + // field in the output — polars-null field values become JSON null, + // matching the nullability contract for scalar columns. + let (values, fields) = *boxed; + let obj: serde_json::Map = fields + .iter() + .zip(values.into_iter()) + .map(|(field, val)| { + ( + field.name().to_string(), + anyvalue_to_json_value(val).unwrap_or(Value::Null), + ) + }) + .collect(); + Some(Value::Object(obj)) + } + // Catch-all: convert to string representation for unknown variants + // (dates, durations, binary, etc.) + other => Some(Value::String(format!("{other}"))), + } +} + +/// Convert a top-level `AnyValue` row (typically from a Struct column) to +/// a JSON string. Returns `None` if the value is a polars null — callers +/// treat that as "skip the row entirely". See [`anyvalue_to_json_value`] +/// for the field-level recursion and the non-finite-float encoding +/// contract. +pub fn anyvalue_to_json(av: AnyValue<'_>) -> Option { + anyvalue_to_json_value(av).map(|v| v.to_string()) +} + +/// Build a ChannelDescriptor from a name + optional tags. +fn descriptor_with_tags(name: &str, tags: Option<&HashMap>) -> ChannelDescriptor { + ChannelDescriptor::with_tags( + name.to_string(), + tags.map_or_else(Vec::new, |t| t.clone().into_iter().collect()), + ) +} + +/// Convert an i64 nanoseconds-since-epoch value to a `Timestamp`. +fn timestamp_from_nanos(nanos: i64) -> Timestamp { + let seconds = nanos.div_euclid(1_000_000_000); + let n = nanos.rem_euclid(1_000_000_000) as i32; + Timestamp { seconds, nanos: n } +} + +impl AvroWriter { + /// Write each non-timestamp column of a polars DataFrame as a separate channel. + /// + /// Per-column dispatch: + /// * `Float64` → `Vec` per column + /// * `Int64` → `Vec` per column + /// * `String` → `Vec` per column + /// * `List(*)` / `Array(*, _)` with String inner → `Vec` per row + /// * `List(*)` / `Array(*, _)` with numeric inner → `Vec` per row (cast to f64) + /// * `Struct(…)` → `Vec` per row (JSON-serialized via serde_json) + /// + /// # Null / NaN / Infinity handling + /// + /// These semantics are deliberate — they match how the Nominal Core + /// ingest backend (`DirectClickHouseFileWriterV2.java`) processes the + /// Avro file it receives. Do not change them without re-checking the + /// backend: the client-side and server-side behaviors are coupled. + /// + /// **Polars null → row skipped** (no avro record emitted, timestamp + /// not paired with a sentinel). The backend drops null-Tuple rows at + /// insert time via a `WHERE Tuple IS NOT NULL` filter, so emitting a + /// null-valued record would produce nothing downstream anyway — we + /// skip client-side to save the wire round-trip and to keep scalar + /// columns free of sentinel confusion (a real 0.0 vs a "was null"). + /// + /// **Float64 NaN / ±Infinity → preserved**. These are real IEEE-754 + /// values, not nulls. They round-trip unchanged through the avro + /// `double` branch and ClickHouse's `Variant(Float64, ...)` column, + /// which stores the IEEE-754 bit patterns as-is. + /// + /// **Complex columns (`Struct`, `List`, `Array`):** a polars-null + /// *row* is skipped (matching the backend's row-level Tuple filter). + /// Inside a struct or list, polars-null *elements* become JSON `null` + /// so the containing object/array structure survives to the reader. + /// Non-finite floats inside a struct field also encode as JSON `null` + /// — **not** as literal `NaN` / `Infinity` tokens. See + /// [`anyvalue_to_json_value`] for the rationale. + /// + /// The timestamp column itself must be `Int64` (nanoseconds since + /// epoch) and must not contain nulls; a null timestamp is a hard + /// error. + pub fn write_dataframe( + &self, + df: &DataFrame, + timestamp_column: &str, + tags: Option>, + ) -> Result<(), Arc> { + let stats = self.inner.stats.clone(); + let t_total = std::time::Instant::now(); + + // --- Extract timestamp column (must be Int64 nanoseconds) --- + let t0 = std::time::Instant::now(); + let ts_col = df + .column(timestamp_column) + .map_err(|e| Arc::new(AvroWriterError::Consumer(format!("timestamp column: {e}"))))?; + let ts_i64 = ts_col.i64().map_err(|_| { + Arc::new(AvroWriterError::Consumer(format!( + "timestamp column {:?} must be Int64, got {:?}", + timestamp_column, + ts_col.dtype() + ))) + })?; + let timestamps_raw: Vec = ts_i64 + .into_iter() + .map(|v| { + v.ok_or_else(|| { + Arc::new(AvroWriterError::Consumer( + "null in timestamp column".to_string(), + )) + }) + }) + .collect::, _>>()?; + let timestamps: Vec = timestamps_raw + .iter() + .map(|&ns| timestamp_from_nanos(ns)) + .collect(); + stats + .extract_ts_ns + .fetch_add(t0.elapsed().as_nanos() as u64, Ordering::Relaxed); + + // --- Column loop: accumulate every (descriptor, points) pair and + // hand the whole frame off to `write_batch` in one shot. This + // amortizes the stream + buffer-lock across all channels. + let t_cols = std::time::Instant::now(); + let mut batch: Vec<(ChannelDescriptor, PointsType)> = + Vec::with_capacity(df.get_columns().len()); + + for col in df.get_columns() { + let name = col.name().as_str(); + if name == timestamp_column { + continue; + } + let desc = descriptor_with_tags(name, tags.as_ref()); + + match col.dtype() { + // Null handling (applies to every scalar arm below): + // - Polars null → row **skipped** (no record emitted; the + // timestamp is dropped for this channel, not paired with + // a sentinel value). This aligns with the backend's + // row-level `WHERE Tuple IS NOT NULL` filter — a + // null-valued record we might emit would be dropped at + // insert anyway, so we skip client-side to save the + // round-trip and keep scalar columns free of sentinel + // ambiguity (a real 0.0 vs "was null"). + // - Float64 NaN / ±Inf → preserved as-is (IEEE-754 + // round-trips through avro's double-branch encoding and + // ClickHouse's Variant(Float64, ...) column). + // + // See the `write_dataframe` docstring for the full contract + // and the backend coupling — do not change without checking + // `DirectClickHouseFileWriterV2.java` in the scout repo. + DataType::Float64 => { + let ch = col.f64().expect("dtype checked above"); + let pts: Vec = timestamps + .iter() + .zip(ch.into_iter()) + .filter_map(|(&t, v)| { + v.map(|value| DoublePoint { + timestamp: Some(t), + value, + }) + }) + .collect(); + if !pts.is_empty() { + batch.push((desc, pts.into_points())); + } + } + DataType::Int64 => { + let ch = col.i64().expect("dtype checked above"); + let pts: Vec = timestamps + .iter() + .zip(ch.into_iter()) + .filter_map(|(&t, v)| { + v.map(|value| IntegerPoint { + timestamp: Some(t), + value, + }) + }) + .collect(); + if !pts.is_empty() { + batch.push((desc, pts.into_points())); + } + } + DataType::String => { + let ch = col.str().expect("dtype checked above"); + let pts: Vec = timestamps + .iter() + .zip(ch.into_iter()) + .filter_map(|(&t, v)| { + v.map(|value| StringPoint { + timestamp: Some(t), + value: value.to_string(), + }) + }) + .collect(); + if !pts.is_empty() { + batch.push((desc, pts.into_points())); + } + } + DataType::Struct(_) => { + // Iterate via as_materialized_series().iter() → AnyValue per row. + // + // `anyvalue_to_json` returns `None` exactly when the row + // is a polars null. We drop it — no record emitted for + // this timestamp. Symmetric with the scalar-null case + // above and with the backend's row-level Tuple filter. + // The non-finite-float → JSON-null mapping happens + // inside `anyvalue_to_json_value`; see its doc comment + // for why that (rather than a literal NaN token) is + // the correct encoding. + let series = col.as_materialized_series(); + let mut pts: Vec = Vec::with_capacity(series.len()); + for (i, row_value) in series.iter().enumerate() { + if let Some(json_str) = anyvalue_to_json(row_value) { + pts.push(StructPoint { + timestamp: Some(timestamps[i]), + json_string: json_str, + }); + } + } + if !pts.is_empty() { + batch.push((desc, pts.into_points())); + } + } + DataType::List(inner) => { + if matches!(**inner, DataType::String) { + // List(String) → StringArrayPoint per row + let ch = col.list().expect("dtype checked above"); + let mut pts: Vec = Vec::with_capacity(ch.len()); + for (i, row) in ch.into_iter().enumerate() { + if let Some(inner_series) = row { + let str_ch = inner_series.str().expect("inner is utf8/string"); + let v: Vec = str_ch + .into_iter() + .map(|s| s.unwrap_or("").to_string()) + .collect(); + pts.push(StringArrayPoint { + timestamp: Some(timestamps[i]), + value: v, + }); + } + // Polars-null row → no record emitted for this + // timestamp. Symmetric with the scalar-null + // case and the backend's row-level Tuple + // filter; see `write_dataframe` docstring. + } + if !pts.is_empty() { + batch.push((desc, pts.into_points())); + } + } else { + // List(numeric) → DoubleArrayPoint per row, cast inner to f64 + let ch = col.list().expect("dtype checked above"); + let mut pts: Vec = Vec::with_capacity(ch.len()); + for (i, row) in ch.into_iter().enumerate() { + if let Some(inner_series) = row { + let f_series = + inner_series.cast(&DataType::Float64).map_err(|e| { + Arc::new(AvroWriterError::Consumer(format!( + "cast inner list to Float64 failed: {e}" + ))) + })?; + let f_ch = f_series.f64().expect("cast succeeded"); + let v: Vec = + f_ch.into_iter().map(|x| x.unwrap_or(f64::NAN)).collect(); + pts.push(DoubleArrayPoint { + timestamp: Some(timestamps[i]), + value: v, + }); + } + // Polars-null row → no record emitted for this + // timestamp. Symmetric with the scalar-null + // case and the backend's row-level Tuple + // filter; see `write_dataframe` docstring. + } + if !pts.is_empty() { + batch.push((desc, pts.into_points())); + } + } + } + DataType::Array(inner, _) => { + if matches!(**inner, DataType::String) { + // Array(String, _) → StringArrayPoint per row + let ch = col.array().expect("dtype checked above"); + let mut pts: Vec = Vec::with_capacity(ch.len()); + for (i, row) in ch.into_iter().enumerate() { + if let Some(inner_series) = row { + let str_ch = inner_series.str().expect("inner is utf8/string"); + let v: Vec = str_ch + .into_iter() + .map(|s| s.unwrap_or("").to_string()) + .collect(); + pts.push(StringArrayPoint { + timestamp: Some(timestamps[i]), + value: v, + }); + } + } + if !pts.is_empty() { + batch.push((desc, pts.into_points())); + } + } else { + // Array(numeric, _) → DoubleArrayPoint per row + let ch = col.array().expect("dtype checked above"); + let mut pts: Vec = Vec::with_capacity(ch.len()); + for (i, row) in ch.into_iter().enumerate() { + if let Some(inner_series) = row { + let f_series = + inner_series.cast(&DataType::Float64).map_err(|e| { + Arc::new(AvroWriterError::Consumer(format!( + "cast inner array to Float64 failed: {e}" + ))) + })?; + let f_ch = f_series.f64().expect("cast succeeded"); + let v: Vec = + f_ch.into_iter().map(|x| x.unwrap_or(f64::NAN)).collect(); + pts.push(DoubleArrayPoint { + timestamp: Some(timestamps[i]), + value: v, + }); + } + } + if !pts.is_empty() { + batch.push((desc, pts.into_points())); + } + } + } + other => { + return Err(Arc::new(AvroWriterError::Consumer(format!( + "unsupported dtype for column {:?}: {:?}", + name, other + )))); + } + } + } + // Close the column-build phase *before* handing off to the stream; + // `enqueue_batch_ns` is measured separately inside `write_batch`. + stats + .column_build_ns + .fetch_add(t_cols.elapsed().as_nanos() as u64, Ordering::Relaxed); + + if !batch.is_empty() { + self.write_batch(batch)?; + } + + stats.write_dataframe_calls.fetch_add(1, Ordering::Relaxed); + stats + .write_dataframe_total_ns + .fetch_add(t_total.elapsed().as_nanos() as u64, Ordering::Relaxed); + Ok(()) + } +} diff --git a/nominal-streaming/src/avro_writer/state.rs b/nominal-streaming/src/avro_writer/state.rs new file mode 100644 index 0000000..876a12c --- /dev/null +++ b/nominal-streaming/src/avro_writer/state.rs @@ -0,0 +1,74 @@ +use std::path::PathBuf; +use std::sync::atomic::AtomicU64; +use std::sync::atomic::AtomicUsize; +use std::sync::Arc; +use std::sync::OnceLock; + +use super::error::AvroWriterError; +use super::opts::AvroWriterOpts; +use super::stats::PipelineStats; +use crate::stream::NominalDatasetStream; + +/// Shared state behind [`super::AvroWriter`]. Held as `Arc` +/// so clones are cheap and `Drop` runs exactly when the last reference goes. +pub(super) struct AvroWriterInner { + /// Template path. When `max_points_per_file == 0`, this IS the output path. + /// When rotating, derived paths follow `_`. + pub(super) base_path: PathBuf, + /// The file path currently being written to. Updated on each rotation. + pub(super) current_path: parking_lot::Mutex, + /// Index of the current file (0-based). Incremented on each rotation. + pub(super) file_index: AtomicUsize, + /// Paths of all fully-closed (finalized) files, in order. + pub(super) finalized_paths: parking_lot::Mutex>, + /// The live stream. `Option` so `close()` / `rotate()` can `.take()` it + /// exclusively and drop it (drop drains + joins the stream's internal + /// threads). The outer Mutex guards the take; reads via `as_ref()` from + /// `write` also hold it. + pub(super) stream: parking_lot::Mutex>, + /// Shared with `ErrorLatchingConsumer` via `Arc` so the consumer can latch + /// from the stream's dispatcher thread without a backchannel to the writer. + /// Layering: outer `Arc` shares the cell across writer + consumer; + /// [`OnceLock`] gives first-wins set semantics with lock-free reads; inner + /// `Arc` lets [`AvroWriter`] callers clone the latched error cheaply out of + /// the cell for return from every subsequent method call. + /// + /// [`AvroWriter`]: super::AvroWriter + pub(super) first_error: Arc>>, + /// Points accepted into the current file since the last rotation (or since + /// construction for the first file). Resets to 0 on each rotation. + pub(super) points_in_current: AtomicU64, + /// Cumulative points accepted across all files (never resets). Returned by + /// [`super::AvroWriter::points_accepted`]. + pub(super) total_points_accepted: AtomicU64, + /// Caches the result of the first `close()` call so subsequent calls return + /// the same `Arc`'d error or `Vec` without repeating the shutdown sequence. + pub(super) close_result: OnceLock, Arc>>, + /// Serializes `close()` (write lock) against in-flight `write` calls + /// (read lock). Prevents a write that has already checked `close_result` + /// from enqueuing into a stream that a concurrent `close()` is about to drop. + pub(super) close_lock: parking_lot::RwLock<()>, + /// Serializes concurrent `Drop` calls across `AvroWriter` clones. + /// Makes the `Arc::strong_count` check + "last clone decides" path atomic + /// so two simultaneous drops don't both believe they are last. + pub(super) drop_mutex: parking_lot::Mutex<()>, + /// The opts the writer was constructed with. Kept so `rotate()` can spin + /// up a fresh stream with the same batching / fsync settings; surfaces + /// `fsync_on_close` / `max_points_per_file` to the hot paths. + pub(super) opts: AvroWriterOpts, + /// Pipeline timing stats. Shared with `ErrorLatchingConsumer` via `Arc`. + pub(super) stats: Arc, +} + +impl AvroWriterInner { + /// Snapshot the latched error, if any. Returns a cheap clone of the `Arc`. + pub(super) fn current_error(&self) -> Option> { + self.first_error.get().cloned() + } + + /// Test-only: forcibly latch an error for testing error-surfacing paths. + #[cfg(test)] + pub(crate) fn __test_latch(&self, err: AvroWriterError) { + let _ = self.first_error.set(Arc::new(err)); + } +} diff --git a/nominal-streaming/src/avro_writer/stats.rs b/nominal-streaming/src/avro_writer/stats.rs new file mode 100644 index 0000000..8c8bc0e --- /dev/null +++ b/nominal-streaming/src/avro_writer/stats.rs @@ -0,0 +1,61 @@ +use std::sync::atomic::AtomicU64; + +/// Atomic counters used to attribute wall-clock time across the pipeline. +/// +/// The `write_dataframe_*`, `df_handoff_ns`, `extract_ts_ns`, and +/// `column_build_ns` fields are populated only by the +/// `write_dataframe` method (available when the `polars` feature is +/// enabled). The remaining fields (`enqueue_batch_ns`, +/// `consumer_consume_ns`, `consumer_consume_calls`) are populated on every +/// write path regardless of feature flags. +/// +/// Shared via `Arc` between [`super::AvroWriter`] and the consumer wrapper so +/// both sides accumulate into the same totals. +/// +/// Producer time is wall-clock on the caller thread; consumer time is +/// wall-clock on the single dispatcher thread and runs in parallel with the +/// producer. If `consumer_consume_ns ≈ wall_total`, the consumer is +/// saturated (bottleneck). If producer CPU fields ≈ `wall_total`, the +/// producer is the bottleneck. +/// +/// Read each field with `.load(Ordering::Relaxed)`. +#[derive(Default, Debug)] +pub struct PipelineStats { + // ── Producer CPU phases (populated by write_dataframe only) ── + /// `py_df_to_rust` via Arrow C Data Interface (per-column rechunk + + /// import). Written by the Python facade's `write_dataframe`, not by + /// the pure-Rust core — a pure-Rust caller that passes a `polars::DataFrame` + /// in-process pays no FFI cost and this counter stays at 0. + pub df_handoff_ns: AtomicU64, + /// Timestamp column extraction + `parse_timestamp`. + pub extract_ts_ns: AtomicU64, + /// Column-loop wall: polars-chunk extraction + point-struct building + + /// `.into_points()` type-erasure + batch accumulation, summed across all + /// non-timestamp columns of every dataframe. + pub column_build_ns: AtomicU64, + + // ── Producer wall time into the stream (every write path) ── + /// Wall time producer spends handing a batch off to the underlying + /// `NominalDatasetStream::enqueue_batch`. Includes buffer-lock + /// acquisition, extends, *and* time blocked waiting for buffer capacity + /// (when both primary and secondary buffers are full → the consumer is + /// too slow). Named after the stream-level method, not the writer's + /// public `write_batch`, because this measures just the hand-off portion + /// inside the critical section. + pub enqueue_batch_ns: AtomicU64, + + // ── Consumer-side (runs on the dispatcher thread, in parallel) ── + /// Wall time spent inside `AvroFileConsumer::consume` per request (avro + /// encode + file write). Summed across all calls on the single dispatcher + /// thread. Compare to wall-clock to see if consumer is saturated. + pub consumer_consume_ns: AtomicU64, + /// Number of `consume` calls — i.e., `WriteRequest`s dispatched. + pub consumer_consume_calls: AtomicU64, + + // ── Meta (write_dataframe only) ── + /// Number of `write_dataframe` invocations so far. + pub write_dataframe_calls: AtomicU64, + /// Total wall time across all `write_dataframe` invocations (sum of + /// producer CPU + `enqueue_batch_ns` for this method only). + pub write_dataframe_total_ns: AtomicU64, +} diff --git a/nominal-streaming/src/avro_writer/writer.rs b/nominal-streaming/src/avro_writer/writer.rs new file mode 100644 index 0000000..b4bcf62 --- /dev/null +++ b/nominal-streaming/src/avro_writer/writer.rs @@ -0,0 +1,1266 @@ +use std::path::PathBuf; +use std::sync::atomic::AtomicU64; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; +use std::sync::Arc; +use std::sync::OnceLock; + +use nominal_api::tonic::io::nominal::scout::api::proto::points::PointsType; +use tracing::debug; +use tracing::error; +use tracing::info; + +use super::error::AvroWriterError; +use super::helpers::ensure_parent_and_truncate; +use super::helpers::open_error_latching_consumer; +use super::helpers::open_stream; +use super::helpers::path_for_index; +use super::opts::AvroWriterOpts; +use super::state::AvroWriterInner; +use super::stats::PipelineStats; +use crate::stream::points_len; +use crate::types::ChannelDescriptor; +use crate::types::IntoPoints; + +/// Purpose-built avro file writer for the Nominal schema. +/// +/// Thin wrapper over [`crate::stream::NominalDatasetStream`] that drives the +/// same battle-tested pipeline (primary/secondary buffer + batch_processor + +/// request_dispatcher) with an [`crate::consumer::AvroFileConsumer`] as the +/// sink. Producers call [`Self::write`] / [`Self::write_batch`] from any +/// thread (it's `Send + Sync`); ordering across threads is unspecified. +/// Errors latch on first failure and surface from every subsequent operation. +/// +/// # Concurrency +/// +/// - Multi-producer: any number of threads may call `write` / `write_batch` +/// concurrently. +/// - Ordering: not guaranteed across threads. Within a single producer, +/// order is preserved. +/// - Backpressure: writes block when the stream's internal buffers are full. +/// +/// # Errors +/// +/// Background encoder errors (I/O, avro encoding, fsync) are latched on first +/// occurrence. Every subsequent `write` / `flush` / `sync` / `close` returns +/// the same latched error. The distinct `SendAfterClose` error is returned +/// only when a write is attempted after `close()` completed. +/// +/// # Note +/// +/// - [`Self::points_accepted`] counts successful enqueues, not on-disk bytes. +/// Call [`Self::close`] to ensure all data is on disk. +/// - Latched errors are **permanent**: there is no reset mechanism. Once an +/// error is latched, every subsequent method call on every clone of this +/// writer returns the same `Arc`. +/// - `Drop` performs a best-effort close; errors are logged via +/// `tracing::error!` but cannot propagate. For error visibility, +/// call [`Self::close`] explicitly. +/// - `flush()` and `sync()` are best-effort: the underlying stream does not +/// expose a force-drain hook, so `flush()` is a no-op aside from the latched +/// error check, and `sync()` fsyncs whatever data has already dispatched. +/// For strong durability, call `close()`. +#[derive(Clone)] +pub struct AvroWriter { + pub(super) inner: Arc, +} + +impl AvroWriter { + /// Construct a writer that will write to `path`. + /// + /// If a file already exists at `path`, it is **truncated** before writing + /// begins. This differs from + /// [`crate::consumer::AvroFileConsumer::new_with_full_path`], which opens + /// without truncation. The `AvroWriter` contract is "write a fresh avro + /// file"; leaving stale tail bytes from a prior run would corrupt the + /// stream. + /// + /// The parent directory is created if it does not exist. + /// + /// Opens the avro file and starts the underlying stream immediately. + /// + /// # Errors + /// + /// Returns `io::Error` if the parent directory cannot be created, the + /// file cannot be truncated, or the underlying consumer cannot open the + /// file for writing. + pub fn new(path: impl Into, opts: AvroWriterOpts) -> std::io::Result { + let base_path: PathBuf = path.into(); + // Decide the initial file path based on rotation mode. + let current_path = if opts.max_points_per_file == 0 { + base_path.clone() + } else { + path_for_index(&base_path, 0) + }; + + // Ensure parent dir exists and truncate any pre-existing file. + // AvroWriter writes a fresh avro stream; leaving stale tail bytes + // from a prior run would corrupt the file for readers. + ensure_parent_and_truncate(¤t_path)?; + + // Build the avro consumer, wrap it in an error-latching layer. + let first_error: Arc>> = Arc::new(OnceLock::new()); + let stats: Arc = Arc::new(PipelineStats::default()); + let wrapped = + open_error_latching_consumer(¤t_path, first_error.clone(), stats.clone())?; + let stream = open_stream(wrapped, opts.max_points_per_batch, opts.max_batch_delay); + + let inner = Arc::new(AvroWriterInner { + base_path: base_path.clone(), + current_path: parking_lot::Mutex::new(current_path.clone()), + file_index: AtomicUsize::new(0), + finalized_paths: parking_lot::Mutex::new(Vec::new()), + stream: parking_lot::Mutex::new(Some(stream)), + first_error, + points_in_current: AtomicU64::new(0), + total_points_accepted: AtomicU64::new(0), + close_result: OnceLock::new(), + close_lock: parking_lot::RwLock::new(()), + drop_mutex: parking_lot::Mutex::new(()), + opts: opts.clone(), + stats, + }); + + info!( + "AvroWriter opened: path={} opts={:?}", + current_path.display(), + opts + ); + + Ok(Self { inner }) + } + + /// Test-only constructor that injects a custom `WriteRequestConsumer` + /// instead of the default `AvroFileConsumer`. The consumer is wrapped + /// in the standard `ErrorLatchingConsumer` so error-latching semantics + /// are identical. Used to exercise end-to-end error propagation + /// without depending on filesystem failures. + #[cfg(test)] + pub(crate) fn new_with_consumer_for_testing( + consumer: C, + path: PathBuf, + opts: AvroWriterOpts, + ) -> Self + where + C: crate::consumer::WriteRequestConsumer + 'static, + { + use super::consumer::ErrorLatchingConsumer; + + let first_error: Arc>> = Arc::new(OnceLock::new()); + let stats: Arc = Arc::new(PipelineStats::default()); + let wrapped = ErrorLatchingConsumer { + inner: consumer, + first_error: first_error.clone(), + stats: stats.clone(), + }; + let stream = open_stream(wrapped, opts.max_points_per_batch, opts.max_batch_delay); + + let current_path = path.clone(); + let inner = Arc::new(AvroWriterInner { + base_path: path.clone(), + current_path: parking_lot::Mutex::new(current_path), + file_index: AtomicUsize::new(0), + finalized_paths: parking_lot::Mutex::new(Vec::new()), + stream: parking_lot::Mutex::new(Some(stream)), + first_error, + points_in_current: AtomicU64::new(0), + total_points_accepted: AtomicU64::new(0), + close_result: OnceLock::new(), + close_lock: parking_lot::RwLock::new(()), + drop_mutex: parking_lot::Mutex::new(()), + // Force rotation off for the test consumer so we don't try to + // reopen a new `AvroFileConsumer` after rotation — the test's + // injected consumer is single-shot. + opts: AvroWriterOpts { + max_points_per_file: 0, + ..opts.clone() + }, + stats, + }); + + Self { inner } + } + + /// The file path currently being written to. + /// + /// When `max_points_per_file == 0`, this is the path given to the constructor. + /// When rotating, this is the path of the currently-open file. + pub fn path(&self) -> PathBuf { + self.inner.current_path.lock().clone() + } + + /// All finalized (fully-closed) file paths, in order. + /// + /// Empty until at least one rotation has occurred (or `close()` is called). + pub fn finalized_paths(&self) -> Vec { + self.inner.finalized_paths.lock().clone() + } + + /// Every file path this writer has opened, in order. + /// + /// Includes the currently-open file while the writer is live; after + /// `close()`, identical to `finalized_paths()`. + pub fn written_files(&self) -> Vec { + let mut v = self.inner.finalized_paths.lock().clone(); + if self.inner.close_result.get().is_none() { + v.push(self.inner.current_path.lock().clone()); + } + v + } + + /// Total points successfully accepted by the writer so far, across all files. + /// + /// Does not include points that failed (latched error, `SendAfterClose`). + /// Does not reflect on-disk state — use [`Self::close`] to ensure durability. + pub fn points_accepted(&self) -> u64 { + self.inner.total_points_accepted.load(Ordering::Relaxed) + } + + /// Typed reference to the pipeline timing counters. + /// Read fields via `.load(Ordering::Relaxed)` on each `AtomicU64`. + pub fn stats(&self) -> &PipelineStats { + &self.inner.stats + } + + /// Drop the current stream (which drains + closes its avro file via its + /// own `Drop`), then optionally fsync the just-finished file path per + /// `opts.fsync_on_close`. Returns the path of the just-finished file on + /// success; on fsync failure, latches the `Io` error and returns it. + /// + /// If an error was already latched during the drain (the consumer + /// propagated it from a failed `consume`), fsync is skipped — the file + /// may be in a half-written state and fsyncing wouldn't produce a + /// meaningful durability guarantee. The caller still sees the latched + /// error via [`AvroWriterInner::current_error`]. + /// + /// Shared by `rotate`, `close`, and `Drop` — each of which follows it up + /// with their own "what next" (open a new shard / record finalized path + /// / log & exit). + /// + /// Must be called without holding `stream.lock()`. + fn take_and_fsync_stream(&self) -> Result> { + // `drop` on the taken stream drains pending batches and joins the + // dispatcher threads — after this line the consumer has processed + // every request the writer has dispatched. + let _ = self.inner.stream.lock().take(); + let current = self.inner.current_path.lock().clone(); + if self.inner.opts.fsync_on_close && self.inner.current_error().is_none() { + if let Err(e) = std::fs::File::open(¤t).and_then(|f| f.sync_all()) { + let wrapped = Arc::new(AvroWriterError::Io(std::io::Error::new( + e.kind(), + e.to_string(), + ))); + let _ = self.inner.first_error.set(wrapped.clone()); + return Err(wrapped); + } + } + Ok(current) + } + + /// Rotate to a new file: finalize the current shard via + /// [`Self::take_and_fsync_stream`], record it in `finalized_paths`, then + /// open a fresh stream writing to the next numbered path. + /// + /// Must be called without holding `stream.lock()`. + fn rotate(&self) -> Result<(), Arc> { + let finalized = self.take_and_fsync_stream()?; + self.inner.finalized_paths.lock().push(finalized); + + // Derive and open the next file. + let new_index = self.inner.file_index.fetch_add(1, Ordering::Relaxed) + 1; + let new_path = path_for_index(&self.inner.base_path, new_index); + ensure_parent_and_truncate(&new_path).map_err(|e| { + let wrapped = Arc::new(AvroWriterError::from(e)); + let _ = self.inner.first_error.set(wrapped.clone()); + wrapped + })?; + *self.inner.current_path.lock() = new_path.clone(); + + let wrapped = open_error_latching_consumer( + &new_path, + self.inner.first_error.clone(), + self.inner.stats.clone(), + ) + .map_err(|e| { + let wrapped = Arc::new(AvroWriterError::from(e)); + let _ = self.inner.first_error.set(wrapped.clone()); + wrapped + })?; + let new_stream = open_stream( + wrapped, + self.inner.opts.max_points_per_batch, + self.inner.opts.max_batch_delay, + ); + *self.inner.stream.lock() = Some(new_stream); + self.inner.points_in_current.store(0, Ordering::Relaxed); + + debug!("AvroWriter: rotated to {}", new_path.display()); + Ok(()) + } + + /// Write points for a single channel. Generic over any `Vec` that + /// can be converted into [`PointsType`] via + /// [`crate::types::IntoPoints`] (e.g. `Vec`, + /// `Vec`, ...). + /// + /// Rotation-aware: if `max_points_per_file > 0` and the batch would + /// overshoot the remaining capacity of the current file, the batch is + /// split along the point axis so each shard stays at or under the + /// threshold. + /// + /// # Errors + /// + /// - Returns any latched encoder error if one was recorded before or + /// during this call. + /// - Returns [`AvroWriterError::SendAfterClose`] if called after close. + pub fn write( + &self, + channel: &ChannelDescriptor, + points: Vec, + ) -> Result<(), Arc> + where + Vec: IntoPoints, + T: Clone, + { + self.write_chunked(channel.clone(), points) + } + + /// Rotation-aware chunking for single-channel batches, implemented as a + /// loop over [`Self::write_batch`]. A large single-channel batch that + /// would overshoot the current file's remaining capacity gets split + /// along the point axis into chunks that each fit. Without this helper + /// the file would overshoot `max_points_per_file` by up to one batch's + /// worth. + pub(super) fn write_chunked( + &self, + descriptor: ChannelDescriptor, + points: Vec, + ) -> Result<(), Arc> + where + Vec: IntoPoints, + { + if self.inner.opts.max_points_per_file == 0 { + return self.write_batch(vec![(descriptor, points.into_points())]); + } + + let max = self.inner.opts.max_points_per_file as u64; + let n = points.len(); + let mut i = 0; + while i < n { + let current = self.inner.points_in_current.load(Ordering::Relaxed); + let remaining = max.saturating_sub(current); + let chunk_size = if remaining == 0 { + max as usize + } else { + remaining as usize + }; + let chunk_end = (i + chunk_size).min(n); + let chunk = points[i..chunk_end].to_vec(); + self.write_batch(vec![(descriptor.clone(), chunk.into_points())])?; + i = chunk_end; + } + Ok(()) + } + + /// Multi-channel batch write. Each entry lands in the same critical + /// section, amortizing the buffer-lock across all channels. + /// + /// Takes the outer `stream` lock and the inner buffer lock each once for + /// the whole batch, amortizing lock-acquisition cost across all channels. + /// Use this when pushing a wide frame (many channels sharing a timestamp + /// span) — it is the path `write_dataframe` uses internally. + /// + /// Rotation is handled rotate-before: if the current file is already at + /// or beyond the limit, a rotation happens before the batch is enqueued. + /// A very large batch may still push the file slightly past + /// `max_points_per_file` — consistent with single-call `write` behavior. + /// + /// # Errors + /// + /// - Returns any latched encoder error if one was recorded before or + /// during this call. + /// - Returns [`AvroWriterError::SendAfterClose`] if called after close. + pub fn write_batch( + &self, + batch: Vec<(ChannelDescriptor, PointsType)>, + ) -> Result<(), Arc> { + if let Some(err) = self.inner.current_error() { + return Err(err); + } + let _close_guard = self.inner.close_lock.read(); + if self.inner.close_result.get().is_some() { + return Err(Arc::new(AvroWriterError::SendAfterClose)); + } + + let total: u64 = batch.iter().map(|(_, p)| points_len(p) as u64).sum(); + if total == 0 { + return Ok(()); + } + + if self.inner.opts.max_points_per_file > 0 + && self.inner.points_in_current.load(Ordering::Relaxed) + >= self.inner.opts.max_points_per_file as u64 + { + self.rotate()?; + } + + // Wall-time instrumentation: time the actual stream interaction + // (lock + enqueue_batch). Any blocking in `when_capacity` shows up here. + let t0 = std::time::Instant::now(); + let stream_guard = self.inner.stream.lock(); + let stream = stream_guard + .as_ref() + .ok_or_else(|| Arc::new(AvroWriterError::SendAfterClose))?; + + stream.enqueue_batch(batch); + drop(stream_guard); + self.inner + .stats + .enqueue_batch_ns + .fetch_add(t0.elapsed().as_nanos() as u64, Ordering::Relaxed); + + self.inner + .points_in_current + .fetch_add(total, Ordering::Relaxed); + self.inner + .total_points_accepted + .fetch_add(total, Ordering::Relaxed); + Ok(()) + } + + /// Best-effort: check for latched errors. Points that the stream has already + /// dispatched to the consumer will be on disk; points still in the stream's + /// internal buffer may not be. For strong durability, call `close()`. + /// + /// # Errors + /// + /// Returns any latched encoder error ([`AvroWriterError::Io`], + /// [`AvroWriterError::Avro`], or [`AvroWriterError::Consumer`]) if one was + /// recorded before or during this call. Once an error is latched it is + /// permanent; all subsequent operations return the same error. + pub fn flush(&self) -> Result<(), Arc> { + // Best-effort: points that the stream has already dispatched to the + // consumer will be on disk; points still in the stream's internal + // buffer may not be. For strong durability, call `close()`. + if let Some(err) = self.inner.current_error() { + return Err(err); + } + Ok(()) + } + + /// [`Self::flush`] plus a best-effort `File::sync_all()` on data that has + /// already been dispatched. + /// + /// # Errors + /// + /// Returns any latched encoder error. The `fsync` failure itself is reported + /// as [`AvroWriterError::Io`] and is also latched, so subsequent calls will + /// return the same error. + pub fn sync(&self) -> Result<(), Arc> { + // See `flush()`. On a best-effort basis, fsync the file path — + // anything already dispatched is now durable. + if let Some(err) = self.inner.current_error() { + return Err(err); + } + let current = self.inner.current_path.lock().clone(); + if let Err(e) = std::fs::File::open(¤t).and_then(|f| f.sync_all()) { + let wrapped = Arc::new(AvroWriterError::from(e)); + let _ = self.inner.first_error.set(wrapped.clone()); + return Err(wrapped); + } + debug!("AvroWriter sync: fsync complete path={}", current.display()); + Ok(()) + } + + /// Gracefully shut down the encoder and return the list of all written files. + /// + /// Idempotent: the first call performs the shutdown and caches the result; + /// subsequent calls return the cached result (same `Arc` on error). Once + /// this returns `Ok`, any further write calls return + /// [`AvroWriterError::SendAfterClose`]. + /// + /// When `max_points_per_file == 0` (no rotation), returns a 1-element + /// `Vec` containing the single output path. When rotating, returns all + /// files written, in order. + /// + /// # Errors + /// + /// Returns any latched encoder error, or an error from the shutdown path + /// (final flush, fsync), whichever came first. + pub fn close(&self) -> Result, Arc> { + if let Some(cached) = self.inner.close_result.get() { + return cached.clone(); + } + + let _close_guard = self.inner.close_lock.write(); + // Re-check after acquiring the lock — another thread may have closed + // while we were waiting. + if let Some(cached) = self.inner.close_result.get() { + return cached.clone(); + } + + info!( + "AvroWriter close: path={} total_points_accepted={}", + self.inner.current_path.lock().display(), + self.inner.total_points_accepted.load(Ordering::Relaxed) + ); + + // Finalize the current shard (drain + fsync). If the consumer also + // latched an error during the drain, `current_error()` reflects it — + // a latched error takes precedence over a successful fsync result, + // matching the pre-refactor behavior. + let result: Result, Arc> = match self.take_and_fsync_stream() + { + Err(e) => Err(e), + Ok(current) => match self.inner.current_error() { + Some(err) => Err(err), + None => { + self.inner.finalized_paths.lock().push(current); + Ok(self.inner.finalized_paths.lock().clone()) + } + }, + }; + + let _ = self.inner.close_result.set(result.clone()); + self.inner.close_result.get().cloned().unwrap_or(result) + } +} + +impl Drop for AvroWriter { + fn drop(&mut self) { + if self.inner.close_result.get().is_some() { + return; + } + let _drop_guard = self.inner.drop_mutex.lock(); + if self.inner.close_result.get().is_some() { + return; + } + // Only the last surviving clone should trigger shutdown. `drop_mutex` + // above makes the count check + the decision to proceed atomic w.r.t. + // other concurrent drops. The `Inner` Arc is held only by external + // `AvroWriter` clones — no background-thread Arcs — so + // `strong_count == 1` reliably means "I am the last." + if Arc::strong_count(&self.inner) > 1 { + return; + } + + let _close_guard = self.inner.close_lock.write(); + + // Best-effort finalize: drain the stream and fsync. Errors go to + // tracing (drop can't propagate Result) but are still latched into + // `first_error` by `take_and_fsync_stream` so any subsequent + // `close()` / `sync()` / `flush()` call on a surviving clone would + // surface them — except there are none, we're the last. + if let Err(e) = self.take_and_fsync_stream() { + error!("AvroWriter drop: finalize failed: {e}"); + } + if let Some(err) = self.inner.current_error() { + error!("AvroWriter drop: latched error: {err}"); + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::AtomicU64; + use std::sync::atomic::Ordering; + use std::sync::Arc; + use std::time::Duration; + + use apache_avro::Reader; + use nominal_api::tonic::google::protobuf::Timestamp; + use nominal_api::tonic::io::nominal::scout::api::proto::DoubleArrayPoint; + use nominal_api::tonic::io::nominal::scout::api::proto::DoublePoint; + use nominal_api::tonic::io::nominal::scout::api::proto::IntegerPoint; + use nominal_api::tonic::io::nominal::scout::api::proto::StringArrayPoint; + use nominal_api::tonic::io::nominal::scout::api::proto::StringPoint; + use nominal_api::tonic::io::nominal::scout::api::proto::StructPoint; + use nominal_api::tonic::io::nominal::scout::api::proto::WriteRequestNominal; + use tempfile::TempDir; + + use super::super::helpers::path_for_index; + use super::*; + use crate::consumer::ConsumerError; + use crate::consumer::ConsumerResult; + use crate::consumer::WriteRequestConsumer; + use crate::types::ChannelDescriptor; + use crate::types::IntoPoints; + + /// A `WriteRequestConsumer` that always fails with a configured + /// `ConsumerError::IoError`, for testing end-to-end error propagation. + #[derive(Debug)] + struct FailingConsumer { + kind: std::io::ErrorKind, + message: String, + } + + impl FailingConsumer { + fn new(kind: std::io::ErrorKind, message: &str) -> Self { + Self { + kind, + message: message.to_string(), + } + } + } + + impl WriteRequestConsumer for FailingConsumer { + fn consume(&self, _request: &WriteRequestNominal) -> ConsumerResult<()> { + Err(ConsumerError::IoError(std::io::Error::new( + self.kind, + self.message.clone(), + ))) + } + } + + fn ts(secs: i64) -> Timestamp { + Timestamp { + seconds: secs, + nanos: 0, + } + } + + fn cd(name: &str) -> ChannelDescriptor { + ChannelDescriptor::new(name) + } + + /// Test-only shim: routes through `write_chunked` (the same code path + /// the public `write` method uses) so tests cover the chunking-across- + /// rotation behavior. + fn enq( + writer: &AvroWriter, + descriptor: ChannelDescriptor, + points: Vec, + ) -> Result<(), Arc> + where + Vec: IntoPoints, + { + writer.write_chunked(descriptor, points) + } + + #[test_log::test] + fn roundtrip_all_value_types() { + let tmp = TempDir::new().unwrap(); + let path = tmp.path().join("all_types.avro"); + let writer = AvroWriter::new(path.clone(), AvroWriterOpts::default()).unwrap(); + + enq( + &writer, + cd("d"), + vec![DoublePoint { + timestamp: Some(ts(1)), + value: 1.5, + }], + ) + .unwrap(); + enq( + &writer, + cd("i"), + vec![IntegerPoint { + timestamp: Some(ts(2)), + value: 42, + }], + ) + .unwrap(); + enq( + &writer, + cd("s"), + vec![StringPoint { + timestamp: Some(ts(3)), + value: "hello".into(), + }], + ) + .unwrap(); + enq( + &writer, + cd("struct"), + vec![StructPoint { + timestamp: Some(ts(4)), + json_string: r#"{"a":1}"#.into(), + }], + ) + .unwrap(); + enq( + &writer, + cd("fa"), + vec![DoubleArrayPoint { + timestamp: Some(ts(5)), + value: vec![1.0, 2.0, 3.0], + }], + ) + .unwrap(); + enq( + &writer, + cd("sa"), + vec![StringArrayPoint { + timestamp: Some(ts(6)), + value: vec!["a".into(), "b".into()], + }], + ) + .unwrap(); + + writer.close().unwrap(); + + let file = std::fs::File::open(&path).unwrap(); + let reader = Reader::new(file).unwrap(); + let records: Vec<_> = reader.collect::, _>>().unwrap(); + assert_eq!(records.len(), 6, "one record per distinct channel"); + } + + #[test_log::test] + fn batch_boundaries_emit_multiple_records() { + let tmp = TempDir::new().unwrap(); + let path = tmp.path().join("batches.avro"); + + // Set batch size to 10; write 35 points via one write call. + // With NominalDatasetStream, record count depends on internal flush + // timing. Assert all 35 points are durably on disk after close. + let writer = AvroWriter::new( + path.clone(), + AvroWriterOpts { + max_points_per_batch: 10, + ..Default::default() + }, + ) + .unwrap(); + + let pts: Vec = (0..35) + .map(|i| IntegerPoint { + timestamp: Some(ts(i)), + value: i, + }) + .collect(); + enq(&writer, cd("x"), pts).unwrap(); + writer.close().unwrap(); + + // All points must still be present on disk. + assert_eq!(count_points_in_file(&path), 35); + } + + #[test_log::test] + fn multi_producer_all_points_land() { + let tmp = TempDir::new().unwrap(); + let path = tmp.path().join("mp.avro"); + let writer = Arc::new(AvroWriter::new(path.clone(), AvroWriterOpts::default()).unwrap()); + + let threads: Vec<_> = (0..8) + .map(|t| { + let w = writer.clone(); + std::thread::spawn(move || { + for i in 0..10_000i64 { + enq( + &w, + cd(&format!("ch{t}")), + vec![IntegerPoint { + timestamp: Some(ts(i)), + value: i, + }], + ) + .unwrap(); + } + }) + }) + .collect(); + for t in threads { + t.join().unwrap(); + } + + assert_eq!(writer.points_accepted(), 80_000); + writer.close().unwrap(); + + let file = std::fs::File::open(&path).unwrap(); + let reader = Reader::new(file).unwrap(); + let record_count = reader.count(); + // 8 channels, at most ceil(10_000 / 250_000) = 1 record per channel + // (since default max_points_per_batch is 250k). Actual count depends + // on delay-based emission interleaving; assert lower bound. + assert!(record_count >= 8, "record_count = {record_count}"); + } + + #[test_log::test] + fn error_latches_and_surfaces() { + let tmp = TempDir::new().unwrap(); + let path = tmp.path().join("latch.avro"); + let writer = AvroWriter::new(path.clone(), AvroWriterOpts::default()).unwrap(); + + // Write one valid point first so we know the writer is functional. + enq( + &writer, + cd("x"), + vec![IntegerPoint { + timestamp: Some(ts(1)), + value: 1, + }], + ) + .unwrap(); + + // Inject a synthetic latched error. + writer + .inner + .__test_latch(AvroWriterError::Consumer("boom".into())); + + // Subsequent write should return the latched error. + let write_err = enq( + &writer, + cd("x"), + vec![IntegerPoint { + timestamp: Some(ts(2)), + value: 2, + }], + ); + assert!( + matches!( + write_err.as_ref().err().map(|e| &**e), + Some(AvroWriterError::Consumer(_)) + ), + "expected latched Consumer error, got {write_err:?}" + ); + + // flush should also surface. + let flush_err = writer.flush(); + assert!( + flush_err.is_err(), + "expected flush to surface latched error" + ); + + // sync too. + let sync_err = writer.sync(); + assert!(sync_err.is_err(), "expected sync to surface latched error"); + + // close returns the latched error. + let close_err = writer.close(); + assert!( + close_err.is_err(), + "expected close to surface latched error" + ); + + // Repeated close returns the same cached error (same Arc). + let close_err2 = writer.close(); + match (close_err.as_ref(), close_err2.as_ref()) { + (Err(a), Err(b)) => assert!(Arc::ptr_eq(a, b), "close() should return same cached Arc"), + _ => panic!("both closes should be Err"), + } + } + + #[test_log::test] + fn drop_without_close_flushes_data() { + let tmp = TempDir::new().unwrap(); + let path = tmp.path().join("drop.avro"); + + { + let writer = AvroWriter::new(path.clone(), AvroWriterOpts::default()).unwrap(); + for i in 0..50i64 { + enq( + &writer, + cd("x"), + vec![IntegerPoint { + timestamp: Some(ts(i)), + value: i, + }], + ) + .unwrap(); + } + let start = std::time::Instant::now(); + drop(writer); + let elapsed = start.elapsed(); + println!("drop elapsed: {elapsed:?}"); + assert!( + elapsed < Duration::from_secs(10), + "drop too slow: {elapsed:?}" + ); + } + + // Contract: drop-without-close must preserve all pushed points. + // Do not assert on record count — the batch_processor may split the + // 50-push loop into multiple records if `max_batch_delay` (100ms) + // elapses mid-loop under CPU contention. That's a timing-dependent + // batching detail, not a data-loss bug. + assert_eq!(count_points_in_file(&path), 50); + } + + #[test_log::test] + fn close_is_idempotent() { + let tmp = TempDir::new().unwrap(); + let path = tmp.path().join("idem.avro"); + let writer = AvroWriter::new(path.clone(), AvroWriterOpts::default()).unwrap(); + enq( + &writer, + cd("x"), + vec![IntegerPoint { + timestamp: Some(ts(1)), + value: 1, + }], + ) + .unwrap(); + + let first = writer.close().unwrap(); + let second = writer.close().unwrap(); + assert_eq!(first, second); + assert_eq!(first, vec![path.clone()]); + + // Post-close write returns SendAfterClose. + let after = enq( + &writer, + cd("x"), + vec![IntegerPoint { + timestamp: Some(ts(2)), + value: 2, + }], + ); + match after { + Err(e) => match &*e { + AvroWriterError::SendAfterClose => {} + other => panic!("expected SendAfterClose, got {other:?}"), + }, + Ok(()) => panic!("write after close should fail"), + } + } + + #[test_log::test] + fn flush_and_sync_both_succeed() { + let tmp = TempDir::new().unwrap(); + let path = tmp.path().join("fs.avro"); + let writer = AvroWriter::new(path.clone(), AvroWriterOpts::default()).unwrap(); + enq( + &writer, + cd("x"), + vec![DoublePoint { + timestamp: Some(ts(1)), + value: 1.0, + }], + ) + .unwrap(); + + // flush() is best-effort with NominalDatasetStream; just verify it returns Ok. + writer.flush().unwrap(); + + // sync() is also best-effort; verify it returns Ok. + writer.sync().unwrap(); + + // close() guarantees all data is durably on disk. + writer.close().unwrap(); + + // Verify the file exists and has content after close. + assert!( + std::fs::metadata(&path).unwrap().len() > 0, + "file should have content after close" + ); + } + + #[test_log::test] + fn roundtrip_single_double() { + let tmp = TempDir::new().unwrap(); + let path = tmp.path().join("out.avro"); + let writer = AvroWriter::new(path.clone(), AvroWriterOpts::default()).unwrap(); + + enq( + &writer, + cd("speed"), + vec![DoublePoint { + timestamp: Some(ts(42)), + value: 3.14, + }], + ) + .unwrap(); + + let closed = writer.close().unwrap(); + assert_eq!(closed, vec![path.clone()]); + + let file = std::fs::File::open(&path).unwrap(); + let reader = Reader::new(file).unwrap(); + let records: Vec<_> = reader.collect::, _>>().unwrap(); + assert_eq!(records.len(), 1, "one record per series"); + } + + fn count_points_in_file(path: &std::path::Path) -> usize { + // A 0-byte file is possible if the writer was closed before any + // data was written (e.g., in the concurrent-race test where the + // writer thread hadn't scheduled yet). Treat as "0 points". + let Ok(file) = std::fs::File::open(path) else { + return 0; + }; + let Ok(reader) = Reader::new(file) else { + return 0; + }; + reader + .filter_map(|r| r.ok()) + .map(|v| match v { + apache_avro::types::Value::Record(fields) => fields + .iter() + .find_map(|(name, val)| match (name.as_str(), val) { + ("timestamps", apache_avro::types::Value::Array(arr)) => Some(arr.len()), + _ => None, + }) + .unwrap_or(0), + _ => 0, + }) + .sum() + } + + #[test_log::test] + fn concurrent_write_and_close_no_silent_drops() { + // Repeat to reliably trigger the race. + for iteration in 0..10 { + let tmp = TempDir::new().unwrap(); + let path = tmp.path().join(format!("race_{iteration}.avro")); + let writer = + Arc::new(AvroWriter::new(path.clone(), AvroWriterOpts::default()).unwrap()); + + let accepted = Arc::new(AtomicU64::new(0)); + let writer_c = writer.clone(); + let accepted_c = accepted.clone(); + let writer_thread = std::thread::spawn(move || { + for i in 0..10_000i64 { + let res = enq( + &writer_c, + cd("x"), + vec![IntegerPoint { + timestamp: Some(ts(i)), + value: i, + }], + ); + if res.is_ok() { + accepted_c.fetch_add(1, Ordering::Relaxed); + } + } + }); + + // Give the writer thread a moment to start producing. + std::thread::sleep(Duration::from_micros(100)); + writer.close().unwrap(); + writer_thread.join().unwrap(); + + let accepted_final = accepted.load(Ordering::Relaxed); + let file_points = count_points_in_file(&path); + + // Every write that returned Ok must be durably in the file. + assert_eq!( + accepted_final as usize, file_points, + "iter {iteration}: accepted {accepted_final} but file has {file_points}" + ); + } + } + + #[test_log::test] + fn simultaneous_drop_triggers_shutdown() { + // Repeat to reliably trigger the race. + for iteration in 0..20 { + let tmp = TempDir::new().unwrap(); + let path = tmp.path().join(format!("drop_race_{iteration}.avro")); + let writer = AvroWriter::new(path.clone(), AvroWriterOpts::default()).unwrap(); + enq( + &writer, + cd("x"), + vec![IntegerPoint { + timestamp: Some(ts(1)), + value: 1, + }], + ) + .unwrap(); + + let c1 = writer.clone(); + let c2 = writer.clone(); + drop(writer); + + let barrier = Arc::new(std::sync::Barrier::new(2)); + let b1 = barrier.clone(); + let b2 = barrier.clone(); + let h1 = std::thread::spawn(move || { + b1.wait(); + drop(c1); + }); + let h2 = std::thread::spawn(move || { + b2.wait(); + drop(c2); + }); + + h1.join().unwrap(); + h2.join().unwrap(); + + // If the encoder hung, we would never reach this point or the file + // would be empty/unreadable. Verify the single point we wrote is + // present. + assert_eq!( + count_points_in_file(&path), + 1, + "iter {iteration}: expected 1 point" + ); + } + } + + #[test_log::test] + fn new_truncates_preexisting_file() { + let tmp = TempDir::new().unwrap(); + let path = tmp.path().join("preexisting.avro"); + + // Plant garbage bytes at the target path — larger than any avro file + // the test will write, so the "tail leaks past new content" failure + // mode would manifest. + std::fs::write(&path, vec![0xffu8; 10_000]).unwrap(); + assert_eq!(std::fs::metadata(&path).unwrap().len(), 10_000); + + let writer = AvroWriter::new(path.clone(), AvroWriterOpts::default()).unwrap(); + enq( + &writer, + cd("x"), + vec![IntegerPoint { + timestamp: Some(ts(1)), + value: 42, + }], + ) + .unwrap(); + writer.close().unwrap(); + + // The file must be a clean avro stream with exactly one record of one + // point — no garbage suffix. + assert_eq!(count_points_in_file(&path), 1); + let file = std::fs::File::open(&path).unwrap(); + let reader = Reader::new(file).unwrap(); + let records: Vec<_> = reader.collect::, _>>().unwrap(); + assert_eq!(records.len(), 1); + } + + #[test_log::test] + fn end_to_end_error_latches_through_dispatcher() { + let tmp = TempDir::new().unwrap(); + let path = tmp.path().join("ignored.avro"); + let writer = AvroWriter::new_with_consumer_for_testing( + FailingConsumer::new(std::io::ErrorKind::PermissionDenied, "denied"), + path, + AvroWriterOpts::default(), + ); + + // Enqueue some points. They travel through the full pipeline: + // write() -> stream.enqueue() -> batch_processor -> request_dispatcher + // -> ErrorLatchingConsumer::consume() -> FailingConsumer::consume() (fails) + // -> ErrorLatchingConsumer latches AvroWriterError::Io(PermissionDenied) + enq( + &writer, + cd("x"), + vec![IntegerPoint { + timestamp: Some(ts(1)), + value: 1, + }], + ) + .unwrap(); + + // close() drops the stream, which drains any pending commands. After + // drop, the consumer has processed (and failed on) the enqueued batch, + // so first_error is populated and close() returns it. + let err = writer.close().unwrap_err(); + + // Variant should be Io (not Consumer) + match &*err { + AvroWriterError::Io(inner) => { + assert_eq!(inner.kind(), std::io::ErrorKind::PermissionDenied); + } + other => panic!("expected AvroWriterError::Io, got {other:?}"), + } + + // close() is idempotent — second call returns the same Arc'd error. + let err2 = writer.close().unwrap_err(); + assert!( + Arc::ptr_eq(&err, &err2), + "close() should cache the error Arc" + ); + } + + // ───────────────────────────────────────────────────────────────────────── + // Rotation-specific tests (max_points_per_file > 0) + // ───────────────────────────────────────────────────────────────────────── + + #[test_log::test] + fn single_file_when_max_is_zero() { + let tmp = TempDir::new().unwrap(); + let path = tmp.path().join("no_rotation.avro"); + let writer = AvroWriter::new(path.clone(), AvroWriterOpts::default()).unwrap(); + + let pts: Vec = (0..1000i64) + .map(|i| IntegerPoint { + timestamp: Some(ts(i)), + value: i, + }) + .collect(); + enq(&writer, cd("x"), pts).unwrap(); + + let paths = writer.close().unwrap(); + assert_eq!(paths.len(), 1, "no rotation: exactly one file"); + assert_eq!(paths[0], path); + assert_eq!(count_points_in_file(&paths[0]), 1000); + } + + #[test_log::test] + fn rotates_at_threshold() { + let tmp = TempDir::new().unwrap(); + let base = tmp.path().join("rot.avro"); + let opts = AvroWriterOpts { + max_points_per_file: 100, + ..Default::default() + }; + let writer = AvroWriter::new(base.clone(), opts).unwrap(); + + // Write 250 points one-at-a-time so single-point enqueue rotation fires. + for i in 0..250i64 { + enq( + &writer, + cd("x"), + vec![IntegerPoint { + timestamp: Some(ts(i)), + value: i, + }], + ) + .unwrap(); + } + + let paths = writer.close().unwrap(); + assert_eq!(paths.len(), 3, "expected 3 files for 250 pts at max=100"); + assert_eq!(paths[0], path_for_index(&base, 0)); + assert_eq!(paths[1], path_for_index(&base, 1)); + assert_eq!(paths[2], path_for_index(&base, 2)); + + let counts: Vec = paths.iter().map(|p| count_points_in_file(p)).collect(); + assert_eq!(counts, vec![100, 100, 50]); + } + + #[test_log::test] + fn write_batch_splits_across_rotation() { + let tmp = TempDir::new().unwrap(); + let base = tmp.path().join("batch_split.avro"); + let opts = AvroWriterOpts { + max_points_per_file: 100, + ..Default::default() + }; + let writer = AvroWriter::new(base.clone(), opts).unwrap(); + + // Pre-fill 80 points into the first file. + let pre: Vec = (0..80i64) + .map(|i| IntegerPoint { + timestamp: Some(ts(i)), + value: i, + }) + .collect(); + enq(&writer, cd("x"), pre).unwrap(); + assert_eq!(writer.inner.points_in_current.load(Ordering::Relaxed), 80); + assert_eq!(writer.finalized_paths().len(), 0); + + // Write a batch of 50 via write_chunked — should split 20 + 30. + let batch: Vec = (80..130i64) + .map(|i| IntegerPoint { + timestamp: Some(ts(i)), + value: i, + }) + .collect(); + enq(&writer, cd("x"), batch).unwrap(); + + assert_eq!( + writer.finalized_paths().len(), + 1, + "first file should be finalized" + ); + assert_eq!(writer.inner.points_in_current.load(Ordering::Relaxed), 30); + + let paths = writer.close().unwrap(); + assert_eq!(paths.len(), 2); + let counts: Vec = paths.iter().map(|p| count_points_in_file(p)).collect(); + assert_eq!(counts, vec![100, 30]); + } +} diff --git a/nominal-streaming/src/consumer.rs b/nominal-streaming/src/consumer.rs index 553528c..b3040b9 100644 --- a/nominal-streaming/src/consumer.rs +++ b/nominal-streaming/src/consumer.rs @@ -224,7 +224,11 @@ impl AvroFileConsumer { } } -fn points_to_avro(points: Option<&Points>) -> (Vec, Vec) { +/// Convert a `Points` payload into parallel `(timestamps, values)` avro `Value` +/// arrays. Exposed so downstream consumers (e.g., a parallel record-builder +/// in py-nominal-streaming) can reuse the dtype-dispatch logic without +/// duplicating the schema match. +pub fn points_to_avro(points: Option<&Points>) -> (Vec, Vec) { let Some(Points { points_type: Some(points), }) = points diff --git a/nominal-streaming/src/lib.rs b/nominal-streaming/src/lib.rs index 1126da4..53aa47e 100644 --- a/nominal-streaming/src/lib.rs +++ b/nominal-streaming/src/lib.rs @@ -155,6 +155,7 @@ let stream = NominalDatasetStreamBuilder::new() ``` */ +pub mod avro_writer; pub mod client; pub mod consumer; pub mod listener; @@ -189,6 +190,10 @@ pub mod prelude { pub use nominal_api::tonic::io::nominal::scout::api::proto::WriteRequest; pub use nominal_api::tonic::io::nominal::scout::api::proto::WriteRequestNominal; + pub use crate::avro_writer::AvroWriter; + pub use crate::avro_writer::AvroWriterError; + pub use crate::avro_writer::AvroWriterOpts; + pub use crate::avro_writer::PipelineStats; pub use crate::consumer::NominalCoreConsumer; pub use crate::stream::NominalDatasetStream; #[expect(deprecated)] @@ -490,10 +495,14 @@ mod tests { let cd = ChannelDescriptor::new("channel_1"); let mut writer = stream.double_writer(cd); + // `max_request_delay` on the test stream is 100ms. Sleep 150ms between + // pushes so the batch_processor thread has a 1.5× margin to emit a + // WriteRequest before the next push — a tighter 101ms margin flaked + // under parallel test load. writer.push(UNIX_EPOCH.elapsed().unwrap(), 1.0); - thread::sleep(Duration::from_millis(101)); + thread::sleep(Duration::from_millis(150)); writer.push(UNIX_EPOCH.elapsed().unwrap(), 2.0); // first flush - thread::sleep(Duration::from_millis(101)); + thread::sleep(Duration::from_millis(150)); writer.push(UNIX_EPOCH.elapsed().unwrap(), 3.0); // second flush drop(writer); diff --git a/nominal-streaming/src/stream.rs b/nominal-streaming/src/stream.rs index 0c90476..7a11f4b 100644 --- a/nominal-streaming/src/stream.rs +++ b/nominal-streaming/src/stream.rs @@ -387,6 +387,25 @@ impl NominalDatasetStream { }); } + /// Enqueue points for multiple channels in a single critical section. + /// + /// Checks combined capacity once and extends every entry under one buffer + /// lock, amortizing the lock-acquisition cost across all channels. Callers + /// who must push a wide frame (many channels for one timestamp span) should + /// prefer this over a loop of [`Self::enqueue`] calls. + pub fn enqueue_batch(&self, batch: Vec<(ChannelDescriptor, PointsType)>) { + let total: usize = batch.iter().map(|(_, p)| points_len(p)).sum(); + if total == 0 { + return; + } + + self.when_capacity(total, move |mut sb| { + for (desc, pts) in batch { + sb.extend(&desc, pts); + } + }); + } + fn when_capacity(&self, new_count: usize, callback: impl FnOnce(SeriesBufferGuard)) { self.unflushed_points .fetch_add(new_count, Ordering::Release); @@ -933,7 +952,7 @@ fn request_dispatcher( ); } -fn points_len(points_type: &PointsType) -> usize { +pub fn points_len(points_type: &PointsType) -> usize { match points_type { PointsType::DoublePoints(points) => points.points.len(), PointsType::StringPoints(points) => points.points.len(), diff --git a/py-nominal-streaming/Cargo.toml b/py-nominal-streaming/Cargo.toml index 5937191..8127aa4 100644 --- a/py-nominal-streaming/Cargo.toml +++ b/py-nominal-streaming/Cargo.toml @@ -10,11 +10,19 @@ repository = "https://github.com/nominal-io/nominal-streaming" [lib] # This becomes the *module* name on the Python side unless overridden by tool.maturin.module-name name = "_nominal_streaming" -crate-type = ["cdylib"] +crate-type = ["cdylib", "rlib"] + +[features] +# `extension-module` is required for the maturin-built wheel but MUST NOT be +# enabled during `cargo test` — the feature tells PyO3 not to link against +# libpython (since the final link happens when Python loads the .so), which +# makes `cargo test`'s test binaries fail with missing-symbol errors. +# Maturin enables it via `[tool.maturin].features` in pyproject.toml. +extension-module = ["pyo3/extension-module"] [dependencies] # PyO3 with a stable ABI across Python 3.10+. -pyo3 = { version = "0.26", features = ["extension-module", "abi3-py310", "time"] } +pyo3 = { version = "0.26", features = ["abi3-py310", "time"] } nominal-api = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "time"] } @@ -27,9 +35,28 @@ openssl = { version = "0.10", features = ["vendored"] } tracing = { workspace = true } tracing-subscriber = { workspace = true } -# Depend on the Rust core crate in the repository root -nominal-streaming = { workspace = true, features = ["logging"] } +# Depend on the Rust core crate in the repository root. +# `polars` feature enables the pure-Rust `AvroWriter::write_dataframe` +# method — this crate adds the Python FFI (polars_ffi.rs) on top. +nominal-streaming = { workspace = true, features = ["logging", "polars"] } anyhow = "1" crossbeam-channel = "0.5" tokio-util = "0.7" +parking_lot = { workspace = true } +thiserror = { workspace = true } +apache-avro = { workspace = true } +serde_json = { workspace = true } + +# Polars Rust crate for write_dataframe — dtype inspection + column iteration. +# We use the Arrow C Data Interface (_export_arrow_to_c on Python's _s object) +# to transfer data from Python polars → Rust polars without requiring pyarrow. +polars = { version = "0.52", default-features = false, features = [ + "dtype-struct", + "dtype-array", +] } +polars-arrow = { version = "0.52", default-features = false } + +[dev-dependencies] +tempfile = { workspace = true } +test-log = { workspace = true } diff --git a/py-nominal-streaming/benchmarks/bench_nominal_avro_writer.py b/py-nominal-streaming/benchmarks/bench_nominal_avro_writer.py new file mode 100644 index 0000000..462b4bd --- /dev/null +++ b/py-nominal-streaming/benchmarks/bench_nominal_avro_writer.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +r"""Benchmark NominalAvroWriter across (dtype × n_cols × total_points × frame_points). + +Reports elapsed write+close time, total bytes on disk, file count, and points/sec. +Writes results incrementally to CSV so partial output survives interrupts. + +Anti-compression-cheating: each config pre-builds a pool DataFrame, and each +iteration slices a different offset from it. Reusing a single sample frame +lets snappy compress repetitions across iterations and produces misleading +bytes/point numbers. + +Tuning: + * ``--max-points-per-batch`` default 1,000,000 (higher than the library + default of 250k — fewer encoder-side emit cycles). + * ``--rotate-every`` default 1,000,000,000 (effectively never rotate within + one bench config). Pass a smaller value to measure rotation overhead. + * fsync on close is ON by default; toggle with ``--no-fsync``. + +Python version note: + Throughput is meaningfully higher on CPython 3.12+ than on 3.10 + (PyO3 boundary crossings + polars calls are both faster in 3.12). + Pin the Python version when comparing runs. + +Example: + # Quick smoke run + uv run python py-nominal-streaming/benchmarks/bench_nominal_avro_writer.py \\ + --dtypes float string --cols 10 100 \\ + --total-points 1_000_000 --frame-points 100_000 + + # Full sweep to CSV + uv run python py-nominal-streaming/benchmarks/bench_nominal_avro_writer.py \\ + --out bench_full.csv +""" + +from __future__ import annotations + +import argparse +import csv +import pathlib +import shutil +import sys +import tempfile +import time +from dataclasses import dataclass + +# Fail loud if run under Python 3.10 / 3.11: PyO3 boundary crossings are +# materially slower there, and silent cross-version comparisons are the +# single most common source of misleading bench numbers on this pipeline. +if sys.version_info < (3, 12): + sys.exit( + f"benchmarks require Python 3.12+ (got {sys.version_info.major}.{sys.version_info.minor}). " + f"Re-run under 3.12+, e.g. `uv run --python 3.12 python {pathlib.Path(__file__).name}`." + ) + +import numpy as np +import polars as pl +from nominal_streaming import NominalAvroWriter, NominalAvroWriterOpts + +DTYPES = ("float", "string", "array", "struct") +COL_COUNTS = (10, 100, 1000) +TOTAL_POINTS = (1_000_000, 10_000_000, 100_000_000, 500_000_000) +FRAME_POINTS = (10_000, 50_000, 100_000, 1_000_000) + +_ARRAY_LEN = 8 +_STRUCT_SCHEMA = pl.Struct([pl.Field("a", pl.Int64), pl.Field("b", pl.Float64)]) + + +def build_pool(dtype: str, pool_rows: int, n_cols: int) -> pl.DataFrame: + """Build a pool DataFrame; iterations slice distinct windows from it.""" + ts = pl.Series("ts", np.arange(pool_rows, dtype=np.int64)) + cols: dict[str, pl.Series] = {"ts": ts} + + if dtype == "float": + data = np.random.default_rng(0).random((pool_rows, n_cols), dtype=np.float64) + for c in range(n_cols): + cols[f"c{c}"] = pl.Series(f"c{c}", data[:, c]) + elif dtype == "string": + sample = [f"v{i}" for i in range(pool_rows)] + for c in range(n_cols): + cols[f"c{c}"] = pl.Series(f"c{c}", sample, dtype=pl.String) + elif dtype == "array": + sample = [[float(i + k) for k in range(_ARRAY_LEN)] for i in range(pool_rows)] + for c in range(n_cols): + cols[f"c{c}"] = pl.Series(f"c{c}", sample, dtype=pl.List(pl.Float64)) + elif dtype == "struct": + sample = [{"a": i, "b": float(i)} for i in range(pool_rows)] + for c in range(n_cols): + cols[f"c{c}"] = pl.Series(f"c{c}", sample, dtype=_STRUCT_SCHEMA) + else: + raise ValueError(f"unknown dtype: {dtype}") + + return pl.DataFrame(cols) + + +def _pool_rows_for(dtype: str, n_cols: int, rows_per_frame: int, budget_mib: int) -> int: + """Pick a pool size under ``budget_mib`` while giving each iteration a distinct slice.""" + per_cell = {"float": 8, "string": 24, "array": 80, "struct": 96}[dtype] + max_rows = max(1, (budget_mib * 1024 * 1024) // (n_cols * per_cell)) + return max(rows_per_frame, min(rows_per_frame * 10, max_rows)) + + +@dataclass +class Result: + dtype: str + n_cols: int + total_points: int + frame_points: int + elapsed_s: float + total_bytes: int + n_files: int + + @property + def mib(self) -> float: + return self.total_bytes / (1024 * 1024) + + @property + def points_per_s(self) -> float: + return self.total_points / self.elapsed_s if self.elapsed_s > 0 else float("inf") + + +def run_one( + dtype: str, + n_cols: int, + total_points: int, + frame_points: int, + out_root: pathlib.Path, + opts: NominalAvroWriterOpts, + rotate_every: int, + pool_budget_mib: int, +) -> Result | None: + rows_per_frame = frame_points // n_cols + if rows_per_frame < 1: + return None + num_frames = total_points // frame_points + if num_frames < 1: + return None + + # Pre-build a pool outside the timed section; each iteration slices a + # different offset so snappy can't cheat by compressing identical frames. + pool_rows = _pool_rows_for(dtype, n_cols, rows_per_frame, pool_budget_mib) + pool = build_pool(dtype, pool_rows, n_cols) + max_offset = pool_rows - rows_per_frame + 1 # > 0 since pool_rows >= rows_per_frame + + out_dir = out_root / f"{dtype}_c{n_cols}_t{total_points}_f{frame_points}" + out_dir.mkdir(parents=True, exist_ok=True) + base = out_dir / "bench.avro" + + run_opts = opts.with_max_points_per_file(rotate_every) + start = time.monotonic() + writer = NominalAvroWriter(base, run_opts) + try: + for i in range(num_frames): + offset = (i * rows_per_frame) % max_offset + frame = pool.slice(offset, rows_per_frame) + writer.write_dataframe(frame, timestamp_column="ts") + finally: + files = list(writer.close()) # returns all paths after shutdown + elapsed = time.monotonic() - start + + total_bytes = sum(pathlib.Path(p).stat().st_size for p in files) + n_files = len(files) + shutil.rmtree(out_dir) + + return Result(dtype, n_cols, total_points, frame_points, elapsed, total_bytes, n_files) + + +class _ExistingDir: + """Context-manager shim so args.workdir and tempfile.TemporaryDirectory share a call site.""" + + def __init__(self, path: pathlib.Path) -> None: + self._path = path + + def __enter__(self) -> str: + self._path.mkdir(parents=True, exist_ok=True) + return str(self._path) + + def __exit__(self, *_: object) -> None: + pass + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", type=pathlib.Path, default=pathlib.Path("nominal_avro_writer_bench.csv")) + parser.add_argument("--dtypes", nargs="+", choices=DTYPES, default=DTYPES) + parser.add_argument("--cols", nargs="+", type=int, default=COL_COUNTS) + parser.add_argument("--total-points", nargs="+", type=int, default=TOTAL_POINTS) + parser.add_argument("--frame-points", nargs="+", type=int, default=FRAME_POINTS) + parser.add_argument( + "--max-points-per-batch", + type=int, + default=1_000_000, + help="NominalAvroWriterOpts.max_points_per_batch (default: 1M; library default is 250k).", + ) + parser.add_argument("--max-batch-delay-secs", type=float, default=0.1) + parser.add_argument( + "--rotate-every", + type=int, + default=1_000_000_000, + help="Rotate to a new file after this many points. Default 1B = effectively never.", + ) + parser.add_argument( + "--pool-budget-mib", type=int, default=256, help="Memory budget per config for the source pool." + ) + parser.add_argument("--no-fsync", action="store_true") + parser.add_argument("--workdir", type=pathlib.Path, default=None) + args = parser.parse_args() + + opts = NominalAvroWriterOpts( + max_points_per_batch=args.max_points_per_batch, + max_batch_delay_secs=args.max_batch_delay_secs, + fsync_on_close=not args.no_fsync, + ) + + print( + f"Writing results to {args.out} " + f"(opts: max_points_per_batch={args.max_points_per_batch:,}, " + f"max_batch_delay_secs={args.max_batch_delay_secs:g}, " + f"fsync_on_close={not args.no_fsync}, " + f"rotate_every={args.rotate_every:,})" + ) + + header = [ + "dtype", + "n_cols", + "total_points", + "frame_points", + "elapsed_s", + "total_bytes", + "total_mib", + "n_files", + "points_per_s", + ] + + tmp_ctx = tempfile.TemporaryDirectory() if args.workdir is None else _ExistingDir(args.workdir) + with tmp_ctx as tmp, open(args.out, "w", newline="") as f: + tmp_root = pathlib.Path(tmp) + writer = csv.writer(f) + writer.writerow(header) + f.flush() + + for dtype in args.dtypes: + for n_cols in args.cols: + for total in args.total_points: + for frame in args.frame_points: + label = f"{dtype:6} cols={n_cols:<5} total={total:>12,} frame={frame:>10,}" + r = run_one( + dtype, + n_cols, + total, + frame, + tmp_root, + opts=opts, + rotate_every=args.rotate_every, + pool_budget_mib=args.pool_budget_mib, + ) + if r is None: + print(f"SKIP {label}: infeasible shape") + continue + writer.writerow( + [ + r.dtype, + r.n_cols, + r.total_points, + r.frame_points, + f"{r.elapsed_s:.3f}", + r.total_bytes, + f"{r.mib:.2f}", + r.n_files, + f"{r.points_per_s:.0f}", + ] + ) + f.flush() + print( + f"{label} elapsed={r.elapsed_s:8.2f}s " + f"size={r.mib:9.1f}MiB files={r.n_files:4} " + f"pts/s={r.points_per_s:>13,.0f}" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/py-nominal-streaming/benchmarks/head_to_head.py b/py-nominal-streaming/benchmarks/head_to_head.py new file mode 100644 index 0000000..700d706 --- /dev/null +++ b/py-nominal-streaming/benchmarks/head_to_head.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +r"""Head-to-head benchmark: NominalAvroWriter vs. PyNominalDatasetStream.to_file. + +Compares the two ways a Python caller can write an avro file with this crate: + + 1. **Old idiom** — build a PyNominalDatasetStream, point it at a file via + ``.to_file(path)``, then call ``enqueue_batch(channel, ts_list, values)`` + once per column per frame. This is the pattern you'd write before this + branch landed — one PyO3 FFI crossing per column per frame, and per-column + Python list materialization via ``Series.to_list()``. + + 2. **New idiom** — ``NominalAvroWriter(path).write_dataframe(df, "ts")``. + One FFI crossing per frame; the per-column iteration runs Rust-side with + the GIL released. + +Both paths write the same avro schema to the same on-disk format. The output +files should be byte-identical modulo non-determinism from snappy + the +per-file ingest_rid tag (neither of which we generate here). + +Run with Python 3.12+ (the guard will fail loud otherwise). + +Example:: + + uv run --python 3.12 python py-nominal-streaming/benchmarks/head_to_head.py + + # Custom shape: + uv run --python 3.12 python py-nominal-streaming/benchmarks/head_to_head.py \ + --cols 1000 --total-points 200_000_000 --frame-points 1_000_000 +""" + +from __future__ import annotations + +import argparse +import pathlib +import sys +import tempfile +import time + +if sys.version_info < (3, 12): + sys.exit( + f"benchmarks require Python 3.12+ (got {sys.version_info.major}.{sys.version_info.minor}). " + f"Re-run under 3.12+, e.g. `uv run --python 3.12 python {pathlib.Path(__file__).name}`." + ) + +import numpy as np +import polars as pl +from nominal_streaming import ( + NominalAvroWriter, + NominalAvroWriterOpts, + PyNominalStreamOpts, +) +from nominal_streaming._nominal_streaming import PyNominalDatasetStream + + +def build_pool(n_cols: int, pool_rows: int) -> pl.DataFrame: + ts = pl.Series("ts", np.arange(pool_rows, dtype=np.int64)) + cols: dict[str, pl.Series] = {"ts": ts} + data = np.random.default_rng(0).random((pool_rows, n_cols), dtype=np.float64) + for c in range(n_cols): + cols[f"c{c}"] = pl.Series(f"c{c}", data[:, c]) + return pl.DataFrame(cols) + + +def run_old( + pool: pl.DataFrame, rows_per_frame: int, num_frames: int, path: pathlib.Path, max_points_per_batch: int +) -> float: + """Old idiom: PyNominalDatasetStream + per-column enqueue_batch.""" + opts = PyNominalStreamOpts( + max_points_per_batch=max_points_per_batch, + max_request_delay_secs=0.1, + ) + stream = PyNominalDatasetStream(opts).to_file(path) + stream.open() + max_offset = max(pool.height - rows_per_frame + 1, 1) + data_cols = [c for c in pool.columns if c != "ts"] + start = time.monotonic() + try: + for i in range(num_frames): + offset = (i * rows_per_frame) % max_offset + frame = pool.slice(offset, rows_per_frame) + ts_list = frame["ts"].to_list() + for col in data_cols: + stream.enqueue_batch(col, ts_list, frame[col].to_list()) + finally: + stream.close() + return time.monotonic() - start + + +def run_new( + pool: pl.DataFrame, rows_per_frame: int, num_frames: int, path: pathlib.Path, max_points_per_batch: int +) -> float: + """New idiom: NominalAvroWriter.write_dataframe.""" + opts = NominalAvroWriterOpts( + max_points_per_batch=max_points_per_batch, + # Match the old-idiom baseline, which doesn't fsync on close — fair comparison. + fsync_on_close=False, + max_points_per_file=0, + ) + max_offset = max(pool.height - rows_per_frame + 1, 1) + start = time.monotonic() + with NominalAvroWriter(path, opts) as w: + for i in range(num_frames): + offset = (i * rows_per_frame) % max_offset + w.write_dataframe(pool.slice(offset, rows_per_frame), timestamp_column="ts") + return time.monotonic() - start + + +def run_pair( + pool: pl.DataFrame, n_cols: int, total_points: int, frame_points: int, max_points_per_batch: int +) -> tuple[float, float]: + rows_per_frame = frame_points // n_cols + num_frames = total_points // frame_points + if rows_per_frame < 1 or num_frames < 1: + return (float("nan"), float("nan")) + + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + t_old = run_old(pool, rows_per_frame, num_frames, tmp_path / "old.avro", max_points_per_batch) + t_new = run_new(pool, rows_per_frame, num_frames, tmp_path / "new.avro", max_points_per_batch) + return (t_old, t_new) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--cols", + nargs="+", + type=int, + default=[10, 100, 1000], + help="Column counts to sweep. Default: 10, 100, 1000.", + ) + parser.add_argument( + "--frame-points", + nargs="+", + type=int, + default=[100_000, 1_000_000], + help="Frame sizes (points per write call). Default: 100K, 1M.", + ) + parser.add_argument("--total-points", type=int, default=50_000_000) + parser.add_argument("--pool-rows", type=int, default=50_000) + parser.add_argument("--max-points-per-batch", type=int, default=1_000_000) + args = parser.parse_args() + + print("Head-to-head: NominalAvroWriter vs PyNominalDatasetStream.to_file") + print(f" total_points={args.total_points:,} pool_rows={args.pool_rows:,}") + print(f" cols={args.cols} frame_points={[f'{f:,}' for f in args.frame_points]}\n") + + header = ( + f"{'cols':>5} {'frame_pts':>11} {'old wall':>10} {'old pts/s':>14} " + f"{'new wall':>10} {'new pts/s':>14} {'speedup':>9}" + ) + print(header) + print("-" * len(header)) + + for n_cols in args.cols: + pool = build_pool(n_cols, args.pool_rows) + for fp in args.frame_points: + t_old, t_new = run_pair(pool, n_cols, args.total_points, fp, args.max_points_per_batch) + old_rate = args.total_points / t_old if t_old > 0 else float("nan") + new_rate = args.total_points / t_new if t_new > 0 else float("nan") + speedup = t_old / t_new if t_new > 0 else float("nan") + print( + f"{n_cols:>5,} {fp:>11,} {t_old:>9.2f}s {old_rate:>14,.0f} " + f"{t_new:>9.2f}s {new_rate:>14,.0f} {speedup:>8.2f}×" + ) + del pool + + print() + print("Legend:") + print(" old idiom = PyNominalDatasetStream(opts).to_file(path) + per-column enqueue_batch.") + print(" new idiom = NominalAvroWriter(path).write_dataframe(df, 'ts').") + print(" speedup = old_wall / new_wall.") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/py-nominal-streaming/benchmarks/pipeline_profile.py b/py-nominal-streaming/benchmarks/pipeline_profile.py new file mode 100644 index 0000000..c9cb918 --- /dev/null +++ b/py-nominal-streaming/benchmarks/pipeline_profile.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +r"""Single-run pipeline attribution for NominalAvroWriter.write_dataframe. + +Runs one workload end-to-end and reads NominalAvroWriter.stats() to attribute +wall-clock time across the pipeline's stages: + + * Producer CPU phases inside write_dataframe (FFI, extract, column build) + — run on the caller thread. + * enqueue_batch wall (producer's handoff into the stream) — includes time + blocked on buffer capacity when the consumer can't keep up. + * Consumer wall (avro encode + snappy + file write) — runs in parallel on + the stream's single dispatcher thread. + +Interpretation: + - If consumer_consume_ns ≈ elapsed → the consumer is saturated (downstream + bottleneck; enqueue_batch_ns grows because the producer is backpressured). + - If producer_cpu_ns ≈ elapsed → the producer is the bottleneck (rare at + reasonable frame sizes; usually signals a Python-side issue). + +Example: + uv run python py-nominal-streaming/benchmarks/pipeline_profile.py \\ + --cols 1000 --frame-points 1_000_000 --total-points 200_000_000 +""" + +from __future__ import annotations + +import argparse +import pathlib +import sys +import tempfile +import time + +# Fail loud if run under Python 3.10 / 3.11: PyO3 boundary crossings are +# materially slower there, and silent cross-version comparisons are the +# single most common source of misleading bench numbers on this pipeline. +if sys.version_info < (3, 12): + sys.exit( + f"benchmarks require Python 3.12+ (got {sys.version_info.major}.{sys.version_info.minor}). " + f"Re-run under 3.12+, e.g. `uv run --python 3.12 python {pathlib.Path(__file__).name}`." + ) + +import numpy as np +import polars as pl +from nominal_streaming import NominalAvroWriter, NominalAvroWriterOpts + + +def build_pool(n_cols: int, pool_rows: int) -> pl.DataFrame: + ts = pl.Series("ts", np.arange(pool_rows, dtype=np.int64)) + cols = {"ts": ts} + data = np.random.default_rng(0).random((pool_rows, n_cols), dtype=np.float64) + for c in range(n_cols): + cols[f"c{c}"] = pl.Series(f"c{c}", data[:, c]) + return pl.DataFrame(cols) + + +def run( + pool: pl.DataFrame, rows_per_frame: int, num_frames: int, out_dir: pathlib.Path, max_points_per_batch: int +) -> tuple[float, dict[str, int]]: + max_offset = max(pool.height - rows_per_frame + 1, 1) + opts = NominalAvroWriterOpts( + max_points_per_batch=max_points_per_batch, + fsync_on_close=True, + max_points_per_file=0, + ) + base = out_dir / "profile.avro" + start = time.monotonic() + with NominalAvroWriter(base, opts) as w: + for i in range(num_frames): + offset = (i * rows_per_frame) % max_offset + w.write_dataframe(pool.slice(offset, rows_per_frame), timestamp_column="ts") + stats = w.stats() + return time.monotonic() - start, stats + + +def fmt_ns(ns: int) -> str: + if ns < 1_000: + return f"{ns}ns" + if ns < 1_000_000: + return f"{ns / 1_000:.1f}µs" + if ns < 1_000_000_000: + return f"{ns / 1_000_000:.1f}ms" + return f"{ns / 1_000_000_000:.3f}s" + + +def bar(pct: float, width: int = 40) -> str: + filled = max(0, min(width, int(round(pct / 100 * width)))) + return "█" * filled + "·" * (width - filled) + + +def print_report(elapsed: float, stats: dict[str, int], total_points: int) -> None: + """Print the flame-graph-style attribution and diagnosis for one run.""" + elapsed_ns = int(elapsed * 1e9) + pts_per_s = total_points / elapsed + + df_handoff = stats["df_handoff_ns"] + extract_ts = stats["extract_ts_ns"] + column_build = stats["column_build_ns"] + enqueue_batch = stats["enqueue_batch_ns"] + consumer = stats["consumer_consume_ns"] + consumer_calls = stats["consumer_consume_calls"] + wdf_calls = stats["write_dataframe_calls"] + wdf_total = stats["write_dataframe_total_ns"] + producer_cpu = df_handoff + extract_ts + column_build + + def row(label: str, ns: int) -> None: + pct = 100 * ns / elapsed_ns + print(f" {label:<26}{fmt_ns(ns):>10}{pct:>8.1f}% {bar(pct)}") + + print("=== Pipeline attribution ===") + print(f"wall_clock: {fmt_ns(elapsed_ns):>10} 100.0% ({pts_per_s:,.0f} pts/s)") + print( + f"write_dataframe total: {fmt_ns(wdf_total):>10} {100 * wdf_total / elapsed_ns:>5.1f}% " + f"({wdf_calls} calls, avg {fmt_ns(wdf_total // max(wdf_calls, 1))})" + ) + print() + print("Producer CPU phases (inside write_dataframe, caller thread):") + row("df_handoff (FFI)", df_handoff) + row("extract timestamps", extract_ts) + row("column_build loop", column_build) + row(" → producer CPU total", producer_cpu) + print() + print("Producer wall inside stream (includes blocked-on-buffer):") + row("enqueue_batch (prod wall)", enqueue_batch) + print() + print("Consumer (parallel, single dispatcher thread):") + row("consumer.consume total", consumer) + print(f" ({consumer_calls} consume calls; avg {fmt_ns(consumer // max(consumer_calls, 1))} per WriteRequest)") + print() + + consumer_pct = 100 * consumer / elapsed_ns + unaccounted = elapsed_ns - (producer_cpu + enqueue_batch) + print("=== Diagnosis ===") + print( + f"producer_cpu + enqueue_batch = {fmt_ns(producer_cpu + enqueue_batch)} " + f"({100 * (producer_cpu + enqueue_batch) / elapsed_ns:.1f}% of wall)" + ) + print(f"unaccounted (GIL / Python loop / misc): {fmt_ns(unaccounted)} ({100 * unaccounted / elapsed_ns:.1f}%)") + saturation = "SATURATED — bottleneck" if consumer_pct > 90 else f"idle {round(100 - consumer_pct, 1)}% of wall" + print(f"consumer utilization: {consumer_pct:.1f}% ({saturation})") + if consumer_pct > 90: + print("→ Consumer (avro encode + snappy + write) is the bottleneck; producer is backpressured.") + elif producer_cpu > enqueue_batch * 0.5: + print("→ Producer CPU is significant; consider bigger frames to amortize FFI.") + else: + print("→ Consumer has headroom; producer is the limiter.") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cols", type=int, default=1000) + parser.add_argument("--total-points", type=int, default=200_000_000) + parser.add_argument("--frame-points", type=int, default=1_000_000) + parser.add_argument("--pool-rows", type=int, default=50_000) + parser.add_argument("--max-points-per-batch", type=int, default=1_000_000) + args = parser.parse_args() + + rows_per_frame = args.frame_points // args.cols + num_frames = args.total_points // args.frame_points + print( + f"Config: {args.cols} cols × {args.total_points:,} points × {args.frame_points:,} frame_points " + f"= {num_frames} frames × {rows_per_frame} rows/frame" + ) + print(f"Building pool ({args.pool_rows} rows × {args.cols} cols)...") + pool = build_pool(args.cols, args.pool_rows) + print("Pool built.\n") + + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + elapsed, stats = run(pool, rows_per_frame, num_frames, tmp_path, args.max_points_per_batch) + + print_report(elapsed, stats, args.total_points) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/py-nominal-streaming/benchmarks/sweep.py b/py-nominal-streaming/benchmarks/sweep.py new file mode 100644 index 0000000..d98469a --- /dev/null +++ b/py-nominal-streaming/benchmarks/sweep.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +r"""1D or 2D sweep over (cols, frame_points) for NominalAvroWriter. + +Prints a throughput matrix + consumer-utilization matrix so you can see how +the pipeline shape shifts across workloads. Both axes default to a small +characterization grid; pass `--cols N1 N2 ...` / `--frame-points N1 N2 ...` +to customize. + +Pool size is chosen per-cell to cap memory at ~400MB. + +Example: + # Default 2D sweep (cols × frame_points) + uv run python py-nominal-streaming/benchmarks/sweep.py + + # Just a cols sweep at one frame size + uv run python py-nominal-streaming/benchmarks/sweep.py \\ + --cols 10 100 1000 --frame-points 1_000_000 +""" + +from __future__ import annotations + +import argparse +import pathlib +import sys +import tempfile +import time + +# Fail loud if run under Python 3.10 / 3.11: PyO3 boundary crossings are +# materially slower there, and silent cross-version comparisons are the +# single most common source of misleading bench numbers on this pipeline. +if sys.version_info < (3, 12): + sys.exit( + f"benchmarks require Python 3.12+ (got {sys.version_info.major}.{sys.version_info.minor}). " + f"Re-run under 3.12+, e.g. `uv run --python 3.12 python {pathlib.Path(__file__).name}`." + ) + +import numpy as np +import polars as pl +from nominal_streaming import NominalAvroWriter, NominalAvroWriterOpts + +DEFAULT_COLS = [10, 100, 1000, 5000] +DEFAULT_FRAME_POINTS = [250_000, 1_000_000, 5_000_000] +POOL_MEM_CAP_BYTES = 400 * 1024 * 1024 # 400MB + + +def build_pool(n_cols: int, pool_rows: int) -> pl.DataFrame: + ts = pl.Series("ts", np.arange(pool_rows, dtype=np.int64)) + cols = {"ts": ts} + data = np.random.default_rng(0).random((pool_rows, n_cols), dtype=np.float64) + for c in range(n_cols): + cols[f"c{c}"] = pl.Series(f"c{c}", data[:, c]) + return pl.DataFrame(cols) + + +def pick_pool_rows(n_cols: int, rows_per_frame: int) -> int: + target = max(rows_per_frame * 10, 10_000) + mem_cap_rows = POOL_MEM_CAP_BYTES // (n_cols * 8) + rows = min(target, mem_cap_rows) + return max(rows, rows_per_frame) + + +def run_bench( + pool: pl.DataFrame, n_cols: int, rows_per_frame: int, total_points: int, out_dir: pathlib.Path +) -> tuple[float, dict[str, int]]: + num_frames = total_points // (rows_per_frame * n_cols) + max_offset = max(pool.height - rows_per_frame + 1, 1) + opts = NominalAvroWriterOpts( + max_points_per_batch=1_000_000, + fsync_on_close=True, + max_points_per_file=0, + ) + base = out_dir / f"sweep_{n_cols}_{rows_per_frame}.avro" + start = time.monotonic() + with NominalAvroWriter(base, opts) as w: + for i in range(num_frames): + offset = (i * rows_per_frame) % max_offset + w.write_dataframe(pool.slice(offset, rows_per_frame), timestamp_column="ts") + stats = w.stats() + return time.monotonic() - start, stats + + +def print_table(title: str, grid: dict, n_cols_grid: list[int], frame_grid: list[int], fmt: str) -> None: + print(f"\n{title}") + header = "cols \\ frame_pts " + "".join(f"{f:>14,}" for f in frame_grid) + print(header) + print("-" * len(header)) + for c in n_cols_grid: + row = f"{c:>7,} " + for f in frame_grid: + v = grid.get((c, f)) + row += f"{v:>13{fmt}} " + print(row) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cols", nargs="+", type=int, default=DEFAULT_COLS) + parser.add_argument("--frame-points", nargs="+", type=int, default=DEFAULT_FRAME_POINTS) + parser.add_argument("--total-points", type=int, default=200_000_000) + args = parser.parse_args() + + print(f"Sweep: total_points={args.total_points:,}") + print(f" cols: {args.cols}") + print(f" frame_pt: {[f'{f:,}' for f in args.frame_points]}\n") + + throughput: dict[tuple[int, int], float] = {} + cons_util: dict[tuple[int, int], float] = {} + elapsed_s: dict[tuple[int, int], float] = {} + + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + for n_cols in args.cols: + for fp in args.frame_points: + rows_per_frame = fp // n_cols + if rows_per_frame < 1: + print(f" SKIP n_cols={n_cols}, frame_pts={fp:,}: rows/frame < 1") + continue + pool_rows = pick_pool_rows(n_cols, rows_per_frame) + pool = build_pool(n_cols, pool_rows) + print( + f" running n_cols={n_cols:>5}, frame_pts={fp:>9,} " + f"(rows/frm={rows_per_frame:,}, pool_rows={pool_rows:,})...", + flush=True, + ) + elapsed, stats = run_bench(pool, n_cols, rows_per_frame, args.total_points, tmp_path) + pts_per_s = args.total_points / elapsed + consumer = stats["consumer_consume_ns"] / 1e9 + cu = 100 * consumer / elapsed + + throughput[(n_cols, fp)] = pts_per_s + cons_util[(n_cols, fp)] = cu + elapsed_s[(n_cols, fp)] = elapsed + + print(f" elapsed={elapsed:.2f}s pts/s={pts_per_s / 1e6:.1f}M cons_util={cu:.1f}%") + del pool + + throughput_m = {k: v / 1e6 for k, v in throughput.items()} + + print_table("=== Throughput (M pts/s) ===", throughput_m, args.cols, args.frame_points, ".1f") + print_table("=== Consumer utilization (% of wall) ===", cons_util, args.cols, args.frame_points, ".1f") + print_table("=== Wall elapsed (seconds) ===", elapsed_s, args.cols, args.frame_points, ".2f") + + if throughput: + best = max(throughput, key=throughput.get) + worst = min(throughput, key=throughput.get) + print() + print(f"Best: cols={best[0]:,} frame_pts={best[1]:,} → {throughput[best] / 1e6:.1f} M pts/s") + print(f"Worst: cols={worst[0]:,} frame_pts={worst[1]:,} → {throughput[worst] / 1e6:.1f} M pts/s") + print(f"Range: {throughput[best] / throughput[worst]:.2f}×") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/py-nominal-streaming/pyproject.toml b/py-nominal-streaming/pyproject.toml index ba3026b..b013a9e 100644 --- a/py-nominal-streaming/pyproject.toml +++ b/py-nominal-streaming/pyproject.toml @@ -20,10 +20,15 @@ dynamic = [ dependencies = [ "typing-extensions>=4,<5", + "polars>=1,<2", # Any version is okay for these dependencies "python-dateutil>=0.0.0", ] +[dependency-groups] +dev = ["pytest>=7", "fastavro>=1", "cramjam>=2"] + + [build-system] requires = ["maturin>=1.6,<2.0"] build-backend = "maturin" @@ -37,10 +42,26 @@ strip = true manifest-path = "./Cargo.toml" features = [ - "pyo3/extension-module", # Required on unix platforms to prevent linking against libpython.so - "pyo3/abi3-py310" # Sets the exact minimum python version supported within multi-version wheels + "extension-module", # Our feature alias; enables pyo3/extension-module for maturin-built wheels but NOT during cargo test. + "pyo3/abi3-py310" # Sets the exact minimum python version supported within multi-version wheels. ] # Tell maturin where the python code lives. # All files under this directory will be included in the wheel. python-source = "python" + +# Keep Python bytecode / caches out of the release wheel. Without this, +# maturin picks up whatever __pycache__ happens to exist in the source +# tree from a prior `pytest` / dev run and ships the .pyc files (plus +# orphaned bytecode from modules that have since been deleted on the +# branch). These are already .gitignored, but maturin doesn't consult +# git's ignore rules. +exclude = [ + { path = "**/__pycache__/**", format = "wheel" }, + { path = "**/*.pyc", format = "wheel" }, + { path = "**/*.pyo", format = "wheel" }, +] + + +[tool.pytest.ini_options] +testpaths = ["python/tests"] diff --git a/py-nominal-streaming/python/nominal_streaming/__init__.py b/py-nominal-streaming/python/nominal_streaming/__init__.py index ca553fe..15fbf0d 100644 --- a/py-nominal-streaming/python/nominal_streaming/__init__.py +++ b/py-nominal-streaming/python/nominal_streaming/__init__.py @@ -1,4 +1,15 @@ -from nominal_streaming._nominal_streaming import PyNominalStreamOpts +from __future__ import annotations + +from nominal_streaming._nominal_streaming import ( + NominalAvroWriter, + NominalAvroWriterOpts, + PyNominalStreamOpts, +) from nominal_streaming.nominal_dataset_stream import NominalDatasetStream -__all__ = ["PyNominalStreamOpts", "NominalDatasetStream"] +__all__ = [ + "PyNominalStreamOpts", + "NominalDatasetStream", + "NominalAvroWriter", + "NominalAvroWriterOpts", +] diff --git a/py-nominal-streaming/python/nominal_streaming/_nominal_streaming.pyi b/py-nominal-streaming/python/nominal_streaming/_nominal_streaming.pyi index acce9c5..2e9a0c3 100644 --- a/py-nominal-streaming/python/nominal_streaming/_nominal_streaming.pyi +++ b/py-nominal-streaming/python/nominal_streaming/_nominal_streaming.pyi @@ -2,12 +2,15 @@ from __future__ import annotations import pathlib from types import TracebackType -from typing import Any, Mapping, Sequence, Type +from typing import TYPE_CHECKING, Any, Mapping, Sequence, Type from typing_extensions import Self from nominal_streaming.nominal_dataset_stream import DataType +if TYPE_CHECKING: + import polars as pl + class PyNominalStreamOpts: """Configuration options for Nominal data streaming. @@ -441,3 +444,392 @@ class PyNominalDatasetStream: def __exit__( self, exc_type: Type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None ) -> None: ... + +class NominalAvroWriterOpts: + """Options for :class:`NominalAvroWriter`. + + Configures how points are batched before being handed to the underlying + avro writer, plus whether close() forces an fsync, and optional per-file + rotation. + """ + + def __init__( + self, + *, + max_points_per_batch: int = 250_000, + max_batch_delay_secs: float = 0.1, + fsync_on_close: bool = True, + max_points_per_file: int = 0, + ) -> None: + """Initialize options. + + Args: + max_points_per_batch: Max points packed into one avro record before + the underlying stream's batch-processor emits it. Default: 250,000. + max_batch_delay_secs: Maximum time the underlying stream buffers a + partial batch before forcing a flush. Default: 0.1 (100 ms). + fsync_on_close: Whether close() calls ``File::sync_all()`` before returning. + max_points_per_file: Maximum points per file before rotating to a new + numbered file. ``0`` (default) means no rotation — all data goes + to the single path given to :class:`NominalAvroWriter`. + When > 0, filenames follow ``_`` + (e.g. ``out.avro`` → ``out_000.avro``, ``out_001.avro``, ...). + """ + + @property + def max_points_per_batch(self) -> int: + """Maximum number of points packed into one avro record before the underlying stream emits it.""" + @property + def max_batch_delay_secs(self) -> float: + """Maximum time the underlying stream buffers a partial batch before forcing a flush, in seconds.""" + @property + def fsync_on_close(self) -> bool: + """Whether :meth:`NominalAvroWriter.close` calls ``File::sync_all()`` before returning.""" + @property + def max_points_per_file(self) -> int: + """Maximum points per file before rotation. ``0`` means no rotation.""" + + def with_max_points_per_batch(self, n: int) -> Self: + """Return a copy with ``max_points_per_batch`` set to *n*.""" + def with_max_batch_delay_secs(self, secs: float) -> Self: + """Return a copy with ``max_batch_delay_secs`` set to *secs*.""" + def with_fsync_on_close(self, b: bool) -> Self: + """Return a copy with ``fsync_on_close`` set to *b*.""" + def with_max_points_per_file(self, n: int) -> Self: + """Return a copy with ``max_points_per_file`` set to *n*.""" + + def __repr__(self) -> str: ... + +class NominalAvroWriter: + """Purpose-built avro file writer for the Nominal schema. + + Writes snappy-compressed avro files in the format consumed by the Nominal + Core ingest API. Use this when you want a local file on disk (optionally + rotated into numbered shards); if you want to upload to Core directly + instead, use :class:`PyNominalDatasetStream`. + + Timestamps throughout the API are ``int`` nanoseconds since the Unix + epoch. No automatic conversion from ``datetime`` / ``pl.Datetime`` is + performed — convert up front. + + Example:: + + from nominal_streaming import NominalAvroWriter, NominalAvroWriterOpts + + with NominalAvroWriter("out.avro") as w: + w.write("temperature", ts_ns=1_700_000_000_000_000_000, value=21.3) + w.write_batch( + "speed", + ts_ns=[1_700_000_000_000_000_000, 1_700_000_000_001_000_000], + values=[0.0, 1.2], + ) + # The file is fully flushed + fsync'd on context-manager exit. + + Choosing a write method: + + * :meth:`write` — one scalar point for one channel. Per-call FFI + overhead; fine for low rates. + * :meth:`write_batch` — many points for one channel. Preferred over + a loop of :meth:`write`. + * :meth:`write_from_dict` — one timestamp, many channels (a "row"). + One FFI crossing per row regardless of channel count. + * :meth:`write_dataframe` — many channels × many rows from a polars + ``DataFrame``. Highest-throughput path — prefer when you have the + data in a frame. See its own docstring for full dtype / null / + NaN semantics. + * :meth:`write_struct` / :meth:`write_float_array` / + :meth:`write_string_array` — non-scalar point types. + + Concurrency: + + * Multi-producer safe — any number of threads may call any + ``write_*`` method concurrently. + * Ordering across threads is unspecified; within one producer, order + is preserved. + * The GIL is released for the duration of the Rust-side enqueue + on every ``write_*`` method. + + Errors: + + * ``RuntimeError`` is raised on background encoder failures (disk + I/O, avro encoding, fsync). The first such error is **latched**: + every subsequent ``write_*`` / :meth:`flush` / :meth:`sync` / + :meth:`close` call returns the same error until the writer is + discarded. There is no reset. + * ``RuntimeError`` with message "send after close" is raised from + any ``write_*`` call made after :meth:`close` completed. This is + always caller misuse, not encoder failure. + * ``ValueError`` / ``TypeError`` are raised for malformed input + (unsupported dtype, null in the timestamp column, non-JSON-native + values to :meth:`write_struct`, etc.) — these do not latch. + * :meth:`__exit__` always calls :meth:`close` — even when the body + raised — and will propagate a latched encoder error if the user's + exception did not already trigger one. + """ + + def __init__( + self, + path: pathlib.Path, + opts: NominalAvroWriterOpts | None = None, + ) -> None: + """Open a new writer at ``path``. + + If a file already exists at ``path``, it is **truncated**. When + rotation is configured (``opts.max_points_per_file > 0``), ``path`` + is the template: actual output goes to ``_000``, + ``_001``, ... + + The parent directory is created if it does not exist. + + Raises: + OSError: Parent directory or file cannot be created / truncated. + """ + + def write( + self, + channel: str, + ts_ns: int, + value: int | float | str, + tags: dict[str, str] | None = None, + ) -> None: + """Write a single scalar point (int, float, or str). + + The Python type of ``value`` determines the avro variant: + ``int`` → ``IntegerPoint``, ``float`` → ``DoublePoint``, + ``str`` → ``StringPoint``. To avoid confusion in mixed-int-and-float + streams, prefer the explicit typed methods below for non-scalars. + """ + + def write_batch( + self, + channel: str, + ts_ns: Sequence[int], + values: Sequence[int | float | str], + tags: dict[str, str] | None = None, + ) -> None: + """Write many points of one channel. Values must be homogeneously typed. + + Note: In a batch, integer values are emitted as ``DoublePoint`` (not + ``IntegerPoint``) — the value-kind classifier checks float first for + compatibility with the dataset-stream path. If you need integer + emission, use :meth:`write` (single points), :meth:`write_from_dict` + (per-timestamp dicts), or :meth:`write_dataframe` with an ``Int64`` + column (which always emits ``IntegerPoint``). + """ + + def write_from_dict( + self, + ts_ns: int, + channel_values: dict[str, int | float | str], + tags: dict[str, str] | None = None, + ) -> None: + """Write many channels at one timestamp from a dict. Values are + emitted per their Python type (int → IntegerPoint, etc.). + """ + + def write_struct( + self, + channel: str, + ts_ns: int, + value: Mapping[str, Any], + tags: dict[str, str] | None = None, + ) -> None: + """Write a struct value (JSON-encoded internally). + + Raises: + TypeError: ``value`` contains a non-JSON-native object. + """ + + def write_float_array( + self, + channel: str, + ts_ns: int, + value: Sequence[float], + tags: dict[str, str] | None = None, + ) -> None: + """Write an array-of-doubles value. NaN / ±Inf elements are preserved.""" + + def write_string_array( + self, + channel: str, + ts_ns: int, + value: Sequence[str], + tags: dict[str, str] | None = None, + ) -> None: + """Write an array-of-strings value.""" + + def write_dataframe( + self, + df: "pl.DataFrame", + timestamp_column: str, + tags: dict[str, str] | None = None, + ) -> None: + """Write each non-timestamp column of ``df`` as a separate channel. + + Highest-throughput entry point. The column loop runs Rust-side with + the GIL released; on Apple-Silicon + CPython 3.12 throughput is + roughly 24–30 M points/second for typical shapes (see the branch's + benchmarks/ directory). + + The timestamp column must be ``Int64`` (nanoseconds since epoch). + Other timestamp dtypes (e.g. ``pl.Datetime``) are **not** auto-cast — + convert first with ``df.with_columns(pl.col("ts").cast(pl.Int64))`` + if your frame uses a datetime representation. + + Per-column dispatch: + + * ``Float64`` → ``DoublePoint`` per row. + * ``Int64`` → ``IntegerPoint`` per row. + * ``String`` → ``StringPoint`` per row. + * ``List(*)`` / ``Array(*, _)`` with ``String`` inner → + ``StringArrayPoint`` per row. + * ``List(*)`` / ``Array(*, _)`` with numeric inner → + ``DoubleArrayPoint`` per row (cast to f64). + * ``Struct(…)`` → ``StructPoint`` per row (JSON-serialized). + + Null / NaN / Infinity handling (deliberate — matches the Nominal + Core ingest backend's filtering semantics): + + * **Polars null → the row is skipped.** No avro record is + emitted for that ``(channel, timestamp)`` — the timestamp is + not paired with a sentinel value on any dtype. Skipping + client-side aligns with the backend's row-level + ``Tuple IS NOT NULL`` filter; a null-valued record would be + dropped at insert anyway. + * **Float64 NaN / ±Infinity → preserved as-is.** These are real + IEEE-754 values, not nulls. They round-trip unchanged through + the avro ``double`` branch and ClickHouse's + ``Variant(Float64, …)`` column. + * **Inside a struct or list:** polars-null *elements* become + JSON ``null`` so the containing object's / array's shape + survives. Non-finite floats inside a struct field **also** + encode as JSON ``null`` (not as literal ``NaN`` / ``Infinity`` + tokens) — emitting literals would corrupt downstream queries + that parse the JSON, because ClickHouse's JSON column accepts + them silently at insert but fails on read. + + Raises: + ValueError: The timestamp column is missing, contains nulls, or + has the wrong dtype. + RuntimeError: An unsupported dtype appears (anything not in the + dispatch list above) — or the writer has a latched error. + """ + + def flush(self) -> None: + """Best-effort error surface. Does NOT force a drain of the underlying stream. + + Returns immediately (aside from the latched-error check). For + durability guarantees, call :meth:`close`. + + Raises: + RuntimeError: A background encoder error has been latched. + """ + + def sync(self) -> None: + """Fsync the file path, durably persisting data the stream has already dispatched. + + In-flight buffered data is NOT forced to disk by this call — the + underlying dataset stream has no synchronous drain hook. For strong + durability guarantees, call :meth:`close`. + + Raises: + RuntimeError: fsync failed, or a background encoder error has + been latched. + """ + + def close(self) -> list[pathlib.Path]: + """Gracefully shut down the encoder. Idempotent; blocks until every + already-enqueued point has reached disk. + + When ``max_points_per_file == 0`` (no rotation), returns a + 1-element list containing the single output path. When rotating, + returns all files written, in order. + + After :meth:`close` returns, any ``write_*`` call raises + ``RuntimeError("send after close")``. + + Raises: + RuntimeError: A background encoder error was latched during + shutdown (final flush or fsync failure). The same error is + cached and re-raised on every subsequent :meth:`close` call. + """ + + def points_accepted(self) -> int: + """Total points successfully accepted by ``write_*`` so far, across all files. + + Counts successful enqueues — not on-disk bytes. Points that are + still in the stream's in-memory buffer are included. For on-disk + durability, call :meth:`close`. + """ + + def path(self) -> pathlib.Path: + """The file path currently being written to. + + Without rotation, this is the path given to ``__init__``. With + rotation, it is the currently-open shard. After :meth:`close`, + this returns the path of the last shard that was open. + """ + + def finalized_paths(self) -> list[pathlib.Path]: + """All fully-closed (finalized) file paths, in order. + + Empty until at least one rotation has occurred (or :meth:`close` + is called). The current in-progress file is NOT included — use + :meth:`written_files` if you want that. + """ + + def written_files(self) -> list[pathlib.Path]: + """Every file path this writer has opened, in order. + + Includes the currently-open file while the writer is live; after + :meth:`close`, identical to :meth:`finalized_paths`. + """ + + def stats(self) -> dict[str, int]: + """Snapshot of pipeline timing counters. + + All ``*_ns`` values are nanoseconds accumulated across the writer's + lifetime; ``*_calls`` are counts. + + Keys: + + * ``df_handoff_ns`` — time inside the Arrow C Data Interface + FFI hop (python polars → rust polars) per :meth:`write_dataframe`. + * ``extract_ts_ns`` — time extracting the timestamp column. + * ``column_build_ns`` — time iterating non-timestamp columns + and building Point structs (the producer's main CPU cost). + * ``enqueue_batch_ns`` — wall time the producer spent handing + the batch to the underlying stream. Includes time blocked + waiting for buffer capacity when the consumer can't drain + fast enough — a large value here means the consumer is the + bottleneck. + * ``consumer_consume_ns`` — wall time the background dispatcher + thread spent inside ``consume`` (avro encode + snappy + file + write). Runs in parallel with the producer. + * ``consumer_consume_calls`` — number of ``WriteRequest``s the + dispatcher dispatched. + * ``write_dataframe_calls`` — number of :meth:`write_dataframe` + invocations. + * ``write_dataframe_total_ns`` — total wall time across all + :meth:`write_dataframe` calls (producer-side only). + + Diagnosis rule of thumb: if ``consumer_consume_ns`` is close to the + total wall clock, the avro encoder is the bottleneck; if + ``column_build_ns + df_handoff_ns`` dominates instead, the producer + is. Includes all dtypes the writer has processed over its lifetime — + the counters are cumulative, not per-call. + """ + + def __enter__(self) -> Self: ... + def __exit__( + self, + exc_type: Type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + """Close the writer (always, even on exception in the body). + + A latched encoder error raised during the final close call will + propagate unless the body already raised. If you want to suppress + close-time errors, wrap the ``with`` in a ``try``/``except``. + """ diff --git a/py-nominal-streaming/python/nominal_streaming/_timestamp.py b/py-nominal-streaming/python/nominal_streaming/_timestamp.py new file mode 100644 index 0000000..802d85f --- /dev/null +++ b/py-nominal-streaming/python/nominal_streaming/_timestamp.py @@ -0,0 +1,23 @@ +"""Shared timestamp parsing used by NominalDatasetStream and NominalAvroWriter.""" + +from __future__ import annotations + +import datetime + +import dateutil.parser + +TimestampLike = str | int | datetime.datetime +ScalarValue = int | float | str + + +def _parse_timestamp(ts: str | int | datetime.datetime) -> int: + """Convert a TimestampLike into nanoseconds since the Unix epoch.""" + if isinstance(ts, int): + return ts + elif isinstance(ts, datetime.datetime): + secs = ts.astimezone(datetime.timezone.utc).timestamp() + return int(secs * 1e9) + else: + # TODO(drake): dateutil loses sub-microsecond precision. + secs = dateutil.parser.parse(ts).astimezone(datetime.timezone.utc).timestamp() + return int(secs * 1e9) diff --git a/py-nominal-streaming/python/nominal_streaming/nominal_dataset_stream.py b/py-nominal-streaming/python/nominal_streaming/nominal_dataset_stream.py index 30a7b5c..e1d63f9 100644 --- a/py-nominal-streaming/python/nominal_streaming/nominal_dataset_stream.py +++ b/py-nominal-streaming/python/nominal_streaming/nominal_dataset_stream.py @@ -30,7 +30,6 @@ from __future__ import annotations -import datetime import logging import pathlib import signal @@ -38,31 +37,18 @@ from types import TracebackType from typing import Any, Mapping, Sequence, Type -import dateutil from typing_extensions import Self from nominal_streaming._nominal_streaming import ( PyNominalDatasetStream, PyNominalStreamOpts, ) +from nominal_streaming._timestamp import ScalarValue, TimestampLike, _parse_timestamp -logger = logging.getLogger(__name__) +# DataType is an alias kept for backwards compatibility. +DataType = ScalarValue -TimestampLike = str | int | datetime.datetime -DataType = int | float | str - - -def _parse_timestamp(ts: str | int | datetime.datetime) -> int: - if isinstance(ts, int): - return ts - elif isinstance(ts, datetime.datetime): - secs = ts.astimezone(datetime.timezone.utc).timestamp() - return int(secs * 1e9) - else: - # TODO(drake): by involving dateutil, this chops off any nano level precision provided - # in the timestamp. Update to not lose precision when converting to absolute nanos. - secs = dateutil.parser.parse(ts).astimezone(datetime.timezone.utc).timestamp() - return int(secs * 1e9) +logger = logging.getLogger(__name__) class NominalDatasetStream: diff --git a/py-nominal-streaming/python/tests/__init__.py b/py-nominal-streaming/python/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/py-nominal-streaming/python/tests/test_nominal_avro_writer.py b/py-nominal-streaming/python/tests/test_nominal_avro_writer.py new file mode 100644 index 0000000..4cde340 --- /dev/null +++ b/py-nominal-streaming/python/tests/test_nominal_avro_writer.py @@ -0,0 +1,326 @@ +"""Tests for NominalAvroWriter (including rotation via max_points_per_file).""" + +from __future__ import annotations + +import pathlib + +import fastavro +import polars as pl +from nominal_streaming import ( + NominalAvroWriter, + NominalAvroWriterOpts, +) + + +def _read_records(path: pathlib.Path) -> list[dict]: + with open(path, "rb") as f: + return list(fastavro.reader(f)) + + +def test_single_file_roundtrip_all_value_types(tmp_path: pathlib.Path) -> None: + path = tmp_path / "roundtrip.avro" + with NominalAvroWriter(path) as w: + w.write("dbl", 1_000_000_000, 1.5) + w.write("int", 2_000_000_000, 42) + w.write("str", 3_000_000_000, "hello") + w.write_struct("st", 4_000_000_000, {"a": 1}) + w.write_float_array("fa", 5_000_000_000, [1.0, 2.0]) + w.write_string_array("sa", 6_000_000_000, ["x", "y"]) + + records = _read_records(path) + channels = {r["channel"] for r in records} + assert channels == {"dbl", "int", "str", "st", "fa", "sa"} + for r in records: + assert len(r["timestamps"]) == 1 + + +def test_rotator_rotates_at_max(tmp_path: pathlib.Path) -> None: + path = tmp_path / "roll.avro" + opts = NominalAvroWriterOpts(max_points_per_file=100) + with NominalAvroWriter(path, opts) as w: + for i in range(250): + w.write("x", (i + 1) * 1_000_000_000, float(i)) + + paths = w.close() + assert len(paths) == 3 + assert paths[0].name == "roll_000.avro" + assert paths[1].name == "roll_001.avro" + assert paths[2].name == "roll_002.avro" + + counts = [sum(len(r["timestamps"]) for r in _read_records(p)) for p in paths] + assert counts == [100, 100, 50] + + +def test_rotator_write_batch_straddles_boundary(tmp_path: pathlib.Path) -> None: + path = tmp_path / "straddle.avro" + opts = NominalAvroWriterOpts(max_points_per_file=100) + with NominalAvroWriter(path, opts) as w: + # Pre-fill the first file to 80 points. + for i in range(80): + w.write("x", (i + 1) * 1_000_000_000, float(i)) + assert w.points_accepted() == 80 + assert len(w.finalized_paths()) == 0 + + # Now write a batch of 50 — should split 20 (fill first) + 30 (new file). + batch_ts = [i * 1_000_000_000 for i in range(80, 130)] + batch_v = [float(i) for i in range(80, 130)] + w.write_batch("x", batch_ts, batch_v) + + assert len(w.finalized_paths()) == 1 + # Total across all files: 80 (first file, finalized) + 20 (fills first to 100) + 30 (second file) = 130 + assert w.points_accepted() == 130 + + paths = w.close() + assert len(paths) == 2 + counts = [sum(len(r["timestamps"]) for r in _read_records(p)) for p in paths] + assert counts == [100, 30] + + +def test_write_dataframe_dispatches_per_dtype(tmp_path: pathlib.Path) -> None: + path = tmp_path / "df.avro" + df = pl.DataFrame( + { + "ts": [1_000_000_000, 2_000_000_000, 3_000_000_000], + "speed": [1.0, 2.0, 3.0], + "label": ["a", "b", "c"], + "arr": [[1.0, 2.0], None, [3.0]], + "meta": [{"k": 1}, None, {"k": 3}], + } + ) + with NominalAvroWriter(path) as w: + w.write_dataframe(df, "ts") + + records = _read_records(path) + by_channel = {} + for r in records: + by_channel.setdefault(r["channel"], []).extend(zip(r["timestamps"], r["values"])) + + assert len(by_channel["speed"]) == 3 + assert len(by_channel["label"]) == 3 + assert len(by_channel["arr"]) == 2 + assert len(by_channel["meta"]) == 2 + + +def test_written_files_observability(tmp_path: pathlib.Path) -> None: + # NominalAvroWriter (no rotation): current path is visible via .path() while open. + single = tmp_path / "single.avro" + w1 = NominalAvroWriter(single) + w1.write("x", 1, 1.0) + assert pathlib.Path(w1.path()) == single + closed_paths = w1.close() + assert closed_paths == [single] + + # Rotating writer: mid-rotation, should see both finalized and current via written_files(). + base = tmp_path / "rot.avro" + opts = NominalAvroWriterOpts(max_points_per_file=100) + w2 = NominalAvroWriter(base, opts) + for i in range(150): + w2.write("x", (i + 1) * 1_000_000_000, float(i)) + files_before_close = list(w2.written_files()) + assert len(files_before_close) == 2 + assert files_before_close[0].name == "rot_000.avro" + assert files_before_close[1].name == "rot_001.avro" + + w2.close() + files_after_close = list(w2.written_files()) + assert files_after_close == files_before_close + + +def test_context_manager_finalizes(tmp_path: pathlib.Path) -> None: + path = tmp_path / "ctx.avro" + with NominalAvroWriter(path) as w: + w.write("x", 1, 1.0) + + # After __exit__, the file is finalized and data is on disk. + assert path.exists() + + records = _read_records(path) + assert len(records) == 1 + + +def test_rotator_context_manager(tmp_path: pathlib.Path) -> None: + path = tmp_path / "ctxr.avro" + opts = NominalAvroWriterOpts(max_points_per_file=10) + with NominalAvroWriter(path, opts) as w: + for i in range(25): + w.write("x", (i + 1) * 1_000_000_000, float(i)) + + paths = w.close() + assert len(paths) == 3 + counts = [sum(len(r["timestamps"]) for r in _read_records(p)) for p in paths] + assert counts == [10, 10, 5] + + +def test_flush_and_sync_are_callable_best_effort(tmp_path: pathlib.Path) -> None: + """flush() and sync() are best-effort after the NominalDatasetStream + delegation — they don't force a drain. Test only confirms that both + calls return without error and the file contains the written data + after close(). + """ + path = tmp_path / "fs.avro" + with NominalAvroWriter(path) as w: + w.write("ch", 1_000_000_000, 1.0) + w.write("ch", 2_000_000_000, 2.0) + w.flush() # no-op aside from error check + w.sync() # fsync whatever has dispatched + w.write("ch", 3_000_000_000, 3.0) + + # The file exists and all 3 points are recoverable after close. + records = _read_records(path) + total = sum(len(r["timestamps"]) for r in records) + assert total == 3 + + +def _read_points_by_channel(path: pathlib.Path) -> dict[str, list[tuple[int, object]]]: + """Helper: flatten an avro file to {channel: [(ts, value), ...]} preserving order.""" + by_channel: dict[str, list[tuple[int, object]]] = {} + for r in _read_records(path): + by_channel.setdefault(r["channel"], []).extend(zip(r["timestamps"], r["values"])) + return by_channel + + +def test_float_null_skipped_nan_preserved_inf_preserved(tmp_path: pathlib.Path) -> None: + """Float64: polars null → row skipped; NaN and ±Inf → preserved as IEEE-754.""" + import math + + path = tmp_path / "float_edges.avro" + df = pl.DataFrame( + { + "ts": [1, 2, 3, 4, 5], # ns since epoch + "x": [1.0, float("nan"), None, float("inf"), float("-inf")], + } + ) + with NominalAvroWriter(path) as w: + w.write_dataframe(df, "ts") + + pts = _read_points_by_channel(path)["x"] + # Null at ts=3 → skipped. Everything else → emitted with its value. + tss = [ts for ts, _ in pts] + assert tss == [1, 2, 4, 5], f"expected nulls skipped, got timestamps {tss}" + + vals = [v for _, v in pts] + assert vals[0] == 1.0 + assert math.isnan(vals[1]), f"NaN should round-trip, got {vals[1]!r}" + assert vals[2] == float("inf"), f"+Inf should round-trip, got {vals[2]!r}" + assert vals[3] == float("-inf"), f"-Inf should round-trip, got {vals[3]!r}" + + +def test_int_null_skipped(tmp_path: pathlib.Path) -> None: + """Int64: polars null → row skipped (no sentinel zero).""" + path = tmp_path / "int_nulls.avro" + df = pl.DataFrame( + { + "ts": [1, 2, 3, 4], + "n": pl.Series("n", [10, None, 20, None], dtype=pl.Int64), + } + ) + with NominalAvroWriter(path) as w: + w.write_dataframe(df, "ts") + + pts = _read_points_by_channel(path)["n"] + assert [ts for ts, _ in pts] == [1, 3] + assert [v for _, v in pts] == [10, 20] + + +def test_string_null_skipped(tmp_path: pathlib.Path) -> None: + """String: polars null → row skipped (no sentinel empty-string).""" + path = tmp_path / "str_nulls.avro" + df = pl.DataFrame( + { + "ts": [1, 2, 3], + "s": pl.Series("s", ["a", None, "c"], dtype=pl.String), + } + ) + with NominalAvroWriter(path) as w: + w.write_dataframe(df, "ts") + + pts = _read_points_by_channel(path)["s"] + assert [ts for ts, _ in pts] == [1, 3] + assert [v for _, v in pts] == ["a", "c"] + + +def test_float_all_nulls_column_emits_nothing(tmp_path: pathlib.Path) -> None: + """Float64 column that is entirely null → no records emitted for that channel.""" + path = tmp_path / "all_null.avro" + df = pl.DataFrame( + { + "ts": [1, 2, 3], + "x": pl.Series("x", [None, None, None], dtype=pl.Float64), + "y": pl.Series("y", [1.0, 2.0, 3.0], dtype=pl.Float64), + } + ) + with NominalAvroWriter(path) as w: + w.write_dataframe(df, "ts") + + by_channel = _read_points_by_channel(path) + assert "x" not in by_channel, "all-null column should emit no records" + assert len(by_channel["y"]) == 3 + + +def test_struct_field_non_finite_is_json_null(tmp_path: pathlib.Path) -> None: + """Struct fields with NaN/Inf emit JSON `null` (preserves structure); they + are NOT silently dropped from the resulting JSON object. + """ + import json + + path = tmp_path / "struct_nonfinite.avro" + schema = pl.Struct([pl.Field("a", pl.Int64), pl.Field("b", pl.Float64)]) + df = pl.DataFrame( + { + "ts": [1, 2, 3], + "s": pl.Series( + "s", + [ + {"a": 1, "b": 1.5}, + {"a": 2, "b": float("nan")}, + {"a": 3, "b": float("inf")}, + ], + dtype=schema, + ), + } + ) + with NominalAvroWriter(path) as w: + w.write_dataframe(df, "ts") + + pts = _read_points_by_channel(path)["s"] + assert len(pts) == 3 + parsed = [json.loads(v["json"]) for _, v in pts] + assert parsed[0] == {"a": 1, "b": 1.5} + # The "b" key is preserved even when the value is non-finite — it becomes null. + assert parsed[1] == {"a": 2, "b": None} + assert parsed[2] == {"a": 3, "b": None} + + +def test_struct_null_row_skipped(tmp_path: pathlib.Path) -> None: + """Struct: polars-null row → no record emitted at that timestamp.""" + import json + + path = tmp_path / "struct_null_row.avro" + schema = pl.Struct([pl.Field("k", pl.Int64)]) + df = pl.DataFrame( + { + "ts": [1, 2, 3], + "s": pl.Series("s", [{"k": 1}, None, {"k": 3}], dtype=schema), + } + ) + with NominalAvroWriter(path) as w: + w.write_dataframe(df, "ts") + + pts = _read_points_by_channel(path)["s"] + assert [ts for ts, _ in pts] == [1, 3] + assert [json.loads(v["json"]) for _, v in pts] == [{"k": 1}, {"k": 3}] + + +def test_timestamp_null_is_hard_error(tmp_path: pathlib.Path) -> None: + """A null in the timestamp column must raise — no silent handling.""" + import pytest + + path = tmp_path / "ts_null.avro" + df = pl.DataFrame( + { + "ts": pl.Series("ts", [1, None, 3], dtype=pl.Int64), + "x": [1.0, 2.0, 3.0], + } + ) + with NominalAvroWriter(path) as w, pytest.raises(ValueError, match="null in timestamp"): + w.write_dataframe(df, "ts") diff --git a/py-nominal-streaming/src/avro_writer/error_map.rs b/py-nominal-streaming/src/avro_writer/error_map.rs new file mode 100644 index 0000000..fa852d8 --- /dev/null +++ b/py-nominal-streaming/src/avro_writer/error_map.rs @@ -0,0 +1,14 @@ +use std::sync::Arc; + +use nominal_streaming::avro_writer::AvroWriterError; +use pyo3::exceptions::PyRuntimeError; +use pyo3::PyErr; + +/// Convert a latched `Arc` into a Python `RuntimeError`. +/// +/// The `Arc` here is the same sticky error shared across all callers of a +/// closed-or-failed writer — we format it to a message; Python code sees +/// a `RuntimeError`. +pub(super) fn map_err(e: Arc) -> PyErr { + PyRuntimeError::new_err(format!("{}", e)) +} diff --git a/py-nominal-streaming/src/avro_writer/mod.rs b/py-nominal-streaming/src/avro_writer/mod.rs new file mode 100644 index 0000000..17faea4 --- /dev/null +++ b/py-nominal-streaming/src/avro_writer/mod.rs @@ -0,0 +1,24 @@ +//! PyO3 facade for [`nominal_streaming::avro_writer::AvroWriter`]. +//! +//! This module provides [`NominalAvroWriter`] and [`NominalAvroWriterOpts`], +//! the Python-facing pyclass wrappers around the pure-Rust `AvroWriter` / +//! `AvroWriterOpts` types that live in the `nominal-streaming` crate. The +//! pure-Rust core is enabled with the `polars` feature (transitively, via +//! py-nominal-streaming's Cargo.toml) so `write_dataframe` is available. +//! +//! # Example (Python) +//! +//! ```python +//! from nominal_streaming import NominalAvroWriter, NominalAvroWriterOpts +//! with NominalAvroWriter("out.avro") as w: +//! w.write("speed", ts_ns, 3.14) +//! ``` + +mod error_map; +mod opts; +mod polars_ffi; +mod pymethods; +mod writer; + +pub use opts::NominalAvroWriterOpts; +pub use writer::NominalAvroWriter; diff --git a/py-nominal-streaming/src/avro_writer/opts.rs b/py-nominal-streaming/src/avro_writer/opts.rs new file mode 100644 index 0000000..44404e2 --- /dev/null +++ b/py-nominal-streaming/src/avro_writer/opts.rs @@ -0,0 +1,103 @@ +use std::time::Duration; + +use nominal_streaming::avro_writer::AvroWriterOpts; +use pyo3::prelude::*; + +/// Configuration for [`super::NominalAvroWriter`]. +/// +/// Thin wrapper around [`nominal_streaming::avro_writer::AvroWriterOpts`] — +/// all fields have sensible defaults. Exposed directly as a PyO3 class. +#[pyclass( + name = "NominalAvroWriterOpts", + module = "nominal_streaming._nominal_streaming" +)] +#[derive(Debug, Clone)] +pub struct NominalAvroWriterOpts { + pub(super) inner: AvroWriterOpts, +} + +impl Default for NominalAvroWriterOpts { + fn default() -> Self { + Self { + inner: AvroWriterOpts::default(), + } + } +} + +#[pymethods] +impl NominalAvroWriterOpts { + #[new] + #[pyo3(signature = (*, max_points_per_batch = 250_000, max_batch_delay_secs = 0.1, fsync_on_close = true, max_points_per_file = 0))] + pub fn py_new( + max_points_per_batch: usize, + max_batch_delay_secs: f64, + fsync_on_close: bool, + max_points_per_file: usize, + ) -> Self { + Self { + inner: AvroWriterOpts { + max_points_per_batch, + max_batch_delay: Duration::from_secs_f64(max_batch_delay_secs), + fsync_on_close, + max_points_per_file, + }, + } + } + + #[getter] + fn max_points_per_batch(&self) -> usize { + self.inner.max_points_per_batch + } + + #[getter] + fn max_batch_delay_secs(&self) -> f64 { + self.inner.max_batch_delay.as_secs_f64() + } + + #[getter] + fn fsync_on_close(&self) -> bool { + self.inner.fsync_on_close + } + + #[getter] + fn max_points_per_file(&self) -> usize { + self.inner.max_points_per_file + } + + fn with_max_points_per_batch(&self, n: usize) -> Self { + Self { + inner: self.inner.clone().with_max_points_per_batch(n), + } + } + + fn with_max_batch_delay_secs(&self, secs: f64) -> Self { + Self { + inner: self + .inner + .clone() + .with_max_batch_delay(Duration::from_secs_f64(secs)), + } + } + + fn with_fsync_on_close(&self, b: bool) -> Self { + Self { + inner: self.inner.clone().with_fsync_on_close(b), + } + } + + fn with_max_points_per_file(&self, n: usize) -> Self { + Self { + inner: self.inner.clone().with_max_points_per_file(n), + } + } + + fn __repr__(&self) -> String { + format!( + "NominalAvroWriterOpts(max_points_per_batch={}, max_batch_delay_secs={:.3}, fsync_on_close={}, max_points_per_file={})", + self.inner.max_points_per_batch, + self.inner.max_batch_delay.as_secs_f64(), + self.inner.fsync_on_close, + self.inner.max_points_per_file, + ) + } +} diff --git a/py-nominal-streaming/src/avro_writer/polars_ffi.rs b/py-nominal-streaming/src/avro_writer/polars_ffi.rs new file mode 100644 index 0000000..87aba20 --- /dev/null +++ b/py-nominal-streaming/src/avro_writer/polars_ffi.rs @@ -0,0 +1,76 @@ +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::*; + +/// Convert a Python polars Series → Rust polars Series via Arrow C Data Interface. +/// +/// This uses `_s._export_arrow_to_c(array_ptr, schema_ptr)` on the Python +/// side — the native Rust object embedded in the Python polars Series. This +/// path does NOT require `pyarrow`; it is a direct memory transfer via the +/// Arrow C Data Interface (capsules / raw pointers). +/// +/// Works with any Python polars version that exposes `_export_arrow_to_c` +/// on its inner `_s` object (i.e., all Python polars 1.x releases). +pub(super) fn py_series_to_rust( + _py: Python<'_>, + py_series: &Bound<'_, PyAny>, +) -> PyResult { + use polars::prelude::*; + use polars_arrow::ffi; + + // Rechunk to ensure a single contiguous Arrow buffer before export. + let py_series = py_series.call_method0("rechunk")?; + // Get the series name for the resulting Rust Series. + let name: String = py_series.getattr("name")?.extract()?; + + // Access the native Rust PySeries object embedded in the Python Series. + let ps = py_series.getattr("_s")?; + + // Allocate empty Arrow C Data Interface structs on the Rust heap. + let array = Box::new(ffi::ArrowArray::empty()); + let schema = Box::new(ffi::ArrowSchema::empty()); + + let array_ptr = &*array as *const ffi::ArrowArray as usize; + let schema_ptr = &*schema as *const ffi::ArrowSchema as usize; + + // Ask polars' native object to fill those structs. + // `_export_arrow_to_c(array_uintptr, schema_uintptr)` is the same C Data + // Interface export that polars uses internally. + ps.call_method1("_export_arrow_to_c", (array_ptr, schema_ptr))?; + + // Now import the filled structs back into Rust types. + let field = unsafe { ffi::import_field_from_c(schema.as_ref()) } + .map_err(|e| PyRuntimeError::new_err(format!("Arrow schema import failed: {e}")))?; + let array = unsafe { ffi::import_array_from_c(*array, field.dtype.clone()) } + .map_err(|e| PyRuntimeError::new_err(format!("Arrow array import failed: {e}")))?; + + // Build a Rust Series from the imported Arrow array + field. + let series = Series::try_from((&field, array)) + .map_err(|e| PyRuntimeError::new_err(format!("Series construction failed: {e}")))?; + + // Rename to match original series name (field name from Arrow may differ). + Ok(series.with_name(PlSmallStr::from(name.as_str()))) +} + +/// Convert a Python polars DataFrame → Rust polars DataFrame column-by-column +/// via `py_series_to_rust`. +pub(super) fn py_df_to_rust( + py: Python<'_>, + py_df: &Bound<'_, PyAny>, +) -> PyResult { + use polars::prelude::*; + + let py_columns = py_df.call_method0("get_columns")?; + let n: usize = py_df.getattr("width")?.extract()?; + let mut columns: Vec = Vec::with_capacity(n); + for py_col in py_columns.try_iter()? { + let py_col = py_col?; + let series = py_series_to_rust(py, &py_col)?; + columns.push(series.into_column()); + } + // SAFETY: we trust the Python polars invariants (same length across columns). + Ok(unsafe { DataFrame::new_no_checks_height_from_first(columns) }) +} + +// `anyvalue_to_json_value` / `anyvalue_to_json` now live in the +// nominal-streaming crate under `avro_writer::polars` (feature-gated) — +// they're pure Rust and don't need to be here. diff --git a/py-nominal-streaming/src/avro_writer/pymethods.rs b/py-nominal-streaming/src/avro_writer/pymethods.rs new file mode 100644 index 0000000..356dfe3 --- /dev/null +++ b/py-nominal-streaming/src/avro_writer/pymethods.rs @@ -0,0 +1,419 @@ +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::atomic::Ordering; + +use nominal_api::tonic::io::nominal::scout::api::proto::points::PointsType; +use nominal_api::tonic::io::nominal::scout::api::proto::DoubleArrayPoint; +use nominal_api::tonic::io::nominal::scout::api::proto::DoublePoint; +use nominal_api::tonic::io::nominal::scout::api::proto::IntegerPoint; +use nominal_api::tonic::io::nominal::scout::api::proto::StringArrayPoint; +use nominal_api::tonic::io::nominal::scout::api::proto::StringPoint; +use nominal_api::tonic::io::nominal::scout::api::proto::StructPoint; +use nominal_streaming::avro_writer::AvroWriter; +use nominal_streaming::types::ChannelDescriptor; +use nominal_streaming::types::IntoPoints; +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::*; + +use super::error_map::map_err; +use super::opts::NominalAvroWriterOpts; +use super::polars_ffi::py_df_to_rust; +use super::writer::NominalAvroWriter; +use crate::point::classify_values; +use crate::point::description_with_tags; +use crate::point::extract_vec_f64; +use crate::point::extract_vec_i64; +use crate::point::extract_vec_string; +use crate::point::extract_vec_ts; +use crate::point::parse_timestamp; +use crate::point::ValueKind; + +#[pymethods] +impl NominalAvroWriter { + #[new] + #[pyo3(signature = (path, opts = None))] + fn py_new(path: PathBuf, opts: Option) -> PyResult { + let opts = opts.unwrap_or_default().inner.clone(); + let inner = AvroWriter::new(path, opts) + .map_err(|e| PyRuntimeError::new_err(format!("failed to open avro writer: {e}")))?; + Ok(Self { inner }) + } + + #[pyo3(name = "close")] + fn py_close(&self, py: Python<'_>) -> PyResult> { + py.detach(|| self.inner.close().map_err(map_err)) + } + + #[pyo3(name = "points_accepted")] + fn py_points_accepted(&self) -> u64 { + self.inner.points_accepted() + } + + #[pyo3(name = "path")] + fn py_path(&self) -> PathBuf { + self.inner.path() + } + + #[pyo3(name = "finalized_paths")] + fn py_finalized_paths(&self) -> Vec { + self.inner.finalized_paths() + } + + #[pyo3(name = "written_files")] + fn py_written_files(&self) -> Vec { + self.inner.written_files() + } + + fn __enter__(slf: Py) -> Py { + slf + } + + fn __exit__( + &self, + py: Python<'_>, + _exc_type: Py, + _exc_value: Py, + _traceback: Py, + ) -> PyResult<()> { + py.detach(|| self.inner.close().map(|_paths| ()).map_err(map_err)) + } + + // All per-point Python writers funnel into the single internal + // `write_batch` entry point: extract values from the Python layer, + // build a `(ChannelDescriptor, PointsType)` pair (or a batch of them + // for wide writes), then one GIL-released `write_batch` call handles + // error-latching, close-guard, rotate-before, stream hand-off, and + // counter updates. No other path reaches the underlying stream. + #[pyo3(signature = (channel, ts_ns, value, tags = None))] + fn write( + &self, + py: Python<'_>, + channel: &str, + ts_ns: u64, + value: Bound<'_, PyAny>, + tags: Option>, + ) -> PyResult<()> { + let descriptor = description_with_tags(channel, tags); + let ts = parse_timestamp(ts_ns); + // Check int first so ints don't get routed to Double via f64 extraction. + let points = if let Ok(i) = value.extract::() { + vec![IntegerPoint { + timestamp: Some(ts), + value: i, + }] + .into_points() + } else if let Ok(f) = value.extract::() { + vec![DoublePoint { + timestamp: Some(ts), + value: f, + }] + .into_points() + } else if let Ok(s) = value.extract::() { + vec![StringPoint { + timestamp: Some(ts), + value: s, + }] + .into_points() + } else { + return Err(pyo3::exceptions::PyTypeError::new_err( + "value must be int, float, or str", + )); + }; + py.detach(|| { + self.inner + .write_batch(vec![(descriptor, points)]) + .map_err(map_err) + }) + } + + #[pyo3(signature = (channel, ts_ns, values, tags = None))] + fn write_batch( + &self, + py: Python<'_>, + channel: &str, + ts_ns: Vec, + values: Bound<'_, PyAny>, + tags: Option>, + ) -> PyResult<()> { + let descriptor = description_with_tags(channel, tags); + let timestamps = extract_vec_ts(ts_ns); + let kind = classify_values(&values)?; + + // Build the typed point vec, then hand it to `AvroWriter::write` + // which internally loops over `write_batch` splitting on rotation + // boundaries. Keeps files tight when callers pass large batches + // with a small `max_points_per_file`. + match kind { + ValueKind::Floats => { + let vs = extract_vec_f64(&values)?; + if vs.len() != timestamps.len() { + return Err(pyo3::exceptions::PyValueError::new_err( + "timestamps and values must have same length", + )); + } + let points: Vec = timestamps + .into_iter() + .zip(vs) + .map(|(t, v)| DoublePoint { + timestamp: Some(t), + value: v, + }) + .collect(); + py.detach(|| self.inner.write(&descriptor, points).map_err(map_err)) + } + ValueKind::Ints => { + let vs = extract_vec_i64(&values)?; + if vs.len() != timestamps.len() { + return Err(pyo3::exceptions::PyValueError::new_err( + "timestamps and values must have same length", + )); + } + let points: Vec = timestamps + .into_iter() + .zip(vs) + .map(|(t, v)| IntegerPoint { + timestamp: Some(t), + value: v, + }) + .collect(); + py.detach(|| self.inner.write(&descriptor, points).map_err(map_err)) + } + ValueKind::Strings => { + let vs = extract_vec_string(&values)?; + if vs.len() != timestamps.len() { + return Err(pyo3::exceptions::PyValueError::new_err( + "timestamps and values must have same length", + )); + } + let points: Vec = timestamps + .into_iter() + .zip(vs) + .map(|(t, v)| StringPoint { + timestamp: Some(t), + value: v, + }) + .collect(); + py.detach(|| self.inner.write(&descriptor, points).map_err(map_err)) + } + } + } + + #[pyo3(signature = (ts_ns, channel_values, tags = None))] + fn write_from_dict( + &self, + py: Python<'_>, + ts_ns: u64, + channel_values: HashMap>, + tags: Option>, + ) -> PyResult<()> { + // Collect every channel's single point into one batch, then dispatch + // in a single `write_batch` call — one outer-lock acquisition total. + let ts = parse_timestamp(ts_ns); + let mut batch: Vec<(ChannelDescriptor, PointsType)> = + Vec::with_capacity(channel_values.len()); + for (name, value) in channel_values { + let descriptor = description_with_tags(&name, tags.clone()); + let points = if let Ok(i) = value.extract::() { + vec![IntegerPoint { + timestamp: Some(ts), + value: i, + }] + .into_points() + } else if let Ok(f) = value.extract::() { + vec![DoublePoint { + timestamp: Some(ts), + value: f, + }] + .into_points() + } else if let Ok(s) = value.extract::() { + vec![StringPoint { + timestamp: Some(ts), + value: s, + }] + .into_points() + } else { + return Err(pyo3::exceptions::PyTypeError::new_err( + "values must be int, float, or str", + )); + }; + batch.push((descriptor, points)); + } + py.detach(|| self.inner.write_batch(batch).map_err(map_err)) + } + + #[pyo3(signature = (channel, ts_ns, value, tags = None))] + fn write_struct( + &self, + py: Python<'_>, + channel: &str, + ts_ns: u64, + value: Bound<'_, PyAny>, + tags: Option>, + ) -> PyResult<()> { + // Serialize the Python mapping to JSON using json.dumps(allow_nan=False). + let json_module = py.import("json")?; + let dumps = json_module.getattr("dumps")?; + let kwargs = pyo3::types::PyDict::new(py); + kwargs.set_item("allow_nan", false)?; + let json_str: String = dumps.call((value,), Some(&kwargs))?.extract()?; + let descriptor = description_with_tags(channel, tags); + let points = vec![StructPoint { + timestamp: Some(parse_timestamp(ts_ns)), + json_string: json_str, + }] + .into_points(); + py.detach(|| { + self.inner + .write_batch(vec![(descriptor, points)]) + .map_err(map_err) + }) + } + + #[pyo3(signature = (channel, ts_ns, value, tags = None))] + fn write_float_array( + &self, + py: Python<'_>, + channel: &str, + ts_ns: u64, + value: Vec, + tags: Option>, + ) -> PyResult<()> { + let descriptor = description_with_tags(channel, tags); + let points = vec![DoubleArrayPoint { + timestamp: Some(parse_timestamp(ts_ns)), + value, + }] + .into_points(); + py.detach(|| { + self.inner + .write_batch(vec![(descriptor, points)]) + .map_err(map_err) + }) + } + + #[pyo3(signature = (channel, ts_ns, value, tags = None))] + fn write_string_array( + &self, + py: Python<'_>, + channel: &str, + ts_ns: u64, + value: Vec, + tags: Option>, + ) -> PyResult<()> { + let descriptor = description_with_tags(channel, tags); + let points = vec![StringArrayPoint { + timestamp: Some(parse_timestamp(ts_ns)), + value, + }] + .into_points(); + py.detach(|| { + self.inner + .write_batch(vec![(descriptor, points)]) + .map_err(map_err) + }) + } + + #[pyo3(name = "flush")] + fn py_flush(&self, py: Python<'_>) -> PyResult<()> { + py.detach(|| self.inner.flush().map_err(map_err)) + } + + #[pyo3(name = "sync")] + fn py_sync(&self, py: Python<'_>) -> PyResult<()> { + py.detach(|| self.inner.sync().map_err(map_err)) + } + + /// Snapshot of pipeline timing counters (nanoseconds, except `*_calls`). + /// Values accumulate across the lifetime of this writer. See + /// [`nominal_streaming::avro_writer::PipelineStats`] for field + /// semantics. + /// + /// Used for throughput diagnosis — compare `consumer_consume_ns` (work on + /// the single dispatcher thread, in parallel with the producer) against + /// `write_dataframe_total_ns` to see whether the consumer is saturated + /// (→ encoder bottleneck) or idle (→ producer bottleneck). + fn stats(&self) -> HashMap<&'static str, u64> { + let s = self.inner.stats(); + HashMap::from([ + ("df_handoff_ns", s.df_handoff_ns.load(Ordering::Relaxed)), + ("extract_ts_ns", s.extract_ts_ns.load(Ordering::Relaxed)), + ("column_build_ns", s.column_build_ns.load(Ordering::Relaxed)), + ( + "enqueue_batch_ns", + s.enqueue_batch_ns.load(Ordering::Relaxed), + ), + ( + "consumer_consume_ns", + s.consumer_consume_ns.load(Ordering::Relaxed), + ), + ( + "consumer_consume_calls", + s.consumer_consume_calls.load(Ordering::Relaxed), + ), + ( + "write_dataframe_calls", + s.write_dataframe_calls.load(Ordering::Relaxed), + ), + ( + "write_dataframe_total_ns", + s.write_dataframe_total_ns.load(Ordering::Relaxed), + ), + ]) + } + + /// Write each non-timestamp column of a polars DataFrame as a separate channel. + /// + /// See [`nominal_streaming::avro_writer::AvroWriter::write_dataframe`] for + /// the full contract, including null / NaN / Infinity handling and the + /// coupling with the Nominal Core ingest backend. + #[pyo3(signature = (df, timestamp_column, tags = None))] + fn write_dataframe( + &self, + py: Python<'_>, + df: Bound<'_, PyAny>, + timestamp_column: &str, + tags: Option>, + ) -> PyResult<()> { + use pyo3::exceptions::PyTypeError; + use pyo3::exceptions::PyValueError; + + let stats = self.inner.stats(); + + // Convert Python polars DataFrame → Rust DataFrame using Arrow C Data Interface. + // This avoids the pyarrow dependency that pyo3-polars' to_arrow() path requires. + let t0 = std::time::Instant::now(); + let df = py_df_to_rust(py, &df)?; + stats + .df_handoff_ns + .fetch_add(t0.elapsed().as_nanos() as u64, Ordering::Relaxed); + + // Pre-validate the timestamp column on the Python side so callers see + // `ValueError` / `TypeError` (not `RuntimeError`) for malformed input. + // The underlying `AvroWriter::write_dataframe` also validates, but its + // `AvroWriterError::Consumer` variant maps to Python `RuntimeError` — + // the public Python contract (established by these tests) is that a + // missing / wrong-dtype / null timestamp raises the Python-standard + // lookup/type/value errors. + { + let ts_col = df + .column(timestamp_column) + .map_err(|e| PyValueError::new_err(format!("timestamp column: {e}")))?; + let ts_i64 = ts_col.i64().map_err(|_| { + PyTypeError::new_err(format!( + "timestamp column {:?} must be Int64, got {:?}", + timestamp_column, + ts_col.dtype() + )) + })?; + if ts_i64.null_count() > 0 { + return Err(PyValueError::new_err("null in timestamp column")); + } + } + + // Release GIL for the column iteration + enqueue (all pure-Rust in + // the core crate). + let inner = self.inner.clone(); + py.detach(move || inner.write_dataframe(&df, timestamp_column, tags)) + .map_err(map_err) + } +} diff --git a/py-nominal-streaming/src/avro_writer/writer.rs b/py-nominal-streaming/src/avro_writer/writer.rs new file mode 100644 index 0000000..c1e78fb --- /dev/null +++ b/py-nominal-streaming/src/avro_writer/writer.rs @@ -0,0 +1,32 @@ +use nominal_streaming::avro_writer::AvroWriter; +use pyo3::prelude::*; + +/// Purpose-built avro file writer for the Nominal schema. +/// +/// Thin PyO3 facade over [`nominal_streaming::avro_writer::AvroWriter`] — +/// all pipeline logic (multi-producer enqueue, error latching, rotation, +/// fsync-on-close) lives in the `nominal-streaming` crate. This struct only +/// exists to carry `#[pyclass]` + host the `#[pymethods]` impl in +/// [`super::pymethods`]. +/// +/// # Concurrency +/// +/// - Multi-producer: any number of threads may call `write` / `write_batch` +/// concurrently. +/// - Ordering: not guaranteed across threads. Within a single producer, +/// order is preserved. +/// - Backpressure: writes block when the stream's internal buffers are full. +/// +/// # Errors +/// +/// Background encoder errors (I/O, avro encoding, fsync) are latched on first +/// occurrence. Every subsequent `write` / `flush` / `sync` / `close` returns +/// the same latched error as a Python `RuntimeError`. +#[pyclass( + name = "NominalAvroWriter", + module = "nominal_streaming._nominal_streaming" +)] +#[derive(Clone)] +pub struct NominalAvroWriter { + pub(super) inner: AvroWriter, +} diff --git a/py-nominal-streaming/src/lib.rs b/py-nominal-streaming/src/lib.rs index 373e09d..80af5eb 100644 --- a/py-nominal-streaming/src/lib.rs +++ b/py-nominal-streaming/src/lib.rs @@ -1,8 +1,16 @@ -//! Top-level entrypoint for exposing Rust streaming code into python +//! Top-level entrypoint for exposing Rust streaming code to Python. +//! +//! The pure-Rust pipeline lives in the `nominal-streaming` crate. Each +//! `#[pyclass]` here is a thin facade that wraps a Rust type from the core +//! and delegates to it. +//! //! Exposes: -//! - PyNominalStreamOpts Settings builder object to pass configuration to rust -//! - PyNominalDatasetStream Wrapper around rust streaming manager, with tweaks to enable pythonic usage +//! - `PyNominalStreamOpts` settings builder for the streaming manager +//! - `PyNominalDatasetStream` wrapper around the streaming manager with Pythonic tweaks +//! - `NominalAvroWriter` pyclass facade over `nominal_streaming::avro_writer::AvroWriter` +//! - `NominalAvroWriterOpts` pyclass facade over `nominal_streaming::avro_writer::AvroWriterOpts` +mod avro_writer; mod lazy_dataset_stream_builder; mod nominal_dataset_stream; mod nominal_stream_opts; @@ -15,5 +23,7 @@ use pyo3::prelude::*; fn nominal_streaming(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; + m.add_class::()?; + m.add_class::()?; Ok(()) } diff --git a/py-nominal-streaming/src/point.rs b/py-nominal-streaming/src/point.rs index 84494c7..3dcfdc2 100644 --- a/py-nominal-streaming/src/point.rs +++ b/py-nominal-streaming/src/point.rs @@ -213,6 +213,11 @@ pub enum ValueKind { /// Peek the first element to decide the homogeneous value kind. /// (Full extraction to Vec will still enforce homogeneity.) +/// +/// Checks `f64` before `i64` for compatibility with +/// `NominalDatasetStream.enqueue_batch`'s historical behavior. Callers that +/// need integer emission in a batch path can't get it through this helper; +/// use single-point or dict emission. pub fn classify_values(values: &Bound<'_, PyAny>) -> PyResult { let seq = values.downcast::()?; let len = seq.len()?; diff --git a/py-nominal-streaming/uv.lock b/py-nominal-streaming/uv.lock new file mode 100644 index 0000000..167d61c --- /dev/null +++ b/py-nominal-streaming/uv.lock @@ -0,0 +1,377 @@ +version = 1 +revision = 3 +requires-python = ">=3.10" + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "cramjam" +version = "2.11.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/14/12/34bf6e840a79130dfd0da7badfb6f7810b8fcfd60e75b0539372667b41b6/cramjam-2.11.0.tar.gz", hash = "sha256:5c82500ed91605c2d9781380b378397012e25127e89d64f460fea6aeac4389b4", size = 99100, upload-time = "2025-07-27T21:25:07.559Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/53/d3/20d0402e4e983b66603117ad3dd3b864a05d7997a830206d3ff9cacef9a2/cramjam-2.11.0-cp310-cp310-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:d0859c65775e8ebf2cbc084bfd51bd0ffda10266da6f9306451123b89f8e5a63", size = 3558999, upload-time = "2025-07-27T21:21:34.105Z" }, + { url = "https://files.pythonhosted.org/packages/f5/a8/a6e2744288938ccd320a5c6f6f3653faa790f933f5edd088c6e5782a2354/cramjam-2.11.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:1d77b9b0aca02a3f6eeeff27fcd315ca5972616c0919ee38e522cce257bcd349", size = 1861558, upload-time = "2025-07-27T21:21:36.624Z" }, + { url = "https://files.pythonhosted.org/packages/96/29/7961e09a849eea7d8302e7baa6f829dd3ef3faf199cb25ed29b318ae799b/cramjam-2.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:66425bc25b5481359b12a6719b6e7c90ffe76d85d0691f1da7df304bfb8ce45c", size = 1699431, upload-time = "2025-07-27T21:21:38.396Z" }, + { url = "https://files.pythonhosted.org/packages/7a/60/6665e52f01a8919bf37c43dcf0e03b6dd3866f5c4e95440b357d508ee14e/cramjam-2.11.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:bd748d3407ec63e049b3aea1595e218814fccab329b7fb10bb51120a30e9fb7e", size = 2025262, upload-time = "2025-07-27T21:21:40.417Z" }, + { url = "https://files.pythonhosted.org/packages/d7/80/79bd84dbeb109e2c6efb74e661b7bd4c3ba393208ebcf69e2ae9454ae80c/cramjam-2.11.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a6d9a23a35b3a105c42a8de60fc2e80281ae6e758f05a3baea0b68eb1ddcb679", size = 1766177, upload-time = "2025-07-27T21:21:42.224Z" }, + { url = "https://files.pythonhosted.org/packages/28/ef/b43280767ebcde022ba31f1e9902137655a956ae30e920d75630fa67e36e/cramjam-2.11.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:40a75b95e05e38a2a055b2446f09994ce1139151721659315151d4ad6289bbff", size = 1854031, upload-time = "2025-07-27T21:21:43.651Z" }, + { url = "https://files.pythonhosted.org/packages/60/1c/79d522757c494dfd9e9b208b0604cc7e97b481483cc477144f5705a06ab7/cramjam-2.11.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e5d042c376d2025300da37d65192d06a457918b63b31140f697f85fd8e310b29", size = 2035812, upload-time = "2025-07-27T21:21:45.473Z" }, + { url = "https://files.pythonhosted.org/packages/c8/70/3bf0670380069b3abd4c6b53f61d3148f4e08935569c08efbeaf7550e87d/cramjam-2.11.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cb148b35ab20c75b19a06c27f05732e2a321adbd86fadc93f9466dbd7b1154a7", size = 2067661, upload-time = "2025-07-27T21:21:47.901Z" }, + { url = "https://files.pythonhosted.org/packages/db/7e/4f6ca98a4b474348e965a529b359184785d1119ab7c4c9ec1280b8bea50a/cramjam-2.11.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ee47c220f0f5179ddc923ab91fc9e282c27b29fabc60c433dfe06f08084f798", size = 1981523, upload-time = "2025-07-27T21:21:49.704Z" }, + { url = "https://files.pythonhosted.org/packages/8a/6c/b241511c7ffd5f1da29641429bb0e19b5fbcffafde5ba1bbcbf9394ea456/cramjam-2.11.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0cf1b5a81b21ea175c976c3ab09e00494258f4b49b7995efc86060cced3f0b2e", size = 2034251, upload-time = "2025-07-27T21:21:51.252Z" }, + { url = "https://files.pythonhosted.org/packages/14/5c/4ef926c8c3c1bf6da96f9c53450ff334cdb6d0fc1efced0aea97e2090803/cramjam-2.11.0-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:360c00338ecf48921492455007f904be607fc7818de3d681acbcc542aae2fb36", size = 2155322, upload-time = "2025-07-27T21:21:53.348Z" }, + { url = "https://files.pythonhosted.org/packages/be/fb/eb2aef7fb2730e56c5a2c9000817ee8fb4a95c92f19cc6e441afed42ec29/cramjam-2.11.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:f31fcc0d30dc3f3e94ea6b4d8e1a855071757c6abf6a7b1e284050ab7d4c299c", size = 2169094, upload-time = "2025-07-27T21:21:55.187Z" }, + { url = "https://files.pythonhosted.org/packages/3b/80/925a5c668dcee1c6f61775067185c5dc9a63c766d5393e5c60d2af4217a7/cramjam-2.11.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:033be66fdceb3d63b2c99b257a98380c4ec22c9e4dca54a2bfec3718cd24e184", size = 2159089, upload-time = "2025-07-27T21:21:57.118Z" }, + { url = "https://files.pythonhosted.org/packages/1f/ac/b2819640eef0592a6de7ca832c0d23c69bd1620f765ce88b60dbc8da9ba2/cramjam-2.11.0-cp310-cp310-win32.whl", hash = "sha256:1c6cea67f6000b81f6bd27d14c8a6f62d00336ca7252fd03ee16f6b70eb5c0d2", size = 1605046, upload-time = "2025-07-27T21:21:58.617Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f4/06af04727b9556721049e2127656d727306d275c518e3d97f9ed4cffd0d8/cramjam-2.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:98aa4a351b047b0f7f9e971585982065028adc2c162c5c23c5d5734c5ccc1077", size = 1710647, upload-time = "2025-07-27T21:22:00.279Z" }, + { url = "https://files.pythonhosted.org/packages/d0/89/8001f6a9b6b6e9fa69bec5319789083475d6f26d52aaea209d3ebf939284/cramjam-2.11.0-cp311-cp311-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:04cfa39118570e70e920a9b75c733299784b6d269733dbc791d9aaed6edd2615", size = 3559272, upload-time = "2025-07-27T21:22:01.988Z" }, + { url = "https://files.pythonhosted.org/packages/0b/f3/001d00070ca92e5fbe6aacc768e455568b0cde46b0eb944561a4ea132300/cramjam-2.11.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:66a18f68506290349a256375d7aa2f645b9f7993c10fc4cc211db214e4e61d2b", size = 1861743, upload-time = "2025-07-27T21:22:03.754Z" }, + { url = "https://files.pythonhosted.org/packages/c9/35/041a3af01bf3f6158f120070f798546d4383b962b63c35cd91dcbf193e17/cramjam-2.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:50e7d65533857736cd56f6509cf2c4866f28ad84dd15b5bdbf2f8a81e77fa28a", size = 1699631, upload-time = "2025-07-27T21:22:05.192Z" }, + { url = "https://files.pythonhosted.org/packages/17/eb/5358b238808abebd0c949c42635c3751204ca7cf82b29b984abe9f5e33c8/cramjam-2.11.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:1f71989668458fc327ac15396db28d92df22f8024bb12963929798b2729d2df5", size = 2025603, upload-time = "2025-07-27T21:22:06.726Z" }, + { url = "https://files.pythonhosted.org/packages/0e/79/19dba7c03a27408d8d11b5a7a4a7908459cfd4e6f375b73264dc66517bf6/cramjam-2.11.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee77ac543f1e2b22af1e8be3ae589f729491b6090582340aacd77d1d757d9569", size = 1766283, upload-time = "2025-07-27T21:22:08.568Z" }, + { url = "https://files.pythonhosted.org/packages/a4/ad/40e4b3408501d886d082db465c33971655fe82573c535428e52ab905f4d0/cramjam-2.11.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ad52784120e7e4d8a0b5b0517d185b8bf7f74f5e17272857ddc8951a628d9be1", size = 1854407, upload-time = "2025-07-27T21:22:10.518Z" }, + { url = "https://files.pythonhosted.org/packages/36/6e/c1b60ceb6d7ea6ff8b0bf197520aefe23f878bf2bfb0de65f2b0c2f82cd1/cramjam-2.11.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b86f8e6d9c1b3f9a75b2af870c93ceee0f1b827cd2507387540e053b35d7459", size = 2035793, upload-time = "2025-07-27T21:22:12.504Z" }, + { url = "https://files.pythonhosted.org/packages/9c/ad/32a8d5f4b1e3717787945ec6d71bd1c6e6bccba4b7e903fc0d9d4e4b08c3/cramjam-2.11.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:320d61938950d95da2371b46c406ec433e7955fae9f396c8e1bf148ffc187d11", size = 2067499, upload-time = "2025-07-27T21:22:14.067Z" }, + { url = "https://files.pythonhosted.org/packages/ff/cd/3b5a662736ea62ff7fa4c4a10a85e050bfdaad375cc53dc80427e8afe41c/cramjam-2.11.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:41eafc8c1653a35a5c7e75ad48138f9f60085cc05cd99d592e5298552d944e9f", size = 1981853, upload-time = "2025-07-27T21:22:15.908Z" }, + { url = "https://files.pythonhosted.org/packages/26/8e/1dbcfaaa7a702ee82ee683ec3a81656934dd7e04a7bc4ee854033686f98a/cramjam-2.11.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:03a7316c6bf763dfa34279335b27702321da44c455a64de58112968c0818ec4a", size = 2034514, upload-time = "2025-07-27T21:22:17.352Z" }, + { url = "https://files.pythonhosted.org/packages/50/62/f11709bfdce74af79a88b410dcb76dedc97612166e759136931bf63cfd7b/cramjam-2.11.0-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:244c2ed8bd7ccbb294a2abe7ca6498db7e89d7eb5e744691dc511a7dc82e65ca", size = 2155343, upload-time = "2025-07-27T21:22:18.854Z" }, + { url = "https://files.pythonhosted.org/packages/8a/6d/3b98b61841a5376d9a9b8468ae58753a8e6cf22be9534a0fa5af4d8621cc/cramjam-2.11.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:405f8790bad36ce0b4bbdb964ad51507bfc7942c78447f25cb828b870a1d86a0", size = 2169367, upload-time = "2025-07-27T21:22:20.389Z" }, + { url = "https://files.pythonhosted.org/packages/11/72/bd5db5c49dbebc8b002f1c4983101b28d2e7fc9419753db1c31ec22b03ef/cramjam-2.11.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6b1b751a5411032b08fb3ac556160229ca01c6bbe4757bb3a9a40b951ebaac23", size = 2159334, upload-time = "2025-07-27T21:22:22.254Z" }, + { url = "https://files.pythonhosted.org/packages/34/32/203c57acdb6eea727e7078b2219984e64ed4ad043c996ed56321301ba167/cramjam-2.11.0-cp311-cp311-win32.whl", hash = "sha256:5251585608778b9ac8effed544933df7ad85b4ba21ee9738b551f17798b215ac", size = 1605313, upload-time = "2025-07-27T21:22:24.126Z" }, + { url = "https://files.pythonhosted.org/packages/a9/bd/102d6deb87a8524ac11cddcd31a7612b8f20bf9b473c3c645045e3b957c7/cramjam-2.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:dca88bc8b68ce6d35dafd8c4d5d59a238a56c43fa02b74c2ce5f9dfb0d1ccb46", size = 1710991, upload-time = "2025-07-27T21:22:25.661Z" }, + { url = "https://files.pythonhosted.org/packages/0b/0d/7c84c913a5fae85b773a9dcf8874390f9d68ba0fcc6630efa7ff1541b950/cramjam-2.11.0-cp312-cp312-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:dba5c14b8b4f73ea1e65720f5a3fe4280c1d27761238378be8274135c60bbc6e", size = 3553368, upload-time = "2025-07-27T21:22:27.162Z" }, + { url = "https://files.pythonhosted.org/packages/2b/cc/4f6d185d8a744776f53035e72831ff8eefc2354f46ab836f4bd3c4f6c138/cramjam-2.11.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:11eb40722b3fcf3e6890fba46c711bf60f8dc26360a24876c85e52d76c33b25b", size = 1860014, upload-time = "2025-07-27T21:22:28.738Z" }, + { url = "https://files.pythonhosted.org/packages/1c/a8/626c76263085c6d5ded0e71823b411e9522bfc93ba6cc59855a5869296e7/cramjam-2.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:aeb26e2898994b6e8319f19a4d37c481512acdcc6d30e1b5ecc9d8ec57e835cb", size = 1693512, upload-time = "2025-07-27T21:22:30.999Z" }, + { url = "https://files.pythonhosted.org/packages/e9/52/0851a16a62447532e30ba95a80e638926fdea869a34b4b5b9d0a020083ba/cramjam-2.11.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:4f8d82081ed7d8fe52c982bd1f06e4c7631a73fe1fb6d4b3b3f2404f87dc40fe", size = 2025285, upload-time = "2025-07-27T21:22:32.954Z" }, + { url = "https://files.pythonhosted.org/packages/98/76/122e444f59dbc216451d8e3d8282c9665dc79eaf822f5f1470066be1b695/cramjam-2.11.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:092a3ec26e0a679305018380e4f652eae1b6dfe3fc3b154ee76aa6b92221a17c", size = 1761327, upload-time = "2025-07-27T21:22:34.484Z" }, + { url = "https://files.pythonhosted.org/packages/a3/bc/3a0189aef1af2b29632c039c19a7a1b752bc21a4053582a5464183a0ad3d/cramjam-2.11.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:529d6d667c65fd105d10bd83d1cd3f9869f8fd6c66efac9415c1812281196a92", size = 1854075, upload-time = "2025-07-27T21:22:36.157Z" }, + { url = "https://files.pythonhosted.org/packages/2e/80/8a6343b13778ce52d94bb8d5365a30c3aa951276b1857201fe79d7e2ad25/cramjam-2.11.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:555eb9c90c450e0f76e27d9ff064e64a8b8c6478ab1a5594c91b7bc5c82fd9f0", size = 2032710, upload-time = "2025-07-27T21:22:38.17Z" }, + { url = "https://files.pythonhosted.org/packages/df/6b/cd1778a207c29eda10791e3dfa018b588001928086e179fc71254793c625/cramjam-2.11.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5edf4c9e32493035b514cf2ba0c969d81ccb31de63bd05490cc8bfe3b431674e", size = 2068353, upload-time = "2025-07-27T21:22:39.615Z" }, + { url = "https://files.pythonhosted.org/packages/dc/f0/5c2a5cd5711032f3b191ca50cb786c17689b4a9255f9f768866e6c9f04d9/cramjam-2.11.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fa2fe41f48c4d58d923803383b0737f048918b5a0d10390de9628bb6272b107", size = 1978104, upload-time = "2025-07-27T21:22:41.106Z" }, + { url = "https://files.pythonhosted.org/packages/f9/8b/b363a5fb2c3347504fe9a64f8d0f1e276844f0e532aa7162c061cd1ffee4/cramjam-2.11.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9ca14cf1cabdb0b77d606db1bb9e9ca593b1dbd421fcaf251ec9a5431ec449f3", size = 2030779, upload-time = "2025-07-27T21:22:42.969Z" }, + { url = "https://files.pythonhosted.org/packages/78/7b/d83dad46adb6c988a74361f81ad9c5c22642be53ad88616a19baedd06243/cramjam-2.11.0-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:309e95bf898829476bccf4fd2c358ec00e7ff73a12f95a3cdeeba4bb1d3683d5", size = 2155297, upload-time = "2025-07-27T21:22:44.6Z" }, + { url = "https://files.pythonhosted.org/packages/1a/be/60d9be4cb33d8740a4aa94c7513f2ef3c4eba4fd13536f086facbafade71/cramjam-2.11.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:86dca35d2f15ef22922411496c220f3c9e315d5512f316fe417461971cc1648d", size = 2169255, upload-time = "2025-07-27T21:22:46.534Z" }, + { url = "https://files.pythonhosted.org/packages/11/b0/4a595f01a243aec8ad272b160b161c44351190c35d98d7787919d962e9e5/cramjam-2.11.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:193c6488bd2f514cbc0bef5c18fad61a5f9c8d059dd56edf773b3b37f0e85496", size = 2155651, upload-time = "2025-07-27T21:22:48.46Z" }, + { url = "https://files.pythonhosted.org/packages/38/47/7776659aaa677046b77f527106e53ddd47373416d8fcdb1e1a881ec5dc06/cramjam-2.11.0-cp312-cp312-win32.whl", hash = "sha256:514e2c008a8b4fa823122ca3ecab896eac41d9aa0f5fc881bd6264486c204e32", size = 1603568, upload-time = "2025-07-27T21:22:50.084Z" }, + { url = "https://files.pythonhosted.org/packages/75/b1/d53002729cfd94c5844ddfaf1233c86d29f2dbfc1b764a6562c41c044199/cramjam-2.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:53fed080476d5f6ad7505883ec5d1ec28ba36c2273db3b3e92d7224fe5e463db", size = 1709287, upload-time = "2025-07-27T21:22:51.534Z" }, + { url = "https://files.pythonhosted.org/packages/0a/8b/406c5dc0f8e82385519d8c299c40fd6a56d97eca3fcd6f5da8dad48de75b/cramjam-2.11.0-cp313-cp313-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:2c289729cc1c04e88bafa48b51082fb462b0a57dbc96494eab2be9b14dca62af", size = 3553330, upload-time = "2025-07-27T21:22:53.124Z" }, + { url = "https://files.pythonhosted.org/packages/00/ad/4186884083d6e4125b285903e17841827ab0d6d0cffc86216d27ed91e91d/cramjam-2.11.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:045201ee17147e36cf43d8ae2fa4b4836944ac672df5874579b81cf6d40f1a1f", size = 1859756, upload-time = "2025-07-27T21:22:54.821Z" }, + { url = "https://files.pythonhosted.org/packages/54/01/91b485cf76a7efef638151e8a7d35784dae2c4ff221b1aec2c083e4b106d/cramjam-2.11.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:619cd195d74c9e1d2a3ad78d63451d35379c84bd851aec552811e30842e1c67a", size = 1693609, upload-time = "2025-07-27T21:22:56.331Z" }, + { url = "https://files.pythonhosted.org/packages/cd/84/d0c80d279b2976870fc7d10f15dcb90a3c10c06566c6964b37c152694974/cramjam-2.11.0-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6eb3ae5ab72edb2ed68bdc0f5710f0a6cad7fd778a610ec2c31ee15e32d3921e", size = 2024912, upload-time = "2025-07-27T21:22:57.915Z" }, + { url = "https://files.pythonhosted.org/packages/d6/70/88f2a5cb904281ed5d3c111b8f7d5366639817a5470f059bcd26833fc870/cramjam-2.11.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:df7da3f4b19e3078f9635f132d31b0a8196accb2576e3213ddd7a77f93317c20", size = 1760715, upload-time = "2025-07-27T21:22:59.528Z" }, + { url = "https://files.pythonhosted.org/packages/b2/06/cf5b02081132537d28964fb385fcef9ed9f8a017dd7d8c59d317e53ba50d/cramjam-2.11.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:57286b289cd557ac76c24479d8ecfb6c3d5b854cce54ccc7671f9a2f5e2a2708", size = 1853782, upload-time = "2025-07-27T21:23:01.07Z" }, + { url = "https://files.pythonhosted.org/packages/57/27/63525087ed40a53d1867021b9c4858b80cc86274ffe7225deed067d88d92/cramjam-2.11.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:28952fbbf8b32c0cb7fa4be9bcccfca734bf0d0989f4b509dc7f2f70ba79ae06", size = 2032354, upload-time = "2025-07-27T21:23:03.021Z" }, + { url = "https://files.pythonhosted.org/packages/c3/ef/dbba082c6ebfb6410da4dd39a64e654d7194fcfd4567f85991a83fa4ec32/cramjam-2.11.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:78ed2e4099812a438b545dfbca1928ec825e743cd253bc820372d6ef8c3adff4", size = 2068007, upload-time = "2025-07-27T21:23:04.526Z" }, + { url = "https://files.pythonhosted.org/packages/35/ce/d902b9358a46a086938feae83b2251720e030f06e46006f4c1fc0ac9da20/cramjam-2.11.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d9aecd5c3845d415bd6c9957c93de8d93097e269137c2ecb0e5a5256374bdc8", size = 1977485, upload-time = "2025-07-27T21:23:06.058Z" }, + { url = "https://files.pythonhosted.org/packages/e8/03/982f54553244b0afcbdb2ad2065d460f0ab05a72a96896a969a1ca136a1e/cramjam-2.11.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:362fcf4d6f5e1242a4540812455f5a594949190f6fbc04f2ffbfd7ae0266d788", size = 2030447, upload-time = "2025-07-27T21:23:07.679Z" }, + { url = "https://files.pythonhosted.org/packages/74/5f/748e54cdb665ec098ec519e23caacc65fc5ae58718183b071e33fc1c45b4/cramjam-2.11.0-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:13240b3dea41b1174456cb9426843b085dc1a2bdcecd9ee2d8f65ac5703374b0", size = 2154949, upload-time = "2025-07-27T21:23:09.366Z" }, + { url = "https://files.pythonhosted.org/packages/69/81/c4e6cb06ed69db0dc81f9a8b1dc74995ebd4351e7a1877143f7031ff2700/cramjam-2.11.0-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:c54eed83726269594b9086d827decc7d2015696e31b99bf9b69b12d9063584fe", size = 2168925, upload-time = "2025-07-27T21:23:10.976Z" }, + { url = "https://files.pythonhosted.org/packages/13/5b/966365523ce8290a08e163e3b489626c5adacdff2b3da9da1b0823dfb14e/cramjam-2.11.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:f8195006fdd0fc0a85b19df3d64a3ef8a240e483ae1dfc7ac6a4316019eb5df2", size = 2154950, upload-time = "2025-07-27T21:23:12.514Z" }, + { url = "https://files.pythonhosted.org/packages/3a/7d/7f8eb5c534b72b32c6eb79d74585bfee44a9a5647a14040bb65c31c2572d/cramjam-2.11.0-cp313-cp313-win32.whl", hash = "sha256:ccf30e3fe6d770a803dcdf3bb863fa44ba5dc2664d4610ba2746a3c73599f2e4", size = 1603199, upload-time = "2025-07-27T21:23:14.38Z" }, + { url = "https://files.pythonhosted.org/packages/37/05/47b5e0bf7c41a3b1cdd3b7c2147f880c93226a6bef1f5d85183040cbdece/cramjam-2.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:ee36348a204f0a68b03400f4736224e9f61d1c6a1582d7f875c1ca56f0254268", size = 1708924, upload-time = "2025-07-27T21:23:16.332Z" }, + { url = "https://files.pythonhosted.org/packages/de/07/a1051cdbbe6d723df16d756b97f09da7c1adb69e29695c58f0392bc12515/cramjam-2.11.0-cp314-cp314-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:7ba5e38c9fbd06f086f4a5a64a1a5b7b417cd3f8fc07a20e5c03651f72f36100", size = 3554141, upload-time = "2025-07-27T21:23:17.938Z" }, + { url = "https://files.pythonhosted.org/packages/74/66/58487d2e16ef3d04f51a7c7f0e69823e806744b4c21101e89da4873074bc/cramjam-2.11.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:b8adeee57b41fe08e4520698a4b0bd3cc76dbd81f99424b806d70a5256a391d3", size = 1860353, upload-time = "2025-07-27T21:23:19.593Z" }, + { url = "https://files.pythonhosted.org/packages/67/b4/67f6254d166ffbcc9d5fa1b56876eaa920c32ebc8e9d3d525b27296b693b/cramjam-2.11.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:b96a74fa03a636c8a7d76f700d50e9a8bc17a516d6a72d28711225d641e30968", size = 1693832, upload-time = "2025-07-27T21:23:21.185Z" }, + { url = "https://files.pythonhosted.org/packages/55/a3/4e0b31c0d454ae70c04684ed7c13d3c67b4c31790c278c1e788cb804fa4a/cramjam-2.11.0-cp314-cp314-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c3811a56fa32e00b377ef79121c0193311fd7501f0fb378f254c7f083cc1fbe0", size = 2027080, upload-time = "2025-07-27T21:23:23.303Z" }, + { url = "https://files.pythonhosted.org/packages/d9/c7/5e8eed361d1d3b8be14f38a54852c5370cc0ceb2c2d543b8ba590c34f080/cramjam-2.11.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5d927e87461f8a0d448e4ab5eb2bca9f31ca5d8ea86d70c6f470bb5bc666d7e", size = 1761543, upload-time = "2025-07-27T21:23:24.991Z" }, + { url = "https://files.pythonhosted.org/packages/09/0c/06b7f8b0ce9fde89470505116a01fc0b6cb92d406c4fb1e46f168b5d3fa5/cramjam-2.11.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f1f5c450121430fd89cb5767e0a9728ecc65997768fd4027d069cb0368af62f9", size = 1854636, upload-time = "2025-07-27T21:23:26.987Z" }, + { url = "https://files.pythonhosted.org/packages/6f/c6/6ebc02c9d5acdf4e5f2b1ec6e1252bd5feee25762246798ae823b3347457/cramjam-2.11.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:724aa7490be50235d97f07e2ca10067927c5d7f336b786ddbc868470e822aa25", size = 2032715, upload-time = "2025-07-27T21:23:28.603Z" }, + { url = "https://files.pythonhosted.org/packages/a2/77/a122971c23f5ca4b53e4322c647ac7554626c95978f92d19419315dddd05/cramjam-2.11.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:54c4637122e7cfd7aac5c1d3d4c02364f446d6923ea34cf9d0e8816d6e7a4936", size = 2069039, upload-time = "2025-07-27T21:23:30.319Z" }, + { url = "https://files.pythonhosted.org/packages/19/0f/f6121b90b86b9093c066889274d26a1de3f29969d45c2ed1ecbe2033cb78/cramjam-2.11.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17eb39b1696179fb471eea2de958fa21f40a2cd8bf6b40d428312d5541e19dc4", size = 1979566, upload-time = "2025-07-27T21:23:32.002Z" }, + { url = "https://files.pythonhosted.org/packages/e0/a3/f95bc57fd7f4166ce6da816cfa917fb7df4bb80e669eb459d85586498414/cramjam-2.11.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:36aa5a798aa34e11813a80425a30d8e052d8de4a28f27bfc0368cfc454d1b403", size = 2030905, upload-time = "2025-07-27T21:23:33.696Z" }, + { url = "https://files.pythonhosted.org/packages/fc/52/e429de4e8bc86ee65e090dae0f87f45abd271742c63fb2d03c522ffde28a/cramjam-2.11.0-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:449fca52774dc0199545fbf11f5128933e5a6833946707885cf7be8018017839", size = 2155592, upload-time = "2025-07-27T21:23:35.375Z" }, + { url = "https://files.pythonhosted.org/packages/6c/6c/65a7a0207787ad39ad804af4da7f06a60149de19481d73d270b540657234/cramjam-2.11.0-cp314-cp314-musllinux_1_1_i686.whl", hash = "sha256:d87d37b3d476f4f7623c56a232045d25bd9b988314702ea01bd9b4a94948a778", size = 2170839, upload-time = "2025-07-27T21:23:37.197Z" }, + { url = "https://files.pythonhosted.org/packages/b2/c5/5c5db505ba692bc844246b066e23901d5905a32baf2f33719c620e65887f/cramjam-2.11.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:26cb45c47d71982d76282e303931c6dd4baee1753e5d48f9a89b3a63e690b3a3", size = 2157236, upload-time = "2025-07-27T21:23:38.854Z" }, + { url = "https://files.pythonhosted.org/packages/b0/22/88e6693e60afe98901e5bbe91b8dea193e3aa7f42e2770f9c3339f5c1065/cramjam-2.11.0-cp314-cp314-win32.whl", hash = "sha256:4efe919d443c2fd112fe25fe636a52f9628250c9a50d9bddb0488d8a6c09acc6", size = 1604136, upload-time = "2025-07-27T21:23:40.56Z" }, + { url = "https://files.pythonhosted.org/packages/cc/f8/01618801cd59ccedcc99f0f96d20be67d8cfc3497da9ccaaad6b481781dd/cramjam-2.11.0-cp314-cp314-win_amd64.whl", hash = "sha256:ccec3524ea41b9abd5600e3e27001fd774199dbb4f7b9cb248fcee37d4bda84c", size = 1710272, upload-time = "2025-07-27T21:23:42.236Z" }, + { url = "https://files.pythonhosted.org/packages/40/81/6cdb3ed222d13ae86bda77aafe8d50566e81a1169d49ed195b6263610704/cramjam-2.11.0-cp314-cp314t-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:966ac9358b23d21ecd895c418c048e806fd254e46d09b1ff0cdad2eba195ea3e", size = 3559671, upload-time = "2025-07-27T21:23:44.504Z" }, + { url = "https://files.pythonhosted.org/packages/cb/43/52b7e54fe5ba1ef0270d9fdc43dabd7971f70ea2d7179be918c997820247/cramjam-2.11.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:387f09d647a0d38dcb4539f8a14281f8eb6bb1d3e023471eb18a5974b2121c86", size = 1867876, upload-time = "2025-07-27T21:23:46.987Z" }, + { url = "https://files.pythonhosted.org/packages/9d/28/30d5b8d10acd30db3193bc562a313bff722888eaa45cfe32aa09389f2b24/cramjam-2.11.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:665b0d8fbbb1a7f300265b43926457ec78385200133e41fef19d85790fc1e800", size = 1695562, upload-time = "2025-07-27T21:23:48.644Z" }, + { url = "https://files.pythonhosted.org/packages/d9/86/ec806f986e01b896a650655024ea52a13e25c3ac8a3a382f493089483cdc/cramjam-2.11.0-cp314-cp314t-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ca905387c7a371531b9622d93471be4d745ef715f2890c3702479cd4fc85aa51", size = 2025056, upload-time = "2025-07-27T21:23:50.404Z" }, + { url = "https://files.pythonhosted.org/packages/09/43/c2c17586b90848d29d63181f7d14b8bd3a7d00975ad46e3edf2af8af7e1f/cramjam-2.11.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c1aa56aef2c8af55a21ed39040a94a12b53fb23beea290f94d19a76027e2ffb", size = 1764084, upload-time = "2025-07-27T21:23:52.265Z" }, + { url = "https://files.pythonhosted.org/packages/2b/a9/68bc334fadb434a61df10071dc8606702aa4f5b6cdb2df62474fc21d2845/cramjam-2.11.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e5db59c1cdfaa2ab85cc988e602d6919495f735ca8a5fd7603608eb1e23c26d5", size = 1854859, upload-time = "2025-07-27T21:23:54.085Z" }, + { url = "https://files.pythonhosted.org/packages/5b/4e/b48e67835b5811ec5e9cb2e2bcba9c3fd76dab3e732569fe801b542c6ca9/cramjam-2.11.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b1f893014f00fe5e89a660a032e813bf9f6d91de74cd1490cdb13b2b59d0c9a3", size = 2035970, upload-time = "2025-07-27T21:23:55.758Z" }, + { url = "https://files.pythonhosted.org/packages/c4/70/d2ac33d572b4d90f7f0f2c8a1d60fb48f06b128fdc2c05f9b49891bb0279/cramjam-2.11.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c26a1eb487947010f5de24943bd7c422dad955b2b0f8650762539778c380ca89", size = 2069320, upload-time = "2025-07-27T21:23:57.494Z" }, + { url = "https://files.pythonhosted.org/packages/1d/4c/85cec77af4a74308ba5fca8e296c4e2f80ec465c537afc7ab1e0ca2f9a00/cramjam-2.11.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d5c8bfb438d94e7b892d1426da5fc4b4a5370cc360df9b8d9d77c33b896c37e", size = 1982668, upload-time = "2025-07-27T21:23:59.126Z" }, + { url = "https://files.pythonhosted.org/packages/55/45/938546d1629e008cc3138df7c424ef892719b1796ff408a2ab8550032e5e/cramjam-2.11.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:cb1fb8c9337ab0da25a01c05d69a0463209c347f16512ac43be5986f3d1ebaf4", size = 2034028, upload-time = "2025-07-27T21:24:00.865Z" }, + { url = "https://files.pythonhosted.org/packages/01/76/b5a53e20505555f1640e66dcf70394bcf51a1a3a072aa18ea35135a0f9ed/cramjam-2.11.0-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:1f6449f6de52dde3e2f1038284910c8765a397a25e2d05083870f3f5e7fc682c", size = 2155513, upload-time = "2025-07-27T21:24:02.92Z" }, + { url = "https://files.pythonhosted.org/packages/84/12/8d3f6ceefae81bbe45a347fdfa2219d9f3ac75ebc304f92cd5fcb4fbddc5/cramjam-2.11.0-cp314-cp314t-musllinux_1_1_i686.whl", hash = "sha256:382dec4f996be48ed9c6958d4e30c2b89435d7c2c4dbf32480b3b8886293dd65", size = 2170035, upload-time = "2025-07-27T21:24:04.558Z" }, + { url = "https://files.pythonhosted.org/packages/4b/85/3be6f0a1398f976070672be64f61895f8839857618a2d8cc0d3ab529d3dc/cramjam-2.11.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:d388bd5723732c3afe1dd1d181e4213cc4e1be210b080572e7d5749f6e955656", size = 2160229, upload-time = "2025-07-27T21:24:06.729Z" }, + { url = "https://files.pythonhosted.org/packages/57/5e/66cfc3635511b20014bbb3f2ecf0095efb3049e9e96a4a9e478e4f3d7b78/cramjam-2.11.0-cp314-cp314t-win32.whl", hash = "sha256:0a70ff17f8e1d13f322df616505550f0f4c39eda62290acb56f069d4857037c8", size = 1610267, upload-time = "2025-07-27T21:24:08.428Z" }, + { url = "https://files.pythonhosted.org/packages/ce/c6/c71e82e041c95ffe6a92ac707785500aa2a515a4339c2c7dd67e3c449249/cramjam-2.11.0-cp314-cp314t-win_amd64.whl", hash = "sha256:028400d699442d40dbda02f74158c73d05cb76587a12490d0bfedd958fd49188", size = 1713108, upload-time = "2025-07-27T21:24:10.147Z" }, + { url = "https://files.pythonhosted.org/packages/bf/8f/82e35ec3c5387f1864f46b3c24bce89a07af8bb3ef242ae47281db2c1848/cramjam-2.11.0-pp310-pypy310_pp73-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:37bed927abc4a7ae2d2669baa3675e21904d8a038ed8e4313326ea7b3be62b2b", size = 3573104, upload-time = "2025-07-27T21:24:40.069Z" }, + { url = "https://files.pythonhosted.org/packages/f0/4e/0c821918080a32ba1e52c040e12dd02dada67728f07305c5f778b808a807/cramjam-2.11.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:50e4a58635fa8c6897d84847d6e065eb69f92811670fc5e9f2d9e3b6279a02b6", size = 1873441, upload-time = "2025-07-27T21:24:42.333Z" }, + { url = "https://files.pythonhosted.org/packages/a8/fd/848d077bf6abc4ce84273d8e3f3a70d61a2240519a339462f699d8acf829/cramjam-2.11.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:3d1ba626dd5f81f7f09bbf59f70b534e2b75e0d6582b056b7bd31b397f1c13e9", size = 1702589, upload-time = "2025-07-27T21:24:44.305Z" }, + { url = "https://files.pythonhosted.org/packages/9d/1c/899818999bbdb59c601756b413e87d37fd65875d1315346c10e367bb3505/cramjam-2.11.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c71e140d5eb3145d61d59d0be0bf72f07cc4cf4b32cb136b09f712a3b1040f5f", size = 1773646, upload-time = "2025-07-27T21:24:46.495Z" }, + { url = "https://files.pythonhosted.org/packages/5f/26/c2813c5422c43b3dcd8b6645bc359f08870737c44325ee4accc18f24eee0/cramjam-2.11.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a6ed7926a5cca28edebad7d0fedd2ad492710ae3524d25fc59a2b20546d9ce1", size = 1994179, upload-time = "2025-07-27T21:24:49.131Z" }, + { url = "https://files.pythonhosted.org/packages/2e/4f/af984f8d7f963f0301812cdd620ddcfd8276461ed7a786c0f89e82b14739/cramjam-2.11.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5eb4ed3cea945b164b0513fd491884993acac2153a27b93a84019c522e8eda82", size = 1714790, upload-time = "2025-07-27T21:24:51.045Z" }, + { url = "https://files.pythonhosted.org/packages/81/da/b3301962ccd6fce9fefa1ecd8ea479edaeaa38fadb1f34d5391d2587216a/cramjam-2.11.0-pp311-pypy311_pp73-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:52d5db3369f95b27b9f3c14d067acb0b183333613363ed34268c9e04560f997f", size = 3573546, upload-time = "2025-07-27T21:24:52.944Z" }, + { url = "https://files.pythonhosted.org/packages/b6/c2/410ddb8ad4b9dfb129284666293cb6559479645da560f7077dc19d6bee9e/cramjam-2.11.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:4820516366d455b549a44d0e2210ee7c4575882dda677564ce79092588321d54", size = 1873654, upload-time = "2025-07-27T21:24:54.958Z" }, + { url = "https://files.pythonhosted.org/packages/d5/99/f68a443c64f7ce7aff5bed369b0aa5b2fac668fa3dfd441837e316e97a1f/cramjam-2.11.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d9e5db525dc0a950a825202f84ee68d89a072479e07da98795a3469df942d301", size = 1702846, upload-time = "2025-07-27T21:24:57.124Z" }, + { url = "https://files.pythonhosted.org/packages/6c/02/0ff358ab773def1ee3383587906c453d289953171e9c92db84fdd01bf172/cramjam-2.11.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62ab4971199b2270005359cdc379bc5736071dc7c9a228581c5122d9ffaac50c", size = 1773683, upload-time = "2025-07-27T21:24:59.28Z" }, + { url = "https://files.pythonhosted.org/packages/e9/31/3298e15f87c9cf2aabdbdd90b153d8644cf989cb42a45d68a1b71e1f7aaf/cramjam-2.11.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24758375cc5414d3035ca967ebb800e8f24604ececcba3c67d6f0218201ebf2d", size = 1994136, upload-time = "2025-07-27T21:25:01.565Z" }, + { url = "https://files.pythonhosted.org/packages/c7/90/20d1747255f1ee69a412e319da51ea594c18cca195e7a4d4c713f045eff5/cramjam-2.11.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:6c2eea545fef1065c7dd4eda991666fd9c783fbc1d226592ccca8d8891c02f23", size = 1714982, upload-time = "2025-07-27T21:25:05.79Z" }, +] + +[[package]] +name = "exceptiongroup" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" }, +] + +[[package]] +name = "fastavro" +version = "1.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/65/8b/fa2d3287fd2267be6261d0177c6809a7fa12c5600ddb33490c8dc29e77b2/fastavro-1.12.1.tar.gz", hash = "sha256:2f285be49e45bc047ab2f6bed040bb349da85db3f3c87880e4b92595ea093b2b", size = 1025661, upload-time = "2025-10-10T15:40:55.41Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/a0/077fd7cbfc143152cb96780cb592ed6cb6696667d8bc1b977745eb2255a8/fastavro-1.12.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:00650ca533907361edda22e6ffe8cf87ab2091c5d8aee5c8000b0f2dcdda7ed3", size = 1000335, upload-time = "2025-10-10T15:40:59.834Z" }, + { url = "https://files.pythonhosted.org/packages/a0/ae/a115e027f3a75df237609701b03ecba0b7f0aa3d77fe0161df533fde1eb7/fastavro-1.12.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ac76d6d95f909c72ee70d314b460b7e711d928845771531d823eb96a10952d26", size = 3221067, upload-time = "2025-10-10T15:41:04.399Z" }, + { url = "https://files.pythonhosted.org/packages/94/4e/c4991c3eec0175af9a8a0c161b88089cb7bf7fe353b3e3be1bc4cf9036b2/fastavro-1.12.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f55eef18c41d4476bd32a82ed5dd86aabc3f614e1b66bdb09ffa291612e1670", size = 3228979, upload-time = "2025-10-10T15:41:06.738Z" }, + { url = "https://files.pythonhosted.org/packages/21/0c/f2afb8eaea38799ccb1ed07d68bf2659f2e313f1902bbd36774cf6a1bef9/fastavro-1.12.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:81563e1f93570e6565487cdb01ba241a36a00e58cff9c5a0614af819d1155d8f", size = 3160740, upload-time = "2025-10-10T15:41:08.731Z" }, + { url = "https://files.pythonhosted.org/packages/0d/1a/f4d367924b40b86857862c1fa65f2afba94ddadf298b611e610a676a29e5/fastavro-1.12.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bec207360f76f0b3de540758a297193c5390e8e081c43c3317f610b1414d8c8f", size = 3235787, upload-time = "2025-10-10T15:41:10.869Z" }, + { url = "https://files.pythonhosted.org/packages/90/ec/8db9331896e3dfe4f71b2b3c23f2e97fbbfd90129777467ca9f8bafccb74/fastavro-1.12.1-cp310-cp310-win_amd64.whl", hash = "sha256:c0390bfe4a9f8056a75ac6785fbbff8f5e317f5356481d2e29ec980877d2314b", size = 449350, upload-time = "2025-10-10T15:41:12.104Z" }, + { url = "https://files.pythonhosted.org/packages/a0/e9/31c64b47cefc0951099e7c0c8c8ea1c931edd1350f34d55c27cbfbb08df1/fastavro-1.12.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6b632b713bc5d03928a87d811fa4a11d5f25cd43e79c161e291c7d3f7aa740fd", size = 1016585, upload-time = "2025-10-10T15:41:13.717Z" }, + { url = "https://files.pythonhosted.org/packages/10/76/111560775b548f5d8d828c1b5285ff90e2d2745643fb80ecbf115344eea4/fastavro-1.12.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eaa7ab3769beadcebb60f0539054c7755f63bd9cf7666e2c15e615ab605f89a8", size = 3404629, upload-time = "2025-10-10T15:41:15.642Z" }, + { url = "https://files.pythonhosted.org/packages/b0/07/6bb93cb963932146c2b6c5c765903a0a547ad9f0f8b769a4a9aad8c06369/fastavro-1.12.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:123fb221df3164abd93f2d042c82f538a1d5a43ce41375f12c91ce1355a9141e", size = 3428594, upload-time = "2025-10-10T15:41:17.779Z" }, + { url = "https://files.pythonhosted.org/packages/d1/67/8115ec36b584197ea737ec79e3499e1f1b640b288d6c6ee295edd13b80f6/fastavro-1.12.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:632a4e3ff223f834ddb746baae0cc7cee1068eb12c32e4d982c2fee8a5b483d0", size = 3344145, upload-time = "2025-10-10T15:41:19.89Z" }, + { url = "https://files.pythonhosted.org/packages/9e/9e/a7cebb3af967e62539539897c10138fa0821668ec92525d1be88a9cd3ee6/fastavro-1.12.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:83e6caf4e7a8717d932a3b1ff31595ad169289bbe1128a216be070d3a8391671", size = 3431942, upload-time = "2025-10-10T15:41:22.076Z" }, + { url = "https://files.pythonhosted.org/packages/c0/d1/7774ddfb8781c5224294c01a593ebce2ad3289b948061c9701bd1903264d/fastavro-1.12.1-cp311-cp311-win_amd64.whl", hash = "sha256:b91a0fe5a173679a6c02d53ca22dcaad0a2c726b74507e0c1c2e71a7c3f79ef9", size = 450542, upload-time = "2025-10-10T15:41:23.333Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f0/10bd1a3d08667fa0739e2b451fe90e06df575ec8b8ba5d3135c70555c9bd/fastavro-1.12.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:509818cb24b98a804fc80be9c5fed90f660310ae3d59382fc811bfa187122167", size = 1009057, upload-time = "2025-10-10T15:41:24.556Z" }, + { url = "https://files.pythonhosted.org/packages/78/ad/0d985bc99e1fa9e74c636658000ba38a5cd7f5ab2708e9c62eaf736ecf1a/fastavro-1.12.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:089e155c0c76e0d418d7e79144ce000524dd345eab3bc1e9c5ae69d500f71b14", size = 3391866, upload-time = "2025-10-10T15:41:26.882Z" }, + { url = "https://files.pythonhosted.org/packages/0d/9e/b4951dc84ebc34aac69afcbfbb22ea4a91080422ec2bfd2c06076ff1d419/fastavro-1.12.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44cbff7518901c91a82aab476fcab13d102e4999499df219d481b9e15f61af34", size = 3458005, upload-time = "2025-10-10T15:41:29.017Z" }, + { url = "https://files.pythonhosted.org/packages/af/f8/5a8df450a9f55ca8441f22ea0351d8c77809fc121498b6970daaaf667a21/fastavro-1.12.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a275e48df0b1701bb764b18a8a21900b24cf882263cb03d35ecdba636bbc830b", size = 3295258, upload-time = "2025-10-10T15:41:31.564Z" }, + { url = "https://files.pythonhosted.org/packages/99/b2/40f25299111d737e58b85696e91138a66c25b7334f5357e7ac2b0e8966f8/fastavro-1.12.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2de72d786eb38be6b16d556b27232b1bf1b2797ea09599507938cdb7a9fe3e7c", size = 3430328, upload-time = "2025-10-10T15:41:33.689Z" }, + { url = "https://files.pythonhosted.org/packages/e0/07/85157a7c57c5f8b95507d7829b5946561e5ee656ff80e9dd9a757f53ddaf/fastavro-1.12.1-cp312-cp312-win_amd64.whl", hash = "sha256:9090f0dee63fe022ee9cc5147483366cc4171c821644c22da020d6b48f576b4f", size = 444140, upload-time = "2025-10-10T15:41:34.902Z" }, + { url = "https://files.pythonhosted.org/packages/bb/57/26d5efef9182392d5ac9f253953c856ccb66e4c549fd3176a1e94efb05c9/fastavro-1.12.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:78df838351e4dff9edd10a1c41d1324131ffecbadefb9c297d612ef5363c049a", size = 1000599, upload-time = "2025-10-10T15:41:36.554Z" }, + { url = "https://files.pythonhosted.org/packages/33/cb/8ab55b21d018178eb126007a56bde14fd01c0afc11d20b5f2624fe01e698/fastavro-1.12.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:780476c23175d2ae457c52f45b9ffa9d504593499a36cd3c1929662bf5b7b14b", size = 3335933, upload-time = "2025-10-10T15:41:39.07Z" }, + { url = "https://files.pythonhosted.org/packages/fe/03/9c94ec9bf873eb1ffb0aa694f4e71940154e6e9728ddfdc46046d7e8ced4/fastavro-1.12.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0714b285160fcd515eb0455540f40dd6dac93bdeacdb03f24e8eac3d8aa51f8d", size = 3402066, upload-time = "2025-10-10T15:41:41.608Z" }, + { url = "https://files.pythonhosted.org/packages/75/c8/cb472347c5a584ccb8777a649ebb28278fccea39d005fc7df19996f41df8/fastavro-1.12.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a8bc2dcec5843d499f2489bfe0747999108f78c5b29295d877379f1972a3d41a", size = 3240038, upload-time = "2025-10-10T15:41:43.743Z" }, + { url = "https://files.pythonhosted.org/packages/e1/77/569ce9474c40304b3a09e109494e020462b83e405545b78069ddba5f614e/fastavro-1.12.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3b1921ac35f3d89090a5816b626cf46e67dbecf3f054131f84d56b4e70496f45", size = 3369398, upload-time = "2025-10-10T15:41:45.719Z" }, + { url = "https://files.pythonhosted.org/packages/4a/1f/9589e35e9ea68035385db7bdbf500d36b8891db474063fb1ccc8215ee37c/fastavro-1.12.1-cp313-cp313-win_amd64.whl", hash = "sha256:5aa777b8ee595b50aa084104cd70670bf25a7bbb9fd8bb5d07524b0785ee1699", size = 444220, upload-time = "2025-10-10T15:41:47.39Z" }, + { url = "https://files.pythonhosted.org/packages/6c/d2/78435fe737df94bd8db2234b2100f5453737cffd29adee2504a2b013de84/fastavro-1.12.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:c3d67c47f177e486640404a56f2f50b165fe892cc343ac3a34673b80cc7f1dd6", size = 1086611, upload-time = "2025-10-10T15:41:48.818Z" }, + { url = "https://files.pythonhosted.org/packages/b6/be/428f99b10157230ddac77ec8cc167005b29e2bd5cbe228345192bb645f30/fastavro-1.12.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5217f773492bac43dae15ff2931432bce2d7a80be7039685a78d3fab7df910bd", size = 3541001, upload-time = "2025-10-10T15:41:50.871Z" }, + { url = "https://files.pythonhosted.org/packages/16/08/a2eea4f20b85897740efe44887e1ac08f30dfa4bfc3de8962bdcbb21a5a1/fastavro-1.12.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:469fecb25cba07f2e1bfa4c8d008477cd6b5b34a59d48715e1b1a73f6160097d", size = 3432217, upload-time = "2025-10-10T15:41:53.149Z" }, + { url = "https://files.pythonhosted.org/packages/87/bb/b4c620b9eb6e9838c7f7e4b7be0762834443adf9daeb252a214e9ad3178c/fastavro-1.12.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d71c8aa841ef65cfab709a22bb887955f42934bced3ddb571e98fdbdade4c609", size = 3366742, upload-time = "2025-10-10T15:41:55.237Z" }, + { url = "https://files.pythonhosted.org/packages/3d/d1/e69534ccdd5368350646fea7d93be39e5f77c614cca825c990bd9ca58f67/fastavro-1.12.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:b81fc04e85dfccf7c028e0580c606e33aa8472370b767ef058aae2c674a90746", size = 3383743, upload-time = "2025-10-10T15:41:57.68Z" }, + { url = "https://files.pythonhosted.org/packages/58/54/b7b4a0c3fb5fcba38128542da1b26c4e6d69933c923f493548bdfd63ab6a/fastavro-1.12.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:9445da127751ba65975d8e4bdabf36bfcfdad70fc35b2d988e3950cce0ec0e7c", size = 1001377, upload-time = "2025-10-10T15:41:59.241Z" }, + { url = "https://files.pythonhosted.org/packages/1e/4f/0e589089c7df0d8f57d7e5293fdc34efec9a3b758a0d4d0c99a7937e2492/fastavro-1.12.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ed924233272719b5d5a6a0b4d80ef3345fc7e84fc7a382b6232192a9112d38a6", size = 3320401, upload-time = "2025-10-10T15:42:01.682Z" }, + { url = "https://files.pythonhosted.org/packages/f9/19/260110d56194ae29d7e423a336fccea8bcd103196d00f0b364b732bdb84e/fastavro-1.12.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3616e2f0e1c9265e92954fa099db79c6e7817356d3ff34f4bcc92699ae99697c", size = 3350894, upload-time = "2025-10-10T15:42:04.073Z" }, + { url = "https://files.pythonhosted.org/packages/d0/96/58b0411e8be9694d5972bee3167d6c1fd1fdfdf7ce253c1a19a327208f4f/fastavro-1.12.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:cb0337b42fd3c047fcf0e9b7597bd6ad25868de719f29da81eabb6343f08d399", size = 3229644, upload-time = "2025-10-10T15:42:06.221Z" }, + { url = "https://files.pythonhosted.org/packages/5b/db/38660660eac82c30471d9101f45b3acfdcbadfe42d8f7cdb129459a45050/fastavro-1.12.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:64961ab15b74b7c168717bbece5660e0f3d457837c3cc9d9145181d011199fa7", size = 3329704, upload-time = "2025-10-10T15:42:08.384Z" }, + { url = "https://files.pythonhosted.org/packages/9d/a9/1672910f458ecb30b596c9e59e41b7c00309b602a0494341451e92e62747/fastavro-1.12.1-cp314-cp314-win_amd64.whl", hash = "sha256:792356d320f6e757e89f7ac9c22f481e546c886454a6709247f43c0dd7058004", size = 452911, upload-time = "2025-10-10T15:42:09.795Z" }, + { url = "https://files.pythonhosted.org/packages/dc/8d/2e15d0938ded1891b33eff252e8500605508b799c2e57188a933f0bd744c/fastavro-1.12.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:120aaf82ac19d60a1016afe410935fe94728752d9c2d684e267e5b7f0e70f6d9", size = 3541999, upload-time = "2025-10-10T15:42:11.794Z" }, + { url = "https://files.pythonhosted.org/packages/a7/1c/6dfd082a205be4510543221b734b1191299e6a1810c452b6bc76dfa6968e/fastavro-1.12.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b6a3462934b20a74f9ece1daa49c2e4e749bd9a35fa2657b53bf62898fba80f5", size = 3433972, upload-time = "2025-10-10T15:42:14.485Z" }, + { url = "https://files.pythonhosted.org/packages/24/90/9de694625a1a4b727b1ad0958d220cab25a9b6cf7f16a5c7faa9ea7b2261/fastavro-1.12.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1f81011d54dd47b12437b51dd93a70a9aa17b61307abf26542fc3c13efbc6c51", size = 3368752, upload-time = "2025-10-10T15:42:16.618Z" }, + { url = "https://files.pythonhosted.org/packages/fa/93/b44f67589e4d439913dab6720f7e3507b0fa8b8e56d06f6fc875ced26afb/fastavro-1.12.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:43ded16b3f4a9f1a42f5970c2aa618acb23ea59c4fcaa06680bdf470b255e5a8", size = 3386636, upload-time = "2025-10-10T15:42:18.974Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "nominal-streaming" +source = { editable = "." } +dependencies = [ + { name = "polars" }, + { name = "python-dateutil" }, + { name = "typing-extensions" }, +] + +[package.dev-dependencies] +dev = [ + { name = "cramjam" }, + { name = "fastavro" }, + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "polars", specifier = ">=1,<2" }, + { name = "python-dateutil", specifier = ">=0.0.0" }, + { name = "typing-extensions", specifier = ">=4,<5" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "cramjam", specifier = ">=2" }, + { name = "fastavro", specifier = ">=1" }, + { name = "pytest", specifier = ">=7" }, +] + +[[package]] +name = "packaging" +version = "26.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/df/de/0d2b39fb4af88a0258f3bac87dfcbb48e73fbdea4a2ed0e2213f9a4c2f9a/packaging-26.1.tar.gz", hash = "sha256:f042152b681c4bfac5cae2742a55e103d27ab2ec0f3d88037136b6bfe7c9c5de", size = 215519, upload-time = "2026-04-14T21:12:49.362Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/c2/920ef838e2f0028c8262f16101ec09ebd5969864e5a64c4c05fad0617c56/packaging-26.1-py3-none-any.whl", hash = "sha256:5d9c0669c6285e491e0ced2eee587eaf67b670d94a19e94e3984a481aba6802f", size = 95831, upload-time = "2026-04-14T21:12:47.56Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "polars" +version = "1.40.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "polars-runtime-32" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/8c/bc9bc948058348ed43117cecc3007cd608f395915dae8a00974579a5dab1/polars-1.40.1.tar.gz", hash = "sha256:ab2694134b137596b5a59bfd7b4c54ebbc9b59f9403127f18e32d363777552e8", size = 733574, upload-time = "2026-04-22T19:15:55.507Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ea/91/74fc60d94488685a92ac9d49d7ec55f3e91fe9b77942a6235a5fa7f249c3/polars-1.40.1-py3-none-any.whl", hash = "sha256:c0f861219d1319cdea45c4ce4d30355a47176b8f98dcedf95ea8269f131b8abd", size = 828723, upload-time = "2026-04-22T19:14:25.452Z" }, +] + +[[package]] +name = "polars-runtime-32" +version = "1.40.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/ba/26d40f039be9f552b5fd7365a621bdfc0f8e912ef77094ae4693491b0bae/polars_runtime_32-1.40.1.tar.gz", hash = "sha256:37f3065615d1bf90d03b5326222df4c5c1f8a5d33e50470aa588e3465e6eb814", size = 2935843, upload-time = "2026-04-22T19:15:57.26Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7d/46/22c8af5eed68ac2eeb556e0fa3ca8a7b798e984ceff4450888f3b5ac61fd/polars_runtime_32-1.40.1-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:b748ef652270cc49e9e69f99a035e0eb4d5f856d42bcd6ac4d9d80a40142aa1e", size = 52098755, upload-time = "2026-04-22T19:14:28.555Z" }, + { url = "https://files.pythonhosted.org/packages/c6/3e/48599a38009ca60ff82a6f38c8a621ce3c0286aa7397c7d79e741bd9060e/polars_runtime_32-1.40.1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:d249b3743e05986060cec0a7aaa542d020df6c6b876e556023a310efd581f9be", size = 46367542, upload-time = "2026-04-22T19:14:32.433Z" }, + { url = "https://files.pythonhosted.org/packages/43/e9/384bc069367a1a36ee31c13782c178dbd039b2b873b772d4a0fc23a2373d/polars_runtime_32-1.40.1-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5987b30e7aa1059d069498496e8dda35afd592b0ac3d46ed87e3ff8df1ad652c", size = 50252104, upload-time = "2026-04-22T19:14:35.945Z" }, + { url = "https://files.pythonhosted.org/packages/15/ef/7d57ceb0651af74194e97ed6583e148d352f03d696090221b8059cdfc90b/polars_runtime_32-1.40.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d7f42a8b3f16fc66002cc0f6516f7dd7653396886ae0ed362ab95c0b3408b59", size = 56250788, upload-time = "2026-04-22T19:14:39.743Z" }, + { url = "https://files.pythonhosted.org/packages/10/0f/e4b3ffc748827a14a474ec9c42e45c066050e440fec57e914091d9adda75/polars_runtime_32-1.40.1-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:e5f7becc237a7ec9d9a10878dc8e54b73bbf4e2d94a2991c37d7a0b38590d8f9", size = 50432590, upload-time = "2026-04-22T19:14:43.388Z" }, + { url = "https://files.pythonhosted.org/packages/d9/0b/b8d95fbed869fa4caabe9c400e4210374913b376e925e96fdcfa9be6416b/polars_runtime_32-1.40.1-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:992d14cf191dde043d36fbdbc98a65e43fbc7e9a5024cecd45f838ac4988c1ee", size = 54155564, upload-time = "2026-04-22T19:14:47.239Z" }, + { url = "https://files.pythonhosted.org/packages/06/d9/d091d8fb5cbed5e9536adfed955c4c89987a4cc3b8e73ae4532402b91c74/polars_runtime_32-1.40.1-cp310-abi3-win_amd64.whl", hash = "sha256:f78bb2abd00101cbb23cc0cb068f7e36e081057a15d2ec2dde3dda280709f030", size = 51829755, upload-time = "2026-04-22T19:14:50.85Z" }, + { url = "https://files.pythonhosted.org/packages/65/ad/b33c3022a394f3eb55c3310597cec615412a8a33880055eee191d154a628/polars_runtime_32-1.40.1-cp310-abi3-win_arm64.whl", hash = "sha256:b5cbfaf6b085b420b4bfcbe24e8f665076d1cccfdb80c0484c02a023ce205537", size = 45822104, upload-time = "2026-04-22T19:14:54.192Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165, upload-time = "2026-04-07T17:16:18.027Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "tomli" +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/de/48c59722572767841493b26183a0d1cc411d54fd759c5607c4590b6563a6/tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f", size = 17543, upload-time = "2026-03-25T20:22:03.828Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/11/db3d5885d8528263d8adc260bb2d28ebf1270b96e98f0e0268d32b8d9900/tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30", size = 154704, upload-time = "2026-03-25T20:21:10.473Z" }, + { url = "https://files.pythonhosted.org/packages/6d/f7/675db52c7e46064a9aa928885a9b20f4124ecb9bc2e1ce74c9106648d202/tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a", size = 149454, upload-time = "2026-03-25T20:21:12.036Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/81c50943cf953efa35bce7646caab3cf457a7d8c030b27cfb40d7235f9ee/tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076", size = 237561, upload-time = "2026-03-25T20:21:13.098Z" }, + { url = "https://files.pythonhosted.org/packages/48/c1/f41d9cb618acccca7df82aaf682f9b49013c9397212cb9f53219e3abac37/tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9", size = 243824, upload-time = "2026-03-25T20:21:14.569Z" }, + { url = "https://files.pythonhosted.org/packages/22/e4/5a816ecdd1f8ca51fb756ef684b90f2780afc52fc67f987e3c61d800a46d/tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c", size = 242227, upload-time = "2026-03-25T20:21:15.712Z" }, + { url = "https://files.pythonhosted.org/packages/6b/49/2b2a0ef529aa6eec245d25f0c703e020a73955ad7edf73e7f54ddc608aa5/tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc", size = 247859, upload-time = "2026-03-25T20:21:17.001Z" }, + { url = "https://files.pythonhosted.org/packages/83/bd/6c1a630eaca337e1e78c5903104f831bda934c426f9231429396ce3c3467/tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049", size = 97204, upload-time = "2026-03-25T20:21:18.079Z" }, + { url = "https://files.pythonhosted.org/packages/42/59/71461df1a885647e10b6bb7802d0b8e66480c61f3f43079e0dcd315b3954/tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e", size = 108084, upload-time = "2026-03-25T20:21:18.978Z" }, + { url = "https://files.pythonhosted.org/packages/b8/83/dceca96142499c069475b790e7913b1044c1a4337e700751f48ed723f883/tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece", size = 95285, upload-time = "2026-03-25T20:21:20.309Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ba/42f134a3fe2b370f555f44b1d72feebb94debcab01676bf918d0cb70e9aa/tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a", size = 155924, upload-time = "2026-03-25T20:21:21.626Z" }, + { url = "https://files.pythonhosted.org/packages/dc/c7/62d7a17c26487ade21c5422b646110f2162f1fcc95980ef7f63e73c68f14/tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085", size = 150018, upload-time = "2026-03-25T20:21:23.002Z" }, + { url = "https://files.pythonhosted.org/packages/5c/05/79d13d7c15f13bdef410bdd49a6485b1c37d28968314eabee452c22a7fda/tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9", size = 244948, upload-time = "2026-03-25T20:21:24.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/90/d62ce007a1c80d0b2c93e02cab211224756240884751b94ca72df8a875ca/tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5", size = 253341, upload-time = "2026-03-25T20:21:25.177Z" }, + { url = "https://files.pythonhosted.org/packages/1a/7e/caf6496d60152ad4ed09282c1885cca4eea150bfd007da84aea07bcc0a3e/tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585", size = 248159, upload-time = "2026-03-25T20:21:26.364Z" }, + { url = "https://files.pythonhosted.org/packages/99/e7/c6f69c3120de34bbd882c6fba7975f3d7a746e9218e56ab46a1bc4b42552/tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1", size = 253290, upload-time = "2026-03-25T20:21:27.46Z" }, + { url = "https://files.pythonhosted.org/packages/d6/2f/4a3c322f22c5c66c4b836ec58211641a4067364f5dcdd7b974b4c5da300c/tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917", size = 98141, upload-time = "2026-03-25T20:21:28.492Z" }, + { url = "https://files.pythonhosted.org/packages/24/22/4daacd05391b92c55759d55eaee21e1dfaea86ce5c571f10083360adf534/tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9", size = 108847, upload-time = "2026-03-25T20:21:29.386Z" }, + { url = "https://files.pythonhosted.org/packages/68/fd/70e768887666ddd9e9f5d85129e84910f2db2796f9096aa02b721a53098d/tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257", size = 95088, upload-time = "2026-03-25T20:21:30.677Z" }, + { url = "https://files.pythonhosted.org/packages/07/06/b823a7e818c756d9a7123ba2cda7d07bc2dd32835648d1a7b7b7a05d848d/tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54", size = 155866, upload-time = "2026-03-25T20:21:31.65Z" }, + { url = "https://files.pythonhosted.org/packages/14/6f/12645cf7f08e1a20c7eb8c297c6f11d31c1b50f316a7e7e1e1de6e2e7b7e/tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a", size = 149887, upload-time = "2026-03-25T20:21:33.028Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e0/90637574e5e7212c09099c67ad349b04ec4d6020324539297b634a0192b0/tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897", size = 243704, upload-time = "2026-03-25T20:21:34.51Z" }, + { url = "https://files.pythonhosted.org/packages/10/8f/d3ddb16c5a4befdf31a23307f72828686ab2096f068eaf56631e136c1fdd/tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f", size = 251628, upload-time = "2026-03-25T20:21:36.012Z" }, + { url = "https://files.pythonhosted.org/packages/e3/f1/dbeeb9116715abee2485bf0a12d07a8f31af94d71608c171c45f64c0469d/tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d", size = 247180, upload-time = "2026-03-25T20:21:37.136Z" }, + { url = "https://files.pythonhosted.org/packages/d3/74/16336ffd19ed4da28a70959f92f506233bd7cfc2332b20bdb01591e8b1d1/tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5", size = 251674, upload-time = "2026-03-25T20:21:38.298Z" }, + { url = "https://files.pythonhosted.org/packages/16/f9/229fa3434c590ddf6c0aa9af64d3af4b752540686cace29e6281e3458469/tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd", size = 97976, upload-time = "2026-03-25T20:21:39.316Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1e/71dfd96bcc1c775420cb8befe7a9d35f2e5b1309798f009dca17b7708c1e/tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36", size = 108755, upload-time = "2026-03-25T20:21:40.248Z" }, + { url = "https://files.pythonhosted.org/packages/83/7a/d34f422a021d62420b78f5c538e5b102f62bea616d1d75a13f0a88acb04a/tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd", size = 95265, upload-time = "2026-03-25T20:21:41.219Z" }, + { url = "https://files.pythonhosted.org/packages/3c/fb/9a5c8d27dbab540869f7c1f8eb0abb3244189ce780ba9cd73f3770662072/tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf", size = 155726, upload-time = "2026-03-25T20:21:42.23Z" }, + { url = "https://files.pythonhosted.org/packages/62/05/d2f816630cc771ad836af54f5001f47a6f611d2d39535364f148b6a92d6b/tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac", size = 149859, upload-time = "2026-03-25T20:21:43.386Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/66341bdb858ad9bd0ceab5a86f90eddab127cf8b046418009f2125630ecb/tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662", size = 244713, upload-time = "2026-03-25T20:21:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/df/6d/c5fad00d82b3c7a3ab6189bd4b10e60466f22cfe8a08a9394185c8a8111c/tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853", size = 252084, upload-time = "2026-03-25T20:21:45.62Z" }, + { url = "https://files.pythonhosted.org/packages/00/71/3a69e86f3eafe8c7a59d008d245888051005bd657760e96d5fbfb0b740c2/tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15", size = 247973, upload-time = "2026-03-25T20:21:46.937Z" }, + { url = "https://files.pythonhosted.org/packages/67/50/361e986652847fec4bd5e4a0208752fbe64689c603c7ae5ea7cb16b1c0ca/tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba", size = 256223, upload-time = "2026-03-25T20:21:48.467Z" }, + { url = "https://files.pythonhosted.org/packages/8c/9a/b4173689a9203472e5467217e0154b00e260621caa227b6fa01feab16998/tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6", size = 98973, upload-time = "2026-03-25T20:21:49.526Z" }, + { url = "https://files.pythonhosted.org/packages/14/58/640ac93bf230cd27d002462c9af0d837779f8773bc03dee06b5835208214/tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7", size = 109082, upload-time = "2026-03-25T20:21:50.506Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2f/702d5e05b227401c1068f0d386d79a589bb12bf64c3d2c72ce0631e3bc49/tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232", size = 96490, upload-time = "2026-03-25T20:21:51.474Z" }, + { url = "https://files.pythonhosted.org/packages/45/4b/b877b05c8ba62927d9865dd980e34a755de541eb65fffba52b4cc495d4d2/tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4", size = 164263, upload-time = "2026-03-25T20:21:52.543Z" }, + { url = "https://files.pythonhosted.org/packages/24/79/6ab420d37a270b89f7195dec5448f79400d9e9c1826df982f3f8e97b24fd/tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c", size = 160736, upload-time = "2026-03-25T20:21:53.674Z" }, + { url = "https://files.pythonhosted.org/packages/02/e0/3630057d8eb170310785723ed5adcdfb7d50cb7e6455f85ba8a3deed642b/tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d", size = 270717, upload-time = "2026-03-25T20:21:55.129Z" }, + { url = "https://files.pythonhosted.org/packages/7a/b4/1613716072e544d1a7891f548d8f9ec6ce2faf42ca65acae01d76ea06bb0/tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41", size = 278461, upload-time = "2026-03-25T20:21:56.228Z" }, + { url = "https://files.pythonhosted.org/packages/05/38/30f541baf6a3f6df77b3df16b01ba319221389e2da59427e221ef417ac0c/tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c", size = 274855, upload-time = "2026-03-25T20:21:57.653Z" }, + { url = "https://files.pythonhosted.org/packages/77/a3/ec9dd4fd2c38e98de34223b995a3b34813e6bdadf86c75314c928350ed14/tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f", size = 283144, upload-time = "2026-03-25T20:21:59.089Z" }, + { url = "https://files.pythonhosted.org/packages/ef/be/605a6261cac79fba2ec0c9827e986e00323a1945700969b8ee0b30d85453/tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8", size = 108683, upload-time = "2026-03-25T20:22:00.214Z" }, + { url = "https://files.pythonhosted.org/packages/12/64/da524626d3b9cc40c168a13da8335fe1c51be12c0a63685cc6db7308daae/tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26", size = 121196, upload-time = "2026-03-25T20:22:01.169Z" }, + { url = "https://files.pythonhosted.org/packages/5a/cd/e80b62269fc78fc36c9af5a6b89c835baa8af28ff5ad28c7028d60860320/tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396", size = 100393, upload-time = "2026-03-25T20:22:02.137Z" }, + { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] diff --git a/uv.lock b/uv.lock index 689615a..db97db1 100644 --- a/uv.lock +++ b/uv.lock @@ -271,6 +271,116 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/eb/f5/af8a71b99fef42507d2bc50e8a532ddcd77717cbcd3da3878e67e36c2405/conjure_python_client-3.1.0-py2.py3-none-any.whl", hash = "sha256:a756e6a1bd3e024340ea1959e209749f73bd9c72c2befd8ae1efd726ddd1ea45", size = 20934, upload-time = "2025-09-04T18:30:17.741Z" }, ] +[[package]] +name = "cramjam" +version = "2.11.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/14/12/34bf6e840a79130dfd0da7badfb6f7810b8fcfd60e75b0539372667b41b6/cramjam-2.11.0.tar.gz", hash = "sha256:5c82500ed91605c2d9781380b378397012e25127e89d64f460fea6aeac4389b4", size = 99100, upload-time = "2025-07-27T21:25:07.559Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/53/d3/20d0402e4e983b66603117ad3dd3b864a05d7997a830206d3ff9cacef9a2/cramjam-2.11.0-cp310-cp310-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:d0859c65775e8ebf2cbc084bfd51bd0ffda10266da6f9306451123b89f8e5a63", size = 3558999, upload-time = "2025-07-27T21:21:34.105Z" }, + { url = "https://files.pythonhosted.org/packages/f5/a8/a6e2744288938ccd320a5c6f6f3653faa790f933f5edd088c6e5782a2354/cramjam-2.11.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:1d77b9b0aca02a3f6eeeff27fcd315ca5972616c0919ee38e522cce257bcd349", size = 1861558, upload-time = "2025-07-27T21:21:36.624Z" }, + { url = "https://files.pythonhosted.org/packages/96/29/7961e09a849eea7d8302e7baa6f829dd3ef3faf199cb25ed29b318ae799b/cramjam-2.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:66425bc25b5481359b12a6719b6e7c90ffe76d85d0691f1da7df304bfb8ce45c", size = 1699431, upload-time = "2025-07-27T21:21:38.396Z" }, + { url = "https://files.pythonhosted.org/packages/7a/60/6665e52f01a8919bf37c43dcf0e03b6dd3866f5c4e95440b357d508ee14e/cramjam-2.11.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:bd748d3407ec63e049b3aea1595e218814fccab329b7fb10bb51120a30e9fb7e", size = 2025262, upload-time = "2025-07-27T21:21:40.417Z" }, + { url = "https://files.pythonhosted.org/packages/d7/80/79bd84dbeb109e2c6efb74e661b7bd4c3ba393208ebcf69e2ae9454ae80c/cramjam-2.11.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a6d9a23a35b3a105c42a8de60fc2e80281ae6e758f05a3baea0b68eb1ddcb679", size = 1766177, upload-time = "2025-07-27T21:21:42.224Z" }, + { url = "https://files.pythonhosted.org/packages/28/ef/b43280767ebcde022ba31f1e9902137655a956ae30e920d75630fa67e36e/cramjam-2.11.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:40a75b95e05e38a2a055b2446f09994ce1139151721659315151d4ad6289bbff", size = 1854031, upload-time = "2025-07-27T21:21:43.651Z" }, + { url = "https://files.pythonhosted.org/packages/60/1c/79d522757c494dfd9e9b208b0604cc7e97b481483cc477144f5705a06ab7/cramjam-2.11.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e5d042c376d2025300da37d65192d06a457918b63b31140f697f85fd8e310b29", size = 2035812, upload-time = "2025-07-27T21:21:45.473Z" }, + { url = "https://files.pythonhosted.org/packages/c8/70/3bf0670380069b3abd4c6b53f61d3148f4e08935569c08efbeaf7550e87d/cramjam-2.11.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cb148b35ab20c75b19a06c27f05732e2a321adbd86fadc93f9466dbd7b1154a7", size = 2067661, upload-time = "2025-07-27T21:21:47.901Z" }, + { url = "https://files.pythonhosted.org/packages/db/7e/4f6ca98a4b474348e965a529b359184785d1119ab7c4c9ec1280b8bea50a/cramjam-2.11.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ee47c220f0f5179ddc923ab91fc9e282c27b29fabc60c433dfe06f08084f798", size = 1981523, upload-time = "2025-07-27T21:21:49.704Z" }, + { url = "https://files.pythonhosted.org/packages/8a/6c/b241511c7ffd5f1da29641429bb0e19b5fbcffafde5ba1bbcbf9394ea456/cramjam-2.11.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0cf1b5a81b21ea175c976c3ab09e00494258f4b49b7995efc86060cced3f0b2e", size = 2034251, upload-time = "2025-07-27T21:21:51.252Z" }, + { url = "https://files.pythonhosted.org/packages/14/5c/4ef926c8c3c1bf6da96f9c53450ff334cdb6d0fc1efced0aea97e2090803/cramjam-2.11.0-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:360c00338ecf48921492455007f904be607fc7818de3d681acbcc542aae2fb36", size = 2155322, upload-time = "2025-07-27T21:21:53.348Z" }, + { url = "https://files.pythonhosted.org/packages/be/fb/eb2aef7fb2730e56c5a2c9000817ee8fb4a95c92f19cc6e441afed42ec29/cramjam-2.11.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:f31fcc0d30dc3f3e94ea6b4d8e1a855071757c6abf6a7b1e284050ab7d4c299c", size = 2169094, upload-time = "2025-07-27T21:21:55.187Z" }, + { url = "https://files.pythonhosted.org/packages/3b/80/925a5c668dcee1c6f61775067185c5dc9a63c766d5393e5c60d2af4217a7/cramjam-2.11.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:033be66fdceb3d63b2c99b257a98380c4ec22c9e4dca54a2bfec3718cd24e184", size = 2159089, upload-time = "2025-07-27T21:21:57.118Z" }, + { url = "https://files.pythonhosted.org/packages/1f/ac/b2819640eef0592a6de7ca832c0d23c69bd1620f765ce88b60dbc8da9ba2/cramjam-2.11.0-cp310-cp310-win32.whl", hash = "sha256:1c6cea67f6000b81f6bd27d14c8a6f62d00336ca7252fd03ee16f6b70eb5c0d2", size = 1605046, upload-time = "2025-07-27T21:21:58.617Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f4/06af04727b9556721049e2127656d727306d275c518e3d97f9ed4cffd0d8/cramjam-2.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:98aa4a351b047b0f7f9e971585982065028adc2c162c5c23c5d5734c5ccc1077", size = 1710647, upload-time = "2025-07-27T21:22:00.279Z" }, + { url = "https://files.pythonhosted.org/packages/d0/89/8001f6a9b6b6e9fa69bec5319789083475d6f26d52aaea209d3ebf939284/cramjam-2.11.0-cp311-cp311-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:04cfa39118570e70e920a9b75c733299784b6d269733dbc791d9aaed6edd2615", size = 3559272, upload-time = "2025-07-27T21:22:01.988Z" }, + { url = "https://files.pythonhosted.org/packages/0b/f3/001d00070ca92e5fbe6aacc768e455568b0cde46b0eb944561a4ea132300/cramjam-2.11.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:66a18f68506290349a256375d7aa2f645b9f7993c10fc4cc211db214e4e61d2b", size = 1861743, upload-time = "2025-07-27T21:22:03.754Z" }, + { url = "https://files.pythonhosted.org/packages/c9/35/041a3af01bf3f6158f120070f798546d4383b962b63c35cd91dcbf193e17/cramjam-2.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:50e7d65533857736cd56f6509cf2c4866f28ad84dd15b5bdbf2f8a81e77fa28a", size = 1699631, upload-time = "2025-07-27T21:22:05.192Z" }, + { url = "https://files.pythonhosted.org/packages/17/eb/5358b238808abebd0c949c42635c3751204ca7cf82b29b984abe9f5e33c8/cramjam-2.11.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:1f71989668458fc327ac15396db28d92df22f8024bb12963929798b2729d2df5", size = 2025603, upload-time = "2025-07-27T21:22:06.726Z" }, + { url = "https://files.pythonhosted.org/packages/0e/79/19dba7c03a27408d8d11b5a7a4a7908459cfd4e6f375b73264dc66517bf6/cramjam-2.11.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee77ac543f1e2b22af1e8be3ae589f729491b6090582340aacd77d1d757d9569", size = 1766283, upload-time = "2025-07-27T21:22:08.568Z" }, + { url = "https://files.pythonhosted.org/packages/a4/ad/40e4b3408501d886d082db465c33971655fe82573c535428e52ab905f4d0/cramjam-2.11.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ad52784120e7e4d8a0b5b0517d185b8bf7f74f5e17272857ddc8951a628d9be1", size = 1854407, upload-time = "2025-07-27T21:22:10.518Z" }, + { url = "https://files.pythonhosted.org/packages/36/6e/c1b60ceb6d7ea6ff8b0bf197520aefe23f878bf2bfb0de65f2b0c2f82cd1/cramjam-2.11.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b86f8e6d9c1b3f9a75b2af870c93ceee0f1b827cd2507387540e053b35d7459", size = 2035793, upload-time = "2025-07-27T21:22:12.504Z" }, + { url = "https://files.pythonhosted.org/packages/9c/ad/32a8d5f4b1e3717787945ec6d71bd1c6e6bccba4b7e903fc0d9d4e4b08c3/cramjam-2.11.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:320d61938950d95da2371b46c406ec433e7955fae9f396c8e1bf148ffc187d11", size = 2067499, upload-time = "2025-07-27T21:22:14.067Z" }, + { url = "https://files.pythonhosted.org/packages/ff/cd/3b5a662736ea62ff7fa4c4a10a85e050bfdaad375cc53dc80427e8afe41c/cramjam-2.11.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:41eafc8c1653a35a5c7e75ad48138f9f60085cc05cd99d592e5298552d944e9f", size = 1981853, upload-time = "2025-07-27T21:22:15.908Z" }, + { url = "https://files.pythonhosted.org/packages/26/8e/1dbcfaaa7a702ee82ee683ec3a81656934dd7e04a7bc4ee854033686f98a/cramjam-2.11.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:03a7316c6bf763dfa34279335b27702321da44c455a64de58112968c0818ec4a", size = 2034514, upload-time = "2025-07-27T21:22:17.352Z" }, + { url = "https://files.pythonhosted.org/packages/50/62/f11709bfdce74af79a88b410dcb76dedc97612166e759136931bf63cfd7b/cramjam-2.11.0-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:244c2ed8bd7ccbb294a2abe7ca6498db7e89d7eb5e744691dc511a7dc82e65ca", size = 2155343, upload-time = "2025-07-27T21:22:18.854Z" }, + { url = "https://files.pythonhosted.org/packages/8a/6d/3b98b61841a5376d9a9b8468ae58753a8e6cf22be9534a0fa5af4d8621cc/cramjam-2.11.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:405f8790bad36ce0b4bbdb964ad51507bfc7942c78447f25cb828b870a1d86a0", size = 2169367, upload-time = "2025-07-27T21:22:20.389Z" }, + { url = "https://files.pythonhosted.org/packages/11/72/bd5db5c49dbebc8b002f1c4983101b28d2e7fc9419753db1c31ec22b03ef/cramjam-2.11.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6b1b751a5411032b08fb3ac556160229ca01c6bbe4757bb3a9a40b951ebaac23", size = 2159334, upload-time = "2025-07-27T21:22:22.254Z" }, + { url = "https://files.pythonhosted.org/packages/34/32/203c57acdb6eea727e7078b2219984e64ed4ad043c996ed56321301ba167/cramjam-2.11.0-cp311-cp311-win32.whl", hash = "sha256:5251585608778b9ac8effed544933df7ad85b4ba21ee9738b551f17798b215ac", size = 1605313, upload-time = "2025-07-27T21:22:24.126Z" }, + { url = "https://files.pythonhosted.org/packages/a9/bd/102d6deb87a8524ac11cddcd31a7612b8f20bf9b473c3c645045e3b957c7/cramjam-2.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:dca88bc8b68ce6d35dafd8c4d5d59a238a56c43fa02b74c2ce5f9dfb0d1ccb46", size = 1710991, upload-time = "2025-07-27T21:22:25.661Z" }, + { url = "https://files.pythonhosted.org/packages/0b/0d/7c84c913a5fae85b773a9dcf8874390f9d68ba0fcc6630efa7ff1541b950/cramjam-2.11.0-cp312-cp312-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:dba5c14b8b4f73ea1e65720f5a3fe4280c1d27761238378be8274135c60bbc6e", size = 3553368, upload-time = "2025-07-27T21:22:27.162Z" }, + { url = "https://files.pythonhosted.org/packages/2b/cc/4f6d185d8a744776f53035e72831ff8eefc2354f46ab836f4bd3c4f6c138/cramjam-2.11.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:11eb40722b3fcf3e6890fba46c711bf60f8dc26360a24876c85e52d76c33b25b", size = 1860014, upload-time = "2025-07-27T21:22:28.738Z" }, + { url = "https://files.pythonhosted.org/packages/1c/a8/626c76263085c6d5ded0e71823b411e9522bfc93ba6cc59855a5869296e7/cramjam-2.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:aeb26e2898994b6e8319f19a4d37c481512acdcc6d30e1b5ecc9d8ec57e835cb", size = 1693512, upload-time = "2025-07-27T21:22:30.999Z" }, + { url = "https://files.pythonhosted.org/packages/e9/52/0851a16a62447532e30ba95a80e638926fdea869a34b4b5b9d0a020083ba/cramjam-2.11.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:4f8d82081ed7d8fe52c982bd1f06e4c7631a73fe1fb6d4b3b3f2404f87dc40fe", size = 2025285, upload-time = "2025-07-27T21:22:32.954Z" }, + { url = "https://files.pythonhosted.org/packages/98/76/122e444f59dbc216451d8e3d8282c9665dc79eaf822f5f1470066be1b695/cramjam-2.11.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:092a3ec26e0a679305018380e4f652eae1b6dfe3fc3b154ee76aa6b92221a17c", size = 1761327, upload-time = "2025-07-27T21:22:34.484Z" }, + { url = "https://files.pythonhosted.org/packages/a3/bc/3a0189aef1af2b29632c039c19a7a1b752bc21a4053582a5464183a0ad3d/cramjam-2.11.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:529d6d667c65fd105d10bd83d1cd3f9869f8fd6c66efac9415c1812281196a92", size = 1854075, upload-time = "2025-07-27T21:22:36.157Z" }, + { url = "https://files.pythonhosted.org/packages/2e/80/8a6343b13778ce52d94bb8d5365a30c3aa951276b1857201fe79d7e2ad25/cramjam-2.11.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:555eb9c90c450e0f76e27d9ff064e64a8b8c6478ab1a5594c91b7bc5c82fd9f0", size = 2032710, upload-time = "2025-07-27T21:22:38.17Z" }, + { url = "https://files.pythonhosted.org/packages/df/6b/cd1778a207c29eda10791e3dfa018b588001928086e179fc71254793c625/cramjam-2.11.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5edf4c9e32493035b514cf2ba0c969d81ccb31de63bd05490cc8bfe3b431674e", size = 2068353, upload-time = "2025-07-27T21:22:39.615Z" }, + { url = "https://files.pythonhosted.org/packages/dc/f0/5c2a5cd5711032f3b191ca50cb786c17689b4a9255f9f768866e6c9f04d9/cramjam-2.11.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fa2fe41f48c4d58d923803383b0737f048918b5a0d10390de9628bb6272b107", size = 1978104, upload-time = "2025-07-27T21:22:41.106Z" }, + { url = "https://files.pythonhosted.org/packages/f9/8b/b363a5fb2c3347504fe9a64f8d0f1e276844f0e532aa7162c061cd1ffee4/cramjam-2.11.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9ca14cf1cabdb0b77d606db1bb9e9ca593b1dbd421fcaf251ec9a5431ec449f3", size = 2030779, upload-time = "2025-07-27T21:22:42.969Z" }, + { url = "https://files.pythonhosted.org/packages/78/7b/d83dad46adb6c988a74361f81ad9c5c22642be53ad88616a19baedd06243/cramjam-2.11.0-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:309e95bf898829476bccf4fd2c358ec00e7ff73a12f95a3cdeeba4bb1d3683d5", size = 2155297, upload-time = "2025-07-27T21:22:44.6Z" }, + { url = "https://files.pythonhosted.org/packages/1a/be/60d9be4cb33d8740a4aa94c7513f2ef3c4eba4fd13536f086facbafade71/cramjam-2.11.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:86dca35d2f15ef22922411496c220f3c9e315d5512f316fe417461971cc1648d", size = 2169255, upload-time = "2025-07-27T21:22:46.534Z" }, + { url = "https://files.pythonhosted.org/packages/11/b0/4a595f01a243aec8ad272b160b161c44351190c35d98d7787919d962e9e5/cramjam-2.11.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:193c6488bd2f514cbc0bef5c18fad61a5f9c8d059dd56edf773b3b37f0e85496", size = 2155651, upload-time = "2025-07-27T21:22:48.46Z" }, + { url = "https://files.pythonhosted.org/packages/38/47/7776659aaa677046b77f527106e53ddd47373416d8fcdb1e1a881ec5dc06/cramjam-2.11.0-cp312-cp312-win32.whl", hash = "sha256:514e2c008a8b4fa823122ca3ecab896eac41d9aa0f5fc881bd6264486c204e32", size = 1603568, upload-time = "2025-07-27T21:22:50.084Z" }, + { url = "https://files.pythonhosted.org/packages/75/b1/d53002729cfd94c5844ddfaf1233c86d29f2dbfc1b764a6562c41c044199/cramjam-2.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:53fed080476d5f6ad7505883ec5d1ec28ba36c2273db3b3e92d7224fe5e463db", size = 1709287, upload-time = "2025-07-27T21:22:51.534Z" }, + { url = "https://files.pythonhosted.org/packages/0a/8b/406c5dc0f8e82385519d8c299c40fd6a56d97eca3fcd6f5da8dad48de75b/cramjam-2.11.0-cp313-cp313-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:2c289729cc1c04e88bafa48b51082fb462b0a57dbc96494eab2be9b14dca62af", size = 3553330, upload-time = "2025-07-27T21:22:53.124Z" }, + { url = "https://files.pythonhosted.org/packages/00/ad/4186884083d6e4125b285903e17841827ab0d6d0cffc86216d27ed91e91d/cramjam-2.11.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:045201ee17147e36cf43d8ae2fa4b4836944ac672df5874579b81cf6d40f1a1f", size = 1859756, upload-time = "2025-07-27T21:22:54.821Z" }, + { url = "https://files.pythonhosted.org/packages/54/01/91b485cf76a7efef638151e8a7d35784dae2c4ff221b1aec2c083e4b106d/cramjam-2.11.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:619cd195d74c9e1d2a3ad78d63451d35379c84bd851aec552811e30842e1c67a", size = 1693609, upload-time = "2025-07-27T21:22:56.331Z" }, + { url = "https://files.pythonhosted.org/packages/cd/84/d0c80d279b2976870fc7d10f15dcb90a3c10c06566c6964b37c152694974/cramjam-2.11.0-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6eb3ae5ab72edb2ed68bdc0f5710f0a6cad7fd778a610ec2c31ee15e32d3921e", size = 2024912, upload-time = "2025-07-27T21:22:57.915Z" }, + { url = "https://files.pythonhosted.org/packages/d6/70/88f2a5cb904281ed5d3c111b8f7d5366639817a5470f059bcd26833fc870/cramjam-2.11.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:df7da3f4b19e3078f9635f132d31b0a8196accb2576e3213ddd7a77f93317c20", size = 1760715, upload-time = "2025-07-27T21:22:59.528Z" }, + { url = "https://files.pythonhosted.org/packages/b2/06/cf5b02081132537d28964fb385fcef9ed9f8a017dd7d8c59d317e53ba50d/cramjam-2.11.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:57286b289cd557ac76c24479d8ecfb6c3d5b854cce54ccc7671f9a2f5e2a2708", size = 1853782, upload-time = "2025-07-27T21:23:01.07Z" }, + { url = "https://files.pythonhosted.org/packages/57/27/63525087ed40a53d1867021b9c4858b80cc86274ffe7225deed067d88d92/cramjam-2.11.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:28952fbbf8b32c0cb7fa4be9bcccfca734bf0d0989f4b509dc7f2f70ba79ae06", size = 2032354, upload-time = "2025-07-27T21:23:03.021Z" }, + { url = "https://files.pythonhosted.org/packages/c3/ef/dbba082c6ebfb6410da4dd39a64e654d7194fcfd4567f85991a83fa4ec32/cramjam-2.11.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:78ed2e4099812a438b545dfbca1928ec825e743cd253bc820372d6ef8c3adff4", size = 2068007, upload-time = "2025-07-27T21:23:04.526Z" }, + { url = "https://files.pythonhosted.org/packages/35/ce/d902b9358a46a086938feae83b2251720e030f06e46006f4c1fc0ac9da20/cramjam-2.11.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d9aecd5c3845d415bd6c9957c93de8d93097e269137c2ecb0e5a5256374bdc8", size = 1977485, upload-time = "2025-07-27T21:23:06.058Z" }, + { url = "https://files.pythonhosted.org/packages/e8/03/982f54553244b0afcbdb2ad2065d460f0ab05a72a96896a969a1ca136a1e/cramjam-2.11.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:362fcf4d6f5e1242a4540812455f5a594949190f6fbc04f2ffbfd7ae0266d788", size = 2030447, upload-time = "2025-07-27T21:23:07.679Z" }, + { url = "https://files.pythonhosted.org/packages/74/5f/748e54cdb665ec098ec519e23caacc65fc5ae58718183b071e33fc1c45b4/cramjam-2.11.0-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:13240b3dea41b1174456cb9426843b085dc1a2bdcecd9ee2d8f65ac5703374b0", size = 2154949, upload-time = "2025-07-27T21:23:09.366Z" }, + { url = "https://files.pythonhosted.org/packages/69/81/c4e6cb06ed69db0dc81f9a8b1dc74995ebd4351e7a1877143f7031ff2700/cramjam-2.11.0-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:c54eed83726269594b9086d827decc7d2015696e31b99bf9b69b12d9063584fe", size = 2168925, upload-time = "2025-07-27T21:23:10.976Z" }, + { url = "https://files.pythonhosted.org/packages/13/5b/966365523ce8290a08e163e3b489626c5adacdff2b3da9da1b0823dfb14e/cramjam-2.11.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:f8195006fdd0fc0a85b19df3d64a3ef8a240e483ae1dfc7ac6a4316019eb5df2", size = 2154950, upload-time = "2025-07-27T21:23:12.514Z" }, + { url = "https://files.pythonhosted.org/packages/3a/7d/7f8eb5c534b72b32c6eb79d74585bfee44a9a5647a14040bb65c31c2572d/cramjam-2.11.0-cp313-cp313-win32.whl", hash = "sha256:ccf30e3fe6d770a803dcdf3bb863fa44ba5dc2664d4610ba2746a3c73599f2e4", size = 1603199, upload-time = "2025-07-27T21:23:14.38Z" }, + { url = "https://files.pythonhosted.org/packages/37/05/47b5e0bf7c41a3b1cdd3b7c2147f880c93226a6bef1f5d85183040cbdece/cramjam-2.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:ee36348a204f0a68b03400f4736224e9f61d1c6a1582d7f875c1ca56f0254268", size = 1708924, upload-time = "2025-07-27T21:23:16.332Z" }, + { url = "https://files.pythonhosted.org/packages/de/07/a1051cdbbe6d723df16d756b97f09da7c1adb69e29695c58f0392bc12515/cramjam-2.11.0-cp314-cp314-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:7ba5e38c9fbd06f086f4a5a64a1a5b7b417cd3f8fc07a20e5c03651f72f36100", size = 3554141, upload-time = "2025-07-27T21:23:17.938Z" }, + { url = "https://files.pythonhosted.org/packages/74/66/58487d2e16ef3d04f51a7c7f0e69823e806744b4c21101e89da4873074bc/cramjam-2.11.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:b8adeee57b41fe08e4520698a4b0bd3cc76dbd81f99424b806d70a5256a391d3", size = 1860353, upload-time = "2025-07-27T21:23:19.593Z" }, + { url = "https://files.pythonhosted.org/packages/67/b4/67f6254d166ffbcc9d5fa1b56876eaa920c32ebc8e9d3d525b27296b693b/cramjam-2.11.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:b96a74fa03a636c8a7d76f700d50e9a8bc17a516d6a72d28711225d641e30968", size = 1693832, upload-time = "2025-07-27T21:23:21.185Z" }, + { url = "https://files.pythonhosted.org/packages/55/a3/4e0b31c0d454ae70c04684ed7c13d3c67b4c31790c278c1e788cb804fa4a/cramjam-2.11.0-cp314-cp314-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c3811a56fa32e00b377ef79121c0193311fd7501f0fb378f254c7f083cc1fbe0", size = 2027080, upload-time = "2025-07-27T21:23:23.303Z" }, + { url = "https://files.pythonhosted.org/packages/d9/c7/5e8eed361d1d3b8be14f38a54852c5370cc0ceb2c2d543b8ba590c34f080/cramjam-2.11.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5d927e87461f8a0d448e4ab5eb2bca9f31ca5d8ea86d70c6f470bb5bc666d7e", size = 1761543, upload-time = "2025-07-27T21:23:24.991Z" }, + { url = "https://files.pythonhosted.org/packages/09/0c/06b7f8b0ce9fde89470505116a01fc0b6cb92d406c4fb1e46f168b5d3fa5/cramjam-2.11.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f1f5c450121430fd89cb5767e0a9728ecc65997768fd4027d069cb0368af62f9", size = 1854636, upload-time = "2025-07-27T21:23:26.987Z" }, + { url = "https://files.pythonhosted.org/packages/6f/c6/6ebc02c9d5acdf4e5f2b1ec6e1252bd5feee25762246798ae823b3347457/cramjam-2.11.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:724aa7490be50235d97f07e2ca10067927c5d7f336b786ddbc868470e822aa25", size = 2032715, upload-time = "2025-07-27T21:23:28.603Z" }, + { url = "https://files.pythonhosted.org/packages/a2/77/a122971c23f5ca4b53e4322c647ac7554626c95978f92d19419315dddd05/cramjam-2.11.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:54c4637122e7cfd7aac5c1d3d4c02364f446d6923ea34cf9d0e8816d6e7a4936", size = 2069039, upload-time = "2025-07-27T21:23:30.319Z" }, + { url = "https://files.pythonhosted.org/packages/19/0f/f6121b90b86b9093c066889274d26a1de3f29969d45c2ed1ecbe2033cb78/cramjam-2.11.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17eb39b1696179fb471eea2de958fa21f40a2cd8bf6b40d428312d5541e19dc4", size = 1979566, upload-time = "2025-07-27T21:23:32.002Z" }, + { url = "https://files.pythonhosted.org/packages/e0/a3/f95bc57fd7f4166ce6da816cfa917fb7df4bb80e669eb459d85586498414/cramjam-2.11.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:36aa5a798aa34e11813a80425a30d8e052d8de4a28f27bfc0368cfc454d1b403", size = 2030905, upload-time = "2025-07-27T21:23:33.696Z" }, + { url = "https://files.pythonhosted.org/packages/fc/52/e429de4e8bc86ee65e090dae0f87f45abd271742c63fb2d03c522ffde28a/cramjam-2.11.0-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:449fca52774dc0199545fbf11f5128933e5a6833946707885cf7be8018017839", size = 2155592, upload-time = "2025-07-27T21:23:35.375Z" }, + { url = "https://files.pythonhosted.org/packages/6c/6c/65a7a0207787ad39ad804af4da7f06a60149de19481d73d270b540657234/cramjam-2.11.0-cp314-cp314-musllinux_1_1_i686.whl", hash = "sha256:d87d37b3d476f4f7623c56a232045d25bd9b988314702ea01bd9b4a94948a778", size = 2170839, upload-time = "2025-07-27T21:23:37.197Z" }, + { url = "https://files.pythonhosted.org/packages/b2/c5/5c5db505ba692bc844246b066e23901d5905a32baf2f33719c620e65887f/cramjam-2.11.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:26cb45c47d71982d76282e303931c6dd4baee1753e5d48f9a89b3a63e690b3a3", size = 2157236, upload-time = "2025-07-27T21:23:38.854Z" }, + { url = "https://files.pythonhosted.org/packages/b0/22/88e6693e60afe98901e5bbe91b8dea193e3aa7f42e2770f9c3339f5c1065/cramjam-2.11.0-cp314-cp314-win32.whl", hash = "sha256:4efe919d443c2fd112fe25fe636a52f9628250c9a50d9bddb0488d8a6c09acc6", size = 1604136, upload-time = "2025-07-27T21:23:40.56Z" }, + { url = "https://files.pythonhosted.org/packages/cc/f8/01618801cd59ccedcc99f0f96d20be67d8cfc3497da9ccaaad6b481781dd/cramjam-2.11.0-cp314-cp314-win_amd64.whl", hash = "sha256:ccec3524ea41b9abd5600e3e27001fd774199dbb4f7b9cb248fcee37d4bda84c", size = 1710272, upload-time = "2025-07-27T21:23:42.236Z" }, + { url = "https://files.pythonhosted.org/packages/40/81/6cdb3ed222d13ae86bda77aafe8d50566e81a1169d49ed195b6263610704/cramjam-2.11.0-cp314-cp314t-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:966ac9358b23d21ecd895c418c048e806fd254e46d09b1ff0cdad2eba195ea3e", size = 3559671, upload-time = "2025-07-27T21:23:44.504Z" }, + { url = "https://files.pythonhosted.org/packages/cb/43/52b7e54fe5ba1ef0270d9fdc43dabd7971f70ea2d7179be918c997820247/cramjam-2.11.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:387f09d647a0d38dcb4539f8a14281f8eb6bb1d3e023471eb18a5974b2121c86", size = 1867876, upload-time = "2025-07-27T21:23:46.987Z" }, + { url = "https://files.pythonhosted.org/packages/9d/28/30d5b8d10acd30db3193bc562a313bff722888eaa45cfe32aa09389f2b24/cramjam-2.11.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:665b0d8fbbb1a7f300265b43926457ec78385200133e41fef19d85790fc1e800", size = 1695562, upload-time = "2025-07-27T21:23:48.644Z" }, + { url = "https://files.pythonhosted.org/packages/d9/86/ec806f986e01b896a650655024ea52a13e25c3ac8a3a382f493089483cdc/cramjam-2.11.0-cp314-cp314t-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ca905387c7a371531b9622d93471be4d745ef715f2890c3702479cd4fc85aa51", size = 2025056, upload-time = "2025-07-27T21:23:50.404Z" }, + { url = "https://files.pythonhosted.org/packages/09/43/c2c17586b90848d29d63181f7d14b8bd3a7d00975ad46e3edf2af8af7e1f/cramjam-2.11.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c1aa56aef2c8af55a21ed39040a94a12b53fb23beea290f94d19a76027e2ffb", size = 1764084, upload-time = "2025-07-27T21:23:52.265Z" }, + { url = "https://files.pythonhosted.org/packages/2b/a9/68bc334fadb434a61df10071dc8606702aa4f5b6cdb2df62474fc21d2845/cramjam-2.11.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e5db59c1cdfaa2ab85cc988e602d6919495f735ca8a5fd7603608eb1e23c26d5", size = 1854859, upload-time = "2025-07-27T21:23:54.085Z" }, + { url = "https://files.pythonhosted.org/packages/5b/4e/b48e67835b5811ec5e9cb2e2bcba9c3fd76dab3e732569fe801b542c6ca9/cramjam-2.11.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b1f893014f00fe5e89a660a032e813bf9f6d91de74cd1490cdb13b2b59d0c9a3", size = 2035970, upload-time = "2025-07-27T21:23:55.758Z" }, + { url = "https://files.pythonhosted.org/packages/c4/70/d2ac33d572b4d90f7f0f2c8a1d60fb48f06b128fdc2c05f9b49891bb0279/cramjam-2.11.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c26a1eb487947010f5de24943bd7c422dad955b2b0f8650762539778c380ca89", size = 2069320, upload-time = "2025-07-27T21:23:57.494Z" }, + { url = "https://files.pythonhosted.org/packages/1d/4c/85cec77af4a74308ba5fca8e296c4e2f80ec465c537afc7ab1e0ca2f9a00/cramjam-2.11.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d5c8bfb438d94e7b892d1426da5fc4b4a5370cc360df9b8d9d77c33b896c37e", size = 1982668, upload-time = "2025-07-27T21:23:59.126Z" }, + { url = "https://files.pythonhosted.org/packages/55/45/938546d1629e008cc3138df7c424ef892719b1796ff408a2ab8550032e5e/cramjam-2.11.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:cb1fb8c9337ab0da25a01c05d69a0463209c347f16512ac43be5986f3d1ebaf4", size = 2034028, upload-time = "2025-07-27T21:24:00.865Z" }, + { url = "https://files.pythonhosted.org/packages/01/76/b5a53e20505555f1640e66dcf70394bcf51a1a3a072aa18ea35135a0f9ed/cramjam-2.11.0-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:1f6449f6de52dde3e2f1038284910c8765a397a25e2d05083870f3f5e7fc682c", size = 2155513, upload-time = "2025-07-27T21:24:02.92Z" }, + { url = "https://files.pythonhosted.org/packages/84/12/8d3f6ceefae81bbe45a347fdfa2219d9f3ac75ebc304f92cd5fcb4fbddc5/cramjam-2.11.0-cp314-cp314t-musllinux_1_1_i686.whl", hash = "sha256:382dec4f996be48ed9c6958d4e30c2b89435d7c2c4dbf32480b3b8886293dd65", size = 2170035, upload-time = "2025-07-27T21:24:04.558Z" }, + { url = "https://files.pythonhosted.org/packages/4b/85/3be6f0a1398f976070672be64f61895f8839857618a2d8cc0d3ab529d3dc/cramjam-2.11.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:d388bd5723732c3afe1dd1d181e4213cc4e1be210b080572e7d5749f6e955656", size = 2160229, upload-time = "2025-07-27T21:24:06.729Z" }, + { url = "https://files.pythonhosted.org/packages/57/5e/66cfc3635511b20014bbb3f2ecf0095efb3049e9e96a4a9e478e4f3d7b78/cramjam-2.11.0-cp314-cp314t-win32.whl", hash = "sha256:0a70ff17f8e1d13f322df616505550f0f4c39eda62290acb56f069d4857037c8", size = 1610267, upload-time = "2025-07-27T21:24:08.428Z" }, + { url = "https://files.pythonhosted.org/packages/ce/c6/c71e82e041c95ffe6a92ac707785500aa2a515a4339c2c7dd67e3c449249/cramjam-2.11.0-cp314-cp314t-win_amd64.whl", hash = "sha256:028400d699442d40dbda02f74158c73d05cb76587a12490d0bfedd958fd49188", size = 1713108, upload-time = "2025-07-27T21:24:10.147Z" }, + { url = "https://files.pythonhosted.org/packages/bf/8f/82e35ec3c5387f1864f46b3c24bce89a07af8bb3ef242ae47281db2c1848/cramjam-2.11.0-pp310-pypy310_pp73-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:37bed927abc4a7ae2d2669baa3675e21904d8a038ed8e4313326ea7b3be62b2b", size = 3573104, upload-time = "2025-07-27T21:24:40.069Z" }, + { url = "https://files.pythonhosted.org/packages/f0/4e/0c821918080a32ba1e52c040e12dd02dada67728f07305c5f778b808a807/cramjam-2.11.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:50e4a58635fa8c6897d84847d6e065eb69f92811670fc5e9f2d9e3b6279a02b6", size = 1873441, upload-time = "2025-07-27T21:24:42.333Z" }, + { url = "https://files.pythonhosted.org/packages/a8/fd/848d077bf6abc4ce84273d8e3f3a70d61a2240519a339462f699d8acf829/cramjam-2.11.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:3d1ba626dd5f81f7f09bbf59f70b534e2b75e0d6582b056b7bd31b397f1c13e9", size = 1702589, upload-time = "2025-07-27T21:24:44.305Z" }, + { url = "https://files.pythonhosted.org/packages/9d/1c/899818999bbdb59c601756b413e87d37fd65875d1315346c10e367bb3505/cramjam-2.11.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c71e140d5eb3145d61d59d0be0bf72f07cc4cf4b32cb136b09f712a3b1040f5f", size = 1773646, upload-time = "2025-07-27T21:24:46.495Z" }, + { url = "https://files.pythonhosted.org/packages/5f/26/c2813c5422c43b3dcd8b6645bc359f08870737c44325ee4accc18f24eee0/cramjam-2.11.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a6ed7926a5cca28edebad7d0fedd2ad492710ae3524d25fc59a2b20546d9ce1", size = 1994179, upload-time = "2025-07-27T21:24:49.131Z" }, + { url = "https://files.pythonhosted.org/packages/2e/4f/af984f8d7f963f0301812cdd620ddcfd8276461ed7a786c0f89e82b14739/cramjam-2.11.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5eb4ed3cea945b164b0513fd491884993acac2153a27b93a84019c522e8eda82", size = 1714790, upload-time = "2025-07-27T21:24:51.045Z" }, + { url = "https://files.pythonhosted.org/packages/81/da/b3301962ccd6fce9fefa1ecd8ea479edaeaa38fadb1f34d5391d2587216a/cramjam-2.11.0-pp311-pypy311_pp73-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:52d5db3369f95b27b9f3c14d067acb0b183333613363ed34268c9e04560f997f", size = 3573546, upload-time = "2025-07-27T21:24:52.944Z" }, + { url = "https://files.pythonhosted.org/packages/b6/c2/410ddb8ad4b9dfb129284666293cb6559479645da560f7077dc19d6bee9e/cramjam-2.11.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:4820516366d455b549a44d0e2210ee7c4575882dda677564ce79092588321d54", size = 1873654, upload-time = "2025-07-27T21:24:54.958Z" }, + { url = "https://files.pythonhosted.org/packages/d5/99/f68a443c64f7ce7aff5bed369b0aa5b2fac668fa3dfd441837e316e97a1f/cramjam-2.11.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d9e5db525dc0a950a825202f84ee68d89a072479e07da98795a3469df942d301", size = 1702846, upload-time = "2025-07-27T21:24:57.124Z" }, + { url = "https://files.pythonhosted.org/packages/6c/02/0ff358ab773def1ee3383587906c453d289953171e9c92db84fdd01bf172/cramjam-2.11.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62ab4971199b2270005359cdc379bc5736071dc7c9a228581c5122d9ffaac50c", size = 1773683, upload-time = "2025-07-27T21:24:59.28Z" }, + { url = "https://files.pythonhosted.org/packages/e9/31/3298e15f87c9cf2aabdbdd90b153d8644cf989cb42a45d68a1b71e1f7aaf/cramjam-2.11.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24758375cc5414d3035ca967ebb800e8f24604ececcba3c67d6f0218201ebf2d", size = 1994136, upload-time = "2025-07-27T21:25:01.565Z" }, + { url = "https://files.pythonhosted.org/packages/c7/90/20d1747255f1ee69a412e319da51ea594c18cca195e7a4d4c713f045eff5/cramjam-2.11.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:6c2eea545fef1065c7dd4eda991666fd9c783fbc1d226592ccca8d8891c02f23", size = 1714982, upload-time = "2025-07-27T21:25:05.79Z" }, +] + [[package]] name = "debugpy" version = "1.8.17" @@ -330,6 +440,53 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl", hash = "sha256:760643d3452b4d777d295bb167ccc74c64a81df23fb5e08eff250c425a4b2017", size = 28317, upload-time = "2025-09-01T09:48:08.5Z" }, ] +[[package]] +name = "fastavro" +version = "1.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/65/8b/fa2d3287fd2267be6261d0177c6809a7fa12c5600ddb33490c8dc29e77b2/fastavro-1.12.1.tar.gz", hash = "sha256:2f285be49e45bc047ab2f6bed040bb349da85db3f3c87880e4b92595ea093b2b", size = 1025661, upload-time = "2025-10-10T15:40:55.41Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/a0/077fd7cbfc143152cb96780cb592ed6cb6696667d8bc1b977745eb2255a8/fastavro-1.12.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:00650ca533907361edda22e6ffe8cf87ab2091c5d8aee5c8000b0f2dcdda7ed3", size = 1000335, upload-time = "2025-10-10T15:40:59.834Z" }, + { url = "https://files.pythonhosted.org/packages/a0/ae/a115e027f3a75df237609701b03ecba0b7f0aa3d77fe0161df533fde1eb7/fastavro-1.12.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ac76d6d95f909c72ee70d314b460b7e711d928845771531d823eb96a10952d26", size = 3221067, upload-time = "2025-10-10T15:41:04.399Z" }, + { url = "https://files.pythonhosted.org/packages/94/4e/c4991c3eec0175af9a8a0c161b88089cb7bf7fe353b3e3be1bc4cf9036b2/fastavro-1.12.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f55eef18c41d4476bd32a82ed5dd86aabc3f614e1b66bdb09ffa291612e1670", size = 3228979, upload-time = "2025-10-10T15:41:06.738Z" }, + { url = "https://files.pythonhosted.org/packages/21/0c/f2afb8eaea38799ccb1ed07d68bf2659f2e313f1902bbd36774cf6a1bef9/fastavro-1.12.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:81563e1f93570e6565487cdb01ba241a36a00e58cff9c5a0614af819d1155d8f", size = 3160740, upload-time = "2025-10-10T15:41:08.731Z" }, + { url = "https://files.pythonhosted.org/packages/0d/1a/f4d367924b40b86857862c1fa65f2afba94ddadf298b611e610a676a29e5/fastavro-1.12.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bec207360f76f0b3de540758a297193c5390e8e081c43c3317f610b1414d8c8f", size = 3235787, upload-time = "2025-10-10T15:41:10.869Z" }, + { url = "https://files.pythonhosted.org/packages/90/ec/8db9331896e3dfe4f71b2b3c23f2e97fbbfd90129777467ca9f8bafccb74/fastavro-1.12.1-cp310-cp310-win_amd64.whl", hash = "sha256:c0390bfe4a9f8056a75ac6785fbbff8f5e317f5356481d2e29ec980877d2314b", size = 449350, upload-time = "2025-10-10T15:41:12.104Z" }, + { url = "https://files.pythonhosted.org/packages/a0/e9/31c64b47cefc0951099e7c0c8c8ea1c931edd1350f34d55c27cbfbb08df1/fastavro-1.12.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6b632b713bc5d03928a87d811fa4a11d5f25cd43e79c161e291c7d3f7aa740fd", size = 1016585, upload-time = "2025-10-10T15:41:13.717Z" }, + { url = "https://files.pythonhosted.org/packages/10/76/111560775b548f5d8d828c1b5285ff90e2d2745643fb80ecbf115344eea4/fastavro-1.12.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eaa7ab3769beadcebb60f0539054c7755f63bd9cf7666e2c15e615ab605f89a8", size = 3404629, upload-time = "2025-10-10T15:41:15.642Z" }, + { url = "https://files.pythonhosted.org/packages/b0/07/6bb93cb963932146c2b6c5c765903a0a547ad9f0f8b769a4a9aad8c06369/fastavro-1.12.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:123fb221df3164abd93f2d042c82f538a1d5a43ce41375f12c91ce1355a9141e", size = 3428594, upload-time = "2025-10-10T15:41:17.779Z" }, + { url = "https://files.pythonhosted.org/packages/d1/67/8115ec36b584197ea737ec79e3499e1f1b640b288d6c6ee295edd13b80f6/fastavro-1.12.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:632a4e3ff223f834ddb746baae0cc7cee1068eb12c32e4d982c2fee8a5b483d0", size = 3344145, upload-time = "2025-10-10T15:41:19.89Z" }, + { url = "https://files.pythonhosted.org/packages/9e/9e/a7cebb3af967e62539539897c10138fa0821668ec92525d1be88a9cd3ee6/fastavro-1.12.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:83e6caf4e7a8717d932a3b1ff31595ad169289bbe1128a216be070d3a8391671", size = 3431942, upload-time = "2025-10-10T15:41:22.076Z" }, + { url = "https://files.pythonhosted.org/packages/c0/d1/7774ddfb8781c5224294c01a593ebce2ad3289b948061c9701bd1903264d/fastavro-1.12.1-cp311-cp311-win_amd64.whl", hash = "sha256:b91a0fe5a173679a6c02d53ca22dcaad0a2c726b74507e0c1c2e71a7c3f79ef9", size = 450542, upload-time = "2025-10-10T15:41:23.333Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f0/10bd1a3d08667fa0739e2b451fe90e06df575ec8b8ba5d3135c70555c9bd/fastavro-1.12.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:509818cb24b98a804fc80be9c5fed90f660310ae3d59382fc811bfa187122167", size = 1009057, upload-time = "2025-10-10T15:41:24.556Z" }, + { url = "https://files.pythonhosted.org/packages/78/ad/0d985bc99e1fa9e74c636658000ba38a5cd7f5ab2708e9c62eaf736ecf1a/fastavro-1.12.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:089e155c0c76e0d418d7e79144ce000524dd345eab3bc1e9c5ae69d500f71b14", size = 3391866, upload-time = "2025-10-10T15:41:26.882Z" }, + { url = "https://files.pythonhosted.org/packages/0d/9e/b4951dc84ebc34aac69afcbfbb22ea4a91080422ec2bfd2c06076ff1d419/fastavro-1.12.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44cbff7518901c91a82aab476fcab13d102e4999499df219d481b9e15f61af34", size = 3458005, upload-time = "2025-10-10T15:41:29.017Z" }, + { url = "https://files.pythonhosted.org/packages/af/f8/5a8df450a9f55ca8441f22ea0351d8c77809fc121498b6970daaaf667a21/fastavro-1.12.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a275e48df0b1701bb764b18a8a21900b24cf882263cb03d35ecdba636bbc830b", size = 3295258, upload-time = "2025-10-10T15:41:31.564Z" }, + { url = "https://files.pythonhosted.org/packages/99/b2/40f25299111d737e58b85696e91138a66c25b7334f5357e7ac2b0e8966f8/fastavro-1.12.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2de72d786eb38be6b16d556b27232b1bf1b2797ea09599507938cdb7a9fe3e7c", size = 3430328, upload-time = "2025-10-10T15:41:33.689Z" }, + { url = "https://files.pythonhosted.org/packages/e0/07/85157a7c57c5f8b95507d7829b5946561e5ee656ff80e9dd9a757f53ddaf/fastavro-1.12.1-cp312-cp312-win_amd64.whl", hash = "sha256:9090f0dee63fe022ee9cc5147483366cc4171c821644c22da020d6b48f576b4f", size = 444140, upload-time = "2025-10-10T15:41:34.902Z" }, + { url = "https://files.pythonhosted.org/packages/bb/57/26d5efef9182392d5ac9f253953c856ccb66e4c549fd3176a1e94efb05c9/fastavro-1.12.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:78df838351e4dff9edd10a1c41d1324131ffecbadefb9c297d612ef5363c049a", size = 1000599, upload-time = "2025-10-10T15:41:36.554Z" }, + { url = "https://files.pythonhosted.org/packages/33/cb/8ab55b21d018178eb126007a56bde14fd01c0afc11d20b5f2624fe01e698/fastavro-1.12.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:780476c23175d2ae457c52f45b9ffa9d504593499a36cd3c1929662bf5b7b14b", size = 3335933, upload-time = "2025-10-10T15:41:39.07Z" }, + { url = "https://files.pythonhosted.org/packages/fe/03/9c94ec9bf873eb1ffb0aa694f4e71940154e6e9728ddfdc46046d7e8ced4/fastavro-1.12.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0714b285160fcd515eb0455540f40dd6dac93bdeacdb03f24e8eac3d8aa51f8d", size = 3402066, upload-time = "2025-10-10T15:41:41.608Z" }, + { url = "https://files.pythonhosted.org/packages/75/c8/cb472347c5a584ccb8777a649ebb28278fccea39d005fc7df19996f41df8/fastavro-1.12.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a8bc2dcec5843d499f2489bfe0747999108f78c5b29295d877379f1972a3d41a", size = 3240038, upload-time = "2025-10-10T15:41:43.743Z" }, + { url = "https://files.pythonhosted.org/packages/e1/77/569ce9474c40304b3a09e109494e020462b83e405545b78069ddba5f614e/fastavro-1.12.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3b1921ac35f3d89090a5816b626cf46e67dbecf3f054131f84d56b4e70496f45", size = 3369398, upload-time = "2025-10-10T15:41:45.719Z" }, + { url = "https://files.pythonhosted.org/packages/4a/1f/9589e35e9ea68035385db7bdbf500d36b8891db474063fb1ccc8215ee37c/fastavro-1.12.1-cp313-cp313-win_amd64.whl", hash = "sha256:5aa777b8ee595b50aa084104cd70670bf25a7bbb9fd8bb5d07524b0785ee1699", size = 444220, upload-time = "2025-10-10T15:41:47.39Z" }, + { url = "https://files.pythonhosted.org/packages/6c/d2/78435fe737df94bd8db2234b2100f5453737cffd29adee2504a2b013de84/fastavro-1.12.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:c3d67c47f177e486640404a56f2f50b165fe892cc343ac3a34673b80cc7f1dd6", size = 1086611, upload-time = "2025-10-10T15:41:48.818Z" }, + { url = "https://files.pythonhosted.org/packages/b6/be/428f99b10157230ddac77ec8cc167005b29e2bd5cbe228345192bb645f30/fastavro-1.12.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5217f773492bac43dae15ff2931432bce2d7a80be7039685a78d3fab7df910bd", size = 3541001, upload-time = "2025-10-10T15:41:50.871Z" }, + { url = "https://files.pythonhosted.org/packages/16/08/a2eea4f20b85897740efe44887e1ac08f30dfa4bfc3de8962bdcbb21a5a1/fastavro-1.12.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:469fecb25cba07f2e1bfa4c8d008477cd6b5b34a59d48715e1b1a73f6160097d", size = 3432217, upload-time = "2025-10-10T15:41:53.149Z" }, + { url = "https://files.pythonhosted.org/packages/87/bb/b4c620b9eb6e9838c7f7e4b7be0762834443adf9daeb252a214e9ad3178c/fastavro-1.12.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d71c8aa841ef65cfab709a22bb887955f42934bced3ddb571e98fdbdade4c609", size = 3366742, upload-time = "2025-10-10T15:41:55.237Z" }, + { url = "https://files.pythonhosted.org/packages/3d/d1/e69534ccdd5368350646fea7d93be39e5f77c614cca825c990bd9ca58f67/fastavro-1.12.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:b81fc04e85dfccf7c028e0580c606e33aa8472370b767ef058aae2c674a90746", size = 3383743, upload-time = "2025-10-10T15:41:57.68Z" }, + { url = "https://files.pythonhosted.org/packages/58/54/b7b4a0c3fb5fcba38128542da1b26c4e6d69933c923f493548bdfd63ab6a/fastavro-1.12.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:9445da127751ba65975d8e4bdabf36bfcfdad70fc35b2d988e3950cce0ec0e7c", size = 1001377, upload-time = "2025-10-10T15:41:59.241Z" }, + { url = "https://files.pythonhosted.org/packages/1e/4f/0e589089c7df0d8f57d7e5293fdc34efec9a3b758a0d4d0c99a7937e2492/fastavro-1.12.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ed924233272719b5d5a6a0b4d80ef3345fc7e84fc7a382b6232192a9112d38a6", size = 3320401, upload-time = "2025-10-10T15:42:01.682Z" }, + { url = "https://files.pythonhosted.org/packages/f9/19/260110d56194ae29d7e423a336fccea8bcd103196d00f0b364b732bdb84e/fastavro-1.12.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3616e2f0e1c9265e92954fa099db79c6e7817356d3ff34f4bcc92699ae99697c", size = 3350894, upload-time = "2025-10-10T15:42:04.073Z" }, + { url = "https://files.pythonhosted.org/packages/d0/96/58b0411e8be9694d5972bee3167d6c1fd1fdfdf7ce253c1a19a327208f4f/fastavro-1.12.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:cb0337b42fd3c047fcf0e9b7597bd6ad25868de719f29da81eabb6343f08d399", size = 3229644, upload-time = "2025-10-10T15:42:06.221Z" }, + { url = "https://files.pythonhosted.org/packages/5b/db/38660660eac82c30471d9101f45b3acfdcbadfe42d8f7cdb129459a45050/fastavro-1.12.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:64961ab15b74b7c168717bbece5660e0f3d457837c3cc9d9145181d011199fa7", size = 3329704, upload-time = "2025-10-10T15:42:08.384Z" }, + { url = "https://files.pythonhosted.org/packages/9d/a9/1672910f458ecb30b596c9e59e41b7c00309b602a0494341451e92e62747/fastavro-1.12.1-cp314-cp314-win_amd64.whl", hash = "sha256:792356d320f6e757e89f7ac9c22f481e546c886454a6709247f43c0dd7058004", size = 452911, upload-time = "2025-10-10T15:42:09.795Z" }, + { url = "https://files.pythonhosted.org/packages/dc/8d/2e15d0938ded1891b33eff252e8500605508b799c2e57188a933f0bd744c/fastavro-1.12.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:120aaf82ac19d60a1016afe410935fe94728752d9c2d684e267e5b7f0e70f6d9", size = 3541999, upload-time = "2025-10-10T15:42:11.794Z" }, + { url = "https://files.pythonhosted.org/packages/a7/1c/6dfd082a205be4510543221b734b1191299e6a1810c452b6bc76dfa6968e/fastavro-1.12.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b6a3462934b20a74f9ece1daa49c2e4e749bd9a35fa2657b53bf62898fba80f5", size = 3433972, upload-time = "2025-10-10T15:42:14.485Z" }, + { url = "https://files.pythonhosted.org/packages/24/90/9de694625a1a4b727b1ad0958d220cab25a9b6cf7f16a5c7faa9ea7b2261/fastavro-1.12.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1f81011d54dd47b12437b51dd93a70a9aa17b61307abf26542fc3c13efbc6c51", size = 3368752, upload-time = "2025-10-10T15:42:16.618Z" }, + { url = "https://files.pythonhosted.org/packages/fa/93/b44f67589e4d439913dab6720f7e3507b0fa8b8e56d06f6fc875ced26afb/fastavro-1.12.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:43ded16b3f4a9f1a42f5970c2aa618acb23ea59c4fcaa06680bdf470b255e5a8", size = 3386636, upload-time = "2025-10-10T15:42:18.974Z" }, +] + [[package]] name = "ffmpeg-python" version = "0.2.0" @@ -360,6 +517,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, ] +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + [[package]] name = "ipykernel" version = "7.0.1" @@ -652,16 +818,32 @@ wheels = [ name = "nominal-streaming" source = { editable = "py-nominal-streaming" } dependencies = [ + { name = "polars" }, { name = "python-dateutil" }, { name = "typing-extensions" }, ] +[package.dev-dependencies] +dev = [ + { name = "cramjam" }, + { name = "fastavro" }, + { name = "pytest" }, +] + [package.metadata] requires-dist = [ + { name = "polars", specifier = ">=1,<2" }, { name = "python-dateutil", specifier = ">=0.0.0" }, { name = "typing-extensions", specifier = ">=4,<5" }, ] +[package.metadata.requires-dev] +dev = [ + { name = "cramjam", specifier = ">=2" }, + { name = "fastavro", specifier = ">=1" }, + { name = "pytest", specifier = ">=7" }, +] + [[package]] name = "numpy" version = "2.2.6" @@ -922,6 +1104,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/73/cb/ac7874b3e5d58441674fb70742e6c374b28b0c7cb988d37d991cde47166c/platformdirs-4.5.0-py3-none-any.whl", hash = "sha256:e578a81bb873cbb89a41fcc904c7ef523cc18284b7e3b3ccf06aca1403b7ebd3", size = 18651, upload-time = "2025-10-08T17:44:47.223Z" }, ] +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + [[package]] name = "polars" version = "1.34.0" @@ -1012,6 +1203,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, ] +[[package]] +name = "pytest" +version = "9.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165, upload-time = "2026-04-07T17:16:18.027Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0"