From 78db6ab15c0d2b94d388fbb535ebec98433b9de9 Mon Sep 17 00:00:00 2001 From: Krisztian Szucs Date: Mon, 4 May 2026 08:57:48 +0200 Subject: [PATCH 1/2] chore(deps): bump opendal to 0.56, migrate to reqsign-core 3.0 --- Cargo.lock | 563 ++++++++++++++++-- Cargo.toml | 2 +- crates/storage/opendal/Cargo.toml | 9 +- crates/storage/opendal/src/lib.rs | 6 +- crates/storage/opendal/src/resolving.rs | 49 +- crates/storage/opendal/src/s3.rs | 46 +- .../storage/opendal/tests/file_io_s3_test.rs | 29 +- .../opendal/tests/resolving_storage_test.rs | 25 +- 8 files changed, 592 insertions(+), 137 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c55c102409..350474c240 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -599,6 +599,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc" dependencies = [ "aws-lc-sys", + "untrusted 0.7.1", "zeroize", ] @@ -1086,7 +1087,7 @@ version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c" dependencies = [ - "darling 0.20.11", + "darling 0.23.0", "ident_case", "prettyplease", "proc-macro2", @@ -1324,7 +1325,17 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", +] + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", ] [[package]] @@ -1549,6 +1560,22 @@ dependencies = [ "memchr", ] +[[package]] +name = "ctor" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "424e0138278faeb2b401f174ad17e715c829512d74f3d1e81eb43365c2e0590e" +dependencies = [ + "ctor-proc-macro", + "dtor", +] + +[[package]] +name = "ctor-proc-macro" +version = "0.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52560adf09603e58c9a7ee1fe1dcb95a16927b17c127f0ac02d6e768a0e25bc1" + [[package]] name = "ctr" version = "0.9.2" @@ -2518,7 +2545,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2559,6 +2586,21 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" +[[package]] +name = "dtor" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "404d02eeb088a82cfd873006cb713fe411306c7d182c344905e101fb1167d301" +dependencies = [ + "dtor-proc-macro", +] + +[[package]] +name = "dtor-proc-macro" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f678cf4a922c215c63e0de95eb1ff08a958a81d47e485cf9da1e27bf6305cfa5" + [[package]] name = "dunce" version = "1.0.5" @@ -2671,7 +2713,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3282,7 +3324,6 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", - "webpki-roots 1.0.6", ] [[package]] @@ -3373,7 +3414,7 @@ dependencies = [ "pretty_assertions", "rand 0.9.4", "regex", - "reqwest", + "reqwest 0.12.28", "roaring", "serde", "serde_arrow", @@ -3452,7 +3493,7 @@ dependencies = [ "iceberg-catalog-sql", "iceberg-storage-opendal", "iceberg_test_utils", - "reqwest", + "reqwest 0.12.28", "rstest", "sqlx", "tempfile", @@ -3470,7 +3511,7 @@ dependencies = [ "iceberg_test_utils", "itertools 0.13.0", "mockito", - "reqwest", + "reqwest 0.12.28", "serde", "serde_derive", "serde_json", @@ -3614,8 +3655,9 @@ dependencies = [ "iceberg", "iceberg_test_utils", "opendal", - "reqsign", - "reqwest", + "reqsign-aws-v4", + "reqsign-core", + "reqwest 0.12.28", "serde", "tokio", "typetag", @@ -3869,11 +3911,13 @@ checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" dependencies = [ "jiff-static", "jiff-tzdb-platform", + "js-sys", "log", "portable-atomic", "portable-atomic-util", "serde_core", - "windows-sys 0.59.0", + "wasm-bindgen", + "windows-sys 0.61.2", ] [[package]] @@ -3902,6 +3946,55 @@ dependencies = [ "jiff-tzdb", ] +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror 2.0.18", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn", +] + [[package]] name = "jobserver" version = "0.1.34" @@ -3924,16 +4017,18 @@ dependencies = [ [[package]] name = "jsonwebtoken" -version = "9.3.1" +version = "10.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde" +checksum = "0529410abe238729a60b108898784df8984c87f6054c9c4fcacc47e4803c1ce1" dependencies = [ + "aws-lc-rs", "base64", + "getrandom 0.2.17", "js-sys", "pem", - "ring", "serde", "serde_json", + "signature", "simple_asn1", ] @@ -4176,6 +4271,15 @@ dependencies = [ "digest", ] +[[package]] +name = "mea" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6747f54621d156e1b47eb6b25f39a941b9fc347f98f67d25d8881ff99e8ed832" +dependencies = [ + "slab", +] + [[package]] name = "memchr" version = "2.8.0" @@ -4422,7 +4526,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4552,7 +4656,7 @@ dependencies = [ "percent-encoding", "quick-xml 0.39.2", "rand 0.10.1", - "reqwest", + "reqwest 0.12.28", "ring", "rustls-pki-types", "serde", @@ -4587,31 +4691,194 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "opendal" -version = "0.55.0" +version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d075ab8a203a6ab4bc1bce0a4b9fe486a72bf8b939037f4b78d95386384bc80a" +checksum = "97b31d3d8e99a85d83b73ec26647f5607b80578ed9375810b6e44ffa3590a236" +dependencies = [ + "ctor", + "opendal-core", + "opendal-layer-concurrent-limit", + "opendal-layer-logging", + "opendal-layer-retry", + "opendal-layer-timeout", + "opendal-service-azdls", + "opendal-service-fs", + "opendal-service-gcs", + "opendal-service-oss", + "opendal-service-s3", +] + +[[package]] +name = "opendal-core" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1849dd2687e173e776d3af5fce1ba3ae47b9dd37a09d1c4deba850ef45fe00ca" dependencies = [ "anyhow", - "backon", "base64", "bytes", - "crc32c", "futures", - "getrandom 0.2.17", "http 1.4.0", "http-body 1.0.1", "jiff", "log", "md-5", + "mea", "percent-encoding", "quick-xml 0.38.4", - "reqsign", - "reqwest", + "reqsign-core", + "reqwest 0.13.3", "serde", "serde_json", "tokio", "url", "uuid", + "web-time", +] + +[[package]] +name = "opendal-layer-concurrent-limit" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "048b1b29c503263bdd80a9afe46a68cd02ea9bd361185b1feab4b151078998e9" +dependencies = [ + "futures", + "http 1.4.0", + "mea", + "opendal-core", +] + +[[package]] +name = "opendal-layer-logging" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2645adc988b12eda106e2679ae529facfbbaa868ceb706f6f8125c6af15c47b" +dependencies = [ + "log", + "opendal-core", +] + +[[package]] +name = "opendal-layer-retry" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eac134ffa4ddda6131a640a84a5315996424b9416c85052f8c64c1a33b70ad4" +dependencies = [ + "backon", + "log", + "opendal-core", +] + +[[package]] +name = "opendal-layer-timeout" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "619586ab7480c2e3009f6d18eabab18957bc094778fd130bcc38924970a90f4c" +dependencies = [ + "opendal-core", + "tokio", +] + +[[package]] +name = "opendal-service-azdls" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f9884c2d8cf8ba2bb077d79c877dac5863ba3bab9e2c9c1e41a2e0491404772" +dependencies = [ + "bytes", + "http 1.4.0", + "log", + "opendal-core", + "opendal-service-azure-common", + "quick-xml 0.38.4", + "reqsign-azure-storage", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", + "serde_json", +] + +[[package]] +name = "opendal-service-azure-common" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffb0e45d6c8dcf66ce2da20e241bcb80e6e540e109a4ff20f318f6c9b4c54e0c" +dependencies = [ + "http 1.4.0", + "opendal-core", +] + +[[package]] +name = "opendal-service-fs" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf0be0417abeeb0053376d816b90fceb9ca98f20dfb54ebf1f2a282729f83663" +dependencies = [ + "bytes", + "log", + "opendal-core", + "serde", + "tokio", + "xattr", +] + +[[package]] +name = "opendal-service-gcs" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70a49477a10163431896d106136117f5670717f9c9e49cf6f710528800c6633a" +dependencies = [ + "async-trait", + "bytes", + "http 1.4.0", + "log", + "opendal-core", + "percent-encoding", + "quick-xml 0.38.4", + "reqsign-core", + "reqsign-file-read-tokio", + "reqsign-google", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "opendal-service-oss" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c8a917829ad06d21b639558532cb0101fe49b040d946d673a73018683fac05" +dependencies = [ + "bytes", + "http 1.4.0", + "log", + "opendal-core", + "quick-xml 0.38.4", + "reqsign-aliyun-oss", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", +] + +[[package]] +name = "opendal-service-s3" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dadddeb9bb50b0d30927dd914c298c4ddca47e4c1cfa7674d311f0cf9b051c8" +dependencies = [ + "base64", + "bytes", + "crc32c", + "http 1.4.0", + "log", + "md-5", + "opendal-core", + "quick-xml 0.38.4", + "reqsign-aws-v4", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", + "url", ] [[package]] @@ -5078,7 +5345,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ "heck", - "itertools 0.13.0", + "itertools 0.14.0", "log", "multimap", "petgraph", @@ -5097,7 +5364,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn", @@ -5148,16 +5415,6 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" -[[package]] -name = "quick-xml" -version = "0.37.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "quick-xml" version = "0.38.4" @@ -5204,6 +5461,7 @@ version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" dependencies = [ + "aws-lc-rs", "bytes", "getrandom 0.3.4", "lru-slab", @@ -5230,7 +5488,7 @@ dependencies = [ "once_cell", "socket2 0.6.3", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -5478,33 +5736,114 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cadadef317c2f20755a64d7fdc48f9e7178ee6b0e1f7fce33fa60f1d68a276e6" [[package]] -name = "reqsign" -version = "0.16.5" +name = "reqsign-aliyun-oss" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43451dbf3590a7590684c25fb8d12ecdcc90ed3ac123433e500447c7d77ed701" +checksum = "57ac2757f3140aa2e213b554148ae0b52733e624fc6723f0cc6bb3d440176c95" +dependencies = [ + "anyhow", + "form_urlencoded", + "http 1.4.0", + "log", + "percent-encoding", + "reqsign-core", + "rust-ini", + "serde", + "serde_json", +] + +[[package]] +name = "reqsign-aws-v4" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44eaca382e94505a49f1a4849658d153aebf79d9c1a58e5dd3b10361511e9f43" +dependencies = [ + "anyhow", + "bytes", + "form_urlencoded", + "http 1.4.0", + "log", + "percent-encoding", + "quick-xml 0.39.2", + "reqsign-core", + "rust-ini", + "serde", + "serde_json", + "serde_urlencoded", + "sha1", +] + +[[package]] +name = "reqsign-azure-storage" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a321980405d596bd34aaf95c4722a3de4128a67fd19e74a81a83aa3fdf082e6" dependencies = [ "anyhow", - "async-trait", "base64", - "chrono", + "bytes", "form_urlencoded", - "getrandom 0.2.17", + "http 1.4.0", + "jsonwebtoken", + "log", + "pem", + "percent-encoding", + "reqsign-core", + "rsa", + "serde", + "serde_json", + "sha1", +] + +[[package]] +name = "reqsign-core" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b10302cf0a7d7e7352ba211fc92c3c5bebf1286153e49cc5aa87348078a8e102" +dependencies = [ + "anyhow", + "base64", + "bytes", + "form_urlencoded", + "futures", "hex", "hmac", - "home", + "http 1.4.0", + "jiff", + "log", + "percent-encoding", + "sha1", + "sha2", + "windows-sys 0.61.2", +] + +[[package]] +name = "reqsign-file-read-tokio" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d89295b3d17abea31851cc8de55d843d89c52132c864963c38d41920613dc5" +dependencies = [ + "anyhow", + "reqsign-core", + "tokio", +] + +[[package]] +name = "reqsign-google" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35cc609b49c69e76ecaceb775a03f792d1ed3e7755ab3548d4534fd801e3242e" +dependencies = [ + "form_urlencoded", "http 1.4.0", "jsonwebtoken", "log", - "once_cell", "percent-encoding", - "quick-xml 0.37.5", - "rand 0.8.5", - "reqwest", + "reqsign-aws-v4", + "reqsign-core", "rsa", - "rust-ini", "serde", "serde_json", - "sha1", "sha2", "tokio", ] @@ -5547,9 +5886,46 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", + "wasm-streams 0.4.2", + "web-sys", +] + +[[package]] +name = "reqwest" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams 0.5.0", "web-sys", - "webpki-roots 1.0.6", ] [[package]] @@ -5562,7 +5938,7 @@ dependencies = [ "cfg-if", "getrandom 0.2.17", "libc", - "untrusted", + "untrusted 0.9.0", "windows-sys 0.52.0", ] @@ -5693,7 +6069,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5733,6 +6109,33 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" version = "0.103.13" @@ -5742,7 +6145,7 @@ dependencies = [ "aws-lc-rs", "ring", "rustls-pki-types", - "untrusted", + "untrusted 0.9.0", ] [[package]] @@ -6149,6 +6552,16 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" +[[package]] +name = "simd_cesu8" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" +dependencies = [ + "rustc_version", + "simdutf8", +] + [[package]] name = "simdutf8" version = "0.1.5" @@ -6656,7 +7069,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7215,6 +7628,12 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "untrusted" version = "0.9.0" @@ -7486,6 +7905,19 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmparser" version = "0.244.0" @@ -7518,6 +7950,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webpki-roots" version = "0.26.11" @@ -7552,7 +7993,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -7957,6 +8398,16 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + [[package]] name = "xmlparser" version = "0.13.6" diff --git a/Cargo.toml b/Cargo.toml index 7f612c44bf..793bb49d86 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -103,7 +103,7 @@ mockito = "1" motore-macros = "0.4.3" murmur3 = "0.5.2" once_cell = "1.20" -opendal = "0.55.0" +opendal = "0.56" ordered-float = "4" parquet = "58" pilota = "0.11.10" diff --git a/crates/storage/opendal/Cargo.toml b/crates/storage/opendal/Cargo.toml index 84f7e1147a..90cc551e35 100644 --- a/crates/storage/opendal/Cargo.toml +++ b/crates/storage/opendal/Cargo.toml @@ -35,7 +35,7 @@ opendal-fs = ["opendal/services-fs"] opendal-gcs = ["opendal/services-gcs"] opendal-memory = ["opendal/services-memory"] opendal-oss = ["opendal/services-oss"] -opendal-s3 = ["opendal/services-s3", "reqsign"] +opendal-s3 = ["opendal/services-s3", "reqsign-aws-v4", "reqsign-core"] [dependencies] anyhow = { workspace = true } @@ -44,8 +44,8 @@ iceberg = { workspace = true } opendal = { workspace = true } async-trait = { workspace = true } bytes = { workspace = true } -reqsign = { version = "0.16.3", optional = true, default-features = false } -reqwest = { workspace = true } +reqsign-aws-v4 = { version = "3.0.0", optional = true } +reqsign-core = { version = "3.0.0", optional = true } serde = { workspace = true } typetag = { workspace = true } url = { workspace = true } @@ -54,6 +54,7 @@ futures = { workspace = true } [dev-dependencies] async-trait = { workspace = true } iceberg_test_utils = { path = "../../test_utils", features = ["tests"] } -reqsign = { version = "0.16.3", default-features = false } +reqsign-aws-v4 = { version = "3.0.0" } +reqsign-core = { version = "3.0.0" } reqwest = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index a0336868e3..65deaa5f44 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -482,7 +482,11 @@ impl Storage for OpenDalStorage { } else { format!("{relative_path}/") }; - Ok(op.remove_all(&path).await.map_err(from_opendal_error)?) + Ok(op + .delete_with(&path) + .recursive(true) + .await + .map_err(from_opendal_error)?) } async fn delete_stream(&self, mut paths: BoxStream<'static, String>) -> Result<()> { diff --git a/crates/storage/opendal/src/resolving.rs b/crates/storage/opendal/src/resolving.rs index 64a16b18d2..4e75e32951 100644 --- a/crates/storage/opendal/src/resolving.rs +++ b/crates/storage/opendal/src/resolving.rs @@ -30,7 +30,6 @@ use iceberg::io::{ StorageFactory, }; use iceberg::{Error, ErrorKind, Result}; -use opendal::Scheme; use serde::{Deserialize, Serialize}; use url::Url; @@ -52,26 +51,24 @@ pub const SCHEME_ABFS: &str = "abfs"; pub const SCHEME_WASBS: &str = "wasbs"; pub const SCHEME_WASB: &str = "wasb"; -/// Parse a URL scheme string into an [`opendal::Scheme`]. -fn parse_scheme(scheme: &str) -> Result { +/// Parse a URL scheme string. +fn parse_scheme(scheme: &str) -> Result<&'static str> { match scheme { - SCHEME_MEMORY => Ok(Scheme::Memory), - SCHEME_FILE | "" => Ok(Scheme::Fs), - SCHEME_S3 | SCHEME_S3A | SCHEME_S3N => Ok(Scheme::S3), - SCHEME_GS | SCHEME_GCS => Ok(Scheme::Gcs), - SCHEME_OSS => Ok(Scheme::Oss), - SCHEME_ABFSS | SCHEME_ABFS | SCHEME_WASBS | SCHEME_WASB => Ok(Scheme::Azdls), - s => s.parse::().map_err(|e| { - Error::new( - ErrorKind::FeatureUnsupported, - format!("Unsupported storage scheme: {s}: {e}"), - ) - }), + SCHEME_MEMORY => Ok("memory"), + SCHEME_FILE | "" => Ok("file"), + SCHEME_S3 | SCHEME_S3A | SCHEME_S3N => Ok("s3"), + SCHEME_GS | SCHEME_GCS => Ok("gcs"), + SCHEME_OSS => Ok("oss"), + SCHEME_ABFSS | SCHEME_ABFS | SCHEME_WASBS | SCHEME_WASB => Ok("azdls"), + s => Err(Error::new( + ErrorKind::FeatureUnsupported, + format!("Unsupported storage scheme: {s}"), + )), } } -/// Extract the [`Scheme`] family from a path URL. -fn extract_scheme(path: &str) -> Result { +/// Extract the scheme from a path URL. +fn extract_scheme(path: &str) -> Result<&'static str> { let url = Url::parse(path).map_err(|e| { Error::new( ErrorKind::DataInvalid, @@ -83,13 +80,13 @@ fn extract_scheme(path: &str) -> Result { /// Build an [`OpenDalStorage`] variant for the given scheme and config properties. fn build_storage_for_scheme( - scheme: Scheme, + scheme: &'static str, props: &HashMap, #[cfg(feature = "opendal-s3")] customized_credential_load: &Option, ) -> Result { match scheme { #[cfg(feature = "opendal-s3")] - Scheme::S3 => { + "s3" => { let config = crate::s3::s3_config_parse(props.clone())?; Ok(OpenDalStorage::S3 { config: Arc::new(config), @@ -97,30 +94,30 @@ fn build_storage_for_scheme( }) } #[cfg(feature = "opendal-gcs")] - Scheme::Gcs => { + "gcs" => { let config = crate::gcs::gcs_config_parse(props.clone())?; Ok(OpenDalStorage::Gcs { config: Arc::new(config), }) } #[cfg(feature = "opendal-oss")] - Scheme::Oss => { + "oss" => { let config = crate::oss::oss_config_parse(props.clone())?; Ok(OpenDalStorage::Oss { config: Arc::new(config), }) } #[cfg(feature = "opendal-azdls")] - Scheme::Azdls => { + "azdls" => { let config = crate::azdls::azdls_config_parse(props.clone())?; Ok(OpenDalStorage::Azdls { config: Arc::new(config), }) } #[cfg(feature = "opendal-fs")] - Scheme::Fs => Ok(OpenDalStorage::LocalFs), + "file" => Ok(OpenDalStorage::LocalFs), #[cfg(feature = "opendal-memory")] - Scheme::Memory => Ok(OpenDalStorage::Memory(crate::memory::memory_config_build()?)), + "memory" => Ok(OpenDalStorage::Memory(crate::memory::memory_config_build()?)), unsupported => Err(Error::new( ErrorKind::FeatureUnsupported, format!("Unsupported storage scheme: {unsupported}"), @@ -201,7 +198,7 @@ pub struct OpenDalResolvingStorage { props: HashMap, /// Cache of scheme to storage mappings. #[serde(skip, default)] - storages: RwLock>>, + storages: RwLock>>, /// Custom AWS credential loader for S3 storage. #[cfg(feature = "opendal-s3")] #[serde(skip)] @@ -286,7 +283,7 @@ impl Storage for OpenDalResolvingStorage { async fn delete_stream(&self, mut paths: BoxStream<'static, String>) -> Result<()> { // Group paths by scheme so each resolved storage receives a batch, // avoiding repeated operator creation per path. - let mut grouped: HashMap> = HashMap::new(); + let mut grouped: HashMap<&'static str, Vec> = HashMap::new(); while let Some(path) = paths.next().await { let scheme = extract_scheme(&path)?; grouped.entry(scheme).or_default().push(path); diff --git a/crates/storage/opendal/src/s3.rs b/crates/storage/opendal/src/s3.rs index 2e21418606..75f7684ccf 100644 --- a/crates/storage/opendal/src/s3.rs +++ b/crates/storage/opendal/src/s3.rs @@ -18,7 +18,6 @@ use std::collections::HashMap; use std::sync::Arc; -use async_trait::async_trait; use iceberg::io::{ CLIENT_REGION, S3_ACCESS_KEY_ID, S3_ALLOW_ANONYMOUS, S3_ASSUME_ROLE_ARN, S3_ASSUME_ROLE_EXTERNAL_ID, S3_ASSUME_ROLE_SESSION_NAME, S3_DISABLE_CONFIG_LOAD, @@ -28,8 +27,11 @@ use iceberg::io::{ use iceberg::{Error, ErrorKind, Result}; use opendal::services::S3Config; use opendal::{Configurator, Operator}; -pub use reqsign::{AwsCredential, AwsCredentialLoad}; -use reqwest::Client; +/// AWS credentials: access key ID, secret access key, and optional session token. +pub use reqsign_aws_v4::Credential as AwsCredential; +/// Trait for types that can asynchronously supply [`AwsCredential`] to a [`CustomAwsCredentialLoader`]. +pub use reqsign_core::ProvideCredential; +use reqsign_core::{ProvideCredentialChain, ProvideCredentialDyn}; use url::Url; use crate::utils::{from_opendal_error, is_truthy}; @@ -143,20 +145,26 @@ pub(crate) fn s3_config_build( // Set bucket name. .bucket(bucket); - if let Some(customized_credential_load) = customized_credential_load { - builder = builder - .customized_credential_load(customized_credential_load.clone().into_opendal_loader()); + if let Some(loader) = customized_credential_load { + let chain = ProvideCredentialChain::new().push(Arc::clone(&loader.0)); + builder = builder.credential_provider_chain(chain); } Ok(Operator::new(builder).map_err(from_opendal_error)?.finish()) } /// Custom AWS credential loader. -/// This can be used to load credentials from a custom source, such as the AWS SDK. /// -/// This should be set as an extension on `FileIOBuilder`. -#[derive(Clone)] -pub struct CustomAwsCredentialLoader(Arc); +/// Wraps any [`ProvideCredential`] implementation for use with the S3 storage backend. +/// Use [`CustomAwsCredentialLoader::new`] to create one, then pass it to +/// [`OpenDalStorageFactory::S3`](crate::OpenDalStorageFactory). +pub struct CustomAwsCredentialLoader(Arc>); + +impl Clone for CustomAwsCredentialLoader { + fn clone(&self) -> Self { + Self(Arc::clone(&self.0)) + } +} impl std::fmt::Debug for CustomAwsCredentialLoader { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -166,21 +174,9 @@ impl std::fmt::Debug for CustomAwsCredentialLoader { } impl CustomAwsCredentialLoader { - /// Create a new custom AWS credential loader. - pub fn new(loader: Arc) -> Self { - Self(loader) - } - - /// Convert this loader into an opendal compatible loader for customized AWS credentials. - pub fn into_opendal_loader(self) -> Box { - Box::new(self) - } -} - -#[async_trait] -impl AwsCredentialLoad for CustomAwsCredentialLoader { - async fn load_credential(&self, client: Client) -> anyhow::Result> { - self.0.load_credential(client).await + /// Create a new custom AWS credential loader from any [`ProvideCredential`] implementation. + pub fn new(provider: impl ProvideCredential + 'static) -> Self { + Self(Arc::new(provider) as Arc>) } } diff --git a/crates/storage/opendal/tests/file_io_s3_test.rs b/crates/storage/opendal/tests/file_io_s3_test.rs index d6dd8a3b45..e11c8aa7af 100644 --- a/crates/storage/opendal/tests/file_io_s3_test.rs +++ b/crates/storage/opendal/tests/file_io_s3_test.rs @@ -23,16 +23,16 @@ mod tests { use std::sync::Arc; - use async_trait::async_trait; use futures::StreamExt; use iceberg::io::{ FileIO, FileIOBuilder, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_PATH_STYLE_ACCESS, S3_REGION, S3_SECRET_ACCESS_KEY, }; - use iceberg_storage_opendal::{CustomAwsCredentialLoader, OpenDalStorageFactory}; + use iceberg_storage_opendal::{ + AwsCredential, CustomAwsCredentialLoader, OpenDalStorageFactory, ProvideCredential, + }; use iceberg_test_utils::{get_minio_endpoint, normalize_test_name_with_parts, set_up}; - use reqsign::{AwsCredential, AwsCredentialLoad}; - use reqwest::Client; + use reqsign_core::Context; async fn get_file_io() -> FileIO { set_up(); @@ -99,6 +99,7 @@ mod tests { } // Mock credential loader for testing + #[derive(Debug)] struct MockCredentialLoader { credential: Option, } @@ -118,9 +119,13 @@ mod tests { } } - #[async_trait] - impl AwsCredentialLoad for MockCredentialLoader { - async fn load_credential(&self, _client: Client) -> anyhow::Result> { + impl ProvideCredential for MockCredentialLoader { + type Credential = AwsCredential; + + async fn provide_credential( + &self, + _ctx: &Context, + ) -> reqsign_core::Result> { Ok(self.credential.clone()) } } @@ -129,7 +134,7 @@ mod tests { fn test_custom_aws_credential_loader_instantiation() { // Test creating CustomAwsCredentialLoader with mock loader let mock_loader = MockCredentialLoader::new_minio(); - let custom_loader = CustomAwsCredentialLoader::new(Arc::new(mock_loader)); + let custom_loader = CustomAwsCredentialLoader::new(mock_loader); // Test that the loader can be used in FileIOBuilder with OpenDalStorageFactory let _builder = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 { @@ -149,7 +154,7 @@ mod tests { // Create a mock credential loader let mock_loader = MockCredentialLoader::new_minio(); - let custom_loader = CustomAwsCredentialLoader::new(Arc::new(mock_loader)); + let custom_loader = CustomAwsCredentialLoader::new(mock_loader); let minio_endpoint = get_minio_endpoint(); @@ -177,7 +182,7 @@ mod tests { // Create a mock credential loader with no credentials let mock_loader = MockCredentialLoader::new(None); - let custom_loader = CustomAwsCredentialLoader::new(Arc::new(mock_loader)); + let custom_loader = CustomAwsCredentialLoader::new(mock_loader); let minio_endpoint = get_minio_endpoint(); @@ -199,8 +204,8 @@ mod tests { ), Err(e) => { assert!( - e.to_string() - .contains("no valid credential found and anonymous access is not allowed") + e.to_string().contains("credential"), + "unexpected error: {e}" ); } } diff --git a/crates/storage/opendal/tests/resolving_storage_test.rs b/crates/storage/opendal/tests/resolving_storage_test.rs index c235089508..1853a796dd 100644 --- a/crates/storage/opendal/tests/resolving_storage_test.rs +++ b/crates/storage/opendal/tests/resolving_storage_test.rs @@ -257,19 +257,21 @@ mod tests { #[cfg(feature = "opendal-s3")] #[tokio::test] async fn test_with_custom_credential_loader() { - use async_trait::async_trait; - use iceberg_storage_opendal::CustomAwsCredentialLoader; - use reqsign::{AwsCredential, AwsCredentialLoad}; - use reqwest::Client; + use iceberg_storage_opendal::{ + AwsCredential, CustomAwsCredentialLoader, ProvideCredential, + }; + use reqsign_core::Context; + #[derive(Debug)] struct MinioCredentialLoader; - #[async_trait] - impl AwsCredentialLoad for MinioCredentialLoader { - async fn load_credential( + impl ProvideCredential for MinioCredentialLoader { + type Credential = AwsCredential; + + async fn provide_credential( &self, - _client: Client, - ) -> anyhow::Result> { + _ctx: &Context, + ) -> reqsign_core::Result> { Ok(Some(AwsCredential { access_key_id: "admin".to_string(), secret_access_key: "password".to_string(), @@ -282,9 +284,8 @@ mod tests { set_up(); let minio_endpoint = get_minio_endpoint(); - let factory = OpenDalResolvingStorageFactory::new().with_s3_credential_loader( - CustomAwsCredentialLoader::new(Arc::new(MinioCredentialLoader)), - ); + let factory = OpenDalResolvingStorageFactory::new() + .with_s3_credential_loader(CustomAwsCredentialLoader::new(MinioCredentialLoader)); let file_io = FileIOBuilder::new(Arc::new(factory)) .with_props(vec![ From 0b76c1241cfc4b4ea645d66bf1cf9f9d5472ec9e Mon Sep 17 00:00:00 2001 From: Krisztian Szucs Date: Tue, 5 May 2026 15:03:54 +0200 Subject: [PATCH 2/2] chore(deps): fix reqsign dev-deps and tighten credential error assertion --- crates/storage/opendal/Cargo.toml | 2 -- crates/storage/opendal/tests/file_io_s3_test.rs | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/storage/opendal/Cargo.toml b/crates/storage/opendal/Cargo.toml index 90cc551e35..80eeaa3d04 100644 --- a/crates/storage/opendal/Cargo.toml +++ b/crates/storage/opendal/Cargo.toml @@ -54,7 +54,5 @@ futures = { workspace = true } [dev-dependencies] async-trait = { workspace = true } iceberg_test_utils = { path = "../../test_utils", features = ["tests"] } -reqsign-aws-v4 = { version = "3.0.0" } -reqsign-core = { version = "3.0.0" } reqwest = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/crates/storage/opendal/tests/file_io_s3_test.rs b/crates/storage/opendal/tests/file_io_s3_test.rs index e11c8aa7af..d5858e18f4 100644 --- a/crates/storage/opendal/tests/file_io_s3_test.rs +++ b/crates/storage/opendal/tests/file_io_s3_test.rs @@ -204,7 +204,7 @@ mod tests { ), Err(e) => { assert!( - e.to_string().contains("credential"), + e.to_string().contains("failed to load signing credential"), "unexpected error: {e}" ); }