diff --git a/Cargo.lock b/Cargo.lock index d64dd85..ee1f040 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -190,9 +190,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "53.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4caf25cdc4a985f91df42ed9e9308e1adbcd341a31a72605c697033fcef163e3" +checksum = "c91839b07e474b3995035fd8ac33ee54f9c9ccbbb1ea33d9909c71bffdf1259d" dependencies = [ "arrow-arith", "arrow-array", @@ -211,9 +211,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "53.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91f2dfd1a7ec0aca967dfaa616096aec49779adc8eccec005e2f5e4111b1192a" +checksum = "855c57c4efd26722b044dcd3e348252560e3e0333087fb9f6479dc0bf744054f" dependencies = [ "arrow-array", "arrow-buffer", @@ -226,9 +226,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "53.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d39387ca628be747394890a6e47f138ceac1aa912eab64f02519fed24b637af8" +checksum = "bd03279cea46569acf9295f6224fbc370c5df184b4d2ecfe97ccb131d5615a7f" dependencies = [ "ahash 0.8.11", "arrow-buffer", @@ -237,15 +237,15 @@ dependencies = [ "chrono", "chrono-tz", "half", - "hashbrown 0.14.5", + "hashbrown 0.15.2", "num", ] [[package]] name = "arrow-buffer" -version = "53.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e51e05228852ffe3eb391ce7178a0f97d2cf80cc6ef91d3c4a6b3cb688049ec" +checksum = "9e4a9b9b1d6d7117f6138e13bc4dd5daa7f94e671b70e8c9c4dc37b4f5ecfc16" dependencies = [ "bytes", "half", @@ -254,9 +254,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "53.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d09aea56ec9fa267f3f3f6cdab67d8a9974cbba90b3aa38c8fe9d0bb071bd8c1" +checksum = "bc70e39916e60c5b7af7a8e2719e3ae589326039e1e863675a008bee5ffe90fd" dependencies = [ "arrow-array", "arrow-buffer", @@ -275,9 +275,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "53.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c07b5232be87d115fde73e32f2ca7f1b353bff1b44ac422d3c6fc6ae38f11f0d" +checksum = "789b2af43c1049b03a8d088ff6b2257cdcea1756cd76b174b1f2600356771b97" dependencies = [ "arrow-array", "arrow-buffer", @@ -294,9 +294,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "53.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b98ae0af50890b494cebd7d6b04b35e896205c1d1df7b29a6272c5d0d0249ef5" +checksum = "e4e75edf21ffd53744a9b8e3ed11101f610e7ceb1a29860432824f1834a1f623" dependencies = [ "arrow-buffer", "arrow-schema", @@ -306,9 +306,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "53.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ed91bdeaff5a1c00d28d8f73466bcb64d32bbd7093b5a30156b4b9f4dba3eee" +checksum = "d186a909dece9160bf8312f5124d797884f608ef5435a36d9d608e0b2a9bcbf8" dependencies = [ "arrow-array", "arrow-buffer", @@ -321,9 +321,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "53.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0471f51260a5309307e5d409c9dc70aede1cd9cf1d4ff0f0a1e8e1a2dd0e0d3c" +checksum = "b66ff2fedc1222942d0bd2fd391cb14a85baa3857be95c9373179bd616753b85" dependencies = [ "arrow-array", "arrow-buffer", @@ -341,9 +341,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "53.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2883d7035e0b600fb4c30ce1e50e66e53d8656aa729f2bfa4b51d359cf3ded52" +checksum = "ece7b5bc1180e6d82d1a60e1688c199829e8842e38497563c3ab6ea813e527fd" dependencies = [ "arrow-array", "arrow-buffer", @@ -356,9 +356,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "53.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "552907e8e587a6fde4f8843fd7a27a576a260f65dab6c065741ea79f633fc5be" +checksum = "745c114c8f0e8ce211c83389270de6fbe96a9088a7b32c2a041258a443fe83ff" dependencies = [ "ahash 0.8.11", "arrow-array", @@ -370,15 +370,15 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "53.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "539ada65246b949bd99ffa0881a9a15a4a529448af1a07a9838dd78617dafab1" +checksum = "b95513080e728e4cec37f1ff5af4f12c9688d47795d17cda80b6ec2cf74d4678" [[package]] name = "arrow-select" -version = "53.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6259e566b752da6dceab91766ed8b2e67bf6270eb9ad8a6e07a33c1bede2b125" +checksum = "8e415279094ea70323c032c6e739c48ad8d80e78a09bef7117b8718ad5bf3722" dependencies = [ "ahash 0.8.11", "arrow-array", @@ -390,9 +390,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "53.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3179ccbd18ebf04277a095ba7321b93fd1f774f18816bd5f6b3ce2f594edb6c" +checksum = "11d956cae7002eb8d83a27dbd34daaea1cf5b75852f0b84deb4d93a276e92bbf" dependencies = [ "arrow-array", "arrow-buffer", @@ -407,9 +407,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.17" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cb8f1d480b0ea3783ab015936d2a55c87e219676f0c0b7dec61494043f21857" +checksum = "df895a515f70646414f4b45c0b79082783b80552b373a68283012928df56f522" dependencies = [ "bzip2", "flate2", @@ -431,7 +431,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -533,9 +533,9 @@ dependencies = [ [[package]] name = "aws-sdk-glue" -version = "1.69.0" +version = "1.70.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a741d4cf8386760213097345aa262ed56154a70733254e138def62b42a602e1" +checksum = "2813f4af56e70572b8d4e3fdd1a5a90562432dbbeaf0faab51780339e18b88e1" dependencies = [ "aws-credential-types", "aws-runtime", @@ -907,7 +907,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -961,9 +961,9 @@ dependencies = [ [[package]] name = "bytemuck" -version = "1.19.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8334215b81e418a0a7bdb8ef0849474f40bb10c8b71f1c4ed315cff49f32494d" +checksum = "8b37c88a63ffd85d15b406896cc343916d7cf57838a847b3a6f2ca5d39a5695a" [[package]] name = "byteorder" @@ -1102,7 +1102,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -1188,6 +1188,16 @@ dependencies = [ "libc", ] +[[package]] +name = "core-foundation" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -1205,9 +1215,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca741a962e1b0bff6d724a1a0958b686406e853bb14061f218562e1896f95e6" +checksum = "16b80225097f2e5ae4e7179dd2266824648f3e2f49d9134d584b76389d31c4c3" dependencies = [ "libc", ] @@ -1333,7 +1343,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -1355,7 +1365,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core 0.20.10", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -1953,7 +1963,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -2146,6 +2156,7 @@ dependencies = [ "datafusion", "datafusion-cli", "datafusion_iceberg", + "iceberg-file-catalog", "iceberg-rest-catalog", "iceberg-rust", "object_store", @@ -2265,7 +2276,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -2354,9 +2365,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" +checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e" dependencies = [ "atomic-waker", "bytes", @@ -2403,9 +2414,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.1" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" [[package]] name = "hashlink" @@ -2567,14 +2578,14 @@ dependencies = [ [[package]] name = "hyper" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbbff0a806a4728c99295b254c8838933b5b082d75e3cb70c8dab21fdfbcfa9a" +checksum = "97818827ef4f364230e16705d4706e2897df2bb60617d6ca15d598025a3c481f" dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.6", + "h2 0.4.7", "http 1.1.0", "http-body 1.0.1", "httparse", @@ -2609,10 +2620,10 @@ checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ "futures-util", "http 1.1.0", - "hyper 1.5.0", + "hyper 1.5.1", "hyper-util", - "rustls 0.23.17", - "rustls-native-certs 0.8.0", + "rustls 0.23.18", + "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", "tokio-rustls 0.26.0", @@ -2643,7 +2654,7 @@ dependencies = [ "futures-util", "http 1.1.0", "http-body 1.0.1", - "hyper 1.5.0", + "hyper 1.5.1", "pin-project-lite", "socket2", "tokio", @@ -2674,6 +2685,21 @@ dependencies = [ "cc", ] +[[package]] +name = "iceberg-file-catalog" +version = "0.5.8" +source = "git+https://github.com/jankaul/iceberg-rust#47c424fdb5ccd3dc3a16f06057db02a1547f6f1d" +dependencies = [ + "async-trait", + "futures", + "iceberg-rust", + "object_store", + "serde_json", + "thiserror 1.0.69", + "url", + "uuid", +] + [[package]] name = "iceberg-glue-catalog" version = "0.5.8" @@ -2896,7 +2922,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -2944,7 +2970,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.15.1", + "hashbrown 0.15.2", "serde", ] @@ -2998,9 +3024,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.11" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" [[package]] name = "jobserver" @@ -3095,9 +3121,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.164" +version = "0.2.165" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "433bfe06b8c75da9b2e3fbea6e5329ff87748f0b144ef75306e674c3f6f7c13f" +checksum = "fcb4d3d38eab6c5239a362fa8bae48c03baf980a6e7079f063942d563ef3533e" [[package]] name = "libflate" @@ -3168,9 +3194,9 @@ checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "litemap" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" +checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" [[package]] name = "lock_api" @@ -3294,7 +3320,7 @@ dependencies = [ "openssl-probe", "openssl-sys", "schannel", - "security-framework", + "security-framework 2.11.1", "security-framework-sys", "tempfile", ] @@ -3458,7 +3484,7 @@ dependencies = [ "chrono", "futures", "humantime", - "hyper 1.5.0", + "hyper 1.5.1", "itertools 0.13.0", "md-5", "parking_lot", @@ -3506,7 +3532,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -3600,9 +3626,9 @@ dependencies = [ [[package]] name = "parquet" -version = "53.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dea02606ba6f5e856561d8d507dba8bac060aefca2a6c0f1aa1d361fed91ff3e" +checksum = "2b449890367085eb65d7d3321540abc3d7babbd179ce31df0016e90719114191" dependencies = [ "ahash 0.8.11", "arrow-array", @@ -3619,7 +3645,7 @@ dependencies = [ "flate2", "futures", "half", - "hashbrown 0.14.5", + "hashbrown 0.15.2", "lz4_flex", "num", "num-bigint", @@ -3729,7 +3755,7 @@ checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -3797,9 +3823,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.89" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" dependencies = [ "unicode-ident", ] @@ -3851,7 +3877,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.23.17", + "rustls 0.23.18", "socket2", "thiserror 2.0.3", "tokio", @@ -3869,7 +3895,7 @@ dependencies = [ "rand", "ring", "rustc-hash", - "rustls 0.23.17", + "rustls 0.23.18", "rustls-pki-types", "slab", "thiserror 2.0.3", @@ -4064,11 +4090,11 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "h2 0.4.6", + "h2 0.4.7", "http 1.1.0", "http-body 1.0.1", "http-body-util", - "hyper 1.5.0", + "hyper 1.5.1", "hyper-rustls 0.27.3", "hyper-util", "ipnet", @@ -4079,14 +4105,14 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.17", - "rustls-native-certs 0.8.0", + "rustls 0.23.18", + "rustls-native-certs 0.8.1", "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", - "sync_wrapper 1.0.1", + "sync_wrapper 1.0.2", "tokio", "tokio-rustls 0.26.0", "tokio-util", @@ -4166,9 +4192,9 @@ checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" [[package]] name = "rsa" -version = "0.9.6" +version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d0e5124fcb30e76a7e79bfee683a2746db83784b86289f6251b54b7950a0dfc" +checksum = "47c75d7c5c6b673e58bf54d8544a9f432e3a925b0e80f7cd3602ab5c50c55519" dependencies = [ "const-oid", "digest", @@ -4223,9 +4249,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.40" +version = "0.38.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99e4ea3e1cdc4b559b8e5650f9c8e5998e3e5c1343b4eaf034565f32318d63c0" +checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6" dependencies = [ "bitflags 2.6.0", "errno", @@ -4248,9 +4274,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.17" +version = "0.23.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f1a745511c54ba6d4465e8d5dfbd81b45791756de28d4981af70d6dca128f1e" +checksum = "9c9cc1d47e243d655ace55ed38201c19ae02c148ae56412ab8750e8f0166ab7f" dependencies = [ "once_cell", "ring", @@ -4269,20 +4295,19 @@ dependencies = [ "openssl-probe", "rustls-pemfile 1.0.4", "schannel", - "security-framework", + "security-framework 2.11.1", ] [[package]] name = "rustls-native-certs" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcaf18a4f2be7326cd874a5fa579fae794320a0f388d365dca7e480e55f83f8a" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" dependencies = [ "openssl-probe", - "rustls-pemfile 2.2.0", "rustls-pki-types", "schannel", - "security-framework", + "security-framework 3.0.1", ] [[package]] @@ -4378,9 +4403,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01227be5826fa0690321a2ba6c5cd57a19cf3f6a09e76973b58e61de6ab9d1c1" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" dependencies = [ "windows-sys 0.59.0", ] @@ -4414,7 +4439,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ "bitflags 2.6.0", - "core-foundation", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1415a607e92bec364ea2cf9264646dcce0f91e6d65281bd6f2819cca3bf39c8" +dependencies = [ + "bitflags 2.6.0", + "core-foundation 0.10.0", "core-foundation-sys", "libc", "security-framework-sys", @@ -4468,7 +4506,7 @@ checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -4491,7 +4529,7 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -4533,7 +4571,7 @@ dependencies = [ "darling 0.20.10", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -4631,7 +4669,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -4697,7 +4735,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -4740,7 +4778,7 @@ dependencies = [ "once_cell", "paste", "percent-encoding", - "rustls 0.23.17", + "rustls 0.23.18", "rustls-pemfile 2.2.0", "serde", "serde_json", @@ -4765,7 +4803,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -4788,7 +4826,7 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn 2.0.87", + "syn 2.0.89", "tempfile", "tokio", "url", @@ -4957,7 +4995,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -4970,7 +5008,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -4992,9 +5030,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.87" +version = "2.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" +checksum = "44d46482f1c1c87acd84dea20c1bf5ebff4c757009ed6bf19cfd36fb10e92c4e" dependencies = [ "proc-macro2", "quote", @@ -5009,9 +5047,9 @@ checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" [[package]] name = "sync_wrapper" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" dependencies = [ "futures-core", ] @@ -5024,7 +5062,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -5034,7 +5072,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" dependencies = [ "bitflags 1.3.2", - "core-foundation", + "core-foundation 0.9.4", "system-configuration-sys", ] @@ -5102,7 +5140,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -5113,7 +5151,7 @@ checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -5218,7 +5256,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -5247,7 +5285,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.23.17", + "rustls 0.23.18", "rustls-pki-types", "tokio", ] @@ -5319,14 +5357,14 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] name = "tracing-core" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" dependencies = [ "once_cell", ] @@ -5374,7 +5412,7 @@ checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -5397,9 +5435,9 @@ checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" [[package]] name = "unicode-ident" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" [[package]] name = "unicode-normalization" @@ -5448,9 +5486,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.3" +version = "2.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" dependencies = [ "form_urlencoded", "idna", @@ -5563,7 +5601,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", "wasm-bindgen-shared", ] @@ -5597,7 +5635,7 @@ checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -5643,9 +5681,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.26.6" +version = "0.26.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958" +checksum = "5d642ff16b7e79272ae451b7322067cdc17cadf68c23264be9d94a32319efe7e" dependencies = [ "rustls-pki-types", ] @@ -5913,9 +5951,9 @@ dependencies = [ [[package]] name = "yoke" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" dependencies = [ "serde", "stable_deref_trait", @@ -5925,13 +5963,13 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", "synstructure", ] @@ -5953,27 +5991,27 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] name = "zerofrom" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", "synstructure", ] @@ -6002,7 +6040,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] diff --git a/frostbow-glue/src/main.rs b/frostbow-glue/src/main.rs index f6da861..436450d 100644 --- a/frostbow-glue/src/main.rs +++ b/frostbow-glue/src/main.rs @@ -5,6 +5,7 @@ use clap::Parser; use datafusion::{ catalog_common::MemoryCatalogProviderList, execution::{context::SessionContext, SessionStateBuilder}, + prelude::SessionConfig, }; use datafusion_cli::{ exec, @@ -14,10 +15,8 @@ use datafusion_cli::{ use datafusion_iceberg::{ catalog::catalog::IcebergCatalog, error::Error, planner::IcebergQueryPlanner, }; -use frostbow::{Args, IcebergContext}; +use frostbow::{get_storage, Args, IcebergContext}; use iceberg_glue_catalog::GlueCatalog; -use iceberg_rust::catalog::bucket::ObjectStoreBuilder; -use object_store::{aws::AmazonS3Builder, local::LocalFileSystem}; #[tokio::main] async fn main() -> ExitCode { @@ -31,21 +30,10 @@ async fn main() -> ExitCode { async fn main_inner() -> Result<(), Error> { let args = Args::parse(); - let bucket = args.bucket; + let storage = args.storage; let command = args.command; - let object_store: ObjectStoreBuilder = match &bucket { - Some(bucket) => { - if bucket.starts_with("s3://") { - let builder = AmazonS3Builder::from_env().with_bucket_name(bucket); - - ObjectStoreBuilder::S3(builder) - } else { - ObjectStoreBuilder::Filesystem(Arc::new(LocalFileSystem::new())) - } - } - _ => ObjectStoreBuilder::memory(), - }; + let object_store = get_storage(storage.as_deref())?; let config = aws_config::load_defaults(BehaviorVersion::v2024_03_28()).await; @@ -64,6 +52,7 @@ async fn main_inner() -> Result<(), Error> { let state = SessionStateBuilder::new() .with_default_features() + .with_config(SessionConfig::default().with_information_schema(true)) .with_catalog_list(iceberg_catalog_list) .with_query_planner(Arc::new(IcebergQueryPlanner {})) .build(); diff --git a/frostbow-sql/src/main.rs b/frostbow-sql/src/main.rs index 09116d8..c1571f9 100644 --- a/frostbow-sql/src/main.rs +++ b/frostbow-sql/src/main.rs @@ -4,6 +4,7 @@ use clap::Parser; use datafusion::{ execution::{context::SessionContext, SessionStateBuilder}, logical_expr::ScalarUDF, + prelude::SessionConfig, }; use datafusion_cli::{ exec, @@ -15,9 +16,7 @@ use datafusion_iceberg::{ error::Error, planner::{IcebergQueryPlanner, RefreshMaterializedView}, }; -use frostbow::{Args, IcebergContext}; -use iceberg_rust::catalog::bucket::ObjectStoreBuilder; -use object_store::{aws::AmazonS3Builder, local::LocalFileSystem, memory::InMemory}; +use frostbow::{get_storage, Args, IcebergContext}; use iceberg_sql_catalog::SqlCatalogList; @@ -38,21 +37,10 @@ async fn main_inner() -> Result<(), Error> { "ICEBERG_CATALOG_URL".to_string(), ))?; - let bucket = args.bucket; + let storage = args.storage; let command = args.command; - let object_store = match &bucket { - Some(bucket) => { - if bucket.starts_with("s3://") { - let builder = AmazonS3Builder::from_env().with_bucket_name(bucket); - - ObjectStoreBuilder::S3(builder) - } else { - ObjectStoreBuilder::Filesystem(Arc::new(LocalFileSystem::new())) - } - } - _ => ObjectStoreBuilder::Memory(Arc::new(InMemory::new())), - }; + let object_store = get_storage(storage.as_deref())?; let iceberg_catalog_list = { Arc::new( @@ -66,6 +54,7 @@ async fn main_inner() -> Result<(), Error> { let state = SessionStateBuilder::new() .with_default_features() + .with_config(SessionConfig::default().with_information_schema(true)) .with_catalog_list(catalog_list) .with_query_planner(Arc::new(IcebergQueryPlanner {})) .build(); diff --git a/frostbow/Cargo.toml b/frostbow/Cargo.toml index 3f77afe..a6c08b8 100644 --- a/frostbow/Cargo.toml +++ b/frostbow/Cargo.toml @@ -19,6 +19,7 @@ async-trait = { workspace = true } reqwest = "0.11" reqwest-middleware = "0.2.0" iceberg-rest-catalog = { git = "https://github.com/jankaul/iceberg-rust", version = "0.5" , optional = true } +iceberg-file-catalog = { git = "https://github.com/jankaul/iceberg-rust", version = "0.5" } url = "2.5" openssl.workspace = true diff --git a/frostbow/src/lib.rs b/frostbow/src/lib.rs index 1ce524d..a4a4a06 100644 --- a/frostbow/src/lib.rs +++ b/frostbow/src/lib.rs @@ -1,5 +1,6 @@ use async_trait::async_trait; use clap::Parser; +use iceberg_rust::{catalog::bucket::ObjectStoreBuilder, error::Error}; use std::sync::Arc; use datafusion::{ @@ -17,19 +18,19 @@ use datafusion_cli::{ object_storage::{AwsOptions, GcpOptions}, }; use datafusion_iceberg::planner::iceberg_transform; -use object_store::ObjectStore; +use object_store::{memory::InMemory, ObjectStore}; #[derive(Debug, Parser)] #[clap(version, about)] pub struct Args { - #[clap(short = 'u', long = "catalog-url")] + #[clap(short = 'u', long = "catalog-url", help = "The url of the catalog.")] pub catalog_url: Option, - #[clap(short = 'b', long)] - pub bucket: Option, - #[clap(short = 'U', long)] - pub username: Option, - #[clap(short = 'W', long)] - pub password: Option, + #[clap( + short = 's', + long, + help = "The storage backend to use. Can be 'aws', 'gcs'. Defaults to 'memory' if not set." + )] + pub storage: Option, #[clap(short = 'c', long)] pub command: Vec, } @@ -78,3 +79,15 @@ impl CliSessionContext for IcebergContext { self.0.execute_logical_plan(plan).await } } + +pub fn get_storage(storage: Option<&str>) -> Result { + match storage { + Some("aws") => Ok(ObjectStoreBuilder::aws()), + Some("gcs") => Ok(ObjectStoreBuilder::gcs()), + None => Ok(ObjectStoreBuilder::Memory(Arc::new(InMemory::new()))), + Some(x) => Err(Error::InvalidFormat(format!( + "Storage {} is not supported.", + x + ))), + } +} diff --git a/frostbow/src/main.rs b/frostbow/src/main.rs index a817e50..e2aad77 100644 --- a/frostbow/src/main.rs +++ b/frostbow/src/main.rs @@ -4,6 +4,7 @@ use clap::Parser; use datafusion::{ execution::{context::SessionContext, SessionStateBuilder}, logical_expr::ScalarUDF, + prelude::SessionConfig, }; use datafusion_cli::{ exec, @@ -15,9 +16,9 @@ use datafusion_iceberg::{ error::Error, planner::{IcebergQueryPlanner, RefreshMaterializedView}, }; -use frostbow::{Args, IcebergContext}; -use iceberg_rust::catalog::bucket::ObjectStoreBuilder; -use object_store::{aws::AmazonS3Builder, memory::InMemory}; +use frostbow::{get_storage, Args, IcebergContext}; +use iceberg_file_catalog::FileCatalogList; +use iceberg_rust::catalog::CatalogList; #[cfg(feature = "rest")] use iceberg_rest_catalog::{apis::configuration::Configuration, catalog::RestCatalogList}; @@ -42,41 +43,39 @@ async fn main_inner() -> Result<(), Error> { "ICEBERG_CATALOG_URL".to_string(), ))?; - let bucket = args.bucket; - - let username = args.username; - let password = args.password; - + let storage = args.storage; let command = args.command; - let object_store = match &bucket { - Some(bucket) => { - let builder = AmazonS3Builder::from_env().with_bucket_name(bucket); - - ObjectStoreBuilder::S3(builder) - } - _ => ObjectStoreBuilder::Memory(Arc::new(InMemory::new())), - }; + let object_store = get_storage(storage.as_deref())?; #[cfg(feature = "rest")] let iceberg_catalog_list = { - let configuration = Configuration { - base_path: catalog_url, - user_agent: None, - client: reqwest_middleware::ClientBuilder::new(reqwest::Client::new()).build(), - basic_auth: username.map(|username| (username, password)), - oauth_access_token: None, - bearer_access_token: None, - api_key: None, - }; - - Arc::new(RestCatalogList::new(configuration, object_store)) + if catalog_url.starts_with("s3://") { + Arc::new( + FileCatalogList::new(&catalog_url, object_store) + .await + .map_err(iceberg_rust::error::Error::from)?, + ) as Arc + } else { + let configuration = Configuration { + base_path: catalog_url, + user_agent: None, + client: reqwest_middleware::ClientBuilder::new(reqwest::Client::new()).build(), + basic_auth: None, + oauth_access_token: None, + bearer_access_token: None, + api_key: None, + }; + + Arc::new(RestCatalogList::new(configuration, object_store)) + } }; let catalog_list = Arc::new(IcebergCatalogList::new(iceberg_catalog_list.clone()).await?); let state = SessionStateBuilder::new() .with_default_features() + .with_config(SessionConfig::default().with_information_schema(true)) .with_catalog_list(catalog_list) .with_query_planner(Arc::new(IcebergQueryPlanner {})) .build();