diff --git a/Cargo.lock b/Cargo.lock index 6bb24978..7491fd32 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -104,9 +104,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" +checksum = "2faccea4cc4ab4a667ce676a30e8ec13922a692c99bb8f5b11f1502c72e04220" [[package]] name = "anyhow" @@ -134,11 +134,10 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "arrow" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bc25126d18a012146a888a0298f2c22e1150327bd2765fc76d710a556b2d614" +checksum = "aa285343fba4d829d49985bdc541e3789cf6000ed0e84be7c039438df4a4e78c" dependencies = [ - "ahash 0.8.7", "arrow-arith", "arrow-array", "arrow-buffer", @@ -156,8 +155,8 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "49.0.0" -source = "git+https://github.com/splitgraph/arrow-rs?branch=49-with-date-fix#3e5d3cb3fd0cd3069ae7c61b4ab3f34d1c1bfc5f" +version = "50.0.0" +source = "git+https://github.com/apache/arrow-rs?rev=72d8a783176219f0864022daba70e84ceab7e221#72d8a783176219f0864022daba70e84ceab7e221" dependencies = [ "arrow-array", "arrow-buffer", @@ -170,8 +169,8 @@ dependencies = [ [[package]] name = "arrow-array" -version = "49.0.0" -source = "git+https://github.com/splitgraph/arrow-rs?branch=49-with-date-fix#3e5d3cb3fd0cd3069ae7c61b4ab3f34d1c1bfc5f" +version = "50.0.0" +source = "git+https://github.com/apache/arrow-rs?rev=72d8a783176219f0864022daba70e84ceab7e221#72d8a783176219f0864022daba70e84ceab7e221" dependencies = [ "ahash 0.8.7", "arrow-buffer", @@ -186,8 +185,8 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "49.0.0" -source = "git+https://github.com/splitgraph/arrow-rs?branch=49-with-date-fix#3e5d3cb3fd0cd3069ae7c61b4ab3f34d1c1bfc5f" +version = "50.0.0" +source = "git+https://github.com/apache/arrow-rs?rev=72d8a783176219f0864022daba70e84ceab7e221#72d8a783176219f0864022daba70e84ceab7e221" dependencies = [ "bytes", "half", @@ -196,8 +195,8 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "49.0.0" -source = "git+https://github.com/splitgraph/arrow-rs?branch=49-with-date-fix#3e5d3cb3fd0cd3069ae7c61b4ab3f34d1c1bfc5f" +version = "50.0.0" +source = "git+https://github.com/apache/arrow-rs?rev=72d8a783176219f0864022daba70e84ceab7e221#72d8a783176219f0864022daba70e84ceab7e221" dependencies = [ "arrow-array", "arrow-buffer", @@ -214,8 +213,8 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "49.0.0" -source = "git+https://github.com/splitgraph/arrow-rs?branch=49-with-date-fix#3e5d3cb3fd0cd3069ae7c61b4ab3f34d1c1bfc5f" +version = "50.0.0" +source = "git+https://github.com/apache/arrow-rs?rev=72d8a783176219f0864022daba70e84ceab7e221#72d8a783176219f0864022daba70e84ceab7e221" dependencies = [ "arrow-array", "arrow-buffer", @@ -232,8 +231,8 @@ dependencies = [ [[package]] name = "arrow-data" -version = "49.0.0" -source = "git+https://github.com/splitgraph/arrow-rs?branch=49-with-date-fix#3e5d3cb3fd0cd3069ae7c61b4ab3f34d1c1bfc5f" +version = "50.0.0" +source = "git+https://github.com/apache/arrow-rs?rev=72d8a783176219f0864022daba70e84ceab7e221#72d8a783176219f0864022daba70e84ceab7e221" dependencies = [ "arrow-buffer", "arrow-schema", @@ -243,9 +242,9 @@ dependencies = [ [[package]] name = "arrow-flight" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624e0dcb6b5a7a06222bfd2be3f7e905ce849a6b714ec989f18cdba330c77d38" +checksum = "1d7f215461ad6346f2e4cc853e377d4e076d533e1ed78d327debe83023e3601f" dependencies = [ "arrow-arith", "arrow-array", @@ -270,9 +269,9 @@ dependencies = [ [[package]] name = "arrow-integration-test" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0a553831ab129cd380680b0d1b8042c61c1df322f0c96ae33b95a999ed28ef" +checksum = "53056074c546e6d0db2932a409c98e17c1bcce66b75add5c30f4aa6fafafb38d" dependencies = [ "arrow", "arrow-buffer", @@ -284,8 +283,8 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "49.0.0" -source = "git+https://github.com/splitgraph/arrow-rs?branch=49-with-date-fix#3e5d3cb3fd0cd3069ae7c61b4ab3f34d1c1bfc5f" +version = "50.0.0" +source = "git+https://github.com/apache/arrow-rs?rev=72d8a783176219f0864022daba70e84ceab7e221#72d8a783176219f0864022daba70e84ceab7e221" dependencies = [ "arrow-array", "arrow-buffer", @@ -293,12 +292,13 @@ dependencies = [ "arrow-data", "arrow-schema", "flatbuffers", + "lz4_flex", ] [[package]] name = "arrow-json" -version = "49.0.0" -source = "git+https://github.com/splitgraph/arrow-rs?branch=49-with-date-fix#3e5d3cb3fd0cd3069ae7c61b4ab3f34d1c1bfc5f" +version = "50.0.0" +source = "git+https://github.com/apache/arrow-rs?rev=72d8a783176219f0864022daba70e84ceab7e221#72d8a783176219f0864022daba70e84ceab7e221" dependencies = [ "arrow-array", "arrow-buffer", @@ -307,7 +307,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap 2.1.0", + "indexmap 2.2.2", "lexical-core", "num", "serde", @@ -316,8 +316,8 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "49.0.0" -source = "git+https://github.com/splitgraph/arrow-rs?branch=49-with-date-fix#3e5d3cb3fd0cd3069ae7c61b4ab3f34d1c1bfc5f" +version = "50.0.0" +source = "git+https://github.com/apache/arrow-rs?rev=72d8a783176219f0864022daba70e84ceab7e221#72d8a783176219f0864022daba70e84ceab7e221" dependencies = [ "arrow-array", "arrow-buffer", @@ -330,8 +330,8 @@ dependencies = [ [[package]] name = "arrow-row" -version = "49.0.0" -source = "git+https://github.com/splitgraph/arrow-rs?branch=49-with-date-fix#3e5d3cb3fd0cd3069ae7c61b4ab3f34d1c1bfc5f" +version = "50.0.0" +source = "git+https://github.com/apache/arrow-rs?rev=72d8a783176219f0864022daba70e84ceab7e221#72d8a783176219f0864022daba70e84ceab7e221" dependencies = [ "ahash 0.8.7", "arrow-array", @@ -344,8 +344,8 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "49.0.0" -source = "git+https://github.com/splitgraph/arrow-rs?branch=49-with-date-fix#3e5d3cb3fd0cd3069ae7c61b4ab3f34d1c1bfc5f" +version = "50.0.0" +source = "git+https://github.com/apache/arrow-rs?rev=72d8a783176219f0864022daba70e84ceab7e221#72d8a783176219f0864022daba70e84ceab7e221" dependencies = [ "bitflags 2.4.2", "serde", @@ -353,8 +353,8 @@ dependencies = [ [[package]] name = "arrow-select" -version = "49.0.0" -source = "git+https://github.com/splitgraph/arrow-rs?branch=49-with-date-fix#3e5d3cb3fd0cd3069ae7c61b4ab3f34d1c1bfc5f" +version = "50.0.0" +source = "git+https://github.com/apache/arrow-rs?rev=72d8a783176219f0864022daba70e84ceab7e221#72d8a783176219f0864022daba70e84ceab7e221" dependencies = [ "ahash 0.8.7", "arrow-array", @@ -366,8 +366,8 @@ dependencies = [ [[package]] name = "arrow-string" -version = "49.0.0" -source = "git+https://github.com/splitgraph/arrow-rs?branch=49-with-date-fix#3e5d3cb3fd0cd3069ae7c61b4ab3f34d1c1bfc5f" +version = "50.0.0" +source = "git+https://github.com/apache/arrow-rs?rev=72d8a783176219f0864022daba70e84ceab7e221#72d8a783176219f0864022daba70e84ceab7e221" dependencies = [ "arrow-array", "arrow-buffer", @@ -555,20 +555,6 @@ dependencies = [ "tower-service", ] -[[package]] -name = "backoff" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" -dependencies = [ - "futures-core", - "getrandom 0.2.12", - "instant", - "pin-project-lite", - "rand 0.8.5", - "tokio", -] - [[package]] name = "backtrace" version = "0.3.69" @@ -674,7 +660,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "digest 0.10.7", + "digest", ] [[package]] @@ -690,15 +676,6 @@ dependencies = [ "constant_time_eq", ] -[[package]] -name = "block-buffer" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" -dependencies = [ - "generic-array", -] - [[package]] name = "block-buffer" version = "0.10.4" @@ -760,7 +737,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c48f0051a4b4c5e0b6d365cd04af53aeaa209e3cc15ec2cdb69e73cc87fbd0dc" dependencies = [ "memchr", - "regex-automata 0.4.4", + "regex-automata 0.4.5", "serde", ] @@ -778,9 +755,9 @@ checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" [[package]] name = "bytecheck" -version = "0.6.11" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6372023ac861f6e6dc89c8344a8f398fb42aaba2b5dbc649ca0c0e9dbcb627" +checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" dependencies = [ "bytecheck_derive", "ptr_meta", @@ -789,9 +766,9 @@ dependencies = [ [[package]] name = "bytecheck_derive" -version = "0.6.11" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7ec4c6f261935ad534c0c22dbef2201b45918860eb1c574b972bd213a76af61" +checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" dependencies = [ "proc-macro2", "quote", @@ -984,9 +961,9 @@ checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" [[package]] name = "chrono" -version = "0.4.32" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41daef31d7a747c5c847246f36de49ced6f7403b4cdabc807a97b5cc184cda7a" +checksum = "9f13690e35a5e4ace198e7beea2895d29f3a9cc55015fcebe6336bd2010af9eb" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1102,8 +1079,8 @@ version = "7.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c64043d6c7b7a4c58e39e7efccfdea7b93d885a795d0c054a69dbbf4dd52686" dependencies = [ - "strum", - "strum_macros", + "strum 0.25.0", + "strum_macros 0.25.3", "unicode-width", ] @@ -1138,7 +1115,7 @@ dependencies = [ [[package]] name = "connectorx" version = "0.3.3-alpha.1" -source = "git+https://github.com/splitgraph/connector-x?branch=datafusion-34-upgrade#7edd597f97141563d5fca34756d8a222702f153c" +source = "git+https://github.com/splitgraph/connector-x?branch=datafusion-35-upgrade#40ed2a27dda290a70d6254142d1b80f535b1f3e5" dependencies = [ "anyhow", "arrow", @@ -1207,14 +1184,14 @@ checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" [[package]] name = "convergence" -version = "0.14.0" -source = "git+https://github.com/splitgraph/convergence?branch=datafusion-34-upgrade#f9ab2785d4ee1ffc65210041d804b548a14382b3" +version = "0.15.0" +source = "git+https://github.com/returnString/convergence?rev=c58ba5c9903e96fd73b65fda8c7b19192fee5cd3#c58ba5c9903e96fd73b65fda8c7b19192fee5cd3" dependencies = [ "async-trait", "bytes", "chrono", "futures", - "sqlparser 0.40.0", + "sqlparser 0.43.1", "thiserror", "tokio", "tokio-util", @@ -1222,8 +1199,8 @@ dependencies = [ [[package]] name = "convergence-arrow" -version = "0.14.0" -source = "git+https://github.com/splitgraph/convergence?branch=datafusion-34-upgrade#f9ab2785d4ee1ffc65210041d804b548a14382b3" +version = "0.15.0" +source = "git+https://github.com/returnString/convergence?rev=c58ba5c9903e96fd73b65fda8c7b19192fee5cd3#c58ba5c9903e96fd73b65fda8c7b19192fee5cd3" dependencies = [ "async-trait", "chrono", @@ -1268,18 +1245,18 @@ dependencies = [ [[package]] name = "cranelift-bforest" -version = "0.103.0" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c22542c0b95bd3302f7ed6839869c561f2324bac2fd5e7e99f5cfa65fdc8b92" +checksum = "d819feeda4c420a18f1e28236ca0ce1177b22bf7c8a44ddee92dfe40de15bcf0" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-codegen" -version = "0.103.0" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b3db903ef2e9c8a4de2ea6db5db052c7857282952f9df604aa55d169e6000d8" +checksum = "e9b8d03d5bdbca7e5f72b0e0a0f69933ed1f09e24be6c075aa6fe3f802b0cc0c" dependencies = [ "bumpalo", "cranelift-bforest", @@ -1298,33 +1275,33 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.103.0" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6590feb5a1d6438f974bf6a5ac4dddf69fca14e1f07f3265d880f69e61a94463" +checksum = "a3fd3664e38e51649b17dc30cfdd561273fe2f590dcd013fb75d9eabc6272dfb" dependencies = [ "cranelift-codegen-shared", ] [[package]] name = "cranelift-codegen-shared" -version = "0.103.0" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7239038c56fafe77fddc8788fc8533dd6c474dc5bdc5637216404f41ba807330" +checksum = "4b031ec5e605828975952622b5a77d49126f20ffe88d33719a0af66b23a0fc36" [[package]] name = "cranelift-control" -version = "0.103.0" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7dc9c595341404d381d27a3d950160856b35b402275f0c3990cd1ad683c8053" +checksum = "fada054d017cf2ed8f7ed2336e0517fc1b19e6825be1790de9eb00c94788362b" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.103.0" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44e3ee532fc4776c69bcedf7e62f9632cbb3f35776fa9a525cdade3195baa3f7" +checksum = "177b6f94ae8de6348eb45bf977c79ab9e3c40fc3ac8cb7ed8109560ea39bee7d" dependencies = [ "serde", "serde_derive", @@ -1332,9 +1309,9 @@ dependencies = [ [[package]] name = "cranelift-frontend" -version = "0.103.0" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a612c94d09e653662ec37681dc2d6fd2b9856e6df7147be0afc9aabb0abf19df" +checksum = "ebebd23a69a23e3ddea78e98ff3a2de222e88c8e045d81ef4a72f042e0d79dbd" dependencies = [ "cranelift-codegen", "log", @@ -1344,15 +1321,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.103.0" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85db9830abeb1170b7d29b536ffd55af1d4d26ac8a77570b5d1aca003bf225cc" +checksum = "1571bfc14df8966d12c6121b5325026591a4b4009e22fea0fe3765ab7cd33b96" [[package]] name = "cranelift-native" -version = "0.103.0" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301ef0edafeaeda5771a5d2db64ac53e1818ae3111220a185677025fe91db4a1" +checksum = "35a69c37e0c10b46fe5527f2397ac821046efbf5f7ec112c8b84df25712f465b" dependencies = [ "cranelift-codegen", "libc", @@ -1361,9 +1338,9 @@ dependencies = [ [[package]] name = "cranelift-wasm" -version = "0.103.0" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "380f0abe8264e4570ac615fc31cef32a3b90a77f7eb97b08331f9dd357b1f500" +checksum = "9b3fef8bbceb8cb56d3f1778b0418d75c5cf12ec571a35fc01eb41abb0227a25" dependencies = [ "cranelift-codegen", "cranelift-entity", @@ -1471,16 +1448,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "crypto-mac" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714" -dependencies = [ - "generic-array", - "subtle", -] - [[package]] name = "csv" version = "1.3.0" @@ -1523,13 +1490,13 @@ checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5" [[package]] name = "datafusion" -version = "34.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "193fd1e7628278d0641c5122860f9a7fd6a1d77d055838d12f55d15bbe28d4d0" +version = "35.0.0" +source = "git+https://github.com/apache/arrow-datafusion?rev=a7a74fa522aaef07e6605f414308f3c99bd1ea06#a7a74fa522aaef07e6605f414308f3c99bd1ea06" dependencies = [ "ahash 0.8.7", "arrow", "arrow-array", + "arrow-ipc", "arrow-schema", "async-compression", "async-trait", @@ -1549,8 +1516,8 @@ dependencies = [ "glob", "half", "hashbrown 0.14.3", - "indexmap 2.1.0", - "itertools 0.12.0", + "indexmap 2.2.2", + "itertools 0.12.1", "log", "num_cpus", "object_store", @@ -1558,7 +1525,7 @@ dependencies = [ "parquet", "pin-project-lite", "rand 0.8.5", - "sqlparser 0.40.0", + "sqlparser 0.43.1", "tempfile", "tokio", "tokio-util", @@ -1570,9 +1537,8 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "34.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "548bc49c4a489e3de474813831ea556dc9d368f9ed8d867b1493da42e8e9f613" +version = "35.0.0" +source = "git+https://github.com/apache/arrow-datafusion?rev=a7a74fa522aaef07e6605f414308f3c99bd1ea06#a7a74fa522aaef07e6605f414308f3c99bd1ea06" dependencies = [ "ahash 0.8.7", "arrow", @@ -1585,14 +1551,13 @@ dependencies = [ "num_cpus", "object_store", "parquet", - "sqlparser 0.40.0", + "sqlparser 0.43.1", ] [[package]] name = "datafusion-execution" -version = "34.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc865657ffcf4da5ff08bdc6436a9a833bc0aa96c3254c8d18ab8a0ad4e437d" +version = "35.0.0" +source = "git+https://github.com/apache/arrow-datafusion?rev=a7a74fa522aaef07e6605f414308f3c99bd1ea06#a7a74fa522aaef07e6605f414308f3c99bd1ea06" dependencies = [ "arrow", "chrono", @@ -1611,25 +1576,23 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "34.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33c473f72d8d81a532e63f6e562ed66dd9209dfd8e433d9712abd42444ee161e" +version = "35.0.0" +source = "git+https://github.com/apache/arrow-datafusion?rev=a7a74fa522aaef07e6605f414308f3c99bd1ea06#a7a74fa522aaef07e6605f414308f3c99bd1ea06" dependencies = [ "ahash 0.8.7", "arrow", "arrow-array", "datafusion-common", "paste", - "sqlparser 0.40.0", - "strum", - "strum_macros", + "sqlparser 0.43.1", + "strum 0.25.0", + "strum_macros 0.25.3", ] [[package]] name = "datafusion-optimizer" -version = "34.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb6218318001d2f6783b7fffa17592318f65f26609d7aab605a3dd0c7c2e2618" +version = "35.0.0" +source = "git+https://github.com/apache/arrow-datafusion?rev=a7a74fa522aaef07e6605f414308f3c99bd1ea06#a7a74fa522aaef07e6605f414308f3c99bd1ea06" dependencies = [ "arrow", "async-trait", @@ -1638,16 +1601,15 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr", "hashbrown 0.14.3", - "itertools 0.12.0", + "itertools 0.12.1", "log", "regex-syntax 0.8.2", ] [[package]] name = "datafusion-physical-expr" -version = "34.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1ca7e35ca22f9dc506c2375b92054b03ccf91afe25c0a90b395a1473a09735" +version = "35.0.0" +source = "git+https://github.com/apache/arrow-datafusion?rev=a7a74fa522aaef07e6605f414308f3c99bd1ea06#a7a74fa522aaef07e6605f414308f3c99bd1ea06" dependencies = [ "ahash 0.8.7", "arrow", @@ -1664,24 +1626,23 @@ dependencies = [ "half", "hashbrown 0.14.3", "hex", - "indexmap 2.1.0", - "itertools 0.12.0", + "indexmap 2.2.2", + "itertools 0.12.1", "log", - "md-5 0.10.6", + "md-5", "paste", "petgraph", "rand 0.8.5", "regex", - "sha2 0.10.8", + "sha2", "unicode-segmentation", "uuid 1.7.0", ] [[package]] name = "datafusion-physical-plan" -version = "34.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddde97adefcca3a55257c646ffee2a95b6cac66f74d1146a6e3a6dbb37830631" +version = "35.0.0" +source = "git+https://github.com/apache/arrow-datafusion?rev=a7a74fa522aaef07e6605f414308f3c99bd1ea06#a7a74fa522aaef07e6605f414308f3c99bd1ea06" dependencies = [ "ahash 0.8.7", "arrow", @@ -1697,8 +1658,8 @@ dependencies = [ "futures", "half", "hashbrown 0.14.3", - "indexmap 2.1.0", - "itertools 0.12.0", + "indexmap 2.2.2", + "itertools 0.12.1", "log", "once_cell", "parking_lot", @@ -1710,9 +1671,8 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "34.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "344700ad4505ffcab6ba0715188ce317ad44e024fe1c5dc7267762113a9dcdf8" +version = "35.0.0" +source = "git+https://github.com/apache/arrow-datafusion?rev=a7a74fa522aaef07e6605f414308f3c99bd1ea06#a7a74fa522aaef07e6605f414308f3c99bd1ea06" dependencies = [ "arrow", "chrono", @@ -1735,7 +1695,7 @@ dependencies = [ "datafusion", "datafusion-common", "datafusion-expr", - "itertools 0.12.0", + "itertools 0.12.1", "rstest", "tokio", "tracing", @@ -1743,16 +1703,15 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "34.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a60d9d6460a64fddb8663db41da97e6b8b0bf79da42f997ebe81722731eaf0e5" +version = "35.0.0" +source = "git+https://github.com/apache/arrow-datafusion?rev=a7a74fa522aaef07e6605f414308f3c99bd1ea06#a7a74fa522aaef07e6605f414308f3c99bd1ea06" dependencies = [ "arrow", "arrow-schema", "datafusion-common", "datafusion-expr", "log", - "sqlparser 0.40.0", + "sqlparser 0.43.1", ] [[package]] @@ -1786,47 +1745,23 @@ dependencies = [ [[package]] name = "deltalake" version = "0.17.0" -source = "git+https://github.com/delta-io/delta-rs?rev=9264edea89a2fc1c35f4a6b9faab125748ff3651#9264edea89a2fc1c35f4a6b9faab125748ff3651" +source = "git+https://github.com/splitgraph/delta-rs?branch=sqlparser-0.43#55915476bf8944f0a87a1b5196bf3ae1332f9b76" dependencies = [ - "deltalake-aws", "deltalake-core", ] -[[package]] -name = "deltalake-aws" -version = "0.1.0" -source = "git+https://github.com/delta-io/delta-rs?rev=9264edea89a2fc1c35f4a6b9faab125748ff3651#9264edea89a2fc1c35f4a6b9faab125748ff3651" -dependencies = [ - "async-trait", - "backoff", - "bytes", - "deltalake-core", - "futures", - "lazy_static", - "maplit", - "object_store", - "regex", - "rusoto_core", - "rusoto_credential", - "rusoto_dynamodb", - "rusoto_sts", - "thiserror", - "tokio", - "tracing", - "url", - "uuid 1.7.0", -] - [[package]] name = "deltalake-core" version = "0.17.0" -source = "git+https://github.com/delta-io/delta-rs?rev=9264edea89a2fc1c35f4a6b9faab125748ff3651#9264edea89a2fc1c35f4a6b9faab125748ff3651" +source = "git+https://github.com/splitgraph/delta-rs?branch=sqlparser-0.43#55915476bf8944f0a87a1b5196bf3ae1332f9b76" dependencies = [ "arrow", "arrow-arith", "arrow-array", "arrow-buffer", "arrow-cast", + "arrow-ipc", + "arrow-json", "arrow-ord", "arrow-row", "arrow-schema", @@ -1846,7 +1781,9 @@ dependencies = [ "errno", "fix-hidden-lifetime-bug", "futures", - "itertools 0.12.0", + "hashbrown 0.14.3", + "indexmap 2.2.2", + "itertools 0.12.1", "lazy_static", "libc", "maplit", @@ -1858,12 +1795,13 @@ dependencies = [ "parking_lot", "parquet", "percent-encoding", + "pin-project-lite", "rand 0.8.5", "regex", "roaring", "serde", "serde_json", - "sqlparser 0.40.0", + "sqlparser 0.43.1", "thiserror", "tokio", "tracing", @@ -1894,9 +1832,9 @@ dependencies = [ [[package]] name = "derive_utils" -version = "0.13.2" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9abcad25e9720609ccb3dcdb795d845e37d8ce34183330a9f48b03a1a71c8e21" +checksum = "61bb5a1014ce6dfc2a378578509abe775a5aa06bff584a547555d9efdb81b926" dependencies = [ "proc-macro2", "quote", @@ -1909,22 +1847,13 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" -[[package]] -name = "digest" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" -dependencies = [ - "generic-array", -] - [[package]] name = "digest" version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer 0.10.4", + "block-buffer", "const-oid", "crypto-common", "subtle", @@ -1949,16 +1878,6 @@ dependencies = [ "dirs-sys", ] -[[package]] -name = "dirs-next" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" -dependencies = [ - "cfg-if", - "dirs-sys-next", -] - [[package]] name = "dirs-sys" version = "0.3.7" @@ -2512,7 +2431,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" dependencies = [ "fallible-iterator 0.3.0", - "indexmap 2.1.0", + "indexmap 2.2.2", "stable_deref_trait", ] @@ -2547,7 +2466,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap 2.1.0", + "indexmap 2.2.2", "slab", "tokio", "tokio-util", @@ -2662,17 +2581,7 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ - "hmac 0.12.1", -] - -[[package]] -name = "hmac" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b" -dependencies = [ - "crypto-mac", - "digest 0.9.0", + "hmac", ] [[package]] @@ -2681,7 +2590,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest 0.10.7", + "digest", ] [[package]] @@ -2819,9 +2728,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.59" +version = "0.1.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6a67363e2aa4443928ce15e57ebae94fd8949958fd1223c4cfc0cd473ad7539" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -2868,9 +2777,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.1.0" +version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +checksum = "824b2ae422412366ba479e8111fd301f7b5faece8149317bb81925979a53f520" dependencies = [ "equivalent", "hashbrown 0.14.3", @@ -2900,12 +2809,11 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] name = "io-enum" -version = "1.1.1" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5305557fa27b460072ae15ce07617e999f5879f14d376c8449f0bfb9f9d8e91e" +checksum = "53b53d712d99a73eec59ee5e4fe6057f8052142d38eeafbbffcb06b36d738a6e" dependencies = [ "derive_utils", - "syn 2.0.48", ] [[package]] @@ -2950,9 +2858,9 @@ dependencies = [ [[package]] name = "itertools" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" dependencies = [ "either", ] @@ -3108,9 +3016,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.152" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libgit2-sys" @@ -3164,9 +3072,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.14" +version = "1.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "295c17e837573c8c821dbaeb3cceb3d745ad082f7572191409e69cbc1b3fd050" +checksum = "037731f5d3aaa87a5675e895b63ddff1a87624bc29f77004ea829809654e48f6" dependencies = [ "cc", "libc", @@ -3267,17 +3175,6 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4facc753ae494aeb6e3c22f839b158aebd4f9270f55cd3c79906c45476c47ab4" -[[package]] -name = "md-5" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15" -dependencies = [ - "block-buffer 0.9.0", - "digest 0.9.0", - "opaque-debug", -] - [[package]] name = "md-5" version = "0.10.6" @@ -3285,7 +3182,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ "cfg-if", - "digest 0.10.7", + "digest", ] [[package]] @@ -3336,9 +3233,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" dependencies = [ "adler", ] @@ -3356,9 +3253,9 @@ dependencies = [ [[package]] name = "moka" -version = "0.12.4" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad9dc9808102655926a6086abd0b9965ebefd4a39ef0d184f074c34ba5049ec6" +checksum = "b1911e88d5831f748a4097a43862d129e3c6fca831eecac9b8db6d01d93c9de2" dependencies = [ "async-lock", "async-trait", @@ -3458,7 +3355,7 @@ dependencies = [ "serde", "serde_json", "sha1", - "sha2 0.10.8", + "sha2", "smallvec", "subprocess", "thiserror", @@ -3593,6 +3490,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-integer" version = "0.1.45" @@ -3654,15 +3557,15 @@ checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ "crc32fast", "hashbrown 0.14.3", - "indexmap 2.1.0", + "indexmap 2.2.2", "memchr", ] [[package]] name = "object_store" -version = "0.8.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2524735495ea1268be33d200e1ee97455096a0846295a21548cd2f3541de7050" +checksum = "d139f545f64630e2e3688fd9f81c470888ab01edeb72d13b4e86c566f1130000" dependencies = [ "async-trait", "base64 0.21.7", @@ -3671,14 +3574,14 @@ dependencies = [ "futures", "humantime", "hyper", - "itertools 0.11.0", + "itertools 0.12.1", "parking_lot", "percent-encoding", "quick-xml", "rand 0.8.5", "reqwest", "ring", - "rustls-pemfile", + "rustls-pemfile 2.0.0", "serde", "serde_json", "snafu", @@ -3694,12 +3597,6 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" -[[package]] -name = "opaque-debug" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" - [[package]] name = "openssl" version = "0.10.63" @@ -3815,9 +3712,9 @@ dependencies = [ [[package]] name = "parquet" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af88740a842787da39b3d69ce5fbf6fce97d20211d3b299fee0a0da6430c74d4" +checksum = "547b92ebf0c1177e3892f44c8f79757ee62e678d564a9834189725f2c5b7a750" dependencies = [ "ahash 0.8.7", "arrow-array", @@ -3833,6 +3730,7 @@ dependencies = [ "chrono", "flate2", "futures", + "half", "hashbrown 0.14.3", "lz4_flex", "num", @@ -3900,9 +3798,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.6" +version = "2.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f200d8d83c44a45b21764d1916299752ca035d15ecd46faca3e9a2a2bf6ad06" +checksum = "219c0dcc30b6a27553f9cc242972b67f75b60eb0db71f0b5462f38b058c41546" dependencies = [ "memchr", "thiserror", @@ -3911,9 +3809,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.6" +version = "2.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcd6ab1236bbdb3a49027e920e693192ebfe8913f6d60e294de57463a493cfde" +checksum = "22e1288dbd7786462961e69bfd4df7848c1e37e8b74303dbdab82c3a9cdd2809" dependencies = [ "pest", "pest_generator", @@ -3921,9 +3819,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.6" +version = "2.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a31940305ffc96863a735bef7c7994a00b325a7138fdbc5bda0f1a0476d3275" +checksum = "1381c29a877c6d34b8c176e734f35d7f7f5b3adaefe940cb4d1bb7af94678e2e" dependencies = [ "pest", "pest_meta", @@ -3934,13 +3832,13 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.7.6" +version = "2.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7ff62f5259e53b78d1af898941cdcdccfae7385cf7d793a6e55de5d05bb4b7d" +checksum = "d0934d6907f148c22a3acbda520c7eed243ad7487a30f51f6ce52b58b7077a8a" dependencies = [ "once_cell", "pest", - "sha2 0.10.8", + "sha2", ] [[package]] @@ -3950,7 +3848,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 2.1.0", + "indexmap 2.2.2", ] [[package]] @@ -3993,18 +3891,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" +checksum = "0302c4a0442c456bd56f841aee5c3bfd17967563f6fadc9ceb9f9c23cf3807e0" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" +checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690" dependencies = [ "proc-macro2", "quote", @@ -4100,11 +3998,11 @@ dependencies = [ "byteorder", "bytes", "fallible-iterator 0.2.0", - "hmac 0.12.1", - "md-5 0.10.6", + "hmac", + "md-5", "memchr", "rand 0.8.5", - "sha2 0.10.8", + "sha2", "stringprep", ] @@ -4299,11 +4197,11 @@ dependencies = [ [[package]] name = "pulldown-cmark" -version = "0.9.3" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a1a2f1f0a7ecff9c31abbe177637be0e97a0aef46cf8738ece09327985d998" +checksum = "57206b407293d2bcd3af849ce869d52068623f19e1b5ff8e8778e3309439682b" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.2", "memchr", "unicase", ] @@ -4541,7 +4439,7 @@ checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.4", + "regex-automata 0.4.5", "regex-syntax 0.8.2", ] @@ -4556,9 +4454,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b7fa1134405e2ec9353fd416b17f8dacd46c473d7d3fd1cf202706a14eb792a" +checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd" dependencies = [ "aho-corasick", "memchr", @@ -4585,18 +4483,18 @@ checksum = "e898588f33fdd5b9420719948f9f2a32c922a246964576f71ba7f24f80610fbc" [[package]] name = "rend" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2571463863a6bd50c32f94402933f03457a3fbaf697a707c5be741e459f08fd" +checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" dependencies = [ "bytecheck", ] [[package]] name = "reqwest" -version = "0.11.23" +version = "0.11.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" +checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251" dependencies = [ "base64 0.21.7", "bytes", @@ -4618,10 +4516,12 @@ dependencies = [ "percent-encoding", "pin-project-lite", "rustls", - "rustls-pemfile", + "rustls-native-certs", + "rustls-pemfile 1.0.4", "serde", "serde_json", "serde_urlencoded", + "sync_wrapper", "system-configuration", "tokio", "tokio-native-tls", @@ -4633,7 +4533,6 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 0.25.3", "winreg", ] @@ -4659,9 +4558,9 @@ dependencies = [ [[package]] name = "rkyv" -version = "0.7.43" +version = "0.7.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "527a97cdfef66f65998b5f3b637c26f5a5ec09cc52a3f9932313ac645f4190f5" +checksum = "5cba464629b3394fc4dbc6f940ff8f5b4ff5c7aef40f29166fd4ad12acbc99c0" dependencies = [ "bitvec", "bytecheck", @@ -4677,9 +4576,9 @@ dependencies = [ [[package]] name = "rkyv_derive" -version = "0.7.43" +version = "0.7.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5c462a1328c8e67e4d6dbad1eb0355dd43e8ab432c6e227a43657f16ade5033" +checksum = "a7dddfff8de25e6f62b9d64e6e432bf1c6736c57d20323e15ee10435fbda7c65" dependencies = [ "proc-macro2", "quote", @@ -4749,7 +4648,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d0e5124fcb30e76a7e79bfee683a2746db83784b86289f6251b54b7950a0dfc" dependencies = [ "const-oid", - "digest 0.10.7", + "digest", "num-bigint-dig", "num-integer", "num-traits", @@ -4791,104 +4690,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "rusoto_core" -version = "0.47.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b4f000e8934c1b4f70adde180056812e7ea6b1a247952db8ee98c94cd3116cc" -dependencies = [ - "async-trait", - "base64 0.13.1", - "bytes", - "crc32fast", - "futures", - "http", - "hyper", - "hyper-tls", - "lazy_static", - "log", - "rusoto_credential", - "rusoto_signature", - "rustc_version", - "serde", - "serde_json", - "tokio", - "xml-rs", -] - -[[package]] -name = "rusoto_credential" -version = "0.47.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a46b67db7bb66f5541e44db22b0a02fed59c9603e146db3a9e633272d3bac2f" -dependencies = [ - "async-trait", - "chrono", - "dirs-next", - "futures", - "hyper", - "serde", - "serde_json", - "shlex", - "tokio", - "zeroize", -] - -[[package]] -name = "rusoto_dynamodb" -version = "0.47.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7935e1f9ca57c4ee92a4d823dcd698eb8c992f7e84ca21976ae72cd2b03016e7" -dependencies = [ - "async-trait", - "bytes", - "futures", - "rusoto_core", - "serde", - "serde_json", -] - -[[package]] -name = "rusoto_signature" -version = "0.47.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6264e93384b90a747758bcc82079711eacf2e755c3a8b5091687b5349d870bcc" -dependencies = [ - "base64 0.13.1", - "bytes", - "chrono", - "digest 0.9.0", - "futures", - "hex", - "hmac 0.11.0", - "http", - "hyper", - "log", - "md-5 0.9.1", - "percent-encoding", - "pin-project-lite", - "rusoto_credential", - "rustc_version", - "serde", - "sha2 0.9.9", - "tokio", -] - -[[package]] -name = "rusoto_sts" -version = "0.47.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7edd42473ac006fd54105f619e480b0a94136e7f53cf3fb73541363678fd92" -dependencies = [ - "async-trait", - "bytes", - "chrono", - "futures", - "rusoto_core", - "serde_urlencoded", - "xml-rs", -] - [[package]] name = "rusqlite" version = "0.29.0" @@ -4916,9 +4717,9 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.33.1" +version = "1.34.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06676aec5ccb8fc1da723cc8c0f9a46549f21ebb8753d3915c6c41db1e7f1dc4" +checksum = "755392e1a2f77afd95580d3f0d0e94ac83eeeb7167552c9b5bca549e61a94d83" dependencies = [ "arrayvec", "borsh", @@ -4933,9 +4734,9 @@ dependencies = [ [[package]] name = "rust_decimal_macros" -version = "1.33.1" +version = "1.34.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e43721f4ef7060ebc2c3ede757733209564ca8207f47674181bcd425dd76945" +checksum = "e418701588729bef95e7a655f2b483ad64bb97c46e8e79fde83efd92aaab6d82" dependencies = [ "quote", "rust_decimal", @@ -4964,9 +4765,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.30" +version = "0.38.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" +checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" dependencies = [ "bitflags 2.4.2", "errno", @@ -4989,6 +4790,18 @@ dependencies = [ "sct", ] +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile 1.0.4", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pemfile" version = "1.0.4" @@ -4998,6 +4811,22 @@ dependencies = [ "base64 0.21.7", ] +[[package]] +name = "rustls-pemfile" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35e4980fa29e4c4b212ffb3db068a564cbf560e51d3944b7c88bd8bf5bec64f4" +dependencies = [ + "base64 0.21.7", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a716eb65e3158e90e17cd93d855216e27bde02745ab842f2cab4a39dba1bacf" + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -5127,7 +4956,7 @@ dependencies = [ "deltalake", "futures", "hex", - "itertools 0.12.0", + "itertools 0.12.1", "lazy_static", "moka", "object_store", @@ -5145,11 +4974,11 @@ dependencies = [ "serde", "serde_json", "serial_test", - "sha2 0.10.8", - "sqlparser 0.40.0", + "sha2", + "sqlparser 0.43.1", "sqlx", - "strum", - "strum_macros", + "strum 0.26.1", + "strum_macros 0.26.1", "tempfile", "thiserror", "tokio", @@ -5214,9 +5043,9 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.195" +version = "1.0.196" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" +checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32" dependencies = [ "serde_derive", ] @@ -5232,9 +5061,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.195" +version = "1.0.196" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" +checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" dependencies = [ "proc-macro2", "quote", @@ -5243,9 +5072,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.111" +version = "1.0.113" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" +checksum = "69801b70b1c3dac963ecb03a364ba0ceda9cf60c71cfe475e99864759c8b8a79" dependencies = [ "itoa", "ryu", @@ -5308,20 +5137,7 @@ checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures", - "digest 0.10.7", -] - -[[package]] -name = "sha2" -version = "0.9.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" -dependencies = [ - "block-buffer 0.9.0", - "cfg-if", - "cpufeatures", - "digest 0.9.0", - "opaque-debug", + "digest", ] [[package]] @@ -5332,7 +5148,7 @@ checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", "cpufeatures", - "digest 0.10.7", + "digest", ] [[package]] @@ -5374,7 +5190,7 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "digest 0.10.7", + "digest", "rand_core 0.6.4", ] @@ -5511,7 +5327,7 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce81b7bd7c4493975347ef60d8c7e8b742d4694f4c49f93e0a12ea263938176c" dependencies = [ - "itertools 0.12.0", + "itertools 0.12.1", "nom", "unicode_categories", ] @@ -5527,9 +5343,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.40.0" +version = "0.43.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c80afe31cdb649e56c0d9bb5503be9166600d68a852c38dd445636d126858e5" +checksum = "f95c4bae5aba7cd30bd506f7140026ade63cff5afd778af8854026f9606bf5d4" dependencies = [ "log", "sqlparser_derive", @@ -5581,17 +5397,17 @@ dependencies = [ "futures-util", "hashlink", "hex", - "indexmap 2.1.0", + "indexmap 2.2.2", "log", "memchr", "once_cell", "paste", "percent-encoding", "rustls", - "rustls-pemfile", + "rustls-pemfile 1.0.4", "serde", "serde_json", - "sha2 0.10.8", + "sha2", "smallvec", "sqlformat", "thiserror", @@ -5600,7 +5416,7 @@ dependencies = [ "tracing", "url", "uuid 1.7.0", - "webpki-roots 0.24.0", + "webpki-roots", ] [[package]] @@ -5631,7 +5447,7 @@ dependencies = [ "quote", "serde", "serde_json", - "sha2 0.10.8", + "sha2", "sqlx-core", "sqlx-mysql", "sqlx-postgres", @@ -5654,7 +5470,7 @@ dependencies = [ "byteorder", "bytes", "crc", - "digest 0.10.7", + "digest", "dotenvy", "either", "futures-channel", @@ -5664,10 +5480,10 @@ dependencies = [ "generic-array", "hex", "hkdf", - "hmac 0.12.1", + "hmac", "itoa", "log", - "md-5 0.10.6", + "md-5", "memchr", "once_cell", "percent-encoding", @@ -5675,7 +5491,7 @@ dependencies = [ "rsa", "serde", "sha1", - "sha2 0.10.8", + "sha2", "smallvec", "sqlx-core", "stringprep", @@ -5704,18 +5520,18 @@ dependencies = [ "futures-util", "hex", "hkdf", - "hmac 0.12.1", + "hmac", "home", "itoa", "log", - "md-5 0.10.6", + "md-5", "memchr", "once_cell", "rand 0.8.5", "serde", "serde_json", "sha1", - "sha2 0.10.8", + "sha2", "smallvec", "sqlx-core", "stringprep", @@ -5783,9 +5599,15 @@ version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" dependencies = [ - "strum_macros", + "strum_macros 0.25.3", ] +[[package]] +name = "strum" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "723b93e8addf9aa965ebe2d11da6d7540fa2283fcea14b3371ff055f7ba13f5f" + [[package]] name = "strum_macros" version = "0.25.3" @@ -5799,6 +5621,19 @@ dependencies = [ "syn 2.0.48", ] +[[package]] +name = "strum_macros" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a3417fc93d76740d974a01654a09777cb500428cc874ca9f45edfe0c4d4cd18" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.48", +] + [[package]] name = "subprocess" version = "0.2.9" @@ -5811,9 +5646,9 @@ dependencies = [ [[package]] name = "subtle" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "syn" @@ -6001,12 +5836,13 @@ dependencies = [ [[package]] name = "time" -version = "0.3.31" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e" +checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" dependencies = [ "deranged", "itoa", + "num-conv", "powerfmt", "serde", "time-core", @@ -6021,10 +5857,11 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26197e33420244aeb70c3e8c78376ca46571bc4e701e4791c2cd9f57dcb3a43f" +checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774" dependencies = [ + "num-conv", "time-core", ] @@ -6054,9 +5891,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.35.1" +version = "1.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" +checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" dependencies = [ "backtrace", "bytes", @@ -6204,11 +6041,11 @@ checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" [[package]] name = "toml_edit" -version = "0.21.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" +checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" dependencies = [ - "indexmap 2.1.0", + "indexmap 2.2.2", "toml_datetime", "winnow", ] @@ -6622,7 +6459,7 @@ dependencies = [ "multer", "percent-encoding", "pin-project", - "rustls-pemfile", + "rustls-pemfile 1.0.4", "scoped-tls", "serde", "serde_json", @@ -6649,9 +6486,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasi-cap-std-sync" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "154528979a211aa28d969846e883df75705809ed9bcc70aba61460683ea7355b" +checksum = "db014d2ced91f17d1f1a8f2b76d6ea8d731bc1dbc8c2bbaec689d6a242568e5d" dependencies = [ "anyhow", "async-trait", @@ -6667,14 +6504,14 @@ dependencies = [ "system-interface", "tracing", "wasi-common", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "wasi-common" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d888b611fee7d273dd057dc009d2dd3132736f36710ffd65657ac83628d1e3b" +checksum = "449d17849e3c83a931374442fe2deee4d6bd1ebf469719ef44192e9e82e19c89" dependencies = [ "anyhow", "bitflags 2.4.2", @@ -6687,7 +6524,7 @@ dependencies = [ "tracing", "wasmtime", "wiggle", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -6767,18 +6604,18 @@ dependencies = [ [[package]] name = "wasm-encoder" -version = "0.40.0" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d162eb64168969ae90e8668ca0593b0e47667e315aa08e717a9c9574d700d826" +checksum = "e09bca7d6388637d27fb5edbeab11f56bfabcef8743c55ae34370e1e5030a071" dependencies = [ "leb128", ] [[package]] name = "wasm-streams" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" +checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" dependencies = [ "futures-util", "js-sys", @@ -6793,36 +6630,36 @@ version = "0.118.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95ee9723b928e735d53000dec9eae7b07a60e490c85ab54abb66659fc61bfcd9" dependencies = [ - "indexmap 2.1.0", + "indexmap 2.2.2", "semver", ] [[package]] name = "wasmparser" -version = "0.120.0" +version = "0.121.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9148127f39cbffe43efee8d5442b16ecdba21567785268daa1ec9e134389705" +checksum = "953cf6a7606ab31382cb1caa5ae403e77ba70c7f8e12eeda167e7040d42bfda8" dependencies = [ "bitflags 2.4.2", - "indexmap 2.1.0", + "indexmap 2.2.2", "semver", ] [[package]] name = "wasmprinter" -version = "0.2.77" +version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8389a95eb0b3165fea0537a6988960cc23a33d9be650e63fc3d63065fe20dcb" +checksum = "05e32c13c59fdc64d3f6998a1d52eb1d362b6904a88b754190ccb85661ad577a" dependencies = [ "anyhow", - "wasmparser 0.120.0", + "wasmparser 0.121.0", ] [[package]] name = "wasmtime" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8e539fded2495422ea3c4dfa7beeddba45904eece182cf315294009e1a323bf" +checksum = "910fabce77e660f0e0e41cfd5f69fc8bf020a025f059718846e918db7177f469" dependencies = [ "anyhow", "async-trait", @@ -6831,7 +6668,7 @@ dependencies = [ "cfg-if", "encoding_rs", "fxprof-processed-profile", - "indexmap 2.1.0", + "indexmap 2.2.2", "libc", "log", "object", @@ -6854,23 +6691,23 @@ dependencies = [ "wasmtime-runtime", "wasmtime-winch", "wat", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "wasmtime-asm-macros" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "660ba9143e15a2acd921820df221b73aee256bd3ca2d208d73d8adc9587ccbb9" +checksum = "37288142e9b4a61655a3bcbdc7316c2e4bb9e776b10ce3dd758f8186b4469572" dependencies = [ "cfg-if", ] [[package]] name = "wasmtime-cache" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3ce373743892002f9391c6741ef0cb0335b55ec899d874f311222b7e36f4594" +checksum = "45cbd74a636f09d2108f9405c79857f061e19323e4abeed22e837cfe7b08a22b" dependencies = [ "anyhow", "base64 0.21.7", @@ -6880,17 +6717,17 @@ dependencies = [ "rustix", "serde", "serde_derive", - "sha2 0.10.8", + "sha2", "toml", - "windows-sys 0.48.0", + "windows-sys 0.52.0", "zstd 0.11.2+zstd.1.5.2", ] [[package]] name = "wasmtime-component-macro" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12ef32643324e564e1c359e9044daa06cbf90d7e2d6c99a738d17a12959f01a5" +checksum = "ad63de18eb42e586386b6091f787c82707cbd5ac5e9343216dba1976190cd03a" dependencies = [ "anyhow", "proc-macro2", @@ -6903,15 +6740,15 @@ dependencies = [ [[package]] name = "wasmtime-component-util" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c87d06c18d21a4818f354c00a85f4ebc62b2270961cd022968452b0e4dbed9d" +checksum = "7e0a160c0c44369aa4bee6d311a8e4366943bab1651040cc8b0fcec2c9eb8906" [[package]] name = "wasmtime-cranelift" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d648c8b4064a7911093b02237cd5569f71ca171d3a0a486bf80600b19e1cba2" +checksum = "3734cc01b7cd37bc62fdbcd9529ca9547440052d4b3886cfdec3b8081a5d3647" dependencies = [ "anyhow", "cfg-if", @@ -6934,9 +6771,9 @@ dependencies = [ [[package]] name = "wasmtime-cranelift-shared" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290a89027688782da8ff60b12bb95695494b1874e0d0ba2ba387d23dace6d70c" +checksum = "e0eb33cd30c47844aa228d4d0030587e65c1108343f311fe9f7248b5bd9cb65c" dependencies = [ "anyhow", "cranelift-codegen", @@ -6950,14 +6787,14 @@ dependencies = [ [[package]] name = "wasmtime-environ" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61eb64fb3e0da883e2df4a13a81d6282e072336e6cb6295021d0f7ab2e352754" +checksum = "9a3a056b041fdea604f0972e2fae97958e7748d629a55180228348baefdfc217" dependencies = [ "anyhow", "cranelift-entity", "gimli", - "indexmap 2.1.0", + "indexmap 2.2.2", "log", "object", "serde", @@ -6973,9 +6810,9 @@ dependencies = [ [[package]] name = "wasmtime-fiber" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ecf1d3a838b0956b71ad3f8cb80069a228339775bf02dd35d86a5a68bbe443" +checksum = "43987d0977c07f15c3608c2f255870c127ffd19e35eeedb1ac1dccedf9932a42" dependencies = [ "anyhow", "cc", @@ -6983,14 +6820,14 @@ dependencies = [ "rustix", "wasmtime-asm-macros", "wasmtime-versioned-export-macros", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "wasmtime-jit" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f485336add49267d8859e8f8084d2d4b9a4b1564496b6f30ba5b168d50c10ceb" +checksum = "9b3e48395ac672b386ed588d97a9612aa13a345008f26466f0dfb2a91628aa9f" dependencies = [ "addr2line", "anyhow", @@ -7010,14 +6847,14 @@ dependencies = [ "wasmtime-jit-debug", "wasmtime-jit-icache-coherence", "wasmtime-runtime", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "wasmtime-jit-debug" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65e119affec40edb2fab9044f188759a00c2df9c3017278d047012a2de1efb4f" +checksum = "dd21fd0f5ca68681d3d5b636eea00f182d0f9d764144469e9257fd7e3f55ae0e" dependencies = [ "object", "once_cell", @@ -7027,26 +6864,26 @@ dependencies = [ [[package]] name = "wasmtime-jit-icache-coherence" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b6d197fcc34ad32ed440e1f9552fd57d1f377d9699d31dee1b5b457322c1f8a" +checksum = "bdc26415bb89e9ccd3bdc498fef63aabf665c4c0dd710c107691deb9694955da" dependencies = [ "cfg-if", "libc", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "wasmtime-runtime" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "794b2bb19b99ef8322ff0dd9fe1ba7e19c41036dfb260b3f99ecce128c42ff92" +checksum = "0abddaf17912aabaf39be0802d5eba9a002e956e902d1ebd438a2fe1c88769a2" dependencies = [ "anyhow", "cc", "cfg-if", "encoding_rs", - "indexmap 2.1.0", + "indexmap 2.2.2", "libc", "log", "mach", @@ -7063,14 +6900,14 @@ dependencies = [ "wasmtime-jit-debug", "wasmtime-versioned-export-macros", "wasmtime-wmemcheck", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "wasmtime-types" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d995db8bb56f2cd8d2dc0ed5ffab94ffb435283b0fe6747f80f7aab40b2d06a1" +checksum = "b35a95cdc1433729085beab42c0a5c742b431f25b17c40d7718e46df63d5ffc7" dependencies = [ "cranelift-entity", "serde", @@ -7081,9 +6918,9 @@ dependencies = [ [[package]] name = "wasmtime-versioned-export-macros" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f55c5565959287c21dd0f4277ae3518dd2ae62679f655ee2dbc4396e19d210db" +checksum = "fad322733fe67e45743784d8b1df452bcb54f581572a4f1a646a4332deecbcc2" dependencies = [ "proc-macro2", "quote", @@ -7092,9 +6929,9 @@ dependencies = [ [[package]] name = "wasmtime-wasi" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccd8370078149d49a3a47e93741553fd79b700421464b6a27ca32718192ab130" +checksum = "902cc299b73655c36679b77efdfce4bb5971992f1a4a8a436dd3809a6848ff0e" dependencies = [ "anyhow", "async-trait", @@ -7122,14 +6959,14 @@ dependencies = [ "wasi-common", "wasmtime", "wiggle", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "wasmtime-winch" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c6f945ff9bad96e0a69973d74f193c19f627c8adbf250e7cb73ae7564b6cc8a" +checksum = "9e63aeca929f84560eec52c5af43bf5d623b92683b0195d9fb06da8ed860e092" dependencies = [ "anyhow", "cranelift-codegen", @@ -7144,21 +6981,21 @@ dependencies = [ [[package]] name = "wasmtime-wit-bindgen" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f328b2d4a690270324756e886ed5be3a4da4c00be0eea48253f4595ad068062b" +checksum = "41e5675998fdc74495afdd90ad2bd221206a258075b23048af0535a969b07893" dependencies = [ "anyhow", "heck", - "indexmap 2.1.0", + "indexmap 2.2.2", "wit-parser", ] [[package]] name = "wasmtime-wmemcheck" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67761d8f8c0b3c13a5d34356274b10a40baba67fe9cfabbfc379a8b414e45de2" +checksum = "b20a19e10d8cb50b45412fb21192982b7ce85c0122dc33bb71f1813e25dc6e52" [[package]] name = "wast" @@ -7171,23 +7008,24 @@ dependencies = [ [[package]] name = "wast" -version = "70.0.1" +version = "70.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5d415036fe747a32b30c76c8bd6c73f69b7705fb7ebca5f16e852eef0c95802" +checksum = "a3d5061300042ff5065123dae1e27d00c03f567d34a2937c8472255148a216dc" dependencies = [ + "bumpalo", "leb128", "memchr", "unicode-width", - "wasm-encoder 0.40.0", + "wasm-encoder 0.41.0", ] [[package]] name = "wat" -version = "1.0.84" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8241f34599d413d2243a21015ab43aef68bfb32a0e447c54eef8d423525ca15e" +checksum = "afd7357b6cc46d46a2509c43dcb1dd4131dafbf4e75562d87017b5a05ffad2d6" dependencies = [ - "wast 70.0.1", + "wast 70.0.2", ] [[package]] @@ -7209,12 +7047,6 @@ dependencies = [ "rustls-webpki", ] -[[package]] -name = "webpki-roots" -version = "0.25.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10" - [[package]] name = "which" version = "4.4.2" @@ -7239,9 +7071,9 @@ dependencies = [ [[package]] name = "wiggle" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0afb26cd3269289bb314a361ff0a6685e5ce793b62181a9fe3f81ace15051697" +checksum = "737728db69a7657a5f6a7bac445c02d8564d603d62c46c95edf928554e67d072" dependencies = [ "anyhow", "async-trait", @@ -7254,9 +7086,9 @@ dependencies = [ [[package]] name = "wiggle-generate" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cef2868fed7584d2b552fa317104858ded80021d23b073b2d682d3c932a027bd" +checksum = "2460c7163b79ffefd9a564eaeab0a5b0e84bb91afdfeeb84d36f304ddbe08982" dependencies = [ "anyhow", "heck", @@ -7269,9 +7101,9 @@ dependencies = [ [[package]] name = "wiggle-macro" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31ae1ec11a17ea481539ee9a5719a278c9790d974060fbf71db4b2c05378780b" +checksum = "fa8d8412375ba8325d61fbae56dead51dabfaec85d620ce36427922fb9cece83" dependencies = [ "proc-macro2", "quote", @@ -7312,9 +7144,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "winch-codegen" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58e58c236a6abdd9ab454552b4f29e16cfa837a86897c1503313b2e62e7609ec" +checksum = "9d2b346bad5397b219b4ff0a8fa7230936061ff07c61f05d589d8d81e06fb7b2" dependencies = [ "anyhow", "cranelift-codegen", @@ -7469,9 +7301,9 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" [[package]] name = "winnow" -version = "0.5.34" +version = "0.5.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7cf47b659b318dccbd69cc4797a39ae128f533dce7902a1096044d1967b9c16" +checksum = "a7cad8365489051ae9f054164e459304af2e7e9bb407c958076c8bf4aef52da5" dependencies = [ "memchr", ] @@ -7526,7 +7358,7 @@ checksum = "df4913a2219096373fd6512adead1fb77ecdaa59d7fc517972a7d30b12f625be" dependencies = [ "anyhow", "id-arena", - "indexmap 2.1.0", + "indexmap 2.2.2", "log", "semver", "serde", @@ -7556,12 +7388,6 @@ dependencies = [ "tap", ] -[[package]] -name = "xml-rs" -version = "0.8.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fcb9cbac069e033553e8bb871be2fbdffcab578eb25bd0f7c508cedc6dcd75a" - [[package]] name = "xz2" version = "0.1.7" diff --git a/Cargo.toml b/Cargo.toml index b6807ce1..c360b537 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,20 +2,20 @@ members = ["clade"] [workspace.dependencies] -arrow = "49.0.0" -arrow-buffer = "49.0.0" -arrow-csv = "49.0.0" -arrow-flight = "49.0.0" +arrow = "50.0.0" +arrow-buffer = "50.0.0" +arrow-csv = "50.0.0" +arrow-flight = "50.0.0" # For the JSON format support # https://github.com/apache/arrow-rs/pull/2868 # https://github.com/apache/arrow-rs/pull/2724 -arrow-integration-test = "49.0.0" -arrow-schema = "49.0.0" +arrow-integration-test = "50.0.0" +arrow-schema = "50.0.0" async-trait = "0.1.64" -datafusion = "34.0.0" -datafusion-common = "34.0.0" -datafusion-expr = "34.0.0" +datafusion = "35.0.0" +datafusion-common = "35.0.0" +datafusion-expr = "35.0.0" itertools = ">=0.10.0" tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread", "signal", "process"] } @@ -51,22 +51,31 @@ object-store-s3 = ["object_store/aws"] remote-tables = ["dep:datafusion-remote-tables"] [patch.crates-io] -# Patch to pick up https://github.com/apache/arrow-rs/pull/5282 from -# https://github.com/splitgraph/arrow-rs/tree/49-with-date-fix - -arrow-arith = { git = "https://github.com/splitgraph/arrow-rs", branch = "49-with-date-fix" } -arrow-array = { git = "https://github.com/splitgraph/arrow-rs", branch = "49-with-date-fix" } -arrow-buffer = { git = "https://github.com/splitgraph/arrow-rs", branch = "49-with-date-fix" } -arrow-cast = { git = "https://github.com/splitgraph/arrow-rs", branch = "49-with-date-fix" } -arrow-csv = { git = "https://github.com/splitgraph/arrow-rs", branch = "49-with-date-fix" } -arrow-data = { git = "https://github.com/splitgraph/arrow-rs", branch = "49-with-date-fix" } -arrow-ipc = { git = "https://github.com/splitgraph/arrow-rs", branch = "49-with-date-fix" } -arrow-json = { git = "https://github.com/splitgraph/arrow-rs", branch = "49-with-date-fix" } -arrow-ord = { git = "https://github.com/splitgraph/arrow-rs", branch = "49-with-date-fix" } -arrow-row = { git = "https://github.com/splitgraph/arrow-rs", branch = "49-with-date-fix" } -arrow-schema = { git = "https://github.com/splitgraph/arrow-rs", branch = "49-with-date-fix" } -arrow-select = { git = "https://github.com/splitgraph/arrow-rs", branch = "49-with-date-fix" } -arrow-string = { git = "https://github.com/splitgraph/arrow-rs", branch = "49-with-date-fix" } +# Pick up https://github.com/apache/arrow-rs/pull/5282 +arrow-arith = { git = "https://github.com/apache/arrow-rs", rev = "72d8a783176219f0864022daba70e84ceab7e221" } +arrow-array = { git = "https://github.com/apache/arrow-rs", rev = "72d8a783176219f0864022daba70e84ceab7e221" } +arrow-buffer = { git = "https://github.com/apache/arrow-rs", rev = "72d8a783176219f0864022daba70e84ceab7e221" } +arrow-cast = { git = "https://github.com/apache/arrow-rs", rev = "72d8a783176219f0864022daba70e84ceab7e221" } +arrow-csv = { git = "https://github.com/apache/arrow-rs", rev = "72d8a783176219f0864022daba70e84ceab7e221" } +arrow-data = { git = "https://github.com/apache/arrow-rs", rev = "72d8a783176219f0864022daba70e84ceab7e221" } +arrow-ipc = { git = "https://github.com/apache/arrow-rs", rev = "72d8a783176219f0864022daba70e84ceab7e221" } +arrow-json = { git = "https://github.com/apache/arrow-rs", rev = "72d8a783176219f0864022daba70e84ceab7e221" } +arrow-ord = { git = "https://github.com/apache/arrow-rs", rev = "72d8a783176219f0864022daba70e84ceab7e221" } +arrow-row = { git = "https://github.com/apache/arrow-rs", rev = "72d8a783176219f0864022daba70e84ceab7e221" } +arrow-schema = { git = "https://github.com/apache/arrow-rs", rev = "72d8a783176219f0864022daba70e84ceab7e221" } +arrow-select = { git = "https://github.com/apache/arrow-rs", rev = "72d8a783176219f0864022daba70e84ceab7e221" } +arrow-string = { git = "https://github.com/apache/arrow-rs", rev = "72d8a783176219f0864022daba70e84ceab7e221" } + +# Pick up https://github.com/apache/arrow-datafusion/pull/8894 and https://github.com/apache/arrow-datafusion/pull/9007 +datafusion = { git = "https://github.com/apache/arrow-datafusion", rev = "a7a74fa522aaef07e6605f414308f3c99bd1ea06" } +datafusion-common = { git = "https://github.com/apache/arrow-datafusion", rev = "a7a74fa522aaef07e6605f414308f3c99bd1ea06" } +datafusion-execution = { git = "https://github.com/apache/arrow-datafusion", rev = "a7a74fa522aaef07e6605f414308f3c99bd1ea06" } +datafusion-expr = { git = "https://github.com/apache/arrow-datafusion", rev = "a7a74fa522aaef07e6605f414308f3c99bd1ea06" } +datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion", rev = "a7a74fa522aaef07e6605f414308f3c99bd1ea06" } +datafusion-physical-expr = { git = "https://github.com/apache/arrow-datafusion", rev = "a7a74fa522aaef07e6605f414308f3c99bd1ea06" } +datafusion-physical-plan = { git = "https://github.com/apache/arrow-datafusion", rev = "a7a74fa522aaef07e6605f414308f3c99bd1ea06" } +datafusion-proto = { git = "https://github.com/apache/arrow-datafusion", rev = "a7a74fa522aaef07e6605f414308f3c99bd1ea06" } +datafusion-sql = { git = "https://github.com/apache/arrow-datafusion", rev = "a7a74fa522aaef07e6605f414308f3c99bd1ea06" } [dependencies] arrow = { workspace = true } @@ -88,8 +97,8 @@ clap = { version = "3.2.19", features = [ "derive" ] } config = "0.13.3" # PG wire protocol support -convergence = { git = "https://github.com/splitgraph/convergence", branch = "datafusion-34-upgrade", optional = true } -convergence-arrow = { git = "https://github.com/splitgraph/convergence", branch = "datafusion-34-upgrade", optional = true } +convergence = { git = "https://github.com/returnString/convergence", rev = "c58ba5c9903e96fd73b65fda8c7b19192fee5cd3", optional = true } +convergence-arrow = { git = "https://github.com/returnString/convergence", rev = "c58ba5c9903e96fd73b65fda8c7b19192fee5cd3", optional = true } dashmap = "5.4.0" @@ -99,14 +108,14 @@ datafusion-expr = { workspace = true } datafusion-remote-tables = { path = "./datafusion_remote_tables", optional = true } -deltalake = { git = "https://github.com/delta-io/delta-rs", rev = "9264edea89a2fc1c35f4a6b9faab125748ff3651", features = ["s3-native-tls", "datafusion-ext"] } +deltalake = { git = "https://github.com/splitgraph/delta-rs", branch = "sqlparser-0.43", features = ["datafusion"] } futures = "0.3" hex = ">=0.4.0" itertools = { workspace = true } lazy_static = ">=1.4.0" moka = { version = "0.12.2", default_features = false, features = ["future", "atomic64", "quanta"] } -object_store = "0.8" +object_store = "0.9" parking_lot = "0.12.1" percent-encoding = "2.2.0" prost = "0.12.1" @@ -123,7 +132,7 @@ rustyline = "13.0" serde = "1.0.156" serde_json = "1.0.93" sha2 = ">=0.10.1" -sqlparser = "0.40" +sqlparser = { version = "0.43", features = ["visitor"] } sqlx = { version = "0.7.1", features = [ "runtime-tokio-rustls", "sqlite", "any", "uuid" ] } strum = ">=0.24" strum_macros = ">=0.24" @@ -139,9 +148,9 @@ uuid = "1.2.1" warp = "0.3.6" # For WASM user-defined functions -wasi-common = "16.0.0" -wasmtime = "16.0.0" -wasmtime-wasi = "16.0.0" +wasi-common = "17.0.0" +wasmtime = "17.0.0" +wasmtime-wasi = "17.0.0" [dev-dependencies] assert_cmd = "2" diff --git a/datafusion_remote_tables/Cargo.toml b/datafusion_remote_tables/Cargo.toml index e942241a..52058253 100644 --- a/datafusion_remote_tables/Cargo.toml +++ b/datafusion_remote_tables/Cargo.toml @@ -19,7 +19,7 @@ arrow-schema = { workspace = true } async-trait = { workspace = true } # Remote query execution for a variety of DBs -connectorx = { git = "https://github.com/splitgraph/connector-x", branch = "datafusion-34-upgrade", features = [ "dst_arrow", "src_postgres", "src_mysql", "src_sqlite" ] } +connectorx = { git = "https://github.com/splitgraph/connector-x", branch = "datafusion-35-upgrade", features = [ "dst_arrow", "src_postgres", "src_mysql", "src_sqlite" ] } datafusion = { workspace = true } datafusion-common = { workspace = true } diff --git a/src/context/delta.rs b/src/context/delta.rs index 746ddd30..4629078e 100644 --- a/src/context/delta.rs +++ b/src/context/delta.rs @@ -25,7 +25,6 @@ use deltalake::writer::create_add; use deltalake::DeltaTable; use futures::{StreamExt, TryStreamExt}; use object_store::path::Path; -use std::collections::HashMap; use std::fs::File; use std::sync::Arc; use tempfile::{NamedTempFile, TempPath}; @@ -258,7 +257,7 @@ pub async fn plan_to_object_store( // Create the corresponding Add action; currently we don't support partition columns // which simplifies things. - let add = create_add(&HashMap::default(), file_name, size, &metadata)?; + let add = create_add(&Default::default(), file_name, size, &metadata)?; Ok(add) }); @@ -400,7 +399,7 @@ impl SeafowlContext { table_log_store.as_ref(), &actions, op, - table.get_state(), + table.state.as_ref(), None, ) .await?; @@ -522,7 +521,6 @@ mod tests { adds[0].path.clone(), adds[0].size, adds[0].partition_values.is_empty(), - adds[0].partition_values_parsed.is_none(), adds[0].data_change, serde_json::from_str::( adds[0].stats.clone().unwrap().as_str() @@ -533,7 +531,6 @@ mod tests { adds[1].path.clone(), adds[1].size, adds[1].partition_values.is_empty(), - adds[1].partition_values_parsed.is_none(), adds[1].data_change, serde_json::from_str::( adds[1].stats.clone().unwrap().as_str() @@ -544,8 +541,7 @@ mod tests { vec![ ( PART_0_FILE_NAME.to_string(), - 1269, - true, + 1298, true, true, json!({ @@ -569,8 +565,7 @@ mod tests { ), ( PART_1_FILE_NAME.to_string(), - 1284, - true, + 1313, true, true, json!({ @@ -599,10 +594,7 @@ mod tests { object_store .get_log_store(&table_uuid.to_string()) .object_store(), - vec![ - Path::from(PART_0_FILE_NAME.to_string()), - Path::from(PART_1_FILE_NAME.to_string()), - ], + vec![PART_0_FILE_NAME.to_string(), PART_1_FILE_NAME.to_string()], ) .await; } diff --git a/src/context/logical.rs b/src/context/logical.rs index 10862292..b5513d2d 100644 --- a/src/context/logical.rs +++ b/src/context/logical.rs @@ -107,6 +107,7 @@ impl SeafowlContext { // Delegate generic queries to the basic DataFusion logical planner // (though note EXPLAIN [our custom query] will mean we have to implement EXPLAIN ourselves) Statement::Explain { .. } + | Statement::ExplainTable { .. } | Statement::ShowVariable { .. } | Statement::ShowTables { .. } | Statement::ShowColumns { .. } @@ -309,7 +310,7 @@ impl SeafowlContext { })), })) } - DFStatement::DescribeTableStmt(_) | DFStatement::CreateExternalTable(_) => { + DFStatement::CreateExternalTable(_) => { self.inner.state().statement_to_plan(stmt).await } DFStatement::CopyTo(_) | DFStatement::Explain(_) => { @@ -483,19 +484,6 @@ mod tests { ); } - #[tokio::test] - async fn test_plan_insert_type_mismatch() { - let ctx = in_memory_context_with_test_db().await; - - // Try inserting a timestamp into a number (note this will work fine for inserting - // e.g. Utf-8 into numbers at plan time but should fail at execution time if the value - // doesn't convert) - let err = ctx - .create_logical_plan("INSERT INTO testcol.some_table SELECT '2022-01-01', to_timestamp('2022-01-01T12:00:00')") - .await.unwrap_err(); - assert_eq!(err.to_string(), "Error during planning: Cannot automatically convert Timestamp(Nanosecond, None) to Float64"); - } - #[tokio::test] async fn test_plan_insert_values_wrong_number() { let ctx = in_memory_context_with_test_db().await; diff --git a/src/context/mod.rs b/src/context/mod.rs index c326cfaf..318afe5c 100644 --- a/src/context/mod.rs +++ b/src/context/mod.rs @@ -133,20 +133,15 @@ impl SeafowlContext { &self, table_name: impl Into>, ) -> Result { - let table_log_store = self - .inner + self.inner .table_provider(table_name) .await? .as_any() .downcast_ref::() .ok_or_else(|| { DataFusionError::Execution("Table {table_name} not found".to_string()) - })? - .log_store(); - - // We can't just keep hold of the downcasted ref from above because of - // `temporary value dropped while borrowed` - Ok(DeltaTable::new(table_log_store, Default::default())) + }) + .cloned() } // Parse the uuid from the Delta table uri if available @@ -264,6 +259,25 @@ mod tests { use super::test_utils::in_memory_context; use super::*; + #[tokio::test] + async fn test_timestamp_to_date_casting() -> Result<()> { + let ctx = in_memory_context().await; + + let plan = ctx.plan_query("SELECT '1998-11-30 00:00:00'::date").await?; + + let results = ctx.collect(plan).await?; + let expected = [ + "+-----------------------------+", + "| Utf8(\"1998-11-30 00:00:00\") |", + "+-----------------------------+", + "| 1998-11-30 |", + "+-----------------------------+", + ]; + assert_batches_eq!(expected, &results); + + Ok(()) + } + #[rstest] #[case::regular_type_names("float", "float")] #[case::legacy_type_names("f32", "f32")] diff --git a/src/context/physical.rs b/src/context/physical.rs index 16cab742..c3f7fded 100644 --- a/src/context/physical.rs +++ b/src/context/physical.rs @@ -259,6 +259,7 @@ impl SeafowlContext { let mut table = self.try_get_delta_table(table_name).await?; table.load().await?; + let snapshot = table.snapshot()?; let schema_ref = TableProvider::schema(&table); let df_schema = DFSchema::try_from_qualified_schema( @@ -297,7 +298,6 @@ impl SeafowlContext { let filter_expr = create_physical_expr( &predicate.clone(), &df_schema, - schema_ref.as_ref(), &ExecutionProps::new(), )?; @@ -305,11 +305,10 @@ impl SeafowlContext { filter_expr.clone(), schema_ref.clone(), )?; - let prune_map = pruning_predicate.prune(&table)?; + let prune_map = pruning_predicate.prune(snapshot)?; - let files_to_prune = table - .get_state() - .files() + let files_to_prune = snapshot + .file_actions()? .iter() .zip(prune_map) .filter_map( @@ -320,7 +319,7 @@ impl SeafowlContext { (Some(filter_expr), files_to_prune) } else { // If no qualifier is specified we're basically updating the whole table. - (None, table.get_state().files().clone()) + (None, snapshot.file_actions()?) }; let uuid = self.get_table_uuid(table_name).await?; @@ -383,7 +382,7 @@ impl SeafowlContext { table.log_store().as_ref(), &actions, op, - table.get_state(), + table.state.as_ref(), None, ) .await?; @@ -404,6 +403,7 @@ impl SeafowlContext { let mut table = self.try_get_delta_table(table_name).await?; table.load().await?; + let snapshot = table.snapshot()?; let schema_ref = SchemaRef::from(table_schema.deref().clone()); let (adds, removes) = @@ -416,16 +416,14 @@ impl SeafowlContext { let prune_expr = create_physical_expr( &predicate.clone(), table_schema, - schema_ref.as_ref(), &ExecutionProps::new(), )?; let pruning_predicate = PruningPredicate::try_new(prune_expr, schema_ref.clone())?; - let prune_map = pruning_predicate.prune(&table)?; - let files_to_prune = table - .get_state() - .files() + let prune_map = pruning_predicate.prune(snapshot)?; + let files_to_prune = snapshot + .file_actions()? .iter() .zip(prune_map) .filter_map( @@ -443,7 +441,6 @@ impl SeafowlContext { let filter_expr = create_physical_expr( &predicate.clone().not(), table_schema, - schema_ref.as_ref(), &ExecutionProps::new(), )?; @@ -472,7 +469,7 @@ impl SeafowlContext { } else { // If no qualifier is specified we're basically truncating the table. // Remove all files. - (vec![], table.get_state().files().clone()) + (vec![], snapshot.file_actions()?) }; let deletion_timestamp = SystemTime::now() @@ -502,7 +499,7 @@ impl SeafowlContext { table.log_store().as_ref(), &actions, op, - table.get_state(), + table.state.as_ref(), None, ) .await?; @@ -712,7 +709,7 @@ impl SeafowlContext { delta_table.load().await?; let plan = VacuumBuilder::new( delta_table.log_store(), - delta_table.state.clone(), + delta_table.snapshot()?.clone(), ) .with_enforce_retention_duration(false) .with_retention_period(Duration::hours(0_i64)); @@ -902,6 +899,7 @@ impl SeafowlContext { #[cfg(test)] mod tests { + use crate::context::test_utils::in_memory_context_with_test_db; use datafusion::assert_batches_eq; use std::sync::Arc; @@ -1024,4 +1022,22 @@ mod tests { Ok(()) } + + #[tokio::test] + async fn test_plan_insert_type_mismatch() { + let ctx = in_memory_context_with_test_db().await; + + // Try inserting a string into a date (note this will work fine for inserting + // e.g. Utf-8 into numbers at plan time but should fail at execution time if the value + // doesn't convert) + let plan = ctx + .create_logical_plan("INSERT INTO testcol.some_table SELECT 'abc', to_timestamp('2022-01-01T12:00:00')") + .await.unwrap(); + + let err = ctx.create_physical_plan(&plan).await.unwrap_err(); + assert_eq!( + err.to_string(), + "Arrow error: Cast error: Cannot cast string 'abc' to value of Date32 type" + ); + } } diff --git a/src/datafusion/parser.rs b/src/datafusion/parser.rs index b69e6c1c..cf0fd105 100644 --- a/src/datafusion/parser.rs +++ b/src/datafusion/parser.rs @@ -26,9 +26,7 @@ //! Declares a SQL parser based on sqlparser that handles custom formats that we need. pub use datafusion::sql::parser::Statement; -use datafusion::sql::parser::{ - CopyToSource, CopyToStatement, CreateExternalTable, DescribeTableStmt, -}; +use datafusion::sql::parser::{CopyToSource, CopyToStatement, CreateExternalTable}; use datafusion_common::parsers::CompressionTypeVariant; use lazy_static::lazy_static; use sqlparser::ast::{CreateFunctionBody, Expr, ObjectName, OrderByExpr, Value}; @@ -151,10 +149,6 @@ impl<'a> DFParser<'a> { self.parser.next_token(); self.parse_copy() } - Keyword::DESCRIBE => { - self.parser.next_token(); - self.parse_describe() - } Keyword::VACUUM => { self.parser.next_token(); self.parse_vacuum() @@ -176,19 +170,12 @@ impl<'a> DFParser<'a> { } } - pub fn parse_describe(&mut self) -> Result { - let table_name = self.parser.parse_object_name()?; - Ok(Statement::DescribeTableStmt(DescribeTableStmt { - table_name, - })) - } - // Parse `CONVERT location TO DELTA table_name` type statement pub fn parse_convert(&mut self) -> Result { let location = self.parser.parse_literal_string()?; self.parser .expect_keywords(&[Keyword::TO, Keyword::DELTA])?; - let table_name = self.parser.parse_object_name()?; + let table_name = self.parser.parse_object_name(true)?; // We'll use the CopyToStatement struct to pass the location and table name // as it's the closest match to what we need. @@ -206,9 +193,9 @@ impl<'a> DFParser<'a> { let mut partitions = None; if self.parser.parse_keyword(Keyword::TABLE) { - table_name = self.parser.parse_object_name()?; + table_name = self.parser.parse_object_name(true)?; } else if self.parser.parse_keyword(Keyword::DATABASE) { - let database_name = self.parser.parse_object_name()?.0[0].clone(); + let database_name = self.parser.parse_object_name(false)?.0[0].clone(); partitions = Some(vec![Expr::Identifier(database_name)]); } else { return self.expected( @@ -233,7 +220,7 @@ impl<'a> DFParser<'a> { CopyToSource::Query(query) } else { // parse as table reference - let table_name = self.parser.parse_object_name()?; + let table_name = self.parser.parse_object_name(true)?; CopyToSource::Relation(table_name) }; @@ -323,7 +310,7 @@ impl<'a> DFParser<'a> { or_replace: bool, temporary: bool, ) -> Result { - let name = self.parser.parse_object_name()?; + let name = self.parser.parse_object_name(false)?; self.parser.expect_keyword(Keyword::AS)?; let class_name = self.parser.parse_function_definition()?; let params = CreateFunctionBody { @@ -354,7 +341,7 @@ impl<'a> DFParser<'a> { loop { if let Token::Word(_) = self.parser.peek_token().token { - let identifier = self.parser.parse_identifier()?; + let identifier = self.parser.parse_identifier(false)?; partitions.push(identifier.to_string()); } else { return self.expected("partition name", self.parser.peek_token()); @@ -456,17 +443,17 @@ impl<'a> DFParser<'a> { } fn parse_column_def(&mut self) -> Result { - let name = self.parser.parse_identifier()?; + let name = self.parser.parse_identifier(false)?; let data_type = self.parser.parse_data_type()?; let collation = if self.parser.parse_keyword(Keyword::COLLATE) { - Some(self.parser.parse_object_name()?) + Some(self.parser.parse_object_name(false)?) } else { None }; let mut options = vec![]; loop { if self.parser.parse_keyword(Keyword::CONSTRAINT) { - let name = Some(self.parser.parse_identifier()?); + let name = Some(self.parser.parse_identifier(false)?); if let Some(option) = self.parser.parse_optional_column_option()? { options.push(ColumnOptionDef { name, option }); } else { @@ -497,7 +484,7 @@ impl<'a> DFParser<'a> { let if_not_exists = self.parser .parse_keywords(&[Keyword::IF, Keyword::NOT, Keyword::EXISTS]); - let table_name = self.parser.parse_object_name()?; + let table_name = self.parser.parse_object_name(true)?; let (columns, constraints) = self.parse_columns()?; #[derive(Default)] diff --git a/src/datafusion/utils.rs b/src/datafusion/utils.rs index 2d199c85..4ade0f96 100644 --- a/src/datafusion/utils.rs +++ b/src/datafusion/utils.rs @@ -138,6 +138,8 @@ pub(crate) fn convert_simple_data_type(sql_type: &SQLDataType) -> Result not_impl_err!( "Unsupported SQL type {sql_type:?}" ), diff --git a/src/frontend/http.rs b/src/frontend/http.rs index fddadbb9..0f3da4b0 100644 --- a/src/frontend/http.rs +++ b/src/frontend/http.rs @@ -123,9 +123,11 @@ struct QueryBody { } /// Convert rows from a `RecordBatch` to their JSON Lines byte representation, with newlines at the end -fn batch_to_json(batch: RecordBatch) -> Result, ArrowError> { +fn batch_to_json( + maybe_batch: Result, +) -> Result, ArrowError> { let mut buf = Vec::new(); - for row in record_batches_to_json_rows(&[&batch])? { + for row in record_batches_to_json_rows(&[&maybe_batch?])? { buf.extend( serde_json::to_vec(&row) .map_err(|error| ArrowError::JsonError(error.to_string()))?, @@ -140,10 +142,14 @@ async fn plan_to_response( context: Arc, plan: Arc, ) -> Result { - let stream = context - .execute_stream(plan) - .await? - .map(|maybe_batch| batch_to_json(maybe_batch?)); + let stream = context.execute_stream(plan).await?.map(|maybe_batch| { + batch_to_json(maybe_batch) + // Seems like at this point wrap/hyper don't really handle the stream error well, + // i.e. the status code returned is 200 even when stream fails. + // To at least make this more transparent convert the error message to payload, + // otherwise the client just gets an opaque empty reply. + .or_else(|e| Ok::, ArrowError>(e.to_string().into_bytes())) + }); let body = hyper::Body::wrap_stream(stream); Ok(Response::new(body)) } @@ -1121,10 +1127,32 @@ pub mod tests { let resp = query_uncached_endpoint(&handler, "SELECT 'notanint'::int", new_db, None) .await; - assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + assert_eq!(resp.status(), StatusCode::OK); let error_msg = String::from_utf8_lossy(resp.body()); - assert!(error_msg.contains("Arrow error: Cast error: Cannot cast string 'notanint' to value of Int32 type")); + assert_eq!( + error_msg, + "Cast error: Cannot cast string 'notanint' to value of Int32 type" + ); + } + + #[rstest] + #[tokio::test] + async fn test_error_json_conversion( + #[values(None, Some("test_db"))] new_db: Option<&str>, + ) { + let context = in_memory_context_with_single_table(new_db).await; + let handler = filters(context, http_config_from_access_policy(free_for_all())); + + let resp = + query_uncached_endpoint(&handler, "SELECT 1::NUMERIC", new_db, None).await; + assert_eq!(resp.status(), StatusCode::OK); + + let error_msg = String::from_utf8_lossy(resp.body()); + assert_eq!( + error_msg, + "Json error: data type Decimal128(38, 10) not supported in nested map for json writer" + ); } #[rstest] diff --git a/src/object_store/http.rs b/src/object_store/http.rs index f6876f93..569d9f2a 100644 --- a/src/object_store/http.rs +++ b/src/object_store/http.rs @@ -21,6 +21,7 @@ use crate::object_store::cache::{ use datafusion::prelude::SessionContext; use lazy_static::lazy_static; use regex::Regex; +use reqwest::header::CONTENT_RANGE; use reqwest::{header, Client, ClientBuilder, RequestBuilder, Response, StatusCode}; use std::env; use std::error::Error; @@ -60,6 +61,7 @@ impl Display for HttpObjectStore { enum HttpObjectStoreError { WritesUnsupported, NoContentLengthResponse, + ContentRangeError, ListingUnsupported, HttpClientError(reqwest::Error), RangesUnsupported, @@ -88,6 +90,7 @@ impl Display for HttpObjectStoreError { Self::NoContentLengthResponse => { writeln!(f, "Server did not respond with a Content-Length header") } + Self::ContentRangeError => writeln!(f, "Error validating content range"), Self::ListingUnsupported => writeln!(f, "HTTP doesn't support listing"), Self::RangesUnsupported => { writeln!(f, "This server does not support byte range fetches") @@ -159,30 +162,29 @@ impl HttpObjectStore { fn request_builder_with_get_options( &self, path: &Path, - options: GetOptions, + options: &GetOptions, ) -> RequestBuilder { let mut request_builder = self.request_builder(path); - if let Some(range) = options.range { - let range = format!("bytes={}-{}", range.start, range.end.saturating_sub(1)); - request_builder = request_builder.header(RANGE, range); + if let Some(ref range) = options.range { + request_builder = request_builder.header(RANGE, range.to_string()); } - if let Some(tag) = options.if_match { + if let Some(ref tag) = options.if_match { request_builder = request_builder.header(IF_MATCH, tag); } - if let Some(tag) = options.if_none_match { + if let Some(ref tag) = options.if_none_match { request_builder = request_builder.header(IF_NONE_MATCH, tag); } const DATE_FORMAT: &str = "%a, %d %b %Y %H:%M:%S GMT"; - if let Some(date) = options.if_unmodified_since { + if let Some(ref date) = options.if_unmodified_since { request_builder = request_builder .header(IF_UNMODIFIED_SINCE, date.format(DATE_FORMAT).to_string()); } - if let Some(date) = options.if_modified_since { + if let Some(ref date) = options.if_modified_since { request_builder = request_builder .header(IF_MODIFIED_SINCE, date.format(DATE_FORMAT).to_string()); } @@ -290,11 +292,11 @@ impl ObjectStore for HttpObjectStore { location: &Path, options: GetOptions, ) -> object_store::Result { - let range = options.range.clone(); let response = self - .send(self.request_builder_with_get_options(location, options)) + .send(self.request_builder_with_get_options(location, &options)) .await?; - let meta = header_meta(location, response.headers())?; + let headers = response.headers().clone(); + let mut meta = header_meta(location, &headers)?; let body = response.bytes_stream(); @@ -302,8 +304,58 @@ impl ObjectStore for HttpObjectStore { body.map(|c| c.map_err(|e| HttpObjectStoreError::HttpClientError(e).into())) .boxed(), ); + + // ensure that we receive the range we asked for + let range = if let Some(_expected) = options.range { + let val = headers + .get(CONTENT_RANGE) + .ok_or(HttpObjectStoreError::ContentRangeError)?; + + // Also copied from object_store, private there + struct ContentRange { + /// The range of the object returned + range: Range, + /// The total size of the object being requested + size: usize, + } + + impl ContentRange { + /// Parse a content range of the form `bytes -/` + /// + /// + fn from_str(s: &str) -> Option { + let rem = s.trim().strip_prefix("bytes ")?; + let (range, size) = rem.split_once('/')?; + let size = size.parse().ok()?; + + let (start_s, end_s) = range.split_once('-')?; + + let start = start_s.parse().ok()?; + let end: usize = end_s.parse().ok()?; + + Some(Self { + size, + range: start..end + 1, + }) + } + } + + let value = val + .to_str() + .map_err(|_| HttpObjectStoreError::ContentRangeError)?; + let value = ContentRange::from_str(value) + .ok_or(HttpObjectStoreError::ContentRangeError)?; + let actual = value.range; + + // Update size to reflect full size of object (#5272) + meta.size = value.size; + actual + } else { + 0..meta.size + }; + Ok(GetResult { - range: range.unwrap_or(0..meta.size), + range, payload: stream, meta, }) diff --git a/src/provider.rs b/src/provider.rs index dd85d657..5516ffd6 100644 --- a/src/provider.rs +++ b/src/provider.rs @@ -86,7 +86,7 @@ impl SchemaProvider for SeafowlSchema { // Ultimately though, since the map gets re-created for each query the only point in // updating the existing table is to optimize potential multi-lookups during processing of // a single query. - let table_log_store = match self.tables.read().get(name) { + let mut delta_table = match self.tables.read().get(name) { None => return None, Some(table) => match table.as_any().downcast_ref::() { // This shouldn't happen since we store only DeltaTable's in the map @@ -98,14 +98,12 @@ impl SchemaProvider for SeafowlSchema { } else { // A negative table version indicates that the table was never loaded; we need // to do it before returning it. - delta_table.log_store() + delta_table.clone() } } }, }; - let mut delta_table = DeltaTable::new(table_log_store, Default::default()); - if let Err(err) = delta_table.load().await { warn!("Failed to load table {name}: {err}"); return None; @@ -156,7 +154,7 @@ pub fn project_expressions( }; let mut proj_expr = - create_physical_expr(expr, df_schema, schema, &ExecutionProps::new())?; + create_physical_expr(expr, df_schema, &ExecutionProps::new())?; let data_type = f.data_type().clone(); if proj_expr.data_type(schema)? != data_type { diff --git a/src/testutils.rs b/src/testutils.rs index c91e5f09..31af07b4 100644 --- a/src/testutils.rs +++ b/src/testutils.rs @@ -130,14 +130,14 @@ pub async fn make_mock_parquet_server( pub async fn assert_uploaded_objects( object_store: Arc, - expected: Vec, + expected: Vec, ) { let actual = object_store .list(None) .map_ok(|meta| meta.location) .try_collect::>() .await - .map(|p| p.into_iter().sorted().collect_vec()) + .map(|p| p.into_iter().sorted().map(String::from).collect_vec()) .unwrap(); assert_eq!(expected.into_iter().sorted().collect_vec(), actual); } diff --git a/src/wasm_udf/wasm.rs b/src/wasm_udf/wasm.rs index 1d80f41a..d5956aaf 100644 --- a/src/wasm_udf/wasm.rs +++ b/src/wasm_udf/wasm.rs @@ -12,8 +12,8 @@ use datafusion::{ logical_expr::{ScalarFunctionImplementation, ScalarUDF, Volatility}, }; +use datafusion::error::Result; use datafusion::prelude::*; -use datafusion::{error::Result, physical_plan::functions::make_scalar_function}; use wasmtime::{Engine, Instance, Memory, Module, Store, TypedFunc, Val, ValType}; @@ -22,6 +22,8 @@ use super::data_types::{get_wasm_type, CreateFunctionDataType, CreateFunctionLan use wasi_common::WasiCtx; use wasmtime_wasi::sync::WasiCtxBuilder; +use arrow::array::Array; +use datafusion_expr::ColumnarValue; use std::sync::Arc; use std::vec; @@ -536,6 +538,23 @@ fn messagepack_decode_results( } } +// This is an alternative variant of `datafusion::physical_plan::functions::columnar_values_to_array`, +// which tries to align scalars with the length of any arrays in the input. +fn columnar_values_to_array(args: &[ColumnarValue]) -> Result> { + let inferred_length = args + .iter() + .map(|v| match v { + ColumnarValue::Array(array) => array.len(), + ColumnarValue::Scalar(_) => 1, + }) + .max() + .unwrap_or(0); + + args.iter() + .map(|arg| arg.clone().into_array(inferred_length)) + .collect::>>() +} + fn make_scalar_function_wasm_messagepack( module_bytes: &[u8], function_name: &str, @@ -553,13 +572,14 @@ fn make_scalar_function_wasm_messagepack( "Error initializing WASM + MessagePack UDF {function_name:?}: {err:?}" )) })?; - let inner = move |args: &[ArrayRef]| { + let inner = move |args: &[ColumnarValue]| { let mut instance = WasmMessagePackUDFInstance::new(&module_bytes, &function_name) .map_err(|err| { DataFusionError::Internal(format!( "Error initializing WASM + MessagePack UDF {function_name:?}: {err:?}" )) })?; + // this is guaranteed by DataFusion based on the function's signature. if args.len() != input_types.len() { return Err(DataFusionError::Internal(format!( @@ -568,6 +588,8 @@ fn make_scalar_function_wasm_messagepack( ))); } + let args = columnar_values_to_array(args)?; + // Length of the vectorized array let array_len = args.first().unwrap().len(); @@ -581,7 +603,7 @@ fn make_scalar_function_wasm_messagepack( for col_ix in 0..args.len() { params.push(messagepack_encode_input_value( input_types.get(col_ix).unwrap(), - args, + &args, row_ix, col_ix, )?); @@ -594,10 +616,11 @@ fn make_scalar_function_wasm_messagepack( })?); } - messagepack_decode_results(&return_type, &encoded_results) + let array = messagepack_decode_results(&return_type, &encoded_results)?; + Ok(ColumnarValue::from(array)) }; - Ok(make_scalar_function(inner)) + Ok(Arc::new(inner)) } /// Build a DataFusion scalar function from WASM module bytecode. @@ -632,7 +655,7 @@ fn make_scalar_function_from_wasm( // Capture the function name and the module code let function_name = function_name.to_owned(); let module_bytes = module_bytes.to_owned(); - let inner = move |args: &[ArrayRef]| { + let inner = move |args: &[ColumnarValue]| { // Load the function again let mut store = Store::<()>::default(); @@ -655,6 +678,8 @@ fn make_scalar_function_from_wasm( // this is guaranteed by DataFusion based on the function's signature. assert_eq!(args.len(), input_types.len()); + let args = columnar_values_to_array(args)?; + // Length of the vectorized array let array_len = args.first().unwrap().len(); @@ -667,21 +692,25 @@ fn make_scalar_function_from_wasm( // Build a slice of WASM Val values to pass to the function for col_ix in 0..args.len() { let wasm_val = match input_types.get(col_ix).unwrap() { - ValType::I32 => Val::I32(get_arrow_value::< - arrow::datatypes::Int32Type, - >(args, row_ix, col_ix)?), - ValType::I64 => Val::I64(get_arrow_value::< - arrow::datatypes::Int64Type, - >(args, row_ix, col_ix)?), + ValType::I32 => { + Val::I32(get_arrow_value::( + &args, row_ix, col_ix, + )?) + } + ValType::I64 => { + Val::I64(get_arrow_value::( + &args, row_ix, col_ix, + )?) + } ValType::F32 => Val::F32( get_arrow_value::( - args, row_ix, col_ix, + &args, row_ix, col_ix, )? .to_bits(), ), ValType::F64 => Val::F64( get_arrow_value::( - args, row_ix, col_ix, + &args, row_ix, col_ix, )? .to_bits(), ), @@ -730,10 +759,10 @@ fn make_scalar_function_from_wasm( ) as ArrayRef, _ => panic!("unexpected type"), }; - Ok(array) + Ok(ColumnarValue::from(array)) }; - Ok(make_scalar_function(inner)) + Ok(Arc::new(inner)) } pub fn create_udf_from_wasm( diff --git a/tests/http/upload.rs b/tests/http/upload.rs index 60a75424..e47bdd6b 100644 --- a/tests/http/upload.rs +++ b/tests/http/upload.rs @@ -208,7 +208,7 @@ async fn test_upload_to_existing_table() { // Verify the newly created table contents let plan = context - .plan_query("SELECT * FROM test_table") + .plan_query("SELECT * FROM test_table ORDER BY col_1") .await .unwrap(); let results = context.collect(plan).await.unwrap(); @@ -279,7 +279,7 @@ async fn test_upload_to_existing_table() { // Verify that the rows have been appended let plan = context - .plan_query("SELECT * FROM test_table") + .plan_query("SELECT * FROM test_table ORDER BY col_1") .await .unwrap(); let results = context.collect(plan).await.unwrap(); diff --git a/tests/statements/convert.rs b/tests/statements/convert.rs index 1a3a1e6b..a2ede504 100644 --- a/tests/statements/convert.rs +++ b/tests/statements/convert.rs @@ -63,10 +63,10 @@ async fn test_convert_from_flat_parquet_table() -> Result<()> { .get_log_store(&table_uuid.to_string()) .object_store(), vec![ - Path::from("_delta_log/00000000000000000000.json"), - Path::from("file_1.parquet"), - Path::from("file_2.parquet"), - Path::from("file_3.parquet"), + String::from("_delta_log/00000000000000000000.json"), + String::from("file_1.parquet"), + String::from("file_2.parquet"), + String::from("file_3.parquet"), ], ) .await; diff --git a/tests/statements/ddl.rs b/tests/statements/ddl.rs index 9b166873..69b4dac7 100644 --- a/tests/statements/ddl.rs +++ b/tests/statements/ddl.rs @@ -377,9 +377,9 @@ async fn test_create_table_drop_schema( .get_log_store(&table_uuid.to_string()) .object_store(), vec![ - Path::from("_delta_log/00000000000000000000.json"), - Path::from("_delta_log/00000000000000000001.json"), - table.get_files_iter().collect_vec()[0].clone(), + String::from("_delta_log/00000000000000000000.json"), + String::from("_delta_log/00000000000000000001.json"), + table.snapshot()?.file_actions()?[0].clone().path, ], ) .await; @@ -533,7 +533,7 @@ async fn test_create_external_table( #[case] location: Option<&str>, #[case] options: &str, #[case] object_store_type: ObjectStoreType, -) { +) -> Result<()> { let url = match location { None => { let (mock_server, _) = testutils::make_mock_parquet_server(true, true).await; @@ -574,8 +574,7 @@ async fn test_create_external_table( ) .as_str(), ) - .await - .unwrap(); + .await?; // Test we see the table in the information_schema let results = list_tables_query(&context).await; @@ -594,11 +593,8 @@ async fn test_create_external_table( assert_batches_eq!(expected, &results); // Test standard query - let plan = context - .plan_query("SELECT * FROM staging.file") - .await - .unwrap(); - let results = context.collect(plan).await.unwrap(); + let plan = context.plan_query("SELECT * FROM staging.file").await?; + let results = context.collect(plan).await?; let expected = if location.is_none() { vec![ "+-------+", @@ -626,8 +622,7 @@ async fn test_create_external_table( // Test dropping the external table works context .collect(context.plan_query("DROP TABLE staging.file").await.unwrap()) - .await - .unwrap(); + .await?; let results = list_tables_query(&context).await; @@ -642,4 +637,6 @@ async fn test_create_external_table( "+--------------------+-------------+", ]; assert_batches_eq!(expected, &results); + + Ok(()) } diff --git a/tests/statements/dml.rs b/tests/statements/dml.rs index 72fb959f..10d949c2 100644 --- a/tests/statements/dml.rs +++ b/tests/statements/dml.rs @@ -25,7 +25,7 @@ async fn test_insert_two_different_schemas( .unwrap(); let plan = context - .plan_query("SELECT * FROM test_table") + .plan_query("SELECT * FROM test_table ORDER BY some_other_value") .await .unwrap(); let results = context.collect(plan).await.unwrap(); @@ -55,7 +55,7 @@ async fn test_delete_statement( ObjectStoreType::S3(Some("/path/to/folder")) )] object_store_type: ObjectStoreType, -) { +) -> Result<()> { let (context, _) = make_context_with_pg(object_store_type).await; create_table_and_some_partitions(&context, "test_table", None).await; @@ -63,20 +63,19 @@ async fn test_delete_statement( // // Ensure we have 4 partitions, and record their file names // - let mut table = context.try_get_delta_table("test_table").await.unwrap(); - table.load().await.unwrap(); - let mut all_partitions = table.get_files_iter().collect_vec().clone(); + let mut table = context.try_get_delta_table("test_table").await?; + table.load().await?; + let mut all_partitions = table.snapshot()?.file_actions()?.clone(); assert_eq!(all_partitions.len(), 4); - let partition_1 = all_partitions.first().unwrap().clone(); - let partition_4 = all_partitions.last().unwrap().clone(); + let partition_1 = all_partitions.last().unwrap().clone(); + let partition_4 = all_partitions.first().unwrap().clone(); // // Check DELETE's query plan to make sure 46 (int) gets cast to a float value by the optimizer // let plan = context .create_logical_plan("DELETE FROM test_table WHERE some_value > 46") - .await - .unwrap(); + .await?; assert_eq!( format!("{}", plan.display_indent()), r#"Dml: op=[Delete] table=[test_table] @@ -89,15 +88,13 @@ async fn test_delete_statement( // context .plan_query("DELETE FROM test_table WHERE some_value > 46") - .await - .unwrap(); + .await?; // Verify results let plan = context .plan_query("SELECT some_value FROM test_table ORDER BY some_value") - .await - .unwrap(); - let results = context.collect(plan).await.unwrap(); + .await?; + let results = context.collect(plan).await?; let expected = vec![ "+------------+", @@ -117,55 +114,52 @@ async fn test_delete_statement( assert_batches_eq!(expected, &results); // Ensure partitions 2 and 3 have been fused into a new partition, and record it. - table.load().await.unwrap(); - match table.get_files_iter().collect_vec().as_slice() { + table.load().await?; + let partition_5 = match table.snapshot()?.file_actions()?.as_slice() { [f_1, f_2, f_3] - if f_1 == &partition_1 + if f_3 == &partition_1 && f_2 == &partition_4 - && !all_partitions.contains(f_3) => + && !all_partitions.contains(f_1) => { - all_partitions.push(f_3.clone()) + f_1.clone() } _ => panic!("Expected exactly 2 inherited and 1 new partition"), }; - let partition_5 = all_partitions.last().unwrap().clone(); + all_partitions.push(partition_5.clone()); // // Add another partition for a new table_version and record the new partition // context .plan_query("INSERT INTO test_table (some_value) VALUES (48), (49), (50)") - .await - .unwrap(); + .await?; // Expect too see a new (6th) partition - table.load().await.unwrap(); - match table.get_files_iter().collect_vec().as_slice() { + table.load().await?; + let partition_6 = match table.snapshot()?.file_actions()?.as_slice() { [f_1, f_2, f_3, f_4] - if f_1 == &partition_1 - && f_2 == &partition_4 - && f_3 == &partition_5 - && !all_partitions.contains(f_4) => + if f_4 == &partition_1 + && f_3 == &partition_4 + && f_2 == &partition_5 + && !all_partitions.contains(f_1) => { - all_partitions.push(f_4.clone()) + f_1.clone() } _ => panic!("Expected exactly 3 inherited and 1 new partition"), }; - let _partition_6 = all_partitions.last().unwrap().clone(); + all_partitions.push(partition_6.clone()); // // Execute DELETE not affecting only partition with id 4, while trimming/combining the rest // context .plan_query("DELETE FROM test_table WHERE some_value IN (43, 45, 49)") - .await - .unwrap(); + .await?; let plan = context .plan_query("SELECT some_value FROM test_table ORDER BY some_value") - .await - .unwrap(); - let results = context.collect(plan).await.unwrap(); + .await?; + let results = context.collect(plan).await?; let expected = vec![ "+------------+", @@ -184,14 +178,12 @@ async fn test_delete_statement( ]; assert_batches_eq!(expected, &results); - table.load().await.unwrap(); - match table.get_files_iter().collect_vec().as_slice() { - [f_1, f_2] if f_1 == &partition_4 && !all_partitions.contains(f_2) => { - all_partitions.push(f_2.clone()) - } + table.load().await?; + let partition_7 = match table.snapshot()?.file_actions()?.as_slice() { + [f_1, f_2] if f_2 == &partition_4 && !all_partitions.contains(f_1) => f_1.clone(), _ => panic!("Expected exactly 1 inherited and 1 new partition"), }; - let partition_7 = all_partitions.last().unwrap().clone(); + all_partitions.push(partition_7.clone()); // // Execute a no-op DELETE, leaving the new table version the same as the prior one @@ -199,20 +191,18 @@ async fn test_delete_statement( context .plan_query("DELETE FROM test_table WHERE some_value < 35") - .await - .unwrap(); + .await?; let plan = context .plan_query("SELECT some_value FROM test_table ORDER BY some_value") - .await - .unwrap(); - let results = context.collect(plan).await.unwrap(); + .await?; + let results = context.collect(plan).await?; assert_batches_eq!(expected, &results); // Both partitions are inherited from the previous version table.load().await.unwrap(); - match table.get_files_iter().collect_vec().as_slice() { - [f_1, f_2] if f_1 == &partition_4 && f_2 == &partition_7 => {} + match table.snapshot()?.file_actions()?.as_slice() { + [f_1, f_2] if f_2 == &partition_4 && f_1 == &partition_7 => {} _ => panic!("Expected same partitions as before"), }; @@ -221,14 +211,12 @@ async fn test_delete_statement( // context .plan_query("DELETE FROM test_table WHERE some_value < 41 OR some_value > 46") - .await - .unwrap(); + .await?; let plan = context .plan_query("SELECT some_value FROM test_table ORDER BY some_value") - .await - .unwrap(); - let results = context.collect(plan).await.unwrap(); + .await?; + let results = context.collect(plan).await?; let expected = [ "+------------+", @@ -246,7 +234,7 @@ async fn test_delete_statement( // Still only one partition file, but different from before table.load().await.unwrap(); - match table.get_files_iter().collect_vec()[..] { + match table.snapshot()?.file_actions()?[..] { [ref f_1] if !all_partitions.contains(f_1) => all_partitions.push(f_1.clone()), _ => panic!("Expected exactly 1 new partition different from the previous one"), }; @@ -254,19 +242,20 @@ async fn test_delete_statement( // // Execute blank DELETE, without qualifiers // - context.plan_query("DELETE FROM test_table").await.unwrap(); + context.plan_query("DELETE FROM test_table").await?; // Verify results let plan = context .plan_query("SELECT some_value FROM test_table") - .await - .unwrap(); - let results = context.collect(plan).await.unwrap(); + .await?; + let results = context.collect(plan).await?; assert!(results.is_empty()); table.load().await.unwrap(); - assert!(table.get_files_iter().collect_vec().is_empty()) + assert!(table.snapshot()?.file_actions()?.is_empty()); + + Ok(()) } #[tokio::test] @@ -348,7 +337,7 @@ async fn test_update_statement( ObjectStoreType::S3(Some("/path/to/folder")) )] object_store_type: ObjectStoreType, -) { +) -> Result<()> { let (context, _) = make_context_with_pg(object_store_type).await; create_table_and_some_partitions(&context, "test_table", None).await; @@ -356,12 +345,12 @@ async fn test_update_statement( // // Ensure we have 4 partitions, and record their file names // - let mut table = context.try_get_delta_table("test_table").await.unwrap(); - table.load().await.unwrap(); - let mut all_partitions = table.get_files_iter().collect_vec().clone(); + let mut table = context.try_get_delta_table("test_table").await?; + table.load().await?; + let mut all_partitions = table.snapshot()?.file_actions()?.clone(); assert_eq!(all_partitions.len(), 4); - let partition_2 = all_partitions[1].clone(); - let partition_3 = all_partitions[2].clone(); + let partition_2 = all_partitions[2].clone(); + let partition_3 = all_partitions[1].clone(); // // Execute UPDATE with a selection, affecting only partitions 1 and 4 @@ -370,7 +359,7 @@ async fn test_update_statement( SET some_time = '2022-01-01 21:21:21Z', some_int_value = 5555, some_value = some_value - 10 WHERE some_value IN (41, 42, 43)"; - let plan = context.create_logical_plan(query).await.unwrap(); + let plan = context.create_logical_plan(query).await?; // Check the UPDATE query plan to make sure IN (41, 42, 43) (int) get cast to a float value assert_eq!( @@ -382,13 +371,12 @@ async fn test_update_statement( ); // Now execute and check the results - context.plan_query(query).await.unwrap(); + context.plan_query(query).await?; let plan = context - .plan_query("SELECT * FROM test_table ORDER BY some_value") - .await - .unwrap(); - let results = context.collect(plan).await.unwrap(); + .plan_query("SELECT * FROM test_table ORDER BY some_value, some_other_value") + .await?; + let results = context.collect(plan).await?; let expected = vec![ "+---------------------+------------+------------------+-----------------+----------------+", "| some_time | some_value | some_other_value | some_bool_value | some_int_value |", @@ -410,39 +398,37 @@ async fn test_update_statement( assert_batches_eq!(expected, &results); // Ensure partitions 1 and 4 have been fused into a new partition, and record it. - table.load().await.unwrap(); - match table.get_files_iter().collect_vec().as_slice() { + table.load().await?; + let partition_5 = match table.snapshot()?.file_actions()?.as_slice() { [f_1, f_2, f_3] - if f_1 == &partition_2 + if f_3 == &partition_2 && f_2 == &partition_3 - && !all_partitions.contains(f_3) => + && !all_partitions.contains(f_1) => { - all_partitions.push(f_3.clone()) + f_1.clone() } _ => panic!("Expected exactly 2 inherited and 1 new partition"), }; - let partition_5 = all_partitions.last().unwrap().clone(); + all_partitions.push(partition_5.clone()); // // Execute UPDATE that doesn't change anything // context .plan_query("UPDATE test_table SET some_bool_value = TRUE WHERE some_value = 200") - .await - .unwrap(); + .await?; let plan = context - .plan_query("SELECT * FROM test_table ORDER BY some_value") - .await - .unwrap(); - let results = context.collect(plan).await.unwrap(); + .plan_query("SELECT * FROM test_table ORDER BY some_value, some_other_value") + .await?; + let results = context.collect(plan).await?; assert_batches_eq!(expected, &results); // Ensure partitions from before are still there - table.load().await.unwrap(); - match table.get_files_iter().collect_vec().as_slice() { + table.load().await?; + match table.snapshot()?.file_actions()?.as_slice() { [f_1, f_2, f_3] - if f_1 == &partition_2 && f_2 == &partition_3 && f_3 == &partition_5 => {} + if f_3 == &partition_2 && f_2 == &partition_3 && f_1 == &partition_5 => {} _ => panic!("Expected 3 inherited partitions"), }; @@ -467,41 +453,39 @@ async fn test_update_statement( "UPDATE test_table SET some_bool_value = FALSE, some_bool_value = (some_int_value = 5555), some_value = 42, \ some_other_value = CASE WHEN some_int_value = 5555 THEN 5.555 WHEN some_int_value = 3333 THEN 3.333 ELSE 0 END" ) - .await - .unwrap(); + .await?; // Verify results - let plan = context - .plan_query("SELECT * FROM test_table") - .await - .unwrap(); - let results = context.collect(plan).await.unwrap(); + let plan = context.plan_query("SELECT * FROM test_table").await?; + let results = context.collect(plan).await?; let expected = vec![ "+---------------------+------------+------------------+-----------------+----------------+", "| some_time | some_value | some_other_value | some_bool_value | some_int_value |", "+---------------------+------------+------------------+-----------------+----------------+", + "| 2022-01-01T21:21:21 | 42.0 | 5.5550000000 | true | 5555 |", + "| 2022-01-01T21:21:21 | 42.0 | 5.5550000000 | true | 5555 |", "| | 42.0 | 0.0000000000 | | |", + "| 2022-01-01T21:21:21 | 42.0 | 5.5550000000 | true | 5555 |", + "| 2022-01-01T21:21:21 | 42.0 | 5.5550000000 | true | 5555 |", + "| 2022-01-01T20:03:03 | 42.0 | 3.3330000000 | false | 3333 |", "| | 42.0 | 0.0000000000 | | |", "| | 42.0 | 0.0000000000 | | |", "| | 42.0 | 0.0000000000 | | |", "| | 42.0 | 0.0000000000 | | |", "| | 42.0 | 0.0000000000 | | |", - "| 2022-01-01T21:21:21 | 42.0 | 5.5550000000 | true | 5555 |", - "| 2022-01-01T21:21:21 | 42.0 | 5.5550000000 | true | 5555 |", - "| 2022-01-01T20:03:03 | 42.0 | 3.3330000000 | false | 3333 |", - "| 2022-01-01T21:21:21 | 42.0 | 5.5550000000 | true | 5555 |", - "| 2022-01-01T21:21:21 | 42.0 | 5.5550000000 | true | 5555 |", "| | 42.0 | 0.0000000000 | | |", "+---------------------+------------+------------------+-----------------+----------------+", ]; assert_batches_eq!(expected, &results); - table.load().await.unwrap(); - match table.get_files_iter().collect_vec().as_slice() { + table.load().await?; + match table.snapshot()?.file_actions()?.as_slice() { [f_1] if !all_partitions.contains(f_1) => {} _ => panic!("Expected only 1 new partition"), }; + + Ok(()) } #[tokio::test] diff --git a/tests/statements/mod.rs b/tests/statements/mod.rs index 159d8a33..668b94c4 100644 --- a/tests/statements/mod.rs +++ b/tests/statements/mod.rs @@ -1,4 +1,3 @@ -use itertools::Itertools; use std::collections::HashMap; use std::env; use std::time::Duration; @@ -9,7 +8,7 @@ use datafusion::assert_batches_eq; use datafusion::datasource::TableProvider; use datafusion_common::{assert_contains, Result}; use itertools::sorted; -use object_store::path::Path; + use seafowl::catalog::{DEFAULT_DB, DEFAULT_SCHEMA}; #[cfg(feature = "object-store-gcs")] use serde_json::json; diff --git a/tests/statements/time_travel.rs b/tests/statements/time_travel.rs index d5bf305d..fdccb679 100644 --- a/tests/statements/time_travel.rs +++ b/tests/statements/time_travel.rs @@ -295,7 +295,7 @@ async fn test_write_time_travel() { .unwrap(); let plan = context - .plan_query("SELECT some_value, some_other_value FROM diff_table") + .plan_query("SELECT some_value, some_other_value FROM diff_table ORDER BY some_other_value, some_value") .await .unwrap(); let results = context.collect(plan).await.unwrap(); @@ -304,15 +304,15 @@ async fn test_write_time_travel() { "+------------+------------------+", "| some_value | some_other_value |", "+------------+------------------+", + "| 42.0 | 1.0000000000 |", + "| 43.0 | 1.0000000000 |", + "| 44.0 | 1.0000000000 |", "| 46.0 | 3.0000000000 |", "| 47.0 | 3.0000000000 |", "| 48.0 | 3.0000000000 |", "| 40.0 | 4.0000000000 |", "| 41.0 | 4.0000000000 |", "| 42.0 | 4.0000000000 |", - "| 42.0 | 1.0000000000 |", - "| 43.0 | 1.0000000000 |", - "| 44.0 | 1.0000000000 |", "+------------+------------------+", ]; assert_batches_eq!(expected, &results); diff --git a/tests/statements/vacuum.rs b/tests/statements/vacuum.rs index 2ca721cd..27055de0 100644 --- a/tests/statements/vacuum.rs +++ b/tests/statements/vacuum.rs @@ -46,14 +46,14 @@ async fn test_vacuum_table() -> Result<()> { let mut table_2 = context.try_get_delta_table("table_2").await?; table_1.load_version(1).await?; - let table_1_v1_file = table_1.get_files_iter().collect_vec()[0].clone(); + let table_1_v1_file = table_1.snapshot()?.file_actions()?[0].clone(); table_1.load_version(2).await?; - let table_1_v2_file = table_1.get_files_iter().collect_vec()[0].clone(); + let table_1_v2_file = table_1.snapshot()?.file_actions()?[0].clone(); table_2.load().await?; - let table_2_v1_file = table_2.get_files_iter().collect_vec()[0].clone(); + let table_2_v1_file = table_2.snapshot()?.file_actions()?[0].clone(); table_2.load_version(2).await?; - let table_2_v2_file = table_2.get_files_iter().collect_vec()[1].clone(); + let table_2_v2_file = table_2.snapshot()?.file_actions()?[1].clone(); // Check initial directory state testutils::assert_uploaded_objects( @@ -62,11 +62,11 @@ async fn test_vacuum_table() -> Result<()> { .get_log_store(&table_1_uuid.to_string()) .object_store(), vec![ - Path::from("_delta_log/00000000000000000000.json"), - Path::from("_delta_log/00000000000000000001.json"), - Path::from("_delta_log/00000000000000000002.json"), - table_1_v1_file, - table_1_v2_file.clone(), + String::from("_delta_log/00000000000000000000.json"), + String::from("_delta_log/00000000000000000001.json"), + String::from("_delta_log/00000000000000000002.json"), + table_1_v1_file.path, + table_1_v2_file.clone().path, ], ) .await; @@ -102,12 +102,12 @@ async fn test_vacuum_table() -> Result<()> { .get_log_store(&table_1_uuid.to_string()) .object_store(), vec![ - Path::from("_delta_log/00000000000000000000.json"), - Path::from("_delta_log/00000000000000000001.json"), - Path::from("_delta_log/00000000000000000002.json"), - Path::from("_delta_log/00000000000000000003.json"), - Path::from("_delta_log/00000000000000000004.json"), - table_1_v2_file, + String::from("_delta_log/00000000000000000000.json"), + String::from("_delta_log/00000000000000000001.json"), + String::from("_delta_log/00000000000000000002.json"), + String::from("_delta_log/00000000000000000003.json"), + String::from("_delta_log/00000000000000000004.json"), + table_1_v2_file.path, ], ) .await; @@ -151,11 +151,11 @@ async fn test_vacuum_table() -> Result<()> { .get_log_store(&table_2_uuid.to_string()) .object_store(), vec![ - Path::from("_delta_log/00000000000000000000.json"), - Path::from("_delta_log/00000000000000000001.json"), - Path::from("_delta_log/00000000000000000002.json"), - table_2_v1_file, - table_2_v2_file, + String::from("_delta_log/00000000000000000000.json"), + String::from("_delta_log/00000000000000000001.json"), + String::from("_delta_log/00000000000000000002.json"), + table_2_v1_file.path, + table_2_v2_file.path, ], ) .await;