diff --git a/Cargo.lock b/Cargo.lock index d49c4445f..27004e950 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -614,16 +614,15 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.24.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8478a5c29ead3f3be14aff8a202ad965cf7da6856860041bfca271becf8ba48b" +checksum = "923ded50f602b3007e5e63e3f094c479d9c8a9b42d7f4034e4afe456aa48bfd2" dependencies = [ "bindgen", "cc", "cmake", "dunce", "fs_extra", - "libc", "paste", ] @@ -3040,6 +3039,16 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "io-uring" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd1e1a01cfb924fd8c5c43b6827965db394f5a3a16c599ce03452266e1cf984c" +dependencies = [ + "bitflags 1.3.2", + "libc", +] + [[package]] name = "ipconfig" version = "0.3.2" @@ -5367,7 +5376,7 @@ dependencies = [ [[package]] name = "spider" -version = "2.22.19" +version = "2.23.1" dependencies = [ "ahash", "aho-corasick", @@ -5422,6 +5431,7 @@ dependencies = [ "tikv-jemallocator", "tokio", "tokio-stream", + "tokio-uring", "tower 0.5.2", "tracing", "ua_generator", @@ -5430,7 +5440,7 @@ dependencies = [ [[package]] name = "spider_chrome" -version = "2.22.19" +version = "2.23.1" dependencies = [ "adblock", "aho-corasick", @@ -5520,7 +5530,7 @@ dependencies = [ [[package]] name = "spider_cli" -version = "2.22.19" +version = "2.23.1" dependencies = [ "clap", "env_logger", @@ -5563,7 +5573,7 @@ dependencies = [ [[package]] name = "spider_transformations" -version = "2.22.19" +version = "2.23.1" dependencies = [ "aho-corasick", "fast_html2md", @@ -5586,7 +5596,7 @@ dependencies = [ [[package]] name = "spider_utils" -version = "2.22.19" +version = "2.23.1" dependencies = [ "indexmap 1.9.3", "serde", @@ -5599,7 +5609,7 @@ dependencies = [ [[package]] name = "spider_worker" -version = "2.22.19" +version = "2.23.1" dependencies = [ "env_logger", "lazy_static", @@ -6491,6 +6501,20 @@ dependencies = [ "tungstenite 0.24.0", ] +[[package]] +name = "tokio-uring" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d5e02bb137e030b3a547c65a3bd2f1836d66a97369fdcc69034002b10e155ef" +dependencies = [ + "io-uring", + "libc", + "scoped-tls", + "slab", + "socket2 0.4.10", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.13" diff --git a/spider/Cargo.toml b/spider/Cargo.toml index 289260e61..cae955793 100644 --- a/spider/Cargo.toml +++ b/spider/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spider" -version = "2.22.19" +version = "2.23.1" authors = [ "j-mendez " ] @@ -81,6 +81,7 @@ features = [ [target.'cfg(target_os = "linux")'.dependencies] libc = "0.2" +tokio-uring = { version = "0.4", optional = true } [target.'cfg(all(not(windows), not(target_os = "android"), not(target_env = "musl")))'.dependencies] tikv-jemallocator = { version = "0.6", optional = true } @@ -115,7 +116,7 @@ reqwest = { version = "0.12", features = [ ] } [features] -default = ["sync", "reqwest_native_tls_native_roots", "disk_native_tls", "cookies", "ua_generator", "encoding", "string_interner_buffer_backend", "balance"] +default = ["sync", "reqwest_native_tls_native_roots", "disk_native_tls", "cookies", "ua_generator", "encoding", "string_interner_buffer_backend", "balance", "io_uring"] disk = ["dep:sqlx", "dep:aho-corasick"] disk_native_tls = ["disk", "sqlx/runtime-tokio-native-tls"] disk_aws = ["disk", "sqlx/tls-rustls-aws-lc-rs"] @@ -128,6 +129,7 @@ jemalloc = ["tikv-jemallocator"] decentralized = ["serde", "flexbuffers"] control = [] time = [] +io_uring = ["dep:tokio-uring"] sync = ["tokio/sync"] flexbuffers = ["dep:flexbuffers"] serde = ["dep:serde", "hashbrown/serde", "string-interner/serde", "dep:serde_regex", "smallvec/serde"] diff --git a/spider/src/utils/connect.rs b/spider/src/utils/connect.rs index d6c433af7..4c09c5588 100644 --- a/spider/src/utils/connect.rs +++ b/spider/src/utils/connect.rs @@ -25,6 +25,29 @@ pub(crate) fn background_connect_threading() -> bool { } /// Init a background thread for request connect handling. +#[cfg(all(target_os = "linux", feature = "io_uring"))] +pub(crate) fn init_background_runtime() { + let _ = CONNECT_THREAD_POOL.set({ + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); + let builder = std::thread::Builder::new(); + + if let Err(_) = builder.spawn(move || { + tokio_uring::builder().start(async { + while let Some(work) = rx.recv().await { + tokio_uring::spawn(work); + } + }) + }) { + let _ = tx.downgrade(); + BACKGROUND_THREAD_CONNECT_ENABLED.store(false, std::sync::atomic::Ordering::Relaxed); + }; + + tx + }); +} + +/// Init a background thread for request connect handling. +#[cfg(any(not(target_os = "linux"), not(feature = "io_uring")))] pub(crate) fn init_background_runtime() { let _ = CONNECT_THREAD_POOL.set({ let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); diff --git a/spider_chrome/Cargo.toml b/spider_chrome/Cargo.toml index 718401048..6f8aea4c1 100644 --- a/spider_chrome/Cargo.toml +++ b/spider_chrome/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spider_chrome" -version = "2.22.19" +version = "2.23.1" rust-version = "1.70" authors = [ "j-mendez " diff --git a/spider_cli/Cargo.toml b/spider_cli/Cargo.toml index 7e1474765..f6074fbc4 100644 --- a/spider_cli/Cargo.toml +++ b/spider_cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spider_cli" -version = "2.22.19" +version = "2.23.1" authors = [ "j-mendez " ] diff --git a/spider_transformations/Cargo.toml b/spider_transformations/Cargo.toml index fc47d3cd9..26867f834 100644 --- a/spider_transformations/Cargo.toml +++ b/spider_transformations/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spider_transformations" -version = "2.22.19" +version = "2.23.1" authors = [ "j-mendez " ] diff --git a/spider_utils/Cargo.toml b/spider_utils/Cargo.toml index 98ac02ca9..800d666d8 100644 --- a/spider_utils/Cargo.toml +++ b/spider_utils/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spider_utils" -version = "2.22.19" +version = "2.23.1" authors = [ "j-mendez " ] diff --git a/spider_worker/Cargo.toml b/spider_worker/Cargo.toml index fe626182c..72842d16e 100644 --- a/spider_worker/Cargo.toml +++ b/spider_worker/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spider_worker" -version = "2.22.19" +version = "2.23.1" authors = [ "j-mendez " ]