perf(page): add max limit streaming chunk size

README.md

@@ -22,6 +22,7 @@ A web crawler and scraper, building blocks for data curation workloads.
 - Subscriptions
 - Smart Mode
 - Anti-Bot mitigation
+- Disk persistence
 - Privacy and Efficiency through Ad, Analytics, and Custom Tiered Network Blocking
 - Blacklisting, Whitelisting, and Budgeting Depth
 - Dynamic AI Prompt Scripting Headless with Step Caching

spider/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider"
-version = "2.22.14"
+version = "2.22.16"
 authors = [
     "j-mendez <[email protected]>"
 ]

spider/src/page.rs

@@ -94,8 +94,8 @@ lazy_static! {
 
     /// The chunk size for the rewriter. Can be adjusted using the env var "SPIDER_STREAMING_CHUNK_SIZE".
     pub(crate) static ref STREAMING_CHUNK_SIZE: usize = {
-        let default_streaming_chunk_size: usize = 8192 * num_cpus::get_physical();
-        let min_streaming_chunk_size: usize = default_streaming_chunk_size / 4;
+        let default_streaming_chunk_size: usize = 8192 * num_cpus::get_physical().min(64);
+        let min_streaming_chunk_size: usize = default_streaming_chunk_size * 2 / 3;
 
         std::env::var("SPIDER_STREAMING_CHUNK_SIZE")
             .ok()
@@ -405,6 +405,7 @@ pub fn validate_empty(content: &Option<Box<Bytes>>, is_success: bool) -> bool {
 }
 
 /// Extract a specific type of error from a chain of errors.
+#[cfg(not(feature = "decentralized"))]
 fn extract_specific_error<'a, T: std::error::Error + 'static>(
     error: &'a (dyn std::error::Error + 'static),
 ) -> Option<&'a T> {
@@ -419,6 +420,7 @@ fn extract_specific_error<'a, T: std::error::Error + 'static>(
 }
 
 /// Determine if the response is goaway and should retry.
+#[cfg(not(feature = "decentralized"))]
 fn should_attempt_retry(error: &(dyn std::error::Error + 'static)) -> bool {
     if let Some(e) = extract_specific_error::<h2::Error>(error) {
         if e.is_go_away() && e.is_remote() && e.reason() == Some(h2::Reason::NO_ERROR) {
@@ -2116,16 +2118,6 @@ impl Page {
     fn abs_path(&self, href: &str) -> Option<Url> {
         self.base.as_ref().map(|b| convert_abs_path(b, href))
     }
-
-    /// Convert a URL to its absolute path without any fragments or params. [unused in the worker atm by default all is returned]
-    #[inline(never)]
-    #[cfg(feature = "decentralized")]
-    fn abs_path(&self, href: &str) -> Option<Url> {
-        match Url::parse(&href) {
-            Ok(u) => Some(convert_abs_path(&u, href)),
-            _ => None,
-        }
-    }
 }
 
 /// Get the content with proper encoding. Pass in a proper encoding label like SHIFT_JIS.

spider_chrome/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_chrome"
-version = "2.22.14"
+version = "2.22.16"
 rust-version = "1.70"
 authors = [
     "j-mendez <[email protected]>"

spider_cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_cli"
-version = "2.22.14"
+version = "2.22.16"
 authors = [
     "j-mendez <[email protected]>"
 ]

spider_transformations/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_transformations"
-version = "2.22.14"
+version = "2.22.16"
 authors = [
     "j-mendez <[email protected]>"
 ]

spider_utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_utils"
-version = "2.22.14"
+version = "2.22.16"
 authors = [
     "j-mendez <[email protected]>"
 ]

spider_worker/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_worker"
-version = "2.22.14"
+version = "2.22.16"
 authors = [
     "j-mendez <[email protected]>"
 ]