chore(website): add connect layer concurrency limit

spider-rs · Dec 30, 2024 · e5e0638 · e5e0638
1 parent d5b69ce
commit e5e0638
Show file tree

Hide file tree

Showing 8 changed files with 29 additions and 12 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/spider/Cargo.toml b/spider/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider"
-version = "2.22.13"
+version = "2.22.14"
 authors = [
     "j-mendez <[email protected]>"
 ]
@@ -67,6 +67,7 @@ tracing = { version = "0.1", default-features = false, features = ["std"], optio
 sysinfo = { version = "0.33", default-features = false, features = ["system"], optional = true }
 sqlx = { version = "0.8", features = [ "runtime-tokio", "sqlite" ], optional = true }
 h2 = "0.4"
+tower = { version = "0.5", features = ["limit"]}
 
 [dependencies.spider_chrome]
 version = "2"

diff --git a/spider/src/website.rs b/spider/src/website.rs
@@ -1056,6 +1056,13 @@ impl Website {
             _ => client,
         };
 
+        let client = match self.configuration.concurrency_limit {
+            Some(limit) => {
+                client.connector_layer(tower::limit::concurrency::ConcurrencyLimitLayer::new(limit))
+            }
+            _ => client,
+        };
+
         self.configure_http_client_cookies(client)
     }
 
@@ -1114,6 +1121,12 @@ impl Website {
         };
 
         let client = self.configure_http_client_cookies(client);
+        let client = match self.configuration.concurrency_limit {
+            Some(limit) => {
+                client.connector_layer(tower::limit::concurrency::ConcurrencyLimitLayer::new(limit))
+            }
+            _ => client,
+        };
         let client = ClientBuilder::new(unsafe { client.build().unwrap_unchecked() });
 
         if self.configuration.cache {

diff --git a/spider_chrome/Cargo.toml b/spider_chrome/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_chrome"
-version = "2.22.13"
+version = "2.22.14"
 rust-version = "1.70"
 authors = [
     "j-mendez <[email protected]>"

diff --git a/spider_cli/Cargo.toml b/spider_cli/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_cli"
-version = "2.22.13"
+version = "2.22.14"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider_transformations/Cargo.toml b/spider_transformations/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_transformations"
-version = "2.22.13"
+version = "2.22.14"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider_utils/Cargo.toml b/spider_utils/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_utils"
-version = "2.22.13"
+version = "2.22.14"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider_worker/Cargo.toml b/spider_worker/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_worker"
-version = "2.22.13"
+version = "2.22.14"
 authors = [
     "j-mendez <[email protected]>"
 ]