chore(website): fix subscribe guard handling queues

spider-rs · Jan 26, 2025 · d377faf · d377faf
1 parent 7a008ad
commit d377faf
Show file tree

Hide file tree

Showing 8 changed files with 15 additions and 17 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/spider/Cargo.toml b/spider/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider"
-version = "2.27.2"
+version = "2.27.3"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider/src/website.rs b/spider/src/website.rs
@@ -2857,14 +2857,13 @@ impl Website {
                             }
                         }
 
+                        self.subscription_guard();
                         self.dequeue(&mut q, &mut links, &mut exceeded_budget).await;
 
                         if links.is_empty() && set.is_empty() {
                             break;
                         }
                     }
-
-                    self.subscription_guard();
                 }
             }
             _ => log::info!("{} - {}", self.url, INVALID_URL),
@@ -3165,15 +3164,14 @@ impl Website {
                                         }
                                     }
 
+                                    self.subscription_guard();
                                     self.dequeue(&mut q, &mut links, &mut exceeded_budget).await;
 
                                     if links.is_empty() && set.is_empty() {
                                         break;
                                     }
                                 }
 
-                                self.subscription_guard();
-
                                 crate::features::chrome::close_browser(
                                     browser_handle,
                                     &shared.5,
@@ -3566,14 +3564,14 @@ impl Website {
                                 }
                             }
 
+                            self.subscription_guard();
                             self.dequeue(&mut q, &mut links, &mut exceeded_budget).await;
 
                             if links.is_empty() && set.is_empty() {
                                 break;
                             }
                         }
 
-                        self.subscription_guard();
                         crate::features::chrome::close_browser(
                             browser_handle,
                             &shared.4,

diff --git a/spider_chrome/Cargo.toml b/spider_chrome/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_chrome"
-version = "2.27.2"
+version = "2.27.3"
 rust-version = "1.70"
 authors = [
     "j-mendez <[email protected]>"

diff --git a/spider_cli/Cargo.toml b/spider_cli/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_cli"
-version = "2.27.2"
+version = "2.27.3"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider_transformations/Cargo.toml b/spider_transformations/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_transformations"
-version = "2.27.2"
+version = "2.27.3"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider_utils/Cargo.toml b/spider_utils/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_utils"
-version = "2.27.2"
+version = "2.27.3"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider_worker/Cargo.toml b/spider_worker/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_worker"
-version = "2.27.2"
+version = "2.27.3"
 authors = [
     "j-mendez <[email protected]>"
 ]