chore(website): fix page empty status

spider-rs · Feb 16, 2025 · a21a1c6 · a21a1c6
1 parent b423812
commit a21a1c6
Show file tree

Hide file tree

Showing 9 changed files with 18 additions and 15 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/spider/Cargo.toml b/spider/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider"
-version = "2.28.1"
+version = "2.28.2"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider/src/website.rs b/spider/src/website.rs
@@ -2425,10 +2425,13 @@ impl Website {
             let links = if !page_links.is_empty() {
                 page_links
             } else {
-                self.status = CrawlStatus::Empty;
                 Default::default()
             };
 
+            if page.is_empty() {
+                self.status = CrawlStatus::Empty;
+            }
+
             self.initial_status_code = page.status_code;
 
             if page.status_code == reqwest::StatusCode::FORBIDDEN {

diff --git a/spider_chrome/Cargo.toml b/spider_chrome/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_chrome"
-version = "2.28.1"
+version = "2.28.2"
 rust-version = "1.70"
 authors = [
     "j-mendez <[email protected]>"

diff --git a/spider_cli/Cargo.toml b/spider_cli/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_cli"
-version = "2.28.1"
+version = "2.28.2"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider_firewall/Cargo.toml b/spider_firewall/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_firewall"
-version = "2.28.1"
+version = "2.28.2"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider_transformations/Cargo.toml b/spider_transformations/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_transformations"
-version = "2.28.1"
+version = "2.28.2"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider_utils/Cargo.toml b/spider_utils/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_utils"
-version = "2.28.1"
+version = "2.28.2"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider_worker/Cargo.toml b/spider_worker/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_worker"
-version = "2.28.1"
+version = "2.28.2"
 authors = [
     "j-mendez <[email protected]>"
 ]