Fix null response in PuppeteerCrawler

apify · Oct 3, 2020 · ec6d66c · ec6d66c
1 parent 6527b30
commit ec6d66c
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 2 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,7 @@
+0.21.7 / 2020/09/03
+====================
+- Fix an error in `PuppeteerCrawler` caused by `page.goto()` randomly returning `null`.
+
 0.21.6 / 2020/09/02
 ====================
 It appears that `CheerioCrawler` was correctly retiring sessions on timeouts

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
     "name": "apify",
-    "version": "0.21.6",
+    "version": "0.21.7",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": "^10.17.0 || ^12.3.0"

diff --git a/src/crawlers/puppeteer_crawler.js b/src/crawlers/puppeteer_crawler.js
@@ -382,7 +382,10 @@ class PuppeteerCrawler {
                 }
             }
 
-            if (this.useSessionPool) this._throwOnBlockedRequest(session, response.status());
+            if (this.useSessionPool) {
+                const statusCode = response && response.status();
+                this._throwOnBlockedRequest(session, statusCode);
+            }
 
             await this.puppeteerPool.serveLiveViewSnapshot(page);
             request.loadedUrl = page.url();