Update scrape.php

Signed-off-by: Nathanael Esayeas <[email protected]>
BMHVillage · Jan 3, 2025 · cb99586 · cb99586
1 parent 37a4eb5
commit cb99586
Showing 1 changed file with 11 additions and 2 deletions.
diff --git a/scrape.php b/scrape.php
@@ -90,8 +90,15 @@
     do {
         $response = $http->get($api, [
             'category' => 'african-american',
-            'page' => $page,
+            'page' => --$page,
         ]);
+
+        if ($response->failed()) {
+            fwrite(STDERR, 'Failed to fetch data from the API' . PHP_EOL);
+            fwrite(STDERR, $response->body() . PHP_EOL);
+            exit(1);
+        }
+
         $crawler = new Crawler($response->body());
         $crawler->filter('div.results-row')
             ->each(static function (Crawler $crawler) {
@@ -147,5 +154,7 @@
                     'hash'=> $hash,
                 ], $payload);
             });
-    } while (--$page);
+
+            sleep(3);
+    } while ($page > 1);
 })(\implode(DIRECTORY_SEPARATOR,[__DIR__ , 'vendor','autoload.php']));