more readable logs

medialab · Sep 26, 2023 · dcf3e2b · dcf3e2b
1 parent 2924dc8
commit dcf3e2b
Showing 1 changed file with 4 additions and 2 deletions.
diff --git a/hyphe_backend/crawler/hcicrawler/spiders/pages.py b/hyphe_backend/crawler/hcicrawler/spiders/pages.py
@@ -139,7 +139,9 @@ def from_crawler(cls, crawler, *args, **kwargs):
 
     def start_requests(self):
         self.log("Starting crawl task - jobid: %s" % self.crawler.settings['JOBID'], logging.INFO)
-        self.log("ARGUMENTS : "+str(self.args), logging.INFO)
+        self.log("ARGUMENTS: ", logging.INFO)
+        for key, value in self.args.items():
+            self.log(" - %s: %s" % (key, value), logging.INFO)
         if self.webarchives:
             self.log("Crawling on Web Archive %s using for prefix %s between %s and %s" % (self.webarchives["option"], self.archiveprefix, self.archivemindate, self.archivemaxdate))
         if self.proxy:
@@ -197,7 +199,7 @@ def spider_closed(self, spider, reason=""):
                         os.remove(fi)
 
     def handle_response(self, response):
-        self.log("RESPONSE %s (%s): %s" % (response.url, response.status, dict(response.headers)), logging.INFO)
+        self.log("RESPONSE (%s) %s: %s" % (response.status, response.url, dict(response.headers)), logging.INFO)
 
         if self.phantom:
             self.phantom.get(response.url)