Adds Vault support

projects/classifieds/classifieds/pipelines.py

@@ -9,6 +9,56 @@
 from itemadapter import ItemAdapter
 from scrapy.exceptions import DropItem
 import mysql.connector
+import pymongo
+
+
+class ClassifiedsRemoveDuplicatesPipeline:
+    def __init__(self):
+        self.titles_seen = set()
+
+    def process_item(self, item, spider):
+        adapter = ItemAdapter(item)
+        if adapter["title"] in self.titles_seen:
+            raise DropItem(f"Duplicate ad detected: {item}")
+        else:
+            self.titles_seen.add(adapter["title"])
+        return item
+
+
+class ClassifiedsRemoveNoPhonesPipeline:
+    def process_item(self, item, spider):
+        adapter = ItemAdapter(item)
+        if not adapter.get("landline") and not adapter.get("mobile"):
+            raise DropItem(f"Ad with no contact info detected: {item}")
+        else:
+            return item
+
+
+class MongoPipeline:
+
+    collection_name = 'classifieds'
+
+    def __init__(self, mongo_uri, mongo_db):
+        self.mongo_uri = mongo_uri
+        self.mongo_db = mongo_db
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(
+            mongo_uri=crawler.settings.get('MONGO_URI'),
+            mongo_db=crawler.settings.get('MONGO_DATABASE')
+        )
+
+    def open_spider(self, spider):
+        self.client = pymongo.MongoClient(self.mongo_uri)
+        self.db = self.client[self.mongo_db]
+
+    def close_spider(self, spider):
+        self.client.close()
+
+    def process_item(self, item, spider):
+        self.db[self.collection_name].insert_one(ItemAdapter(item).asdict())
+        return item
 
 
 class ClassifiedsRemoveDuplicatesPipeline:

projects/classifieds/classifieds/settings.py

@@ -6,6 +6,8 @@
 #     https://docs.scrapy.org/en/latest/topics/settings.html
 #     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
 #     https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+import os
+import hvac
 
 BOT_NAME = 'classifieds'
 
@@ -86,12 +88,22 @@
 #HTTPCACHE_DIR = 'httpcache'
 #HTTPCACHE_IGNORE_HTTP_CODES = []
 #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
-MYSQL_USERNAME="root"
-MYSQL_PASSWORD="admin"
-MYSQL_HOST="localhost"
-MYSQL_DATABASE="scrapy"
+client = hvac.Client(
+    url='http://127.0.0.1:8200',
+    token=os.environ["VAULT_ROOT_TOKEN"]
+)
+MYSQL_USERNAME = client.secrets.kv.read_secret_version(
+    path='mysql')['data']['data']['username']
+MYSQL_PASSWORD = client.secrets.kv.read_secret_version(
+    path='mysql')['data']['data']['password']
+MYSQL_HOST = "localhost"
+MYSQL_DATABASE = "scrapy"
+MONGO_URI = client.secrets.kv.read_secret_version(
+    path='mongodb')['data']['data']['uri']
+MONGO_DATABASE = "scrapy"
 ITEM_PIPELINES = {
     'classifieds.pipelines.ClassifiedsRemoveDuplicatesPipeline': 1,
     'classifieds.pipelines.ClassifiedsRemoveNoPhonesPipeline': 2,
-    'classifieds.pipelines.MySQLPipeline': 3
+    'classifieds.pipelines.MongoPipeline': 3,
+    'classifieds.pipelines.MySQLPipeline': 4
 }