BITMAKER-723: add kafka-producer

.gitignore

@@ -5,3 +5,5 @@ dist
 .eggs/
 .tox
 /.coverage
+.idea/
+venv/

bm_scrapy/extensions.py

@@ -1,7 +1,24 @@
+import os
+from json import dumps
 from scrapy import signals
 from scrapy.exporters import PythonItemExporter
 
 from bm_scrapy.writer import pipe_writer
+from kafka import KafkaProducer
+
+
+def connect_kafka_producer():
+    _producer = None
+    bootstrap_server = [
+        '{}:{}'.format(os.getenv('KAFKA_ADVERTISED_HOST_NAME', '127.0.0.1'),
+                       os.getenv('KAFKA_ADVERTISED_PORT', '9092'))
+    ]
+    _producer = KafkaProducer(bootstrap_servers=bootstrap_server, api_version=(0, 10),
+                              value_serializer=lambda x: dumps(x).encode('utf-8'))
+    return _producer
+
+
+producer = connect_kafka_producer()
 
 
 class ItemStorageExtension:
@@ -19,6 +36,8 @@ def from_crawler(cls, crawler):
 
     def item_scraped(self, item):
         item = self.exporter.export_item(item)
+        producer.send('spider-items', value=dict(item))
+        producer.flush()
         self.writer.write_item(item)
 
     def spider_closed(self, spider, reason):

setup.py

@@ -7,6 +7,7 @@
     packages=find_packages(),
     install_requires=[
         'Scrapy>=1.0',
+        'kafka-python'
     ],
     entry_points={
         'console_scripts': [