Release (#267)

* refactoring deployment

* refactoring deployment

* fixing tests

* cleanup

* fixing malware domains url

* bump release

* more cleanup

* ignore

* bugfix

* re-factoring out bambenek feeds

* expanding gatherer send timeout

* re-introducing backend polling

* cleaning up deploy

* perf tweaks

* bumping smrt ver

* bumping indicator version, thanks @ventz !

* changing default alexa/cisco rules

* tweaking some rules

* tweaking sqlite

* tweaking sqlite

* making a comment

* bumping smrt dep

* version bump of smrt, updating smrt cache reqs

* removing python 3.4 test

* fixing spamhuas hunter confidence

* adding new sqlite datatypes

* bumping smrt req

* fixing tests

* fixing an index

* bumping version

.gitignore

@@ -1,4 +1,7 @@
 .DS_Store
+Vagrantfile_*
+Vagrantfile\.*
+Dockerfile\.*
 /*.yml
 *.retry
 coverage_html_report/

.travis.yml

@@ -7,7 +7,6 @@ cache:
 
 python:
     - 2.7
-    - 3.4
     - 3.5
 
 install:

Vagrantfile

@@ -5,22 +5,24 @@
 VAGRANTFILE_API_VERSION = "2"
 VAGRANTFILE_LOCAL = 'Vagrantfile.local'
 
+sdist=ENV['CIF_ANSIBLE_SDIST']
+es=ENV['CIF_ANSIBLE_ES']
+
 $script = <<SCRIPT
-echo 'yes' | sudo add-apt-repository 'ppa:fkrull/deadsnakes-python2.7'
-sudo apt-get update && sudo apt-get install -y python2.7 python-pip python-dev git libffi-dev libssl-dev sqlite3
-sudo pip install pip --upgrade
-sudo pip install 'setuptools>=11.3' 'ansible>=2.1' versioneer markupsafe
-cd /vagrant/deployment/ubuntu14
-sudo ansible-playbook -i "localhost," -c local vagrant.yml -vv
+export CIF_ANSIBLE_SDIST=#{sdist}
+export CIF_ANSIBLE_ES=#{es}
+
+cd /vagrant/deployment
+bash easybutton.sh
 SCRIPT
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.vm.provision "shell", inline: $script
-  config.vm.box = 'ubuntu/trusty64'
+  config.vm.box = 'ubuntu/xenial64'
 
-  config.vm.network :forwarded_port, guest: 5000, host: 5000
-  config.vm.network :forwarded_port, guest: 9200, host:9200
+  #config.vm.network :forwarded_port, guest: 5000, host: 5000
   config.vm.network :forwarded_port, guest: 443, host: 8443
+
   config.vm.provider :virtualbox do |vb|
     vb.customize ["modifyvm", :id, "--cpus", "2", "--ioapic", "on", "--memory", "1024" ]
   end

cif/gatherer/__init__.py

@@ -11,7 +11,7 @@
 from cif.constants import GATHERER_ADDR, GATHERER_SINK_ADDR
 from csirtg_indicator import Indicator
 
-SNDTIMEO = 15000
+SNDTIMEO = 30000
 LINGER = 0
 
 logger = logging.getLogger(__name__)

cif/gatherer/geo.py

@@ -56,7 +56,7 @@ def _resolve(self, indicator):
                 indicator.timezone = g.location.time_zone
 
     def process(self, indicator):
-        if indicator.itype != 'ipv4' and indicator.itype != 'ipv6':
+        if indicator.itype not in ['ipv4', 'ipv6']:
             return indicator
 
         if indicator.is_private():

cif/hunter/spamhaus_fqdn.py

@@ -78,11 +78,15 @@ def process(self, i, router):
                     r = None
 
                 if r:
+                    confidence = CONFIDENCE
+                    if ' legit ' in r['description']:
+                        confidence = 6
+
                     f = Indicator(**i.__dict__())
 
                     f.tags = [r['tags']]
                     f.description = r['description']
-                    f.confidence = CONFIDENCE
+                    f.confidence = confidence
                     f.provider = PROVIDER
                     f.reference_tlp = 'white'
                     f.reference = 'http://www.spamhaus.org/query/dbl?domain={}'.format(f.indicator)

cif/router.py

@@ -29,6 +29,7 @@
 ZMQ_SNDTIMEO = 5000
 ZMQ_RCVTIMEO = 5000
 FRONTEND_TIMEOUT = os.environ.get('CIF_FRONTEND_TIMEOUT', 100)
+BACKEND_TIMEOUT = os.environ.get('CIF_BACKEND_TIMEOUT', 10)
 
 HUNTER_TOKEN = os.environ.get('CIF_HUNTER_TOKEN', None)
 
@@ -92,6 +93,7 @@ def __init__(self, listen=ROUTER_ADDR, hunter=HUNTER_ADDR, store_type=STORE_DEFA
         self.count_start = time.time()
 
         self.poller = zmq.Poller()
+        self.poller_backend = zmq.Poller()
 
         self.terminate = False
 
@@ -133,8 +135,8 @@ def stop(self):
     def start(self):
         self.logger.debug('starting loop')
 
-        self.poller.register(self.hunter_sink_s, zmq.POLLIN)
-        self.poller.register(self.gatherer_sink_s, zmq.POLLIN)
+        self.poller_backend.register(self.hunter_sink_s, zmq.POLLIN)
+        self.poller_backend.register(self.gatherer_sink_s, zmq.POLLIN)
         self.poller.register(self.store_s, zmq.POLLIN)
         self.poller.register(self.frontend_s, zmq.POLLIN)
 
@@ -150,6 +152,8 @@ def start(self):
             if self.store_s in items and items[self.store_s] == zmq.POLLIN:
                 self.handle_message_store(self.store_s)
 
+            items = dict(self.poller_backend.poll(BACKEND_TIMEOUT))
+
             if self.gatherer_sink_s in items and items[self.gatherer_sink_s] == zmq.POLLIN:
                 self.handle_message_gatherer(self.gatherer_sink_s)
 

cif/store/__init__.py

@@ -215,7 +215,9 @@ def handle_indicators_create(self, token, data, id=None, client_id=None):
             r = self.store.indicators.upsert(t, data)
 
             if len(data) > 1:
-                logger.info('Upserting %d indicators.. took %0.2f seconds', len(data), time.time() - start_time)
+                n = len(data)
+                t = time.time() - start_time
+                logger.info('Upserting %d indicators.. took %0.2f seconds (%0.2f/sec)', n, t, (n/t))
 
             return r
 
@@ -252,7 +254,7 @@ def _log_search(self, t, data):
             lasttime=ts,
             reporttime=ts,
             group=t['groups'][0],
-            count=1
+            count=1,
         )
         self.store.indicators.upsert(t, s.__dict__())
 

cif/store/sqlite/__init__.py

@@ -20,6 +20,12 @@
 logger = logging.getLogger(__name__)
 TRACE = os.environ.get('CIF_STORE_SQLITE_TRACE')
 
+# http://stackoverflow.com/q/9671490/7205341
+SYNC = os.environ.get('CIF_STORE_SQLITE_SYNC', 'NORMAL')
+
+# https://www.sqlite.org/pragma.html#pragma_cache_size
+CACHE_SIZE = os.environ.get('CIF_STORE_SQLITE_CACHE_SIZE', 256000000)  # 256MB
+
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
 
@@ -36,6 +42,10 @@
 def set_sqlite_pragma(dbapi_connection, connection_record):
     cursor = dbapi_connection.cursor()
     cursor.execute("PRAGMA foreign_keys=ON")
+    cursor.execute("PRAGMA journal_mode = MEMORY")
+    cursor.execute("PRAGMA synchronous = {}".format(SYNC))
+    cursor.execute("PRAGMA temp_store = MEMORY")
+    cursor.execute("PRAGMA cache_size = {}".format(CACHE_SIZE))
     cursor.close()
 
 

cif/store/sqlite/fqdn.py

@@ -0,0 +1,37 @@
+from sqlalchemy.types import UserDefinedType
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy import types
+Base = declarative_base()
+
+
+class Fqdn(UserDefinedType):
+
+    impl = types.BINARY(16)
+
+    def __init__(self, version=4):
+        self.version = version
+
+    def get_col_spec(self, **kw):
+        return "FQDN"
+
+    def bind_processor(self, dialect):
+
+        DBAPIBinary = dialect.dbapi.Binary
+
+        def process(value):
+            if isinstance(value, str):
+                value = value.encode('utf-8')
+
+            return DBAPIBinary(value)
+
+        return process
+
+    def result_processor(self, dialect, coltype):
+        def process(value):
+            return value
+
+        return process
+
+    @property
+    def python_type(self):
+        return self.impl.type.python_type

cif/store/sqlite/hash.py

@@ -0,0 +1,36 @@
+from sqlalchemy.types import UserDefinedType
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy import types
+import socket
+
+Base = declarative_base()
+
+
+class Hash(UserDefinedType):
+
+    impl = types.BINARY(16)
+
+    def __init__(self, version=4):
+        self.version = version
+
+    def get_col_spec(self, **kw):
+        return "HASH"
+
+    def bind_processor(self, dialect):
+
+        DBAPIBinary = dialect.dbapi.Binary
+
+        def process(value):
+            return DBAPIBinary(value)
+
+        return process
+
+    def result_processor(self, dialect, coltype):
+        def process(value):
+            return value
+
+        return process
+
+    @property
+    def python_type(self):
+        return self.impl.type.python_type