Release (#292)

* tweaking for feed performance

* bugfixes

* fixes

* deployment fixes

Vagrantfile

@@ -9,6 +9,7 @@ sdist=ENV['CIF_ANSIBLE_SDIST']
 es=ENV['CIF_ANSIBLE_ES']
 hunter_threads=ENV['CIF_HUNTER_THREADS']
 geo_fqdn=ENV['CIF_GATHERER_GEO_FQDN']
+csirtg_token=ENV['CSIRTG_TOKEN']
 
 unless File.directory?('deploymentkit')
     puts "Please unzip the latest release of the deploymentkit before continuing..."
@@ -24,6 +25,9 @@ export CIF_ANSIBLE_ES=#{es}
 export CIF_HUNTER_THREADS=#{hunter_threads}
 export CIF_GATHERER_GEO_FQDN=#{geo_fqdn}
 export CIF_BOOTSTRAP_TEST=1
+export CSIRTG_TOKEN=#{csirtg_token}
+
+echo "export CSIRTG_TOKEN='${CSIRTG_TOKEN}'" >> /home/ubuntu/.profile
 
 cd /vagrant/deploymentkit
 bash easybutton.sh
@@ -36,7 +40,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.vm.network :forwarded_port, guest: 443, host: 8443
 
   config.vm.provider :virtualbox do |vb|
-    vb.customize ["modifyvm", :id, "--cpus", "2", "--ioapic", "on", "--memory", "1024" ]
+    vb.customize ["modifyvm", :id, "--cpus", "2", "--ioapic", "on", "--memory", "2048" ]
   end
 
   if File.file?(VAGRANTFILE_LOCAL)

cif/constants.py

@@ -43,3 +43,5 @@
 FEEDS_DAYS = 21
 FEEDS_LIMIT = 50000
 FEEDS_WHITELIST_LIMIT = 25000
+
+HTTPD_FEED_WHITELIST_CONFIDENCE = os.getenv('CIF_HTTPD_FEED_WHITELIST_CONFIDENCE', 5)

cif/httpd/views/feed/__init__.py

@@ -5,7 +5,7 @@
 from flask import request, current_app, jsonify
 from cifsdk.client.zeromq import ZMQ as Client
 from cifsdk.client.dummy import Dummy as DummyClient
-from cif.constants import ROUTER_ADDR, FEEDS_DAYS, FEEDS_LIMIT, FEEDS_WHITELIST_LIMIT
+from cif.constants import ROUTER_ADDR, FEEDS_DAYS, FEEDS_LIMIT, FEEDS_WHITELIST_LIMIT, HTTPD_FEED_WHITELIST_CONFIDENCE
 from cifsdk.exceptions import InvalidSearch, AuthError
 import logging
 import copy
@@ -85,11 +85,15 @@ def get(self):
             logger.error(e)
             return jsonify_unknown('invalid search', 400)
 
+        except Exception as e:
+            logger.error(e)
+            return jsonify_unknown(msg='search failed')
+
         r = aggregate(r)
 
         wl_filters = copy.deepcopy(filters)
         wl_filters['tags'] = 'whitelist'
-        wl_filters['confidence'] = 25
+        wl_filters['confidence'] = HTTPD_FEED_WHITELIST_CONFIDENCE
 
         wl_filters['nolog'] = True
         wl_filters['limit'] = FEEDS_WHITELIST_LIMIT
@@ -98,7 +102,7 @@ def get(self):
             wl = Client(remote, pull_token()).indicators_search(wl_filters)
         except Exception as e:
             logger.error(e)
-            return jsonify_unknown('feed failed', 503)
+            return jsonify_unknown('feed query failed', 503)
 
         wl = aggregate(wl)
 

cif/httpd/views/indicators.py

@@ -36,14 +36,18 @@ def get(self):
 
         except RuntimeError as e:
             logger.error(e)
-            return jsonify_unknown(msg='search failed', code=403)
+            return jsonify_unknown(msg='search failed')
 
         except InvalidSearch as e:
             return jsonify_unknown(msg='invalid search', code=400)
 
         except AuthError:
             return jsonify_unauth()
 
+        except Exception as e:
+            logger.error(e)
+            return jsonify_unknown(msg='search failed, system may be too busy, check back later')
+
         response = current_app.response_class(r, mimetype='application/json')
 
         if isinstance(r, basestring):

cif/store/__init__.py

@@ -51,10 +51,10 @@
 TRACE = os.environ.get('CIF_ROUTER_TRACE') or os.environ.get('CIF_STORE_TRACE')
 
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.ERROR)
+#logger.setLevel(logging.ERROR)
 
-if TRACE:
-    logger.setLevel(logging.DEBUG)
+#if TRACE:
+#    logger.setLevel(logging.DEBUG)
 
 
 class Store(multiprocessing.Process):

cif/store/sqlite/__init__.py

@@ -24,13 +24,15 @@
 SYNC = os.environ.get('CIF_STORE_SQLITE_SYNC', 'NORMAL')
 
 # https://www.sqlite.org/pragma.html#pragma_cache_size
-CACHE_SIZE = os.environ.get('CIF_STORE_SQLITE_CACHE_SIZE', 256000000)  # 256MB
+CACHE_SIZE = os.environ.get('CIF_STORE_SQLITE_CACHE_SIZE', 512000000)  # 256MB
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
+logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
 
 if not TRACE:
     logger.setLevel(logging.ERROR)
+    logging.getLogger('sqlalchemy.engine').setLevel(logging.ERROR)
 
 VALID_FILTERS = ['indicator', 'confidence', 'provider', 'itype', 'group', 'tags']
 
@@ -63,7 +65,7 @@ def __init__(self, dbfile=DB_FILE, autocommit=False, dictrows=True, **kwargs):
 
         echo = False
         if TRACE:
-            echo = True
+            echo = False
 
         # http://docs.sqlalchemy.org/en/latest/orm/contextual.html
         self.engine = create_engine(self.path, echo=echo)

cif/store/sqlite/indicator.py

@@ -1,8 +1,8 @@
 import os
 
 import arrow
-from sqlalchemy import Column, Integer, String, Float, DateTime, UnicodeText, desc, ForeignKey, or_
-from sqlalchemy.orm import relationship, backref, class_mapper, lazyload
+from sqlalchemy import Column, Integer, String, Float, DateTime, UnicodeText, desc, ForeignKey, or_, Index
+from sqlalchemy.orm import relationship, backref, class_mapper, lazyload, joinedload, subqueryload
 
 from cifsdk.constants import RUNTIME_PATH, PYVERSION
 import json
@@ -205,6 +205,8 @@ class Tag(Base):
         Indicator,
     )
 
+    __table_args__ = (Index('ix_tags_indicator', "tag", "indicator_id"),)
+
 
 class Message(Base):
     __tablename__ = 'messages'
@@ -351,15 +353,15 @@ def _filter_terms(self, filters, s):
                 s = s.filter(Indicator.reporttime <= filters[k])
 
             elif k == 'tags':
-                s = s.join(Tag).filter(Tag.tag == filters[k])
+                s = s.outerjoin(Tag).filter(Tag.tag == filters[k])
 
             elif k == 'confidence':
                 if ',' in str(filters[k]):
                     start, end = str(filters[k]).split(',')
                     s = s.filter(Indicator.confidence >= float(start))
                     s = s.filter(Indicator.confidence <= float(end))
                 else:
-                    s = s.filter(Indicator.confidence >= filters[k])
+                    s = s.filter(Indicator.confidence >= float(filters[k]))
 
             elif k == 'itype':
                 s = s.filter(Indicator.itype == filters[k])

requirements.txt

@@ -1,6 +1,6 @@
 cython>=0.2
 pyzmq>=16.0.2,<17.0
-csirtg_indicator==0.0.0b16
+csirtg_indicator==0.0.0b18
 cifsdk>=3.0.0a24,<4.0
 Flask-Limiter>=0.9.3,<1.0
 limits>=1.1.1,<1.2
@@ -17,7 +17,8 @@ setuptools>=18.5
 ujson>=1.35
 html5lib==1.0b8 # bug in csirtg-smrt upstream
 msgpack-python>=0.4.8,<0.5.0
-csirtg_smrt==0.0.0a42
+apwgsdk==0.0.0a6
+csirtg_smrt==0.0.0a45
 csirtg_dnsdb==0.0.0a4
 tornado>=4.4.1,<5.0
 faker==0.7.11

rules/default/csirtg.yml

@@ -11,7 +11,8 @@
 # To remove the limits, sign up for an API key at https://csirtg.io
 
 parser: csv
-#token: < token here -> get one at https://csirtg.io >
+token: 'CSIRTG_TOKEN'  # ENV['CSIRTG_TOKEN'] <get one at https://csirtg.io >
+limit: 250
 defaults:
   provider: csirtg.io
   altid_tlp: white

rules/default/danger_rules_sk.yml

@@ -1,3 +1,4 @@
+parser: pattern
 defaults:
   provider: 'danger.rulez.sk'
   confidence: 9