Release 3.0.0a18 (#285)

* bugfixes

* updating for deploymentkit

* removing old deployment dir

* fixes

* bugfix to geo

* bugfixes and cleanup

* bugfix to geo

.gitignore

@@ -1,4 +1,5 @@
 .DS_Store
+deploymentkit/*
 Vagrantfile_*
 Vagrantfile\.*
 Dockerfile\.*

Vagrantfile

@@ -7,20 +7,32 @@ VAGRANTFILE_LOCAL = 'Vagrantfile.local'
 
 sdist=ENV['CIF_ANSIBLE_SDIST']
 es=ENV['CIF_ANSIBLE_ES']
+hunter_threads=ENV['CIF_HUNTER_THREADS']
+geo_fqdn=ENV['CIF_GATHERER_GEO_FQDN']
+
+unless File.directory?('deploymentkit')
+    puts "Please unzip the latest release of the deploymentkit before continuing..."
+    puts ""
+    puts "https://github.com/csirtgadgets/bearded-avenger-deploymentkit/wiki"
+    puts ""
+    exit
+end
 
 $script = <<SCRIPT
 export CIF_ANSIBLE_SDIST=#{sdist}
 export CIF_ANSIBLE_ES=#{es}
+export CIF_HUNTER_THREADS=#{hunter_threads}
+export CIF_GATHERER_GEO_FQDN=#{geo_fqdn}
+export CIF_BOOTSTRAP_TEST=1
 
-cd /vagrant/deployment
+cd /vagrant/deploymentkit
 bash easybutton.sh
 SCRIPT
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.vm.provision "shell", inline: $script
   config.vm.box = 'ubuntu/xenial64'
 
-  #config.vm.network :forwarded_port, guest: 5000, host: 5000
   config.vm.network :forwarded_port, guest: 443, host: 8443
 
   config.vm.provider :virtualbox do |vb|

cif/gatherer/geo.py

@@ -21,10 +21,12 @@
     '/usr/local/share/GeoIP'
 ]
 
+ENABLE_FQDN = os.getenv('CIF_GATHERER_GEO_FQDN')
 DB_FILE = 'GeoLite2-City.mmdb'
 DB_PATH = os.environ.get('CIF_GEO_PATH')
 
 ASN_DB_PATH = 'GeoIPASNum.dat'
+ASN_DB_PATH2 = 'GeoLiteASNum.dat'
 CITY_DB_PATH = 'GeoLiteCity.dat'
 
 
@@ -56,6 +58,10 @@ def __init__(self, path=DB_SEARCH_PATHS, db=DB_FILE):
                 self.asn_db = pygeoip.GeoIP(os.path.join(p, ASN_DB_PATH), pygeoip.MMAP_CACHE)
                 break
 
+            if os.path.isfile(os.path.join(p, ASN_DB_PATH2)):
+                self.asn_db = pygeoip.GeoIP(os.path.join(p, ASN_DB_PATH2), pygeoip.MMAP_CACHE)
+                break
+
         for p in DB_SEARCH_PATHS:
             if os.path.isfile(os.path.join(p, CITY_DB_PATH)):
                 self.city_db = pygeoip.GeoIP(os.path.join(p, CITY_DB_PATH), pygeoip.MMAP_CACHE)
@@ -80,6 +86,9 @@ def _resolve(self, indicator):
 
         i = indicator.indicator
         if indicator.itype in ['url', 'fqdn']:
+            if not ENABLE_FQDN:
+                return
+
             if indicator.itype == 'url':
                 u = urlparse(i)
                 i = u.hostname
@@ -91,7 +100,11 @@ def _resolve(self, indicator):
             if not indicator.rdata:
                 indicator.rdata = i
 
-        i = self._ip_to_prefix(i)
+        try:
+            i = self._ip_to_prefix(i)
+        except IndexError:
+            self.logger.error('unable to determine geo for %s' % indicator.indicator)
+            return
 
         g = self.db.city(i)
 
@@ -112,7 +125,7 @@ def _resolve(self, indicator):
 
         g = self.city_db.record_by_addr(i)
 
-        if g.get('region_code'):
+        if g and g.get('region_code'):
             indicator.region = g['region_code']
 
         g = self.asn_db.asn_by_addr(i)

cif/router.py

@@ -20,7 +20,7 @@
 from cifsdk.msg import Msg
 
 HUNTER_MIN_CONFIDENCE = 4
-HUNTER_THREADS = os.getenv('CIF_HUNTER_THREADS', 2)
+HUNTER_THREADS = os.getenv('CIF_HUNTER_THREADS', 0)
 HUNTER_ADVANCED = os.getenv('CIF_HUNTER_ADVANCED', 0)
 GATHERER_THREADS = os.getenv('CIF_GATHERER_THREADS', 2)
 STORE_DEFAULT = 'sqlite'
@@ -79,7 +79,7 @@ def __init__(self, listen=ROUTER_ADDR, hunter=HUNTER_ADDR, store_type=STORE_DEFA
 
         self.hunters = []
         self.hunters_s = None
-        if int(hunter_threads):
+        if hunter_threads and int(hunter_threads):
             self.hunters_s = self.context.socket(zmq.PUSH)
             self.logger.debug('binding hunter: {}'.format(hunter))
             self.hunters_s.bind(hunter)

cif/store/sqlite/indicator.py

@@ -459,6 +459,12 @@ def upsert(self, token, data):
         for d in data:
             logger.debug(d)
 
+            if not d.get('group'):
+                raise InvalidIndicator('missing group')
+
+            if isinstance(d['group'], list):
+                d['group'] = d['group'][0]
+
             # raises AuthError if invalid group
             self._check_token_groups(token, d)