Develop

.travis.yml

@@ -2,8 +2,8 @@ language: python
 sudo: required
 services:
 - docker
-python:
-- '2.7'
+python: 3.7
+dist: xenial
 env:
   global:
   - secure: YDiEDTMd1/lodlkCMJsctiL9PGjtAlPokPoNdux1YT4Qlb5sr5mhqg75lp61bstc2NXGExXHBtCKJdNH3xo06g2vJz+5nVIGYSH7Dbmx37+9xeS6H8TRoGf2EvnT+tFsJ3fAyKUFC8aXxB2UBgY8LOwk09xOJsNPsfa1hqIe3f3tBWqCpMiPLy3fsapNlQZIycq70JCt7rza4N0NuJKOaoJvhedg8E/0IHleWbHfXJ6kmnB83W3cFAO9p5quFB/h+UiIjPGwYtIRODunbb/dsJfaH60wbZqC3705ppF9d6/FMmL874npNIc+ZfnCqIP1RCqZgDGUer5qf9DuohCUH/X1d47rw6Ec0WtPPt5lf4AVbax2zGbOUzXQ99TZTdOTxde7n4XnVZq1P/+w8Bqgf0DuiTpMPxdGEGnfMlNysXTrwvGlDpv/XVA0jJjCxV6z9fnGfWNmxZWBqfQIE5/ZbaHY3BtMjbeTXPQvZ1EXglSbV7CshK+TWQG4o9KiYOdKI+JiQvIsvA9AW96+x2A7bOtyLn1PLsfntEUpyjih6g4d9r+H75qbPR7CGVcd9pNhavKy04gB7B6nh07xgaKLXlZ4L6rIf5ygecdq4jMcfcIEQXzxEfUfhU5VSFadTain2KXD6+l1LB7hXGOEPDwsa2mxXdkkNgDQ8vW6CyJgj7E=
@@ -13,14 +13,9 @@ before_install:
   -in resin_deploy.enc -out ~/.ssh/resin_deploy -d
 - chmod 600 ~/.ssh/resin_deploy
 install:
-- sudo apt-get update && sudo apt-get install -y lshw
-- pip install pytest==3.0.4 pytest-cov pytest-pep8 python-coveralls codeclimate-test-reporter
-- pip install -r ./requirements.txt
-- sudo mkdir /data/
 jobs:
   include:
-    - script: py.test sitch/tests/ --cov sitchlib
-    - script: docker build -t throwaway -f Dravisfile .
+    - script: docker build -t throwaway --build-arg ARCH=amd64 .
     - stage: deploy
       script: if [ $TRAVIS_BRANCH == 'test' ] && [ $TRAVIS_EVENT_TYPE != 'pull_request' ];
               then echo "Host git.resin.io" >> ~/.ssh/config;

Dockerfile

@@ -1,5 +1,78 @@
-FROM resin/armv7hf-debian:jessie
-MAINTAINER http://sitch.io
+ARG ARCH=armv7hf
+FROM balenalib/${ARCH}-debian-golang:1.13.12-buster as mmdb
+
+ENV MMDB_CONFIG_FILE=/usr/local/etc/GeoIP.conf
+
+RUN env GO111MODULE=on go get -u github.com/maxmind/geoipupdate/v4/cmd/geoipupdate
+
+# MMDB Config
+RUN echo "AccountID ${MMDB_ACCOUNT_ID:-UNSET}" > ${MMDB_CONFIG_FILE} && \
+    echo "LicenseKey ${MMDB_LICENSE_KEY:-UNSET}" >> ${MMDB_CONFIG_FILE} && \
+    echo "EditionIDs GeoLite2-City" >> ${MMDB_CONFIG_FILE} && \
+    echo "DatabaseDirectory /var/mmdb/" >> ${MMDB_CONFIG_FILE}
+
+RUN mkdir -p /var/mmdb/
+
+RUN $GOPATH/bin/geoipupdate || echo "Unable to download GeoIP DB!!" && touch /var/mmdb/.placeholder
+
+#############################################
+###### Build the unit test image
+FROM balenalib/${ARCH}-debian-python:3.6-jessie
+
+# Install requirements
+COPY apt-install /
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    `cat /apt-install` \
+    build-essential \
+    curl \
+    libffi-dev \
+    libssl-dev \
+    ca-certificates \
+    zlib1g-dev && \
+    apt-get clean && \
+    apt-get -y autoclean && \
+    apt-get -y autoremove && \
+    rm -rf /var/lib/apt/lists/*
+
+
+
+# Copy forward the GeoLite2 DB
+COPY --from=mmdb /var/mmdb/* /var/mmdb/
+
+# Place Kalibrate
+COPY binaries/kal-linux-arm /usr/local/bin/
+
+# Place Filebeat
+COPY binaries/filebeat-linux-arm /usr/local/bin
+
+# Place config templates
+RUN mkdir -p /etc/templates
+COPY configs/filebeat.json /etc/templates
+
+# Place schema file
+RUN mkdir /etc/schemas
+COPY configs/feed_db_translation.yaml /etc/schemas
+COPY configs/feed_db_schema.yaml /etc/schemas
+
+# Get the scripts in place
+COPY sitch/ /app/sitch
+
+RUN mkdir /data/
+
+COPY requirements*.txt /
+
+WORKDIR /app/sitch
+
+RUN /usr/local/bin/python -m pip install virtualenv==15.1.0 && \
+    virtualenv venv && \
+    . ./venv/bin/activate && \
+    pip install -r /requirements-test.txt && \
+    py.test tests/ --cov sitchlib
+
+
+##########################################
+####### Build the final image
+FROM balenalib/${ARCH}-debian-python:3.6-jessie
 
 ENV FEED_RADIO_TARGETS="GSM"
 ENV GSM_MODEM_BAND="ALL_BAND"
@@ -11,15 +84,16 @@ ENV MCC_LIST="310,311,312,316"
 
 ENV MODE="full"
 
+# Install all the packages
 COPY apt-install /
-
 RUN apt-get update && apt-get install -y --no-install-recommends \
     `cat /apt-install` && \
     apt-get clean && \
     apt-get -y autoclean && \
     apt-get -y autoremove && \
     rm -rf /var/lib/apt/lists/*
 
+
 # Place Kalibrate
 COPY binaries/kal-linux-arm /usr/local/bin/
 
@@ -35,6 +109,8 @@ RUN mkdir /etc/schemas
 COPY configs/feed_db_translation.yaml /etc/schemas
 COPY configs/feed_db_schema.yaml /etc/schemas
 
+# Bring forward the GeoLite2 DB
+COPY --from=mmdb /var/mmdb/* /var/mmdb/
 
 # Get the scripts in place
 COPY sitch/ /app/sitch
@@ -43,7 +119,10 @@ COPY requirements.txt /requirements.txt
 
 WORKDIR /app/sitch
 
-RUN pip install virtualenv==15.1.0 && \
+RUN dpkg -l | grep gcc
+RUN which gcc
+
+RUN /usr/local/bin/python -m pip install virtualenv==15.1.0 && \
     virtualenv venv && \
     . ./venv/bin/activate && \
     pip install -r /requirements.txt

Dravisfile

@@ -1,6 +1,12 @@
 FROM debian:jessie
 MAINTAINER http://sitch.io
 
+# Tests the availability of GeoIP DB
+
+WORKDIR /var/mmdb/
+wget http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz
+gunzip ./GeoLite2-City.mmdb.gz
+
 # This is just to regularly test the package version pins
 
 COPY apt-install /apt-install

README.md

@@ -16,7 +16,7 @@
 ## Prerequisites
 
 * Accounts with the following providers:
-  * Resin.io
+  * Balena.io
   * Github
 * Access to the following services (See Service configuration for more
   information)
@@ -33,18 +33,18 @@
 
 ## Step by step...
 
-1. Create an application in Resin.
+1. Create an application in Balena.
 1. Fork this project and clone it on your workstation.  Or clone it directly...
 but forking makes modifications and PRs easier to deal with.
-1. Add the Resin application as a remote repo (`git remote add resin myusername@git.resin.io/myusername/myapplicationname.git`)
-1. Push to your Resin application: `git push resin master`
+1. Add the Balena application as a remote repo (`git remote add balena myusername@git.balena.io/myusername/myapplicationname.git`)
+1. Push to your Balena application: `git push balena master`
 
-We expect (at least) the following environment variables to be set in Resin:
+We expect (at least) the following environment variables to be set in Balena:
 
 
 | Variable              | Purpose                                                 |
 |-----------------------|---------------------------------------------------------|
-| LOCATION_NAME         | Override the default device name (Resin UUID)           |
+| LOCATION_NAME         | Override the default device name (Balena UUID)           |
 | LOG_HOST              |  hostname:port                                          |
 | STATE_LIST            | List of states (in caps) for FCC feed.  ex: "CA,TX"     |
 | VAULT_PATH            | Path to logstash cert/keys in Vault                     |
@@ -62,8 +62,8 @@ Testing is done with pytest.  Coverage module optional.
 
 Testing requirements (local testing possible only on Linux):
 * lshw
-* pip packages: pytest-cov pytest-pep8 pyserial hvac kalibrate haversine
-python-geoip python-geoip-geolite2 pyudev gps3 LatLon python-dateutil
+* pip packages: pytest-cov pytest-pep8 pyserial hvac kalibrate
+python-geoip python-geoip-geolite2 pyudev gps3 geopy python-dateutil
 
 
 1. Navigate to the base directory of the repository.
@@ -87,6 +87,12 @@ This repository contains pre-built binaries for Filebeat and Kalibrate.  The
 licenses which apply to these two tools can be found
 [here](./filebeat-license.txt) and [here](./kalibrate-license.txt), respectively.
 
+Building the Docker container will cause the retrieval of the MaxMind GeoLite2
+database...
+
+This tool includes the geoipupdate tool created by MaxMind, also available
+[here](https://github.com/maxmind/geoipupdate)
+
 ## Contributing
 
 * Please do PRs against the `test` branch.

apt-install

@@ -1,14 +1,12 @@
 expect=5.45-6
 logrotate=3.8.7-1+b1
-gcc=4:4.9.2-2
-gpsd=3.11-3
-gpsd-clients=3.11-3
+gcc
+gpsd=3.11-3+deb8u1
+gpsd-clients=3.11-3+deb8u1
 kmod=18-3
 lshw=02.17-1.1
 libfftw3-double3=3.3.4-2
 librtlsdr0=0.5.3-3
-libc6=2.19-18+deb8u10
-libudev1=215-17+deb8u8
-python-pip=1.5.6-5
-python-dev=2.7.9-1
+libudev1=215-17+deb8u13
 tcl=8.6.0+8
+libssl-dev

requirements-test.txt

@@ -0,0 +1,6 @@
+pytest~=3.6
+pytest-cov
+pytest-pep8
+mock
+codeclimate-test-reporter
+-r requirements.txt

requirements.txt

@@ -1,12 +1,10 @@
-psutil==5.4.8
+psutil==5.6.6
 pyserial==3.4
-pyyaml==4.2b1
+pyyaml==5.3.1
+geopy==1.18.1
 gps3==0.33.3
 hvac==0.7.1
-kalibrate==2.1.0
-haversine==2.0.0
+kalibrate==2.2.1
 python-dateutil==2.7.5
-python-geoip==1.2
-python-geoip-geolite2==2015.303
+geoip2==2.9.0
 pyudev==0.21.0
-LatLon==1.0.2

sitch/sitchlib/__init__.py

@@ -1,18 +1,18 @@
-from alert_manager import AlertManager  # NOQA
-from config_helper import ConfigHelper  # NOQA
-from feed_schema_translator import FeedSchemaTranslator  # NOQA
-from gps_device import GpsListener  # NOQA
-from gsm_modem import GsmModem  # NOQA
-from location_tool import LocationTool  # NOQA
-from logger import LogHandler  # NOQA
-from geo_ip import GeoIp  # NOQA
-from utility import Utility  # NOQA
-from feed_manager import FeedManager  # NOQA
-from device_detector import DeviceDetector  # NOQA
-from decomposer import Decomposer  # NOQA
-from arfcn_correlator import ArfcnCorrelator  # NOQA
-from cgi_correlator import CgiCorrelator  # NOQA
-from geo_correlator import GeoCorrelator  # NOQA
+from .alert_manager import AlertManager  # NOQA
+from .config_helper import ConfigHelper  # NOQA
+from .feed_schema_translator import FeedSchemaTranslator  # NOQA
+from .gps_device import GpsListener  # NOQA
+from .gsm_modem import GsmModem  # NOQA
+from .location_tool import LocationTool  # NOQA
+from .logger import LogHandler  # NOQA
+from .geo_ip import GeoIp  # NOQA
+from .utility import Utility  # NOQA
+from .feed_manager import FeedManager  # NOQA
+from .device_detector import DeviceDetector  # NOQA
+from .decomposer import Decomposer  # NOQA
+from .arfcn_correlator import ArfcnCorrelator  # NOQA
+from .cgi_correlator import CgiCorrelator  # NOQA
+from .geo_correlator import GeoCorrelator  # NOQA
 
 __author__ = "Ash Wilson"
-__version__ = "4.0"
+__version__ = "4.1"

sitch/sitchlib/alert_manager.py

@@ -1,8 +1,8 @@
 """Alert Manager."""
-from utility import Utility
+from .utility import Utility
 
 
-class AlertManager(object):
+class AlertManager:
     """AlertManager is used to ensure alerts are consistently formatted."""
 
     def __init__(self, device_id):
@@ -20,7 +20,6 @@ def __init__(self, device_id):
             310: "GPS time delta over threshold.",
             400: "Failed to locate a valid license for ARFCN in this area."
         }
-        return
 
     def get_alert_type(self, alert_id):
         """Return the alert description for alert_id."""

sitch/sitchlib/arfcn_correlator.py

@@ -1,12 +1,12 @@
 """ARFCN Correlator."""
 
-import alert_manager
 import os
 import sqlite3
-from utility import Utility
+from . import alert_manager
+from .utility import Utility
 
 
-class ArfcnCorrelator(object):
+class ArfcnCorrelator:
     """The ArfcnCorrelator compares ARFCN metadata against feeds and threshold.
 
     The feed data is put in place by the FeedManager class, prior to
@@ -34,7 +34,6 @@ def __init__(self, feed_dir, whitelist, power_threshold, device_id):
         self.observed_arfcn = whitelist
         self.arfcn_threshold = []
         self.arfcn_range = []
-        return
 
     def correlate(self, scan_bolus):
         """Entrypoint for correlation, wraps individual checks.

sitch/sitchlib/cgi_correlator.py

@@ -2,11 +2,11 @@
 
 import os
 import sqlite3
-import alert_manager
-from utility import Utility
+from . import alert_manager
+from .utility import Utility
 
 
-class CgiCorrelator(object):
+class CgiCorrelator:
     """The CgiCorrelator compares CGI addressing against the OpenCellID DB.
 
     The feed data is put in place by the FeedManager class, prior to
@@ -33,7 +33,6 @@ def __init__(self, feed_dir, cgi_whitelist, mcc_list, device_id):
         self.cgi_db = os.path.join(feed_dir, "cgi.db")
         self.alarm_140_cache = ""
         print(CgiCorrelator.cgi_whitelist_message(self.cgi_whitelist))
-        return
 
     def correlate(self, scan_bolus):
         """Entrypoint for the CGI correlation component.

sitch/sitchlib/config_helper.py

@@ -6,8 +6,8 @@
 import pprint
 import sys
 import yaml
-from device_detector import DeviceDetector as dd
-from utility import Utility as utility
+from .device_detector import DeviceDetector as dd
+from .utility import Utility as utility
 
 
 class ConfigHelper:
@@ -71,7 +71,6 @@ def print_devices_as_detected(self):
         pp.pprint(self.detector.gsm_radios)
         print("Configurator: Detected GPS devices:")
         pp.pprint(self.detector.gps_devices)
-        return
 
     def get_gsm_modem_port(self):
         """Get GSM modem port from detector, override with env var."""
@@ -129,7 +128,6 @@ def write_filebeat_config(self):
         fb = self.set_filebeat_logfile_paths(self.log_prefix, fb)
         with open(self.filebeat_config_file_path, 'w') as out_file:
             yaml.safe_dump(fb, out_file)
-        return
 
     @classmethod
     def set_filebeat_logfile_paths(cls, log_prefix, filebeat_config):