Merge pull request #66 from anclrii/dev

Adding logging and catching some exceptions

.github/workflows/docker-publish.yml

@@ -8,6 +8,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+      - name: Set up Python environment
+        uses: actions/setup-python@v3
+        with:
+          python-version: '3.8.5'
       - name: Install pip requirements
         run: pip install --no-cache-dir -r requirements.txt -r requirements_tests.txt
       - name: Run flake8 lint

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.7-alpine3.12
+FROM python:3.8.5-alpine3.12
 
 COPY requirements.txt /
 RUN pip install --no-cache-dir -r /requirements.txt

README.md

@@ -1,12 +1,16 @@
 ![Build](https://github.com/anclrii/Storj-Exporter/workflows/Build/badge.svg)
 ![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/anclrii/Storj-exporter)
+[![Stand With Ukraine](https://raw.githubusercontent.com/vshymanskyy/StandWithUkraine/main/badges/StandWithUkraine.svg)](https://stand-with-ukraine.pp.ua)
+
+[![Stand With Ukraine](https://raw.githubusercontent.com/vshymanskyy/StandWithUkraine/main/banner2-direct.svg)](https://stand-with-ukraine.pp.ua)
+
 ## About
 
 Storj exporter for Prometheus written in python. It pulls information from storj node api for `node`, `satellite` and `payout` metrics.
 
 Also check out [Storj-Exporter-dashboard](https://github.com/anclrii/Storj-Exporter-dashboard) for Grafana to visualise metrics for multiple storj nodes.
 
-Tested with storj node versions listed in under `tests/api_mock/`
+Tested with storj node versions listed under `tests/api_mock/`
 
 <img src="https://github.com/anclrii/Storj-Exporter-dashboard/raw/master/storj-exporter-boom-table.png" alt="0x187C8C43890fe4C91aFabbC62128D383A90548Dd" hight=490 width=490 align="right"/> 
 

storj_exporter/__main__.py

@@ -5,13 +5,16 @@
 import threading
 import time
 import json
+import logging
 from wsgiref.simple_server import make_server
 from prometheus_client import MetricsHandler, make_wsgi_app
 from prometheus_client.core import REGISTRY
 from prometheus_client.exposition import ThreadingWSGIServer
 from api_wrapper import ApiClient
 from collectors import NodeCollector, SatCollector, PayoutCollector
 
+logger = logging.getLogger(__name__)
+
 
 class GracefulKiller:
     kill_now = False
@@ -40,11 +43,12 @@ def do_GET(self):
             return MetricsHandler.do_GET(self)
 
     def log_message(self, format, *args):
-        """Log nothing."""
+        logger.debug("Client request: %s %s" % (self.address_string(), format % args))
 
 
 def start_wsgi_server(port, addr='', registry=REGISTRY):
     """Starts a WSGI server for prometheus metrics as a daemon thread."""
+    logger.info(f'Starting WSGI server on port {port}')
     app = make_wsgi_app(registry)
     httpd = make_server(addr, port, app, ThreadingWSGIServer,
                         handler_class=HTTPRequestHandler)
@@ -54,6 +58,7 @@ def start_wsgi_server(port, addr='', registry=REGISTRY):
     killer = GracefulKiller()
     while not killer.kill_now:
         time.sleep(1)
+    logger.info("Shutting down WSGI server")
 
 
 def main():
@@ -62,19 +67,32 @@ def main():
     storj_api_port = os.environ.get('STORJ_API_PORT', '14002')
     storj_exporter_port = int(os.environ.get('STORJ_EXPORTER_PORT', '9651'))
     storj_collectors = os.environ.get('STORJ_COLLECTORS', 'payout sat').split()
+    log_level = os.environ.get('STORJ_EXPORTER_LOG_LEVEL', 'INFO').upper()
+
+    """Setup logging."""
+    logging.basicConfig(
+        level=log_level,
+        format="%(asctime)s [%(levelname)s]: %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S"
+    )
 
     """Instantiate api client and node collector"""
     baseurl = 'http://' + storj_host_address + ':' + storj_api_port
+    logger.info(f'Starting storj exporter on port {storj_exporter_port}, '
+                f'connecting to {baseurl} with collectors {storj_collectors} enabled')
     client = ApiClient(baseurl)
     node_collector = NodeCollector(client)
+    logger.info('Registering node collector')
     REGISTRY.register(node_collector)
 
     """Instantiate and register optional collectors"""
     if 'payout' in storj_collectors:
         payout_collector = PayoutCollector(client)
+        logger.info('Registering payout collector')
         REGISTRY.register(payout_collector)
     if 'sat' in storj_collectors:
         sat_collector = SatCollector(client)
+        logger.info('Registering sat collector')
         REGISTRY.register(sat_collector)
 
     start_wsgi_server(storj_exporter_port, '')

storj_exporter/api_wrapper.py

@@ -1,30 +1,45 @@
 import requests
+import logging
+from requests.adapters import HTTPAdapter, Retry
+from json.decoder import JSONDecodeError
+
+logger = logging.getLogger(__name__)
+
 
 class ApiClient(object):
     """
     Storagenode (Storj) api client.
     (https://github.com/storj/storj/blob/main/storagenode/console/consoleserver/server.go)
     """
-    def __init__(self, base_url, path="/api/", timeout=10, retries=3):
+    def __init__(self, base_url, path="/api/", timeout=10, retries=2, backoff_factor=1):
         self._api_url = base_url + path
         self._timeout = timeout
-        self._session = self._make_session(retries)
+        self._retries = Retry(total=retries, backoff_factor=backoff_factor)
+        self._session = self._make_session()
 
-    def _make_session(self, retries):
+    def _make_session(self):
         session = requests.Session()
-        http_adapter = requests.adapters.HTTPAdapter(max_retries=retries)
+        http_adapter = HTTPAdapter(max_retries=self._retries)
         session.mount('http://', http_adapter)
         session.mount('https://', http_adapter)
         return session
 
     def _get(self, endpoint, default=None):
-        response = default
+        response_json = default
         try:
             url = self._api_url + endpoint
-            response = self._session.get(url=url, timeout=self._timeout).json()
-        except Exception:
+            response = self._session.get(url=url, timeout=self._timeout)
+            response.raise_for_status()
+            response_json = response.json()
+        except requests.exceptions.RequestException:
+            logger.debug(f"Error while getting data from {url}", exc_info=True)
+            pass
+        except JSONDecodeError:
+            logger.error(f"Failed to parse json response from {url}", exc_info=True)
             pass
-        return response
+        else:
+            logger.debug(f"Got response from {url}")
+        return response_json
 
     def node(self):
         return self._get('sno/', {})

storj_exporter/collectors.py

@@ -1,6 +1,9 @@
+import logging
 from metric_templates import GaugeMetricTemplate, InfoMetricTemplate
 from utils import sum_list_of_dicts, safe_list_get
 
+logger = logging.getLogger(__name__)
+
 
 class StorjCollector(object):
     def __init__(self, client):
@@ -22,7 +25,9 @@ def _refresh_data(self):
         self._node = self.client.node()
 
     def collect(self):
+        logger.debug(f'{self.__class__.__name__}.collect() called, refreshing data ...')
         self._refresh_data()
+        logger.debug(f'Creating metrics objects for {self.__class__.__name__}')
         _metric_template_map = self._get_metric_template_map()
         for template in _metric_template_map:
             template.add_metric_samples()
@@ -60,16 +65,20 @@ def _refresh_data(self):
         self._node = self.client.node()
 
     def collect(self):
+        logger.debug(f'{self.__class__.__name__}.collect() called, refreshing data ...')
         self._refresh_data()
-        self._node = self.client.node()
+        logger.debug(f'Creating metrics objects for {self.__class__.__name__}')
         _metric_template_map = self._get_metric_template_map({}, 'id', 'url')
         for satellite in self._node.get('satellites', []):
+            logger.debug(f'Processing satellite {satellite}')
             if satellite and isinstance(satellite, dict):
                 _sat_data, _sat_id, _sat_url = self._get_sat_data(satellite)
                 for template in _metric_template_map:
                     template.data_dict = _sat_data
                     template.extra_labels_values = [_sat_id, _sat_url]
                     template.add_metric_samples()
+            else:
+                logger.debug('Node data for satellite is invalid, skipping satellite')
         for template in _metric_template_map:
             yield template.metric_object
 
@@ -78,12 +87,19 @@ def _get_sat_data(self, satellite):
         _sat_id = satellite.get('id', None)
         _sat_url = satellite.get('url', None)
         if _sat_id and _sat_url:
+            logger.debug(f'Getting data for satellite {_sat_url} ({_sat_id})')
             _sat_data = self.client.satellite(_sat_id)
             if _sat_data and isinstance(_sat_data, dict):
                 _sat_data = self._prepare_sat_data(satellite, _sat_data)
+            else:
+                logger.debug('Satellite data is invalid, skipping satellite')
+        else:
+            logger.debug(f'_sat_id = {_sat_id} and _sat_url = {_sat_url}, '
+                         'skipping satellite ...')
         return _sat_data, _sat_id, _sat_url
 
     def _prepare_sat_data(self, satellite, _sat_data):
+        logger.debug('Preparing satellite data for adding samples ...')
         _suspended = 1 if satellite.get('suspended', None) else 0
         _sat_data.update({'suspended': _suspended})
 
@@ -163,7 +179,9 @@ def _refresh_data(self):
         self._payout = self.client.payout().get('currentMonth', {})
 
     def collect(self):
+        logger.debug(f'{self.__class__.__name__}.collect() called, refreshing data ...')
         self._refresh_data()
+        logger.debug(f'Creating metrics objects for {self.__class__.__name__}')
         _metric_template_map = self._get_metric_template_map()
         for template in _metric_template_map:
             template.add_metric_samples()

storj_exporter/metric_templates.py

@@ -1,3 +1,4 @@
+import logging
 from dataclasses import dataclass, field
 from prometheus_client.core import (
     GaugeMetricFamily,
@@ -6,6 +7,8 @@
 )
 from utils import to_float, nested_get
 
+logger = logging.getLogger(__name__)
+
 
 @dataclass
 class MetricTemplate(object):
@@ -22,11 +25,13 @@ class MetricTemplate(object):
     def __post_init__(self):
         self.metric_object = self._metric_class(
             name=self.metric_name, documentation=self.documentation, labels=self.labels)
+        logger.debug(f'... {self.metric_name} metric object created')
 
     def _get_value(self, key):
         return self.data_dict.get(key, None)
 
     def add_metric_samples(self):
+        logger.debug(f'... adding samples to {self.metric_name}')
         if self.nested_path:
             self.data_dict = nested_get(self.data_dict, self.nested_path)
         if self.data_dict and isinstance(self.data_dict, dict):
@@ -35,6 +40,12 @@ def add_metric_samples(self):
                 labels_list = [key] + self.extra_labels_values
                 if labels_list and value is not None:
                     self.metric_object.add_metric(labels_list, value)
+                else:
+                    logger.debug(f'{self.metric_name} sample not added for {key}, '
+                                 f'labels: {labels_list}, value: {value}')
+        else:
+            logger.debug(f'{self.metric_name} data is empty or invalid, '
+                         'not adding samples')
         self.data_dict = {}
 
 
@@ -46,6 +57,8 @@ def _get_value(self, key):
         value = self.data_dict.get(key, None)
         if value is not None:
             value = to_float(value)
+        else:
+            logger.debug(f'{self.metric_name} value for key {key} is None or not found')
         return value
 
 
@@ -57,4 +70,6 @@ def _get_value(self, key):
         value = self.data_dict.get(key, None)
         if value is not None:
             value = {key: str(value)}
+        else:
+            logger.debug(f'{self.metric_name} value for key {key} is None or not found')
         return value

tests/test_collectors.py

@@ -106,6 +106,7 @@ def test_collect(self, client, expected_samples):
         for metric in res_list:
             assert len(metric.samples) >= expected_samples
 
+    @pytest.mark.usefixtures("mock_get_sno")
     @pytest.mark.usefixtures("mock_get_satellite")
     @pytest.mark.parametrize("mock_get_satellite",
                              [("success"), ("notfound"), ("timeout")],