Adding basic monitor endpoint (#2211)

Adds /api/monitor/ endpoint to perform a simple check on influx, rabbit, postgres, and redis services.

Co-authored-by: Till! <[email protected]>

galaxy/api/urls.py

@@ -325,6 +325,9 @@
     # url(r'^v1/', include('galaxy.api.v1.urls')),
     path('v2/', include('galaxy.api.v2.urls', namespace='v2')),
     path('internal/', include('galaxy.api.internal.urls', namespace='int')),
+
+    # Monitoring endpoint
+    url(r'^monitor/', views.MonitorRootView.as_view()),
 ]
 
 

galaxy/api/views/__init__.py

@@ -34,3 +34,4 @@
 from .survey import *               # noqa
 from .influx import *               # noqa
 from .active_user import *          # noqa
+from .monitor import *              # noqa

galaxy/api/views/monitor.py

@@ -0,0 +1,134 @@
+# (c) 2012-2018, Ansible by Red Hat
+#
+# This file is part of Ansible Galaxy
+#
+# Ansible Galaxy is free software: you can redistribute it and/or modify
+# it under the terms of the Apache License as published by
+# the Apache Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# Ansible Galaxy is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# Apache License for more details.
+#
+# You should have received a copy of the Apache License
+# along with Galaxy.  If not, see <http://www.apache.org/licenses/>.
+
+
+import logging
+
+from django.conf import settings
+from django.db.migrations.executor import MigrationExecutor
+from django.db import connections, DatabaseError, DEFAULT_DB_ALIAS
+import influxdb.client
+import kombu
+from rest_framework.permissions import AllowAny
+from rest_framework.response import Response
+import requests.exceptions
+import redis
+
+from galaxy.api.views import base_views
+
+
+__all__ = [
+    'MonitorRootView',
+]
+
+logger = logging.getLogger(__name__)
+
+
+def get_influx_status():
+    if not settings.GALAXY_METRICS_ENABLED:
+        return None
+
+    client = influxdb.InfluxDBClient(
+        host=settings.INFLUX_DB_HOST,
+        port=settings.INFLUX_DB_PORT,
+        username=settings.INFLUX_DB_USERNAME,
+        password=settings.INFLUX_DB_PASSWORD,
+        timeout=5,
+        retries=1,
+    )
+
+    try:
+        client.ping()
+    except (
+            influxdb.client.InfluxDBClientError,
+            requests.exceptions.ConnectionError,
+            requests.exceptions.Timeout
+    ):
+        return 'error'
+    except Exception:
+        logger.exception('Unexpected error when trying to connect to InfluxDB')
+        return 'error'
+    finally:
+        client.close()
+    return 'ok'
+
+
+def get_postgres_status():
+    status = 'ok'
+    migrations = 'ok'
+    try:
+        connection = connections[DEFAULT_DB_ALIAS]
+        connection.prepare_database()
+        executor = MigrationExecutor(connection)
+        targets = executor.loader.graph.leaf_nodes()
+        if executor.migration_plan(targets):
+            migrations = 'needed'
+    except DatabaseError:
+        status = 'error'
+    except Exception:
+        logger.exception('Unexpected error when trying to connect to Postgres')
+        status = 'error'
+    return status, migrations
+
+
+def get_redis_status():
+    pool = redis.ConnectionPool(
+        host=settings.REDIS_HOST,
+        port=settings.REDIS_PORT
+    )
+    client = redis.Redis(connection_pool=pool)
+    try:
+        client.ping()
+    except redis.RedisError:
+        return 'error'
+    except Exception:
+        logger.exception('Unexpected error when trying to connect to Redis')
+        return 'error'
+    finally:
+        pool.disconnect()
+    return 'ok'
+
+
+def get_rabbit_status():
+    client = kombu.Connection(settings.BROKER_URL)
+    try:
+        client.connect()
+    except client.connection_errors:
+        return 'error'
+    except Exception:
+        logger.exception('Unexpected error when trying to connect to Rabbit')
+        return 'error'
+    finally:
+        client.close()
+    return 'ok'
+
+
+class MonitorRootView(base_views.APIView):
+    """ Monitor resources """
+    permission_classes = (AllowAny,)
+
+    def get(self, request, format=None):
+        response = {
+            'influx': get_influx_status(),
+            'migrations': None,
+            'postgresql': None,
+            'rabbit': get_rabbit_status(),
+            'redis': get_redis_status()
+        }
+        (response['postgresql'], response['migrations']) = \
+            get_postgres_status()
+        return Response(response)

galaxy/api/views/views.py

@@ -51,7 +51,6 @@
 from galaxy.main import models
 from galaxy.common import version, sanitize_content_name
 
-
 logger = logging.getLogger(__name__)
 
 __all__ = [

testing/simple/roles/check_service_states/tasks/main.yml

@@ -0,0 +1,28 @@
+---
+- name: A service is expected to be down
+  block:
+    - name: Check service is down
+      assert:
+        that:
+          - item.value == 'error'
+      loop: "{{ monitor_response.json|dict2items }}"
+      when: item.key == expected_down
+
+    - name: Check remaining services are up
+      assert:
+        that:
+          - item.value == 'ok'
+      loop: "{{ monitor_response.json|dict2items }}"
+      when: item.key != expected_down
+
+  when: expected_down != ''
+
+- name: All services should be up
+  block:
+    - name: Check service is up
+      assert:
+        that:
+          - item.value == 'ok'
+      loop: "{{ monitor_response.json|dict2items }}"
+
+  when: expected_down == ''

testing/simple/roles/get_container_name/tasks/main.yml

@@ -0,0 +1,9 @@
+---
+- shell: "docker ps --format {% raw %}'{{.Names}}' {% endraw %} | grep {{ name_contains }}"
+  register: docker_output
+
+- set_fact:
+    container_name: "{{ docker_output.stdout_lines[0] }}"
+
+- debug:
+    msg: "Service container name is: {{ container_name }}"

testing/simple/roles/get_monitor_response/tasks/main.yml

@@ -0,0 +1,7 @@
+---
+- name: Access the monitor endpoint
+  uri:
+    url: "http://{{ galaxy_host }}:{{ galaxy_port }}/api/monitor"
+    return_content: true
+    body_format: json
+  register: monitor_response

testing/simple/roles/make_monitor_assertions/tasks/main.yml

@@ -0,0 +1,18 @@
+---
+- include_role:
+    name: get_monitor_response
+
+- include_role:
+    name: check_service_states
+  vars:
+    expected_down: ''
+
+- include_role:
+    name: test_service
+  loop:
+    - {service_name: 'postgres', expected_down: 'postgresql'}
+    - {service_name: 'influxdb', expected_down: 'influx'}
+    - {service_name: 'redis', expected_down: 'redis'}
+    - {service_name: 'rabbit', expected_down: 'rabbit'}
+  loop_control:
+    loop_var: service_item

testing/simple/roles/test_service/tasks/main.yml

@@ -0,0 +1,27 @@
+---
+- debug:
+    msg: "Testing service {{ service_item.service_name }}"
+
+- name: Get the container name
+  include_role:
+    name: get_container_name
+  vars:
+    name_contains: "{{ service_item.service_name }}"
+
+- name: Stop the service
+  command: docker stop {{ container_name }}
+
+- include_role:
+    name: get_monitor_response
+
+- include_role:
+    name: check_service_states
+  vars:
+    expected_down: "{{ service_item.expected_down }}"
+
+- name: Restart the service
+  command: docker restart {{ container_name }}
+
+- name: Wait for service to start
+  wait_for:
+    timeout: 15

testing/simple/test.yml

@@ -22,3 +22,7 @@
       assert:
         that:
           - "'<title>Ansible Galaxy</title>' in homeout.content"
+
+    # Test the /api/monitor endpoint
+    - include_role:
+        name: make_monitor_assertions