Add Celery Integration to Django, Update Dockerfile, and Upgrade Library in Requirements

deployment/docker-compose.override.slim-sample.yml

@@ -0,0 +1,79 @@
+version: '3.9'
+volumes:
+  nginx-conf-volume:
+    driver_opts:
+      type: none
+      device: ${PWD}/sites-enabled
+      o: bind
+  static-volume:
+    driver_opts:
+      type: none
+      device: ${PWD}/static
+      o: bind
+  media-volume:
+    driver_opts:
+      type: none
+      device: ${PWD}/media
+      o: bind
+  reports-volume:
+    driver_opts:
+      type: none
+      device: ${PWD}/reports
+      o: bind
+  db-backup-volume:
+    driver_opts:
+      type: none
+      device: ${PWD}/backups
+      o: bind
+  db-sql-volume:
+    driver_opts:
+      type: none
+      device: ${PWD}/sql
+      o: bind
+
+x-common-dev:
+  &default-common-django-dev
+  image: geocontext-devweb
+  build:
+    context: ./docker
+    dockerfile: Dockerfile-slim-dev
+  environment:
+    - REDIS_HOST=${REDIS_HOST:-redis}
+    - REDIS_PASSWORD=${REDIS_PASSWORD:-redis_password}
+    - DATABASE_NAME=gis
+    - DATABASE_USERNAME=docker
+    - DATABASE_PASSWORD=docker
+    - DATABASE_HOST=db
+    - DJANGO_SETTINGS_MODULE=core.settings.prod_docker
+    - PYTHONPATH=/usr/src/geocontext
+    - VIRTUAL_HOST=geocontext.kartoza.com
+    - VIRTUAL_PORT=8080
+  volumes:
+    - ../django_project:/usr/src/geocontext
+    - static-volume:/home/web/static:rw
+    - media-volume:/home/web/media:rw
+    - reports-volume:/home/web/reports
+    - ./logs:/var/log/
+  links:
+    - db:db
+
+services:
+  devweb:
+    <<: *default-common-django-dev
+    links:
+      - db:db
+      - worker:worker
+    ports:
+      - "${HTTP_DEBUG_PORT}:8080"
+
+  worker:
+    <<: *default-common-django-dev
+    entrypoint: []
+    command: 'celery -A geocontext worker -l info'
+    links:
+      - db
+      - redis
+
+  db:
+    ports:
+      - "${POSTGRES_PORT}:5432"

deployment/docker-compose.yml

@@ -1,11 +1,35 @@
-version: '3'
+version: '3.9'
 volumes:
   nginx-conf-volume:
   static-volume:
   media-volume:
   reports-volume:
   db-backup-volume:
   db-sql-volume:
+
+x-common-django:
+  &default-common-django
+  build:
+    context: ../
+    dockerfile: deployment/docker/Dockerfile
+  environment:
+    - REDIS_HOST=${REDIS_HOST:-redis}
+    - REDIS_PASSWORD=${REDIS_PASSWORD:-redis_password}
+    - DATABASE_NAME=gis
+    - DATABASE_USERNAME=docker
+    - DATABASE_PASSWORD=docker
+    - DATABASE_HOST=db
+    - DJANGO_SETTINGS_MODULE=core.settings.prod_docker
+    - PYTHONPATH=/usr/src/geocontext
+    - VIRTUAL_HOST=geocontext.kartoza.com
+    - VIRTUAL_PORT=8080
+  volumes:
+    - static-volume:/home/web/static:rw
+    - media-volume:/home/web/media:rw
+    - reports-volume:/home/web/reports
+  links:
+    - db:db
+
 services:
   web:
     image: nginx
@@ -21,29 +45,24 @@ services:
     restart: unless-stopped
 
   uwsgi:
-    build: 
-      context: ../
-      dockerfile: deployment/docker/Dockerfile
-    hostname: uwsgi
+    <<: *default-common-django
+    user: root
+
+  redis:
+    image: bitnami/redis:7.0.2
     environment:
-      - DATABASE_NAME=gis
-      - DATABASE_USERNAME=docker
-      - DATABASE_PASSWORD=docker
-      - DATABASE_HOST=db
-      - DJANGO_SETTINGS_MODULE=core.settings.prod_docker
-      - PYTHONPATH=/usr/src/geocontext
-      - VIRTUAL_HOST=geocontext.kartoza.com
-      - VIRTUAL_PORT=8080
-    volumes:
-      - static-volume:/home/web/static:rw
-      - media-volume:/home/web/media:rw
-      - reports-volume:/home/web/reports
+      - REDIS_PASSWORD=${REDIS_PASSWORD:-redis_password}
+
+  worker:
+    <<: *default-common-django
+    entrypoint: []
+    command: 'celery -A geocontext worker -l info'
     links:
-      - db:db
-    user: root
+      - db
+      - redis
 
   db:
-    image: kartoza/postgis:13.0
+    image: kartoza/postgis:15-3.3
     volumes:
       - db-backup-volume:/backups
       - db-sql-volume:/sql

deployment/docker/Dockerfile-slim-dev

@@ -0,0 +1,32 @@
+FROM python:3.8-slim-buster
+
+LABEL maintainer="Andre Theron<[email protected]>"
+
+RUN DEBIAN_FRONTEND=noninteractive \
+    && apt-get update -y \
+    && apt-get install -y --no-install-recommends \
+    binutils \
+    gcc \
+    gdal-bin \
+    git \
+    libproj-dev \
+    libgdal-dev \
+    python3-gdal \
+    python3-geoip \
+    python3-setuptools \
+    && rm -rf /var/lib/apt/lists/*
+
+ADD requirements.txt /etc/geocontext/requirements.txt
+RUN pip3 install --upgrade pip setuptools
+RUN pip3 install --no-cache-dir -r /etc/geocontext/requirements.txt
+
+ADD requirements-dev.txt /etc/geocontext/requirements-dev.txt
+RUN pip install -r /etc/geocontext/requirements-dev.txt
+ADD bashrc /root/.bashrc
+
+# Open port 8080 as we will be running our django dev server on
+EXPOSE 8080
+# Open port 22 as we will be using a remote interpreter from pycharm
+EXPOSE 22
+
+CMD ["tail", "-f", "/dev/null"]

deployment/docker/requirements.txt

@@ -1,7 +1,7 @@
-aiohttp[speedups]==3.6.*
+aiohttp[speedups]==3.9.1
 arcgis2geojson==2.0.*
 coreapi==2.3.*
-Django==3.1.*
+Django==3.2.*
 django-allauth==0.42.*
 django-braces==1.14.*
 django-cors-headers==3.2.*
@@ -17,9 +17,13 @@ djangorestframework==3.13.*
 geopy==1.22.*
 gunicorn==20.0.*
 markdown==3.2.*
-psycopg2-binary==2.8.*
+psycopg2-binary==2.9.9
 pytz==2020.*
 raven==6.10.*
-requests==2.24.*
+requests==2.25.*
 uwsgi==2.0.*
-uvicorn==0.11.*
+uvicorn==0.24.0
+redis==4.4.4
+Celery==5.3.6
+django-celery-results==2.5.0
+django-redis==5.4.0

django_project/core/settings/base.py

@@ -179,3 +179,13 @@
         },
     }
 }
+
+CACHES = {
+    'default': {
+        'BACKEND': 'django_redis.cache.RedisCache',
+        'LOCATION': (
+            f'redis://default:{os.environ.get("REDIS_PASSWORD", "")}'
+            f'@{os.environ.get("REDIS_HOST", "")}',
+        ),
+    }
+}

django_project/core/settings/contrib.py

@@ -36,7 +36,8 @@
     'rest_framework_gis',
     'rest_framework.authtoken',
     'corsheaders',
-    'leaflet'
+    'leaflet',
+    'django_celery_results'
 )
 
 MIDDLEWARE += (

django_project/geocontext/__init__.py

@@ -0,0 +1 @@
+from .celery import app as celery_app  # noqa

django_project/geocontext/celery.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+"""A celery config for the project.
+
+"""
+from __future__ import absolute_import, unicode_literals
+
+import os
+from celery import Celery
+from celery.schedules import crontab
+
+# set the default Django settings module for the 'celery' program.
+# this is also used in manage.py
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings')
+
+# Get the base REDIS URL, default to redis' default
+BASE_REDIS_URL = (
+    f'redis://default:{os.environ.get("REDIS_PASSWORD", "")}'
+    f'@{os.environ.get("REDIS_HOST", "")}',
+)
+
+app = Celery('sanbi')
+
+# Using a string here means the worker don't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+#   should have a `CELERY_` prefix.
+app.config_from_object('django.conf:settings', namespace='CELERY')
+
+# Load task modules from all registered Django app configs.
+app.autodiscover_tasks()
+
+app.conf.broker_url = BASE_REDIS_URL
+
+# for scheduling task.
+# app.conf.beat_scheduler = 'django_celery_beat.schedulers.DatabaseScheduler'

django_project/geocontext/tasks/__init__.py

@@ -0,0 +1 @@
+from .fetch_service import *  # noqa

django_project/geocontext/tasks/fetch_service.py

@@ -0,0 +1,26 @@
+import logging
+from celery import shared_task
+
+logger = logging.getLogger(__name__)
+
+
+@shared_task(name="fetch_service_data")
+def fetch_service_data(service_id: str, x: str, y: str):
+    from geocontext.models.service import Service
+    from geocontext.utilities.geometry import parse_coord
+    from geocontext.utilities.async_service import async_retrieve_services
+    from geocontext.utilities.async_service import AsyncService
+
+    service = Service.objects.get(id=service_id)
+    point = parse_coord(
+        x=x, y=y, srid=service.srid)
+    async_service = AsyncService(
+        service.key, point, service.tolerance)
+    new_async_service = async_retrieve_services([async_service])
+    if new_async_service[0].value != service.test_value:
+        service.status = False
+        logger.warning(f'Service: {service.name} status offline')
+    else:
+        service.status = True
+        logger.info(f'Service: {service.name} status online')
+    service.save()

django_project/geocontext/utilities/geometry.py

@@ -69,7 +69,7 @@ def get_bbox(point: Point, tolerance: float = 10, order_latlon: bool = True) ->
     return ','.join([str(i) for i in bbox])
 
 
-def parse_coord(x: str, y: str, srid: str = '4326') -> float:
+def parse_coord(x: str, y: str, srid: str = '4326') -> Point:
     """Parse string DD/DM/DMS coordinate input. Split by °,',".
     Signed degrees or suffix E/W/N/S.
 

django_project/geocontext/utilities/worker.py

@@ -70,7 +70,6 @@ def retrieve_all(self) -> dict:
 
         if len(req_s) > 0:
             async_services = [AsyncService(s, self.point, self.tolerance) for s in req_s]
-            logger.info('test', async_services)
             new_async_services = async_retrieve_services(async_services)
             caches.extend(self.bulk_create_caches(new_async_services))