From bd3417260750643b5093332b6a21d27d28fa45de Mon Sep 17 00:00:00 2001 From: Dimas Date: Fri, 1 Dec 2023 07:05:39 +0700 Subject: [PATCH 1/2] Add celery, update dockerfile --- .../docker-compose.override.slim-sample.yml | 79 +++++++++++++++++++ deployment/docker-compose.yml | 59 +++++++++----- deployment/docker/Dockerfile-slim-dev | 32 ++++++++ deployment/docker/requirements.txt | 14 ++-- django_project/core/settings/base.py | 10 +++ django_project/core/settings/contrib.py | 3 +- django_project/geocontext/__init__.py | 1 + django_project/geocontext/celery.py | 35 ++++++++ django_project/geocontext/tasks/__init__.py | 1 + .../geocontext/tasks/fetch_service.py | 26 ++++++ .../geocontext/utilities/geometry.py | 2 +- django_project/geocontext/utilities/worker.py | 3 +- 12 files changed, 236 insertions(+), 29 deletions(-) create mode 100644 deployment/docker-compose.override.slim-sample.yml create mode 100644 deployment/docker/Dockerfile-slim-dev create mode 100644 django_project/geocontext/celery.py create mode 100644 django_project/geocontext/tasks/__init__.py create mode 100644 django_project/geocontext/tasks/fetch_service.py diff --git a/deployment/docker-compose.override.slim-sample.yml b/deployment/docker-compose.override.slim-sample.yml new file mode 100644 index 0000000..a339161 --- /dev/null +++ b/deployment/docker-compose.override.slim-sample.yml @@ -0,0 +1,79 @@ +version: '3.9' +volumes: + nginx-conf-volume: + driver_opts: + type: none + device: ${PWD}/sites-enabled + o: bind + static-volume: + driver_opts: + type: none + device: ${PWD}/static + o: bind + media-volume: + driver_opts: + type: none + device: ${PWD}/media + o: bind + reports-volume: + driver_opts: + type: none + device: ${PWD}/reports + o: bind + db-backup-volume: + driver_opts: + type: none + device: ${PWD}/backups + o: bind + db-sql-volume: + driver_opts: + type: none + device: ${PWD}/sql + o: bind + +x-common-dev: + &default-common-django-dev + image: geocontext-devweb + build: + context: ./docker + dockerfile: Dockerfile-slim-dev + environment: + - REDIS_HOST=${REDIS_HOST:-redis} + - REDIS_PASSWORD=${REDIS_PASSWORD:-redis_password} + - DATABASE_NAME=gis + - DATABASE_USERNAME=docker + - DATABASE_PASSWORD=docker + - DATABASE_HOST=db + - DJANGO_SETTINGS_MODULE=core.settings.prod_docker + - PYTHONPATH=/usr/src/geocontext + - VIRTUAL_HOST=geocontext.kartoza.com + - VIRTUAL_PORT=8080 + volumes: + - ../django_project:/usr/src/geocontext + - static-volume:/home/web/static:rw + - media-volume:/home/web/media:rw + - reports-volume:/home/web/reports + - ./logs:/var/log/ + links: + - db:db + +services: + devweb: + <<: *default-common-django-dev + links: + - db:db + - worker:worker + ports: + - "${HTTP_DEBUG_PORT}:8080" + + worker: + <<: *default-common-django-dev + entrypoint: [] + command: 'celery -A geocontext worker -l info' + links: + - db + - redis + + db: + ports: + - "${POSTGRES_PORT}:5432" diff --git a/deployment/docker-compose.yml b/deployment/docker-compose.yml index 3d5dc8f..fd4a863 100644 --- a/deployment/docker-compose.yml +++ b/deployment/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3' +version: '3.9' volumes: nginx-conf-volume: static-volume: @@ -6,6 +6,30 @@ volumes: reports-volume: db-backup-volume: db-sql-volume: + +x-common-django: + &default-common-django + build: + context: ../ + dockerfile: deployment/docker/Dockerfile + environment: + - REDIS_HOST=${REDIS_HOST:-redis} + - REDIS_PASSWORD=${REDIS_PASSWORD:-redis_password} + - DATABASE_NAME=gis + - DATABASE_USERNAME=docker + - DATABASE_PASSWORD=docker + - DATABASE_HOST=db + - DJANGO_SETTINGS_MODULE=core.settings.prod_docker + - PYTHONPATH=/usr/src/geocontext + - VIRTUAL_HOST=geocontext.kartoza.com + - VIRTUAL_PORT=8080 + volumes: + - static-volume:/home/web/static:rw + - media-volume:/home/web/media:rw + - reports-volume:/home/web/reports + links: + - db:db + services: web: image: nginx @@ -21,29 +45,24 @@ services: restart: unless-stopped uwsgi: - build: - context: ../ - dockerfile: deployment/docker/Dockerfile - hostname: uwsgi + <<: *default-common-django + user: root + + redis: + image: bitnami/redis:7.0.2 environment: - - DATABASE_NAME=gis - - DATABASE_USERNAME=docker - - DATABASE_PASSWORD=docker - - DATABASE_HOST=db - - DJANGO_SETTINGS_MODULE=core.settings.prod_docker - - PYTHONPATH=/usr/src/geocontext - - VIRTUAL_HOST=geocontext.kartoza.com - - VIRTUAL_PORT=8080 - volumes: - - static-volume:/home/web/static:rw - - media-volume:/home/web/media:rw - - reports-volume:/home/web/reports + - REDIS_PASSWORD=${REDIS_PASSWORD:-redis_password} + + worker: + <<: *default-common-django + entrypoint: [] + command: 'celery -A geocontext worker -l info' links: - - db:db - user: root + - db + - redis db: - image: kartoza/postgis:13.0 + image: kartoza/postgis:15-3.3 volumes: - db-backup-volume:/backups - db-sql-volume:/sql diff --git a/deployment/docker/Dockerfile-slim-dev b/deployment/docker/Dockerfile-slim-dev new file mode 100644 index 0000000..4096785 --- /dev/null +++ b/deployment/docker/Dockerfile-slim-dev @@ -0,0 +1,32 @@ +FROM python:3.8-slim-buster + +LABEL maintainer="Andre Theron" + +RUN DEBIAN_FRONTEND=noninteractive \ + && apt-get update -y \ + && apt-get install -y --no-install-recommends \ + binutils \ + gcc \ + gdal-bin \ + git \ + libproj-dev \ + libgdal-dev \ + python3-gdal \ + python3-geoip \ + python3-setuptools \ + && rm -rf /var/lib/apt/lists/* + +ADD requirements.txt /etc/geocontext/requirements.txt +RUN pip3 install --upgrade pip setuptools +RUN pip3 install --no-cache-dir -r /etc/geocontext/requirements.txt + +ADD requirements-dev.txt /etc/geocontext/requirements-dev.txt +RUN pip install -r /etc/geocontext/requirements-dev.txt +ADD bashrc /root/.bashrc + +# Open port 8080 as we will be running our django dev server on +EXPOSE 8080 +# Open port 22 as we will be using a remote interpreter from pycharm +EXPOSE 22 + +CMD ["tail", "-f", "/dev/null"] diff --git a/deployment/docker/requirements.txt b/deployment/docker/requirements.txt index 2e2afb0..0ba9d34 100644 --- a/deployment/docker/requirements.txt +++ b/deployment/docker/requirements.txt @@ -1,7 +1,7 @@ -aiohttp[speedups]==3.6.* +aiohttp[speedups]==3.9.1 arcgis2geojson==2.0.* coreapi==2.3.* -Django==3.1.* +Django==3.2.* django-allauth==0.42.* django-braces==1.14.* django-cors-headers==3.2.* @@ -17,9 +17,13 @@ djangorestframework==3.13.* geopy==1.22.* gunicorn==20.0.* markdown==3.2.* -psycopg2-binary==2.8.* +psycopg2-binary==2.9.9 pytz==2020.* raven==6.10.* -requests==2.24.* +requests==2.25.* uwsgi==2.0.* -uvicorn==0.11.* \ No newline at end of file +uvicorn==0.24.0 +redis==4.4.4 +Celery==5.3.6 +django-celery-results==2.5.0 +django-redis==5.4.0 diff --git a/django_project/core/settings/base.py b/django_project/core/settings/base.py index 50c9b5a..b28a947 100644 --- a/django_project/core/settings/base.py +++ b/django_project/core/settings/base.py @@ -179,3 +179,13 @@ }, } } + +CACHES = { + 'default': { + 'BACKEND': 'django_redis.cache.RedisCache', + 'LOCATION': ( + f'redis://default:{os.environ.get("REDIS_PASSWORD", "")}' + f'@{os.environ.get("REDIS_HOST", "")}', + ), + } +} diff --git a/django_project/core/settings/contrib.py b/django_project/core/settings/contrib.py index 9f2b06d..86944b2 100644 --- a/django_project/core/settings/contrib.py +++ b/django_project/core/settings/contrib.py @@ -36,7 +36,8 @@ 'rest_framework_gis', 'rest_framework.authtoken', 'corsheaders', - 'leaflet' + 'leaflet', + 'django_celery_results' ) MIDDLEWARE += ( diff --git a/django_project/geocontext/__init__.py b/django_project/geocontext/__init__.py index e69de29..0a8d3da 100644 --- a/django_project/geocontext/__init__.py +++ b/django_project/geocontext/__init__.py @@ -0,0 +1 @@ +from .celery import app as celery_app # noqa diff --git a/django_project/geocontext/celery.py b/django_project/geocontext/celery.py new file mode 100644 index 0000000..baea7f7 --- /dev/null +++ b/django_project/geocontext/celery.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +"""A celery config for the project. + +""" +from __future__ import absolute_import, unicode_literals + +import os +from celery import Celery +from celery.schedules import crontab + +# set the default Django settings module for the 'celery' program. +# this is also used in manage.py +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings') + +# Get the base REDIS URL, default to redis' default +BASE_REDIS_URL = ( + f'redis://default:{os.environ.get("REDIS_PASSWORD", "")}' + f'@{os.environ.get("REDIS_HOST", "")}', +) + +app = Celery('sanbi') + +# Using a string here means the worker don't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +app.config_from_object('django.conf:settings', namespace='CELERY') + +# Load task modules from all registered Django app configs. +app.autodiscover_tasks() + +app.conf.broker_url = BASE_REDIS_URL + +# for scheduling task. +# app.conf.beat_scheduler = 'django_celery_beat.schedulers.DatabaseScheduler' diff --git a/django_project/geocontext/tasks/__init__.py b/django_project/geocontext/tasks/__init__.py new file mode 100644 index 0000000..9bcd85f --- /dev/null +++ b/django_project/geocontext/tasks/__init__.py @@ -0,0 +1 @@ +from .fetch_service import * # noqa diff --git a/django_project/geocontext/tasks/fetch_service.py b/django_project/geocontext/tasks/fetch_service.py new file mode 100644 index 0000000..5cbe65b --- /dev/null +++ b/django_project/geocontext/tasks/fetch_service.py @@ -0,0 +1,26 @@ +import logging +from celery import shared_task + +logger = logging.getLogger(__name__) + + +@shared_task(name="fetch_service_data") +def fetch_service_data(service_id: str, x: str, y: str): + from geocontext.models.service import Service + from geocontext.utilities.geometry import parse_coord + from geocontext.utilities.async_service import async_retrieve_services + from geocontext.utilities.async_service import AsyncService + + service = Service.objects.get(id=service_id) + point = parse_coord( + x=x, y=y, srid=service.srid) + async_service = AsyncService( + service.key, point, service.tolerance) + new_async_service = async_retrieve_services([async_service]) + if new_async_service[0].value != service.test_value: + service.status = False + logger.warning(f'Service: {service.name} status offline') + else: + service.status = True + logger.info(f'Service: {service.name} status online') + service.save() diff --git a/django_project/geocontext/utilities/geometry.py b/django_project/geocontext/utilities/geometry.py index a487488..e51d240 100644 --- a/django_project/geocontext/utilities/geometry.py +++ b/django_project/geocontext/utilities/geometry.py @@ -69,7 +69,7 @@ def get_bbox(point: Point, tolerance: float = 10, order_latlon: bool = True) -> return ','.join([str(i) for i in bbox]) -def parse_coord(x: str, y: str, srid: str = '4326') -> float: +def parse_coord(x: str, y: str, srid: str = '4326') -> Point: """Parse string DD/DM/DMS coordinate input. Split by °,',". Signed degrees or suffix E/W/N/S. diff --git a/django_project/geocontext/utilities/worker.py b/django_project/geocontext/utilities/worker.py index 4d44719..051b9c4 100644 --- a/django_project/geocontext/utilities/worker.py +++ b/django_project/geocontext/utilities/worker.py @@ -70,7 +70,6 @@ def retrieve_all(self) -> dict: if len(req_s) > 0: async_services = [AsyncService(s, self.point, self.tolerance) for s in req_s] - logger.info('test', async_services) new_async_services = async_retrieve_services(async_services) caches.extend(self.bulk_create_caches(new_async_services)) @@ -131,7 +130,7 @@ def retrieve_caches(self, services: QuerySet) -> list: last_subsrt=Right('service__key', StrIndex(Reverse('service__key'), Value('_')) - 1, output_field=CharField()), ) - return sorted(caches, key=lambda cache: self.get_order(cache.service.id)) + return sorted(Cache.objects.none(), key=lambda cache: self.get_order(cache.service.id)) def bulk_create_caches(self, new_async_services: list) -> list: """Bulk update cache with new AsyncService values. From 03b296b447f3fc49c8aec81564c939072c99a6ca Mon Sep 17 00:00:00 2001 From: Dimas Date: Fri, 1 Dec 2023 07:08:02 +0700 Subject: [PATCH 2/2] Revert changes --- django_project/geocontext/utilities/worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_project/geocontext/utilities/worker.py b/django_project/geocontext/utilities/worker.py index 051b9c4..f06ca8e 100644 --- a/django_project/geocontext/utilities/worker.py +++ b/django_project/geocontext/utilities/worker.py @@ -130,7 +130,7 @@ def retrieve_caches(self, services: QuerySet) -> list: last_subsrt=Right('service__key', StrIndex(Reverse('service__key'), Value('_')) - 1, output_field=CharField()), ) - return sorted(Cache.objects.none(), key=lambda cache: self.get_order(cache.service.id)) + return sorted(caches, key=lambda cache: self.get_order(cache.service.id)) def bulk_create_caches(self, new_async_services: list) -> list: """Bulk update cache with new AsyncService values.