Merge branch 'master2release' into release

.env.example

@@ -3,7 +3,7 @@ ENVIRONMENT=test
 
 QFIELDCLOUD_HOST=localhost
 DJANGO_SETTINGS_MODULE=qfieldcloud.settings
-DJANGO_ALLOWED_HOSTS=localhost 127.0.0.1 0.0.0.0 app
+DJANGO_ALLOWED_HOSTS=localhost 127.0.0.1 0.0.0.0 app nginx
 
 SECRET_KEY=change_me
 
@@ -81,6 +81,10 @@ QFIELDCLOUD_WORKER_QFIELDCLOUD_URL=http://app:8000/api/v1/
 # DEFAULT: 1
 QFIELDCLOUD_WORKER_REPLICAS=1
 
+# QFieldCloud subscription model
+# DEFAULT: subscription.Subscription
+QFIELDCLOUD_SUBSCRIPTION_MODEL=subscription.Subscription
+
 # The Django development port. Not used in production.
 # DEFAULT: 8011
 DJANGO_DEV_PORT=8011
@@ -102,7 +106,18 @@ SMTP4DEV_SMTP_PORT=25
 # DEFAULT: 143
 SMTP4DEV_IMAP_PORT=143
 
+# Prefix used by docker compose for each of the containers, e.g. app will be `qfieldcloud_app`
+# DEFAULT: qfieldcloud
 COMPOSE_PROJECT_NAME=qfieldcloud
+
+# List of docker compose files
+# DEFAULT: docker-compose.yml:docker-compose.override.local.yml
 COMPOSE_FILE=docker-compose.yml:docker-compose.override.local.yml
-# required for making COMPOSE_FILE above cross-platform (do not change)
+
+# Separator in `COMPOSE_FILE` between filenames. Required for making COMPOSE_FILE above cross-platform (do not change)
+# DEFAULT: :
 COMPOSE_PATH_SEPARATOR=:
+
+# Path to the nginx, letsencrypt, etc configuration files, used by script in `./scripts/`.
+# DEFAULT: ./conf
+CONFIG_PATH=./conf

.github/disabled-workflows/deploy_dev.yml

@@ -7,7 +7,7 @@ on:
 jobs:
   deploy:
     name: Deploy on dev.qfield.cloud
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
     steps:
       - name: Deploy
         uses: appleboy/ssh-action@master
@@ -23,15 +23,15 @@ jobs:
           script_stop: true
           script: |
             cd /opt/qfieldcloud
-            docker-compose -f docker-compose.dev.yml stop
+            docker compose -f docker-compose.dev.yml stop
             echo "$DEV_PASSWORD" | sudo -S git pull https://"$REPO_USERNAME":"$REPO_TOKEN"@github.com/opengisch/qfieldcloud
-            docker-compose -f docker-compose.dev.yml up -d --build
-            docker-compose -f docker-compose.dev.yml exec -T web python manage.py collectstatic --no-input --clear
-            docker-compose -f docker-compose.dev.yml exec -T web python manage.py migrate --noinput
+            docker compose -f docker-compose.dev.yml up -d --build
+            docker compose -f docker-compose.dev.yml exec -T web python manage.py collectstatic --no-input --clear
+            docker compose -f docker-compose.dev.yml exec -T web python manage.py migrate --noinput
 
   status:
     name: Check dev.qfield.cloud status
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
     needs: deploy
     steps:
       - name: Check

.github/workflows/backport.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   backport:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     name: Backport
     steps:
       - name: Backport

.github/workflows/build_and_push.yml

@@ -8,7 +8,7 @@ on:
       - "v*.*.*"
 jobs:
   build_and_push:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout
         uses: actions/checkout@v2

.github/workflows/release_drafter.yml

@@ -8,7 +8,7 @@ on:
 
 jobs:
   update_release_draft:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       # Drafts your next Release notes as Pull Requests are merged into "master"
       - uses: release-drafter/release-drafter@v5

.github/workflows/stale.yml

@@ -5,7 +5,7 @@ on:
 
 jobs:
   stale:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
     - uses: actions/stale@v4
       with:

.github/workflows/test.yml

@@ -5,7 +5,7 @@ on:
 jobs:
   test:
     name: Code check and tests
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout repo
         uses: actions/checkout@v2
@@ -31,19 +31,34 @@ jobs:
           sed -ri 's/^COMPOSE_FILE=(.*)/COMPOSE_FILE=\1:docker-compose.override.test.yml/g' .env
           eval $(egrep "^[^#;]" .env | xargs -d'\n' -n1 | sed -E 's/(\w+)=(.*)/export \1='"'"'\2'"'"'/g')
       - name: Pull docker containers
-        run: docker-compose pull
+        run: docker compose pull
       - name: Build and run docker containers
         run: |
-          docker-compose up -d --build
+          docker compose up -d --build
       - name: Initial manage.py commands
         run: |
-          docker-compose run app python manage.py migrate
-          docker-compose run app python manage.py collectstatic
+          docker compose run app python manage.py migrate
+          docker compose run app python manage.py collectstatic
       - name: Run unit and integration tests
         run: |
-          docker-compose run app python manage.py test --keepdb -v2 qfieldcloud
+          docker compose run app python manage.py test --keepdb -v2 qfieldcloud
 
       - name: "failure logs"
         if: failure()
         run: |
-          docker-compose logs
+          docker compose logs
+
+      - name: Post Google Chat message on failure
+        if: ${{ failure() }}
+        uses: julb/action-post-googlechat-message@v1
+        with:
+          message: |
+            Failed job run for branch `${{ github.head_ref || github.ref_name }}`, check ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} .
+          gchat_webhook_url: ${{ secrets.GOOGLE_CHAT_WEBHOOK_URL }}
+
+      - name: Setup tmate session
+        if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3
+        timeout-minutes: 30
+        with:
+          limit-access-to-actor: true

README.md

@@ -37,41 +37,41 @@ desire with a good editor:
 
 To build development images and run the containers:
 
-    docker-compose up -d --build
+    docker compose up -d --build
 
 It will read the `docker-compose*.yml` files specified in the `COMPOSE_FILE`
-variable and start a django built-in server at `http://localhost:8111`.
+variable and start a django built-in server at `http://localhost:8011`.
 
 Run the django database migrations.
 
-    docker-compose exec app python manage.py migrate
+    docker compose exec app python manage.py migrate
 
 And collect the static files (CSS, JS etc):
 
-    docker-compose run app python manage.py collectstatic --noinput
+    docker compose run app python manage.py collectstatic --noinput
 
 You can check if everything seems to work correctly using the
 `status` command:
 
-    docker-compose exec app python manage.py status
+    docker compose exec app python manage.py status
 
 Now you can get started by adding the first user that would also be a super user:
 
-    docker-compose run app python manage.py createsuperuser --username super_user --email [email protected]
+    docker compose run app python manage.py createsuperuser --username super_user --email [email protected]
 
 ### Tests
 
 To run all the unit and functional tests (on a throwaway test
 database and a throwaway test storage directory):
 
     export COMPOSE_FILE=docker-compose.yml:docker-compose.override.local.yml:docker-compose.override.test.yml
-    docker-compose up -d
-    docker-compose run app python manage.py migrate
-    docker-compose run app python manage.py test --keepdb
+    docker compose up -d
+    docker compose run app python manage.py migrate
+    docker compose run app python manage.py test --keepdb
 
 To run only a test module (e.g. `test_permission.py`)
 
-    docker-compose run app python manage.py test qfieldcloud.core.tests.test_permission
+    docker compose run app python manage.py test qfieldcloud.core.tests.test_permission
 
 ### Debugging
 
@@ -88,10 +88,9 @@ debugpy.wait_for_client()  # optional
 ```
 
 Or alternativley, prefix your commands with `python -m debugpy --listen 0.0.0.0:5680 --wait-for-client`.
-```shell
-docker-compose run app -p 5680:5680 python -m debugpy --listen 0.0.0.0:5680 --wait-for-client manage.py test
-docker-compose run worker_wrapper -p 5681:5681 python -m debugpy --listen 0.0.0.0:5681 --wait-for-client manage.py test
-```
+
+    docker compose run app -p 5680:5680 python -m debugpy --listen 0.0.0.0:5680 --wait-for-client manage.py test
+    docker compose run worker_wrapper -p 5681:5681 python -m debugpy --listen 0.0.0.0:5681 --wait-for-client manage.py test
 
 Then, configure your IDE to connect (example given for VSCode's `.vscode/launch.json`, triggered with `F5`):
 ```
@@ -203,11 +202,11 @@ Create the directory for qfieldcloud logs and supervisor socket file
 
 Run and build the docker containers
 
-    docker-compose up -d --build
+    docker compose up -d --build
 
 Run the django database migrations
 
-    docker-compose exec app python manage.py migrate
+    docker compose exec app python manage.py migrate
 
 
 ## Create a certificate using Let's Encrypt
@@ -243,15 +242,14 @@ Based on this example
 
 Docker logs are managed by docker in the default way. To read the logs:
 
-    docker-compose logs
+    docker compose logs
 
 
 For great `nginx` logs, use:
 
     QFC_JQ='[.ts, .ip, (.method + " " + (.status|tostring) + " " + (.resp_time|tostring) + "s"), .uri, "I " + (.request_length|tostring) + " O " + (.resp_body_size|tostring), "C " + (.upstream_connect_time|tostring) + "s", "H " + (.upstream_header_time|tostring) + "s", "R " + (.upstream_response_time|tostring) + "s", .user_agent] | @tsv'
     docker compose logs nginx -f --no-log-prefix | grep ':"nginx"' | jq -r $QFC_JQ
 
-
 ### Geodb
 
 The geodb (database for the users projects data) is installed on

conf/nginx/templates/default.conf.template

@@ -58,7 +58,7 @@ server {
 
   access_log /var/log/nginx/access.log json-logger;
 
-  server_name ${QFIELDCLOUD_HOST};
+  server_name ${QFIELDCLOUD_HOST} nginx;
   client_max_body_size 10G;
   keepalive_timeout 5;
 

docker-app/Dockerfile

@@ -1,50 +1,110 @@
-###########
-# BUILDER #
-###########
+##########################
+# BUILDER                #
+##########################
+
+# pull a builder image, the same as the base
+FROM python:3.10.8-slim-buster as build
+
+# Disable annoying pip version check, we don't care if pip is slightly older
+ARG PIP_DISABLE_PIP_VERSION_CHECK 1
+
+# Do not create and use redundant cache dir in the current user home
+ARG PIP_NO_CACHE_DIR 1
+
+# install psycopg2 requirements
+RUN apt update && apt install -y libpq-dev python3-dev gcc
+
+# install dependencies
+COPY ./requirements.txt .
+RUN pip3 install -r requirements.txt
+
+# save 60 MB of data... Safe to delete according to botocore contributor https://github.com/boto/botocore/issues/1629#issuecomment-451309885
+RUN ls -Q /usr/local/lib/python3.10/site-packages/botocore/data | grep -xv "endpoints.json" | xargs rm -rf
+
+##########################
+# BASE                   #
+##########################
 
 # pull official base image
-FROM python:3.8.0-buster as builder
+FROM python:3.10.8-slim-buster as base
 
 # set work directory
 WORKDIR /usr/src/app
 
 # set environment variables
 ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONUNBUFFERED 1
+# check `build` stage for more info
+# NOTE while using ARG would be more appropriate, the following vars would have to be redifined for each build stage.
+ENV PIP_DISABLE_PIP_VERSION_CHECK 1
+ENV PIP_NO_CACHE_DIR 1
 
 # install dependencies
-RUN apt update \
-    && apt install -y netcat python3-gdal
+RUN apt update && apt-get install -y \
+# GeoDjango as recommended at https://docs.djangoproject.com/en/4.1/ref/contrib/gis/install/geolibs/#installing-geospatial-libraries
+    binutils libproj-dev gdal-bin
 
-# install dependencies
-COPY ./requirements.txt .
-RUN pip install -r requirements.txt
+# copy the dependencies
+COPY --from=build /usr/local/lib/python3.10/site-packages/ /usr/local/lib/python3.10/site-packages/
 
 # install debug dependencies
 ARG DEBUG_BUILD
-RUN if [ "$DEBUG_BUILD" = "1" ]; then pip install debugpy; fi
+RUN if [ "$DEBUG_BUILD" = "1" ]; then pip3 install debugpy; fi
 
-# copy project
-COPY . .
+# add app group
+RUN addgroup --system app && adduser --system app --ingroup app
 
 # create the appropriate directories
 RUN mkdir staticfiles
 RUN mkdir mediafiles
 
-# chown all the files to the app user
-RUN addgroup --system app && adduser --system app --ingroup app
-RUN chown -R app:app .
+# create an empty JSON fixture for the sole purpose of Django's testserver
+RUN echo '{}' > fixture.json
+
+# Do not uncomment unless you want to share the user id between the host and the container
 # RUN groupmod -g 1000 app
 # RUN usermod -u 1000 app
 
-# change to the app user
+COPY ./entrypoint.sh /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]
+
+##########################
+# WEBSERVER RUNTIME      #
+##########################
+
+# a separate stage for webserver runtime environment
+FROM base as webserver_runtime
+COPY ./requirements_runtime.txt .
+RUN pip3 install -r requirements_runtime.txt
+EXPOSE 8000
+
+COPY . .
+RUN chown -R app:app .
 USER app
 
+##########################
+# WEBSERVER TEST         #
+##########################
+
+# a separate stage for webserver test environment
+FROM base as webserver_test
+COPY ./requirements_test.txt .
+RUN pip3 install -r requirements_test.txt
 EXPOSE 8000
 
-# create an empty JSON fixture for the sole purpose of Django's testserver
-RUN echo '{}' > fixture.json
+COPY . .
+RUN chown -R app:app .
+USER app
 
-# run entrypoint.prod.sh
-COPY ./entrypoint.sh /entrypoint.sh
-ENTRYPOINT ["/entrypoint.sh"]
+##########################
+# WORKER WRAPPER RUNTIME #
+##########################
+
+# a separate stage for worker wrapper runtime environment
+FROM base as worker_wrapper_runtime
+COPY ./requirements_worker_wrapper.txt .
+RUN pip3 install -r requirements_worker_wrapper.txt
+
+COPY . .
+RUN chown -R app:app .
+USER app