New docker compose configuration (#187)

* Refactored Compose files

* Removed unnecessary overrides

* Configurable default network name

* Added override for using prebuilt development version image

* Tagging release candidates build with 'rc' tag

* Removed service name from RC override

* Updated README

* Fixed condition in Actions workflow

* Fixed PR CI workflow

* Fixed PR CI workflow once again

* Fixed condition in CI workflow

Also removed RUN_ON_MASTER variable because we don't need it: on merge to master the workflow is not triggered, and on tag push the event will be 'semver', not 'branch'

* Added depends_on to the api service

* Fixed selenoid network name for connection

* Added restart policy

Stop services only after they are explicitly stopped.
This enables automatic application startup after system reboot

* Removed unnecessary chrome service

* Using start_celery.sh script by default

* Extended .dockerignore

* Added external network for Selenoid to compensate removal of Chrome service

* Storing Mongo data in a Docker volume instead of project directory

* Fixed PR CI workflow

* Should be working without new network

* Legacy directory removed from .gitignore

.dockerignore

@@ -36,3 +36,14 @@ data/uploaded_dataset
 data/other
 
 **/notebooks
+
+# Git data is not needed in Docker container
+.git/
+.gitignore
+.gitmodules
+
+# These scripts are unnecessary in Docker container
+build_docker.sh
+create-env.bat
+remove-env.bat
+run_docker.sh

.github/workflows/docker-image-update.yml

@@ -9,6 +9,9 @@ on:
     branches:
       - master
 
+env:
+  RUN_ON_RC: ${{ startsWith( github.head_ref, 'release/' ) }}
+
 jobs:
   build-and-push:
     runs-on: ubuntu-latest
@@ -28,10 +31,11 @@ jobs:
             org.opencontainers.image.title=JDI QASP ML
             org.opencontainers.image.description=Backend for JDN Chrome plugin
           tags: |
-            type=ref,event=branch,enable=${{ github.ref != 'refs/heads/master' }}
-            type=ref,event=pr
+            type=ref,event=branch
+            type=ref,event=pr,enable=${{ env.RUN_ON_RC == 'false' }}
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
+            type=raw,value=rc,enable=${{ env.RUN_ON_RC == 'true' }}
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v3
         with:

.github/workflows/pull-request.yml

@@ -9,7 +9,7 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     env:
-      COMPOSE_FILE: docker-compose.dev.yaml
+      COMPOSE_FILE: docker-compose.yaml:docker-compose.override.dev.yaml
       SELENOID_PARALLEL_SESSIONS_COUNT: 2
     steps:
       - name: Checkout repository

.gitignore

@@ -53,3 +53,5 @@ coverage.xml
 .env
 .env.rc
 .env.dev
+
+docker-compose.override.yaml

Dockerfile

@@ -11,13 +11,15 @@ RUN apt install -y curl wget mc gcc make
 ENV APP_HOME=/jdi-qasp-ml
 WORKDIR ${APP_HOME}
 
-COPY Pipfile* ${APP_HOME}/
+COPY Pipfile* ./
 
 RUN pip install -U pip && \
     pip install pipenv
 
 RUN pipenv install --ignore-pipfile --system --deploy
 
-COPY . ${APP_HOME}
+COPY . ./
+
+RUN chmod +x start_celery.sh
 
 ENV PYTHONPATH=${PYTHONPATH}:/jdi-qasp-ml

README.md

@@ -167,33 +167,34 @@ To validate models quality we use test web-pages, placed in directory **notebook
 
 # Docker
 ## Take docker image from github:
-### RC version
+### Release version
 #### macOS
 ```shell
-export SELENOID_PARALLEL_SESSIONS_COUNT=$(sysctl -n hw.ncpu) && docker rm --force jdi-qasp-ml-api && docker image rm ghcr.io/jdi-testing/jdi-qasp-ml:rc --force && curl --output docker-compose.yaml --url https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/rc/docker-compose-rc.yaml && curl --output browsers.json https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/rc/browsers.json && docker-compose up
+export SELENOID_PARALLEL_SESSIONS_COUNT=$(sysctl -n hw.ncpu) && docker rm --force jdi-qasp-ml-api && docker image rm ghcr.io/jdi-testing/jdi-qasp-ml:latest --force && curl --output docker-compose.yaml --url https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/master/docker-compose.yaml && curl --output browsers.json https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/master/browsers.json && docker-compose up
 ```
 #### Linux
 ```shell
-export SELENOID_PARALLEL_SESSIONS_COUNT=$(nproc) && docker rm --force jdi-qasp-ml-api && docker image rm ghcr.io/jdi-testing/jdi-qasp-ml:rc --force && curl --output docker-compose.yaml https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/rc/docker-compose-rc.yaml && curl --output browsers.json https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/rc/browsers.json && docker compose up
+export SELENOID_PARALLEL_SESSIONS_COUNT=$(nproc) && docker rm --force jdi-qasp-ml-api && docker image rm ghcr.io/jdi-testing/jdi-qasp-ml:latest --force && curl --output docker-compose.yaml https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/master/docker-compose.yaml && curl --output browsers.json https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/master/browsers.json && docker compose up
 ```
 #### Windows
 ```shell
-set SELENOID_PARALLEL_SESSIONS_COUNT=%NUMBER_OF_PROCESSORS%&& docker rm --force jdi-qasp-ml-api && docker image rm ghcr.io/jdi-testing/jdi-qasp-ml:rc --force && curl.exe --output docker-compose.yaml --url https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/rc/docker-compose-rc.yaml && curl.exe --output browsers.json --url https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/rc/browsers.json && docker compose up
+set SELENOID_PARALLEL_SESSIONS_COUNT=%NUMBER_OF_PROCESSORS%&& docker rm --force jdi-qasp-ml-api && docker image rm ghcr.io/jdi-testing/jdi-qasp-ml:latest --force && curl.exe --output docker-compose.yaml --url https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/master/docker-compose.yaml && curl.exe --output browsers.json --url https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/master/browsers.json && docker compose up
 ```
 
 ### Development version
 #### macOS
 ```shell
-export SELENOID_PARALLEL_SESSIONS_COUNT=$(sysctl -n hw.ncpu) && docker rm --force jdi-qasp-ml-api && docker image rm ghcr.io/jdi-testing/jdi-qasp-ml:latest --force && curl --output docker-compose.yaml https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/develop/docker-compose.yaml && curl --output browsers.json https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/develop/browsers.json && docker-compose up
+export SELENOID_PARALLEL_SESSIONS_COUNT=$(sysctl -n hw.ncpu) && docker rm --force jdi-qasp-ml-api && docker image rm ghcr.io/jdi-testing/jdi-qasp-ml:develop --force && curl --output docker-compose.yaml https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/develop/docker-compose.yaml && curl --output docker-compose.override.yaml https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/develop/docker-compose.override.prebuilt-dev.yaml && curl --output browsers.json https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/develop/browsers.json && docker-compose up
 ```
 #### Linux
 ```shell
-export SELENOID_PARALLEL_SESSIONS_COUNT=$(nproc) && docker rm --force jdi-qasp-ml-api && docker image rm ghcr.io/jdi-testing/jdi-qasp-ml:latest --force && curl --output docker-compose.yaml https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/develop/docker-compose.yaml && curl --output browsers.json https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/develop/browsers.json && docker compose up
+export SELENOID_PARALLEL_SESSIONS_COUNT=$(nproc) && docker rm --force jdi-qasp-ml-api && docker image rm ghcr.io/jdi-testing/jdi-qasp-ml:develop --force && curl --output docker-compose.yaml https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/develop/docker-compose.yaml && curl --output docker-compose.override.yaml https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/develop/docker-compose.override.prebuilt-dev.yaml && curl --output browsers.json https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/develop/browsers.json && docker compose up
 ```
 #### Windows
 ```shell
-set SELENOID_PARALLEL_SESSIONS_COUNT=%NUMBER_OF_PROCESSORS%&& docker rm --force jdi-qasp-ml-api && docker image rm ghcr.io/jdi-testing/jdi-qasp-ml:latest --force && curl.exe --output docker-compose.yaml --url https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/develop/docker-compose.yaml && curl.exe --output browsers.json --url https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/develop/browsers.json && docker compose up
+set SELENOID_PARALLEL_SESSIONS_COUNT=%NUMBER_OF_PROCESSORS%&& docker rm --force jdi-qasp-ml-api && docker image rm ghcr.io/jdi-testing/jdi-qasp-ml:develop --force && curl.exe --output docker-compose.yaml --url https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/develop/docker-compose.yaml && curl.exe --output docker-compose.override.yaml https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/develop/docker-compose.override.prebuilt-dev.yaml && curl.exe --output browsers.json --url https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/develop/browsers.json && docker compose up
 ```
+
 ## Installing version from any other repository branch:
 Example with branch "branch_name":
 

app/selenium_app.py

@@ -28,7 +28,7 @@ def get_webdriver() -> webdriver.Remote:
     }
 
     return webdriver.Remote(
-        command_executor="http://jdi-qasp-ml-selenoid:4444/wd/hub",
+        command_executor="http://selenoid:4444/wd/hub",
         desired_capabilities=capabilities,
         options=chrome_options,
     )

docker-compose.override.dev.yaml

@@ -0,0 +1,18 @@
+services:
+  selenoid:
+    ports:
+      - "${SELENOID_PORT:-4445}:4444"
+
+  api:
+    image: !reset null
+    build:
+      context: .
+      dockerfile: Dockerfile
+    volumes:
+      - .:/jdi-qasp-ml
+
+  celery:
+    image: !reset null
+    build:
+      context: .
+      dockerfile: Dockerfile

docker-compose.override.prebuilt-dev.yaml

@@ -0,0 +1,6 @@
+services:
+  api:
+    image: "ghcr.io/jdi-testing/jdi-qasp-ml:develop"
+
+  celery:
+    image: "ghcr.io/jdi-testing/jdi-qasp-ml:develop"

docker-compose.override.rc.yaml

@@ -0,0 +1,10 @@
+services:
+  api:
+    image: "ghcr.io/jdi-testing/jdi-qasp-ml:rc"
+    environment:
+      ENV: LOCAL
+
+  celery:
+    image: "ghcr.io/jdi-testing/jdi-qasp-ml:rc"
+
+  mongodb: !reset []