Add environment variable configuration

.dvc/config

@@ -1,7 +1,7 @@
 [core]
     remote = myremote
 ['remote "myremote"']
-    url = webdavs://tubcloud.tu-berlin.de/remote.php/dav/files/cf531c5e-2043-103b-8745-111da40a61ee/dvcR
+    url = webdavs://tubcloud.tu-berlin.de/remote.php/dav/files/cf531c5e-2043-103b-8745-111da40a61ee/rDVC
     ask_password = true
     timeout = 600
 

.github/workflows/docker-image.yml

@@ -20,6 +20,10 @@ jobs:
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v1
 
+    - name: Load Environment Variables from global.env
+      run: |
+        cat global.env >> $GITHUB_ENV
+
     - name: Login to DockerHub
       uses: docker/login-action@v1
       with:
@@ -32,4 +36,4 @@ jobs:
         context: .
         file: ./Dockerfile
         push: true
-        tags: ${{ secrets.DOCKER_USERNAME }}/ml-pipeline-image:latest
+        tags: ${{ secrets.DOCKER_USERNAME }}/$TUSTU_PROJECT_NAME-image:latest

.gitignore

@@ -28,3 +28,4 @@ NOTES.md
 # Ignore singularity image
 ml-pipeline-image_latest.sif
 
+local.env

Dockerfile

@@ -16,10 +16,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libbz2-dev \
     && rm -rf /var/lib/apt/lists/*
 
-# Install Python 3.12.4
-RUN wget --no-check-certificate https://www.python.org/ftp/python/3.12.4/Python-3.12.4.tgz \
-    && tar -xf Python-3.12.4.tgz \
-    && cd Python-3.12.4 \
+# Install Python Version
+RUN wget --no-check-certificate https://www.python.org/ftp/python/$TUSTU_PYTHON_VERSION/Python-$TUSTU_PYTHON_VERSION.tgz \
+    && tar -xf Python-$TUSTU_PYTHON_VERSION.tgz \
+    && cd Python-$TUSTU_PYTHON_VERSION\
     && ./configure --enable-optimizations \
     && make -j$(nproc) \
     && make altinstall \

batchjob.sh

@@ -17,11 +17,14 @@ ulimit -u 512
 module load singularity/4.0.2 
 module load nvidia/cuda/12.2 
 
+chmod +x unix_set_env.sh
+source unix_set_env.sh
+
 # Check and pull the latest container image
-if [ -f ml-pipeline-image_latest.sif ]; then
-    rm ml-pipeline-image_latest.sif
+if [ -f $TUSTU_PROJECT_NAME-image_latest.sif ]; then
+    rm $TUSTU_PROJECT_NAME-image_latest.sif
 fi
-singularity pull docker://michaelwitte/ml-pipeline-image:latest 
+singularity pull docker://$TUSTU_DOCKERHUB_USERNAME/$TUSTU_PROJECT_NAME-image:latest 
 
 echo "Cleaning up the logs directory..."
 find ./logs -type f ! -name "slurm-$SLURM_JOB_ID.out" -delete
@@ -33,8 +36,8 @@ STORAGE_DEFAULT_DIRECTORY="$PWD" singularity exec --nv --bind $(pwd):/usr/src/ap
   # Add the github.com host key to the known hosts file
   ssh-keyscan github.com >> /root/.ssh/known_hosts &&
   # Set the git user name and email
-  git config --global user.name "michaelwitte" &&     
-  git config --global user.email "[email protected]" && 
+  git config --global user.name $TUSTU_GITHUB_USERNAME &&     
+  git config --global user.email $TUSTU_GITHUB_EMAIL && 
   # Activate the virtual environment, located at a different path than the containers working directory 
   source /usr/src/cntnrvenv/bin/activate &&                
   # Pull the latest raw data for the pipeline and run the experiment

dvc.yaml

@@ -16,6 +16,7 @@ stages:
     cmd: python3 source/train.py
     deps:
     - source/train.py
+    - source/model.py
     - data/processed/
     params:
     - train.name

global.env

@@ -0,0 +1,5 @@
+TUSTU_PROJECT_NAME=ml-pipeline
+TUSTU_PYTHON_VERSION=3.12.4
+TUSTU_GITHUB_EMAIL=[email protected]
+TUSTU_GITHUB_USERNAME=michaelwitte
+TUSTU_DOCKERHUB_USERNAME=michaelwitte

unix_set_env.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+load_env() {
+  while IFS= read -r line; do
+    if [ ! -z "$line" ]; then
+      export "$line"
+    fi
+  done < "$1"
+  echo "Environment variables set from $1."
+}
+
+if [ -f "global.env" ]; then
+  load_env "global.env"
+fi
+