add standard bash scripts

AfricasVoices · Apr 27, 2022 · b6ca831 · b6ca831
1 parent e56a0c0
commit b6ca831
Show file tree

Hide file tree

Showing 10 changed files with 621 additions and 0 deletions.
diff --git a/archive_data_dir.sh b/archive_data_dir.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -e
+
+if [[ $# -ne 2 ]]; then
+    echo "Usage: ./archive_data_dir <data-dir> <archive-file>"
+    echo "Backs-up the data root directory to a compressed file in at the specified location"
+    exit
+fi
+
+DATA_DIR=$1
+ARCHIVE_FILE=$2
+
+mkdir -p "$(dirname "$ARCHIVE_FILE")"
+find "$DATA_DIR" -type f -name '.DS_Store' -delete
+cd "$DATA_DIR"
+echo "tarring into -czvf $DATA_DIR"
+tar -czvf "$ARCHIVE_FILE" .
diff --git a/docker-run-engagement-db-to-analysis.sh b/docker-run-engagement-db-to-analysis.sh
@@ -0,0 +1,74 @@
+#!/bin/bash
+
+set -e
+
+IMAGE_NAME="$(<configurations/docker_image_name.txt)"
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --incremental-cache-volume)
+            INCREMENTAL_ARG="--incremental-cache-path /cache"
+            INCREMENTAL_CACHE_VOLUME_NAME="$2"
+            shift 2;;
+        --)
+            shift
+            break;;
+        *)
+            break;;
+    esac
+done
+
+# Check that the correct number of arguments were provided.
+if [[ $# -ne 5 ]]; then
+    echo "Usage: $0 
+    [--incremental-cache-volume <incremental-cache-volume>] 
+    <user> <google-cloud-credentials-file-path> <configuration-file> <code-schemes-dir> <data-dir>"
+    exit
+fi
+
+# Assign the program arguments to bash variables.
+USER=$1
+GOOGLE_CLOUD_CREDENTIALS_PATH=$2
+CONFIGURATION_FILE=$3
+CODE_SCHEMES_DIR=$4
+DATA_DIR=$5
+
+CMD="pipenv run python -u engagement_db_to_analysis.py ${INCREMENTAL_ARG} ${USER} \
+    /credentials/google-cloud-credentials.json configuration /data/membership-groups /data/analysis-outputs"
+
+if [[ "$INCREMENTAL_ARG" ]]; then
+    container="$(docker container create -w /app --mount source="$INCREMENTAL_CACHE_VOLUME_NAME",target=/cache "$IMAGE_NAME" /bin/bash -c "$CMD")"
+else
+    container="$(docker container create -w /app "$IMAGE_NAME" /bin/bash -c "$CMD")"
+fi
+
+echo "Created container $container"
+container_short_id=${container:0:7}
+
+# Copy input data into the container
+echo "Copying $GOOGLE_CLOUD_CREDENTIALS_PATH -> $container_short_id:/credentials/google-cloud-credentials.json"
+docker cp "$GOOGLE_CLOUD_CREDENTIALS_PATH" "$container:/credentials/google-cloud-credentials.json"
+
+echo "Copying $CODE_SCHEMES_DIR -> $container_short_id:/app/code_schemes"
+docker cp "$CODE_SCHEMES_DIR" "$container:/app/code_schemes"
+
+echo "Copying $CONFIGURATION_FILE -> $container_short_id:/app/configuration.py"
+docker cp "$CONFIGURATION_FILE" "$container:/app/configuration.py"
+
+# Run the container
+echo "Starting container $container_short_id"
+docker start -a -i "$container"
+
+# Copy the output data back out of the container
+echo "Copying $container_short_id:/data/. -> $DATA_DIR"
+docker cp "$container:/data/." "$DATA_DIR"
+
+# Copy cache data out of the container for backup
+if [[ "$INCREMENTAL_ARG" ]]; then
+    echo "Copying $container_short_id:/cache/. -> $DATA_DIR/Cache"
+    mkdir -p "$DATA_DIR/Cache"
+    docker cp "$container:/cache/." "$DATA_DIR/Cache"
+fi
+
+# Tear down the container when it has run successfully
+docker container rm "$container" >/dev/null
diff --git a/docker-run-log-pipeline-event.sh b/docker-run-log-pipeline-event.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+set -e
+
+IMAGE_NAME="$(<configurations/docker_image_name.txt)"
+
+# Check that the correct number of arguments were provided.
+if [[ $# -ne 5 ]]; then
+    echo "Usage: ./docker-run-log-pipeline-event.sh
+    [--profile-cpu <cpu-profile-output-path>] <configuration-file> <code-schemes-dir>
+     <google-cloud-credentials-file-path> <run-id> <event-key>"
+    echo "Updates pipeline event/status to a firebase table to aid in monitoring"
+    exit 1
+fi
+
+# Assign the program arguments to bash variables.
+CONFIGURATION_FILE=$1
+CODE_SCHEMES_DIR=$2
+GOOGLE_CLOUD_CREDENTIALS_PATH=$3
+RUN_ID=$4
+EVENT_KEY=$5
+
+CMD="pipenv run python -u log_pipeline_event.py configuration /credentials/google-cloud-credentials.json \
+       ${RUN_ID} ${EVENT_KEY}"
+
+container="$(docker container create -w /app "$IMAGE_NAME" /bin/bash -c "$CMD")"
+
+echo "Created container $container"
+container_short_id=${container:0:7}
+
+# Copy input data into the container
+echo "Copying $GOOGLE_CLOUD_CREDENTIALS_PATH -> $container_short_id:/credentials/google-cloud-credentials.json"
+docker cp "$GOOGLE_CLOUD_CREDENTIALS_PATH" "$container:/credentials/google-cloud-credentials.json"
+
+echo "Copying $CODE_SCHEMES_DIR -> $container_short_id:/app/code_schemes"
+docker cp "$CODE_SCHEMES_DIR" "$container:/app/code_schemes"
+
+echo "Copying $CONFIGURATION_FILE -> $container_short_id:/app/configuration.py"
+docker cp "$CONFIGURATION_FILE" "$container:/app/configuration.py"
+
+# Run the container
+echo "Starting container $container_short_id"
+docker start -a -i "$container"
+
+# Tear down the container when it has run successfully
+docker container rm "$container" >/dev/null
diff --git a/docker-run-telegram-group-to-engagement-db.sh b/docker-run-telegram-group-to-engagement-db.sh
@@ -0,0 +1,75 @@
+#!/bin/bash
+
+set -e
+
+PROJECT_NAME="$(<configurations/docker_image_project_name.txt)"
+IMAGE_NAME=$PROJECT_NAME-telegram-group-to-engagement-db
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --incremental-cache-volume)
+            INCREMENTAL_ARG="--incremental-cache-path /cache"
+            INCREMENTAL_CACHE_VOLUME_NAME="$2"
+            shift 2;;
+        --)
+            shift
+            break;;
+        *)
+            break;;
+    esac
+done
+
+# Check that the correct number of arguments were provided.
+if [[ $# -ne 5 ]]; then
+    echo "Usage: $0
+    [--incremental-cache-volume <incremental-cache-volume>]
+    <user> <google-cloud-credentials-file-path> <configuration-file> <code-schemes-dir> <data-dir>"
+    exit
+fi
+
+# Assign the program arguments to bash variables.
+USER=$1
+GOOGLE_CLOUD_CREDENTIALS_PATH=$2
+CONFIGURATION_FILE=$3
+CODE_SCHEMES_DIR=$4
+DATA_DIR=$5
+
+# Build an image for this pipeline stage.
+docker build -t "$IMAGE_NAME" .
+
+# Create a container from the image that was just built.
+CMD="pipenv run python -u sync_telegram_group_to_engagement_db.py ${INCREMENTAL_ARG} ${USER} \
+    /credentials/google-cloud-credentials.json configuration"
+
+if [[ "$INCREMENTAL_ARG" ]]; then
+    container="$(docker container create -t -w /app --mount source="$INCREMENTAL_CACHE_VOLUME_NAME",target=/cache "$IMAGE_NAME" /bin/bash -c "$CMD")"
+else
+    container="$(docker container create -t -w /app "$IMAGE_NAME" /bin/bash -c "$CMD")"
+fi
+
+echo "Created container $container"
+container_short_id=${container:0:7}
+
+# Copy input data into the container
+echo "Copying $GOOGLE_CLOUD_CREDENTIALS_PATH -> $container_short_id:/credentials/google-cloud-credentials.json"
+docker cp "$GOOGLE_CLOUD_CREDENTIALS_PATH" "$container:/credentials/google-cloud-credentials.json"
+
+echo "Copying $CODE_SCHEMES_DIR -> $container_short_id:/app/code_schemes"
+docker cp "$CODE_SCHEMES_DIR" "$container:/app/code_schemes"
+
+echo "Copying $CONFIGURATION_FILE -> $container_short_id:/app/configuration.py"
+docker cp "$CONFIGURATION_FILE" "$container:/app/configuration.py"
+
+# Run the container
+echo "Starting container $container_short_id"
+docker start -a "$container"
+
+# Copy cache data out of the container for backup
+if [[ "$INCREMENTAL_ARG" ]]; then
+    echo "Copying $container_short_id:/cache/. -> $DATA_DIR/Cache"
+    mkdir -p "$DATA_DIR/Cache"
+    docker cp "$container:/cache/." "$DATA_DIR/Cache"
+fi
+
+# Tear down the container when it has run successfully
+docker container rm "$container" >/dev/null
diff --git a/docker-run-upload-archive-files.sh b/docker-run-upload-archive-files.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+set -e
+
+IMAGE_NAME="$(<configurations/docker_image_name.txt)"
+
+# Check that the correct number of arguments were provided.
+if [[ $# -ne 5 ]]; then
+    echo "Usage: ./docker-run-upload-archive-files.sh
+     <user> <google-cloud-credentials-file-path> <configuration-module> <code-schemes-dir> <archive-dir>"
+    exit 1
+fi
+# Assign the program arguments to bash variables.
+USER=$1
+GOOGLE_CLOUD_CREDENTIALS_PATH=$2
+CONFIGURATION_FILE=$3
+CODE_SCHEMES_DIR=$4
+ARCHIVE_DIR=$5
+
+CMD="pipenv run python -u upload_archive_files.py ${USER} /credentials/google-cloud-credentials.json \
+    configuration /archives"
+
+# Create the container. Note that we use a bind mount here rather than a volume or docker cp so we can directly
+# edit the archive files on the host system.
+container="$(docker container create -w /app --mount type=bind,source="$ARCHIVE_DIR",target=/archives "$IMAGE_NAME" /bin/bash -c "$CMD")"
+
+echo "Created container $container"
+container_short_id=${container:0:7}
+
+# Copy input data into the container
+echo "Copying $GOOGLE_CLOUD_CREDENTIALS_PATH -> $container_short_id:/credentials/google-cloud-credentials.json"
+docker cp "$GOOGLE_CLOUD_CREDENTIALS_PATH" "$container:/credentials/google-cloud-credentials.json"
+
+echo "Copying $CONFIGURATION_FILE -> $container_short_id:/app/configuration.py"
+docker cp "$CONFIGURATION_FILE" "$container:/app/configuration.py"
+
+echo "Copying $CODE_SCHEMES_DIR -> $container_short_id:/app/code_schemes"
+docker cp "$CODE_SCHEMES_DIR" "$container:/app/code_schemes"
+
+# Run the container
+echo "Starting container $container_short_id"
+docker start -a -i "$container"
+
+# Tear down the container when it has run successfully
+docker container rm "$container" >/dev/null
diff --git a/docker-sync-coda-to-engagement-db.sh b/docker-sync-coda-to-engagement-db.sh
@@ -0,0 +1,73 @@
+#!/bin/bash
+
+set -e
+
+IMAGE_NAME="$(<configurations/docker_image_name.txt)"
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --dry-run)
+            DRY_RUN="--dry-run"
+            shift;;
+        --incremental-cache-volume)
+            INCREMENTAL_ARG="--incremental-cache-path /cache"
+            INCREMENTAL_CACHE_VOLUME_NAME="$2"
+            shift 2;;
+        --)
+            shift
+            break;;
+        *)
+            break;;
+    esac
+done
+
+# Check that the correct number of arguments were provided.
+if [[ $# -ne 5 ]]; then
+    echo "Usage: $0 
+    [--dry-run] [--incremental-cache-volume <incremental-cache-volume>]
+    <user> <google-cloud-credentials-file-path> <configuration-file> <code-schemes-dir> <data-dir>"
+    exit
+fi
+
+# Assign the program arguments to bash variables.
+USER=$1
+GOOGLE_CLOUD_CREDENTIALS_PATH=$2
+CONFIGURATION_FILE=$3
+CODE_SCHEMES_DIR=$4
+DATA_DIR=$5
+
+CMD="pipenv run python -u sync_coda_to_engagement_db.py ${DRY_RUN} ${INCREMENTAL_ARG} \
+    ${USER} /credentials/google-cloud-credentials.json configuration"
+
+if [[ "$INCREMENTAL_ARG" ]]; then
+    container="$(docker container create -w /app --mount source="$INCREMENTAL_CACHE_VOLUME_NAME",target=/cache "$IMAGE_NAME" /bin/bash -c "$CMD")"
+else
+    container="$(docker container create -w /app "$IMAGE_NAME" /bin/bash -c "$CMD")"
+fi
+
+echo "Created container $container"
+container_short_id=${container:0:7}
+
+# Copy input data into the container
+echo "Copying $GOOGLE_CLOUD_CREDENTIALS_PATH -> $container_short_id:/credentials/google-cloud-credentials.json"
+docker cp "$GOOGLE_CLOUD_CREDENTIALS_PATH" "$container:/credentials/google-cloud-credentials.json"
+
+echo "Copying $CODE_SCHEMES_DIR -> $container_short_id:/app/code_schemes"
+docker cp "$CODE_SCHEMES_DIR" "$container:/app/code_schemes"
+
+echo "Copying $CONFIGURATION_FILE -> $container_short_id:/app/configuration.py"
+docker cp "$CONFIGURATION_FILE" "$container:/app/configuration.py"
+
+# Run the container
+echo "Starting container $container_short_id"
+docker start -a -i "$container"
+
+# Copy cache data out of the container for backup
+if [[ "$INCREMENTAL_ARG" ]]; then
+    echo "Copying $container_short_id:/cache/. -> $DATA_DIR/Cache"
+    mkdir -p "$DATA_DIR/Cache"
+    docker cp "$container:/cache/." "$DATA_DIR/Cache"
+fi
+
+# Tear down the container when it has run successfully
+docker container rm "$container" >/dev/null