diff --git a/.env.template b/.env.template index 5d32dcd..11281e9 100644 --- a/.env.template +++ b/.env.template @@ -2,4 +2,13 @@ DATABASE_HOST=localhost DATABASE_PORT=5432 DATABASE_NAME=databaseofbabel DATABASE_USERNAME=metakgp_user -DATABASE_PASSWORD=somerandomstringfordevelopment \ No newline at end of file +DATABASE_PASSWORD=somerandomstringfordevelopment +# Path to secret file containing database access information(username, password, etc.) +DBPASSFILE= +# Dropbox api variables(used for storing backups) +DROPBOX_APP_KEY= +DROPBOX_APP_SECRET= +DROPBOX_ACCESS_TOKEN= +DROPBOX_REFRESH_TOKEN= +# Slack Webhook URL used to send incident reports and errors(like Dropbox backup failure) +SLACK_INCIDENTS_WH_URL= diff --git a/.gitignore b/.gitignore index 2eea525..129fc64 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -.env \ No newline at end of file +.env +.pgpass \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index add3340..9a1a0f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1 +1 @@ -FROM postgres:latest +FROM postgres:16 diff --git a/backup/Dockerfile b/backup/Dockerfile new file mode 100644 index 0000000..713ded3 --- /dev/null +++ b/backup/Dockerfile @@ -0,0 +1,28 @@ +FROM postgres:16 + +RUN apt-get update && apt-get install -y curl python3 python3-pip gzip + +ENV TZ=Asia/Kolkata +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +# install supercronic as cron replacement : https://github.com/aptible/supercronic?tab=readme-ov-file#why-supercronic +ENV SUPERCRONIC_URL=https://github.com/aptible/supercronic/releases/download/v0.2.30/supercronic-linux-amd64 \ + SUPERCRONIC=supercronic-linux-amd64 \ + SUPERCRONIC_SHA1SUM=9f27ad28c5c57cd133325b2a66bba69ba2235799 + +RUN curl -fsSLO "$SUPERCRONIC_URL" \ + && echo "${SUPERCRONIC_SHA1SUM} ${SUPERCRONIC}" | sha1sum -c - \ + && chmod +x "$SUPERCRONIC" \ + && mv "$SUPERCRONIC" "/usr/local/bin/${SUPERCRONIC}" \ + && ln -s "/usr/local/bin/${SUPERCRONIC}" /usr/local/bin/supercronic + +WORKDIR /root + +ARG DBPASSFILE + +COPY ./ ./backup +COPY ${DBPASSFILE} ./.pgpass + +RUN pip3 install -qr ./backup/requirements.txt --break-system-packages + +CMD ["supercronic", "/root/backup/crontab"] diff --git a/backup/backup_to_dropbox.py b/backup/backup_to_dropbox.py new file mode 100755 index 0000000..4722e65 --- /dev/null +++ b/backup/backup_to_dropbox.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 + +import dropbox +import traceback +import sys +import os + +file_name = sys.argv[1] + +app_key = os.environ["DROPBOX_APP_KEY"] +app_secret = os.environ["DROPBOX_APP_SECRET"] +access_token = os.environ["DROPBOX_ACCESS_TOKEN"] +refresh_token = os.environ["DROPBOX_REFRESH_TOKEN"] +client = dropbox.Dropbox( + app_key=app_key, + app_secret=app_secret, + oauth2_access_token=access_token, + oauth2_refresh_token=refresh_token, +) + +with open(file_name, "rb") as f: + chunksize = 32 * 1024 * 1024 + + try: + if os.path.getsize(file_name) < chunksize: + result = client.files_upload(f.read(), "/" + file_name) + print(result) + + else: + next_chunk = f.read(chunksize) + + session = client.files_upload_session_start(next_chunk) + uploaded = len(next_chunk) + + next_chunk = f.read(chunksize) + cursor = dropbox.files.UploadSessionCursor(session.session_id, uploaded) + + print("Uploaded: ", float(uploaded) / (1024 * 1024), "MB") + while next_chunk: + client.files_upload_session_append_v2(next_chunk, cursor) + uploaded += len(next_chunk) + cursor = dropbox.files.UploadSessionCursor(session.session_id, uploaded) + print("Uploaded: ", float(uploaded) / (1024 * 1024), "MB") + + next_chunk = f.read(chunksize) + + commit_info = dropbox.files.CommitInfo(path="/" + file_name) + result = client.files_upload_session_finish(f.read(), cursor, commit_info) + print(result) + except Exception as e: + sys.exit(traceback.format_exc()) diff --git a/backup/crontab b/backup/crontab new file mode 100644 index 0000000..7e8dcd7 --- /dev/null +++ b/backup/crontab @@ -0,0 +1 @@ +20 4 * * * /root/backup/run_backup.sh >> /var/log/backups.log 2>&1 diff --git a/backup/requirements.txt b/backup/requirements.txt new file mode 100644 index 0000000..650a4c8 --- /dev/null +++ b/backup/requirements.txt @@ -0,0 +1 @@ +dropbox \ No newline at end of file diff --git a/backup/rotate_backups.py b/backup/rotate_backups.py new file mode 100755 index 0000000..5c4af3c --- /dev/null +++ b/backup/rotate_backups.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 + +import dropbox +import os +import traceback +import sys +import datetime + +app_key = os.environ["DROPBOX_APP_KEY"] +app_secret = os.environ["DROPBOX_APP_SECRET"] +access_token = os.environ["DROPBOX_ACCESS_TOKEN"] +refresh_token = os.environ["DROPBOX_REFRESH_TOKEN"] +client = dropbox.Dropbox( + app_key=app_key, + app_secret=app_secret, + oauth2_access_token=access_token, + oauth2_refresh_token=refresh_token, +) + +BACKUP_FOLDER_PATH = "" +has_more_files = True +cursor = None +result = None +files = [] +now = datetime.datetime.now() +counter = 0 + +try: + print("Fetching files from Dropbox...") + while has_more_files: + if cursor is None: + result = client.files_list_folder(BACKUP_FOLDER_PATH) + else: + result = client.files_list_folder_continue(cursor=cursor) + cursor = result.cursor + has_more_files = result.has_more + files.extend(result.entries) + + for file in files: + if file.name.find("dob_dump") == -1: + files.remove(file) + + number_of_files = len(files) + print(f"{number_of_files} backup files found.") + + print("Starting rotation") + for file in files: + file_timestamp = file.client_modified + days_old = (now - file_timestamp).days + if days_old > 30 and (number_of_files - counter) > 30: + client.files_delete(file.path_display) + counter += 1 + + print(f"{counter} backup file(s) successfully deleted.") +except Exception as e: + sys.exit(traceback.format_exc()) diff --git a/backup/run_backup.sh b/backup/run_backup.sh new file mode 100755 index 0000000..ca5661a --- /dev/null +++ b/backup/run_backup.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +echo -e "\n-- Database of Babel Backup and Rotation START --\n" + +timestamp=$(date +%Y_%m_%d_%H_%M_%S) +backup_to_dropbox="python3 /root/backup/backup_to_dropbox.py" +rotate_backup="python3 /root/backup/rotate_backups.py" +backups_dir="/root/backup/backups" +dump_file="dob_dump_$timestamp.sql" +backup_file="$dump_file.gz" + +mkdir -p "${backups_dir}" +cd "$backups_dir" + +if ! pg_dumpall -U $POSTGRES_USER --no-password >$dump_file; then + echo "pgdump failure!" + exit 1 +fi + +gzip $dump_file + +# Dropbox backup rotation +if ! $rotate_backup; then + echo "failed to rotate backups!" + # Notify Slack + if [[ -n "$SLACK_INCIDENTS_WH_URL" ]]; then + curl -s -H 'content-type: application/json' \ + -d "{ \"text\": \"🔴DoB Dropbox backup rotation failure on ${timestamp}\" }" \ + "$SLACK_INCIDENTS_WH_URL" + fi + exit 1 +fi + +# Delete local backups older than one week +for file in ./*.gz; do + if [ $(($(date +%s) - $(date -r $file +%s))) -gt 604800 ]; then + rm "$file" + fi +done + +# Backup to Dropbox +if ! $backup_to_dropbox $backup_file; then + echo "Dropbox backup failure!" + # Notify Slack + if [[ -n "$SLACK_INCIDENTS_WH_URL" ]]; then + curl -s -H 'content-type: application/json' \ + -d "{ \"text\": \"🔴DoB Dropbox backup failure\nCould not upload \`${backup_file}\`.\" }" \ + "$SLACK_INCIDENTS_WH_URL" + fi + exit 1 +fi + +echo -e "\n-- Database of Babel Backup and Rotation END --\n" \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 0fec1f0..3fd7868 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,26 +1,48 @@ services: - database: - image: metakgporg/dob - container_name: dob - build: . - networks: - metaploy-private-network: - aliases: - - postgres-of-babel - environment: - - POSTGRES_USER=${DATABASE_USERNAME} - - POSTGRES_PASSWORD=${DATABASE_PASSWORD} - - POSTGRES_DB=${DATABASE_NAME} - - PGPORT=${DATABASE_PORT} - - PGHOST=${DATABASE_HOST} - volumes: - - db-volume:/var/lib/postgresql/data - restart: always + database: + image: metakgporg/dob + container_name: dob + build: . + networks: + metaploy-private-network: + aliases: + - postgres-of-babel + environment: + - POSTGRES_USER=${DATABASE_USERNAME} + - POSTGRES_PASSWORD=${DATABASE_PASSWORD} + - POSTGRES_DB=${DATABASE_NAME} + - PGPORT=${DATABASE_PORT} + - PGHOST=${DATABASE_HOST} + volumes: + - db-volume:/var/lib/postgresql/data + restart: always + + backup: + build: + context: ./backup + args: + - DBPASSFILE=${DBPASSFILE} + restart: always + networks: + metaploy-private-network: + depends_on: + - database + environment: + - PGHOST=database + - POSTGRES_USER=${DATABASE_USERNAME} + - PGPASSFILE=/root/.pgpass + - DROPBOX_APP_KEY=${DROPBOX_APP_KEY} + - DROPBOX_APP_SECRET=${DROPBOX_APP_SECRET} + - DROPBOX_ACCESS_TOKEN=${DROPBOX_ACCESS_TOKEN} + - DROPBOX_REFRESH_TOKEN=${DROPBOX_REFRESH_TOKEN} + - SLACK_INCIDENTS_WH_URL=${SLACK_INCIDENTS_WH_URL} + volumes: + - db-volume:/var/lib/postgresql/data networks: - metaploy-private-network: - external: true - name: metaploy-private-network + metaploy-private-network: + external: true + name: metaployprivate- -network volumes: - db-volume: + db-volume: