From bd97faa3b8f7a299a0e8d8a02ccb238b3a95358e Mon Sep 17 00:00:00 2001 From: Eric Blau Date: Wed, 26 Jun 2024 10:26:55 -0500 Subject: [PATCH 01/11] Make minimal backup for info_django as well as a backup for serviceindex --- sbin/database_backup.sh | 54 ++++++++++++++---------- sbin/serviceindex_backup.crontab | 1 + sbin/serviceindex_backup.sh | 70 ++++++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+), 22 deletions(-) create mode 100644 sbin/serviceindex_backup.crontab create mode 100755 sbin/serviceindex_backup.sh diff --git a/sbin/database_backup.sh b/sbin/database_backup.sh index 6f6bcd6..7f602d2 100755 --- a/sbin/database_backup.sh +++ b/sbin/database_backup.sh @@ -1,38 +1,48 @@ #!/bin/bash -MY_BASE=/soft/warehouse-apps-1.0/Management-Tools -PYTHON=python3 -PYTHON_BASE=/soft/python/python-3.8.11-base +### +# Run warehouse_management: Database backup +### + +APP_NAME=database_backup +APP_HOME=/soft/applications-2.0/warehouse_management +DBNAME=warehouse2 +DBHOST=opsdb-dev.cluster-clabf5kcvwmz.us-east-2.rds.amazonaws.com +DBUSER=info_django +S3DIR=s3://backup.operations.access-ci.org/operations-api.access-ci.org/rds.backup/ + +# Override in shell environment +if [ -z "$PYTHON_BASE" ]; then + PYTHON_BASE=/usr +fi + +####### Everything else should be standard ####### + +PYTHON_BIN=python3 export LD_LIBRARY_PATH=${PYTHON_BASE}/lib -PYTHON_ROOT=/soft/awscli/python -source ${PYTHON_ROOT}/bin/activate +source ${APP_HOME}/python/bin/activate -BACKUP_DIR=${MY_BASE}/backups/ +BACKUP_DIR=${APP_HOME}/backups/ [ ! -d ${BACKUP_DIR} ] && mkdir ${BACKUP_DIR} -exec 1>> ${BACKUP_DIR}/database_backup.log +exec 1>> ${BACKUP_DIR}/${APP_NAME}.log echo Starting at `date` -DBHOST=information-warehouse-prod-cluster.cluster-clabf5kcvwmz.us-east-2.rds.amazonaws.com DATE=`date +'%s'` -/usr/pgsql-13/bin/pg_dump -h ${DBHOST} -U django_owner -n django -d warehouse \ +# Using OS installed PostgreSQL tools +pg_dump -h ${DBHOST} -U ${DBUSER} -n public -d ${DBNAME} \ >${BACKUP_DIR}/django.dump.${DATE} - -/usr/pgsql-13/bin/pg_dump -h ${DBHOST} -U xcsr_owner -n xcsr -d warehouse \ - >${BACKUP_DIR}/xcsr.dump.${DATE} - -/usr/pgsql-13/bin/pg_dump -h ${DBHOST} -U glue2_owner -n glue2 -d warehouse --exclude-table-data=glue2_db_entityhistory \ - >${BACKUP_DIR}/glue2.dump.${DATE} +# Minimum backup without history for development environments +pg_dump -h ${DBHOST} -U ${DBUSER} -n public -d ${DBNAME} --exclude-table=public.glue2_entityhistory --exclude-table=public.warehouse_state_processingerror \ + >${BACKUP_DIR}/django.mindump.${DATE} #zip all dumps to save disk gzip -9 ${BACKUP_DIR}/django.dump.${DATE} -gzip -9 ${BACKUP_DIR}/xcsr.dump.${DATE} -gzip -9 ${BACKUP_DIR}/glue2.dump.${DATE} +gzip -9 ${BACKUP_DIR}/django.mindump.${DATE} -aws s3 cp ${BACKUP_DIR}/django.dump.${DATE}.gz s3://xci.xsede.org/info.xsede.org/rds.backup/ --only-show-errors --profile newbackup -aws s3 cp ${BACKUP_DIR}/xcsr.dump.${DATE}.gz s3://xci.xsede.org/info.xsede.org/rds.backup/ --only-show-errors --profile newbackup -aws s3 cp ${BACKUP_DIR}/glue2.dump.${DATE}.gz s3://xci.xsede.org/info.xsede.org/rds.backup/ --only-show-errors --profile newbackup +aws s3 cp ${BACKUP_DIR}/django.dump.${DATE}.gz ${S3DIR} --only-show-errors --profile newbackup +aws s3 cp ${BACKUP_DIR}/django.mindump.${DATE}.gz ${S3DIR} --only-show-errors --profile newbackup #aws s3 ls s3://xci.xsede.org/info.xsede.org/rds.backup/\*.${DATE} --profile newbackup @@ -41,14 +51,14 @@ find ${BACKUP_DIR} -mtime +2 -name \*dump\* -exec rm {} \; # Delete s3 files older than seven days let maxage=60*60*24*7 -aws s3 ls s3://xci.xsede.org/info.xsede.org/rds.backup/ --profile newbackup | awk '{print $4}' | while read filename +aws s3 ls ${S3DIR} --profile newbackup | awk '{print $4}' | while read filename do echo "${filename}" fileepoch="$(cut -d'.' -f3 <<<"${filename}")" if [ -n "${fileepoch}" ] && [ "${fileepoch}" -eq "${fileepoch}" ] 2>/dev/null; then let fileage=${DATE}-${fileepoch} if [ "${fileage}" -gt "${maxage}" ]; then - aws s3 rm s3://xci.xsede.org/info.xsede.org/rds.backup/${filename} --profile newbackup + aws s3 rm ${S3DIR}/${filename} --profile newbackup fi fi done diff --git a/sbin/serviceindex_backup.crontab b/sbin/serviceindex_backup.crontab new file mode 100644 index 0000000..e9ecad7 --- /dev/null +++ b/sbin/serviceindex_backup.crontab @@ -0,0 +1 @@ +0 41 * * * /bin/bash -l -c /soft/warehouse-apps-1.0/Management-Tools/PROD/sbin/serviceindex_backup.sh diff --git a/sbin/serviceindex_backup.sh b/sbin/serviceindex_backup.sh new file mode 100755 index 0000000..7f4dbfb --- /dev/null +++ b/sbin/serviceindex_backup.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +### +# Run warehouse_management: Database backup +### + +APP_NAME=service-index +APP_HOME=/soft/applications-2.0/warehouse_management +DBNAME=serviceindex1 +DBHOST=opsdb-dev.cluster-clabf5kcvwmz.us-east-2.rds.amazonaws.com +DBUSER=serviceindex_django +S3DIR=s3://backup.operations.access-ci.org/service-index.operations.access-ci.org/rds.backup/ + +# Override in shell environment +if [ -z "$PYTHON_BASE" ]; then + PYTHON_BASE=/usr +fi + +####### Everything else should be standard ####### + +PYTHON_BIN=python3 +export LD_LIBRARY_PATH=${PYTHON_BASE}/lib +source ${APP_HOME}/python/bin/activate + +BACKUP_DIR=${APP_HOME}/backups/serviceindex/ +[ ! -d ${BACKUP_DIR} ] && mkdir -p ${BACKUP_DIR} + +exec 1>> ${BACKUP_DIR}/${APP_NAME}.log +echo Starting at `date` + +DATE=`date +'%s'` + +# Using OS installed PostgreSQL tools +pg_dump -h ${DBHOST} -U ${DBUSER} -n public -d ${DBNAME} \ + -t serviceindex.availability -t serviceindex.site -t serviceindex.staff -t serviceindex.support \ + -t serviceindex.service -t serviceindex.host -t serviceindex.link -t serviceindex.logentry \ + -t serviceindex.event -t serviceindex.hosteventlog -t serviceindex.hosteventstatus \ + >${BACKUP_DIR}/django.dump.${DATE} + +#pg_dump -h ${DBHOST} -U ${DBUSER} -n public -d ${DBNAME} \ +# >${BACKUP_DIR}/django.dump.${DATE} +# Minimum backup without history for development environments +#pg_dump -h ${DBHOST} -U ${DBUSER} -n public -d ${DBNAME} --exclude-table=public.glue2_entityhistory --exclude-table=public.warehouse_state_processingerror \ +# >${BACKUP_DIR}/django.mindump.${DATE} + +#zip all dumps to save disk +gzip -9 ${BACKUP_DIR}/django.dump.${DATE} +#gzip -9 ${BACKUP_DIR}/django.mindump.${DATE} + +aws s3 cp ${BACKUP_DIR}/django.dump.${DATE}.gz ${S3DIR} --only-show-errors --profile newbackup +#aws s3 cp ${BACKUP_DIR}/django.mindump.${DATE}.gz ${S3DIR} --only-show-errors --profile newbackup + +#aws s3 ls s3://xci.xsede.org/info.xsede.org/rds.backup/\*.${DATE} --profile newbackup + +#Cleanup backups older than 2 days +#find ${BACKUP_DIR} -mtime +2 -name \*dump\* -exec rm {} \; + +# Delete s3 files older than seven days +let maxage=60*60*24*7 +aws s3 ls ${S3DIR} --profile newbackup | awk '{print $4}' | while read filename +do + echo "${filename}" + fileepoch="$(cut -d'.' -f3 <<<"${filename}")" + if [ -n "${fileepoch}" ] && [ "${fileepoch}" -eq "${fileepoch}" ] 2>/dev/null; then + let fileage=${DATE}-${fileepoch} + if [ "${fileage}" -gt "${maxage}" ]; then + aws s3 rm ${S3DIR}/${filename} --profile newbackup + fi + fi +done From 4ff060b0473a029d25339a213caa016fd990ab63 Mon Sep 17 00:00:00 2001 From: Eric Blau Date: Wed, 26 Jun 2024 10:52:47 -0500 Subject: [PATCH 02/11] fixed path to backup scripts in crontabs --- sbin/database_backup.crontab | 2 +- sbin/serviceindex_backup.crontab | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sbin/database_backup.crontab b/sbin/database_backup.crontab index b1f50ad..5caec73 100644 --- a/sbin/database_backup.crontab +++ b/sbin/database_backup.crontab @@ -1 +1 @@ -0 21 * * * /bin/bash -l -c /soft/warehouse-apps-1.0/Management-Tools/PROD/sbin/database_backup.sh +0 21 * * * /bin/bash -l -c /soft/warehouse-apps-1.0/management-tools/PROD/sbin/database_backup.sh diff --git a/sbin/serviceindex_backup.crontab b/sbin/serviceindex_backup.crontab index e9ecad7..ee50e11 100644 --- a/sbin/serviceindex_backup.crontab +++ b/sbin/serviceindex_backup.crontab @@ -1 +1 @@ -0 41 * * * /bin/bash -l -c /soft/warehouse-apps-1.0/Management-Tools/PROD/sbin/serviceindex_backup.sh +0 41 * * * /bin/bash -l -c /soft/warehouse-apps-1.0/management-tools/PROD/sbin/serviceindex_backup.sh From 3bd58d441691d0d4cb680239d5432b5b4c9614b7 Mon Sep 17 00:00:00 2001 From: Eric Blau Date: Wed, 26 Jun 2024 11:00:35 -0500 Subject: [PATCH 03/11] fix crontabs again --- sbin/database_backup.crontab | 2 +- sbin/serviceindex_backup.crontab | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sbin/database_backup.crontab b/sbin/database_backup.crontab index 5caec73..794aaec 100644 --- a/sbin/database_backup.crontab +++ b/sbin/database_backup.crontab @@ -1 +1 @@ -0 21 * * * /bin/bash -l -c /soft/warehouse-apps-1.0/management-tools/PROD/sbin/database_backup.sh +0 21 * * * /bin/bash -l -c /soft/applications-2.0/management-tools/PROD/sbin/database_backup.sh diff --git a/sbin/serviceindex_backup.crontab b/sbin/serviceindex_backup.crontab index ee50e11..81b831d 100644 --- a/sbin/serviceindex_backup.crontab +++ b/sbin/serviceindex_backup.crontab @@ -1 +1 @@ -0 41 * * * /bin/bash -l -c /soft/warehouse-apps-1.0/management-tools/PROD/sbin/serviceindex_backup.sh +0 41 * * * /bin/bash -l -c /soft/applications-2.0/management-tools/PROD/sbin/serviceindex_backup.sh From 0b281390cb35c94f03ba0adf803bcbd323f49d17 Mon Sep 17 00:00:00 2001 From: John-Paul Navarro Date: Wed, 23 Oct 2024 06:33:22 -0500 Subject: [PATCH 04/11] Add model serviceindex.misc_urls to backup script --- CHANGELOG | 3 +++ sbin/serviceindex_backup.sh | 1 + 2 files changed, 4 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index e7ec7cd..57a7031 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,6 @@ +tag-0.7-20241023 JP + - Add model serviceindex.misc_urls to backup script + tag-0.6-20220301 JP - Implement sbin/database_backup* diff --git a/sbin/serviceindex_backup.sh b/sbin/serviceindex_backup.sh index 7f4dbfb..990b63e 100755 --- a/sbin/serviceindex_backup.sh +++ b/sbin/serviceindex_backup.sh @@ -35,6 +35,7 @@ pg_dump -h ${DBHOST} -U ${DBUSER} -n public -d ${DBNAME} \ -t serviceindex.availability -t serviceindex.site -t serviceindex.staff -t serviceindex.support \ -t serviceindex.service -t serviceindex.host -t serviceindex.link -t serviceindex.logentry \ -t serviceindex.event -t serviceindex.hosteventlog -t serviceindex.hosteventstatus \ + -t serviceindex.misc_urls \ >${BACKUP_DIR}/django.dump.${DATE} #pg_dump -h ${DBHOST} -U ${DBUSER} -n public -d ${DBNAME} \ From a32a9dd85776b09a02b8a1a06279e1df4aaa61a1 Mon Sep 17 00:00:00 2001 From: John-Paul Navarro Date: Fri, 1 Nov 2024 06:54:47 -0500 Subject: [PATCH 05/11] Backup the development Service Index database also --- CHANGELOG | 3 +++ sbin/serviceindex_backup.sh | 45 ++++++++++++++++++++----------------- 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 57a7031..5d9b35a 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,6 @@ +tag-0.8-20241101 JP + - Backup the development Service Index database also + tag-0.7-20241023 JP - Add model serviceindex.misc_urls to backup script diff --git a/sbin/serviceindex_backup.sh b/sbin/serviceindex_backup.sh index 990b63e..ce51002 100755 --- a/sbin/serviceindex_backup.sh +++ b/sbin/serviceindex_backup.sh @@ -1,14 +1,20 @@ #!/bin/bash ### -# Run warehouse_management: Database backup +# Run warehouse_management: Service Index databases backup ### APP_NAME=service-index APP_HOME=/soft/applications-2.0/warehouse_management -DBNAME=serviceindex1 -DBHOST=opsdb-dev.cluster-clabf5kcvwmz.us-east-2.rds.amazonaws.com -DBUSER=serviceindex_django + +DBNAME1=serviceindex1 +DBHOST1=opsdb-dev.cluster-clabf5kcvwmz.us-east-2.rds.amazonaws.com +DBUSER1=serviceindex_django + +DBNAME2=serviceindex2 +DBHOST2=${DBHOST1} +DBUSER2=${DBUSER1} + S3DIR=s3://backup.operations.access-ci.org/service-index.operations.access-ci.org/rds.backup/ # Override in shell environment @@ -30,28 +36,25 @@ echo Starting at `date` DATE=`date +'%s'` -# Using OS installed PostgreSQL tools -pg_dump -h ${DBHOST} -U ${DBUSER} -n public -d ${DBNAME} \ +DUMPNAME=django.${DBNAME1}.dump.${DATE} +pg_dump -h ${DBHOST1} -U ${DBUSER1} -n public -d ${DBNAME1} \ -t serviceindex.availability -t serviceindex.site -t serviceindex.staff -t serviceindex.support \ -t serviceindex.service -t serviceindex.host -t serviceindex.link -t serviceindex.logentry \ -t serviceindex.event -t serviceindex.hosteventlog -t serviceindex.hosteventstatus \ -t serviceindex.misc_urls \ - >${BACKUP_DIR}/django.dump.${DATE} - -#pg_dump -h ${DBHOST} -U ${DBUSER} -n public -d ${DBNAME} \ -# >${BACKUP_DIR}/django.dump.${DATE} -# Minimum backup without history for development environments -#pg_dump -h ${DBHOST} -U ${DBUSER} -n public -d ${DBNAME} --exclude-table=public.glue2_entityhistory --exclude-table=public.warehouse_state_processingerror \ -# >${BACKUP_DIR}/django.mindump.${DATE} - -#zip all dumps to save disk -gzip -9 ${BACKUP_DIR}/django.dump.${DATE} -#gzip -9 ${BACKUP_DIR}/django.mindump.${DATE} + >${BACKUP_DIR}/${DUMPNAME} +gzip -9 ${BACKUP_DIR}/${DUMPNAME} +aws s3 cp ${BACKUP_DIR}/${DUMPNAME}.gz ${S3DIR} --only-show-errors --profile newbackup -aws s3 cp ${BACKUP_DIR}/django.dump.${DATE}.gz ${S3DIR} --only-show-errors --profile newbackup -#aws s3 cp ${BACKUP_DIR}/django.mindump.${DATE}.gz ${S3DIR} --only-show-errors --profile newbackup - -#aws s3 ls s3://xci.xsede.org/info.xsede.org/rds.backup/\*.${DATE} --profile newbackup +DUMPNAME=django.${DBNAME2}.dump.${DATE} +pg_dump -h ${DBHOST2} -U ${DBUSER2} -n public -d ${DBNAME2} \ + -t serviceindex.availability -t serviceindex.site -t serviceindex.staff -t serviceindex.support \ + -t serviceindex.service -t serviceindex.host -t serviceindex.link -t serviceindex.logentry \ + -t serviceindex.event -t serviceindex.hosteventlog -t serviceindex.hosteventstatus \ + -t serviceindex.misc_urls \ + >${BACKUP_DIR}/${DUMPNAME} +gzip -9 ${BACKUP_DIR}/${DUMPNAME} +aws s3 cp ${BACKUP_DIR}/${DUMPNAME}.gz ${S3DIR} --only-show-errors --profile newbackup #Cleanup backups older than 2 days #find ${BACKUP_DIR} -mtime +2 -name \*dump\* -exec rm {} \; From 5376c6d00eebdff8db5711731330257cedc4886e Mon Sep 17 00:00:00 2001 From: John-Paul Navarro Date: Fri, 10 Nov 2023 11:24:21 -0600 Subject: [PATCH 06/11] See CHANGELOG tag v1.0.0 rebased from Router_Management_Tools --- CHANGELOG | 6 ++++++ README | 5 +---- conf/es_reload.conf.example | 4 ++-- sbin/database_backup.crontab | 2 +- sbin/es_reload.sh | 33 +++++++++++++++++++++++---------- 5 files changed, 33 insertions(+), 17 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 5d9b35a..b83fca3 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,9 @@ +v1.0.0-20231110 JP + - Fork of https://github.com/access-ci-org/Operations_Router_Management-Tools + - Rename since these tools are for managing the warehouse and not routers per-se + - Primarily cleaning up the wareouse backup tool + - TODO: review/convert bin/es_reload.py + tag-0.8-20241101 JP - Backup the development Service Index database also diff --git a/README b/README index b393295..31c9fba 100644 --- a/README +++ b/README @@ -1,4 +1 @@ -Scripts to manage AMIE queues and users have been removed from this package. -In ACCESS-CI, AMIE no longer uses rabbitmq queues. -The scripts can still be found in the original XSEDE repository at -https://github.com/XSEDE/Discovery_Management-Tools +Tools for managing the Information Sharing Platform Warehouse diff --git a/conf/es_reload.conf.example b/conf/es_reload.conf.example index 23bbf40..ba17877 100644 --- a/conf/es_reload.conf.example +++ b/conf/es_reload.conf.example @@ -2,6 +2,6 @@ "DEBUG": true, "ELASTIC_HOSTS": ["localhost:9200"], "LOG_LEVEL": "info", - "LOG_FILE": "/soft/warehouse-apps-1.0/Management-Tools/var/es_reload.log", - "PID_FILE": "/soft/warehouse-apps-1.0/Management-Tools/var/es_reload.pid" + "LOG_FILE": "/soft/applications-2.0/warehouse_management/var/es_reload.log", + "PID_FILE": "/soft/applications-2.0/warehouse_management/var/es_reload.pid" } diff --git a/sbin/database_backup.crontab b/sbin/database_backup.crontab index 794aaec..67fc97d 100644 --- a/sbin/database_backup.crontab +++ b/sbin/database_backup.crontab @@ -1 +1 @@ -0 21 * * * /bin/bash -l -c /soft/applications-2.0/management-tools/PROD/sbin/database_backup.sh +0 21 * * * /bin/bash -l -c /soft/applications-2.0/warehouse_management/PROD/sbin/database_backup.sh diff --git a/sbin/es_reload.sh b/sbin/es_reload.sh index 66cf7be..93fee68 100755 --- a/sbin/es_reload.sh +++ b/sbin/es_reload.sh @@ -1,18 +1,31 @@ #!/bin/bash -### RePublish Tool -MY_BASE=/soft/warehouse-apps-1.0/Management-Tools +### +# Run %APP_NAME%: OpenSearch reload +### +APP_NAME=es_reload +APP_HOME=%APP_HOME% -PYTHON=python3 -PYTHON_BASE=/soft/python/python-3.7.7-base -PYTHON_ROOT=/soft/warehouse-apps-1.0/Management-Tools/python -source ${PYTHON_ROOT}/bin/activate +# Override in shell environment +if [ -z "$PYTHON_BASE" ]; then + PYTHON_BASE=%PYTHON_BASE% +fi -export PYTHONPATH=$DAEMON_DIR/lib:/soft/warehouse-1.0/PROD/django_xsede_warehouse -export DJANGO_CONF=/soft/warehouse-apps-1.0/Management-Tools/conf/django_xsede_warehouse.conf -export DJANGO_SETTINGS_MODULE=xsede_warehouse.settings +####### Everything else should be standard ####### +APP_SOURCE=${APP_HOME}/PROD +APP_BIN=${APP_SOURCE}/bin/${APP_NAME}.py +APP_OPTS="-c ${APP_HOME}/conf/${APP_NAME}.conf" -$MY_BASE/PROD/bin/es_reload.py -c $MY_BASE/conf/es_reload.conf "$@" +PYTHON_BIN=python3 +export LD_LIBRARY_PATH=${PYTHON_BASE}/lib +source ${APP_HOME}/python/bin/activate + +export PYTHONPATH=${APP_SOURCE}/lib:${WAREHOUSE_DJANGO} +export APP_CONFIG=${APP_HOME}/conf/django_prod_router.conf +export DJANGO_SETTINGS_MODULE=Operations_Warehouse_Django.settings + +echo "Starting: ${PYTHON_BIN} ${APP_BIN} $@ ${APP_OPTS}" +${PYTHON_BIN} ${APP_BIN} $@ ${APP_OPTS} RETVAL=$? echo rc=$RETVAL exit $RETVAL From dec62e1557850ac1d8e7d26c861cf0b7ea51fde4 Mon Sep 17 00:00:00 2001 From: John-Paul Navarro Date: Fri, 10 Nov 2023 12:33:33 -0600 Subject: [PATCH 07/11] Enable backup crontab --- CHANGELOG | 3 +++ sbin/create_amie_source_user.sh | 12 ------------ 2 files changed, 3 insertions(+), 12 deletions(-) delete mode 100755 sbin/create_amie_source_user.sh diff --git a/CHANGELOG b/CHANGELOG index b83fca3..81f1624 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,6 @@ +v1.0.1-20231110 JP + - Enable backup crontab + v1.0.0-20231110 JP - Fork of https://github.com/access-ci-org/Operations_Router_Management-Tools - Rename since these tools are for managing the warehouse and not routers per-se diff --git a/sbin/create_amie_source_user.sh b/sbin/create_amie_source_user.sh deleted file mode 100755 index 590cee6..0000000 --- a/sbin/create_amie_source_user.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -#this is only for amie.xsede.org, that is, the source of the amie packets -#coming through RabbitMQ. All other amie users should use create_amie_user.sh - -input=$(sed -r 's/street=/2.5.4.9=/g' <<<"$1") -input=$(sed -r 's/postalCode=/2.5.4.17=/g' <<<"$input") -echo creating user $input -sudo rabbitmqctl add_user "$input" foo - -sudo rabbitmqctl list_user_permissions "$1" - -sudo rabbitmqctl set_permissions -p xsede "$1" "^amq.gen.*" "^amq.gen.*|^amie.to.*" "^amq.gen.*|^amie.from.* From 2ad45d2568e00c82eb56f7efed8b775daaa3baa8 Mon Sep 17 00:00:00 2001 From: John-Paul Navarro Date: Fri, 10 Nov 2023 13:58:58 -0600 Subject: [PATCH 08/11] Fix a bunch of sbin/* file problems --- CHANGELOG | 3 +++ sbin/database_backup.crontab | 2 +- sbin/es_reload.sh | 1 + sbin/es_reload.sh.dev | 16 ++++++++++------ sbin/repub.sh | 22 +++++++++++++--------- sbin/repub.sh.dev | 18 ------------------ 6 files changed, 28 insertions(+), 34 deletions(-) delete mode 100755 sbin/repub.sh.dev diff --git a/CHANGELOG b/CHANGELOG index 81f1624..75c02d0 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,6 @@ +v1.0.2-20231110 JP + - Fix a bunch of sbin/* file problems + v1.0.1-20231110 JP - Enable backup crontab diff --git a/sbin/database_backup.crontab b/sbin/database_backup.crontab index 67fc97d..a9fd8c6 100644 --- a/sbin/database_backup.crontab +++ b/sbin/database_backup.crontab @@ -1 +1 @@ -0 21 * * * /bin/bash -l -c /soft/applications-2.0/warehouse_management/PROD/sbin/database_backup.sh +0 21 * * * /bin/bash -l -c %APP_HOME%/sbin/database_backup.sh diff --git a/sbin/es_reload.sh b/sbin/es_reload.sh index 93fee68..a947689 100755 --- a/sbin/es_reload.sh +++ b/sbin/es_reload.sh @@ -3,6 +3,7 @@ ### # Run %APP_NAME%: OpenSearch reload ### + APP_NAME=es_reload APP_HOME=%APP_HOME% diff --git a/sbin/es_reload.sh.dev b/sbin/es_reload.sh.dev index 70112d0..ddbf012 100755 --- a/sbin/es_reload.sh.dev +++ b/sbin/es_reload.sh.dev @@ -1,18 +1,22 @@ #!/bin/bash ### RePublish Tool -MY_BASE=/soft/warehouse-apps-1.0/Management-Tools +MY_BASE=%APP_HOME% -PYTHON=python3 -PYTHON_BASE=/Library/Frameworks/Python.framework/Versions/3.7 -PYTHON_ROOT=/soft/warehouse-apps-1.0/Management-Tools/python -source ${PYTHON_ROOT}/bin/activate +# Override in shell environment +if [ -z "$PYTHON_BASE" ]; then + PYTHON_BASE=%PYTHON_BASE% +fi + +PYTHON_BIN=python3 +export LD_LIBRARY_PATH=${PYTHON_BASE}/lib +source ${APP_HOME}/python/bin/activate export PYTHONPATH=$DAEMON_DIR/lib:/soft/warehouse-1.0/PROD/django_xsede_warehouse export DJANGO_CONF=/soft/warehouse-apps-1.0/Management-Tools/conf/django_xsede_warehouse.conf export DJANGO_SETTINGS_MODULE=xsede_warehouse.settings -$MY_BASE/PROD/bin/es_reload.py -c $MY_BASE/conf/es_reload.conf $@ +${PYTHON_BIN} ${MY_BASE}/PROD/bin/es_reload.py -c ${MY_BASE}/conf/es_reload.conf $@ RETVAL=$? echo rc=$RETVAL exit $RETVAL diff --git a/sbin/repub.sh b/sbin/repub.sh index d086d48..30891a7 100755 --- a/sbin/repub.sh +++ b/sbin/repub.sh @@ -1,18 +1,22 @@ #!/bin/bash ### RePublish Tool -MY_BASE=/soft/warehouse-apps-1.0/Management-Tools +APP_HOME=%APP_HOME% -PYTHON=python3 -PYTHON_BASE=/soft/python/python-3.7.7-base -PYTHON_ROOT=/soft/warehouse-apps-1.0/Management-Tools/python -source ${PYTHON_ROOT}/bin/activate +# Override in shell environment +if [ -z "$PYTHON_BASE" ]; then + PYTHON_BASE=%PYTHON_BASE% +fi -export PYTHONPATH=$DAEMON_DIR/lib:/soft/warehouse-1.0/PROD/django_xsede_warehouse -export DJANGO_CONF=/soft/warehouse-apps-1.0/Management-Tools/conf/django_xsede_warehouse.conf -export DJANGO_SETTINGS_MODULE=xsede_warehouse.settings +PYTHON_BIN=python3 +export LD_LIBRARY_PATH=${PYTHON_BASE}/lib +source ${APP_HOME}/python/bin/activate -$MY_BASE/PROD/bin/repub.py -c $MY_BASE/conf/repub.conf -r dummy.test.xsede.org -i $@ +export PYTHONPATH=${APP_HOME}/PROS/lib:${WAREHOUSE_DJANGO} +export DJANGO_CONF=${APP_HOME}/conf/django_prod_router.conf +export DJANGO_SETTINGS_MODULE=Operations_Warehouse_Django.settinge + +${PYTHON_BIN} ${APP_HOME}/PROD/bin/repub.py -c ${APP_HOME}/conf/repub.conf -r dummy.test.access-ci.org -i $@ RETVAL=$? echo rc=$RETVAL exit $RETVAL diff --git a/sbin/repub.sh.dev b/sbin/repub.sh.dev deleted file mode 100755 index 855179a..0000000 --- a/sbin/repub.sh.dev +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -### RePublish Tool -MY_BASE=/soft/warehouse-apps-1.0/Management-Tools - -PYTHON=python3 -PYTHON_BASE=/Library/Frameworks/Python.framework/Versions/3.7 -PYTHON_ROOT=/soft/warehouse-apps-1.0/Management-Tools/python -source ${PYTHON_ROOT}/bin/activate - -export PYTHONPATH=$DAEMON_DIR/lib:/soft/warehouse-1.0/PROD/django_xsede_warehouse -export DJANGO_CONF=/soft/warehouse-apps-1.0/Management-Tools/conf/django_xsede_warehouse.conf -export DJANGO_SETTINGS_MODULE=xsede_warehouse.settings - -$MY_BASE/PROD/bin/repub.py -c $MY_BASE/conf/repub.conf -r dummy.test.xsede.org -i $@ -RETVAL=$? -echo rc=$RETVAL -exit $RETVAL From 657906d63cf23c24d2b00a3e0ad0fd352d760b33 Mon Sep 17 00:00:00 2001 From: John-Paul Navarro Date: Fri, 1 Nov 2024 08:03:20 -0500 Subject: [PATCH 09/11] See CHANGELOG tag v1.0.3-20241101 rebased from Router_Management_Tools --- .gitignore | 137 ++++++++++++++++++++++++++++++++++++++++ CHANGELOG | 4 ++ sbin/database_backup.sh | 34 +++++----- 3 files changed, 160 insertions(+), 15 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b3b0c4b --- /dev/null +++ b/.gitignore @@ -0,0 +1,137 @@ +# Repo specific ignores +*_customized.conf +*_customized.sql + + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# Mac +.DS_Store diff --git a/CHANGELOG b/CHANGELOG index 75c02d0..eb29dc2 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,7 @@ +v1.0.3-20241101 JP + - Backup the development Service Index database also + - Backup the development Warehouse database also + v1.0.2-20231110 JP - Fix a bunch of sbin/* file problems diff --git a/sbin/database_backup.sh b/sbin/database_backup.sh index 7f602d2..1e5a594 100755 --- a/sbin/database_backup.sh +++ b/sbin/database_backup.sh @@ -6,9 +6,15 @@ APP_NAME=database_backup APP_HOME=/soft/applications-2.0/warehouse_management -DBNAME=warehouse2 -DBHOST=opsdb-dev.cluster-clabf5kcvwmz.us-east-2.rds.amazonaws.com -DBUSER=info_django + +DBNAME1=warehouse2 +DBHOST1=opsdb-dev.cluster-clabf5kcvwmz.us-east-2.rds.amazonaws.com +DBUSER1=info_django + +DBNAME2=warehouse3 +DBHOST2=opsdb-dev.cluster-clabf5kcvwmz.us-east-2.rds.amazonaws.com +DBUSER2=info_django + S3DIR=s3://backup.operations.access-ci.org/operations-api.access-ci.org/rds.backup/ # Override in shell environment @@ -30,19 +36,17 @@ echo Starting at `date` DATE=`date +'%s'` -# Using OS installed PostgreSQL tools -pg_dump -h ${DBHOST} -U ${DBUSER} -n public -d ${DBNAME} \ - >${BACKUP_DIR}/django.dump.${DATE} -# Minimum backup without history for development environments -pg_dump -h ${DBHOST} -U ${DBUSER} -n public -d ${DBNAME} --exclude-table=public.glue2_entityhistory --exclude-table=public.warehouse_state_processingerror \ - >${BACKUP_DIR}/django.mindump.${DATE} - -#zip all dumps to save disk -gzip -9 ${BACKUP_DIR}/django.dump.${DATE} -gzip -9 ${BACKUP_DIR}/django.mindump.${DATE} +DUMPNAME=django.${DBNAME1}.dump.${DATE} +pg_dump -h ${DBHOST1} -U ${DBUSER1} -n public -d ${DBNAME1} \ + >${BACKUP_DIR}/${DUMPNAME} +gzip -9 ${BACKUP_DIR}/${DUMPNAME} +aws s3 cp ${BACKUP_DIR}/${DUMPNAME}.gz ${S3DIR} --only-show-errors --profile newbackup -aws s3 cp ${BACKUP_DIR}/django.dump.${DATE}.gz ${S3DIR} --only-show-errors --profile newbackup -aws s3 cp ${BACKUP_DIR}/django.mindump.${DATE}.gz ${S3DIR} --only-show-errors --profile newbackup +DUMPNAME=django.${DBNAME2}.dump.${DATE} +pg_dump -h ${DBHOST2} -U ${DBUSER2} -n public -d ${DBNAME2} \ + >${BACKUP_DIR}/${DUMPNAME} +gzip -9 ${BACKUP_DIR}/${DUMPNAME} +aws s3 cp ${BACKUP_DIR}/${DUMPNAME}.gz ${S3DIR} --only-show-errors --profile newbackup #aws s3 ls s3://xci.xsede.org/info.xsede.org/rds.backup/\*.${DATE} --profile newbackup From 3cd6a4b44c247f74062a6f41d55e51fddf6283ea Mon Sep 17 00:00:00 2001 From: John-Paul Navarro Date: Fri, 1 Nov 2024 08:37:30 -0500 Subject: [PATCH 10/11] Exclude resource_v4 objects from backups --- CHANGELOG | 3 +++ sbin/database_backup.sh | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index eb29dc2..95514ea 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,6 @@ +v1.0.4-20241101 JP + - Exclude resource_v4 objects from backups + v1.0.3-20241101 JP - Backup the development Service Index database also - Backup the development Warehouse database also diff --git a/sbin/database_backup.sh b/sbin/database_backup.sh index 1e5a594..a4b6465 100755 --- a/sbin/database_backup.sh +++ b/sbin/database_backup.sh @@ -38,16 +38,48 @@ DATE=`date +'%s'` DUMPNAME=django.${DBNAME1}.dump.${DATE} pg_dump -h ${DBHOST1} -U ${DBUSER1} -n public -d ${DBNAME1} \ + --exclude-table=public.resource_v4_resourcev4 \ + --exclude-table=public.resource_v4_resourcev4local \ + --exclude-table=public.resource_v4_resourcev4relation \ >${BACKUP_DIR}/${DUMPNAME} gzip -9 ${BACKUP_DIR}/${DUMPNAME} aws s3 cp ${BACKUP_DIR}/${DUMPNAME}.gz ${S3DIR} --only-show-errors --profile newbackup +# Minimum backup without history for development environments +MINDUMPNAME=django.${DBNAME1}.mindump.${DATE} +pg_dump -h ${DBHOST1} -U ${DBUSER1} -n public -d ${DBNAME1} \ + --exclude-table=public.resource_v4_resourcev4 \ + --exclude-table=public.resource_v4_resourcev4local \ + --exclude-table=public.resource_v4_resourcev4relation \ + --exclude-table=public.glue2_entityhistory \ + --exclude-table=public.warehouse_state_processingerror \ + >${BACKUP_DIR}/${MINDUMPNAME} +gzip -9 ${BACKUP_DIR}/${MINDUMPNAME} +aws s3 cp ${BACKUP_DIR}/${MINDUMPNAME}.gz ${S3DIR} --only-show-errors --profile newbackup + +### + DUMPNAME=django.${DBNAME2}.dump.${DATE} pg_dump -h ${DBHOST2} -U ${DBUSER2} -n public -d ${DBNAME2} \ + --exclude-table=public.resource_v4_resourcev4 \ + --exclude-table=public.resource_v4_resourcev4local \ + --exclude-table=public.resource_v4_resourcev4relation \ >${BACKUP_DIR}/${DUMPNAME} gzip -9 ${BACKUP_DIR}/${DUMPNAME} aws s3 cp ${BACKUP_DIR}/${DUMPNAME}.gz ${S3DIR} --only-show-errors --profile newbackup +# Minimum backup without history for development environments +MINDUMPNAME=django.${DBNAME2}.mindump.${DATE} +pg_dump -h ${DBHOST2} -U ${DBUSER2} -n public -d ${DBNAME2} \ + --exclude-table=public.resource_v4_resourcev4 \ + --exclude-table=public.resource_v4_resourcev4local \ + --exclude-table=public.resource_v4_resourcev4relation \ + --exclude-table=public.glue2_entityhistory \ + --exclude-table=public.warehouse_state_processingerror \ + >${BACKUP_DIR}/${MINDUMPNAME} +gzip -9 ${BACKUP_DIR}/${MINDUMPNAME} +aws s3 cp ${BACKUP_DIR}/${MINDUMPNAME}.gz ${S3DIR} --only-show-errors --profile newbackup + #aws s3 ls s3://xci.xsede.org/info.xsede.org/rds.backup/\*.${DATE} --profile newbackup #Cleanup backups older than 2 days From 0e0411200dc3554cb9f975571e7602584e4c9652 Mon Sep 17 00:00:00 2001 From: Eric Blau Date: Wed, 11 Dec 2024 16:53:02 +0000 Subject: [PATCH 11/11] reconciled with in-situ changes --- sbin/database_backup.sh | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/sbin/database_backup.sh b/sbin/database_backup.sh index a4b6465..6832511 100755 --- a/sbin/database_backup.sh +++ b/sbin/database_backup.sh @@ -1,11 +1,11 @@ #!/bin/bash ### -# Run warehouse_management: Database backup +# Run %APP_NAME%: Database backup ### APP_NAME=database_backup -APP_HOME=/soft/applications-2.0/warehouse_management +APP_HOME=%APP_HOME% DBNAME1=warehouse2 DBHOST1=opsdb-dev.cluster-clabf5kcvwmz.us-east-2.rds.amazonaws.com @@ -19,7 +19,7 @@ S3DIR=s3://backup.operations.access-ci.org/operations-api.access-ci.org/rds.back # Override in shell environment if [ -z "$PYTHON_BASE" ]; then - PYTHON_BASE=/usr + PYTHON_BASE=%PYTHON_BASE% fi ####### Everything else should be standard ####### @@ -60,22 +60,30 @@ aws s3 cp ${BACKUP_DIR}/${MINDUMPNAME}.gz ${S3DIR} --only-show-errors --profile ### DUMPNAME=django.${DBNAME2}.dump.${DATE} -pg_dump -h ${DBHOST2} -U ${DBUSER2} -n public -d ${DBNAME2} \ - --exclude-table=public.resource_v4_resourcev4 \ - --exclude-table=public.resource_v4_resourcev4local \ - --exclude-table=public.resource_v4_resourcev4relation \ +pg_dump -h ${DBHOST2} -U ${DBUSER2} -n info -n info_django -d ${DBNAME2} \ + --exclude-table=info.resource_v4_resourcev4 \ + --exclude-table=info.resource_v4_resourcev4local \ + --exclude-table=info.resource_v4_resourcev4relation \ + --exclude-table=info_django.resource_v4_resourcev4 \ + --exclude-table=info_django.resource_v4_resourcev4local \ + --exclude-table=info_django.resource_v4_resourcev4relation \ >${BACKUP_DIR}/${DUMPNAME} gzip -9 ${BACKUP_DIR}/${DUMPNAME} aws s3 cp ${BACKUP_DIR}/${DUMPNAME}.gz ${S3DIR} --only-show-errors --profile newbackup # Minimum backup without history for development environments MINDUMPNAME=django.${DBNAME2}.mindump.${DATE} -pg_dump -h ${DBHOST2} -U ${DBUSER2} -n public -d ${DBNAME2} \ - --exclude-table=public.resource_v4_resourcev4 \ - --exclude-table=public.resource_v4_resourcev4local \ - --exclude-table=public.resource_v4_resourcev4relation \ - --exclude-table=public.glue2_entityhistory \ - --exclude-table=public.warehouse_state_processingerror \ +pg_dump -h ${DBHOST2} -U ${DBUSER2} -n info -n info_django -d ${DBNAME2} \ + --exclude-table=info.resource_v4_resourcev4 \ + --exclude-table=info.resource_v4_resourcev4local \ + --exclude-table=info.resource_v4_resourcev4relation \ + --exclude-table=info.glue2_entityhistory \ + --exclude-table=info.warehouse_state_processingerror \ + --exclude-table=info_django.resource_v4_resourcev4 \ + --exclude-table=info_django.resource_v4_resourcev4local \ + --exclude-table=info_django.resource_v4_resourcev4relation \ + --exclude-table=info_django.glue2_entityhistory \ + --exclude-table=info_django.warehouse_state_processingerror \ >${BACKUP_DIR}/${MINDUMPNAME} gzip -9 ${BACKUP_DIR}/${MINDUMPNAME} aws s3 cp ${BACKUP_DIR}/${MINDUMPNAME}.gz ${S3DIR} --only-show-errors --profile newbackup