-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #768 from openva/deploy
Merge in from Deploy
- Loading branch information
Showing
213 changed files
with
7,842 additions
and
15,205 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,52 +1,58 @@ | ||
| #!/bin/bash -e | ||
| #!/bin/bash | ||
|
|
||
| set -e | ||
|
|
||
| HOST="{PDO_SERVER}" | ||
| USERNAME="{PDO_USERNAME}" | ||
| PASSWORD="{PDO_PASSWORD}" | ||
|
|
||
| export MYSQL_PWD="$PASSWORD" | ||
|
|
||
| # All database tables that we want to export the structure of | ||
| STRUCTURE=(bills bills_copatrons bills_full_text bills_places bills_section_numbers bills_status bills_views blacklist chamber_status comments comments_subscriptions committees committee_members dashboard_bills dashboard_portfolios dashboard_user_data dashboard_watch_lists districts dockets files gazetteer meetings minutes polls representatives representatives_districts representatives_fundraising representatives_terms representatives_votes sessions tags users vacode video_clips video_index video_index_faces video_transcript votes) | ||
| STRUCTURE=(bills bills_copatrons bills_full_text bills_places bills_places_test bills_section_numbers bills_status bills_views blacklist chamber_status comments comments_subscriptions committees committee_members dashboard_bills dashboard_portfolios dashboard_user_data dashboard_watch_lists districts dockets files gazetteer lobbyists meetings minutes polls representatives representatives_districts representatives_terms representatives_votes sessions tags users vacode video_clips video_index video_index_faces video_transcript votes) | ||
|
|
||
| # All database tables that we want to export the contents of | ||
| ALL_CONTENTS=(committees committee_members districts files representatives representatives_districts representatives_fundraising representatives_terms sessions) | ||
| # All database tables that we want to export all contents of | ||
| ALL_CONTENTS=(committees committee_members districts files representatives representatives_districts sessions) | ||
|
|
||
| # All database tables that we want to export some contents of, as test data | ||
| SOME_CONTENTS=(bills_copatrons bills_full_text bills_places bills_section_numbers bills_status bills_views comments dockets polls tags video_clips votes) | ||
|
|
||
| # The ID of the bills to use to generate test data | ||
| BILL_IDS=(45618 45663 46308 46058 45113 44355 44599 45136 45453 45811) | ||
| BILL_IDS=(73202 73894 74728 74513 73272) | ||
|
|
||
| # Change to the directory this script is in | ||
| cd "$(dirname "$0")" | ||
| mkdir -p mysql | ||
|
|
||
| # Export all of the structural data | ||
| # Export the structural data | ||
| truncate --size 0 mysql/structure.sql | ||
| for TABLE in ${STRUCTURE[*]}; do | ||
| mysqldump {MYSQL_DATABASE} -d --routines --triggers -u "$USERNAME" -p"$PASSWORD" \ | ||
| --host "$HOST" --tables "$TABLE" >> mysql/structure.sql | ||
| done | ||
| STRUCTURE_LIST=$(printf "%s " "${STRUCTURE[@]}") | ||
| mysqldump -d --routines --triggers --set-gtid-purged=OFF -u "$USERNAME" \ | ||
| --host "$HOST" {MYSQL_DATABASE} $STRUCTURE_LIST > mysql/structure.sql | ||
|
|
||
| # Export all of the tables for which we want complete contents | ||
| # Export the tables for which we want complete contents | ||
| truncate --size 0 mysql/basic-contents.sql | ||
| for TABLE in ${ALL_CONTENTS[*]}; do | ||
| mysqldump {MYSQL_DATABASE} --no-create-info -u "$USERNAME" -p"$PASSWORD" --host "$HOST" \ | ||
| --tables "$TABLE" >> mysql/basic-contents.sql | ||
| done | ||
| ALL_CONTENTS_LIST=$(printf "%s " "${ALL_CONTENTS[@]}") | ||
| mysqldump --no-create-info --skip-lock-tables --set-gtid-purged=OFF -u "$USERNAME" \ | ||
| --host "$HOST" {MYSQL_DATABASE} $ALL_CONTENTS_LIST > mysql/basic-contents.sql | ||
|
|
||
| # Export selected contents from the remaining tables | ||
| truncate --size 0 mysql/test-records.sql | ||
| for BILL_ID in "${BILL_IDS[@]}"; do | ||
| mysqldump {MYSQL_DATABASE} --no-create-info -u "$USERNAME" -p"$PASSWORD" --host "$HOST" bills \ | ||
| --where "id=$BILL_ID" >> mysql/test-records.sql | ||
| mysqldump {MYSQL_DATABASE} --no-create-info --skip-lock-tables -u "$USERNAME" \ | ||
| --set-gtid-purged=OFF --host "$HOST" bills --where "id=$BILL_ID" >> mysql/test-records.sql | ||
| done | ||
|
|
||
| for TABLE in ${SOME_CONTENTS[*]}; do | ||
| for TABLE in "${SOME_CONTENTS[@]}"; do | ||
| for BILL_ID in "${BILL_IDS[@]}"; do | ||
| # Genericize all IP addresses and email addresses, to maintain privacy. | ||
| mysqldump {MYSQL_DATABASE} --no-create-info -u "$USERNAME" -p"$PASSWORD" --host "$HOST" "$TABLE" \ | ||
| mysqldump {MYSQL_DATABASE} --no-create-info --skip-lock-tables --set-gtid-purged=OFF \ | ||
| -u "$USERNAME" --host "$HOST" "$TABLE" \ | ||
| --where "bill_id=$BILL_ID" |perl -pe 's{[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}}{ sprintf "127.%01d.%01d.%01d", int(255*rand()), int(255*rand()), int(255*rand()) }ge' \ | ||
| |sed -E "s/\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b/[email protected]/g" \ | ||
| >> mysql/test-records.sql | ||
| done | ||
| done | ||
| done | ||
|
|
||
| # Remove the environment variable, now that we're done with it | ||
| unset MYSQL_PWD |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.