From 07c2935925b7ebaab0d107fb821c848f9c8e8118 Mon Sep 17 00:00:00 2001 From: Jonas Kaninda Date: Tue, 3 Sep 2024 06:49:26 +0200 Subject: [PATCH] chore: rename environment variable for database migration operation --- .github/workflows/release.yml | 2 +- docker/Dockerfile | 14 +++---- docs/how-tos/migrate.md | 72 +++++++++++++++++++---------------- docs/reference/index.md | 10 ++--- pkg/config.go | 34 ++++++++--------- pkg/migrate.go | 30 ++++++++------- pkg/restore.go | 4 +- pkg/var.go | 14 +++---- utils/s3.go | 2 +- 9 files changed, 96 insertions(+), 86 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cf5ffff..8a19c2c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,4 @@ -name: Release +name: CI on: push: tags: diff --git a/docker/Dockerfile b/docker/Dockerfile index dd1641a..e4cb184 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -10,7 +10,7 @@ RUN go mod download RUN CGO_ENABLED=0 GOOS=linux go build -o /app/mysql-bkup FROM ubuntu:24.04 -ENV DB_HOST="" +ENV DB_HOST="localhost" ENV DB_NAME="" ENV DB_USERNAME="" ENV DB_PASSWORD="" @@ -30,13 +30,13 @@ ENV SSH_PASSWORD="" ENV SSH_HOST_NAME="" ENV SSH_IDENTIFY_FILE="" ENV SSH_PORT="22" -ENV SOURCE_DB_HOST="" -ENV SOURCE_DB_PORT=3306 -ENV SOURCE_DB_NAME="" -ENV SOURCE_DB_USERNAME="" -ENV SOURCE_DB_PASSWORD="" +ENV TARGET_DB_HOST="" +ENV TARGET_DB_PORT=3306 +ENV TARGET_DB_NAME="localhost" +ENV TARGET_DB_USERNAME="" +ENV TARGET_DB_PASSWORD="" ARG DEBIAN_FRONTEND=noninteractive -ENV VERSION="v1.2.3" +ENV VERSION="v1.2.4" ENV BACKUP_CRON_EXPRESSION="" ARG WORKDIR="/config" ARG BACKUPDIR="/backup" diff --git a/docs/how-tos/migrate.md b/docs/how-tos/migrate.md index 9d553ba..8117bcc 100644 --- a/docs/how-tos/migrate.md +++ b/docs/how-tos/migrate.md @@ -10,11 +10,13 @@ nav_order: 9 To migrate the database, you need to add `migrate` command. {: .note } -The Mysql backup has another great feature: migrating your database from a source database to another. +The Mysql backup has another great feature: migrating your database from a source database to a target. As you know, to restore a database from a source to a target database, you need 2 operations: which is to start by backing up the source database and then restoring the source backed database to the target database. Instead of proceeding like that, you can use the integrated feature `(migrate)`, which will help you migrate your database by doing only one operation. +{: .warning } +The `migrate` operation is irreversible, please backup your target database before this action. ### Docker compose ```yml @@ -30,18 +32,18 @@ services: volumes: - ./backup:/backup environment: - ## Target database + ## Source database - DB_PORT=3306 - DB_HOST=mysql - DB_NAME=database - DB_USERNAME=username - DB_PASSWORD=password - ## Source database - - SOURCE_DB_HOST=mysql2 - - SOURCE_DB_PORT=3306 - - SOURCE_DB_NAME=sourcedb - - SOURCE_DB_USERNAME=jonas - - SOURCE_DB_PASSWORD=password + ## Target database + - TARGET_DB_HOST=target-mysql + - TARGET_DB_PORT=3306 + - TARGET_DB_NAME=dbname + - TARGET_DB_USERNAME=username + - TARGET_DB_PASSWORD=password # mysql-bkup container must be connected to the same network with your database networks: - web @@ -49,30 +51,31 @@ networks: web: ``` + ### Migrate database using Docker CLI ``` -## Target database -DB_PORT=3306 +## Source database DB_HOST=mysql -DB_NAME=targetdb -DB_USERNAME=targetuser +DB_PORT=3306 +DB_NAME=dbname +DB_USERNAME=username DB_PASSWORD=password -## Source database -SOURCE_DB_HOST=mysql2 -SOURCE_DB_PORT=3306 -SOURCE_DB_NAME=sourcedb -SOURCE_DB_USERNAME=sourceuser -SOURCE_DB_PASSWORD=password +## Taget database +TARGET_DB_HOST=target-mysql +TARGET_DB_PORT=3306 +TARGET_DB_NAME=dbname +TARGET_DB_USERNAME=username +TARGET_DB_PASSWORD=password ``` ```shell docker run --rm --network your_network_name \ --env-file your-env -v $PWD/backup:/backup/ \ - jkaninda/mysql-bkup migrate -d database_name + jkaninda/mysql-bkup migrate ``` ## Kubernetes @@ -96,28 +99,33 @@ spec: command: - /bin/sh - -c - - migrate -d targetdb + - migrate resources: limits: memory: "128Mi" cpu: "500m" env: - ## Target DB + ## Source Database - name: DB_HOST - value: "postgres-target" + value: "mysql" + - name: DB_PORT + value: "3306" + - name: DB_NAME + value: "dbname" - name: DB_USERNAME - value: "mysql" + value: "username" - name: DB_PASSWORD value: "password" - ## Source DB - - name: SOURCE_DB_HOST - value: "postgres-source" - - name: SOURCE_DB_NAME - value: "sourcedb" - - name: SOURCE_DB_USERNAME - value: "postgres" - # Please use secret! - - name: SOURCE_DB_PASSWORD + ## Target Database + - name: TARGET_DB_HOST + value: "target-mysql" + - name: TARGET_DB_PORT + value: "3306" + - name: TARGET_DB_NAME + value: "dbname" + - name: TARGET_DB_USERNAME + value: "username" + - name: TARGET_DB_PASSWORD value: "password" restartPolicy: Never ``` \ No newline at end of file diff --git a/docs/reference/index.md b/docs/reference/index.md index bec4272..be21b22 100644 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -57,11 +57,11 @@ Backup, restore and migrate targets, schedule and retention are configured using | SSH_IDENTIFY_FILE | Optional, required for SSH storage | ssh remote user's private key | | SSH_PORT | Optional, required for SSH storage | ssh remote server port | | SSH_REMOTE_PATH | Optional, required for SSH storage | ssh remote path (/home/toto/backup) | -| SOURCE_DB_HOST | Optional, required for database migration | Source database host | -| SOURCE_DB_PORT | Optional, required for database migration | Source database port | -| SOURCE_DB_NAME | Optional, required for database migration | Source database name | -| SOURCE_DB_USERNAME | Optional, required for database migration | Source database username | -| SOURCE_DB_PASSWORD | Optional, required for database migration | Source database password | +| TARGET_DB_HOST | Optional, required for database migration | Target database host | +| TARGET_DB_PORT | Optional, required for database migration | Target database port | +| TARGET_DB_NAME | Optional, required for database migration | Target database name | +| TARGET_DB_USERNAME | Optional, required for database migration | Target database username | +| TARGET_DB_PASSWORD | Optional, required for database migration | Target database password | --- ## Run in Scheduled mode diff --git a/pkg/config.go b/pkg/config.go index a0236c5..721a23a 100644 --- a/pkg/config.go +++ b/pkg/config.go @@ -16,12 +16,12 @@ type dbConfig struct { dbUserName string dbPassword string } -type dbSourceConfig struct { - sourceDbHost string - sourceDbPort string - sourceDbUserName string - sourceDbPassword string - sourceDbName string +type targetDbConfig struct { + targetDbHost string + targetDbPort string + targetDbUserName string + targetDbPassword string + targetDbName string } func getDbConfig(cmd *cobra.Command) *dbConfig { @@ -41,18 +41,18 @@ func getDbConfig(cmd *cobra.Command) *dbConfig { } return &dConf } -func getSourceDbConfig() *dbSourceConfig { - sdbConfig := dbSourceConfig{} - sdbConfig.sourceDbHost = os.Getenv("SOURCE_DB_HOST") - sdbConfig.sourceDbPort = os.Getenv("SOURCE_DB_PORT") - sdbConfig.sourceDbName = os.Getenv("SOURCE_DB_NAME") - sdbConfig.sourceDbUserName = os.Getenv("SOURCE_DB_USERNAME") - sdbConfig.sourceDbPassword = os.Getenv("SOURCE_DB_PASSWORD") +func getTargetDbConfig() *targetDbConfig { + tdbConfig := targetDbConfig{} + tdbConfig.targetDbHost = os.Getenv("TARGET_DB_HOST") + tdbConfig.targetDbPort = os.Getenv("TARGET_DB_PORT") + tdbConfig.targetDbName = os.Getenv("TARGET_DB_NAME") + tdbConfig.targetDbUserName = os.Getenv("TARGET_DB_USERNAME") + tdbConfig.targetDbPassword = os.Getenv("TARGET_DB_PASSWORD") - err := utils.CheckEnvVars(sdbRVars) + err := utils.CheckEnvVars(tdbRVars) if err != nil { - utils.Error("Please make sure all required environment variables for source database are set") - utils.Fatal("Error checking environment variables: %s", err) + utils.Error("Please make sure all required environment variables for the target database are set") + utils.Fatal("Error checking target database environment variables: %s", err) } - return &sdbConfig + return &tdbConfig } diff --git a/pkg/migrate.go b/pkg/migrate.go index fd80703..26453e6 100644 --- a/pkg/migrate.go +++ b/pkg/migrate.go @@ -11,21 +11,23 @@ func StartMigration(cmd *cobra.Command) { utils.Info("Starting database migration...") //Get DB config dbConf = getDbConfig(cmd) - sDbConf = getSourceDbConfig() + targetDbConf = getTargetDbConfig() - //Generate file name - backupFileName := fmt.Sprintf("%s_%s.sql", sDbConf.sourceDbName, time.Now().Format("20060102_150405")) - //Backup Source Database + //Defining the target database variables newDbConfig := dbConfig{} - newDbConfig.dbHost = sDbConf.sourceDbHost - newDbConfig.dbPort = sDbConf.sourceDbPort - newDbConfig.dbName = sDbConf.sourceDbName - newDbConfig.dbUserName = sDbConf.sourceDbUserName - newDbConfig.dbPassword = sDbConf.sourceDbPassword - BackupDatabase(&newDbConfig, backupFileName, true) + newDbConfig.dbHost = targetDbConf.targetDbHost + newDbConfig.dbPort = targetDbConf.targetDbPort + newDbConfig.dbName = targetDbConf.targetDbName + newDbConfig.dbUserName = targetDbConf.targetDbUserName + newDbConfig.dbPassword = targetDbConf.targetDbPassword + + //Generate file name + backupFileName := fmt.Sprintf("%s_%s.sql", dbConf.dbName, time.Now().Format("20060102_150405")) + //Backup source Database + BackupDatabase(dbConf, backupFileName, true) //Restore source database into target database - utils.Info("Restoring [%s] database into [%s] database...", sDbConf.sourceDbName, dbConf.dbName) - RestoreDatabase(dbConf, backupFileName) - utils.Info("[%s] database has been restored into [%s] database", sDbConf.sourceDbName, dbConf.dbName) - utils.Info("Database migration completed!") + utils.Info("Restoring [%s] database into [%s] database...", dbConf.dbName, targetDbConf.targetDbName) + RestoreDatabase(&newDbConfig, backupFileName) + utils.Info("[%s] database has been restored into [%s] database", dbConf.dbName, targetDbConf.targetDbName) + utils.Info("Database migration completed.") } diff --git a/pkg/restore.go b/pkg/restore.go index bb40d4a..6745579 100644 --- a/pkg/restore.go +++ b/pkg/restore.go @@ -95,7 +95,7 @@ func RestoreDatabase(db *dbConfig, file string) { extension := filepath.Ext(fmt.Sprintf("%s/%s", tmpPath, file)) // Restore from compressed file / .sql.gz if extension == ".gz" { - str := "zcat " + fmt.Sprintf("%s/%s", tmpPath, file) + " | mysql -h " + os.Getenv("DB_HOST") + " -P " + os.Getenv("DB_PORT") + " -u " + os.Getenv("DB_USERNAME") + " --password=" + os.Getenv("DB_PASSWORD") + " " + os.Getenv("DB_NAME") + str := "zcat " + fmt.Sprintf("%s/%s", tmpPath, file) + " | mysql -h " + db.dbHost + " -P " + db.dbPort + " -u " + db.dbUserName + " --password=" + db.dbPassword + " " + db.dbName _, err := exec.Command("bash", "-c", str).Output() if err != nil { utils.Fatal("Error, in restoring the database %v", err) @@ -107,7 +107,7 @@ func RestoreDatabase(db *dbConfig, file string) { } else if extension == ".sql" { //Restore from sql file - str := "cat " + fmt.Sprintf("%s/%s", tmpPath, file) + " | mysql -h " + os.Getenv("DB_HOST") + " -P " + os.Getenv("DB_PORT") + " -u " + os.Getenv("DB_USERNAME") + " --password=" + os.Getenv("DB_PASSWORD") + " " + os.Getenv("DB_NAME") + str := "cat " + fmt.Sprintf("%s/%s", tmpPath, file) + " | mysql -h " + db.dbHost + " -P " + db.dbPort + " -u " + db.dbUserName + " --password=" + db.dbPassword + " " + db.dbName _, err := exec.Command("bash", "-c", str).Output() if err != nil { utils.Fatal(fmt.Sprintf("Error in restoring the database %s", err)) diff --git a/pkg/var.go b/pkg/var.go index a7afd00..b5f8bda 100644 --- a/pkg/var.go +++ b/pkg/var.go @@ -23,16 +23,16 @@ var dbHVars = []string{ "DB_USERNAME", "DB_NAME", } -var sdbRVars = []string{ - "SOURCE_DB_HOST", - "SOURCE_DB_PORT", - "SOURCE_DB_NAME", - "SOURCE_DB_USERNAME", - "SOURCE_DB_PASSWORD", +var tdbRVars = []string{ + "TARGET_DB_HOST", + "TARGET_DB_PORT", + "TARGET_DB_NAME", + "TARGET_DB_USERNAME", + "TARGET_DB_PASSWORD", } var dbConf *dbConfig -var sDbConf *dbSourceConfig +var targetDbConf *targetDbConfig // sshHVars Required environment variables for SSH remote server storage var sshHVars = []string{ diff --git a/utils/s3.go b/utils/s3.go index a9513ee..a72c986 100644 --- a/utils/s3.go +++ b/utils/s3.go @@ -122,7 +122,7 @@ func DownloadFile(destinationPath, key, bucket, prefix string) error { fmt.Println("Failed to download file", err) return err } - Info(fmt.Sprintf("Backup downloaded: ", file.Name(), " bytes size ", numBytes)) + Info("Backup downloaded: %s bytes size %s ", file.Name(), numBytes) return nil }