Skip to content

Commit

Permalink
chore: rename environment variable for database migration operation
Browse files Browse the repository at this point in the history
  • Loading branch information
jkaninda committed Sep 3, 2024
1 parent f3c5585 commit 07c2935
Show file tree
Hide file tree
Showing 9 changed files with 96 additions and 86 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: Release
name: CI
on:
push:
tags:
Expand Down
14 changes: 7 additions & 7 deletions docker/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ RUN go mod download
RUN CGO_ENABLED=0 GOOS=linux go build -o /app/mysql-bkup

FROM ubuntu:24.04
ENV DB_HOST=""
ENV DB_HOST="localhost"
ENV DB_NAME=""
ENV DB_USERNAME=""
ENV DB_PASSWORD=""
Expand All @@ -30,13 +30,13 @@ ENV SSH_PASSWORD=""
ENV SSH_HOST_NAME=""
ENV SSH_IDENTIFY_FILE=""
ENV SSH_PORT="22"
ENV SOURCE_DB_HOST=""
ENV SOURCE_DB_PORT=3306
ENV SOURCE_DB_NAME=""
ENV SOURCE_DB_USERNAME=""
ENV SOURCE_DB_PASSWORD=""
ENV TARGET_DB_HOST=""
ENV TARGET_DB_PORT=3306
ENV TARGET_DB_NAME="localhost"
ENV TARGET_DB_USERNAME=""
ENV TARGET_DB_PASSWORD=""
ARG DEBIAN_FRONTEND=noninteractive
ENV VERSION="v1.2.3"
ENV VERSION="v1.2.4"
ENV BACKUP_CRON_EXPRESSION=""
ARG WORKDIR="/config"
ARG BACKUPDIR="/backup"
Expand Down
72 changes: 40 additions & 32 deletions docs/how-tos/migrate.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,13 @@ nav_order: 9
To migrate the database, you need to add `migrate` command.

{: .note }
The Mysql backup has another great feature: migrating your database from a source database to another.
The Mysql backup has another great feature: migrating your database from a source database to a target.

As you know, to restore a database from a source to a target database, you need 2 operations: which is to start by backing up the source database and then restoring the source backed database to the target database.
Instead of proceeding like that, you can use the integrated feature `(migrate)`, which will help you migrate your database by doing only one operation.

{: .warning }
The `migrate` operation is irreversible, please backup your target database before this action.

### Docker compose
```yml
Expand All @@ -30,49 +32,50 @@ services:
volumes:
- ./backup:/backup
environment:
## Target database
## Source database
- DB_PORT=3306
- DB_HOST=mysql
- DB_NAME=database
- DB_USERNAME=username
- DB_PASSWORD=password
## Source database
- SOURCE_DB_HOST=mysql2
- SOURCE_DB_PORT=3306
- SOURCE_DB_NAME=sourcedb
- SOURCE_DB_USERNAME=jonas
- SOURCE_DB_PASSWORD=password
## Target database
- TARGET_DB_HOST=target-mysql
- TARGET_DB_PORT=3306
- TARGET_DB_NAME=dbname
- TARGET_DB_USERNAME=username
- TARGET_DB_PASSWORD=password
# mysql-bkup container must be connected to the same network with your database
networks:
- web
networks:
web:
```
### Migrate database using Docker CLI
```
## Target database
DB_PORT=3306
## Source database
DB_HOST=mysql
DB_NAME=targetdb
DB_USERNAME=targetuser
DB_PORT=3306
DB_NAME=dbname
DB_USERNAME=username
DB_PASSWORD=password

## Source database
SOURCE_DB_HOST=mysql2
SOURCE_DB_PORT=3306
SOURCE_DB_NAME=sourcedb
SOURCE_DB_USERNAME=sourceuser
SOURCE_DB_PASSWORD=password
## Taget database
TARGET_DB_HOST=target-mysql
TARGET_DB_PORT=3306
TARGET_DB_NAME=dbname
TARGET_DB_USERNAME=username
TARGET_DB_PASSWORD=password
```

```shell
docker run --rm --network your_network_name \
--env-file your-env
-v $PWD/backup:/backup/ \
jkaninda/mysql-bkup migrate -d database_name
jkaninda/mysql-bkup migrate
```

## Kubernetes
Expand All @@ -96,28 +99,33 @@ spec:
command:
- /bin/sh
- -c
- migrate -d targetdb
- migrate
resources:
limits:
memory: "128Mi"
cpu: "500m"
env:
## Target DB
## Source Database
- name: DB_HOST
value: "postgres-target"
value: "mysql"
- name: DB_PORT
value: "3306"
- name: DB_NAME
value: "dbname"
- name: DB_USERNAME
value: "mysql"
value: "username"
- name: DB_PASSWORD
value: "password"
## Source DB
- name: SOURCE_DB_HOST
value: "postgres-source"
- name: SOURCE_DB_NAME
value: "sourcedb"
- name: SOURCE_DB_USERNAME
value: "postgres"
# Please use secret!
- name: SOURCE_DB_PASSWORD
## Target Database
- name: TARGET_DB_HOST
value: "target-mysql"
- name: TARGET_DB_PORT
value: "3306"
- name: TARGET_DB_NAME
value: "dbname"
- name: TARGET_DB_USERNAME
value: "username"
- name: TARGET_DB_PASSWORD
value: "password"
restartPolicy: Never
```
10 changes: 5 additions & 5 deletions docs/reference/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -57,11 +57,11 @@ Backup, restore and migrate targets, schedule and retention are configured using
| SSH_IDENTIFY_FILE | Optional, required for SSH storage | ssh remote user's private key |
| SSH_PORT | Optional, required for SSH storage | ssh remote server port |
| SSH_REMOTE_PATH | Optional, required for SSH storage | ssh remote path (/home/toto/backup) |
| SOURCE_DB_HOST | Optional, required for database migration | Source database host |
| SOURCE_DB_PORT | Optional, required for database migration | Source database port |
| SOURCE_DB_NAME | Optional, required for database migration | Source database name |
| SOURCE_DB_USERNAME | Optional, required for database migration | Source database username |
| SOURCE_DB_PASSWORD | Optional, required for database migration | Source database password |
| TARGET_DB_HOST | Optional, required for database migration | Target database host |
| TARGET_DB_PORT | Optional, required for database migration | Target database port |
| TARGET_DB_NAME | Optional, required for database migration | Target database name |
| TARGET_DB_USERNAME | Optional, required for database migration | Target database username |
| TARGET_DB_PASSWORD | Optional, required for database migration | Target database password |

---
## Run in Scheduled mode
Expand Down
34 changes: 17 additions & 17 deletions pkg/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,12 @@ type dbConfig struct {
dbUserName string
dbPassword string
}
type dbSourceConfig struct {
sourceDbHost string
sourceDbPort string
sourceDbUserName string
sourceDbPassword string
sourceDbName string
type targetDbConfig struct {
targetDbHost string
targetDbPort string
targetDbUserName string
targetDbPassword string
targetDbName string
}

func getDbConfig(cmd *cobra.Command) *dbConfig {
Expand All @@ -41,18 +41,18 @@ func getDbConfig(cmd *cobra.Command) *dbConfig {
}
return &dConf
}
func getSourceDbConfig() *dbSourceConfig {
sdbConfig := dbSourceConfig{}
sdbConfig.sourceDbHost = os.Getenv("SOURCE_DB_HOST")
sdbConfig.sourceDbPort = os.Getenv("SOURCE_DB_PORT")
sdbConfig.sourceDbName = os.Getenv("SOURCE_DB_NAME")
sdbConfig.sourceDbUserName = os.Getenv("SOURCE_DB_USERNAME")
sdbConfig.sourceDbPassword = os.Getenv("SOURCE_DB_PASSWORD")
func getTargetDbConfig() *targetDbConfig {
tdbConfig := targetDbConfig{}
tdbConfig.targetDbHost = os.Getenv("TARGET_DB_HOST")
tdbConfig.targetDbPort = os.Getenv("TARGET_DB_PORT")
tdbConfig.targetDbName = os.Getenv("TARGET_DB_NAME")
tdbConfig.targetDbUserName = os.Getenv("TARGET_DB_USERNAME")
tdbConfig.targetDbPassword = os.Getenv("TARGET_DB_PASSWORD")

err := utils.CheckEnvVars(sdbRVars)
err := utils.CheckEnvVars(tdbRVars)
if err != nil {
utils.Error("Please make sure all required environment variables for source database are set")
utils.Fatal("Error checking environment variables: %s", err)
utils.Error("Please make sure all required environment variables for the target database are set")
utils.Fatal("Error checking target database environment variables: %s", err)
}
return &sdbConfig
return &tdbConfig
}
30 changes: 16 additions & 14 deletions pkg/migrate.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,21 +11,23 @@ func StartMigration(cmd *cobra.Command) {
utils.Info("Starting database migration...")
//Get DB config
dbConf = getDbConfig(cmd)
sDbConf = getSourceDbConfig()
targetDbConf = getTargetDbConfig()

//Generate file name
backupFileName := fmt.Sprintf("%s_%s.sql", sDbConf.sourceDbName, time.Now().Format("20060102_150405"))
//Backup Source Database
//Defining the target database variables
newDbConfig := dbConfig{}
newDbConfig.dbHost = sDbConf.sourceDbHost
newDbConfig.dbPort = sDbConf.sourceDbPort
newDbConfig.dbName = sDbConf.sourceDbName
newDbConfig.dbUserName = sDbConf.sourceDbUserName
newDbConfig.dbPassword = sDbConf.sourceDbPassword
BackupDatabase(&newDbConfig, backupFileName, true)
newDbConfig.dbHost = targetDbConf.targetDbHost
newDbConfig.dbPort = targetDbConf.targetDbPort
newDbConfig.dbName = targetDbConf.targetDbName
newDbConfig.dbUserName = targetDbConf.targetDbUserName
newDbConfig.dbPassword = targetDbConf.targetDbPassword

//Generate file name
backupFileName := fmt.Sprintf("%s_%s.sql", dbConf.dbName, time.Now().Format("20060102_150405"))
//Backup source Database
BackupDatabase(dbConf, backupFileName, true)
//Restore source database into target database
utils.Info("Restoring [%s] database into [%s] database...", sDbConf.sourceDbName, dbConf.dbName)
RestoreDatabase(dbConf, backupFileName)
utils.Info("[%s] database has been restored into [%s] database", sDbConf.sourceDbName, dbConf.dbName)
utils.Info("Database migration completed!")
utils.Info("Restoring [%s] database into [%s] database...", dbConf.dbName, targetDbConf.targetDbName)
RestoreDatabase(&newDbConfig, backupFileName)
utils.Info("[%s] database has been restored into [%s] database", dbConf.dbName, targetDbConf.targetDbName)
utils.Info("Database migration completed.")
}
4 changes: 2 additions & 2 deletions pkg/restore.go
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ func RestoreDatabase(db *dbConfig, file string) {
extension := filepath.Ext(fmt.Sprintf("%s/%s", tmpPath, file))
// Restore from compressed file / .sql.gz
if extension == ".gz" {
str := "zcat " + fmt.Sprintf("%s/%s", tmpPath, file) + " | mysql -h " + os.Getenv("DB_HOST") + " -P " + os.Getenv("DB_PORT") + " -u " + os.Getenv("DB_USERNAME") + " --password=" + os.Getenv("DB_PASSWORD") + " " + os.Getenv("DB_NAME")
str := "zcat " + fmt.Sprintf("%s/%s", tmpPath, file) + " | mysql -h " + db.dbHost + " -P " + db.dbPort + " -u " + db.dbUserName + " --password=" + db.dbPassword + " " + db.dbName
_, err := exec.Command("bash", "-c", str).Output()
if err != nil {
utils.Fatal("Error, in restoring the database %v", err)
Expand All @@ -107,7 +107,7 @@ func RestoreDatabase(db *dbConfig, file string) {

} else if extension == ".sql" {
//Restore from sql file
str := "cat " + fmt.Sprintf("%s/%s", tmpPath, file) + " | mysql -h " + os.Getenv("DB_HOST") + " -P " + os.Getenv("DB_PORT") + " -u " + os.Getenv("DB_USERNAME") + " --password=" + os.Getenv("DB_PASSWORD") + " " + os.Getenv("DB_NAME")
str := "cat " + fmt.Sprintf("%s/%s", tmpPath, file) + " | mysql -h " + db.dbHost + " -P " + db.dbPort + " -u " + db.dbUserName + " --password=" + db.dbPassword + " " + db.dbName
_, err := exec.Command("bash", "-c", str).Output()
if err != nil {
utils.Fatal(fmt.Sprintf("Error in restoring the database %s", err))
Expand Down
14 changes: 7 additions & 7 deletions pkg/var.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,16 @@ var dbHVars = []string{
"DB_USERNAME",
"DB_NAME",
}
var sdbRVars = []string{
"SOURCE_DB_HOST",
"SOURCE_DB_PORT",
"SOURCE_DB_NAME",
"SOURCE_DB_USERNAME",
"SOURCE_DB_PASSWORD",
var tdbRVars = []string{
"TARGET_DB_HOST",
"TARGET_DB_PORT",
"TARGET_DB_NAME",
"TARGET_DB_USERNAME",
"TARGET_DB_PASSWORD",
}

var dbConf *dbConfig
var sDbConf *dbSourceConfig
var targetDbConf *targetDbConfig

// sshHVars Required environment variables for SSH remote server storage
var sshHVars = []string{
Expand Down
2 changes: 1 addition & 1 deletion utils/s3.go
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ func DownloadFile(destinationPath, key, bucket, prefix string) error {
fmt.Println("Failed to download file", err)
return err
}
Info(fmt.Sprintf("Backup downloaded: ", file.Name(), " bytes size ", numBytes))
Info("Backup downloaded: %s bytes size %s ", file.Name(), numBytes)

return nil
}
Expand Down

0 comments on commit 07c2935

Please sign in to comment.