diff --git a/README.md b/README.md index 5770d8d..c494df3 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,11 @@ Besides creating backups, `brudi` can also be used to restore your data from bac - [Redis](#redis) - [Restic](#restic) - [Forget](#forget) + - [Snapshots](#snapshots) + - [Check](#check) + - [Prune](#prune) + - [Rebuild-Index](#rebuild-index) + - [Tags](#tags) - [Sensitive data: Environment variables](#sensitive-data--environment-variables) - [Restoring from backup](#restoring-from-backup) - [TarRestore](#tarrestore) @@ -277,6 +282,82 @@ restic: ids: [] ``` +##### Snapshots + +It's possible to run `restic snapshots`-cmd after executing `restic backup` with `brudi` by using `--restic-snapshots`. +The `snapshots`-options are defined in the configuration `.yaml` for brudi. + +```yaml +restic: + global: + flags: + # you can provide the repository also via RESTIC_REPOSITORY + repo: "s3:s3.eu-central-1.amazonaws.com/your.s3.bucket/myResticRepo" + backup: + flags: + # in case there is no hostname given, the hostname from source backup is used + hostname: "MyHost" + # these paths are backuped additionally to your given source backup + paths: [] + snapshots: + flags: + host: "MyHost" +``` + +##### Check + +It's possible to run `restic check`-cmd after executing `restic backup` with `brudi` by using `--restic-check`. +The `check`-options are defined in the configuration `.yaml` for brudi. + +```yaml +restic: + global: + flags: + # you can provide the repository also via RESTIC_REPOSITORY + repo: "s3:s3.eu-central-1.amazonaws.com/your.s3.bucket/myResticRepo" + backup: + flags: + # in case there is no hostname given, the hostname from source backup is used + hostname: "MyHost" + # these paths are backuped additionally to your given source backup + paths: [] + check: + checkUnused: true +``` + +##### Prune + +It's possible to run `restic prune`-cmd after executing `restic backup` with `brudi` by using `--restic-prune`. +The `prune`-cmd has no specific options + +##### Rebuild-Index +It's possible to run `restic rebuild-index`-cmd after executing `restic backup` with `brudi` by using `--restic-rebuild-index`. +he `rebuild-index`-cmd has no specific options + +##### Tags + +It's possible to run `restic tags`-cmd after executing `restic backup` with `brudi` by using `--restic-tags`. +The `tags`-options are defined in the configuration `.yaml` for brudi. + +```yaml +restic: + global: + flags: + # you can provide the repository also via RESTIC_REPOSITORY + repo: "s3:s3.eu-central-1.amazonaws.com/your.s3.bucket/myResticRepo" + backup: + flags: + # in case there is no hostname given, the hostname from source backup is used + hostname: "MyHost" + # these paths are backuped additionally to your given source backup + paths: [] + tags: + flags: + add: ["TestTag"] + host: "MyHost" + ids: [] +``` + #### Sensitive data: Environment variables In case you don't want to provide data directly in the `.yaml`-file, e.g. sensitive data like passwords, you can use environment-variables. @@ -478,5 +559,10 @@ It is also possible to specify concrete snapshot-ids instead of `latest`. - [x] `restic backup` - [x] `restic forget` - [x] `restic restore` + - [x] `restic snapshots` + - [x] `restic prune` + - [x] `restic check` + - [x] `restic rebuild index` + - [x] `restic taga` - [x] `storage` - [x] `s3` diff --git a/cmd/mongodump.go b/cmd/mongodump.go index fe1e858..0e98460 100644 --- a/cmd/mongodump.go +++ b/cmd/mongodump.go @@ -19,7 +19,7 @@ var ( ctx, cancel := context.WithCancel(context.Background()) defer cancel() - err := source.DoBackupForKind(ctx, mongodump.Kind, cleanup, useRestic, useResticForget) + err := source.DoBackupForKind(ctx, mongodump.Kind, extaResticFlags, cleanup, useRestic, useResticForget) if err != nil { panic(err) } diff --git a/cmd/mysqldump.go b/cmd/mysqldump.go index 41f6126..72f3fc2 100644 --- a/cmd/mysqldump.go +++ b/cmd/mysqldump.go @@ -18,7 +18,7 @@ var ( ctx, cancel := context.WithCancel(context.Background()) defer cancel() - err := source.DoBackupForKind(ctx, mysqldump.Kind, cleanup, useRestic, useResticForget) + err := source.DoBackupForKind(ctx, mysqldump.Kind, extaResticFlags, cleanup, useRestic, useResticForget) if err != nil { panic(err) } diff --git a/cmd/pgdump.go b/cmd/pgdump.go index 15f8256..8482704 100644 --- a/cmd/pgdump.go +++ b/cmd/pgdump.go @@ -19,7 +19,7 @@ var ( ctx, cancel := context.WithCancel(context.Background()) defer cancel() - err := source.DoBackupForKind(ctx, pgdump.Kind, cleanup, useRestic, useResticForget) + err := source.DoBackupForKind(ctx, pgdump.Kind, extaResticFlags, cleanup, useRestic, useResticForget) if err != nil { panic(err) } diff --git a/cmd/redisdump.go b/cmd/redisdump.go index e400f40..aef3a03 100644 --- a/cmd/redisdump.go +++ b/cmd/redisdump.go @@ -19,7 +19,7 @@ var ( ctx, cancel := context.WithCancel(context.Background()) defer cancel() - err := source.DoBackupForKind(ctx, redisdump.Kind, cleanup, useRestic, useResticForget) + err := source.DoBackupForKind(ctx, redisdump.Kind, extaResticFlags, cleanup, useRestic, useResticForget) if err != nil { panic(err) } diff --git a/cmd/root.go b/cmd/root.go index 7a2bd82..2677dbd 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -19,6 +19,7 @@ var ( useRestic bool useResticForget bool cleanup bool + extaResticFlags config.ExtraResticFlags rootCmd = &cobra.Command{ Use: "brudi", @@ -37,6 +38,21 @@ func init() { rootCmd.PersistentFlags().BoolVar(&cleanup, "cleanup", false, "cleanup backup files afterwards") + rootCmd.PersistentFlags().BoolVar(&extaResticFlags.ResticList, "restic-snapshots", false, + "list snapshots in restic repository afterwards") + + rootCmd.PersistentFlags().BoolVar(&extaResticFlags.ResticCheck, "restic-check", false, + "perform 'restic check' on the repository") + + rootCmd.PersistentFlags().BoolVar(&extaResticFlags.ResticPrune, "restic-prune", false, + "perform 'restic prune' on the repository") + + rootCmd.PersistentFlags().BoolVar(&extaResticFlags.ResticRebuild, "restic-rebuild-index", false, + "perform 'restic rebuild-index' on the repository") + + rootCmd.PersistentFlags().BoolVar(&extaResticFlags.ResticTags, "restic-tags", false, + "executes 'restic tags' after backing up things with restic") + rootCmd.PersistentFlags().StringSliceVarP(&cfgFiles, "config", "c", []string{}, "config file (default is ${HOME}/.brudi.yaml)") } diff --git a/cmd/tar.go b/cmd/tar.go index ff4f5a4..1f1df9f 100644 --- a/cmd/tar.go +++ b/cmd/tar.go @@ -19,7 +19,7 @@ var ( ctx, cancel := context.WithCancel(context.Background()) defer cancel() - err := source.DoBackupForKind(ctx, tar.Kind, cleanup, useRestic, useResticForget) + err := source.DoBackupForKind(ctx, tar.Kind, extaResticFlags, cleanup, useRestic, useResticForget) if err != nil { panic(err) } diff --git a/pkg/config/types.go b/pkg/config/types.go index 7b76045..bfa8aad 100644 --- a/pkg/config/types.go +++ b/pkg/config/types.go @@ -4,3 +4,11 @@ const ( KeyOptionsFlags = "options.flags" KeyOptionsAdditionalArgs = "options.additionalArgs" ) + +type ExtraResticFlags struct { + ResticList bool + ResticCheck bool + ResticPrune bool + ResticRebuild bool + ResticTags bool +} diff --git a/pkg/restic/client.go b/pkg/restic/client.go index f4988e9..126c84f 100644 --- a/pkg/restic/client.go +++ b/pkg/restic/client.go @@ -13,6 +13,7 @@ type Client struct { Config *Config } +// NewResticClient creates a new restic client with the given hostname and backup paths func NewResticClient(logger *log.Entry, hostname string, backupPaths ...string) (*Client, error) { conf := &Config{ Global: &GlobalOptions{ @@ -26,6 +27,15 @@ func NewResticClient(logger *log.Entry, hostname string, backupPaths ...string) Flags: &ForgetFlags{}, IDs: []string{}, }, + Snapshots: &SnapshotOptions{ + Flags: &SnapshotFlags{}, + IDs: []string{}, + }, + Tags: &TagOptions{ + Flags: &TagFlags{}, + IDs: []string{}, + }, + Check: &CheckFlags{}, Restore: &RestoreOptions{ Flags: &RestoreFlags{}, ID: "", @@ -51,6 +61,7 @@ func NewResticClient(logger *log.Entry, hostname string, backupPaths ...string) }, nil } +// DoResticBackup executes initBackup and CreateBackup with the settings from c func (c *Client) DoResticBackup(ctx context.Context) error { c.Logger.Info("running 'restic backup'") @@ -73,6 +84,7 @@ func (c *Client) DoResticBackup(ctx context.Context) error { return nil } +// DoResticRestore executes RestoreBackup with the settings from c func (c *Client) DoResticRestore(ctx context.Context, backupPath string) error { c.Logger.Info("running 'restic restore'") _, err := RestoreBackup(ctx, c.Config.Global, c.Config.Restore, false) @@ -82,6 +94,7 @@ func (c *Client) DoResticRestore(ctx context.Context, backupPath string) error { return nil } +// DoResticForget executes Forget with the settings of c func (c *Client) DoResticForget(ctx context.Context) error { c.Logger.Info("running 'restic forget'") @@ -96,3 +109,67 @@ func (c *Client) DoResticForget(ctx context.Context) error { return nil } + +// DoResticListSnapshots executes ListSnapshots with the settings from c +func (c *Client) DoResticListSnapshots(ctx context.Context) error { + c.Logger.Info("running 'restic snapshots'") + + output, err := ListSnapshots(ctx, c.Config.Global, c.Config.Snapshots) + if err != nil { + return errors.WithStack(err) + } + fmt.Println("output of 'restic snapshots':") + for index := range output { + fmt.Printf("ID: %d; Time: %s; Host: %s; Tags: %s; Paths: %s\n", + output[index].ID, output[index].Time, output[index].Hostname, output[index].Tags, output[index].Paths) + } + return nil +} + +// DoResticCheck executes Check with the settings from c +func (c *Client) DoResticCheck(ctx context.Context) error { + c.Logger.Info("running 'restic check'") + + output, err := Check(ctx, c.Config.Global, c.Config.Check) + if err != nil { + return errors.WithStack(fmt.Errorf("%s - %s", err.Error(), output)) + } + fmt.Println(string(output)) + return nil +} + +// DoResticPruneRepo executes Prune with the settings from c +func (c *Client) DoResticPruneRepo(ctx context.Context) error { + c.Logger.Info("running 'restic prune") + + output, err := Prune(ctx, c.Config.Global) + if err != nil { + return errors.WithStack(fmt.Errorf("%s - %s", err.Error(), output)) + } + fmt.Println(string(output)) + return nil +} + +// DoResticRebuildIndex executes RebuildIndex with the settings from c +func (c *Client) DoResticRebuildIndex(ctx context.Context) error { + c.Logger.Info("running 'restic rebuild-index'") + + output, err := RebuildIndex(ctx, c.Config.Global) + if err != nil { + return errors.WithStack(fmt.Errorf("%s - %s", err.Error(), output)) + } + fmt.Println(string(output)) + return nil +} + +// DoResticTag executes Tag with the settings from c +func (c *Client) DoResticTag(ctx context.Context) error { + c.Logger.Info("running 'restic rebuild-index'") + + output, err := Tag(ctx, c.Config.Global, c.Config.Tags) + if err != nil { + return errors.WithStack(fmt.Errorf("%s - %s", err.Error(), output)) + } + fmt.Println(string(output)) + return nil +} diff --git a/pkg/restic/commands.go b/pkg/restic/commands.go index 8ac4322..a592626 100644 --- a/pkg/restic/commands.go +++ b/pkg/restic/commands.go @@ -25,7 +25,7 @@ var ( cmdTimeout = 6 * time.Hour ) -// InitBackup executes "restic init" +// initBackup executes "restic init" func initBackup(ctx context.Context, globalOpts *GlobalOptions) ([]byte, error) { cmd := newCommand("init", cli.StructToCLI(globalOpts)...) @@ -256,11 +256,17 @@ func GetSnapshotSizeByPath(ctx context.Context, snapshotID, path string) (size u } // ListSnapshots executes "restic snapshots" -func ListSnapshots(ctx context.Context, opts *SnapshotOptions) ([]Snapshot, error) { - cmd := newCommand("snapshots", cli.StructToCLI(&opts)...) - +func ListSnapshots(ctx context.Context, glob *GlobalOptions, opts *SnapshotOptions) ([]Snapshot, error) { + args := append([]string{"--json"}, cli.StructToCLI(opts)...) + args = append(args, cli.StructToCLI(glob)...) + cmd := cli.CommandType{ + Binary: binary, + Command: "snapshots", + Args: args, + } out, err := cli.Run(ctx, cmd) if err != nil { + fmt.Println(string(out)) return nil, err } var snapshots []Snapshot @@ -288,8 +294,12 @@ func Find(ctx context.Context, opts *FindOptions) ([]FindResult, error) { } // Check executes "restic check" -func Check(ctx context.Context, flags *CheckFlags) ([]byte, error) { - cmd := newCommand("check", cli.StructToCLI(flags)...) +func Check(ctx context.Context, glob *GlobalOptions, flags *CheckFlags) ([]byte, error) { + cmd := cli.CommandType{ + Binary: binary, + Command: "check", + Args: append(cli.StructToCLI(glob), cli.StructToCLI(flags)...), + } return cli.Run(ctx, cmd) } @@ -332,20 +342,23 @@ func Forget( } // Prune executes "restic prune" -func Prune(ctx context.Context) ([]byte, error) { - cmd := newCommand("prune", nil...) - +func Prune(ctx context.Context, glob *GlobalOptions) ([]byte, error) { + cmd := cli.CommandType{ + Binary: binary, + Command: "prune", + Args: cli.StructToCLI(glob), + } return cli.Run(ctx, cmd) } // RebuildIndex executes "restic rebuild-index" -func RebuildIndex(ctx context.Context) ([]byte, error) { +func RebuildIndex(ctx context.Context, glob *GlobalOptions) ([]byte, error) { nice := 19 ionice := 2 cmd := cli.CommandType{ Binary: binary, Command: "rebuild-index", - Args: nil, + Args: cli.StructToCLI(glob), Nice: &nice, IONice: &ionice, } @@ -385,8 +398,11 @@ func Unlock(ctx context.Context, globalOpts *GlobalOptions, unlockOpts *UnlockOp } // Tag executes "restic tag" -func Tag(ctx context.Context, opts *TagOptions) ([]byte, error) { - cmd := newCommand("tag", cli.StructToCLI(opts)...) - +func Tag(ctx context.Context, glob *GlobalOptions, opts *TagOptions) ([]byte, error) { + cmd := cli.CommandType{ + Binary: binary, + Command: "tag", + Args: append(cli.StructToCLI(glob), cli.StructToCLI(opts)...), + } return cli.Run(ctx, cmd) } diff --git a/pkg/restic/config.go b/pkg/restic/config.go index ae95f89..d85d6f2 100644 --- a/pkg/restic/config.go +++ b/pkg/restic/config.go @@ -9,10 +9,13 @@ const ( ) type Config struct { - Global *GlobalOptions - Backup *BackupOptions - Forget *ForgetOptions - Restore *RestoreOptions + Global *GlobalOptions + Backup *BackupOptions + Forget *ForgetOptions + Restore *RestoreOptions + Snapshots *SnapshotOptions + Tags *TagOptions + Check *CheckFlags } func (c *Config) InitFromViper() error { diff --git a/pkg/source/backup.go b/pkg/source/backup.go index 320920e..4ec529f 100644 --- a/pkg/source/backup.go +++ b/pkg/source/backup.go @@ -4,6 +4,8 @@ import ( "context" "fmt" + "github.com/mittwald/brudi/pkg/config" + "github.com/mittwald/brudi/pkg/restic" "github.com/mittwald/brudi/pkg/source/pgdump" @@ -34,7 +36,10 @@ func getGenericBackendForKind(kind string) (Generic, error) { } } -func DoBackupForKind(ctx context.Context, kind string, cleanup, useRestic, useResticForget bool) error { +// DoBackupForKind performs the appropriate backup action for given arguments. +// It also executes any given restic commands after files have been backed up, +func DoBackupForKind(ctx context.Context, kind string, resticFlags config.ExtraResticFlags, + cleanup, useRestic, useResticForget bool) error { logKind := log.WithFields( log.Fields{ "kind": kind, @@ -79,14 +84,51 @@ func DoBackupForKind(ctx context.Context, kind string, cleanup, useRestic, useRe return err } + // execute any applicable restic commands + err = resticClient.DoResticBackup(ctx) if err != nil { return err } - if !useResticForget { + if useResticForget { + err = resticClient.DoResticForget(ctx) + if err != nil { + return err + } + } + + if resticFlags.ResticList { + err = resticClient.DoResticListSnapshots(ctx) + if err != nil { + return err + } + } + + if resticFlags.ResticPrune { + err = resticClient.DoResticPruneRepo(ctx) + if err != nil { + return err + } + } + + if resticFlags.ResticTags { + err = resticClient.DoResticTag(ctx) + if err != nil { + return err + } + } + + if resticFlags.ResticRebuild { + err = resticClient.DoResticRebuildIndex(ctx) + if err != nil { + return err + } + } + + if !resticFlags.ResticCheck { return nil } - return resticClient.DoResticForget(ctx) + return resticClient.DoResticCheck(ctx) } diff --git a/test/pkg/source/internal/testcommons.go b/test/pkg/source/internal/testcommons.go index e9ddb32..7f222fe 100644 --- a/test/pkg/source/internal/testcommons.go +++ b/test/pkg/source/internal/testcommons.go @@ -3,13 +3,15 @@ package internal import ( "context" "fmt" - "github.com/pkg/errors" - "github.com/spf13/viper" "os" "os/exec" "strings" + "github.com/mittwald/brudi/pkg/config" + "github.com/docker/go-connections/nat" + "github.com/pkg/errors" + "github.com/spf13/viper" "github.com/testcontainers/testcontainers-go" ) @@ -33,6 +35,14 @@ var ResticReq = testcontainers.ContainerRequest{ }, } +var ExtraFlags = config.ExtraResticFlags{ + ResticCheck: false, + ResticRebuild: false, + ResticTags: false, + ResticPrune: false, + ResticList: false, +} + // NewTestContainerSetup creates a TestContainerSetup which acts as a wrapper for the testcontainer specified by request func NewTestContainerSetup(ctx context.Context, request *testcontainers.ContainerRequest, port nat.Port) (TestContainerSetup, error) { result := TestContainerSetup{} diff --git a/test/pkg/source/mongodbtest/mongodb_test.go b/test/pkg/source/mongodbtest/mongodb_test.go index b3551bb..5518c7c 100644 --- a/test/pkg/source/mongodbtest/mongodb_test.go +++ b/test/pkg/source/mongodbtest/mongodb_test.go @@ -150,7 +150,7 @@ func mongoDoBackup(ctx context.Context, useRestic bool, } // perform backup action on mongodb-container - err = source.DoBackupForKind(ctx, dumpKind, false, useRestic, false) + err = source.DoBackupForKind(ctx, dumpKind, commons.ExtraFlags, false, useRestic, false) if err != nil { return []interface{}{}, err } diff --git a/test/pkg/source/mysqltest/mysql_test.go b/test/pkg/source/mysqltest/mysql_test.go index c6f4886..63e80b5 100644 --- a/test/pkg/source/mysqltest/mysql_test.go +++ b/test/pkg/source/mysqltest/mysql_test.go @@ -172,7 +172,7 @@ func mySQLDoBackup(ctx context.Context, useRestic bool, } // use brudi to create dump - err = source.DoBackupForKind(ctx, dumpKind, false, useRestic, false) + err = source.DoBackupForKind(ctx, dumpKind, commons.ExtraFlags, false, useRestic, false) if err != nil { return []TestStruct{}, err } diff --git a/test/pkg/source/postgrestest/postgres_test.go b/test/pkg/source/postgrestest/postgres_test.go index 183468b..99264cc 100644 --- a/test/pkg/source/postgrestest/postgres_test.go +++ b/test/pkg/source/postgrestest/postgres_test.go @@ -192,7 +192,7 @@ func pgDoBackup(ctx context.Context, useRestic bool, } // perform backup action on database - err = source.DoBackupForKind(ctx, dumpKind, false, useRestic, false) + err = source.DoBackupForKind(ctx, dumpKind, commons.ExtraFlags, false, useRestic, false) if err != nil { return []testStruct{}, err } diff --git a/test/pkg/source/redisdump/redisdump_test.go b/test/pkg/source/redisdump/redisdump_test.go index 83d0b1e..f8f1274 100644 --- a/test/pkg/source/redisdump/redisdump_test.go +++ b/test/pkg/source/redisdump/redisdump_test.go @@ -160,7 +160,7 @@ func redisDoBackup(ctx context.Context, useRestic bool, } // perform backup action on first redis container - err = source.DoBackupForKind(ctx, dumpKind, false, useRestic, false) + err = source.DoBackupForKind(ctx, dumpKind, commons.ExtraFlags, false, useRestic, false) if err != nil { return testStruct{}, errors.WithStack(err) } diff --git a/test/pkg/source/tar/tar_test.go b/test/pkg/source/tar/tar_test.go index 430fb4f..1f2fc82 100644 --- a/test/pkg/source/tar/tar_test.go +++ b/test/pkg/source/tar/tar_test.go @@ -76,7 +76,7 @@ func TestTarTestSuite(t *testing.T) { // tarDoBackup uses brudi to compress a test file into a tar.gz archive and returns the uncompressed files md5 hash func tarDoBackup(ctx context.Context) (string, error) { hash, err := hashFile(backupPath) - err = source.DoBackupForKind(ctx, "tar", false, false, false) + err = source.DoBackupForKind(ctx, "tar", commons.ExtraFlags, false, false, false) if err != nil { return "", err }