Merge pull request #5 from rqlite/topic/autobackup

Add chart support for auto backup and restore

charts/rqlite/Chart.yaml

@@ -1,8 +1,8 @@
 apiVersion: v2
 name: rqlite
-version: 1.0.1
-appVersion: 8.14.1
-description: The lightweight, distributed relational database built on SQLite 
+version: 1.1.0
+appVersion: 8.16.0
+description: The lightweight, distributed relational database built on SQLite
 type: application
 home: https://rqlite.io/
 sources:

charts/rqlite/README.md

@@ -232,6 +232,48 @@ and can thereafter be dynamically scaled, either by running `kubectl scale`, usi
 some other orchestrator.
 
 
+## Auto Backup and Restore
+
+rqlite provides support for [automatically backing up its data to S3-compatible
+storage](https://rqlite.io/docs/guides/backup/#automatic-backups), and even automatically
+restoring from that backup when the cluster is being bootstrapped from a clean (data-less)
+state.
+
+The Helm chart exposes this capability under the `config.backup` dict.  Here, the S3
+storage details are provided in `config.backup.storage`, which applies to both backup and
+restore, and then automatic backup and restore can be independently enabled using
+`config.backup.autoBackup.enabled` and `config.backup.autoRestore.enabled` respectively.
+
+The example below configures both automatic backup and restore to a MinIO deployment at
+`s3.example.com` with credentials for a MinIO Service Account with a policy that grants
+read/write access to a bucket called `rqlite`:
+
+```yaml
+config:
+  backup:
+    storage:
+      # MinIO Service Account credentials
+      accessKeyId: cdRtR5mRJMvtMJz51Cts
+      secretAccessKey: YY6RieQEwbbek3rhjOPwbwEUIkg8kYhhbxrL0h3R
+      bucket: rqlite
+      # region is a required field, but the value doesn't generally matter with MinIO
+      region: us-east-1
+      path: backups/mydatabase.sqlite.gz
+      # Endpoint must be defined for any non-Amazon-native S3 storage
+      endpoint: s3.example.com
+      # Most MinIO deployments use path-style requests, so unlike Amazon S3 (where this
+      # is not recommended to be set), for MinIO we set it to true
+      forcePathStyle: true
+    autoBackup:
+      # Enable automatic backups every 30 minutes
+      enabled: true
+      interval: 30m
+    autoRestore:
+      # Enable automatic restoration when the cluster is being bootstrapped
+      # with no existing data.
+      enabled: true
+```
+
 ## Versioning
 
 Helm charts use semantic versioning, and rqlite's chart offers the following guarantees:

charts/rqlite/templates/_statefulset.tpl

@@ -189,6 +189,12 @@ spec:
             # that changes to replicaCount doesn't result in a rolling restart of all pods
             # in the StatefulSet.
             - -bootstrap-expect=$(RQLITE_VOTER_REPLICA_COUNT)
+            {{- if dig "backup" "autoBackup" "enabled" false $config }}
+            - -auto-backup=/config/sensitive/backup.json
+            {{- end }}
+            {{- if dig "backup" "autoRestore" "enabled" false $config }}
+            - -auto-restore=/config/sensitive/restore.json
+            {{- end }}
             {{- end }}
             - -join-interval=1s
             - -join-attempts=120

charts/rqlite/templates/secret.yaml

@@ -1,15 +1,47 @@
 {{- $name := tpl (include "rqlite.fullname" .) $ -}}
 {{- include "rqlite.generateSecrets" . }}
+{{- $config := .Values.config }}
 {{/*
 List of additional generated system users which will be concatenated with the
 user-suppplied list below
 */}}
-{{ $internalUsers := list
-     (dict "username" "_system_probes" "perms" (list "ready") 
+{{- $internalUsers := list
+     (dict "username" "_system_probes" "perms" (list "ready")
       "password" (.Release.rqlite.probePassword))
-     (dict "username" "_system_rqlite" "perms" (list "all") 
+     (dict "username" "_system_rqlite" "perms" (list "all")
       "password" (.Release.rqlite.rqlitePassword))
 }}
+{{/* Construct the "sub" key for backup.json and restore.json */}}
+{{- $backupStorage := dict
+      "access_key_id" (dig "backup" "storage" "accessKeyId" nil $config)
+      "secret_access_key" (dig "backup" "storage" "secretAccessKey" nil $config)
+      "endpoint" (dig "backup" "storage" "endpoint" nil $config)
+      "region" (dig "backup" "storage" "region" nil $config)
+      "bucket" (dig "backup" "storage" "bucket" nil $config)
+      "path" (dig "backup" "storage" "path" nil $config)
+      "force_path_style" (dig "backup" "storage" "forcePathStyle" false $config)
+}}
+{{- $backup := dict }}
+{{- $restore := dict }}
+{{- if dig "backup" "autoBackup" "enabled" false $config }}
+  {{- $backup = dict
+        "version" 1
+        "type" (dig "backup" "storage" "type" "s3" $config)
+        "interval" (dig "backup" "autoBackup" "interval" nil $config)
+        "vacuum" (dig "backup" "autoBackup" "vacuum" false $config)
+        "no_compress" (dig "backup" "autoBackup" "noCompress" false $config)
+        "sub" $backupStorage
+  }}
+{{- end }}
+{{- if dig "backup" "autoRestore" "enabled" false $config }}
+  {{- $restore = dict
+        "version" 1
+        "type" (dig "backup" "storage" "type" "s3" $config)
+        "timeout" (dig "backup" "autoRestore" "timeout" "60s" $config)
+        "continue_on_failure" (dig "backup" "autoRestore" "continueOnFailure" false $config)
+        "sub" $backupStorage
+  }}
+{{- end }}
 apiVersion: v1
 kind: Secret
 metadata:
@@ -19,11 +51,21 @@ metadata:
 type: Opaque
 stringData:
   users.json: |-
-    {{- if .Values.config.users }}
-    {{- $users := concat $internalUsers .Values.config.users }}
+    {{- if $config.users }}
+    {{- $users := concat $internalUsers $config.users }}
     {{- tpl ($users | toPrettyJson) . | nindent 4}}
     {{- end }}
 
+  {{- with $backup }}
+  backup.json: |-
+    {{- tpl (. | toPrettyJson) $ | nindent 4}}
+  {{- end }}
+
+  {{- with $restore }}
+  restore.json: |-
+    {{- tpl (. | toPrettyJson) $ | nindent 4}}
+  {{- end }}
+
 {{- include "rqlite.renderTLSFiles" (dict "value" "node" "ctx" $) | nindent 2 }}
 {{- include "rqlite.renderTLSFiles" (dict "value" "client" "ctx" $) | nindent 2 }}
 ---

charts/rqlite/values.yaml

@@ -91,7 +91,7 @@ updateStrategy:
 # This value is inherited by read-only nodes but may be overridden (see "readonly" below).
 terminationGracePeriodSeconds: 15
 
-# The pod-level security context. 
+# The pod-level security context.
 #
 # By default, all rqlite containers are run as non-root. rqlite isn't opinionated about
 # the UID it runs under.
@@ -301,7 +301,7 @@ ingress:
   # to rewrite the path. For example, for ingress-nginx, see
   # https://kubernetes.github.io/ingress-nginx/examples/rewrite/
   path: /
-  # If using path rewriting with custom ingress annotations, set to ImplementationSpecific 
+  # If using path rewriting with custom ingress annotations, set to ImplementationSpecific
   pathType: Prefix
   # Custom annotations for the Ingress resource. Note that if config.client.tls.enabled is
   # true, you will need to add the appropriate annotation for your ingress to have it
@@ -393,7 +393,7 @@ config:
       # connections are then established to the pod IPs, but pod IPs are unpredictable
       # (unless you leverage special constructs like Calico IP Pools). This setting allows
       # provisioning the certificate with a well-known SAN which can be verified.
-      # 
+      #
       # If for some reason you want to disable this and force chart deployment anyway,
       # set this value to the empty string ("") instead of null.
       verifyServerName: null
@@ -461,3 +461,59 @@ config:
   #  - username: myapp
   #    password: 9uyYgs2NugvpSrEcZCmsu4mYdm1FBPZ9
   #    perms: [execute, query]
+
+  # This section configures rqlite's automatic backup/restore functionality, which supports
+  # S3-compatible cloud storage providers.
+  #
+  # See also https://rqlite.io/docs/guides/backup/
+  #
+  # Be aware that apart from the "enable" field, any other change will require a
+  # rolling restart of rqlite to pick up the changes.
+  backup:
+    # The storage map describes the S3 bucket that applies to both backups and restores
+    # (if either is enabled).
+    storage:
+      # These are the credentials used to access the bucket.
+      accessKeyId: ""
+      secretAccessKey: ""
+      # The S3 endpoint URL is optional when using native Amazon S3, but for non-Amazon
+      # providers such as Wasabi, Backblaze B2, or self-hosted solutions like MinIO, you
+      # will need to specify it.
+      #
+      # HTTPS is assumed if no URL scheme is given, so in most cases you can just specify
+      # the FQDN here, e.g. "s3.eu-central-1.wasabisys.com". However if non-SSL HTTP is
+      # required, for example with self-hosted MinIO on the same network, you can prefix
+      # the endpoint with http://
+      endpoint: null
+      # The name of the S3 bucket backups are written to (or pulled from)
+      bucket: rqlite-bucket
+      # The region of the S3 bucket.  With MinIO this is usually just "us-east-1"
+      region: us-east-1
+      # Full path and object name of the backup file
+      path: backups/rqlite.sqlite.gz
+    autoBackup:
+      # Whether to enable automatic backups
+      enabled: false
+      # Backup frequency. This is a duration string as defined at
+      # https://pkg.go.dev/maze.io/x/duration#ParseDuration
+      interval: 1h
+      # Whether to run a VACUUM on the SQLite database prior to exporting for backup.
+      # This requires having sufficient free disk space.  See also
+      # https://www.sqlite.org/lang_vacuum.html
+      vacuum: false
+      # Normally backups are compressed prior to uploading. Set to true to disable
+      # compression.
+      noCompress: false
+    autoRestore:
+      # Whether to enable automatic restoration on startup. This will only occur when the
+      # PV has no existing data, and only by the elected rqlite leader.
+      enabled: false
+      # How long to wait for backup files to be downloaded from cloud storage before
+      # giving up. Note this isn't related to network timeouts or download stalls: this
+      # applies equally to successful downloads as well. For larger data sets, you may
+      # want to increase this from the chart default.
+      timeout: 5m
+      # By default, download failures (including timeouts) will cause rqlite to exit, and
+      # the pod will be restarted. Set this to true if you want rqlite to continue startup
+      # despite a download failure.
+      continueOnFailure: false