GCS native support (GCS implemented using Google Cloud Storage libraries)

README.md

@@ -104,20 +104,22 @@ Below is a summary of the configurations supported in `backup.yaml`:
 |                   | `tlsMode`                        | TLS mode (0: none, 1: one-way, 2: two-way).                                                          | `0`                                           |
 |                   | `user`                           | Username for Milvus.                                                                                  | `root`                                        |
 |                   | `password`                       | Password for Milvus.                                                                                  | `Milvus`                                      |
-| `minio`           | `storageType`                    | Storage type for Milvus (e.g., `local`, `minio`, `s3`, `aws`, `gcp`, `ali(aliyun)`, `azure`, `tc(tencent)`).                                                 | `minio`                                       |
+| `minio`           | `storageType`                    | Storage type for Milvus (e.g., `local`, `minio`, `s3`, `aws`, `gcp`, `ali(aliyun)`, `azure`, `tc(tencent)`, `gcpnative`). Use `gcpnative` for the Google Cloud Platform provider.                                                 | `minio`                                       |
 |                   | `address`                        | MinIO/S3 address.                                                                                     | `localhost`                                   |
 |                   | `port`                           | MinIO/S3 port.                                                                                        | `9000`                                        |
 |                   | `accessKeyID`                    | MinIO/S3 access key ID.                                                                               | `minioadmin`                                  |
 |                   | `secretAccessKey`                | MinIO/S3 secret access key.                                                                           | `minioadmin`                                  |
+|                   | `gcpCredentialJSON`              | Path to your GCP credential JSON key file. Used only for the "gcpnative" cloud provider.                                                                                                                                                      |             `/path/to/file`                               |
 |                   | `useSSL`                         | Whether to use SSL for MinIO/S3.                                                                      | `false`                                       |
 |                   | `bucketName`                     | Bucket name in MinIO/S3.                                                                              | `a-bucket`                                    |
 |                   | `rootPath`                       | Storage root path in MinIO/S3.                                                                        | `files`                                       |
-| `minio (backup)`  | `backupStorageType`              | Backup storage type (e.g., `local`, `minio`, `s3`, `aws`, `gcp`, `ali(aliyun)`, `azure`, `tc(tencent)`).                                                     | `minio`                                       |
+| `minio (backup)`  | `backupStorageType`              | Backup storage type for Milvus (e.g., `local`, `minio`, `s3`, `aws`, `gcp`, `ali(aliyun)`, `azure`, `tc(tencent)`, `gcpnative`). Use `gcpnative` for the Google Cloud Platform provider.                                                     | `minio`                                       |
 |                   | `backupAddress`                  | Address of backup storage.                                                                            | `localhost`                                   |
 |                   | `backupPort`                     | Port of backup storage.                                                                               | `9000`                                        |
 |                   | `backupUseSSL`                   | Whether to use SSL for backup storage.                                                                | `false`                                       |
 |                   | `backupAccessKeyID`              | Backup storage access key ID.                                                                         | `minioadmin`                                  |
 |                   | `backupSecretAccessKey`          | Backup storage secret access key.                                                                     | `minioadmin`                                  |
+|                   | `backupGcpCredentialJSON`        | Path to your GCP credential JSON key file. Used only for the "gcpnative" cloud provider.                                                                                                                                                      |             `/path/to/file`                                        |
 |                   | `backupBucketName`               | Bucket name for backups.                                                                              | `a-bucket`                                    |
 |                   | `backupRootPath`                 | Root path to store backup data.                                                                       | `backup`                                      |
 |                   | `crossStorage`                   | Enable cross-storage backups (e.g., MinIO to AWS S3).                                                 | `false`                                       |
@@ -153,6 +155,7 @@ backupAddress: s3.{your-aws-region}.amazonaws.com  # Address of AWS S3 (replace
 backupPort: 443                              # Default port for AWS S3
 backupAccessKeyID: <your-access-key-id>      # Access key ID for your AWS S3
 backupSecretAccessKey: <your-secret-key>     # Secret access key for AWS S3
+backupGcpCredentialJSON: "/path/to/file"     # Path to your GCP credential JSON key file. Used only for the "gcpnative" cloud provider.
 backupBucketName: "your-bucket-name"         # Bucket name where the backups will be stored
 backupRootPath: "backups"                    # Root path inside the bucket to store backups
 backupUseSSL: true                           # Use SSL for secure connections (Required)

cmd/backup_yaml.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"fmt"
 	"strings"
+
 	"github.com/spf13/cobra"
 	"github.com/zilliztech/milvus-backup/core/paramtable"
 	"gopkg.in/yaml.v3"
@@ -41,18 +42,20 @@ type YAMLConFig struct {
 		Password             string `yaml:"password"`
 	} `yaml:"milvus"`
 	Minio struct {
-		Address          string `yaml:"address"`
-		Port             int    `yaml:"port"`
-		AccessKeyID      string `yaml:"accessKeyID"`
-		secretAccessKey  string `yaml:"secretAccessKey"`
-		UseSSL           bool   `yaml:"useSSL"`
-		UseIAM           bool   `yaml:"useIAM"`
-		CloudProvider    string `yaml:"cloudProvider"`
-		IamEndpoint      string `yaml:"iamEndpoint"`
-		BucketName       string `yaml:"bucketName"`
-		RootPath         string `yaml:"rootPath"`
-		BackupBucketName string `yaml:"backupBucketName"`
-		BackupRootPath   string `yaml:"backupRootPath"`
+		Address                 string `yaml:"address"`
+		Port                    int    `yaml:"port"`
+		AccessKeyID             string `yaml:"accessKeyID"`
+		secretAccessKey         string `yaml:"secretAccessKey"`
+		GcpCredentialJSON       string `yaml:"gcpCredentialJSON"`
+		UseSSL                  bool   `yaml:"useSSL"`
+		UseIAM                  bool   `yaml:"useIAM"`
+		CloudProvider           string `yaml:"cloudProvider"`
+		IamEndpoint             string `yaml:"iamEndpoint"`
+		BucketName              string `yaml:"bucketName"`
+		RootPath                string `yaml:"rootPath"`
+		BackupGcpCredentialJSON string `yaml:"backupGcpCredentialJSON"`
+		BackupBucketName        string `yaml:"backupBucketName"`
+		BackupRootPath          string `yaml:"backupRootPath"`
 	} `yaml:"minio"`
 	Backup struct {
 		MaxSegmentGroupSize string `yaml:"maxSegmentGroupSize"`
@@ -82,12 +85,14 @@ func printParams(base *paramtable.BackupParams) {
 	yml.Minio.Port = base.ParseIntWithDefault("minio.port", 9000)
 	yml.Minio.AccessKeyID = base.BaseTable.LoadWithDefault("minio.accessKeyID", "")
 	yml.Minio.secretAccessKey = base.BaseTable.LoadWithDefault("minio.secretAccessKey", "")
+	yml.Minio.GcpCredentialJSON = base.BaseTable.LoadWithDefault("minio.gcpCredentialJSON", "")
 	yml.Minio.UseSSL = base.ParseBool("minio.useSSL", false)
 	yml.Minio.UseIAM = base.ParseBool("minio.useIAM", false)
 	yml.Minio.CloudProvider = base.BaseTable.LoadWithDefault("minio.cloudProvider", "aws")
 	yml.Minio.IamEndpoint = base.BaseTable.LoadWithDefault("minio.iamEndpoint", "")
 	yml.Minio.BucketName = base.BaseTable.LoadWithDefault("minio.bucketName", "")
 	yml.Minio.RootPath = base.LoadWithDefault("minio.rootPath", "")
+	yml.Minio.BackupGcpCredentialJSON = base.BaseTable.LoadWithDefault("minio.backupGcpCredentialJSON", "")
 	yml.Minio.BackupBucketName = base.LoadWithDefault("minio.backupBucketName", "")
 	yml.Minio.BackupRootPath = base.LoadWithDefault("minio.backupRootPath", "")
 

configs/backup.yaml

@@ -22,23 +22,26 @@ milvus:
 # Related configuration of minio, which is responsible for data persistence for Milvus.
 minio:
   # Milvus storage configs, make them the same with milvus config
-  storageType: "minio" # support storage type: local, minio, s3, aws, gcp, ali(aliyun), azure, tc(tencent)
+  storageType: "minio" # support storage type: local, minio, s3, aws, gcp, ali(aliyun), azure, tc(tencent), gcpnative
+  # You can use "gcpnative" for the Google Cloud Platform provider. Uses service account credentials for authentication.
   address: localhost # Address of MinIO/S3
   port: 9000   # Port of MinIO/S3
   accessKeyID: minioadmin  # accessKeyID of MinIO/S3
   secretAccessKey: minioadmin # MinIO/S3 encryption string
+  gcpCredentialJSON: "" # The JSON content contains the gcs service account credentials. Used only for the "gcpnative" cloud provider.
   useSSL: false # Access to MinIO/S3 with SSL
   useIAM: false
   iamEndpoint: ""
   bucketName: "a-bucket" # Milvus Bucket name in MinIO/S3, make it the same as your milvus instance
   rootPath: "files" # Milvus storage root path in MinIO/S3, make it the same as your milvus instance
 
   # Backup storage configs, the storage you want to put the backup data
-  backupStorageType: "minio" # support storage type: local, minio, s3, aws, gcp, ali(aliyun), azure, tc(tencent)
+  backupStorageType: "minio" # support storage type: local, minio, s3, aws, gcp, ali(aliyun), azure, tc(tencent), gcpnative
   backupAddress: localhost # Address of MinIO/S3
   backupPort: 9000   # Port of MinIO/S3
   backupAccessKeyID: minioadmin  # accessKeyID of MinIO/S3
   backupSecretAccessKey: minioadmin # MinIO/S3 encryption string
+  backupGcpCredentialJSON: "" # The JSON content contains the gcs service account credentials. Used only for the "gcpnative" cloud provider.
   backupBucketName: "a-bucket" # Bucket name to store backup data. Backup data will store to backupBucketName/backupRootPath
   backupRootPath: "backup" # Rootpath to store backup data. Backup data will store to backupBucketName/backupRootPath
 

core/backup_context.go

@@ -101,6 +101,7 @@ func createStorageClient(ctx context.Context, params paramtable.BackupParams) (s
 		BucketName:        params.MinioCfg.BucketName,
 		AccessKeyID:       params.MinioCfg.AccessKeyID,
 		SecretAccessKeyID: params.MinioCfg.SecretAccessKey,
+		GcpCredentialJSON: params.MinioCfg.GcpCredentialJSON,
 		UseSSL:            params.MinioCfg.UseSSL,
 		UseIAM:            params.MinioCfg.UseIAM,
 		IAMEndpoint:       params.MinioCfg.IAMEndpoint,
@@ -169,6 +170,7 @@ func (b *BackupContext) getMilvusStorageClient() storage.ChunkManager {
 			BucketName:        b.params.MinioCfg.BucketName,
 			AccessKeyID:       b.params.MinioCfg.AccessKeyID,
 			SecretAccessKeyID: b.params.MinioCfg.SecretAccessKey,
+			GcpCredentialJSON: b.params.MinioCfg.GcpCredentialJSON,
 			UseSSL:            b.params.MinioCfg.UseSSL,
 			UseIAM:            b.params.MinioCfg.UseIAM,
 			IAMEndpoint:       b.params.MinioCfg.IAMEndpoint,
@@ -200,6 +202,7 @@ func (b *BackupContext) getBackupStorageClient() storage.ChunkManager {
 			BucketName:        b.params.MinioCfg.BackupBucketName,
 			AccessKeyID:       b.params.MinioCfg.BackupAccessKeyID,
 			SecretAccessKeyID: b.params.MinioCfg.BackupSecretAccessKey,
+			GcpCredentialJSON: b.params.MinioCfg.BackupGcpCredentialJSON,
 			UseSSL:            b.params.MinioCfg.BackupUseSSL,
 			UseIAM:            b.params.MinioCfg.BackupUseIAM,
 			IAMEndpoint:       b.params.MinioCfg.BackupIAMEndpoint,

core/paramtable/base_table.go

@@ -37,6 +37,7 @@ const (
 	DefaultMinioPort            = "9000"
 	DefaultMinioAccessKey       = "minioadmin"
 	DefaultMinioSecretAccessKey = "minioadmin"
+	DefaultGcpCredentialJSON    = ""
 	DefaultMinioUseSSL          = "false"
 	DefaultMinioBucketName      = "a-bucket"
 	DefaultMinioRootPath        = "files"
@@ -428,6 +429,11 @@ func (gp *BaseTable) loadMinioConfig() {
 		_ = gp.Save("minio.secretAccessKey", minioSecretKey)
 	}
 
+	gcpCredentialJSON := os.Getenv("GCP_KEY_JSON")
+	if gcpCredentialJSON != "" {
+		_ = gp.Save("minio.gcpCredentialJSON", gcpCredentialJSON)
+	}
+
 	minioUseSSL := os.Getenv("MINIO_USE_SSL")
 	if minioUseSSL != "" {
 		_ = gp.Save("minio.useSSL", minioUseSSL)
@@ -483,6 +489,11 @@ func (gp *BaseTable) loadMinioConfig() {
 		_ = gp.Save("minio.backupSecretAccessKey", minioBackupSecretKey)
 	}
 
+	backupGcpCredentialJSON := os.Getenv("BACKUP_GCP_KEY_JSON")
+	if backupGcpCredentialJSON != "" {
+		_ = gp.Save("minio.backupGcpCredentialJSON", backupGcpCredentialJSON)
+	}
+
 	minioBackupUseSSL := os.Getenv("MINIO_BACKUP_USE_SSL")
 	if minioBackupUseSSL != "" {
 		_ = gp.Save("minio.backupUseSSL", minioBackupUseSSL)

core/paramtable/params.go

@@ -171,6 +171,7 @@ const (
 	S3                        = "s3"
 	CloudProviderAWS          = "aws"
 	CloudProviderGCP          = "gcp"
+	CloudProviderGCPNative    = "gcpnative"
 	CloudProviderAli          = "ali"
 	CloudProviderAliyun       = "aliyun"
 	CloudProviderAzure        = "azure"
@@ -184,6 +185,7 @@ var supportedStorageType = map[string]bool{
 	S3:                        true,
 	CloudProviderAWS:          true,
 	CloudProviderGCP:          true,
+	CloudProviderGCPNative:    true,
 	CloudProviderAli:          true,
 	CloudProviderAliyun:       true,
 	CloudProviderAzure:        true,
@@ -196,27 +198,29 @@ type MinioConfig struct {
 
 	StorageType string
 	// Deprecated
-	CloudProvider   string
-	Address         string
-	Port            string
-	AccessKeyID     string
-	SecretAccessKey string
-	UseSSL          bool
-	BucketName      string
-	RootPath        string
-	UseIAM          bool
-	IAMEndpoint     string
-
-	BackupStorageType     string
-	BackupAddress         string
-	BackupPort            string
-	BackupAccessKeyID     string
-	BackupSecretAccessKey string
-	BackupUseSSL          bool
-	BackupBucketName      string
-	BackupRootPath        string
-	BackupUseIAM          bool
-	BackupIAMEndpoint     string
+	CloudProvider     string
+	Address           string
+	Port              string
+	AccessKeyID       string
+	SecretAccessKey   string
+	GcpCredentialJSON string
+	UseSSL            bool
+	BucketName        string
+	RootPath          string
+	UseIAM            bool
+	IAMEndpoint       string
+
+	BackupStorageType       string
+	BackupAddress           string
+	BackupPort              string
+	BackupAccessKeyID       string
+	BackupSecretAccessKey   string
+	BackupGcpCredentialJSON string
+	BackupUseSSL            bool
+	BackupBucketName        string
+	BackupRootPath          string
+	BackupUseIAM            bool
+	BackupIAMEndpoint       string
 
 	CrossStorage bool
 }
@@ -229,6 +233,7 @@ func (p *MinioConfig) init(base *BaseTable) {
 	p.initPort()
 	p.initAccessKeyID()
 	p.initSecretAccessKey()
+	p.initGcpCredentialJSON()
 	p.initUseSSL()
 	p.initBucketName()
 	p.initRootPath()
@@ -241,6 +246,7 @@ func (p *MinioConfig) init(base *BaseTable) {
 	p.initBackupPort()
 	p.initBackupAccessKeyID()
 	p.initBackupSecretAccessKey()
+	p.initBackupGcpCredentialJSON()
 	p.initBackupUseSSL()
 	p.initBackupBucketName()
 	p.initBackupRootPath()
@@ -270,6 +276,11 @@ func (p *MinioConfig) initSecretAccessKey() {
 	p.SecretAccessKey = key
 }
 
+func (p *MinioConfig) initGcpCredentialJSON() {
+	gcpCredentialJSON := p.Base.LoadWithDefault("minio.gcpCredentialJSON", DefaultGcpCredentialJSON)
+	p.GcpCredentialJSON = gcpCredentialJSON
+}
+
 func (p *MinioConfig) initUseSSL() {
 	usessl := p.Base.LoadWithDefault("minio.useSSL", DefaultMinioUseSSL)
 	var err error
@@ -380,6 +391,11 @@ func (p *MinioConfig) initBackupSecretAccessKey() {
 	p.BackupSecretAccessKey = key
 }
 
+func (p *MinioConfig) initBackupGcpCredentialJSON() {
+	gcpCredentialJSON := p.Base.LoadWithDefault("minio.backupGcpCredentialJSON", DefaultGcpCredentialJSON)
+	p.BackupGcpCredentialJSON = gcpCredentialJSON
+}
+
 func (p *MinioConfig) initBackupBucketName() {
 	bucketName := p.Base.LoadWithDefault("minio.backupBucketName",
 		p.Base.LoadWithDefault("minio.bucketName", DefaultMinioBackupBucketName))

core/storage/chunk_manager.go

@@ -42,6 +42,8 @@ func NewChunkManager(ctx context.Context, params paramtable.BackupParams, config
 	case paramtable.CloudProviderAzure:
 		// todo @wayblink
 		return newAzureChunkManagerWithParams(ctx, params, config)
+	case paramtable.CloudProviderGCPNative:
+		return newGCPNativeChunkManagerWithConfig(ctx, config)
 	default:
 		return NewMinioChunkManagerWithConfig(ctx, config)
 	}