From 058acbc4d0867640e0ee467f8db9cc28d348ef49 Mon Sep 17 00:00:00 2001 From: Toan Nguyen Date: Thu, 27 Feb 2025 20:07:57 +0700 Subject: [PATCH] support hybrid mode --- compose.yaml | 2 +- connector/collection/bucket.go | 8 +- connector/collection/object.go | 24 +- connector/collection/predicate.go | 62 +- connector/connector.go | 10 + connector/functions/bucket.go | 6 +- connector/schema.generated.go | 12 + connector/storage/bucket.go | 16 +- connector/storage/common/arguments.go | 42 +- connector/storage/manager.go | 36 +- connector/storage/object.go | 18 +- tests/configuration/configuration.yaml | 2 +- .../app/metadata/CreateStorageBucket.hml | 8 + .../app/metadata/DownloadStorageObject.hml | 8 + .../metadata/DownloadStorageObjectText.hml | 8 + .../RemoveIncompleteStorageUpload.hml | 8 + .../app/metadata/RemoveStorageBucket.hml | 8 + .../app/metadata/RemoveStorageObject.hml | 8 + .../app/metadata/RemoveStorageObjects.hml | 8 + .../app/metadata/RestoreStorageObject.hml | 8 + tests/engine/app/metadata/StorageBucket.hml | 8 + .../app/metadata/StorageBucketExists.hml | 8 + tests/engine/app/metadata/StorageBuckets.hml | 12 + .../app/metadata/StorageDeletedObjects.hml | 8 + .../app/metadata/StorageIncompleteUploads.hml | 8 + tests/engine/app/metadata/StorageObject.hml | 8 + .../app/metadata/StorageObjectConnections.hml | 8 + tests/engine/app/metadata/StorageObjects.hml | 14 + .../metadata/StoragePresignedDownloadUrl.hml | 8 + .../metadata/StoragePresignedUploadUrl.hml | 8 + .../app/metadata/UpdateStorageBucket.hml | 8 + .../app/metadata/UpdateStorageObject.hml | 8 + .../app/metadata/UploadStorageObject.hml | 8 + .../app/metadata/UploadStorageObjectText.hml | 8 + tests/engine/app/metadata/storage-types.hml | 38 ++ tests/engine/app/metadata/storage.hml | 548 +++++++++++++++++- 36 files changed, 927 insertions(+), 83 deletions(-) diff --git a/compose.yaml b/compose.yaml index dbdfd59..f20a959 100644 --- a/compose.yaml +++ b/compose.yaml @@ -32,7 +32,7 @@ services: GOOGLE_STORAGE_PUBLIC_HOST: $GOOGLE_STORAGE_PUBLIC_HOST GOOGLE_PROJECT_ID: $GOOGLE_PROJECT_ID GOOGLE_STORAGE_CREDENTIALS_FILE: /service_account.json - HASURA_LOG_LEVEL: debug + # HASURA_LOG_LEVEL: debug OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4317 OTEL_METRICS_EXPORTER: prometheus diff --git a/connector/collection/bucket.go b/connector/collection/bucket.go index ca559f4..928ea2c 100644 --- a/connector/collection/bucket.go +++ b/connector/collection/bucket.go @@ -31,7 +31,7 @@ func (coe *CollectionBucketExecutor) Execute(ctx context.Context) (*schema.RowSe }, nil } - request, err := EvalBucketPredicate(nil, "", coe.Request.Query.Predicate, coe.Variables) + request, err := EvalBucketPredicate(common.StorageClientCredentialArguments{}, "", coe.Request.Query.Predicate, coe.Variables) if err != nil { return nil, schema.UnprocessableContentError(err.Error(), nil) } @@ -45,6 +45,10 @@ func (coe *CollectionBucketExecutor) Execute(ctx context.Context) (*schema.RowSe }, nil } + if err := request.EvalArguments(coe.Arguments); err != nil { + return nil, err + } + request.evalQuerySelectionFields(coe.Request.Query.Fields) predicate := request.BucketPredicate.CheckPostPredicate @@ -82,7 +86,7 @@ func (coe *CollectionBucketExecutor) Execute(ctx context.Context) (*schema.RowSe options.MaxResults = &maxResults } - response, err := coe.Storage.ListBuckets(ctx, request.ClientID, options, predicate) + response, err := coe.Storage.ListBuckets(ctx, request.GetBucketArguments().StorageClientCredentialArguments, options, predicate) if err != nil { return nil, err } diff --git a/connector/collection/object.go b/connector/collection/object.go index 1df5827..945f6c7 100644 --- a/connector/collection/object.go +++ b/connector/collection/object.go @@ -61,28 +61,8 @@ func (coe *CollectionObjectExecutor) Execute(ctx context.Context) (*schema.RowSe NumThreads: coe.Concurrency, } - if clientType, err := utils.GetNullableString(coe.Arguments, ArgumentClientType); err != nil { - return nil, schema.UnprocessableContentError(err.Error(), nil) - } else if clientType != nil { - request.ClientType = (*common.StorageProviderType)(clientType) - } - - if endpoint, err := utils.GetNullableString(coe.Arguments, ArgumentEndpoint); err != nil { - return nil, schema.UnprocessableContentError(err.Error(), nil) - } else if endpoint != nil { - request.Endpoint = *endpoint - } - - if accessKey, err := utils.GetNullableString(coe.Arguments, ArgumentAccessKeyID); err != nil { - return nil, schema.UnprocessableContentError(err.Error(), nil) - } else if accessKey != nil { - request.AccessKeyID = *accessKey - } - - if secretAccessKey, err := utils.GetNullableString(coe.Arguments, ArgumentSecretAccessKey); err != nil { - return nil, schema.UnprocessableContentError(err.Error(), nil) - } else if secretAccessKey != nil { - request.SecretAccessKey = *secretAccessKey + if err := request.EvalArguments(coe.Arguments); err != nil { + return nil, err } if hierarchy, err := utils.GetNullableBoolean(coe.Arguments, argumentHierarchy); err != nil { diff --git a/connector/collection/predicate.go b/connector/collection/predicate.go index e1ab71d..bd9f26a 100644 --- a/connector/collection/predicate.go +++ b/connector/collection/predicate.go @@ -6,12 +6,13 @@ import ( "strings" "github.com/hasura/ndc-sdk-go/schema" + "github.com/hasura/ndc-sdk-go/utils" "github.com/hasura/ndc-storage/connector/storage/common" ) // PredicateEvaluator the structured predicate result which is evaluated from the raw expression. type PredicateEvaluator struct { - common.StorageBucketArguments + common.StorageClientCredentialArguments IsValid bool Include common.StorageObjectIncludeOptions @@ -23,13 +24,11 @@ type PredicateEvaluator struct { } // EvalBucketPredicate evaluates the predicate bucket condition of the query request. -func EvalBucketPredicate(clientID *common.StorageClientID, prefix string, predicate schema.Expression, variables map[string]any) (*PredicateEvaluator, error) { +func EvalBucketPredicate(bucketArguments common.StorageClientCredentialArguments, prefix string, predicate schema.Expression, variables map[string]any) (*PredicateEvaluator, error) { result := &PredicateEvaluator{ - StorageBucketArguments: common.StorageBucketArguments{ - ClientID: clientID, - }, - Include: common.StorageObjectIncludeOptions{}, - variables: variables, + StorageClientCredentialArguments: bucketArguments, + Include: common.StorageObjectIncludeOptions{}, + variables: variables, } if prefix != "" { @@ -58,9 +57,9 @@ func EvalBucketPredicate(clientID *common.StorageClientID, prefix string, predic // EvalObjectPredicate evaluates the predicate object condition of the query request. func EvalObjectPredicate(bucketInfo common.StorageBucketArguments, preOperator *StringComparisonOperator, predicate schema.Expression, variables map[string]any) (*PredicateEvaluator, error) { result := &PredicateEvaluator{ - StorageBucketArguments: bucketInfo, - Include: common.StorageObjectIncludeOptions{}, - variables: variables, + StorageClientCredentialArguments: bucketInfo.StorageClientCredentialArguments, + Include: common.StorageObjectIncludeOptions{}, + variables: variables, } if bucketInfo.Bucket != "" { @@ -96,17 +95,48 @@ func EvalObjectPredicate(bucketInfo common.StorageBucketArguments, preOperator * // GetBucketArguments get bucket arguments information func (pe PredicateEvaluator) GetBucketArguments() common.StorageBucketArguments { result := common.StorageBucketArguments{ - ClientID: pe.ClientID, - ClientType: pe.ClientType, - Endpoint: pe.Endpoint, - AccessKeyID: pe.AccessKeyID, - SecretAccessKey: pe.SecretAccessKey, - Bucket: pe.BucketPredicate.GetPrefix(), + StorageClientCredentialArguments: common.StorageClientCredentialArguments{ + ClientID: pe.ClientID, + ClientType: pe.ClientType, + Endpoint: pe.Endpoint, + AccessKeyID: pe.AccessKeyID, + SecretAccessKey: pe.SecretAccessKey, + }, + Bucket: pe.BucketPredicate.GetPrefix(), } return result } +// EvalArguments evaluate other request arguments +func (pe *PredicateEvaluator) EvalArguments(arguments map[string]any) error { + if clientType, err := utils.GetNullableString(arguments, ArgumentClientType); err != nil { + return schema.UnprocessableContentError(err.Error(), nil) + } else if clientType != nil { + pe.ClientType = (*common.StorageProviderType)(clientType) + } + + if endpoint, err := utils.GetNullableString(arguments, ArgumentEndpoint); err != nil { + return schema.UnprocessableContentError(err.Error(), nil) + } else if endpoint != nil { + pe.Endpoint = *endpoint + } + + if accessKey, err := utils.GetNullableString(arguments, ArgumentAccessKeyID); err != nil { + return schema.UnprocessableContentError(err.Error(), nil) + } else if accessKey != nil { + pe.AccessKeyID = *accessKey + } + + if secretAccessKey, err := utils.GetNullableString(arguments, ArgumentSecretAccessKey); err != nil { + return schema.UnprocessableContentError(err.Error(), nil) + } else if secretAccessKey != nil { + pe.SecretAccessKey = *secretAccessKey + } + + return nil +} + func (pe *PredicateEvaluator) EvalSelection(selection schema.NestedField) error { if len(selection) == 0 { return nil diff --git a/connector/connector.go b/connector/connector.go index 73a122c..6f9a2da 100644 --- a/connector/connector.go +++ b/connector/connector.go @@ -206,4 +206,14 @@ func (c *Connector) evalSchema(connectorSchema *schema.SchemaResponse) { bytesScalar.Representation = schema.NewTypeRepresentationString().Encode() connectorSchema.ScalarTypes["Bytes"] = *bytesScalar } + + if !c.config.Generator.DynamicCredentials { + for name, object := range connectorSchema.ObjectTypes { + for _, key := range dynamicCredentialArguments { + delete(object.Fields, key) + } + + connectorSchema.ObjectTypes[name] = object + } + } } diff --git a/connector/functions/bucket.go b/connector/functions/bucket.go index 1fd903c..5d20911 100644 --- a/connector/functions/bucket.go +++ b/connector/functions/bucket.go @@ -25,7 +25,7 @@ func FunctionStorageBucketConnections(ctx context.Context, state *types.State, a return StorageConnection[common.StorageBucket]{}, schema.UnprocessableContentError("$first argument must be larger than 0", nil) } - request, err := collection.EvalBucketPredicate(args.ClientID, args.Prefix, args.Where, types.QueryVariablesFromContext(ctx)) + request, err := collection.EvalBucketPredicate(args.StorageClientCredentialArguments, args.Prefix, args.Where, types.QueryVariablesFromContext(ctx)) if err != nil { return StorageConnection[common.StorageBucket]{}, err } @@ -45,7 +45,9 @@ func FunctionStorageBucketConnections(ctx context.Context, state *types.State, a predicate = nil } - buckets, err := state.Storage.ListBuckets(ctx, request.ClientID, &common.ListStorageBucketsOptions{ + bucketArguments := request.GetBucketArguments() + + buckets, err := state.Storage.ListBuckets(ctx, bucketArguments.StorageClientCredentialArguments, &common.ListStorageBucketsOptions{ Prefix: request.BucketPredicate.GetPrefix(), MaxResults: args.First, StartAfter: args.After, diff --git a/connector/schema.generated.go b/connector/schema.generated.go index 481a933..54c4ab4 100644 --- a/connector/schema.generated.go +++ b/connector/schema.generated.go @@ -1642,9 +1642,18 @@ func GetConnectorSchema() *schema.SchemaResponse { Description: toPtr("creates a new bucket."), ResultType: schema.NewNamedType("SuccessResponse").Encode(), Arguments: map[string]schema.ArgumentInfo{ + "accessKeyId": { + Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), + }, "clientId": { Type: schema.NewNullableType(schema.NewNamedType("StorageClientID")).Encode(), }, + "clientType": { + Type: schema.NewNullableType(schema.NewNamedType("StorageProviderType")).Encode(), + }, + "endpoint": { + Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), + }, "name": { Type: schema.NewNamedType("String").Encode(), }, @@ -1654,6 +1663,9 @@ func GetConnectorSchema() *schema.SchemaResponse { "region": { Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), }, + "secretAccessKey": { + Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), + }, "tags": { Type: schema.NewNullableType(schema.NewArrayType(schema.NewNamedType("StorageKeyValue"))).Encode(), }, diff --git a/connector/storage/bucket.go b/connector/storage/bucket.go index 86f8452..2475840 100644 --- a/connector/storage/bucket.go +++ b/connector/storage/bucket.go @@ -9,8 +9,10 @@ import ( // MakeBucket creates a new bucket. func (m *Manager) MakeBucket(ctx context.Context, clientID *common.StorageClientID, args *common.MakeStorageBucketOptions) error { client, bucketName, err := m.GetClientAndBucket(ctx, common.StorageBucketArguments{ - ClientID: clientID, - Bucket: args.Name, + StorageClientCredentialArguments: common.StorageClientCredentialArguments{ + ClientID: clientID, + }, + Bucket: args.Name, }) if err != nil { return err @@ -36,9 +38,13 @@ func (m *Manager) UpdateBucket(ctx context.Context, args *common.UpdateBucketArg } // ListBuckets list all buckets. -func (m *Manager) ListBuckets(ctx context.Context, clientID *common.StorageClientID, options *common.ListStorageBucketsOptions, predicate func(string) bool) (*common.StorageBucketListResults, error) { - client, ok := m.GetClient(clientID) - if !ok { +func (m *Manager) ListBuckets(ctx context.Context, clientInfo common.StorageClientCredentialArguments, options *common.ListStorageBucketsOptions, predicate func(string) bool) (*common.StorageBucketListResults, error) { + client, err := m.GetOrCreateClient(ctx, clientInfo) + if err != nil { + return nil, err + } + + if client == nil { return &common.StorageBucketListResults{ Buckets: []common.StorageBucket{}, }, nil diff --git a/connector/storage/common/arguments.go b/connector/storage/common/arguments.go index 6d46de7..90b8d07 100644 --- a/connector/storage/common/arguments.go +++ b/connector/storage/common/arguments.go @@ -15,8 +15,8 @@ type StorageKeyValue struct { // ListStorageBucketArguments represent the input arguments for the ListBuckets methods. type ListStorageBucketArguments struct { - // The storage client ID. - ClientID *StorageClientID `json:"clientId,omitempty"` + StorageClientCredentialArguments + // Returns list of bucket with the prefix. Prefix string `json:"prefix,omitempty"` // The maximum number of objects requested per batch. @@ -35,15 +35,30 @@ type GetStorageBucketArguments struct { // StorageBucketArguments represent the common input arguments for bucket-related methods. type StorageBucketArguments struct { - // The storage client ID. - ClientID *StorageClientID `json:"clientId,omitempty"` - ClientType *StorageProviderType `json:"clientType,omitempty"` - Endpoint string `json:"endpoint,omitempty"` - // The bucket name. - Bucket string `json:"bucket,omitempty"` - AccessKeyID string `json:"accessKeyId,omitempty"` - SecretAccessKey string `json:"secretAccessKey,omitempty"` + Bucket string `json:"bucket,omitempty"` + + StorageClientCredentialArguments +} + +// StorageClientCredentials hold common storage client credential arguments. +type StorageClientCredentialArguments struct { + ClientID *StorageClientID `json:"clientId,omitempty"` + ClientType *StorageProviderType `json:"clientType,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + AccessKeyID string `json:"accessKeyId,omitempty"` + SecretAccessKey string `json:"secretAccessKey,omitempty"` +} + +// IsEmpty checks if all properties are empty. +func (ca StorageClientCredentialArguments) IsEmpty() bool { + return ca.ClientType == nil || !ca.ClientType.IsValid() || (ca.AccessKeyID == "" && ca.SecretAccessKey == "" && ca.Endpoint == "") +} + +// MakeStorageBucketArguments holds all arguments to tweak bucket creation. +type MakeStorageBucketArguments struct { + StorageClientCredentialArguments + MakeStorageBucketOptions } // CopyStorageObjectArguments represent input arguments of the CopyObject method. @@ -62,13 +77,6 @@ type ComposeStorageObjectArguments struct { Sources []StorageCopySrcOptions `json:"sources"` } -// MakeStorageBucketArguments holds all arguments to tweak bucket creation. -type MakeStorageBucketArguments struct { - ClientID *StorageClientID `json:"clientId,omitempty"` - - MakeStorageBucketOptions -} - // MakeStorageBucketOptions holds all options to tweak bucket creation. type MakeStorageBucketOptions struct { // Bucket name diff --git a/connector/storage/manager.go b/connector/storage/manager.go index b4b4ec2..9103d98 100644 --- a/connector/storage/manager.go +++ b/connector/storage/manager.go @@ -8,13 +8,17 @@ import ( "strconv" "time" + "github.com/hasura/ndc-sdk-go/connector" "github.com/hasura/ndc-sdk-go/schema" "github.com/hasura/ndc-sdk-go/utils" "github.com/hasura/ndc-storage/connector/storage/azblob" "github.com/hasura/ndc-storage/connector/storage/common" "github.com/hasura/ndc-storage/connector/storage/minio" + "go.opentelemetry.io/otel/attribute" ) +var tracer = connector.NewTracer("connector/storage") + // RuntimeSettings hold runtime settings for the connector. type RuntimeSettings struct { // Maximum size in MB of the object is allowed to download content in the GraphQL response @@ -63,7 +67,7 @@ func NewManager(ctx context.Context, configs []ClientConfig, runtimeSettings Run if baseConfig.DefaultPresignedExpiry != nil { presignedExpiry, err := time.ParseDuration(*baseConfig.DefaultPresignedExpiry) if err != nil { - return nil, fmt.Errorf("failied to parse defaultPresignedExpiry in client %s: %w", configID, err) + return nil, fmt.Errorf("failed to parse defaultPresignedExpiry in client %s: %w", configID, err) } c.defaultPresignedExpiry = &presignedExpiry @@ -109,14 +113,27 @@ func (m *Manager) GetClientIDs() []string { return results } +func (m *Manager) GetOrCreateClient(ctx context.Context, arguments common.StorageClientCredentialArguments) (*Client, error) { + if len(m.clients) == 0 || !arguments.IsEmpty() { + return m.createTemporaryClient(ctx, arguments) + } + + client, ok := m.GetClient(arguments.ClientID) + if !ok { + return nil, nil + } + + return client, nil +} + // GetClient gets the inner client by key and bucket name. func (m *Manager) GetClientAndBucket(ctx context.Context, arguments common.StorageBucketArguments) (*Client, string, error) { - if len(m.clients) == 0 { + if len(m.clients) == 0 || !arguments.StorageClientCredentialArguments.IsEmpty() { if arguments.Bucket == "" { return nil, "", schema.UnprocessableContentError("bucket is required", nil) } - client, err := m.createTemporaryClient(ctx, arguments) + client, err := m.createTemporaryClient(ctx, arguments.StorageClientCredentialArguments) if err != nil { return nil, "", err } @@ -155,20 +172,21 @@ func (m *Manager) GetClientAndBucket(ctx context.Context, arguments common.Stora return &m.clients[0], arguments.Bucket, nil } -func (m *Manager) createTemporaryClient(ctx context.Context, arguments common.StorageBucketArguments) (*Client, error) { +func (m *Manager) createTemporaryClient(ctx context.Context, arguments common.StorageClientCredentialArguments) (*Client, error) { + _, span := tracer.Start(ctx, "createTemporaryClient") + defer span.End() + clientType := common.S3 if arguments.ClientType != nil && *arguments.ClientType != "" { - if err := arguments.ClientType.Validate(); err != nil { - return nil, err - } - clientType = *arguments.ClientType } clientId := common.StorageClientID(fmt.Sprintf("%s-temp", clientType)) defaultPresignedExpiry := 24 * time.Hour + span.SetAttributes(attribute.String("storage.client.type", string(clientType))) + switch clientType { case common.AzureBlobStore: if arguments.Endpoint == "" { @@ -214,6 +232,8 @@ func (m *Manager) createTemporaryClient(ctx context.Context, arguments common.St StorageClient: client, defaultPresignedExpiry: &defaultPresignedExpiry, }, nil + case common.S3, common.GoogleStorage: + fallthrough default: if arguments.AccessKeyID == "" && arguments.SecretAccessKey == "" { return nil, schema.UnprocessableContentError("accessKeyId and secretAccessKey arguments are required", nil) diff --git a/connector/storage/object.go b/connector/storage/object.go index 892aacc..6bf21fa 100644 --- a/connector/storage/object.go +++ b/connector/storage/object.go @@ -119,8 +119,10 @@ func (m *Manager) PutObject(ctx context.Context, bucketInfo common.StorageBucket // To copy multiple source objects into a single destination object see the ComposeObject API. func (m *Manager) CopyObject(ctx context.Context, args *common.CopyStorageObjectArguments) (*common.StorageUploadInfo, error) { client, bucketName, err := m.GetClientAndBucket(ctx, common.StorageBucketArguments{ - ClientID: args.ClientID, - Bucket: args.Dest.Bucket, + StorageClientCredentialArguments: common.StorageClientCredentialArguments{ + ClientID: args.ClientID, + }, + Bucket: args.Dest.Bucket, }) if err != nil { return nil, err @@ -145,8 +147,10 @@ func (m *Manager) CopyObject(ctx context.Context, args *common.CopyStorageObject // ComposeObject creates an object by concatenating a list of source objects using server-side copying. func (m *Manager) ComposeObject(ctx context.Context, args *common.ComposeStorageObjectArguments) (*common.StorageUploadInfo, error) { client, bucketName, err := m.GetClientAndBucket(ctx, common.StorageBucketArguments{ - ClientID: args.ClientID, - Bucket: args.Dest.Bucket, + StorageClientCredentialArguments: common.StorageClientCredentialArguments{ + ClientID: args.ClientID, + }, + Bucket: args.Dest.Bucket, }) if err != nil { return nil, err @@ -243,8 +247,10 @@ func (m *Manager) RestoreObject(ctx context.Context, bucketInfo common.StorageBu // RemoveIncompleteUpload removes a partially uploaded object. func (m *Manager) RemoveIncompleteUpload(ctx context.Context, args *common.RemoveIncompleteUploadArguments) error { client, bucketName, err := m.GetClientAndBucket(ctx, common.StorageBucketArguments{ - ClientID: args.ClientID, - Bucket: args.Bucket, + StorageClientCredentialArguments: common.StorageClientCredentialArguments{ + ClientID: args.ClientID, + }, + Bucket: args.Bucket, }) if err != nil { return err diff --git a/tests/configuration/configuration.yaml b/tests/configuration/configuration.yaml index 5668d2c..7ef3160 100644 --- a/tests/configuration/configuration.yaml +++ b/tests/configuration/configuration.yaml @@ -101,4 +101,4 @@ concurrency: mutation: 10 generator: promptqlCompatible: true - dynamicCredentials: false + dynamicCredentials: true diff --git a/tests/engine/app/metadata/CreateStorageBucket.hml b/tests/engine/app/metadata/CreateStorageBucket.hml index 54683ff..221e570 100644 --- a/tests/engine/app/metadata/CreateStorageBucket.hml +++ b/tests/engine/app/metadata/CreateStorageBucket.hml @@ -42,6 +42,14 @@ definition: type: String - name: tags type: "[StorageKeyValue!]" + - name: accessKeyId + type: String + - name: clientType + type: StorageProviderType + - name: endpoint + type: String + - name: secretAccessKey + type: String source: dataConnectorName: storage dataConnectorCommand: diff --git a/tests/engine/app/metadata/DownloadStorageObject.hml b/tests/engine/app/metadata/DownloadStorageObject.hml index 057d184..1c44bdd 100644 --- a/tests/engine/app/metadata/DownloadStorageObject.hml +++ b/tests/engine/app/metadata/DownloadStorageObject.hml @@ -46,6 +46,14 @@ definition: type: "[StorageKeyValue!]" - name: versionId type: String + - name: accessKeyId + type: String + - name: clientType + type: StorageProviderType + - name: endpoint + type: String + - name: secretAccessKey + type: String source: dataConnectorName: storage dataConnectorCommand: diff --git a/tests/engine/app/metadata/DownloadStorageObjectText.hml b/tests/engine/app/metadata/DownloadStorageObjectText.hml index d955497..af4366f 100644 --- a/tests/engine/app/metadata/DownloadStorageObjectText.hml +++ b/tests/engine/app/metadata/DownloadStorageObjectText.hml @@ -46,6 +46,14 @@ definition: type: "[StorageKeyValue!]" - name: versionId type: String + - name: accessKeyId + type: String + - name: clientType + type: StorageProviderType + - name: endpoint + type: String + - name: secretAccessKey + type: String source: dataConnectorName: storage dataConnectorCommand: diff --git a/tests/engine/app/metadata/RemoveIncompleteStorageUpload.hml b/tests/engine/app/metadata/RemoveIncompleteStorageUpload.hml index 64946a9..4bd3987 100644 --- a/tests/engine/app/metadata/RemoveIncompleteStorageUpload.hml +++ b/tests/engine/app/metadata/RemoveIncompleteStorageUpload.hml @@ -11,6 +11,14 @@ definition: type: StorageClientId - name: object type: String! + - name: accessKeyId + type: String + - name: clientType + type: StorageProviderType + - name: endpoint + type: String + - name: secretAccessKey + type: String source: dataConnectorName: storage dataConnectorCommand: diff --git a/tests/engine/app/metadata/RemoveStorageBucket.hml b/tests/engine/app/metadata/RemoveStorageBucket.hml index b2d8c0e..d073934 100644 --- a/tests/engine/app/metadata/RemoveStorageBucket.hml +++ b/tests/engine/app/metadata/RemoveStorageBucket.hml @@ -9,6 +9,14 @@ definition: type: String - name: clientId type: StorageClientId + - name: accessKeyId + type: String + - name: clientType + type: StorageProviderType + - name: endpoint + type: String + - name: secretAccessKey + type: String source: dataConnectorName: storage dataConnectorCommand: diff --git a/tests/engine/app/metadata/RemoveStorageObject.hml b/tests/engine/app/metadata/RemoveStorageObject.hml index d315359..9094ccb 100644 --- a/tests/engine/app/metadata/RemoveStorageObject.hml +++ b/tests/engine/app/metadata/RemoveStorageObject.hml @@ -19,6 +19,14 @@ definition: type: Boolean - name: versionId type: String + - name: accessKeyId + type: String + - name: clientType + type: StorageProviderType + - name: endpoint + type: String + - name: secretAccessKey + type: String source: dataConnectorName: storage dataConnectorCommand: diff --git a/tests/engine/app/metadata/RemoveStorageObjects.hml b/tests/engine/app/metadata/RemoveStorageObjects.hml index 232020d..fd34c27 100644 --- a/tests/engine/app/metadata/RemoveStorageObjects.hml +++ b/tests/engine/app/metadata/RemoveStorageObjects.hml @@ -52,6 +52,14 @@ definition: type: Boolean - name: prefix type: String + - name: accessKeyId + type: String + - name: clientType + type: StorageProviderType + - name: endpoint + type: String + - name: secretAccessKey + type: String source: dataConnectorName: storage dataConnectorCommand: diff --git a/tests/engine/app/metadata/RestoreStorageObject.hml b/tests/engine/app/metadata/RestoreStorageObject.hml index 9f8d0a2..4f8eff1 100644 --- a/tests/engine/app/metadata/RestoreStorageObject.hml +++ b/tests/engine/app/metadata/RestoreStorageObject.hml @@ -11,6 +11,14 @@ definition: type: StorageClientId - name: object type: String! + - name: accessKeyId + type: String + - name: clientType + type: StorageProviderType + - name: endpoint + type: String + - name: secretAccessKey + type: String source: dataConnectorName: storage dataConnectorCommand: diff --git a/tests/engine/app/metadata/StorageBucket.hml b/tests/engine/app/metadata/StorageBucket.hml index dc33d3f..424d56f 100644 --- a/tests/engine/app/metadata/StorageBucket.hml +++ b/tests/engine/app/metadata/StorageBucket.hml @@ -9,6 +9,14 @@ definition: type: String - name: clientId type: StorageClientId + - name: accessKeyId + type: String + - name: clientType + type: StorageProviderType + - name: endpoint + type: String + - name: secretAccessKey + type: String source: dataConnectorName: storage dataConnectorCommand: diff --git a/tests/engine/app/metadata/StorageBucketExists.hml b/tests/engine/app/metadata/StorageBucketExists.hml index 49ef07b..321cd5d 100644 --- a/tests/engine/app/metadata/StorageBucketExists.hml +++ b/tests/engine/app/metadata/StorageBucketExists.hml @@ -36,6 +36,14 @@ definition: type: String - name: clientId type: StorageClientId + - name: accessKeyId + type: String + - name: clientType + type: StorageProviderType + - name: endpoint + type: String + - name: secretAccessKey + type: String source: dataConnectorName: storage dataConnectorCommand: diff --git a/tests/engine/app/metadata/StorageBuckets.hml b/tests/engine/app/metadata/StorageBuckets.hml index b4d8dc1..8a41a89 100644 --- a/tests/engine/app/metadata/StorageBuckets.hml +++ b/tests/engine/app/metadata/StorageBuckets.hml @@ -877,6 +877,18 @@ definition: arguments: - name: after type: String + - name: accessKeyId + type: String + description: Access key ID or Account name credential + - name: clientType + type: StorageProviderType + description: The cloud storage provider type + - name: endpoint + type: String + description: Endpoint of the cloud storage service + - name: secretAccessKey + type: String + description: Secret Access key ID or Account key credential source: dataConnectorName: storage collection: storageBuckets diff --git a/tests/engine/app/metadata/StorageDeletedObjects.hml b/tests/engine/app/metadata/StorageDeletedObjects.hml index 65c197a..989cefe 100644 --- a/tests/engine/app/metadata/StorageDeletedObjects.hml +++ b/tests/engine/app/metadata/StorageDeletedObjects.hml @@ -47,6 +47,14 @@ definition: type: Boolean - name: prefix type: String + - name: accessKeyId + type: String + - name: clientType + type: StorageProviderType + - name: endpoint + type: String + - name: secretAccessKey + type: String source: dataConnectorName: storage dataConnectorCommand: diff --git a/tests/engine/app/metadata/StorageIncompleteUploads.hml b/tests/engine/app/metadata/StorageIncompleteUploads.hml index 27f82d1..783ecb1 100644 --- a/tests/engine/app/metadata/StorageIncompleteUploads.hml +++ b/tests/engine/app/metadata/StorageIncompleteUploads.hml @@ -50,6 +50,14 @@ definition: type: StorageClientId - name: prefix type: String + - name: accessKeyId + type: String + - name: clientType + type: StorageProviderType + - name: endpoint + type: String + - name: secretAccessKey + type: String source: dataConnectorName: storage dataConnectorCommand: diff --git a/tests/engine/app/metadata/StorageObject.hml b/tests/engine/app/metadata/StorageObject.hml index d9c4774..062c0dc 100644 --- a/tests/engine/app/metadata/StorageObject.hml +++ b/tests/engine/app/metadata/StorageObject.hml @@ -19,6 +19,14 @@ definition: type: "[StorageKeyValue!]" - name: versionId type: String + - name: accessKeyId + type: String + - name: clientType + type: StorageProviderType + - name: endpoint + type: String + - name: secretAccessKey + type: String source: dataConnectorName: storage dataConnectorCommand: diff --git a/tests/engine/app/metadata/StorageObjectConnections.hml b/tests/engine/app/metadata/StorageObjectConnections.hml index 5321a12..54666ab 100644 --- a/tests/engine/app/metadata/StorageObjectConnections.hml +++ b/tests/engine/app/metadata/StorageObjectConnections.hml @@ -133,6 +133,14 @@ definition: type: String - name: where type: StorageObjectFilterBoolExp + - name: accessKeyId + type: String + - name: clientType + type: StorageProviderType + - name: endpoint + type: String + - name: secretAccessKey + type: String source: dataConnectorName: storage dataConnectorCommand: diff --git a/tests/engine/app/metadata/StorageObjects.hml b/tests/engine/app/metadata/StorageObjects.hml index 53a116b..9f412ef 100644 --- a/tests/engine/app/metadata/StorageObjects.hml +++ b/tests/engine/app/metadata/StorageObjects.hml @@ -634,6 +634,20 @@ definition: type: String - name: hierarchy type: Boolean + - name: accessKeyId + type: String + description: Access key ID or Account name credential + - name: bucket + type: String + - name: clientType + type: StorageProviderType + description: The cloud storage provider type + - name: endpoint + type: String + description: Endpoint of the cloud storage service + - name: secretAccessKey + type: String + description: Secret Access key ID or Account key credential source: dataConnectorName: storage collection: storageObjects diff --git a/tests/engine/app/metadata/StoragePresignedDownloadUrl.hml b/tests/engine/app/metadata/StoragePresignedDownloadUrl.hml index 3fe8b15..e8fb6fe 100644 --- a/tests/engine/app/metadata/StoragePresignedDownloadUrl.hml +++ b/tests/engine/app/metadata/StoragePresignedDownloadUrl.hml @@ -45,6 +45,14 @@ definition: type: String! - name: requestParams type: "[StorageKeyValue!]" + - name: accessKeyId + type: String + - name: clientType + type: StorageProviderType + - name: endpoint + type: String + - name: secretAccessKey + type: String source: dataConnectorName: storage dataConnectorCommand: diff --git a/tests/engine/app/metadata/StoragePresignedUploadUrl.hml b/tests/engine/app/metadata/StoragePresignedUploadUrl.hml index 91d3fb5..a794cb3 100644 --- a/tests/engine/app/metadata/StoragePresignedUploadUrl.hml +++ b/tests/engine/app/metadata/StoragePresignedUploadUrl.hml @@ -13,6 +13,14 @@ definition: type: DurationString - name: object type: String! + - name: accessKeyId + type: String + - name: clientType + type: StorageProviderType + - name: endpoint + type: String + - name: secretAccessKey + type: String source: dataConnectorName: storage dataConnectorCommand: diff --git a/tests/engine/app/metadata/UpdateStorageBucket.hml b/tests/engine/app/metadata/UpdateStorageBucket.hml index c24466f..04a2baa 100644 --- a/tests/engine/app/metadata/UpdateStorageBucket.hml +++ b/tests/engine/app/metadata/UpdateStorageBucket.hml @@ -52,6 +52,14 @@ definition: type: "[StorageKeyValue!]" - name: versioningEnabled type: Boolean + - name: accessKeyId + type: String + - name: clientType + type: StorageProviderType + - name: endpoint + type: String + - name: secretAccessKey + type: String source: dataConnectorName: storage dataConnectorCommand: diff --git a/tests/engine/app/metadata/UpdateStorageObject.hml b/tests/engine/app/metadata/UpdateStorageObject.hml index f585c81..4f00835 100644 --- a/tests/engine/app/metadata/UpdateStorageObject.hml +++ b/tests/engine/app/metadata/UpdateStorageObject.hml @@ -54,6 +54,14 @@ definition: type: "[StorageKeyValue!]" - name: versionId type: String + - name: accessKeyId + type: String + - name: clientType + type: StorageProviderType + - name: endpoint + type: String + - name: secretAccessKey + type: String source: dataConnectorName: storage dataConnectorCommand: diff --git a/tests/engine/app/metadata/UploadStorageObject.hml b/tests/engine/app/metadata/UploadStorageObject.hml index b1bf66e..54ba962 100644 --- a/tests/engine/app/metadata/UploadStorageObject.hml +++ b/tests/engine/app/metadata/UploadStorageObject.hml @@ -132,6 +132,14 @@ definition: type: String! - name: options type: PutStorageObjectOptions + - name: accessKeyId + type: String + - name: clientType + type: StorageProviderType + - name: endpoint + type: String + - name: secretAccessKey + type: String source: dataConnectorName: storage dataConnectorCommand: diff --git a/tests/engine/app/metadata/UploadStorageObjectText.hml b/tests/engine/app/metadata/UploadStorageObjectText.hml index 5c461ce..eab6c68 100644 --- a/tests/engine/app/metadata/UploadStorageObjectText.hml +++ b/tests/engine/app/metadata/UploadStorageObjectText.hml @@ -15,6 +15,14 @@ definition: type: String! - name: options type: PutStorageObjectOptions + - name: accessKeyId + type: String + - name: clientType + type: StorageProviderType + - name: endpoint + type: String + - name: secretAccessKey + type: String source: dataConnectorName: storage dataConnectorCommand: diff --git a/tests/engine/app/metadata/storage-types.hml b/tests/engine/app/metadata/storage-types.hml index 3e99177..5ceb100 100644 --- a/tests/engine/app/metadata/storage-types.hml +++ b/tests/engine/app/metadata/storage-types.hml @@ -783,3 +783,41 @@ definition: graphql: comparisonExpressionTypeName: ChecksumTypeComparisonExp +--- +kind: ScalarType +version: v1 +definition: + name: StorageProviderType + graphql: + typeName: StorageProviderType + +--- +kind: BooleanExpressionType +version: v1 +definition: + name: StorageProviderTypeBoolExp + operand: + scalar: + type: StorageProviderType + comparisonOperators: [] + dataConnectorOperatorMapping: + - dataConnectorName: storage + dataConnectorScalarType: StorageProviderType + operatorMapping: {} + logicalOperators: + enable: true + isNull: + enable: true + graphql: + typeName: StorageProviderTypeBoolExp + +--- +kind: DataConnectorScalarRepresentation +version: v1 +definition: + dataConnectorName: storage + dataConnectorScalarType: StorageProviderType + representation: StorageProviderType + graphql: + comparisonExpressionTypeName: StorageProviderTypeComparisonExp + diff --git a/tests/engine/app/metadata/storage.hml b/tests/engine/app/metadata/storage.hml index 26e9296..7c71a0e 100644 --- a/tests/engine/app/metadata/storage.hml +++ b/tests/engine/app/metadata/storage.hml @@ -2145,30 +2145,92 @@ definition: - name: storageBuckets description: List storage buckets arguments: + accessKeyId: + description: Access key ID or Account name credential + type: + type: nullable + underlying_type: + type: named + name: String after: type: type: nullable underlying_type: type: named name: String + clientType: + description: The cloud storage provider type + type: + type: nullable + underlying_type: + type: named + name: StorageProviderType + endpoint: + description: Endpoint of the cloud storage service + type: + type: nullable + underlying_type: + type: named + name: String + secretAccessKey: + description: Secret Access key ID or Account key credential + type: + type: nullable + underlying_type: + type: named + name: String type: StorageBucket uniqueness_constraints: {} foreign_keys: {} - name: storageObjects description: List storage objects arguments: + accessKeyId: + description: Access key ID or Account name credential + type: + type: nullable + underlying_type: + type: named + name: String after: type: type: nullable underlying_type: type: named name: String + bucket: + type: + type: nullable + underlying_type: + type: named + name: String + clientType: + description: The cloud storage provider type + type: + type: nullable + underlying_type: + type: named + name: StorageProviderType + endpoint: + description: Endpoint of the cloud storage service + type: + type: nullable + underlying_type: + type: named + name: String hierarchy: type: type: nullable underlying_type: type: named name: Boolean + secretAccessKey: + description: Secret Access key ID or Account key credential + type: + type: nullable + underlying_type: + type: named + name: String type: StorageObject uniqueness_constraints: {} foreign_keys: {} @@ -2176,6 +2238,12 @@ definition: - name: downloadStorageObject description: returns a stream of the object data. Most of the common errors occur when reading the stream. arguments: + accessKeyId: + type: + type: nullable + underlying_type: + type: named + name: String bucket: type: type: nullable @@ -2188,6 +2256,18 @@ definition: underlying_type: type: named name: StorageClientID + clientType: + type: + type: nullable + underlying_type: + type: named + name: StorageProviderType + endpoint: + type: + type: nullable + underlying_type: + type: named + name: String headers: type: type: nullable @@ -2214,6 +2294,12 @@ definition: element_type: type: named name: StorageKeyValue + secretAccessKey: + type: + type: nullable + underlying_type: + type: named + name: String versionId: type: type: nullable @@ -2226,6 +2312,12 @@ definition: - name: downloadStorageObjectText description: returns the object content in plain text. Use this function only if you know exactly the file as an text file. arguments: + accessKeyId: + type: + type: nullable + underlying_type: + type: named + name: String bucket: type: type: nullable @@ -2238,6 +2330,18 @@ definition: underlying_type: type: named name: StorageClientID + clientType: + type: + type: nullable + underlying_type: + type: named + name: StorageProviderType + endpoint: + type: + type: nullable + underlying_type: + type: named + name: String headers: type: type: nullable @@ -2264,6 +2368,12 @@ definition: element_type: type: named name: StorageKeyValue + secretAccessKey: + type: + type: nullable + underlying_type: + type: named + name: String versionId: type: type: nullable @@ -2276,6 +2386,12 @@ definition: - name: storageBucket description: gets a bucket by name. arguments: + accessKeyId: + type: + type: nullable + underlying_type: + type: named + name: String bucket: type: type: nullable @@ -2288,6 +2404,24 @@ definition: underlying_type: type: named name: StorageClientID + clientType: + type: + type: nullable + underlying_type: + type: named + name: StorageProviderType + endpoint: + type: + type: nullable + underlying_type: + type: named + name: String + secretAccessKey: + type: + type: nullable + underlying_type: + type: named + name: String result_type: type: nullable underlying_type: @@ -2332,6 +2466,12 @@ definition: - name: storageBucketExists description: checks if a bucket exists. arguments: + accessKeyId: + type: + type: nullable + underlying_type: + type: named + name: String bucket: type: type: nullable @@ -2344,12 +2484,36 @@ definition: underlying_type: type: named name: StorageClientID + clientType: + type: + type: nullable + underlying_type: + type: named + name: StorageProviderType + endpoint: + type: + type: nullable + underlying_type: + type: named + name: String + secretAccessKey: + type: + type: nullable + underlying_type: + type: named + name: String result_type: type: named name: ExistsResponse - name: storageDeletedObjects description: list deleted objects in a bucket. arguments: + accessKeyId: + type: + type: nullable + underlying_type: + type: named + name: String after: type: type: nullable @@ -2368,6 +2532,18 @@ definition: underlying_type: type: named name: StorageClientID + clientType: + type: + type: nullable + underlying_type: + type: named + name: StorageProviderType + endpoint: + type: + type: nullable + underlying_type: + type: named + name: String first: type: type: nullable @@ -2386,12 +2562,24 @@ definition: underlying_type: type: named name: String + secretAccessKey: + type: + type: nullable + underlying_type: + type: named + name: String result_type: type: named name: StorageObjectListResults - name: storageIncompleteUploads description: list partially uploaded objects in a bucket. arguments: + accessKeyId: + type: + type: nullable + underlying_type: + type: named + name: String bucket: type: type: nullable @@ -2404,12 +2592,30 @@ definition: underlying_type: type: named name: StorageClientID + clientType: + type: + type: nullable + underlying_type: + type: named + name: StorageProviderType + endpoint: + type: + type: nullable + underlying_type: + type: named + name: String prefix: type: type: nullable underlying_type: type: named name: String + secretAccessKey: + type: + type: nullable + underlying_type: + type: named + name: String result_type: type: array element_type: @@ -2418,6 +2624,12 @@ definition: - name: storageObject description: fetches metadata of an object. arguments: + accessKeyId: + type: + type: nullable + underlying_type: + type: named + name: String bucket: type: type: nullable @@ -2430,6 +2642,18 @@ definition: underlying_type: type: named name: StorageClientID + clientType: + type: + type: nullable + underlying_type: + type: named + name: StorageProviderType + endpoint: + type: + type: nullable + underlying_type: + type: named + name: String headers: type: type: nullable @@ -2456,6 +2680,12 @@ definition: element_type: type: named name: StorageKeyValue + secretAccessKey: + type: + type: nullable + underlying_type: + type: named + name: String versionId: type: type: nullable @@ -2470,6 +2700,12 @@ definition: - name: storageObjectConnections description: lists objects in a bucket using the relay style. arguments: + accessKeyId: + type: + type: nullable + underlying_type: + type: named + name: String after: type: type: nullable @@ -2488,6 +2724,18 @@ definition: underlying_type: type: named name: StorageClientID + clientType: + type: + type: nullable + underlying_type: + type: named + name: StorageProviderType + endpoint: + type: + type: nullable + underlying_type: + type: named + name: String first: type: type: nullable @@ -2506,6 +2754,12 @@ definition: underlying_type: type: named name: String + secretAccessKey: + type: + type: nullable + underlying_type: + type: named + name: String where: type: type: nullable @@ -2518,6 +2772,12 @@ definition: - name: storagePresignedDownloadUrl description: generates a presigned URL for HTTP GET operations. Browsers/Mobile clients may point to this URL to directly download objects even if the bucket is private. This presigned URL can have an associated expiration time in seconds after which it is no longer operational. The maximum expiry is 604800 seconds (i.e. 7 days) and minimum is 1 second. arguments: + accessKeyId: + type: + type: nullable + underlying_type: + type: named + name: String bucket: type: type: nullable @@ -2530,6 +2790,18 @@ definition: underlying_type: type: named name: StorageClientID + clientType: + type: + type: nullable + underlying_type: + type: named + name: StorageProviderType + endpoint: + type: + type: nullable + underlying_type: + type: named + name: String expiry: type: type: nullable @@ -2548,6 +2820,12 @@ definition: element_type: type: named name: StorageKeyValue + secretAccessKey: + type: + type: nullable + underlying_type: + type: named + name: String result_type: type: nullable underlying_type: @@ -2556,18 +2834,36 @@ definition: - name: storagePresignedUploadUrl description: generates a presigned URL for HTTP PUT operations. Browsers/Mobile clients may point to this URL to upload objects directly to a bucket even if it is private. This presigned URL can have an associated expiration time in seconds after which it is no longer operational. The default expiry is set to 7 days. arguments: + accessKeyId: + type: + type: nullable + underlying_type: + type: named + name: String bucket: type: type: nullable underlying_type: type: named - name: String - clientId: + name: String + clientId: + type: + type: nullable + underlying_type: + type: named + name: StorageClientID + clientType: + type: + type: nullable + underlying_type: + type: named + name: StorageProviderType + endpoint: type: type: nullable underlying_type: type: named - name: StorageClientID + name: String expiry: type: type: nullable @@ -2578,6 +2874,12 @@ definition: type: type: named name: String + secretAccessKey: + type: + type: nullable + underlying_type: + type: named + name: String result_type: type: nullable underlying_type: @@ -2629,12 +2931,30 @@ definition: - name: createStorageBucket description: creates a new bucket. arguments: + accessKeyId: + type: + type: nullable + underlying_type: + type: named + name: String clientId: type: type: nullable underlying_type: type: named name: StorageClientID + clientType: + type: + type: nullable + underlying_type: + type: named + name: StorageProviderType + endpoint: + type: + type: nullable + underlying_type: + type: named + name: String name: type: type: named @@ -2651,6 +2971,12 @@ definition: underlying_type: type: named name: String + secretAccessKey: + type: + type: nullable + underlying_type: + type: named + name: String tags: type: type: nullable @@ -2665,6 +2991,12 @@ definition: - name: removeIncompleteStorageUpload description: removes a partially uploaded object. arguments: + accessKeyId: + type: + type: nullable + underlying_type: + type: named + name: String bucket: type: type: nullable @@ -2677,16 +3009,40 @@ definition: underlying_type: type: named name: StorageClientID + clientType: + type: + type: nullable + underlying_type: + type: named + name: StorageProviderType + endpoint: + type: + type: nullable + underlying_type: + type: named + name: String object: type: type: named name: String + secretAccessKey: + type: + type: nullable + underlying_type: + type: named + name: String result_type: type: named name: SuccessResponse - name: removeStorageBucket description: removes a bucket, bucket should be empty to be successfully removed. arguments: + accessKeyId: + type: + type: nullable + underlying_type: + type: named + name: String bucket: type: type: nullable @@ -2699,12 +3055,36 @@ definition: underlying_type: type: named name: StorageClientID + clientType: + type: + type: nullable + underlying_type: + type: named + name: StorageProviderType + endpoint: + type: + type: nullable + underlying_type: + type: named + name: String + secretAccessKey: + type: + type: nullable + underlying_type: + type: named + name: String result_type: type: named name: SuccessResponse - name: removeStorageObject description: removes an object with some specified options. arguments: + accessKeyId: + type: + type: nullable + underlying_type: + type: named + name: String bucket: type: type: nullable @@ -2717,6 +3097,18 @@ definition: underlying_type: type: named name: StorageClientID + clientType: + type: + type: nullable + underlying_type: + type: named + name: StorageProviderType + endpoint: + type: + type: nullable + underlying_type: + type: named + name: String forceDelete: type: type: nullable @@ -2733,6 +3125,12 @@ definition: type: type: named name: String + secretAccessKey: + type: + type: nullable + underlying_type: + type: named + name: String softDelete: type: type: nullable @@ -2751,6 +3149,12 @@ definition: - name: removeStorageObjects description: remove a list of objects obtained from an input channel. The call sends a delete request to the server up to 1000 objects at a time. The errors observed are sent over the error channel. arguments: + accessKeyId: + type: + type: nullable + underlying_type: + type: named + name: String after: type: type: nullable @@ -2769,6 +3173,18 @@ definition: underlying_type: type: named name: StorageClientID + clientType: + type: + type: nullable + underlying_type: + type: named + name: StorageProviderType + endpoint: + type: + type: nullable + underlying_type: + type: named + name: String first: type: type: nullable @@ -2793,6 +3209,12 @@ definition: underlying_type: type: named name: String + secretAccessKey: + type: + type: nullable + underlying_type: + type: named + name: String result_type: type: array element_type: @@ -2801,6 +3223,12 @@ definition: - name: restoreStorageObject description: restore a soft-deleted object. arguments: + accessKeyId: + type: + type: nullable + underlying_type: + type: named + name: String bucket: type: type: nullable @@ -2813,16 +3241,40 @@ definition: underlying_type: type: named name: StorageClientID + clientType: + type: + type: nullable + underlying_type: + type: named + name: StorageProviderType + endpoint: + type: + type: nullable + underlying_type: + type: named + name: String object: type: type: named name: String + secretAccessKey: + type: + type: nullable + underlying_type: + type: named + name: String result_type: type: named name: SuccessResponse - name: updateStorageBucket description: updates the bucket's configuration. arguments: + accessKeyId: + type: + type: nullable + underlying_type: + type: named + name: String bucket: type: type: nullable @@ -2835,12 +3287,24 @@ definition: underlying_type: type: named name: StorageClientID + clientType: + type: + type: nullable + underlying_type: + type: named + name: StorageProviderType encryption: type: type: nullable underlying_type: type: named name: ServerSideEncryptionConfiguration + endpoint: + type: + type: nullable + underlying_type: + type: named + name: String lifecycle: type: type: nullable @@ -2853,6 +3317,12 @@ definition: underlying_type: type: named name: SetStorageObjectLockConfig + secretAccessKey: + type: + type: nullable + underlying_type: + type: named + name: String tags: type: type: nullable @@ -2873,6 +3343,12 @@ definition: - name: updateStorageObject description: updates the object's configuration. arguments: + accessKeyId: + type: + type: nullable + underlying_type: + type: named + name: String bucket: type: type: nullable @@ -2885,6 +3361,18 @@ definition: underlying_type: type: named name: StorageClientID + clientType: + type: + type: nullable + underlying_type: + type: named + name: StorageProviderType + endpoint: + type: + type: nullable + underlying_type: + type: named + name: String legalHold: type: type: nullable @@ -2909,6 +3397,12 @@ definition: underlying_type: type: named name: SetStorageObjectRetentionOptions + secretAccessKey: + type: + type: nullable + underlying_type: + type: named + name: String tags: type: type: nullable @@ -2929,6 +3423,12 @@ definition: - name: uploadStorageObject description: uploads object that are less than 128MiB in a single PUT operation. For objects that are greater than 128MiB in size, PutObject seamlessly uploads the object as parts of 128MiB or more depending on the actual file size. The max upload size for an object is 5TB. arguments: + accessKeyId: + type: + type: nullable + underlying_type: + type: named + name: String bucket: type: type: nullable @@ -2941,10 +3441,22 @@ definition: underlying_type: type: named name: StorageClientID + clientType: + type: + type: nullable + underlying_type: + type: named + name: StorageProviderType data: type: type: named name: Bytes + endpoint: + type: + type: nullable + underlying_type: + type: named + name: String object: type: type: named @@ -2955,12 +3467,24 @@ definition: underlying_type: type: named name: PutStorageObjectOptions + secretAccessKey: + type: + type: nullable + underlying_type: + type: named + name: String result_type: type: named name: StorageUploadInfo - name: uploadStorageObjectText description: uploads object in plain text to the storage server. The file content is not encoded to base64 so the input size is smaller than 30%. arguments: + accessKeyId: + type: + type: nullable + underlying_type: + type: named + name: String bucket: type: type: nullable @@ -2973,10 +3497,22 @@ definition: underlying_type: type: named name: StorageClientID + clientType: + type: + type: nullable + underlying_type: + type: named + name: StorageProviderType data: type: type: named name: String + endpoint: + type: + type: nullable + underlying_type: + type: named + name: String object: type: type: named @@ -2987,6 +3523,12 @@ definition: underlying_type: type: named name: PutStorageObjectOptions + secretAccessKey: + type: + type: nullable + underlying_type: + type: named + name: String result_type: type: named name: StorageUploadInfo