initial

helm/transfer/Chart.yaml → helm/Chart.yaml

@@ -1,24 +1,15 @@
 apiVersion: v2
 name: transfer
-description: A Helm chart for Transfer
-
-# A chart can be either an 'application' or a 'library' chart.
-#
-# Application charts are a collection of templates that can be packaged into versioned archives
-# to be deployed.
-#
-# Library charts provide useful utilities or functions for the chart developer. They're included as
-# a dependency of application charts to inject those utilities and functions into the rendering
-# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+description: A Helm chart for running multiple TRCLI instances in a Kubernetes cluster
 type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.0.7
+version: 0.0.5
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "v0.0.0-rc9"
+appVersion: "0.0.0-rc20"

helm/Secrets/clickhouse.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: transfer-clickhouse
+  namespace: altinity-cloud-managed-clickhouse
+type: Opaque
+stringData:
+  CH_PASSWORD: "XXX"

helm/Secrets/mongodb.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: transfer-mongodb
+  namespace: altinity-cloud-managed-clickhouse
+type: Opaque
+stringData:
+  mongodb_PASSWORD: "XX"
+  mongodb_TLSFILE: |
+      -----BEGIN CERTIFICATE-----
+      
+      -----END CERTIFICATE-----

helm/Secrets/postgresdb.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: postgresdb-database
+  namespace: altinity-cloud-managed-clickhouse
+type: Opaque
+stringData:
+  postgresdb_PASSWORD: "XXX"

helm/configs/defaults.yaml

@@ -0,0 +1,4 @@
+type_system_version: 9
+id: {{ .name }}   # Unique ID of a transfer, used for coordinator dir name on s3 and for monitoring label
+transfername: {{ .name }}  # Human friendly name for a transfer
+type: {{ default "INCREMENT_ONLY" .type }}

helm/configs/test1.yaml

@@ -0,0 +1,25 @@
+src:
+  type: mongo
+  params:
+    Collections:
+    - DatabaseName: DB
+      CollectionName: COL
+dst:
+  type: ch
+  params:
+    Database: db2
+    IsUpdateable: true
+    InsertParams:
+      MaterializedViewsIgnoreErrors: true
+    altnameslist:
+      - from: COL
+        to: collection
+transformation:
+  debugmode: false
+  transformers:
+  - sql:
+      query: "SELECT id, JSONExtractString(document, 'channelId') as channelId
+       from table"
+data_objects:
+  include_objects:
+  - '"DB"."COL"'

helm/configs/test2.yaml

@@ -0,0 +1,27 @@
+src:
+  type: mongo
+  params:
+    Collections:
+    - DatabaseName: DB
+      CollectionName: COL2
+dst:
+  type: ch
+  params:
+    Database: db2
+    IsUpdateable: true
+    InsertParams:
+      MaterializedViewsIgnoreErrors: true
+transformation:
+  debugmode: false
+  transformers:
+  - renameTables:
+      renameTables:
+      - newName:
+          name: channel
+          nameSpace: ""
+        originalName:
+          name: COL2
+          nameSpace: DB
+data_objects:
+  include_objects:
+  - DB.COL2

helm/configs/test3.yaml

@@ -0,0 +1,25 @@
+src:
+  type: pg
+  params:
+    slotid: dtt9qqh008j2ega4rh90_t
+dst:
+  type: ch
+  params:
+    database: db_ch
+transformation:
+  debugmode: false
+  transformers:
+  - renameTables:
+      renameTables:
+      - newName:
+          name: info_in_ch
+          nameSpace: ""
+        originalName:
+          name: info
+          nameSpace: db_pg
+    transformerId: ""
+  errorsoutput: null
+data_objects:
+  include_objects:
+  - db_pg.info
+

helm/db-hosts/clickhouse.yaml

@@ -0,0 +1,34 @@
+MdbClusterId: ""
+ChClusterName: "{{ $.Values.global.cluster }}"
+User: transfer
+Password: "${CH_PASSWORD}"    # ref to Secrets
+Database: ""
+Partition: ""
+SslEnabled: false
+HttpPort: 8123
+NativePort: 9000
+Ttl: ""
+InferSchema: false
+MigrationOptions:
+  AddNewColumns: true
+ProtoColUnspecified: true
+AnyAsString: false
+SystemColumnsFirst: false
+IsUpdateable: false
+UpsertAbsentToastedRows: false
+InsertParams:
+  MaterializedViewsIgnoreErrors: false
+RetryCount: 20
+UseSchemaInTableName: false
+ShardCol: ""
+ShardByTransferId: false
+ShardByRoundRobin: false
+Rotation: null
+Cleanup: Disabled
+BufferTriggeringSize: 268435456
+Interval: 1000000000
+ShardsList:
+- Name: s2
+  Hosts:
+  - chi-{{ $.Values.global.cluster }}-{{ $.Values.global.cluster }}-0-0.altinity-cloud-managed-clickhouse.svc
+  - chi-{{ $.Values.global.cluster }}-{{ $.Values.global.cluster }}-0-1.altinity-cloud-managed-clickhouse.svc

helm/db-hosts/mongodb.yaml

@@ -0,0 +1,28 @@
+Hosts:
+- mongodb-shard-00-00.XXX.mongodb.net
+- mongodb-shard-00-01.XXX.mongodb.net
+Port: 27017
+ReplicaSet: ""
+AuthSource: admin
+User: clickhouse_user
+Password: "${mongodb_PASSWORD}"
+ExcludedCollections: []
+SubNetworkId: ""
+SecurityGroupIds: []
+TechnicalDatabase: ""
+IsHomo: false
+SlotId: ""
+SecondaryPreferredMode: false
+TlsFile: |
+  ${mongodb_TLSFILE}
+ReplicationSource: ""
+BatchingParams:
+  BatchSizeLimit: 5000
+  KeySizeThreshold: 4194304
+  BatchFlushInterval: 5000000000
+DesiredPartSize: 1073741824
+PreventJsonRepack: false
+FilterOpLogWithRegexp: false
+Direct: false
+RootCAFiles: []
+SrvMode: false

helm/db-hosts/postgresdb.yaml

@@ -0,0 +1,79 @@
+Hosts:
+- postgresdb.us-east-1.rds.amazonaws.com
+User: postgres
+Password: "${postgresdb_PASSWORD}"
+Port: 5432
+Database: db1
+DbTables: []
+BufferTriggingSize: 5000
+BufferTriggingInterval: 5000000000
+SlotByteLagLimit: 53687091200
+TlsFile: ""
+EnableTls: false
+KeeperSchema: public
+SubNetworkId: ""
+SecurityGroupIds: []
+CollapseInheritTables: false
+UsePolling: false
+ExcludedTables: []
+IsHomo: false
+NoHomo: false
+AutoActivate: false
+PreSteps:
+  Table: true
+  PrimaryKey: true
+  View: true
+  Sequence: true
+  SequenceOwnedBy: true
+  Rule: true
+  Type: true
+  Constraint: false
+  FkConstraint: false
+  Index: false
+  Function: true
+  Collation: true
+  Trigger: false
+  Policy: true
+  Cast: false
+  Default: true
+  MaterializedView: true
+  SequenceSet: true
+  TableAttach: true
+  IndexAttach: false
+PostSteps:
+  Table: false
+  PrimaryKey: false
+  View: false
+  Sequence: false
+  SequenceOwnedBy: false
+  Rule: false
+  Type: false
+  Constraint: true
+  FkConstraint: true
+  Index: true
+  Function: false
+  Collation: false
+  Trigger: true
+  Policy: false
+  Cast: false
+  Default: false
+  MaterializedView: false
+  SequenceSet: false
+  TableAttach: false
+  IndexAttach: true
+UseFakePrimaryKey: false
+IgnoreUserTypes: false
+IgnoreUnknownTables: false
+MaxBufferSize: 0
+ExcludeDescendants: false
+DesiredTableSize: 1073741824
+SnapshotDegreeOfParallelism: 4
+EmitTimeTypes: false
+DbLogEnabled: false
+ChunkSize: 0
+SnapshotSerializationFormat: ""
+ShardingKeyFields: {}
+PgDumpCommand: []
+ConnectionId: ""
+ClusterId: ""
+Host: ""

helm/templates/configmap.yaml

@@ -0,0 +1,38 @@
+{{- range .Values.transfers }}
+{{- if ne .mode "disabled" }}
+
+  {{- $cfg := $.Files.Get (printf "configs/%s.yaml" .name) | fromYaml }}
+  {{- $cfg = merge (deepCopy ( fromYaml (tpl ($.Files.Get "configs/defaults.yaml") .) )) $cfg }}
+
+  {{- $srcDefaults := $.Files.Get (printf "db-hosts/%s.yaml" .src) | fromYaml }}
+  {{- $cfg = merge $cfg
+      (dict "src"
+        (merge
+          (default dict $cfg.src)
+          (dict "params" (merge $srcDefaults (default dict $cfg.src.params)))
+        )
+      )
+  }}
+
+  {{- $dstDefaults := $.Files.Get (printf "db-hosts/%s.yaml" .dst) | fromYaml }}
+  {{- $cfg = merge $cfg
+      (dict "dst"
+        (merge
+          (default dict $cfg.dst)
+          (dict "params" (merge $dstDefaults (default dict $cfg.dst.params)))
+        )
+      )
+  }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ $.Release.Name }}-{{ $.Values.global.cluster }}-{{ .name }}
+  namespace: {{ $.Values.global.namespace }}
+  labels:
+    app.kubernetes.io/component: {{ $.Release.Name }}-{{ $.Values.global.cluster }}-{{ .name }}
+data:
+  transfer.yaml: |
+{{ tpl ($cfg | toYaml) $ | nindent 4 }}
+{{- end }}
+{{- end }}