add tidb-binlog support (#106)

* add tidb-binlog support

charts/tidb-cluster/templates/drainer-configmap.yaml

@@ -0,0 +1,104 @@
+{{- if .Values.binlog.drainer.create }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Values.clusterName }}-drainer
+  labels:
+    app.kubernetes.io/name: {{ template "tidb-cluster.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Values.clusterName }}
+    app.kubernetes.io/component: drainer
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
+data:
+  drainer-config: |-
+    # drainer Configuration.
+
+    # addr (i.e. 'host:port') to listen on for drainer connections
+    # will register this addr into etcd
+    # addr = "127.0.0.1:8249"
+
+    # the interval time (in seconds) of detect pumps' status
+    detect-interval = {{ .Values.binlog.drainer.detectInterval | default 10 }}
+
+    # drainer meta data directory path
+    data-dir = "/data"
+
+    # a comma separated list of PD endpoints
+    pd-urls = "http://{{ .Values.clusterName }}-pd:2379"
+
+    #[security]
+    # Path of file that contains list of trusted SSL CAs for connection with cluster components.
+    # ssl-ca = "/path/to/ca.pem"
+    # Path of file that contains X509 certificate in PEM format for connection with cluster components.
+    # ssl-cert = "/path/to/pump.pem"
+    # Path of file that contains X509 key in PEM format for connection with cluster components.
+    # ssl-key = "/path/to/pump-key.pem"
+
+    # syncer Configuration.
+    [syncer]
+
+    # disable sync these schema
+    ignore-schemas = "{{ .Values.binlog.drainer.ignoreSchemas | default "INFORMATION_SCHEMA,PERFORMANCE_SCHEMA,mysql,test" }}"
+
+    # number of binlog events in a transaction batch
+    txn-batch = {{ .Values.binlog.drainer.txnBatch | default 1 }}
+
+    # work count to execute binlogs
+    worker-count = {{ .Values.binlog.drainer.workerCount | default 1 }}
+
+    disable-dispatch = {{ .Values.binlog.drainer.disableDispatch | default false }}
+
+    # safe mode will split update to delete and insert
+    safe-mode = {{ .Values.binlog.drainer.safeMode | default false }}
+
+    # downstream storage, equal to --dest-db-type
+    # valid values are "mysql", "pb", "tidb", "flash", "kafka"
+    db-type = "{{ .Values.binlog.drainer.destDBType }}"
+
+    ##replicate-do-db priority over replicate-do-table if have same db name
+    ##and we support regex expression , start with '~' declare use regex expression.
+    #
+    #replicate-do-db = ["~^b.*","s1"]
+    #[[syncer.replicate-do-table]]
+    #db-name ="test"
+    #tbl-name = "log"
+
+    #[[syncer.replicate-do-table]]
+    #db-name ="test"
+    #tbl-name = "~^a.*"
+
+{{- if eq .Values.binlog.drainer.destDBType "mysql" }}
+    # the downstream mysql protocol database
+    [syncer.to]
+    host = "{{ .Values.binlog.drainer.mysql.host }}"
+    user = "{{ .Values.binlog.drainer.mysql.user }}"
+    password = "{{ .Values.binlog.drainer.mysql.password }}"
+    port = {{ .Values.binlog.drainer.mysql.port }}
+    # Time and size limits for flash batch write
+    time-limit = "{{ .Values.binlog.drainer.mysql.timeLimit | default "30s" }}"
+    size-limit = "{{ .Values.binlog.drainer.mysql.sizeLimit | default "100000" }}"
+{{- end }}
+
+{{- if eq .Values.binlog.drainer.destDBType "pb" }}
+    # Uncomment this if you want to use pb or sql as db-type.
+    # Compress compresses output file, like pb and sql file. Now it supports "gzip" algorithm only.
+    # Values can be "gzip". Leave it empty to disable compression.
+    [syncer.to]
+    dir = "/data/pb"
+    compression = "gzip"
+{{- end }}
+
+
+{{- if eq .Values.binlog.drainer.destDBType "kafka" }}
+    # when db-type is kafka, you can uncomment this to config the down stream kafka, it will be the globle config kafka default
+    [syncer.to]
+    # only need config one of zookeeper-addrs and kafka-addrs, will get kafka address if zookeeper-addrs is configed.
+    {{- if .Values.binlog.drainer.kafka.zookeeperAddrs }}
+    zookeeper-addrs = {{ .Values.binlog.drainer.kafka.zookeeperAddrs }}
+    {{- end }}
+    {{- if .Values.binlog.drainer.kafka.kafkaAddrs }}
+    kafka-addrs = {{ .Values.binlog.drainer.kafka.kafkaAddrs }}
+    {{- end }}
+    kafka-version = {{ .Values.binlog.drainer.kafka.kafkaVersion | default "0.8.2.0" }}
+{{- end }}
+{{- end }}

charts/tidb-cluster/templates/drainer-service.yaml

@@ -0,0 +1,21 @@
+{{- if .Values.binlog.drainer.create }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Values.clusterName }}-drainer
+  labels:
+    app.kubernetes.io/name: {{ template "tidb-cluster.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Values.clusterName }}
+    app.kubernetes.io/component: drainer
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
+spec:
+  clusterIP: None
+  ports:
+  - name: drainer
+    port: 8249
+  selector:
+    app.kubernetes.io/name: {{ template "tidb-cluster.name" . }}
+    app.kubernetes.io/instance: {{ .Values.clusterName }}
+    app.kubernetes.io/component: drainer
+{{- end }}

charts/tidb-cluster/templates/drainer-statefulset.yaml

@@ -0,0 +1,96 @@
+{{- if .Values.binlog.drainer.create }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ .Values.clusterName }}-drainer
+  labels:
+    app.kubernetes.io/name: {{ template "tidb-cluster.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Values.clusterName }}
+    app.kubernetes.io/component: drainer
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ template "tidb-cluster.name" . }}
+      app.kubernetes.io/instance: {{ .Values.clusterName }}
+      app.kubernetes.io/component: drainer
+  serviceName: {{ .Values.clusterName }}-drainer
+  replicas: 1
+  template:
+    metadata:
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/path: "/metrics"
+        prometheus.io/port: "8249"
+      labels:
+        app.kubernetes.io/name: {{ template "tidb-cluster.name" . }}
+        app.kubernetes.io/instance: {{ .Values.clusterName }}
+        app.kubernetes.io/component: drainer
+    spec:
+      containers:
+      - name: drainer
+        image: {{ .Values.binlog.drainer.image }}
+        imagePullPolicy: {{ .Values.binlog.drainer.imagePullPolicy | default "IfNotPresent" }}
+        command:
+        - /bin/sh
+        - -c
+        - |
+          set -euo pipefail
+
+          domain=`echo ${HOSTNAME}`.{{ .Values.clusterName }}-drainer
+
+          elapseTime=0
+          period=1
+          threshold=30
+          while true; do
+              sleep ${period}
+              elapseTime=$(( elapseTime+period ))
+
+              if [[ ${elapseTime} -ge ${threshold} ]]
+              then
+                  echo "waiting for drainer domain ready timeout" >&2
+                  exit 1
+              fi
+
+              if nslookup ${domain} 2>/dev/null
+              then
+                  echo "nslookup domain ${domain} success"
+                  break
+              else
+                  echo "nslookup domain ${domain} failed" >&2
+              fi
+          done
+
+          /drainer \
+          -L={{ .Values.binlog.drainer.logLevel | default "info" }} \
+          -addr=`echo ${HOSTNAME}`.{{ .Values.clusterName }}-drainer:8249 \
+          -config=/etc/drainer/drainer.toml \
+          -disable-detect={{ .Values.binlog.drainer.disableDetect | default false }} \
+          -initial-commit-ts={{ .Values.binlog.drainer.initialCommitTs | default 0 }} \
+          -log-file=
+        ports:
+        - containerPort: 8249
+          name: drainer
+        volumeMounts:
+        - name: data
+          mountPath: /data
+        - name: config
+          mountPath: /etc/drainer
+      volumes:
+      - name: config
+        configMap:
+          name: {{ .Values.clusterName }}-drainer
+          items:
+          - key: drainer-config
+            path: drainer.toml
+  volumeClaimTemplates:
+  - metadata:
+      name: data
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      storageClassName: {{ .Values.binlog.drainer.storageClassName }}
+      resources:
+        requests:
+          storage: {{ .Values.binlog.drainer.storage }}
+{{- end }}

charts/tidb-cluster/templates/pump-configmap.yaml

@@ -0,0 +1,42 @@
+{{- if .Values.binlog.pump.create }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Values.clusterName }}-pump
+  labels:
+    app.kubernetes.io/name: {{ template "tidb-cluster.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Values.clusterName }}
+    app.kubernetes.io/component: pump
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
+data:
+  pump-config: |-
+    # pump Configuration.
+
+    # addr(i.e. 'host:port') to listen on for client traffic
+    addr = "0.0.0.0:8250"
+
+    # addr(i.e. 'host:port') to advertise to the public
+    # advertise-addr = ""
+
+    # a integer value to control expiry date of the binlog data, indicates for how long (in days) the binlog data would be stored.
+    # must bigger than 0
+    gc = {{ .Values.binlog.pump.gc | default 7 }}
+
+    # path to the data directory of pump's data
+    data-dir = "/data"
+
+    # number of seconds between heartbeat ticks (in 2 seconds)
+    heartbeat-interval = {{ .Values.binlog.pump.heartbeatInterval | default 2 }}
+
+    # a comma separated list of PD endpoints
+    pd-urls = "http://{{ .Values.clusterName }}-pd:2379"
+
+    #[security]
+    # Path of file that contains list of trusted SSL CAs for connection with cluster components.
+    # ssl-ca = "/path/to/ca.pem"
+    # Path of file that contains X509 certificate in PEM format for connection with cluster components.
+    # ssl-cert = "/path/to/drainer.pem"
+    # Path of file that contains X509 key in PEM format for connection with cluster components.
+    # ssl-key = "/path/to/drainer-key.pem"
+{{- end }}

charts/tidb-cluster/templates/pump-service.yaml

@@ -0,0 +1,21 @@
+{{- if .Values.binlog.pump.create }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Values.clusterName }}-pump
+  labels:
+    app.kubernetes.io/name: {{ template "tidb-cluster.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Values.clusterName }}
+    app.kubernetes.io/component: pump
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
+spec:
+  clusterIP: None
+  ports:
+  - name: pump
+    port: 8250
+  selector:
+    app.kubernetes.io/name: {{ template "tidb-cluster.name" . }}
+    app.kubernetes.io/instance: {{ .Values.clusterName }}
+    app.kubernetes.io/component: pump
+{{- end }}

charts/tidb-cluster/templates/pump-statefulset.yaml

@@ -0,0 +1,70 @@
+{{- if .Values.binlog.pump.create }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ .Values.clusterName }}-pump
+  labels:
+    app.kubernetes.io/name: {{ template "tidb-cluster.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Values.clusterName }}
+    app.kubernetes.io/component: pump
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ template "tidb-cluster.name" . }}
+      app.kubernetes.io/instance: {{ .Values.clusterName }}
+      app.kubernetes.io/component: pump
+  serviceName: {{ .Values.clusterName }}-pump
+  replicas: {{ .Values.binlog.pump.replicas }}
+  template:
+    metadata:
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/path: "/metrics"
+        prometheus.io/port: "8250"
+      labels:
+        app.kubernetes.io/name: {{ template "tidb-cluster.name" . }}
+        app.kubernetes.io/instance: {{ .Values.clusterName }}
+        app.kubernetes.io/component: pump
+    spec:
+      containers:
+      - name: pump
+        image: {{ .Values.binlog.pump.image }}
+        imagePullPolicy: {{ .Values.binlog.pump.imagePullPolicy }}
+        command:
+        - /bin/sh
+        - -c
+        - |
+          set -euo pipefail
+          /pump \
+          -L={{ .Values.binlog.pump.logLevel | default "info" }} \
+          -advertise-addr=`echo ${HOSTNAME}`.{{ .Values.clusterName }}-pump:8250 \
+          -config=/etc/pump/pump.toml \
+          -enable-tolerant={{ .Values.binlog.pump.enableTolerant | default true }} \
+          -log-file=
+        ports:
+        - containerPort: 8250
+          name: pump
+        volumeMounts:
+        - name: data
+          mountPath: /data
+        - name: config
+          mountPath: /etc/pump
+      volumes:
+      - name: config
+        configMap:
+          name: {{ .Values.clusterName }}-pump
+          items:
+          - key: pump-config
+            path: pump.toml
+  volumeClaimTemplates:
+  - metadata:
+      name: data
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      storageClassName: {{ .Values.binlog.pump.storageClassName }}
+      resources:
+        requests:
+          storage: {{ .Values.binlog.pump.storage }}
+{{- end }}

charts/tidb-cluster/templates/tidb-cluster.yaml

@@ -69,3 +69,4 @@ spec:
     tolerations:
 {{ toYaml .Values.tidb.tolerations | indent 4 }}
   {{- end }}
+    binlogEnabled: {{ .Values.binlog.pump.create | default false }}

charts/tidb-cluster/templates/tidb-configmap.yaml

@@ -42,6 +42,11 @@ data:
     --config=/etc/tidb/tidb.toml
     "
 
+    if [[ X${BINLOG_ENABLED} == Xtrue ]]
+    then
+        ARGS="${ARGS} --enable-binlog=true"
+    fi
+
     echo "start tidb-server ..."
     echo "/tidb-server ${ARGS}"
     exec /tidb-server ${ARGS}