Merge branch 'feature/ZENKO-1065-crr-parallelism' into q/0.9

scality · Aug 27, 2018 · 33316ec · 33316ec
2 parents cdb3338 + 8263099
commit 33316ec
Show file tree

Hide file tree

Showing 10 changed files with 38 additions and 4 deletions.
diff --git a/eve/ci-values.yml b/eve/ci-values.yml
@@ -26,9 +26,10 @@ zenko-queue:
     enabled: false
   configurationOverrides:
     "offsets.topic.replication.factor": 1 # - replication factor for the offsets topic
-    "num.partitions": 1                   # - default number of log partitions per topic
     "auto.create.topics.enable": true     # - enable auto creation of topic on the server
     "min.insync.replicas": 1              # - min number of replicas that must acknowledge a write
+  partitionCount: 1
+  partitionFactor: 2
 
 redis-ha:
   rbac:
@@ -53,6 +54,8 @@ cloudserver:
     PUSH_STATS: 'false'
 
 backbeat:
+  replication:
+    replicaFactor: 2
   lifecycle:
     conductor:
       cronRule: "0 */1 * * * *"

diff --git a/kubernetes/zenko/charts/backbeat/templates/_helpers.tpl b/kubernetes/zenko/charts/backbeat/templates/_helpers.tpl
@@ -48,3 +48,11 @@ Create the default redis sentinels hosts string
 {{- $release := .Release.Name -}}
 {{- range $v := until $count }}{{ $release }}-redis-ha-server-{{ $v }}.{{ $release }}-redis-ha:26379{{ if ne $v (sub $count 1) }},{{- end -}}{{- end -}}
 {{- end -}}
+
+{{/*
+Create the default replicaCount for backbeat replication data processors
+*/}}
+{{- define "backbeat.replication.dataProcessor.replicaCount" -}}
+{{- $count := mul .Values.replication.dataProcessor.replicaFactor .Values.replication.dataProcessor.replicaCount -}}
+{{- printf "%d" $count }}
+{{- end -}}
diff --git a/kubernetes/zenko/charts/backbeat/templates/replication/data_processor_deployment.yaml b/kubernetes/zenko/charts/backbeat/templates/replication/data_processor_deployment.yaml
@@ -8,7 +8,7 @@ metadata:
     release: {{ .Release.Name }}
     heritage: {{ .Release.Service }}
 spec:
-  replicas: {{ .Values.replication.dataProcessor.replicaCount }}
+  replicas: {{ template "backbeat.replication.dataProcessor.replicaCount" . }}
   template:
     metadata:
       {{- if not .Values.global.orbit.enabled }}

diff --git a/kubernetes/zenko/charts/backbeat/values.yaml b/kubernetes/zenko/charts/backbeat/values.yaml
@@ -116,6 +116,7 @@ lifecycle:
 replication:
   dataProcessor:
     replicaCount: 1
+    replicaFactor: 1
 
     resources: {}
     nodeSelector: {}

diff --git a/kubernetes/zenko/charts/zenko-queue/Chart.yaml b/kubernetes/zenko/charts/zenko-queue/Chart.yaml
@@ -14,4 +14,4 @@ sources:
 - https://github.com/Yolean/kubernetes-kafka
 - https://github.com/confluentinc/cp-docker-images
 - https://github.com/apache/kafka
-version: 0.7.1-Z
+version: 0.7.2-Z
diff --git a/kubernetes/zenko/charts/zenko-queue/README.md b/kubernetes/zenko/charts/zenko-queue/README.md
@@ -72,6 +72,8 @@ following configurable parameters:
 | `external.domain`              | Domain in which to advertise Kafka external listeners.                                                          | `cluster.local`                                            |
 | `external.init`                | External init container settings.                                                                               | (see `values.yaml`)                                        |
 | `rbac.enabled`                 | Enable a service account and role for the init container to use in an RBAC enabled cluster                      | `false`                                                    |
+| `partitionCount`                 | Unmodified number of partitions                      | `1`                                                    |
+| `partitionFactor`                 | Partition count modifier                      | `1`                                                    |
 | `configurationOverrides`       | `Kafka ` [configuration setting][brokerconfigs] overrides in the dictionary format                              | `{ offsets.topic.replication.factor: 3 }`                  |
 | `additionalPorts`              | Additional ports to expose on brokers.  Useful when the image exposes metrics (like prometheus, etc.) through a javaagent instead of a sidecar   | `{}`                                 |
 | `readinessProbe.initialDelaySeconds` | Number of seconds before probe is initiated.                                                              | `30`                                                       |

diff --git a/kubernetes/zenko/charts/zenko-queue/templates/_helpers.tpl b/kubernetes/zenko/charts/zenko-queue/templates/_helpers.tpl
@@ -24,6 +24,14 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 
+{{/*
+Create the default number of Kafka partitions.
+*/}}
+{{- define "kafka.num.partitions" -}}
+{{- $num := mul .Values.partitionFactor .Values.partitionCount -}}
+{{- printf "%d" $num -}}
+{{- end -}}
+
 {{/*
 Form the Zookeeper URL. If zookeeper is installed as part of this chart, use k8s service discovery,
 else use user-provided URL

diff --git a/kubernetes/zenko/charts/zenko-queue/templates/statefulset.yaml b/kubernetes/zenko/charts/zenko-queue/templates/statefulset.yaml
@@ -143,6 +143,8 @@ spec:
         - name: KAFKA_LOG_DIRS
           value: {{ printf "%s/%s" .Values.persistence.mountPath .Values.logSubPath | quote }}
         {{- end }}
+        - name: KAFKA_NUM_PARTITIONS
+          value: {{ include "kafka.num.partitions" . | quote }}
         {{- range $key, $value := .Values.configurationOverrides }}
         - name: {{ printf "KAFKA_%s" $key | replace "." "_" | upper | quote }}
           value: {{ $value | quote }}

diff --git a/kubernetes/zenko/charts/zenko-queue/values.yaml b/kubernetes/zenko/charts/zenko-queue/values.yaml
@@ -122,6 +122,11 @@ external:
     imageTag: "0.4"
     imagePullPolicy: "IfNotPresent"
 
+## The number of partitions is equal to partitionCount times partitionFactor.
+##
+partitionCount: 1
+partitionFactor: 1
+
 ## Configuration Overrides. Specify any Kafka settings you would like set on the StatefulSet
 ## here in map format, as defined in the official docs.
 ## ref: https://kafka.apache.org/documentation/#brokerconfigs

diff --git a/kubernetes/zenko/values.yaml b/kubernetes/zenko/values.yaml
@@ -7,6 +7,9 @@
 # is equivalent to the number of nodes in a Kubernetes Cluster.
 nodeCount: &nodeCount 3
 
+# Factor of Backbeat Replication Data Processors and Kafka partitions.
+crrParallelism: &crrParallelism 2
+
 ingress:
   enabled: false
   # Used to create an Ingress record.
@@ -48,6 +51,7 @@ backbeat:
   replication:
     dataProcessor:
       replicaCount: *nodeCount
+      replicaFactor: *crrParallelism
     statusProcessor:
       replicaCount: *nodeCount
   lifecycle:
@@ -113,9 +117,10 @@ zenko-queue:
     enabled: true
   configurationOverrides:
     "offsets.topic.replication.factor": 3 # - replication factor for the offsets topic
-    "num.partitions": *nodeCount          # - default number of log partitions per topic
     "auto.create.topics.enable": true     # - enable auto creation of topic on the server
     "min.insync.replicas": 2              # - min number of replicas that must acknowledge a write
+  partitionCount: *nodeCount
+  partitionFactor: *crrParallelism
   prometheus:
     jmx:
       enabled: true