update kafka version used in testing to 3.6.0

libbeat/docker-compose.yml

@@ -1,4 +1,3 @@
-version: '2.3'
 services:
   # This is a proxy used to block "docker-compose up" until all services are healthy.
   # See: https://github.com/docker/compose/issues/4369

metricbeat/docker-compose.yml

@@ -1,5 +1,3 @@
-version: '2.3'
-
 services:
   beat:
     build: ${PWD}/.

metricbeat/docs/modules/kafka.asciidoc

@@ -35,7 +35,7 @@ kafka-acls --authorizer-properties zookeeper.connect=localhost:2181 --add --allo
 [float]
 === Compatibility
 
-This module is tested with Kafka 0.10.2.1, 1.1.0, 2.1.1, and 2.2.2.
+This module is tested with Kafka 0.10.2.1, 1.1.0, 2.1.1, 2.2.2 and 3.6.0.
 
 The Broker, Producer, Consumer metricsets require <<metricbeat-module-jolokia,Jolokia>> to fetch JMX metrics. Refer to the link for Jolokia's compatibility notes.
 

metricbeat/module/kafka/README.md

@@ -20,10 +20,10 @@ To bring this container up simply run the tests for Kafka module:
 After the tests have been completed, the Kafka container should be still running. Verify with:
 
 ```console
-707b50334835    docker.elastic.co/integrations-ci/beats-kafka:2.2.2-2  "/run.sh"        2 minutes ago    Up 2 minutes (healthy)  2181/tcp, 0.0.0.0:32785->8774/tcp, 0.0.0.0:32784->8775/tcp, 0.0.0.0:32783->8779/tcp, 0.0.0.0:32782->9092/tcp  kafka_a035cf4c6889705a_kafka_1
+707b50334835    docker.elastic.co/integrations-ci/beats-kafka:3.6.0-2  "/run.sh"        2 minutes ago    Up 2 minutes (healthy)  2181/tcp, 0.0.0.0:32785->8774/tcp, 0.0.0.0:32784->8775/tcp, 0.0.0.0:32783->8779/tcp, 0.0.0.0:32782->9092/tcp  kafka_a035cf4c6889705a_kafka_1
 ```
 
-In order to identify to which port the Broker is listening on one should check in the logs of the container and find 
+In order to identify to which port the Broker is listening on one should check in the logs of the container and find
 the advertised address:
 
 ```console
@@ -39,7 +39,7 @@ listeners = INSIDE://localhost:9091,OUTSIDE://0.0.0.0:9092
 ```
 
 So here in this example the host we should in the module's config is `localhost:32778`.
-Note that this is different between MAC and Linux machines. The above is the case for the MAC machine, and here is how 
+Note that this is different between MAC and Linux machines. The above is the case for the MAC machine, and here is how
 the respective address for a LINUX machine should look like:
 
 ```console
@@ -60,7 +60,7 @@ This was needed before moving the metricbeat docker used in CI to host network,
 
 #### Configuring Kafka module
 In order to configure the Module we will use the advertised addressed to connect to the broker and the credentials
-that are also used for the tests 
+that are also used for the tests
 (see [test config](https://github.com/elastic/beats/blob/6c279ebf2789655725889f37820c959a8f2ea969/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go#L39)).
 Here is how the config should look like (in a MAC):
 

metricbeat/module/kafka/_meta/Dockerfile

@@ -18,12 +18,13 @@ RUN apt-get update && apt-get install -y curl openjdk-8-jre-headless netcat-open
 RUN mkdir -p ${KAFKA_LOGS_DIR} && mkdir -p ${KAFKA_HOME} && \
     curl -J -L -s -f -o - https://github.com/kadwanev/retry/releases/download/1.0.1/retry-1.0.1.tar.gz | tar xfz - -C /usr/local/bin && \
     retry --min 1 --max 180 -- curl -J -L -s -f --show-error -o $INSTALL_DIR/kafka.tgz \
-        "https://archive.apache.org/dist/kafka/${KAFKA_VERSION}/kafka_2.11-${KAFKA_VERSION}.tgz" && \
+        "https://archive.apache.org/dist/kafka/${KAFKA_VERSION}/kafka_2.13-${KAFKA_VERSION}.tgz" && \
     tar xzf ${INSTALL_DIR}/kafka.tgz -C ${KAFKA_HOME} --strip-components 1
 
 RUN retry --min 1 --max 180 -- curl -J -L -s -f --show-error -o /opt/jolokia-jvm-1.5.0-agent.jar \
     http://search.maven.org/remotecontent\?filepath\=org/jolokia/jolokia-jvm/1.5.0/jolokia-jvm-1.5.0-agent.jar
 
+ADD adminclient.properties /kafka/bin/adminclient.properties
 ADD kafka_server_jaas.conf /etc/kafka/server_jaas.conf
 ADD jaas-kafka-client-producer.conf /kafka/bin/jaas-kafka-client-producer.conf
 ADD sasl-producer.properties /kafka/bin/sasl-producer.properties

metricbeat/module/kafka/_meta/adminclient.properties

@@ -0,0 +1,3 @@
+security.protocol=SASL_PLAINTEXT
+sasl.mechanism=PLAIN
+sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username=admin password=admin-secret;

metricbeat/module/kafka/_meta/docs.asciidoc

@@ -24,7 +24,7 @@ kafka-acls --authorizer-properties zookeeper.connect=localhost:2181 --add --allo
 [float]
 === Compatibility
 
-This module is tested with Kafka 0.10.2.1, 1.1.0, 2.1.1, and 2.2.2.
+This module is tested with Kafka 0.10.2.1, 1.1.0, 2.1.1, 2.2.2 and 3.6.0.
 
 The Broker, Producer, Consumer metricsets require <<metricbeat-module-jolokia,Jolokia>> to fetch JMX metrics. Refer to the link for Jolokia's compatibility notes.
 

metricbeat/module/kafka/_meta/healthcheck.sh

@@ -4,11 +4,11 @@
 
 TOPIC="foo-`date '+%s-%N'`"
 
-${KAFKA_HOME}/bin/kafka-topics.sh --zookeeper=127.0.0.1:2181 --create --partitions 1 --topic "${TOPIC}" --replication-factor 1
+${KAFKA_HOME}/bin/kafka-topics.sh --bootstrap-server localhost:9091 --command-config ${KAFKA_HOME}/bin/adminclient.properties --create --partitions 1 --topic "${TOPIC}" --replication-factor 1
 rc=$?
 if [[ $rc != 0 ]]; then
 	exit $rc
 fi
 
-${KAFKA_HOME}/bin/kafka-topics.sh --zookeeper=127.0.0.1:2181 --delete --topic "${TOPIC}"
+${KAFKA_HOME}/bin/kafka-topics.sh --bootstrap-server localhost:9091 --command-config ${KAFKA_HOME}/bin/adminclient.properties --delete --topic "${TOPIC}"
 exit 0

metricbeat/module/kafka/_meta/run.sh

@@ -34,7 +34,7 @@ wait_for_port() {
     nc -z localhost $port
 }
 
-${KAFKA_HOME}/bin/kafka-topics.sh --zookeeper=127.0.0.1:2181 --create --partitions 1 --topic test --replication-factor 1
+${KAFKA_HOME}/bin/kafka-topics.sh  --bootstrap-server localhost:9091  --create --partitions 1 --topic test --replication-factor 1 --command-config ${KAFKA_HOME}/bin/adminclient.properties
 
 echo "Starting ZooKeeper"
 ${KAFKA_HOME}/bin/zookeeper-server-start.sh ${KAFKA_HOME}/config/zookeeper.properties &
@@ -44,7 +44,7 @@ echo "Starting Kafka broker"
 mkdir -p ${KAFKA_LOGS_DIR}
 export KAFKA_OPTS="-Djava.security.auth.login.config=/etc/kafka/server_jaas.conf -javaagent:/opt/jolokia-jvm-1.5.0-agent.jar=port=8779,host=0.0.0.0"
 ${KAFKA_HOME}/bin/kafka-server-start.sh ${KAFKA_HOME}/config/server.properties \
-    --override authorizer.class.name=kafka.security.auth.SimpleAclAuthorizer \
+    --override authorizer.class.name=kafka.security.authorizer.AclAuthorizer \
     --override super.users=User:admin \
     --override sasl.enabled.mechanisms=PLAIN \
     --override sasl.mechanism.inter.broker.protocol=PLAIN \
@@ -53,6 +53,7 @@ ${KAFKA_HOME}/bin/kafka-server-start.sh ${KAFKA_HOME}/config/server.properties \
     --override advertised.listeners=INSIDE://localhost:9091,OUTSIDE://$KAFKA_ADVERTISED_HOST \
     --override listener.security.protocol.map=INSIDE:SASL_PLAINTEXT,OUTSIDE:SASL_PLAINTEXT \
     --override inter.broker.listener.name=INSIDE \
+    --override zookeeper.set.acl=false \
     --override logs.dir=${KAFKA_LOGS_DIR} &
 
 wait_for_port 9092
@@ -62,22 +63,23 @@ wait_for_port 8779
 echo "Kafka load status code $?"
 
 # ACLS used to prepare tests
-${KAFKA_HOME}/bin/kafka-acls.sh --authorizer-properties zookeeper.connect=localhost:2181 --add --allow-principal User:producer --operation All --cluster --topic '*' --group '*'
-${KAFKA_HOME}/bin/kafka-acls.sh --authorizer-properties zookeeper.connect=localhost:2181 --add --allow-principal User:consumer --operation All --cluster --topic '*' --group '*'
+${KAFKA_HOME}/bin/kafka-acls.sh --bootstrap-server localhost:9091 --command-config ${KAFKA_HOME}/bin/adminclient.properties --add --allow-principal User:producer --operation All --cluster --topic '*' --group '*'
+${KAFKA_HOME}/bin/kafka-acls.sh --bootstrap-server localhost:9091 --command-config ${KAFKA_HOME}/bin/adminclient.properties --add --allow-principal User:consumer --operation All --cluster --topic '*' --group '*'
 
 # Minimal ACLs required by metricbeat. If this needs to be changed, please update docs too
-${KAFKA_HOME}/bin/kafka-acls.sh --authorizer-properties zookeeper.connect=localhost:2181 --add --allow-principal User:stats --operation Describe --group '*'
-${KAFKA_HOME}/bin/kafka-acls.sh --authorizer-properties zookeeper.connect=localhost:2181 --add --allow-principal User:stats --operation Read --topic '*'
+${KAFKA_HOME}/bin/kafka-acls.sh --bootstrap-server localhost:9091 --command-config ${KAFKA_HOME}/bin/adminclient.properties --add --allow-principal User:stats --operation Describe --group '*'
+${KAFKA_HOME}/bin/kafka-acls.sh --bootstrap-server localhost:9091 --command-config ${KAFKA_HOME}/bin/adminclient.properties --add --allow-principal User:stats --operation Read --topic '*'
 
 touch /tmp/.acls_loaded
 
-
+echo "Kafka producer start"
 # Start a forever producer
 { while sleep 1; do echo message; done } | KAFKA_OPTS="-Djava.security.auth.login.config=/kafka/bin/jaas-kafka-client-producer.conf -javaagent:/opt/jolokia-jvm-1.5.0-agent.jar=port=8775,host=0.0.0.0" \
  ${KAFKA_HOME}/bin/kafka-console-producer.sh --topic test --broker-list localhost:9091 --producer.config ${KAFKA_HOME}/bin/sasl-producer.properties > /dev/null &
 
 wait_for_port 8775
 
+echo "Kafka consumer start"
 # Start a forever consumer
 KAFKA_OPTS="-Djava.security.auth.login.config=/kafka/bin/jaas-kafka-client-consumer.conf -javaagent:/opt/jolokia-jvm-1.5.0-agent.jar=port=8774,host=0.0.0.0" \
  ${KAFKA_HOME}/bin/kafka-console-consumer.sh --topic=test --bootstrap-server=localhost:9091 --consumer.config ${KAFKA_HOME}/bin/sasl-producer.properties > /dev/null &

metricbeat/module/kafka/_meta/supported-versions.yml

@@ -1,4 +1,5 @@
 variants:
+  - KAFKA_VERSION: 3.6.0
   - KAFKA_VERSION: 2.2.2
   - KAFKA_VERSION: 2.1.1
   - KAFKA_VERSION: 2.0.0

metricbeat/module/kafka/broker/_meta/docs.asciidoc

@@ -2,7 +2,7 @@ This metricset periodically fetches JMX metrics from Kafka Broker JMX.
 
 [float]
 === Compatibility
-The module has been tested with Kafka 2.1.1 and 2.2.2. Other versions are expected to work.
+The module has been tested with Kafka 2.1.1, 2.2.2 and 3.6.0. Other versions are expected to work.
 
 [float]
 === Usage

metricbeat/module/kafka/consumer/_meta/docs.asciidoc

@@ -2,7 +2,7 @@ This metricset periodically fetches JMX metrics from Kafka Consumers implemented
 
 [float]
 === Compatibility
-The module has been tested with Kafka 2.1.1 and 2.2.2. Other versions are expected to work.
+The module has been tested with Kafka 2.1.1, 2.2.2 and 3.6.0. Other versions are expected to work.
 
 [float]
 === Usage

metricbeat/module/kafka/consumergroup/consumergroup.go

@@ -52,7 +52,7 @@ var debugf = logp.MakeDebug("kafka")
 // New creates a new instance of the MetricSet.
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 	opts := kafka.MetricSetOptions{
-		Version: "2.2.0",
+		Version: "3.6.0",
 	}
 
 	ms, err := kafka.NewMetricSet(base, opts)

metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go

@@ -107,7 +107,7 @@ func startConsumer(t *testing.T, host string, groupID string) (io.Closer, error)
 	// Create a new consumer group
 	consumerGroup, err := sarama.NewConsumerGroup(brokers, groupID, config)
 	if err != nil {
-		t.Fatalf("Error creating consumer group: %v", err)
+		t.Fatalf("Error creating consumer group: %v, brokers: %s", err, brokers)
 		return nil, err
 	}
 

metricbeat/module/kafka/docker-compose.yml

@@ -1,10 +1,10 @@
 services:
   kafka:
-    image: docker.elastic.co/integrations-ci/beats-kafka:${KAFKA_VERSION:-2.2.2}-2
+    image: docker.elastic.co/integrations-ci/beats-kafka:${KAFKA_VERSION:-3.6.0}-2
     build:
       context: ./_meta
       args:
-        KAFKA_VERSION: ${KAFKA_VERSION:-2.2.2}
+        KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0}
     ports:
       - 9092
       - 8779

metricbeat/module/kafka/partition/partition.go

@@ -51,7 +51,7 @@ var debugf = logp.MakeDebug("kafka")
 // New creates a new instance of the partition MetricSet.
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 	opts := kafka.MetricSetOptions{
-		Version: "2.2.0",
+		Version: "3.6.0",
 	}
 
 	ms, err := kafka.NewMetricSet(base, opts)

metricbeat/module/kafka/producer/_meta/docs.asciidoc

@@ -2,7 +2,7 @@ This metricset periodically fetches JMX metrics from Kafka Producers implemented
 
 [float]
 === Compatibility
-The module has been tested with Kafka 2.1.1 and 2.2.2. Other versions are expected to work.
+The module has been tested with Kafka 2.1.1, 2.2.2 and 3.6.0. Other versions are expected to work.
 
 [float]
 === Usage

testing/environments/docker/kafka/Dockerfile

@@ -6,7 +6,7 @@ ENV KAFKA_HOME=/kafka
 ENV KAFKA_ADVERTISED_HOST=kafka
 
 ENV KAFKA_LOGS_DIR="/kafka-logs"
-ENV KAFKA_VERSION=2.2.2
+ENV KAFKA_VERSION=3.6.0
 ENV _JAVA_OPTIONS="-Djava.net.preferIPv4Stack=true"
 ENV TERM=linux
 
@@ -15,7 +15,7 @@ RUN apt-get update && apt-get install -y curl openjdk-11-jre-headless netcat-ope
 RUN mkdir -p ${KAFKA_LOGS_DIR} && mkdir -p ${KAFKA_HOME} && \
     curl -J -L -s -f -o - https://github.com/kadwanev/retry/releases/download/1.0.1/retry-1.0.1.tar.gz | tar xfz - -C /usr/local/bin && \
     retry --min 1 --max 180 -- curl -J -L -s -f --show-error -o $INSTALL_DIR/kafka.tgz \
-        "https://archive.apache.org/dist/kafka/${KAFKA_VERSION}/kafka_2.11-${KAFKA_VERSION}.tgz" && \
+        "https://archive.apache.org/dist/kafka/${KAFKA_VERSION}/kafka_2.13-${KAFKA_VERSION}.tgz" && \
     tar xzf ${INSTALL_DIR}/kafka.tgz -C ${KAFKA_HOME} --strip-components 1
 
 ADD run.sh /run.sh

testing/environments/docker/kafka/healthcheck.sh

@@ -2,11 +2,11 @@
 
 TOPIC="foo-`date '+%s-%N'`"
 
-${KAFKA_HOME}/bin/kafka-topics.sh --zookeeper=127.0.0.1:2181 --create --partitions 1 --topic "${TOPIC}" --replication-factor 1
+${KAFKA_HOME}/bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --partitions 1 --topic "${TOPIC}" --replication-factor 1
 rc=$?
 if [[ $rc != 0 ]]; then
 	exit $rc
 fi
 
-${KAFKA_HOME}/bin/kafka-topics.sh --zookeeper=127.0.0.1:2181 --delete --topic "${TOPIC}"
+${KAFKA_HOME}/bin/kafka-topics.sh --bootstrap-server localhost:9092  --delete --topic "${TOPIC}"
 exit 0

testing/environments/docker/kafka/run.sh

@@ -16,13 +16,6 @@ echo "Starting ZooKeeper"
 ${KAFKA_HOME}/bin/zookeeper-server-start.sh ${KAFKA_HOME}/config/zookeeper.properties &
 wait_for_port 2181
 
-# create a user beats with password KafkaTest, for use in client SASL authentication
-/kafka/bin/kafka-configs.sh \
-	--zookeeper localhost:2181 \
-	--alter --add-config 'SCRAM-SHA-512=[password=KafkaTest]' \
-	--entity-type users \
-	--entity-name beats
-
 # Start Kafka with three listeners. The INSIDE listener makes Kafka reachable inside of docker
 # networks when the container hostname matches KAFKA_ADVERTISED_HOST. The OUTSIDE and SASL_SSL both
 # bind to localhost and are reachable from the host machine on the loopback interface.
@@ -49,5 +42,12 @@ wait_for_port 9092
 
 echo "Kafka load status code $?"
 
+# create a user beats with password KafkaTest, for use in client SASL authentication
+/kafka/bin/kafka-configs.sh \
+	--bootstrap-server localhost:9092 \
+	--alter --add-config 'SCRAM-SHA-512=[password=KafkaTest]' \
+	--entity-type users \
+	--entity-name beats
+
 # Make sure the container keeps running
 tail -f /dev/null