docker-compose.yaml

version: "3.8"

services:
  ############################
  # KAFKA (ZOOKEEPER)
  ############################
  zoo1:
    image: confluentinc/cp-zookeeper:6.2.1@sha256:5598d44649da42ab5ff926830ecd502303e1ba992fcedb65e03fab0ec2bfe976
    hostname: zoo1
    container_name: zoo1
    ports:
      - "2181:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_SERVERS: zoo1:2888:3888
    volumes:
      - type: volume
        source: zksingle-ksingle_zoo1-data
        target: /data
      - type: volume
        source: zksingle-ksingle_zoo1-log
        target: /datalog
  ############################
  # KAFKA (KAFKA BROKER 1)
  ############################
  kafka1:
    image: confluentinc/cp-kafka:6.2.1@sha256:870c760a9e184260c159e4f7e5b4e715be0b89726b397f709fd6ce43add34456
    hostname: kafka1
    user: "appuser:appuser"
    ports:
      - "9092:9092"
      - "9999:9999"
    environment:
      KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
      KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
      KAFKA_BROKER_ID: 1
      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_JMX_PORT: 9999
      KAFKA_JMX_HOSTNAME: ${DOCKER_HOST_IP:-127.0.0.1}
    volumes:
      - type: "volume"
        source: zksingle-ksingle_kafka1-data
        target: /var/lib/kafka/data
    depends_on:
      - zoo1

  ############################
  # KAFKA (KAFKA CONNECT)
  ############################
  kafka-connect:
    image: confluentinc/cp-kafka-connect-base:6.2.1@sha256:f4cadb139b4b9e1daafb7ffb3350652132360ac959ac40d0e9d4a0ead1c0b13e
    container_name: kafka-connect
    depends_on:
      - kafka1
    ports:
      - 8083:8083
    environment:
      CONNECT_BOOTSTRAP_SERVERS: "kafka1:19092"
      CONNECT_REST_PORT: 8083
      CONNECT_GROUP_ID: kafka-connect
      CONNECT_CONFIG_STORAGE_TOPIC: _connect-configs
      CONNECT_OFFSET_STORAGE_TOPIC: _connect-offsets
      CONNECT_STATUS_STORAGE_TOPIC: _connect-status
      CONNECT_KEY_CONVERTER: org.apache.kafka.connect.converters.IntegerConverter
      CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "false"
      CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "false"
      CONNECT_VALUE_CONVERTER_SCHEMA_IGNORE: "true"
      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: "http://schemaregistry:8081"
      CONNECT_REST_ADVERTISED_HOST_NAME: "kafka-connect"
      CONNECT_LOG4J_APPENDER_STDOUT_LAYOUT_CONVERSIONPATTERN: "[%d] %p %X{connector.context}%m (%c:%L)%n"
      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "1"
    command:
      - bash
      - -c
      - |
        echo "Installing Connector"
        confluent-hub install --no-prompt debezium/debezium-connector-mysql:1.7.0
        confluent-hub install --no-prompt confluentinc/kafka-connect-elasticsearch:5.4.0
        confluent-hub install --no-prompt confluentinc/kafka-connect-datagen:0.5.2
        confluent-hub install --no-prompt neo4j/kafka-connect-neo4j:2.0.0
        #
        echo "Launching Kafka Connect worker"
        /etc/confluent/docker/run &
        #
        sleep infinity      

#  ############################
#  # KAFKA (CONFLUENT SCHEMA REGISTRY)
#  ############################
#  schemaregistry:
#    image: confluentinc/cp-schema-registry:6.2.1@sha256:d0478567133996be2298d7b237e0cbcb937753db6d82836e5a8737550397b493
#    depends_on:
#      - zoo1
#    environment:
#      - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=PLAINTEXT://kafka1:19092
#      - SCHEMA_REGISTRY_HOST_NAME=schemaregistry
#      - SCHEMA_REGISTRY_LISTENERS=http://0.0.0.0:8081
#    ports:
#      - 8081:8081
#
#  ############################
#  # KAFKA (KAFKA REST PROXY)
#  ############################
#  restproxy:
#    image: confluentinc/cp-kafka-rest:6.2.1@sha256:3b7fb2823c4e65a6998f43ae37f0e289268b3f4059a77a52081ffce6cf591820
#    restart: always
#    depends_on:
#      - kafka1
#    environment:
#      KAFKA_REST_ZOOKEEPER_CONNECT: "zoo1:2181"
#      KAFKA_REST_LISTENERS: "http://0.0.0.0:8082"
#      KAFKA_REST_SCHEMA_REGISTRY_URL: "http://schemaregistry:8085"
#      KAFKA_REST_HOST_NAME: restproxy
#      KAFKA_REST_DEBUG: "true"
#    ports:
#      - 8082:8082
  ############################
  # KAFDROP
  ############################
  kafdrop:
    image: obsidiandynamics/kafdrop:latest@sha256:b7ba8577ce395b1975b0ed98bb53cb6b13e7d32d5442188da1ce41c0838d1ce9
    environment:
      KAFKA_BROKERCONNECT: "kafka1:19092"
      SERVER_SERVLET_CONTEXTPATH: "/"
    ports:
      - 9000:9000
    profiles:
      - monitoring

  prometheus:
    image: prom/prometheus@sha256:a8779cfe553e0331e9046268e26c539fa39ecf90d59836d828163e65e8f4fa35
    volumes:
      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
    profiles:
      - monitoring

  grafana:
    image: grafana/grafana@sha256:2f6e30f0998a190cd4c97225690f1fd8702498185385cb943233b9fa778c0a7b
    ports:
      - "3000:3000"
    volumes:
      - ./grafana/custom.ini:/usr/share/grafana/conf/custom.ini
      - grafana-storage:/var/lib/grafana
    profiles:
      - monitoring

  ############################
  # OPEN SEARCH
  ############################
  opensearch-node1:
    image: opensearchproject/opensearch:latest@sha256:967d7f57f72f5df159380159b7eb63fe0b4a1cf860d92dd44d359e2db448a8f9
    container_name: opensearch-node1
    environment:
      - cluster.name=opensearch-cluster
      - node.name=opensearch-node1
      - discovery.seed_hosts=opensearch-node1
      - plugins.security.disabled=true
      - cluster.initial_master_nodes=opensearch-node1
      - bootstrap.memory_lock=true # along with the memlock settings below, disables swapping
      - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 265000 # maximum number of open files for the OpenSearch user, set to at least 65536 on modern systems
        hard: 265000
    volumes:
      - opensearch-data1:/usr/share/opensearch/data
      #- ./custom-opensearch.yml:/usr/share/opensearch/config/opensearch.yml
    ports:
      - 9200:9200
      - 9600:9600 # required for Performance Analyzer

  ############################
  # OPEN SEARCH DASHBOARD (NO SECURITY)
  ############################
  opensearch-dashboards:
    image: opensearchproject/opensearch-dashboards-no-security:latest
    build:
      context: .
      dockerfile: Dockerfile-opensearch-dashboard-nosec
    container_name: opensearch-dashboards
    environment:
      - opensearch.ssl.verificationMode=none
      - OPENSEARCH_HOSTS=["http://opensearch-node1:9200"]
    ports:
      - 5601:5601
    expose:
      - "5601"
      #volumes:
      #- ./custom-opensearch_dashboards.yml:/usr/share/opensearch-dashboards/config/opensearch_dashboards.yml

  ############################
  # KAFKA MOVIE STREAMS
  ############################
  kafka-movie-streams:
    image: swiedenfeld/kafka-movie-streams
    build:
      context: .
      dockerfile: Dockerfile
    depends_on:
      - kafka1
    ports:
      - 8080:8080
    environment:
      - SPRING_KAFKA_STREAMS_BOOTSTRAP_SERVERS=kafka1:19092
      - SPRING_KAFKA_BOOTSTRAP_SERVERS=kafka1:19092

volumes:
  zksingle-ksingle_kafka1-data:
  zksingle-ksingle_zoo1-data:
  zksingle-ksingle_zoo1-log:
  opensearch-data1:
  grafana-storage: