docker-compose.yml

version: "3.8"

services:
  kafka:
    image: docker.io/bitnami/kafka:3.4
    container_name: kafka
    ports:
      - "9093:9093"
      - "9092:9092"
    environment:
      KAFKA_CFG_NODE_ID: 0
      ALLOW_PLAINTEXT_LISTENER: 'yes'
      KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'true'
      KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@kafka:9093
      KAFKA_KRAFT_CLUSTER_ID: abcdefghijklmnopqrstuv
      KAFKA_CFG_PROCESS_ROLES: controller,broker
      KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
      KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093
      EXTRA_ARGS: "-Xms128m -Xmx256m -javaagent:/opt/jmx-exporter/jmx_prometheus_javaagent-0.19.0.jar=9404:/opt/jmx-exporter/kafka-2_0_0.yml"
    volumes:
      - ./jmx-exporter:/opt/jmx-exporter
    healthcheck:
      test: "bash -c 'printf \"\" > /dev/tcp/127.0.0.1/9092; exit $$?;'"
      interval: 5s
      timeout: 10s
      retries: 3
      start_period: 30s

  kafka-exporter:
    container_name: kafka-exporter
    image: docker.io/bitnami/kafka-exporter:latest
    depends_on:
      kafka:
        condition: service_healthy
    command: --kafka.server=kafka:9092

  producer:
    build:
      context: ./container/producer/
    container_name: producer
    command: python /opt/producer/logs_producer.py
    volumes:
      - ./producer:/opt/producer
    restart: on-failure
    depends_on:
      - kafka

  logstash:
    build:
      context: ./container/logstash/
    container_name: logstash
    volumes:
      - "./logstash:/usr/share/logstash/pipeline"
    environment:
      - "LS_JAVA_OPTS=-Xmx256m -Xms256m"
      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD}
      - API_HTTP_PORT=9675 # assign API HTTP port for metrics to avoid conflict with default port (9600)
      - PIPELINE_ECS_COMPATIBILITY=disabled
      - PIPELINE_BATCH_SIZE=250
    ports:
      - "9675:9675"
    depends_on:
      - opensearch-node1
      - opensearch-node2
      - kafka
   
  opensearch-node1:
    build:
      context: ./container/opensearch
    container_name: opensearch-node1
    environment:
      - cluster.name=opensearch-cluster
      - node.name=opensearch-node1
      - discovery.seed_hosts=opensearch-node1,opensearch-node2 # Nodes to look for when discovering the cluster
      - cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2 # Nodes eligible to serve as cluster manager
      - bootstrap.memory_lock=true # Disable JVM heap memory swapping
      - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" # Set min and max JVM heap sizes to at least 50% of system RAM
      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD}    # Sets the demo admin user password when using demo configuration, required for OpenSearch 2.12 and later
      - "DISABLE_INSTALL_DEMO_CONFIG=true" # Prevents execution of bundled demo script which installs demo certificates and security configurations to OpenSearch
      - "DISABLE_SECURITY_PLUGIN=true" # Disables Security plugin
    ulimits:
      memlock:
        soft: -1 # Set memlock to unlimited (no soft or hard limit)
        hard: -1
      nofile:
        soft: 65536 # Maximum number of open files for the opensearch user - set to at least 65536
        hard: 65536
    volumes:
      - opensearch-data1:/usr/share/opensearch/data # Creates volume called opensearch-data1 and mounts it to the container
    ports:
      - 9200:9200 # REST API
      - 9600:9600 # Performance Analyzer

  opensearch-node2:
    build:
      context: ./container/opensearch # This should be the same image used for opensearch-node1 to avoid issues
    container_name: opensearch-node2
    environment:
      - cluster.name=opensearch-cluster
      - node.name=opensearch-node2
      - discovery.seed_hosts=opensearch-node1,opensearch-node2
      - cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2
      - bootstrap.memory_lock=true
      - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD}
      - "DISABLE_INSTALL_DEMO_CONFIG=true" # Prevents execution of bundled demo script which installs demo certificates and security configurations to OpenSearch
      - "DISABLE_SECURITY_PLUGIN=true" # Disables Security plugin
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 65536
        hard: 65536
    volumes:
      - opensearch-data2:/usr/share/opensearch/data

  opensearch-dashboards:
    image: opensearchproject/opensearch-dashboards:2.16.0
    container_name: opensearch-dashboards
    ports:
      - 5601:5601
    expose:
      - "5601"
    environment:
      - 'OPENSEARCH_HOSTS=["http://opensearch-node1:9200","http://opensearch-node2:9200"]' # Define the OpenSearch nodes that OpenSearch Dashboards will query
      - "DISABLE_SECURITY_DASHBOARDS_PLUGIN=true" # disables security dashboards plugin in OpenSearch Dashboards

  json-exporter:
    image: prometheuscommunity/json-exporter:latest
    container_name: json_exporter
    volumes:
      - ./metrics/config.yml:/config.yml:ro
    ports:
      - "7979:7979"

  prometheus:
    image: prom/prometheus:v2.37.1
    container_name: prometheus
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml

  grafana:
    image: grafana/grafana:8.4.0
    container_name: grafana
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD}
    volumes:
      - ./grafana/provisioning/:/etc/grafana/provisioning/

volumes:
  opensearch-data1:
  opensearch-data2: