docker-compose.yaml

version: '3.7'
services:
  portainer:
    container_name: portainer
    image: portainer/portainer
    networks:
      - dockernet   
    restart: "always"
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
    ports:
      - 9000:9000
  mysql:
    image: mysql:5.7
    networks:
      - dockernet   
    ports:
      - "3306:3306"
    container_name: mysql
    environment:
          - MYSQL_DATABASE=openmrs
          - MYSQL_ROOT_PASSWORD=debezium
          - MYSQL_USER=mysqluser
          - MYSQL_PASSWORD=mysqlpw
          - MYSQL_ROOT_HOST=%    # Allow docker contatiners to connect to mysql
    volumes:
        - ./mysql.cnf:/etc/mysql/conf.d/custom.cnf
        - ./dbdump:/docker-entrypoint-initdb.d
  phpmyadmin:
    depends_on:
      - mysql
    image: phpmyadmin/phpmyadmin
    container_name: phpmyadmin
    networks:
      - dockernet   
    restart: always
    ports:
      - '7070:80'
    environment:
      PMA_HOST: mysql
      MYSQL_ROOT_PASSWORD: debezium 

  openmrs:
    restart: "always"
    image: openmrs/openmrs-reference-application-distro:latest
    container_name: openmrs
    networks:
      - dockernet   
    depends_on:
      - mysql
    ports:
      - "8080:8080"
    environment:
      DB_DATABASE: 'openmrs'
      DB_HOST: mysql
      DB_USERNAME: 'mysqluser'
      DB_PASSWORD: 'mysqlpw'
      #DB_CREATE_TABLES: 'true'
      #DB_AUTO_UPDATE: 'true'
      #MODULE_WEB_ADMIN: 'true'
  zookeeper:
    image: debezium/zookeeper:${DEBEZIUM_VERSION}
    networks:
      - dockernet   
    ports:
     - 2181:2181
     - 2888:2888
     - 3888:3888
    container_name: zookeeper
  kafka:
    image: debezium/kafka:${DEBEZIUM_VERSION}
    container_name: kafka
    networks:
      - dockernet   
    ports:
     - 9092:9092
     - 9999:9999
    expose:
     - 9092
     - 9999
    depends_on:
     - zookeeper
    environment:
     - ZOOKEEPER_CONNECT=zookeeper:2181
     - CREATE_TOPICS= "input:1:3,output:1:3"

  connect:
    image: debezium/connect:${DEBEZIUM_VERSION}
    restart: always
    networks:
      - dockernet   
    depends_on:
     - kafka
    ports:
     - 8083:8083
    environment:
     - BOOTSTRAP_SERVERS=kafka:9092
     - GROUP_ID=1
     - CONFIG_STORAGE_TOPIC=my_connect_configs
     - OFFSET_STORAGE_TOPIC=my_connect_offsets
     - STATUS_STORAGE_TOPIC=my_connect_statuses
    container_name: mysql-connect

# other important containers - uncomment to install
  kafka-schema-registry:
    container_name: kafka-schema-registry
    image: confluentinc/cp-schema-registry:4.0.0
    hostname: kafka-schema-registry
    networks:
      - dockernet   
    restart: unless-stopped
    ports:
      - "8081:8081"
    environment:
      SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper:2181
      SCHEMA_REGISTRY_HOST_NAME: kafka-schema-registry
      SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
    depends_on:
      - zookeeper
      - kafka

  schema-registry-ui:
    container_name: schema-registry-ui
    image: landoop/schema-registry-ui:0.9.3
    hostname: kafka-schema-registry-ui
    networks:
      - dockernet   
    ports:
      - "8001:8000"
    environment:
      SCHEMAREGISTRY_URL: http://kafka-schema-registry:8081/
      PROXY: "true"
    depends_on:
      - kafka-schema-registry

  kafka-rest-proxy:
    container_name: kafka-rest-proxy
    image: confluentinc/cp-kafka-rest:4.0.0
    hostname: kafka-rest-proxy
    networks:
      - dockernet   
    ports:
      - "8082:8082"
    environment:
      KAFKA_REST_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_REST_LISTENERS: http://0.0.0.0:8082/
      KAFKA_REST_SCHEMA_REGISTRY_URL: http://kafka-schema-registry:8081/
      KAFKA_REST_HOST_NAME: kafka-rest-proxy
      KAFKA_REST_BOOTSTRAP_SERVERS: kafka:9092
    depends_on:
      - zookeeper
      - kafka
      - kafka-schema-registry

  kafka-topics-ui:
    container_name: kafka-topics-ui
    image: landoop/kafka-topics-ui:0.9.3
    hostname: kafka-topics-ui
    ports:
      - "8000:8000"
    environment:
      KAFKA_REST_PROXY_URL: "http://kafka-rest-proxy:8082/"
      PROXY: "true"
    depends_on:
      - zookeeper
      - kafka
      - kafka-schema-registry
      - kafka-rest-proxy

  ksql-server:
    container_name: ksql-server
    image: "confluentinc/cp-ksql-server:5.2.2"
    networks:
      - dockernet   
    ports:
      - 8088:8088
    depends_on:
      - kafka
      - kafka-schema-registry
    environment:
      KSQL_BOOTSTRAP_SERVERS: kafka:9092
      KSQL_LISTENERS: http://ksql-server:8088
      KSQL_KSQL_SCHEMA_REGISTRY_URL: http://kafka-schema-registry:8081/

  kafka_manager:
    image: hlebalbau/kafka-manager:1.3.3.18
    container_name: kafka-manager
    networks:
      - dockernet   
    ports:
      - "8089:8089"
    expose:
      - 8089  
    depends_on:
      - zookeeper
      - kafka
    command: -Dconfig.file=/kafka-manager/conf/application.conf -Dapplication.home=/kafkamanager  
    environment:
      ZK_HOSTS: zookeeper:2181
      APPLICATION_SECRET: letmein
      KM_ARGS: -Djava.net.preferIPv4Stack=true

  polynote:
    image: lolhens/polynote
    networks:
      - dockernet   
    environment:
      POLYNOTE_CONFIG: |
        listen:
          host: 0.0.0.0
          port: 8192
    ports:
      - "8192:8192"
    volumes:
      - notebooks:/opt/polynote/notebooks

  spark:
    image: bitnami/spark:2.4.5
    container_name: spark-master
    networks:
      - dockernet   
    environment:
      - SPARK_MODE=master
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
      - SPARK_SSL_ENABLED=no
    ports:
      - "7077:7077"
      - "4040:8080"
  spark-worker-1:
    image: bitnami/spark:2.4.5
    container_name: spark-worker-1
    hostname: localhost
    depends_on:
      - spark
    networks:
      - dockernet   
    ports:
      - "8100:8081"
    environment:
      - SPARK_MODE=worker
      - SPARK_MASTER_URL=spark://spark-master:7077
    #  - SPARK_WORKER_MEMORY=1G
    #  - SPARK_WORKER_CORES=1
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
      - SPARK_SSL_ENABLED=no
  spark-worker-2:
    image: bitnami/spark:2.4.5
    hostname: localhost
    container_name: spark-worker-2
    depends_on:
      - spark
    ports:
      - "8200:8081"
    networks:
      - dockernet     
    environment:
      - SPARK_MODE=worker
      - SPARK_MASTER_URL=spark://spark-master:7077
    #  - SPARK_WORKER_MEMORY=1G
    #  - SPARK_WORKER_CORES=1
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
      - SPARK_SSL_ENABLED=no

  spark-worker-3:
    image: bitnami/spark:2.4.5
    container_name: spark-worker-3
    hostname: localhost
    depends_on:
      - spark
    networks:
      - dockernet   
    ports:
      - "8300:8081"
    environment:
      - SPARK_MODE=worker
      - SPARK_MASTER_URL=spark://spark-master:7077
    #  - SPARK_WORKER_MEMORY=1G
    #  - SPARK_WORKER_CORES=1
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
      - SPARK_SSL_ENABLED=no
  spark-worker-4:
    image: bitnami/spark:2.4.5
    hostname: localhost
    container_name: spark-worker-4
    depends_on:
      - spark
    ports:
      - "8400:8081"
    networks:
      - dockernet     
    environment:
      - SPARK_MODE=worker
      - SPARK_MASTER_URL=spark://spark-master:7077
    #  - SPARK_WORKER_MEMORY=1G
    #  - SPARK_WORKER_CORES=1
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
      - SPARK_SSL_ENABLED=no

 
 
# TODO: https://github.com/EthicalML/kafka-spark-streaming-zeppelin-docker, https://github.com/jupyter-incubator/sparkmagic/blob/master/docker-compose.yml


volumes:
  notebooks:

networks:
    dockernet: