docker-compose.yaml

version: '3'
services:
  spark:
    image: bitnami/spark:3.5.0
    hostname: spark
    environment:
      - SPARK_MODE=master
    ports:
      - '8080:8080'
      - '7077:7077'
    volumes:
      - ./data:/data
      # - /Volumes/development/spark-fires-data:/data
      - ./src:/src
    networks:
      - spark_network
  spark-worker-1:
    image: bitnami/spark:3.5.0
    environment:
      - SPARK_MODE=worker
      - SPARK_MASTER_URL=spark://spark:7077
      - SPARK_WORKER_MEMORY=2G
      - SPARK_EXECUTOR_MEMORY=2G
      - SPARK_WORKER_CORES=2
    ports:
      - '8081:8081'
    volumes:
      - ./data:/data
      # - /Volumes/development/spark-fires-data:/data
      - ./src:/src
    depends_on:
      - spark
    networks:
      - spark_network
  spark-worker-2:
    image: bitnami/spark:3.5.0
    environment:
      - SPARK_MODE=worker
      - SPARK_MASTER_URL=spark://spark:7077
      - SPARK_WORKER_MEMORY=2G
      - SPARK_EXECUTOR_MEMORY=2G
      - SPARK_WORKER_CORES=2
    ports:
      - '8082:8081'
    volumes:
      - ./data:/data
      # - /Volumes/development/spark-fires-data:/data
      - ./src:/src
    depends_on:
      - spark
    networks:
      - spark_network
  spark-worker-3:
    image: bitnami/spark:3.5.0
    environment:
      - SPARK_MODE=worker
      - SPARK_MASTER_URL=spark://spark:7077
      - SPARK_WORKER_MEMORY=2G
      - SPARK_EXECUTOR_MEMORY=2G
      - SPARK_WORKER_CORES=2
      - HOT_NODE=true
    ports:
      - '8083:8081'
    volumes:
      - ./data:/data
      # - /Volumes/development/spark-fires-data:/data
      - ./src:/src
    depends_on:
      - spark
    networks:
      - spark_network
  jupyter:
    image: jupyter/pyspark-notebook:spark-3.5.0
    ports:
      # - '8890:8888'
      - '8888:8888'
      - '4040:4040'
    volumes:
      - ./data:/data
      # - /Volumes/development/spark-fires-data:/data
      - ./src:/src
      - ./notebooks:/home/jovyan/work
    depends_on:
      - spark
    networks:
      - spark_network

networks:
  spark_network:
    driver: bridge
    name: spark_network