docker-compose.yml

version: '3.8'

networks:
  internal:
    driver: bridge

x-spark-common: &spark-common
  build: 
    dockerfile: docker/spark/Dockerfile
  environment: 
    - SPARK_MODE=worker
    - SPARK_MASTER_URL=spark://spark-master:7077
    - SPARK_WORKER_MEMORY=2G
    - SPARK_WORKER_CORES=1
    - SPARK_RPC_AUTHENTICATION_ENABLED=no
    - SPARK_RPC_ENCRYPTION_ENABLED=no
    - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
    - SPARK_SSL_ENABLED=no
    - SPARK_USER=spark
    # More environment variables natively supported by Apache Spark
    # can be found here -> https://spark.apache.org/docs/latest/spark-standalone.html#cluster-launch-scripts
  volumes:
    - ./src:/app/src
    - ./jobs:/app/jobs
    - ./spark:/app/spark
    - ./.env:/app/.env
    - ./config.yaml:/app/config.yaml
    - ./pyproject.toml:/app/pyproject.toml
    - ./poetry.lock:/app/poetry.lock
    - ./config/spark/log4j2.properties:/opt/bitnami/spark/conf/log4j2.properties
  restart: unless-stopped
  networks:
    - internal

services:
  spark-master:
    <<: *spark-common
    container_name: spark-master
    environment:
      - SPARK_MODE=master
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
      - SPARK_SSL_ENABLED=no
      - SPARK_USER=spark
    ports:
      - "8080:8080"
      - "7077:7077"

  spark-worker-1:
    <<: *spark-common
    container_name: spark-worker-1
    ports:
      - "8081:8081"
    depends_on:
      - spark-master

  spark-worker-2:
    <<: *spark-common
    container_name: spark-worker-2
    ports:
      - "8082:8081"
    depends_on:
      - spark-master

  spark-worker-3:
    <<: *spark-common
    container_name: spark-worker-3
    ports:
      - "8083:8081"
    depends_on:
      - spark-master

  spark-worker-4:
    <<: *spark-common
    container_name: spark-worker-4
    ports:
      - "8084:8081"
    depends_on:
      - spark-master

  spark-worker-5:
    <<: *spark-common
    container_name: spark-worker-5
    ports:
      - "8085:8081"
    depends_on:
      - spark-master

# https://github.com/bitnami/containers/tree/main/bitnami/kafka
  kafka:
    image: docker.io/bitnami/kafka:3.4.1-debian-11-r13
    container_name: kafka
    ports:
      - 9092:9092
    environment:
      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://51.250.28.37:9092
      - ALLOW_PLAINTEXT_LISTENER=yes
      - KAFKA_CFG_BROKER_ID=1
      - KAFKA_CFG_DEFAULT_REPLICATION_FACTOR=1
      - KAFKA_LOG_RETENTION_HOURS=24
    volumes:
      - ./config/kafka/log4j.properties:/opt/bitnami/kafka/config/log4j.properties 
    restart: unless-stopped
    networks:
      - internal

  producer:
    build: 
      dockerfile: docker/producer/Dockerfile
    container_name: producer
    command: /bin/bash /app/producer/run.sh
    volumes:
      - ./src:/app/src
      - ./.env:/app/.env
      - ./producer:/app/producer
      - ./config.yaml:/app/config.yaml
    depends_on:
      - kafka
    restart: on-failure
    networks:
      - internal

  generator:
    build: 
      dockerfile: docker/generator/Dockerfile
    container_name: generator
    command: /bin/bash /app/generator/run.sh
    volumes:
      - ./src:/app/src
      - ./.env:/app/.env
      - ./generator:/app/generator 
      - ./config.yaml:/app/config.yaml
    restart: on-failure
    networks:
      - internal