docker-compose.yml

# Copyright VMware, Inc.
# SPDX-License-Identifier: APACHE-2.0

version: '2'

services:
  spark:
    image: my/spark:3.4_gcp
    environment:
      - SPARK_MODE=master
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
      - SPARK_SSL_ENABLED=no
      - SPARK_USER=spark
    ports:
      - '8080:8080'
      - '8082:8082'
  spark-worker-a:
    image: my/spark:3.4_gcp
    environment:
      - SPARK_MODE=worker
      - SPARK_MASTER_URL=spark://spark:7077
      - SPARK_WORKER_MEMORY=4G
      - SPARK_WORKER_CORES=4
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
      - SPARK_SSL_ENABLED=no
      - SPARK_USER=spark
  # spark-worker-b:
  #   image: my/spark:3.4_gcp
  #   environment:
  #     - SPARK_MODE=worker
  #     - SPARK_MASTER_URL=spark://spark:7077
  #     - SPARK_WORKER_MEMORY=4G
  #     - SPARK_WORKER_CORES=4
  #     - SPARK_RPC_AUTHENTICATION_ENABLED=no
  #     - SPARK_RPC_ENCRYPTION_ENABLED=no
  #     - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
  #     - SPARK_SSL_ENABLED=no
  #     - SPARK_USER=spark
  airflow:
    image: apache/airflow:2.7.0
    ports:
      - "8081:8080"
    command:
      bash -c "airflow standalone"