docker-compose.yml

version: '3.8'
services:
  # zookeeper:
  #   image: bitnami/zookeeper:3.7
  #   container_name: zookeeper
  #   environment:
  #     - ALLOW_ANONYMOUS_LOGIN=yes
  #   ports:
  #     - "2181:2181"
  kafka:
    image: bitnami/kafka:3.5.0
    container_name: kafka
    ports:
      - '9092:9092'
    environment:
      - KAFKA_CFG_NODE_ID=0
      - KAFKA_CFG_PROCESS_ROLES=controller,broker
      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093
      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
      - ALLOW_PLAINTEXT_LISTENER=YES
    volumes:
      - ./data/kafka:/bitnami/kafka
    networks:
      - bigdata
  db:
    image: mysql:5.7 #mysql版本
    container_name: mysql
    hostname: mysql
    volumes:
      - ./data/mysql/db:/var/lib/mysql
      - ./conf/mysql/my.cnf:/etc/mysql/my.cnf
    restart: always
    ports:
      - 13306:3306
    environment:
      MYSQL_ROOT_PASSWORD: 123456 #访问密码
      secure_file_priv:
    healthcheck:
      test: ["CMD-SHELL", "mysql -hlocalhost -uroot -p123456 -e 'create database if not exists hive'"]
      interval: 10s
      timeout: 5s
      retries: 3
    networks:
      - bigdata
  canal:
    image: canal:2.0
    container_name: canal
    hostname: canal
    stdin_open: true
    tty: true
    command: bash
    depends_on:
      - kafka
      - db
    volumes:
      - ./logs:/opt/canal/logs
    ports:
      - "11111:11111"
    networks:
      - bigdata
  master1:
    image: hadoop:2.0
    stdin_open: true
    tty: true
    command: bash
    hostname: master1
    volumes:
      - ./data/hdfs/master1:/app/hdfs
    ports:
      - "9000:9000"
      - "9870:9870"
      - "8088:8088"
    restart: always
    depends_on:
      - db
      - master2
      - worker1
      - worker2
      - worker3
    healthcheck:
      test: ["CMD", "curl localhost:9870/jmx"]
      interval: 30s
      timeout: 5s
      retries: 5
      start_period: 30s
    networks:
      - bigdata
  master2:
    image: hadoop:2.0
    stdin_open: true
    tty: true
    command: bash
    volumes:
      - ./data/hdfs/master2:/app/hdfs
    restart: always
    networks:
      - bigdata
  worker1:
    image: hadoop:2.0
    stdin_open: true
    tty: true
    command: bash
    volumes:
      - ./data/hdfs/worker1:/app/hdfs
    restart: always
    networks:
      - bigdata
  worker2:
    image: hadoop:2.0
    stdin_open: true
    tty: true
    command: bash
    volumes:
      - ./data/hdfs/worker2:/app/hdfs
    restart: always
    networks:
      - bigdata
  worker3:
    image: hadoop:2.0
    stdin_open: true
    tty: true
    command: bash
    volumes:
      - ./data/hdfs/worker3:/app/hdfs
    restart: always
    networks:
      - bigdata
  hive:
    image: hadoop:2.0
    stdin_open: true
    tty: true
    command: bash
    hostname: hive
    networks:
      - bigdata
    ports:
      - "9083:9083"
      - "10000:10000"
    depends_on:
      - db
      - master1
  spark:
    image: spark:2.0
    stdin_open: true
    tty: true
    command: bash
    ports:
      - "10086:10086"
      - "10085:10085"
    depends_on:
      - hive
    healthcheck:
      test: ["CMD-SHELL", "/app/hive/bin/beeline -u jdbc:hive2://localhost:10086 -n root -e 'create database if not exists test'"]
      interval: 10s
      timeout: 5s
      retries: 3
      start_period: 30s
    networks:
      - bigdata
  flink:
    image: flink:2.0
    stdin_open: true
    tty: true
    command: bash
    ports:
      - "8081:8081"
    depends_on:
      - spark
    networks:
      - bigdata
  # zeppelin:
  #   image: hadoop:2.0
  #   stdin_open: true
  #   tty: true
  #   command: bash
  #   ports:
  #     - "8080:8080"
  #   depends_on:
  #     - hive
  #   volumes:
  #     - ./conf/zeppelin:/app/zeppelin/conf
  #     - ./data/zeppelin/local-repo:/app/zeppelin/local-repo
  #     - ./data/zeppelin/notebook:/app/zeppelin/notebook
    # healthcheck:
    #   test: ["CMD-SHELL", "curl localhost:8080"]
    #   interval: 10s
    #   timeout: 5s
    #   retries: 3
    #   start_period: 30s
networks:
  bigdata:
    driver: bridge
    name: bigdata