compose.yaml

---
version: '3.7'

# networks:
#   net-ksql-con:
#     driver: bridge

services:
  postgres:
    image: debezium/postgres:13-alpine
    # image: postgres:13-alpine
    hostname: postgres
    container_name: postgres
    ports:
      - "5432:5432"
    environment:
      POSTGRES_USER: postgres_u
      POSTGRES_PASSWORD: postpass
      POSTGRES_DB: db_postgres
    volumes:
      - ./postgres/custom-config.conf:/etc/postgresql/postgresql.conf
    command: postgres -c config_file=/etc/postgresql/postgresql.conf

  zookeeper:
    image: confluentinc/cp-zookeeper:7.3.0
    # image: debezium/zookeeper
    hostname: zookeeper
    container_name: zookeeper
    ports:
      - "2181:2181"
    environment:
      ALLOW_ANONYMOUS_LOGIN: yes
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000

  broker:
    image: confluentinc/cp-kafka:7.3.0
    # image: debezium/kafka
    hostname: broker
    container_name: broker
    depends_on:
      - zookeeper
    # networks:
    #   - net-ksql-con
    ports:
      - "29092:29092"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:9092,PLAINTEXT_HOST://localhost:29092
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
    command:
      - bash
      - -c 
      - |
        echo '127.0.0.1 broker' >> /etc/hosts
        /etc/confluent/docker/run
        sleep infinity

  schema-registry:
    image: confluentinc/cp-schema-registry:7.3.0
    container_name: schema-registry
    depends_on:
      - zookeeper
      - broker
    # networks:
    #   - net-ksql-con
    ports:
      - "8081:8081"
    environment:
      SCHEMA_REGISTRY_HOST_NAME: schema-registry
      SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper:2181
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: "PLAINTEXT://broker:9092"
  
  kafka-connect:
    image: confluentinc/cp-kafka-connect:7.3.0
    # image: debezium/connect:latest
    container_name: kafka-connect
    depends_on:
      - broker
      - schema-registry
    # networks:
    #   - net-ksql-con
    ports:
      - "8083:8083"
    environment:
      # CONNECT_ADVERTISED_HOST_NAME: "$(echo $DOCKER_HOST | cut -f3 -d’/’ | cut -f1 -d’:’)"
      CONNECT_BOOTSTRAP_SERVERS: "broker:9092"
      CONNECT_REST_ADVERTISED_HOST_NAME: "kafka-connect"
      CONNECT_REST_PORT: 8083
      CONNECT_GROUP_ID: "kafka-connect"
      CONNECT_CONFIG_STORAGE_TOPIC: "_connect-configs"
      CONNECT_OFFSET_STORAGE_TOPIC: "_connect-offsets"
      CONNECT_STATUS_STORAGE_TOPIC: "_connect-status"
      CONNECT_KEY_CONVERTER: "org.apache.kafka.connect.storage.StringConverter"
      CONNECT_VALUE_CONVERTER: "io.confluent.connect.avro.AvroConverter"
      # CONNECT_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      # CONNECT_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081'
      CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.storage.StringConverter"
      CONNECT_INTERNAL_VALUE_CONVERTER: "io.confluent.connect.avro.AvroConverter"
      CONNECT_LOG4J_ROOT_LOGLEVEL: "INFO"
      CONNECT_LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR"
      CONNECT_LOG4J_APPENDER_STDOUT_LAYOUT_CONVERSIONPATTERN: "[%d] %p %X{connector.context}%m (%c:%L)%n"
      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_PLUGIN_PATH: /usr/share/java,/usr/share/confluent-hub-components,/data/connect-jars
    # volumes: # this volume must be active if you want to provide the connectors via local storage
    #   - $PWD/data:/data
    command:
      - bash
      - -c
      - |
        echo '127.0.0.1 kafka-connect' >> /etc/hosts
        echo "Installing Connector"
        confluent-hub install --no-prompt debezium/debezium-connector-postgresql:1.9.7
        #
        echo "Launching Kafka Connect worker"
        /etc/confluent/docker/run &
        #
        sleep infinity

  ksqldb-server:
    image: confluentinc/cp-ksqldb-server:7.3.0
    hostname: ksqldb-server
    container_name: ksqldb-server
    # networks:
    #   - net-ksql-con
    depends_on:
      - schema-registry
      - broker
      - kafka-connect
    ports:
      - "8088:8088"
    environment:
      KSQL_LISTENERS: "http://0.0.0.0:8088"
      KSQL_BOOTSTRAP_SERVERS: "broker:9092"
      KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: "true"
      KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE: "true"
      KSQL_KSQL_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081'
      KSQL_CONNECT_GROUP_ID: "ksql-connect-cluster"
      KSQL_STREAMS_PRODUCER_MAX_BLOCK_MS: "9223372036854775807"
      KSQL_KSQL_CONNECT_URL: "http://kafka-connect:8083"
      KSQL_CONNECT_REST_ADVERTISED_HOST_NAME: "ksql-connect"
      KSQL_CONNECT_BOOTSTRAP_SERVERS: "broker:9092"
      KSQL_CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "false"
      KSQL_CONNECT_KEY_CONVERTER: "org.apache.kafka.connect.storage.StringConverter"
      KSQL_CONNECT_VALUE_CONVERTER: "io.confluent.connect.avro.AvroConverter"
      # KSQL_CONNECT_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      # KSQL_CONNECT_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      KSQL_CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081'
      KSQL_CONNECT_CONFIG_STORAGE_TOPIC: "_ksql-connect-configs"
      KSQL_CONNECT_OFFSET_STORAGE_TOPIC: "_ksql-connect-offsets"
      KSQL_CONNECT_STATUS_STORAGE_TOPIC: "_ksql-connect-statuses"
      KSQL_CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
      KSQL_CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
      KSQL_CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
      # KSQL_CONNECT_PLUGIN_PATH: /usr/share/java,/usr/share/confluent-hub-components,/data/connect-jars
      # KSQL_CONFIG_DIR: "/etc/ksqldb"
      KSQL_KSQL_SERVICE_ID: "ksql_01"
      KSQL_KSQL_HIDDEN_TOPICS: '^_.*'
      # put inside command: like the kafka-connect if the server also act as kafka-connect
      # confluent-hub install --no-prompt debezium/debezium-connector-postgresql:1.9.7 --component-dir /usr/share/confluent-hub-components

  ksqldb-cli:
    image: confluentinc/cp-ksqldb-cli:7.3.0
    container_name: ksqldb-cli
    # networks:
    #   - net-ksql-con
    depends_on:
      - broker
      - ksqldb-server
    entrypoint: /bin/sh
    tty: true