compose/docker-compose-kafka-cluster.yml

###########################################################################################
#
# Deploy a Zookeeper/Kafka messaging cluster
#
# It is meant to be deployed within a swarm cluster using the docker stack functionality.
# This file will deploy 3 zookeeper containers and 1 kafka container per swarm node.
#
# This file will also create an overlay network which can be referred by other services.
#
# Example:
#
# docker stack deploy -c docker-compose-kafka-ha-cluster.yml kafka
#
###########################################################################################

version: '3'
services:
    zk1:
        image: "${REGISTRY}wurstmeister/zookeeper:${ZK_TAG:-latest}"
        logging:
            driver: "json-file"
            options:
               max-size: "10m"
               max-file: 3
        networks:
            - kafka-net
        environment:
            ZOO_MY_ID: 1
            ZOO_SERVERS: server.1=0.0.0.0:2888:3888 server.2=zk2:2888:3888 server.3=zk3:2888:3888

    zk2:
        image: "${REGISTRY}wurstmeister/zookeeper:${ZK_TAG:-latest}"
        logging:
            driver: "json-file"
            options:
                max-size: "10m"
                max-file: 3
        networks:
            - kafka-net
        environment:
            ZOO_MY_ID: 2
            ZOO_SERVERS: server.1=zk1:2888:3888 server.2=0.0.0.0:2888:3888 server.3=zk3:2888:3888

    zk3:
        image: "${REGISTRY}wurstmeister/zookeeper:${ZK_TAG:-latest}"
        logging:
            driver: "json-file"
            options:
                max-size: "10m"
                max-file: 3
        networks:
            - kafka-net
        environment:
            ZOO_MY_ID: 3
            ZOO_SERVERS: server.1=zk1:2888:3888 server.2=zk2:2888:3888 server.3=0.0.0.0:2888:3888

    kafka:
        deploy:
            mode: global
        image: "${REGISTRY}wurstmeister/kafka:${KAFKA_TAG:-latest}"
        logging:
            driver: "json-file"
            options:
                max-size: "10m"
                max-file: 3
        environment:
            KAFKA_LISTENERS: PLAINTEXT://:9092
            KAFKA_ZOOKEEPER_CONNECT: zk1:2181,zk2:2181,zk3:2181
            KAFKA_HEAP_OPTS: -Xmx256M -Xms128M
        volumes:
            - /var/run/docker.sock:/var/run/docker.sock
        networks:
            - kafka-net
            - voltha-net
        ports:
            - 9092:9092

networks:
    kafka-net:
        external:
            name: kafka_net

    voltha-net:
        external:
            name: voltha_net