VOL-582: Update the Voltha core to support either consul or etcd in a kubernetes cluster
The voltha core is refactored such that it interworks with either consul or etcd.
The refactoring should not have changed any of voltha's behaviour with consul. In
cluster mode the update was tested with only the vcli, envoy, vcore, and KV-store
(consul or etcd) containers:
- executed itests on a single-node system
- tested with consul on a kubernetes 3-node cluster
- tested with etcd on a kubernetes 3-node cluster
- tested with consul on a 3-node docker swarm
- container and node failure testing not performed
I believe Consul is still required in single-node operation because the combination of
consul and registrator is used to provide a DNS-like service. This service would have
to be migrated to etcd.
This update enables some experimentation of voltha in the kubernetes environment.
There is more work to be done. Future updates could:
- replace the current polling of etcd keys with an asynchronous technique
Change-Id: Ieea5b34acd9bcf70b760a4d7d2be9b1f88026cd5
diff --git a/compose/docker-compose-system-test-with-etcd.yml b/compose/docker-compose-system-test-with-etcd.yml
new file mode 100644
index 0000000..a02a9cf
--- /dev/null
+++ b/compose/docker-compose-system-test-with-etcd.yml
@@ -0,0 +1,340 @@
+version: '2'
+services:
+ #
+ # Single-node zookeeper service
+ #
+ zookeeper:
+ image: wurstmeister/zookeeper
+ ports:
+ - 2181
+ environment:
+ SERVICE_2181_NAME: "zookeeper"
+ #
+ # Single-node kafka service
+ #
+ kafka:
+ image: wurstmeister/kafka
+ ports:
+ - 9092
+ environment:
+ KAFKA_ADVERTISED_HOST_NAME: ${DOCKER_HOST_IP}
+ KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+ KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+ KAFKA_HEAP_OPTS: "-Xmx256M -Xms128M"
+ SERVICE_9092_NAME: "kafka"
+ depends_on:
+ - consul
+ volumes:
+ - /var/run/docker.sock:/var/run/docker.sock
+ #
+ # Single-node consul agent
+ #
+ consul:
+ image: consul:latest
+ command: agent -server -bootstrap -client 0.0.0.0 -ui
+ ports:
+ - "8300:8300"
+ - "8400:8400"
+ - "8500:8500"
+ - "8600:8600/udp"
+ environment:
+ #SERVICE_53_IGNORE: "yes"
+ SERVICE_8300_IGNORE: "yes"
+ SERVICE_8400_IGNORE: "yes"
+ SERVICE_8500_NAME: "consul-rest"
+ #
+ # Single-node etcd server
+ #
+ etcd:
+ image: quay.io/coreos/etcd:v3.2.9
+ command: [
+ "etcd",
+ "--name=etcd0",
+ "--advertise-client-urls=http://${DOCKER_HOST_IP}:2379,http://${DOCKER_HOST_IP}:4001",
+ "--listen-client-urls=http://0.0.0.0:2379,http://0.0.0.0:4001",
+ "--initial-advertise-peer-urls=http://${DOCKER_HOST_IP}:2380",
+ "--listen-peer-urls=http://0.0.0.0:2380",
+ "--initial-cluster-token=etcd-cluster-1",
+ "--initial-cluster=etcd0=http://${DOCKER_HOST_IP}:2380",
+ "--initial-cluster-state=new"
+ ]
+ ports:
+ - "2379:2379"
+ - 2380
+ - 4001
+ #
+ # Registrator
+ #
+ registrator:
+ image: gliderlabs/registrator:latest
+ command: [
+ "-ip=${DOCKER_HOST_IP}",
+ "-retry-attempts", "100",
+ "-cleanup",
+ # "-internal",
+ "consul://consul:8500"
+ ]
+ links:
+ - consul
+ volumes:
+ - "/var/run/docker.sock:/tmp/docker.sock"
+
+ #
+ # Fluentd log server
+ #
+ fluentd:
+ image: fluent/fluentd
+ ports:
+ - "24224:24224"
+ volumes:
+ - "/tmp/fluentd:/fluentd/log"
+ environment:
+ SERVICE_24224_NAME: "fluentd-intake"
+
+ #
+ # Graphite-Grafana-statsd service instance
+ # (demo place-holder for external KPI system)
+ #
+ grafana:
+ image: voltha/grafana
+ ports:
+ - "8883:80"
+ - "2003:2003"
+ - "2004:2004"
+ - "8126:8126"
+ - "8125:8125/udp"
+ environment:
+ SERVICE_80_NAME: "grafana-web-ui"
+ SERVICE_2003_NAME: "carbon-plain-text-intake"
+ SERVICE_2004_NAME: "carbon-pickle-intake"
+ SERVICE_8126_NAME: "statsd-tcp-intake"
+ SERVICE_8125_NAME: "statsd-udp-intake"
+ GR_SERVER_ROOT_URL: "http://localhost:80/grafana/"
+
+ #
+ # Shovel (Kafka-graphite-gateway)
+ #
+ shovel:
+ image: cord/shovel
+ command: [
+ "/shovel/shovel/main.py",
+ "--kafka=@kafka",
+ "--consul=${DOCKER_HOST_IP}:8500",
+ "--topic=voltha.kpis",
+ "--host=${DOCKER_HOST_IP}"
+ ]
+ depends_on:
+ - consul
+ - kafka
+ - grafana
+ restart: unless-stopped
+
+ #
+ # Voltha server instance(s)
+ #
+ voltha:
+ image: cord/voltha
+ command: [
+ "/voltha/voltha/main.py",
+ "-v",
+ "--consul=${DOCKER_HOST_IP}:8500",
+ "--etcd=${DOCKER_HOST_IP}:2379",
+ "--fluentd=fluentd:24224",
+ "--rest-port=8880",
+ "--grpc-port=50556",
+ "--kafka=@kafka",
+ "--instance-id-is-container-name",
+ "--interface=eth1",
+ "--backend=etcd",
+ "-v"
+ ]
+ ports:
+ - 8880
+ - 50556
+ - 18880
+ - "60001:60001"
+ depends_on:
+ - consul
+ - etcd
+ links:
+ - consul
+ - etcd
+ - fluentd
+ environment:
+ SERVICE_8880_NAME: "voltha-health"
+ SERVICE_8880_CHECK_HTTP: "/health"
+ SERVICE_8880_CHECK_INTERVAL: "5s"
+ SERVICE_8880_CHECK_TIMEOUT: "1s"
+ SERVICE_18880_NAME: "voltha-sim-rest"
+ SERVICE_HOST_IP: "${DOCKER_HOST_IP}"
+ volumes:
+ - "/var/run/docker.sock:/tmp/docker.sock"
+ networks:
+ - default
+ - ponmgmt
+
+ envoy:
+ image: voltha/envoy
+ entrypoint:
+ - /usr/local/bin/envoyd
+ - -envoy-cfg-template
+ - "/envoy/voltha-grpc-proxy.template.json"
+ - -envoy-config
+ - "/envoy/voltha-grpc-proxy.json"
+ - -kv
+ - "etcd"
+ - -kv-svc-name
+ - "etcd"
+ - -kv-port
+ - "2379"
+
+
+ ports:
+ - "50555:50555"
+ - "8882:8882"
+ - "8443:8443"
+ - "8001:8001"
+ environment:
+ SERVICE_50555_NAME: "voltha-grpc"
+ volumes:
+ - "/var/run/docker.sock:/tmp/docker.sock"
+ networks:
+ - default
+ - ponmgmt
+ links:
+ - voltha:vcore
+ #
+ # Voltha cli container
+ #
+ vcli:
+ image: cord/vcli
+ command: [
+ "/cli/cli/setup.sh",
+ "-L",
+ "-G"
+ ]
+ environment:
+ DOCKER_HOST_IP: "${DOCKER_HOST_IP}"
+ ports:
+ - "5022:22"
+ depends_on:
+ - voltha
+
+#############################################
+# Item below this line will soon be removed.#
+#############################################
+
+ #
+ # ofagent server instance
+ #
+ ofagent:
+ image: cord/ofagent
+ command: [
+ "/ofagent/ofagent/main.py",
+ "-v",
+ "--consul=${DOCKER_HOST_IP}:8500",
+ "--fluentd=fluentd:24224",
+ "--controller=${DOCKER_HOST_IP}:6653",
+ "--grpc-endpoint=@voltha-grpc",
+ "--instance-id-is-container-name",
+ "--enable-tls",
+ "--key-file=/ofagent/pki/voltha.key",
+ "--cert-file=/ofagent/pki/voltha.crt",
+ "-v"
+ ]
+ depends_on:
+ - consul
+ - voltha
+ links:
+ - consul
+ - fluentd
+ volumes:
+ - "/var/run/docker.sock:/tmp/docker.sock"
+ restart: unless-stopped
+
+ #
+ # Netconf server instance(s)
+ #
+ netconf:
+ image: cord/netconf
+ privileged: true
+ command: [
+ "/netconf/netconf/main.py",
+ "-v",
+ "--consul=${DOCKER_HOST_IP}:8500",
+ "--fluentd=fluentd:24224",
+ "--grpc-endpoint=@voltha-grpc",
+ "--instance-id-is-container-name",
+ "-v"
+ ]
+ ports:
+ - "830:1830"
+ depends_on:
+ - consul
+ - voltha
+ links:
+ - consul
+ - fluentd
+ environment:
+ SERVICE_1830_NAME: "netconf-server"
+ volumes:
+ - "/var/run/docker.sock:/tmp/docker.sock"
+
+ #
+ # Dashboard daemon
+ #
+ dashd:
+ image: cord/dashd
+ command: [
+ "/dashd/dashd/main.py",
+ "--kafka=@kafka",
+ "--consul=${DOCKER_HOST_IP}:8500",
+ "--grafana_url=http://admin:admin@${DOCKER_HOST_IP}:8883/api",
+ "--topic=voltha.kpis",
+ "--docker_host=${DOCKER_HOST_IP}"
+ ]
+ depends_on:
+ - consul
+ - kafka
+ - grafana
+ restart: unless-stopped
+
+ #
+ # Nginx service consolidation
+ #
+ nginx:
+ image: voltha/nginx
+ ports:
+ - "80:80"
+ environment:
+ CONSUL_ADDR: "${DOCKER_HOST_IP}:8500"
+ command: [
+ "/nginx_config/start_service.sh"
+ ]
+ depends_on:
+ - consul
+ - grafana
+ - portainer
+ restart: unless-stopped
+
+ #
+ # Docker ui
+ #
+ portainer:
+ image: voltha/portainer
+ ports:
+ - "9000:9000"
+ environment:
+ CONSUL_ADDR: "${DOCKER_HOST_IP}:8500"
+ restart: unless-stopped
+ entrypoint: ["/portainer", "--logo", "/docker/images/logo_alt.png"]
+ volumes:
+ - "/var/run/docker.sock:/var/run/docker.sock"
+
+networks:
+ default:
+ driver: bridge
+ ponmgmt:
+ driver: bridge
+ driver_opts:
+ com.docker.network.bridge.name: "ponmgmt"