Updated dashd to automatically create a "Voltha Stats" data source in
Grafana to simplify use of Grafana as a performance management graphing
tool. Users of dashd no longer need to add the data source manually.
Also added a compose file that uses fixed ports which is mainly used
for developent.
Change-Id: I07927f27608c0b6601f266d2bac138844840e8f9
diff --git a/compose/docker-compose-fixed-port.yml b/compose/docker-compose-fixed-port.yml
new file mode 100644
index 0000000..9f816ca
--- /dev/null
+++ b/compose/docker-compose-fixed-port.yml
@@ -0,0 +1,258 @@
+version: '2'
+services:
+ #
+ # Single-node zookeeper service
+ #
+ zookeeper:
+ image: wurstmeister/zookeeper
+ ports:
+ - "2181:2181"
+ environment:
+ SERVICE_2181_NAME: "zookeeper"
+ #
+ # Single-node kafka service
+ #
+ kafka:
+ image: wurstmeister/kafka
+ ports:
+ - "9092:9092"
+ environment:
+ KAFKA_ADVERTISED_HOST_NAME: ${DOCKER_HOST_IP}
+ KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+ KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+ KAFKA_HEAP_OPTS: "-Xmx256M -Xms128M"
+ SERVICE_9092_NAME: "kafka"
+ depends_on:
+ - consul
+ volumes:
+ - /var/run/docker.sock:/var/run/docker.sock
+ #
+ # Single-node consul agent
+ #
+ consul:
+ image: consul:latest
+ command: agent -server -bootstrap -client 0.0.0.0 -ui
+ ports:
+ - "8300:8300"
+ - "8400:8400"
+ - "8500:8500"
+ - "8600:8600/udp"
+ environment:
+ #SERVICE_53_IGNORE: "yes"
+ SERVICE_8300_IGNORE: "yes"
+ SERVICE_8400_IGNORE: "yes"
+ SERVICE_8500_NAME: "consul-rest"
+ #
+ # Registrator
+ #
+ registrator:
+ image: gliderlabs/registrator:latest
+ command: [
+ "-ip=${DOCKER_HOST_IP}",
+ "-retry-attempts", "100",
+ # "-internal",
+ "consul://consul:8500"
+ ]
+ links:
+ - consul
+ volumes:
+ - "/var/run/docker.sock:/tmp/docker.sock"
+
+ #
+ # Fluentd log server
+ #
+ fluentd:
+ image: fluent/fluentd
+ ports:
+ - "24224:24224"
+ volumes:
+ - "/tmp/fluentd:/fluentd/log"
+ environment:
+ SERVICE_24224_NAME: "fluentd-intake"
+
+ #
+ # Graphite-Grafana-statsd service instance
+ # (demo place-holder for external KPI system)
+ #
+ grafana:
+ image: kamon/grafana_graphite
+ ports:
+ - "8882:80"
+ - "2003:2003"
+ - "2004:2004"
+ - "8126:8126"
+ - "8125:8125/udp"
+ environment:
+ SERVICE_80_NAME: "grafana-web-ui"
+ SERVICE_2003_NAME: "carbon-plain-text-intake"
+ SERVICE_2004_NAME: "carbon-pickle-intake"
+ SERVICE_8126_NAME: "statsd-tcp-intake"
+ SERVICE_8125_NAME: "statsd-udp-intake"
+
+ #
+ # Shovel (Kafka-graphite-gateway)
+ #
+ shovel:
+ image: cord/shovel
+ command: [
+ "/shovel/shovel/main.py",
+ "--kafka=@kafka",
+ "--consul=${DOCKER_HOST_IP}:8500",
+ "--topic=voltha.kpis",
+ "--host=${DOCKER_HOST_IP}"
+ ]
+ depends_on:
+ - consul
+ - kafka
+ - grafana
+ restart: unless-stopped
+
+ #
+ # Voltha server instance(s)
+ #
+ voltha:
+ image: cord/voltha
+ command: [
+ "/voltha/voltha/main.py",
+ "-v",
+ "--consul=${DOCKER_HOST_IP}:8500",
+ "--fluentd=fluentd:24224",
+ "--rest-port=8880",
+ "--grpc-port=50555",
+ "--kafka=@kafka",
+ "--instance-id-is-container-name",
+ "--interface=eth1",
+ "-v"
+ ]
+ ports:
+ - 8880
+ - 50555
+ - 18880
+ depends_on:
+ - consul
+ links:
+ - consul
+ - fluentd
+ environment:
+ SERVICE_8880_NAME: "voltha-health"
+ SERVICE_8880_CHECK_HTTP: "/health"
+ SERVICE_8880_CHECK_INTERVAL: "5s"
+ SERVICE_8880_CHECK_TIMEOUT: "1s"
+ SERVICE_50555_NAME: "voltha-grpc"
+ SERVICE_18880_NAME: "voltha-sim-rest"
+ volumes:
+ - "/var/run/docker.sock:/tmp/docker.sock"
+ networks:
+ - default
+ - ponmgmt
+
+#############################################
+# Item below this line will soon be removed.#
+#############################################
+
+ #
+ # Chameleon server instance(s)
+ #
+ chameleon:
+ image: cord/chameleon
+ command: [
+ "/chameleon/chameleon/main.py",
+ "-v",
+ "--consul=consul:8500",
+ "--fluentd=fluentd:24224",
+ "--rest-port=8881",
+ "--grpc-endpoint=@voltha-grpc",
+ "--instance-id-is-container-name",
+ "-v"
+ ]
+ ports:
+ - "33533:8881"
+ depends_on:
+ - consul
+ - voltha
+ links:
+ - consul
+ - fluentd
+ environment:
+ SERVICE_8881_NAME: "chameleon-rest"
+ volumes:
+ - "/var/run/docker.sock:/tmp/docker.sock"
+ #
+ # ofagent server instance
+ #
+ ofagent:
+ image: cord/ofagent
+ command: [
+ "/ofagent/ofagent/main.py",
+ "-v",
+ "--consul=${DOCKER_HOST_IP}:8500",
+ "--fluentd=fluentd:24224",
+ "--controller=${DOCKER_HOST_IP}:6653",
+ "--grpc-endpoint=@voltha-grpc",
+ "--instance-id-is-container-name",
+ "-v"
+ ]
+ depends_on:
+ - consul
+ - voltha
+ links:
+ - consul
+ - fluentd
+ volumes:
+ - "/var/run/docker.sock:/tmp/docker.sock"
+ restart: unless-stopped
+
+ #
+ # Netconf server instance(s)
+ #
+ netconf:
+ image: cord/netconf
+ privileged: true
+ command: [
+ "/netconf/netconf/main.py",
+ "-v",
+ "--consul=${DOCKER_HOST_IP}:8500",
+ "--fluentd=fluentd:24224",
+ "--grpc-endpoint=@voltha-grpc",
+ "--instance-id-is-container-name",
+ "-v"
+ ]
+ ports:
+ - "830:1830"
+ depends_on:
+ - consul
+ - voltha
+ links:
+ - consul
+ - fluentd
+ environment:
+ SERVICE_1830_NAME: "netconf-server"
+ volumes:
+ - "/var/run/docker.sock:/tmp/docker.sock"
+
+ #
+ # Dashboard daemon
+ #
+ dashd:
+ image: cord/dashd
+ command: [
+ "/dashd/dashd/main.py",
+ "--kafka=@kafka",
+ "--consul=${DOCKER_HOST_IP}:8500",
+ "--grafana_url=http://admin:admin@${DOCKER_HOST_IP}:8882/api",
+ "--topic=voltha.kpis",
+ "--docker_host=${DOCKER_HOST_IP}"
+ ]
+ depends_on:
+ - consul
+ - kafka
+ - grafana
+ restart: unless-stopped
+
+networks:
+ default:
+ driver: bridge
+ ponmgmt:
+ driver: bridge
+ driver_opts:
+ com.docker.network.bridge.name: "ponmgmt"
diff --git a/dashd/dashd_impl.py b/dashd/dashd_impl.py
index 2be471a..6d1ef2f 100755
--- a/dashd/dashd_impl.py
+++ b/dashd/dashd_impl.py
@@ -79,6 +79,7 @@
import json
import re
import sys
+import time
from dashd.dash_template import DashTemplate
log = get_logger()
@@ -97,9 +98,16 @@
self.topic = topic
self.dash_template = DashTemplate(grafana_url)
self.grafana_url = grafana_url
- self.kafka_endpoint = get_endpoint_from_consul(consul_endpoint,
- 'kafka')
- # print('kafka endpoint: ', self.kafka_endpoint)
+ self.kafka_endpoint = None
+ self.consul_endpoint = consul_endpoint
+ while True:
+ try:
+ self.kafka_endpoint = get_endpoint_from_consul(self.consul_endpoint,
+ 'kafka')
+ break
+ except:
+ log.error("unable-to-communicate-with-consul")
+ time.sleep(10)
self.on_start_callback = None
self._client = KafkaClient(self.kafka_endpoint)
@@ -118,12 +126,15 @@
try:
while not partitions:
yield self._client.load_metadata_for_topics(self.topic)
+ #self._client.load_metadata_for_topics(self.topic)
e = self._client.metadata_error_for_topic(self.topic)
if e:
log.warning('no-metadata-for-topic', error=e,
topic=self.topic)
else:
partitions = self._client.topic_partitions[self.topic]
+ break
+ time.sleep(20)
except KafkaUnavailableError:
log.error("unable-to-communicate-with-Kafka-brokers")
self.stop()
@@ -147,7 +158,30 @@
# they'll be deleted. If they are valid then they'll persist.
#print("Starting main loop")
try:
- r = requests.get(self.grafana_url + "/search?")
+ while True:
+ r = requests.get(self.grafana_url + "/datasources")
+ if r.status_code == requests.codes.ok:
+ break
+ else:
+ time.sleep(10)
+ j = r.json()
+ data_source = False
+ for i in j:
+ if i["name"] == "Voltha Stats":
+ data_source = True
+ break
+ if not data_source:
+ r = requests.post(self.grafana_url + "/datasources",
+ data = {"name":"Voltha Stats","type":"graphite",
+ "access":"proxy","url":"http://localhost:81"})
+ log.info('data-source-added',status=r.status_code, text=r.text)
+
+ while True:
+ r = requests.get(self.grafana_url + "/search?")
+ if r.status_code == requests.codes.ok:
+ break
+ else:
+ time.sleep(10)
j = r.json()
for i in j:
# Look for dashboards that have a title of *olt.[[:hexidgit:]].