Fixing build.md tests
Also adding some utility commands in consulhelpers, namely:
- verify_all_services_healthy
- checks all services registered in consul are healthy
- can also check a single service by name
- connect_to_consult
- connects to consul, unsurprisingly.
- get_all_services
- returns all the services
Change-Id: I5aaa4b7f5567fb6d7e8b1a9887bcc7592f9e2847
diff --git a/common/utils/consulhelpers.py b/common/utils/consulhelpers.py
index a828b3f..3efc7b9 100644
--- a/common/utils/consulhelpers.py
+++ b/common/utils/consulhelpers.py
@@ -25,16 +25,65 @@
log = get_logger()
-def get_endpoint_from_consul(consul_endpoint, service_name):
- """Look up, from consul, the service name specified by service-name
- """
- log.debug('getting-service-endpoint', consul=consul_endpoint,
- service=service_name)
+def connect_to_consult(consul_endpoint):
+ log.debug('getting-service-endpoint', consul=consul_endpoint)
host = consul_endpoint.split(':')[0].strip()
port = int(consul_endpoint.split(':')[1].strip())
- consul = Consul(host=host, port=port)
+ return Consul(host=host, port=port)
+
+
+def verify_all_services_healthy(consul_endpoint, service_name=None,
+ number_of_expected_services=None):
+ """
+ Verify in consul if any service is healthy
+ :param consul_endpoint: a <host>:<port> string
+ :param service_name: name of service to check, optional
+ :param number_of_expected_services number of services to check for, optional
+ :return: true if healthy, false otherwise
+ """
+
+ def check_health(service):
+ _, serv_health = consul.health.service(service, passing=True)
+ return not serv_health == []
+
+ consul = connect_to_consult(consul_endpoint)
+
+ if service_name is not None:
+ return check_health(service_name)
+
+ services = get_all_services(consul_endpoint)
+
+ items = services.keys()
+
+ if number_of_expected_services is not None and \
+ len(items) != number_of_expected_services:
+ return False
+
+ for item in items:
+ if not check_health(item):
+ return False
+
+ return True
+
+
+def get_all_services(consul_endpoint):
+
+ log.debug('getting-service-verify-health')
+
+ consul = connect_to_consult(consul_endpoint)
+ _, services = consul.catalog.services()
+
+ return services
+
+
+def get_endpoint_from_consul(consul_endpoint, service_name):
+ """Look up, from consul, the service name specified by service-name
+ """
+ log.debug('getting-service-info', service=service_name)
+
+ consul = connect_to_consult(consul_endpoint)
_, services = consul.catalog.service(service_name)
if len(services) == 0:
diff --git a/tests/itests/docutests/build_md_test.py b/tests/itests/docutests/build_md_test.py
index 22e0f22..1e42a3b 100644
--- a/tests/itests/docutests/build_md_test.py
+++ b/tests/itests/docutests/build_md_test.py
@@ -17,6 +17,7 @@
import subprocess
import time
import logging
+from common.utils.consulhelpers import verify_all_services_healthy
import os
import json
from unittest import TestCase
@@ -32,7 +33,8 @@
log = logging.getLogger(__name__)
-LOCAL_CONSUL_URL = "http://localhost:8500"
+LOCAL_CONSUL = "localhost:8500"
+LOCAL_CONSUL_URL = "http://%s" % LOCAL_CONSUL
LOCAL_CONSUL_DNS = "@localhost -p 8600"
DOCKER_COMPOSE_FILE = "compose/docker-compose-system-test.yml"
DOCKER_COMPOSE_FILE_SERVICES_COUNT = 7
@@ -370,10 +372,11 @@
self.assertEqual(rc, 0)
print "Waiting for all containers to be ready ..."
- rc, not_found_list = self._wait_for_all_containers_to_ready()
- if rc:
- print "Not found patterns:{}".format(not_found_list)
- self.assertEqual(rc, 0)
+ time.sleep(10)
+ rc = verify_all_services_healthy(LOCAL_CONSUL)
+ if not rc:
+ print "Not all services are up"
+ self.assertEqual(rc, True)
# verify that all containers are running
print "Verify all services are running using docker command ..."
@@ -541,10 +544,10 @@
print "Waiting for all containers to be ready ..."
time.sleep(10)
- rc, not_found_list = self._wait_for_all_containers_to_ready()
- if rc:
- print "Not found patterns:{}".format(not_found_list)
- self.assertEqual(rc, 0)
+ rc = verify_all_services_healthy(LOCAL_CONSUL)
+ if not rc:
+ print "Not all services are up"
+ self.assertEqual(rc, True)
# Get the IP address(es) for voltha's REST interface
print "Get IP of Voltha REST interface..."
@@ -689,99 +692,3 @@
return -1 # consul should have come up by this time
else:
time.sleep(2) # constant sleep for testing
-
- def _wait_for_all_containers_to_ready(self):
- # After the containers have been started using docker-compose, look
- # at the logs for the following patterns to decide if the containers
- # are up and in sync:
- #
- # For registrator, look for
- # "(.*)registrator_1(.*)Listening for Docker events"
- #
- # For voltha, zookeeper and kafka look for these patterns
- # "(.*)voltha_1(.*)main.heartbeat {status: up, uptime:"
- # "(.*)voltha_1(.*)kafka_proxy.send_message {event: Successfully sent message Heartbeat message"
- #
- # For fluentd, look for
- # "(.*)fluentd_1(.*)listening fluent socket on"
- #
- # For chameleon, look for
- # "(.*)chameleon_1(.*)main.startup_components {event:
- # started-internal-services, instance_id: compose_chameleon_1}"
- #
- # For consul, look for
- # "(.*)consul_1(.*)agent: Synced service(.*)consul(.*)8500"
- # "(.*)consul_1(.*)agent: Synced service(.*)consul(.*)8600:udp"
- # "(.*)consul_1(.*)agent: Synced service(.*)fluentd"
- # "(.*)consul_1(.*)agent: Synced service(.*)voltha"
- # "(.*)consul_1(.*)agent: Synced service(.*)voltha(.*):8880"
- # "(.*)consul_1(.*)agent: Synced service(.*)chameleon(.*):8881"
- # "(.*)consul_1(.*)agent: Synced service(.*)zookeeper(.*):2181"
- # "(.*)consul_1(.*)agent: Synced service(.*)kafka(.*):9092"
- #
- expected_output = [
- "(.*)registrator_1(.*)Listening for Docker events",
- "(.*)voltha_1(.*)main.heartbeat {status: up, uptime:",
- "(.*)voltha_1(.*)kafka_proxy.send_message(.*)event: "
- "sent-kafka-msg",
- "(.*)fluentd_1(.*)listening fluent socket on",
- "(.*)chameleon_1(.*)main.startup_components {event: "
- "started-internal-services, instance_id: compose_chameleon_1}",
- "(.*)consul_1(.*)agent: Synced service(.*)consul(.*)8500",
- "(.*)consul_1(.*)agent: Synced service(.*)consul(.*)8600:udp",
- "(.*)consul_1(.*)agent: Synced service(.*)fluentd",
- "(.*)consul_1(.*)agent: Synced service(.*)voltha(.*):(?!8880)",
- "(.*)consul_1(.*)agent: Synced service(.*)voltha(.*):8880",
- "(.*)consul_1(.*)agent: Synced service(.*)chameleon(.*):8881",
- "(.*)consul_1(.*)agent: Synced service(.*)zookeeper(.*):2181",
- "(.*)consul_1(.*)agent: Synced service(.*)kafka(.*):9092"
- ]
- pattern_found = []
- max_wait_time = 60 # wait up to 1 minute before declaring a failure
-
- def _stop_process(proc):
- try:
- proc.terminate()
- proc.wait()
- # In principle this 'reset' should not be required.
- # However, without it, the terminal is left in a funny
- # state and required
- subprocess.Popen(['reset']).wait()
- except Exception as e:
- print "Received exception {} when killing process " \
- .format(repr(e), )
-
- try:
- t0 = time.time()
- env = os.environ.copy()
- proc = subprocess.Popen(
- command_defs['docker_compose_logs'],
- env=env,
- shell=True,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- bufsize=1
- )
- for line in iter(proc.stdout.readline, b''):
- ansi_escape = re.compile(r'\x1b[^m]*m')
- line = ansi_escape.sub('', line)
- for pattern in expected_output:
- if re.match(pattern, line, re.I):
- if pattern not in pattern_found:
- pattern_found.append(pattern)
- break
- # Check if we found all patterns yet
- if len(pattern_found) == len(expected_output):
- _stop_process(proc)
- return 0, [] # success
- elif time.time() - t0 > max_wait_time:
- _stop_process(proc)
- not_found = [p for p in expected_output if p not in
- pattern_found]
- return -1, not_found # failure
- else:
- return -1, []
-
- except Exception as e:
- print 'Exception {} '.format(repr(e))
- return -1, []