Adding a wait for containers condition
Change-Id: I4601b9f9e2f17ccfaaa1fcc6c24eaf9e13dd3b38
diff --git a/Makefile b/Makefile
index c4855cc..865c255 100644
--- a/Makefile
+++ b/Makefile
@@ -73,8 +73,7 @@
@echo "utest : Run all unit tests"
@echo
-build: protos docker-base
-#build: utest protos docker-base
+build: utest protos docker-base
docker build -t cord/voltha -f Dockerfile.voltha .
docker build -t cord/chameleon -f Dockerfile.chameleon .
@@ -132,7 +131,7 @@
itest: venv
@ echo "Executing all integration tests"
. ${VENVDIR}/bin/activate && \
- nosetests tests/itests -s --exclude-dir=./tests/utests/
+ nosetests tests/itests -s
flake8: $(DIRS_FLAKE8)
diff --git a/tests/itests/docutests/build_md_test.py b/tests/itests/docutests/build_md_test.py
index 562dc8e..b75da23 100644
--- a/tests/itests/docutests/build_md_test.py
+++ b/tests/itests/docutests/build_md_test.py
@@ -186,7 +186,7 @@
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
finally:
- print "Test_03_make_Start:------------------ took {} secs \n\n"\
+ print "Test_03_make_Start:------------------ took {} secs \n\n" \
.format(time.time() - t0)
def test_04_run_voltha_standalone_without_consul(self):
@@ -250,9 +250,9 @@
print "Start consul ..."
self._run_consul()
- # sleep until consul is ready
print "Waiting for consul to be ready ..."
- time.sleep(10)
+ rc = self._wait_for_consul_to_be_ready()
+ self.assertEqual(rc, 0)
# Get the docker IP address and port number of the consul instance
print "Get consul leader IP ..."
@@ -370,6 +370,12 @@
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
+ print "Waiting for all containers to be ready ..."
+ rc, not_found_list = self._wait_for_all_containers_to_ready()
+ if rc:
+ print "Not found patterns:{}".format(not_found_list)
+ self.assertEqual(rc, 0)
+
# verify that all containers are running
print "Verify all services are running using docker command ..."
for service in docker_service_list:
@@ -437,7 +443,7 @@
expected_pattern = ['voltha-heartbeat', 'Heartbeat message']
cmd = command_defs['kafka_client_run_10_secs']
kafka_client_output = run_long_running_command_with_timeout(cmd,
- 10)
+ 20)
# Verify the kafka client output
# instance id
@@ -532,9 +538,12 @@
print "Start all containers..."
self._start_all_containers()
- # sleep until all containers are sync up and ready
- print "Waiting for consul to be ready ..."
+ print "Waiting for all containers to be ready ..."
time.sleep(10)
+ rc, not_found_list = self._wait_for_all_containers_to_ready()
+ if rc:
+ print "Not found patterns:{}".format(not_found_list)
+ self.assertEqual(rc, 0)
# Get the IP address(es) for voltha's REST interface
print "Get IP of Voltha REST interface..."
@@ -576,6 +585,13 @@
print "Start all containers..."
self._start_all_containers()
+ print "Waiting for all containers to be ready ..."
+ time.sleep(10)
+ rc, not_found_list = self._wait_for_all_containers_to_ready()
+ if rc:
+ print "Not found patterns:{}".format(not_found_list)
+ self.assertEqual(rc, 0)
+
# Scale voltha to 10 instances
print "Scale voltha to 10 instances ..."
cmd = command_defs['docker_compose_scale_voltha_to_10']
@@ -650,3 +666,118 @@
env = os.environ.copy()
return env
+
+ def _wait_for_consul_to_be_ready(self):
+ # Consul is ready when it's leader ip and port is set. The maximum
+ # time to wait of 60 secs as consul should be ready by then
+ max_wait_time = 60
+ t0 = time.time()
+
+ while True:
+ # Get the docker IP address and port number of the consul instance
+ print "waiting for consul to be ready ..."
+ cmd = command_defs['consul_get_leader_ip_port']
+ out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
+ out = out.strip()
+ if rc != 0:
+ # Something is wrong, return
+ return -1 # error
+ elif out is not None and out != '':
+ return 0 # found something
+ elif time.time() - t0 > max_wait_time:
+ return -1 # consul should have come up by this time
+ else:
+ time.sleep(2) # constant sleep for testing
+
+ def _wait_for_all_containers_to_ready(self):
+ # After the containers have been started using docker-compose, look
+ # at the logs for the following patterns to decide if the containers
+ # are up and in sync:
+ #
+ # For registrator, look for
+ # "(.*)registrator_1(.*)Listening for Docker events"
+ #
+ # For voltha, zookeeper and kafka look for these patterns
+ # "(.*)voltha_1(.*)main.heartbeat {status: up, uptime:"
+ # "(.*)voltha_1(.*)kafka_proxy.send_message {event: Successfully sent message Heartbeat message"
+ #
+ # For fluentd, look for
+ # "(.*)fluentd_1(.*)listening fluent socket on"
+ #
+ # For chameleon, look for
+ # "(.*)chameleon_1(.*)main.startup_components {event:
+ # started-internal-services, instance_id: compose_chameleon_1}"
+ #
+ # For consul, look for
+ # "(.*)consul_1(.*)agent: Synced service(.*)consul(.*)8500"
+ # "(.*)consul_1(.*)agent: Synced service(.*)consul(.*)8600:udp"
+ # "(.*)consul_1(.*)agent: Synced service(.*)fluentd"
+ # "(.*)consul_1(.*)agent: Synced service(.*)voltha"
+ # "(.*)consul_1(.*)agent: Synced service(.*)voltha(.*):8880"
+ # "(.*)consul_1(.*)agent: Synced service(.*)chameleon(.*):8881"
+ # "(.*)consul_1(.*)agent: Synced service(.*)zookeeper(.*):2181"
+ # "(.*)consul_1(.*)agent: Synced service(.*)kafka(.*):9092"
+ #
+ expected_output = [
+ "(.*)registrator_1(.*)Listening for Docker events",
+ "(.*)voltha_1(.*)main.heartbeat {status: up, uptime:",
+ "(.*)voltha_1(.*)kafka_proxy.send_message {event: Successfully "
+ "sent message Heartbeat message",
+ "(.*)fluentd_1(.*)listening fluent socket on",
+ "(.*)chameleon_1(.*)main.startup_components {event: "
+ "started-internal-services, instance_id: compose_chameleon_1}",
+ "(.*)consul_1(.*)agent: Synced service(.*)consul(.*)8500",
+ "(.*)consul_1(.*)agent: Synced service(.*)consul(.*)8600:udp",
+ "(.*)consul_1(.*)agent: Synced service(.*)fluentd",
+ "(.*)consul_1(.*)agent: Synced service(.*)voltha(.*):(?!8880)",
+ "(.*)consul_1(.*)agent: Synced service(.*)voltha(.*):8880",
+ "(.*)consul_1(.*)agent: Synced service(.*)chameleon(.*):8881",
+ "(.*)consul_1(.*)agent: Synced service(.*)zookeeper(.*):2181",
+ "(.*)consul_1(.*)agent: Synced service(.*)kafka(.*):9092"
+ ]
+ pattern_found = []
+ max_wait_time = 120 # wait 2 mins as a maximum
+
+ def _stop_process(proc):
+ try:
+ proc.terminate()
+ proc.wait()
+ # In principle this 'reset' should not be required.
+ # However, without it, the terminal is left in a funny
+ # state and required
+ subprocess.Popen(['reset']).wait()
+ except Exception as e:
+ print "Received exception {} when killing process " \
+ .format(repr(e), )
+
+ try:
+ t0 = time.time()
+ env = os.environ.copy()
+ proc = subprocess.Popen(
+ command_defs['docker_compose_logs'],
+ env=env,
+ shell=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ bufsize=1
+ )
+ for line in iter(proc.stdout.readline, b''):
+ ansi_escape = re.compile(r'\x1b[^m]*m')
+ line = ansi_escape.sub('', line)
+ for pattern in expected_output:
+ if re.match(pattern, line, re.I):
+ if pattern not in pattern_found:
+ pattern_found.append(pattern)
+ break
+ # Check if we found all patterns yet
+ if len(pattern_found) == len(expected_output):
+ _stop_process(proc)
+ return 0, [] # success
+ elif time.time() - t0 > max_wait_time:
+ _stop_process(proc)
+ not_found = [p for p in expected_output if p not in
+ pattern_found]
+ return -1, not_found # failure
+ except Exception as e:
+ print 'Exception {} '.format(repr(e))
+ return -1, []