blob: 6a00f7169e5650547a08687feb44e96d552bcc86 [file] [log] [blame]
Khen Nursimulu37a9bf82016-10-16 20:11:31 -04001#!/usr/bin/env python
2#
3# Copyright 2016 the original author or authors.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17import subprocess
khenaidoo1243ee92017-07-17 15:54:06 -040018import select
Khen Nursimulu37a9bf82016-10-16 20:11:31 -040019import time
20import logging
alshabib8b734be2017-02-06 14:56:05 -080021from common.utils.consulhelpers import verify_all_services_healthy, get_endpoint_from_consul
Khen Nursimulu37a9bf82016-10-16 20:11:31 -040022import os
23import json
24from unittest import TestCase
25import re
khenaidoo1243ee92017-07-17 15:54:06 -040026import simplejson
27import sys
28import traceback
Khen Nursimulu37a9bf82016-10-16 20:11:31 -040029
30this_dir = os.path.abspath(os.path.dirname(__file__))
31
32from test_utils import run_command_to_completion_with_raw_stdout, \
33 is_open, \
34 is_valid_ip, \
35 run_long_running_command_with_timeout, \
36 run_command_to_completion_with_stdout_in_list
37
38log = logging.getLogger(__name__)
39
alshabib16c0da72017-01-19 12:26:02 -060040LOCAL_CONSUL = "localhost:8500"
41LOCAL_CONSUL_URL = "http://%s" % LOCAL_CONSUL
Khen Nursimulu37a9bf82016-10-16 20:11:31 -040042LOCAL_CONSUL_DNS = "@localhost -p 8600"
David K. Bainbridge10a7a7e2018-01-29 09:54:40 -080043DOCKER_COMPOSE_PROJECT = "compose"
khenaidoofe874ae2017-07-14 18:07:27 -040044DOCKER_COMPOSE_FILE = "compose/docker-compose-docutests.yml"
Khen Nursimulu37a9bf82016-10-16 20:11:31 -040045DOCKER_COMPOSE_FILE_SERVICES_COUNT = 7
46
47command_defs = dict(
48 makefile_fetch_images="grep \"docker pull\" Makefile",
49 make="make",
David K. Bainbridge10a7a7e2018-01-29 09:54:40 -080050 make_clean_build="make -e DOCKER_CACHE_ARG=--no-cache build",
Khen Nursimulu37a9bf82016-10-16 20:11:31 -040051 make_fetch="make fetch",
52 remove_env_directory="rm -rf venv-linux",
53 make_clean="make clean",
54 docker_images="docker images",
55 docker_stop="docker stop",
56 docker_rm="docker rm",
57 fluentd_logs="less /tmp/fluentd/data.log",
David K. Bainbridge10a7a7e2018-01-29 09:54:40 -080058 docker_voltha_logs="docker-compose -p {} -f {} logs voltha"
59 .format(DOCKER_COMPOSE_PROJECT, DOCKER_COMPOSE_FILE),
60 docker_compose_logs="docker-compose -p {} -f {} logs"
61 .format(DOCKER_COMPOSE_PROJECT, DOCKER_COMPOSE_FILE),
Khen Nursimulu37a9bf82016-10-16 20:11:31 -040062 docker_stop_and_remove_all_containers="docker stop `docker ps -q` ; "
63 "docker rm `docker ps -a -q`",
David K. Bainbridgebba65ff2018-01-19 09:26:09 -080064 docker_start_voltha="docker run -ti --rm voltha/voltha",
Khen Nursimulu37a9bf82016-10-16 20:11:31 -040065 docker_start_voltha_with_consul_ip="docker run -ti --rm --net="
David K. Bainbridgebba65ff2018-01-19 09:26:09 -080066 "compose_default voltha/voltha "
Khen Nursimulu96bb5322016-11-09 20:16:03 -080067 "/voltha/voltha/main.py --consul=",
Khen Nursimulu37a9bf82016-10-16 20:11:31 -040068 docker_get_consul_ip="docker inspect "
69 "compose_consul_1 | jq -r "
70 "'.[0].NetworkSettings.Networks."
71 "compose_default.IPAddress'",
David K. Bainbridge10a7a7e2018-01-29 09:54:40 -080072 docker_compose_start_consul="docker-compose -p {} -f {} up -d "
73 "consul".format(DOCKER_COMPOSE_PROJECT, DOCKER_COMPOSE_FILE),
74 docker_compose_start_all="docker-compose -p {} -f {} up -d "
75 .format(DOCKER_COMPOSE_PROJECT, DOCKER_COMPOSE_FILE),
76 docker_compose_stop="docker-compose -p {} -f {} stop"
77 .format(DOCKER_COMPOSE_PROJECT, DOCKER_COMPOSE_FILE),
78 docker_compose_rm_f="docker-compose -p {} -f {} rm -f"
79 .format(DOCKER_COMPOSE_PROJECT, DOCKER_COMPOSE_FILE),
80 docker_compose_down="docker-compose -p {} -f {} down"
81 .format(DOCKER_COMPOSE_PROJECT, DOCKER_COMPOSE_FILE),
82 docker_compose_ps="docker-compose -p {} -f {} ps"
83 .format(DOCKER_COMPOSE_PROJECT, DOCKER_COMPOSE_FILE),
Khen Nursimulu37a9bf82016-10-16 20:11:31 -040084 docker_ps="docker ps",
85 docker_ps_count="docker ps -q | wc -l",
David K. Bainbridge10a7a7e2018-01-29 09:54:40 -080086 docker_compose_is_consul_up="docker-compose -p {} -f {} ps | grep consul"
87 .format(DOCKER_COMPOSE_PROJECT, DOCKER_COMPOSE_FILE),
Khen Nursimulu37a9bf82016-10-16 20:11:31 -040088 consul_get_leader_ip_port="curl -s {}/v1/status/leader | jq -r ."
89 .format(LOCAL_CONSUL_URL),
David K. Bainbridge10a7a7e2018-01-29 09:54:40 -080090 docker_compose_services_running="docker-compose -p {} -f {} ps -q"
91 .format(DOCKER_COMPOSE_PROJECT, DOCKER_COMPOSE_FILE),
92 docker_compose_services_running_count="docker-compose -p {} -f {} ps -q | "
93 "grep Up | wc -l"
94 .format(DOCKER_COMPOSE_PROJECT, DOCKER_COMPOSE_FILE),
95 docker_compose_services="docker-compose -p {} -f {} config --services"
96 .format(DOCKER_COMPOSE_PROJECT, DOCKER_COMPOSE_FILE),
Khen Nursimulu37a9bf82016-10-16 20:11:31 -040097 consul_get_services="curl -s {}/v1/catalog/services | jq -r ."
98 .format(LOCAL_CONSUL_URL),
99 consul_get_srv_voltha_health="curl -s {}/v1/catalog/service/voltha-health "
100 "| jq -r .".format(LOCAL_CONSUL_URL),
alshabib8b734be2017-02-06 14:56:05 -0800101 kafka_client_run="kafkacat -b {} -L",
khenaidoo1243ee92017-07-17 15:54:06 -0400102 kafka_client_heart_check="kafkacat -o end -b {} -C -t voltha.heartbeat -c 1",
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400103 consul_get_voltha_rest_a_record="dig {} voltha-health.service.consul"
104 .format(LOCAL_CONSUL_DNS),
105 consul_get_voltha_rest_ip="dig {} +short voltha-health.service.consul"
106 .format(LOCAL_CONSUL_DNS),
107 consul_get_voltha_service_port="dig {} +short "
108 "voltha-health.service.consul SRV | "
109 " awk \'{{print $3}}'"
110 .format(LOCAL_CONSUL_DNS),
David K. Bainbridge10a7a7e2018-01-29 09:54:40 -0800111 docker_compose_scale_voltha_to_10="docker-compose -p {} -f {} scale "
112 "voltha=10"
113 .format(DOCKER_COMPOSE_PROJECT, DOCKER_COMPOSE_FILE),
114 docker_compose_scaled_voltha_ps="docker-compose -p {} -f {} ps voltha | "
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400115 "grep Up | wc -l"
David K. Bainbridge10a7a7e2018-01-29 09:54:40 -0800116 .format(DOCKER_COMPOSE_PROJECT, DOCKER_COMPOSE_FILE),
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400117 consul_verify_voltha_registration="curl -s {}"
118 "/v1/kv/service/voltha/members?recurse |"
119 " jq -r .".format(LOCAL_CONSUL_DNS)
120)
121
122
123class BuildMdTests(TestCase):
124 # docker_client = Client(base_url='unix://var/run/docker.sock')
125
khenaidoo88518e82017-06-21 15:42:26 -0400126 def wait_till(self, msg, predicate, interval=0.1, timeout=5.0):
127 deadline = time.time() + timeout
128 while time.time() < deadline:
129 if predicate():
130 return
131 time.sleep(interval)
132 self.fail('Timed out while waiting for condition: {}'.format(msg))
133
134
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400135 def test_01_setup(self):
136 print "Test_01_setup_Start:------------------"
137 t0 = time.time()
138
139 try:
140 # remove the venv-linux directory
141 print "Remove venv-linux ..."
142 cmd = command_defs['remove_env_directory']
143 rm_venv, err, rc = run_command_to_completion_with_raw_stdout(cmd)
144 self.assertEqual(rc, 0)
145
146 # make clean
147 print "Make clean ..."
148 cmd = command_defs['make_clean']
149 mk_clean, err, rc = run_command_to_completion_with_raw_stdout(cmd)
150 self.assertEqual(rc, 0)
151
152 # source the env
153 print "Source environment ..."
154 self._source_env()
155
156 finally:
157 print "Test_01_setup_End:------------------ took {} " \
158 "secs\n\n".format(time.time() - t0)
159
160 def test_02_make_fetch(self):
161 print "Test_02_make_fetch_Start:------------------"
162 t0 = time.time()
163
164 try:
165 # Get list of images to fetch from the Makefile
166 print "Get list of images to fetch ..."
167 cmd = command_defs['makefile_fetch_images']
168 makefile_images_to_fetch, err, rc \
169 = run_command_to_completion_with_stdout_in_list(cmd)
170 self.assertEqual(rc, 0)
171
172 images_to_fetch = []
173 for image in makefile_images_to_fetch:
174 tmp = ''.join(image.split())
175 images_to_fetch.append(tmp[len('dockerpull'):])
176
177 # make fetch
178 print "Fetching images {} ...".format(images_to_fetch)
179 cmd = command_defs['make_fetch']
180 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
181 self.assertEqual(rc, 0)
182
183 # verify that the images have been downloaded
184 print "Verify images downloaded and present locally ..."
185 cmd = command_defs['docker_images']
186 local_images, err, rc = \
187 run_command_to_completion_with_stdout_in_list(cmd)
188 self.assertEqual(rc, 0)
189
190 local_images_list = []
191 for local_image in local_images:
192 words = local_image.split()
193 local_images_list.append('{}:{}'.format(words[0], words[1]))
194
195 intersection_list = [i for i in images_to_fetch if
196 i in local_images_list]
197 assert len(intersection_list) == len(images_to_fetch)
198
199 finally:
200 print "Test_02_make_fetch_End:------------------ took {} " \
201 "secs \n\n".format(time.time() - t0)
202
203 def test_03_make(self):
David K. Bainbridge10a7a7e2018-01-29 09:54:40 -0800204 print "Test_03_make_build_Start:------------------"
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400205 t0 = time.time()
206 try:
David K. Bainbridge10a7a7e2018-01-29 09:54:40 -0800207 cmd = command_defs['make_clean_build']
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400208 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
209 self.assertEqual(rc, 0)
210 finally:
David K. Bainbridge10a7a7e2018-01-29 09:54:40 -0800211 print "Test_03_make_build_Start:------------------ took {} secs \n\n" \
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400212 .format(time.time() - t0)
213
214 def test_04_run_voltha_standalone_without_consul(self):
215 print "Test_04_run_voltha_standalone_without_consul_Start:------------" \
216 "------"
217 t0 = time.time()
218
219 try:
220 # Run voltha for 10 secs and verity the following lines are displayed
221 # (a subset of output messages along with a flag when found)
222 print "Start voltha ..."
223 expected_output_subset = [
224 'main.print_banner {event: (to stop: press Ctrl-C), '
225 'instance_id:',
Khen Nursimulu96bb5322016-11-09 20:16:03 -0800226 'coordinator.__init__ {event: initializing-coordinator,',
227 'grpc_server.start {event: started',
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400228 'main.<lambda> {event: twisted-reactor-started',
229 'main.startup_components {event: started-internal-services,',
Khen Nursimulu9b9f1ad2017-01-10 15:43:32 -0500230 'kafka_proxy.start {event: started,',
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400231 'coordinator._backoff {retry_in: 5, event: consul-not-up,'
232 ]
233
234 cmd = command_defs['docker_start_voltha']
235 command_output = run_long_running_command_with_timeout(cmd, 10)
236
237 # There should at least be 1 line in the output
238 self.assertGreater(len(command_output), 0)
239
240 # Verify that the output contained the expected_output_subset -
241 # save the docker instance id
242 print "Verify voltha started correctly ..."
243 instance_id = None
244 for ext_output in expected_output_subset:
245 match_str = next(
246 (out for out in command_output if ext_output in out),
247 None)
248 self.assertIsNotNone(match_str)
249 if "instance_id" in ext_output:
250 instance_id = re.findall(r'[0-9a-f]+', match_str)[-1]
251
252 # Now stop the voltha docker that was created
253 print "Stop voltha ..."
254 self._stop_docker_container_by_id(instance_id)
255
256
257 finally:
258 # Remove any created container
259 self._stop_and_remove_all_containers()
260
261 print "Test_04_run_voltha_standalone_without_consul_End" \
262 ":------------------ took {} secs \n\n".format(
263 time.time() - t0)
264
265 def test_05_run_consul_only(self):
266 print "Test_05_run_consul_only_Start:------------------ "
267 t0 = time.time()
268
269 try:
270 # run consul
271 print "Start consul ..."
272 self._run_consul()
273
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400274 print "Waiting for consul to be ready ..."
Khen Nursimulua54b6632016-10-18 18:01:25 -0400275 rc = self._wait_for_consul_to_be_ready()
276 self.assertEqual(rc, 0)
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400277
278 # Get the docker IP address and port number of the consul instance
279 print "Get consul leader IP ..."
280 cmd = command_defs['consul_get_leader_ip_port']
281 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
282 self.assertEqual(rc, 0)
283
284 # validate that the returned ip:port is valid and open
285 print "Verify consul IP and port is reachable ..."
286 self.assertTrue(is_open(out))
287
288 finally:
289 # clean up all created containers for this test
290 print "Stop consul ..."
291 self._stop_and_remove_all_containers()
292
293 print "Test_05_run_consul_only_End:------------------ took {} secs" \
294 "\n\n".format(time.time() - t0)
295
296 def test_06_run_voltha_standalone_with_consul_only(self):
297 print "Test_06_run_voltha_standalone_with_consul_only_Start:----------" \
298 "-------- "
299 t0 = time.time()
300
301 try:
302 # run consul first
303 print "Start consul ..."
304 self._run_consul()
305
306 # get consul ip
307 print "Get consul IP ..."
308 cmd = command_defs['docker_get_consul_ip']
309 consul_ip, err, rc = run_command_to_completion_with_raw_stdout(cmd)
310 self.assertEqual(rc, 0)
311 self.assertIsNotNone(consul_ip)
312
313 # start voltha now for 15 secs and verify it can now connect to
314 # consul - following message in the output
315 print "Start voltha with consul IP ..."
316 expected_pattern = ['coordinator', 'event: created-consul-session']
317 cmd = command_defs['docker_start_voltha_with_consul_ip'] + \
318 '{}:8500'.format(consul_ip.strip())
319 command_output = run_long_running_command_with_timeout(cmd, 10)
320
321 # Verify the output of voltha and get the container instance id
322 print "Verify voltha is registered with consul ..."
323 instance_id = None
324 for out in command_output:
325 if all(ep for ep in expected_pattern if ep in out):
326 self.assertTrue(True)
327 instance_id = re.findall(r'[0-9a-f]+', out)[-1]
328 break
329
330 self.assertIsNotNone(instance_id)
331
332 # Verify Voltha's self-registration with consul
333 expected_output = ['ModifyIndex', 'CreateIndex', 'Session',
334 'Value',
335 'Flags', 'Key', 'LockIndex']
336
337 cmd = command_defs['consul_verify_voltha_registration']
338 registration_info, err, rc = \
339 run_command_to_completion_with_raw_stdout(cmd)
340 self.assertEqual(rc, 0)
341 try:
342 jr_info = json.loads(registration_info)
343 intersect_elems = [e for e in jr_info[0] if
344 e in expected_output]
345 self.assertEqual(len(expected_output), len(intersect_elems))
346 except Exception as e:
347 self.assertRaises(e)
348
349 # stop voltha
350 print "Stop voltha ..."
351 self._stop_docker_container_by_id(instance_id)
352
353 # check the service has deregistered
354 print "Verify voltha is no longer registered in consul..."
355 cmd = command_defs['consul_verify_voltha_registration']
356 registration_info, err, rc = \
357 run_command_to_completion_with_raw_stdout(cmd)
358 self.assertEqual(rc, 0)
359 self.assertEqual(registration_info, '')
360
361 finally:
362 # clean up all created containers for this test
363 print "Stop consul ..."
364 self._stop_and_remove_all_containers()
365
366 print "Test_06_run_voltha_standalone_with_consul_only_End:--------" \
367 "---------- took {} " \
368 "secs \n\n".format(time.time() - t0)
369
370 def test_07_start_all_containers(self):
371 print "Test_07_start_all_containers_Start:------------------ "
372 t0 = time.time()
373
374 try:
375 # Pre-test - clean up all running docker containers
376 print "Pre-test: Removing all running containers ..."
David K. Bainbridge10a7a7e2018-01-29 09:54:40 -0800377 cmd = command_defs['docker_compose_stop']
378 _, err, rc = run_command_to_completion_with_raw_stdout(cmd)
379 self.assertEqual(rc, 0)
380 cmd = command_defs['docker_compose_rm_f']
381 _, err, rc = run_command_to_completion_with_raw_stdout(cmd)
382 self.assertEqual(rc, 0)
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400383
384 # get a list of services in the docker-compose file
385 print "Getting list of services in docker compose file ..."
386 cmd = command_defs['docker_compose_services']
387 services, err, rc = run_command_to_completion_with_raw_stdout(cmd)
388 self.assertEqual(rc, 0)
389 docker_service_list = services.split()
390 self.assertGreaterEqual(len(docker_service_list),
391 DOCKER_COMPOSE_FILE_SERVICES_COUNT)
392
393 # start all the containers
394 print "Starting all containers ..."
395 cmd = command_defs['docker_compose_start_all']
396 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
397 self.assertEqual(rc, 0)
398
khenaidoo88518e82017-06-21 15:42:26 -0400399 # Instead of using only a fixed timeout:
400 # 1) wait until the services are ready (polling per second)
401 # 2) bail out after a longer timeout.
Khen Nursimulua54b6632016-10-18 18:01:25 -0400402 print "Waiting for all containers to be ready ..."
khenaidoo88518e82017-06-21 15:42:26 -0400403 self.wait_till('Not all services are up',
khenaidoo1243ee92017-07-17 15:54:06 -0400404 self._is_voltha_ensemble_ready,
khenaidoo88518e82017-06-21 15:42:26 -0400405 interval=1,
406 timeout=30)
Khen Nursimulua54b6632016-10-18 18:01:25 -0400407
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400408 # verify that all containers are running
409 print "Verify all services are running using docker command ..."
410 for service in docker_service_list:
411 cmd = command_defs['docker_compose_ps'] + ' {} | wc -l'.format(
412 service)
413 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
414 self.assertEqual(rc, 0)
415 self.assertGreaterEqual(out, 3) # 2 are for headers
416
417 # Verify that 'docker ps' return the same number of running process
418 cmd = command_defs['docker_ps_count']
419 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
420 self.assertEqual(rc, 0)
Jonathan Hart87314cd2018-02-12 17:15:35 -0800421 self.assertGreaterEqual(out, (len(docker_service_list)))
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400422
423 # Retrieve the list of services from consul and validate against
424 # the list obtained from docker composed
425 print "Verify all services are registered in consul ..."
426 expected_services = ['consul-rest', 'fluentd-intake',
khenaidoo079a7762017-10-26 21:42:05 -0400427 'voltha-grpc',
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400428 'voltha-health',
Khen Nursimulu9b9f1ad2017-01-10 15:43:32 -0500429 'consul-8600', 'zookeeper', 'consul',
430 'kafka']
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400431
432 cmd = command_defs['consul_get_services']
433 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
434 self.assertEqual(rc, 0)
435 try:
436 consul_services = json.loads(out)
437 intersected_services = [s for s in expected_services if
438 s in consul_services]
439 self.assertEqual(len(intersected_services),
440 len(expected_services))
441 # services_match = 0
442 # for d_service in docker_service_list:
443 # for c_service in consul_services:
444 # if c_service.find(d_service) != -1:
445 # services_match += 1
446 # print d_service, c_service
447 # break
448 # self.assertEqual(services_match, len(docker_service_list))
449 except Exception as e:
450 self.assertRaises(e)
451
452 # Verify the service record of the voltha service
453 print "Verify the service record of voltha in consul ..."
454 expected_srv_elements = ['ModifyIndex', 'CreateIndex',
455 'ServiceEnableTagOverride', 'Node',
456 'Address', 'TaggedAddresses', 'ServiceID',
457 'ServiceName', 'ServiceTags',
458 'ServiceAddress', 'ServicePort']
459 cmd = command_defs['consul_get_srv_voltha_health']
460 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
461 self.assertEqual(rc, 0)
462 try:
463 srv = json.loads(out)
464 intersect_elems = [e for e in srv[0] if
465 e in expected_srv_elements]
466 self.assertEqual(len(expected_srv_elements),
467 len(intersect_elems))
468 except Exception as e:
469 self.assertRaises(e)
470
471 # Verify kafka client is receiving the messages
alshabib8b734be2017-02-06 14:56:05 -0800472 print "Verify kafka client has heartbeat topic ..."
473 expected_pattern = ['voltha.heartbeat']
474 kafka_endpoint = get_endpoint_from_consul(LOCAL_CONSUL,'kafka')
475 cmd = command_defs['kafka_client_run'].format(kafka_endpoint)
476 kafka_client_output = run_long_running_command_with_timeout(cmd, 20)
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400477
alshabib8b734be2017-02-06 14:56:05 -0800478 # Verify the kafka client output
479 # instance id
480 found = False
481 for out in kafka_client_output:
482 if all(ep for ep in expected_pattern if ep in out):
483 found = True
484 break
485 self.assertTrue(found)
486
khenaidoo1243ee92017-07-17 15:54:06 -0400487 # Commented the heartbeat messages from voltha as on Jenkins this
488 # test fails more often than not. On local or cluster environment
489 # the kafka event bus works well.
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400490
491 # verify docker-compose logs are being produced - just get the
492 # first work of each line
493 print "Verify docker compose logs has output from all the services " \
494 "..."
Jonathan Hart87314cd2018-02-12 17:15:35 -0800495 expected_output = ['voltha_1', 'fluentd_1', 'vconsul_1',
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400496 'registrator_1', 'kafka_1', 'zookeeper_1',
khenaidoo079a7762017-10-26 21:42:05 -0400497 'ofagent_1', 'netconf_1']
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400498 cmd = command_defs['docker_compose_logs']
David K. Bainbridge10a7a7e2018-01-29 09:54:40 -0800499 docker_compose_logs = run_long_running_command_with_timeout(cmd, 5, 0)
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400500 intersected_logs = [l for l in expected_output if
501 l in docker_compose_logs]
502 self.assertEqual(len(intersected_logs), len(expected_output))
503
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400504 # verify docker voltha logs are being produced - we will just verify
505 # some
506 # key messages in the logs
507 print "Verify docker voltha logs are produced ..."
khenaidoo3be3dbc2018-02-14 12:22:45 -0500508 self.wait_till('Basic voltha logs are absent',
509 self._is_basic_voltha_logs_produced,
510 interval=1,
511 timeout=30)
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400512
513 finally:
514 print "Stopping all containers ..."
515 # clean up all created containers for this test
David K. Bainbridge10a7a7e2018-01-29 09:54:40 -0800516 #self._stop_and_remove_all_containers()
517 cmd = command_defs['docker_compose_down']
518 _, err, rc = run_command_to_completion_with_raw_stdout(cmd)
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400519
520 print "Test_07_start_all_containers_End:------------------ took {}" \
521 " secs \n\n".format(time.time() - t0)
522
523 def test_08_stop_all_containers_started_using_docker_compose(self):
524 print "Test_08_stop_all_containers_started_using_docker_compose_Start:" \
525 "------------------ "
526 t0 = time.time()
527
528 try:
529 # commands to stop and clear the docker images
530 cmds = [command_defs['docker_compose_stop'],
531 command_defs['docker_compose_rm_f']]
532
533 print "Stopping all containers ..."
534 for cmd in cmds:
535 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
536 self.assertEqual(rc, 0)
537
538 # Verify that no docker process is running
539 print "Verify no containers is running..."
540 cmd = command_defs['docker_compose_services_running']
541 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
542 self.assertEqual(rc, 0)
543
544 finally:
545 print "Test_08_stop_all_containers_started_using_docker_compose_:" \
546 "------------------ took {} secs \n\n".format(
547 time.time() - t0)
548
549 def test_09_dig_consul_command(self):
550 print "Test_09_dig_consul_command_Start:------------------"
551 t0 = time.time()
552
553 try:
554 # start all containers
555 print "Start all containers..."
556 self._start_all_containers()
557
Khen Nursimulua54b6632016-10-18 18:01:25 -0400558 print "Waiting for all containers to be ready ..."
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400559 time.sleep(10)
alshabib16c0da72017-01-19 12:26:02 -0600560 rc = verify_all_services_healthy(LOCAL_CONSUL)
561 if not rc:
562 print "Not all services are up"
563 self.assertEqual(rc, True)
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400564
565 # Get the IP address(es) for voltha's REST interface
566 print "Get IP of Voltha REST interface..."
567 cmd = command_defs['consul_get_voltha_rest_a_record']
568 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
569 self.assertEqual(rc, 0)
570 self.assertGreaterEqual(out.find("voltha-health.service.consul"),
571 0)
572
573 # Get only the ip address
574 cmd = command_defs['consul_get_voltha_rest_ip']
575 ip, err, rc = run_command_to_completion_with_raw_stdout(cmd)
576 self.assertEqual(rc, 0)
577 self.assertTrue(is_valid_ip(ip))
578
579 # Get the exposed service port
580 print "Get Voltha exposed service port..."
581 cmd = command_defs['consul_get_voltha_service_port']
582 port, err, rc = run_command_to_completion_with_raw_stdout(cmd)
583 self.assertEqual(rc, 0)
584 # Verify that we can connect to the port using the previously
585 # acquired ip
586 print "Verify connectivity with voltha ip and port..."
587 self.assertTrue(is_open('{}:{}'.format(ip, port)))
588 finally:
589 print "Stopping all containers ..."
590 # clean up all created containers for this test
591 self._stop_and_remove_all_containers()
592
593 print "Test_09_dig_consul_command_Start_End:------------------" \
594 "took {} secs \n\n".format(time.time() - t0)
595
596 def test_10_scale_voltha(self):
597 print "Test_10_scale_voltha_Start:------------------"
598 t0 = time.time()
599
600 try:
601 # start all containers
602 print "Start all containers..."
603 self._start_all_containers()
604
khenaidoo1243ee92017-07-17 15:54:06 -0400605 # Instead of using only a fixed timeout:
606 # 1) wait until the services are ready (polling per second)
607 # 2) bail out after a longer timeout.
Khen Nursimulua54b6632016-10-18 18:01:25 -0400608 print "Waiting for all containers to be ready ..."
khenaidoo1243ee92017-07-17 15:54:06 -0400609 self.wait_till('Not all services are up',
610 self._is_voltha_ensemble_ready,
611 interval=1,
612 timeout=30)
Khen Nursimulua54b6632016-10-18 18:01:25 -0400613
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400614 # Scale voltha to 10 instances
615 print "Scale voltha to 10 instances ..."
616 cmd = command_defs['docker_compose_scale_voltha_to_10']
617 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
618 self.assertEqual(rc, 0)
619
620 # Verify that 10 instances are running
621 print "Verify 10 instances of voltha are running ..."
622 cmd = command_defs['docker_compose_scaled_voltha_ps']
623 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
624 self.assertEqual(rc, 0)
625 self.assertEqual(out.split(), ['10'])
626 finally:
627 print "Stopping all containers ..."
628 # clean up all created containers for this test
629 self._stop_and_remove_all_containers()
630
631 print "Test_10_scale_voltha_End:------------------took {} secs " \
632 "\n\n".format(time.time() - t0)
633
634 def _start_all_containers(self):
635 cmd = command_defs['docker_compose_start_all']
636 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
637 self.assertEqual(rc, 0)
638
khenaidoo1243ee92017-07-17 15:54:06 -0400639 def _is_voltha_ensemble_ready(self):
640 res = verify_all_services_healthy(LOCAL_CONSUL)
641 if not res:
642 print "Not all consul services are ready ..."
643 return res
644
khenaidoo3be3dbc2018-02-14 12:22:45 -0500645 def _is_basic_voltha_logs_produced(self):
646 expected_output = ['coordinator._renew_session', 'main.heartbeat']
647 cmd = command_defs['docker_voltha_logs']
648 docker_voltha_logs = run_long_running_command_with_timeout(cmd,
649 10, 5)
650 intersected_logs = [l for l in expected_output if
651 l in docker_voltha_logs]
652 return len(intersected_logs) == len(expected_output)
653
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400654 def _run_consul(self):
655 # run consul
656 cmd = command_defs['docker_compose_start_consul']
657 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
658 self.assertEqual(rc, 0)
659
660 # verify consul is up
661 cmd = command_defs['docker_compose_is_consul_up']
662 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
663 self.assertEqual(rc, 0)
664 self.assertIn('compose_consul_1', out)
665
666 def _stop_and_remove_all_containers(self):
667 # check if there are any running containers first
668 cmd = command_defs['docker_ps']
669 out, err, rc = run_command_to_completion_with_stdout_in_list(cmd)
670 self.assertEqual(rc, 0)
671 if len(out) > 1: # not counting docker ps header
672 cmd = command_defs['docker_stop_and_remove_all_containers']
673 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
674 self.assertEqual(rc, 0)
675
676 def _stop_docker_container_by_id(self, instance_id):
677 # stop
678 cmd = command_defs['docker_stop'] + " {}".format(instance_id)
679 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
680 self.assertEqual(rc, 0)
681
682 # remove
683 cmd = command_defs['docker_rm'] + " {}".format(instance_id)
684 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
685 self.assertEqual(rc, 0)
686
687 def _source_env(self):
688 # Go to voltha root directory
689 res = os.system('cd {}'.format(this_dir))
690 assert res == 0
691
692 # set the env
693 command = ['bash', '-c', '. env.sh']
694 proc = subprocess.Popen(command, stdout=subprocess.PIPE,
695 stderr=subprocess.PIPE)
696
697 if proc.wait() != 0:
698 err_msg = "Failed to source the environment'"
699 raise RuntimeError(err_msg)
700
701 env = os.environ.copy()
702 return env
Khen Nursimulua54b6632016-10-18 18:01:25 -0400703
704 def _wait_for_consul_to_be_ready(self):
705 # Consul is ready when it's leader ip and port is set. The maximum
706 # time to wait of 60 secs as consul should be ready by then
707 max_wait_time = 60
708 t0 = time.time()
709
710 while True:
711 # Get the docker IP address and port number of the consul instance
712 print "waiting for consul to be ready ..."
713 cmd = command_defs['consul_get_leader_ip_port']
714 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
715 out = out.strip()
716 if rc != 0:
717 # Something is wrong, return
718 return -1 # error
719 elif out is not None and out != '':
720 return 0 # found something
721 elif time.time() - t0 > max_wait_time:
722 return -1 # consul should have come up by this time
723 else:
724 time.sleep(2) # constant sleep for testing