blob: 133736e787d480fe7c285cb340af676db200dae7 [file] [log] [blame]
Khen Nursimulu37a9bf82016-10-16 20:11:31 -04001#!/usr/bin/env python
2#
3# Copyright 2016 the original author or authors.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17import subprocess
khenaidoo1243ee92017-07-17 15:54:06 -040018import select
Khen Nursimulu37a9bf82016-10-16 20:11:31 -040019import time
20import logging
alshabib8b734be2017-02-06 14:56:05 -080021from common.utils.consulhelpers import verify_all_services_healthy, get_endpoint_from_consul
Khen Nursimulu37a9bf82016-10-16 20:11:31 -040022import os
23import json
24from unittest import TestCase
25import re
khenaidoo1243ee92017-07-17 15:54:06 -040026import simplejson
27import sys
28import traceback
Khen Nursimulu37a9bf82016-10-16 20:11:31 -040029
30this_dir = os.path.abspath(os.path.dirname(__file__))
31
32from test_utils import run_command_to_completion_with_raw_stdout, \
33 is_open, \
34 is_valid_ip, \
35 run_long_running_command_with_timeout, \
36 run_command_to_completion_with_stdout_in_list
37
38log = logging.getLogger(__name__)
39
alshabib16c0da72017-01-19 12:26:02 -060040LOCAL_CONSUL = "localhost:8500"
41LOCAL_CONSUL_URL = "http://%s" % LOCAL_CONSUL
Khen Nursimulu37a9bf82016-10-16 20:11:31 -040042LOCAL_CONSUL_DNS = "@localhost -p 8600"
khenaidoofe874ae2017-07-14 18:07:27 -040043DOCKER_COMPOSE_FILE = "compose/docker-compose-docutests.yml"
Khen Nursimulu37a9bf82016-10-16 20:11:31 -040044DOCKER_COMPOSE_FILE_SERVICES_COUNT = 7
45
46command_defs = dict(
47 makefile_fetch_images="grep \"docker pull\" Makefile",
48 make="make",
49 make_fetch="make fetch",
50 remove_env_directory="rm -rf venv-linux",
51 make_clean="make clean",
52 docker_images="docker images",
53 docker_stop="docker stop",
54 docker_rm="docker rm",
55 fluentd_logs="less /tmp/fluentd/data.log",
56 docker_voltha_logs="docker logs -f compose_voltha_1",
57 docker_compose_logs="docker-compose -f {} logs".format(
58 DOCKER_COMPOSE_FILE),
59 docker_stop_and_remove_all_containers="docker stop `docker ps -q` ; "
60 "docker rm `docker ps -a -q`",
61 docker_start_voltha="docker run -ti --rm cord/voltha",
62 docker_start_voltha_with_consul_ip="docker run -ti --rm --net="
63 "compose_default cord/voltha "
Khen Nursimulu96bb5322016-11-09 20:16:03 -080064 "/voltha/voltha/main.py --consul=",
Khen Nursimulu37a9bf82016-10-16 20:11:31 -040065 docker_get_consul_ip="docker inspect "
66 "compose_consul_1 | jq -r "
67 "'.[0].NetworkSettings.Networks."
68 "compose_default.IPAddress'",
69 docker_compose_start_consul="docker-compose -f {} up -d "
70 "consul".format(DOCKER_COMPOSE_FILE),
71 docker_compose_start_all="docker-compose -f {} up -d "
72 .format(DOCKER_COMPOSE_FILE),
73 docker_compose_stop="docker-compose -f {} stop"
74 .format(DOCKER_COMPOSE_FILE),
75 docker_compose_rm_f="docker-compose -f {} rm -f"
76 .format(DOCKER_COMPOSE_FILE),
77 docker_compose_ps="docker-compose -f {} ps".format(DOCKER_COMPOSE_FILE),
78 docker_ps="docker ps",
79 docker_ps_count="docker ps -q | wc -l",
80 docker_compose_is_consul_up="docker-compose -f {} ps | grep consul"
81 .format(DOCKER_COMPOSE_FILE),
82 consul_get_leader_ip_port="curl -s {}/v1/status/leader | jq -r ."
83 .format(LOCAL_CONSUL_URL),
84 docker_compose_services_running="docker-compose -f {} ps -q"
85 .format(DOCKER_COMPOSE_FILE),
86 docker_compose_services_running_count="docker-compose -f {} ps -q | "
87 "grep Up "
88 "| wc -l".format(
89 DOCKER_COMPOSE_FILE),
90 docker_compose_services="docker-compose -f {} config --services"
91 .format(DOCKER_COMPOSE_FILE),
92 consul_get_services="curl -s {}/v1/catalog/services | jq -r ."
93 .format(LOCAL_CONSUL_URL),
94 consul_get_srv_voltha_health="curl -s {}/v1/catalog/service/voltha-health "
95 "| jq -r .".format(LOCAL_CONSUL_URL),
alshabib8b734be2017-02-06 14:56:05 -080096 kafka_client_run="kafkacat -b {} -L",
khenaidoo1243ee92017-07-17 15:54:06 -040097 kafka_client_heart_check="kafkacat -o end -b {} -C -t voltha.heartbeat -c 1",
Khen Nursimulu37a9bf82016-10-16 20:11:31 -040098 consul_get_voltha_rest_a_record="dig {} voltha-health.service.consul"
99 .format(LOCAL_CONSUL_DNS),
100 consul_get_voltha_rest_ip="dig {} +short voltha-health.service.consul"
101 .format(LOCAL_CONSUL_DNS),
102 consul_get_voltha_service_port="dig {} +short "
103 "voltha-health.service.consul SRV | "
104 " awk \'{{print $3}}'"
105 .format(LOCAL_CONSUL_DNS),
106 docker_compose_scale_voltha_to_10="docker-compose -f {} scale "
107 "voltha=10".format(DOCKER_COMPOSE_FILE),
108 docker_compose_scaled_voltha_ps="docker-compose -f {} ps voltha | "
109 "grep Up | wc -l"
110 .format(DOCKER_COMPOSE_FILE),
111 consul_verify_voltha_registration="curl -s {}"
112 "/v1/kv/service/voltha/members?recurse |"
113 " jq -r .".format(LOCAL_CONSUL_DNS)
114)
115
116
117class BuildMdTests(TestCase):
118 # docker_client = Client(base_url='unix://var/run/docker.sock')
119
khenaidoo88518e82017-06-21 15:42:26 -0400120 def wait_till(self, msg, predicate, interval=0.1, timeout=5.0):
121 deadline = time.time() + timeout
122 while time.time() < deadline:
123 if predicate():
124 return
125 time.sleep(interval)
126 self.fail('Timed out while waiting for condition: {}'.format(msg))
127
128
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400129 def test_01_setup(self):
130 print "Test_01_setup_Start:------------------"
131 t0 = time.time()
132
133 try:
134 # remove the venv-linux directory
135 print "Remove venv-linux ..."
136 cmd = command_defs['remove_env_directory']
137 rm_venv, err, rc = run_command_to_completion_with_raw_stdout(cmd)
138 self.assertEqual(rc, 0)
139
140 # make clean
141 print "Make clean ..."
142 cmd = command_defs['make_clean']
143 mk_clean, err, rc = run_command_to_completion_with_raw_stdout(cmd)
144 self.assertEqual(rc, 0)
145
146 # source the env
147 print "Source environment ..."
148 self._source_env()
149
150 finally:
151 print "Test_01_setup_End:------------------ took {} " \
152 "secs\n\n".format(time.time() - t0)
153
154 def test_02_make_fetch(self):
155 print "Test_02_make_fetch_Start:------------------"
156 t0 = time.time()
157
158 try:
159 # Get list of images to fetch from the Makefile
160 print "Get list of images to fetch ..."
161 cmd = command_defs['makefile_fetch_images']
162 makefile_images_to_fetch, err, rc \
163 = run_command_to_completion_with_stdout_in_list(cmd)
164 self.assertEqual(rc, 0)
165
166 images_to_fetch = []
167 for image in makefile_images_to_fetch:
168 tmp = ''.join(image.split())
169 images_to_fetch.append(tmp[len('dockerpull'):])
170
171 # make fetch
172 print "Fetching images {} ...".format(images_to_fetch)
173 cmd = command_defs['make_fetch']
174 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
175 self.assertEqual(rc, 0)
176
177 # verify that the images have been downloaded
178 print "Verify images downloaded and present locally ..."
179 cmd = command_defs['docker_images']
180 local_images, err, rc = \
181 run_command_to_completion_with_stdout_in_list(cmd)
182 self.assertEqual(rc, 0)
183
184 local_images_list = []
185 for local_image in local_images:
186 words = local_image.split()
187 local_images_list.append('{}:{}'.format(words[0], words[1]))
188
189 intersection_list = [i for i in images_to_fetch if
190 i in local_images_list]
191 assert len(intersection_list) == len(images_to_fetch)
192
193 finally:
194 print "Test_02_make_fetch_End:------------------ took {} " \
195 "secs \n\n".format(time.time() - t0)
196
197 def test_03_make(self):
198 print "Test_03_make_Start:------------------"
199 t0 = time.time()
200 try:
201 cmd = command_defs['make']
202 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
203 self.assertEqual(rc, 0)
204 finally:
Khen Nursimulua54b6632016-10-18 18:01:25 -0400205 print "Test_03_make_Start:------------------ took {} secs \n\n" \
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400206 .format(time.time() - t0)
207
208 def test_04_run_voltha_standalone_without_consul(self):
209 print "Test_04_run_voltha_standalone_without_consul_Start:------------" \
210 "------"
211 t0 = time.time()
212
213 try:
214 # Run voltha for 10 secs and verity the following lines are displayed
215 # (a subset of output messages along with a flag when found)
216 print "Start voltha ..."
217 expected_output_subset = [
218 'main.print_banner {event: (to stop: press Ctrl-C), '
219 'instance_id:',
Khen Nursimulu96bb5322016-11-09 20:16:03 -0800220 'coordinator.__init__ {event: initializing-coordinator,',
221 'grpc_server.start {event: started',
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400222 'main.<lambda> {event: twisted-reactor-started',
223 'main.startup_components {event: started-internal-services,',
Khen Nursimulu9b9f1ad2017-01-10 15:43:32 -0500224 'kafka_proxy.start {event: started,',
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400225 'coordinator._backoff {retry_in: 5, event: consul-not-up,'
226 ]
227
228 cmd = command_defs['docker_start_voltha']
229 command_output = run_long_running_command_with_timeout(cmd, 10)
230
231 # There should at least be 1 line in the output
232 self.assertGreater(len(command_output), 0)
233
234 # Verify that the output contained the expected_output_subset -
235 # save the docker instance id
236 print "Verify voltha started correctly ..."
237 instance_id = None
238 for ext_output in expected_output_subset:
239 match_str = next(
240 (out for out in command_output if ext_output in out),
241 None)
242 self.assertIsNotNone(match_str)
243 if "instance_id" in ext_output:
244 instance_id = re.findall(r'[0-9a-f]+', match_str)[-1]
245
246 # Now stop the voltha docker that was created
247 print "Stop voltha ..."
248 self._stop_docker_container_by_id(instance_id)
249
250
251 finally:
252 # Remove any created container
253 self._stop_and_remove_all_containers()
254
255 print "Test_04_run_voltha_standalone_without_consul_End" \
256 ":------------------ took {} secs \n\n".format(
257 time.time() - t0)
258
259 def test_05_run_consul_only(self):
260 print "Test_05_run_consul_only_Start:------------------ "
261 t0 = time.time()
262
263 try:
264 # run consul
265 print "Start consul ..."
266 self._run_consul()
267
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400268 print "Waiting for consul to be ready ..."
Khen Nursimulua54b6632016-10-18 18:01:25 -0400269 rc = self._wait_for_consul_to_be_ready()
270 self.assertEqual(rc, 0)
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400271
272 # Get the docker IP address and port number of the consul instance
273 print "Get consul leader IP ..."
274 cmd = command_defs['consul_get_leader_ip_port']
275 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
276 self.assertEqual(rc, 0)
277
278 # validate that the returned ip:port is valid and open
279 print "Verify consul IP and port is reachable ..."
280 self.assertTrue(is_open(out))
281
282 finally:
283 # clean up all created containers for this test
284 print "Stop consul ..."
285 self._stop_and_remove_all_containers()
286
287 print "Test_05_run_consul_only_End:------------------ took {} secs" \
288 "\n\n".format(time.time() - t0)
289
290 def test_06_run_voltha_standalone_with_consul_only(self):
291 print "Test_06_run_voltha_standalone_with_consul_only_Start:----------" \
292 "-------- "
293 t0 = time.time()
294
295 try:
296 # run consul first
297 print "Start consul ..."
298 self._run_consul()
299
300 # get consul ip
301 print "Get consul IP ..."
302 cmd = command_defs['docker_get_consul_ip']
303 consul_ip, err, rc = run_command_to_completion_with_raw_stdout(cmd)
304 self.assertEqual(rc, 0)
305 self.assertIsNotNone(consul_ip)
306
307 # start voltha now for 15 secs and verify it can now connect to
308 # consul - following message in the output
309 print "Start voltha with consul IP ..."
310 expected_pattern = ['coordinator', 'event: created-consul-session']
311 cmd = command_defs['docker_start_voltha_with_consul_ip'] + \
312 '{}:8500'.format(consul_ip.strip())
313 command_output = run_long_running_command_with_timeout(cmd, 10)
314
315 # Verify the output of voltha and get the container instance id
316 print "Verify voltha is registered with consul ..."
317 instance_id = None
318 for out in command_output:
319 if all(ep for ep in expected_pattern if ep in out):
320 self.assertTrue(True)
321 instance_id = re.findall(r'[0-9a-f]+', out)[-1]
322 break
323
324 self.assertIsNotNone(instance_id)
325
326 # Verify Voltha's self-registration with consul
327 expected_output = ['ModifyIndex', 'CreateIndex', 'Session',
328 'Value',
329 'Flags', 'Key', 'LockIndex']
330
331 cmd = command_defs['consul_verify_voltha_registration']
332 registration_info, err, rc = \
333 run_command_to_completion_with_raw_stdout(cmd)
334 self.assertEqual(rc, 0)
335 try:
336 jr_info = json.loads(registration_info)
337 intersect_elems = [e for e in jr_info[0] if
338 e in expected_output]
339 self.assertEqual(len(expected_output), len(intersect_elems))
340 except Exception as e:
341 self.assertRaises(e)
342
343 # stop voltha
344 print "Stop voltha ..."
345 self._stop_docker_container_by_id(instance_id)
346
347 # check the service has deregistered
348 print "Verify voltha is no longer registered in consul..."
349 cmd = command_defs['consul_verify_voltha_registration']
350 registration_info, err, rc = \
351 run_command_to_completion_with_raw_stdout(cmd)
352 self.assertEqual(rc, 0)
353 self.assertEqual(registration_info, '')
354
355 finally:
356 # clean up all created containers for this test
357 print "Stop consul ..."
358 self._stop_and_remove_all_containers()
359
360 print "Test_06_run_voltha_standalone_with_consul_only_End:--------" \
361 "---------- took {} " \
362 "secs \n\n".format(time.time() - t0)
363
364 def test_07_start_all_containers(self):
365 print "Test_07_start_all_containers_Start:------------------ "
366 t0 = time.time()
367
368 try:
369 # Pre-test - clean up all running docker containers
370 print "Pre-test: Removing all running containers ..."
371 self._stop_and_remove_all_containers()
372
373 # get a list of services in the docker-compose file
374 print "Getting list of services in docker compose file ..."
375 cmd = command_defs['docker_compose_services']
376 services, err, rc = run_command_to_completion_with_raw_stdout(cmd)
377 self.assertEqual(rc, 0)
378 docker_service_list = services.split()
379 self.assertGreaterEqual(len(docker_service_list),
380 DOCKER_COMPOSE_FILE_SERVICES_COUNT)
381
382 # start all the containers
383 print "Starting all containers ..."
384 cmd = command_defs['docker_compose_start_all']
385 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
386 self.assertEqual(rc, 0)
387
khenaidoo88518e82017-06-21 15:42:26 -0400388 # Instead of using only a fixed timeout:
389 # 1) wait until the services are ready (polling per second)
390 # 2) bail out after a longer timeout.
Khen Nursimulua54b6632016-10-18 18:01:25 -0400391 print "Waiting for all containers to be ready ..."
khenaidoo88518e82017-06-21 15:42:26 -0400392 self.wait_till('Not all services are up',
khenaidoo1243ee92017-07-17 15:54:06 -0400393 self._is_voltha_ensemble_ready,
khenaidoo88518e82017-06-21 15:42:26 -0400394 interval=1,
395 timeout=30)
Khen Nursimulua54b6632016-10-18 18:01:25 -0400396
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400397 # verify that all containers are running
398 print "Verify all services are running using docker command ..."
399 for service in docker_service_list:
400 cmd = command_defs['docker_compose_ps'] + ' {} | wc -l'.format(
401 service)
402 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
403 self.assertEqual(rc, 0)
404 self.assertGreaterEqual(out, 3) # 2 are for headers
405
406 # Verify that 'docker ps' return the same number of running process
407 cmd = command_defs['docker_ps_count']
408 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
409 self.assertEqual(rc, 0)
Khen Nursimulu283d7682016-11-11 16:37:32 -0500410 self.assertGreaterEqual(out.split(), [str(len(
411 docker_service_list))])
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400412
413 # Retrieve the list of services from consul and validate against
414 # the list obtained from docker composed
415 print "Verify all services are registered in consul ..."
416 expected_services = ['consul-rest', 'fluentd-intake',
khenaidoo079a7762017-10-26 21:42:05 -0400417 'voltha-grpc',
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400418 'voltha-health',
Khen Nursimulu9b9f1ad2017-01-10 15:43:32 -0500419 'consul-8600', 'zookeeper', 'consul',
420 'kafka']
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400421
422 cmd = command_defs['consul_get_services']
423 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
424 self.assertEqual(rc, 0)
425 try:
426 consul_services = json.loads(out)
427 intersected_services = [s for s in expected_services if
428 s in consul_services]
429 self.assertEqual(len(intersected_services),
430 len(expected_services))
431 # services_match = 0
432 # for d_service in docker_service_list:
433 # for c_service in consul_services:
434 # if c_service.find(d_service) != -1:
435 # services_match += 1
436 # print d_service, c_service
437 # break
438 # self.assertEqual(services_match, len(docker_service_list))
439 except Exception as e:
440 self.assertRaises(e)
441
442 # Verify the service record of the voltha service
443 print "Verify the service record of voltha in consul ..."
444 expected_srv_elements = ['ModifyIndex', 'CreateIndex',
445 'ServiceEnableTagOverride', 'Node',
446 'Address', 'TaggedAddresses', 'ServiceID',
447 'ServiceName', 'ServiceTags',
448 'ServiceAddress', 'ServicePort']
449 cmd = command_defs['consul_get_srv_voltha_health']
450 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
451 self.assertEqual(rc, 0)
452 try:
453 srv = json.loads(out)
454 intersect_elems = [e for e in srv[0] if
455 e in expected_srv_elements]
456 self.assertEqual(len(expected_srv_elements),
457 len(intersect_elems))
458 except Exception as e:
459 self.assertRaises(e)
460
461 # Verify kafka client is receiving the messages
alshabib8b734be2017-02-06 14:56:05 -0800462 print "Verify kafka client has heartbeat topic ..."
463 expected_pattern = ['voltha.heartbeat']
464 kafka_endpoint = get_endpoint_from_consul(LOCAL_CONSUL,'kafka')
465 cmd = command_defs['kafka_client_run'].format(kafka_endpoint)
466 kafka_client_output = run_long_running_command_with_timeout(cmd, 20)
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400467
alshabib8b734be2017-02-06 14:56:05 -0800468 # Verify the kafka client output
469 # instance id
470 found = False
471 for out in kafka_client_output:
472 if all(ep for ep in expected_pattern if ep in out):
473 found = True
474 break
475 self.assertTrue(found)
476
khenaidoo1243ee92017-07-17 15:54:06 -0400477 # Commented the heartbeat messages from voltha as on Jenkins this
478 # test fails more often than not. On local or cluster environment
479 # the kafka event bus works well.
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400480
481 # verify docker-compose logs are being produced - just get the
482 # first work of each line
483 print "Verify docker compose logs has output from all the services " \
484 "..."
485 expected_output = ['voltha_1', 'fluentd_1', 'consul_1',
486 'registrator_1', 'kafka_1', 'zookeeper_1',
khenaidoo079a7762017-10-26 21:42:05 -0400487 'ofagent_1', 'netconf_1']
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400488 cmd = command_defs['docker_compose_logs']
489 docker_compose_logs = run_long_running_command_with_timeout(cmd, 5,
490 0)
491 intersected_logs = [l for l in expected_output if
492 l in docker_compose_logs]
493 self.assertEqual(len(intersected_logs), len(expected_output))
494
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400495 # verify docker voltha logs are being produced - we will just verify
496 # some
497 # key messages in the logs
498 print "Verify docker voltha logs are produced ..."
khenaidoo1243ee92017-07-17 15:54:06 -0400499 expected_output = ['coordinator._renew_session', 'main.heartbeat']
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400500 cmd = command_defs['docker_voltha_logs']
501 docker_voltha_logs = run_long_running_command_with_timeout(cmd,
Khen Nursimulu96bb5322016-11-09 20:16:03 -0800502 0.5, 3)
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400503 intersected_logs = [l for l in expected_output if
504 l in docker_voltha_logs]
505 self.assertEqual(len(intersected_logs), len(expected_output))
506
507 finally:
508 print "Stopping all containers ..."
509 # clean up all created containers for this test
510 self._stop_and_remove_all_containers()
511
512 print "Test_07_start_all_containers_End:------------------ took {}" \
513 " secs \n\n".format(time.time() - t0)
514
515 def test_08_stop_all_containers_started_using_docker_compose(self):
516 print "Test_08_stop_all_containers_started_using_docker_compose_Start:" \
517 "------------------ "
518 t0 = time.time()
519
520 try:
521 # commands to stop and clear the docker images
522 cmds = [command_defs['docker_compose_stop'],
523 command_defs['docker_compose_rm_f']]
524
525 print "Stopping all containers ..."
526 for cmd in cmds:
527 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
528 self.assertEqual(rc, 0)
529
530 # Verify that no docker process is running
531 print "Verify no containers is running..."
532 cmd = command_defs['docker_compose_services_running']
533 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
534 self.assertEqual(rc, 0)
535
536 finally:
537 print "Test_08_stop_all_containers_started_using_docker_compose_:" \
538 "------------------ took {} secs \n\n".format(
539 time.time() - t0)
540
541 def test_09_dig_consul_command(self):
542 print "Test_09_dig_consul_command_Start:------------------"
543 t0 = time.time()
544
545 try:
546 # start all containers
547 print "Start all containers..."
548 self._start_all_containers()
549
Khen Nursimulua54b6632016-10-18 18:01:25 -0400550 print "Waiting for all containers to be ready ..."
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400551 time.sleep(10)
alshabib16c0da72017-01-19 12:26:02 -0600552 rc = verify_all_services_healthy(LOCAL_CONSUL)
553 if not rc:
554 print "Not all services are up"
555 self.assertEqual(rc, True)
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400556
557 # Get the IP address(es) for voltha's REST interface
558 print "Get IP of Voltha REST interface..."
559 cmd = command_defs['consul_get_voltha_rest_a_record']
560 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
561 self.assertEqual(rc, 0)
562 self.assertGreaterEqual(out.find("voltha-health.service.consul"),
563 0)
564
565 # Get only the ip address
566 cmd = command_defs['consul_get_voltha_rest_ip']
567 ip, err, rc = run_command_to_completion_with_raw_stdout(cmd)
568 self.assertEqual(rc, 0)
569 self.assertTrue(is_valid_ip(ip))
570
571 # Get the exposed service port
572 print "Get Voltha exposed service port..."
573 cmd = command_defs['consul_get_voltha_service_port']
574 port, err, rc = run_command_to_completion_with_raw_stdout(cmd)
575 self.assertEqual(rc, 0)
576 # Verify that we can connect to the port using the previously
577 # acquired ip
578 print "Verify connectivity with voltha ip and port..."
579 self.assertTrue(is_open('{}:{}'.format(ip, port)))
580 finally:
581 print "Stopping all containers ..."
582 # clean up all created containers for this test
583 self._stop_and_remove_all_containers()
584
585 print "Test_09_dig_consul_command_Start_End:------------------" \
586 "took {} secs \n\n".format(time.time() - t0)
587
588 def test_10_scale_voltha(self):
589 print "Test_10_scale_voltha_Start:------------------"
590 t0 = time.time()
591
592 try:
593 # start all containers
594 print "Start all containers..."
595 self._start_all_containers()
596
khenaidoo1243ee92017-07-17 15:54:06 -0400597 # Instead of using only a fixed timeout:
598 # 1) wait until the services are ready (polling per second)
599 # 2) bail out after a longer timeout.
Khen Nursimulua54b6632016-10-18 18:01:25 -0400600 print "Waiting for all containers to be ready ..."
khenaidoo1243ee92017-07-17 15:54:06 -0400601 self.wait_till('Not all services are up',
602 self._is_voltha_ensemble_ready,
603 interval=1,
604 timeout=30)
Khen Nursimulua54b6632016-10-18 18:01:25 -0400605
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400606 # Scale voltha to 10 instances
607 print "Scale voltha to 10 instances ..."
608 cmd = command_defs['docker_compose_scale_voltha_to_10']
609 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
610 self.assertEqual(rc, 0)
611
612 # Verify that 10 instances are running
613 print "Verify 10 instances of voltha are running ..."
614 cmd = command_defs['docker_compose_scaled_voltha_ps']
615 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
616 self.assertEqual(rc, 0)
617 self.assertEqual(out.split(), ['10'])
618 finally:
619 print "Stopping all containers ..."
620 # clean up all created containers for this test
621 self._stop_and_remove_all_containers()
622
623 print "Test_10_scale_voltha_End:------------------took {} secs " \
624 "\n\n".format(time.time() - t0)
625
626 def _start_all_containers(self):
627 cmd = command_defs['docker_compose_start_all']
628 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
629 self.assertEqual(rc, 0)
630
khenaidoo1243ee92017-07-17 15:54:06 -0400631 def _is_voltha_ensemble_ready(self):
632 res = verify_all_services_healthy(LOCAL_CONSUL)
633 if not res:
634 print "Not all consul services are ready ..."
635 return res
636
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400637 def _run_consul(self):
638 # run consul
639 cmd = command_defs['docker_compose_start_consul']
640 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
641 self.assertEqual(rc, 0)
642
643 # verify consul is up
644 cmd = command_defs['docker_compose_is_consul_up']
645 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
646 self.assertEqual(rc, 0)
647 self.assertIn('compose_consul_1', out)
648
649 def _stop_and_remove_all_containers(self):
650 # check if there are any running containers first
651 cmd = command_defs['docker_ps']
652 out, err, rc = run_command_to_completion_with_stdout_in_list(cmd)
653 self.assertEqual(rc, 0)
654 if len(out) > 1: # not counting docker ps header
655 cmd = command_defs['docker_stop_and_remove_all_containers']
656 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
657 self.assertEqual(rc, 0)
658
659 def _stop_docker_container_by_id(self, instance_id):
660 # stop
661 cmd = command_defs['docker_stop'] + " {}".format(instance_id)
662 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
663 self.assertEqual(rc, 0)
664
665 # remove
666 cmd = command_defs['docker_rm'] + " {}".format(instance_id)
667 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
668 self.assertEqual(rc, 0)
669
670 def _source_env(self):
671 # Go to voltha root directory
672 res = os.system('cd {}'.format(this_dir))
673 assert res == 0
674
675 # set the env
676 command = ['bash', '-c', '. env.sh']
677 proc = subprocess.Popen(command, stdout=subprocess.PIPE,
678 stderr=subprocess.PIPE)
679
680 if proc.wait() != 0:
681 err_msg = "Failed to source the environment'"
682 raise RuntimeError(err_msg)
683
684 env = os.environ.copy()
685 return env
Khen Nursimulua54b6632016-10-18 18:01:25 -0400686
687 def _wait_for_consul_to_be_ready(self):
688 # Consul is ready when it's leader ip and port is set. The maximum
689 # time to wait of 60 secs as consul should be ready by then
690 max_wait_time = 60
691 t0 = time.time()
692
693 while True:
694 # Get the docker IP address and port number of the consul instance
695 print "waiting for consul to be ready ..."
696 cmd = command_defs['consul_get_leader_ip_port']
697 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
698 out = out.strip()
699 if rc != 0:
700 # Something is wrong, return
701 return -1 # error
702 elif out is not None and out != '':
703 return 0 # found something
704 elif time.time() - t0 > max_wait_time:
705 return -1 # consul should have come up by this time
706 else:
707 time.sleep(2) # constant sleep for testing