blob: a16efe70fd71abfe789d1d5ffd12a45504bf335f [file] [log] [blame]
Khen Nursimulu37a9bf82016-10-16 20:11:31 -04001#!/usr/bin/env python
2#
3# Copyright 2016 the original author or authors.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17import subprocess
18import time
19import logging
20import os
21import json
22from unittest import TestCase
23import re
24
25this_dir = os.path.abspath(os.path.dirname(__file__))
26
27from test_utils import run_command_to_completion_with_raw_stdout, \
28 is_open, \
29 is_valid_ip, \
30 run_long_running_command_with_timeout, \
31 run_command_to_completion_with_stdout_in_list
32
33log = logging.getLogger(__name__)
34
35LOCAL_CONSUL_URL = "http://localhost:8500"
36LOCAL_CONSUL_DNS = "@localhost -p 8600"
37DOCKER_COMPOSE_FILE = "compose/docker-compose-system-test.yml"
38DOCKER_COMPOSE_FILE_SERVICES_COUNT = 7
39
40command_defs = dict(
41 makefile_fetch_images="grep \"docker pull\" Makefile",
42 make="make",
43 make_fetch="make fetch",
44 remove_env_directory="rm -rf venv-linux",
45 make_clean="make clean",
46 docker_images="docker images",
47 docker_stop="docker stop",
48 docker_rm="docker rm",
49 fluentd_logs="less /tmp/fluentd/data.log",
50 docker_voltha_logs="docker logs -f compose_voltha_1",
51 docker_compose_logs="docker-compose -f {} logs".format(
52 DOCKER_COMPOSE_FILE),
53 docker_stop_and_remove_all_containers="docker stop `docker ps -q` ; "
54 "docker rm `docker ps -a -q`",
55 docker_start_voltha="docker run -ti --rm cord/voltha",
56 docker_start_voltha_with_consul_ip="docker run -ti --rm --net="
57 "compose_default cord/voltha "
Khen Nursimulu96bb5322016-11-09 20:16:03 -080058 "/voltha/voltha/main.py --consul=",
Khen Nursimulu37a9bf82016-10-16 20:11:31 -040059 docker_get_consul_ip="docker inspect "
60 "compose_consul_1 | jq -r "
61 "'.[0].NetworkSettings.Networks."
62 "compose_default.IPAddress'",
63 docker_compose_start_consul="docker-compose -f {} up -d "
64 "consul".format(DOCKER_COMPOSE_FILE),
65 docker_compose_start_all="docker-compose -f {} up -d "
66 .format(DOCKER_COMPOSE_FILE),
67 docker_compose_stop="docker-compose -f {} stop"
68 .format(DOCKER_COMPOSE_FILE),
69 docker_compose_rm_f="docker-compose -f {} rm -f"
70 .format(DOCKER_COMPOSE_FILE),
71 docker_compose_ps="docker-compose -f {} ps".format(DOCKER_COMPOSE_FILE),
72 docker_ps="docker ps",
73 docker_ps_count="docker ps -q | wc -l",
74 docker_compose_is_consul_up="docker-compose -f {} ps | grep consul"
75 .format(DOCKER_COMPOSE_FILE),
76 consul_get_leader_ip_port="curl -s {}/v1/status/leader | jq -r ."
77 .format(LOCAL_CONSUL_URL),
78 docker_compose_services_running="docker-compose -f {} ps -q"
79 .format(DOCKER_COMPOSE_FILE),
80 docker_compose_services_running_count="docker-compose -f {} ps -q | "
81 "grep Up "
82 "| wc -l".format(
83 DOCKER_COMPOSE_FILE),
84 docker_compose_services="docker-compose -f {} config --services"
85 .format(DOCKER_COMPOSE_FILE),
86 consul_get_services="curl -s {}/v1/catalog/services | jq -r ."
87 .format(LOCAL_CONSUL_URL),
88 consul_get_srv_voltha_health="curl -s {}/v1/catalog/service/voltha-health "
89 "| jq -r .".format(LOCAL_CONSUL_URL),
90 kafka_client_run_10_secs="python kafka/kafka-consumer.py -r 10",
91 consul_get_voltha_rest_a_record="dig {} voltha-health.service.consul"
92 .format(LOCAL_CONSUL_DNS),
93 consul_get_voltha_rest_ip="dig {} +short voltha-health.service.consul"
94 .format(LOCAL_CONSUL_DNS),
95 consul_get_voltha_service_port="dig {} +short "
96 "voltha-health.service.consul SRV | "
97 " awk \'{{print $3}}'"
98 .format(LOCAL_CONSUL_DNS),
99 docker_compose_scale_voltha_to_10="docker-compose -f {} scale "
100 "voltha=10".format(DOCKER_COMPOSE_FILE),
101 docker_compose_scaled_voltha_ps="docker-compose -f {} ps voltha | "
102 "grep Up | wc -l"
103 .format(DOCKER_COMPOSE_FILE),
104 consul_verify_voltha_registration="curl -s {}"
105 "/v1/kv/service/voltha/members?recurse |"
106 " jq -r .".format(LOCAL_CONSUL_DNS)
107)
108
109
110class BuildMdTests(TestCase):
111 # docker_client = Client(base_url='unix://var/run/docker.sock')
112
113 def test_01_setup(self):
114 print "Test_01_setup_Start:------------------"
115 t0 = time.time()
116
117 try:
118 # remove the venv-linux directory
119 print "Remove venv-linux ..."
120 cmd = command_defs['remove_env_directory']
121 rm_venv, err, rc = run_command_to_completion_with_raw_stdout(cmd)
122 self.assertEqual(rc, 0)
123
124 # make clean
125 print "Make clean ..."
126 cmd = command_defs['make_clean']
127 mk_clean, err, rc = run_command_to_completion_with_raw_stdout(cmd)
128 self.assertEqual(rc, 0)
129
130 # source the env
131 print "Source environment ..."
132 self._source_env()
133
134 finally:
135 print "Test_01_setup_End:------------------ took {} " \
136 "secs\n\n".format(time.time() - t0)
137
138 def test_02_make_fetch(self):
139 print "Test_02_make_fetch_Start:------------------"
140 t0 = time.time()
141
142 try:
143 # Get list of images to fetch from the Makefile
144 print "Get list of images to fetch ..."
145 cmd = command_defs['makefile_fetch_images']
146 makefile_images_to_fetch, err, rc \
147 = run_command_to_completion_with_stdout_in_list(cmd)
148 self.assertEqual(rc, 0)
149
150 images_to_fetch = []
151 for image in makefile_images_to_fetch:
152 tmp = ''.join(image.split())
153 images_to_fetch.append(tmp[len('dockerpull'):])
154
155 # make fetch
156 print "Fetching images {} ...".format(images_to_fetch)
157 cmd = command_defs['make_fetch']
158 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
159 self.assertEqual(rc, 0)
160
161 # verify that the images have been downloaded
162 print "Verify images downloaded and present locally ..."
163 cmd = command_defs['docker_images']
164 local_images, err, rc = \
165 run_command_to_completion_with_stdout_in_list(cmd)
166 self.assertEqual(rc, 0)
167
168 local_images_list = []
169 for local_image in local_images:
170 words = local_image.split()
171 local_images_list.append('{}:{}'.format(words[0], words[1]))
172
173 intersection_list = [i for i in images_to_fetch if
174 i in local_images_list]
175 assert len(intersection_list) == len(images_to_fetch)
176
177 finally:
178 print "Test_02_make_fetch_End:------------------ took {} " \
179 "secs \n\n".format(time.time() - t0)
180
181 def test_03_make(self):
182 print "Test_03_make_Start:------------------"
183 t0 = time.time()
184 try:
185 cmd = command_defs['make']
186 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
187 self.assertEqual(rc, 0)
188 finally:
Khen Nursimulua54b6632016-10-18 18:01:25 -0400189 print "Test_03_make_Start:------------------ took {} secs \n\n" \
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400190 .format(time.time() - t0)
191
192 def test_04_run_voltha_standalone_without_consul(self):
193 print "Test_04_run_voltha_standalone_without_consul_Start:------------" \
194 "------"
195 t0 = time.time()
196
197 try:
198 # Run voltha for 10 secs and verity the following lines are displayed
199 # (a subset of output messages along with a flag when found)
200 print "Start voltha ..."
201 expected_output_subset = [
202 'main.print_banner {event: (to stop: press Ctrl-C), '
203 'instance_id:',
Khen Nursimulu96bb5322016-11-09 20:16:03 -0800204 'coordinator.__init__ {event: initializing-coordinator,',
205 'grpc_server.start {event: started',
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400206 'main.<lambda> {event: twisted-reactor-started',
207 'main.startup_components {event: started-internal-services,',
208 'kafka_proxy.send_message {event: Sending message Heartbeat '
209 'message',
210 'coordinator._backoff {retry_in: 5, event: consul-not-up,'
211 ]
212
213 cmd = command_defs['docker_start_voltha']
214 command_output = run_long_running_command_with_timeout(cmd, 10)
215
216 # There should at least be 1 line in the output
217 self.assertGreater(len(command_output), 0)
218
219 # Verify that the output contained the expected_output_subset -
220 # save the docker instance id
221 print "Verify voltha started correctly ..."
222 instance_id = None
223 for ext_output in expected_output_subset:
224 match_str = next(
225 (out for out in command_output if ext_output in out),
226 None)
227 self.assertIsNotNone(match_str)
228 if "instance_id" in ext_output:
229 instance_id = re.findall(r'[0-9a-f]+', match_str)[-1]
230
231 # Now stop the voltha docker that was created
232 print "Stop voltha ..."
233 self._stop_docker_container_by_id(instance_id)
234
235
236 finally:
237 # Remove any created container
238 self._stop_and_remove_all_containers()
239
240 print "Test_04_run_voltha_standalone_without_consul_End" \
241 ":------------------ took {} secs \n\n".format(
242 time.time() - t0)
243
244 def test_05_run_consul_only(self):
245 print "Test_05_run_consul_only_Start:------------------ "
246 t0 = time.time()
247
248 try:
249 # run consul
250 print "Start consul ..."
251 self._run_consul()
252
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400253 print "Waiting for consul to be ready ..."
Khen Nursimulua54b6632016-10-18 18:01:25 -0400254 rc = self._wait_for_consul_to_be_ready()
255 self.assertEqual(rc, 0)
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400256
257 # Get the docker IP address and port number of the consul instance
258 print "Get consul leader IP ..."
259 cmd = command_defs['consul_get_leader_ip_port']
260 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
261 self.assertEqual(rc, 0)
262
263 # validate that the returned ip:port is valid and open
264 print "Verify consul IP and port is reachable ..."
265 self.assertTrue(is_open(out))
266
267 finally:
268 # clean up all created containers for this test
269 print "Stop consul ..."
270 self._stop_and_remove_all_containers()
271
272 print "Test_05_run_consul_only_End:------------------ took {} secs" \
273 "\n\n".format(time.time() - t0)
274
275 def test_06_run_voltha_standalone_with_consul_only(self):
276 print "Test_06_run_voltha_standalone_with_consul_only_Start:----------" \
277 "-------- "
278 t0 = time.time()
279
280 try:
281 # run consul first
282 print "Start consul ..."
283 self._run_consul()
284
285 # get consul ip
286 print "Get consul IP ..."
287 cmd = command_defs['docker_get_consul_ip']
288 consul_ip, err, rc = run_command_to_completion_with_raw_stdout(cmd)
289 self.assertEqual(rc, 0)
290 self.assertIsNotNone(consul_ip)
291
292 # start voltha now for 15 secs and verify it can now connect to
293 # consul - following message in the output
294 print "Start voltha with consul IP ..."
295 expected_pattern = ['coordinator', 'event: created-consul-session']
296 cmd = command_defs['docker_start_voltha_with_consul_ip'] + \
297 '{}:8500'.format(consul_ip.strip())
298 command_output = run_long_running_command_with_timeout(cmd, 10)
299
300 # Verify the output of voltha and get the container instance id
301 print "Verify voltha is registered with consul ..."
302 instance_id = None
303 for out in command_output:
304 if all(ep for ep in expected_pattern if ep in out):
305 self.assertTrue(True)
306 instance_id = re.findall(r'[0-9a-f]+', out)[-1]
307 break
308
309 self.assertIsNotNone(instance_id)
310
311 # Verify Voltha's self-registration with consul
312 expected_output = ['ModifyIndex', 'CreateIndex', 'Session',
313 'Value',
314 'Flags', 'Key', 'LockIndex']
315
316 cmd = command_defs['consul_verify_voltha_registration']
317 registration_info, err, rc = \
318 run_command_to_completion_with_raw_stdout(cmd)
319 self.assertEqual(rc, 0)
320 try:
321 jr_info = json.loads(registration_info)
322 intersect_elems = [e for e in jr_info[0] if
323 e in expected_output]
324 self.assertEqual(len(expected_output), len(intersect_elems))
325 except Exception as e:
326 self.assertRaises(e)
327
328 # stop voltha
329 print "Stop voltha ..."
330 self._stop_docker_container_by_id(instance_id)
331
332 # check the service has deregistered
333 print "Verify voltha is no longer registered in consul..."
334 cmd = command_defs['consul_verify_voltha_registration']
335 registration_info, err, rc = \
336 run_command_to_completion_with_raw_stdout(cmd)
337 self.assertEqual(rc, 0)
338 self.assertEqual(registration_info, '')
339
340 finally:
341 # clean up all created containers for this test
342 print "Stop consul ..."
343 self._stop_and_remove_all_containers()
344
345 print "Test_06_run_voltha_standalone_with_consul_only_End:--------" \
346 "---------- took {} " \
347 "secs \n\n".format(time.time() - t0)
348
349 def test_07_start_all_containers(self):
350 print "Test_07_start_all_containers_Start:------------------ "
351 t0 = time.time()
352
353 try:
354 # Pre-test - clean up all running docker containers
355 print "Pre-test: Removing all running containers ..."
356 self._stop_and_remove_all_containers()
357
358 # get a list of services in the docker-compose file
359 print "Getting list of services in docker compose file ..."
360 cmd = command_defs['docker_compose_services']
361 services, err, rc = run_command_to_completion_with_raw_stdout(cmd)
362 self.assertEqual(rc, 0)
363 docker_service_list = services.split()
364 self.assertGreaterEqual(len(docker_service_list),
365 DOCKER_COMPOSE_FILE_SERVICES_COUNT)
366
367 # start all the containers
368 print "Starting all containers ..."
369 cmd = command_defs['docker_compose_start_all']
370 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
371 self.assertEqual(rc, 0)
372
Khen Nursimulua54b6632016-10-18 18:01:25 -0400373 print "Waiting for all containers to be ready ..."
374 rc, not_found_list = self._wait_for_all_containers_to_ready()
375 if rc:
376 print "Not found patterns:{}".format(not_found_list)
377 self.assertEqual(rc, 0)
378
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400379 # verify that all containers are running
380 print "Verify all services are running using docker command ..."
381 for service in docker_service_list:
382 cmd = command_defs['docker_compose_ps'] + ' {} | wc -l'.format(
383 service)
384 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
385 self.assertEqual(rc, 0)
386 self.assertGreaterEqual(out, 3) # 2 are for headers
387
388 # Verify that 'docker ps' return the same number of running process
389 cmd = command_defs['docker_ps_count']
390 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
391 self.assertEqual(rc, 0)
Khen Nursimulu283d7682016-11-11 16:37:32 -0500392 self.assertGreaterEqual(out.split(), [str(len(
393 docker_service_list))])
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400394
395 # Retrieve the list of services from consul and validate against
396 # the list obtained from docker composed
397 print "Verify all services are registered in consul ..."
398 expected_services = ['consul-rest', 'fluentd-intake',
399 'chameleon-rest', 'voltha-grpc',
400 'voltha-health',
401 'consul-8600', 'zookeeper', 'consul', 'kafka']
402
403 cmd = command_defs['consul_get_services']
404 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
405 self.assertEqual(rc, 0)
406 try:
407 consul_services = json.loads(out)
408 intersected_services = [s for s in expected_services if
409 s in consul_services]
410 self.assertEqual(len(intersected_services),
411 len(expected_services))
412 # services_match = 0
413 # for d_service in docker_service_list:
414 # for c_service in consul_services:
415 # if c_service.find(d_service) != -1:
416 # services_match += 1
417 # print d_service, c_service
418 # break
419 # self.assertEqual(services_match, len(docker_service_list))
420 except Exception as e:
421 self.assertRaises(e)
422
423 # Verify the service record of the voltha service
424 print "Verify the service record of voltha in consul ..."
425 expected_srv_elements = ['ModifyIndex', 'CreateIndex',
426 'ServiceEnableTagOverride', 'Node',
427 'Address', 'TaggedAddresses', 'ServiceID',
428 'ServiceName', 'ServiceTags',
429 'ServiceAddress', 'ServicePort']
430 cmd = command_defs['consul_get_srv_voltha_health']
431 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
432 self.assertEqual(rc, 0)
433 try:
434 srv = json.loads(out)
435 intersect_elems = [e for e in srv[0] if
436 e in expected_srv_elements]
437 self.assertEqual(len(expected_srv_elements),
438 len(intersect_elems))
439 except Exception as e:
440 self.assertRaises(e)
441
442 # Verify kafka client is receiving the messages
443 print "Verify kafka client is receiving the heartbeat messages ..."
444 expected_pattern = ['voltha-heartbeat', 'Heartbeat message']
445 cmd = command_defs['kafka_client_run_10_secs']
446 kafka_client_output = run_long_running_command_with_timeout(cmd,
Khen Nursimulua54b6632016-10-18 18:01:25 -0400447 20)
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400448
449 # Verify the kafka client output
450 # instance id
451 found = False
452 for out in kafka_client_output:
453 if all(ep for ep in expected_pattern if ep in out):
454 found = True
455 break
456 self.assertTrue(found)
457
458 # verify docker-compose logs are being produced - just get the
459 # first work of each line
460 print "Verify docker compose logs has output from all the services " \
461 "..."
462 expected_output = ['voltha_1', 'fluentd_1', 'consul_1',
463 'registrator_1', 'kafka_1', 'zookeeper_1',
464 'chameleon_1']
465 cmd = command_defs['docker_compose_logs']
466 docker_compose_logs = run_long_running_command_with_timeout(cmd, 5,
467 0)
468 intersected_logs = [l for l in expected_output if
469 l in docker_compose_logs]
470 self.assertEqual(len(intersected_logs), len(expected_output))
471
472 # TODO: file in /tmp/fluentd/ cannot be found
473 # # verify fluentd logs are being produced - we will just verify
474 # that there are "voltha.logging" in the logs
475 # os.environ["PYTHONPATH"] += os.pathsep + "/tmp/fluentd/"
476 # os.environ['PATH'] += os.pathsep + "/tmp/fluentd/"
477 # expected_output=['voltha.logging']
478 # cmd = command_defs['fluentd_logs']
479 # fluentd_logs, err = run_command_to_completion_with_raw_stdout(cmd)
480 # # self.assertIsNone(err)
481 # print err
482 # intersected_logs = [l for l in expected_output if
483 # l in fluentd_logs]
484 # self.assertEqual(len(intersected_logs), len(expected_output))
485
486 # verify docker voltha logs are being produced - we will just verify
487 # some
488 # key messages in the logs
489 print "Verify docker voltha logs are produced ..."
490 expected_output = ['kafka_proxy.send_message',
491 'coordinator._renew_session', 'main.heartbeat']
492 cmd = command_defs['docker_voltha_logs']
493 docker_voltha_logs = run_long_running_command_with_timeout(cmd,
Khen Nursimulu96bb5322016-11-09 20:16:03 -0800494 0.5, 3)
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400495 intersected_logs = [l for l in expected_output if
496 l in docker_voltha_logs]
497 self.assertEqual(len(intersected_logs), len(expected_output))
498
499 finally:
500 print "Stopping all containers ..."
501 # clean up all created containers for this test
502 self._stop_and_remove_all_containers()
503
504 print "Test_07_start_all_containers_End:------------------ took {}" \
505 " secs \n\n".format(time.time() - t0)
506
507 def test_08_stop_all_containers_started_using_docker_compose(self):
508 print "Test_08_stop_all_containers_started_using_docker_compose_Start:" \
509 "------------------ "
510 t0 = time.time()
511
512 try:
513 # commands to stop and clear the docker images
514 cmds = [command_defs['docker_compose_stop'],
515 command_defs['docker_compose_rm_f']]
516
517 print "Stopping all containers ..."
518 for cmd in cmds:
519 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
520 self.assertEqual(rc, 0)
521
522 # Verify that no docker process is running
523 print "Verify no containers is running..."
524 cmd = command_defs['docker_compose_services_running']
525 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
526 self.assertEqual(rc, 0)
527
528 finally:
529 print "Test_08_stop_all_containers_started_using_docker_compose_:" \
530 "------------------ took {} secs \n\n".format(
531 time.time() - t0)
532
533 def test_09_dig_consul_command(self):
534 print "Test_09_dig_consul_command_Start:------------------"
535 t0 = time.time()
536
537 try:
538 # start all containers
539 print "Start all containers..."
540 self._start_all_containers()
541
Khen Nursimulua54b6632016-10-18 18:01:25 -0400542 print "Waiting for all containers to be ready ..."
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400543 time.sleep(10)
Khen Nursimulua54b6632016-10-18 18:01:25 -0400544 rc, not_found_list = self._wait_for_all_containers_to_ready()
545 if rc:
546 print "Not found patterns:{}".format(not_found_list)
547 self.assertEqual(rc, 0)
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400548
549 # Get the IP address(es) for voltha's REST interface
550 print "Get IP of Voltha REST interface..."
551 cmd = command_defs['consul_get_voltha_rest_a_record']
552 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
553 self.assertEqual(rc, 0)
554 self.assertGreaterEqual(out.find("voltha-health.service.consul"),
555 0)
556
557 # Get only the ip address
558 cmd = command_defs['consul_get_voltha_rest_ip']
559 ip, err, rc = run_command_to_completion_with_raw_stdout(cmd)
560 self.assertEqual(rc, 0)
561 self.assertTrue(is_valid_ip(ip))
562
563 # Get the exposed service port
564 print "Get Voltha exposed service port..."
565 cmd = command_defs['consul_get_voltha_service_port']
566 port, err, rc = run_command_to_completion_with_raw_stdout(cmd)
567 self.assertEqual(rc, 0)
568 # Verify that we can connect to the port using the previously
569 # acquired ip
570 print "Verify connectivity with voltha ip and port..."
571 self.assertTrue(is_open('{}:{}'.format(ip, port)))
572 finally:
573 print "Stopping all containers ..."
574 # clean up all created containers for this test
575 self._stop_and_remove_all_containers()
576
577 print "Test_09_dig_consul_command_Start_End:------------------" \
578 "took {} secs \n\n".format(time.time() - t0)
579
580 def test_10_scale_voltha(self):
581 print "Test_10_scale_voltha_Start:------------------"
582 t0 = time.time()
583
584 try:
585 # start all containers
586 print "Start all containers..."
587 self._start_all_containers()
588
Khen Nursimulua54b6632016-10-18 18:01:25 -0400589 print "Waiting for all containers to be ready ..."
590 time.sleep(10)
591 rc, not_found_list = self._wait_for_all_containers_to_ready()
592 if rc:
593 print "Not found patterns:{}".format(not_found_list)
594 self.assertEqual(rc, 0)
595
Khen Nursimulu37a9bf82016-10-16 20:11:31 -0400596 # Scale voltha to 10 instances
597 print "Scale voltha to 10 instances ..."
598 cmd = command_defs['docker_compose_scale_voltha_to_10']
599 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
600 self.assertEqual(rc, 0)
601
602 # Verify that 10 instances are running
603 print "Verify 10 instances of voltha are running ..."
604 cmd = command_defs['docker_compose_scaled_voltha_ps']
605 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
606 self.assertEqual(rc, 0)
607 self.assertEqual(out.split(), ['10'])
608 finally:
609 print "Stopping all containers ..."
610 # clean up all created containers for this test
611 self._stop_and_remove_all_containers()
612
613 print "Test_10_scale_voltha_End:------------------took {} secs " \
614 "\n\n".format(time.time() - t0)
615
616 def _start_all_containers(self):
617 cmd = command_defs['docker_compose_start_all']
618 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
619 self.assertEqual(rc, 0)
620
621 def _run_consul(self):
622 # run consul
623 cmd = command_defs['docker_compose_start_consul']
624 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
625 self.assertEqual(rc, 0)
626
627 # verify consul is up
628 cmd = command_defs['docker_compose_is_consul_up']
629 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
630 self.assertEqual(rc, 0)
631 self.assertIn('compose_consul_1', out)
632
633 def _stop_and_remove_all_containers(self):
634 # check if there are any running containers first
635 cmd = command_defs['docker_ps']
636 out, err, rc = run_command_to_completion_with_stdout_in_list(cmd)
637 self.assertEqual(rc, 0)
638 if len(out) > 1: # not counting docker ps header
639 cmd = command_defs['docker_stop_and_remove_all_containers']
640 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
641 self.assertEqual(rc, 0)
642
643 def _stop_docker_container_by_id(self, instance_id):
644 # stop
645 cmd = command_defs['docker_stop'] + " {}".format(instance_id)
646 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
647 self.assertEqual(rc, 0)
648
649 # remove
650 cmd = command_defs['docker_rm'] + " {}".format(instance_id)
651 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
652 self.assertEqual(rc, 0)
653
654 def _source_env(self):
655 # Go to voltha root directory
656 res = os.system('cd {}'.format(this_dir))
657 assert res == 0
658
659 # set the env
660 command = ['bash', '-c', '. env.sh']
661 proc = subprocess.Popen(command, stdout=subprocess.PIPE,
662 stderr=subprocess.PIPE)
663
664 if proc.wait() != 0:
665 err_msg = "Failed to source the environment'"
666 raise RuntimeError(err_msg)
667
668 env = os.environ.copy()
669 return env
Khen Nursimulua54b6632016-10-18 18:01:25 -0400670
671 def _wait_for_consul_to_be_ready(self):
672 # Consul is ready when it's leader ip and port is set. The maximum
673 # time to wait of 60 secs as consul should be ready by then
674 max_wait_time = 60
675 t0 = time.time()
676
677 while True:
678 # Get the docker IP address and port number of the consul instance
679 print "waiting for consul to be ready ..."
680 cmd = command_defs['consul_get_leader_ip_port']
681 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
682 out = out.strip()
683 if rc != 0:
684 # Something is wrong, return
685 return -1 # error
686 elif out is not None and out != '':
687 return 0 # found something
688 elif time.time() - t0 > max_wait_time:
689 return -1 # consul should have come up by this time
690 else:
691 time.sleep(2) # constant sleep for testing
692
693 def _wait_for_all_containers_to_ready(self):
694 # After the containers have been started using docker-compose, look
695 # at the logs for the following patterns to decide if the containers
696 # are up and in sync:
697 #
698 # For registrator, look for
699 # "(.*)registrator_1(.*)Listening for Docker events"
700 #
701 # For voltha, zookeeper and kafka look for these patterns
702 # "(.*)voltha_1(.*)main.heartbeat {status: up, uptime:"
703 # "(.*)voltha_1(.*)kafka_proxy.send_message {event: Successfully sent message Heartbeat message"
704 #
705 # For fluentd, look for
706 # "(.*)fluentd_1(.*)listening fluent socket on"
707 #
708 # For chameleon, look for
709 # "(.*)chameleon_1(.*)main.startup_components {event:
710 # started-internal-services, instance_id: compose_chameleon_1}"
711 #
712 # For consul, look for
713 # "(.*)consul_1(.*)agent: Synced service(.*)consul(.*)8500"
714 # "(.*)consul_1(.*)agent: Synced service(.*)consul(.*)8600:udp"
715 # "(.*)consul_1(.*)agent: Synced service(.*)fluentd"
716 # "(.*)consul_1(.*)agent: Synced service(.*)voltha"
717 # "(.*)consul_1(.*)agent: Synced service(.*)voltha(.*):8880"
718 # "(.*)consul_1(.*)agent: Synced service(.*)chameleon(.*):8881"
719 # "(.*)consul_1(.*)agent: Synced service(.*)zookeeper(.*):2181"
720 # "(.*)consul_1(.*)agent: Synced service(.*)kafka(.*):9092"
721 #
722 expected_output = [
723 "(.*)registrator_1(.*)Listening for Docker events",
724 "(.*)voltha_1(.*)main.heartbeat {status: up, uptime:",
725 "(.*)voltha_1(.*)kafka_proxy.send_message {event: Successfully "
726 "sent message Heartbeat message",
727 "(.*)fluentd_1(.*)listening fluent socket on",
728 "(.*)chameleon_1(.*)main.startup_components {event: "
729 "started-internal-services, instance_id: compose_chameleon_1}",
730 "(.*)consul_1(.*)agent: Synced service(.*)consul(.*)8500",
731 "(.*)consul_1(.*)agent: Synced service(.*)consul(.*)8600:udp",
732 "(.*)consul_1(.*)agent: Synced service(.*)fluentd",
733 "(.*)consul_1(.*)agent: Synced service(.*)voltha(.*):(?!8880)",
734 "(.*)consul_1(.*)agent: Synced service(.*)voltha(.*):8880",
735 "(.*)consul_1(.*)agent: Synced service(.*)chameleon(.*):8881",
736 "(.*)consul_1(.*)agent: Synced service(.*)zookeeper(.*):2181",
737 "(.*)consul_1(.*)agent: Synced service(.*)kafka(.*):9092"
738 ]
739 pattern_found = []
Khen Nursimulu94026692016-11-10 13:52:00 -0800740 max_wait_time = 60 # wait up to 1 minute before declaring a failure
Khen Nursimulua54b6632016-10-18 18:01:25 -0400741
742 def _stop_process(proc):
743 try:
744 proc.terminate()
745 proc.wait()
746 # In principle this 'reset' should not be required.
747 # However, without it, the terminal is left in a funny
748 # state and required
749 subprocess.Popen(['reset']).wait()
750 except Exception as e:
751 print "Received exception {} when killing process " \
752 .format(repr(e), )
753
754 try:
755 t0 = time.time()
756 env = os.environ.copy()
757 proc = subprocess.Popen(
758 command_defs['docker_compose_logs'],
759 env=env,
760 shell=True,
761 stdout=subprocess.PIPE,
762 stderr=subprocess.PIPE,
763 bufsize=1
764 )
765 for line in iter(proc.stdout.readline, b''):
766 ansi_escape = re.compile(r'\x1b[^m]*m')
767 line = ansi_escape.sub('', line)
768 for pattern in expected_output:
769 if re.match(pattern, line, re.I):
770 if pattern not in pattern_found:
771 pattern_found.append(pattern)
772 break
773 # Check if we found all patterns yet
774 if len(pattern_found) == len(expected_output):
775 _stop_process(proc)
776 return 0, [] # success
777 elif time.time() - t0 > max_wait_time:
778 _stop_process(proc)
779 not_found = [p for p in expected_output if p not in
780 pattern_found]
781 return -1, not_found # failure
782 except Exception as e:
783 print 'Exception {} '.format(repr(e))
784 return -1, []