tests/itests/docutests/build_md_test.py

#!/usr/bin/env python
#
# Copyright 2016 the original author or authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import subprocess
import select
import time
import logging
from common.utils.consulhelpers import verify_all_services_healthy, get_endpoint_from_consul
import os
import json
from unittest import TestCase
import re
import simplejson
import sys
import traceback

this_dir = os.path.abspath(os.path.dirname(__file__))

from test_utils import run_command_to_completion_with_raw_stdout, \
    is_open, \
    is_valid_ip, \
    run_long_running_command_with_timeout, \
    run_command_to_completion_with_stdout_in_list

log = logging.getLogger(__name__)

LOCAL_CONSUL = "localhost:8500"
LOCAL_CONSUL_URL = "http://%s" % LOCAL_CONSUL
LOCAL_CONSUL_DNS = "@localhost -p 8600"
DOCKER_COMPOSE_PROJECT = "compose"
DOCKER_COMPOSE_FILE = "compose/docker-compose-docutests.yml"
DOCKER_COMPOSE_FILE_SERVICES_COUNT = 7

command_defs = dict(
    makefile_fetch_images="grep \"docker pull\" Makefile",
    make="make",
    make_clean_build="make -e DOCKER_CACHE_ARG=--no-cache build",
    make_fetch="make fetch",
    remove_env_directory="rm -rf venv-linux",
    make_clean="make clean",
    docker_images="docker images",
    docker_stop="docker stop",
    docker_rm="docker rm",
    fluentd_logs="less /tmp/fluentd/data.log",
    docker_voltha_logs="docker-compose -p {} -f {} logs voltha"
        .format(DOCKER_COMPOSE_PROJECT, DOCKER_COMPOSE_FILE),
    docker_compose_logs="docker-compose -p {} -f {} logs"
        .format(DOCKER_COMPOSE_PROJECT, DOCKER_COMPOSE_FILE),
    docker_stop_and_remove_all_containers="docker stop `docker ps -q` ; "
                                          "docker rm `docker ps -a -q`",
    docker_start_voltha="docker run -ti --rm voltha/voltha",
    docker_start_voltha_with_consul_ip="docker run -ti --rm --net="
                                       "compose_default voltha/voltha "
                                       "/voltha/voltha/main.py --consul=",
    docker_get_consul_ip="docker inspect "
                         "compose_consul_1 | jq -r "
                         "'.[0].NetworkSettings.Networks."
                         "compose_default.IPAddress'",
    docker_compose_start_consul="docker-compose -p {} -f {} up -d "
                                "consul".format(DOCKER_COMPOSE_PROJECT, DOCKER_COMPOSE_FILE),
    docker_compose_start_all="docker-compose -p {} -f {} up -d "
        .format(DOCKER_COMPOSE_PROJECT, DOCKER_COMPOSE_FILE),
    docker_compose_stop="docker-compose -p {} -f {} stop"
        .format(DOCKER_COMPOSE_PROJECT, DOCKER_COMPOSE_FILE),
    docker_compose_rm_f="docker-compose -p {} -f {} rm -f"
        .format(DOCKER_COMPOSE_PROJECT, DOCKER_COMPOSE_FILE),
    docker_compose_down="docker-compose -p {} -f {} down"
        .format(DOCKER_COMPOSE_PROJECT, DOCKER_COMPOSE_FILE),
    docker_compose_ps="docker-compose -p {} -f {} ps"
        .format(DOCKER_COMPOSE_PROJECT, DOCKER_COMPOSE_FILE),
    docker_ps="docker ps",
    docker_ps_count="docker ps -q | wc -l",
    docker_compose_is_consul_up="docker-compose -p {} -f {} ps | grep consul"
        .format(DOCKER_COMPOSE_PROJECT, DOCKER_COMPOSE_FILE),
    consul_get_leader_ip_port="curl -s {}/v1/status/leader | jq -r ."
        .format(LOCAL_CONSUL_URL),
    docker_compose_services_running="docker-compose -p {} -f {} ps -q"
        .format(DOCKER_COMPOSE_PROJECT, DOCKER_COMPOSE_FILE),
    docker_compose_services_running_count="docker-compose -p {} -f {} ps -q | "
                                          "grep Up | wc -l"
        .format(DOCKER_COMPOSE_PROJECT, DOCKER_COMPOSE_FILE),
    docker_compose_services="docker-compose -p {} -f {} config --services"
        .format(DOCKER_COMPOSE_PROJECT, DOCKER_COMPOSE_FILE),
    consul_get_services="curl -s {}/v1/catalog/services | jq -r ."
        .format(LOCAL_CONSUL_URL),
    consul_get_srv_voltha_health="curl -s {}/v1/catalog/service/voltha-health "
                                 "| jq -r .".format(LOCAL_CONSUL_URL),
    kafka_client_run="kafkacat -b {} -L",
    kafka_client_heart_check="kafkacat -o end -b {} -C -t voltha.heartbeat -c 1",
    consul_get_voltha_rest_a_record="dig {} voltha-health.service.consul"
        .format(LOCAL_CONSUL_DNS),
    consul_get_voltha_rest_ip="dig {} +short voltha-health.service.consul"
        .format(LOCAL_CONSUL_DNS),
    consul_get_voltha_service_port="dig {} +short "
                                   "voltha-health.service.consul SRV | "
                                   " awk \'{{print $3}}'"
        .format(LOCAL_CONSUL_DNS),
    docker_compose_scale_voltha_to_10="docker-compose -p {} -f {} scale "
                                      "voltha=10"
        .format(DOCKER_COMPOSE_PROJECT, DOCKER_COMPOSE_FILE),
    docker_compose_scaled_voltha_ps="docker-compose -p {} -f {} ps voltha | "
                                    "grep Up | wc -l"
        .format(DOCKER_COMPOSE_PROJECT, DOCKER_COMPOSE_FILE),
    consul_verify_voltha_registration="curl -s {}"
                                      "/v1/kv/service/voltha/members?recurse |"
                                      " jq -r .".format(LOCAL_CONSUL_DNS)
)


class BuildMdTests(TestCase):
    # docker_client = Client(base_url='unix://var/run/docker.sock')

    def wait_till(self, msg, predicate, interval=0.1, timeout=5.0):
        deadline = time.time() + timeout
        while time.time() < deadline:
            if predicate():
                return
            time.sleep(interval)
        self.fail('Timed out while waiting for condition: {}'.format(msg))


    def test_01_setup(self):
        print "Test_01_setup_Start:------------------"
        t0 = time.time()

        try:
            # remove the venv-linux directory
            print "Remove venv-linux ..."
            cmd = command_defs['remove_env_directory']
            rm_venv, err, rc = run_command_to_completion_with_raw_stdout(cmd)
            self.assertEqual(rc, 0)

            # make clean
            print "Make clean ..."
            cmd = command_defs['make_clean']
            mk_clean, err, rc = run_command_to_completion_with_raw_stdout(cmd)
            self.assertEqual(rc, 0)

            # source the env
            print "Source environment ..."
            self._source_env()

        finally:
            print "Test_01_setup_End:------------------ took {} " \
                  "secs\n\n".format(time.time() - t0)

    def test_02_make_fetch(self):
        print "Test_02_make_fetch_Start:------------------"
        t0 = time.time()

        try:
            # Get list of images to fetch from the Makefile
            print "Get list of images to fetch ..."
            cmd = command_defs['makefile_fetch_images']
            makefile_images_to_fetch, err, rc \
                = run_command_to_completion_with_stdout_in_list(cmd)
            self.assertEqual(rc, 0)

            images_to_fetch = []
            for image in makefile_images_to_fetch:
                tmp = ''.join(image.split())
                images_to_fetch.append(tmp[len('dockerpull'):])

            # make fetch
            print "Fetching images {} ...".format(images_to_fetch)
            cmd = command_defs['make_fetch']
            out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
            self.assertEqual(rc, 0)

            # verify that the images have been downloaded
            print "Verify images downloaded and present locally ..."
            cmd = command_defs['docker_images']
            local_images, err, rc = \
                run_command_to_completion_with_stdout_in_list(cmd)
            self.assertEqual(rc, 0)

            local_images_list = []
            for local_image in local_images:
                words = local_image.split()
                local_images_list.append('{}:{}'.format(words[0], words[1]))

            intersection_list = [i for i in images_to_fetch if
                                 i in local_images_list]
            assert len(intersection_list) == len(images_to_fetch)

        finally:
            print "Test_02_make_fetch_End:------------------ took {} " \
                  "secs \n\n".format(time.time() - t0)

    def test_03_make(self):
        print "Test_03_make_build_Start:------------------"
        t0 = time.time()
        try:
            cmd = command_defs['make_clean_build']
            out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
            self.assertEqual(rc, 0)
        finally:
            print "Test_03_make_build_Start:------------------ took {} secs \n\n" \
                .format(time.time() - t0)

    def test_04_run_voltha_standalone_without_consul(self):
        print "Test_04_run_voltha_standalone_without_consul_Start:------------" \
              "------"
        t0 = time.time()

        try:
            # Run voltha for 10 secs and verity the following lines are displayed
            #  (a subset of output messages along with a flag when found)
            print "Start voltha ..."
            expected_output_subset = [
                'main.print_banner {event: (to stop: press Ctrl-C), '
                'instance_id:',
                'coordinator.__init__ {event: initializing-coordinator,',
                'grpc_server.start {event: started',
                'main.<lambda> {event: twisted-reactor-started',
                'main.startup_components {event: started-internal-services,',
                'kafka_proxy.start {event: started,',
                'coordinator._backoff {retry_in: 5, event: consul-not-up,'
            ]

            cmd = command_defs['docker_start_voltha']
            command_output = run_long_running_command_with_timeout(cmd, 10)

            # There should at least be 1 line in the output
            self.assertGreater(len(command_output), 0)

            # Verify that the output contained the expected_output_subset -
            # save the docker instance id
            print "Verify voltha started correctly ..."
            instance_id = None
            for ext_output in expected_output_subset:
                match_str = next(
                    (out for out in command_output if ext_output in out),
                    None)
                self.assertIsNotNone(match_str)
                if "instance_id" in ext_output:
                    instance_id = re.findall(r'[0-9a-f]+', match_str)[-1]

            # Now stop the voltha docker that was created
            print "Stop voltha ..."
            self._stop_docker_container_by_id(instance_id)


        finally:
            # Remove any created container
            self._stop_and_remove_all_containers()

            print "Test_04_run_voltha_standalone_without_consul_End" \
                  ":------------------ took {} secs \n\n".format(
                time.time() - t0)

    def test_05_run_consul_only(self):
        print "Test_05_run_consul_only_Start:------------------ "
        t0 = time.time()

        try:
            # run consul
            print "Start consul ..."
            self._run_consul()

            print "Waiting for consul to be ready ..."
            rc = self._wait_for_consul_to_be_ready()
            self.assertEqual(rc, 0)

            # Get the docker IP address and port number of the consul instance
            print "Get consul leader IP ..."
            cmd = command_defs['consul_get_leader_ip_port']
            out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
            self.assertEqual(rc, 0)

            # validate that the returned ip:port is valid and open
            print "Verify consul IP and port is reachable ..."
            self.assertTrue(is_open(out))

        finally:
            # clean up all created containers for this test
            print "Stop consul ..."
            self._stop_and_remove_all_containers()

            print "Test_05_run_consul_only_End:------------------ took {} secs" \
                  "\n\n".format(time.time() - t0)

    def test_06_run_voltha_standalone_with_consul_only(self):
        print "Test_06_run_voltha_standalone_with_consul_only_Start:----------" \
              "-------- "
        t0 = time.time()

        try:
            # run consul first
            print "Start consul ..."
            self._run_consul()

            # get consul ip
            print "Get consul IP ..."
            cmd = command_defs['docker_get_consul_ip']
            consul_ip, err, rc = run_command_to_completion_with_raw_stdout(cmd)
            self.assertEqual(rc, 0)
            self.assertIsNotNone(consul_ip)

            # start voltha now for 15 secs and verify it can now connect to
            # consul - following message in the output
            print "Start voltha with consul IP ..."
            expected_pattern = ['coordinator', 'event: created-consul-session']
            cmd = command_defs['docker_start_voltha_with_consul_ip'] + \
                  '{}:8500'.format(consul_ip.strip())
            command_output = run_long_running_command_with_timeout(cmd, 10)

            # Verify the output of voltha and get the container instance id
            print "Verify voltha is registered with consul ..."
            instance_id = None
            for out in command_output:
                if all(ep for ep in expected_pattern if ep in out):
                    self.assertTrue(True)
                    instance_id = re.findall(r'[0-9a-f]+', out)[-1]
                    break

            self.assertIsNotNone(instance_id)

            # Verify Voltha's self-registration with consul
            expected_output = ['ModifyIndex', 'CreateIndex', 'Session',
                               'Value',
                               'Flags', 'Key', 'LockIndex']

            cmd = command_defs['consul_verify_voltha_registration']
            registration_info, err, rc = \
                run_command_to_completion_with_raw_stdout(cmd)
            self.assertEqual(rc, 0)
            try:
                jr_info = json.loads(registration_info)
                intersect_elems = [e for e in jr_info[0] if
                                   e in expected_output]
                self.assertEqual(len(expected_output), len(intersect_elems))
            except Exception as e:
                self.assertRaises(e)

            # stop voltha
            print "Stop voltha ..."
            self._stop_docker_container_by_id(instance_id)

            # check the service has deregistered
            print "Verify voltha is no longer registered in consul..."
            cmd = command_defs['consul_verify_voltha_registration']
            registration_info, err, rc = \
                run_command_to_completion_with_raw_stdout(cmd)
            self.assertEqual(rc, 0)
            self.assertEqual(registration_info, '')

        finally:
            # clean up all created containers for this test
            print "Stop consul ..."
            self._stop_and_remove_all_containers()

            print "Test_06_run_voltha_standalone_with_consul_only_End:--------" \
                  "---------- took {} " \
                  "secs \n\n".format(time.time() - t0)

    def test_07_start_all_containers(self):
        print "Test_07_start_all_containers_Start:------------------ "
        t0 = time.time()

        try:
            # Pre-test - clean up all running docker containers
            print "Pre-test: Removing all running containers ..."
            cmd = command_defs['docker_compose_stop']
            _, err, rc = run_command_to_completion_with_raw_stdout(cmd)
            self.assertEqual(rc, 0)
            cmd = command_defs['docker_compose_rm_f']
            _, err, rc = run_command_to_completion_with_raw_stdout(cmd)
            self.assertEqual(rc, 0)

            # get a list of services in the docker-compose file
            print "Getting list of services in docker compose file ..."
            cmd = command_defs['docker_compose_services']
            services, err, rc = run_command_to_completion_with_raw_stdout(cmd)
            self.assertEqual(rc, 0)
            docker_service_list = services.split()
            self.assertGreaterEqual(len(docker_service_list),
                                    DOCKER_COMPOSE_FILE_SERVICES_COUNT)

            # start all the containers
            print "Starting all containers ..."
            cmd = command_defs['docker_compose_start_all']
            out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
            self.assertEqual(rc, 0)

            # Instead of using only a fixed timeout:
            #   1) wait until the services are ready (polling per second)
            #   2) bail out after a longer timeout.
            print "Waiting for all containers to be ready ..."
            self.wait_till('Not all services are up',
                           self._is_voltha_ensemble_ready,
                           interval=1,
                           timeout=30)

            # verify that all containers are running
            print "Verify all services are running using docker command ..."
            for service in docker_service_list:
                cmd = command_defs['docker_compose_ps'] + ' {} | wc -l'.format(
                    service)
                out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
                self.assertEqual(rc, 0)
                self.assertGreaterEqual(out, 3)  # 2 are for headers

            # Verify that 'docker ps' return the same number of running process
            cmd = command_defs['docker_ps_count']
            out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
            self.assertEqual(rc, 0)
            self.assertGreaterEqual(out.split(), [str(len(
                docker_service_list))])

            # Retrieve the list of services from consul and validate against
            # the list obtained from docker composed
            print "Verify all services are registered in consul ..."
            expected_services = ['consul-rest', 'fluentd-intake',
                                 'voltha-grpc',
                                 'voltha-health',
                                 'consul-8600', 'zookeeper', 'consul',
                                 'kafka']

            cmd = command_defs['consul_get_services']
            out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
            self.assertEqual(rc, 0)
            try:
                consul_services = json.loads(out)
                intersected_services = [s for s in expected_services if
                                        s in consul_services]
                self.assertEqual(len(intersected_services),
                                 len(expected_services))
                # services_match = 0
                # for d_service in docker_service_list:
                #     for c_service in consul_services:
                #         if c_service.find(d_service) != -1:
                #             services_match += 1
                #             print d_service, c_service
                #             break
                # self.assertEqual(services_match, len(docker_service_list))
            except Exception as e:
                self.assertRaises(e)

            # Verify the service record of the voltha service
            print "Verify the service record of voltha in consul ..."
            expected_srv_elements = ['ModifyIndex', 'CreateIndex',
                                     'ServiceEnableTagOverride', 'Node',
                                     'Address', 'TaggedAddresses', 'ServiceID',
                                     'ServiceName', 'ServiceTags',
                                     'ServiceAddress', 'ServicePort']
            cmd = command_defs['consul_get_srv_voltha_health']
            out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
            self.assertEqual(rc, 0)
            try:
                srv = json.loads(out)
                intersect_elems = [e for e in srv[0] if
                                   e in expected_srv_elements]
                self.assertEqual(len(expected_srv_elements),
                                 len(intersect_elems))
            except Exception as e:
                self.assertRaises(e)

            # Verify kafka client is receiving the messages
            print "Verify kafka client has heartbeat topic ..."
            expected_pattern = ['voltha.heartbeat']
            kafka_endpoint = get_endpoint_from_consul(LOCAL_CONSUL,'kafka')
            cmd = command_defs['kafka_client_run'].format(kafka_endpoint)
            kafka_client_output = run_long_running_command_with_timeout(cmd, 20)

            # Verify the kafka client output
            # instance id
            found = False
            for out in kafka_client_output:
                if all(ep for ep in expected_pattern if ep in out):
                    found = True
                    break
            self.assertTrue(found)

            # Commented the heartbeat messages from voltha as on Jenkins this
            # test fails more often than not.   On local or cluster environment
            # the kafka event bus works well.

            # verify docker-compose logs are being produced - just get the
            # first work of each line
            print "Verify docker compose logs has output from all the services " \
                  "..."
            expected_output = ['voltha_1', 'fluentd_1', 'consul_1',
                               'registrator_1', 'kafka_1', 'zookeeper_1',
                               'ofagent_1', 'netconf_1']
            cmd = command_defs['docker_compose_logs']
            docker_compose_logs = run_long_running_command_with_timeout(cmd, 5, 0)
            intersected_logs = [l for l in expected_output if
                                l in docker_compose_logs]
            self.assertEqual(len(intersected_logs), len(expected_output))

            # verify docker voltha logs are being produced - we will just verify
            # some
            # key messages in the logs
            print "Verify docker voltha logs are produced ..."
            expected_output = ['coordinator._renew_session', 'main.heartbeat']
            cmd = command_defs['docker_voltha_logs']
            docker_voltha_logs = run_long_running_command_with_timeout(cmd,
                                                                       0.5, 5)
            intersected_logs = [l for l in expected_output if
                                l in docker_voltha_logs]
            self.assertEqual(len(intersected_logs), len(expected_output))

        finally:
            print "Stopping all containers ..."
            # clean up all created containers for this test
            #self._stop_and_remove_all_containers()
            cmd = command_defs['docker_compose_down']
            _, err, rc = run_command_to_completion_with_raw_stdout(cmd)

            print "Test_07_start_all_containers_End:------------------ took {}" \
                  " secs \n\n".format(time.time() - t0)

    def test_08_stop_all_containers_started_using_docker_compose(self):
        print "Test_08_stop_all_containers_started_using_docker_compose_Start:" \
              "------------------ "
        t0 = time.time()

        try:
            # commands to stop and clear the docker images
            cmds = [command_defs['docker_compose_stop'],
                    command_defs['docker_compose_rm_f']]

            print "Stopping all containers ..."
            for cmd in cmds:
                out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
                self.assertEqual(rc, 0)

            # Verify that no docker process is running
            print "Verify no containers is running..."
            cmd = command_defs['docker_compose_services_running']
            out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
            self.assertEqual(rc, 0)

        finally:
            print "Test_08_stop_all_containers_started_using_docker_compose_:" \
                  "------------------ took {} secs \n\n".format(
                time.time() - t0)

    def test_09_dig_consul_command(self):
        print "Test_09_dig_consul_command_Start:------------------"
        t0 = time.time()

        try:
            # start all containers
            print "Start all containers..."
            self._start_all_containers()

            print "Waiting for all containers to be ready ..."
            time.sleep(10)
            rc = verify_all_services_healthy(LOCAL_CONSUL)
            if not  rc:
                print "Not all services are up"
            self.assertEqual(rc, True)

            # Get the IP address(es) for voltha's REST interface
            print "Get IP of Voltha REST interface..."
            cmd = command_defs['consul_get_voltha_rest_a_record']
            out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
            self.assertEqual(rc, 0)
            self.assertGreaterEqual(out.find("voltha-health.service.consul"),
                                    0)

            # Get only the ip address
            cmd = command_defs['consul_get_voltha_rest_ip']
            ip, err, rc = run_command_to_completion_with_raw_stdout(cmd)
            self.assertEqual(rc, 0)
            self.assertTrue(is_valid_ip(ip))

            # Get the exposed service port
            print "Get Voltha exposed service port..."
            cmd = command_defs['consul_get_voltha_service_port']
            port, err, rc = run_command_to_completion_with_raw_stdout(cmd)
            self.assertEqual(rc, 0)
            # Verify that we can connect to the port using the previously
            # acquired ip
            print "Verify connectivity with voltha ip and port..."
            self.assertTrue(is_open('{}:{}'.format(ip, port)))
        finally:
            print "Stopping all containers ..."
            # clean up all created containers for this test
            self._stop_and_remove_all_containers()

            print "Test_09_dig_consul_command_Start_End:------------------" \
                  "took {} secs \n\n".format(time.time() - t0)

    def test_10_scale_voltha(self):
        print "Test_10_scale_voltha_Start:------------------"
        t0 = time.time()

        try:
            # start all containers
            print "Start all containers..."
            self._start_all_containers()

            # Instead of using only a fixed timeout:
            #   1) wait until the services are ready (polling per second)
            #   2) bail out after a longer timeout.
            print "Waiting for all containers to be ready ..."
            self.wait_till('Not all services are up',
                           self._is_voltha_ensemble_ready,
                           interval=1,
                           timeout=30)

            # Scale voltha to 10 instances
            print "Scale voltha to 10 instances ..."
            cmd = command_defs['docker_compose_scale_voltha_to_10']
            out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
            self.assertEqual(rc, 0)

            # Verify that 10 instances are running
            print "Verify 10 instances of voltha are running ..."
            cmd = command_defs['docker_compose_scaled_voltha_ps']
            out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
            self.assertEqual(rc, 0)
            self.assertEqual(out.split(), ['10'])
        finally:
            print "Stopping all containers ..."
            # clean up all created containers for this test
            self._stop_and_remove_all_containers()

            print "Test_10_scale_voltha_End:------------------took {} secs " \
                  "\n\n".format(time.time() - t0)

    def _start_all_containers(self):
        cmd = command_defs['docker_compose_start_all']
        out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
        self.assertEqual(rc, 0)

    def _is_voltha_ensemble_ready(self):
        res = verify_all_services_healthy(LOCAL_CONSUL)
        if not res:
            print "Not all consul services are ready ..."
        return res

    def _run_consul(self):
        # run consul
        cmd = command_defs['docker_compose_start_consul']
        out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
        self.assertEqual(rc, 0)

        # verify consul is up
        cmd = command_defs['docker_compose_is_consul_up']
        out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
        self.assertEqual(rc, 0)
        self.assertIn('compose_consul_1', out)

    def _stop_and_remove_all_containers(self):
        # check if there are any running containers first
        cmd = command_defs['docker_ps']
        out, err, rc = run_command_to_completion_with_stdout_in_list(cmd)
        self.assertEqual(rc, 0)
        if len(out) > 1:  # not counting docker ps header
            cmd = command_defs['docker_stop_and_remove_all_containers']
            out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
            self.assertEqual(rc, 0)

    def _stop_docker_container_by_id(self, instance_id):
        # stop
        cmd = command_defs['docker_stop'] + " {}".format(instance_id)
        out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
        self.assertEqual(rc, 0)

        # remove
        cmd = command_defs['docker_rm'] + " {}".format(instance_id)
        out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
        self.assertEqual(rc, 0)

    def _source_env(self):
        # Go to voltha root directory
        res = os.system('cd {}'.format(this_dir))
        assert res == 0

        # set the env
        command = ['bash', '-c', '. env.sh']
        proc = subprocess.Popen(command, stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE)

        if proc.wait() != 0:
            err_msg = "Failed to source the environment'"
            raise RuntimeError(err_msg)

        env = os.environ.copy()
        return env

    def _wait_for_consul_to_be_ready(self):
        # Consul is ready when it's leader ip and port is set.  The maximum
        # time to wait of 60 secs as consul should be ready by then
        max_wait_time = 60
        t0 = time.time()

        while True:
            # Get the docker IP address and port number of the consul instance
            print "waiting for consul to be ready ..."
            cmd = command_defs['consul_get_leader_ip_port']
            out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
            out = out.strip()
            if rc != 0:
                # Something is wrong, return
                return -1  # error
            elif out is not None and out != '':
                return 0  # found something
            elif time.time() - t0 > max_wait_time:
                return -1  # consul should have come up by this time
            else:
                time.sleep(2)  # constant sleep for testing