Prep for docker-only tests

Chnages to support more production-like tests where all
components are running as Docker containers. Specific
changes:
- Expanded Vagrant memory to 6G to have room
- Chameleon to cope with premature service lookup before
  Voltha is self-registered (can happen when docker-compose
  starts the "world"
- Add missing termcolor package
- Allow CLI to use Consul for endpoints
- 2nd external ponmgmt bridge for Voltha

Change-Id: Ib2471784a5aafbfd9c611ebf293d7f81f61dd75b
diff --git a/Vagrantfile b/Vagrantfile
index 342262d..9ea9dc3 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -18,7 +18,7 @@
     d.vm.provision :shell, inline: "PYTHONUNBUFFERED=1 ansible-playbook /voltha/ansible/voltha.yml -c local"
     d.vm.provision :shell, inline: "cd /voltha && source env.sh && make install-protoc && chmod 777 /tmp/fluentd"
     d.vm.provider "virtualbox" do |v|
-      v.memory = 4096
+      v.memory = 6144
     end
   end
 
diff --git a/chameleon/grpc_client/grpc_client.py b/chameleon/grpc_client/grpc_client.py
index 05a4dd7..48cfa75 100644
--- a/chameleon/grpc_client/grpc_client.py
+++ b/chameleon/grpc_client/grpc_client.py
@@ -95,7 +95,8 @@
 
         try:
             if self.endpoint.startswith('@'):
-                _endpoint = self._get_endpoint_from_consul(self.endpoint[1:])
+                _endpoint = yield self._get_endpoint_from_consul(
+                    self.endpoint[1:])
             else:
                 _endpoint = self.endpoint
 
@@ -138,6 +139,7 @@
             log.info('reconnected', after_retries=self.retries)
             self.retries = 0
 
+    @inlineCallbacks
     def _get_endpoint_from_consul(self, service_name):
         """
         Look up an appropriate grpc endpoint (host, port) from
@@ -146,11 +148,16 @@
         host = self.consul_endpoint.split(':')[0].strip()
         port = int(self.consul_endpoint.split(':')[1].strip())
 
-        consul = Consul(host=host, port=port)
-        _, services = consul.catalog.service(service_name)
-
-        if len(services) == 0:
-            raise Exception('Cannot find service %s in consul' % service_name)
+        while True:
+            log.debug('consul-lookup', host=host, port=port)
+            consul = Consul(host=host, port=port)
+            _, services = consul.catalog.service(service_name)
+            log.debug('consul-response', services=services)
+            if services:
+                break
+            log.warning('no-service', consul_host=host, consul_port=port,
+                        service_name=service_name)
+            yield asleep(1.0)
 
         # pick a random entry
         # TODO should we prefer local IP addresses? Probably.
@@ -158,7 +165,7 @@
         service = services[randint(0, len(services) - 1)]
         endpoint = '{}:{}'.format(service['ServiceAddress'],
                                   service['ServicePort'])
-        return endpoint
+        returnValue(endpoint)
 
     def _retrieve_schema(self):
         """
diff --git a/chameleon/main.py b/chameleon/main.py
index 805acba..4d3939d 100755
--- a/chameleon/main.py
+++ b/chameleon/main.py
@@ -219,15 +219,18 @@
 
     @inlineCallbacks
     def startup_components(self):
-        self.log.info('starting-internal-components')
-        args = self.args
-        self.grpc_client = yield \
-            GrpcClient(args.consul, args.work_dir, args.grpc_endpoint)
-        self.web_server = yield \
-            WebServer(args.rest_port, args.work_dir, self.grpc_client).start()
-        self.grpc_client.set_reconnect_callback(
-            self.web_server.reload_generated_routes).start()
-        self.log.info('started-internal-services')
+        try:
+            self.log.info('starting-internal-components')
+            args = self.args
+            self.grpc_client = yield \
+                GrpcClient(args.consul, args.work_dir, args.grpc_endpoint)
+            self.rest_server = yield \
+                WebServer(args.rest_port, args.work_dir, self.grpc_client).start()
+            self.grpc_client.set_reconnect_callback(
+                self.rest_server.reload_generated_routes).start()
+            self.log.info('started-internal-services')
+        except Exception, e:
+            self.log.exception('startup-failed', e=e)
 
     @inlineCallbacks
     def shutdown_components(self):
diff --git a/cli/device.py b/cli/device.py
index 3ab4b0d..cd3e3ce 100644
--- a/cli/device.py
+++ b/cli/device.py
@@ -38,6 +38,9 @@
         self.prompt = '(' + self.colorize(
             self.colorize('device {}'.format(device_id), 'red'), 'bold') + ') '
 
+    def cmdloop(self):
+        self._cmdloop()
+
     def get_device(self, depth=0):
         stub = voltha_pb2.VolthaLocalServiceStub(self.get_channel())
         res = stub.GetDevice(voltha_pb2.ID(id=self.device_id),
diff --git a/cli/logical_device.py b/cli/logical_device.py
index 3e90f41..46b1865 100644
--- a/cli/logical_device.py
+++ b/cli/logical_device.py
@@ -41,6 +41,9 @@
             self.colorize('logical device {}'.format(logical_device_id), 'red'),
             'bold') + ') '
 
+    def cmdloop(self):
+        self._cmdloop()
+
     def get_logical_device(self, depth=0):
         stub = voltha_pb2.VolthaLocalServiceStub(self.get_channel())
         res = stub.GetLogicalDevice(voltha_pb2.ID(id=self.logical_device_id),
diff --git a/cli/main.py b/cli/main.py
index 49fc812..3d02beb 100755
--- a/cli/main.py
+++ b/cli/main.py
@@ -14,11 +14,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
+import argparse
+import os
 import readline
 from optparse import make_option
 from time import sleep, time
 
+import sys
+from consul import Consul
 import grpc
 import requests
 from cmd2 import Cmd, options
@@ -36,6 +39,16 @@
 _ = third_party
 from cli.utils import pb2dict, dict2line
 
+
+defs = dict(
+    # config=os.environ.get('CONFIG', './cli.yml'),
+    consul=os.environ.get('CONSUL', 'localhost:8500'),
+    voltha_grpc_endpoint=os.environ.get('VOLTHA_GRPC_ENDPOINT',
+                                        'localhost:50055'),
+    voltha_sim_rest_endpoint=os.environ.get('VOLTHA_SIM_REST_ENDPOINT',
+                                            'localhost:18880'),
+)
+
 banner = """\
          _ _   _              _ _
 __ _____| | |_| |_  __ _   __| (_)
@@ -67,10 +80,9 @@
                                   'is specified',
     ))
 
-    # cleanup of superflous commands from cmd2
+    # cleanup of superfluous commands from cmd2
     del Cmd.do_cmdenvironment
     # del Cmd.do_eof
-    del Cmd.do_exit
     del Cmd.do_q
     del Cmd.do_hi
     del Cmd.do_l
@@ -80,10 +92,10 @@
     del Cmd.do__relative_load
     Cmd.do_edit = Cmd.do_ed
 
-
-    def __init__(self, *args, **kw):
-
-        Cmd.__init__(self, *args, **kw)
+    def __init__(self, voltha_grpc, voltha_sim_rest):
+        VolthaCli.voltha_grpc = voltha_grpc
+        VolthaCli.voltha_sim_rest = voltha_sim_rest
+        Cmd.__init__(self)
         self.prompt = '(' + self.colorize(
             self.colorize(self.prompt, 'blue'), 'bold') + ') '
         self.channel = None
@@ -92,6 +104,11 @@
         self.logical_device_ids_cache = None
         self.logical_device_ids_cache_ts = time()
 
+    # we override cmd2's method to avoid its optparse conflicting with our
+    # command line parsing
+    def cmdloop(self):
+        self._cmdloop()
+
     def load_history(self):
         """Load saved command history from local history file"""
         try:
@@ -111,7 +128,7 @@
             self.perror('Could not save history in {}: {}'.format(
                 self.history_file_name, e))
         else:
-            self.perror('History saved as {}'.format(
+            self.poutput('History saved as {}'.format(
                 self.history_file_name))
 
     def perror(self, errmsg, statement=None):
@@ -244,14 +261,15 @@
 
     def do_test(self, line):
         """Enter test mode, which makes a bunch on new commands available"""
-        sub = TestCli(self.history, self.get_channel)
+        sub = TestCli(self.history, self.get_channel, self.voltha_grpc,
+                      self.voltha_sim_rest)
         sub.cmdloop()
 
 
 class TestCli(VolthaCli):
 
-    def __init__(self, history, get_channel):
-        VolthaCli.__init__(self)
+    def __init__(self, history, get_channel, voltha_grpc, voltha_sim_rest):
+        VolthaCli.__init__(self, voltha_grpc, voltha_sim_rest)
         self.history = history
         self.get_channel = get_channel
         self.prompt = '(' + self.colorize(self.colorize('test', 'cyan'),
@@ -644,7 +662,50 @@
 
 
 if __name__ == '__main__':
-    c = VolthaCli()
+
+    parser = argparse.ArgumentParser()
+
+    _help = '<hostname>:<port> to consul agent (default: %s)' % defs['consul']
+    parser.add_argument(
+        '-C', '--consul', action='store', default=defs['consul'], help=_help)
+
+    _help = 'Lookup Voltha endpoints based on service entries in Consul'
+    parser.add_argument(
+        '-L', '--lookup', action='store_true', help=_help)
+
+    _help = '<hostname>:<port> of Voltha gRPC service (default={})'.format(
+        defs['voltha_grpc_endpoint'])
+    parser.add_argument('-g', '--grpc-endpoint', action='store',
+                        default=defs['voltha_grpc_endpoint'], help=_help)
+
+    _help = '<hostname>:<port> of Voltha simulated adapter backend for ' \
+            'testing (default={})'.format(
+        defs['voltha_sim_rest_endpoint'])
+    parser.add_argument('-s', '--sim-rest-endpoint', action='store',
+                        default=defs['voltha_sim_rest_endpoint'], help=_help)
+
+    args = parser.parse_args()
+
+    if args.lookup:
+        host = args.consul.split(':')[0].strip()
+        port = int(args.consul.split(':')[1].strip())
+        consul = Consul(host=host, port=port)
+
+        _, services = consul.catalog.service('voltha-grpc')
+        if not services:
+            print('No voltha-grpc service registered in consul; exiting')
+            sys.exit(1)
+        args.grpc_endpoint = '{}:{}'.format(services[0]['ServiceAddress'],
+                                            services[0]['ServicePort'])
+
+        _, services = consul.catalog.service('voltha-sim-rest')
+        if not services:
+            print('No voltha-sim-rest service registered in consul; exiting')
+            sys.exit(1)
+        args.sim_rest_endpoint = '{}:{}'.format(services[0]['ServiceAddress'],
+                                                services[0]['ServicePort'])
+
+    c = VolthaCli(args.grpc_endpoint, args.sim_rest_endpoint)
     c.poutput(banner)
     c.load_history()
     c.cmdloop()
diff --git a/compose/docker-compose-system-test-with-podder.yml b/compose/docker-compose-system-test-with-podder.yml
index 140688e..2295848 100644
--- a/compose/docker-compose-system-test-with-podder.yml
+++ b/compose/docker-compose-system-test-with-podder.yml
@@ -79,6 +79,42 @@
     restart: unless-stopped
 
   #
+  # Graphite-Grafana-statsd service instance
+  # (demo place-holder for external KPI system)
+  #
+  grafana:
+    image: kamon/grafana_graphite
+    ports:
+    - "8882:80"
+    - "2003:2003"
+    - "2004:2004"
+    - "8126:8126"
+    - "8125:8125/udp"
+    environment:
+      SERVICE_80_NAME:   "grafana-web-ui"
+      SERVICE_2003_NAME: "carbon-plain-text-intake"
+      SERVICE_2004_NAME: "carbon-pickle-intake"
+      SERVICE_8126_NAME: "statsd-tcp-intake"
+      SERVICE_8125_NAME: "statsd-udp-intake"
+
+  #
+  # Shovel (Kafka-graphite-gateway)
+  #
+  shovel:
+    image: cord/shovel
+    command: [
+      "/shovel/shovel/main.py",
+      "--kafka=@kafka",
+      "--consul=${DOCKER_HOST_IP}:8500",
+      "--topic=voltha.kpis",
+      "--host=${DOCKER_HOST_IP}"
+    ]
+    depends_on:
+    - consul
+    - kafka
+    - grafana
+
+  #
   # Voltha server instance(s)
   #
   voltha:
@@ -97,6 +133,7 @@
     ports:
     - 8880
     - 50555
+    - 18880
     depends_on:
     - consul
     - podder
@@ -109,6 +146,15 @@
       SERVICE_8880_CHECK_INTERVAL: "5s"
       SERVICE_8880_CHECK_TIMEOUT: "1s"
       SERVICE_50555_NAME: "voltha-grpc"
+      SERVICE_18880_NAME: "voltha-sim-rest"
     volumes:
     - "/var/run/docker.sock:/tmp/docker.sock"
+    networks:
+    - default
+    - ponmgmt
 
+networks:
+  default:
+    driver: bridge
+  ponmgmt:
+    driver: bridge
diff --git a/compose/docker-compose-system-test.yml b/compose/docker-compose-system-test.yml
index 443fc01..c6038cb 100644
--- a/compose/docker-compose-system-test.yml
+++ b/compose/docker-compose-system-test.yml
@@ -129,11 +129,13 @@
       "--grpc-port=50555",
       "--kafka=@kafka",
       "--instance-id-is-container-name",
+      "--interface=eth1",
       "-v"
     ]
     ports:
     - 8880
     - 50555
+    - 18880
     depends_on:
     - consul
     links:
@@ -145,8 +147,12 @@
       SERVICE_8880_CHECK_INTERVAL: "5s"
       SERVICE_8880_CHECK_TIMEOUT: "1s"
       SERVICE_50555_NAME: "voltha-grpc"
+      SERVICE_18880_NAME: "voltha-sim-rest"
     volumes:
     - "/var/run/docker.sock:/tmp/docker.sock"
+    networks:
+    - default
+    - ponmgmt
 
 #############################################
 # Item below this line will soon be removed.#
@@ -230,3 +236,10 @@
     volumes:
     - "/var/run/docker.sock:/tmp/docker.sock"
 
+networks:
+  default:
+    driver: bridge
+  ponmgmt:
+    driver: bridge
+    driver_opts:
+      com.docker.network.bridge.name: "ponmgmt"
diff --git a/requirements.txt b/requirements.txt
index 325306c..fdbbc32 100755
--- a/requirements.txt
+++ b/requirements.txt
@@ -35,6 +35,7 @@
 simplejson>=3.8.1
 six>=1.10.0
 structlog>=16.1.0
+termcolor>=1.1.0
 treq>=15.1.0
 Twisted>=13.2.0
 urllib3>=1.7.1