Support to specify karaf_version in manifest and cord-test command line options for run and setup subcommands.
Fix cord-tester cluster bootup ONOS device connectivity and flow test verification specifically for the cluster instantiate.
Fix prerequisites to delay before verification of docker install.

Change-Id: Ide02adae4fd8224cf4d1e200f1153be901e794d6
diff --git a/src/test/setup/cord-test.py b/src/test/setup/cord-test.py
index ef516f3..08eb528 100755
--- a/src/test/setup/cord-test.py
+++ b/src/test/setup/cord-test.py
@@ -109,13 +109,14 @@
         egress_map = { 'ether': '00:00:00:00:00:03', 'ip': '192.168.30.1' }
         ingress_map = { 'ether': '00:00:00:00:00:04', 'ip': '192.168.40.1' }
         device_id = 'of:{}'.format(get_mac(switch))
+        ctlr = self.ctlr_ip.split(',')[0]
         flow = OnosFlowCtrl(deviceId = device_id,
                             egressPort = egress,
                             ingressPort = ingress,
                             ethType = '0x800',
                             ipSrc = ('IPV4_SRC', ingress_map['ip']+'/32'),
                             ipDst = ('IPV4_DST', egress_map['ip']+'/32'),
-                            controller = self.ctlr_ip
+                            controller = ctlr
                             )
         result = flow.addFlow()
         if result != True:
@@ -136,7 +137,8 @@
         if not self.olt:
             return False
         device_id = 'of:{}'.format(get_mac(switch))
-        devices = OnosCtrl.get_devices(controller = self.ctlr_ip)
+        ctlr = self.ctlr_ip.split(',')[0]
+        devices = OnosCtrl.get_devices(controller = ctlr)
         if devices:
             device = filter(lambda d: d['id'] == device_id, devices)
             return True
@@ -448,6 +450,8 @@
     service_profile = test_manifest.service_profile
     synchronizer = test_manifest.synchronizer
     onos_cord = None
+    Onos.update_data_dir(test_manifest.karaf_version)
+
     if onos_cord_loc:
         if onos_cord_loc.find(os.path.sep) < 0:
             onos_cord_loc = os.path.join(os.getenv('HOME'), onos_cord_loc)
@@ -582,7 +586,8 @@
                      'LOG_LEVEL': test_manifest.log_level,
                      'MANIFEST': int(use_manifest),
                      'HEAD_NODE': head_node if head_node else CORD_TEST_HOST,
-                     'MAAS_API_KEY': maas_api_key
+                     'MAAS_API_KEY': maas_api_key,
+                     'KARAF_VERSION' : test_manifest.karaf_version
                    }
 
     if ssh_key_file:
@@ -708,6 +713,8 @@
     synchronizer = test_manifest.synchronizer
     onos_cord = None
     onos_cord_loc = test_manifest.onos_cord
+    Onos.update_data_dir(test_manifest.karaf_version)
+
     if onos_cord_loc:
         if onos_cord_loc.find(os.path.sep) < 0:
             onos_cord_loc = os.path.join(os.getenv('HOME'), onos_cord_loc)
@@ -842,7 +849,8 @@
                          'LOG_LEVEL': test_manifest.log_level,
                          'MANIFEST': int(use_manifest),
                          'HEAD_NODE': head_node if head_node else CORD_TEST_HOST,
-                         'MAAS_API_KEY': maas_api_key
+                         'MAAS_API_KEY': maas_api_key,
+                         'KARAF_VERSION' : test_manifest.karaf_version
                        }
 
         if ssh_key_file:
@@ -1145,6 +1153,7 @@
     parser_run.add_argument('-synchronizer', '--synchronizer', default='', type=str,
                             help='Specify the synchronizer to use for ONOS cord instance when running on podd.'
                             'Eg: vtn,fabric,cord')
+    parser_run.add_argument('-karaf', '--karaf', default='3.0.5', type=str, help='Karaf version for ONOS')
     parser_run.set_defaults(func=runTest)
 
     parser_setup = subparser.add_parser('setup', help='Setup cord tester environment')
@@ -1186,6 +1195,7 @@
     parser_setup.add_argument('-f', '--foreground', action='store_true', help='Run in foreground')
     parser_setup.add_argument('-jvm-heap-size', '--jvm-heap-size', default='', type=str, help='ONOS JVM heap size')
     parser_setup.add_argument('-network', '--network', default='', type=str, help='Docker network to attach')
+    parser_setup.add_argument('-karaf', '--karaf', default='3.0.5', type=str, help='Karaf version for ONOS')
     parser_setup.set_defaults(func=setupCordTester)
 
     parser_xos = subparser.add_parser('xos', help='Building xos into cord tester environment')
diff --git a/src/test/setup/manifest.json b/src/test/setup/manifest.json
index 862662d..d9bb2e4 100644
--- a/src/test/setup/manifest.json
+++ b/src/test/setup/manifest.json
@@ -4,5 +4,6 @@
     "start_switch": true,
     "onos_image": "onosproject/onos:latest",
     "log_level" : "INFO",
-    "jvm_heap_size" : "1G"
+    "jvm_heap_size" : "1G",
+    "karaf_version" : "3.0.5"
 }
diff --git a/src/test/setup/prerequisites.sh b/src/test/setup/prerequisites.sh
index 6a8a0c1..8a3c06d 100755
--- a/src/test/setup/prerequisites.sh
+++ b/src/test/setup/prerequisites.sh
@@ -19,6 +19,7 @@
     echo "Installing Docker"
     apt-get -y install docker-engine
     service docker restart
+    sleep 5
     echo "Verifying Docker installation"
     docker run --rm hello-world || exit 127
     docker rmi hello-world
diff --git a/src/test/utils/CordContainer.py b/src/test/utils/CordContainer.py
index bda9359..81df366 100644
--- a/src/test/utils/CordContainer.py
+++ b/src/test/utils/CordContainer.py
@@ -465,6 +465,7 @@
     host_config_dir = os.path.join(setup_dir, 'onos-config')
     guest_config_dir = '/root/onos/config'
     guest_data_dir = '/root/onos/apache-karaf-3.0.5/data'
+    guest_log_file = '/root/onos/apache-karaf-3.0.5/data/log/karaf.log'
     onos_gen_partitions = os.path.join(setup_dir, 'onos-gen-partitions')
     onos_form_cluster = os.path.join(setup_dir, 'onos-form-cluster')
     cord_apps_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'apps')
@@ -521,6 +522,11 @@
         if os.path.exists(host_volume_dir):
             shutil.rmtree(host_volume_dir)
 
+    @classmethod
+    def update_data_dir(cls, karaf):
+        Onos.guest_data_dir = '/root/onos/apache-karaf-{}/data'.format(karaf)
+        Onos.guest_log_file = '/root/onos/apache-karaf-{}/data/log/karaf.log'.format(karaf)
+
     def remove_data_volume(self):
         if self.data_map is not None:
             self.remove_data_map(*self.data_map)
@@ -643,7 +649,7 @@
 
     @classmethod
     def wait_for_onos_start(cls, ip, tries = 30):
-        onos_log = OnosLog(host = ip)
+        onos_log = OnosLog(host = ip, log_file = Onos.guest_log_file)
         num_tries = 0
         started = None
         while not started and num_tries < tries:
diff --git a/src/test/utils/OnosLog.py b/src/test/utils/OnosLog.py
index 0a31b80..15fb216 100644
--- a/src/test/utils/OnosLog.py
+++ b/src/test/utils/OnosLog.py
@@ -6,10 +6,14 @@
     CLI_USER = 'karaf'
     CLI_PASSWD = 'karaf'
     CLI_PORT = 8101
+    KARAF_VERSION = os.getenv('KARAF_VERSION', '3.0.5')
     HOST = os.getenv('ONOS_CONTROLLER_IP', '172.17.0.2').split(',')[0]
     last_snapshot_map = {}
 
-    def __init__(self, host = HOST):
+    def __init__(self, host = HOST, log_file = None):
+        if log_file is None:
+            log_file = '/root/onos/apache-karaf-{}/data/log/karaf.log'.format(self.KARAF_VERSION)
+        self.log_file = log_file
         self.ssh_agent = SSHTestAgent(host = host, user = self.CLI_USER,
                                       password = self.CLI_PASSWD, port = self.CLI_PORT)
         if not OnosLog.last_snapshot_map.has_key(host):
@@ -27,7 +31,7 @@
 
     def get_log(self, search_terms = None, exception = True, cache_result = True):
         """Run the command on the test host"""
-        cmd = 'cat /root/onos/apache-karaf-3.0.5/data/log/karaf.log'
+        cmd = 'cat {}'.format(self.log_file)
         st, output = self.ssh_agent.run_cmd(cmd)
         if st is False:
             return st, output
@@ -64,7 +68,7 @@
 
     def search_log_pattern(self, pattern):
         r_pat = re.compile(pattern)
-        cmd = 'cat /root/onos/apache-karaf-3.0.5/data/log/karaf.log'
+        cmd = 'cat {}'.format(self.log_file)
         st, output = self.ssh_agent.run_cmd(cmd)
         if st is False:
             return None
diff --git a/src/test/utils/TestManifest.py b/src/test/utils/TestManifest.py
index b5ec2d4..143727c 100644
--- a/src/test/utils/TestManifest.py
+++ b/src/test/utils/TestManifest.py
@@ -48,6 +48,7 @@
             self.iterations = None
             self.server = args.server
             self.jvm_heap_size = args.jvm_heap_size if args.jvm_heap_size else None
+            self.karaf_version = args.karaf
         else:
             with open(self.manifest, 'r') as fd:
                 data = json.load(fd)
@@ -72,3 +73,4 @@
             self.server = data.get('test_server', '{}:{}'.format(CORD_TEST_HOST, CORD_TEST_PORT))
             self.iterations = data.get('iterations', None)
             self.jvm_heap_size = data.get('jvm_heap_size', None)
+            self.karaf_version = data.get('karaf_version', '3.0.5')