Test: Additional changes to support cluster based test scenarios.

Change-Id: I8464d2c775809a723ef46c0f5d3625bbbe5d0d48
diff --git a/src/test/cli/onosclidriver.py b/src/test/cli/onosclidriver.py
index cc3d8c0..d9c6011 100644
--- a/src/test/cli/onosclidriver.py
+++ b/src/test/cli/onosclidriver.py
@@ -48,8 +48,11 @@
         self.name = None
         self.home = None
         self.handle = None
-        self.controller = os.getenv('ONOS_CONTROLLER_IP') or 'localhost'
-        self.controller = self.controller.split(',')[0]
+	if controller is not None:
+            self.controller = controller
+	else:
+            self.controller = os.getenv('ONOS_CONTROLLER_IP') or 'localhost'
+            self.controller = self.controller.split(',')[0]
         super( CLI, self ).__init__()
         if connect == True:
             self.connect_cli()
diff --git a/src/test/setup/cord-test.py b/src/test/setup/cord-test.py
index 6fc35a8..6ced05b 100755
--- a/src/test/setup/cord-test.py
+++ b/src/test/setup/cord-test.py
@@ -389,6 +389,7 @@
             radius_ip = None
 
     Container.IMAGE_PREFIX = args.prefix
+    cluster_mode = True if args.onos_instances > 1 else False
     if onos_ip is None:
         image_names = args.onos.rsplit(':', 1)
         onos_cnt['image'] = image_names[0]
@@ -405,6 +406,23 @@
         onos = Onos(image = Onos.IMAGE,
                     tag = Onos.TAG, boot_delay = 60)
         onos_ip = onos.ip()
+    onos_ips = [ onos_ip ]
+    num_onos_instances = args.onos_instances
+    if num_onos_instances > 1 and onos is not None:
+        onos_instances = []
+        onos_instances.append(onos)
+        for i in range(1, num_onos_instances):
+            name = '{}-{}'.format(Onos.NAME, i+1)
+            onos = Onos(name = name, image = Onos.IMAGE, tag = Onos.TAG, boot_delay = 60, cluster = cluster_mode)
+            onos_instances.append(onos)
+            onos_ips.append(onos.ipaddr)
+        try:
+            for ip in onos_ips:
+                print('Installing cord tester ONOS app %s in ONOS instance %s' %(args.app,ip))
+                OnosCtrl.install_app(args.app, onos_ip = ip)
+        except: pass
+        Onos.setup_cluster(onos_instances)
+    ctlr_addr = ','.join(onos_ips)
 
     print('Onos IP %s, Test type %s' %(onos_ip, args.test_type))
     if use_manifest or args.test_controller:
@@ -413,10 +431,12 @@
             Onos.install_cord_apps(onos_ip = onos_ip)
         except: pass
 
-    print('Installing cord tester ONOS app %s' %args.app)
-    try:
-        OnosCtrl.install_app(args.app, onos_ip = onos_ip)
-    except: pass
+    if not cluster_mode:
+        print('Installing cord tester ONOS app %s' %args.app)
+        try:
+	    for ip in onos_ips:
+                OnosCtrl.install_app(args.app, onos_ip = onos_ip)
+        except: pass
 
     if radius_ip is None:
         ##Start Radius container
@@ -435,7 +455,7 @@
         maas_api_key = 'UNKNOWN'
 
     ssh_key_file = set_ssh_key_file(args.identity_file)
-    test_cnt_env = { 'ONOS_CONTROLLER_IP' : onos_ip,
+    test_cnt_env = { 'ONOS_CONTROLLER_IP' : ctlr_addr,
                      'ONOS_AAA_IP' : radius_ip if radius_ip is not None else '',
                      'QUAGGA_IP': test_host,
                      'CORD_TEST_HOST' : test_host,
@@ -472,7 +492,7 @@
     for container in range(num_test_containers):
         test_cnt = CordTester(tests_parallel[test_slice_start:test_slice_end],
                               instance = container, num_instances = num_test_containers,
-                              ctlr_ip = onos_ip,
+                              ctlr_ip = ctlr_addr,
                               name = args.container,
                               image = nose_cnt['image'],
                               prefix = Container.IMAGE_PREFIX,
@@ -494,10 +514,13 @@
 
     status = 0
     if len(test_containers) > 1:
-        thread_pool = ThreadPool(len(test_containers), queue_size = 1, wait_timeout=1)
-        for test_cnt in test_containers:
-            thread_pool.addTask(test_cnt.run_tests)
-        thread_pool.cleanUpThreads()
+	if True:
+	    status = test_containers[0].run_tests()
+	else:
+            thread_pool = ThreadPool(len(test_containers), queue_size = 1, wait_timeout=1)
+            for test_cnt in test_containers:
+                thread_pool.addTask(test_cnt.run_tests)
+                thread_pool.cleanUpThreads()
     else:
         if test_containers:
             status = test_containers[0].run_tests()
@@ -505,7 +528,7 @@
     ##Run the linear tests
     if tests_not_parallel:
         test_cnt = CordTester(tests_not_parallel,
-                              ctlr_ip = onos_ip,
+                              ctlr_ip = ctlr_addr,
                               name = args.container,
                               image = nose_cnt['image'],
                               prefix = Container.IMAGE_PREFIX,
@@ -520,7 +543,7 @@
                 test_cnt.start_switch()
         if test_cnt.create and test_cnt.olt:
             test_cnt.setup_intfs(port_num = port_num)
-        status = test_cnt.run_tests()
+        test_cnt.run_tests()
 
     if test_server:
         cord_test_server_stop(test_server)
@@ -910,6 +933,8 @@
     parser_run.add_argument('-d', '--no-switch', action='store_true', help='Dont start test switch.')
     parser_run.add_argument('-i', '--identity-file', default=identity_file_default,
                             type=str, help='ssh identity file to access compute nodes from test container')
+    parser_run.add_argument('-j', '--onos-instances', default=1, type=int,
+                            help='Specify number to test onos instances to form cluster')
     parser_run.set_defaults(func=runTest)
 
 
diff --git a/src/test/utils/OnosFlowCtrl.py b/src/test/utils/OnosFlowCtrl.py
index aa5437e..844bc3c 100644
--- a/src/test/utils/OnosFlowCtrl.py
+++ b/src/test/utils/OnosFlowCtrl.py
@@ -53,7 +53,8 @@
 		  ipv6_target="",
 		  ipv6_sll="",
 		  ipv6_tll="",
-		  ipv6_extension=""):
+		  ipv6_extension="",
+		  controller=None):
         self.deviceId = deviceId
         self.appId = appId
         self.ingressPort = ingressPort
@@ -81,10 +82,13 @@
 	self.ipv6_sll = ipv6_sll
 	self.ipv6_tll = ipv6_tll
 	self.ipv6_extension = ipv6_extension
+	if controller is not None:
+		self.controller=controller
+		self.cfg_url = 'http://%s:8181/onos/v1/flows/' %(self.controller)
 
     @classmethod
-    def get_flows(cls, device_id):
-        return OnosCtrl.get_flows(device_id)
+    def get_flows(cls, device_id,controller=None):
+        return OnosCtrl.get_flows(device_id,controller=controller)
 
     def addFlow(self):
         """
@@ -257,7 +261,7 @@
         return True
 
     def findFlow(self, deviceId, **criterias):
-        flows = self.get_flows(deviceId)
+        flows = self.get_flows(deviceId,controller=self.controller)
         match_keys = criterias.keys()
         matches = len(match_keys)
         num_matched = 0