Test: Changes to cordSubscriber to delete ovs groups before the test.
This is done through a remote rpc or local as appropriate.
This also seems to make the single channel N subscriber tests receive packets after join as OVS seems flaky otherwise.
Support for cord-test.py run to execute tests with container name setup with:
cord-test.py setup --olt --start-switch
option.

Change-Id: Ifca43e9b7e76bb231ed8ad84b6971d985b66b9e1
diff --git a/src/test/cordSubscriber/cordSubscriberTest.py b/src/test/cordSubscriber/cordSubscriberTest.py
index 6606ec5..27dbc97 100644
--- a/src/test/cordSubscriber/cordSubscriberTest.py
+++ b/src/test/cordSubscriber/cordSubscriberTest.py
@@ -19,7 +19,7 @@
 from threadPool import ThreadPool
 from portmaps import g_subscriber_port_map
 from OltConfig import *
-from CordTestServer import cord_test_onos_restart
+from CordTestServer import cord_test_onos_restart, cord_test_shell
 
 log.setLevel('INFO')
 
@@ -122,6 +122,10 @@
             log.info('Subscriber %s on port %s receiving from group %s, channel %d' %
                      (self.name, self.rx_intf, self.gaddr(chan), chan))
             r = self.recv(chan, cb = cb, count = count, timeout = timeout)
+            if len(r) == 0:
+                  log.info('Subscriber %s on port %s timed out' %(self.name, self.rx_intf))
+            else:
+                  log.info('Subscriber %s on port %s received %d packets' %(self.name, self.rx_intf, len(r)))
             if self.recv_timeout:
                   ##Negative test case is disabled for now
                   assert_equal(len(r), 0)
@@ -156,7 +160,7 @@
                            self.test_status = True
                            log.info('This service is failed and other services will not run for this subscriber')
                            break
-            log.info('This Subscriber is tested for multiple service elgibility ')
+            log.info('This Subscriber is tested for multiple service eligibility ')
             self.test_status = True
 
 
@@ -164,6 +168,7 @@
 
       apps = ('org.opencord.aaa', 'org.onosproject.dhcp')
       olt_apps = () #'org.opencord.cordmcast')
+      vtn_app = 'org.opencord.vtn'
       table_app = 'org.ciena.cordigmp'
       dhcp_server_config = {
         "ip": "10.1.11.50",
@@ -301,6 +306,8 @@
             log.info('Installing the multi table app %s for subscriber test' %(cls.table_app_file))
             OnosCtrl.install_app(cls.table_app_file)
             time.sleep(3)
+            onos_ctrl = OnosCtrl(cls.vtn_app)
+            onos_ctrl.deactivate()
 
       @classmethod
       def uninstall_app_table(cls):
@@ -309,6 +316,8 @@
             time.sleep(2)
             log.info('Installing back the cord igmp app %s for subscriber test on exit' %(cls.app_file))
             OnosCtrl.install_app(cls.app_file)
+            onos_ctrl = OnosCtrl(cls.vtn_app)
+            onos_ctrl.activate()
 
       @classmethod
       def start_onos(cls, network_cfg = None):
@@ -352,6 +361,16 @@
             assert_equal(ret, 0)
             time.sleep(30)
 
+      @classmethod
+      def ovs_cleanup(cls):
+            ##For every test case, delete all the OVS groups
+            cmd = 'ovs-ofctl del-groups br-int -OOpenFlow11 >/dev/null 2>&1'
+            cord_test_shell(cmd)
+            ##Since olt config is used for this test, we just fire a careless local cmd as well
+            try:
+                  os.system(cmd)
+            except: pass
+
       def onos_aaa_load(self):
             if self.aaa_loaded:
                   return
@@ -418,6 +437,7 @@
                   else:
                         log.info('GET request from %s succeeded for subscriber %s'
                                  %(url, subscriber.name))
+                  return self.test_status
 
       def tls_verify(self, subscriber):
             if subscriber.has_service('TLS'):
@@ -556,8 +576,8 @@
 
       def subscriber_join_verify( self, num_subscribers = 10, num_channels = 1,
                                   channel_start = 0, cbs = None, port_list = [], negative_subscriber_auth = None):
-
           self.test_status = False
+          self.ovs_cleanup()
           subscribers_count = num_subscribers
           sub_loop_count =  num_subscribers
           self.subscriber_load(create = True, num = num_subscribers,
diff --git a/src/test/setup/cord-test.py b/src/test/setup/cord-test.py
index d29b9d2..31d72ab 100755
--- a/src/test/setup/cord-test.py
+++ b/src/test/setup/cord-test.py
@@ -40,12 +40,13 @@
     IMAGE = 'cord-test/nose'
     ALL_TESTS = ('tls', 'dhcp', 'dhcprelay','igmp', 'subscriber', 'cordSubscriber', 'vrouter', 'flows', 'proxyarp', 'acl')
 
-    def __init__(self, tests, instance = 0, num_instances = 1, ctlr_ip = None, image = IMAGE, tag = 'latest',
+    def __init__(self, tests, instance = 0, num_instances = 1, ctlr_ip = None,
+                 name = '', image = IMAGE, tag = 'latest',
                  env = None, rm = False, update = False):
         self.tests = tests
         self.ctlr_ip = ctlr_ip
         self.rm = rm
-        self.name = self.get_name()
+        self.name = name or self.get_name()
         super(CordTester, self).__init__(self.name, image = image, tag = tag)
         host_config = self.create_host_config(host_guest_map = self.host_guest_map, privileged = True)
         volumes = []
@@ -53,8 +54,13 @@
             volumes.append(g)
         if update is True or not self.img_exists():
             self.build_image(image)
-        ##Remove test container if any
-        self.remove_container(self.name, force=True)
+        self.create = True
+        #check if are trying to run tests on existing container
+        if not name or not self.exists():
+            ##Remove test container if any
+            self.remove_container(self.name, force=True)
+        else:
+            self.create = False
         self.olt = False
         if env is not None and env.has_key('OLT_CONFIG'):
             self.olt = True
@@ -71,9 +77,10 @@
             env['TEST_HOST'] = self.name
             env['TEST_INSTANCE'] = instance
             env['TEST_INSTANCES'] = num_instances
-        print('Starting test container %s, image %s, tag %s' %(self.name, self.image, self.tag))
-        self.start(rm = False, volumes = volumes, environment = env,
-                   host_config = host_config, tty = True)
+        if self.create:
+            print('Starting test container %s, image %s, tag %s' %(self.name, self.image, self.tag))
+            self.start(rm = False, volumes = volumes, environment = env,
+                       host_config = host_config, tty = True)
 
     def execute_switch(self, cmd, shell = False):
         if self.olt:
@@ -270,6 +277,7 @@
             cmd = 'nosetests -v --collect-only {0}/../{1}/{2}'.format(cls.tester_base, test, test_file)
             os.system(cmd)
 
+
 ##default onos/radius/test container images and names
 onos_image_default='onosproject/onos:latest'
 nose_image_default= '{}:latest'.format(CordTester.IMAGE)
@@ -360,6 +368,11 @@
         olt_conf_test_loc = os.path.join(CordTester.sandbox_setup, 'olt_config.json')
         test_cnt_env['OLT_CONFIG'] = olt_conf_test_loc
 
+    if args.num_containers > 1 and args.container:
+        print('Cannot specify number of containers with container option')
+        sys.exit(1)
+    if args.container:
+        args.keep = True
     port_num = 0
     num_tests = len(tests_parallel)
     tests_per_container = max(1, num_tests/args.num_containers)
@@ -371,7 +384,8 @@
     for container in range(num_test_containers):
         test_cnt = CordTester(tests_parallel[test_slice_start:test_slice_end],
                               instance = container, num_instances = num_test_containers,
-                              ctlr_ip = onos_ip, image = nose_cnt['image'], tag = nose_cnt['tag'],
+                              ctlr_ip = onos_ip,
+                              name = args.container, image = nose_cnt['image'], tag = nose_cnt['tag'],
                               env = test_cnt_env,
                               rm = False if args.keep else True,
                               update = update_map['test'])
@@ -379,9 +393,11 @@
         test_slice_end = test_slice_start + tests_per_container
         update_map['test'] = False
         test_containers.append(test_cnt)
-        if args.start_switch or not args.olt:
+        if not test_cnt.create:
+            continue
+        if test_cnt.create and (args.start_switch or not args.olt):
             test_cnt.start_switch()
-        if test_cnt.olt:
+        if test_cnt.create and test_cnt.olt:
             _, port_num = test_cnt.setup_intfs(port_num = port_num)
 
     thread_pool = ThreadPool(len(test_containers), queue_size = 1, wait_timeout=1)
@@ -392,13 +408,14 @@
     ##Run the linear tests
     if tests_not_parallel:
         test_cnt = CordTester(tests_not_parallel,
-                              ctlr_ip = onos_ip, image = nose_cnt['image'], tag = nose_cnt['tag'],
+                              ctlr_ip = onos_ip,
+                              name = args.container, image = nose_cnt['image'], tag = nose_cnt['tag'],
                               env = test_cnt_env,
                               rm = False if args.keep else True,
                               update = update_map['test'])
-        if args.start_switch or not args.olt:
+        if test_cnt.create and (args.start_switch or not args.olt):
             test_cnt.start_switch()
-        if test_cnt.olt:
+        if test_cnt.create and test_cnt.olt:
             test_cnt.setup_intfs(port_num = port_num)
         test_cnt.run_tests()
 
@@ -577,6 +594,7 @@
                         '    --update=all to rebuild all cord tester images.')
     parser_run.add_argument('-n', '--num-containers', default=1, type=int,
                             help='Specify number of test containers to spawn for tests')
+    parser_run.add_argument('-c', '--container', default='', type=str, help='Test container name for running tests')
     parser_run.set_defaults(func=runTest)
 
 
diff --git a/src/test/setup/of-bridge-local.sh b/src/test/setup/of-bridge-local.sh
index 8ad125d..5bd3ba7 100755
--- a/src/test/setup/of-bridge-local.sh
+++ b/src/test/setup/of-bridge-local.sh
@@ -9,7 +9,7 @@
 fi
 pkill -9 ofdatapath
 pkill -9 ofprotocol
-service openvswitch-switch start
+service openvswitch-switch restart
 echo "Configuring ovs bridge $bridge"
 ovs-vsctl del-br $bridge
 ovs-vsctl add-br $bridge
diff --git a/src/test/utils/CordTestServer.py b/src/test/utils/CordTestServer.py
index d946acb..3b97814 100644
--- a/src/test/utils/CordTestServer.py
+++ b/src/test/utils/CordTestServer.py
@@ -89,9 +89,18 @@
             ret = os.system(exec_cmd)
         return ret
 
+    def __run_shell(self, cmd = None):
+        ret = 0
+        if cmd is not None:
+            ret = os.system(cmd)
+        return ret
+
     def run_shell_quagga(self, kwargs):
         return self.__run_shell_quagga(**kwargs)
 
+    def run_shell(self, kwargs):
+        return self.__run_shell(**kwargs)
+
     def restart_radius(self):
         print('Restarting RADIUS Server')
         Radius(restart = True)
@@ -167,6 +176,15 @@
     return __cord_test_quagga_shell(cmd = cmd)
 
 @nottest
+def __cord_test_shell(**kwargs):
+    return rpc_server_instance().run_shell(kwargs)
+
+@nottest
+def cord_test_shell(cmd = None):
+    '''Send shell cmd to run remotely'''
+    return __cord_test_shell(cmd = cmd)
+
+@nottest
 def cord_test_quagga_stop():
     data = rpc_server_instance().stop_quagga()
     if data == 'DONE':