New test to restart entire ONOS cluster before verifying.
New cord tester api to restart entire cluster used by the test.
Changed the restarts to check for onos start using wait_for_onos_start instead of a hard delay.
Added a robot test for restart cluster.

Change-Id: I8dbc163462570a6a8eaf8e7684c790fc3fea8f48
diff --git a/src/test/utils/CordContainer.py b/src/test/utils/CordContainer.py
index a5dbdc6..b64c8fa 100644
--- a/src/test/utils/CordContainer.py
+++ b/src/test/utils/CordContainer.py
@@ -524,7 +524,7 @@
         cls.setup_cluster(cls.cluster_instances)
 
     @classmethod
-    def restart_cluster(cls, network_cfg = None):
+    def restart_cluster(cls, network_cfg = None, timeout = 10, setup = False):
         if cls.cluster_mode is False:
             return
         if not cls.cluster_instances:
@@ -535,19 +535,21 @@
             with open('{}/network-cfg.json'.format(cls.host_config_dir), 'w') as f:
                 f.write(json_data)
 
+        cls.cleanup_cluster()
+        if timeout > 0:
+            time.sleep(timeout)
+
         for onos in cls.cluster_instances:
-            if onos.exists():
-                onos.kill()
-            onos.remove_container(onos.name, force=True)
             print('Restarting ONOS container %s' %onos.name)
             onos.start(ports = onos.ports, environment = onos.env,
                        host_config = onos.host_config, volumes = onos.volumes, tty = True)
-            print('Waiting %d seconds for ONOS %s to boot' %(onos.boot_delay, onos.name))
-            time.sleep(onos.boot_delay)
             onos.ipaddr = onos.ip()
+            onos.wait_for_onos_start(onos.ipaddr)
+            onos.install_cord_apps(onos.ipaddr)
 
-        ##form the cluster
-        cls.setup_cluster(cls.cluster_instances)
+        ##form the cluster as appropriate
+        if setup is True:
+            cls.setup_cluster(cls.cluster_instances)
 
     @classmethod
     def cluster_ips(cls):
@@ -586,11 +588,9 @@
                 print('Restarting ONOS container %s' %onos.name)
                 onos.start(ports = onos.ports, environment = onos.env,
                            host_config = onos.host_config, volumes = onos.volumes, tty = True)
-                #onos.ipaddr = onos.ip()
-                #onos.wait_for_onos_start(onos.ipaddr)
-                print('Waiting %d seconds for ONOS %s to boot' %(onos.boot_delay, onos.name))
-                time.sleep(onos.boot_delay)
                 onos.ipaddr = onos.ip()
+                onos.wait_for_onos_start(onos.ipaddr)
+                onos.install_cord_apps(onos.ipaddr)
 
     @classmethod
     def install_cord_apps(cls, onos_ip = None):
diff --git a/src/test/utils/CordTestServer.py b/src/test/utils/CordTestServer.py
index e529219..de17261 100644
--- a/src/test/utils/CordTestServer.py
+++ b/src/test/utils/CordTestServer.py
@@ -64,6 +64,13 @@
     def shutdown_onos(self, kwargs):
         return self.__shutdown_onos(**kwargs)
 
+    def __restart_cluster(self, config = None, timeout = 10, setup = False):
+        Onos.restart_cluster(network_cfg = config, timeout = timeout, setup = setup)
+        return 'DONE'
+
+    def restart_cluster(self, kwargs):
+        return self.__restart_cluster(**kwargs)
+
     def __add_cluster_onos(self, count = 1, config = None):
         Onos.add_cluster(count = count, network_cfg = config)
         return 'DONE'
@@ -213,6 +220,17 @@
     return False
 
 @nottest
+def __cord_test_restart_cluster(**kwargs):
+    return rpc_server_instance().restart_cluster(kwargs)
+
+@nottest
+def cord_test_restart_cluster(config = None, timeout = 10, setup = False):
+    data = __cord_test_restart_cluster(config = config, timeout = timeout, setup = setup)
+    if data == 'DONE':
+        return True
+    return False
+
+@nottest
 def __cord_test_onos_add_cluster(**kwargs):
     return rpc_server_instance().add_cluster_onos(kwargs)