New test to restart entire ONOS cluster before verifying.
New cord tester api to restart entire cluster used by the test.
Changed the restarts to check for onos start using wait_for_onos_start instead of a hard delay.
Added a robot test for restart cluster.

Change-Id: I8dbc163462570a6a8eaf8e7684c790fc3fea8f48
diff --git a/src/test/utils/CordContainer.py b/src/test/utils/CordContainer.py
index a5dbdc6..b64c8fa 100644
--- a/src/test/utils/CordContainer.py
+++ b/src/test/utils/CordContainer.py
@@ -524,7 +524,7 @@
         cls.setup_cluster(cls.cluster_instances)
 
     @classmethod
-    def restart_cluster(cls, network_cfg = None):
+    def restart_cluster(cls, network_cfg = None, timeout = 10, setup = False):
         if cls.cluster_mode is False:
             return
         if not cls.cluster_instances:
@@ -535,19 +535,21 @@
             with open('{}/network-cfg.json'.format(cls.host_config_dir), 'w') as f:
                 f.write(json_data)
 
+        cls.cleanup_cluster()
+        if timeout > 0:
+            time.sleep(timeout)
+
         for onos in cls.cluster_instances:
-            if onos.exists():
-                onos.kill()
-            onos.remove_container(onos.name, force=True)
             print('Restarting ONOS container %s' %onos.name)
             onos.start(ports = onos.ports, environment = onos.env,
                        host_config = onos.host_config, volumes = onos.volumes, tty = True)
-            print('Waiting %d seconds for ONOS %s to boot' %(onos.boot_delay, onos.name))
-            time.sleep(onos.boot_delay)
             onos.ipaddr = onos.ip()
+            onos.wait_for_onos_start(onos.ipaddr)
+            onos.install_cord_apps(onos.ipaddr)
 
-        ##form the cluster
-        cls.setup_cluster(cls.cluster_instances)
+        ##form the cluster as appropriate
+        if setup is True:
+            cls.setup_cluster(cls.cluster_instances)
 
     @classmethod
     def cluster_ips(cls):
@@ -586,11 +588,9 @@
                 print('Restarting ONOS container %s' %onos.name)
                 onos.start(ports = onos.ports, environment = onos.env,
                            host_config = onos.host_config, volumes = onos.volumes, tty = True)
-                #onos.ipaddr = onos.ip()
-                #onos.wait_for_onos_start(onos.ipaddr)
-                print('Waiting %d seconds for ONOS %s to boot' %(onos.boot_delay, onos.name))
-                time.sleep(onos.boot_delay)
                 onos.ipaddr = onos.ip()
+                onos.wait_for_onos_start(onos.ipaddr)
+                onos.install_cord_apps(onos.ipaddr)
 
     @classmethod
     def install_cord_apps(cls, onos_ip = None):