Enable logging and log archiving for cluster tests.
Retry on cord_test_onos_restart/cluster_restart rpc failures.
Change-Id: I549056d72c25c08933a4e9bebfd36f951a608b1c
diff --git a/src/test/cluster/clusterTest.py b/src/test/cluster/clusterTest.py
index 64ae73f..5fa6d36 100644
--- a/src/test/cluster/clusterTest.py
+++ b/src/test/cluster/clusterTest.py
@@ -81,10 +81,10 @@
controllers = os.getenv('ONOS_CONTROLLER_IP') or ''
return controllers.split(',')
- def cliEnter(self,controller = None):
+ def cliEnter(self, controller = None):
retries = 0
- while retries < 3:
- self.cli = OnosCliDriver(controller = controller,connect = True)
+ while retries < 30:
+ self.cli = OnosCliDriver(controller = controller, connect = True)
if self.cli.handle:
break
else:
@@ -439,7 +439,7 @@
if controller in ips and inclusive is False:
log.info('Controller %s still ACTIVE on Node %s after it was shutdown' %(controller, ctlr))
if controller not in ips and inclusive is True:
- log.info('Controller %s still INACTIVE on Node %s after it was shutdown' %(controller, ctlr))
+ log.info('Controller %s still INACTIVE on Node %s after it was restarted' %(controller, ctlr))
return controller
@@ -447,8 +447,10 @@
#chose a random controller for shutdown/restarts
controller = controllers[random.randrange(0, ctlr_len)]
controller_name = onos_map[controller]
+ ##enable the log level for the controllers
+ self.log_set(controllers = controllers)
+ self.log_set(app = 'io.atomix', controllers = controllers)
for num in range(tries):
- index = num % ctlr_len
log.info('ITERATION: %d. Shutting down Controller %s' %(num + 1, controller_name))
try:
cord_test_onos_shutdown(node = controller_name)
@@ -461,6 +463,8 @@
#Now restart the controller back
log.info('Restarting back the controller %s' %controller_name)
cord_test_onos_restart(node = controller_name)
+ self.log_set(controllers = controller)
+ self.log_set(app = 'io.atomix', controllers = controller)
time.sleep(60)
#archive the logs for this run
CordLogger.archive_results('test_cluster_single_controller_restarts',
@@ -514,6 +518,8 @@
log.info('ITERATION: %d. Restarting cluster with controllers at %s' %(num+1, controllers))
try:
cord_test_restart_cluster()
+ self.log_set(controllers = controllers)
+ self.log_set(app = 'io.atomix', controllers = controllers)
log.info('Delaying before verifying cluster status')
time.sleep(60)
except:
diff --git a/src/test/utils/CordLogger.py b/src/test/utils/CordLogger.py
index 51c7dc8..b026ab0 100644
--- a/src/test/utils/CordLogger.py
+++ b/src/test/utils/CordLogger.py
@@ -22,7 +22,7 @@
if not controller:
continue
retries = 0
- while retries < 3:
+ while retries < 30:
cli = OnosCliDriver(controller = controller, connect = True)
if cli.handle:
cls.cliSessions[controller] = cli
diff --git a/src/test/utils/CordTestServer.py b/src/test/utils/CordTestServer.py
index de17261..1359850 100644
--- a/src/test/utils/CordTestServer.py
+++ b/src/test/utils/CordTestServer.py
@@ -203,9 +203,14 @@
@nottest
def cord_test_onos_restart(node = None, config = None, timeout = 10):
'''Send ONOS restart to server'''
- data = __cord_test_onos_restart(node = node, config = config, timeout = timeout)
- if data == 'DONE':
- return True
+ for i in range(3):
+ try:
+ data = __cord_test_onos_restart(node = node, config = config, timeout = timeout)
+ if data == 'DONE':
+ return True
+ except:
+ time.sleep(2)
+
return False
@nottest
@@ -225,9 +230,14 @@
@nottest
def cord_test_restart_cluster(config = None, timeout = 10, setup = False):
- data = __cord_test_restart_cluster(config = config, timeout = timeout, setup = setup)
- if data == 'DONE':
- return True
+ for i in range(3):
+ try:
+ data = __cord_test_restart_cluster(config = config, timeout = timeout, setup = setup)
+ if data == 'DONE':
+ return True
+ except:
+ time.sleep(2)
+
return False
@nottest