More support for ONOS in cluster mode
Add a ONOS cluster test case to randomly kill controller and check for exceptions.
Also verify leader status on all the nodes.
Change-Id: I0c5f9e35069c249dcee7135df6f59d2b6e8ad2b4
diff --git a/src/test/utils/CordContainer.py b/src/test/utils/CordContainer.py
index fb71403..c4a73a0 100644
--- a/src/test/utils/CordContainer.py
+++ b/src/test/utils/CordContainer.py
@@ -201,7 +201,9 @@
def restart(self, timeout =10):
return self.dckr.restart(self.name, timeout)
-def get_mem():
+def get_mem(instances = 1):
+ if instances <= 0:
+ instances = 1
with open('/proc/meminfo', 'r') as fd:
meminfo = fd.readlines()
mem = 0
@@ -209,7 +211,7 @@
if m.startswith('MemTotal:') or m.startswith('SwapTotal:'):
mem += int(m.split(':')[1].strip().split()[0])
- mem = max(mem/1024/1024/2, 1)
+ mem = max(mem/1024/1024/2/instances, 1)
mem = min(mem, 16)
return str(mem) + 'G'
@@ -299,7 +301,9 @@
quagga_config = ( { 'bridge' : 'quagga-br', 'ip': '10.10.0.4', 'mask' : 16 }, )
SYSTEM_MEMORY = (get_mem(),) * 2
+ INSTANCE_MEMORY = (get_mem(instances=3),) * 2
JAVA_OPTS = '-Xms{} -Xmx{} -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode'.format(*SYSTEM_MEMORY)#-XX:+PrintGCDetails -XX:+PrintGCTimeStamps'
+ JAVA_OPTS_CLUSTER = '-Xms{} -Xmx{} -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode'.format(*INSTANCE_MEMORY)
env = { 'ONOS_APPS' : 'drivers,openflow,proxyarp,vrouter', 'JAVA_OPTS' : JAVA_OPTS }
onos_cord_apps = ( ('cord-config', '1.0-SNAPSHOT'),
('aaa', '1.0-SNAPSHOT'),
@@ -369,6 +373,7 @@
self.boot_delay = boot_delay
if cluster is True:
self.ports = []
+ self.env['JAVA_OPTS'] = self.JAVA_OPTS_CLUSTER
if os.access(self.cluster_cfg, os.F_OK):
try:
os.unlink(self.cluster_cfg)
diff --git a/src/test/utils/OnosLog.py b/src/test/utils/OnosLog.py
index 5baa6f1..f5dcb77 100644
--- a/src/test/utils/OnosLog.py
+++ b/src/test/utils/OnosLog.py
@@ -6,7 +6,7 @@
CLI_USER = 'karaf'
CLI_PASSWD = 'karaf'
CLI_PORT = 8101
- HOST = os.getenv('ONOS_CONTROLLER_IP', '172.17.0.2')
+ HOST = os.getenv('ONOS_CONTROLLER_IP', '172.17.0.2').split(',')[0]
last_snapshot_map = {}
def __init__(self, host = HOST):
@@ -30,7 +30,7 @@
cmd = 'cat /root/onos/apache-karaf-3.0.5/data/log/karaf.log'
st, output = self.ssh_agent.run_cmd(cmd)
if st is False:
- return output
+ return st, output
exception_map = {'Exception' : [] }
last_snapshot = self.get_last_snapshot(self.ssh_agent.host)
lines = output.splitlines()