Wait for ONOS to start by checking ONOS logs instead of a static delay.
Enable exception logs in onos log.
Change-Id: Iff967d1daab63e2d3d8f7ac90fe76e2cd7abfa39
diff --git a/src/test/setup/prerequisites.sh b/src/test/setup/prerequisites.sh
index 21920c4..efa374c 100755
--- a/src/test/setup/prerequisites.sh
+++ b/src/test/setup/prerequisites.sh
@@ -41,7 +41,7 @@
pip install -U robotframework
pip install -U robotframework-requests
pip install -U robotframework-sshlibrary
-pip install paramiko
+pip install paramiko==1.10.1
( cd /tmp && git clone https://github.com/jpetazzo/pipework.git && cp -v pipework/pipework /usr/bin && rm -rf pipework )
## Special mode to pull cord-tester repo in case prereqs was installed by hand instead of repo
if [ "$1" = "--test" ]; then
diff --git a/src/test/utils/CordContainer.py b/src/test/utils/CordContainer.py
index 81addcb..4d0c5bb 100644
--- a/src/test/utils/CordContainer.py
+++ b/src/test/utils/CordContainer.py
@@ -23,6 +23,7 @@
from docker import Client
from shutil import copy
from OnosCtrl import OnosCtrl
+from OnosLog import OnosLog
class docker_netns(object):
@@ -353,7 +354,7 @@
except: pass
def __init__(self, name = NAME, image = IMAGE, prefix = PREFIX, tag = TAG,
- boot_delay = 60, restart = False, network_cfg = None, cluster = False):
+ boot_delay = 20, restart = False, network_cfg = None, cluster = False):
if restart is True:
##Find the right image to restart
running_image = filter(lambda c: c['Names'][0] == '/{}'.format(name), self.dckr.containers())
@@ -411,13 +412,32 @@
print('Restarting ONOS container %s' %self.name)
self.start(ports = self.ports, environment = self.env,
host_config = self.host_config, volumes = self.volumes, tty = True)
- print('Waiting %d seconds for ONOS to boot' %(boot_delay))
+ print('Waiting for ONOS to boot')
time.sleep(boot_delay)
+ self.wait_for_onos_start(self.ip())
+
self.ipaddr = self.ip()
if cluster is False:
self.install_cord_apps(self.ipaddr)
@classmethod
+ def wait_for_onos_start(cls, ip, tries = 30):
+ onos_log = OnosLog(host = ip)
+ num_tries = 0
+ started = None
+ while not started and num_tries < tries:
+ time.sleep(3)
+ started = onos_log.search_log_pattern('ApplicationManager .* Started')
+ num_tries += 1
+
+ onos_log.close()
+ if not started:
+ print('ONOS did not start')
+ else:
+ print('ONOS started')
+ return started
+
+ @classmethod
def setup_cluster_deprecated(cls, onos_instances, image_name = None):
if not onos_instances or len(onos_instances) < 2:
return
@@ -922,3 +942,6 @@
@classmethod
def build_image(cls, image = IMAGE):
Xos.build_image(image, cls.dockerfile_path)
+
+if __name__ == '__main__':
+ onos = Onos(boot_delay = 10, restart = True)
diff --git a/src/test/utils/OnosLog.py b/src/test/utils/OnosLog.py
index e876c67..6bdef9f 100644
--- a/src/test/utils/OnosLog.py
+++ b/src/test/utils/OnosLog.py
@@ -1,4 +1,5 @@
import os
+import re
from SSHTestAgent import SSHTestAgent
class OnosLog(object):
@@ -28,19 +29,22 @@
def update_last_snapshot(cls, host, res):
cls.last_snapshot_map[host] = res
- def get_log(self, search_terms = None):
+ def get_log(self, search_terms = None, exception = True):
"""Run the command on the test host"""
cmd = 'cat /root/onos/apache-karaf-3.0.5/data/log/karaf.log'
st, output = self.ssh_agent.run_cmd(cmd)
if st is False:
return output
+ exception_map = {'Exception' : [] }
last_snapshot = self.get_last_snapshot(self.ssh_agent.host)
lines = output.splitlines()
if search_terms:
if type(search_terms) is str:
terms = [ search_terms ]
else:
- terms = search_terms
+ terms = list(search_terms)
+ if exception is True and 'Exception' not in terms:
+ terms.append('Exception')
match_lines = []
last_len = len(last_snapshot)
for i in xrange(0, len(lines)):
@@ -50,13 +54,23 @@
for t in terms:
if lines[i].find(t) >= 0:
match_lines.append(lines[i])
-
+ if t == 'Exception':
+ exception_map[t] = lines[i+1:i+1+10]
output = '\n'.join(match_lines)
+ output += '\n'.join(exception_map['Exception'])
#update the last snapshot
self.update_last_snapshot(self.ssh_agent.host, lines)
return st, output
+ def search_log_pattern(self, pattern):
+ r_pat = re.compile(pattern)
+ cmd = 'cat /root/onos/apache-karaf-3.0.5/data/log/karaf.log'
+ st, output = self.ssh_agent.run_cmd(cmd)
+ if st is False:
+ return None
+ return r_pat.findall(output)
+
if __name__ == '__main__':
onos = os.getenv('ONOS_CONTROLLER_IP', '172.17.0.2')
onos_log = OnosLog(host = onos)
@@ -70,3 +84,8 @@
st, output = onos_log.get_log(('ERROR', 'INFO'))
print(st, output)
onos_log.close()
+ pat = onos_log.search_log_pattern('ApplicationManager .* Started')
+ if pat:
+ print(pat)
+ else:
+ print('Onos did not start')
diff --git a/src/test/utils/SSHTestAgent.py b/src/test/utils/SSHTestAgent.py
index 12d3530..4cd7666 100644
--- a/src/test/utils/SSHTestAgent.py
+++ b/src/test/utils/SSHTestAgent.py
@@ -1,11 +1,11 @@
import os, sys
from paramiko import SSHClient, WarningPolicy, AutoAddPolicy
-from CordTestServer import CORD_TEST_HOST
from scapy.all import *
class SSHTestAgent(object):
key_file = os.getenv('SSH_KEY_FILE', None)
- host = os.getenv('CORD_TEST_HOST', CORD_TEST_HOST)
+ host = os.getenv('CORD_TEST_HOST', '172.17.0.1')
+ hosts_file = os.path.join(os.getenv('HOME'), '.ssh', 'known_hosts')
user = 'ubuntu'
password = None
@@ -19,7 +19,9 @@
def run_cmd(self, cmd, timeout = 5):
"""Run the command on the test host"""
+ host_remove = 'ssh-keygen -f "%s" -R [%s]:8101' %(self.hosts_file, self.host)
try:
+ os.system(host_remove)
self.client.connect(self.host, username = self.user, password = self.password,
key_filename = self.key_file, timeout=timeout, port = self.port)
except: