Implement support to report error,warning logs after every test case run.

Change-Id: Id5710e9ee3497b4bc1030d7fa69c182d2e1348c6
diff --git a/src/test/cordSubscriber/cordSubscriberTest.py b/src/test/cordSubscriber/cordSubscriberTest.py
index d7c3e04..2494d5c 100644
--- a/src/test/cordSubscriber/cordSubscriberTest.py
+++ b/src/test/cordSubscriber/cordSubscriberTest.py
@@ -35,6 +35,7 @@
 from portmaps import g_subscriber_port_map
 from OltConfig import *
 from CordTestServer import cord_test_onos_restart, cord_test_shell
+from CordLogger import CordLogger
 
 log.setLevel('INFO')
 
@@ -179,7 +180,7 @@
             self.test_status = True
 
 
-class subscriber_exchange(unittest.TestCase):
+class subscriber_exchange(CordLogger):
 
       apps = ('org.opencord.aaa', 'org.onosproject.dhcp')
       olt_apps = () #'org.opencord.cordmcast')
diff --git a/src/test/dhcp/dhcpTest.py b/src/test/dhcp/dhcpTest.py
index 38eb839..e117d68 100644
--- a/src/test/dhcp/dhcpTest.py
+++ b/src/test/dhcp/dhcpTest.py
@@ -24,9 +24,10 @@
 from OltConfig import *
 from OnosCtrl import OnosCtrl
 from portmaps import g_subscriber_port_map
+from CordLogger import CordLogger
 log.setLevel('INFO')
 
-class dhcp_exchange(unittest.TestCase):
+class dhcp_exchange(CordLogger):
 
     dhcp_server_config = {
         "ip": "10.1.11.50",
@@ -64,15 +65,17 @@
 
     def setUp(self):
         ''' Activate the dhcp app'''
+        super(dhcp_exchange, self).setUp()
         self.maxDiff = None ##for assert_equal compare outputs on failure
         self.onos_ctrl = OnosCtrl(self.app)
         status, _ = self.onos_ctrl.activate()
         assert_equal(status, True)
         time.sleep(3)
 
-    def teardown(self):
+    def tearDown(self):
         '''Deactivate the dhcp app'''
         self.onos_ctrl.deactivate()
+        super(dhcp_exchange, self).tearDown()
 
     def onos_load_config(self, config):
         status, code = OnosCtrl.config(config)
diff --git a/src/test/dhcprelay/dhcprelayTest.py b/src/test/dhcprelay/dhcprelayTest.py
index dee0aa4..ea0ff47 100644
--- a/src/test/dhcprelay/dhcprelayTest.py
+++ b/src/test/dhcprelay/dhcprelayTest.py
@@ -24,12 +24,13 @@
 from OnosCtrl import OnosCtrl, get_mac
 from OltConfig import OltConfig
 from CordTestServer import cord_test_onos_restart
+from CordLogger import CordLogger
 from portmaps import g_subscriber_port_map
 import threading, random
 from threading import current_thread
 log.setLevel('INFO')
 
-class dhcprelay_exchange(unittest.TestCase):
+class dhcprelay_exchange(CordLogger):
 
     app = 'org.onosproject.dhcprelay'
     app_dhcp = 'org.onosproject.dhcp'
diff --git a/src/test/flows/flowsTest.py b/src/test/flows/flowsTest.py
index 8fa51fb..54d1ec7 100644
--- a/src/test/flows/flowsTest.py
+++ b/src/test/flows/flowsTest.py
@@ -25,12 +25,13 @@
 from OnosCtrl import OnosCtrl
 from OnosFlowCtrl import OnosFlowCtrl
 from OltConfig import OltConfig
+from CordLogger import CordLogger
 import random
 from threading import current_thread
 import collections
 log.setLevel('INFO')
 
-class flows_exchange(unittest.TestCase):
+class flows_exchange(CordLogger):
 
     #Use the first available device id as our device id to program flows
     app = 'org.onosproject.cli'
diff --git a/src/test/igmp/igmpTest.py b/src/test/igmp/igmpTest.py
index b42af1a..676565b 100644
--- a/src/test/igmp/igmpTest.py
+++ b/src/test/igmp/igmpTest.py
@@ -29,6 +29,7 @@
 from OnosCtrl import OnosCtrl
 from OltConfig import OltConfig
 from Channels import IgmpChannel
+from CordLogger import CordLogger
 log.setLevel('INFO')
 
 class IGMPTestState:
@@ -52,7 +53,7 @@
       def update_state(self):
           self.state = self.state ^ 1
 
-class igmp_exchange(unittest.TestCase):
+class igmp_exchange(CordLogger):
 
     V_INF1 = 'veth0'
     V_INF2 = 'veth1'
@@ -92,11 +93,13 @@
 
     def setUp(self):
         ''' Activate the igmp app'''
+        super(igmp_exchange, self).setUp()
         self.onos_ctrl = OnosCtrl(self.app)
 	self.onos_ctrl.activate()
         self.igmp_channel = IgmpChannel()
 
-    def tearDown(self): pass
+    def tearDown(self):
+        super(igmp_exchange, self).tearDown()
 
     def onos_load_config(self, config):
 	log.info('onos load config is %s'%config)
diff --git a/src/test/netCondition/netConditionTest.py b/src/test/netCondition/netConditionTest.py
index 5d64d62..6e04207 100644
--- a/src/test/netCondition/netConditionTest.py
+++ b/src/test/netCondition/netConditionTest.py
@@ -44,6 +44,7 @@
 import scapy
 from CordTestBase import CordTester
 from CordContainer import *
+from CordLogger import CordLogger
 import re
 from random import randint
 from time import sleep
@@ -75,7 +76,7 @@
       def update_state(self):
           self.state = self.state ^ 1
 
-class netCondition_exchange(unittest.TestCase):
+class netCondition_exchange(CordLogger):
 
     V_INF1 = 'veth0'
     V_INF2 = 'veth1'
diff --git a/src/test/subscriber/subscriberTest.py b/src/test/subscriber/subscriberTest.py
index ab1e6a7..1a65193 100644
--- a/src/test/subscriber/subscriberTest.py
+++ b/src/test/subscriber/subscriberTest.py
@@ -33,6 +33,8 @@
 from portmaps import g_subscriber_port_map
 from OltConfig import *
 from CordContainer import *
+from CordTestServer import cord_test_radius_restart
+from CordLogger import CordLogger
 import copy
 log.setLevel('INFO')
 DEFAULT_NO_CHANNELS = 1
@@ -167,7 +169,7 @@
             log.info('This Subscriber is tested for multiple service elgibility ')
             self.test_status = True
 
-class subscriber_exchange(unittest.TestCase):
+class subscriber_exchange(CordLogger):
 
       apps = [ 'org.opencord.aaa', 'org.onosproject.dhcp' ]
 
@@ -236,6 +238,7 @@
 
       def setUp(self):
           '''Load the OLT config and activate relevant apps'''
+          super(subscriber_exchange, self).setUp()
           self.olt = OltConfig()
           self.port_map, _ = self.olt.olt_port_map()
           ##if no olt config, fall back to ovs port map
@@ -249,20 +252,16 @@
 
           self.activate_apps(self.apps)
 
-      def teardown(self):
+      def tearDown(self):
           '''Deactivate the dhcp app'''
+          super(subscriber_exchange, self).tearDown()
           for app in self.apps:
               onos_ctrl = OnosCtrl(app)
               onos_ctrl.deactivate()
           log.info('Restarting the Radius container in the setup after running every subscriber test cases by default')
-          rest = Container('cord-radius', 'cord-test/radius',)
-          rest.restart('cord-radius','10')
-          radius = Radius()
-          radius_ip = radius.ip()
-          print('Radius server is running with IP %s' %radius_ip)
+          cord_test_radius_restart()
           #os.system('ifconfig '+INTF_RX_DEFAULT+' up')
 
-
       def activate_apps(self, apps):
             for app in apps:
                   onos_ctrl = OnosCtrl(app)
diff --git a/src/test/tls/tlsTest.py b/src/test/tls/tlsTest.py
index 15547e2..1dcf7a2 100644
--- a/src/test/tls/tlsTest.py
+++ b/src/test/tls/tlsTest.py
@@ -21,12 +21,13 @@
 from twisted.internet import defer
 from EapTLS import TLSAuthTest
 from OnosCtrl import OnosCtrl
+from CordLogger import CordLogger
 from scapy.all import *
 from scapy_ssl_tls.ssl_tls import *
 from scapy_ssl_tls.ssl_tls_crypto import *
 log.setLevel('INFO')
 
-class eap_auth_exchange(unittest.TestCase):
+class eap_auth_exchange(CordLogger):
 
     app = 'org.opencord.aaa'
     TLS_TIMEOUT = 20
@@ -72,6 +73,7 @@
                              'TLS_DH_anon_WITH_AES_256_CBC_SHA256']
 
     def setUp(self):
+        super(eap_auth_exchange, self).setUp()
         self.onos_ctrl = OnosCtrl(self.app)
         self.onos_aaa_config()
 
diff --git a/src/test/utils/CordLogger.py b/src/test/utils/CordLogger.py
new file mode 100644
index 0000000..2d9242c
--- /dev/null
+++ b/src/test/utils/CordLogger.py
@@ -0,0 +1,26 @@
+from OnosLog import OnosLog
+from scapy.all import log
+import unittest
+
+class CordLogger(unittest.TestCase):
+
+    def setUp(self):
+        '''Read the log buffer'''
+        try:
+            onosLog = OnosLog()
+            st, output = onosLog.get_log()
+            onosLog.close()
+        except: pass
+
+    def tearDown(self):
+        '''Dump the log buffer for ERRORS/warnings'''
+        try:
+            onosLog = OnosLog()
+            st, output = onosLog.get_log( ('ERROR','WARN') )
+            onosLog.close()
+            if st and output:
+                log.info('\nTest %s has errors and warnings\n' %self._testMethodName)
+                log.info('%s' %output)
+            else:
+                log.info('\nTest %s has no errors and warnings in the logs' %self._testMethodName)
+        except: pass
diff --git a/src/test/utils/OnosLog.py b/src/test/utils/OnosLog.py
new file mode 100644
index 0000000..e876c67
--- /dev/null
+++ b/src/test/utils/OnosLog.py
@@ -0,0 +1,72 @@
+import os
+from SSHTestAgent import SSHTestAgent
+
+class OnosLog(object):
+    CLI_USER = 'karaf'
+    CLI_PASSWD = 'karaf'
+    CLI_PORT = 8101
+    HOST = os.getenv('ONOS_CONTROLLER_IP', '172.17.0.2')
+    last_snapshot_map = {}
+
+    def __init__(self, host = HOST):
+        self.ssh_agent = SSHTestAgent(host = host, user = self.CLI_USER,
+                                      password = self.CLI_PASSWD, port = self.CLI_PORT)
+        if not OnosLog.last_snapshot_map.has_key(host):
+            OnosLog.last_snapshot_map[host] = []
+
+    #should be the last function to call for tearing down the log session
+    def close(self):
+        self.ssh_agent.client.close()
+
+    @classmethod
+    def get_last_snapshot(cls, host):
+        if cls.last_snapshot_map.has_key(host):
+            return cls.last_snapshot_map[host]
+        return []
+
+    @classmethod
+    def update_last_snapshot(cls, host, res):
+        cls.last_snapshot_map[host] = res
+
+    def get_log(self, search_terms = None):
+        """Run the command on the test host"""
+        cmd = 'cat /root/onos/apache-karaf-3.0.5/data/log/karaf.log'
+        st, output = self.ssh_agent.run_cmd(cmd)
+        if st is False:
+            return output
+        last_snapshot = self.get_last_snapshot(self.ssh_agent.host)
+        lines = output.splitlines()
+        if search_terms:
+            if type(search_terms) is str:
+                terms = [ search_terms ]
+            else:
+                terms = search_terms
+            match_lines = []
+            last_len = len(last_snapshot)
+            for i in xrange(0, len(lines)):
+                if i < last_len and lines[i] in last_snapshot:
+                    ##skip lines matching the last snapshot
+                    continue
+                for t in terms:
+                    if lines[i].find(t) >= 0:
+                        match_lines.append(lines[i])
+
+            output = '\n'.join(match_lines)
+
+        #update the last snapshot
+        self.update_last_snapshot(self.ssh_agent.host, lines)
+        return st, output
+
+if __name__ == '__main__':
+    onos = os.getenv('ONOS_CONTROLLER_IP', '172.17.0.2')
+    onos_log = OnosLog(host = onos)
+    print('Checking for INFO')
+    st, output = onos_log.get_log('INFO')
+    print(st, output)
+    print('\n\nChecking for ERROR\n\n')
+    st, output = onos_log.get_log('ERROR')
+    print(st, output)
+    print('Checking for ERROR and INFO')
+    st, output = onos_log.get_log(('ERROR', 'INFO'))
+    print(st, output)
+    onos_log.close()
diff --git a/src/test/utils/SSHTestAgent.py b/src/test/utils/SSHTestAgent.py
index 6e89ffb..12d3530 100644
--- a/src/test/utils/SSHTestAgent.py
+++ b/src/test/utils/SSHTestAgent.py
@@ -9,25 +9,29 @@
     user = 'ubuntu'
     password = None
 
-    def __init__(self, user = user, password = password):
+    def __init__(self, host = host, user = user, password = password, port = 22):
+        self.host = host
         self.user = user
         self.password = password
+        self.port = port
         self.client = SSHClient()
-        self.client.load_system_host_keys()
         self.client.set_missing_host_key_policy(AutoAddPolicy())
 
     def run_cmd(self, cmd, timeout = 5):
         """Run the command on the test host"""
         try:
             self.client.connect(self.host, username = self.user, password = self.password,
-                                key_filename = self.key_file, timeout=timeout)
+                                key_filename = self.key_file, timeout=timeout, port = self.port)
         except:
             log.error('Unable to connect to test host %s' %self.host)
             return False, None
         
         channel = self.client.get_transport().open_session()
         channel.exec_command(cmd)
-        status = channel.recv_exit_status()
+        if channel.exit_status_ready():
+            status = channel.recv_exit_status()
+        else:
+            status = 0
         output = None
         st = status == 0
         if st:
diff --git a/src/test/vrouter/vrouterTest.py b/src/test/vrouter/vrouterTest.py
index 9ca696a..7b416a6 100644
--- a/src/test/vrouter/vrouterTest.py
+++ b/src/test/vrouter/vrouterTest.py
@@ -24,6 +24,7 @@
 from CordContainer import Container, Onos, Quagga
 from CordTestServer import cord_test_onos_restart, cord_test_quagga_restart, cord_test_quagga_stop, cord_test_quagga_shell
 from portmaps import g_subscriber_port_map
+from CordLogger import CordLogger
 import threading
 import time
 import os
@@ -38,7 +39,7 @@
 
 log.setLevel('INFO')
 
-class vrouter_exchange(unittest.TestCase):
+class vrouter_exchange(CordLogger):
 
     apps = ('org.onosproject.vrouter', 'org.onosproject.fwd')
     device_id = 'of:' + get_mac()