src/test/cluster/clusterTest.py

#copyright 2016-present Ciena Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import unittest
from nose.tools import *
from scapy.all import *
from OnosCtrl import OnosCtrl, get_mac
from OltConfig import OltConfig
from socket import socket
from OnosFlowCtrl import OnosFlowCtrl
from nose.twistedtools import reactor, deferred
from twisted.internet import defer
from onosclidriver import OnosCliDriver
from CordContainer import Container, Onos, Quagga
from CordTestServer import cord_test_onos_restart, cord_test_onos_shutdown, cord_test_onos_add_cluster, cord_test_quagga_restart, cord_test_restart_cluster
from portmaps import g_subscriber_port_map
from scapy.all import *
import time, monotonic
import threading
from threading import current_thread
from Cluster import *
from EapTLS import TLSAuthTest
from ACL import ACLTest
from OnosLog import OnosLog
from CordLogger import CordLogger
import os
import json
import random
import collections
log.setLevel('INFO')

class cluster_exchange(CordLogger):
    test_path = os.path.dirname(os.path.realpath(__file__))
    onos_config_path = os.path.join(test_path, '..', 'setup/onos-config')
    mac = RandMAC()._fix()
    flows_eth = Ether(src = RandMAC()._fix(), dst = RandMAC()._fix())
    igmp_eth = Ether(dst = '01:00:5e:00:00:16', type = ETH_P_IP)
    igmp_ip = IP(dst = '224.0.0.22')
    ONOS_INSTANCES = 3
    V_INF1 = 'veth0'
    TLS_TIMEOUT = 100
    device_id = 'of:' + get_mac()
    igmp = cluster_igmp()
    igmp_groups = igmp.mcast_ip_range(start_ip = '224.1.8.10',end_ip = '224.1.10.49')
    igmp_sources = igmp.source_ip_range(start_ip = '38.24.29.35',end_ip='38.24.35.56')
    tls = cluster_tls()
    flows = cluster_flows()
    proxyarp = cluster_proxyarp()
    vrouter = cluster_vrouter()
    acl = cluster_acl()
    dhcprelay = cluster_dhcprelay()
    subscriber = cluster_subscriber()
    testcaseLoggers = ('test_cluster_controller_restarts', 'test_cluster_graceful_controller_restarts',
                       'test_cluster_single_controller_restarts', 'test_cluster_restarts')

    def setUp(self):
        if self._testMethodName not in self.testcaseLoggers:
            super(cluster_exchange, self).setUp()

    def tearDown(self):
        if self._testMethodName not in self.testcaseLoggers:
            super(cluster_exchange, self).tearDown()

    def get_controller(self):
        controller = os.getenv('ONOS_CONTROLLER_IP') or 'localhost'
        controller = controller.split(',')[0]
        return controller

    @classmethod
    def get_controllers(cls):
        controllers = os.getenv('ONOS_CONTROLLER_IP') or ''
        return controllers.split(',')

    def cliEnter(self, controller = None):
        retries = 0
        while retries < 30:
            self.cli = OnosCliDriver(controller = controller, connect = True)
            if self.cli.handle:
                break
            else:
                retries += 1
                time.sleep(2)

    def cliExit(self):
        self.cli.disconnect()

    def get_leader(self, controller = None):
        self.cliEnter(controller = controller)
        try:
            result = json.loads(self.cli.leaders(jsonFormat = True))
        except:
            result = None

        if result is None:
            log.info('Leaders command failure for controller %s' %controller)
        else:
            log.info('Leaders returned: %s' %result)
        self.cliExit()
        return result

    def onos_shutdown(self, controller = None):
        status = True
        self.cliEnter(controller = controller)
        try:
            self.cli.shutdown(timeout = 10)
        except:
            log.info('Graceful shutdown of ONOS failed for controller: %s' %controller)
            status = False

        self.cliExit()
        return status

    def log_set(self, level = None, app = 'org.onosproject', controllers = None):
        CordLogger.logSet(level = level, app = app, controllers = controllers, forced = True)

    def get_leaders(self, controller = None):
        result_map = {}
        if controller is None:
            controller = self.get_controller()
        if type(controller) in [ list, tuple ]:
            for c in controller:
                leaders = self.get_leader(controller = c)
                result_map[c] = leaders
        else:
            leaders = self.get_leader(controller = controller)
            result_map[controller] = leaders
        return result_map

    def verify_leaders(self, controller = None):
        leaders_map = self.get_leaders(controller = controller)
        failed = [ k for k,v in leaders_map.items() if v == None ]
        return failed

    def verify_cluster_status(self,controller = None,onos_instances=ONOS_INSTANCES,verify=False):
	tries = 0
	try:
            self.cliEnter(controller = controller)
	    while tries <= 10:
                cluster_summary = json.loads(self.cli.summary(jsonFormat = True))
                if cluster_summary:
	            log.info("cluster 'summary' command output is %s"%cluster_summary)
		    nodes = cluster_summary['nodes']
		    if verify:
		        if nodes == onos_instances:
		            self.cliExit()
		            return True
		        else:
		            tries += 1
		            time.sleep(1)
		    else:
			if nodes >= onos_instances:
                            self.cliExit()
                            return True
                        else:
                            tries += 1
                            time.sleep(1)
	        else:
	            tries += 1
	            time.sleep(1)
	    self.cliExit()
	    return False
        except:
            raise Exception('Failed to get cluster members')
	    return False

    def get_cluster_current_member_ips(self, controller = None, nodes_filter = None):
        tries = 0
	cluster_ips = []
        try:
            self.cliEnter(controller = controller)
            while tries <= 10:
                cluster_nodes = json.loads(self.cli.nodes(jsonFormat = True))
                if cluster_nodes:
                    log.info("cluster 'nodes' output is %s"%cluster_nodes)
                    if nodes_filter:
                        cluster_nodes = nodes_filter(cluster_nodes)
                    cluster_ips = map(lambda c: c['id'], cluster_nodes)
		    self.cliExit()
                    cluster_ips.sort(lambda i1,i2: int(i1.split('.')[-1]) - int(i2.split('.')[-1]))
		    return cluster_ips
		else:
		    tries += 1
	    self.cliExit()
	    return cluster_ips
        except:
            raise Exception('Failed to get cluster members')
            return cluster_ips

    def get_cluster_container_names_ips(self,controller=None):
        onos_names_ips = {}
        onos_ips = self.get_cluster_current_member_ips(controller=controller)
        onos_names_ips[onos_ips[0]] = Onos.NAME
        onos_names_ips[Onos.NAME] = onos_ips[0]
        for i in range(1,len(onos_ips)):
            name = '{0}-{1}'.format(Onos.NAME,i+1)
            onos_names_ips[onos_ips[i]] = name
            onos_names_ips[name] = onos_ips[i]

        return onos_names_ips

    #identifying current master of a connected device, not tested
    def get_cluster_current_master_standbys(self,controller=None,device_id=device_id):
	master = None
	standbys = []
	tries = 0
	try:
	    cli = self.cliEnter(controller = controller)
	    while tries <= 10:
	        roles = json.loads(self.cli.roles(jsonFormat = True))
	        log.info("cluster 'roles' command output is %s"%roles)
	        if roles:
	            for device in roles:
	                log.info('Verifying device info in line %s'%device)
	                if device['id'] == device_id:
	                    master = str(device['master'])
		            standbys = map(lambda d: str(d), device['standbys'])
		            log.info('Master and standbys for device %s are %s and %s'%(device_id, master, standbys))
			    self.cliExit()
		            return master, standbys
		    self.cliExit()
		    return master, standbys
	        else:
		    tries += 1
		    time.sleep(1)
	    self.cliExit()
	    return master,standbys
	except:
            raise Exception('Failed to get cluster members')
	    return master,standbys

    def get_cluster_current_master_standbys_of_connected_devices(self,controller=None):
	''' returns master and standbys of all the connected devices to ONOS cluster instance'''
        device_dict = {}
        tries = 0
        try:
            cli = self.cliEnter(controller = controller)
            while tries <= 10:
		device_dict = {}
                roles = json.loads(self.cli.roles(jsonFormat = True))
                log.info("cluster 'roles' command output is %s"%roles)
                if roles:
                    for device in roles:
			device_dict[str(device['id'])]= {'master':str(device['master']),'standbys':device['standbys']}
                        for i in range(len(device_dict[device['id']]['standbys'])):
			    device_dict[device['id']]['standbys'][i] = str(device_dict[device['id']]['standbys'][i])
                        log.info('master and standbys for device %s are %s and %s'%(device['id'],device_dict[device['id']]['master'],device_dict[device['id']]['standbys']))
                    self.cliExit()
                    return device_dict
                else:
                    tries += 1
                    time.sleep(1)
            self.cliExit()
            return device_dict
        except:
            raise Exception('Failed to get cluster members')
            return device_dict

    #identify current master of a connected device, not tested
    def get_cluster_connected_devices(self,controller=None):
	'''returns all the devices connected to ONOS cluster'''
        device_list = []
        tries = 0
        try:
            cli = self.cliEnter(controller = controller)
            while tries <= 10:
		device_list = []
                devices = json.loads(self.cli.devices(jsonFormat = True))
                log.info("cluster 'devices' command output is %s"%devices)
                if devices:
                    for device in devices:
			log.info('device id is %s'%device['id'])
			device_list.append(str(device['id']))
                    self.cliExit()
                    return device_list
                else:
                    tries += 1
                    time.sleep(1)
            self.cliExit()
            return device_list
        except:
            raise Exception('Failed to get cluster members')
            return device_list

    def get_number_of_devices_of_master(self,controller=None):
	'''returns master-device pairs, which master having what devices'''
	master_count = {}
	try:
	    cli = self.cliEnter(controller = controller)
	    masters = json.loads(self.cli.masters(jsonFormat = True))
	    if masters:
		for master in masters:
		    master_count[str(master['id'])] = {'size':int(master['size']),'devices':master['devices']}
		return master_count
	    else:
		return master_count
	except:
            raise Exception('Failed to get cluster members')
            return master_count

    def change_master_current_cluster(self,new_master=None,device_id=device_id,controller=None):
	if new_master is None: return False
	self.cliEnter(controller=controller)
        cmd = 'device-role' + ' ' + device_id + ' ' + new_master + ' ' + 'master'
        command = self.cli.command(cmd = cmd, jsonFormat = False)
        self.cliExit()
        time.sleep(60)
        master, standbys = self.get_cluster_current_master_standbys(controller=controller,device_id=device_id)
        assert_equal(master,new_master)
	log.info('Cluster master changed to %s successfully'%new_master)

    def withdraw_cluster_current_mastership(self,master_ip=None,device_id=device_id,controller=None):
	'''current master looses its mastership and hence new master will be elected'''
        self.cliEnter(controller=controller)
        cmd = 'device-role' + ' ' + device_id + ' ' + master_ip + ' ' + 'none'
        command = self.cli.command(cmd = cmd, jsonFormat = False)
        self.cliExit()
        time.sleep(60)
        new_master_ip, standbys = self.get_cluster_current_master_standbys(controller=controller,device_id=device_id)
        assert_not_equal(new_master_ip,master_ip)
	log.info('Device-role of device %s successfully changed to none for controller %s'%(device_id,master_ip))
	log.info('Cluster new master is %s'%new_master_ip)
	return True

    def cluster_controller_restarts(self, graceful = False):
        controllers = self.get_controllers()
        ctlr_len = len(controllers)
        if ctlr_len <= 1:
            log.info('ONOS is not running in cluster mode. This test only works for cluster mode')
            assert_greater(ctlr_len, 1)

        #this call would verify the cluster for once
        onos_map = self.get_cluster_container_names_ips()

        def check_exception(controller = None):
            adjacent_controller = None
            adjacent_controllers = None
            if controller:
                adjacent_controllers = list(set(controllers) - set([controller]))
                adjacent_controller = adjacent_controllers[0]
            for node in controllers:
                onosLog = OnosLog(host = node)
                ##check the logs for storage exception
                _, output = onosLog.get_log(('ERROR', 'Exception',))
                if output and output.find('StorageException$Timeout') >= 0:
                    log.info('\nStorage Exception Timeout found on node: %s\n' %node)
                    log.info('Dumping the ERROR and Exception logs for node: %s\n' %node)
                    log.info('\n' + '-' * 50 + '\n')
                    log.info('%s' %output)
                    log.info('\n' + '-' * 50 + '\n')
                    failed = self.verify_leaders(controllers)
                    if failed:
                        log.info('Leaders command failed on nodes: %s' %failed)
                        assert_equal(len(failed), 0)
                    return controller

            try:
                ips = self.get_cluster_current_member_ips(controller = adjacent_controller)
                log.info('ONOS cluster formed with controllers: %s' %ips)
                st = True
            except:
                st = False

            failed = self.verify_leaders(controllers)
            assert_equal(len(failed), 0)
            if st is False:
                log.info('No storage exception and ONOS cluster was not formed successfully')
            else:
                controller = None

            return controller

        next_controller = None
        tries = 10
        for num in range(tries):
            index = num % ctlr_len
            #index = random.randrange(0, ctlr_len)
            controller_name = onos_map[controllers[index]] if next_controller is None else onos_map[next_controller]
            controller = onos_map[controller_name]
            log.info('ITERATION: %d. Restarting Controller %s' %(num + 1, controller_name))
            try:
                #enable debug log for the other controllers before restarting this controller
                adjacent_controllers = list( set(controllers) - set([controller]) )
                self.log_set(controllers = adjacent_controllers)
                self.log_set(app = 'io.atomix', controllers = adjacent_controllers)
                if graceful is True:
                    self.onos_shutdown(controller)
                cord_test_onos_restart(node = controller, timeout = 0)
                self.log_set(controllers = controller)
                self.log_set(app = 'io.atomix', controllers = controller)
                time.sleep(60)
            except:
                time.sleep(5)
                continue

            #first archive the test case logs for this run
            CordLogger.archive_results(self._testMethodName,
                                       controllers = controllers,
                                       iteration = 'iteration_{}'.format(num+1))
            next_controller = check_exception(controller = controller)

    def test_cluster_controller_restarts(self):
        '''Test the cluster by repeatedly killing the controllers'''
        self.cluster_controller_restarts()

    def test_cluster_graceful_controller_restarts(self):
        '''Test the cluster by repeatedly restarting the controllers gracefully'''
        self.cluster_controller_restarts(graceful = True)

    def test_cluster_single_controller_restarts(self):
        '''Test the cluster by repeatedly restarting the same controller'''
        controllers = self.get_controllers()
        ctlr_len = len(controllers)
        if ctlr_len <= 1:
            log.info('ONOS is not running in cluster mode. This test only works for cluster mode')
            assert_greater(ctlr_len, 1)

        #this call would verify the cluster for once
        onos_map = self.get_cluster_container_names_ips()

        def check_exception(controller, inclusive = False):
            adjacent_controllers = list(set(controllers) - set([controller]))
            adjacent_controller = adjacent_controllers[0]
            controller_list = adjacent_controllers if inclusive == False else controllers
            storage_exceptions = []
            for node in controller_list:
                onosLog = OnosLog(host = node)
                ##check the logs for storage exception
                _, output = onosLog.get_log(('ERROR', 'Exception',))
                if output and output.find('StorageException$Timeout') >= 0:
                    log.info('\nStorage Exception Timeout found on node: %s\n' %node)
                    log.info('Dumping the ERROR and Exception logs for node: %s\n' %node)
                    log.info('\n' + '-' * 50 + '\n')
                    log.info('%s' %output)
                    log.info('\n' + '-' * 50 + '\n')
                    storage_exceptions.append(node)

            failed = self.verify_leaders(controller_list)
            if failed:
                log.info('Leaders command failed on nodes: %s' %failed)
                if storage_exceptions:
                    log.info('Storage exception seen on nodes: %s' %storage_exceptions)
                    assert_equal(len(failed), 0)
                    return controller

            for ctlr in controller_list:
                ips = self.get_cluster_current_member_ips(controller = ctlr,
                                                          nodes_filter = \
                                                          lambda nodes: [ n for n in nodes if n['state'] in [ 'ACTIVE', 'READY'] ])
                log.info('ONOS cluster on node %s formed with controllers: %s' %(ctlr, ips))
                if controller in ips and inclusive is False:
                    log.info('Controller %s still ACTIVE on Node %s after it was shutdown' %(controller, ctlr))
                if controller not in ips and inclusive is True:
                    log.info('Controller %s still INACTIVE on Node %s after it was restarted' %(controller, ctlr))

            return controller

        tries = 10
        #chose a random controller for shutdown/restarts
        controller = controllers[random.randrange(0, ctlr_len)]
        controller_name = onos_map[controller]
        ##enable the log level for the controllers
        self.log_set(controllers = controllers)
        self.log_set(app = 'io.atomix', controllers = controllers)
        for num in range(tries):
            log.info('ITERATION: %d. Shutting down Controller %s' %(num + 1, controller_name))
            try:
                cord_test_onos_shutdown(node = controller)
                time.sleep(20)
            except:
                time.sleep(5)
                continue
            #check for exceptions on the adjacent nodes
            check_exception(controller)
            #Now restart the controller back
            log.info('Restarting back the controller %s' %controller_name)
            cord_test_onos_restart(node = controller)
            self.log_set(controllers = controller)
            self.log_set(app = 'io.atomix', controllers = controller)
            time.sleep(60)
            #archive the logs for this run
            CordLogger.archive_results('test_cluster_single_controller_restarts',
                                       controllers = controllers,
                                       iteration = 'iteration_{}'.format(num+1))
            check_exception(controller, inclusive = True)

    def test_cluster_restarts(self):
        '''Test the cluster by repeatedly restarting the entire cluster'''
        controllers = self.get_controllers()
        ctlr_len = len(controllers)
        if ctlr_len <= 1:
            log.info('ONOS is not running in cluster mode. This test only works for cluster mode')
            assert_greater(ctlr_len, 1)

        #this call would verify the cluster for once
        onos_map = self.get_cluster_container_names_ips()

        def check_exception():
            controller_list = controllers
            storage_exceptions = []
            for node in controller_list:
                onosLog = OnosLog(host = node)
                ##check the logs for storage exception
                _, output = onosLog.get_log(('ERROR', 'Exception',))
                if output and output.find('StorageException$Timeout') >= 0:
                    log.info('\nStorage Exception Timeout found on node: %s\n' %node)
                    log.info('Dumping the ERROR and Exception logs for node: %s\n' %node)
                    log.info('\n' + '-' * 50 + '\n')
                    log.info('%s' %output)
                    log.info('\n' + '-' * 50 + '\n')
                    storage_exceptions.append(node)

            failed = self.verify_leaders(controller_list)
            if failed:
                log.info('Leaders command failed on nodes: %s' %failed)
                if storage_exceptions:
                    log.info('Storage exception seen on nodes: %s' %storage_exceptions)
                    assert_equal(len(failed), 0)
                    return

            for ctlr in controller_list:
                ips = self.get_cluster_current_member_ips(controller = ctlr,
                                                          nodes_filter = \
                                                          lambda nodes: [ n for n in nodes if n['state'] in [ 'ACTIVE', 'READY'] ])
                log.info('ONOS cluster on node %s formed with controllers: %s' %(ctlr, ips))
                assert_equal(len(ips), len(controllers))

        tries = 10
        for num in range(tries):
            log.info('ITERATION: %d. Restarting cluster with controllers at %s' %(num+1, controllers))
            try:
                cord_test_restart_cluster()
                self.log_set(controllers = controllers)
                self.log_set(app = 'io.atomix', controllers = controllers)
                log.info('Delaying before verifying cluster status')
                time.sleep(60)
            except:
                time.sleep(10)
                continue

            #archive the logs for this run before verification
            CordLogger.archive_results('test_cluster_restarts',
                                       controllers = controllers,
                                       iteration = 'iteration_{}'.format(num+1))
            #check for exceptions on the adjacent nodes
            check_exception()

    #pass
    def test_cluster_formation_and_verification(self,onos_instances = ONOS_INSTANCES):
	status = self.verify_cluster_status(onos_instances = onos_instances)
	assert_equal(status, True)
	log.info('Cluster exists with %d ONOS instances'%onos_instances)

    #nottest cluster not coming up properly if member goes down
    def test_cluster_adding_members(self, add = 2, onos_instances = ONOS_INSTANCES):
	status = self.verify_cluster_status(onos_instances = onos_instances)
	assert_equal(status, True)
        onos_ips = self.get_cluster_current_member_ips()
	onos_instances = len(onos_ips)+add
        log.info('Adding %d nodes to the ONOS cluster' %add)
        cord_test_onos_add_cluster(count = add)
	status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status, True)

    def test_cluster_removing_master(self, onos_instances = ONOS_INSTANCES):
        status = self.verify_cluster_status(onos_instances = onos_instances)
        assert_equal(status, True)
	master, standbys = self.get_cluster_current_master_standbys()
        assert_equal(len(standbys),(onos_instances-1))
        onos_names_ips =  self.get_cluster_container_names_ips()
        master_onos_name = onos_names_ips[master]
        log.info('Removing cluster current master %s'%(master))
        cord_test_onos_shutdown(node = master)
        time.sleep(60)
        onos_instances -= 1
        status = self.verify_cluster_status(onos_instances = onos_instances,controller=standbys[0])
        assert_equal(status, True)
	new_master, standbys = self.get_cluster_current_master_standbys(controller=standbys[0])
	assert_not_equal(master,new_master)
	log.info('Successfully removed clusters master instance')

    def test_cluster_removing_one_member(self, onos_instances = ONOS_INSTANCES):
        status = self.verify_cluster_status(onos_instances = onos_instances)
        assert_equal(status, True)
	master, standbys = self.get_cluster_current_master_standbys()
        assert_equal(len(standbys),(onos_instances-1))
        onos_names_ips =  self.get_cluster_container_names_ips()
        member_onos_name = onos_names_ips[standbys[0]]
	log.info('Removing cluster member %s'%standbys[0])
        cord_test_onos_shutdown(node = standbys[0])
	time.sleep(60)
	onos_instances -= 1
        status = self.verify_cluster_status(onos_instances = onos_instances,controller=master)
        assert_equal(status, True)

    def test_cluster_removing_two_members(self,onos_instances = ONOS_INSTANCES):
        status = self.verify_cluster_status(onos_instances = onos_instances)
        assert_equal(status, True)
       	master, standbys = self.get_cluster_current_master_standbys()
        assert_equal(len(standbys),(onos_instances-1))
        onos_names_ips =  self.get_cluster_container_names_ips()
        member1_onos_name = onos_names_ips[standbys[0]]
        member2_onos_name = onos_names_ips[standbys[1]]
        log.info('Removing cluster member %s'%standbys[0])
        cord_test_onos_shutdown(node = standbys[0])
        log.info('Removing cluster member %s'%standbys[1])
        cord_test_onos_shutdown(node = standbys[1])
        time.sleep(60)
        onos_instances = onos_instances - 2
        status = self.verify_cluster_status(onos_instances = onos_instances,controller=master)
        assert_equal(status, True)

    def test_cluster_removing_N_members(self,remove = 2, onos_instances = ONOS_INSTANCES):
        status = self.verify_cluster_status(onos_instances = onos_instances)
        assert_equal(status, True)
	master, standbys = self.get_cluster_current_master_standbys()
        assert_equal(len(standbys),(onos_instances-1))
        onos_names_ips =  self.get_cluster_container_names_ips()
        for i in range(remove):
	    member_onos_name = onos_names_ips[standbys[i]]
            log.info('Removing onos container with name %s'%standbys[i])
            cord_test_onos_shutdown(node = standbys[i])
        time.sleep(60)
        onos_instances = onos_instances - remove
        status = self.verify_cluster_status(onos_instances = onos_instances, controller=master)
        assert_equal(status, True)

    #nottest test cluster not coming up properly if member goes down
    def test_cluster_adding_and_removing_members(self,onos_instances = ONOS_INSTANCES , add = 2, remove = 2):
        status = self.verify_cluster_status(onos_instances = onos_instances)
        assert_equal(status, True)
        onos_ips = self.get_cluster_current_member_ips()
        onos_instances = len(onos_ips)+add
        log.info('Adding %d ONOS instances to the cluster'%add)
        cord_test_onos_add_cluster(count = add)
        status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status, True)
        log.info('Removing %d ONOS instances from the cluster'%remove)
        for i in range(remove):
            name = '{}-{}'.format(Onos.NAME, onos_instances - i)
            log.info('Removing onos container with name %s'%name)
            cord_test_onos_shutdown(node = name)
        time.sleep(60)
        onos_instances = onos_instances-remove
        status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status, True)

    #nottest cluster not coming up properly if member goes down
    def test_cluster_removing_and_adding_member(self,onos_instances = ONOS_INSTANCES,add = 1, remove = 1):
        status = self.verify_cluster_status(onos_instances = onos_instances)
        assert_equal(status, True)
        onos_ips = self.get_cluster_current_member_ips()
        onos_instances = onos_instances-remove
        log.info('Removing %d ONOS instances from the cluster'%remove)
        for i in range(remove):
            name = '{}-{}'.format(Onos.NAME, len(onos_ips)-i)
            log.info('Removing onos container with name %s'%name)
            cord_test_onos_shutdown(node = name)
        time.sleep(60)
        status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status, True)
        log.info('Adding %d ONOS instances to the cluster'%add)
        cord_test_onos_add_cluster(count = add)
        onos_instances = onos_instances+add
        status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status, True)

    def test_cluster_restart(self, onos_instances = ONOS_INSTANCES):
	status = self.verify_cluster_status(onos_instances = onos_instances)
        assert_equal(status, True)
	log.info('Restarting cluster')
	cord_test_onos_restart()
	status = self.verify_cluster_status(onos_instances = onos_instances)
        assert_equal(status, True)

    def test_cluster_master_restart(self,onos_instances = ONOS_INSTANCES):
        status = self.verify_cluster_status(onos_instances = onos_instances)
        assert_equal(status, True)
	master, standbys = self.get_cluster_current_master_standbys()
        onos_names_ips =  self.get_cluster_container_names_ips()
        master_onos_name = onos_names_ips[master]
        log.info('Restarting cluster master %s'%master)
        cord_test_onos_restart(node = master)
        status = self.verify_cluster_status(onos_instances = onos_instances)
        assert_equal(status, True)
	log.info('Cluster came up after master restart as expected')

    #test fail. master changing after restart. Need to check correct behavior.
    def test_cluster_master_ip_after_master_restart(self,onos_instances = ONOS_INSTANCES):
        status = self.verify_cluster_status(onos_instances = onos_instances)
        assert_equal(status, True)
        master1, standbys = self.get_cluster_current_master_standbys()
        onos_names_ips =  self.get_cluster_container_names_ips()
        master_onos_name = onos_names_ips[master1]
        log.info('Restarting cluster master %s'%master1)
        cord_test_onos_restart(node = master1)
        status = self.verify_cluster_status(onos_instances = onos_instances)
        assert_equal(status, True)
	master2, standbys = self.get_cluster_current_master_standbys()
	assert_equal(master1,master2)
        log.info('Cluster master is same before and after cluster master restart as expected')

    def test_cluster_one_member_restart(self,onos_instances = ONOS_INSTANCES):
        status = self.verify_cluster_status(onos_instances = onos_instances)
        assert_equal(status, True)
	master, standbys = self.get_cluster_current_master_standbys()
	assert_equal(len(standbys),(onos_instances-1))
        onos_names_ips =  self.get_cluster_container_names_ips()
	member_onos_name = onos_names_ips[standbys[0]]
        log.info('Restarting cluster member %s'%standbys[0])
        cord_test_onos_restart(node = standbys[0])
        status = self.verify_cluster_status(onos_instances = onos_instances)
        assert_equal(status, True)
	log.info('Cluster came up as expected after restarting one member')

    def test_cluster_two_members_restart(self,onos_instances = ONOS_INSTANCES):
        status = self.verify_cluster_status(onos_instances = onos_instances)
        assert_equal(status, True)
	master, standbys = self.get_cluster_current_master_standbys()
        assert_equal(len(standbys),(onos_instances-1))
        onos_names_ips =  self.get_cluster_container_names_ips()
        member1_onos_name = onos_names_ips[standbys[0]]
        member2_onos_name = onos_names_ips[standbys[1]]
        log.info('Restarting cluster members %s and %s'%(standbys[0],standbys[1]))
        cord_test_onos_restart(node = standbys[0])
        cord_test_onos_restart(node = standbys[1])
        status = self.verify_cluster_status(onos_instances = onos_instances)
        assert_equal(status, True)
	log.info('Cluster came up as expected after restarting two members')

    def test_cluster_state_with_N_members_restart(self, members = 2, onos_instances = ONOS_INSTANCES):
        status = self.verify_cluster_status(onos_instances = onos_instances)
        assert_equal(status,True)
	master, standbys = self.get_cluster_current_master_standbys()
        assert_equal(len(standbys),(onos_instances-1))
        onos_names_ips =  self.get_cluster_container_names_ips()
	for i in range(members):
            member_onos_name = onos_names_ips[standbys[i]]
	    log.info('Restarting cluster member %s'%standbys[i])
            cord_test_onos_restart(node = standbys[i])

        status = self.verify_cluster_status(onos_instances = onos_instances)
        assert_equal(status, True)
	log.info('Cluster came up as expected after restarting %d members'%members)

    def test_cluster_state_with_master_change(self,onos_instances = ONOS_INSTANCES):
        status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status, True)
        master, standbys = self.get_cluster_current_master_standbys()
	assert_equal(len(standbys),(onos_instances-1))
        log.info('Cluster current master of devices is %s'%master)
	self.change_master_current_cluster(new_master=standbys[0])
        log.info('Cluster master changed successfully')

    #tested on single onos setup.
    def test_cluster_with_vrouter_routes_in_cluster_members(self,networks = 5,onos_instances = ONOS_INSTANCES):
        status = self.verify_cluster_status(onos_instances = onos_instances)
        assert_equal(status, True)
	onos_ips = self.get_cluster_current_member_ips()
        self.vrouter.setUpClass()
        res = self.vrouter.vrouter_network_verify(networks, peers = 1)
        assert_equal(res, True)
        for onos_ip in onos_ips:
            tries = 0
            flag = False
            try:
                self.cliEnter(controller = onos_ip)
                while tries <= 5:
                    routes = json.loads(self.cli.routes(jsonFormat = True))
                    if routes:
                        assert_equal(len(routes['routes4']), networks)
                        self.cliExit()
                        flag = True
                        break
                    else:
                        tries += 1
                        time.sleep(1)
                assert_equal(flag, True)
            except:
                log.info('Exception occured while checking routes in onos instance %s'%onos_ip)
                raise

    #tested on single onos setup.
    def test_cluster_with_vrouter_and_master_down(self,networks = 5, onos_instances = ONOS_INSTANCES):
        status = self.verify_cluster_status(onos_instances = onos_instances)
        assert_equal(status, True)
        onos_ips = self.get_cluster_current_member_ips()
	master, standbys = self.get_cluster_current_master_standbys()
	onos_names_ips =  self.get_cluster_container_names_ips()
	master_onos_name = onos_names_ips[master]
        self.vrouter.setUpClass()
        res = self.vrouter.vrouter_network_verify(networks, peers = 1)
	assert_equal(res,True)
        cord_test_onos_shutdown(node = master)
	time.sleep(60)
	log.info('Verifying vrouter traffic after cluster master is down')
	self.vrouter.vrouter_traffic_verify()

    #tested on single onos setup.
    def test_cluster_with_vrouter_and_restarting_master(self,networks = 5,onos_instances = ONOS_INSTANCES):
        status = self.verify_cluster_status(onos_instances = onos_instances)
        assert_equal(status, True)
        onos_ips = self.get_cluster_current_member_ips()
        master, standbys = self.get_cluster_current_master_standbys()
        onos_names_ips =  self.get_cluster_container_names_ips()
        master_onos_name = onos_names_ips[master]
        self.vrouter.setUpClass()
        res = self.vrouter.vrouter_network_verify(networks, peers = 1)
        assert_equal(res, True)
        cord_test_onos_restart()
	self.vrouter.vrouter_traffic_verify()

    #tested on single onos setup.
    def test_cluster_deactivating_vrouter_app(self,networks = 5, onos_instances = ONOS_INSTANCES):
        status = self.verify_cluster_status(onos_instances = onos_instances)
        assert_equal(status, True)
        self.vrouter.setUpClass()
        res = self.vrouter.vrouter_network_verify(networks, peers = 1)
        assert_equal(res, True)
	self.vrouter.vrouter_activate(deactivate=True)
        time.sleep(15)
	self.vrouter.vrouter_traffic_verify(positive_test=False)
	self.vrouter.vrouter_activate(deactivate=False)

    #tested on single onos setup.
    def test_cluster_deactivating_vrouter_app_and_making_master_down(self,networks = 5,onos_instances = ONOS_INSTANCES):
        status = self.verify_cluster_status(onos_instances = onos_instances)
        assert_equal(status, True)
	master, standbys = self.get_cluster_current_master_standbys()
        onos_names_ips =  self.get_cluster_container_names_ips()
        master_onos_name = onos_names_ips[master]
        self.vrouter.setUpClass()
	log.info('Verifying vrouter before master down')
        res = self.vrouter.vrouter_network_verify(networks, peers = 1)
        assert_equal(res, True)
	self.vrouter.vrouter_activate(deactivate=True)
	log.info('Verifying vrouter traffic after app deactivated')
        time.sleep(15) ## Expecting vrouter should work properly if master of cluster goes down
        self.vrouter.vrouter_traffic_verify(positive_test=False)
	log.info('Verifying vrouter traffic after master down')
        cord_test_onos_shutdown(node = master)
	time.sleep(60)
	self.vrouter.vrouter_traffic_verify(positive_test=False)
        self.vrouter.vrouter_activate(deactivate=False)

    #tested on single onos setup.
    def test_cluster_for_vrouter_app_and_making_member_down(self, networks = 5,onos_instances = ONOS_INSTANCES):
        status = self.verify_cluster_status(onos_instances = onos_instances)
        assert_equal(status, True)
        master, standbys = self.get_cluster_current_master_standbys()
        onos_names_ips =  self.get_cluster_container_names_ips()
        member_onos_name = onos_names_ips[standbys[0]]
        self.vrouter.setUpClass()
        log.info('Verifying vrouter before cluster member down')
        res = self.vrouter.vrouter_network_verify(networks, peers = 1)
        assert_equal(res, True) # Expecting vrouter should work properly
        log.info('Verifying vrouter after cluster member down')
        cord_test_onos_shutdown(node = standbys[0])
	time.sleep(60)
	self.vrouter.vrouter_traffic_verify()# Expecting vrouter should work properly if member of cluster goes down

    #tested on single onos setup.
    def test_cluster_for_vrouter_app_and_restarting_member(self,networks = 5, onos_instances = ONOS_INSTANCES):
        status = self.verify_cluster_status(onos_instances = onos_instances)
        assert_equal(status, True)
        master, standbys = self.get_cluster_current_master_standbys()
        onos_names_ips =  self.get_cluster_container_names_ips()
        member_onos_name = onos_names_ips[standbys[1]]
        self.vrouter.setUpClass()
        log.info('Verifying vrouter traffic before cluster member restart')
        res = self.vrouter.vrouter_network_verify(networks, peers = 1)
        assert_equal(res, True) # Expecting vrouter should work properly
        cord_test_onos_restart(node = standbys[1])
	log.info('Verifying vrouter traffic after cluster member restart')
        self.vrouter.vrouter_traffic_verify()# Expecting vrouter should work properly if member of cluster restarts

    #tested on single onos setup.
    def test_cluster_for_vrouter_app_restarting_cluster(self,networks = 5, onos_instances = ONOS_INSTANCES):
        status = self.verify_cluster_status(onos_instances = onos_instances)
        assert_equal(status, True)
        self.vrouter.setUpClass()
        log.info('Verifying vrouter traffic before cluster restart')
        res = self.vrouter.vrouter_network_verify(networks, peers = 1)
        assert_equal(res, True) # Expecting vrouter should work properly
	cord_test_onos_restart()
        log.info('Verifying vrouter traffic after cluster restart')
        self.vrouter.vrouter_traffic_verify()# Expecting vrouter should work properly if member of cluster restarts


    #test fails because flow state is in pending_add in onos
    def test_cluster_for_flows_of_udp_port_and_making_master_down(self, onos_instances = ONOS_INSTANCES):
        status = self.verify_cluster_status(onos_instances = onos_instances)
        assert_equal(status, True)
	master, standbys = self.get_cluster_current_master_standbys()
	onos_names_ips =  self.get_cluster_container_names_ips()
        master_onos_name = onos_names_ips[master]
        self.flows.setUpClass()
        egress = 1
        ingress = 2
        egress_map = { 'ip': '192.168.30.1', 'udp_port': 9500 }
        ingress_map = { 'ip': '192.168.40.1', 'udp_port': 9000 }
        flow = OnosFlowCtrl(deviceId = self.device_id,
                            egressPort = egress,
                            ingressPort = ingress,
                            udpSrc = ingress_map['udp_port'],
                            udpDst = egress_map['udp_port'],
			    controller=master
                            )
        result = flow.addFlow()
        assert_equal(result, True)
        time.sleep(1)
        self.success = False
        def mac_recv_task():
            def recv_cb(pkt):
                log.info('Pkt seen with ingress UDP port %s, egress UDP port %s' %(pkt[UDP].sport, pkt[UDP].dport))
                self.success = True
            sniff(timeout=2,
             lfilter = lambda p: UDP in p and p[UDP].dport == egress_map['udp_port']
                                and p[UDP].sport == ingress_map['udp_port'], prn = recv_cb, iface = self.flows.port_map[egress])

	for i in [0,1]:
	    if i == 1:
                cord_test_onos_shutdown(node = master)
                log.info('Verifying flows traffic after master killed')
                time.sleep(45)
	    else:
		log.info('Verifying flows traffic before master killed')
            t = threading.Thread(target = mac_recv_task)
            t.start()
            L2 = self.flows_eth #Ether(src = ingress_map['ether'], dst = egress_map['ether'])
            L3 = IP(src = ingress_map['ip'], dst = egress_map['ip'])
            L4 = UDP(sport = ingress_map['udp_port'], dport = egress_map['udp_port'])
            pkt = L2/L3/L4
            log.info('Sending packets to verify if flows are correct')
            sendp(pkt, count=50, iface = self.flows.port_map[ingress])
            t.join()
            assert_equal(self.success, True)

    def test_cluster_state_changing_master_and_flows_of_ecn(self,onos_instances = ONOS_INSTANCES):
        status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status, True)
	master, standbys = self.get_cluster_current_master_standbys()
        self.flows.setUpClass()
        egress = 1
        ingress = 2
        egress_map = { 'ip': '192.168.30.1' }
        ingress_map = { 'ip': '192.168.40.1' }
        flow = OnosFlowCtrl(deviceId = self.device_id,
                            egressPort = egress,
                            ingressPort = ingress,
                            ecn = 1,
			    controller=master
                            )
        result = flow.addFlow()
        assert_equal(result, True)
        ##wait for flows to be added to ONOS
        time.sleep(1)
        self.success = False
        def mac_recv_task():
            def recv_cb(pkt):
                log.info('Pkt seen with ingress ip %s, egress ip %s and Type of Service %s' %(pkt[IP].src, pkt[IP].dst, pkt[IP].tos))
                self.success = True
            sniff(count=2, timeout=5,
                  lfilter = lambda p: IP in p and p[IP].dst == egress_map['ip'] and p[IP].src == ingress_map['ip']
                        and int(bin(p[IP].tos).split('b')[1][-2:],2) == 1,prn = recv_cb,
                                iface = self.flows.port_map[egress])
	for i in [0,1]:
	    if i == 1:
		log.info('Changing cluster master to %s'%standbys[0])
		self.change_master_current_cluster(new_master=standbys[0])
		log.info('Verifying flow traffic after cluster master chnaged')
	    else:
		log.info('Verifying flow traffic  before cluster master changed')
            t = threading.Thread(target = mac_recv_task)
            t.start()
            L2 = self.flows_eth # Ether(src = ingress_map['ether'], dst = egress_map['ether'])
            L3 = IP(src = ingress_map['ip'], dst = egress_map['ip'], tos = 1)
            pkt = L2/L3
            log.info('Sending a packet to verify if flows are correct')
            sendp(pkt, count=50, iface = self.flows.port_map[ingress])
            t.join()
            assert_equal(self.success, True)

    #pass
    def test_cluster_flow_for_ipv6_extension_header_and_master_restart(self,onos_instances = ONOS_INSTANCES):
        status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status, True)
	master,standbys = self.get_cluster_current_master_standbys()
	onos_names_ips =  self.get_cluster_container_names_ips()
        master_onos_name = onos_names_ips[master]
        self.flows.setUpClass()
        egress = 1
        ingress = 2
        egress_map = { 'ipv6': '2001:db8:a0b:12f0:1010:1010:1010:1001' }
        ingress_map = { 'ipv6': '2001:db8:a0b:12f0:1010:1010:1010:1002' }
        flow = OnosFlowCtrl(deviceId = self.device_id,
                            egressPort = egress,
                            ingressPort = ingress,
                            ipv6_extension = 0,
			    controller=master
                            )

        result = flow.addFlow()
        assert_equal(result, True)
        ##wait for flows to be added to ONOS
        time.sleep(1)
        self.success = False
        def mac_recv_task():
            def recv_cb(pkt):
                log.info('Pkt seen with ingress ip %s, egress ip %s, Extension Header Type %s'%(pkt[IPv6].src, pkt[IPv6].dst, pkt[IPv6].nh))
                self.success = True
            sniff(timeout=2,count=5,
                  lfilter = lambda p: IPv6 in p and p[IPv6].nh == 0, prn = recv_cb, iface = self.flows.port_map[egress])
	for i in [0,1]:
	    if i == 1:
		log.info('Restart cluster current master %s'%master)
                Container(master_onos_name,Onos.IMAGE).restart()
                time.sleep(45)
	        log.info('Verifying flow traffic after master restart')
	    else:
		log.info('Verifying flow traffic before master restart')
            t = threading.Thread(target = mac_recv_task)
            t.start()
            L2 = self.flows_eth
            L3 = IPv6(src = ingress_map['ipv6'] , dst = egress_map['ipv6'], nh = 0)
            pkt = L2/L3
            log.info('Sending packets to verify if flows are correct')
            sendp(pkt, count=50, iface = self.flows.port_map[ingress])
            t.join()
            assert_equal(self.success, True)

    def send_multicast_data_traffic(self, group, intf= 'veth2',source = '1.2.3.4'):
        dst_mac = self.igmp.iptomac(group)
        eth = Ether(dst= dst_mac)
        ip = IP(dst=group,src=source)
        data = repr(monotonic.monotonic())
        sendp(eth/ip/data,count=20, iface = intf)
        pkt = (eth/ip/data)
        log.info('multicast traffic packet %s'%pkt.show())

    def verify_igmp_data_traffic(self, group, intf='veth0', source='1.2.3.4' ):
        log.info('verifying multicast traffic for group %s from source %s'%(group,source))
        self.success = False
        def recv_task():
            def igmp_recv_cb(pkt):
                log.info('multicast data received for group %s from source %s'%(group,source))
                self.success = True
            sniff(prn = igmp_recv_cb,lfilter = lambda p: IP in p and p[IP].dst == group and p[IP].src == source, count=1,timeout = 2, iface='veth0')
        t = threading.Thread(target = recv_task)
        t.start()
        self.send_multicast_data_traffic(group,source=source)
        t.join()
        return self.success

    #pass
    def test_cluster_with_igmp_include_exclude_modes_and_restarting_master(self, onos_instances=ONOS_INSTANCES):
	status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status, True)
	master, standbys = self.get_cluster_current_master_standbys()
        assert_equal(len(standbys), (onos_instances-1))
	onos_names_ips =  self.get_cluster_container_names_ips()
        master_onos_name = onos_names_ips[master]
	self.igmp.setUp(controller=master)
        groups = ['224.2.3.4','230.5.6.7']
        src_list = ['2.2.2.2','3.3.3.3']
        self.igmp.onos_ssm_table_load(groups, src_list=src_list, controller=master)
        self.igmp.send_igmp_join(groups = [groups[0]], src_list = src_list,record_type = IGMP_V3_GR_TYPE_INCLUDE,
                             iface = self.V_INF1, delay = 2)
        self.igmp.send_igmp_join(groups = [groups[1]], src_list = src_list,record_type = IGMP_V3_GR_TYPE_EXCLUDE,
                             iface = self.V_INF1, delay = 2)
        status = self.verify_igmp_data_traffic(groups[0],intf=self.V_INF1,source=src_list[0])
        assert_equal(status,True)
        status = self.verify_igmp_data_traffic(groups[1],intf = self.V_INF1,source= src_list[1])
        assert_equal(status,False)
	log.info('restarting cluster master %s'%master)
	Container(master_onos_name,Onos.IMAGE).restart()
	time.sleep(60)
	log.info('verifying multicast data traffic after master restart')
	status = self.verify_igmp_data_traffic(groups[0],intf=self.V_INF1,source=src_list[0])
        assert_equal(status,True)
        status = self.verify_igmp_data_traffic(groups[1],intf = self.V_INF1,source= src_list[1])
        assert_equal(status,False)

    #pass
    def test_cluster_with_igmp_include_exclude_modes_and_making_master_down(self, onos_instances=ONOS_INSTANCES):
        status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status, True)
        master, standbys = self.get_cluster_current_master_standbys()
        assert_equal(len(standbys), (onos_instances-1))
        onos_names_ips =  self.get_cluster_container_names_ips()
        master_onos_name = onos_names_ips[master]
        self.igmp.setUp(controller=master)
        groups = [self.igmp.random_mcast_ip(),self.igmp.random_mcast_ip()]
        src_list = [self.igmp.randomsourceip()]
        self.igmp.onos_ssm_table_load(groups, src_list=src_list,controller=master)
        self.igmp.send_igmp_join(groups = [groups[0]], src_list = src_list,record_type = IGMP_V3_GR_TYPE_INCLUDE,
                             iface = self.V_INF1, delay = 2)
        self.igmp.send_igmp_join(groups = [groups[1]], src_list = src_list,record_type = IGMP_V3_GR_TYPE_EXCLUDE,
                             iface = self.V_INF1, delay = 2)
        status = self.verify_igmp_data_traffic(groups[0],intf=self.V_INF1,source=src_list[0])
        assert_equal(status,True)
        status = self.verify_igmp_data_traffic(groups[1],intf = self.V_INF1,source= src_list[0])
        assert_equal(status,False)
        log.info('Killing cluster master %s'%master)
        Container(master_onos_name,Onos.IMAGE).kill()
        time.sleep(60)
	status = self.verify_cluster_status(onos_instances=onos_instances-1,controller=standbys[0])
        assert_equal(status, True)
        log.info('Verifying multicast data traffic after cluster master down')
        status = self.verify_igmp_data_traffic(groups[0],intf=self.V_INF1,source=src_list[0])
        assert_equal(status,True)
        status = self.verify_igmp_data_traffic(groups[1],intf = self.V_INF1,source= src_list[0])
        assert_equal(status,False)

    def test_cluster_with_igmp_include_mode_checking_traffic_recovery_time_after_master_is_down(self, onos_instances=ONOS_INSTANCES):
        status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status, True)
        master, standbys = self.get_cluster_current_master_standbys()
        assert_equal(len(standbys), (onos_instances-1))
        onos_names_ips =  self.get_cluster_container_names_ips()
        master_onos_name = onos_names_ips[master]
        self.igmp.setUp(controller=master)
        groups = [self.igmp.random_mcast_ip()]
        src_list = [self.igmp.randomsourceip()]
        self.igmp.onos_ssm_table_load(groups, src_list=src_list,controller=master)
        self.igmp.send_igmp_join(groups = [groups[0]], src_list = src_list,record_type = IGMP_V3_GR_TYPE_INCLUDE,
                             iface = self.V_INF1, delay = 2)
        status = self.verify_igmp_data_traffic(groups[0],intf=self.V_INF1,source=src_list[0])
        assert_equal(status,True)
        log.info('Killing clusters master %s'%master)
        Container(master_onos_name,Onos.IMAGE).kill()
	count = 0
	for i in range(60):
            log.info('Verifying multicast data traffic after cluster master down')
            status = self.verify_igmp_data_traffic(groups[0],intf=self.V_INF1,source=src_list[0])
	    if status:
		break
	    else:
		count += 1
	        time.sleep(1)
	assert_equal(status, True)
	log.info('Time taken to recover traffic after clusters master down is %d seconds'%count)


    #pass
    def test_cluster_state_with_igmp_leave_group_after_master_change(self, onos_instances=ONOS_INSTANCES):
        status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status, True)
        master, standbys = self.get_cluster_current_master_standbys()
	assert_equal(len(standbys), (onos_instances-1))
        self.igmp.setUp(controller=master)
        groups = [self.igmp.random_mcast_ip()]
        src_list = [self.igmp.randomsourceip()]
        self.igmp.onos_ssm_table_load(groups, src_list=src_list,controller=master)
        self.igmp.send_igmp_join(groups = [groups[0]], src_list = src_list,record_type = IGMP_V3_GR_TYPE_INCLUDE,
                             iface = self.V_INF1, delay = 2)
        status = self.verify_igmp_data_traffic(groups[0],intf=self.V_INF1,source=src_list[0])
        assert_equal(status,True)
	log.info('Changing cluster master %s to %s'%(master,standbys[0]))
	self.change_cluster_current_master(new_master=standbys[0])
	log.info('Verifying multicast traffic after cluster master change')
	status = self.verify_igmp_data_traffic(groups[0],intf=self.V_INF1,source=src_list[0])
        assert_equal(status,True)
        log.info('Sending igmp TO_EXCLUDE message to leave the group %s'%groups[0])
        self.igmp.send_igmp_join(groups = [groups[0]], src_list = src_list,record_type = IGMP_V3_GR_TYPE_CHANGE_TO_EXCLUDE,
                             iface = self.V_INF1, delay = 1)
	time.sleep(10)
        status = self.verify_igmp_data_traffic(groups[0],intf = self.V_INF1,source= src_list[0])
        assert_equal(status,False)

    #pass
    def test_cluster_state_with_igmp_join_before_and_after_master_change(self,onos_instances=ONOS_INSTANCES):
        status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status, True)
	master,standbys = self.get_cluster_current_master_standbys()
	assert_equal(len(standbys), (onos_instances-1))
        self.igmp.setUp(controller=master)
        groups = [self.igmp.random_mcast_ip()]
        src_list = [self.igmp.randomsourceip()]
        self.igmp.onos_ssm_table_load(groups, src_list=src_list,controller=master)
	log.info('Changing cluster master %s to %s'%(master,standbys[0]))
	self.change_cluster_current_master(new_master = standbys[0])
        self.igmp.send_igmp_join(groups = [groups[0]], src_list = src_list,record_type = IGMP_V3_GR_TYPE_INCLUDE,
                             iface = self.V_INF1, delay = 2)
	time.sleep(1)
	self.change_cluster_current_master(new_master = master)
        status = self.verify_igmp_data_traffic(groups[0],intf=self.V_INF1,source=src_list[0])
        assert_equal(status,True)

    #pass
    @deferred(TLS_TIMEOUT)
    def test_cluster_with_eap_tls_traffic(self,onos_instances=ONOS_INSTANCES):
	status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status, True)
	master, standbys = self.get_cluster_current_master_standbys()
        assert_equal(len(standbys), (onos_instances-1))
	self.tls.setUp(controller=master)
        df = defer.Deferred()
        def eap_tls_verify(df):
            tls = TLSAuthTest()
            tls.runTest()
            df.callback(0)
        reactor.callLater(0, eap_tls_verify, df)
        return df

    @deferred(120)
    def test_cluster_for_eap_tls_traffic_before_and_after_master_change(self,onos_instances=ONOS_INSTANCES):
	master, standbys = self.get_cluster_current_master_standbys()
        assert_equal(len(standbys), (onos_instances-1))
        self.tls.setUp()
        df = defer.Deferred()
	def eap_tls_verify2(df2):
            tls = TLSAuthTest()
            tls.runTest()
            df.callback(0)
        for i in [0,1]:
	    if i == 1:
		log.info('Changing cluster master %s to %s'%(master, standbys[0]))
		self.change_master_current_cluster(new_master=standbys[0])
                log.info('Verifying tls authentication after cluster master changed to %s'%standbys[0])
	    else:
		log.info('Verifying tls authentication before cluster master change')
            reactor.callLater(0, eap_tls_verify, df)
        return df

    @deferred(TLS_TIMEOUT)
    def test_cluster_for_eap_tls_traffic_before_and_after_making_master_down(self,onos_instances=ONOS_INSTANCES):
        status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status, True)
        master, standbys = self.get_cluster_current_master_standbys()
        assert_equal(len(standbys), (onos_instances-1))
	onos_names_ips =  self.get_cluster_container_names_ips()
        master_onos_name = onos_names_ips[master]
        self.tls.setUp()
        df = defer.Deferred()
        def eap_tls_verify(df):
            tls = TLSAuthTest()
            tls.runTest()
            df.callback(0)
        for i in [0,1]:
            if i == 1:
                log.info('Killing cluster current master %s'%master)
                cord_test_onos_shutdown(node = master)
		time.sleep(20)
                status = self.verify_cluster_status(controller=standbys[0],onos_instances=onos_instances-1,verify=True)
		assert_equal(status, True)
		log.info('Cluster came up with %d instances after killing master'%(onos_instances-1))
                log.info('Verifying tls authentication after killing cluster master')
            reactor.callLater(0, eap_tls_verify, df)
        return df

    @deferred(TLS_TIMEOUT)
    def test_cluster_for_eap_tls_with_no_cert_before_and_after_member_is_restarted(self,onos_instances=ONOS_INSTANCES):
	status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status, True)
	master, standbys = self.get_cluster_current_master_standbys()
        assert_equal(len(standbys), (onos_instances-1))
        onos_names_ips =  self.get_cluster_container_names_ips()
        member_onos_name = onos_names_ips[standbys[0]]
	self.tls.setUp()
        df = defer.Deferred()
        def eap_tls_no_cert(df):
            def tls_no_cert_cb():
                log.info('TLS authentication failed with no certificate')
            tls = TLSAuthTest(fail_cb = tls_no_cert_cb, client_cert = '')
            tls.runTest()
            assert_equal(tls.failTest, True)
            df.callback(0)
	for i in [0,1]:
	    if i == 1:
	        log.info('Restart cluster member %s'%standbys[0])
                Container(member_onos_name,Onos.IMAGE).restart()
                time.sleep(20)
                status = self.verify_cluster_status(onos_instances=onos_instances)
                assert_equal(status, True)
                log.info('Cluster came up with %d instances after member restart'%(onos_instances))
                log.info('Verifying tls authentication after member restart')
        reactor.callLater(0, eap_tls_no_cert, df)
        return df

    #pass
    def test_cluster_proxyarp_master_change_and_app_deactivation(self,onos_instances=ONOS_INSTANCES,hosts = 3):
	status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status,True)
	master,standbys = self.get_cluster_current_master_standbys()
        assert_equal(len(standbys),(onos_instances-1))
	self.proxyarp.setUpClass()
        ports_map, egress_map,hosts_config = self.proxyarp.proxyarp_config(hosts = hosts,controller=master)
        ingress = hosts+1
        for hostip, hostmac in hosts_config:
            self.proxyarp.proxyarp_arpreply_verify(ingress,hostip,hostmac,PositiveTest = True)
            time.sleep(1)
	log.info('changing cluster current master from %s to %s'%(master,standbys[0]))
	self.change_cluster_current_master(new_master=standbys[0])
	log.info('verifying proxyarp after master change')
	for hostip, hostmac in hosts_config:
            self.proxyarp.proxyarp_arpreply_verify(ingress,hostip,hostmac,PositiveTest = True)
            time.sleep(1)
        log.info('Deactivating proxyarp  app and expecting proxyarp functionality not to work')
        self.proxyarp.proxyarp_activate(deactivate = True,controller=standbys[0])
	time.sleep(3)
        for hostip, hostmac in hosts_config:
            self.proxyarp.proxyarp_arpreply_verify(ingress,hostip,hostmac,PositiveTest = False)
            time.sleep(1)
        log.info('activating proxyarp  app and expecting to get arp reply from ONOS')
        self.proxyarp.proxyarp_activate(deactivate = False,controller=standbys[0])
	time.sleep(3)
        for hostip, hostmac in hosts_config:
            self.proxyarp.proxyarp_arpreply_verify(ingress,hostip,hostmac,PositiveTest = True)
            time.sleep(1)

    #pass
    def test_cluster_with_proxyarp_and_one_member_down(self,hosts=3,onos_instances=ONOS_INSTANCES):
	status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status, True)
	master, standbys = self.get_cluster_current_master_standbys()
        assert_equal(len(standbys), (onos_instances-1))
        onos_names_ips =  self.get_cluster_container_names_ips()
        member_onos_name = onos_names_ips[standbys[1]]
	self.proxyarp.setUpClass()
        ports_map, egress_map,hosts_config = self.proxyarp.proxyarp_config(hosts = hosts,controller=master)
        ingress = hosts+1
        for hostip, hostmac in hosts_config:
            self.proxyarp.proxyarp_arpreply_verify(ingress,hostip,hostmac,PositiveTest = True)
            time.sleep(1)
	log.info('killing cluster member %s'%standbys[1])
        Container(member_onos_name,Onos.IMAGE).kill()
        time.sleep(20)
        status = self.verify_cluster_status(onos_instances=onos_instances-1,controller=master,verify=True)
        assert_equal(status, True)
        log.info('cluster came up with %d instances after member down'%(onos_instances-1))
        log.info('verifying proxy arp functionality after cluster member down')
	for hostip, hostmac in hosts_config:
            self.proxyarp.proxyarp_arpreply_verify(ingress,hostip,hostmac,PositiveTest = True)
            time.sleep(1)

    #pass
    def test_cluster_with_proxyarp_and_concurrent_requests_with_multiple_host_and_different_interfaces(self,hosts=10,onos_instances=ONOS_INSTANCES):
	status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status, True)
	self.proxyarp.setUpClass()
	master, standbys = self.get_cluster_current_master_standbys()
        assert_equal(len(standbys), (onos_instances-1))
        ports_map, egress_map, hosts_config = self.proxyarp.proxyarp_config(hosts = hosts, controller=master)
        self.success = True
        ingress = hosts+1
        ports = range(ingress,ingress+10)
        hostmac = []
        hostip = []
        for ip,mac in hosts_config:
            hostmac.append(mac)
            hostip.append(ip)
        success_dir = {}
        def verify_proxyarp(*r):
            ingress, hostmac, hostip = r[0],r[1],r[2]
            def mac_recv_task():
                def recv_cb(pkt):
                    log.info('Arp Reply seen with source Mac is %s' %(pkt[ARP].hwsrc))
                    success_dir[current_thread().name] = True
                sniff(count=1, timeout=5,lfilter = lambda p: ARP in p and p[ARP].op == 2 and p[ARP].hwsrc == hostmac,
                    prn = recv_cb, iface = self.proxyarp.port_map[ingress])
            t = threading.Thread(target = mac_recv_task)
            t.start()
            pkt = (Ether(dst = 'ff:ff:ff:ff:ff:ff')/ARP(op=1,pdst= hostip))
            log.info('Sending arp request  for dest ip %s on interface %s' %
                 (hostip,self.proxyarp.port_map[ingress]))
            sendp(pkt, count = 10,iface = self.proxyarp.port_map[ingress])
            t.join()
        t = []
        for i in range(10):
            t.append(threading.Thread(target = verify_proxyarp, args = [ports[i],hostmac[i],hostip[i]]))
        for i in range(10):
            t[i].start()
	time.sleep(2)
        for i in range(10):
            t[i].join()
        if len(success_dir) != 10:
                self.success = False
        assert_equal(self.success, True)

    #pass
    def test_cluster_with_acl_rule_before_master_change_and_remove_acl_rule_after_master_change(self,onos_instances=ONOS_INSTANCES):
	status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status, True)
	master,standbys = self.get_cluster_current_master_standbys()
        assert_equal(len(standbys),(onos_instances-1))
	self.acl.setUp()
        acl_rule = ACLTest()
        status,code = acl_rule.adding_acl_rule('v4', srcIp=self.acl.ACL_SRC_IP, dstIp =self.acl.ACL_DST_IP, action = 'allow',controller=master)
        if status is False:
            log.info('JSON request returned status %d' %code)
            assert_equal(status, True)
        result = acl_rule.get_acl_rules(controller=master)
        aclRules1 = result.json()['aclRules']
	log.info('Added acl rules is %s'%aclRules1)
        acl_Id = map(lambda d: d['id'], aclRules1)
	log.info('Changing cluster current master from %s to %s'%(master,standbys[0]))
	self.change_cluster_current_master(new_master=standbys[0])
        status,code = acl_rule.remove_acl_rule(acl_Id[0],controller=standbys[0])
        if status is False:
            log.info('JSON request returned status %d' %code)
            assert_equal(status, True)

    #pass
    def test_cluster_verifying_acl_rule_in_new_master_after_current_master_is_down(self,onos_instances=ONOS_INSTANCES):
        status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status, True)
        master,standbys = self.get_cluster_current_master_standbys()
        assert_equal(len(standbys),(onos_instances-1))
	onos_names_ips =  self.get_cluster_container_names_ips()
	master_onos_name = onos_names_ips[master]
        self.acl.setUp()
        acl_rule = ACLTest()
        status,code = acl_rule.adding_acl_rule('v4', srcIp=self.acl.ACL_SRC_IP, dstIp =self.acl.ACL_DST_IP, action = 'allow',controller=master)
        if status is False:
            log.info('JSON request returned status %d' %code)
            assert_equal(status, True)
        result1 = acl_rule.get_acl_rules(controller=master)
        aclRules1 = result1.json()['aclRules']
        log.info('Added acl rules is %s'%aclRules1)
        acl_Id1 = map(lambda d: d['id'], aclRules1)
        log.info('Killing cluster current master %s'%master)
	Container(master_onos_name,Onos.IMAGE).kill()
	time.sleep(45)
	status = self.verify_cluster_status(onos_instances=onos_instances,controller=standbys[0])
        assert_equal(status, True)
        new_master,standbys = self.get_cluster_current_master_standbys(controller=standbys[0])
	assert_equal(len(standbys),(onos_instances-2))
	assert_not_equal(new_master,master)
        result2 = acl_rule.get_acl_rules(controller=new_master)
        aclRules2 = result2.json()['aclRules']
	acl_Id2 = map(lambda d: d['id'], aclRules2)
	log.info('Acl Ids before and after master down are %s and %s'%(acl_Id1,acl_Id2))
	assert_equal(acl_Id2,acl_Id1)

    #acl traffic scenario not working as acl rule is not getting added to onos
    def test_cluster_with_acl_traffic_before_and_after_two_members_down(self,onos_instances=ONOS_INSTANCES):
	status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status, True)
	master,standbys = self.get_cluster_current_master_standbys()
        assert_equal(len(standbys),(onos_instances-1))
	onos_names_ips =  self.get_cluster_container_names_ips()
        member1_onos_name = onos_names_ips[standbys[0]]
        member2_onos_name = onos_names_ips[standbys[1]]
        ingress = self.acl.ingress_iface
        egress = self.acl.CURRENT_PORT_NUM
        acl_rule = ACLTest()
        status, code, host_ip_mac = acl_rule.generate_onos_interface_config(iface_num= self.acl.CURRENT_PORT_NUM, iface_name = 'b1',iface_count = 1, iface_ip = self.acl.HOST_DST_IP)
        self.acl.CURRENT_PORT_NUM += 1
        time.sleep(5)
        if status is False:
            log.info('JSON request returned status %d' %code)
            assert_equal(status, True)
        srcMac = '00:00:00:00:00:11'
        dstMac = host_ip_mac[0][1]
        self.acl.acl_hosts_add(dstHostIpMac = host_ip_mac, egress_iface_count = 1,  egress_iface_num = egress )
        status, code = acl_rule.adding_acl_rule('v4', srcIp=self.acl.ACL_SRC_IP, dstIp =self.acl.ACL_DST_IP, action = 'deny',controller=master)
        time.sleep(10)
        if status is False:
            log.info('JSON request returned status %d' %code)
            assert_equal(status, True)
        self.acl.acl_rule_traffic_send_recv(srcMac = srcMac, dstMac = dstMac ,srcIp =self.acl.ACL_SRC_IP, dstIp = self.acl.ACL_DST_IP,ingress =ingress, egress = egress, ip_proto = 'UDP', positive_test = False)
	log.info('killing cluster members %s and %s'%(standbys[0],standbys[1]))
        Container(member1_onos_name, Onos.IMAGE).kill()
        Container(member2_onos_name, Onos.IMAGE).kill()
	time.sleep(40)
	status = self.verify_cluster_status(onos_instances=onos_instances-2,verify=True,controller=master)
        assert_equal(status, True)
	self.acl.acl_rule_traffic_send_recv(srcMac = srcMac, dstMac = dstMac ,srcIp =self.acl.ACL_SRC_IP, dstIp = self.acl.ACL_DST_IP,ingress =ingress, egress = egress, ip_proto = 'UDP', positive_test = False)
        self.acl.acl_hosts_remove(egress_iface_count = 1,  egress_iface_num = egress)

    #pass
    def test_cluster_with_dhcpRelay_releasing_dhcp_ip_after_master_change(self, iface = 'veth0',onos_instances=ONOS_INSTANCES):
	status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status, True)
	master,standbys = self.get_cluster_current_master_standbys()
        assert_equal(len(standbys),(onos_instances-1))
	self.dhcprelay.setUpClass(controller=master)
        mac = self.dhcprelay.get_mac(iface)
        self.dhcprelay.host_load(iface)
        ##we use the defaults for this test that serves as an example for others
        ##You don't need to restart dhcpd server if retaining default config
        config = self.dhcprelay.default_config
        options = self.dhcprelay.default_options
        subnet = self.dhcprelay.default_subnet_config
        dhcpd_interface_list = self.dhcprelay.relay_interfaces
        self.dhcprelay.dhcpd_start(intf_list = dhcpd_interface_list,
                         config = config,
                         options = options,
                         subnet = subnet,
			 controller=master)
        self.dhcprelay.dhcp = DHCPTest(seed_ip = '10.10.100.10', iface = iface)
        cip, sip = self.dhcprelay.send_recv(mac)
	log.info('Changing cluster current master from %s to %s'%(master, standbys[0]))
	self.change_master_current_cluster(new_master=standbys[0])
        log.info('Releasing ip %s to server %s' %(cip, sip))
        assert_equal(self.dhcprelay.dhcp.release(cip), True)
        log.info('Triggering DHCP discover again after release')
        cip2, sip2 = self.dhcprelay.send_recv(mac)
        log.info('Verifying released IP was given back on rediscover')
        assert_equal(cip, cip2)
        log.info('Test done. Releasing ip %s to server %s' %(cip2, sip2))
        assert_equal(self.dhcprelay.dhcp.release(cip2), True)
	self.dhcprelay.tearDownClass(controller=standbys[0])


    def test_cluster_with_dhcpRelay_and_verify_dhcp_ip_after_master_down(self, iface = 'veth0',onos_instances=ONOS_INSTANCES):
	status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status, True)
        master,standbys = self.get_cluster_current_master_standbys()
        assert_equal(len(standbys),(onos_instances-1))
	onos_names_ips =  self.get_cluster_container_names_ips()
        master_onos_name = onos_names_ips[master]
        self.dhcprelay.setUpClass(controller=master)
        mac = self.dhcprelay.get_mac(iface)
        self.dhcprelay.host_load(iface)
        ##we use the defaults for this test that serves as an example for others
        ##You don't need to restart dhcpd server if retaining default config
        config = self.dhcprelay.default_config
        options = self.dhcprelay.default_options
        subnet = self.dhcprelay.default_subnet_config
        dhcpd_interface_list = self.dhcprelay.relay_interfaces
        self.dhcprelay.dhcpd_start(intf_list = dhcpd_interface_list,
                         config = config,
                         options = options,
                         subnet = subnet,
			 controller=master)
        self.dhcprelay.dhcp = DHCPTest(seed_ip = '10.10.10.1', iface = iface)
        log.info('Initiating dhcp process from client %s'%mac)
        cip, sip = self.dhcprelay.send_recv(mac)
        log.info('Killing cluster current master %s'%master)
	Container(master_onos_name, Onos.IMAGE).kill()
	time.sleep(60)
	status = self.verify_cluster_status(onos_instances=onos_instances-1,verify=True,controller=standbys[0])
        assert_equal(status, True)
	mac = self.dhcprelay.dhcp.get_mac(cip)[0]
        log.info("Verifying dhcp clients gets same IP after cluster master restarts")
        new_cip, new_sip = self.dhcprelay.dhcp.only_request(cip, mac)
        assert_equal(new_cip, cip)
	self.dhcprelay.tearDownClass(controller=standbys[0])

    #pass
    def test_cluster_with_dhcpRelay_and_simulate_client_by_changing_master(self, iface = 'veth0',onos_instances=ONOS_INSTANCES):
	status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status, True)
	master,standbys = self.get_cluster_current_master_standbys()
        assert_equal(len(standbys),(onos_instances-1))
	self.dhcprelay.setUpClass(controller=master)
        macs = ['e4:90:5e:a3:82:c1','e4:90:5e:a3:82:c2','e4:90:5e:a3:82:c3']
        self.dhcprelay.host_load(iface)
        ##we use the defaults for this test that serves as an example for others
        ##You don't need to restart dhcpd server if retaining default config
        config = self.dhcprelay.default_config
        options = self.dhcprelay.default_options
        subnet = self.dhcprelay.default_subnet_config
        dhcpd_interface_list = self.dhcprelay.relay_interfaces
        self.dhcprelay.dhcpd_start(intf_list = dhcpd_interface_list,
                         config = config,
                         options = options,
                         subnet = subnet,
			 controller=master)
        self.dhcprelay.dhcp = DHCPTest(seed_ip = '10.10.10.1', iface = iface)
	cip1, sip1 = self.dhcprelay.send_recv(macs[0])
	assert_not_equal(cip1,None)
        log.info('Got dhcp client IP %s for mac %s when cluster master is %s'%(cip1,macs[0],master))
        log.info('Changing cluster master from %s to %s'%(master, standbys[0]))
	self.change_master_current_cluster(new_master=standbys[0])
	cip2, sip2 = self.dhcprelay.send_recv(macs[1])
	assert_not_equal(cip2,None)
	log.info('Got dhcp client IP %s for mac %s when cluster master is %s'%(cip2,macs[1],standbys[0]))
	self.change_master_current_cluster(new_master=master)
        log.info('Changing cluster master from %s to %s'%(standbys[0],master))
        cip3, sip3 = self.dhcprelay.send_recv(macs[2])
	assert_not_equal(cip3,None)
	log.info('Got dhcp client IP %s for mac %s when cluster master is %s'%(cip2,macs[2],master))
	self.dhcprelay.tearDownClass(controller=standbys[0])

    def test_cluster_with_cord_subscriber_joining_next_channel_before_and_after_cluster_restart(self,onos_instances=ONOS_INSTANCES):
	status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status, True)
	self.subscriber.setUpClass(controller=master)
        self.subscriber.num_subscribers = 5
        self.subscriber.num_channels = 10
	for i in [0,1]:
	    if i == 1:
		cord_test_onos_restart()
		time.sleep(45)
		status = self.verify_cluster_status(onos_instances=onos_instances)
		assert_equal(status, True)
		log.info('Verifying cord subscriber functionality after cluster restart')
	    else:
		log.info('Verifying cord subscriber functionality before cluster restart')
            test_status = self.subscriber.subscriber_join_verify(num_subscribers = self.subscriber.num_subscribers,
                                                    num_channels = self.subscriber.num_channels,
                                                    cbs = (self.subscriber.tls_verify, self.subscriber.dhcp_next_verify,
                                                           self.subscriber.igmp_next_verify, self.subscriber.traffic_verify),
                                                    port_list = self.subscriber.generate_port_list(self.subscriber.num_subscribers,
                                                                                        self.subscriber.num_channels))
            assert_equal(test_status, True)
	self.subscriber.tearDownClass(controller=master)

    #not validated on cluster setup because ciena-cordigmp-multitable-2.0 app installation fails on cluster
    def test_cluster_with_cord_subscriber_join_next_channel_before_and_after_cluster_mastership_is_withdrawn(self,onos_instances=ONOS_INSTANCES):
        status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status, True)
	master,standbys = self.get_cluster_current_master_standbys()
        assert_equal(len(standbys),(onos_instances-1))
        self.subscriber.setUpClass(controller=master)
        self.subscriber.num_subscribers = 5
        self.subscriber.num_channels = 10
        for i in [0,1]:
            if i == 1:
		status=self.withdraw_cluster_current_mastership(master_ip=master)
		asser_equal(status, True)
		master,standbys = self.get_cluster_current_master_standbys()
                log.info('verifying cord subscriber functionality after cluster current master withdraw mastership')
            else:
		 log.info('verifying cord subscriber functionality before cluster master withdraw mastership')
            test_status = self.subscriber.subscriber_join_verify(num_subscribers = self.subscriber.num_subscribers,
                                                    num_channels = self.subscriber.num_channels,
                                                    cbs = (self.subscriber.tls_verify, self.subscriber.dhcp_next_verify,
                                                           self.subscriber.igmp_next_verify, self.subscriber.traffic_verify),
                                                    port_list = self.subscriber.generate_port_list(self.subscriber.num_subscribers,
                                                                                        self.subscriber.num_channels),controller=master)
            assert_equal(test_status, True)
        self.subscriber.tearDownClass(controller=master)

    #not validated on cluster setup because ciena-cordigmp-multitable-2.0 app installation fails on cluster
    def test_cluster_with_cord_subscriber_join_recv_traffic_from_10channels_and_making_one_cluster_member_down(self,onos_instances=ONOS_INSTANCES):
	status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status, True)
	master, standbys = self.get_cluster_current_master_standbys()
        assert_equal(len(standbys),(onos_instances-1))
        onos_names_ips =  self.get_cluster_container_names_ips()
        member_onos_name = onos_names_ips[standbys[0]]
	self.subscriber.setUpClass(controller=master)
	num_subscribers = 1
        num_channels = 10
	for i in [0,1]:
	    if i == 1:
                cord_test_onos_shutdown(node = standbys[0])
		time.sleep(30)
		status = self.verify_cluster_status(onos_instances=onos_instances-1,verify=True,controller=master)
                assert_equal(status, True)
		log.info('Verifying cord subscriber functionality after cluster member %s is down'%standbys[0])
	    else:
		log.info('Verifying cord subscriber functionality before cluster member %s is down'%standbys[0])
            test_status = self.subscriber.subscriber_join_verify(num_subscribers = num_subscribers,
                                                    num_channels = num_channels,
                                                    cbs = (self.subscriber.tls_verify, self.subscriber.dhcp_verify,
                                                              self.subscriber.igmp_verify, self.subscriber.traffic_verify),
                                                    port_list = self.subscriber.generate_port_list(num_subscribers, num_channels),
                                                    negative_subscriber_auth = 'all',controller=master)
            assert_equal(test_status, True)
	self.subscriber.tearDownClass(controller=master)

    def test_cluster_with_cord_subscriber_joining_next_10channels_making_two_cluster_members_down(self,onos_instances=ONOS_INSTANCES):
	status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status, True)
	master, standbys = self.get_cluster_current_master_standbys()
        assert_equal(len(standbys),(onos_instances-1))
        onos_names_ips =  self.get_cluster_container_names_ips()
        member1_onos_name = onos_names_ips[standbys[0]]
	member2_onos_name = onos_names_ips[standbys[1]]
	self.subscriber.setUpClass(controller=master)
        num_subscribers = 1
        num_channels = 10
	for i in [0,1]:
	    if i == 1:
                cord_test_onos_shutdown(node = standbys[0])
                cord_test_onos_shutdown(node = standbys[1])
		time.sleep(60)
		status = self.verify_cluster_status(onos_instances=onos_instances-2)
                assert_equal(status, True)
		log.info('Verifying cord subscriber funtionality after cluster two members %s and %s down'%(standbys[0],standbys[1]))
	    else:
		log.info('Verifying cord subscriber funtionality before cluster two members %s and %s down'%(standbys[0],standbys[1]))
	    test_status = self.subscriber.subscriber_join_verify(num_subscribers = num_subscribers,
                                                    num_channels = num_channels,
                                                    cbs = (self.subscriber.tls_verify, self.subscriber.dhcp_next_verify,
                                                           self.subscriber.igmp_next_verify, self.subscriber.traffic_verify),
                                                    port_list = self.subscriber.generate_port_list(num_subscribers, num_channels),
                                                    negative_subscriber_auth = 'all')
            assert_equal(test_status, True)
	self.subscriber.tearDownClass(controller=master)

    #pass
    def test_cluster_with_multiple_ovs_switches(self,onos_instances = ONOS_INSTANCES):
	status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status, True)
	device_dict = self.get_cluster_current_master_standbys_of_connected_devices()
	for device in device_dict.keys():
	    log.info("Device is %s"%device_dict[device])
	    assert_not_equal(device_dict[device]['master'],'none')
	    log.info('Master and standbys for device %s are %s and %s'%(device,device_dict[device]['master'],device_dict[device]['standbys']))
	    assert_equal(len(device_dict[device]['standbys']), onos_instances-1)

    #pass
    def test_cluster_state_in_multiple_ovs_switches(self,onos_instances = ONOS_INSTANCES):
        status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status, True)
        device_dict = self.get_cluster_current_master_standbys_of_connected_devices()
	cluster_ips = self.get_cluster_current_member_ips()
	for ip in cluster_ips:
	    device_dict= self.get_cluster_current_master_standbys_of_connected_devices(controller = ip)
	    assert_equal(len(device_dict.keys()),onos_instances)
            for device in device_dict.keys():
                log.info("Device is %s"%device_dict[device])
                assert_not_equal(device_dict[device]['master'],'none')
                log.info('Master and standbys for device %s are %s and %s'%(device,device_dict[device]['master'],device_dict[device]['standbys']))
                assert_equal(len(device_dict[device]['standbys']), onos_instances-1)

    #pass
    def test_cluster_verifying_multiple_ovs_switches_after_master_is_restarted(self,onos_instances = ONOS_INSTANCES):
        status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status, True)
	onos_names_ips =  self.get_cluster_container_names_ips()
	master_count = self.get_number_of_devices_of_master()
        log.info('Master count information is %s'%master_count)
	total_devices = 0
	for master in master_count.keys():
	    total_devices += master_count[master]['size']
	    if master_count[master]['size'] != 0:
		restart_ip = master
	assert_equal(total_devices,onos_instances)
	member_onos_name = onos_names_ips[restart_ip]
	log.info('Restarting cluster member %s having ip %s'%(member_onos_name,restart_ip))
        Container(member_onos_name, Onos.IMAGE).restart()
	time.sleep(40)
	master_count = self.get_number_of_devices_of_master()
	log.info('Master count information after restart is %s'%master_count)
	total_devices = 0
        for master in master_count.keys():
            total_devices += master_count[master]['size']
	    if master == restart_ip:
		assert_equal(master_count[master]['size'], 0)
	assert_equal(total_devices,onos_instances)

    #pass
    def test_cluster_verifying_multiple_ovs_switches_with_one_master_down(self,onos_instances = ONOS_INSTANCES):
        status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status, True)
        onos_names_ips =  self.get_cluster_container_names_ips()
        master_count = self.get_number_of_devices_of_master()
        log.info('Master count information is %s'%master_count)
        total_devices = 0
        for master in master_count.keys():
            total_devices += master_count[master]['size']
            if master_count[master]['size'] != 0:
                restart_ip = master
        assert_equal(total_devices,onos_instances)
        master_onos_name = onos_names_ips[restart_ip]
        log.info('Shutting down cluster member %s having ip %s'%(master_onos_name,restart_ip))
        Container(master_onos_name, Onos.IMAGE).kill()
        time.sleep(40)
	for ip in onos_names_ips.keys():
	    if ip != restart_ip:
		controller_ip = ip
	status = self.verify_cluster_status(onos_instances=onos_instances-1,controller=controller_ip)
        assert_equal(status, True)
        master_count = self.get_number_of_devices_of_master(controller=controller_ip)
        log.info('Master count information after restart is %s'%master_count)
        total_devices = 0
        for master in master_count.keys():
            total_devices += master_count[master]['size']
            if master == restart_ip:
                assert_equal(master_count[master]['size'], 0)
        assert_equal(total_devices,onos_instances)

    #pass
    def test_cluster_verifying_multiple_ovs_switches_with_current_master_withdrawing_mastership(self,onos_instances = ONOS_INSTANCES):
        status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status, True)
        master_count = self.get_number_of_devices_of_master()
        log.info('Master count information is %s'%master_count)
        total_devices = 0
        for master in master_count.keys():
            total_devices += int(master_count[master]['size'])
            if master_count[master]['size'] != 0:
                master_ip = master
		log.info('Devices of master %s are %s'%(master_count[master]['devices'],master))
		device_id = str(master_count[master]['devices'][0])
		device_count = master_count[master]['size']
        assert_equal(total_devices,onos_instances)
	log.info('Withdrawing mastership of device %s for controller %s'%(device_id,master_ip))
	status=self.withdraw_cluster_current_mastership(master_ip=master_ip,device_id = device_id)
        assert_equal(status, True)
        master_count = self.get_number_of_devices_of_master()
        log.info('Master count information after cluster mastership withdraw is %s'%master_count)
        total_devices = 0
        for master in master_count.keys():
            total_devices += int(master_count[master]['size'])
            if master == master_ip:
                assert_equal(master_count[master]['size'], device_count-1)
        assert_equal(total_devices,onos_instances)

    #pass
    def test_cluster_verifying_multiple_ovs_switches_and_restarting_cluster(self,onos_instances = ONOS_INSTANCES):
        status = self.verify_cluster_status(onos_instances=onos_instances)
        assert_equal(status, True)
        master_count = self.get_number_of_devices_of_master()
        log.info('Master count information is %s'%master_count)
        total_devices = 0
        for master in master_count.keys():
            total_devices += master_count[master]['size']
        assert_equal(total_devices,onos_instances)
        log.info('Restarting cluster')
	cord_test_onos_restart()
	time.sleep(60)
        master_count = self.get_number_of_devices_of_master()
        log.info('Master count information after restart is %s'%master_count)
        total_devices = 0
        for master in master_count.keys():
            total_devices += master_count[master]['size']
        assert_equal(total_devices,onos_instances)