Support for configuring/starting multiple switches in cord-tester.
Useful to fire tests with multiple switches configured with num_ports as per olt_config.json.

Change-Id: Ifbe65c75ad008db75ac5f4e3554e36499cb194de
diff --git a/src/test/setup/cord-test.py b/src/test/setup/cord-test.py
index 4f76d43..95e1cc9 100755
--- a/src/test/setup/cord-test.py
+++ b/src/test/setup/cord-test.py
@@ -81,12 +81,10 @@
         olt_config = OltConfig(olt_conf_file)
         self.port_map, _ = olt_config.olt_port_map()
         #Try using the host interface in olt conf to setup the switch
-        if self.port_map.has_key('host'):
-            self.switch = self.port_map['host']
-        else:
-            self.switch = 'ovsbr0'
+        self.switches = self.port_map['switches']
         if env is not None:
-            env['TEST_SWITCH'] = self.switch
+            env['TEST_SWITCH'] = self.switches[0]
+            env['TEST_SWITCHES'] = ','.join(self.switches)
             env['TEST_HOST'] = self.name
             env['TEST_INSTANCE'] = instance
             env['TEST_INSTANCES'] = num_instances
@@ -104,32 +102,36 @@
         """Start OVS"""
         ##Determine if OVS has to be started locally or not
         s_file,s_sandbox = ('of-bridge-local.sh',self.tester_base) if self.olt else ('of-bridge.sh',self.sandbox_setup)
-        ovs_cmd = os.path.join(s_sandbox, s_file) + ' {0}'.format(self.switch)
+        ovs_cmd = os.path.join(s_sandbox, s_file)
         if self.olt:
             if CordTester.switch_on_olt is True:
                 return
             CordTester.switch_on_olt = True
-            ovs_cmd += ' {0}'.format(self.ctlr_ip)
-            print('Starting OVS on the host with controller: %s' %(self.ctlr_ip))
+            ovs_cmd += ' {} {}'.format(len(self.switches), self.ctlr_ip)
+            print('Starting OVS on the host with %d switches for controller: %s' %(len(self.switches), self.ctlr_ip))
         else:
-            print('Starting OVS on test container %s' %self.name)
+            ovs_cmd += ' {}'.format(self.switches[0])
+            print('Starting OVS on test container %s for controller: %s' %(self.name, self.ctlr_ip))
         self.execute_switch(ovs_cmd)
-        status = 1
         ## Wait for the LLDP flows to be added to the switch
-        tries = 0
-        while status != 0 and tries < 500:
-            cmd = 'sudo ovs-ofctl dump-flows {0} | grep \"type=0x8942\"'.format(self.switch)
-            status = self.execute_switch(cmd, shell = True)
-            tries += 1
-            if tries % 10 == 0:
-                print('Waiting for test switch to be connected to ONOS controller ...')
+        for switch in self.switches:
+            status = 1
+            tries = 0
+            while status != 0 and tries < 500:
+                cmd = 'sudo ovs-ofctl dump-flows {0} | grep \"type=0x8942\"'.format(switch)
+                status = self.execute_switch(cmd, shell = True)
+                tries += 1
+                if tries % 10 == 0:
+                    print('Waiting for test switch %s to be connected to ONOS controller ...' %switch)
 
-        if status != 0:
-            print('Test Switch not connected to ONOS container.'
-                  'Please remove ONOS container and restart the test')
-            if self.rm:
-                self.kill()
-            sys.exit(1)
+            if status != 0:
+                print('Test Switch %s not connected to ONOS container.'
+                      'Please remove ONOS container and restart the test' %switch)
+                if self.rm:
+                    self.kill()
+                sys.exit(1)
+            else:
+                print('Test Switch %s connected to ONOS container.' %switch)
 
         if boot_delay:
             time.sleep(boot_delay)
@@ -137,71 +139,78 @@
     def setup_intfs(self, port_num = 0):
         tester_intf_subnet = '192.168.100'
         res = 0
-        host_intf = self.port_map['host']
+        switches = self.port_map['switches']
         start_vlan = self.port_map['start_vlan']
         start_vlan += port_num
         uplink = self.port_map['uplink']
         wan = self.port_map['wan']
-        port_list = self.port_map['ports'] + self.port_map['relay_ports']
-        for port in port_list:
-            guest_if = port
-            local_if = '{0}_{1}'.format(guest_if, port_num+1)
-            guest_ip = '{0}.{1}/24'.format(tester_intf_subnet, port_num+1)
-            ##Use pipeworks to configure container interfaces on host/bridge interfaces
-            pipework_cmd = 'pipework {0} -i {1} -l {2} {3} {4}'.format(host_intf, guest_if,
-                                                                       local_if, self.name, guest_ip)
-            #if the wan interface is specified for uplink, then use it instead
-            if wan and port == self.port_map[uplink]:
-                pipework_cmd = 'pipework {0} -i {1} -l {2} {3} {4}'.format(wan, guest_if,
+        port_list = self.port_map['switch_port_list'] + self.port_map['switch_relay_port_list']
+        for host_intf, ports in port_list:
+            uplink = self.port_map[host_intf]['uplink']
+            for port in ports:
+                guest_if = port
+                local_if = '{0}_{1}'.format(guest_if, port_num+1)
+                guest_ip = '{0}.{1}/24'.format(tester_intf_subnet, port_num+1)
+                ##Use pipeworks to configure container interfaces on host/bridge interfaces
+                pipework_cmd = 'pipework {0} -i {1} -l {2} {3} {4}'.format(host_intf, guest_if,
                                                                            local_if, self.name, guest_ip)
-            else:
-                if start_vlan != 0:
-                    pipework_cmd += ' @{}'.format(start_vlan)
-                    start_vlan += 1
-
-            res += os.system(pipework_cmd)
-            port_num += 1
+                #if the wan interface is specified for uplink, then use it instead
+                if wan and port == self.port_map[uplink]:
+                    pipework_cmd = 'pipework {0} -i {1} -l {2} {3} {4}'.format(wan, guest_if,
+                                                                               local_if, self.name, guest_ip)
+                else:
+                    if start_vlan != 0:
+                        pipework_cmd += ' @{}'.format(start_vlan)
+                        start_vlan += 1
+                #print('Running PIPEWORK cmd: %s' %pipework_cmd)
+                res += os.system(pipework_cmd)
+                port_num += 1
 
         return res, port_num
 
     @classmethod
+    def get_intf_type(cls, intf):
+        intf_type = 0
+        if os.path.isdir('/sys/class/net/{}/bridge'.format(intf)):
+            intf_type = 1 ##linux bridge
+        else:
+            cmd = 'ovs-vsctl list-br | grep -q "^{0}$"'.format(intf)
+            res = os.system(cmd)
+            if res == 0: ##ovs bridge
+                intf_type = 2
+
+        return intf_type
+
+    @classmethod
     def cleanup_intfs(cls):
         olt_conf_file = os.path.join(cls.tester_base, 'olt_config.json')
         olt_config = OltConfig(olt_conf_file)
         port_map, _ = olt_config.olt_port_map()
         port_num = 0
-        intf_host = port_map['host']
         start_vlan = port_map['start_vlan']
-        uplink = port_map['uplink']
         wan = port_map['wan']
-        intf_type = 0
-        if os.path.isdir('/sys/class/net/{}/bridge'.format(intf_host)):
-            intf_type = 1 ##linux bridge
-        else:
-            cmd = 'ovs-vsctl list-br | grep -q "^{0}$"'.format(intf_host)
-            res = os.system(cmd)
-            if res == 0: ##ovs bridge
-                intf_type = 2
         cmds = ()
         res = 0
-        for port in port_map['ports'] + port_map['relay_ports']:
-            local_if = '{0}_{1}'.format(port, port_num+1)
-            if intf_type == 0:
-                if start_vlan != 0:
-                    cmds = ('ip link del {}.{}'.format(intf_host, start_vlan),)
-                    start_vlan += 1
-            else:
-                if intf_type == 1:
-                    cmds = ('brctl delif {} {}'.format(intf_host, local_if),
-                            'ip link del {}'.format(local_if))
+        port_list = port_map['switch_port_list'] + port_map['switch_relay_port_list']
+        for intf_host, ports in port_list:
+            intf_type = cls.get_intf_type(intf_host)
+            for port in ports:
+                local_if = '{0}_{1}'.format(port, port_num+1)
+                if intf_type == 0:
+                    if start_vlan != 0:
+                        cmds = ('ip link del {}.{}'.format(intf_host, start_vlan),)
+                        start_vlan += 1
                 else:
-                    cmds = ('ovs-vsctl del-port {} {}'.format(intf_host, local_if),
-                            'ip link del {}'.format(local_if))
+                    if intf_type == 1:
+                        cmds = ('brctl delif {} {}'.format(intf_host, local_if),
+                                'ip link del {}'.format(local_if))
+                    else:
+                        cmds = ('ovs-vsctl del-port {} {}'.format(intf_host, local_if),
+                                'ip link del {}'.format(local_if))
 
-            for cmd in cmds:
-                res += os.system(cmd)
-
-            port_num += 1
+                for cmd in cmds:
+                    res += os.system(cmd)
+                port_num += 1
 
     @classmethod
     def get_name(cls):
diff --git a/src/test/setup/of-bridge-local.sh b/src/test/setup/of-bridge-local.sh
index 6abfae6..da946dc 100755
--- a/src/test/setup/of-bridge-local.sh
+++ b/src/test/setup/of-bridge-local.sh
@@ -10,15 +10,45 @@
 pkill -9 ofdatapath
 pkill -9 ofprotocol
 service openvswitch-switch restart
-echo "Configuring ovs bridge $bridge"
-ovs-vsctl del-br $bridge
-ovs-vsctl add-br $bridge
+bridges=()
+num_bridges=1
+if [[ $bridge =~ ^[0-9]+$ ]]; then
+    num_bridges=$bridge
+    if [ $num_bridges -eq 0 ]; then
+        num_bridges=1
+    fi
+    for num in $(seq $num_bridges); do
+        if [ $num -eq 1 ]; then
+            br=br-int
+        else
+            br=br-int$num
+        fi
+        n=$(($num-1))
+        bridges[$n]=$br
+    done
+else
+    bridges[0]=$bridge
+fi
+
 ctlr=""
 for ip in `echo $controller | tr ',' '\n'`; do
   ctlr="$ctlr tcp:$ip:6653"
 done
-ovs-vsctl set-controller $bridge $ctlr
-ovs-vsctl set controller $bridge max_backoff=1000
-ovs-vsctl set bridge $bridge protocols=OpenFlow10,OpenFlow11,OpenFlow12,OpenFlow13
-ovs-vsctl show
-ovs-ofctl show $bridge
+
+#Delete existing bridges if any
+for br in "${bridges[@]}"; do
+    ovs-vsctl del-br $br
+done
+
+for br in "${bridges[@]}"; do
+    echo "Configuring OVS bridge:$br"
+    ovs-vsctl add-br $br
+    ovs-vsctl set-controller $br $ctlr
+    ovs-vsctl set controller $br max_backoff=1000
+    ovs-vsctl set bridge $br protocols=OpenFlow10,OpenFlow11,OpenFlow12,OpenFlow13
+done
+
+for br in "${bridges[@]}"; do
+    ovs-vsctl show
+    ovs-ofctl show $br
+done
diff --git a/src/test/setup/olt_config.json b/src/test/setup/olt_config.json
index 36070f7..844b4fe 100644
--- a/src/test/setup/olt_config.json
+++ b/src/test/setup/olt_config.json
@@ -1 +1 @@
-{ "olt" : false , "port_map" : { "num_ports" : 11, "host" : "br-int", "start_vlan" : 1000 }, "uplink" : 2, "vlan" : 0 }
+{ "olt" : false , "port_map" : { "num_ports" : 11, "start_vlan" : 1000, "nr_switches": 1 }, "uplink" : 2, "vlan" : 0 }
diff --git a/src/test/utils/OltConfig.py b/src/test/utils/OltConfig.py
index 730b23d..acf2339 100644
--- a/src/test/utils/OltConfig.py
+++ b/src/test/utils/OltConfig.py
@@ -38,19 +38,57 @@
     def olt_port_map(self):
         if self.on_olt() and self.olt_conf.has_key('port_map'):
             port_map = {}
+            if self.olt_conf['port_map'].has_key('switches'):
+                port_map['switches'] = self.olt_conf['port_map']['switches']
+            else:
+                port_map['switches'] = []
+                nr_switches = 1
+                if self.olt_conf['port_map'].has_key('nr_switches'):
+                    nr_switches = int(self.olt_conf['port_map']['nr_switches'])
+                for sw in xrange(nr_switches):
+                    switch = 'br-int{}'.format(sw+1) if sw > 0 else 'br-int'
+                    port_map['switches'].append(switch)
+            #if we have a host interface enabled, invalidate the switches config
+            if self.olt_conf['port_map'].has_key('host'):
+                #if host interface is specified, then use the host instead of ovs switch
+                port_map['host'] = self.olt_conf['port_map']['host']
+                port_map['switches'] = [ port_map['host'] ]
+            else:
+                port_map['host'] = port_map['switches'][0]
+            nr_switches = len(port_map['switches'])
+            port_map['switch_port_list'] = []
             if self.olt_conf['port_map'].has_key('ports'):
                 port_map['ports'] = self.olt_conf['port_map']['ports']
                 num_ports = len(port_map['ports'])
+                port_map['switch_port_list'].append( (port_map['switches'][0], port_map['ports']) )
             else:
                 port_map['ports'] = []
                 num_ports = int(self.olt_conf['port_map']['num_ports'])
-                for port in xrange(0, num_ports*2, 2):
-                    port_map['ports'].append('veth{}'.format(port))
+                for sw in xrange(nr_switches):
+                    port_list = []
+                    switch = port_map['switches'][sw]
+                    port_start = sw * num_ports * 2
+                    port_end = port_start + num_ports * 2
+                    for port in xrange(port_start, port_end, 2):
+                        port_name = 'veth{}'.format(port)
+                        port_map['ports'].append(port_name)
+                        port_list.append(port_name)
+                    port_map['switch_port_list'].append( (switch, port_list) )
             ##also add dhcprelay ports. We add as many relay ports as subscriber ports
+            port_map['num_ports'] = num_ports
             relay_ports = num_ports
             port_map['relay_ports'] = []
-            for port in xrange(relay_ports*2, relay_ports*4, 2):
-                port_map['relay_ports'].append('veth{}'.format(port))
+            port_map['switch_relay_port_list'] = []
+            for sw in xrange(nr_switches):
+                port_list = []
+                switch = port_map['switches'][sw]
+                port_start = (nr_switches + sw) * relay_ports * 2
+                port_end = port_start + relay_ports * 2
+                for port in xrange(port_start, port_end, 2):
+                    port_name = 'veth{}'.format(port)
+                    port_map['relay_ports'].append(port_name)
+                    port_list.append(port_name)
+                port_map['switch_relay_port_list'].append( (switch, port_list) )
             port_num = 1
             port_map['uplink'] = int(self.olt_conf['uplink'])
             port_map['wan'] = None
@@ -58,23 +96,26 @@
                 port_map['wan'] = self.olt_conf['wan']
             port_list = []
             ##build the port map and inverse port map
-            for port in port_map['ports']:
-                port_map[port_num] = port
-                port_map[port] = port_num
-                if port_num != port_map['uplink']:
-                    ##create tx,rx map
-                    port_list.append( (port_map['uplink'], port_num ) )
-                port_num += 1
+            for sw in xrange(nr_switches):
+                sw_portnum = 1
+                switch, ports = port_map['switch_port_list'][sw]
+                uplink = sw * num_ports + port_map['uplink']
+                port_map[switch] = {}
+                port_map[switch]['uplink'] = uplink
+                for p in ports:
+                    port_map[port_num] = p
+                    port_map[p] = port_num
+                    if sw_portnum != port_map['uplink']:
+                        #create tx, rx map
+                        port_list.append( (uplink, port_num) )
+                    port_num += 1
+                    sw_portnum += 1
             ##build the port and inverse map for relay ports
             for port in port_map['relay_ports']:
                 port_map[port_num] = port
                 port_map[port] = port_num
                 port_num += 1
             port_map['start_vlan'] = 0
-            if self.olt_conf['port_map'].has_key('host'):
-                port_map['host'] = self.olt_conf['port_map']['host']
-            else:
-                port_map['host'] = 'ovsbr0'
             if self.olt_conf['port_map'].has_key('start_vlan'):
                 port_map['start_vlan'] = int(self.olt_conf['port_map']['start_vlan'])