Fixing the flows.EcmpGroupMod test.
Disabling a bunch of flows tests that fail when run with other tests.
Adding a few flows tests which simplify the test to run in a single direction.
Updating README to reflect these changes.

Change-Id: Id5e42984dcf9d26cd2deeaa3bf8b311d4f256c5a
diff --git a/README.md b/README.md
index 691ed64..0ffe6fe 100755
--- a/README.md
+++ b/README.md
@@ -8,7 +8,7 @@
 
 In this project we use OFTest to ensure OpenFlow conformance of the switches used to implement the network infrastructure of the OpenCord project.
 
-At this moment the switches supported are Accton switches. For more information abot that check the wiki page at wiki.opencord.org
+At this moment the switches supported are OF-DPA based Accton switches. For more information, check the wiki page at wiki.opencord.org/display/CORD/Underlay+Fabric
 
 This code base was forked from [macauleycheng](github.com/macauleycheng/oftest) which was forked from [floodlight](github.com/floodlight/oftest). This documentationn aims to describe the specific test cases developed for OpenCord. To get started on the basics of OFTest please check their documentation.
 
@@ -24,7 +24,7 @@
 i12           | i12        | Archived
 i12_1.7       | cord-1.0   | Released
 2.0 GA        | 2.0-ga     | Archived
-**3.0 EA0**   | **master** | **Developing <- current branch**
+**3.0 EA4**   | **master** | **Developing <- current branch**
 
 The test case collection for OpenCord are under the folder ofdpa. They are listed under the section Test case collection.
 
@@ -41,18 +41,12 @@
 
 ## Start Testing
 
-On the switch side:
+On the switch side (for OF-DPA version 3.0 EA4):
 
-1. Purge the switch flow/group table by running
+1. Connect the switch to the VM running OFTest on the management network
 
 	```
-	client_cfg_purge
-	```
-
-2. Connect the switch to the testing controller
-
-	```
-	brcm-indigo-ofdpa-ofagent -t <controller_ip>
+	launcher ofagentapp  -t <controller_ip>
 	```
 
 On the controller side:
@@ -63,22 +57,23 @@
 	git clone -b master git://github.com/opencord/fabric-oftest
 	```
 
-2. Run **all test cases** in OFTest
+2. Run all test cases (that are not disabled) in a sequence
 
 	```
 	sudo ./oft -V1.3 --test-dir=ofdpa flows -i 24@eth1 -i 12@eth2
 	```
-	This command assumes you connected the switch port 24 to interface eth1 on the OFtest server and port 12 to eth2
+	This command assumes you connected the switch port 24 to interface eth1 on the OFtest server and port 12 to eth2.
+	It runs all the tests that are in ofdpa/flows.py
 
 ## Useful commands
 
 * List all available test cases
 
 	```
-	./oft --list --test-dir=ofdpa flows
+	./oft -V1.3 --list --test-dir=ofdpa flows
 	```
 
-* Run only specific test case
+* Run only specific test case. Can also be used to run tests that are currently disabled.
 
 	```
 	sudo ./oft -V1.3 --test-dir=ofdpa flows.PacketInArp -i 24@eth1 -i 12@eth2
@@ -97,15 +92,15 @@
 /32UnicastTagged   | ok      | ok     | ok      | ok      |
 /24ECMPL3          | ok      | ok     | ok      | ok      |
 /32ECMPL3          | ok      | ok     | ok      | ok      |
-/24ECMPVPN         | ok      | ok     | ok      | ok      |
-/32ECMPVPN         | ok      | ok     | ok      | ok      |
-/32VPN             | ok      | ok     | ok      | ok      |
-/24VPN             | ok      | ok     | ok      | ok      |
+/24ECMPVPN         | ok      | ok     | ok      | ok~      |
+/32ECMPVPN         | ok      | ok     | ok      | ok~      |
+/32VPN             | ok      | ok     | ok      | ok~      |
+/24VPN             | ok      | ok     | ok      | ok~      |
 EcmpGroupMod       | X       | X      | ok      | ok      |
 PacketInArp        | ok      | ok     | ok      | ok      |
 MTU1500            | ok      | ok     | ok      | ok      |
-MplsTermination    | ok      | ok     | ok      | ok      |
-MplsFwd            | X       | ok     | ok      | ok      |
+MplsTermination    | ok      | ok     | ok      | ok~      |
+MplsFwd            | X       | ok     | ok      | ok~      |
 L2FloodQinQ        | ok      | ok     | ok      | ok      |
 L2UnicastTagged    | ok      | ok     | ok      | ok      |
 L3McastToL3        | ok      | X      | ok      | ok      |
@@ -118,13 +113,17 @@
 PacketInUDP        | ok      | ok     | ok      | ok      |
 Unfiltered         | X       | ok     | X       | ok      |
 Untagged           | ok      | n/a    | ok      | ok      |
-PacketInIPTable    | X       | X      | ok      | ok      |
+PacketInIPTable    | X       | X      | ok      | ok~      |
 
+```
+~       Tests marked with tilda are currently disabled because of a bug which causes
+        interference with other tests. The @disabled flag will be removed once the bug is fixed.
 *       Untag -> Untag (4094 as internal vlan)
 **      Untag -> Tag
 ***     Tag   -> Untag
 ****    Tag   -> Tag
 *****   Tag   -> Tag (Translated)
+```
 
 n/a means test is not available for that version of the pipeline.
 
diff --git a/ofdpa/flows.py b/ofdpa/flows.py
index b888105..b4b2dae 100755
--- a/ofdpa/flows.py
+++ b/ofdpa/flows.py
@@ -8,6 +8,7 @@
 import logging
 import oftest.base_tests as base_tests
 import ofp
+import time
 from oftest.testutils import *
 from accton_util import *
 from utils import *
@@ -65,6 +66,9 @@
 
 @disabled
 class ArpNL2( base_tests.SimpleDataPlane ):
+    """
+    Needs a description, disabled for now. Also needs try/finally
+    """
     def runTest( self ):
         delete_all_flows( self.controller )
         delete_all_groups( self.controller )
@@ -125,12 +129,12 @@
                     ofp.action.output( port=ofp.OFPP_CONTROLLER, max_len=ofp.OFPCML_NO_BUFFER ) ] ), ],
                     buffer_id=ofp.OFP_NO_BUFFER, priority=1 )
 
-            logging.info( "Inserting packet in flow to controller" )
+            logging.info( "Inserting arp flow " )
             self.controller.message_send( request )
             do_barrier( self.controller )
 
             for of_port in config[ "port_map" ].keys( ):
-                logging.info( "PacketInMiss test, port %d", of_port )
+                logging.info( "PacketInArp test, sending arp packet to port %d", of_port )
                 self.dataplane.send( of_port, arp_pkt )
 
                 verify_packet_in( self, arp_pkt, of_port, ofp.OFPR_ACTION )
@@ -140,7 +144,7 @@
             delete_all_flows( self.controller )
             delete_all_groups( self.controller )
 
-
+@disabled
 class PacketInIPTable( base_tests.SimpleDataPlane ):
     """
     Verify Packet-in message from IP table when controller action is used
@@ -244,6 +248,7 @@
 @disabled
 class L2FloodTagged( base_tests.SimpleDataPlane ):
     """
+    currently disabled; fix with try/finally
     Test L2 flood to a vlan
     Send a packet with unknown dst_mac and check if the packet is flooded to all ports except inport
     """
@@ -319,7 +324,10 @@
 
 
 class Mtu1500( base_tests.SimpleDataPlane ):
-    """V erifies basic mtu limits"""
+    """
+    Verifies basic mtu limits
+    """
+
     def runTest( self ):
         Groups = Queue.LifoQueue( )
         try:
@@ -355,7 +363,7 @@
 
 
 class _32UcastTagged( base_tests.SimpleDataPlane ):
-    """ Verify /32 IP forwarding to L3 Interface"""
+    """ Verify /32 IP forwarding to L3 Unicast-> L2Interface"""
 
     def runTest( self ):
         Groups = Queue.LifoQueue( )
@@ -414,9 +422,12 @@
             delete_groups( self.controller, Groups )
             delete_all_groups( self.controller )
 
-
+@disabled
 class _32VPN( base_tests.SimpleDataPlane ):
-    """ Verify MPLS IP VPN Initiation from /32 rule   """
+    """
+    Verify /32 routing rule -> MPLS_VPN_Label -> MPLSInterface -> L2Interface
+    No ECMP group used
+    """
 
     def runTest( self ):
         Groups = Queue.LifoQueue( )
@@ -442,7 +453,6 @@
                 mpls_label_gid, mpls_label_msg = add_mpls_label_group( self.controller,
                         subtype=OFDPA_MPLS_GROUP_SUBTYPE_L3_VPN_LABEL, index=id, ref_gid=mpls_gid,
                         push_mpls_header=True, set_mpls_label=port, set_bos=1, set_ttl=32 )
-                # ecmp_msg=add_l3_ecmp_group(self.controller, vlan_id, [mpls_label_gid])
                 do_barrier( self.controller )
                 # add vlan flow table
                 add_one_vlan_table_flow( self.controller, port, 1, vlan_id, vrf=2,
@@ -482,9 +492,11 @@
             delete_groups( self.controller, Groups )
             delete_all_groups( self.controller )
 
-
+@disabled
 class _32EcmpVpn( base_tests.SimpleDataPlane ):
-    """  Verify MPLS IP VPN Initiation from /32 rule using ECMP  """
+    """
+    Verify /32 routing rule -> L3 ECMP -> MPLS_VPN_Label -> MPLSInterface -> L2Interface
+    """
 
     def runTest( self ):
         Groups = Queue.LifoQueue( )
@@ -546,14 +558,92 @@
                     pkt = str( exp_pkt )
                     verify_packet( self, pkt, out_port )
                     verify_no_other_packets( self )
+
         finally:
             delete_all_flows( self.controller )
             delete_groups( self.controller, Groups )
             delete_all_groups( self.controller )
 
 
+@disabled
+class One_32EcmpVpn( base_tests.SimpleDataPlane ):
+    """
+    Verify /32 routing rule -> L3 ECMP -> MPLS_VPN_Label -> MPLSInterface -> L2Interface
+    in only one direction
+    """
+
+    def runTest( self ):
+        Groups = Queue.LifoQueue( )
+        try:
+            if len( config[ "port_map" ] ) < 2:
+                logging.info( "Port count less than 2, can't run this case" )
+                return
+
+            intf_src_mac = [ 0x00, 0x00, 0x00, 0xcc, 0xcc, 0xcc ]
+            dst_mac = [ 0x00, 0x00, 0x00, 0x22, 0x22, 0x00 ]
+            dip = 0xc0a80001
+            ports = config[ "port_map" ].keys( )
+            # add l2 interface group
+            id = ports[1]
+            vlan_id = ports[1] + 20
+            l2_gid, l2_msg = add_one_l2_interface_group( self.controller, ports[1], vlan_id, True, True )
+            dst_mac[ 5 ] = vlan_id
+            # add MPLS interface group
+            mpls_gid, mpls_msg = add_mpls_intf_group( self.controller, l2_gid, dst_mac, intf_src_mac,
+                                                      vlan_id, id )
+            # add MPLS L3 VPN group
+            mpls_label_gid, mpls_label_msg = add_mpls_label_group( self.controller,
+                        subtype=OFDPA_MPLS_GROUP_SUBTYPE_L3_VPN_LABEL, index=id, ref_gid=mpls_gid,
+                        push_mpls_header=True, set_mpls_label=ports[1] + 20, set_bos=1, set_ttl=32 )
+            # add ECMP group
+            ecmp_msg = add_l3_ecmp_group( self.controller, vlan_id, [ mpls_label_gid ] )
+            do_barrier( self.controller )
+            # add vlan flow table
+            add_one_vlan_table_flow( self.controller, ports[0], 1, vlan_id=ports[0] + 10, vrf=0,
+                                     flag=VLAN_TABLE_FLAG_ONLY_TAG )
+            # add termination flow
+            add_termination_flow( self.controller, ports[0], 0x0800, intf_src_mac, vlanid=ports[0] + 10 )
+            # add routing flow
+            dst_ip = dip + (vlan_id << 8)
+            add_unicast_routing_flow( self.controller, 0x0800, dst_ip, 0xffffffff, ecmp_msg.group_id, send_barrier=True )
+            Groups._put( l2_gid )
+            Groups._put( mpls_gid )
+            Groups._put( mpls_label_gid )
+            Groups._put( ecmp_msg.group_id )
+
+
+            switch_mac = ':'.join( [ '%02X' % x for x in intf_src_mac ] )
+            in_port = ports[0]
+            out_port = ports[1]
+            ip_src = '192.168.%02d.1' % (in_port)
+            ip_dst = '192.168.%02d.1' % (out_port)
+            parsed_pkt = simple_tcp_packet( pktlen=100, dl_vlan_enable=True, vlan_vid=(in_port + 20),
+                                            eth_dst=switch_mac, ip_ttl=64, ip_src=ip_src, ip_dst=ip_dst )
+            pkt = str( parsed_pkt )
+            self.dataplane.send( in_port, pkt )
+            # build expect packet
+            mac_dst = '00:00:00:22:22:%02X' % (out_port)
+            label = (out_port, 0, 1, 32)
+            exp_pkt = mpls_packet( pktlen=104, dl_vlan_enable=True, vlan_vid=(out_port + 20), ip_ttl=63,
+                                   ip_src=ip_src, ip_dst=ip_dst, eth_dst=mac_dst, eth_src=switch_mac,
+                                   label=[ label+20 ] )
+            pkt = str( exp_pkt )
+            verify_packet( self, pkt, out_port )
+            #verify_no_other_packets( self )
+
+        finally:
+            delete_all_flows( self.controller )
+            delete_group(self.controller, ecmp_msg.group_id)
+            delete_group(self.controller, mpls_label_gid)
+            delete_group(self.controller, mpls_gid)
+            delete_group(self.controller, l2_gid)
+
+
 class _32ECMPL3( base_tests.SimpleDataPlane ):
-    """ Verifies /32 IP routing and ECMP """
+    """
+    Verifies /32 IP routing and ECMP with no label push
+    IP -> L3ECMP -> L3Unicast -> L2Interface
+    """
 
     def runTest( self ):
         Groups = Queue.LifoQueue( )
@@ -613,9 +703,75 @@
             delete_groups( self.controller, Groups )
             delete_all_groups( self.controller )
 
+@disabled
+class One_32ECMPL3( base_tests.SimpleDataPlane ):
+    """
+    Verifies /32 IP routing and ECMP with no label push
+    IP -> L3ECMP -> L3Unicast -> L2Interface
+    in only one direction
+    """
 
+    def runTest( self ):
+        Groups = Queue.LifoQueue( )
+        try:
+            if len( config[ "port_map" ] ) < 2:
+                logging.info( "Port count less than 2, can't run this case" )
+                return
+
+            intf_src_mac = [ 0x00, 0x00, 0x00, 0xcc, 0xcc, 0xcc ]
+            dst_mac = [ 0x00, 0x00, 0x00, 0x22, 0x22, 0x00 ]
+            dip = 0xc0a80001
+            # Hashes Test Name and uses it as id for installing unique groups
+            ports = config[ "port_map" ].keys( )
+            inport = ports[0]
+            outport = ports[1]
+
+            vlan_id = outport + 20
+            id = outport
+            # add l2 interface group, l3 unicast and ecmp group for outport
+            l2_gid, msg = add_one_l2_interface_group( self.controller, outport, vlan_id=vlan_id,
+                                                      is_tagged=True, send_barrier=False )
+            dst_mac[ 5 ] = vlan_id
+            l3_msg = add_l3_unicast_group( self.controller, outport, vlanid=vlan_id, id=id,
+                                           src_mac=intf_src_mac, dst_mac=dst_mac )
+            ecmp_msg = add_l3_ecmp_group( self.controller, id, [ l3_msg.group_id ] )
+            # add vlan flow table
+            add_one_vlan_table_flow( self.controller, of_port=inport, vlan_id=inport+20, flag=VLAN_TABLE_FLAG_ONLY_TAG )
+            # add termination flow
+            add_termination_flow( self.controller, in_port=inport, eth_type=0x0800, dst_mac=intf_src_mac, vlanid=inport+20 )
+            # add unicast routing flow
+            dst_ip = dip + (vlan_id << 8)
+            add_unicast_routing_flow( self.controller, 0x0800, dst_ip, 0xffffffff, ecmp_msg.group_id, send_barrier=True )
+            Groups._put( l2_gid )
+            Groups._put( l3_msg.group_id )
+            Groups._put( ecmp_msg.group_id )
+
+            switch_mac = ':'.join( [ '%02X' % x for x in intf_src_mac ] )
+            mac_src = '00:00:00:22:22:%02X' % inport
+            ip_src = '192.168.%02d.1' % inport
+            ip_dst = '192.168.%02d.1' % outport
+            parsed_pkt = simple_tcp_packet( pktlen=100, dl_vlan_enable=True, vlan_vid=inport+20,
+                                            eth_dst=switch_mac, eth_src=mac_src, ip_ttl=64, ip_src=ip_src, ip_dst=ip_dst )
+            pkt = str( parsed_pkt )
+            self.dataplane.send( inport, pkt )
+            # build expected packet
+            mac_dst = '00:00:00:22:22:%02X' % outport
+            exp_pkt = simple_tcp_packet( pktlen=100, dl_vlan_enable=True, vlan_vid=outport+20,
+                                         eth_dst=mac_dst, eth_src=switch_mac, ip_ttl=63, ip_src=ip_src, ip_dst=ip_dst )
+            pkt = str( exp_pkt )
+            verify_packet( self, pkt, outport )
+            verify_no_other_packets( self )
+        finally:
+            delete_all_flows( self.controller )
+            delete_groups( self.controller, Groups )
+            delete_all_groups( self.controller )
+
+
+
+
+@disabled
 class _24VPN( base_tests.SimpleDataPlane ):
-    """  Verify MPLS IP VPN Initiation from /32 rule using ECMP  """
+    """  Verify MPLS IP VPN Initiation from /32 rule without using ECMP  """
 
     def runTest( self ):
         Groups = Queue.LifoQueue( )
@@ -681,7 +837,7 @@
             delete_groups( self.controller, Groups )
             delete_all_groups( self.controller )
 
-
+@disabled
 class _24EcmpVpn( base_tests.SimpleDataPlane ):
     """  Verify MPLS IP VPN Initiation from /24 rule using ECMP  """
 
@@ -755,7 +911,7 @@
 
 
 class FloodGroupMod( base_tests.SimpleDataPlane ):
-    """ Modify referenced group test """
+    """ Modify a referenced flood group """
 
     def runTest( self ):
         Groups = Queue.LifoQueue( )
@@ -803,7 +959,7 @@
 
 
 class _24ECMPL3( base_tests.SimpleDataPlane ):
-    """ Verifies /24 IP routing using ECMP """
+    """ Verifies /24 IP routing using ECMP -> L3U -> L2I """
 
     def runTest( self ):
         Groups = Queue.LifoQueue( )
@@ -866,6 +1022,9 @@
 
 @disabled
 class MPLSBUG( base_tests.SimpleDataPlane ):
+    """
+    Needs a description or needs to be removed
+    """
     def runTest( self ):
         if len( config[ "port_map" ] ) < 2:
             logging.info( "Port count less than 2, can't run this case" )
@@ -1345,9 +1504,9 @@
             delete_groups( self.controller, Groups )
             delete_all_groups( self.controller )
 
-
+@disabled
 class _MplsFwd( base_tests.SimpleDataPlane ):
-    """ Verify basic MPLS forwarding: Label switch router """
+    """ Verify basic MPLS forwarding: Label switch router  """
 
     def runTest( self ):
         Groups = Queue.LifoQueue( )
@@ -1374,8 +1533,8 @@
                 mpls_label_gid, mpls_label_msg = add_mpls_label_group( self.controller,
                         subtype=OFDPA_MPLS_GROUP_SUBTYPE_SWAP_LABEL, index=id, ref_gid=mpls_gid,
                         push_mpls_header=False, set_mpls_label=mpls_label, set_bos=1 )
-                ecmp_gid, ecmp_msg = add_mpls_forwarding_group( self.controller,
-                        subtype=OFDPA_MPLS_GROUP_SUBTYPE_ECMP, index=id, ref_gids=[mpls_label_gid] )
+                #ecmp_gid, ecmp_msg = add_mpls_forwarding_group( self.controller,
+                #        subtype=OFDPA_MPLS_GROUP_SUBTYPE_ECMP, index=id, ref_gids=[mpls_label_gid] )
                 # add vlan flow table
                 add_one_vlan_table_flow( self.controller, port, 1, vlan_id, flag=VLAN_TABLE_FLAG_ONLY_TAG )
                 # add termination flow
@@ -1386,7 +1545,7 @@
                 Groups._put( l2_gid )
                 Groups._put( mpls_gid )
                 Groups._put( mpls_label_gid )
-                Groups._put( ecmp_gid )
+                #Groups._put( ecmp_gid )
             do_barrier( self.controller )
 
             switch_mac = ':'.join( [ '%02X' % x for x in intf_src_mac ] )
@@ -1422,7 +1581,7 @@
             delete_groups( self.controller, Groups )
             delete_all_groups( self.controller )
 
-
+@disabled
 class _MplsTermination( base_tests.SimpleDataPlane ):
     """ Verify MPLS VPN Termination at penultimate hop """
 
@@ -1493,6 +1652,76 @@
             delete_all_flows( self.controller )
             delete_groups( self.controller, Groups )
             delete_all_groups( self.controller )
+            print("Done")
+
+@disabled
+class One_MplsTermination( base_tests.SimpleDataPlane ):
+    """
+    Verify MPLS VPN Termination at penultimate hop in only one direction
+    """
+
+    def runTest( self ):
+        Groups = Queue.LifoQueue( )
+        try:
+            if len( config[ "port_map" ] ) < 2:
+                logging.info( "Port count less than 2, can't run this case" )
+                return
+            dip = 0xc0a80001
+            intf_src_mac = [ 0x00, 0x00, 0x00, 0xcc, 0xcc, 0xcc ]
+            dst_mac = [ 0x00, 0x00, 0x00, 0x22, 0x22, 0x00 ]
+            # Assigns unique hardcoded test_id to make sure tests don't overlap when writing rules
+            ports = config[ "port_map" ].keys( )
+            inport = ports[0]
+            outport = ports[1]
+
+            # Shift MPLS label and VLAN ID by 16 to avoid reserved values
+            invlan_id = inport + 16
+            outvlan_id = outport + 16
+            mpls_label = outport + 16
+
+            # add l2 interface group
+            id, dst_mac[ 5 ] = inport, outport
+            l2_gid, l2_msg = add_one_l2_interface_group( self.controller, outport, outvlan_id, True, False )
+            # add L3 Unicast  group
+            l3_msg = add_l3_unicast_group( self.controller, outport, vlanid=outvlan_id, id=id,
+                                           src_mac=intf_src_mac, dst_mac=dst_mac )
+            # add L3 ecmp group
+            ecmp_msg = add_l3_ecmp_group( self.controller, id, [ l3_msg.group_id ] )
+            # add vlan flow table
+            add_one_vlan_table_flow( self.controller, inport, 1, invlan_id, flag=VLAN_TABLE_FLAG_ONLY_TAG )
+            # add tmac flow
+            add_termination_flow( self.controller, inport, 0x8847, intf_src_mac, invlan_id, goto_table=24 )
+            # add mpls termination flow
+            add_mpls_flow( self.controller, ecmp_msg.group_id, mpls_label, send_barrier=True )
+            Groups._put( l2_gid )
+            Groups._put( l3_msg.group_id )
+            Groups._put( ecmp_msg.group_id )
+
+            time.sleep(0.1)
+            switch_mac = ':'.join( [ '%02X' % x for x in intf_src_mac ] )
+            ip_src = '192.168.%02d.1' % (inport)
+            # Shift MPLS label and VLAN ID by 16 to avoid reserved values
+            out_mpls_label = outport + 16
+            in_vlan_vid = inport + 16
+            out_vlan_vid = outport + 16
+
+            ip_dst = '192.168.%02d.1' % (outport)
+            label = (out_mpls_label, 0, 1, 32)
+            parsed_pkt = mpls_packet( pktlen=104, dl_vlan_enable=True, vlan_vid=(in_vlan_vid),
+                                      ip_src=ip_src, ip_dst=ip_dst, eth_dst=switch_mac, label=[ label ] )
+            pkt = str( parsed_pkt )
+            self.dataplane.send( inport, pkt )
+            # build expect packet
+            mac_dst = '00:00:00:22:22:%02X' % (outport)
+            exp_pkt = simple_tcp_packet( pktlen=100, dl_vlan_enable=True, vlan_vid=(out_vlan_vid),
+                                         eth_dst=mac_dst, eth_src=switch_mac, ip_ttl=31, ip_src=ip_src, ip_dst=ip_dst )
+            pkt = str( exp_pkt )
+            verify_packet( self, pkt, outport )
+            verify_no_other_packets( self )
+        finally:
+            delete_all_flows( self.controller )
+            delete_groups( self.controller, Groups )
+            delete_all_groups( self.controller )
 
 
 class _24UcastTagged( base_tests.SimpleDataPlane ):
@@ -1644,7 +1873,7 @@
             delete_all_flows( self.controller )
             delete_all_groups( self.controller )
 
-
+@disabled
 class L3McastToVPN( base_tests.SimpleDataPlane ):
     """
     Mcast routing and VPN initiation
@@ -1726,7 +1955,7 @@
             delete_all_flows( self.controller )
             delete_all_groups( self.controller )
 
-
+@disabled
 class PacketInSrcMacMiss( base_tests.SimpleDataPlane ):
     """
     Test packet in function on a src-mac miss
@@ -1759,9 +1988,7 @@
 
 class EcmpGroupMod( base_tests.SimpleDataPlane ):
     """
-        Verify referenced group can be modified adding or removing buckets
-        Attention the hashing behavior may vary according to your porting assigment
-        Current values are hardcoded for our topology
+        Verify referenced group can be modified by adding or removing buckets
     """
 
     def runTest( self ):
@@ -1777,6 +2004,9 @@
             # Hashes Test Name and uses it as id for installing unique groups
             ports = config[ "port_map" ].keys( )
             ecmp = [ ]
+            dst_ips = []
+            # add flows for all ports but include only the egress switchport (connected to ports[1])
+            # in the ecmp group
             for port in ports:
                 vlan_id = port
                 id = port
@@ -1786,7 +2016,8 @@
                 dst_mac[ 5 ] = vlan_id
                 l3_msg = add_l3_unicast_group( self.controller, port, vlanid=vlan_id, id=id,
                         src_mac=intf_src_mac, dst_mac=dst_mac )
-                ecmp += [ l3_msg.group_id ]
+                if port == ports[1]:
+                    ecmp += [ l3_msg.group_id ]
                 Groups._put( l2_gid )
                 Groups._put( l3_msg.group_id )
                 ecmp_msg = add_l3_ecmp_group( self.controller, ports[ 0 ], [ l3_msg.group_id ] )
@@ -1796,67 +2027,109 @@
                 add_termination_flow( self.controller, port, 0x0800, intf_src_mac, vlan_id )
                 # add unicast routing flow
                 dst_ip = dip + (vlan_id << 8)
-                add_unicast_routing_flow( self.controller, 0x0800, dst_ip, 0xffffff00, ecmp_msg.group_id )
+                dst_ips += [dst_ip]
                 Groups._put( ecmp_msg.group_id )
             mod_l3_ecmp_group( self.controller, ports[ 0 ], ecmp )
-
+            for dst_ip in dst_ips:
+                add_unicast_routing_flow( self.controller, 0x0800, dst_ip, 0xffffff00, ecmp_msg.group_id )
+            time.sleep(0.1)
+            # first part of the test: send packet from ingress switchport and expect it at egress switchport
             switch_mac = ':'.join( [ '%02X' % x for x in intf_src_mac ] )
             parsed_pkt = exp_pkt = 0
-            for out_port in ports:
-                mac_src = '00:00:00:22:22:%02X' % ports[ 0 ]
-                ip_src = '192.168.%02d.%02d' % (ports[ 0 ], 1)
-                ip_dst = '192.168.%02d.%02d' % (ports[ 1 ], 1)
-                tcp = out_port if out_port == 24 else 25
-                parsed_pkt = simple_tcp_packet( pktlen=100, dl_vlan_enable=True, vlan_vid=ports[ 0 ],
-                        eth_dst=switch_mac, eth_src=mac_src, ip_ttl=64, ip_src=ip_src, ip_dst=ip_dst,
-                        tcp_dport=tcp )
-                pkt = str( parsed_pkt )
-                self.dataplane.send( ports[ 0 ], pkt )
-                # build expected packet
-                mac_dst = '00:00:00:22:22:%02X' % out_port
-                exp_pkt = simple_tcp_packet( pktlen=100, dl_vlan_enable=True, vlan_vid=out_port,
-                        eth_dst=mac_dst, eth_src=switch_mac, ip_ttl=63, ip_src=ip_src, ip_dst=ip_dst,
-                        tcp_dport=tcp )
-                pkt = str( exp_pkt )
-                verify_packet( self, pkt, out_port )
-                verify_no_other_packets( self )
+            in_port = ports[0]
+            out_port = ports[1]
+            logging.info("\nSending packet to port: " + str(in_port) + ", expected egress on port: " + str(out_port))
+            mac_src = '00:00:00:22:22:%02X' % ports[ 0 ]
+            ip_src = '192.168.%02d.%02d' % (ports[ 0 ], 1)
+            ip_dst = '192.168.%02d.%02d' % (ports[ 1 ], 1)
+            tcp = out_port if out_port == 24 else 25
+            parsed_pkt = simple_tcp_packet( pktlen=100, dl_vlan_enable=True, vlan_vid=ports[ 0 ],
+                                            eth_dst=switch_mac, eth_src=mac_src, ip_ttl=64, ip_src=ip_src,
+                                            ip_dst=ip_dst, tcp_dport=tcp )
+            pkt = str( parsed_pkt )
+            self.dataplane.send( ports[ 0 ], pkt )
+            # build expected packet at egress switchport
+            mac_dst = '00:00:00:22:22:%02X' % out_port
+            exp_pkt = simple_tcp_packet( pktlen=100, dl_vlan_enable=True, vlan_vid=out_port,
+                                         eth_dst=mac_dst, eth_src=switch_mac, ip_ttl=63, ip_src=ip_src,
+                                         ip_dst=ip_dst, tcp_dport=tcp )
+            pkt = str( exp_pkt )
+            verify_packet( self, pkt, out_port )
+            verify_no_other_packets( self )
+
+            # second part of the test - edit the ecmp group to remove the orginal egress switchport
+            # and instead add the ingress switchport. Send packet from ingress switchport, and expect
+            # it back on the ingress switchport
             l3_gid = encode_l3_unicast_group_id( ports[ 0 ] )
             mod_l3_ecmp_group( self.controller, ports[ 0 ], [ l3_gid ] )
-            for port in ports:
-                mac_src = '00:00:00:22:22:%02X' % ports[ 0 ]
-                ip_src = '192.168.%02d.%02d' % (ports[ 0 ], 1)
-                ip_dst = '192.168.%02d.%02d' % (ports[ 1 ], 1)
-                tcp = port if port == 24 else 25
-                parsed_pkt = simple_tcp_packet( pktlen=100, dl_vlan_enable=True, vlan_vid=ports[ 0 ],
-                        eth_dst=switch_mac, eth_src=mac_src, ip_ttl=64, ip_src=ip_src, ip_dst=ip_dst,
-                        tcp_dport=tcp )
-                pkt = str( parsed_pkt )
-                self.dataplane.send( ports[ 0 ], pkt )
-                # build expected packet
-                mac_dst = '00:00:00:22:22:%02X' % ports[ 0 ]
-                exp_pkt = simple_tcp_packet( pktlen=100, dl_vlan_enable=True, vlan_vid=ports[ 0 ],
-                        eth_dst=mac_dst, eth_src=switch_mac, ip_ttl=63, ip_src=ip_src, ip_dst=ip_dst,
-                        tcp_dport=tcp )
-                pkt = str( exp_pkt )
-                verify_packet( self, pkt, ports[ 0 ] )
-                verify_no_other_packets( self )
+            time.sleep(0.1)
+            logging.info("Sending packet to port: " + str(ports[0]) + ", expected egress on port: " + str(ports[0]))
+            mac_src = '00:00:00:22:22:%02X' % ports[ 0 ]
+            ip_src = '192.168.%02d.%02d' % (ports[ 0 ], 1)
+            ip_dst = '192.168.%02d.%02d' % (ports[ 1 ], 1)
+            tcp = port if port == 24 else 25
+            parsed_pkt = simple_tcp_packet( pktlen=100, dl_vlan_enable=True, vlan_vid=ports[ 0 ],
+                                            eth_dst=switch_mac, eth_src=mac_src, ip_ttl=64, ip_src=ip_src,
+                                            ip_dst=ip_dst,tcp_dport=tcp )
+            pkt = str( parsed_pkt )
+            self.dataplane.send( ports[ 0 ], pkt )
+            # build expected packet
+            mac_dst = '00:00:00:22:22:%02X' % ports[ 0 ]
+            exp_pkt = simple_tcp_packet( pktlen=100, dl_vlan_enable=True, vlan_vid=ports[ 0 ],
+                                         eth_dst=mac_dst, eth_src=switch_mac, ip_ttl=63, ip_src=ip_src,
+                                         ip_dst=ip_dst,tcp_dport=tcp )
+            pkt = str( exp_pkt )
+            verify_packet( self, pkt, ports[ 0 ] )
+            verify_no_other_packets( self )
+
+            # third part of the test - edit the group to completely remove bucket. Packet sent
+            # should be dropped by the switch
             mod_l3_ecmp_group( self.controller, ports[ 0 ], [ ] )
-            for port in ports:
-                mac_src = '00:00:00:22:22:%02X' % ports[ 0 ]
-                ip_src = '192.168.%02d.%02d' % (ports[ 0 ], 1)
-                ip_dst = '192.168.%02d.%02d' % (ports[ 1 ], 1)
-                tcp = port if port == 24 else 25
-                parsed_pkt = simple_tcp_packet( pktlen=100, dl_vlan_enable=True, vlan_vid=ports[ 0 ],
-                        eth_dst=switch_mac, eth_src=mac_src, ip_ttl=64, ip_src=ip_src, ip_dst=ip_dst,
-                        tcp_dport=tcp )
-                pkt = str( parsed_pkt )
-                self.dataplane.send( ports[ 0 ], pkt )
-                verify_no_other_packets( self )
+            time.sleep(0.1)
+            logging.info("Sending packet to port: " + str(ports[0]) + ", expected drop")
+            mac_src = '00:00:00:22:22:%02X' % ports[ 0 ]
+            ip_src = '192.168.%02d.%02d' % (ports[ 0 ], 1)
+            ip_dst = '192.168.%02d.%02d' % (ports[ 1 ], 1)
+            tcp = port if port == 24 else 25
+            parsed_pkt = simple_tcp_packet( pktlen=100, dl_vlan_enable=True, vlan_vid=ports[ 0 ],
+                                            eth_dst=switch_mac, eth_src=mac_src, ip_ttl=64, ip_src=ip_src,
+                                            ip_dst=ip_dst,tcp_dport=tcp )
+            pkt = str( parsed_pkt )
+            self.dataplane.send( ports[ 0 ], pkt )
+            verify_no_other_packets( self )
+
+            # final part of the test - edit the empty group to add back the bucket for the
+            # original egress port, and verify packet is received on egress switch port
+            l3_gid = encode_l3_unicast_group_id( ports[ 1 ] )
+            mod_l3_ecmp_group( self.controller, ports[ 0 ], [ l3_gid ] )
+            time.sleep(0.1)
+            in_port = ports[0]
+            out_port = ports[1]
+            logging.info("Sending packet to port: " + str(in_port) + ", expected egress on port: " + str(out_port))
+            mac_src = '00:00:00:22:22:%02X' % ports[ 0 ]
+            ip_src = '192.168.%02d.%02d' % (ports[ 0 ], 1)
+            ip_dst = '192.168.%02d.%02d' % (ports[ 1 ], 1)
+            tcp = out_port if out_port == 24 else 25
+            parsed_pkt = simple_tcp_packet( pktlen=100, dl_vlan_enable=True, vlan_vid=ports[ 0 ],
+                                            eth_dst=switch_mac, eth_src=mac_src, ip_ttl=64, ip_src=ip_src,
+                                            ip_dst=ip_dst, tcp_dport=tcp )
+            pkt = str( parsed_pkt )
+            self.dataplane.send( ports[ 0 ], pkt )
+            # build expected packet at egress switchport
+            mac_dst = '00:00:00:22:22:%02X' % out_port
+            exp_pkt = simple_tcp_packet( pktlen=100, dl_vlan_enable=True, vlan_vid=out_port,
+                                         eth_dst=mac_dst, eth_src=switch_mac, ip_ttl=63, ip_src=ip_src,
+                                         ip_dst=ip_dst, tcp_dport=tcp )
+            pkt = str( exp_pkt )
+            verify_packet( self, pkt, out_port )
+            verify_no_other_packets( self )
+
         finally:
             delete_all_flows( self.controller )
             delete_groups( self.controller, Groups )
             delete_all_groups( self.controller )
 
+
 class Untagged( base_tests.SimpleDataPlane ):
     """
         Verify VLAN filtering table does not require OFPVID_PRESENT bit to be 0.
diff --git a/src/python/oftest/testutils.py b/src/python/oftest/testutils.py
index cd501ac..5e56589 100755
--- a/src/python/oftest/testutils.py
+++ b/src/python/oftest/testutils.py
@@ -73,6 +73,17 @@
         ctrl.message_send(msg)
         do_barrier(ctrl)
 
+def delete_group(ctrl, group_id):
+    """
+    Delete a single group
+    @param ctrl The controller object for the test
+    :param group_id
+    """
+    logging.info("Deleting a single group with groupId:" + str(group_id))
+    msg = ofp.message.group_delete(group_id=group_id)
+    ctrl.message_send(msg)
+    do_barrier(ctrl)
+
 def required_wildcards(parent):
     w = test_param_get('required_wildcards', default='default')
     if w == 'l3-l4':