[ 3195 ] Remove duplicate method add_port

Update to the device operations as follows:
1) Add a few test scenarios to test the device state transitions
2) Check whether there is a callback before removing it from the queue
3) Fix a port reference issue when disabling an ONU
4) Update the CLI to handle exceptions from the grpc server

Change-Id: Ic7f41e80279f41d9a4575da5dd49de11294a22d5
diff --git a/cli/main.py b/cli/main.py
index 05cee3b..f125634 100755
--- a/cli/main.py
+++ b/cli/main.py
@@ -285,23 +285,26 @@
         """
         device_id = line or self.default_device_id
         self.poutput('enabling {}'.format(device_id))
-        stub = voltha_pb2.VolthaLocalServiceStub(self.get_channel())
-        stub.EnableDevice(voltha_pb2.ID(id=device_id))
+        try:
+            stub = voltha_pb2.VolthaLocalServiceStub(self.get_channel())
+            stub.EnableDevice(voltha_pb2.ID(id=device_id))
 
-        while True:
-            device = stub.GetDevice(voltha_pb2.ID(id=device_id))
-            # If this is an OLT then acquire logical device id
-            if device.oper_status == voltha_pb2.OperStatus.ACTIVE:
-                if device.type.endswith('_olt'):
-                    assert device.parent_id
-                    self.default_logical_device_id = device.parent_id
-                    self.poutput('success (logical device id = {})'.format(
-                        self.default_logical_device_id))
-                else:
-                    self.poutput('success (device id = {})'.format(device.id))
-                break
-            self.poutput('waiting for device to be enabled...')
-            sleep(.5)
+            while True:
+                device = stub.GetDevice(voltha_pb2.ID(id=device_id))
+                # If this is an OLT then acquire logical device id
+                if device.oper_status == voltha_pb2.OperStatus.ACTIVE:
+                    if device.type.endswith('_olt'):
+                        assert device.parent_id
+                        self.default_logical_device_id = device.parent_id
+                        self.poutput('success (logical device id = {})'.format(
+                            self.default_logical_device_id))
+                    else:
+                        self.poutput('success (device id = {})'.format(device.id))
+                    break
+                self.poutput('waiting for device to be enabled...')
+                sleep(.5)
+        except Exception, e:
+            self.poutput('Error enabling {}.  Error:{}'.format(device_id, e))
 
     complete_activate_olt = complete_device
 
@@ -311,9 +314,12 @@
         """
         device_id = line or self.default_device_id
         self.poutput('rebooting {}'.format(device_id))
-        stub = voltha_pb2.VolthaLocalServiceStub(self.get_channel())
-        stub.RebootDevice(voltha_pb2.ID(id=device_id))
-        self.poutput('rebooted {}'.format(device_id))
+        try:
+            stub = voltha_pb2.VolthaLocalServiceStub(self.get_channel())
+            stub.RebootDevice(voltha_pb2.ID(id=device_id))
+            self.poutput('rebooted {}'.format(device_id))
+        except Exception, e:
+            self.poutput('Error rebooting {}.  Error:{}'.format(device_id, e))
 
     def do_delete(self, line):
         """
@@ -321,9 +327,12 @@
         """
         device_id = line or self.default_device_id
         self.poutput('deleting {}'.format(device_id))
-        stub = voltha_pb2.VolthaLocalServiceStub(self.get_channel())
-        stub.DeleteDevice(voltha_pb2.ID(id=device_id))
-        self.poutput('deleted {}'.format(device_id))
+        try:
+            stub = voltha_pb2.VolthaLocalServiceStub(self.get_channel())
+            stub.DeleteDevice(voltha_pb2.ID(id=device_id))
+            self.poutput('deleted {}'.format(device_id))
+        except Exception, e:
+            self.poutput('Error deleting {}.  Error:{}'.format(device_id, e))
 
     def do_disable(self, line):
         """
@@ -331,20 +340,23 @@
         """
         device_id = line
         self.poutput('disabling {}'.format(device_id))
-        stub = voltha_pb2.VolthaLocalServiceStub(self.get_channel())
-        stub.DisableDevice(voltha_pb2.ID(id=device_id))
+        try:
+            stub = voltha_pb2.VolthaLocalServiceStub(self.get_channel())
+            stub.DisableDevice(voltha_pb2.ID(id=device_id))
 
-        # Do device query and verify that the device admin status is
-        # DISABLED and Operational Status is unknown
-        device = stub.GetDevice(voltha_pb2.ID(id=device_id))
-        if device.oper_status == voltha_pb2.OperStatus.UNKNOWN and \
-                        device.admin_state == voltha_pb2.AdminState.DISABLED:
-            self.poutput('disabled successfully {}'.format(device_id))
-        else:
-            self.poutput('disabling failed {}.  Admin State:{} '
-                         'Operation State: {}'.format(device_id,
-                                                      device.admin_state,
-                                                      device.oper_status))
+            # Do device query and verify that the device admin status is
+            # DISABLED and Operational Status is unknown
+            device = stub.GetDevice(voltha_pb2.ID(id=device_id))
+            if device.oper_status == voltha_pb2.OperStatus.UNKNOWN and \
+                            device.admin_state == voltha_pb2.AdminState.DISABLED:
+                self.poutput('disabled successfully {}'.format(device_id))
+            else:
+                self.poutput('disabling failed {}.  Admin State:{} '
+                             'Operation State: {}'.format(device_id,
+                                                          device.admin_state,
+                                                          device.oper_status))
+        except Exception, e:
+            self.poutput('Error disabling {}.  Error:{}'.format(device_id, e))
 
     def do_test(self, line):
         """Enter test mode, which makes a bunch on new commands available"""
diff --git a/cli/utils.py b/cli/utils.py
index f4111e3..5ca0da5 100644
--- a/cli/utils.py
+++ b/cli/utils.py
@@ -153,7 +153,7 @@
     table.print_table(header, printfn)
 
     # see CORD-817 (https://jira.opencord.org/browse/CORD-817)
-    assert len(groups) == 0
+    # assert len(groups) == 0
 
 
 def dict2line(d):
diff --git a/tests/itests/voltha/test_device_state_changes.py b/tests/itests/voltha/test_device_state_changes.py
new file mode 100644
index 0000000..9b3bc76
--- /dev/null
+++ b/tests/itests/voltha/test_device_state_changes.py
@@ -0,0 +1,350 @@
+from time import time, sleep
+
+from google.protobuf.json_format import MessageToDict
+
+from voltha.core.flow_decomposer import *
+from voltha.protos.device_pb2 import Device
+from voltha.protos.common_pb2 import AdminState, OperStatus
+from voltha.protos import openflow_13_pb2 as ofp
+from tests.itests.voltha.rest_base import RestBase
+from common.utils.consulhelpers import get_endpoint_from_consul
+
+LOCAL_CONSUL = "localhost:8500"
+
+
+class TestDeviceStateChangeSequence(RestBase):
+    """
+    The prerequisite for this test are:
+     1. voltha ensemble is running
+          docker-compose -f compose/docker-compose-system-test.yml up -d
+     2. ponsim olt is running with 1 OLT and 4 ONUs
+          sudo -s
+          . ./env.sh
+          ./ponsim/main.py -v -o 4
+    """
+
+    # Retrieve details of the REST entry point
+    rest_endpoint = get_endpoint_from_consul(LOCAL_CONSUL, 'chameleon-rest')
+
+    # Construct the base_url
+    base_url = 'http://' + rest_endpoint
+
+    def wait_till(self, msg, predicate, interval=0.1, timeout=5.0):
+        deadline = time() + timeout
+        while time() < deadline:
+            if predicate():
+                return
+            sleep(interval)
+        self.fail('Timed out while waiting for condition: {}'.format(msg))
+
+    def test_device_state_changes_scenarios(self):
+
+        self.verify_prerequisites()
+        # Test basic scenario
+
+        self.basic_scenario()
+        self.failure_scenario()
+
+    def basic_scenario(self):
+        """
+        Test the enable -> disable -> enable -> disable -> delete for OLT
+        and ONU.
+        """
+        self.assert_no_device_present()
+        olt_id = self.add_olt_device()
+        self.verify_device_preprovisioned_state(olt_id)
+        self.enable_device(olt_id)
+        ldev_id = self.wait_for_logical_device(olt_id)
+        onu_ids = self.wait_for_onu_discovery(olt_id)
+        self.verify_logical_ports(ldev_id, 5)
+        self.simulate_eapol_flow_install(ldev_id, olt_id, onu_ids)
+        self.verify_olt_eapol_flow(olt_id)
+        olt_ids, onu_ids = self.get_devices()
+        self.disable_device(onu_ids[0])
+        self.verify_logical_ports(ldev_id, 4)
+        self.enable_device(onu_ids[0])
+        self.verify_logical_ports(ldev_id, 5)
+        self.simulate_eapol_flow_install(ldev_id, olt_id, onu_ids)
+        self.verify_olt_eapol_flow(olt_id)
+        self.disable_device(olt_ids[0])
+        self.assert_all_onus_state(olt_ids[0], 'DISABLED', 'UNKNOWN')
+        self.assert_no_logical_device()
+        self.enable_device(olt_ids[0])
+        self.assert_all_onus_state(olt_ids[0], 'ENABLED', 'ACTIVE')
+        self.wait_for_logical_device(olt_ids[0])
+        self.simulate_eapol_flow_install(ldev_id, olt_id, onu_ids)
+        self.verify_olt_eapol_flow(olt_id)
+        self.disable_device(onu_ids[0])
+        self.delete_device(onu_ids[0])
+        self.verify_logical_ports(ldev_id, 4)
+        self.disable_device(olt_ids[0])
+        self.delete_device(olt_ids[0])
+        self.assert_no_device_present()
+
+    def failure_scenario(self):
+        self.assert_no_device_present()
+        olt_id = self.add_olt_device()
+        self.verify_device_preprovisioned_state(olt_id)
+        self.enable_device(olt_id)
+        ldev_id = self.wait_for_logical_device(olt_id)
+        onu_ids = self.wait_for_onu_discovery(olt_id)
+        self.verify_logical_ports(ldev_id, 5)
+        self.simulate_eapol_flow_install(ldev_id, olt_id, onu_ids)
+        self.verify_olt_eapol_flow(olt_id)
+        self.delete_device_incorrect_state(olt_id)
+        self.delete_device_incorrect_state(onu_ids[0])
+        unknown_id = '9999999999'
+        self.enable_unknown_device(unknown_id)
+        self.disable_unknown_device(unknown_id)
+        self.delete_unknown_device(unknown_id)
+        latest_olt_ids, latest_onu_ids = self.get_devices()
+        self.assertEqual(len(latest_olt_ids), 1)
+        self.assertEqual(len(latest_onu_ids), 4)
+        self.verify_logical_ports(ldev_id, 5)
+        self.simulate_eapol_flow_install(ldev_id, olt_id, onu_ids)
+        # Cleanup
+        self.disable_device(olt_id)
+        self.delete_device(olt_id)
+        self.assert_no_device_present()
+
+    def verify_prerequisites(self):
+        # all we care is that Voltha is available via REST using the base uri
+        self.get('/api/v1')
+
+    def get_devices(self):
+        devices = self.get('/api/v1/devices')['items']
+        olt_ids = []
+        onu_ids = []
+        for d in devices:
+            if d['adapter'] == 'ponsim_olt':
+                olt_ids.append(d['id'])
+            elif d['adapter'] == 'ponsim_onu':
+                onu_ids.append(d['id'])
+            else:
+                onu_ids.append(d['id'])
+        return olt_ids, onu_ids
+
+    def add_olt_device(self):
+        device = Device(
+            type='ponsim_olt',
+            host_and_port='172.17.0.1:50060'
+        )
+        device = self.post('/api/v1/devices', MessageToDict(device),
+                           expected_code=200)
+        return device['id']
+
+    def verify_device_preprovisioned_state(self, olt_id):
+        # we also check that so far what we read back is same as what we get
+        # back on create
+        device = self.get('/api/v1/devices/{}'.format(olt_id))
+        self.assertNotEqual(device['id'], '')
+        self.assertEqual(device['adapter'], 'ponsim_olt')
+        self.assertEqual(device['admin_state'], 'PREPROVISIONED')
+        self.assertEqual(device['oper_status'], 'UNKNOWN')
+
+    def enable_device(self, olt_id):
+        path = '/api/v1/devices/{}'.format(olt_id)
+        self.post(path + '/enable', expected_code=200)
+        device = self.get(path)
+        self.assertEqual(device['admin_state'], 'ENABLED')
+
+        self.wait_till(
+            'admin state moves to ACTIVATING or ACTIVE',
+            lambda: self.get(path)['oper_status'] in ('ACTIVATING', 'ACTIVE'),
+            timeout=0.5)
+
+        # eventually, it shall move to active state and by then we shall have
+        # device details filled, connect_state set, and device ports created
+        self.wait_till(
+            'admin state ACTIVE',
+            lambda: self.get(path)['oper_status'] == 'ACTIVE',
+            timeout=0.5)
+        device = self.get(path)
+        self.assertEqual(device['connect_status'], 'REACHABLE')
+
+        ports = self.get(path + '/ports')['items']
+        self.assertEqual(len(ports), 2)
+
+    def wait_for_logical_device(self, olt_id):
+        # we shall find the logical device id from the parent_id of the olt
+        # (root) device
+        device = self.get(
+            '/api/v1/devices/{}'.format(olt_id))
+        self.assertNotEqual(device['parent_id'], '')
+        logical_device = self.get(
+            '/api/v1/logical_devices/{}'.format(device['parent_id']))
+
+        # the logical device shall be linked back to the hard device,
+        # its ports too
+        self.assertEqual(logical_device['root_device_id'], device['id'])
+
+        logical_ports = self.get(
+            '/api/v1/logical_devices/{}/ports'.format(
+                logical_device['id'])
+        )['items']
+        self.assertGreaterEqual(len(logical_ports), 1)
+        logical_port = logical_ports[0]
+        self.assertEqual(logical_port['id'], 'nni')
+        self.assertEqual(logical_port['ofp_port']['name'], 'nni')
+        self.assertEqual(logical_port['ofp_port']['port_no'], 0)
+        self.assertEqual(logical_port['device_id'], device['id'])
+        self.assertEqual(logical_port['device_port_no'], 2)
+        return logical_device['id']
+
+    def find_onus(self, olt_id):
+        devices = self.get('/api/v1/devices')['items']
+        return [
+            d for d in devices
+            if d['parent_id'] == olt_id
+            ]
+
+    def wait_for_onu_discovery(self, olt_id):
+        # shortly after we shall see the discovery of four new onus, linked to
+        # the olt device
+        self.wait_till(
+            'find four ONUs linked to the olt device',
+            lambda: len(self.find_onus(olt_id)) >= 4,
+            2
+        )
+        # verify that they are properly set
+        onus = self.find_onus(olt_id)
+        for onu in onus:
+            self.assertEqual(onu['admin_state'], 'ENABLED')
+            self.assertEqual(onu['oper_status'], 'ACTIVE')
+
+        return [onu['id'] for onu in onus]
+
+    def assert_all_onus_state(self, olt_id, admin_state, oper_state):
+        # verify all onus are in a given state
+        onus = self.find_onus(olt_id)
+        for onu in onus:
+            self.assertEqual(onu['admin_state'], admin_state)
+            self.assertEqual(onu['oper_status'], oper_state)
+
+        return [onu['id'] for onu in onus]
+
+    def assert_onu_state(self, onu_id, admin_state, oper_state):
+        # Verify the onu states are correctly set
+        onu = self.get('/api/v1/devices/{}'.format(onu_id))
+        self.assertEqual(onu['admin_state'], admin_state)
+        self.assertEqual(onu['oper_status'], oper_state)
+
+    def verify_logical_ports(self, ldev_id, num_ports):
+
+        # at this point we shall see num_ports logical ports on the
+        # logical device
+        logical_ports = self.get(
+            '/api/v1/logical_devices/{}/ports'.format(ldev_id)
+        )['items']
+        self.assertGreaterEqual(len(logical_ports), num_ports)
+
+        # verify that all logical ports are LIVE (state=4)
+        for lport in logical_ports:
+            self.assertEqual(lport['ofp_port']['state'], 4)
+
+    def simulate_eapol_flow_install(self, ldev_id, olt_id, onu_ids):
+
+        # emulate the flow mod requests that shall arrive from the SDN
+        # controller, one for each ONU
+        lports = self.get(
+            '/api/v1/logical_devices/{}/ports'.format(ldev_id)
+        )['items']
+
+        # device_id -> logical port map, which we will use to construct
+        # our flows
+        lport_map = dict((lp['device_id'], lp) for lp in lports)
+        for onu_id in onu_ids:
+            # if eth_type == 0x888e => send to controller
+            _in_port = lport_map[onu_id]['ofp_port']['port_no']
+            req = ofp.FlowTableUpdate(
+                id='ponsim1',
+                flow_mod=mk_simple_flow_mod(
+                    match_fields=[
+                        in_port(_in_port),
+                        vlan_vid(ofp.OFPVID_PRESENT | 0),
+                        eth_type(0x888e)],
+                    actions=[
+                        output(ofp.OFPP_CONTROLLER)
+                    ],
+                    priority=1000
+                )
+            )
+            res = self.post('/api/v1/logical_devices/{}/flows'.format(ldev_id),
+                            MessageToDict(req,
+                                          preserving_proto_field_name=True),
+                            expected_code=200)
+
+        # for sanity, verify that flows are in flow table of logical device
+        flows = self.get(
+            '/api/v1/logical_devices/{}/flows'.format(ldev_id))['items']
+        self.assertGreaterEqual(len(flows), 4)
+
+    def verify_olt_eapol_flow(self, olt_id):
+        # olt shall have two flow rules, one is the default and the
+        # second is the result of eapol forwarding with rule:
+        # if eth_type == 0x888e => push vlan(1000); out_port=nni_port
+        flows = self.get('/api/v1/devices/{}/flows'.format(olt_id))['items']
+        self.assertEqual(len(flows), 2)
+        flow = flows[1]
+        self.assertEqual(flow['table_id'], 0)
+        self.assertEqual(flow['priority'], 1000)
+
+        # TODO refine this
+        # self.assertEqual(flow['match'], {})
+        # self.assertEqual(flow['instructions'], [])
+
+    def disable_device(self, id):
+        path = '/api/v1/devices/{}'.format(id)
+        self.post(path + '/disable', expected_code=200)
+        device = self.get(path)
+        self.assertEqual(device['admin_state'], 'DISABLED')
+
+        self.wait_till(
+            'operational state moves to UNKNOWN',
+            lambda: self.get(path)['oper_status'] == 'UNKNOWN',
+            timeout=0.5)
+
+        # eventually, the connect_state should be UNREACHABLE
+        self.wait_till(
+            'connest status UNREACHABLE',
+            lambda: self.get(path)['connect_status'] == 'UNREACHABLE',
+            timeout=0.5)
+
+        # Device's ports should be INACTIVE
+        ports = self.get(path + '/ports')['items']
+        self.assertEqual(len(ports), 2)
+        for p in ports:
+            self.assertEqual(p['admin_state'], 'DISABLED')
+            self.assertEqual(p['oper_status'], 'UNKNOWN')
+
+    def delete_device(self, id):
+        path = '/api/v1/devices/{}'.format(id)
+        self.delete(path + '/delete', expected_code=200)
+        device = self.get(path, expected_code=404)
+        self.assertIsNone(device)
+
+    def assert_no_device_present(self):
+        path = '/api/v1/devices'
+        devices = self.get(path)['items']
+        self.assertEqual(devices, [])
+
+    def assert_no_logical_device(self):
+        path = '/api/v1/logical_devices'
+        ld = self.get(path)['items']
+        self.assertEqual(ld, [])
+
+    def delete_device_incorrect_state(self, id):
+        path = '/api/v1/devices/{}'.format(id)
+        self.delete(path + '/delete', expected_code=400)
+
+    def enable_unknown_device(self, id):
+        path = '/api/v1/devices/{}'.format(id)
+        self.post(path + '/enable', expected_code=404)
+
+    def disable_unknown_device(self, id):
+        path = '/api/v1/devices/{}'.format(id)
+        self.post(path + '/disable', expected_code=404)
+
+    def delete_unknown_device(self, id):
+        path = '/api/v1/devices/{}'.format(id)
+        self.delete(path + '/delete', expected_code=404)
diff --git a/voltha/adapters/ponsim_olt/ponsim_olt.py b/voltha/adapters/ponsim_olt/ponsim_olt.py
index 3fed5c4..aecac9f 100644
--- a/voltha/adapters/ponsim_olt/ponsim_olt.py
+++ b/voltha/adapters/ponsim_olt/ponsim_olt.py
@@ -185,6 +185,7 @@
         self.io_port = None
         self.logical_device_id = None
         self.nni_port = None
+        self.ofp_port_no = None
         self.interface = registry('main').get_args().interface
 
     def __del__(self):
@@ -257,6 +258,7 @@
         )
         ld_initialized = self.adapter_agent.create_logical_device(ld)
         cap = OFPPF_1GB_FD | OFPPF_FIBER
+        self.ofp_port_no = info.nni_port
         self.adapter_agent.add_logical_port(ld_initialized.id, LogicalPort(
             id='nni',
             ofp_port=ofp_port(
@@ -401,12 +403,12 @@
         # Disable all child devices first
         self.adapter_agent.disable_all_child_devices(self.device_id)
 
-        # # Remove all child devices
-        # self.adapter_agent.remove_all_child_devices(self.device_id)
-
         # Remove the peer references from this device
         self.adapter_agent.delete_all_peer_references(self.device_id)
 
+        # Set all ports to disabled
+        self.adapter_agent.disable_all_ports(self.device_id)
+
         # close the frameio port
         registry('frameio').close_port(self.io_port)
 
@@ -427,6 +429,9 @@
         device.connect_status = ConnectStatus.REACHABLE
         self.adapter_agent.update_device(device)
 
+        # Set all ports to enabled
+        self.adapter_agent.enable_all_ports(self.device_id)
+
         ld = LogicalDevice(
             # not setting id and datapth_id will let the adapter agent pick id
             desc=ofp_desc(
@@ -453,11 +458,9 @@
         self.adapter_agent.add_logical_port(ld_initialized.id, LogicalPort(
             id='nni',
             ofp_port=ofp_port(
-                # port_no=info.nni_port,
-                # hw_addr=mac_str_to_tuple('00:00:00:00:00:%02x' % info.nni_port),
-                port_no=self.nni_port.port_no,
+                port_no=self.ofp_port_no,
                 hw_addr=mac_str_to_tuple(
-                    '00:00:00:00:00:%02x' % self.nni_port.port_no),
+                    '00:00:00:00:00:%02x' % self.ofp_port_no),
                 name='nni',
                 config=0,
                 state=OFPPS_LIVE,
@@ -481,7 +484,6 @@
         # Reenable all child devices
         self.adapter_agent.reenable_all_child_devices(device.id)
 
-
         # finally, open the frameio port to receive in-band packet_in messages
         self.log.info('registering-frameio')
         self.io_port = registry('frameio').open_port(
diff --git a/voltha/adapters/ponsim_onu/ponsim_onu.py b/voltha/adapters/ponsim_onu/ponsim_onu.py
index 493145c..a131052 100644
--- a/voltha/adapters/ponsim_onu/ponsim_onu.py
+++ b/voltha/adapters/ponsim_onu/ponsim_onu.py
@@ -45,7 +45,6 @@
 
 @implementer(IAdapterInterface)
 class PonSimOnuAdapter(object):
-
     name = 'ponsim_onu'
 
     supported_device_types = [
@@ -123,7 +122,7 @@
 
     def update_flows_bulk(self, device, flows, groups):
         log.info('bulk-flow-update', device_id=device.id,
-                  flows=flows, groups=groups)
+                 flows=flows, groups=groups)
         assert len(groups.items) == 0
         handler = self.devices_handlers[device.id]
         return handler.update_flow_table(flows.items)
@@ -146,7 +145,6 @@
 
 
 class PonSimOnuHandler(object):
-
     def __init__(self, adapter, device_id):
         self.adapter = adapter
         self.adapter_agent = adapter.adapter_agent
@@ -177,7 +175,7 @@
         # populate device info
         device.root = True
         device.vendor = 'ponsim'
-        device.model ='n/a'
+        device.model = 'n/a'
         device.connect_status = ConnectStatus.REACHABLE
         self.adapter_agent.update_device(device)
 
@@ -250,7 +248,6 @@
 
         yield self.incoming_messages.get()
 
-
     @inlineCallbacks
     def reboot(self):
         self.log.info('rebooting', device_id=self.device_id)
@@ -265,7 +262,7 @@
         self.adapter_agent.update_device(device)
 
         # Sleep 10 secs, simulating a reboot
-        #TODO: send alert and clear alert after the reboot
+        # TODO: send alert and clear alert after the reboot
         yield asleep(10)
 
         # Change the operational status back to its previous state.  With a
@@ -278,7 +275,6 @@
         self.adapter_agent.update_device(device)
         self.log.info('rebooted', device_id=self.device_id)
 
-
     def disable(self):
         self.log.info('disabling', device_id=self.device_id)
 
@@ -301,7 +297,8 @@
         port_no = device.proxy_address.channel_id
         port_id = 'uni-{}'.format(port_no)
         try:
-            port = self.adapter_agent.get_logical_port(logical_device_id, port_id)
+            port = self.adapter_agent.get_logical_port(logical_device_id,
+                                                       port_id)
             self.adapter_agent.delete_logical_port(logical_device_id, port)
         except KeyError:
             self.log.info('logical-port-not-found', device_id=self.device_id,
@@ -316,7 +313,8 @@
         # yield self.adapter_agent.update_logical_port(logical_device_id,
         #                                             port)
         # Unregister for proxied message
-        self.adapter_agent.unregister_for_proxied_messages(device.proxy_address)
+        self.adapter_agent.unregister_for_proxied_messages(
+            device.proxy_address)
 
         # TODO:
         # 1) Remove all flows from the device
@@ -324,7 +322,6 @@
 
         self.log.info('disabled', device_id=device.id)
 
-
     def reenable(self):
         self.log.info('re-enabling', device_id=self.device_id)
 
@@ -342,7 +339,11 @@
         self.adapter_agent.register_for_proxied_messages(device.proxy_address)
 
         # Re-enable the ports on that device
-        self.adapter_agent.reenable_all_ports(self.device_id)
+        self.adapter_agent.enable_all_ports(self.device_id)
+
+        # Add the pon port reference to the parent
+        self.adapter_agent.add_port_reference_to_parent(device.id,
+                                                        self.pon_port)
 
         # Update the connect status to REACHABLE
         device.connect_status = ConnectStatus.REACHABLE
@@ -378,7 +379,6 @@
 
         self.log.info('re-enabled', device_id=device.id)
 
-
     def delete(self):
         self.log.info('deleting', device_id=self.device_id)
 
diff --git a/voltha/core/adapter_agent.py b/voltha/core/adapter_agent.py
index d6e32af..e926d9d 100644
--- a/voltha/core/adapter_agent.py
+++ b/voltha/core/adapter_agent.py
@@ -221,9 +221,7 @@
     def update_adapter_pm_config(self, device, device_pm_config):
         self.adapter.update_pm_config(device, device_pm_config)
 
-    def add_port(self, device_id, port):
-        assert isinstance(port, Port)
-
+    def _add_peer_reference(self, device_id, port):
         # for referential integrity, add/augment references
         port.device_id = device_id
         me_as_peer = Port.PeerPort(device_id=device_id, port_no=port.port_no)
@@ -236,13 +234,30 @@
                 new.CopyFrom(me_as_peer)
             self.root_proxy.update(peer_port_path, peer_port)
 
+    def _del_peer_reference(self, device_id, port):
+        me_as_peer = Port.PeerPort(device_id=device_id, port_no=port.port_no)
+        for peer in port.peers:
+            peer_port_path = '/devices/{}/ports/{}'.format(
+                peer.device_id, peer.port_no)
+            peer_port = self.root_proxy.get(peer_port_path)
+            if me_as_peer in peer_port.peers:
+                peer_port.peers.remove(me_as_peer)
+            self.root_proxy.update(peer_port_path, peer_port)
+
+    def add_port(self, device_id, port):
+        assert isinstance(port, Port)
+
+        # for referential integrity, add/augment references
+        self._add_peer_reference(device_id, port)
+
+        # Add port
         self._make_up_to_date('/devices/{}/ports'.format(device_id),
                               port.port_no, port)
 
     def disable_all_ports(self, device_id):
         """
         Disable all ports on that device, i.e. change the admin status to
-        disable and operational status to UNKNOWN
+        disable and operational status to UNKNOWN.
         :param device_id: device id
         :return: None
         """
@@ -255,7 +270,8 @@
             self._make_up_to_date('/devices/{}/ports'.format(device_id),
                                   port.port_no, port)
 
-    def reenable_all_ports(self, device_id):
+
+    def enable_all_ports(self, device_id):
         """
         Re-enable all ports on that device, i.e. change the admin status to
         enabled and operational status to ACTIVE
@@ -293,16 +309,18 @@
         """
         assert isinstance(port, Port)
         self.log.info('delete-port-reference', device_id=device_id, port=port)
+        self._del_peer_reference(device_id, port)
 
-        # for referential integrity, remove references
-        me_as_peer = Port.PeerPort(device_id=device_id, port_no=port.port_no)
-        for peer in port.peers:
-            peer_port_path = '/devices/{}/ports/{}'.format(
-                peer.device_id, peer.port_no)
-            peer_port = self.root_proxy.get(peer_port_path)
-            if me_as_peer in peer_port.peers:
-                peer_port.peers.remove(me_as_peer)
-            self.root_proxy.update(peer_port_path, peer_port)
+    def add_port_reference_to_parent(self, device_id, port):
+        """
+        Add the port reference to the parent device
+        :param device_id: id of device containing the port
+        :param port: port to add
+        :return: None
+        """
+        assert isinstance(port, Port)
+        self.log.info('add-port-reference', device_id=device_id, port=port)
+        self._add_peer_reference(device_id, port)
 
     def _find_first_available_id(self):
         logical_devices = self.root_proxy.get('/logical_devices')
diff --git a/voltha/core/config/config_node.py b/voltha/core/config/config_node.py
index 56518bb..f1c8fc0 100644
--- a/voltha/core/config/config_node.py
+++ b/voltha/core/config/config_node.py
@@ -309,7 +309,7 @@
                     self._mk_event_bus().advertise,
                     change_type,
                     data,
-                    hash=rev.hash,
+                    hash=rev.hash
                 )
 
     # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ add operation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/voltha/core/config/config_proxy.py b/voltha/core/config/config_proxy.py
index fa6ec57..57d8150 100644
--- a/voltha/core/config/config_proxy.py
+++ b/voltha/core/config/config_proxy.py
@@ -135,7 +135,8 @@
 
     def unregister_callback(self, callback_type, callback, *args, **kw):
         lst = self._callbacks.setdefault(callback_type, [])
-        lst.remove((callback, args, kw))
+        if (callback, args, kw) in lst:
+            lst.remove((callback, args, kw))
 
     # ~~~~~~~~~~~~~~~~~~~~~ Callback dispatch ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/voltha/core/local_handler.py b/voltha/core/local_handler.py
index 7360ca8..4a0fde8 100644
--- a/voltha/core/local_handler.py
+++ b/voltha/core/local_handler.py
@@ -262,7 +262,7 @@
                 'in admin state \'{}\''.format(device.admin_state)
 
         except AssertionError, e:
-            context.set_details(e.msg)
+            context.set_details(e.message)
             context.set_code(StatusCode.INVALID_ARGUMENT)
             return Device()
 
@@ -287,7 +287,7 @@
             context.set_details(
                 'Malformed device id \'{}\''.format(request.id))
             context.set_code(StatusCode.INVALID_ARGUMENT)
-            return Device()
+            return Empty()
 
         try:
             path = '/devices/{}'.format(request.id)
@@ -300,9 +300,8 @@
             self.root.update(path, device, strict=True)
 
         except AssertionError, e:
-            context.set_details(e.msg)
+            context.set_details(e.message)
             context.set_code(StatusCode.INVALID_ARGUMENT)
-            return Device()
 
         except KeyError:
             context.set_details(
@@ -319,8 +318,7 @@
             context.set_details(
                 'Malformed device id \'{}\''.format(request.id))
             context.set_code(StatusCode.INVALID_ARGUMENT)
-            return Device()
-
+            return Empty()
         try:
             path = '/devices/{}'.format(request.id)
             device = self.root.get(path)
@@ -331,9 +329,8 @@
             self.root.update(path, device, strict=True)
 
         except AssertionError, e:
-            context.set_details(e.msg)
+            context.set_details(e.message)
             context.set_code(StatusCode.INVALID_ARGUMENT)
-            return Device()
 
         except KeyError:
             context.set_details(
@@ -350,7 +347,7 @@
             context.set_details(
                 'Malformed device id \'{}\''.format(request.id))
             context.set_code(StatusCode.INVALID_ARGUMENT)
-            return Device()
+            return Empty()
 
         try:
             path = '/devices/{}'.format(request.id)
@@ -359,11 +356,6 @@
             agent = self.core.get_device_agent(device.id)
             agent.reboot_device(device)
 
-        except AssertionError, e:
-            context.set_details(e.msg)
-            context.set_code(StatusCode.INVALID_ARGUMENT)
-            return Device()
-
         except KeyError:
             context.set_details(
                 'Device \'{}\' not found'.format(request.id))
@@ -379,7 +371,7 @@
             context.set_details(
                 'Malformed device id \'{}\''.format(request.id))
             context.set_code(StatusCode.INVALID_ARGUMENT)
-            return Device()
+            return Empty()
 
         try:
             path = '/devices/{}'.format(request.id)
@@ -391,9 +383,8 @@
             self.root.remove(path)
 
         except AssertionError, e:
-            context.set_details(e.msg)
+            context.set_details(e.message)
             context.set_code(StatusCode.INVALID_ARGUMENT)
-            return Device()
 
         except KeyError:
             context.set_details(
@@ -432,9 +423,10 @@
             return PmConfigs()
 
         try:
-            pm_configs = self.root.get('/devices/{}/pm_configs'.format(request.id))
+            pm_configs = self.root.get(
+                '/devices/{}/pm_configs'.format(request.id))
             pm_configs.id = request.id
-            log.info('device-for-pms',pm_configs=pm_configs)
+            log.info('device-for-pms', pm_configs=pm_configs)
             return pm_configs
         except KeyError:
             context.set_details(
diff --git a/voltha/core/logical_device_agent.py b/voltha/core/logical_device_agent.py
index 1ecd032..dc5d1c0 100644
--- a/voltha/core/logical_device_agent.py
+++ b/voltha/core/logical_device_agent.py
@@ -85,17 +85,21 @@
 
     def stop(self):
         self.log.debug('stopping')
-        self.flows_proxy.unregister_callback(
-            CallbackType.POST_UPDATE, self._flow_table_updated)
-        self.groups_proxy.unregister_callback(
-            CallbackType.POST_UPDATE, self._group_table_updated)
-        self.self_proxy.unregister_callback(
-            CallbackType.POST_ADD, self._port_list_updated)
-        self.self_proxy.unregister_callback(
-            CallbackType.POST_REMOVE, self._port_list_updated)
+        try:
+            self.flows_proxy.unregister_callback(
+                CallbackType.POST_UPDATE, self._flow_table_updated)
+            self.groups_proxy.unregister_callback(
+                CallbackType.POST_UPDATE, self._group_table_updated)
+            self.self_proxy.unregister_callback(
+                CallbackType.POST_ADD, self._port_list_updated)
+            self.self_proxy.unregister_callback(
+                CallbackType.POST_REMOVE, self._port_list_updated)
 
-        # Remove subscription to the event bus
-        self.event_bus.unsubscribe(self.packet_in_subscription)
+            # Remove subscription to the event bus
+            self.event_bus.unsubscribe(self.packet_in_subscription)
+        except Exception, e:
+            self.log.info('stop-exception', e=e)
+
         self.log.info('stopped')
 
     def announce_flows_deleted(self, flows):
@@ -115,15 +119,6 @@
     def signal_group_mod_error(self, code, group_mod):
         pass  # TODO
 
-    def delete_all_flows(self):
-        self.update_flow_table(mk_simple_flow_mod(
-            command=ofp.OFPFC_DELETE,
-            out_port=ofp.OFPP_ANY,
-            out_group=ofp.OFPG_ANY,
-            match_fields=[],
-            actions=[]
-        ))
-
     def update_flow_table(self, flow_mod):
 
         command = flow_mod.command