ADTRAN OLT: Fixes for ONU deletion after deprecation of xPON support

Change-Id: Ib162376e2739e908fc862154b8c7d30cc8026476
diff --git a/voltha/adapters/adtran_olt/adtran_device_handler.py b/voltha/adapters/adtran_olt/adtran_device_handler.py
index 7f3f225..049cd65 100644
--- a/voltha/adapters/adtran_olt/adtran_device_handler.py
+++ b/voltha/adapters/adtran_olt/adtran_device_handler.py
@@ -873,6 +873,7 @@
         # NOTE: Override this in your derived class for any device startup completion
         return defer.succeed('NOP')
 
+    @inlineCallbacks
     def disable(self):
         """
         This is called when a previously enabled device needs to be disabled based on a NBI call.
@@ -894,7 +895,6 @@
 
         # Drop registration for ONU detection
         # self.adapter_agent.unregister_for_onu_detect_state(self.device.id)
-
         # Suspend any active healthchecks / pings
 
         h, self.heartbeat = self.heartbeat, None
@@ -933,23 +933,18 @@
         # Wait for completion
 
         self.startup = defer.gatherResults(dl, consumeErrors=True)
+        yield self.startup
 
-        def _drop_netconf():
-            return self.netconf_client.close() if \
-                self.netconf_client is not None else defer.succeed('NOP')
+        if self.netconf_client:
+            self.netconf_client.close()
 
-        def _null_clients():
-            self._netconf_client = None
-            self._rest_client = None
-
-        # Shutdown communications with OLT
-        self.startup.addCallbacks(_drop_netconf, _null_clients)
-        self.startup.addCallbacks(_null_clients, _null_clients)
+        self._netconf_client = None
+        self._rest_client = None
 
         device.reason = ''
         self.adapter_agent.update_device(device)
         self.log.info('disabled', device_id=device.id)
-        return self.startup
+        returnValue(None)
 
     @inlineCallbacks
     def reenable(self, done_deferred=None):
@@ -982,19 +977,8 @@
 
         # Reenable any previously configured southbound ports
         for port in self.southbound_ports.itervalues():
-            self.log.debug('reenable-checking-pon-port', pon_id=port.pon_id)
-
-            # TODO: Need to implement this now that XPON is deprecated
-            #gpon_info = self.get_xpon_info(port.pon_id)         # SEBA
-            gpon_info = None
-            if gpon_info is not None and \
-                gpon_info['channel-terminations'] is not None and \
-                len(gpon_info['channel-terminations']) > 0:
-
-                cterms = gpon_info['channel-terminations']
-                if any(term.get('enabled') for term in cterms.itervalues()):
-                    self.log.info('reenable', pon_id=port.pon_id)
-                    port.enabled = True
+            self.log.debug('reenable-pon-port', pon_id=port.pon_id)
+            port.enabled = True
 
         # Flows should not exist on re-enable. They are re-pushed
         if len(self._evcs):
diff --git a/voltha/adapters/adtran_olt/adtran_olt.py b/voltha/adapters/adtran_olt/adtran_olt.py
index 6b765e6..32e271a 100644
--- a/voltha/adapters/adtran_olt/adtran_olt.py
+++ b/voltha/adapters/adtran_olt/adtran_olt.py
@@ -52,7 +52,7 @@
         self.descriptor = Adapter(
             id=self.name,
             vendor='ADTRAN, Inc.',
-            version='1.33',
+            version='1.34',
             config=AdapterConfig(log_level=LogLevel.INFO)
         )
         log.debug('adtran_olt.__init__', adapter_agent=adapter_agent)
diff --git a/voltha/adapters/adtran_olt/adtran_olt_handler.py b/voltha/adapters/adtran_olt/adtran_olt_handler.py
index 6ea7d0d..056e5be 100644
--- a/voltha/adapters/adtran_olt/adtran_olt_handler.py
+++ b/voltha/adapters/adtran_olt/adtran_olt_handler.py
@@ -755,6 +755,7 @@
                 exceptiontype = None
                 if pkt.type == FlowEntry.EtherType.EAPOL:
                     exceptiontype = 'eapol'
+                    ctag = self.utility_vlan
                 elif pkt.type == 2:
                     exceptiontype = 'igmp'
                 elif pkt.type == FlowEntry.EtherType.IPv4:
diff --git a/voltha/adapters/adtran_olt/flow/evc.py b/voltha/adapters/adtran_olt/flow/evc.py
index 2e1914c..6aa8990 100644
--- a/voltha/adapters/adtran_olt/flow/evc.py
+++ b/voltha/adapters/adtran_olt/flow/evc.py
@@ -268,8 +268,7 @@
             xml += '<enabled>{}</enabled>'.format('true' if self._enabled else 'false')
 
             if self._ce_vlan_preservation is not None:
-                xml += '<ce-vlan-preservation>{}</ce-vlan-preservation>'.\
-                    format('true' if self._ce_vlan_preservation else 'false')
+                xml += '<ce-vlan-preservation>{}</ce-vlan-preservation>'.format('false')
 
             if self._s_tag is not None:
                 xml += '<stag>{}</stag>'.format(self._s_tag)
diff --git a/voltha/adapters/adtran_olt/flow/evc_map.py b/voltha/adapters/adtran_olt/flow/evc_map.py
index 92da980..dc8cc2d 100644
--- a/voltha/adapters/adtran_olt/flow/evc_map.py
+++ b/voltha/adapters/adtran_olt/flow/evc_map.py
@@ -202,7 +202,7 @@
         return '</evc-map></evc-maps>'
 
     def get_evcmap_name(self, onu_id, gem_id):
-        return'{}.{}.{}.{}'.format(self.name, onu_id, self.pon_id, gem_id)
+        return'{}.{}.{}.{}'.format(self.name, self.pon_id, onu_id, gem_id)
 
     def _common_install_xml(self):
         xml = '<enabled>{}</enabled>'.format('true' if self._enabled else 'false')
diff --git a/voltha/adapters/adtran_olt/flow/flow_entry.py b/voltha/adapters/adtran_olt/flow/flow_entry.py
index 02b112e..04aa9eb 100644
--- a/voltha/adapters/adtran_olt/flow/flow_entry.py
+++ b/voltha/adapters/adtran_olt/flow/flow_entry.py
@@ -18,7 +18,7 @@
 from utility_evc import UtilityEVC
 import voltha.core.flow_decomposer as fd
 from voltha.core.flow_decomposer import *
-from voltha.protos.openflow_13_pb2 import OFPP_MAX, OFPP_CONTROLLER
+from voltha.protos.openflow_13_pb2 import OFPP_MAX, OFPP_CONTROLLER, OFPVID_PRESENT, OFPXMC_OPENFLOW_BASIC
 from twisted.internet.defer import returnValue, inlineCallbacks, gatherResults
 
 log = structlog.get_logger()
@@ -443,7 +443,7 @@
                     self._logical_port = self.in_port
 
             elif field.type == VLAN_VID:
-                if field.vlan_vid >= ofp.OFPVID_PRESENT + 4095:
+                if field.vlan_vid >= OFPVID_PRESENT + 4095:
                     self.vlan_id = None             # pre-ONOS v1.13.5 or old EAPOL Rule
                 else:
                     self.vlan_id = field.vlan_vid & 0xfff
@@ -479,22 +479,23 @@
                 self.udp_src = field.udp_src
 
             elif field.type == METADATA:
-                log.debug('*** field.type == METADATA', value=field.table_metadata)
+                if self._handler.is_nni_port(self.in_port):
+                    # Downstream flow
+                    log.debug('*** field.type == METADATA', value=field.table_metadata)
 
-                if 0xFFFFFFFF >= field.table_metadata > ofp.OFPVID_PRESENT + 4095:
-                    # Default flows for old-style controller flows
-                    self.inner_vid = None
+                    if 0xFFFFFFFF >= field.table_metadata > OFPVID_PRESENT + 4095:
+                        # Default flows for old-style controller flows
+                        self.inner_vid = None
 
-                elif field.table_metadata > 0xFFFFFFFF:
-                    # ONOS v1.13.5 or later. c-vid in upper 32-bits
-                    self.inner_vid = field.table_metadata >> 32
+                    elif field.table_metadata > 0xFFFFFFFF:
+                        # ONOS v1.13.5 or later. c-vid in upper 32-bits
+                        self.inner_vid = field.table_metadata >> 32
 
+                    log.debug('*** field.type == METADATA', value=field.table_metadata,
+                              inner_vid=self.inner_vid)
                 else:
-                    # Pre- ONOS v1.13.5
-                    self.inner_vid = field.table_metadata
-
-                log.debug('*** field.type == METADATA', value=field.table_metadata,
-                          inner_vid=self.inner_vid)
+                    # Upstream flow
+                    pass   # Not used upstream at this time
             else:
                 log.warn('unsupported-selection-field', type=field.type)
                 self._status_message = 'Unsupported field.type={}'.format(field.type)
@@ -519,7 +520,7 @@
 
             elif act.type == SET_FIELD:
                 log.debug('*** action.type == SET_FIELD', value=act.set_field.field)
-                assert (act.set_field.field.oxm_class == ofp.OFPXMC_OPENFLOW_BASIC)
+                assert (act.set_field.field.oxm_class == OFPXMC_OPENFLOW_BASIC)
                 field = act.set_field.field.ofb_field
 
                 if field.type == VLAN_VID:
diff --git a/voltha/adapters/adtran_olt/flow/flow_tables.py b/voltha/adapters/adtran_olt/flow/flow_tables.py
index aa90ee3..48e2e7e 100644
--- a/voltha/adapters/adtran_olt/flow/flow_tables.py
+++ b/voltha/adapters/adtran_olt/flow/flow_tables.py
@@ -62,7 +62,7 @@
 
     def remove(self, item):
         flow_id = item.flow_id if isinstance(item, FlowEntry) else item
-        return self._flow_table.pop(flow_id, default=None)
+        return self._flow_table.pop(flow_id, None)
 
     def clear_all(self):
         self._flow_table = dict()
diff --git a/voltha/adapters/adtran_olt/net/adtran_rest.py b/voltha/adapters/adtran_olt/net/adtran_rest.py
index a478d95..9020e82 100644
--- a/voltha/adapters/adtran_olt/net/adtran_rest.py
+++ b/voltha/adapters/adtran_olt/net/adtran_rest.py
@@ -54,13 +54,14 @@
     HTTP_NO_CONTENT = 204
     HTTP_RESET_CONTENT = 205
     HTTP_PARTIAL_CONTENT = 206
+    HTTP_NOT_FOUND = 404
 
     _valid_methods = {'GET', 'POST', 'PATCH', 'DELETE'}
     _valid_results = {'GET': [HTTP_OK, HTTP_NO_CONTENT],
                       'POST': [HTTP_OK, HTTP_CREATED, HTTP_NO_CONTENT],
                       'PUT': [HTTP_OK, HTTP_CREATED, HTTP_NO_CONTENT],
                       'PATCH': [HTTP_OK],
-                      'DELETE': [HTTP_OK, HTTP_ACCEPTED, HTTP_NO_CONTENT]
+                      'DELETE': [HTTP_OK, HTTP_ACCEPTED, HTTP_NO_CONTENT, HTTP_NOT_FOUND]
                       }
 
     for _method in _valid_methods:
@@ -162,7 +163,7 @@
                 log.error(message)
             raise RestInvalidResponseCode(message, url, response.code)
 
-        if response.code == self.HTTP_NO_CONTENT:
+        if response.code in {self.HTTP_NO_CONTENT, self.HTTP_NOT_FOUND}:
             returnValue(None)
 
         else:
diff --git a/voltha/adapters/adtran_olt/onu.py b/voltha/adapters/adtran_olt/onu.py
index 37be2d8..a8eb44c 100644
--- a/voltha/adapters/adtran_olt/onu.py
+++ b/voltha/adapters/adtran_olt/onu.py
@@ -396,7 +396,6 @@
 
     def stop(self):
         self._cancel_deferred()
-        self._sync_deferred = reactor.callLater(0, self._sync_hardware)
 
     def restart(self):
         if not self._valid:
diff --git a/voltha/adapters/adtran_olt/pon_port.py b/voltha/adapters/adtran_olt/pon_port.py
index f6cef70..6a76fd9 100644
--- a/voltha/adapters/adtran_olt/pon_port.py
+++ b/voltha/adapters/adtran_olt/pon_port.py
@@ -92,7 +92,6 @@
         self._v_ont_anis = {}             # Name -> dict
         self._ont_anis = {}               # Name -> dict
         self._tconts = {}                 # Name -> dict
-        self._traffic_descriptors = {}    # Name -> dict
         self._gem_ports = {}              # Name -> dict
 
     def __str__(self):
@@ -385,6 +384,7 @@
         super(PonPort, self).finish_startup()
         returnValue('Enabled')
 
+    @inlineCallbacks
     def finish_stop(self):
         # Remove all existing ONUs. They will need to be re-discovered
         dl = []
@@ -397,7 +397,8 @@
                 self.log.exception('onu-cleanup', onu_id=onu_id, e=e)
 
         dl.append(self._set_pon_config("enabled", False))
-        return defer.gatherResults(dl, consumeErrors=True)
+        results = yield defer.gatherResults(dl, consumeErrors=True)
+        returnValue(results)
 
     @inlineCallbacks
     def reset(self):
@@ -909,13 +910,30 @@
 
         if onu is not None:
             try:
+                # Remove from xPON config    (TODO: Deprecate this by refactoring ONU add steps)
+                name = 'customer-{}-{}'.format(self.pon_id, onu_id)
+                self._v_ont_anis.pop(name, None)
+                self._ont_anis.pop(name, None)
+
+                tcont_name = 'tcont-{}-{}-data'.format(self.pon_id, onu_id)
+                self._tconts.pop(tcont_name, None)
+
+                gem_ids = {gem_port.gem_id for gem_port in onu.gem_ports}
+                for gem_id in gem_ids:
+                    gem_port_name = 'gem-{}-{}-{}'.format(self.pon_id, onu_id, gem_id)
+                    self._gem_ports.pop(gem_port_name, None)
+
+            except Exception as e:
+                self.log.exception('onu-delete-cleanup', serial_number=onu.serial_number, e=e)
+
+            try:
+                # Remove from hardware
+                onu.delete()
+
                 # And removal from VOLTHA adapter agent
                 if not hw_only:
                     self._parent.delete_child_device(onu.proxy_address)
 
-                # Remove from hardware
-                onu.delete()
-
             except Exception as e:
                 self.log.exception('onu-delete', serial_number=onu.serial_number, e=e)
 
@@ -927,7 +945,7 @@
                 self.log.debug('onu-remove', serial_number=onu.serial_number, e=e)
 
         # Remove from LOS list if needed
-        if onu.id in self._active_los_alarms:
+        if onu is not None and onu.id in self._active_los_alarms:
             self._active_los_alarms.remove(onu.id)
 
     def add_mcast_gem_port(self, mcast_gem, vlan):
@@ -953,10 +971,6 @@
         return self._tconts
 
     @property
-    def traffic_descriptors(self):
-        return self._traffic_descriptors
-
-    @property
     def gem_ports(self):
         return self._gem_ports
 
diff --git a/voltha/adapters/adtran_olt/port.py b/voltha/adapters/adtran_olt/port.py
index 9509556..5dbb543 100644
--- a/voltha/adapters/adtran_olt/port.py
+++ b/voltha/adapters/adtran_olt/port.py
@@ -216,20 +216,18 @@
 
             self.state = AdtnPort.State.STOPPED
 
-            self.sync_deferred = reactor.callLater(self.sync_tick,
-                                                   self.sync_hardware)
-
             self.deferred = self.finish_stop()
             yield self.deferred
 
         except Exception as e:
             self.log.exception('stop-failed', e=e)
-            raise
 
         returnValue('Stopped')
 
+    @inlineCallbacks
     def finish_stop(self):
         pass   # Add to your derived class if needed
+        returnValue(None)
 
     def restart(self):
         if self.state == AdtnPort.State.RUNNING or self.state == AdtnPort.State.STOPPED:
diff --git a/voltha/adapters/adtran_onu/adtran_onu.py b/voltha/adapters/adtran_onu/adtran_onu.py
index 2d6b8a3..ecec76f 100755
--- a/voltha/adapters/adtran_onu/adtran_onu.py
+++ b/voltha/adapters/adtran_onu/adtran_onu.py
@@ -42,13 +42,16 @@
                                                device_handler_class=AdtranOnuHandler,
                                                name='adtran_onu',
                                                vendor='ADTRAN, Inc.',
-                                               version='1.22',
+                                               version='1.23',
                                                device_type='adtran_onu',
                                                vendor_id='ADTN',
                                                accepts_add_remove_flow_updates=False),  # TODO: Support flow-mods
         # Customize OpenOMCI for Adtran ONUs
         self.adtran_omci = deepcopy(OpenOmciAgentDefaults)
 
+        from voltha.extensions.omci.database.mib_db_dict import MibDbVolatileDict
+        self.adtran_omci['mib-synchronizer']['database'] = MibDbVolatileDict
+
         self.adtran_omci['mib-synchronizer']['state-machine'] = AdtnMibSynchronizer
         self.adtran_omci['mib-synchronizer']['tasks']['get-mds'] = AdtnGetMdsTask
         self.adtran_omci['mib-synchronizer']['tasks']['mib-audit'] = AdtnGetMdsTask
diff --git a/voltha/adapters/adtran_onu/uni_port.py b/voltha/adapters/adtran_onu/uni_port.py
index 5451e8a..0f2c10f 100644
--- a/voltha/adapters/adtran_onu/uni_port.py
+++ b/voltha/adapters/adtran_onu/uni_port.py
@@ -150,7 +150,9 @@
             # adapter_agent add_port also does an update of existing port
             self._handler.adapter_agent.add_port(self._handler.device_id,
                                                  self.get_port())
-        except Exception as e:
+        except KeyError:  # Expected exception during ONU disabling
+            pass
+        except Exception as e:  # Expected exception during ONU disabling
             self.log.exception('update-port', e=e)
 
     def get_port(self):