Support for PON,ONU,TCONT,GEM Port hardware reflow after unexpected power cycle
Change-Id: Ifea153d79530db357ec1da47a12bf37771b24b7b
diff --git a/voltha/adapters/adtran_olt/codec/olt_config.py b/voltha/adapters/adtran_olt/codec/olt_config.py
index 3db361c..5997682 100644
--- a/voltha/adapters/adtran_olt/codec/olt_config.py
+++ b/voltha/adapters/adtran_olt/codec/olt_config.py
@@ -78,7 +78,7 @@
@property
def enabled(self):
"""The desired state of the interface"""
- return self._packet.get('enabled', True)
+ return self._packet.get('enabled', False)
@property
def downstream_fec_enable(self):
diff --git a/voltha/adapters/adtran_olt/onu.py b/voltha/adapters/adtran_olt/onu.py
index d7fd0cf..8d43761 100644
--- a/voltha/adapters/adtran_olt/onu.py
+++ b/voltha/adapters/adtran_olt/onu.py
@@ -190,9 +190,13 @@
pass
@inlineCallbacks
- def create(self, tconts, gem_ports):
+ def create(self, tconts, gem_ports, reflow=False):
"""
- POST -> /restconf/data/gpon-olt-hw:olt/pon=<pon-id>/onus/onu ->
+ Create (or reflow) this ONU to hardware
+ :param tconts: (TCont) Current TCONT information
+ :param gem_ports: (GemPort) Current GEM Port configuration information
+ :param reflow: (boolean) Flag, if True, indicating if this is a reflow ONU
+ information after an unmanaged OLT hardware reboot
"""
self.log.debug('create')
self._cancel_deferred()
@@ -207,7 +211,7 @@
try:
results = yield self.olt.rest_client.request('POST', uri, data=data, name=name)
- except Exception as e:
+ except Exception as e: # TODO: Add breakpoint here during unexpected reboot test
self.log.exception('onu-create', e=e)
raise
@@ -215,17 +219,17 @@
for _, tcont in tconts.items():
try:
- results = yield self.add_tcont(tcont)
+ results = yield self.add_tcont(tcont, reflow=reflow)
except Exception as e:
self.log.exception('add-tcont', tcont=tcont, e=e)
for _, gem_port in gem_ports.items():
try:
- results = yield self.add_gem_port(gem_port)
+ results = yield self.add_gem_port(gem_port, reflow=reflow)
except Exception as e:
- self.log.exception('add-gem_port', gem_port=gem_port, e=e)
+ self.log.exception('add-gem-port', gem_port=gem_port, reflow=reflow, e=e)
self._sync_deferred = reactor.callLater(self._sync_tick, self._sync_hardware)
@@ -333,7 +337,7 @@
return [self.remove_tcont(alloc_id) for alloc_id in alloc_ids]
def sync_add_missing_tconts(alloc_ids):
- return [self.add_tcont(self._tconts[alloc_id], add_always=True) for alloc_id in alloc_ids]
+ return [self.add_tcont(self._tconts[alloc_id], reflow=True) for alloc_id in alloc_ids]
def sync_matching_tconts(hw_tconts):
from tcont import TrafficDescriptor
@@ -399,7 +403,8 @@
return [self.remove_gem_id(gem_id) for gem_id in gem_ids]
def sync_add_missing_gem_ports(gem_ids):
- return [self.add_gem_port(self._gem_ports[gem_id], add_always=True) for gem_id in gem_ids]
+ return [self.add_gem_port(self._gem_ports[gem_id], reflow=True)
+ for gem_id in gem_ids]
def sync_matching_gem_ports(hw_gem_ports):
dl = []
@@ -467,18 +472,18 @@
return frozenset(self._tconts.keys())
@inlineCallbacks
- def add_tcont(self, tcont, add_always=False):
+ def add_tcont(self, tcont, reflow=False):
"""
Creates/ a T-CONT with the given alloc-id
:param tcont: (TCont) Object that maintains the TCONT properties
- :param add_always: (boolean) If true, force add (used during h/w resync)
+ :param reflow: (boolean) If true, force add (used during h/w resync)
:return: (deferred)
"""
if not self._valid:
returnValue(succeed('Deleting'))
- if not add_always and tcont.alloc_id in self._tconts:
+ if not reflow and tcont.alloc_id in self._tconts:
returnValue(succeed('already created'))
try:
@@ -487,7 +492,7 @@
self._tconts[tcont.alloc_id] = tcont
except Exception as e:
- self.log.exception('tcont', tcont=tcont, e=e)
+ self.log.exception('tcont', tcont=tcont, reflow=reflow, e=e)
raise
returnValue(results)
@@ -522,18 +527,18 @@
if not gem.multicast and not gem.exception]) # FIXED_ONU
@inlineCallbacks
- def add_gem_port(self, gem_port, add_always=False):
+ def add_gem_port(self, gem_port, reflow=False):
"""
Add a GEM Port to this ONU
:param gem_port: (GemPort) GEM Port to add
- :param add_always: (boolean) If true, force add (used during h/w resync)
+ :param reflow: (boolean) If true, force add (used during h/w resync)
:return: (deferred)
"""
if not self._valid:
returnValue(succeed('Deleting'))
- if not add_always and gem_port.gem_id in self._gem_ports:
+ if not reflow and gem_port.gem_id in self._gem_ports:
returnValue(succeed('already created'))
try:
@@ -547,10 +552,10 @@
# GEM-IDs are a sorted list (ascending). First gemport handles downstream traffic
from flow.flow_entry import FlowEntry
evc_maps = FlowEntry.find_evc_map_flows(self._device_id, self._pon_id, self._onu_id)
- pass
+ pass # TODO: Start here Tuesday
except Exception as e:
- self.log.exception('gem-port', e=e)
+ self.log.exception('gem-port', gem_port=gem_port, reflow=reflow, e=e)
raise
returnValue(results)
diff --git a/voltha/adapters/adtran_olt/pon_port.py b/voltha/adapters/adtran_olt/pon_port.py
index aae1e20..275236e 100644
--- a/voltha/adapters/adtran_olt/pon_port.py
+++ b/voltha/adapters/adtran_olt/pon_port.py
@@ -68,6 +68,9 @@
self._discovered_onus = [] # List of serial numbers
self._sync_tick = 20.0
self._in_sync = False
+ self._expedite_sync = False
+ self._expedite_count = 0
+
self._onus = {} # serial_number-base64 -> ONU (allowed list)
self._onu_by_id = {} # onu-id -> ONU
self._next_onu_id = Onu.MIN_ONU_ID + 128
@@ -581,27 +584,31 @@
config = config[self.pon_id]
self._in_sync = True
- dl = [defer.succeed(config)] # Forward get_config on to ont SYNC
+ dl = []
if self.enabled != config.enabled:
self._in_sync = False
+ self._expedite_sync = True
dl.append(self._set_pon_config("enabled", self.enabled))
- if self._state == PonPort.State.RUNNING:
+ elif self._state == PonPort.State.RUNNING:
+ if self.deployment_range != config.deployment_range:
+ self._in_sync = False
+ self._expedite_sync = True
+ dl.append(self._set_pon_config("deployment-range",
+ self.deployment_range))
+
if self.downstream_fec_enable != config.downstream_fec_enable:
self._in_sync = False
+ self._expedite_sync = True
dl.append(self._set_pon_config("downstream-fec-enable",
self.downstream_fec_enable))
if self.upstream_fec_enable != config.upstream_fec_enable:
self._in_sync = False
+ self._expedite_sync = True
dl.append(self._set_pon_config("upstream-fec-enable",
self.upstream_fec_enable))
-
- if self.deployment_range != config.deployment_range:
- self._in_sync = False
- dl.append(self._set_pon_config("deployment-range",
- self.deployment_range))
return defer.gatherResults(dl)
def sync_onus(results):
@@ -620,18 +627,25 @@
extra_onus = hw_onu_ids - my_onu_ids
dl = [self.delete_onu(onu_id) for onu_id in extra_onus]
- missing_onus = my_onu_ids - hw_onu_ids
- # TODO: Need to remove from this PONs dicts so discovery and 'Add' work
- # properly. May be able to just call add_onu?
-
return defer.gatherResults(dl, consumeErrors=True)
def failure(reason, what):
self.log.error('hardware-sync-{}-failed'.format(what), reason=reason)
self._in_sync = False
+ self._expedite_sync = False
def reschedule(_):
+ # Speed up sequential resync a limited number of times if out of sync.
+
delay = self._sync_tick
+
+ if self._expedite_sync:
+ self._expedite_count += 1
+ if self._expedite_count < 5:
+ delay = 1
+ else:
+ self._expedite_count = 0
+
delay += random.uniform(-delay / 10, delay / 10)
self._sync_deferred = reactor.callLater(delay, self._sync_hardware)
@@ -651,27 +665,14 @@
if self._admin_state != AdminState.ENABLED:
return
- # Process the ONU list in for this PON, may have previously provisioned ones there
- # were discovered on an earlier boot
+ # Get new/missing from the discovered ONU leaf. Stale ONUs from previous
+ # configs are now cleaned up during h/w re-sync/reflow.
- new = self._process_status_onu_list(status.onus)
+ new, rediscovered_onus = self._process_status_onu_discovered_list(status.discovered_onu)
- for onu_id in new:
- # self.add_new_onu(serial_number, status)
- self.log.info('found-ONU', onu_id=onu_id)
- raise NotImplementedError('TODO: Adding ONUs from existing ONU (status list) not supported')
+ # Process newly discovered ONU list and rediscovered ONUs
- # Get new/missing from the discovered ONU leaf
-
- new, missing = self._process_status_onu_discovered_list(status.discovered_onu)
-
- # TODO: Do something useful (Does the discovery list clear out activated ONU's?)
- # if len(missing):
- # self.log.info('missing-ONUs', missing=missing)
-
- # Process discovered ONU list
-
- for serial_number in new:
+ for serial_number in new | rediscovered_onus:
reactor.callLater(0, self.add_onu, serial_number, status)
# Process LOS list
@@ -727,25 +728,6 @@
self._active_los_alarms.add(onu_id)
los_alarm(True, onu_id)
- def _process_status_onu_list(self, onus):
- """
- Look for new or missing ONUs
-
- :param onus: (dict) Set of known ONUs
- """
- self.log.debug('ONU-list', onus=onus)
-
- my_onu_ids = frozenset([o.onu_id for o in self._onus.itervalues()])
- discovered_onus = frozenset(onus.keys())
-
- new_onus_ids = discovered_onus - my_onu_ids
- missing_onus_ids = my_onu_ids - discovered_onus
-
- new = {o: v for o, v in onus.iteritems() if o in new_onus_ids}
- missing_onus = {o: v for o, v in onus.iteritems() if o in missing_onus_ids}
-
- return new # , missing_onus # TODO: Support ONU removal
-
def _process_status_onu_discovered_list(self, discovered_onus):
"""
Look for new ONUs
@@ -763,9 +745,9 @@
my_onus = frozenset(self._onus.keys())
new_onus = discovered_onus - my_onus
- # TODO: Remove later if not needed -> missing_onus = my_onus - discovered_onus
+ rediscovered_onus = my_onus & discovered_onus
- return new_onus, None # , missing_onus
+ return new_onus, rediscovered_onus
def _get_onu_info(self, serial_number):
"""
@@ -841,42 +823,55 @@
if serial_number not in status.onus:
# Newly found and not enabled ONU, enable it now if not at max
- if len(self._onus) >= self.MAX_ONUS_SUPPORTED:
- self.log.warning('max-onus-provisioned')
+ onu_info = self._get_onu_info(Onu.serial_number_to_string(serial_number))
+
+ if onu_info is None:
+ self.log.info('lookup-failure', serial_number=serial_number)
+
+ elif serial_number in self._onus or onu_info['onu-id'] in self._onu_by_id:
+ # May be here due to unmanaged power-cycle on OLT
+ self.log.info('onu-already-added', serial_number=serial_number)
+ assert serial_number in self._onus and\
+ onu_info['onu-id'] in self._onu_by_id, \
+ 'ONU not in both lists'
+
+ # Recover ONU information and attempt to reflow TCONT/GEM-PORT
+ # information as well
+
+ onu = self._onus[serial_number]
+ reflow = True
+
+ elif len(self._onus) >= self.MAX_ONUS_SUPPORTED:
+ self.log.warning('max-onus-provisioned', count=len(self._onus))
else:
- onu_info = self._get_onu_info(Onu.serial_number_to_string(serial_number))
+ # TODO: Make use of upstream_channel_speed variable
+ onu = Onu(onu_info)
+ reflow = False
+ self._onus[serial_number] = onu
+ self._onu_by_id[onu.onu_id] = onu
- if onu_info is None:
- self.log.info('lookup-failure', serial_number=serial_number)
+ try:
+ tconts = onu_info['t-conts']
+ gem_ports = onu_info['gem-ports']
- elif serial_number in self._onus or onu_info['onu-id'] in self._onu_by_id:
- self.log.warning('onu-already-added', serial_number=serial_number)
+ # Add Multicast to PON on a per-ONU basis until xPON multicast support is ready
+ # In xPON/BBF, mcast gems tie back to the channel-pair
+ # MCAST VLAN IDs stored as a negative value
- else:
- # TODO: Make use of upstream_channel_speed variable
- onu = Onu(onu_info)
- self._onus[serial_number] = onu
- self._onu_by_id[onu.onu_id] = onu
+ for id_or_vid, gem_port in gem_ports.iteritems(): # TODO: Deprecate this when BBF ready
+ if gem_port.multicast:
+ self.add_mcast_gem_port(gem_port, -id_or_vid)
- try:
- tconts = onu_info['t-conts']
- gem_ports = onu_info['gem-ports']
+ yield onu.create(tconts, gem_ports, reflow=reflow)
+ if not reflow:
+ self.activate_onu(onu)
- # Add Multicast to PON on a per-ONU basis until xPON multicast support is ready
- # In xPON/BBF, mcast gems tie back to the channel-pair
- # MCAST VLAN IDs stored as a negative value
+ except Exception as e:
+ self.log.exception('add-onu', serial_number=serial_number, reflow=reflow, e=e)
- for id_or_vid, gem_port in gem_ports.iteritems(): # TODO: Deprecate this when BBF ready
- if gem_port.multicast:
- self.add_mcast_gem_port(gem_port, -id_or_vid)
-
- yield onu.create(tconts, gem_ports)
- self.activate_onu(onu)
-
- except Exception as e:
- del self._onus[serial_number]
- del self._onu_by_id[onu.onu_id]
- self.log.exception('add-onu', serial_number=serial_number, e=e)
+ if not reflow:
+ del self._onus[serial_number]
+ del self._onu_by_id[onu.onu_id]
def activate_onu(self, onu):
"""