Add restore of PON ports after reboot request

Change-Id: Ice56e3692736057a4a2883e96d5c4817424c342c
diff --git a/voltha/adapters/adtran_olt/adtran_device_handler.py b/voltha/adapters/adtran_olt/adtran_device_handler.py
index 04adb42..26c411b 100644
--- a/voltha/adapters/adtran_olt/adtran_device_handler.py
+++ b/voltha/adapters/adtran_olt/adtran_device_handler.py
@@ -51,12 +51,12 @@
 _MANAGEMENT_VLAN = 4093
 _is_inband_frame = BpfProgramFilter('(ether[14:2] & 0xfff) = 0x{:03x}'.format(_PACKET_IN_VLAN))
 
-_DEFAULT_RESTCONF_USERNAME = "ADMIN"
-_DEFAULT_RESTCONF_PASSWORD = "PASSWORD"
+_DEFAULT_RESTCONF_USERNAME = ""
+_DEFAULT_RESTCONF_PASSWORD = ""
 _DEFAULT_RESTCONF_PORT = 8081
 
-_DEFAULT_NETCONF_USERNAME = "hsvroot"
-_DEFAULT_NETCONF_PASSWORD = "BOSCO"
+_DEFAULT_NETCONF_USERNAME = ""
+_DEFAULT_NETCONF_PASSWORD = ""
 _DEFAULT_NETCONF_PORT = 830
 
 
@@ -172,10 +172,10 @@
         h, self.heartbeat = self.heartbeat, None
         ldi, self.logical_device_id = self.logical_device_id, None
 
-        if d is not None:
+        if d is not None and not d.called:
             d.cancel()
 
-        if h is not None:
+        if h is not None and not h.called:
             h.cancel()
 
         self._deactivate_io_port()
@@ -742,7 +742,7 @@
         # Kill any heartbeat poll
         h, self.heartbeat = self.heartbeat, None
 
-        if h is not None:
+        if h is not None and not h.called:
             h.cancel()
 
         # TODO: What else (delete logical device, ???)
@@ -755,7 +755,7 @@
 
         # Cancel any running enable/disable/... in progress
         d, self.startup = self.startup, None
-        if d is not None:
+        if d is not None and not d.called:
             d.cancel()
 
         # Get the latest device reference
@@ -768,7 +768,7 @@
 
         h, self.heartbeat = self.heartbeat, None
 
-        if h is not None:
+        if h is not None and not h.called:
             h.cancel()
 
         # Update the operational status to UNKNOWN
@@ -835,7 +835,7 @@
 
         # Cancel any running enable/disable/... in progress
         d, self.startup = self.startup, None
-        if d is not None:
+        if d is not None and not d.called:
             d.cancel()
 
         # Get the latest device reference
@@ -917,7 +917,7 @@
 
         # Cancel any running enable/disable/... in progress
         d, self.startup = self.startup, None
-        if d is not None:
+        if d is not None and not d.called:
             d.cancel()
 
         # Update the operational status to ACTIVATING and connect status to
@@ -1075,11 +1075,11 @@
         # Cancel any outstanding tasks
 
         d, self.startup = self.startup, None
-        if d is not None:
+        if d is not None and not d.called:
             d.cancel()
 
         h, self.heartbeat = self.heartbeat, None
-        if h is not None:
+        if h is not None and not h.called:
             h.cancel()
 
         # Remove all flows from the device
diff --git a/voltha/adapters/adtran_olt/adtran_olt.py b/voltha/adapters/adtran_olt/adtran_olt.py
index a7daa52..e4f70a4 100644
--- a/voltha/adapters/adtran_olt/adtran_olt.py
+++ b/voltha/adapters/adtran_olt/adtran_olt.py
@@ -51,7 +51,7 @@
         self.descriptor = Adapter(
             id=self.name,
             vendor='Adtran Inc.',
-            version='0.2',
+            version='0.3',
             config=AdapterConfig(log_level=LogLevel.INFO)
         )
         log.debug('adtran_olt.__init__', adapter_agent=adapter_agent)
diff --git a/voltha/adapters/adtran_olt/adtran_olt_handler.py b/voltha/adapters/adtran_olt/adtran_olt_handler.py
index 0cd919b..fd90bf2 100644
--- a/voltha/adapters/adtran_olt/adtran_olt_handler.py
+++ b/voltha/adapters/adtran_olt/adtran_olt_handler.py
@@ -282,7 +282,7 @@
                 pass
 
         d, self.status_poll = self.status_poll, None
-        if d is not None:
+        if d is not None and not d.called:
             try:
                 d.cancel()
             except:
@@ -302,7 +302,7 @@
             c.shutdown()
 
         d, self.status_poll = self.status_poll, None
-        if d is not None:
+        if d is not None and not d.called:
             d.cancel()
 
         super(AdtranOltHandler, self).reboot()
@@ -319,7 +319,7 @@
             c.shutdown()
 
         d, self.status_poll = self.status_poll, None
-        if d is not None:
+        if d is not None and not d.called:
             d.cancel()
 
         super(AdtranOltHandler, self).delete()
@@ -390,7 +390,7 @@
         # OLT Specific things here
 
         d, self.startup = self.startup, None
-        if d is not None:
+        if d is not None and not d.called:
             d.cancel()
 
         # self.pons.clear()
diff --git a/voltha/adapters/adtran_olt/nni_port.py b/voltha/adapters/adtran_olt/nni_port.py
index 1867f6f..751f8bb 100644
--- a/voltha/adapters/adtran_olt/nni_port.py
+++ b/voltha/adapters/adtran_olt/nni_port.py
@@ -111,7 +111,7 @@
 
     def _cancel_deferred(self):
         d, self._deferred = self._deferred, None
-        if d is not None:
+        if d is not None and not d.called:
             d.cancel()
 
     def _update_adapter_agent(self):
@@ -168,45 +168,42 @@
         self.log.info('Starting NNI port')
         self._cancel_deferred()
 
-        # TODO: Start up any watchdog/polling tasks here
-
-        self._admin_state = AdminState.ENABLED
-        self._oper_status = OperStatus.ACTIVE
+        self._oper_status = OperStatus.ACTIVATING
         self._update_adapter_agent()
 
         # Do the rest of the startup in an async method
         self._deferred = reactor.callLater(0, self._finish_startup)
         return self._deferred
 
+    @inlineCallbacks
     def _finish_startup(self):
         if self._state != NniPort.State.INITIAL:
             returnValue('Done')
 
-        # returnValue('TODO: Implement startup of each NNI port')
+        self._enabled = True
+        self._admin_state = AdminState.ENABLED
+        self._oper_status = OperStatus.ACTIVE  # TODO: is this correct, how do we tell GRPC
+        self._update_adapter_agent()
 
-        if self._enabled:
-            self._admin_state = AdminState.ENABLED
-            self._oper_status = OperStatus.ACTIVE  # TODO: is this correct, how do we tell GRPC
-            self._update_adapter_agent()
+        try:
+            results = yield self.set_config('enabled', True)
 
-            # TODO: Start status polling of NNI interfaces
-            self._deferred = None  # = reactor.callLater(3, self.do_stuff)
+        except Exception as e:
+            self.log.exception('nni-start', e=e)
+            self._admin_state = AdminState.UNKNOWN
+            raise
 
-            self._state = NniPort.State.RUNNING
-        else:
-            # Startup failed. Could be due to object creation with an invalid initial admin_status
-            #                 state.  May want to schedule a start to occur again if this happens
-            self._admin_state = AdminState.DISABLED
-            self._oper_status = OperStatus.UNKNOWN
-            self._update_adapter_agent()
+        # TODO: Start status polling of NNI interfaces
+        self._deferred = None  # = reactor.callLater(3, self.do_stuff)
+        self._state = NniPort.State.RUNNING
+        returnValue(self._deferred)
 
-            self._state = NniPort.State.STOPPED
-
+    @inlineCallbacks
     def stop(self):
         if self._state == NniPort.State.STOPPED:
-            return succeed('Stopped')
+            returnValue(succeed('Stopped'))
 
-        self.log.info('Stopping NNI port')
+        self.log.info('stopping-nni')
         self._cancel_deferred()
 
         # NOTE: Leave all NNI ports active (may have inband management)
@@ -219,8 +216,16 @@
         self._oper_status = OperStatus.UNKNOWN
         self._update_adapter_agent()
 
+        try:
+            results = yield self.set_config('enabled', False)
+
+        except Exception as e:
+            self.log.exception('nni-start', e=e)
+            self._admin_state = AdminState.UNKNOWN
+            raise
+
         self._state = NniPort.State.STOPPED
-        return self._deferred
+        returnValue(self._deferred)
 
     def restart(self):
         if self._state == NniPort.State.RUNNING or self._state == NniPort.State.STOPPED:
diff --git a/voltha/adapters/adtran_olt/onu.py b/voltha/adapters/adtran_olt/onu.py
index 403460b..35d60ec 100644
--- a/voltha/adapters/adtran_olt/onu.py
+++ b/voltha/adapters/adtran_olt/onu.py
@@ -51,7 +51,7 @@
         #     'onu-id': None,  # Set later (mandatory)
         #     'enabled': True,
         #     'upstream-channel-speed': 0,
-        #     't-cont': get_tconts(self.pon_id, serial_number),
+        #     't-conts': get_tconts(self.pon_id, serial_number),
         #     'gem-ports': get_gem_ports(self.pon_id, serial_number),
         # }
         self._onu_id = onu_info['onu-id']
@@ -67,6 +67,7 @@
         self._gem_ports = {}                           # gem-id -> GemPort
         self._tconts = {}                              # alloc-id -> TCont
         self._onu_vid = onu_info['onu-vid']
+        self._enabled = onu_info['enabled']
 
         # TODO: enable and upstream-channel-speed not yet supported
 
@@ -123,7 +124,7 @@
         return self._vendor_id
 
     @inlineCallbacks
-    def create(self, onu_info):
+    def create(self, tconts, gem_ports):
         """
         POST -> /restconf/data/gpon-olt-hw:olt/pon=<pon-id>/onus/onu ->
         """
@@ -132,10 +133,10 @@
         pon_id = self.pon.pon_id
         data = json.dumps({'onu-id': self._onu_id,
                            'serial-number': self._serial_number_base64,
-                           'enable': onu_info['enabled']})
+                           'enable': self._enabled})
         uri = AdtranOltHandler.GPON_ONU_CONFIG_LIST_URI.format(pon_id)
         name = 'onu-create-{}-{}-{}: {}'.format(pon_id, self._onu_id,
-                                                self._serial_number_base64, onu_info['enabled'])
+                                                self._serial_number_base64, self._enabled)
 
         try:
             results = yield self.olt.rest_client.request('POST', uri, data=data, name=name)
@@ -146,14 +147,14 @@
 
         # Now set up all tconts & gem-ports
 
-        for _, tcont in onu_info['t-conts'].items():
+        for _, tcont in tconts.items():
             try:
                 results = yield self.add_tcont(tcont)
 
             except Exception as e:
                 self.log.exception('add-tcont', tcont=tcont, e=e)
 
-        for _, gem_port in onu_info['gem-ports'].items():
+        for _, gem_port in gem_ports.items():
             try:
                 if gem_port.multicast:
                     self.log.warning('multicast-not-yet-supported', gem_port=gem_port)  # TODO Support it
@@ -165,6 +166,11 @@
 
         returnValue(results)
 
+    def restart(self):
+        tconts, self._tconts = self._tconts, {}
+        gem_ports, self._gem_ports = self._gem_ports, {}
+        return self.create(tconts, gem_ports)
+
     def set_config(self, leaf, value):
         self.log.debug('set-config', leaf=leaf, value=value)
 
diff --git a/voltha/adapters/adtran_olt/pon_port.py b/voltha/adapters/adtran_olt/pon_port.py
index 48f0260..47bf18f 100644
--- a/voltha/adapters/adtran_olt/pon_port.py
+++ b/voltha/adapters/adtran_olt/pon_port.py
@@ -18,7 +18,7 @@
 
 import structlog
 from enum import Enum
-from twisted.internet import reactor
+from twisted.internet import reactor, defer
 from twisted.internet.defer import inlineCallbacks, returnValue, succeed
 
 from adtran_olt_handler import AdtranOltHandler
@@ -69,7 +69,7 @@
 
         self._admin_state = AdminState.DISABLED
         self._oper_status = OperStatus.DISCOVERED
-        self._deferred = None                   # General purpose
+        self._deferred = None                     # General purpose
         self._discovery_deferred = None           # Specifically for ONU discovery
         self._state = PonPort.State.INITIAL
 
@@ -194,10 +194,17 @@
         d1, self._deferred = self._deferred, None
         d2, self._discovery_deferred = self._discovery_deferred, None
         
-        if d1 is not None:
-            d1.cancel()            
-        if d2 is not None:
-            d2.cancel()
+        if d1 is not None and not d1.called:
+            try:
+                d1.cancel()
+            except Exception as e:
+                pass
+
+        if d2 is not None and not d2.called:
+            try:
+                d2.cancel()
+            except Exception as e:
+                pass
 
     def _update_adapter_agent(self):
         # TODO: Currently the adapter_agent does not allow 'update' of port status
@@ -290,6 +297,14 @@
             self._oper_status = OperStatus.ACTIVE  # TODO: is this correct, how do we tell GRPC
             self._state = PonPort.State.RUNNING
 
+            # Restart any ONU's in case here due to reboot
+
+            if len(self._onus) > 0:
+                dl = []
+                for onu in self._onus.itervalues():
+                    dl.append(onu.restart())
+                yield defer.gatherResults(dl)
+
             # Begin to ONU discovery
 
             self._discovery_deferred = reactor.callLater(5, self._discover_onus)
@@ -391,6 +406,10 @@
         if self._state == PonPort.State.RUNNING or self._state == PonPort.State.STOPPED:
             start_it = (self._state == PonPort.State.RUNNING)
             self._state = PonPort.State.INITIAL
+            self._enabled = None
+            self._downstream_fec_enable = None
+            self._upstream_fec_enable = None
+
             return self.start() if start_it else self.stop()
         return succeed('nop')
 
@@ -616,7 +635,9 @@
                     self._onu_by_id[onu.onu_id] = onu
 
                     try:
-                        yield onu.create(onu_info)
+                        tconts = onu_info['t-conts']
+                        gem_ports = onu_info['gem-ports']
+                        yield onu.create(tconts, gem_ports)
                         self.activate_onu(onu)
 
                     except Exception as e: