VOL-1185: Support for new/refactored KPI metrics format

Change-Id: I2b378208a6f6c9fba744b8ad46a5fd83e8b09607
diff --git a/voltha/adapters/adtran_olt/adtran_device_handler.py b/voltha/adapters/adtran_olt/adtran_device_handler.py
index 3dbbb98..f6796ed 100644
--- a/voltha/adapters/adtran_olt/adtran_device_handler.py
+++ b/voltha/adapters/adtran_olt/adtran_device_handler.py
@@ -475,8 +475,8 @@
                             'pon-ports': self.southbound_ports.values()
                         }
                         self.pm_metrics = OltPmMetrics(self.adapter_agent, self.device_id,
-                                                       grouped=True, freq_override=False,
-                                                       **kwargs)
+                                                       ld_initialized.id, grouped=True,
+                                                       freq_override=False, **kwargs)
 
                         pm_config = self.pm_metrics.make_proto()
                         self.log.debug("initial-pm-config", pm_config=pm_config)
@@ -788,7 +788,6 @@
     def enumerate_northbound_ports(self, device):
         """
         Enumerate all northbound ports of a device. You should override
-        this method in your derived class as necessary. Should you encounter
         a non-recoverable error, throw an appropriate exception.
 
         :param device: A voltha.Device object, with possible device-type
diff --git a/voltha/adapters/adtran_olt/adtran_olt.py b/voltha/adapters/adtran_olt/adtran_olt.py
index 6b867bf..84a39c5 100644
--- a/voltha/adapters/adtran_olt/adtran_olt.py
+++ b/voltha/adapters/adtran_olt/adtran_olt.py
@@ -52,7 +52,7 @@
         self.descriptor = Adapter(
             id=self.name,
             vendor='Adtran Inc.',
-            version='0.19',
+            version='0.20',
             config=AdapterConfig(log_level=LogLevel.INFO)
         )
         log.debug('adtran_olt.__init__', adapter_agent=adapter_agent)
diff --git a/voltha/adapters/adtran_olt/flow/flow_entry.py b/voltha/adapters/adtran_olt/flow/flow_entry.py
index c578190..dac6e1e 100644
--- a/voltha/adapters/adtran_olt/flow/flow_entry.py
+++ b/voltha/adapters/adtran_olt/flow/flow_entry.py
@@ -193,19 +193,25 @@
             downstream_sig_table = _existing_downstream_flow_entries[flow_entry.device_id]
             upstream_flow_table = _existing_upstream_flow_entries[flow_entry.device_id]
 
+            log.debug('flow-entry-decoded', flow=flow_entry, signature=flow_entry.signature,
+                      downstream_signature=flow_entry.downstream_signature)
+
             if flow_entry.flow_direction == FlowEntry.FlowDirection.UPSTREAM and\
                     flow_entry.flow_id in upstream_flow_table:
+                log.debug('flow-entry-upstream-exists', flow=flow_entry)
                 return flow_entry, None
 
             if flow_entry.flow_direction == FlowEntry.FlowDirection.DOWNSTREAM and\
                     flow_entry.signature in downstream_sig_table and\
                     flow_entry.flow_id in downstream_sig_table[flow_entry.signature]:
+                log.debug('flow-entry-upstream-exists', flow=flow_entry)
                 return flow_entry, None
 
             # Look for any matching flows in the other direction that might help make an EVC
             # and then save it off in the device specific flow table
             # TODO: For now, only support for E-LINE services between NNI and UNI
 
+            log.debug('flow-entry-search-for-match', flow=flow_entry)
             downstream_flow = None
             upstream_flows = None
             downstream_sig = None
@@ -250,6 +256,9 @@
                 if len(upstream_flows) == 0 and not downstream_flow.is_multicast_flow:
                     upstream_flows = None
 
+            log.debug('flow-entry-search-results', flow=flow_entry, downstream_flow=downstream_flow,
+                      upstream_flows=upstream_flows)
+
             # Compute EVC and and maps
             evc = FlowEntry._create_evc_and_maps(evc, downstream_flow, upstream_flows)
             if evc is not None and evc.valid and downstream_flow_table['evc'] is None:
diff --git a/voltha/adapters/adtran_olt/nni_port.py b/voltha/adapters/adtran_olt/nni_port.py
index 83f113a..b0a704e 100644
--- a/voltha/adapters/adtran_olt/nni_port.py
+++ b/voltha/adapters/adtran_olt/nni_port.py
@@ -15,11 +15,12 @@
 #
 
 import random
+import arrow
 
 import structlog
 import xmltodict
 from port import AdtnPort
-from twisted.internet import reactor, defer
+from twisted.internet import reactor
 from twisted.internet.defer import inlineCallbacks, returnValue, succeed, fail
 from twisted.python.failure import Failure
 from voltha.core.logical_device_agent import mac_str_to_tuple
@@ -51,7 +52,6 @@
         self._stats_deferred = None
 
         # Local cache of NNI configuration
-
         self._ianatype = '<type xmlns:ianaift="urn:ietf:params:xml:ns:yang:iana-if-type">ianaift:ethernetCsmacd</type>'
 
         # And optional parameters
@@ -75,12 +75,14 @@
 
         # Statistics
         self.rx_dropped = 0
-        self.rx_errors = 0
-        self.rx_bcast = 0
-        self.rx_mcast = 0
+        self.rx_error_packets = 0
+        self.rx_ucast_packets = 0
+        self.rx_bcast_packets = 0
+        self.rx_mcast_packets = 0
         self.tx_dropped = 0
-        self.tx_bcast = 0
-        self.tx_mcast = 0
+        self.rx_ucast_packets = 0
+        self.tx_bcast_packets = 0
+        self.tx_mcast_packets = 0
 
     def __str__(self):
         return "NniPort-{}: Admin: {}, Oper: {}, parent: {}".format(self._port_no,
@@ -106,7 +108,7 @@
            self._port.oper_status != self._oper_status:
 
             self.log.debug('get-port-status-update', admin_state=self._admin_state,
-                           oper_status = self._oper_status)
+                           oper_status=self._oper_status)
             self._port.admin_state = self._admin_state
             self._port.oper_status = self._oper_status
 
@@ -194,9 +196,9 @@
         Set the NNI Port to a known good state on initial port startup.  Actual
         NNI 'Start' is done elsewhere
         """
-        #if self.state != AdtnPort.State.INITIAL:
-        #    self.log.error('reset-ignored', state=self.state)
-        #    returnValue('Ignored')
+        # if self.state != AdtnPort.State.INITIAL:
+        #     self.log.error('reset-ignored', state=self.state)
+        #     returnValue('Ignored')
 
         self.log.info('resetting', label=self._label)
 
@@ -296,26 +298,29 @@
             self.sync_deferred.addBoth(reschedule)
 
     def _decode_nni_statistics(self, entry):
-        admin_status = entry.get('admin-status')
-        oper_status = entry.get('oper-status')
-        admin_status = entry.get('admin-status')
-        phys_address = entry.get('phys-address')
+        # admin_status = entry.get('admin-status')
+        # oper_status = entry.get('oper-status')
+        # admin_status = entry.get('admin-status')
+        # phys_address = entry.get('phys-address')
 
         stats = entry.get('statistics')
         if stats is not None:
+            self.timestamp = arrow.utcnow().float_timestamp
             self.rx_bytes = int(stats.get('in-octets', 0))
-            self.rx_dropped = int(stats.get('in-discards', 0))
-            self.rx_errors = int(stats.get('in-errors', 0))
-            self.rx_bcast = int(stats.get('in-broadcast-pkts', 0))
-            self.rx_mcast = int(stats.get('in-multicast-pkts', 0))
+            self.rx_ucast_packets = int(stats.get('in-unicast-pkts', 0))
+            self.rx_bcast_packets = int(stats.get('in-broadcast-pkts', 0))
+            self.rx_mcast_packets = int(stats.get('in-multicast-pkts', 0))
+            self.rx_error_packets = int(stats.get('in-errors', 0)) + int(stats.get('in-discards', 0))
 
             self.tx_bytes = int(stats.get('out-octets', 0))
-            self.tx_bcast = int(stats.get('out-broadcast-pkts', 0))
-            self.tx_mcast = int(stats.get('out-multicast-pkts', 0))
-            self.tx_dropped = int(stats.get('out-discards', 0)) + int(stats.get('out-errors', 0))
+            self.tx_ucast_packets = int(stats.get('out-unicast-pkts', 0))
+            self.tx_bcast_packets = int(stats.get('out-broadcast-pkts', 0))
+            self.tx_mcasy_packets = int(stats.get('out-multicast-pkts', 0))
+            self.tx_error_packets = int(stats.get('out-errors', 0)) + int(stats.get('out-discards', 0))
 
-            self.rx_packets = int(stats.get('in-unicast-pkts', 0)) + self.rx_mcast + self.rx_bcast
-            self.tx_packets = int(stats.get('out-unicast-pkts', 0)) + self.tx_mcast + self.tx_bcast
+            self.rx_packets = self.rx_ucast_packets + self.rx_mcast_packets + self.rx_bcast_packets
+            self.tx_packets = self.tx_ucast_packets + self.tx_mcast_packets + self.tx_bcast_packets
+            # No support for rx_crc_errors or bip_errors
 
     def _update_statistics(self):
         if self.state == AdtnPort.State.RUNNING:
diff --git a/voltha/adapters/adtran_olt/onu.py b/voltha/adapters/adtran_olt/onu.py
index 2681d00..b855218 100644
--- a/voltha/adapters/adtran_olt/onu.py
+++ b/voltha/adapters/adtran_olt/onu.py
@@ -75,6 +75,7 @@
         self._resync_flows = False
         self._sync_deferred = None     # For sync of ONT config to hardware
         self._password = None
+        self._timestamp = None
 
         if onu_info['venet'] is not None:
             port_no, subscriber_vlan, self.untagged_vlan = Onu.decode_venet(onu_info['venet'],
@@ -149,6 +150,10 @@
         return self.olt.southbound_ports[self._pon_id]
 
     @property
+    def intf_id(self):
+        return self.pon.intf_id
+
+    @property
     def pon_id(self):
         return self._pon_id
 
@@ -315,6 +320,14 @@
         return self._serial_number_string
 
     @property
+    def timestamp(self):
+        return self._timestamp
+
+    @timestamp.setter
+    def timestamp(self, value):
+        self._timestamp = value
+
+    @property
     def rssi(self):
         """The received signal strength indication of the ONU"""
         return self._rssi
@@ -814,6 +827,8 @@
 
         gem_port.pon_id = self.pon_id
         gem_port.onu_id = self.onu_id if self.onu_id is not None else -1
+        gem_port.intf_id = self.intf_id
+
         self.log.info('add', gem_port=gem_port, reflow=reflow)
         self._gem_ports[gem_port.gem_id] = gem_port
 
diff --git a/voltha/adapters/adtran_olt/pon_port.py b/voltha/adapters/adtran_olt/pon_port.py
index 82dcd9b..e0f7f61 100644
--- a/voltha/adapters/adtran_olt/pon_port.py
+++ b/voltha/adapters/adtran_olt/pon_port.py
@@ -14,6 +14,7 @@
 
 import json
 import random
+import arrow
 
 import structlog
 from port import AdtnPort
@@ -643,14 +644,15 @@
             reactor.callLater(0, self.add_onu, serial_number, status)
 
         # PON Statistics
-        self._process_statistics(status)
+        timestamp = arrow.utcnow().float_timestamp
+        self._process_statistics(status, timestamp)
 
         # Process ONU info. Note that newly added ONUs will not be processed
         # until the next pass
-        self._update_onu_status(status.onus)
+        self._update_onu_status(status.onus, timestamp)
 
         # Process GEM Port information
-        self._update_gem_status(status.gems)
+        self._update_gem_status(status.gems, timestamp)
 
     def _handle_discovered_onu(self, child_device, ind_info):
         pon_id = ind_info['_pon_id']
@@ -691,14 +693,15 @@
             self.log.info('Invalid-ONU-event', olt_id=olt_id,
                           pon_ni=ind_info['_pon_id'], onu_data=ind_info)
 
-    def _process_statistics(self, status):
+    def _process_statistics(self, status, timestamp):
+        self.timestamp = timestamp
         self.rx_packets = status.rx_packets
         self.rx_bytes = status.rx_bytes
         self.tx_packets = status.tx_packets
         self.tx_bytes = status.tx_bytes
         self.tx_bip_errors = status.tx_bip_errors
 
-    def _update_onu_status(self, onus):
+    def _update_onu_status(self, onus, timestamp):
         """
         Process ONU status for this PON
         :param onus: (dict) onu_id: ONU State
@@ -706,18 +709,20 @@
         for onu_id, onu_status in onus.iteritems():
             if onu_id in self._onu_by_id:
                 onu = self._onu_by_id[onu_id]
+                onu.timestamp = timestamp
                 onu.rssi = onu_status.rssi
                 onu.equalization_delay = onu_status.equalization_delay
                 onu.equalization_delay = onu_status.equalization_delay
                 onu.fiber_length = onu_status.fiber_length
                 onu.password = onu_status.reported_password
 
-    def _update_gem_status(self, gems):
+    def _update_gem_status(self, gems, timestamp):
         for gem_id, gem_status in gems.iteritems():
             onu = self._onu_by_id.get(gem_status.onu_id)
             if onu is not None:
                 gem_port = onu.gem_port(gem_status.gem_id)
                 if gem_port is not None:
+                    gem_port.timestamp = timestamp
                     gem_port.rx_packets = gem_status.rx_packets
                     gem_port.rx_bytes = gem_status.rx_bytes
                     gem_port.tx_packets = gem_status.tx_packets
diff --git a/voltha/adapters/adtran_olt/port.py b/voltha/adapters/adtran_olt/port.py
index 69597a7..0373724 100644
--- a/voltha/adapters/adtran_olt/port.py
+++ b/voltha/adapters/adtran_olt/port.py
@@ -63,6 +63,7 @@
         self.rx_bytes = 0
         self.tx_packets = 0
         self.tx_bytes = 0
+        self.timestamp = 0
 
     def __del__(self):
         self.stop()
@@ -79,6 +80,10 @@
         return self._port_no
 
     @property
+    def intf_id(self):
+        return self.port_no
+
+    @property
     def name(self):
         return self._name
 
@@ -216,6 +221,7 @@
             yield self.deferred
 
         except Exception as e:
+            self.log.exception('stop-failed', e=e)
             raise
 
         returnValue('Stopped')
diff --git a/voltha/adapters/adtran_olt/xpon/gem_port.py b/voltha/adapters/adtran_olt/xpon/gem_port.py
index 4801a7f..5cc84b2 100644
--- a/voltha/adapters/adtran_olt/xpon/gem_port.py
+++ b/voltha/adapters/adtran_olt/xpon/gem_port.py
@@ -44,6 +44,7 @@
         #       from hardware methods
         self._pon_id = None
         self._onu_id = None
+        self._intf_id = None
 
         # Statistics
         self.rx_packets = 0
@@ -75,6 +76,15 @@
         self._onu_id = onu_id
 
     @property
+    def intf_id(self):
+        return self._intf_id
+
+    @intf_id.setter
+    def intf_id(self, intf_id):
+        assert self._intf_id is None or self._intf_id == intf_id, 'Port Number can only be set once'
+        self._intf_id = intf_id
+
+    @property
     def alloc_id(self):
         if self._alloc_id is None and self._handler is not None:
             try:
diff --git a/voltha/adapters/adtran_olt/xpon/olt_gem_port.py b/voltha/adapters/adtran_olt/xpon/olt_gem_port.py
index c5b72a1..1f1c916 100644
--- a/voltha/adapters/adtran_olt/xpon/olt_gem_port.py
+++ b/voltha/adapters/adtran_olt/xpon/olt_gem_port.py
@@ -48,6 +48,7 @@
                                          name=name,
                                          handler=handler)
         self._is_mock = is_mock
+        self._timestamp = None
 
     @staticmethod
     def create(handler, gem_port):
@@ -69,6 +70,14 @@
                           untagged=untagged)
 
     @property
+    def timestamp(self):
+        return self._timestamp
+
+    @timestamp.setter
+    def timestamp(self, value):
+        self._timestamp = value
+
+    @property
     def encryption(self):
         return self._encryption