VOL-698: Implement MIB Comparison step for the Synchronizer MIB Audit state

Change-Id: I8662d76c7b16769da878df7cce8c5b0a98693596
diff --git a/voltha/extensions/omci/omci_cc.py b/voltha/extensions/omci/omci_cc.py
index 5a4ad6c..69f7646 100644
--- a/voltha/extensions/omci/omci_cc.py
+++ b/voltha/extensions/omci/omci_cc.py
@@ -26,6 +26,7 @@
 from voltha.extensions.omci.omci_me import OntGFrame, OntDataFrame
 from common.event_bus import EventBusClient
 from enum import IntEnum
+from binascii import hexlify
 
 
 _MAX_INCOMING_ALARM_MESSAGES = 256
diff --git a/voltha/extensions/omci/onu_device_entry.py b/voltha/extensions/omci/onu_device_entry.py
index 0169dd1..c02f86e 100644
--- a/voltha/extensions/omci/onu_device_entry.py
+++ b/voltha/extensions/omci/onu_device_entry.py
@@ -27,6 +27,7 @@
 OP = EntityOperations
 RC = ReasonCodes
 
+ACTIVE_KEY = 'active'
 IN_SYNC_KEY = 'in-sync'
 LAST_IN_SYNC_KEY = 'last-in-sync-time'
 
@@ -47,8 +48,12 @@
         """
         Class initializer
 
+        :param omci_agent: (OpenOMCIAgent) Reference to OpenOMCI Agent
         :param device_id: (str) ONU Device ID
+        :param adapter_agent: (AdapterAgent) Adapter agent for ONU
         :param custom_me_map: (dict) Additional/updated ME to add to class map
+        :param mib_synchronizer_info: (dict) MIB Synchronization State Machine & Task information
+        :param mib_db: (MibDbApi) MIB Database reference
         """
         self.log = structlog.get_logger(device_id=device_id)
 
@@ -154,6 +159,9 @@
             self.event_bus.publish(topic=topic, msg=msg)
 
     def start(self):
+        """
+        Start the ONU Device Entry state machines
+        """
         if self._started:
             return
 
@@ -175,6 +183,11 @@
         self._publish_device_status_event()
 
     def stop(self):
+        """
+        Stop the ONU Device Entry state machines
+
+        When the ONU Device Entry is stopped,
+        """
         if not self._started:
             return
 
@@ -198,12 +211,20 @@
         """
         topic = OnuDeviceEntry.event_bus_topic(self.device_id,
                                                OnuDeviceEvents.DeviceStatusEvent)
-        msg = {'active': self._started}
+        msg = {ACTIVE_KEY: self._started}
         self.event_bus.publish(topic=topic, msg=msg)
 
     def delete(self):
+        """
+        Stop the ONU Device's state machine and remove the ONU, and any related
+        OMCI state information from the OpenOMCI Framework
+        """
         self.stop()
 
+        # OpenOMCI cleanup
+        if self._omci_agent is not None:
+            self._omci_agent.remove_device(self._device_id, cleanup=True)
+
     def query_mib(self, class_id=None, instance_id=None, attributes=None):
         """
         Get MIB database information.
diff --git a/voltha/extensions/omci/openomci_agent.py b/voltha/extensions/omci/openomci_agent.py
index 9973444..22d56b4 100644
--- a/voltha/extensions/omci/openomci_agent.py
+++ b/voltha/extensions/omci/openomci_agent.py
@@ -82,6 +82,10 @@
         """ Return a reference to the VOLTHA Core component"""
         return self._core
 
+    @property
+    def database_class(self):
+        return self._mib_database_cls
+
     def start(self):
         """
         Start OpenOMCI
diff --git a/voltha/extensions/omci/state_machines/mib_sync.py b/voltha/extensions/omci/state_machines/mib_sync.py
index b0beb48..f121033 100644
--- a/voltha/extensions/omci/state_machines/mib_sync.py
+++ b/voltha/extensions/omci/state_machines/mib_sync.py
@@ -59,6 +59,7 @@
         {'trigger': 'force_resync', 'source': 'auditing', 'dest': 'resynchronizing'},
 
         {'trigger': 'success', 'source': 'resynchronizing', 'dest': 'in_sync'},
+        {'trigger': 'diffs_found', 'source': 'resynchronizing', 'dest': 'out_of_sync'},
         {'trigger': 'timeout', 'source': 'resynchronizing', 'dest': 'out_of_sync'},
 
         # Do wildcard 'stop' trigger last so it covers all previous states
@@ -114,6 +115,10 @@
         self._last_mib_db_sync_value = None
         self._device_in_db = False
 
+        self._on_olt_only_diffs = None
+        self._on_onu_only_diffs = None
+        self._attr_diffs = None
+
         self._event_bus = EventBusClient()
         self._subscriptions = {               # RxEvent.enum -> Subscription Object
             RxEvent.MIB_Reset: None,
@@ -270,7 +275,7 @@
         Begin full MIB data sync, starting with a MIB RESET
         """
         def success(results):
-            self.log.info('mib-upload-success: {}'.format(results))
+            self.log.debug('mib-upload-success: {}'.format(results))
             self._current_task = None
             self._deferred = reactor.callLater(0, self.success)
 
@@ -293,7 +298,7 @@
         self._mib_data_sync = self._database.get_mib_data_sync(self._device_id) or 0
 
         def success(onu_mds_value):
-            self.log.info('examine-mds-success: {}'.format(onu_mds_value))
+            self.log.debug('examine-mds-success: {}'.format(onu_mds_value))
             self._current_task = None
 
             # Examine MDS value
@@ -326,13 +331,69 @@
 
     def on_enter_out_of_sync(self):
         """
+        The MIB in OpenOMCI and the ONU are out of sync.  This can happen if:
+
+           o the MIB_Data_Sync values are not equal, or
+           o the MIBs were compared and differences were found.
+
+        If all of the *_diff properties are allNone, then we are here after initial
+        startup and MDS did not match, or the MIB Audit/Resync state failed.
+
+        In the second case, one or more of our *_diff properties will be non-None.
+        If that is true, we need to update the ONU accordingly.
+
         Schedule a tick to occur to in the future to request an audit
         """
         self.log.debug('state-transition', audit_delay=self._audit_delay)
         self._device.mib_db_in_sync = False
 
-        if self._audit_delay > 0:
-            self._deferred = reactor.callLater(self._audit_delay, self.audit_mib)
+        if all(diff is None for diff in [self._on_olt_only_diffs,
+                                         self._on_onu_only_diffs,
+                                         self._attr_diffs]):
+            # Retry the Audit process
+            self._deferred = reactor.callLater(1, self.audit_mib)
+
+        else:
+            step = 'Nothing'
+            class_id = 0
+            instance_id = 0
+            attribute = ''
+
+            try:
+                # Need to update the ONU accordingly
+                if self._attr_diffs is not None:
+                    assert self._attr_diffs is not None, 'Should match'
+                    step = 'attribute-update'
+                    pass    # TODO: Perform the 'set' commands needed
+
+                if self._on_onu_only_diffs is not None:
+                    step = 'onu-cleanup'
+                    #
+                    # TODO: May want to watch for ONU only attributes
+                    #    It is possible that if they are the 'default' value or
+                    #    are not used if another attribute is set a specific way.
+                    #
+                    #    For instance, no one may set the gal_loopback_configuration
+                    #    in the GEM Interworking Termination point since its default
+                    #    values is '0' disable, but when we audit, the ONU will report zer
+                    #
+                    #    A good way to perhaps fix this is to update our database with the
+                    #    default.  Or perhaps set all defaults in the database in the first
+                    #    place when we do the initial create/set
+                    #
+                    pass  # TODO: Perform 'delete' commands as needed, see 'default' note above
+
+                if self._on_olt_only_diffs is not None:
+                    step = 'olt-push'
+                    pass    # TODO: Perform 'create' commands as needed
+
+                self._deferred = reactor.callLater(1, self.audit_mib)
+
+            except Exception as e:
+                self.log.exception('onu-update', e=e, step=step, class_id=class_id,
+                                   instance_id=instance_id, attribute=attribute)
+                # Retry the Audit process
+                self._deferred = reactor.callLater(1, self.audit_mib)
 
     def on_enter_auditing(self):
         """
@@ -350,6 +411,7 @@
             def success(onu_mds_value):
                 self.log.debug('get-mds-success: {}'.format(onu_mds_value))
                 self._current_task = None
+
                 # Examine MDS value
                 if self._mib_data_sync == onu_mds_value:
                     self._deferred = reactor.callLater(0, self.success)
@@ -369,15 +431,40 @@
     def on_enter_resynchronizing(self):
         """
         Perform a resynchronization of the MIB database
+
+        First calculate any differences
         """
         def success(results):
-            self.log.info('resync-success: {}'.format(results))
+            self.log.debug('resync-success: {}'.format(results))
+
+            on_olt_only = results.get('on-olt-only')
+            on_onu_only = results.get('on-onu-only')
+            attr_diffs = results.get('attr-diffs')
+
             self._current_task = None
-            self._deferred = reactor.callLater(0, self.success)
+            self._on_olt_only_diffs = on_olt_only if len(on_olt_only) else None
+            self._on_onu_only_diffs = on_onu_only if len(on_onu_only) else None
+            self._attr_diffs = attr_diffs if len(attr_diffs) else None
+
+            if all(diff is None for diff in [self._on_olt_only_diffs,
+                                             self._on_onu_only_diffs,
+                                             self._attr_diffs]):
+                # TODO: If here, do we need to make sure OpenOMCI mib_data_sync matches
+                #       the ONU.  Remember we compared against an ONU snapshot, it may
+                #       be different now.  Best thing to do is perhaps set it to our
+                #       MDS value if different. Also remember that setting the MDS on
+                #       the ONU to 'n' is a set command and it will be 'n+1' after the
+                #       set.
+                self._deferred = reactor.callLater(0, self.success)
+            else:
+                self._deferred = reactor.callLater(0, self.diffs_found)
 
         def failure(reason):
             self.log.info('resync-failure', reason=reason)
             self._current_task = None
+            self._on_olt_only_diffs = None
+            self._on_onu_only_diffs = None
+            self._attr_diffs = None
             self._deferred = reactor.callLater(self._timeout_delay, self.timeout)
 
         self._current_task = self._resync_task(self._agent, self._device_id)
@@ -391,7 +478,7 @@
         :param _topic: (str) OMCI-RX topic
         :param msg: (dict) Dictionary with 'rx-response' and 'tx-request' (if any)
         """
-        self.log.info('on-mib-reset-response', state=self.state)
+        self.log.debug('on-mib-reset-response', state=self.state)
         try:
             response = msg[RX_RESPONSE_KEY]
 
@@ -426,7 +513,7 @@
         :param _topic: (str) OMCI-RX topic
         :param msg: (dict) Dictionary with 'rx-response' and 'tx-request' (if any)
         """
-        self.log.info('on-avc-notification', state=self.state)
+        self.log.debug('on-avc-notification', state=self.state)
 
         if self._subscriptions[RxEvent.AVC_Notification]:
             try:
@@ -454,8 +541,8 @@
 
                     if changed:
                         # Autonomous creation and deletion of managed entities do not
-                        # result in an incrwment of the MIB data sync value. However,
-                        # AVC's in response to a change by the Operater do incur an
+                        # result in an increment of the MIB data sync value. However,
+                        # AVC's in response to a change by the Operator do incur an
                         # increment of the MIB Data Sync
                         pass
 
@@ -532,7 +619,7 @@
         :param _topic: (str) OMCI-RX topic
         :param msg: (dict) Dictionary with 'rx-response' and 'tx-request' (if any)
         """
-        self.log.info('on-create-response', state=self.state)
+        self.log.debug('on-create-response', state=self.state)
 
         if self._subscriptions[RxEvent.Create]:
             if self.state in ['disabled', 'uploading']:
@@ -541,15 +628,16 @@
             try:
                 request = msg[TX_REQUEST_KEY]
                 response = msg[RX_RESPONSE_KEY]
+                status = response.fields['omci_message'].fields['success_code']
 
-                if response.fields['omci_message'].fields['success_code'] != RC.Success:
+                if status != RC.Success and status != RC.InstanceExists:
                     # TODO: Support offline ONTs in post VOLTHA v1.3.0
                     omci_msg = response.fields['omci_message']
                     self.log.warn('set-response-failure',
                                   class_id=omci_msg.fields['entity_class'],
                                   instance_id=omci_msg.fields['entity_id'],
-                                  status=omci_msg.fields['status_code'],
-                                  status_text=self._status_to_text(omci_msg.fields['status_code']),
+                                  status=omci_msg.fields['success_code'],
+                                  status_text=self._status_to_text(omci_msg.fields['success_code']),
                                   parameter_error_attributes_mask=omci_msg.fields['parameter_error_attributes_mask'])
                 else:
                     omci_msg = request.fields['omci_message'].fields
@@ -575,7 +663,7 @@
         :param _topic: (str) OMCI-RX topic
         :param msg: (dict) Dictionary with 'rx-response' and 'tx-request' (if any)
         """
-        self.log.info('on-delete-response', state=self.state)
+        self.log.debug('on-delete-response', state=self.state)
 
         if self._subscriptions[RxEvent.Delete]:
             if self.state in ['disabled', 'uploading']:
@@ -591,8 +679,8 @@
                     self.log.warn('set-response-failure',
                                   class_id=omci_msg.fields['entity_class'],
                                   instance_id=omci_msg.fields['entity_id'],
-                                  status=omci_msg.fields['status_code'],
-                                  status_text=self._status_to_text(omci_msg.fields['status_code']))
+                                  status=omci_msg.fields['success_code'],
+                                  status_text=self._status_to_text(omci_msg.fields['success_code']))
                 else:
                     omci_msg = request.fields['omci_message'].fields
                     class_id = omci_msg['entity_class']
@@ -616,7 +704,7 @@
         :param _topic: (str) OMCI-RX topic
         :param msg: (dict) Dictionary with 'rx-response' and 'tx-request' (if any)
         """
-        self.log.info('on-set-response', state=self.state)
+        self.log.debug('on-set-response', state=self.state)
 
         if self._subscriptions[RxEvent.Set]:
             if self.state in ['disabled', 'uploading']:
@@ -631,8 +719,8 @@
                     self.log.warn('set-response-failure',
                                   class_id=omci_msg.fields['entity_class'],
                                   instance_id=omci_msg.fields['entity_id'],
-                                  status=omci_msg.fields['status_code'],
-                                  status_text=self._status_to_text(omci_msg.fields['status_code']),
+                                  status=omci_msg.fields['success_code'],
+                                  status_text=self._status_to_text(omci_msg.fields['success_code']),
                                   unsupported_attribute_mask=omci_msg.fields['unsupported_attributes_mask'],
                                   failed_attribute_mask=omci_msg.fields['failed_attributes_mask'])
                 else:
diff --git a/voltha/extensions/omci/tasks/get_mds_task.py b/voltha/extensions/omci/tasks/get_mds_task.py
index b042877..4722eb5 100644
--- a/voltha/extensions/omci/tasks/get_mds_task.py
+++ b/voltha/extensions/omci/tasks/get_mds_task.py
@@ -87,7 +87,10 @@
 
             omci_msg = results.fields['omci_message'].fields
             status = omci_msg['success_code']
-            self.log.debug('ont-data-mds', status=status)
+            self.log.debug('ont-data-mds', status=status,
+                           mib_data_sync=omci_msg['data']['mib_data_sync']
+                           if 'data' in omci_msg and 'mib_data_sync' in omci_msg['data']
+                           else None)
 
             assert status == RC.Success, 'Unexpected Response Status: {}'.format(status)
 
diff --git a/voltha/extensions/omci/tasks/mib_resync_task.py b/voltha/extensions/omci/tasks/mib_resync_task.py
index 923c4ef..e39e96b 100644
--- a/voltha/extensions/omci/tasks/mib_resync_task.py
+++ b/voltha/extensions/omci/tasks/mib_resync_task.py
@@ -14,13 +14,13 @@
 # limitations under the License.
 #
 from task import Task
-from datetime import datetime
 from twisted.internet.defer import inlineCallbacks, TimeoutError, failure, returnValue
 from twisted.internet import reactor
 from common.utils.asleep import asleep
 from voltha.extensions.omci.database.mib_db_dict import *
-from voltha.extensions.omci.omci_defs import ReasonCodes
 from voltha.extensions.omci.omci_entities import OntData
+from voltha.extensions.omci.omci_defs import AttributeAccess
+AA = AttributeAccess
 
 
 class MibCopyException(Exception):
@@ -64,7 +64,7 @@
         self._local_deferred = None
         self._device = omci_agent.get_device(device_id)
         self._db_active = MibDbVolatileDict(omci_agent)
-        self._db_active.add(device_id)
+        self._db_active.start()
 
     def cancel_deferred(self):
         super(MibResyncTask, self).cancel_deferred()
@@ -78,21 +78,23 @@
 
     def start(self):
         """
-        Start MIB Synchronization tasks
+        Start MIB Re-Synchronization task
         """
         super(MibResyncTask, self).start()
         self._local_deferred = reactor.callLater(0, self.perform_mib_resync)
         self._db_active.start()
+        self._db_active.add(self.device_id)
 
     def stop(self):
         """
-        Shutdown MIB Synchronization tasks
+        Shutdown MIB Re-Synchronization task
         """
         self.log.debug('stopping')
 
         self.cancel_deferred()
         self._device = None
         self._db_active.stop()
+        self._db_active = None
         super(MibResyncTask, self).stop()
 
     @inlineCallbacks
@@ -100,7 +102,7 @@
         """
         Perform the MIB Resynchronization sequence
 
-        The sequence to be performed are:
+        The sequence to be performed is:
             - get a copy of the current MIB database (db_copy)
 
             - perform MIB upload commands to get ONU's database and save this
@@ -115,47 +117,39 @@
         """
         self.log.info('perform-mib-resync')
 
-        # Try at least 3 times to snapshot the current MIB and get the
-        # MIB upload request out so ONU snapshots its database
-
-        db_copy = None
-        number_of_commands = None
-        commands_retrieved = 0
-
         try:
             results = yield self.snapshot_mib()
             db_copy = results[0]
-            number_of_commands = results[1]
 
-            # Start the MIB upload sequence
-            commands_retrieved = yield self.upload_mib(number_of_commands)
+            if db_copy is None:
+                e = MibCopyException('Failed to get local database copy')
+                self.deferred.errback(failure.Failure(e))
+
+            else:
+                number_of_commands = results[1]
+
+                # Start the MIB upload sequence
+                commands_retrieved = yield self.upload_mib(number_of_commands)
+
+                if commands_retrieved < number_of_commands:
+                    e = MibDownloadException('Only retrieved {} of {} instances'.
+                                             format(commands_retrieved, number_of_commands))
+                    self.deferred.errback(failure.Failure(e))
+                else:
+                    # Compare the databases
+                    on_olt_only, on_onu_only, attr_diffs = \
+                        self.compare_mibs(db_copy, self._db_active.query(self.device_id))
+
+                    self.deferred.callback(
+                            {
+                                'on-olt-only': on_olt_only if len(on_olt_only) else None,
+                                'on-onu-only': on_onu_only if len(on_onu_only) else None,
+                                'attr-diffs': attr_diffs if len(attr_diffs) else None
+                            })
 
         except Exception as e:
+            self.log.exception('resync', e=e)
             self.deferred.errback(failure.Failure(e))
-            returnValue(None)
-
-        if db_copy is None:
-            e = MibCopyException('Failed to get local database copy')
-            self.deferred.errback(failure.Failure(e))
-            returnValue('FAILED')
-
-        if commands_retrieved < number_of_commands:
-            e = MibDownloadException('Only retrieved {} of {} instances'.
-                                     format(commands_retrieved, number_of_commands))
-            self.deferred.errback(failure.Failure(e))
-            returnValue('FAILED')
-
-        # Compare the database
-
-        mib_differences = self.compare_mibs(db_copy,
-                                            self._db_active.query(self.device_id))
-
-        if mib_differences is None:
-            self.deferred.callback('success')
-            self.deferred.callback('TODO: This task has not been coded.')
-
-        # TODO: Handle mismatches
-        pass
 
     @inlineCallbacks
     def snapshot_mib(self):
@@ -173,11 +167,11 @@
             for retries in xrange(0, max_tries + 1):
                 # Send MIB Upload so ONU snapshots its MIB
                 try:
-                    mib_upload_time = datetime.utcnow()
                     number_of_commands = yield self.send_mib_upload()
 
                     if number_of_commands is None:
                         if retries >= max_tries:
+                            db_copy = None
                             break
 
                 except TimeoutError as e:
@@ -191,14 +185,6 @@
                 # Get a snapshot of the local MIB database
                 db_copy = self._device.query_mib()
 
-                if db_copy is None or db_copy[MODIFIED_KEY] > mib_upload_time:
-                    if retries >= max_tries:
-                        break
-
-                    yield asleep(MibResyncTask.db_copy_retry_delay)
-                    continue
-                break
-
         except Exception as e:
             self.log.exception('mib-resync', e=e)
             raise
@@ -237,13 +223,12 @@
     def upload_mib(self, number_of_commands):
         ########################################
         # Begin MIB Upload
-
         seq_no = None
 
         for seq_no in xrange(number_of_commands):
-            max_tries = MibResyncTask.max_mib_upload_next_retries - 1
+            max_tries = MibResyncTask.max_mib_upload_next_retries
 
-            for retries in xrange(0, max_tries + 1):
+            for retries in xrange(0, max_tries):
                 try:
                     response = yield self._device.omci_cc.send_mib_upload_next(seq_no)
 
@@ -253,37 +238,146 @@
 
                     # Filter out the 'mib_data_sync' from the database. We save that at
                     # the device level and do not want it showing up during a re-sync
-                    # during data compares
+                    # during data comparison
 
                     if class_id == OntData.class_id:
-                        pass      # TODO: Save to a local variable
+                        break
 
                     attributes = {k: v for k, v in omci_msg['object_data'].items()}
 
                     # Save to the database
                     self._db_active.set(self.device_id, class_id, entity_id, attributes)
+                    break
 
-                except TimeoutError as e:
-                    self.log.warn('mib-resync-timeout', e=e, seq_no=seq_no,
+                except TimeoutError:
+                    self.log.warn('mib-resync-timeout', seq_no=seq_no,
                                   number_of_commands=number_of_commands)
-                    if retries >= max_tries:
+
+                    if retries < max_tries - 1:
+                        yield asleep(MibResyncTask.mib_upload_next_delay)
+                    else:
                         raise
 
-                    yield asleep(MibResyncTask.mib_upload_next_delay)
-                    continue
+                except Exception as e:
+                    self.log.exception('resync', e=e, seq_no=seq_no,
+                                       number_of_commands=number_of_commands)
 
-        returnValue(seq_no)
+        returnValue(seq_no + 1)     # seq_no is zero based.
 
     def compare_mibs(self, db_copy, db_active):
         """
         Compare the our db_copy with the ONU's active copy
+
         :param db_copy: (dict) OpenOMCI's copy of the database
         :param db_active: (dict) ONU's database snapshot
-        :return: (dict) Difference dictionary
+        :return: (dict), (dict), dict()  Differences
         """
-        return None        # TODO: Do this
+        # Class & Entities only in local copy (OpenOMCI)
+        on_olt_only = self.get_lsh_only_dict(db_copy, db_active)
+
+        # Class & Entities only on remote (ONU)
+        on_onu_only = self.get_lsh_only_dict(db_active, db_copy)
+
+        # Class & Entities on both local & remote, but one or more attributes
+        # are different on the ONU.  This is the value that the local (OpenOMCI)
+        # thinks should be on the remote (ONU)
+
+        me_map = self.omci_agent.get_device(self.device_id).me_map
+        attr_diffs = self.get_attribute_diffs(db_copy, db_active, me_map)
+
         # TODO: Note that certain MEs are excluded from the MIB upload.  In particular,
-        #       instances of some gneeral purpose MEs, such as the Managed Entity ME and
+        #       instances of some general purpose MEs, such as the Managed Entity ME and
         #       and the Attribute ME are not included in the MIB upload.  Also all table
         #       attributes are not included in the MIB upload (but we do not yet support
         #       tables in this OpenOMCI implementation (VOLTHA v1.3.0)
+
+        return on_olt_only, on_onu_only, attr_diffs
+
+    def get_lsh_only_dict(self, lhs, rhs):
+        """
+        Compare two MIB database dictionaries and return the ME Class ID and
+        instances that are unique to the lhs dictionary. Both parameters
+        should be in the common MIB Database output dictionary format that
+        is returned by the mib 'query' command.
+
+        :param lhs: (dict) Left-hand-side argument.
+        :param rhs: (dict) Right-hand-side argument
+
+        return: (list(int,int)) List of tuples where (class_id, inst_id)
+        """
+        results = list()
+
+        for cls_id, cls_data in lhs.items():
+            # Get unique classes
+            #
+            # Skip keys that are not class IDs
+            if not isinstance(cls_id, int):
+                continue
+
+            if cls_id not in rhs:
+                results.extend([(cls_id, inst_id) for inst_id in cls_data.keys()
+                                if isinstance(inst_id, int)])
+            else:
+                # Get unique instances of a class
+                lhs_cls = cls_data
+                rhs_cls = rhs[cls_id]
+
+                for inst_id, _ in lhs_cls.items():
+                    # Skip keys that are not instance IDs
+                    if isinstance(cls_id, int) and inst_id not in rhs_cls:
+                        results.extend([(cls_id, inst_id)])
+
+        return results
+
+    def get_attribute_diffs(self, omci_copy, onu_copy, me_map):
+        """
+        Compare two OMCI MIBs and return the ME class and instance IDs that exists
+        on both the local copy and the remote ONU that have different attribute
+        values. Both parameters should be in the common MIB Database output
+        dictionary format that is returned by the mib 'query' command.
+
+        :param omci_copy: (dict) OpenOMCI copy (OLT-side) of the MIB Database
+        :param onu_copy: (dict) active ONU latest copy its database
+        :param me_map: (dict) ME Class ID MAP for this ONU
+
+        return: (list(int,int,str)) List of tuples where (class_id, inst_id, attribute)
+                                    points to the specific ME instance where attributes
+                                    are different
+        """
+        results = list()
+        ro_set = {AA.R}
+
+        # Get class ID's that are in both
+        class_ids = {cls_id for cls_id, _ in omci_copy.items()
+                     if isinstance(cls_id, int) and cls_id in onu_copy}
+
+        for cls_id in class_ids:
+            # Get unique instances of a class
+            olt_cls = omci_copy[cls_id]
+            onu_cls = onu_copy[cls_id]
+
+            # Weed out read-only attributes. Attributes on onu may be read-only. These
+            # will only show up it the OpenOMCI (OLT-side) database if it changed and
+            # an AVC Notification was sourced by the ONU
+            # TODO: These could be calculated once at ONU startup (device add)
+            ro_attrs = {attr.field.name for attr in me_map[cls_id].attributes
+                        if attr.access == ro_set}
+
+            # Get set of common instance IDs
+            inst_ids = {inst_id for inst_id, _ in olt_cls.items()
+                        if isinstance(inst_id, int) and inst_id in onu_cls}
+
+            for inst_id in inst_ids:
+                omci_attributes = {k for k in olt_cls[inst_id][ATTRIBUTES_KEY].iterkeys()}
+                onu_attributes = {k for k in onu_cls[inst_id][ATTRIBUTES_KEY].iterkeys()}
+
+                # Get attributes that exist in one database, but not the other
+                sym_diffs = (omci_attributes ^ onu_attributes) - ro_attrs
+                results.extend([(cls_id, inst_id, attr) for attr in sym_diffs])
+
+                # Get common attributes with different values
+                common_attributes = (omci_attributes & onu_attributes) - ro_attrs
+                results.extend([(cls_id, inst_id, attr) for attr in common_attributes
+                               if olt_cls[inst_id][ATTRIBUTES_KEY][attr] !=
+                                onu_cls[inst_id][ATTRIBUTES_KEY][attr]])
+        return results