VOL-697: Improvement to MIB database output - prep for MIB Audit work
Try to get jenkins label now that omci_entities.py conflict resolved

Change-Id: Ic553172087a6b64811cd7372f837a71270df9320
diff --git a/tests/utests/voltha/extensions/omci/test_mib_db_dict.py b/tests/utests/voltha/extensions/omci/test_mib_db_dict.py
new file mode 100644
index 0000000..be42733
--- /dev/null
+++ b/tests/utests/voltha/extensions/omci/test_mib_db_dict.py
@@ -0,0 +1,502 @@
+#
+# Copyright 2017 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from unittest import main, TestCase
+
+from voltha.extensions.omci.omci_entities import *
+from voltha.extensions.omci.database.mib_db_dict import *
+from voltha.extensions.omci.database.mib_db_api import MODIFIED_KEY, CREATED_KEY,\
+    DEVICE_ID_KEY, MDS_KEY, LAST_SYNC_KEY
+from mock.mock_adapter_agent import MockAdapterAgent, MockDevice
+from nose.tools import raises, assert_raises
+import time
+
+_DEVICE_ID = 'br-549'
+
+
+class TestOmciMibDbDict(TestCase):
+
+    def setUp(self):
+        self.adapter_agent = MockAdapterAgent()
+        self.adapter_agent.add_device(MockDevice(_DEVICE_ID))  # For Entity class lookups
+        self.db = MibDbVolatileDict(self.adapter_agent)
+
+    def tearDown(self):
+        self.db.stop()
+
+    def test_start_stop(self):
+        # Simple start stop
+        self.assertFalse(self.db.active)
+        self.db.start()
+        self.assertTrue(self.db.active)
+        self.db.stop()
+        self.assertFalse(self.db.active)
+
+        # Start after start still okay
+        self.db.start()
+        self.db.start()
+        self.assertTrue(self.db.active)
+
+        self.db.stop()
+        self.db.stop()
+        self.assertFalse(self.db.active)
+
+    @raises(DatabaseStateError)
+    def test_bad_state_add(self):
+        self.db.add(_DEVICE_ID)
+
+    @raises(DatabaseStateError)
+    def test_bad_state_remove(self):
+        self.db.remove(_DEVICE_ID)
+
+    @raises(DatabaseStateError)
+    def test_bad_state_query_1(self):
+        self.db.query(_DEVICE_ID, 0)
+
+    @raises(DatabaseStateError)
+    def test_bad_state_query_2(self):
+        self.db.query(_DEVICE_ID, 0, 0)
+
+    @raises(DatabaseStateError)
+    def test_bad_state_query_3(self):
+        self.db.query(_DEVICE_ID, 0, 0, 'test')
+
+    @raises(DatabaseStateError)
+    def test_bad_state_set(self):
+        self.db.set(_DEVICE_ID, 0, 0, {'test': 123})
+
+    @raises(DatabaseStateError)
+    def test_bad_state_delete(self):
+        self.db.delete(_DEVICE_ID, 0, 0)
+
+    @raises(KeyError)
+    def test_no_device_query(self):
+        self.db.start()
+        self.db.query(_DEVICE_ID)
+
+    def test_no_device_last_sync(self):
+        self.db.start()
+        # Returns None, not a KeyError
+        value = self.db.get_last_sync(_DEVICE_ID)
+        self.assertIsNone(value)
+
+    def test_no_device_mds(self):
+        self.db.start()
+        # Returns None, not a KeyError
+        value = self.db.get_mib_data_sync(_DEVICE_ID)
+        self.assertIsNone(value)
+
+    @raises(KeyError)
+    def test_no_device_save_last_sync(self):
+        self.db.start()
+        self.db.save_last_sync(_DEVICE_ID, datetime.utcnow())
+
+    @raises(KeyError)
+    def test_no_device_save_mds(self):
+        self.db.start()
+        self.db.save_mib_data_sync(_DEVICE_ID, 123)
+
+    def test_param_types(self):
+        self.db.start()
+        assert_raises(TypeError, self.db.add, 123)
+        assert_raises(TypeError, self.db.remove, 123)
+        assert_raises(TypeError, self.db.query, 123)
+
+        assert_raises(TypeError, self.db.get_mib_data_sync, 123)
+        assert_raises(TypeError, self.db.save_mib_data_sync, 123, 0)
+        assert_raises(TypeError, self.db.save_mib_data_sync, _DEVICE_ID, 'zero')
+
+        assert_raises(TypeError, self.db.get_last_sync, 123)
+        assert_raises(TypeError, self.db.save_last_sync, 123, datetime.utcnow())
+        assert_raises(TypeError, self.db.save_last_sync, _DEVICE_ID, 'bad-date')
+
+        assert_raises(TypeError, self.db.set, 123, 0, 0, {'test': 0})
+        assert_raises(TypeError, self.db.set, None, 0, 0, {'test': 0})
+        assert_raises(ValueError, self.db.set, _DEVICE_ID, None, 0, {'test': 0})
+        assert_raises(ValueError, self.db.set, _DEVICE_ID, 0, None, {'test': 0})
+        assert_raises(TypeError, self.db.set, _DEVICE_ID, 0, 0, None)
+        assert_raises(TypeError, self.db.set, _DEVICE_ID, 0, 0, 'not-a-dict')
+
+        assert_raises(ValueError, self.db.set, _DEVICE_ID, -1, 0, {'test': 0})
+        assert_raises(ValueError, self.db.set, _DEVICE_ID, 0x10000, 0, {'test': 0})
+        assert_raises(ValueError, self.db.set, _DEVICE_ID, 0, -1, {'test': 0})
+        assert_raises(ValueError, self.db.set, _DEVICE_ID, 0, 0x10000, {'test': 0})
+
+        assert_raises(TypeError, self.db.delete, 123, 0, 0)
+        assert_raises(ValueError, self.db.delete, _DEVICE_ID, -1, 0)
+        assert_raises(ValueError, self.db.delete, _DEVICE_ID, 0x10000, 0)
+        assert_raises(ValueError, self.db.delete, _DEVICE_ID, 0, -1)
+        assert_raises(ValueError, self.db.delete, _DEVICE_ID, 0, 0x10000)
+
+    def test_add_remove_device(self):
+        self.db.start()
+
+        # Remove of non-existent device is not an error
+        assert_raises(KeyError, self.db.query, _DEVICE_ID)
+        self.db.remove(_DEVICE_ID)
+
+        start_time = datetime.utcnow()
+        self.db.add(_DEVICE_ID)
+        dev_data = self.db.query(_DEVICE_ID)
+        end_time = datetime.utcnow()
+
+        self.assertEqual(dev_data[DEVICE_ID_KEY], _DEVICE_ID)
+        self.assertEquals(dev_data[MDS_KEY], 0)
+        self.assertIsNone(dev_data[LAST_SYNC_KEY])
+        self.assertEqual(dev_data[VERSION_KEY], MibDbVolatileDict.CURRENT_VERSION)
+
+        # Remove it
+        self.db.remove(_DEVICE_ID)
+        assert_raises(KeyError, self.db.query, _DEVICE_ID)
+
+        # Remove of non-existant dev okay
+        self.db.remove(_DEVICE_ID +'abcd')
+
+        # Overwrite tests
+        self.db.add(_DEVICE_ID)
+        assert_raises(KeyError, self.db.add, _DEVICE_ID)
+        self.db.add(_DEVICE_ID, overwrite=True)  # This is okay
+
+    def test_mib_data_sync(self):
+        self.db.start()
+        self.db.add(_DEVICE_ID)
+        self.assertEquals(self.db.get_mib_data_sync(_DEVICE_ID), 0)
+
+        self.db.save_mib_data_sync(_DEVICE_ID, 100)
+        self.assertEqual(self.db.get_mib_data_sync(_DEVICE_ID), 100)
+
+        assert_raises(ValueError, self.db.save_mib_data_sync, _DEVICE_ID, -1)
+        assert_raises(ValueError, self.db.save_mib_data_sync, _DEVICE_ID, 256)
+
+    def test_last_sync(self):
+        self.db.start()
+        self.assertIsNone(self.db.get_last_sync(_DEVICE_ID))
+
+        self.db.add(_DEVICE_ID)
+        self.assertIsNone(self.db.get_last_sync(_DEVICE_ID))
+
+        now = datetime.utcnow()
+
+        self.db.save_last_sync(_DEVICE_ID, now)
+        self.assertEqual(self.db.get_last_sync(_DEVICE_ID), now)
+
+        assert_raises(TypeError, self.db.save_last_sync, _DEVICE_ID, 'hello')
+
+    def test_set_and_query(self):
+        self.db.start()
+        self.db.add(_DEVICE_ID)     # Base device DB created here
+        time.sleep(0.1)
+
+        class_id = OntG.class_id
+        inst_id = 0
+        attributes = {'vendor_id': 'ABCD'}
+
+        start_time = datetime.utcnow()
+        set_occurred = self.db.set(_DEVICE_ID, class_id, inst_id, attributes)
+        self.assertTrue(set_occurred)
+        end_time = datetime.utcnow()
+
+        dev_data = self.db.query(_DEVICE_ID)
+        self.assertEqual(dev_data[DEVICE_ID_KEY], _DEVICE_ID)
+
+        dev_classes = [v for k, v in dev_data.items() if isinstance(k, int)]
+
+        self.assertEqual(len(dev_classes), 1)
+        class_data = dev_classes[0]
+
+        self.assertEqual(class_data[CLASS_ID_KEY], class_id)
+
+        class_insts = [v for k, v in class_data.items() if isinstance(k, int)]
+
+        self.assertEqual(len(class_insts), 1)
+        inst_data = class_insts[0]
+
+        self.assertEqual(inst_data[INSTANCE_ID_KEY], inst_id)
+        self.assertGreaterEqual(inst_data[MODIFIED_KEY], start_time)
+        self.assertLessEqual(inst_data[MODIFIED_KEY], end_time)
+        self.assertLessEqual(inst_data[CREATED_KEY], inst_data[MODIFIED_KEY])
+
+        inst_attributes = inst_data[ATTRIBUTES_KEY]
+        self.assertEqual(len(inst_attributes), 1)
+
+        self.assertTrue('vendor_id' in inst_attributes)
+        self.assertEqual(inst_attributes['vendor_id'], attributes['vendor_id'])
+
+        ########################################
+        # Query with device and class. Should be same as from full device query
+        cls_2_data = self.db.query(_DEVICE_ID, class_id)
+
+        self.assertEqual(class_data[CLASS_ID_KEY], cls_2_data[CLASS_ID_KEY])
+
+        cl2_insts = {k:v for k, v in cls_2_data.items() if isinstance(k, int)}
+        self.assertEqual(len(cl2_insts), len(class_insts))
+
+        # Bad class id query
+        cls_no_data = self.db.query(_DEVICE_ID, class_id + 1)
+        self.assertTrue(isinstance(cls_no_data, dict))
+        self.assertEqual(len(cls_no_data), 0)
+
+        ########################################
+        # Query with device, class, instance
+        inst_2_data = self.db.query(_DEVICE_ID, class_id, inst_id)
+
+        self.assertEqual(inst_data[INSTANCE_ID_KEY], inst_2_data[INSTANCE_ID_KEY])
+        self.assertEqual(inst_data[MODIFIED_KEY], inst_2_data[MODIFIED_KEY])
+        self.assertEqual(inst_data[CREATED_KEY], inst_2_data[CREATED_KEY])
+
+        inst2_attr = inst_2_data[ATTRIBUTES_KEY]
+        self.assertEqual(len(inst2_attr), len(inst_attributes))
+
+        # Bad instance id query
+        inst_no_data = self.db.query(_DEVICE_ID, class_id, inst_id + 100)
+        self.assertTrue(isinstance(inst_no_data, dict))
+        self.assertEqual(len(inst_no_data), 0)
+
+        ########################################
+        # Attribute queries
+        attr_2_data = self.db.query(_DEVICE_ID, class_id, inst_id, 'vendor_id')
+        self.assertEqual(attr_2_data['vendor_id'], attributes['vendor_id'])
+
+        attr_3_data = self.db.query(_DEVICE_ID, class_id, inst_id, ['vendor_id'])
+        self.assertEqual(attr_3_data['vendor_id'], attributes['vendor_id'])
+
+        attr_4_data = self.db.query(_DEVICE_ID, class_id, inst_id, {'vendor_id'})
+        self.assertEqual(attr_4_data['vendor_id'], attributes['vendor_id'])
+
+        attr_no_data = self.db.query(_DEVICE_ID, class_id, inst_id, 'no_such_thing')
+        self.assertTrue(isinstance(attr_no_data, dict))
+        self.assertEqual(len(attr_no_data), 0)
+
+        # Set to same value does not change modified data.  The modified is
+        # at the instance level
+
+        class_id = OntG.class_id
+        inst_id = 0
+        attributes = {'vendor_id': 'ABCD'}
+        set_occurred = self.db.set(_DEVICE_ID, class_id, inst_id, attributes)
+        self.assertFalse(set_occurred)
+
+        inst_3_data = self.db.query(_DEVICE_ID, class_id, inst_id)
+        self.assertEqual(inst_data[MODIFIED_KEY], inst_3_data[MODIFIED_KEY])
+        self.assertEqual(inst_data[CREATED_KEY], inst_3_data[CREATED_KEY])
+
+        # But set to new value does
+        time.sleep(0.1)
+        attributes = {'vendor_id': 'WXYZ'}
+        set_occurred = self.db.set(_DEVICE_ID, class_id, inst_id, attributes)
+        self.assertTrue(set_occurred)
+
+        inst_4_data = self.db.query(_DEVICE_ID, class_id, inst_id)
+        self.assertLess(inst_3_data[MODIFIED_KEY], inst_4_data[MODIFIED_KEY])
+        self.assertEqual(inst_3_data[CREATED_KEY], inst_4_data[CREATED_KEY])
+
+    def test_delete_instances(self):
+        self.db.start()
+        self.db.add(_DEVICE_ID)
+        create_time = datetime.utcnow()
+
+        class_id = GalEthernetProfile.class_id
+        inst_id_1 = 0x100
+        inst_id_2 = 0x200
+        attributes = {'max_gem_payload_size': 1500}
+
+        self.db.set(_DEVICE_ID, class_id, inst_id_1, attributes)
+        self.db.set(_DEVICE_ID, class_id, inst_id_2, attributes)
+        set_time = datetime.utcnow()
+        time.sleep(0.1)
+
+        dev_data = self.db.query(_DEVICE_ID)
+        cls_data = self.db.query(_DEVICE_ID, class_id)
+        inst_data = {k: v for k, v in cls_data.items() if isinstance(k, int)}
+        self.assertEqual(len(inst_data), 2)
+
+        self.assertLessEqual(dev_data[CREATED_KEY], create_time)
+        self.assertLessEqual(self.db.created, create_time)
+
+        # Delete one instance
+        time.sleep(0.1)
+        del_time = datetime.utcnow()
+        result = self.db.delete(_DEVICE_ID, class_id, inst_id_1)
+        self.assertTrue(result)     # True returned if a del actually happened
+
+        dev_data = self.db.query(_DEVICE_ID)
+        cls_data = self.db.query(_DEVICE_ID, class_id)
+        inst_data = {k: v for k, v in cls_data.items() if isinstance(k, int)}
+        self.assertEqual(len(inst_data), 1)
+
+        self.assertLessEqual(dev_data[CREATED_KEY], create_time)
+        self.assertLessEqual(self.db.created, create_time)
+
+        # Delete remaining instance
+        time.sleep(0.1)
+        result = self.db.delete(_DEVICE_ID, class_id, inst_id_2)
+        self.assertTrue(result)     # True returned if a del actually happened
+
+        dev_data = self.db.query(_DEVICE_ID)
+        cls_data = {k: v for k, v in dev_data.items() if isinstance(k, int)}
+        self.assertEqual(len(cls_data), 0)
+        self.assertLessEqual(dev_data[CREATED_KEY], create_time)
+
+        # Delete returns false if not instance
+        self.assertFalse(self.db.delete(_DEVICE_ID, class_id, inst_id_1))
+        self.assertFalse(self.db.delete(_DEVICE_ID, class_id, inst_id_2))
+
+    def test_on_mib_reset_listener(self):
+        self.db.start()
+        self.db.add(_DEVICE_ID)
+        time.sleep(0.1)
+
+        class_id = OntG.class_id
+        inst_id = 0
+        attributes = {'vendor_id': 'ABCD'}
+
+        set_time = datetime.utcnow()
+        self.db.set(_DEVICE_ID, class_id, inst_id, attributes)
+
+        time.sleep(0.1)
+        self.db.on_mib_reset(_DEVICE_ID)
+
+        dev_data = self.db.query(_DEVICE_ID)
+        self.assertEqual(dev_data[DEVICE_ID_KEY], _DEVICE_ID)
+        self.assertLessEqual(dev_data[CREATED_KEY], set_time)
+        self.assertLessEqual(self.db.created, set_time)
+
+        self.assertFalse(any(isinstance(cls, int) for cls in dev_data.iterkeys()))
+
+    def test_str_field_serialization(self):
+        self.db.start()
+        self.db.add(_DEVICE_ID)
+
+        class_id = OltG.class_id
+        inst_id = 0
+        attributes = {
+            'olt_vendor_id': 'ABCD',             # StrFixedLenField(4)
+        }
+        self.db.set(_DEVICE_ID, class_id, inst_id, attributes)
+        data = self.db.query(_DEVICE_ID, class_id, inst_id, attributes.keys())
+        self.assertTrue(all(isinstance(data[k], basestring) for k in attributes.keys()))
+        self.assertTrue(all(data[k] == attributes[k] for k in attributes.keys()))
+
+    def test_mac_address_ip_field_serialization(self):
+        self.db.start()
+        self.db.add(_DEVICE_ID)
+
+        class_id = IpHostConfigData.class_id
+        inst_id = 0
+        attributes = {
+            'mac_address': '00:01:02:03:04:05',             # MACField
+            'ip_address': '1.2.3.4',                        # IPField
+        }
+        self.db.set(_DEVICE_ID, class_id, inst_id, attributes)
+        data = self.db.query(_DEVICE_ID, class_id, inst_id, attributes.keys())
+        self.assertTrue(all(isinstance(data[k], basestring) for k in attributes.keys()))
+        self.assertTrue(all(data[k] == attributes[k] for k in attributes.keys()))
+
+    def test_byte_and_short_field_serialization(self):
+        self.db.start()
+        self.db.add(_DEVICE_ID)
+
+        class_id = UniG.class_id
+        inst_id = 0
+        attributes = {
+            'administrative_state': int(1),                # ByteField
+            'non_omci_management_identifier': int(12345)   # IPField
+        }
+        self.db.set(_DEVICE_ID, class_id, inst_id, attributes)
+        data = self.db.query(_DEVICE_ID, class_id, inst_id, attributes.keys())
+        self.assertTrue(all(isinstance(data[k], type(attributes[k])) for k in attributes.keys()))
+        self.assertTrue(all(data[k] == attributes[k] for k in attributes.keys()))
+
+    def test_int_field_serialization(self):
+        self.db.start()
+        self.db.add(_DEVICE_ID)
+
+        class_id = PriorityQueueG.class_id
+        inst_id = 0
+        attributes = {
+            'related_port': int(1234567)    # IntField
+        }
+        self.db.set(_DEVICE_ID, class_id, inst_id, attributes)
+        data = self.db.query(_DEVICE_ID, class_id, inst_id, attributes.keys())
+        self.assertTrue(all(isinstance(data[k], type(attributes[k])) for k in attributes.keys()))
+        self.assertTrue(all(data[k] == attributes[k] for k in attributes.keys()))
+
+    def test_long_field_serialization(self):
+        self.db.start()
+        self.db.add(_DEVICE_ID)
+
+        class_id = PriorityQueueG.class_id
+        inst_id = 0
+        attributes = {
+            'packet_drop_queue_thresholds': int(0x1234)        # LongField
+        }
+        self.db.set(_DEVICE_ID, class_id, inst_id, attributes)
+        data = self.db.query(_DEVICE_ID, class_id, inst_id, attributes.keys())
+        self.assertTrue(all(isinstance(data[k], type(attributes[k])) for k in attributes.keys()))
+        self.assertTrue(all(data[k] == attributes[k] for k in attributes.keys()))
+
+    def test_bit_field_serialization(self):
+        self.db.start()
+        self.db.add(_DEVICE_ID)
+
+        class_id = OntG.class_id
+        inst_id = 0
+        attributes = {
+            'extended_tc_layer_options': long(0x1234),        # BitField(16)
+        }
+        self.db.set(_DEVICE_ID, class_id, inst_id, attributes)
+        data = self.db.query(_DEVICE_ID, class_id, inst_id, attributes.keys())
+        self.assertTrue(all(isinstance(data[k], type(attributes[k])) for k in attributes.keys()))
+        self.assertTrue(all(data[k] == attributes[k] for k in attributes.keys()))
+
+    def test_complex_json_serialization(self):
+        self.db.start()
+        self.db.add(_DEVICE_ID)
+
+        class_id = ExtendedVlanTaggingOperationConfigurationData.class_id
+        inst_id = 0x202
+        table_data = VlanTaggingOperation(
+            filter_outer_priority=15,
+            filter_inner_priority=8,
+            filter_inner_vid=1024,
+            filter_inner_tpid_de=5,
+            filter_ether_type=0,
+            treatment_tags_to_remove=1,
+            pad3=2,
+            treatment_outer_priority=15,
+            treatment_inner_priority=8,
+            treatment_inner_vid=1024,
+            treatment_inner_tpid_de=4
+        )
+        attributes = dict(
+            received_frame_vlan_tagging_operation_table=table_data
+        )
+        self.db.set(_DEVICE_ID, class_id, inst_id, attributes)
+
+        data = self.db.query(_DEVICE_ID, class_id, inst_id, attributes.keys())
+        table_as_dict = json.loads(table_data.to_json())
+
+        self.assertTrue(all(isinstance(data['received_frame_vlan_tagging_operation_table'][k],
+                                       type(attributes['received_frame_vlan_tagging_operation_table'].fields[k]))
+                            for k in attributes['received_frame_vlan_tagging_operation_table'].fields.keys()))
+        self.assertTrue(all(data['received_frame_vlan_tagging_operation_table'][k] ==
+                            attributes['received_frame_vlan_tagging_operation_table'].fields[k]
+                            for k in attributes['received_frame_vlan_tagging_operation_table'].fields.keys()))
+        self.assertTrue(all(data['received_frame_vlan_tagging_operation_table'][k] == table_as_dict[k]
+                            for k in table_as_dict.keys()))
+
+
+if __name__ == '__main__':
+    main()
diff --git a/tests/utests/voltha/extensions/omci/test_mib_db_ext.py b/tests/utests/voltha/extensions/omci/test_mib_db_ext.py
index b28e027..725da65 100644
--- a/tests/utests/voltha/extensions/omci/test_mib_db_ext.py
+++ b/tests/utests/voltha/extensions/omci/test_mib_db_ext.py
@@ -1,9 +1,20 @@
-
+#
+# Copyright 2017 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
 from unittest import main, TestCase
 
-from voltha.protos.omci_mib_db_pb2 import MibInstanceData, MibClassData, \
-    MibDeviceData, MibAttributeData
-from datetime import datetime
 from voltha.extensions.omci.database.mib_db_ext import *
 from voltha.extensions.omci.database.mib_db_api import MODIFIED_KEY, CREATED_KEY,\
     DEVICE_ID_KEY, MDS_KEY, LAST_SYNC_KEY
@@ -14,7 +25,7 @@
 _DEVICE_ID = 'br-549'
 
 
-class TestOmciMibDb(TestCase):
+class TestOmciMibDbExt(TestCase):
 
     def setUp(self):
         self.adapter_agent = MockAdapterAgent()
@@ -376,14 +387,6 @@
         self.assertTrue(all(isinstance(data[k], basestring) for k in attributes.keys()))
         self.assertTrue(all(data[k] == attributes[k] for k in attributes.keys()))
 
-    def test_object_as_a_str_field_serialization(self):
-        # Some entity classes such as ExtendedVlanTaggingOperationConfigurationData
-        # (class-id = 171) have a Scapy Packet derived object (VlanTaggingOperation)
-        # that in the parent object is a StrFixedLenField.
-        # These classes should encode into a JSON string.  Test this serialization
-        # as well as deserialization
-        pass    # TODO: More tests here
-
     def test_mac_address_ip_field_serialization(self):
         self.db.start()
         self.db.add(_DEVICE_ID)
@@ -457,5 +460,42 @@
         self.assertTrue(all(data[k] == attributes[k] for k in attributes.keys()))
 
 
+    def test_complex_json_serialization(self):
+        self.db.start()
+        self.db.add(_DEVICE_ID)
+
+        class_id = ExtendedVlanTaggingOperationConfigurationData.class_id
+        inst_id = 0x202
+        table_data = VlanTaggingOperation(
+            filter_outer_priority=15,
+            filter_inner_priority=8,
+            filter_inner_vid=1024,
+            filter_inner_tpid_de=5,
+            filter_ether_type=0,
+            treatment_tags_to_remove=1,
+            pad3=2,
+            treatment_outer_priority=15,
+            treatment_inner_priority=8,
+            treatment_inner_vid=1024,
+            treatment_inner_tpid_de=4
+        )
+        attributes = dict(
+            received_frame_vlan_tagging_operation_table=table_data
+        )
+        self.db.set(_DEVICE_ID, class_id, inst_id, attributes)
+
+        data = self.db.query(_DEVICE_ID, class_id, inst_id, attributes.keys())
+        table_as_dict = json.loads(table_data.to_json())
+
+        self.assertTrue(all(isinstance(data['received_frame_vlan_tagging_operation_table'][k],
+                                       type(attributes['received_frame_vlan_tagging_operation_table'].fields[k]))
+                            for k in attributes['received_frame_vlan_tagging_operation_table'].fields.keys()))
+        self.assertTrue(all(data['received_frame_vlan_tagging_operation_table'][k] ==
+                            attributes['received_frame_vlan_tagging_operation_table'].fields[k]
+                            for k in attributes['received_frame_vlan_tagging_operation_table'].fields.keys()))
+        self.assertTrue(all(data['received_frame_vlan_tagging_operation_table'][k] == table_as_dict[k]
+                            for k in table_as_dict.keys()))
+
+
 if __name__ == '__main__':
     main()
diff --git a/voltha/extensions/omci/database/mib_db_dict.py b/voltha/extensions/omci/database/mib_db_dict.py
index d0cf98d..58d81c2 100644
--- a/voltha/extensions/omci/database/mib_db_dict.py
+++ b/voltha/extensions/omci/database/mib_db_dict.py
@@ -15,6 +15,7 @@
 #
 import copy
 from mib_db_api import *
+import json
 
 
 class MibDbVolatileDict(MibDbApi):
@@ -79,9 +80,8 @@
         self._data[device_id] = {
             DEVICE_ID_KEY: device_id,
             CREATED_KEY: now,
-            MODIFIED_KEY: now,
-            MDS_KEY: 0,
             LAST_SYNC_KEY: None,
+            MDS_KEY: 0,
             VERSION_KEY: MibDbVolatileDict.CURRENT_VERSION
         }
 
@@ -101,6 +101,7 @@
 
         if device_id in self._data:
             del self._data[device_id]
+            self._modified = datetime.utcnow()
 
     def on_mib_reset(self, device_id):
         """
@@ -108,33 +109,24 @@
 
         :param device_id: (str) ONU Device ID
         :raises DatabaseStateError: If the database is not enabled
+        :raises KeyError: If the device does not exist in the database
         """
         if not self._started:
             raise DatabaseStateError('The Database is not currently active')
 
-        now = datetime.utcnow()
-        device_db = self._data.get(device_id)
+        if not isinstance(device_id, basestring):
+            raise TypeError('Device ID should be an string')
 
-        if device_db is None:
-            self._data[device_id] = {
-                CREATED_KEY: now,
-                LAST_SYNC_KEY: None
-            }
-            device_db = self._data[device_id]
-        else:
-            created = device_db[CREATED_KEY]
-            last_sync = device_db[LAST_SYNC_KEY]
+        device_db = self._data[device_id]
+        self._modified = datetime.utcnow()
 
-            self._data[device_id] = {
-                CREATED_KEY: created,
-                LAST_SYNC_KEY: last_sync
-            }
-
-        device_db[DEVICE_ID_KEY] = device_id
-        device_db[MODIFIED_KEY] = now
-        device_db[MDS_KEY] = 0
-        device_db[VERSION_KEY] = MibDbVolatileDict.CURRENT_VERSION
-        self._modified = now
+        self._data[device_id] = {
+            DEVICE_ID_KEY: device_id,
+            CREATED_KEY: device_db[CREATED_KEY],
+            LAST_SYNC_KEY: device_db[LAST_SYNC_KEY],
+            MDS_KEY: 0,
+            VERSION_KEY: MibDbVolatileDict.CURRENT_VERSION
+        }
 
     def save_mib_data_sync(self, device_id, value):
         """
@@ -143,12 +135,18 @@
         :param device_id: (str) ONU Device ID
         :param value: (int) Value to save
         """
-        if device_id in self._data:
-            assert 0 <= value <= 255,\
-                'Invalid MIB Data Sync Value: {}'.format(value)
+        if not isinstance(device_id, basestring):
+            raise TypeError('Device ID should be an string')
 
-            self._data[device_id][MODIFIED_KEY] = datetime.utcnow()
-            self._data[device_id][MDS_KEY] = value
+        if not isinstance(value, int):
+            raise TypeError('MIB Data Sync is an integer')
+
+        if not 0 <= value <= 255:
+            raise ValueError('Invalid MIB-data-sync value {}.  Must be 0..255'.
+                             format(value))
+
+        self._data[device_id][MDS_KEY] = value
+        self._modified = datetime.utcnow()
 
     def get_mib_data_sync(self, device_id):
         """
@@ -157,6 +155,9 @@
         :param device_id: (str) ONU Device ID
         :return: (int) The Value or None if not found
         """
+        if not isinstance(device_id, basestring):
+            raise TypeError('Device ID should be an string')
+
         if device_id not in self._data:
             return None
 
@@ -169,9 +170,15 @@
         :param device_id: (str) ONU Device ID
         :param value: (DateTime) Value to save
         """
-        if device_id in self._data:
-            self._data[device_id][MODIFIED_KEY] = datetime.utcnow()
-            self._data[device_id][LAST_SYNC_KEY] = value
+        if not isinstance(device_id, basestring):
+            raise TypeError('Device ID should be an string')
+
+        if not isinstance(value, datetime):
+            raise TypeError('Expected a datetime object, got {}'.
+                            format(type(datetime)))
+
+        self._data[device_id][LAST_SYNC_KEY] = value
+        self._modified = datetime.utcnow()
 
     def get_last_sync(self, device_id):
         """
@@ -180,6 +187,9 @@
         :param device_id: (str) ONU Device ID
         :return: (int) The Value or None if not found
         """
+        if not isinstance(device_id, basestring):
+            raise TypeError('Device ID should be an string')
+
         if device_id not in self._data:
             return None
 
@@ -201,6 +211,18 @@
         :raises KeyError: If device does not exist
         :raises DatabaseStateError: If the database is not enabled
         """
+        if not isinstance(device_id, basestring):
+            raise TypeError('Device ID should be a string')
+
+        if not 0 <= class_id <= 0xFFFF:
+            raise ValueError("Invalid Class ID: {}, should be 0..65535".format(class_id))
+
+        if not 0 <= instance_id <= 0xFFFF:
+            raise ValueError("Invalid Instance ID: {}, should be 0..65535".format(instance_id))
+
+        if not isinstance(attributes, dict):
+            raise TypeError("Attributes should be a dictionary")
+
         if not self._started:
             raise DatabaseStateError('The Database is not currently active')
 
@@ -211,9 +233,7 @@
 
             if class_db is None:
                 device_db[class_id] = {
-                    CLASS_ID_KEY: class_id,
-                    CREATED_KEY: now,
-                    MODIFIED_KEY: now
+                    CLASS_ID_KEY: class_id
                 }
                 class_db = device_db[class_id]
                 self._modified = now
@@ -227,8 +247,6 @@
                     ATTRIBUTES_KEY: dict()
                 }
                 instance_db = class_db[instance_id]
-                class_db[MODIFIED_KEY] = now
-                device_db[MODIFIED_KEY] = now
                 self._modified = now
 
             changed = False
@@ -238,10 +256,22 @@
                 assert value is not None, "Attribute '{}' value cannot be 'None'".\
                     format(attribute)
 
-                db_value = instance_db.get(attribute)
+                # Complex packet types may have an attribute encoded as an object, this
+                # can be check by seeing if there is a to_json() conversion callable
+                # defined
+                if hasattr(value, 'to_json'):
+                    value = value.to_json()
+
+                # Other complex packet types may be a repeated list field (FieldListField)
+                elif isinstance(value, list):
+                    value = json.dumps(value, separators=(',', ':'))
+
+                db_value = instance_db[ATTRIBUTES_KEY].get(attribute) \
+                    if ATTRIBUTES_KEY in instance_db else None
+
                 assert db_value is None or isinstance(value, type(db_value)), \
                     "New value for attribute '{}' type is changing from '{}' to '{}'".\
-                        format(attribute, type(db_value), type(value))
+                    format(attribute, type(db_value), type(value))
 
                 if db_value is None or db_value != value:
                     instance_db[ATTRIBUTES_KEY][attribute] = value
@@ -249,8 +279,6 @@
 
             if changed:
                 instance_db[MODIFIED_KEY] = now
-                class_db[MODIFIED_KEY] = now
-                device_db[MODIFIED_KEY] = now
                 self._modified = now
 
             return changed
@@ -277,6 +305,15 @@
         if not self._started:
             raise DatabaseStateError('The Database is not currently active')
 
+        if not isinstance(device_id, basestring):
+            raise TypeError('Device ID should be an string')
+
+        if not 0 <= class_id <= 0xFFFF:
+            raise ValueError('class-id is 0..0xFFFF')
+
+        if not 0 <= instance_id <= 0xFFFF:
+            raise ValueError('instance-id is 0..0xFFFF')
+
         try:
             device_db = self._data[device_id]
             class_db = device_db.get(class_id)
@@ -291,14 +328,10 @@
             now = datetime.utcnow()
             del class_db[instance_id]
 
-            if len(class_db) == len([CREATED_KEY, MODIFIED_KEY]):
+            if len(class_db) == 1:      # Is only 'CLASS_ID_KEY' remaining
                 del device_db[class_id]
-            else:
-                class_db[MODIFIED_KEY] = now
 
-            device_db[MODIFIED_KEY] = now
             self._modified = now
-
             return True
 
         except Exception as e:
@@ -333,21 +366,21 @@
 
         device_db = self._data[device_id]
         if class_id is None:
-            return device_db            # TODO: copy.deepcopy(device_db)
+            return self._fix_dev_json_attributes(copy.copy(device_db))
 
         if not isinstance(class_id, int):
             raise TypeError('Class ID is an integer')
 
         class_db = device_db.get(class_id, dict())
         if instance_id is None or len(class_db) == 0:
-            return class_db         # TODO: copy.deepcopy(class_db)
+            return self._fix_cls_json_attributes(copy.copy(class_db))
 
         if not isinstance(instance_id, int):
             raise TypeError('Instance ID is an integer')
 
         instance_db = class_db.get(instance_id, dict())
         if attributes is None or len(instance_db) == 0:
-            return instance_db              # TODO: copy.deepcopy(instance_db)
+            return self._fix_inst_json_attributes(copy.copy(instance_db))
 
         if not isinstance(attributes, (basestring, list, set)):
             raise TypeError('Attributes should be a string or list/set of strings')
@@ -355,5 +388,45 @@
         if not isinstance(attributes, (list, set)):
             attributes = [attributes]
 
-        return {attr: val for attr, val in instance_db[ATTRIBUTES_KEY].iteritems()
-                if attr in attributes}
+        results = {attr: val for attr, val in instance_db[ATTRIBUTES_KEY].iteritems()
+                   if attr in attributes}
+
+        for attr, attr_data in results.items():
+            results[attr] = self._fix_attr_json_attribute(copy.copy(attr_data))
+
+        return results
+
+    #########################################################################
+    # Following routines are used to fix-up JSON encoded complex data. A
+    # nice side effect is that the values returned will be a deep-copy of
+    # the class/instance/attribute data of what is in the database. Note
+    # That other database values (created, modified, ...) will still reference
+    # back to the original DB.
+
+    def _fix_dev_json_attributes(self, dev_data):
+        for cls_id, cls_data in dev_data.items():
+            if isinstance(cls_id, int):
+                dev_data[cls_id] = self._fix_cls_json_attributes(copy.copy(cls_data))
+        return dev_data
+
+    def _fix_cls_json_attributes(self, cls_data):
+        for inst_id, inst_data in cls_data.items():
+            if isinstance(inst_id, int):
+                cls_data[inst_id] = self._fix_inst_json_attributes(copy.copy(inst_data))
+        return cls_data
+
+    def _fix_inst_json_attributes(self, inst_data):
+        if ATTRIBUTES_KEY in inst_data:
+            for attr, attr_data in inst_data[ATTRIBUTES_KEY].items():
+                inst_data[ATTRIBUTES_KEY][attr] = self._fix_attr_json_attribute(copy.copy(attr_data))
+        return inst_data
+
+    def _fix_attr_json_attribute(self, attr_data):
+        try:
+            return json.loads(attr_data) if isinstance(attr_data, basestring) else attr_data
+
+        except ValueError:
+            return attr_data
+
+        except Exception as e:
+            pass
diff --git a/voltha/extensions/omci/database/mib_db_ext.py b/voltha/extensions/omci/database/mib_db_ext.py
index f93f14b..832a821 100644
--- a/voltha/extensions/omci/database/mib_db_ext.py
+++ b/voltha/extensions/omci/database/mib_db_ext.py
@@ -13,7 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-from datetime import datetime
 from mib_db_api import *
 from voltha.protos.omci_mib_db_pb2 import MibInstanceData, MibClassData, \
     MibDeviceData, MibAttributeData
@@ -105,15 +104,19 @@
             eca = entity.attributes[attr_index]
             field = eca.field
 
-            if isinstance(field, (StrField, MACField, IPField)):
-                #  For StrField, value is an str already (or possibly JSON encoded object)
-                #  For MACField, value is a string in ':' delimited form
-                #  For IPField, value is a string in '.' delimited form
+            if isinstance(field, StrFixedLenField):
+                #  For StrFixedLenField, value is an str already (or possibly JSON encoded object)
                 if hasattr(value, 'to_json'):
                     str_value = value.to_json()
                 else:
                     str_value = str(value)
 
+            elif isinstance(field, (StrField, MACField, IPField)):
+                #  For StrField, value is an str already
+                #  For MACField, value is a string in ':' delimited form
+                #  For IPField, value is a string in '.' delimited form
+                str_value = str(value)
+
             elif isinstance(field, (ByteField, ShortField, IntField, LongField)):
                 #  For ByteField, ShortField, IntField, and LongField value is an int
                 str_value = str(value)
@@ -156,7 +159,12 @@
             field = eca.field
 
             if isinstance(field, StrFixedLenField):
-                value = str_value
+                from scapy.base_classes import Packet_metaclass
+                if isinstance(field.default, Packet_metaclass) and \
+                        hasattr(field.default, 'to_json'):
+                    value = json.loads(str_value)
+                else:
+                    value = str_value
 
             elif isinstance(field, MACField):
                 value = str_value
@@ -165,6 +173,8 @@
                 value = str_value
 
             elif isinstance(field, (ByteField, ShortField, IntField, LongField)):
+                if str_value.lower() in ('true', 'false'):
+                    str_value = '1' if str_value.lower() == 'true' else '0'
                 value = int(str_value)
 
             elif isinstance(field, BitField):
@@ -359,6 +369,7 @@
 
         :param device_id: (str) ONU Device ID
         :raises DatabaseStateError: If the database is not enabled
+        :raises KeyError: If the device does not exist in the database
         """
         self.log.debug('on-mib-reset', device_id=device_id)
 
@@ -747,7 +758,6 @@
 
                     else:
                         # Specific attribute(s)
-
                         if isinstance(attributes, basestring):
                             attributes = {attributes}
 
diff --git a/voltha/extensions/omci/omci_entities.py b/voltha/extensions/omci/omci_entities.py
index fb5f43a..65e5359 100644
--- a/voltha/extensions/omci/omci_entities.py
+++ b/voltha/extensions/omci/omci_entities.py
@@ -564,7 +564,7 @@
     ]
 
     def to_json(self):
-        return json.dumps(self.fields)
+        return json.dumps(self.fields, separators=(',', ':'))
 
 
 class ExtendedVlanTaggingOperationConfigurationData(EntityClass):
@@ -849,7 +849,7 @@
     ]
 
     def to_json(self):
-        return json.dumps(self.fields)
+        return json.dumps(self.fields, separators=(',', ':'))
 
 
 class AccessControlRow1(Packet):
@@ -869,7 +869,7 @@
     ]
 
     def to_json(self):
-        return json.dumps(self.fields)
+        return json.dumps(self.fields, separators=(',', ':'))
 
 
 class AccessControlRow2(Packet):
@@ -885,7 +885,7 @@
     ]
 
     def to_json(self):
-        return json.dumps(self.fields)
+        return json.dumps(self.fields, separators=(',', ':'))
 
 
 class DownstreamIgmpMulticastTci(Packet):
@@ -896,7 +896,7 @@
     ]
 
     def to_json(self):
-        return json.dumps(self.fields)
+        return json.dumps(self.fields, separators=(',', ':'))
 
 
 class MulticastOperationsProfile(EntityClass):
@@ -950,7 +950,7 @@
     ]
 
     def to_json(self):
-        return json.dumps(self.fields)
+        return json.dumps(self.fields, separators=(',', ':'))
 
 
 class AllowedPreviewGroupsRow0(Packet):
@@ -968,7 +968,7 @@
     ]
 
     def to_json(self):
-        return json.dumps(self.fields)
+        return json.dumps(self.fields, separators=(',', ':'))
 
 
 class AllowedPreviewGroupsRow1(Packet):
@@ -986,7 +986,7 @@
     ]
 
     def to_json(self):
-        return json.dumps(self.fields)
+        return json.dumps(self.fields, separators=(',', ':'))
 
 
 class MulticastSubscriberConfigInfo(EntityClass):