VOL-697: Improvement to MIB database output - prep for MIB Audit work
Try to get jenkins label now that omci_entities.py conflict resolved
Change-Id: Ic553172087a6b64811cd7372f837a71270df9320
diff --git a/tests/utests/voltha/extensions/omci/test_mib_db_dict.py b/tests/utests/voltha/extensions/omci/test_mib_db_dict.py
new file mode 100644
index 0000000..be42733
--- /dev/null
+++ b/tests/utests/voltha/extensions/omci/test_mib_db_dict.py
@@ -0,0 +1,502 @@
+#
+# Copyright 2017 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from unittest import main, TestCase
+
+from voltha.extensions.omci.omci_entities import *
+from voltha.extensions.omci.database.mib_db_dict import *
+from voltha.extensions.omci.database.mib_db_api import MODIFIED_KEY, CREATED_KEY,\
+ DEVICE_ID_KEY, MDS_KEY, LAST_SYNC_KEY
+from mock.mock_adapter_agent import MockAdapterAgent, MockDevice
+from nose.tools import raises, assert_raises
+import time
+
+_DEVICE_ID = 'br-549'
+
+
+class TestOmciMibDbDict(TestCase):
+
+ def setUp(self):
+ self.adapter_agent = MockAdapterAgent()
+ self.adapter_agent.add_device(MockDevice(_DEVICE_ID)) # For Entity class lookups
+ self.db = MibDbVolatileDict(self.adapter_agent)
+
+ def tearDown(self):
+ self.db.stop()
+
+ def test_start_stop(self):
+ # Simple start stop
+ self.assertFalse(self.db.active)
+ self.db.start()
+ self.assertTrue(self.db.active)
+ self.db.stop()
+ self.assertFalse(self.db.active)
+
+ # Start after start still okay
+ self.db.start()
+ self.db.start()
+ self.assertTrue(self.db.active)
+
+ self.db.stop()
+ self.db.stop()
+ self.assertFalse(self.db.active)
+
+ @raises(DatabaseStateError)
+ def test_bad_state_add(self):
+ self.db.add(_DEVICE_ID)
+
+ @raises(DatabaseStateError)
+ def test_bad_state_remove(self):
+ self.db.remove(_DEVICE_ID)
+
+ @raises(DatabaseStateError)
+ def test_bad_state_query_1(self):
+ self.db.query(_DEVICE_ID, 0)
+
+ @raises(DatabaseStateError)
+ def test_bad_state_query_2(self):
+ self.db.query(_DEVICE_ID, 0, 0)
+
+ @raises(DatabaseStateError)
+ def test_bad_state_query_3(self):
+ self.db.query(_DEVICE_ID, 0, 0, 'test')
+
+ @raises(DatabaseStateError)
+ def test_bad_state_set(self):
+ self.db.set(_DEVICE_ID, 0, 0, {'test': 123})
+
+ @raises(DatabaseStateError)
+ def test_bad_state_delete(self):
+ self.db.delete(_DEVICE_ID, 0, 0)
+
+ @raises(KeyError)
+ def test_no_device_query(self):
+ self.db.start()
+ self.db.query(_DEVICE_ID)
+
+ def test_no_device_last_sync(self):
+ self.db.start()
+ # Returns None, not a KeyError
+ value = self.db.get_last_sync(_DEVICE_ID)
+ self.assertIsNone(value)
+
+ def test_no_device_mds(self):
+ self.db.start()
+ # Returns None, not a KeyError
+ value = self.db.get_mib_data_sync(_DEVICE_ID)
+ self.assertIsNone(value)
+
+ @raises(KeyError)
+ def test_no_device_save_last_sync(self):
+ self.db.start()
+ self.db.save_last_sync(_DEVICE_ID, datetime.utcnow())
+
+ @raises(KeyError)
+ def test_no_device_save_mds(self):
+ self.db.start()
+ self.db.save_mib_data_sync(_DEVICE_ID, 123)
+
+ def test_param_types(self):
+ self.db.start()
+ assert_raises(TypeError, self.db.add, 123)
+ assert_raises(TypeError, self.db.remove, 123)
+ assert_raises(TypeError, self.db.query, 123)
+
+ assert_raises(TypeError, self.db.get_mib_data_sync, 123)
+ assert_raises(TypeError, self.db.save_mib_data_sync, 123, 0)
+ assert_raises(TypeError, self.db.save_mib_data_sync, _DEVICE_ID, 'zero')
+
+ assert_raises(TypeError, self.db.get_last_sync, 123)
+ assert_raises(TypeError, self.db.save_last_sync, 123, datetime.utcnow())
+ assert_raises(TypeError, self.db.save_last_sync, _DEVICE_ID, 'bad-date')
+
+ assert_raises(TypeError, self.db.set, 123, 0, 0, {'test': 0})
+ assert_raises(TypeError, self.db.set, None, 0, 0, {'test': 0})
+ assert_raises(ValueError, self.db.set, _DEVICE_ID, None, 0, {'test': 0})
+ assert_raises(ValueError, self.db.set, _DEVICE_ID, 0, None, {'test': 0})
+ assert_raises(TypeError, self.db.set, _DEVICE_ID, 0, 0, None)
+ assert_raises(TypeError, self.db.set, _DEVICE_ID, 0, 0, 'not-a-dict')
+
+ assert_raises(ValueError, self.db.set, _DEVICE_ID, -1, 0, {'test': 0})
+ assert_raises(ValueError, self.db.set, _DEVICE_ID, 0x10000, 0, {'test': 0})
+ assert_raises(ValueError, self.db.set, _DEVICE_ID, 0, -1, {'test': 0})
+ assert_raises(ValueError, self.db.set, _DEVICE_ID, 0, 0x10000, {'test': 0})
+
+ assert_raises(TypeError, self.db.delete, 123, 0, 0)
+ assert_raises(ValueError, self.db.delete, _DEVICE_ID, -1, 0)
+ assert_raises(ValueError, self.db.delete, _DEVICE_ID, 0x10000, 0)
+ assert_raises(ValueError, self.db.delete, _DEVICE_ID, 0, -1)
+ assert_raises(ValueError, self.db.delete, _DEVICE_ID, 0, 0x10000)
+
+ def test_add_remove_device(self):
+ self.db.start()
+
+ # Remove of non-existent device is not an error
+ assert_raises(KeyError, self.db.query, _DEVICE_ID)
+ self.db.remove(_DEVICE_ID)
+
+ start_time = datetime.utcnow()
+ self.db.add(_DEVICE_ID)
+ dev_data = self.db.query(_DEVICE_ID)
+ end_time = datetime.utcnow()
+
+ self.assertEqual(dev_data[DEVICE_ID_KEY], _DEVICE_ID)
+ self.assertEquals(dev_data[MDS_KEY], 0)
+ self.assertIsNone(dev_data[LAST_SYNC_KEY])
+ self.assertEqual(dev_data[VERSION_KEY], MibDbVolatileDict.CURRENT_VERSION)
+
+ # Remove it
+ self.db.remove(_DEVICE_ID)
+ assert_raises(KeyError, self.db.query, _DEVICE_ID)
+
+ # Remove of non-existant dev okay
+ self.db.remove(_DEVICE_ID +'abcd')
+
+ # Overwrite tests
+ self.db.add(_DEVICE_ID)
+ assert_raises(KeyError, self.db.add, _DEVICE_ID)
+ self.db.add(_DEVICE_ID, overwrite=True) # This is okay
+
+ def test_mib_data_sync(self):
+ self.db.start()
+ self.db.add(_DEVICE_ID)
+ self.assertEquals(self.db.get_mib_data_sync(_DEVICE_ID), 0)
+
+ self.db.save_mib_data_sync(_DEVICE_ID, 100)
+ self.assertEqual(self.db.get_mib_data_sync(_DEVICE_ID), 100)
+
+ assert_raises(ValueError, self.db.save_mib_data_sync, _DEVICE_ID, -1)
+ assert_raises(ValueError, self.db.save_mib_data_sync, _DEVICE_ID, 256)
+
+ def test_last_sync(self):
+ self.db.start()
+ self.assertIsNone(self.db.get_last_sync(_DEVICE_ID))
+
+ self.db.add(_DEVICE_ID)
+ self.assertIsNone(self.db.get_last_sync(_DEVICE_ID))
+
+ now = datetime.utcnow()
+
+ self.db.save_last_sync(_DEVICE_ID, now)
+ self.assertEqual(self.db.get_last_sync(_DEVICE_ID), now)
+
+ assert_raises(TypeError, self.db.save_last_sync, _DEVICE_ID, 'hello')
+
+ def test_set_and_query(self):
+ self.db.start()
+ self.db.add(_DEVICE_ID) # Base device DB created here
+ time.sleep(0.1)
+
+ class_id = OntG.class_id
+ inst_id = 0
+ attributes = {'vendor_id': 'ABCD'}
+
+ start_time = datetime.utcnow()
+ set_occurred = self.db.set(_DEVICE_ID, class_id, inst_id, attributes)
+ self.assertTrue(set_occurred)
+ end_time = datetime.utcnow()
+
+ dev_data = self.db.query(_DEVICE_ID)
+ self.assertEqual(dev_data[DEVICE_ID_KEY], _DEVICE_ID)
+
+ dev_classes = [v for k, v in dev_data.items() if isinstance(k, int)]
+
+ self.assertEqual(len(dev_classes), 1)
+ class_data = dev_classes[0]
+
+ self.assertEqual(class_data[CLASS_ID_KEY], class_id)
+
+ class_insts = [v for k, v in class_data.items() if isinstance(k, int)]
+
+ self.assertEqual(len(class_insts), 1)
+ inst_data = class_insts[0]
+
+ self.assertEqual(inst_data[INSTANCE_ID_KEY], inst_id)
+ self.assertGreaterEqual(inst_data[MODIFIED_KEY], start_time)
+ self.assertLessEqual(inst_data[MODIFIED_KEY], end_time)
+ self.assertLessEqual(inst_data[CREATED_KEY], inst_data[MODIFIED_KEY])
+
+ inst_attributes = inst_data[ATTRIBUTES_KEY]
+ self.assertEqual(len(inst_attributes), 1)
+
+ self.assertTrue('vendor_id' in inst_attributes)
+ self.assertEqual(inst_attributes['vendor_id'], attributes['vendor_id'])
+
+ ########################################
+ # Query with device and class. Should be same as from full device query
+ cls_2_data = self.db.query(_DEVICE_ID, class_id)
+
+ self.assertEqual(class_data[CLASS_ID_KEY], cls_2_data[CLASS_ID_KEY])
+
+ cl2_insts = {k:v for k, v in cls_2_data.items() if isinstance(k, int)}
+ self.assertEqual(len(cl2_insts), len(class_insts))
+
+ # Bad class id query
+ cls_no_data = self.db.query(_DEVICE_ID, class_id + 1)
+ self.assertTrue(isinstance(cls_no_data, dict))
+ self.assertEqual(len(cls_no_data), 0)
+
+ ########################################
+ # Query with device, class, instance
+ inst_2_data = self.db.query(_DEVICE_ID, class_id, inst_id)
+
+ self.assertEqual(inst_data[INSTANCE_ID_KEY], inst_2_data[INSTANCE_ID_KEY])
+ self.assertEqual(inst_data[MODIFIED_KEY], inst_2_data[MODIFIED_KEY])
+ self.assertEqual(inst_data[CREATED_KEY], inst_2_data[CREATED_KEY])
+
+ inst2_attr = inst_2_data[ATTRIBUTES_KEY]
+ self.assertEqual(len(inst2_attr), len(inst_attributes))
+
+ # Bad instance id query
+ inst_no_data = self.db.query(_DEVICE_ID, class_id, inst_id + 100)
+ self.assertTrue(isinstance(inst_no_data, dict))
+ self.assertEqual(len(inst_no_data), 0)
+
+ ########################################
+ # Attribute queries
+ attr_2_data = self.db.query(_DEVICE_ID, class_id, inst_id, 'vendor_id')
+ self.assertEqual(attr_2_data['vendor_id'], attributes['vendor_id'])
+
+ attr_3_data = self.db.query(_DEVICE_ID, class_id, inst_id, ['vendor_id'])
+ self.assertEqual(attr_3_data['vendor_id'], attributes['vendor_id'])
+
+ attr_4_data = self.db.query(_DEVICE_ID, class_id, inst_id, {'vendor_id'})
+ self.assertEqual(attr_4_data['vendor_id'], attributes['vendor_id'])
+
+ attr_no_data = self.db.query(_DEVICE_ID, class_id, inst_id, 'no_such_thing')
+ self.assertTrue(isinstance(attr_no_data, dict))
+ self.assertEqual(len(attr_no_data), 0)
+
+ # Set to same value does not change modified data. The modified is
+ # at the instance level
+
+ class_id = OntG.class_id
+ inst_id = 0
+ attributes = {'vendor_id': 'ABCD'}
+ set_occurred = self.db.set(_DEVICE_ID, class_id, inst_id, attributes)
+ self.assertFalse(set_occurred)
+
+ inst_3_data = self.db.query(_DEVICE_ID, class_id, inst_id)
+ self.assertEqual(inst_data[MODIFIED_KEY], inst_3_data[MODIFIED_KEY])
+ self.assertEqual(inst_data[CREATED_KEY], inst_3_data[CREATED_KEY])
+
+ # But set to new value does
+ time.sleep(0.1)
+ attributes = {'vendor_id': 'WXYZ'}
+ set_occurred = self.db.set(_DEVICE_ID, class_id, inst_id, attributes)
+ self.assertTrue(set_occurred)
+
+ inst_4_data = self.db.query(_DEVICE_ID, class_id, inst_id)
+ self.assertLess(inst_3_data[MODIFIED_KEY], inst_4_data[MODIFIED_KEY])
+ self.assertEqual(inst_3_data[CREATED_KEY], inst_4_data[CREATED_KEY])
+
+ def test_delete_instances(self):
+ self.db.start()
+ self.db.add(_DEVICE_ID)
+ create_time = datetime.utcnow()
+
+ class_id = GalEthernetProfile.class_id
+ inst_id_1 = 0x100
+ inst_id_2 = 0x200
+ attributes = {'max_gem_payload_size': 1500}
+
+ self.db.set(_DEVICE_ID, class_id, inst_id_1, attributes)
+ self.db.set(_DEVICE_ID, class_id, inst_id_2, attributes)
+ set_time = datetime.utcnow()
+ time.sleep(0.1)
+
+ dev_data = self.db.query(_DEVICE_ID)
+ cls_data = self.db.query(_DEVICE_ID, class_id)
+ inst_data = {k: v for k, v in cls_data.items() if isinstance(k, int)}
+ self.assertEqual(len(inst_data), 2)
+
+ self.assertLessEqual(dev_data[CREATED_KEY], create_time)
+ self.assertLessEqual(self.db.created, create_time)
+
+ # Delete one instance
+ time.sleep(0.1)
+ del_time = datetime.utcnow()
+ result = self.db.delete(_DEVICE_ID, class_id, inst_id_1)
+ self.assertTrue(result) # True returned if a del actually happened
+
+ dev_data = self.db.query(_DEVICE_ID)
+ cls_data = self.db.query(_DEVICE_ID, class_id)
+ inst_data = {k: v for k, v in cls_data.items() if isinstance(k, int)}
+ self.assertEqual(len(inst_data), 1)
+
+ self.assertLessEqual(dev_data[CREATED_KEY], create_time)
+ self.assertLessEqual(self.db.created, create_time)
+
+ # Delete remaining instance
+ time.sleep(0.1)
+ result = self.db.delete(_DEVICE_ID, class_id, inst_id_2)
+ self.assertTrue(result) # True returned if a del actually happened
+
+ dev_data = self.db.query(_DEVICE_ID)
+ cls_data = {k: v for k, v in dev_data.items() if isinstance(k, int)}
+ self.assertEqual(len(cls_data), 0)
+ self.assertLessEqual(dev_data[CREATED_KEY], create_time)
+
+ # Delete returns false if not instance
+ self.assertFalse(self.db.delete(_DEVICE_ID, class_id, inst_id_1))
+ self.assertFalse(self.db.delete(_DEVICE_ID, class_id, inst_id_2))
+
+ def test_on_mib_reset_listener(self):
+ self.db.start()
+ self.db.add(_DEVICE_ID)
+ time.sleep(0.1)
+
+ class_id = OntG.class_id
+ inst_id = 0
+ attributes = {'vendor_id': 'ABCD'}
+
+ set_time = datetime.utcnow()
+ self.db.set(_DEVICE_ID, class_id, inst_id, attributes)
+
+ time.sleep(0.1)
+ self.db.on_mib_reset(_DEVICE_ID)
+
+ dev_data = self.db.query(_DEVICE_ID)
+ self.assertEqual(dev_data[DEVICE_ID_KEY], _DEVICE_ID)
+ self.assertLessEqual(dev_data[CREATED_KEY], set_time)
+ self.assertLessEqual(self.db.created, set_time)
+
+ self.assertFalse(any(isinstance(cls, int) for cls in dev_data.iterkeys()))
+
+ def test_str_field_serialization(self):
+ self.db.start()
+ self.db.add(_DEVICE_ID)
+
+ class_id = OltG.class_id
+ inst_id = 0
+ attributes = {
+ 'olt_vendor_id': 'ABCD', # StrFixedLenField(4)
+ }
+ self.db.set(_DEVICE_ID, class_id, inst_id, attributes)
+ data = self.db.query(_DEVICE_ID, class_id, inst_id, attributes.keys())
+ self.assertTrue(all(isinstance(data[k], basestring) for k in attributes.keys()))
+ self.assertTrue(all(data[k] == attributes[k] for k in attributes.keys()))
+
+ def test_mac_address_ip_field_serialization(self):
+ self.db.start()
+ self.db.add(_DEVICE_ID)
+
+ class_id = IpHostConfigData.class_id
+ inst_id = 0
+ attributes = {
+ 'mac_address': '00:01:02:03:04:05', # MACField
+ 'ip_address': '1.2.3.4', # IPField
+ }
+ self.db.set(_DEVICE_ID, class_id, inst_id, attributes)
+ data = self.db.query(_DEVICE_ID, class_id, inst_id, attributes.keys())
+ self.assertTrue(all(isinstance(data[k], basestring) for k in attributes.keys()))
+ self.assertTrue(all(data[k] == attributes[k] for k in attributes.keys()))
+
+ def test_byte_and_short_field_serialization(self):
+ self.db.start()
+ self.db.add(_DEVICE_ID)
+
+ class_id = UniG.class_id
+ inst_id = 0
+ attributes = {
+ 'administrative_state': int(1), # ByteField
+ 'non_omci_management_identifier': int(12345) # IPField
+ }
+ self.db.set(_DEVICE_ID, class_id, inst_id, attributes)
+ data = self.db.query(_DEVICE_ID, class_id, inst_id, attributes.keys())
+ self.assertTrue(all(isinstance(data[k], type(attributes[k])) for k in attributes.keys()))
+ self.assertTrue(all(data[k] == attributes[k] for k in attributes.keys()))
+
+ def test_int_field_serialization(self):
+ self.db.start()
+ self.db.add(_DEVICE_ID)
+
+ class_id = PriorityQueueG.class_id
+ inst_id = 0
+ attributes = {
+ 'related_port': int(1234567) # IntField
+ }
+ self.db.set(_DEVICE_ID, class_id, inst_id, attributes)
+ data = self.db.query(_DEVICE_ID, class_id, inst_id, attributes.keys())
+ self.assertTrue(all(isinstance(data[k], type(attributes[k])) for k in attributes.keys()))
+ self.assertTrue(all(data[k] == attributes[k] for k in attributes.keys()))
+
+ def test_long_field_serialization(self):
+ self.db.start()
+ self.db.add(_DEVICE_ID)
+
+ class_id = PriorityQueueG.class_id
+ inst_id = 0
+ attributes = {
+ 'packet_drop_queue_thresholds': int(0x1234) # LongField
+ }
+ self.db.set(_DEVICE_ID, class_id, inst_id, attributes)
+ data = self.db.query(_DEVICE_ID, class_id, inst_id, attributes.keys())
+ self.assertTrue(all(isinstance(data[k], type(attributes[k])) for k in attributes.keys()))
+ self.assertTrue(all(data[k] == attributes[k] for k in attributes.keys()))
+
+ def test_bit_field_serialization(self):
+ self.db.start()
+ self.db.add(_DEVICE_ID)
+
+ class_id = OntG.class_id
+ inst_id = 0
+ attributes = {
+ 'extended_tc_layer_options': long(0x1234), # BitField(16)
+ }
+ self.db.set(_DEVICE_ID, class_id, inst_id, attributes)
+ data = self.db.query(_DEVICE_ID, class_id, inst_id, attributes.keys())
+ self.assertTrue(all(isinstance(data[k], type(attributes[k])) for k in attributes.keys()))
+ self.assertTrue(all(data[k] == attributes[k] for k in attributes.keys()))
+
+ def test_complex_json_serialization(self):
+ self.db.start()
+ self.db.add(_DEVICE_ID)
+
+ class_id = ExtendedVlanTaggingOperationConfigurationData.class_id
+ inst_id = 0x202
+ table_data = VlanTaggingOperation(
+ filter_outer_priority=15,
+ filter_inner_priority=8,
+ filter_inner_vid=1024,
+ filter_inner_tpid_de=5,
+ filter_ether_type=0,
+ treatment_tags_to_remove=1,
+ pad3=2,
+ treatment_outer_priority=15,
+ treatment_inner_priority=8,
+ treatment_inner_vid=1024,
+ treatment_inner_tpid_de=4
+ )
+ attributes = dict(
+ received_frame_vlan_tagging_operation_table=table_data
+ )
+ self.db.set(_DEVICE_ID, class_id, inst_id, attributes)
+
+ data = self.db.query(_DEVICE_ID, class_id, inst_id, attributes.keys())
+ table_as_dict = json.loads(table_data.to_json())
+
+ self.assertTrue(all(isinstance(data['received_frame_vlan_tagging_operation_table'][k],
+ type(attributes['received_frame_vlan_tagging_operation_table'].fields[k]))
+ for k in attributes['received_frame_vlan_tagging_operation_table'].fields.keys()))
+ self.assertTrue(all(data['received_frame_vlan_tagging_operation_table'][k] ==
+ attributes['received_frame_vlan_tagging_operation_table'].fields[k]
+ for k in attributes['received_frame_vlan_tagging_operation_table'].fields.keys()))
+ self.assertTrue(all(data['received_frame_vlan_tagging_operation_table'][k] == table_as_dict[k]
+ for k in table_as_dict.keys()))
+
+
+if __name__ == '__main__':
+ main()
diff --git a/tests/utests/voltha/extensions/omci/test_mib_db_ext.py b/tests/utests/voltha/extensions/omci/test_mib_db_ext.py
index b28e027..725da65 100644
--- a/tests/utests/voltha/extensions/omci/test_mib_db_ext.py
+++ b/tests/utests/voltha/extensions/omci/test_mib_db_ext.py
@@ -1,9 +1,20 @@
-
+#
+# Copyright 2017 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
from unittest import main, TestCase
-from voltha.protos.omci_mib_db_pb2 import MibInstanceData, MibClassData, \
- MibDeviceData, MibAttributeData
-from datetime import datetime
from voltha.extensions.omci.database.mib_db_ext import *
from voltha.extensions.omci.database.mib_db_api import MODIFIED_KEY, CREATED_KEY,\
DEVICE_ID_KEY, MDS_KEY, LAST_SYNC_KEY
@@ -14,7 +25,7 @@
_DEVICE_ID = 'br-549'
-class TestOmciMibDb(TestCase):
+class TestOmciMibDbExt(TestCase):
def setUp(self):
self.adapter_agent = MockAdapterAgent()
@@ -376,14 +387,6 @@
self.assertTrue(all(isinstance(data[k], basestring) for k in attributes.keys()))
self.assertTrue(all(data[k] == attributes[k] for k in attributes.keys()))
- def test_object_as_a_str_field_serialization(self):
- # Some entity classes such as ExtendedVlanTaggingOperationConfigurationData
- # (class-id = 171) have a Scapy Packet derived object (VlanTaggingOperation)
- # that in the parent object is a StrFixedLenField.
- # These classes should encode into a JSON string. Test this serialization
- # as well as deserialization
- pass # TODO: More tests here
-
def test_mac_address_ip_field_serialization(self):
self.db.start()
self.db.add(_DEVICE_ID)
@@ -457,5 +460,42 @@
self.assertTrue(all(data[k] == attributes[k] for k in attributes.keys()))
+ def test_complex_json_serialization(self):
+ self.db.start()
+ self.db.add(_DEVICE_ID)
+
+ class_id = ExtendedVlanTaggingOperationConfigurationData.class_id
+ inst_id = 0x202
+ table_data = VlanTaggingOperation(
+ filter_outer_priority=15,
+ filter_inner_priority=8,
+ filter_inner_vid=1024,
+ filter_inner_tpid_de=5,
+ filter_ether_type=0,
+ treatment_tags_to_remove=1,
+ pad3=2,
+ treatment_outer_priority=15,
+ treatment_inner_priority=8,
+ treatment_inner_vid=1024,
+ treatment_inner_tpid_de=4
+ )
+ attributes = dict(
+ received_frame_vlan_tagging_operation_table=table_data
+ )
+ self.db.set(_DEVICE_ID, class_id, inst_id, attributes)
+
+ data = self.db.query(_DEVICE_ID, class_id, inst_id, attributes.keys())
+ table_as_dict = json.loads(table_data.to_json())
+
+ self.assertTrue(all(isinstance(data['received_frame_vlan_tagging_operation_table'][k],
+ type(attributes['received_frame_vlan_tagging_operation_table'].fields[k]))
+ for k in attributes['received_frame_vlan_tagging_operation_table'].fields.keys()))
+ self.assertTrue(all(data['received_frame_vlan_tagging_operation_table'][k] ==
+ attributes['received_frame_vlan_tagging_operation_table'].fields[k]
+ for k in attributes['received_frame_vlan_tagging_operation_table'].fields.keys()))
+ self.assertTrue(all(data['received_frame_vlan_tagging_operation_table'][k] == table_as_dict[k]
+ for k in table_as_dict.keys()))
+
+
if __name__ == '__main__':
main()
diff --git a/voltha/extensions/omci/database/mib_db_dict.py b/voltha/extensions/omci/database/mib_db_dict.py
index d0cf98d..58d81c2 100644
--- a/voltha/extensions/omci/database/mib_db_dict.py
+++ b/voltha/extensions/omci/database/mib_db_dict.py
@@ -15,6 +15,7 @@
#
import copy
from mib_db_api import *
+import json
class MibDbVolatileDict(MibDbApi):
@@ -79,9 +80,8 @@
self._data[device_id] = {
DEVICE_ID_KEY: device_id,
CREATED_KEY: now,
- MODIFIED_KEY: now,
- MDS_KEY: 0,
LAST_SYNC_KEY: None,
+ MDS_KEY: 0,
VERSION_KEY: MibDbVolatileDict.CURRENT_VERSION
}
@@ -101,6 +101,7 @@
if device_id in self._data:
del self._data[device_id]
+ self._modified = datetime.utcnow()
def on_mib_reset(self, device_id):
"""
@@ -108,33 +109,24 @@
:param device_id: (str) ONU Device ID
:raises DatabaseStateError: If the database is not enabled
+ :raises KeyError: If the device does not exist in the database
"""
if not self._started:
raise DatabaseStateError('The Database is not currently active')
- now = datetime.utcnow()
- device_db = self._data.get(device_id)
+ if not isinstance(device_id, basestring):
+ raise TypeError('Device ID should be an string')
- if device_db is None:
- self._data[device_id] = {
- CREATED_KEY: now,
- LAST_SYNC_KEY: None
- }
- device_db = self._data[device_id]
- else:
- created = device_db[CREATED_KEY]
- last_sync = device_db[LAST_SYNC_KEY]
+ device_db = self._data[device_id]
+ self._modified = datetime.utcnow()
- self._data[device_id] = {
- CREATED_KEY: created,
- LAST_SYNC_KEY: last_sync
- }
-
- device_db[DEVICE_ID_KEY] = device_id
- device_db[MODIFIED_KEY] = now
- device_db[MDS_KEY] = 0
- device_db[VERSION_KEY] = MibDbVolatileDict.CURRENT_VERSION
- self._modified = now
+ self._data[device_id] = {
+ DEVICE_ID_KEY: device_id,
+ CREATED_KEY: device_db[CREATED_KEY],
+ LAST_SYNC_KEY: device_db[LAST_SYNC_KEY],
+ MDS_KEY: 0,
+ VERSION_KEY: MibDbVolatileDict.CURRENT_VERSION
+ }
def save_mib_data_sync(self, device_id, value):
"""
@@ -143,12 +135,18 @@
:param device_id: (str) ONU Device ID
:param value: (int) Value to save
"""
- if device_id in self._data:
- assert 0 <= value <= 255,\
- 'Invalid MIB Data Sync Value: {}'.format(value)
+ if not isinstance(device_id, basestring):
+ raise TypeError('Device ID should be an string')
- self._data[device_id][MODIFIED_KEY] = datetime.utcnow()
- self._data[device_id][MDS_KEY] = value
+ if not isinstance(value, int):
+ raise TypeError('MIB Data Sync is an integer')
+
+ if not 0 <= value <= 255:
+ raise ValueError('Invalid MIB-data-sync value {}. Must be 0..255'.
+ format(value))
+
+ self._data[device_id][MDS_KEY] = value
+ self._modified = datetime.utcnow()
def get_mib_data_sync(self, device_id):
"""
@@ -157,6 +155,9 @@
:param device_id: (str) ONU Device ID
:return: (int) The Value or None if not found
"""
+ if not isinstance(device_id, basestring):
+ raise TypeError('Device ID should be an string')
+
if device_id not in self._data:
return None
@@ -169,9 +170,15 @@
:param device_id: (str) ONU Device ID
:param value: (DateTime) Value to save
"""
- if device_id in self._data:
- self._data[device_id][MODIFIED_KEY] = datetime.utcnow()
- self._data[device_id][LAST_SYNC_KEY] = value
+ if not isinstance(device_id, basestring):
+ raise TypeError('Device ID should be an string')
+
+ if not isinstance(value, datetime):
+ raise TypeError('Expected a datetime object, got {}'.
+ format(type(datetime)))
+
+ self._data[device_id][LAST_SYNC_KEY] = value
+ self._modified = datetime.utcnow()
def get_last_sync(self, device_id):
"""
@@ -180,6 +187,9 @@
:param device_id: (str) ONU Device ID
:return: (int) The Value or None if not found
"""
+ if not isinstance(device_id, basestring):
+ raise TypeError('Device ID should be an string')
+
if device_id not in self._data:
return None
@@ -201,6 +211,18 @@
:raises KeyError: If device does not exist
:raises DatabaseStateError: If the database is not enabled
"""
+ if not isinstance(device_id, basestring):
+ raise TypeError('Device ID should be a string')
+
+ if not 0 <= class_id <= 0xFFFF:
+ raise ValueError("Invalid Class ID: {}, should be 0..65535".format(class_id))
+
+ if not 0 <= instance_id <= 0xFFFF:
+ raise ValueError("Invalid Instance ID: {}, should be 0..65535".format(instance_id))
+
+ if not isinstance(attributes, dict):
+ raise TypeError("Attributes should be a dictionary")
+
if not self._started:
raise DatabaseStateError('The Database is not currently active')
@@ -211,9 +233,7 @@
if class_db is None:
device_db[class_id] = {
- CLASS_ID_KEY: class_id,
- CREATED_KEY: now,
- MODIFIED_KEY: now
+ CLASS_ID_KEY: class_id
}
class_db = device_db[class_id]
self._modified = now
@@ -227,8 +247,6 @@
ATTRIBUTES_KEY: dict()
}
instance_db = class_db[instance_id]
- class_db[MODIFIED_KEY] = now
- device_db[MODIFIED_KEY] = now
self._modified = now
changed = False
@@ -238,10 +256,22 @@
assert value is not None, "Attribute '{}' value cannot be 'None'".\
format(attribute)
- db_value = instance_db.get(attribute)
+ # Complex packet types may have an attribute encoded as an object, this
+ # can be check by seeing if there is a to_json() conversion callable
+ # defined
+ if hasattr(value, 'to_json'):
+ value = value.to_json()
+
+ # Other complex packet types may be a repeated list field (FieldListField)
+ elif isinstance(value, list):
+ value = json.dumps(value, separators=(',', ':'))
+
+ db_value = instance_db[ATTRIBUTES_KEY].get(attribute) \
+ if ATTRIBUTES_KEY in instance_db else None
+
assert db_value is None or isinstance(value, type(db_value)), \
"New value for attribute '{}' type is changing from '{}' to '{}'".\
- format(attribute, type(db_value), type(value))
+ format(attribute, type(db_value), type(value))
if db_value is None or db_value != value:
instance_db[ATTRIBUTES_KEY][attribute] = value
@@ -249,8 +279,6 @@
if changed:
instance_db[MODIFIED_KEY] = now
- class_db[MODIFIED_KEY] = now
- device_db[MODIFIED_KEY] = now
self._modified = now
return changed
@@ -277,6 +305,15 @@
if not self._started:
raise DatabaseStateError('The Database is not currently active')
+ if not isinstance(device_id, basestring):
+ raise TypeError('Device ID should be an string')
+
+ if not 0 <= class_id <= 0xFFFF:
+ raise ValueError('class-id is 0..0xFFFF')
+
+ if not 0 <= instance_id <= 0xFFFF:
+ raise ValueError('instance-id is 0..0xFFFF')
+
try:
device_db = self._data[device_id]
class_db = device_db.get(class_id)
@@ -291,14 +328,10 @@
now = datetime.utcnow()
del class_db[instance_id]
- if len(class_db) == len([CREATED_KEY, MODIFIED_KEY]):
+ if len(class_db) == 1: # Is only 'CLASS_ID_KEY' remaining
del device_db[class_id]
- else:
- class_db[MODIFIED_KEY] = now
- device_db[MODIFIED_KEY] = now
self._modified = now
-
return True
except Exception as e:
@@ -333,21 +366,21 @@
device_db = self._data[device_id]
if class_id is None:
- return device_db # TODO: copy.deepcopy(device_db)
+ return self._fix_dev_json_attributes(copy.copy(device_db))
if not isinstance(class_id, int):
raise TypeError('Class ID is an integer')
class_db = device_db.get(class_id, dict())
if instance_id is None or len(class_db) == 0:
- return class_db # TODO: copy.deepcopy(class_db)
+ return self._fix_cls_json_attributes(copy.copy(class_db))
if not isinstance(instance_id, int):
raise TypeError('Instance ID is an integer')
instance_db = class_db.get(instance_id, dict())
if attributes is None or len(instance_db) == 0:
- return instance_db # TODO: copy.deepcopy(instance_db)
+ return self._fix_inst_json_attributes(copy.copy(instance_db))
if not isinstance(attributes, (basestring, list, set)):
raise TypeError('Attributes should be a string or list/set of strings')
@@ -355,5 +388,45 @@
if not isinstance(attributes, (list, set)):
attributes = [attributes]
- return {attr: val for attr, val in instance_db[ATTRIBUTES_KEY].iteritems()
- if attr in attributes}
+ results = {attr: val for attr, val in instance_db[ATTRIBUTES_KEY].iteritems()
+ if attr in attributes}
+
+ for attr, attr_data in results.items():
+ results[attr] = self._fix_attr_json_attribute(copy.copy(attr_data))
+
+ return results
+
+ #########################################################################
+ # Following routines are used to fix-up JSON encoded complex data. A
+ # nice side effect is that the values returned will be a deep-copy of
+ # the class/instance/attribute data of what is in the database. Note
+ # That other database values (created, modified, ...) will still reference
+ # back to the original DB.
+
+ def _fix_dev_json_attributes(self, dev_data):
+ for cls_id, cls_data in dev_data.items():
+ if isinstance(cls_id, int):
+ dev_data[cls_id] = self._fix_cls_json_attributes(copy.copy(cls_data))
+ return dev_data
+
+ def _fix_cls_json_attributes(self, cls_data):
+ for inst_id, inst_data in cls_data.items():
+ if isinstance(inst_id, int):
+ cls_data[inst_id] = self._fix_inst_json_attributes(copy.copy(inst_data))
+ return cls_data
+
+ def _fix_inst_json_attributes(self, inst_data):
+ if ATTRIBUTES_KEY in inst_data:
+ for attr, attr_data in inst_data[ATTRIBUTES_KEY].items():
+ inst_data[ATTRIBUTES_KEY][attr] = self._fix_attr_json_attribute(copy.copy(attr_data))
+ return inst_data
+
+ def _fix_attr_json_attribute(self, attr_data):
+ try:
+ return json.loads(attr_data) if isinstance(attr_data, basestring) else attr_data
+
+ except ValueError:
+ return attr_data
+
+ except Exception as e:
+ pass
diff --git a/voltha/extensions/omci/database/mib_db_ext.py b/voltha/extensions/omci/database/mib_db_ext.py
index f93f14b..832a821 100644
--- a/voltha/extensions/omci/database/mib_db_ext.py
+++ b/voltha/extensions/omci/database/mib_db_ext.py
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-from datetime import datetime
from mib_db_api import *
from voltha.protos.omci_mib_db_pb2 import MibInstanceData, MibClassData, \
MibDeviceData, MibAttributeData
@@ -105,15 +104,19 @@
eca = entity.attributes[attr_index]
field = eca.field
- if isinstance(field, (StrField, MACField, IPField)):
- # For StrField, value is an str already (or possibly JSON encoded object)
- # For MACField, value is a string in ':' delimited form
- # For IPField, value is a string in '.' delimited form
+ if isinstance(field, StrFixedLenField):
+ # For StrFixedLenField, value is an str already (or possibly JSON encoded object)
if hasattr(value, 'to_json'):
str_value = value.to_json()
else:
str_value = str(value)
+ elif isinstance(field, (StrField, MACField, IPField)):
+ # For StrField, value is an str already
+ # For MACField, value is a string in ':' delimited form
+ # For IPField, value is a string in '.' delimited form
+ str_value = str(value)
+
elif isinstance(field, (ByteField, ShortField, IntField, LongField)):
# For ByteField, ShortField, IntField, and LongField value is an int
str_value = str(value)
@@ -156,7 +159,12 @@
field = eca.field
if isinstance(field, StrFixedLenField):
- value = str_value
+ from scapy.base_classes import Packet_metaclass
+ if isinstance(field.default, Packet_metaclass) and \
+ hasattr(field.default, 'to_json'):
+ value = json.loads(str_value)
+ else:
+ value = str_value
elif isinstance(field, MACField):
value = str_value
@@ -165,6 +173,8 @@
value = str_value
elif isinstance(field, (ByteField, ShortField, IntField, LongField)):
+ if str_value.lower() in ('true', 'false'):
+ str_value = '1' if str_value.lower() == 'true' else '0'
value = int(str_value)
elif isinstance(field, BitField):
@@ -359,6 +369,7 @@
:param device_id: (str) ONU Device ID
:raises DatabaseStateError: If the database is not enabled
+ :raises KeyError: If the device does not exist in the database
"""
self.log.debug('on-mib-reset', device_id=device_id)
@@ -747,7 +758,6 @@
else:
# Specific attribute(s)
-
if isinstance(attributes, basestring):
attributes = {attributes}
diff --git a/voltha/extensions/omci/omci_entities.py b/voltha/extensions/omci/omci_entities.py
index fb5f43a..65e5359 100644
--- a/voltha/extensions/omci/omci_entities.py
+++ b/voltha/extensions/omci/omci_entities.py
@@ -564,7 +564,7 @@
]
def to_json(self):
- return json.dumps(self.fields)
+ return json.dumps(self.fields, separators=(',', ':'))
class ExtendedVlanTaggingOperationConfigurationData(EntityClass):
@@ -849,7 +849,7 @@
]
def to_json(self):
- return json.dumps(self.fields)
+ return json.dumps(self.fields, separators=(',', ':'))
class AccessControlRow1(Packet):
@@ -869,7 +869,7 @@
]
def to_json(self):
- return json.dumps(self.fields)
+ return json.dumps(self.fields, separators=(',', ':'))
class AccessControlRow2(Packet):
@@ -885,7 +885,7 @@
]
def to_json(self):
- return json.dumps(self.fields)
+ return json.dumps(self.fields, separators=(',', ':'))
class DownstreamIgmpMulticastTci(Packet):
@@ -896,7 +896,7 @@
]
def to_json(self):
- return json.dumps(self.fields)
+ return json.dumps(self.fields, separators=(',', ':'))
class MulticastOperationsProfile(EntityClass):
@@ -950,7 +950,7 @@
]
def to_json(self):
- return json.dumps(self.fields)
+ return json.dumps(self.fields, separators=(',', ':'))
class AllowedPreviewGroupsRow0(Packet):
@@ -968,7 +968,7 @@
]
def to_json(self):
- return json.dumps(self.fields)
+ return json.dumps(self.fields, separators=(',', ':'))
class AllowedPreviewGroupsRow1(Packet):
@@ -986,7 +986,7 @@
]
def to_json(self):
- return json.dumps(self.fields)
+ return json.dumps(self.fields, separators=(',', ':'))
class MulticastSubscriberConfigInfo(EntityClass):