VOL-1333 - OpenOMCI exception caused by missing set-table to ONU and read-table from DB
Added support to complete the set transaction for an OMCI table attribute. If using
the OmcitableField, the omci will automatically handle converting a set of a single row
and upon completion of the set, cause a table "update" to occur to augment the new
row into the existing table. The supplier must provide an index() and is_delete() method
to support determination if the row set() is updating an existing row, deleting
an existing row, or adding a new row. Rows are sorted by index() order.
The ramification is also a change in the query() contract for a table attribute. It now
returns a list[] of objects rather than a single scaler object. Presently, this
only affects query of the ExtendedVlanTaggingOperationData table.
Change-Id: I2b24f747beb79013f078bbb8c37006e75fda0712
diff --git a/common/pon_resource_manager/resource_manager.py b/common/pon_resource_manager/resource_manager.py
index f2c082d..aa1b6ca 100644
--- a/common/pon_resource_manager/resource_manager.py
+++ b/common/pon_resource_manager/resource_manager.py
@@ -412,7 +412,7 @@
# delegate to the master instance if sharing enabled across instances
shared_resource_mgr = self.shared_resource_mgrs[self.shared_idx_by_type[resource_type]]
if shared_resource_mgr is not None and shared_resource_mgr is not self:
- return shared_resource_mgr.get_resource_id(pon_intf_id, resource_type)
+ return shared_resource_mgr.get_resource_id(pon_intf_id, resource_type, num_of_id)
path = self._get_path(pon_intf_id, resource_type)
if path is None:
diff --git a/tests/utests/voltha/extensions/omci/test_mib_db_dict.py b/tests/utests/voltha/extensions/omci/test_mib_db_dict.py
index 229809f..cd5cadf 100644
--- a/tests/utests/voltha/extensions/omci/test_mib_db_dict.py
+++ b/tests/utests/voltha/extensions/omci/test_mib_db_dict.py
@@ -507,13 +507,13 @@
data = self.db.query(_DEVICE_ID, class_id, inst_id, attributes.keys())
table_as_dict = json.loads(table_data.to_json())
- self.assertTrue(all(isinstance(data['received_frame_vlan_tagging_operation_table'][k],
+ self.assertTrue(all(isinstance(data['received_frame_vlan_tagging_operation_table'][0].fields[k],
type(attributes['received_frame_vlan_tagging_operation_table'].fields[k]))
for k in attributes['received_frame_vlan_tagging_operation_table'].fields.keys()))
- self.assertTrue(all(data['received_frame_vlan_tagging_operation_table'][k] ==
+ self.assertTrue(all(data['received_frame_vlan_tagging_operation_table'][0].fields[k] ==
attributes['received_frame_vlan_tagging_operation_table'].fields[k]
for k in attributes['received_frame_vlan_tagging_operation_table'].fields.keys()))
- self.assertTrue(all(data['received_frame_vlan_tagging_operation_table'][k] == table_as_dict[k]
+ self.assertTrue(all(data['received_frame_vlan_tagging_operation_table'][0].fields[k] == table_as_dict[k]
for k in table_as_dict.keys()))
diff --git a/tests/utests/voltha/extensions/omci/test_mib_db_ext.py b/tests/utests/voltha/extensions/omci/test_mib_db_ext.py
index 60dedef..f1469a0 100644
--- a/tests/utests/voltha/extensions/omci/test_mib_db_ext.py
+++ b/tests/utests/voltha/extensions/omci/test_mib_db_ext.py
@@ -506,13 +506,13 @@
data = self.db.query(_DEVICE_ID, class_id, inst_id, attributes.keys())
table_as_dict = json.loads(table_data.to_json())
- self.assertTrue(all(isinstance(data['received_frame_vlan_tagging_operation_table'][k],
+ self.assertTrue(all(isinstance(data['received_frame_vlan_tagging_operation_table'][0].fields[k],
type(attributes['received_frame_vlan_tagging_operation_table'].fields[k]))
for k in attributes['received_frame_vlan_tagging_operation_table'].fields.keys()))
- self.assertTrue(all(data['received_frame_vlan_tagging_operation_table'][k] ==
+ self.assertTrue(all(data['received_frame_vlan_tagging_operation_table'][0].fields[k] ==
attributes['received_frame_vlan_tagging_operation_table'].fields[k]
for k in attributes['received_frame_vlan_tagging_operation_table'].fields.keys()))
- self.assertTrue(all(data['received_frame_vlan_tagging_operation_table'][k] == table_as_dict[k]
+ self.assertTrue(all(data['received_frame_vlan_tagging_operation_table'][0].fields[k] == table_as_dict[k]
for k in table_as_dict.keys()))
def test_unknown_me_serialization(self):
diff --git a/voltha/adapters/tellabs_openomci_onu/tellabs_openomci_onu.py b/voltha/adapters/tellabs_openomci_onu/tellabs_openomci_onu.py
index c1cf775..0506606 100755
--- a/voltha/adapters/tellabs_openomci_onu/tellabs_openomci_onu.py
+++ b/voltha/adapters/tellabs_openomci_onu/tellabs_openomci_onu.py
@@ -73,6 +73,8 @@
)
log.info('tellabs_openomci_onu.__init__', adapter=self.descriptor)
+ self.broadcom_omci['mib-synchronizer']['state-machine'] = BrcmMibSynchronizer
+ #self.broadcom_omci['mib-synchronizer']['database'] = MibDbVolatileDict
self.broadcom_omci['mib-synchronizer']['database'] = MibDbExternal
def device_types(self):
diff --git a/voltha/extensions/omci/database/mib_db_dict.py b/voltha/extensions/omci/database/mib_db_dict.py
index 19294b2..fddbf60 100644
--- a/voltha/extensions/omci/database/mib_db_dict.py
+++ b/voltha/extensions/omci/database/mib_db_dict.py
@@ -265,6 +265,9 @@
assert value is not None, "Attribute '{}' value cannot be 'None'".\
format(attribute)
+ db_value = instance_db[ATTRIBUTES_KEY].get(attribute) \
+ if ATTRIBUTES_KEY in instance_db else None
+
if entity is not None and isinstance(value, basestring):
from scapy.fields import StrFixedLenField
attr_index = entity.attribute_name_to_index_map[attribute]
@@ -278,6 +281,14 @@
# Value/hex of Packet Class to string
value = field.default.json_from_value(value)
+ if entity is not None and attribute in entity.attribute_name_to_index_map:
+ attr_index = entity.attribute_name_to_index_map[attribute]
+ eca = entity.attributes[attr_index]
+ field = eca.field
+
+ if hasattr(field, 'to_json'):
+ value = field.to_json(value, db_value)
+
# Complex packet types may have an attribute encoded as an object, this
# can be check by seeing if there is a to_json() conversion callable
# defined
@@ -288,9 +299,6 @@
elif isinstance(value, (list, dict)):
value = json.dumps(value, separators=(',', ':'))
- db_value = instance_db[ATTRIBUTES_KEY].get(attribute) \
- if ATTRIBUTES_KEY in instance_db else None
-
assert db_value is None or isinstance(value, type(db_value)), \
"New value type for attribute '{}' type is changing from '{}' to '{}'".\
format(attribute, type(db_value), type(value))
@@ -389,21 +397,24 @@
device_db = self._data[device_id]
if class_id is None:
- return self._fix_dev_json_attributes(copy.copy(device_db))
+ return self._fix_dev_json_attributes(copy.copy(device_db), device_id)
if not isinstance(class_id, int):
raise TypeError('Class ID is an integer')
+ me_map = self._omci_agent.get_device(device_id).me_map
+ entity = me_map.get(class_id)
+
class_db = device_db.get(class_id, dict())
if instance_id is None or len(class_db) == 0:
- return self._fix_cls_json_attributes(copy.copy(class_db))
+ return self._fix_cls_json_attributes(copy.copy(class_db), entity)
if not isinstance(instance_id, int):
raise TypeError('Instance ID is an integer')
instance_db = class_db.get(instance_id, dict())
if attributes is None or len(instance_db) == 0:
- return self._fix_inst_json_attributes(copy.copy(instance_db))
+ return self._fix_inst_json_attributes(copy.copy(instance_db), entity)
if not isinstance(attributes, (basestring, list, set)):
raise TypeError('Attributes should be a string or list/set of strings')
@@ -415,7 +426,9 @@
if attr in attributes}
for attr, attr_data in results.items():
- results[attr] = self._fix_attr_json_attribute(copy.copy(attr_data))
+ attr_index = entity.attribute_name_to_index_map[attr]
+ eca = entity.attributes[attr_index]
+ results[attr] = self._fix_attr_json_attribute(copy.copy(attr_data), eca)
return results
@@ -426,26 +439,38 @@
# That other database values (created, modified, ...) will still reference
# back to the original DB.
- def _fix_dev_json_attributes(self, dev_data):
+ def _fix_dev_json_attributes(self, dev_data, device_id):
for cls_id, cls_data in dev_data.items():
if isinstance(cls_id, int):
- dev_data[cls_id] = self._fix_cls_json_attributes(copy.copy(cls_data))
+ me_map = self._omci_agent.get_device(device_id).me_map
+ entity = me_map.get(cls_id)
+ dev_data[cls_id] = self._fix_cls_json_attributes(copy.copy(cls_data), entity)
return dev_data
- def _fix_cls_json_attributes(self, cls_data):
+ def _fix_cls_json_attributes(self, cls_data, entity):
for inst_id, inst_data in cls_data.items():
if isinstance(inst_id, int):
- cls_data[inst_id] = self._fix_inst_json_attributes(copy.copy(inst_data))
+ cls_data[inst_id] = self._fix_inst_json_attributes(copy.copy(inst_data), entity)
return cls_data
- def _fix_inst_json_attributes(self, inst_data):
+ def _fix_inst_json_attributes(self, inst_data, entity):
if ATTRIBUTES_KEY in inst_data:
for attr, attr_data in inst_data[ATTRIBUTES_KEY].items():
- inst_data[ATTRIBUTES_KEY][attr] = self._fix_attr_json_attribute(copy.copy(attr_data))
+ attr_index = entity.attribute_name_to_index_map[attr] \
+ if entity is not None and attr in entity.attribute_name_to_index_map else None
+ eca = entity.attributes[attr_index] if attr_index is not None else None
+ inst_data[ATTRIBUTES_KEY][attr] = self._fix_attr_json_attribute(copy.copy(attr_data), eca)
return inst_data
- def _fix_attr_json_attribute(self, attr_data):
+ def _fix_attr_json_attribute(self, attr_data, eca):
+
try:
+ if eca is not None:
+ field = eca.field
+ if hasattr(field, 'load_json'):
+ value = field.load_json(attr_data)
+ return value
+
return json.loads(attr_data) if isinstance(attr_data, basestring) else attr_data
except ValueError:
diff --git a/voltha/extensions/omci/database/mib_db_ext.py b/voltha/extensions/omci/database/mib_db_ext.py
index 13d19dc..cf7ad1d 100644
--- a/voltha/extensions/omci/database/mib_db_ext.py
+++ b/voltha/extensions/omci/database/mib_db_ext.py
@@ -136,7 +136,7 @@
def _string_to_time(self, time):
return datetime.strptime(time, MibDbExternal._TIME_FORMAT) if len(time) else None
- def _attribute_to_string(self, device_id, class_id, attr_name, value):
+ def _attribute_to_string(self, device_id, class_id, attr_name, value, old_value = None):
"""
Convert an ME's attribute value to string representation
@@ -161,9 +161,8 @@
from voltha.extensions.omci.omci_cc import UNKNOWN_CLASS_ATTRIBUTE_KEY
field = StrFixedLenField(UNKNOWN_CLASS_ATTRIBUTE_KEY, None, 24)
- if isinstance(field, StrFixedLenField) or isinstance(field, MultipleTypeField):
+ if isinstance(field, StrFixedLenField):
from scapy.base_classes import Packet_metaclass
- # For StrFixedLenField, value is a str already (or possibly JSON encoded)
if hasattr(value, 'to_json') and not isinstance(value, basestring):
# Packet Class to string
str_value = value.to_json()
@@ -190,6 +189,9 @@
#
str_value = str(value)
+ elif hasattr(field, 'to_json'):
+ str_value = field.to_json(value, old_value)
+
elif isinstance(field, FieldListField):
str_value = json.dumps(value, separators=(',', ':'))
@@ -231,11 +233,9 @@
from voltha.extensions.omci.omci_cc import UNKNOWN_CLASS_ATTRIBUTE_KEY
field = StrFixedLenField(UNKNOWN_CLASS_ATTRIBUTE_KEY, None, 24)
- if isinstance(field, StrFixedLenField) or isinstance(field, MultipleTypeField):
+ if isinstance(field, StrFixedLenField):
from scapy.base_classes import Packet_metaclass
default = field.default
- if isinstance(field.default, PacketField):
- default = default.cls
if isinstance(default, Packet_metaclass) and \
hasattr(default, 'to_json'):
value = json.loads(str_value)
@@ -256,6 +256,9 @@
elif isinstance(field, BitField):
value = long(str_value)
+ elif hasattr(field, 'load_json'):
+ value = field.load_json(str_value)
+
elif isinstance(field, FieldListField):
value = json.loads(str_value)
@@ -740,7 +743,10 @@
for k, v in attributes.items():
try:
- str_value = self._attribute_to_string(device_id, class_id, k, v)
+ old_value = None if k not in exist_attr_indexes \
+ else new_attributes[exist_attr_indexes[k]].value
+
+ str_value = self._attribute_to_string(device_id, class_id, k, v, old_value)
if k not in exist_attr_indexes:
new_attributes.append(MibAttributeData(name=k, value=str_value))
diff --git a/voltha/extensions/omci/omci_cc.py b/voltha/extensions/omci/omci_cc.py
index 22babf0..baacd41 100644
--- a/voltha/extensions/omci/omci_cc.py
+++ b/voltha/extensions/omci/omci_cc.py
@@ -440,14 +440,14 @@
},
}),
# OmciAlarmNotification.message_id: (OmciAlarmNotification, None),
- # OmciAttributeValueChange.message_id: (OmciAttributeValueChange,
- # {
- # 'entity_class': unpack('!H', msg[0:2])[0],
- # 'entity_id': unpack('!H', msg[2:4])[0],
- # 'data': {
- # UNKNOWN_CLASS_ATTRIBUTE_KEY: hexlify(msg[4:-8])
- # },
- # }),
+ OmciAttributeValueChange.message_id: (OmciAttributeValueChange,
+ {
+ 'entity_class': unpack('!H', msg[0:2])[0],
+ 'entity_id': unpack('!H', msg[2:4])[0],
+ 'data': {
+ UNKNOWN_CLASS_ATTRIBUTE_KEY: hexlify(msg[4:-8])
+ },
+ }),
# OmciTestResult.message_id: (OmciTestResult, None),
}.get(msg_type, None)
diff --git a/voltha/extensions/omci/omci_entities.py b/voltha/extensions/omci/omci_entities.py
index f97fc2d..3968224 100644
--- a/voltha/extensions/omci/omci_entities.py
+++ b/voltha/extensions/omci/omci_entities.py
@@ -618,6 +618,37 @@
)
return json.dumps(temp.fields, separators=(',', ':'))
+ def index(self):
+ return '{:02}'.format(self.fields.get('filter_outer_priority',0)) + \
+ '{:03}'.format(self.fields.get('filter_outer_vid',0)) + \
+ '{:01}'.format(self.fields.get('filter_outer_tpid_de',0)) + \
+ '{:03}'.format(self.fields.get('filter_inner_priority',0)) + \
+ '{:04}'.format(self.fields.get('filter_inner_vid',0)) + \
+ '{:01}'.format(self.fields.get('filter_inner_tpid_de',0)) + \
+ '{:02}'.format(self.fields.get('filter_ether_type',0))
+
+ def is_delete(self):
+ return self.fields.get('treatment_tags_to_remove',0) == 0x3 and \
+ self.fields.get('pad3',0) == 0x3ff and \
+ self.fields.get('treatment_outer_priority',0) == 0xf and \
+ self.fields.get('treatment_outer_vid',0) == 0x1fff and \
+ self.fields.get('treatment_outer_tpid_de',0) == 0x7 and \
+ self.fields.get('pad4',0) == 0xfff and \
+ self.fields.get('treatment_inner_priority',0) == 0xf and \
+ self.fields.get('treatment_inner_vid',0) == 0x1fff and \
+ self.fields.get('treatment_inner_tpid_de',0) == 0x7
+
+ def delete(self):
+ self.fields['treatment_tags_to_remove'] = 0x3
+ self.fields['pad3'] = 0x3ff
+ self.fields['treatment_outer_priority'] = 0xf
+ self.fields['treatment_outer_vid'] = 0x1fff
+ self.fields['treatment_outer_tpid_de'] = 0x7
+ self.fields['pad4'] = 0xfff
+ self.fields['treatment_inner_priority'] = 0xf
+ self.fields['treatment_inner_vid'] = 0x1fff
+ self.fields['treatment_inner_tpid_de'] = 0x7
+ return self
class ExtendedVlanTaggingOperationConfigurationData(EntityClass):
class_id = 171
diff --git a/voltha/extensions/omci/omci_fields.py b/voltha/extensions/omci/omci_fields.py
index b7241bf..b6ccf5e 100644
--- a/voltha/extensions/omci/omci_fields.py
+++ b/voltha/extensions/omci/omci_fields.py
@@ -14,7 +14,9 @@
# limitations under the License.
#
import binascii
-from scapy.fields import Field, StrFixedLenField, PadField, IntField, FieldListField, ByteField, StrField, StrFixedLenField
+import json
+from scapy.fields import Field, StrFixedLenField, PadField, IntField, FieldListField, ByteField, StrField, \
+ StrFixedLenField, PacketField
from scapy.packet import Raw
class FixedLenField(PadField):
@@ -171,6 +173,9 @@
class OmciTableField(MultipleTypeField):
def __init__(self, tblfld):
+ assert isinstance(tblfld, PacketField)
+ assert hasattr(tblfld.cls, 'index'), 'No index() method defined for OmciTableField row object'
+ assert hasattr(tblfld.cls, 'is_delete'), 'No delete() method defined for OmciTableField row object'
super(OmciTableField, self).__init__(
[
(IntField('table_length', 0), (self.cond_pkt, self.cond_pkt_val)),
@@ -193,3 +198,42 @@
def cond_pkt_val2(self, pkt, val):
return pkt is not None and pkt.message_id == self.OmciGetNextResponseMessageId
+
+ def to_json(self, new_values, old_values_json):
+ if not isinstance(new_values, list): new_values = [new_values] # If setting a scalar, augment the old table
+ else: old_values_json = None # If setting a vector of new values, erase all old_values
+
+ key_value_pairs = dict()
+
+ old_table = self.load_json(old_values_json)
+ for old in old_table:
+ index = old.index()
+ key_value_pairs[index] = old
+ for new in new_values:
+ index = new.index()
+ if new.is_delete():
+ del key_value_pairs[index]
+ else:
+ key_value_pairs[index] = new
+
+ new_table = []
+ for k, v in sorted(key_value_pairs.iteritems()):
+ assert isinstance(v, self.default.cls), 'object type for Omci Table row object invalid'
+ new_table.append(v.fields)
+
+ str_values = json.dumps(new_table, separators=(',', ':'))
+
+ return str_values
+
+ def load_json(self, json_str):
+ if json_str is None: json_str = '[]'
+ json_values = json.loads(json_str)
+ key_value_pairs = dict()
+ for json_value in json_values:
+ v = self.default.cls(**json_value)
+ index = v.index()
+ key_value_pairs[index] = v
+ table = []
+ for k, v in sorted(key_value_pairs.iteritems()):
+ table.append(v)
+ return table
\ No newline at end of file
diff --git a/voltha/extensions/omci/state_machines/mib_sync.py b/voltha/extensions/omci/state_machines/mib_sync.py
index 16f29b2..2a8b535 100644
--- a/voltha/extensions/omci/state_machines/mib_sync.py
+++ b/voltha/extensions/omci/state_machines/mib_sync.py
@@ -385,7 +385,7 @@
self._mib_data_sync = self._database.get_mib_data_sync(self._device_id) or 0
def success(onu_mds_value):
- self.log.debug('examine-mds-success', mds_value=onu_mds_value)
+ self.log.debug('examine-mds-success', onu_mds_value=onu_mds_value, olt_mds_value=self.mib_data_sync)
self._current_task = None
# Examine MDS value
@@ -437,13 +437,13 @@
self._on_olt_only_diffs is None
def success(onu_mds_value):
- self.log.debug('examine-mds-success', mds_value=onu_mds_value)
+ self.log.debug('reconcile-success', mds_value=onu_mds_value)
self._current_task = None
self._next_resync = datetime.utcnow() + timedelta(seconds=self._resync_delay)
self._deferred = reactor.callLater(0, self.success)
def failure(reason):
- self.log.info('examine-mds-failure', reason=reason)
+ self.log.info('reconcile-failure', reason=reason)
self._current_task = None
self._deferred = reactor.callLater(self._timeout_delay, self.timeout)
@@ -480,7 +480,7 @@
self._deferred = reactor.callLater(0, self.force_resync)
else:
def success(onu_mds_value):
- self.log.debug('get-mds-success', mds_value=onu_mds_value)
+ self.log.debug('audit-success', onu_mds_value=onu_mds_value, olt_mds_value=self.mib_data_sync)
self._current_task = None
# Examine MDS value
@@ -491,7 +491,7 @@
self._deferred = reactor.callLater(0, self.mismatch)
def failure(reason):
- self.log.info('get-mds-failure', reason=reason)
+ self.log.info('audit-failure', reason=reason)
self._current_task = None
self._deferred = reactor.callLater(self._timeout_delay, self.timeout)