Fix persisted config data generating different hashes and tags not being persisted
Data loaded from the key value store could generate different hashes depending on the order python decides to give us keys back
Everytime tags are updated, push the tag to the datastore if one is provided and properly load the tags back
Change-Id: I148c01b13009a038187c4aeec3080b105a7d8956
diff --git a/tests/utests/voltha/core/config/test_persistence.py b/tests/utests/voltha/core/config/test_persistence.py
index fe91919..6ff3ac2 100644
--- a/tests/utests/voltha/core/config/test_persistence.py
+++ b/tests/utests/voltha/core/config/test_persistence.py
@@ -2,6 +2,7 @@
from random import randint, seed
from time import time
from unittest import main, TestCase
+import json
from voltha.core.config.config_root import ConfigRoot
from voltha.protos.openflow_13_pb2 import ofp_desc
@@ -54,9 +55,11 @@
# create node and pump data
node = ConfigRoot(VolthaInstance(), kv_store=kv_store)
+ node.tag('original')
pt('init')
self.pump_some_data(node)
pt('pump')
+ node.tag('pumped')
# check that content of kv_store looks ok
size1 = len(kv_store)
@@ -67,22 +70,26 @@
pt('prunning')
size2 = len(kv_store)
- self.assertEqual(size2, 7 + 2 * (1 + 1 + n_adapters + n_logical_nodes))
+ self.assertEqual(size2, 7 + 2 * (1 + 1 + n_adapters + n_logical_nodes) + 2)
all_latest_data = node.get('/', deep=1)
pt('deep get')
# save dict so that deleting the node will not wipe it
+ latest_hash = node.latest.hash
kv_store = copy(kv_store)
pt('copy kv store')
del node
pt('delete node')
# self.assertEqual(size2, 1 + 2 * (1 + 1 + n_adapters + n_logical_nodes))
+ self.assertEqual(json.loads(kv_store['root'])['latest'], latest_hash)
# recreate tree from persistence
node = ConfigRoot.load(VolthaInstance, kv_store)
pt('load from kv store')
self.assertEqual(node.get('/', deep=1), all_latest_data)
pt('deep get')
+ self.assertEqual(latest_hash, node.latest.hash)
+ self.assertEqual(node.tags, ['original', 'pumped'])
if __name__ == '__main__':
diff --git a/voltha/core/config/config_node.py b/voltha/core/config/config_node.py
index b588808..cba6d88 100644
--- a/voltha/core/config/config_node.py
+++ b/voltha/core/config/config_node.py
@@ -471,6 +471,7 @@
branch = self._branches[None] # tag only what has been committed
rev = branch._latest if hash is None else branch._revs[hash]
self._tags[tag] = rev
+ self.persist_tags()
return self
@property
@@ -490,10 +491,12 @@
def delete_tag(self, tag):
del self._tags[tag]
+ self.persist_tags()
def delete_tags(self, *tags):
for tag in tags:
del self._tags[tag]
+ self.persist_tags()
def prune_untagged(self):
branch = self._branches[None]
@@ -504,6 +507,11 @@
del branch._revs[hash]
return self
+ def persist_tags(self):
+ """
+ Persist tag information to the backend
+ """
+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Internals ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def _test_no_children(self, data):
diff --git a/voltha/core/config/config_rev.py b/voltha/core/config/config_rev.py
index e2b1f24..01c1ae2 100644
--- a/voltha/core/config/config_rev.py
+++ b/voltha/core/config/config_rev.py
@@ -215,7 +215,7 @@
def _hash_data(data):
"""Hash function to be used to track version changes of config nodes"""
if isinstance(data, (dict, list)):
- to_hash = dumps(data)
+ to_hash = dumps(data, sort_keys=True)
elif is_proto_message(data):
to_hash = ':'.join((
data.__class__.__module__,
@@ -262,7 +262,8 @@
# hash is derived from config hash and hashes of all children
m = md5('' if self._config is None else self._config._hash)
if self._children is not None:
- for children in self._children.itervalues():
+ for child_field in sorted(self._children.keys()):
+ children = self._children[child_field]
assert isinstance(children, list)
m.update(''.join(c._hash for c in children))
return m.hexdigest()[:12]
diff --git a/voltha/core/config/config_root.py b/voltha/core/config/config_root.py
index a5bc229..012ce75 100644
--- a/voltha/core/config/config_root.py
+++ b/voltha/core/config/config_root.py
@@ -176,13 +176,24 @@
blob = dumps(root_data)
self._kv_store['root'] = blob
+ def persist_tags(self):
+ if self._kv_store is not None:
+ root_data = loads(self.kv_store['root'])
+ root_data = dict(
+ latest=root_data['latest'],
+ tags=dict((k, v._hash) for k, v in self._tags.iteritems())
+ )
+ blob = dumps(root_data)
+ self._kv_store['root'] = blob
+
def load_from_persistence(self, root_msg_cls):
self._loading = True
blob = self._kv_store['root']
root_data = loads(blob)
for tag, hash in root_data['tags'].iteritems():
- raise NotImplementedError()
+ self.load_latest(hash)
+ self._tags[tag] = self.latest
self.load_latest(root_data['latest'])