SEBA-301 Add diff methods to ORM
Change-Id: I47622670bd6cb96b5574352670e24a111ff616a0
diff --git a/VERSION b/VERSION
index f316ecb..dfb6396 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.1.19
+2.1.20
diff --git a/containers/chameleon/Dockerfile.chameleon b/containers/chameleon/Dockerfile.chameleon
index 0b197e7..c844f73 100644
--- a/containers/chameleon/Dockerfile.chameleon
+++ b/containers/chameleon/Dockerfile.chameleon
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/chameleon
-FROM xosproject/xos-base:2.1.18
+FROM xosproject/xos-base:2.1.20
# xos-base already has protoc and dependencies installed
diff --git a/containers/xos/Dockerfile.client b/containers/xos/Dockerfile.client
index 19d6163..90fe477 100644
--- a/containers/xos/Dockerfile.client
+++ b/containers/xos/Dockerfile.client
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/xos-client
-FROM xosproject/xos-libraries:2.1.19
+FROM xosproject/xos-libraries:2.1.20
# Install XOS client
COPY xos/xos_client /tmp/xos_client
diff --git a/containers/xos/Dockerfile.libraries b/containers/xos/Dockerfile.libraries
index d3808f1..5d289f8 100644
--- a/containers/xos/Dockerfile.libraries
+++ b/containers/xos/Dockerfile.libraries
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/xos-libraries
-FROM xosproject/xos-base:2.1.19
+FROM xosproject/xos-base:2.1.20
# Add libraries
COPY lib /opt/xos/lib
diff --git a/containers/xos/Dockerfile.synchronizer-base b/containers/xos/Dockerfile.synchronizer-base
index 7caf211..57a7ba5 100644
--- a/containers/xos/Dockerfile.synchronizer-base
+++ b/containers/xos/Dockerfile.synchronizer-base
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/xos-synchronizer-base
-FROM xosproject/xos-client:2.1.19
+FROM xosproject/xos-client:2.1.20
COPY xos/synchronizers/new_base /opt/xos/synchronizers/new_base
COPY xos/xos/logger.py /opt/xos/xos/logger.py
diff --git a/containers/xos/Dockerfile.xos-core b/containers/xos/Dockerfile.xos-core
index 0edef02..1f52a75 100644
--- a/containers/xos/Dockerfile.xos-core
+++ b/containers/xos/Dockerfile.xos-core
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/xos-core
-FROM xosproject/xos-libraries:2.1.19
+FROM xosproject/xos-libraries:2.1.20
# Install XOS
ADD xos /opt/xos
diff --git a/lib/xos-genx/xosgenx/targets/mock_classes.xtarget b/lib/xos-genx/xosgenx/targets/mock_classes.xtarget
index 86deac2..69efe92 100644
--- a/lib/xos-genx/xosgenx/targets/mock_classes.xtarget
+++ b/lib/xos-genx/xosgenx/targets/mock_classes.xtarget
@@ -74,27 +74,39 @@
else:
self.get_items().append(o)
-class MockObject:
+class MockObject(object):
objects = None
id = None
deleted = False
+ field_names = []
+
def __init__(self, **kwargs):
+ object.__setattr__(self, 'is_set', {})
+
setattr(self, 'backend_code', 0)
setattr(self, 'id', 98052)
setattr(self, 'pk', random.randint(0, 1<<30))
-
+
+ self.leaf_model = self
+
+ # reset is_set
+ self.is_set = {}
+
for (k,v) in kwargs.items():
setattr(self,k,v)
+ self.is_new = True
+ self._initial = self._dict
+
+ def __setattr__(self, name, value):
+ self.is_set[name] = True
+ object.__setattr__(self, name, value)
+
@property
def self_content_type_id(self):
return self.__class__.__name__
- @property
- def leaf_model(self):
- return self
-
def save(self, update_fields=[], always_update_timestamp=False):
if self.objects:
self.objects.save(self)
@@ -105,6 +117,53 @@
def tologdict(self):
return {}
+ @property
+ def _dict(self):
+ d={}
+ for name in self.field_names:
+ if self.is_set.get(name, False):
+ d[name] = getattr(self, name)
+ return d
+
+ @property
+ def diff(self):
+ d1 = self._initial
+ d2 = self._dict
+ all_field_names = self.field_names
+ diffs=[]
+ for k in all_field_names:
+ if (d1.get(k,None) != d2.get(k,None)):
+ diffs.append( (k, (d1.get(k,None), d2.get(k,None))) )
+
+ return dict(diffs)
+
+ @property
+ def has_changed(self):
+ return bool(self.diff)
+
+ @property
+ def changed_fields(self):
+ if self.is_new:
+ return self._dict.keys()
+ return self.diff.keys()
+
+ def has_field_changed(self, field_name):
+ return field_name in self.diff.keys()
+
+ def get_field_diff(self, field_name):
+ return self.diff.get(field_name, None)
+
+ def recompute_initial(self):
+ self._initial = self._dict
+
+ def save_changed_fields(self, always_update_timestamp=False):
+ if self.has_changed:
+ update_fields = self.changed_fields
+ if always_update_timestamp and "updated" not in update_fields:
+ update_fields.append("updated")
+ self.save(update_fields=sorted(update_fields), always_update_timestamp=always_update_timestamp)
+
+
def get_MockObjectStore(x):
store = globals()["Mock%sObjects" % x]()
if not store in AllMockObjectStores:
@@ -163,4 +222,11 @@
{% if f.link -%}{{ f.name }}_id = None{% endif %}
{% endfor %}
leaf_model_name = "{{ m.name }}"
+
+ field_names = ["id", \
+ {% for f in xproto_base_fields(m, proto.message_table) + m.fields -%}
+ "{{ f.name }}",
+ {% endfor %}
+ ]
+
{% endfor %}
diff --git a/xos/synchronizers/new_base/tests/test_diffs.py b/xos/synchronizers/new_base/tests/test_diffs.py
new file mode 100644
index 0000000..6ae2b37
--- /dev/null
+++ b/xos/synchronizers/new_base/tests/test_diffs.py
@@ -0,0 +1,110 @@
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+from mock import patch, call, Mock, PropertyMock
+import json
+
+import os, sys
+
+# Hack to load synchronizer framework
+test_path=os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
+xos_dir=os.path.join(test_path, "../../..")
+if not os.path.exists(os.path.join(test_path, "new_base")):
+ xos_dir=os.path.join(test_path, "../../../../../../orchestration/xos/xos")
+ services_dir = os.path.join(xos_dir, "../../xos_services")
+sys.path.append(xos_dir)
+sys.path.append(os.path.join(xos_dir, 'synchronizers', 'new_base'))
+# END Hack to load synchronizer framework
+
+class TestDiffs(unittest.TestCase):
+
+ """ These tests are for the mock modelaccessor, to make sure it behaves like the real one """
+
+ def setUp(self):
+
+ self.sys_path_save = sys.path
+ sys.path.append(xos_dir)
+ sys.path.append(os.path.join(xos_dir, 'synchronizers', 'new_base'))
+
+ # Setting up the config module
+ from xosconfig import Config
+ config = os.path.join(test_path, "test_config.yaml")
+ Config.clear()
+ Config.init(config, "synchronizer-config-schema.yaml")
+ # END Setting up the config module
+
+ from synchronizers.new_base.mock_modelaccessor_build import build_mock_modelaccessor
+
+ # FIXME this is to get jenkins to pass the tests, somehow it is running tests in a different order
+ # and apparently it is not overriding the generated model accessor
+ build_mock_modelaccessor(xos_dir, services_dir, [])
+ import synchronizers.new_base.modelaccessor
+
+ # import all class names to globals
+ for (k, v) in synchronizers.new_base.modelaccessor.model_accessor.all_model_classes.items():
+ globals()[k] = v
+
+ self.log = Mock()
+
+
+ def tearDown(self):
+ sys.path = self.sys_path_save
+
+
+ def test_new_diff(self):
+ site = Site(name="mysite")
+
+ self.assertEqual(site.is_new, True)
+ self.assertEqual(site._dict, {"name": "mysite"})
+ self.assertEqual(site.diff, {})
+ self.assertEqual(site.changed_fields, ["name"])
+ self.assertEqual(site.has_field_changed("name"), False)
+ self.assertEqual(site.has_field_changed("login_base"), False)
+
+ site.login_base = "bar"
+
+ self.assertEqual(site._dict, {'login_base': 'bar', 'name': 'mysite'})
+ self.assertEqual(site.diff, {'login_base': (None, 'bar')})
+ self.assertIn("name", site.changed_fields)
+ self.assertIn("login_base", site.changed_fields)
+ self.assertEqual(site.has_field_changed("name"), False)
+ self.assertEqual(site.has_field_changed("login_base"), True)
+ self.assertEqual(site.get_field_diff("login_base"), (None, "bar"))
+
+ def test_existing_diff(self):
+ site = Site(name="mysite", login_base="foo")
+
+ # this is what would happen after saving and re-loading
+ site.is_new = False
+ site.id = 1
+ site._initial = site._dict
+
+ self.assertEqual(site.is_new, False)
+ self.assertEqual(site._dict, {"id": 1, "name": "mysite", "login_base": "foo"})
+ self.assertEqual(site.diff, {})
+ self.assertEqual(site.changed_fields, [])
+ self.assertEqual(site.has_field_changed("name"), False)
+ self.assertEqual(site.has_field_changed("login_base"), False)
+
+ site.login_base = "bar"
+
+ self.assertEqual(site._dict, {'id': 1, 'login_base': 'bar', 'name': 'mysite'})
+ self.assertEqual(site.diff, {'login_base': ("foo", 'bar')})
+ self.assertIn("login_base", site.changed_fields)
+ self.assertEqual(site.has_field_changed("name"), False)
+ self.assertEqual(site.has_field_changed("login_base"), True)
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/xos/xos_client/xosapi/fake_stub.py b/xos/xos_client/xosapi/fake_stub.py
index 3cd8094..8cc60f7 100644
--- a/xos/xos_client/xosapi/fake_stub.py
+++ b/xos/xos_client/xosapi/fake_stub.py
@@ -72,6 +72,14 @@
return self.is_set.get(name, False)
+ def ListFields(self):
+ fbn = self.DESCRIPTOR.fields_by_name
+ l = []
+ for (k,v) in fbn.items():
+ if self.is_set.get(k, False):
+ l.append( (v, getattr(self, k)) )
+ return l
+
@property
def self_content_type_id(self):
return "xos.%s" % self.__class__.__name__.lower()
@@ -99,6 +107,7 @@
extensions = {}
self.field_decl = field
+ self.name = field["name"]
fk_model = field.get("fk_model", None)
if fk_model:
@@ -192,6 +201,7 @@
class Site(FakeObj):
FIELDS = ( {"name": "id", "default": 0},
{"name": "name", "default": ""},
+ {"name": "login_base", "default": ""},
{"name": "slices_ids", "default": [], "fk_reverse": "Slice"},
{"name": "leaf_model_name", "default": "Site"},
{"name": "class_names", "default": "Site"})
diff --git a/xos/xos_client/xosapi/orm.py b/xos/xos_client/xosapi/orm.py
index 392dd84..0ea714b 100644
--- a/xos/xos_client/xosapi/orm.py
+++ b/xos/xos_client/xosapi/orm.py
@@ -69,6 +69,67 @@
super(ORMWrapper, self).__setattr__("_fkmap", fkmap)
reverse_fkmap=self.gen_reverse_fkmap()
super(ORMWrapper, self).__setattr__("_reverse_fkmap", reverse_fkmap)
+ super(ORMWrapper, self).__setattr__("_initial", self._dict)
+
+ def fields_differ(self,f1,f2):
+ return (f1 != f2)
+
+ @property
+ def _dict(self):
+ """ Return a dictionary of {fieldname: fieldvalue} for the object.
+
+ This differs for the xos-core implementation of XOSBase. For new object, XOSBase will include field names
+ that are set to default values. ORM ignores fields that are set to default values.
+ """
+ d={}
+ for (fieldDesc, val) in self._wrapped_class.ListFields():
+ name = fieldDesc.name
+ d[name] = val
+ return d
+
+ @property
+ def diff(self):
+ d1 = self._initial
+ d2 = self._dict
+ all_field_names = self._wrapped_class.DESCRIPTOR.fields_by_name.keys()
+ diffs=[]
+ for k in all_field_names:
+ if (d1.get(k,None) != d2.get(k,None)):
+ diffs.append( (k, (d1.get(k,None), d2.get(k,None))) )
+
+ #diffs = [(k, (v, d2[k])) for k, v in d1.items() if self.fields_differ(v,d2[k])]
+ return dict(diffs)
+
+ @property
+ def has_changed(self):
+ return bool(self.diff)
+
+ @property
+ def changed_fields(self):
+ """ Return the list of changed fields.
+
+ This differs for the xos-core implementation of XOSBase. For new object, XOSBase will include field names
+ that are set to default values.
+ """
+ if self.is_new:
+ return self._dict.keys()
+ return self.diff.keys()
+
+ def has_field_changed(self, field_name):
+ return field_name in self.diff.keys()
+
+ def get_field_diff(self, field_name):
+ return self.diff.get(field_name, None)
+
+ def recompute_initial(self):
+ self._initial = self._dict
+
+ def save_changed_fields(self, always_update_timestamp=False):
+ if self.has_changed:
+ update_fields = self.changed_fields
+ if always_update_timestamp and "updated" not in update_fields:
+ update_fields.append("updated")
+ self.save(update_fields=sorted(update_fields), always_update_timestamp=always_update_timestamp)
def create_attr(self, name, value=None):
""" setattr(self, ...) will fail for attributes that don't exist in the
@@ -309,6 +370,9 @@
self.stub.invoke("Update%s" % self._wrapped_class.__class__.__name__, self._wrapped_class, metadata=metadata)
self.do_post_save_fixups()
+ # Now that object has saved, reset our initial state for diff calculation
+ self.recompute_initial()
+
def delete(self):
id = self.stub.make_ID(id=self._wrapped_class.id)
self.stub.invoke("Delete%s" % self._wrapped_class.__class__.__name__, id)
@@ -502,6 +566,7 @@
o = make_ORMWrapper(cls(), self._stub, is_new=True)
for (k,v) in kwargs.items():
setattr(o, k, v)
+ o.recompute_initial()
return o
class ORMModelClass(object):
diff --git a/xos/xos_client/xosapi/test_orm.py b/xos/xos_client/xosapi/test_orm.py
index ddeab89..4c3cd32 100644
--- a/xos/xos_client/xosapi/test_orm.py
+++ b/xos/xos_client/xosapi/test_orm.py
@@ -479,6 +479,65 @@
self.assertEqual(q.elements[0].operator, q.elements[0].EQUAL)
self.assertEqual(q.elements[0].sValue, "foo")
+ def test_ORMWrapper_new_diff(self):
+ orm = self.make_coreapi()
+ site = orm.Site(name="mysite")
+
+ self.assertEqual(site.is_new, True)
+ self.assertEqual(site._dict, {"name": "mysite"})
+ self.assertEqual(site.diff, {})
+ self.assertEqual(site.changed_fields, ["name"])
+ self.assertEqual(site.has_field_changed("name"), False)
+ self.assertEqual(site.has_field_changed("login_base"), False)
+
+ site.login_base = "bar"
+
+ self.assertEqual(site._dict, {'login_base': 'bar', 'name': 'mysite'})
+ self.assertEqual(site.diff, {'login_base': (None, 'bar')})
+ self.assertIn("name", site.changed_fields)
+ self.assertIn("login_base", site.changed_fields)
+ self.assertEqual(site.has_field_changed("name"), False)
+ self.assertEqual(site.has_field_changed("login_base"), True)
+ self.assertEqual(site.get_field_diff("login_base"), (None, "bar"))
+
+ def test_ORMWrapper_existing_diff(self):
+ orm = self.make_coreapi()
+ site = orm.Site(name="mysite", login_base="foo")
+ site.save()
+ site = orm.Site.objects.first()
+
+ self.assertEqual(site.is_new, False)
+ self.assertEqual(site._dict, {"id": 1, "name": "mysite", "login_base": "foo"})
+ self.assertEqual(site.diff, {})
+ self.assertEqual(site.changed_fields, [])
+ self.assertEqual(site.has_field_changed("name"), False)
+ self.assertEqual(site.has_field_changed("login_base"), False)
+
+ site.login_base = "bar"
+
+ self.assertEqual(site._dict, {'id': 1, 'login_base': 'bar', 'name': 'mysite'})
+ self.assertEqual(site.diff, {'login_base': ("foo", 'bar')})
+ self.assertIn("login_base", site.changed_fields)
+ self.assertEqual(site.has_field_changed("name"), False)
+ self.assertEqual(site.has_field_changed("login_base"), True)
+
+ def test_ORMWrapper_diff_after_save(self):
+ orm = self.make_coreapi()
+ site = orm.Site(name="mysite", login_base="foo")
+ site.save()
+ site = orm.Site.objects.first()
+
+ self.assertEqual(site.diff, {})
+
+ site.login_base = "bar"
+
+ self.assertEqual(site.diff, {'login_base': ("foo", 'bar')})
+
+ site.save()
+
+ self.assertEqual(site.diff, {})
+
+
def main():
global USE_FAKE_STUB
global xos_grpc_client