blob: 41d775f9947737f5f430594c4eaec618018bac0e [file] [log] [blame]
Matteo Scandolod2044a42017-08-07 16:08:28 -07001# Copyright 2017-present Open Networking Foundation
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
Zack Williams045b63d2019-01-22 16:30:57 -070015from __future__ import print_function
16import os
17import sys
18import threading
19import time
20import imp
Andy Bavier04ee1912019-01-30 14:17:16 -070021import traceback
Zack Williams045b63d2019-01-22 16:30:57 -070022from xosconfig import Config
23from multistructlog import create_logger
Matteo Scandolod2044a42017-08-07 16:08:28 -070024
Scott Baker96b995a2017-02-15 16:21:12 -080025"""
26Django-like ORM layer for gRPC
27
28Usage:
29 api = ORMStub(stub)
30
31 api.Slices.all() ... list all slices
32
33 someSlice = api.Slices.get(id=1) ... get slice #1
34
35 someSlice.site ... automatically resolves site_id into a site object
36 someSlice.instances ... automatically resolves instances_ids into instance objects
37 someSlice.save() ... saves the slice object
38"""
39
40"""
41import grpc_client, orm
42c=grpc_client.SecureClient("xos-core.cord.lab", username="padmin@vicci.org", password="letmein")
Scott Bakera1eae7a2017-06-06 09:20:15 -070043u=c.xos_orm.User.objects.get(id=1)
44"""
Scott Baker96b995a2017-02-15 16:21:12 -080045
Matteo Scandoloe3d2f262018-06-05 17:45:39 -070046
Zack Williams045b63d2019-01-22 16:30:57 -070047log = create_logger(Config().get("logging"))
Scott Baker96b995a2017-02-15 16:21:12 -080048
Scott Baker22796cc2017-02-23 16:53:34 -080049convenience_wrappers = {}
50
Andy Bavier04ee1912019-01-30 14:17:16 -070051# Find the topmost synchronizer-specific function in the call stack
52def get_synchronizer_function():
53 result = None
54 for file,line,func,stmt in traceback.extract_stack():
55 if file.startswith("/opt/xos/synchronizers"):
56 if not result:
57 result = "%s:%s()" % (file,func)
58 if not file.startswith("/opt/xos/synchronizers/new_base"):
59 result = "%s:%s()" % (file,func)
60 break
61 return result
Zack Williams045b63d2019-01-22 16:30:57 -070062
Scott Bakerd0f1dc12018-04-23 12:05:32 -070063class ORMGenericContentNotFoundException(Exception):
64 pass
65
Zack Williams045b63d2019-01-22 16:30:57 -070066
Scott Bakerd0f1dc12018-04-23 12:05:32 -070067class ORMGenericObjectNotFoundException(Exception):
68 pass
69
Zack Williams045b63d2019-01-22 16:30:57 -070070
Scott Baker96b995a2017-02-15 16:21:12 -080071class ORMWrapper(object):
72 """ Wraps a protobuf object to provide ORM features """
73
74 def __init__(self, wrapped_class, stub, is_new=False):
75 super(ORMWrapper, self).__setattr__("_wrapped_class", wrapped_class)
76 super(ORMWrapper, self).__setattr__("stub", stub)
77 super(ORMWrapper, self).__setattr__("cache", {})
78 super(ORMWrapper, self).__setattr__("reverse_cache", {})
Sapan Bhatia71f57682017-08-23 20:09:08 -040079 super(ORMWrapper, self).__setattr__("synchronizer_step", None)
Sapan Bhatia2b307f72017-11-02 11:39:17 -040080 super(ORMWrapper, self).__setattr__("dependent", None)
Scott Baker96b995a2017-02-15 16:21:12 -080081 super(ORMWrapper, self).__setattr__("is_new", is_new)
Scott Bakerc4156c32017-12-08 10:58:21 -080082 super(ORMWrapper, self).__setattr__("post_save_fixups", [])
Zack Williams045b63d2019-01-22 16:30:57 -070083 fkmap = self.gen_fkmap()
Scott Baker96b995a2017-02-15 16:21:12 -080084 super(ORMWrapper, self).__setattr__("_fkmap", fkmap)
Zack Williams045b63d2019-01-22 16:30:57 -070085 reverse_fkmap = self.gen_reverse_fkmap()
Scott Baker96b995a2017-02-15 16:21:12 -080086 super(ORMWrapper, self).__setattr__("_reverse_fkmap", reverse_fkmap)
Scott Baker5b7fba02018-10-17 08:46:46 -070087 super(ORMWrapper, self).__setattr__("_initial", self._dict)
88
Zack Williams045b63d2019-01-22 16:30:57 -070089 def fields_differ(self, f1, f2):
90 return f1 != f2
Scott Baker5b7fba02018-10-17 08:46:46 -070091
92 @property
93 def _dict(self):
94 """ Return a dictionary of {fieldname: fieldvalue} for the object.
95
96 This differs for the xos-core implementation of XOSBase. For new object, XOSBase will include field names
97 that are set to default values. ORM ignores fields that are set to default values.
98 """
Zack Williams045b63d2019-01-22 16:30:57 -070099 d = {}
Scott Baker5b7fba02018-10-17 08:46:46 -0700100 for (fieldDesc, val) in self._wrapped_class.ListFields():
101 name = fieldDesc.name
102 d[name] = val
103 return d
104
105 @property
106 def diff(self):
107 d1 = self._initial
108 d2 = self._dict
109 all_field_names = self._wrapped_class.DESCRIPTOR.fields_by_name.keys()
Zack Williams045b63d2019-01-22 16:30:57 -0700110 diffs = []
Scott Baker5b7fba02018-10-17 08:46:46 -0700111 for k in all_field_names:
Zack Williams045b63d2019-01-22 16:30:57 -0700112 if d1.get(k, None) != d2.get(k, None):
113 diffs.append((k, (d1.get(k, None), d2.get(k, None))))
Scott Baker5b7fba02018-10-17 08:46:46 -0700114
Zack Williams045b63d2019-01-22 16:30:57 -0700115 # diffs = [(k, (v, d2[k])) for k, v in d1.items() if self.fields_differ(v,d2[k])]
Scott Baker5b7fba02018-10-17 08:46:46 -0700116 return dict(diffs)
117
118 @property
119 def has_changed(self):
120 return bool(self.diff)
121
122 @property
123 def changed_fields(self):
124 """ Return the list of changed fields.
125
126 This differs for the xos-core implementation of XOSBase. For new object, XOSBase will include field names
127 that are set to default values.
128 """
129 if self.is_new:
130 return self._dict.keys()
131 return self.diff.keys()
132
133 def has_field_changed(self, field_name):
134 return field_name in self.diff.keys()
135
136 def get_field_diff(self, field_name):
137 return self.diff.get(field_name, None)
138
139 def recompute_initial(self):
140 self._initial = self._dict
141
142 def save_changed_fields(self, always_update_timestamp=False):
143 if self.has_changed:
144 update_fields = self.changed_fields
145 if always_update_timestamp and "updated" not in update_fields:
146 update_fields.append("updated")
Zack Williams045b63d2019-01-22 16:30:57 -0700147 self.save(
148 update_fields=sorted(update_fields),
149 always_update_timestamp=always_update_timestamp,
150 )
Scott Baker96b995a2017-02-15 16:21:12 -0800151
Scott Bakerd78f6472017-03-14 17:30:14 -0700152 def create_attr(self, name, value=None):
153 """ setattr(self, ...) will fail for attributes that don't exist in the
154 wrapped grpc class. This is by design. However, if someone really
155 does want to attach a new attribute to this class, then they can
156 call create_attr()
157 """
158 super(ORMWrapper, self).__setattr__(name, value)
159
Scott Baker03a163f2017-05-17 09:21:47 -0700160 def get_generic_foreignkeys(self):
161 """ this is a placeholder until generic foreign key support is added
162 to xproto.
163 """
164 return []
165
Scott Baker96b995a2017-02-15 16:21:12 -0800166 def gen_fkmap(self):
167 fkmap = {}
168
Scott Bakeraa556b02017-03-07 16:07:34 -0800169 all_field_names = self._wrapped_class.DESCRIPTOR.fields_by_name.keys()
170
Scott Baker96b995a2017-02-15 16:21:12 -0800171 for (name, field) in self._wrapped_class.DESCRIPTOR.fields_by_name.items():
Zack Williams045b63d2019-01-22 16:30:57 -0700172 if name.endswith("_id"):
173 foreignKey = field.GetOptions().Extensions._FindExtensionByName(
174 "xos.foreignKey"
175 )
176 fk = field.GetOptions().Extensions[foreignKey]
177 if fk and fk.modelName:
178 fkdict = {
179 "src_fieldName": name,
180 "modelName": fk.modelName,
181 "kind": "fk",
182 }
183 if fk.reverseFieldName:
184 fkdict["reverse_fieldName"] = fk.reverseFieldName
185 fkmap[name[:-3]] = fkdict
186 else:
187 # If there's a corresponding _type_id field, then see if this
188 # is a generic foreign key.
189 type_name = name[:-3] + "_type_id"
190 if type_name in all_field_names:
191 fkmap[name[:-3]] = {
192 "src_fieldName": name,
193 "ct_fieldName": type_name,
194 "kind": "generic_fk",
195 }
Scott Baker96b995a2017-02-15 16:21:12 -0800196
Scott Baker03a163f2017-05-17 09:21:47 -0700197 for gfk in self.get_generic_foreignkeys():
Zack Williams045b63d2019-01-22 16:30:57 -0700198 fkmap[gfk["name"]] = {
199 "src_fieldName": gfk["id"],
200 "ct_fieldName": gfk["content_type"],
201 "kind": "generic_fk",
202 }
Scott Baker03a163f2017-05-17 09:21:47 -0700203
Scott Baker96b995a2017-02-15 16:21:12 -0800204 return fkmap
205
206 def gen_reverse_fkmap(self):
207 reverse_fkmap = {}
208
209 for (name, field) in self._wrapped_class.DESCRIPTOR.fields_by_name.items():
Zack Williams045b63d2019-01-22 16:30:57 -0700210 if name.endswith("_ids"):
211 reverseForeignKey = field.GetOptions().Extensions._FindExtensionByName(
212 "xos.reverseForeignKey"
213 )
214 fk = field.GetOptions().Extensions[reverseForeignKey]
215 if fk and fk.modelName:
216 reverse_fkmap[name[:-4]] = {
217 "src_fieldName": name,
218 "modelName": fk.modelName,
219 "writeable": False,
220 }
221 else:
222 manyToManyForeignKey = field.GetOptions().Extensions._FindExtensionByName(
223 "xos.manyToManyForeignKey"
224 )
225 fk = field.GetOptions().Extensions[manyToManyForeignKey]
226 if fk and fk.modelName:
227 reverse_fkmap[name[:-4]] = {
228 "src_fieldName": name,
229 "modelName": fk.modelName,
230 "writeable": True,
231 }
Scott Baker96b995a2017-02-15 16:21:12 -0800232
233 return reverse_fkmap
234
235 def fk_resolve(self, name):
236 if name in self.cache:
Scott Bakerc4156c32017-12-08 10:58:21 -0800237 return self.cache[name]
Scott Baker96b995a2017-02-15 16:21:12 -0800238
239 fk_entry = self._fkmap[name]
Scott Bakeraa556b02017-03-07 16:07:34 -0800240 fk_kind = fk_entry["kind"]
241 fk_id = getattr(self, fk_entry["src_fieldName"])
242
243 if not fk_id:
244 return None
245
Zack Williams045b63d2019-01-22 16:30:57 -0700246 if fk_kind == "fk":
247 id = self.stub.make_ID(id=fk_id)
Scott Bakeraa556b02017-03-07 16:07:34 -0800248 dest_model = self.stub.invoke("Get%s" % fk_entry["modelName"], id)
249
Zack Williams045b63d2019-01-22 16:30:57 -0700250 elif fk_kind == "generic_fk":
251 dest_model = self.stub.genericForeignKeyResolve(
252 getattr(self, fk_entry["ct_fieldName"]), fk_id
253 )._wrapped_class
Scott Bakeraa556b02017-03-07 16:07:34 -0800254
255 else:
256 raise Exception("unknown fk_kind")
Scott Baker96b995a2017-02-15 16:21:12 -0800257
Scott Bakerc4156c32017-12-08 10:58:21 -0800258 dest_model = make_ORMWrapper(dest_model, self.stub)
Scott Baker96b995a2017-02-15 16:21:12 -0800259 self.cache[name] = dest_model
260
Scott Bakerc4156c32017-12-08 10:58:21 -0800261 return dest_model
Scott Baker96b995a2017-02-15 16:21:12 -0800262
263 def reverse_fk_resolve(self, name):
264 if name not in self.reverse_cache:
265 fk_entry = self._reverse_fkmap[name]
Zack Williams045b63d2019-01-22 16:30:57 -0700266 self.reverse_cache[name] = ORMLocalObjectManager(
267 self.stub,
268 fk_entry["modelName"],
269 getattr(self, fk_entry["src_fieldName"]),
270 fk_entry["writeable"],
271 )
Scott Baker96b995a2017-02-15 16:21:12 -0800272
Scott Baker7ab456b2019-01-08 14:58:13 -0800273 return self.reverse_cache[name]
Scott Baker96b995a2017-02-15 16:21:12 -0800274
Scott Bakere72e7612017-02-20 10:07:09 -0800275 def fk_set(self, name, model):
276 fk_entry = self._fkmap[name]
Scott Bakeraa556b02017-03-07 16:07:34 -0800277 fk_kind = fk_entry["kind"]
Scott Bakera1eae7a2017-06-06 09:20:15 -0700278 if model:
279 id = model.id
280 else:
281 id = 0
Scott Bakere72e7612017-02-20 10:07:09 -0800282 setattr(self._wrapped_class, fk_entry["src_fieldName"], id)
283
Zack Williams045b63d2019-01-22 16:30:57 -0700284 if fk_kind == "generic_fk":
285 setattr(
286 self._wrapped_class,
287 fk_entry["ct_fieldName"],
288 model.self_content_type_id,
289 )
Scott Bakeraa556b02017-03-07 16:07:34 -0800290
Scott Bakerc4156c32017-12-08 10:58:21 -0800291 if name in self.cache:
292 old_model = self.cache[name]
293 if fk_entry.get("reverse_fieldName"):
294 # Note this fk change so that we can update the destination model after we save.
Zack Williams045b63d2019-01-22 16:30:57 -0700295 self.post_save_fixups.append(
296 {
297 "src_fieldName": fk_entry["src_fieldName"],
298 "dest_id": id,
299 "dest_model": old_model,
300 "remove": True,
301 "reverse_fieldName": fk_entry.get("reverse_fieldName"),
302 }
303 )
Scott Bakerc4156c32017-12-08 10:58:21 -0800304 del self.cache[name]
Scott Bakere72e7612017-02-20 10:07:09 -0800305
Scott Bakerc4156c32017-12-08 10:58:21 -0800306 if model:
307 self.cache[name] = model
308 if fk_entry.get("reverse_fieldName"):
309 # Note this fk change so that we can update the destination model after we save.
Zack Williams045b63d2019-01-22 16:30:57 -0700310 self.post_save_fixups.append(
311 {
312 "src_fieldName": fk_entry["src_fieldName"],
313 "dest_id": id,
314 "dest_model": model,
315 "remove": False,
316 "reverse_fieldName": fk_entry.get("reverse_fieldName"),
317 }
318 )
Scott Bakerc4156c32017-12-08 10:58:21 -0800319 elif name in self.cache:
320 del self.cache[name]
Scott Bakere72e7612017-02-20 10:07:09 -0800321
Scott Bakerc4156c32017-12-08 10:58:21 -0800322 def do_post_save_fixups(self):
323 # Perform post-save foreign key fixups.
324 # Fixup the models that we've set a foreign key to so that their in-memory representation has the correct
325 # reverse foreign key back to us. We can only do this after a save, because self.id isn't known until
326 # after save.
327 # See unit test test_foreign_key_set_without_invalidate
328 for fixup in self.post_save_fixups:
329 model = fixup["dest_model"]
330 reverse_fieldName_ids = fixup["reverse_fieldName"] + "_ids"
331 if not hasattr(model, reverse_fieldName_ids):
332 continue
333 if fixup["remove"]:
334 reverse_ids = getattr(model, reverse_fieldName_ids)
335 if self.id in reverse_ids:
336 reverse_ids.remove(self.id)
337 else:
338 reverse_ids = getattr(model, reverse_fieldName_ids)
339 if self.id not in reverse_ids:
340 reverse_ids.append(self.id)
341 model.invalidate_cache(fixup["reverse_fieldName"])
342 self.post_save_fixups = []
Scott Bakere72e7612017-02-20 10:07:09 -0800343
Scott Baker96b995a2017-02-15 16:21:12 -0800344 def __getattr__(self, name, *args, **kwargs):
345 # note: getattr is only called for attributes that do not exist in
346 # self.__dict__
347
Scott Baker186372f2017-02-23 13:49:36 -0800348 # pk is a synonym for id
Zack Williams045b63d2019-01-22 16:30:57 -0700349 if name == "pk":
Scott Baker186372f2017-02-23 13:49:36 -0800350 name = "id"
351
Scott Baker96b995a2017-02-15 16:21:12 -0800352 if name in self._fkmap.keys():
353 return self.fk_resolve(name)
354
355 if name in self._reverse_fkmap.keys():
356 return self.reverse_fk_resolve(name)
357
Scott Baker37cf9e22018-08-20 14:39:33 -0700358 try:
359 # When sending a reply, XOS will leave the field unset if it is None in the data model. If
360 # HasField(<fieldname>)==False for an existing object, then the caller can infer that field was set to
361 # None.
362 if (not self.is_new) and (not self._wrapped_class.HasField(name)):
363 return None
364 except ValueError:
365 # ValueError is thrown if the field does not exist. We will handle that case in the getattr() below.
366 pass
367
Scott Baker96b995a2017-02-15 16:21:12 -0800368 return getattr(self._wrapped_class, name, *args, **kwargs)
369
370 def __setattr__(self, name, value):
Scott Bakere72e7612017-02-20 10:07:09 -0800371 if name in self._fkmap.keys():
372 self.fk_set(name, value)
373 elif name in self.__dict__:
Zack Williams045b63d2019-01-22 16:30:57 -0700374 super(ORMWrapper, self).__setattr__(name, value)
Scott Baker37cf9e22018-08-20 14:39:33 -0700375 elif value is None:
376 # When handling requests, XOS interprets gRPC HasField(<fieldname>)==False to indicate that the caller
377 # has not set the field and wants it to continue to use its existing (or default) value. That leaves us
378 # with no easy way to support setting a field to None.
379 raise ValueError("Setting a non-foreignkey field to None is not supported")
Scott Baker96b995a2017-02-15 16:21:12 -0800380 else:
381 setattr(self._wrapped_class, name, value)
382
383 def __repr__(self):
Scott Bakerd1940972017-05-01 15:45:32 -0700384 class_name = self._wrapped_class.__class__.__name__
385 id = getattr(self._wrapped_class, "id", "noid")
386 name = getattr(self._wrapped_class, "name", None)
387 if name:
388 return "<%s: %s>" % (class_name, name)
389 else:
390 return "<%s: id-%s>" % (class_name, id)
391
392 def __str__(self):
393 class_name = self._wrapped_class.__class__.__name__
394 id = getattr(self._wrapped_class, "id", "noid")
395 name = getattr(self._wrapped_class, "name", None)
396 if name:
397 return name
398 else:
399 return "%s-%s" % (class_name, id)
400
401 def dumpstr(self):
Scott Baker96b995a2017-02-15 16:21:12 -0800402 return self._wrapped_class.__repr__()
403
Scott Bakerd1940972017-05-01 15:45:32 -0700404 def dump(self):
Zack Williams045b63d2019-01-22 16:30:57 -0700405 print(self.dumpstr())
Scott Bakerd1940972017-05-01 15:45:32 -0700406
Scott Bakere72e7612017-02-20 10:07:09 -0800407 def invalidate_cache(self, name=None):
408 if name:
409 if name in self.cache:
410 del self.cache[name]
411 if name in self.reverse_cache:
412 del self.reverse_cache[name]
Scott Bakere72e7612017-02-20 10:07:09 -0800413 else:
414 self.cache.clear()
415 self.reverse_cache.clear()
Scott Bakere72e7612017-02-20 10:07:09 -0800416
Zack Williams045b63d2019-01-22 16:30:57 -0700417 def save(
418 self,
419 update_fields=None,
420 always_update_timestamp=False,
421 is_sync_save=False,
422 is_policy_save=False,
423 ):
Andy Bavier04ee1912019-01-30 14:17:16 -0700424 classname = self._wrapped_class.__class__.__name__
Scott Baker96b995a2017-02-15 16:21:12 -0800425 if self.is_new:
Andy Bavier04ee1912019-01-30 14:17:16 -0700426 log.debug("save(): is new", classname=classname, syncstep=get_synchronizer_function())
Zack Williams045b63d2019-01-22 16:30:57 -0700427 new_class = self.stub.invoke(
Andy Bavier04ee1912019-01-30 14:17:16 -0700428 "Create%s" % classname, self._wrapped_class
Zack Williams045b63d2019-01-22 16:30:57 -0700429 )
430 self._wrapped_class = new_class
431 self.is_new = False
Scott Baker96b995a2017-02-15 16:21:12 -0800432 else:
Andy Bavier04ee1912019-01-30 14:17:16 -0700433 if self.has_changed:
434 log.debug("save(): updated", classname=classname, changed_fields=self.changed_fields, syncstep=get_synchronizer_function())
435 else:
436 log.debug("save(): no changes", classname=classname, syncstep=get_synchronizer_function())
Zack Williams045b63d2019-01-22 16:30:57 -0700437 metadata = []
438 if update_fields:
439 metadata.append(("update_fields", ",".join(update_fields)))
440 if always_update_timestamp:
441 metadata.append(("always_update_timestamp", "1"))
442 if is_policy_save:
443 metadata.append(("is_policy_save", "1"))
444 if is_sync_save:
445 metadata.append(("is_sync_save", "1"))
446 self.stub.invoke(
Andy Bavier04ee1912019-01-30 14:17:16 -0700447 "Update%s" % classname,
Zack Williams045b63d2019-01-22 16:30:57 -0700448 self._wrapped_class,
449 metadata=metadata,
450 )
Scott Bakerc4156c32017-12-08 10:58:21 -0800451 self.do_post_save_fixups()
Scott Baker96b995a2017-02-15 16:21:12 -0800452
Scott Baker5b7fba02018-10-17 08:46:46 -0700453 # Now that object has saved, reset our initial state for diff calculation
454 self.recompute_initial()
455
Scott Baker96b995a2017-02-15 16:21:12 -0800456 def delete(self):
457 id = self.stub.make_ID(id=self._wrapped_class.id)
458 self.stub.invoke("Delete%s" % self._wrapped_class.__class__.__name__, id)
459
Scott Baker22796cc2017-02-23 16:53:34 -0800460 def tologdict(self):
461 try:
Zack Williams045b63d2019-01-22 16:30:57 -0700462 d = {"model_name": self._wrapped_class.__class__.__name__, "pk": self.pk}
463 except BaseException:
Scott Baker22796cc2017-02-23 16:53:34 -0800464 d = {}
465
466 return d
467
Scott Bakerbb81e152017-03-02 15:28:36 -0800468 @property
Scott Bakerff104cc2017-08-14 15:24:41 -0700469 def leaf_model(self):
470 # Easy case - this model is already the leaf
471 if self.leaf_model_name == self._wrapped_class.__class__.__name__:
472 return self
473
474 # This model is not the leaf, so use the stub to fetch the leaf model
475 return getattr(self.stub, self.leaf_model_name).objects.get(id=self.id)
476
477 @property
Scott Bakerd2543ed2017-03-07 21:46:48 -0800478 def model_name(self):
479 return self._wrapped_class.__class__.__name__
480
481 @property
Scott Bakerbb81e152017-03-02 15:28:36 -0800482 def ansible_tag(self):
483 return "%s_%s" % (self._wrapped_class.__class__.__name__, self.id)
484
Zack Williams045b63d2019-01-22 16:30:57 -0700485
Scott Bakerb05393b2017-03-01 14:59:55 -0800486class ORMQuerySet(list):
487 """ Makes lists look like django querysets """
Zack Williams045b63d2019-01-22 16:30:57 -0700488
Scott Bakerb05393b2017-03-01 14:59:55 -0800489 def first(self):
Zack Williams045b63d2019-01-22 16:30:57 -0700490 if len(self) > 0:
Scott Bakerb05393b2017-03-01 14:59:55 -0800491 return self[0]
492 else:
493 return None
494
Scott Baker8c7267d2017-03-14 19:34:13 -0700495 def exists(self):
Zack Williams045b63d2019-01-22 16:30:57 -0700496 return len(self) > 0
497
Scott Baker8c7267d2017-03-14 19:34:13 -0700498
Scott Baker96b995a2017-02-15 16:21:12 -0800499class ORMLocalObjectManager(object):
500 """ Manages a local list of objects """
501
Scott Bakerc59f1bc2017-12-04 16:55:05 -0800502 def __init__(self, stub, modelName, idList, writeable):
Scott Baker96b995a2017-02-15 16:21:12 -0800503 self._stub = stub
504 self._modelName = modelName
505 self._idList = idList
Scott Bakerc59f1bc2017-12-04 16:55:05 -0800506 self._writeable = writeable
Scott Baker96b995a2017-02-15 16:21:12 -0800507 self._cache = None
508
509 def resolve_queryset(self):
510 if self._cache is not None:
511 return self._cache
512
513 models = []
514 for id in self._idList:
Zack Williams045b63d2019-01-22 16:30:57 -0700515 models.append(
516 self._stub.invoke("Get%s" % self._modelName, self._stub.make_ID(id=id))
517 )
Scott Baker96b995a2017-02-15 16:21:12 -0800518
519 self._cache = models
520
521 return models
522
523 def all(self):
524 models = self.resolve_queryset()
Zack Williams045b63d2019-01-22 16:30:57 -0700525 return [make_ORMWrapper(x, self._stub) for x in models]
Scott Baker96b995a2017-02-15 16:21:12 -0800526
Scott Baker8c7267d2017-03-14 19:34:13 -0700527 def exists(self):
Zack Williams045b63d2019-01-22 16:30:57 -0700528 return len(self._idList) > 0
Scott Baker8c7267d2017-03-14 19:34:13 -0700529
Scott Bakera1eae7a2017-06-06 09:20:15 -0700530 def count(self):
531 return len(self._idList)
532
Scott Baker8c7267d2017-03-14 19:34:13 -0700533 def first(self):
534 if self._idList:
Zack Williams045b63d2019-01-22 16:30:57 -0700535 model = make_ORMWrapper(
536 self._stub.invoke(
537 "Get%s" % self._modelName, self._stub.make_ID(id=self._idList[0])
538 ),
539 self._stub,
540 )
Scott Baker8c7267d2017-03-14 19:34:13 -0700541 return model
542 else:
543 return None
544
Scott Bakerc59f1bc2017-12-04 16:55:05 -0800545 def add(self, model):
546 if not self._writeable:
547 raise Exception("Only ManyToMany lists are writeable")
548
549 if isinstance(model, int):
550 id = model
551 else:
552 if not model.id:
553 raise Exception("Model %s has no id" % model)
554 id = model.id
555
556 self._idList.append(id)
557
558 def remove(self, model):
559 if not self._writeable:
560 raise Exception("Only ManyToMany lists are writeable")
561
562 if isinstance(model, int):
563 id = model
564 else:
565 if not model.id:
566 raise Exception("Model %s has no id" % model)
567 id = model.id
568
569 self._idList.remove(id)
570
Zack Williams045b63d2019-01-22 16:30:57 -0700571
Scott Baker96b995a2017-02-15 16:21:12 -0800572class ORMObjectManager(object):
573 """ Manages a remote list of objects """
574
Scott Bakerac2f2b52017-02-21 14:53:23 -0800575 # constants better agree with common.proto
Scott Bakerea1f4d02018-12-17 10:21:50 -0800576 DEFAULT = 0
577 ALL = 1
Scott Bakerbae9d842017-03-21 10:44:10 -0700578 SYNCHRONIZER_DIRTY_OBJECTS = 2
579 SYNCHRONIZER_DELETED_OBJECTS = 3
580 SYNCHRONIZER_DIRTY_POLICIES = 4
581 SYNCHRONIZER_DELETED_POLICIES = 5
Scott Bakerac2f2b52017-02-21 14:53:23 -0800582
Scott Bakerea1f4d02018-12-17 10:21:50 -0800583 def __init__(self, stub, modelName, packageName, kind=0):
Scott Baker96b995a2017-02-15 16:21:12 -0800584 self._stub = stub
585 self._modelName = modelName
586 self._packageName = packageName
Scott Bakerea1f4d02018-12-17 10:21:50 -0800587 self._kind = kind
Scott Baker96b995a2017-02-15 16:21:12 -0800588
589 def wrap_single(self, obj):
Scott Baker22796cc2017-02-23 16:53:34 -0800590 return make_ORMWrapper(obj, self._stub)
Scott Baker96b995a2017-02-15 16:21:12 -0800591
592 def wrap_list(self, obj):
Zack Williams045b63d2019-01-22 16:30:57 -0700593 result = []
Scott Baker96b995a2017-02-15 16:21:12 -0800594 for item in obj.items:
Scott Baker22796cc2017-02-23 16:53:34 -0800595 result.append(make_ORMWrapper(item, self._stub))
Scott Bakerb05393b2017-03-01 14:59:55 -0800596 return ORMQuerySet(result)
Scott Baker96b995a2017-02-15 16:21:12 -0800597
598 def all(self):
Zack Williams045b63d2019-01-22 16:30:57 -0700599 if self._kind == self.DEFAULT:
600 return self.wrap_list(
601 self._stub.invoke("List%s" % self._modelName, self._stub.make_empty())
602 )
Scott Bakerea1f4d02018-12-17 10:21:50 -0800603 else:
604 return self.filter()
Scott Baker96b995a2017-02-15 16:21:12 -0800605
Scott Baker22796cc2017-02-23 16:53:34 -0800606 def first(self):
Scott Bakerea1f4d02018-12-17 10:21:50 -0800607 objs = self.all()
Scott Baker22796cc2017-02-23 16:53:34 -0800608 if not objs:
609 return None
610 return objs[0]
611
Scott Bakerac2f2b52017-02-21 14:53:23 -0800612 def filter(self, **kwargs):
613 q = self._stub.make_Query()
Scott Bakerea1f4d02018-12-17 10:21:50 -0800614 q.kind = self._kind
Scott Bakerac2f2b52017-02-21 14:53:23 -0800615
616 for (name, val) in kwargs.items():
617 el = q.elements.add()
618
619 if name.endswith("__gt"):
620 name = name[:-4]
621 el.operator = el.GREATER_THAN
622 elif name.endswith("__gte"):
623 name = name[:-5]
624 el.operator = el.GREATER_THAN_OR_EQUAL
625 elif name.endswith("__lt"):
626 name = name[:-4]
627 el.operator = el.LESS_THAN
628 elif name.endswith("__lte"):
629 name = name[:-5]
630 el.operator = el.LESS_THAN_OR_EQUAL
Scott Bakere1607b82018-09-20 14:10:59 -0700631 elif name.endswith("__iexact"):
632 name = name[:-8]
633 el.operator = el.IEXACT
Scott Bakerac2f2b52017-02-21 14:53:23 -0800634 else:
635 el.operator = el.EQUAL
636
637 el.name = name
638 if isinstance(val, int):
639 el.iValue = val
640 else:
641 el.sValue = val
642
643 return self.wrap_list(self._stub.invoke("Filter%s" % self._modelName, q))
644
645 def filter_special(self, kind):
646 q = self._stub.make_Query()
647 q.kind = kind
648 return self.wrap_list(self._stub.invoke("Filter%s" % self._modelName, q))
649
Scott Baker22796cc2017-02-23 16:53:34 -0800650 def get(self, **kwargs):
651 if kwargs.keys() == ["id"]:
652 # the fast and easy case, look it up by id
Zack Williams045b63d2019-01-22 16:30:57 -0700653 return self.wrap_single(
654 self._stub.invoke(
655 "Get%s" % self._modelName, self._stub.make_ID(id=kwargs["id"])
656 )
657 )
Scott Baker22796cc2017-02-23 16:53:34 -0800658 else:
659 # the slightly more difficult case, filter and return the first item
660 objs = self.filter(**kwargs)
661 return objs[0]
Scott Baker96b995a2017-02-15 16:21:12 -0800662
663 def new(self, **kwargs):
Zack Williams045b63d2019-01-22 16:30:57 -0700664 if self._kind != ORMObjectManager.DEFAULT:
665 raise Exception(
666 "Creating objects is only supported by the DEFAULT object manager"
667 )
Scott Bakerea1f4d02018-12-17 10:21:50 -0800668
Scott Bakeraa556b02017-03-07 16:07:34 -0800669 cls = self._stub.all_grpc_classes[self._modelName]
Scott Bakerfe42a6f2017-03-18 09:11:31 -0700670 o = make_ORMWrapper(cls(), self._stub, is_new=True)
Zack Williams045b63d2019-01-22 16:30:57 -0700671 for (k, v) in kwargs.items():
Scott Bakerfe42a6f2017-03-18 09:11:31 -0700672 setattr(o, k, v)
Scott Baker5b7fba02018-10-17 08:46:46 -0700673 o.recompute_initial()
Scott Bakerfe42a6f2017-03-18 09:11:31 -0700674 return o
Scott Baker96b995a2017-02-15 16:21:12 -0800675
Zack Williams045b63d2019-01-22 16:30:57 -0700676
Scott Baker96b995a2017-02-15 16:21:12 -0800677class ORMModelClass(object):
678 def __init__(self, stub, model_name, package_name):
Scott Baker22796cc2017-02-23 16:53:34 -0800679 self.model_name = model_name
Scott Bakeraa556b02017-03-07 16:07:34 -0800680 self._stub = stub
Scott Baker96b995a2017-02-15 16:21:12 -0800681 self.objects = ORMObjectManager(stub, model_name, package_name)
Zack Williams045b63d2019-01-22 16:30:57 -0700682 self.deleted_objects = ORMObjectManager(
683 stub,
684 model_name,
685 package_name,
686 ORMObjectManager.SYNCHRONIZER_DELETED_OBJECTS,
687 )
Scott Baker96b995a2017-02-15 16:21:12 -0800688
Scott Bakerbb81e152017-03-02 15:28:36 -0800689 @property
Scott Baker22796cc2017-02-23 16:53:34 -0800690 def __name__(self):
691 return self.model_name
692
Scott Bakeraa556b02017-03-07 16:07:34 -0800693 @property
694 def content_type_id(self):
695 return self._stub.reverse_content_type_map[self.model_name]
696
Scott Baker8a6d91f2017-03-22 11:23:11 -0700697 def __call__(self, *args, **kwargs):
698 return self.objects.new(*args, **kwargs)
699
Zack Williams045b63d2019-01-22 16:30:57 -0700700
Scott Baker96b995a2017-02-15 16:21:12 -0800701class ORMStub(object):
Zack Williams045b63d2019-01-22 16:30:57 -0700702 def __init__(
703 self,
704 stub,
705 protos,
706 package_name,
707 invoker=None,
708 caller_kind="grpcapi",
709 empty=None,
710 enable_backoff=True,
711 restart_on_disconnect=False,
712 ):
Scott Baker96b995a2017-02-15 16:21:12 -0800713 self.grpc_stub = stub
Scott Bakerb96ba432018-02-26 09:53:48 -0800714 self.protos = protos
715 self.common_protos = protos.common__pb2
Scott Baker96b995a2017-02-15 16:21:12 -0800716 self.all_model_names = []
Scott Bakeraa556b02017-03-07 16:07:34 -0800717 self.all_grpc_classes = {}
718 self.content_type_map = {}
719 self.reverse_content_type_map = {}
Scott Bakeref8d85d2017-02-21 16:44:28 -0800720 self.invoker = invoker
Scott Baker22796cc2017-02-23 16:53:34 -0800721 self.caller_kind = caller_kind
Scott Baker500f8c72017-05-19 09:41:50 -0700722 self.enable_backoff = enable_backoff
Scott Bakerb06e3e02017-12-12 11:05:53 -0800723 self.restart_on_disconnect = restart_on_disconnect
Scott Baker96b995a2017-02-15 16:21:12 -0800724
Scott Bakerf0ee0dc2017-05-15 10:10:05 -0700725 if not empty:
Scott Bakerb96ba432018-02-26 09:53:48 -0800726 empty = self.protos.google_dot_protobuf_dot_empty__pb2.Empty
Scott Bakerf0ee0dc2017-05-15 10:10:05 -0700727 self._empty = empty
728
Scott Baker96b995a2017-02-15 16:21:12 -0800729 for name in dir(stub):
Zack Williams045b63d2019-01-22 16:30:57 -0700730 if name.startswith("Get"):
731 model_name = name[3:]
732 setattr(self, model_name, ORMModelClass(self, model_name, package_name))
Scott Baker96b995a2017-02-15 16:21:12 -0800733
Zack Williams045b63d2019-01-22 16:30:57 -0700734 self.all_model_names.append(model_name)
Scott Baker96b995a2017-02-15 16:21:12 -0800735
Zack Williams045b63d2019-01-22 16:30:57 -0700736 grpc_class = getattr(self.protos, model_name)
737 self.all_grpc_classes[model_name] = grpc_class
Scott Bakeraa556b02017-03-07 16:07:34 -0800738
Zack Williams045b63d2019-01-22 16:30:57 -0700739 ct = grpc_class.DESCRIPTOR.GetOptions().Extensions._FindExtensionByName(
740 "xos.contentTypeId"
741 )
742 if ct:
743 ct = grpc_class.DESCRIPTOR.GetOptions().Extensions[ct]
744 if ct:
745 self.content_type_map[ct] = model_name
746 self.reverse_content_type_map[model_name] = ct
Scott Bakeraa556b02017-03-07 16:07:34 -0800747
748 def genericForeignKeyResolve(self, content_type_id, id):
Scott Bakerd0f1dc12018-04-23 12:05:32 -0700749 if content_type_id.endswith("_decl"):
750 content_type_id = content_type_id[:-5]
751
752 if content_type_id not in self.content_type_map:
Zack Williams045b63d2019-01-22 16:30:57 -0700753 raise ORMGenericContentNotFoundException(
754 "Content_type %s not found in self.content_type_map" % content_type_id
755 )
Scott Bakerd0f1dc12018-04-23 12:05:32 -0700756
Scott Bakeraa556b02017-03-07 16:07:34 -0800757 model_name = self.content_type_map[content_type_id]
Scott Bakerd0f1dc12018-04-23 12:05:32 -0700758
Scott Bakeraa556b02017-03-07 16:07:34 -0800759 model = getattr(self, model_name)
Scott Bakerd0f1dc12018-04-23 12:05:32 -0700760 objs = model.objects.filter(id=id)
761 if not objs:
Zack Williams045b63d2019-01-22 16:30:57 -0700762 raise ORMGenericObjectNotFoundException(
763 "Object %s of model %s was not found" % (id, model_name)
764 )
Scott Bakerd0f1dc12018-04-23 12:05:32 -0700765
Scott Bakeraa556b02017-03-07 16:07:34 -0800766 return model.objects.get(id=id)
767
Scott Baker22796cc2017-02-23 16:53:34 -0800768 def add_default_metadata(self, metadata):
Zack Williams045b63d2019-01-22 16:30:57 -0700769 default_metadata = [("caller_kind", self.caller_kind)]
Scott Baker22796cc2017-02-23 16:53:34 -0800770
Scott Bakerd8246712018-07-12 18:08:31 -0700771 # introspect to see if we're running from a synchronizer thread
772 if getattr(threading.current_thread(), "is_sync_thread", False):
Zack Williams045b63d2019-01-22 16:30:57 -0700773 default_metadata.append(("is_sync_save", "1"))
Scott Bakerd8246712018-07-12 18:08:31 -0700774
775 # introspect to see if we're running from a model_policy thread
776 if getattr(threading.current_thread(), "is_policy_thread", False):
Zack Williams045b63d2019-01-22 16:30:57 -0700777 default_metadata.append(("is_policy_save", "1"))
Scott Bakerd8246712018-07-12 18:08:31 -0700778
Scott Baker22796cc2017-02-23 16:53:34 -0800779 # build up a list of metadata keys we already have
Zack Williams045b63d2019-01-22 16:30:57 -0700780 md_keys = [x[0] for x in metadata]
Scott Baker22796cc2017-02-23 16:53:34 -0800781
782 # add any defaults that we don't already have
783 for md in default_metadata:
784 if md[0] not in md_keys:
Zack Williams045b63d2019-01-22 16:30:57 -0700785 metadata.append((md[0], md[1]))
Scott Baker22796cc2017-02-23 16:53:34 -0800786
Scott Baker57c74822017-02-23 11:13:04 -0800787 def invoke(self, name, request, metadata=[]):
Scott Baker22796cc2017-02-23 16:53:34 -0800788 self.add_default_metadata(metadata)
789
Scott Bakeref8d85d2017-02-21 16:44:28 -0800790 if self.invoker:
791 # Hook in place to call Chameleon's invoke method, as soon as we
792 # have rewritten the synchronizer to use reactor.
Zack Williams045b63d2019-01-22 16:30:57 -0700793 return self.invoker.invoke(
794 self.grpc_stub.__class__, name, request, metadata={}
795 ).result[0]
Scott Baker500f8c72017-05-19 09:41:50 -0700796 elif self.enable_backoff:
Scott Bakeref8d85d2017-02-21 16:44:28 -0800797 # Our own retry mechanism. This works fine if there is a temporary
798 # failure in connectivity, but does not re-download gRPC schema.
Scott Baker500f8c72017-05-19 09:41:50 -0700799 import grpc
Zack Williams045b63d2019-01-22 16:30:57 -0700800
Scott Bakerb06e3e02017-12-12 11:05:53 -0800801 backoff = [0.5, 1, 2, 4, 8]
Scott Bakeref8d85d2017-02-21 16:44:28 -0800802 while True:
Scott Bakeref8d85d2017-02-21 16:44:28 -0800803 try:
804 method = getattr(self.grpc_stub, name)
Scott Baker57c74822017-02-23 11:13:04 -0800805 return method(request, metadata=metadata)
Zack Williams045b63d2019-01-22 16:30:57 -0700806 except grpc._channel._Rendezvous as e:
Scott Bakeref8d85d2017-02-21 16:44:28 -0800807 code = e.code()
808 if code == grpc.StatusCode.UNAVAILABLE:
Scott Bakerb06e3e02017-12-12 11:05:53 -0800809 if self.restart_on_disconnect:
810 # This is a blunt technique... We lost connectivity to the core, and we don't know that
811 # the core is still serving up the same models it was when we established connectivity,
812 # so restart the synchronizer.
813 # TODO: Hash check on the core models to tell if something changed would be better.
Zack Williams045b63d2019-01-22 16:30:57 -0700814 os.execv(sys.executable, ["python"] + sys.argv)
Scott Bakeref8d85d2017-02-21 16:44:28 -0800815 if not backoff:
816 raise Exception("No more retries on %s" % name)
817 time.sleep(backoff.pop(0))
818 else:
819 raise
Zack Williams045b63d2019-01-22 16:30:57 -0700820 except BaseException:
Scott Bakeref8d85d2017-02-21 16:44:28 -0800821 raise
Scott Baker500f8c72017-05-19 09:41:50 -0700822 else:
823 method = getattr(self.grpc_stub, name)
824 return method(request, metadata=metadata)
825
Scott Baker96b995a2017-02-15 16:21:12 -0800826 def make_ID(self, id):
Scott Bakerb96ba432018-02-26 09:53:48 -0800827 return getattr(self.common_protos, "ID")(id=id)
Scott Bakerf0ee0dc2017-05-15 10:10:05 -0700828
829 def make_empty(self):
830 return self._empty()
Scott Baker96b995a2017-02-15 16:21:12 -0800831
Scott Bakerac2f2b52017-02-21 14:53:23 -0800832 def make_Query(self):
Scott Bakerb96ba432018-02-26 09:53:48 -0800833 return getattr(self.common_protos, "Query")()
Scott Bakerac2f2b52017-02-21 14:53:23 -0800834
Scott Bakerf6145a22017-03-29 14:50:25 -0700835 def listObjects(self):
836 return self.all_model_names
837
Zack Williams045b63d2019-01-22 16:30:57 -0700838
Scott Baker22796cc2017-02-23 16:53:34 -0800839def register_convenience_wrapper(class_name, wrapper):
840 global convenience_wrappers
Scott Baker96b995a2017-02-15 16:21:12 -0800841
Scott Baker22796cc2017-02-23 16:53:34 -0800842 convenience_wrappers[class_name] = wrapper
843
Zack Williams045b63d2019-01-22 16:30:57 -0700844
Scott Baker22796cc2017-02-23 16:53:34 -0800845def make_ORMWrapper(wrapped_class, *args, **kwargs):
Scott Baker2f314d52018-08-24 08:31:19 -0700846 cls = None
847
848 if (not cls) and wrapped_class.__class__.__name__ in convenience_wrappers:
Scott Baker22796cc2017-02-23 16:53:34 -0800849 cls = convenience_wrappers[wrapped_class.__class__.__name__]
Scott Baker2f314d52018-08-24 08:31:19 -0700850
Zack Williams045b63d2019-01-22 16:30:57 -0700851 if not cls:
Scott Baker2f314d52018-08-24 08:31:19 -0700852 # Search the list of class names for this model to see if we have any applicable wrappers. The list is always
853 # sorted from most specific to least specific, so the first one we find will automatically be the most relevant
854 # one. If we don't find any, then default to ORMWrapper
855
856 # Note: Only works on objects that have been fetched from the server, not objects that are created on the
857 # client. This is because wrapped_class.class_names is filled in by the server.
858
859 # TODO(smbaker): Ought to be able to make this work with newly created objects after they are saved.
860
861 for name in wrapped_class.class_names.split(","):
862 if name in convenience_wrappers:
863 cls = convenience_wrappers[name]
864
Zack Williams045b63d2019-01-22 16:30:57 -0700865 if not cls:
Scott Baker22796cc2017-02-23 16:53:34 -0800866 cls = ORMWrapper
867
868 return cls(wrapped_class, *args, **kwargs)
869
Zack Williams045b63d2019-01-22 16:30:57 -0700870
Matteo Scandolo10a2f3c2018-04-20 16:59:38 +0200871def import_convenience_methods():
872
873 log.info("Loading convenience methods")
874
875 cwd = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
876 api_convenience_dir = os.path.join(cwd, "convenience")
877 for file in os.listdir(api_convenience_dir):
Zack Williams045b63d2019-01-22 16:30:57 -0700878 if file.endswith(".py") and "test" not in file:
Matteo Scandolo10a2f3c2018-04-20 16:59:38 +0200879 pathname = os.path.join(api_convenience_dir, file)
880 try:
881 log.debug("Loading: %s" % file)
882 imp.load_source(file[:-3], pathname)
Zack Williams045b63d2019-01-22 16:30:57 -0700883 except Exception:
884 log.exception(
885 "Cannot import api convenience method for: %s, %s"
886 % (file[:-3], pathname)
887 )