blob: 398d19996f12aa82fa838b5f41ba625180256234 [file] [log] [blame]
Matteo Scandolod2044a42017-08-07 16:08:28 -07001# Copyright 2017-present Open Networking Foundation
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
Zack Williams045b63d2019-01-22 16:30:57 -070015from __future__ import print_function
16import os
17import sys
18import threading
19import time
20import imp
21from xosconfig import Config
22from multistructlog import create_logger
Matteo Scandolod2044a42017-08-07 16:08:28 -070023
Scott Baker96b995a2017-02-15 16:21:12 -080024"""
25Django-like ORM layer for gRPC
26
27Usage:
28 api = ORMStub(stub)
29
30 api.Slices.all() ... list all slices
31
32 someSlice = api.Slices.get(id=1) ... get slice #1
33
34 someSlice.site ... automatically resolves site_id into a site object
35 someSlice.instances ... automatically resolves instances_ids into instance objects
36 someSlice.save() ... saves the slice object
37"""
38
39"""
40import grpc_client, orm
41c=grpc_client.SecureClient("xos-core.cord.lab", username="padmin@vicci.org", password="letmein")
Scott Bakera1eae7a2017-06-06 09:20:15 -070042u=c.xos_orm.User.objects.get(id=1)
43"""
Scott Baker96b995a2017-02-15 16:21:12 -080044
Matteo Scandoloe3d2f262018-06-05 17:45:39 -070045
Zack Williams045b63d2019-01-22 16:30:57 -070046log = create_logger(Config().get("logging"))
Scott Baker96b995a2017-02-15 16:21:12 -080047
Scott Baker22796cc2017-02-23 16:53:34 -080048convenience_wrappers = {}
49
Zack Williams045b63d2019-01-22 16:30:57 -070050
Scott Bakerd0f1dc12018-04-23 12:05:32 -070051class ORMGenericContentNotFoundException(Exception):
52 pass
53
Zack Williams045b63d2019-01-22 16:30:57 -070054
Scott Bakerd0f1dc12018-04-23 12:05:32 -070055class ORMGenericObjectNotFoundException(Exception):
56 pass
57
Zack Williams045b63d2019-01-22 16:30:57 -070058
Scott Baker96b995a2017-02-15 16:21:12 -080059class ORMWrapper(object):
60 """ Wraps a protobuf object to provide ORM features """
61
62 def __init__(self, wrapped_class, stub, is_new=False):
63 super(ORMWrapper, self).__setattr__("_wrapped_class", wrapped_class)
64 super(ORMWrapper, self).__setattr__("stub", stub)
65 super(ORMWrapper, self).__setattr__("cache", {})
66 super(ORMWrapper, self).__setattr__("reverse_cache", {})
Sapan Bhatia71f57682017-08-23 20:09:08 -040067 super(ORMWrapper, self).__setattr__("synchronizer_step", None)
Sapan Bhatia2b307f72017-11-02 11:39:17 -040068 super(ORMWrapper, self).__setattr__("dependent", None)
Scott Baker96b995a2017-02-15 16:21:12 -080069 super(ORMWrapper, self).__setattr__("is_new", is_new)
Scott Bakerc4156c32017-12-08 10:58:21 -080070 super(ORMWrapper, self).__setattr__("post_save_fixups", [])
Zack Williams045b63d2019-01-22 16:30:57 -070071 fkmap = self.gen_fkmap()
Scott Baker96b995a2017-02-15 16:21:12 -080072 super(ORMWrapper, self).__setattr__("_fkmap", fkmap)
Zack Williams045b63d2019-01-22 16:30:57 -070073 reverse_fkmap = self.gen_reverse_fkmap()
Scott Baker96b995a2017-02-15 16:21:12 -080074 super(ORMWrapper, self).__setattr__("_reverse_fkmap", reverse_fkmap)
Scott Baker5b7fba02018-10-17 08:46:46 -070075 super(ORMWrapper, self).__setattr__("_initial", self._dict)
76
Zack Williams045b63d2019-01-22 16:30:57 -070077 def fields_differ(self, f1, f2):
78 return f1 != f2
Scott Baker5b7fba02018-10-17 08:46:46 -070079
80 @property
81 def _dict(self):
82 """ Return a dictionary of {fieldname: fieldvalue} for the object.
83
84 This differs for the xos-core implementation of XOSBase. For new object, XOSBase will include field names
85 that are set to default values. ORM ignores fields that are set to default values.
86 """
Zack Williams045b63d2019-01-22 16:30:57 -070087 d = {}
Scott Baker5b7fba02018-10-17 08:46:46 -070088 for (fieldDesc, val) in self._wrapped_class.ListFields():
89 name = fieldDesc.name
90 d[name] = val
91 return d
92
93 @property
94 def diff(self):
95 d1 = self._initial
96 d2 = self._dict
97 all_field_names = self._wrapped_class.DESCRIPTOR.fields_by_name.keys()
Zack Williams045b63d2019-01-22 16:30:57 -070098 diffs = []
Scott Baker5b7fba02018-10-17 08:46:46 -070099 for k in all_field_names:
Zack Williams045b63d2019-01-22 16:30:57 -0700100 if d1.get(k, None) != d2.get(k, None):
101 diffs.append((k, (d1.get(k, None), d2.get(k, None))))
Scott Baker5b7fba02018-10-17 08:46:46 -0700102
Zack Williams045b63d2019-01-22 16:30:57 -0700103 # diffs = [(k, (v, d2[k])) for k, v in d1.items() if self.fields_differ(v,d2[k])]
Scott Baker5b7fba02018-10-17 08:46:46 -0700104 return dict(diffs)
105
106 @property
107 def has_changed(self):
108 return bool(self.diff)
109
110 @property
111 def changed_fields(self):
112 """ Return the list of changed fields.
113
114 This differs for the xos-core implementation of XOSBase. For new object, XOSBase will include field names
115 that are set to default values.
116 """
117 if self.is_new:
118 return self._dict.keys()
119 return self.diff.keys()
120
121 def has_field_changed(self, field_name):
122 return field_name in self.diff.keys()
123
124 def get_field_diff(self, field_name):
125 return self.diff.get(field_name, None)
126
127 def recompute_initial(self):
128 self._initial = self._dict
129
130 def save_changed_fields(self, always_update_timestamp=False):
131 if self.has_changed:
132 update_fields = self.changed_fields
133 if always_update_timestamp and "updated" not in update_fields:
134 update_fields.append("updated")
Zack Williams045b63d2019-01-22 16:30:57 -0700135 self.save(
136 update_fields=sorted(update_fields),
137 always_update_timestamp=always_update_timestamp,
138 )
Scott Baker96b995a2017-02-15 16:21:12 -0800139
Scott Bakerd78f6472017-03-14 17:30:14 -0700140 def create_attr(self, name, value=None):
141 """ setattr(self, ...) will fail for attributes that don't exist in the
142 wrapped grpc class. This is by design. However, if someone really
143 does want to attach a new attribute to this class, then they can
144 call create_attr()
145 """
146 super(ORMWrapper, self).__setattr__(name, value)
147
Scott Baker03a163f2017-05-17 09:21:47 -0700148 def get_generic_foreignkeys(self):
149 """ this is a placeholder until generic foreign key support is added
150 to xproto.
151 """
152 return []
153
Scott Baker96b995a2017-02-15 16:21:12 -0800154 def gen_fkmap(self):
155 fkmap = {}
156
Scott Bakeraa556b02017-03-07 16:07:34 -0800157 all_field_names = self._wrapped_class.DESCRIPTOR.fields_by_name.keys()
158
Scott Baker96b995a2017-02-15 16:21:12 -0800159 for (name, field) in self._wrapped_class.DESCRIPTOR.fields_by_name.items():
Zack Williams045b63d2019-01-22 16:30:57 -0700160 if name.endswith("_id"):
161 foreignKey = field.GetOptions().Extensions._FindExtensionByName(
162 "xos.foreignKey"
163 )
164 fk = field.GetOptions().Extensions[foreignKey]
165 if fk and fk.modelName:
166 fkdict = {
167 "src_fieldName": name,
168 "modelName": fk.modelName,
169 "kind": "fk",
170 }
171 if fk.reverseFieldName:
172 fkdict["reverse_fieldName"] = fk.reverseFieldName
173 fkmap[name[:-3]] = fkdict
174 else:
175 # If there's a corresponding _type_id field, then see if this
176 # is a generic foreign key.
177 type_name = name[:-3] + "_type_id"
178 if type_name in all_field_names:
179 fkmap[name[:-3]] = {
180 "src_fieldName": name,
181 "ct_fieldName": type_name,
182 "kind": "generic_fk",
183 }
Scott Baker96b995a2017-02-15 16:21:12 -0800184
Scott Baker03a163f2017-05-17 09:21:47 -0700185 for gfk in self.get_generic_foreignkeys():
Zack Williams045b63d2019-01-22 16:30:57 -0700186 fkmap[gfk["name"]] = {
187 "src_fieldName": gfk["id"],
188 "ct_fieldName": gfk["content_type"],
189 "kind": "generic_fk",
190 }
Scott Baker03a163f2017-05-17 09:21:47 -0700191
Scott Baker96b995a2017-02-15 16:21:12 -0800192 return fkmap
193
194 def gen_reverse_fkmap(self):
195 reverse_fkmap = {}
196
197 for (name, field) in self._wrapped_class.DESCRIPTOR.fields_by_name.items():
Zack Williams045b63d2019-01-22 16:30:57 -0700198 if name.endswith("_ids"):
199 reverseForeignKey = field.GetOptions().Extensions._FindExtensionByName(
200 "xos.reverseForeignKey"
201 )
202 fk = field.GetOptions().Extensions[reverseForeignKey]
203 if fk and fk.modelName:
204 reverse_fkmap[name[:-4]] = {
205 "src_fieldName": name,
206 "modelName": fk.modelName,
207 "writeable": False,
208 }
209 else:
210 manyToManyForeignKey = field.GetOptions().Extensions._FindExtensionByName(
211 "xos.manyToManyForeignKey"
212 )
213 fk = field.GetOptions().Extensions[manyToManyForeignKey]
214 if fk and fk.modelName:
215 reverse_fkmap[name[:-4]] = {
216 "src_fieldName": name,
217 "modelName": fk.modelName,
218 "writeable": True,
219 }
Scott Baker96b995a2017-02-15 16:21:12 -0800220
221 return reverse_fkmap
222
223 def fk_resolve(self, name):
224 if name in self.cache:
Scott Bakerc4156c32017-12-08 10:58:21 -0800225 return self.cache[name]
Scott Baker96b995a2017-02-15 16:21:12 -0800226
227 fk_entry = self._fkmap[name]
Scott Bakeraa556b02017-03-07 16:07:34 -0800228 fk_kind = fk_entry["kind"]
229 fk_id = getattr(self, fk_entry["src_fieldName"])
230
231 if not fk_id:
232 return None
233
Zack Williams045b63d2019-01-22 16:30:57 -0700234 if fk_kind == "fk":
235 id = self.stub.make_ID(id=fk_id)
Scott Bakeraa556b02017-03-07 16:07:34 -0800236 dest_model = self.stub.invoke("Get%s" % fk_entry["modelName"], id)
237
Zack Williams045b63d2019-01-22 16:30:57 -0700238 elif fk_kind == "generic_fk":
239 dest_model = self.stub.genericForeignKeyResolve(
240 getattr(self, fk_entry["ct_fieldName"]), fk_id
241 )._wrapped_class
Scott Bakeraa556b02017-03-07 16:07:34 -0800242
243 else:
244 raise Exception("unknown fk_kind")
Scott Baker96b995a2017-02-15 16:21:12 -0800245
Scott Bakerc4156c32017-12-08 10:58:21 -0800246 dest_model = make_ORMWrapper(dest_model, self.stub)
Scott Baker96b995a2017-02-15 16:21:12 -0800247 self.cache[name] = dest_model
248
Scott Bakerc4156c32017-12-08 10:58:21 -0800249 return dest_model
Scott Baker96b995a2017-02-15 16:21:12 -0800250
251 def reverse_fk_resolve(self, name):
252 if name not in self.reverse_cache:
253 fk_entry = self._reverse_fkmap[name]
Zack Williams045b63d2019-01-22 16:30:57 -0700254 self.reverse_cache[name] = ORMLocalObjectManager(
255 self.stub,
256 fk_entry["modelName"],
257 getattr(self, fk_entry["src_fieldName"]),
258 fk_entry["writeable"],
259 )
Scott Baker96b995a2017-02-15 16:21:12 -0800260
Scott Baker7ab456b2019-01-08 14:58:13 -0800261 return self.reverse_cache[name]
Scott Baker96b995a2017-02-15 16:21:12 -0800262
Scott Bakere72e7612017-02-20 10:07:09 -0800263 def fk_set(self, name, model):
264 fk_entry = self._fkmap[name]
Scott Bakeraa556b02017-03-07 16:07:34 -0800265 fk_kind = fk_entry["kind"]
Scott Bakera1eae7a2017-06-06 09:20:15 -0700266 if model:
267 id = model.id
268 else:
269 id = 0
Scott Bakere72e7612017-02-20 10:07:09 -0800270 setattr(self._wrapped_class, fk_entry["src_fieldName"], id)
271
Zack Williams045b63d2019-01-22 16:30:57 -0700272 if fk_kind == "generic_fk":
273 setattr(
274 self._wrapped_class,
275 fk_entry["ct_fieldName"],
276 model.self_content_type_id,
277 )
Scott Bakeraa556b02017-03-07 16:07:34 -0800278
Scott Bakerc4156c32017-12-08 10:58:21 -0800279 if name in self.cache:
280 old_model = self.cache[name]
281 if fk_entry.get("reverse_fieldName"):
282 # Note this fk change so that we can update the destination model after we save.
Zack Williams045b63d2019-01-22 16:30:57 -0700283 self.post_save_fixups.append(
284 {
285 "src_fieldName": fk_entry["src_fieldName"],
286 "dest_id": id,
287 "dest_model": old_model,
288 "remove": True,
289 "reverse_fieldName": fk_entry.get("reverse_fieldName"),
290 }
291 )
Scott Bakerc4156c32017-12-08 10:58:21 -0800292 del self.cache[name]
Scott Bakere72e7612017-02-20 10:07:09 -0800293
Scott Bakerc4156c32017-12-08 10:58:21 -0800294 if model:
295 self.cache[name] = model
296 if fk_entry.get("reverse_fieldName"):
297 # Note this fk change so that we can update the destination model after we save.
Zack Williams045b63d2019-01-22 16:30:57 -0700298 self.post_save_fixups.append(
299 {
300 "src_fieldName": fk_entry["src_fieldName"],
301 "dest_id": id,
302 "dest_model": model,
303 "remove": False,
304 "reverse_fieldName": fk_entry.get("reverse_fieldName"),
305 }
306 )
Scott Bakerc4156c32017-12-08 10:58:21 -0800307 elif name in self.cache:
308 del self.cache[name]
Scott Bakere72e7612017-02-20 10:07:09 -0800309
Scott Bakerc4156c32017-12-08 10:58:21 -0800310 def do_post_save_fixups(self):
311 # Perform post-save foreign key fixups.
312 # Fixup the models that we've set a foreign key to so that their in-memory representation has the correct
313 # reverse foreign key back to us. We can only do this after a save, because self.id isn't known until
314 # after save.
315 # See unit test test_foreign_key_set_without_invalidate
316 for fixup in self.post_save_fixups:
317 model = fixup["dest_model"]
318 reverse_fieldName_ids = fixup["reverse_fieldName"] + "_ids"
319 if not hasattr(model, reverse_fieldName_ids):
320 continue
321 if fixup["remove"]:
322 reverse_ids = getattr(model, reverse_fieldName_ids)
323 if self.id in reverse_ids:
324 reverse_ids.remove(self.id)
325 else:
326 reverse_ids = getattr(model, reverse_fieldName_ids)
327 if self.id not in reverse_ids:
328 reverse_ids.append(self.id)
329 model.invalidate_cache(fixup["reverse_fieldName"])
330 self.post_save_fixups = []
Scott Bakere72e7612017-02-20 10:07:09 -0800331
Scott Baker96b995a2017-02-15 16:21:12 -0800332 def __getattr__(self, name, *args, **kwargs):
333 # note: getattr is only called for attributes that do not exist in
334 # self.__dict__
335
Scott Baker186372f2017-02-23 13:49:36 -0800336 # pk is a synonym for id
Zack Williams045b63d2019-01-22 16:30:57 -0700337 if name == "pk":
Scott Baker186372f2017-02-23 13:49:36 -0800338 name = "id"
339
Scott Baker96b995a2017-02-15 16:21:12 -0800340 if name in self._fkmap.keys():
341 return self.fk_resolve(name)
342
343 if name in self._reverse_fkmap.keys():
344 return self.reverse_fk_resolve(name)
345
Scott Baker37cf9e22018-08-20 14:39:33 -0700346 try:
347 # When sending a reply, XOS will leave the field unset if it is None in the data model. If
348 # HasField(<fieldname>)==False for an existing object, then the caller can infer that field was set to
349 # None.
350 if (not self.is_new) and (not self._wrapped_class.HasField(name)):
351 return None
352 except ValueError:
353 # ValueError is thrown if the field does not exist. We will handle that case in the getattr() below.
354 pass
355
Scott Baker96b995a2017-02-15 16:21:12 -0800356 return getattr(self._wrapped_class, name, *args, **kwargs)
357
358 def __setattr__(self, name, value):
Scott Bakere72e7612017-02-20 10:07:09 -0800359 if name in self._fkmap.keys():
360 self.fk_set(name, value)
361 elif name in self.__dict__:
Zack Williams045b63d2019-01-22 16:30:57 -0700362 super(ORMWrapper, self).__setattr__(name, value)
Scott Baker37cf9e22018-08-20 14:39:33 -0700363 elif value is None:
364 # When handling requests, XOS interprets gRPC HasField(<fieldname>)==False to indicate that the caller
365 # has not set the field and wants it to continue to use its existing (or default) value. That leaves us
366 # with no easy way to support setting a field to None.
367 raise ValueError("Setting a non-foreignkey field to None is not supported")
Scott Baker96b995a2017-02-15 16:21:12 -0800368 else:
369 setattr(self._wrapped_class, name, value)
370
371 def __repr__(self):
Scott Bakerd1940972017-05-01 15:45:32 -0700372 class_name = self._wrapped_class.__class__.__name__
373 id = getattr(self._wrapped_class, "id", "noid")
374 name = getattr(self._wrapped_class, "name", None)
375 if name:
376 return "<%s: %s>" % (class_name, name)
377 else:
378 return "<%s: id-%s>" % (class_name, id)
379
380 def __str__(self):
381 class_name = self._wrapped_class.__class__.__name__
382 id = getattr(self._wrapped_class, "id", "noid")
383 name = getattr(self._wrapped_class, "name", None)
384 if name:
385 return name
386 else:
387 return "%s-%s" % (class_name, id)
388
389 def dumpstr(self):
Scott Baker96b995a2017-02-15 16:21:12 -0800390 return self._wrapped_class.__repr__()
391
Scott Bakerd1940972017-05-01 15:45:32 -0700392 def dump(self):
Zack Williams045b63d2019-01-22 16:30:57 -0700393 print(self.dumpstr())
Scott Bakerd1940972017-05-01 15:45:32 -0700394
Scott Bakere72e7612017-02-20 10:07:09 -0800395 def invalidate_cache(self, name=None):
396 if name:
397 if name in self.cache:
398 del self.cache[name]
399 if name in self.reverse_cache:
400 del self.reverse_cache[name]
Scott Bakere72e7612017-02-20 10:07:09 -0800401 else:
402 self.cache.clear()
403 self.reverse_cache.clear()
Scott Bakere72e7612017-02-20 10:07:09 -0800404
Zack Williams045b63d2019-01-22 16:30:57 -0700405 def save(
406 self,
407 update_fields=None,
408 always_update_timestamp=False,
409 is_sync_save=False,
410 is_policy_save=False,
411 ):
Scott Baker96b995a2017-02-15 16:21:12 -0800412 if self.is_new:
Zack Williams045b63d2019-01-22 16:30:57 -0700413 new_class = self.stub.invoke(
414 "Create%s" % self._wrapped_class.__class__.__name__, self._wrapped_class
415 )
416 self._wrapped_class = new_class
417 self.is_new = False
Scott Baker96b995a2017-02-15 16:21:12 -0800418 else:
Zack Williams045b63d2019-01-22 16:30:57 -0700419 metadata = []
420 if update_fields:
421 metadata.append(("update_fields", ",".join(update_fields)))
422 if always_update_timestamp:
423 metadata.append(("always_update_timestamp", "1"))
424 if is_policy_save:
425 metadata.append(("is_policy_save", "1"))
426 if is_sync_save:
427 metadata.append(("is_sync_save", "1"))
428 self.stub.invoke(
429 "Update%s" % self._wrapped_class.__class__.__name__,
430 self._wrapped_class,
431 metadata=metadata,
432 )
Scott Bakerc4156c32017-12-08 10:58:21 -0800433 self.do_post_save_fixups()
Scott Baker96b995a2017-02-15 16:21:12 -0800434
Scott Baker5b7fba02018-10-17 08:46:46 -0700435 # Now that object has saved, reset our initial state for diff calculation
436 self.recompute_initial()
437
Scott Baker96b995a2017-02-15 16:21:12 -0800438 def delete(self):
439 id = self.stub.make_ID(id=self._wrapped_class.id)
440 self.stub.invoke("Delete%s" % self._wrapped_class.__class__.__name__, id)
441
Scott Baker22796cc2017-02-23 16:53:34 -0800442 def tologdict(self):
443 try:
Zack Williams045b63d2019-01-22 16:30:57 -0700444 d = {"model_name": self._wrapped_class.__class__.__name__, "pk": self.pk}
445 except BaseException:
Scott Baker22796cc2017-02-23 16:53:34 -0800446 d = {}
447
448 return d
449
Scott Bakerbb81e152017-03-02 15:28:36 -0800450 @property
Scott Bakerff104cc2017-08-14 15:24:41 -0700451 def leaf_model(self):
452 # Easy case - this model is already the leaf
453 if self.leaf_model_name == self._wrapped_class.__class__.__name__:
454 return self
455
456 # This model is not the leaf, so use the stub to fetch the leaf model
457 return getattr(self.stub, self.leaf_model_name).objects.get(id=self.id)
458
459 @property
Scott Bakerd2543ed2017-03-07 21:46:48 -0800460 def model_name(self):
461 return self._wrapped_class.__class__.__name__
462
463 @property
Scott Bakerbb81e152017-03-02 15:28:36 -0800464 def ansible_tag(self):
465 return "%s_%s" % (self._wrapped_class.__class__.__name__, self.id)
466
Zack Williams045b63d2019-01-22 16:30:57 -0700467
Scott Bakerb05393b2017-03-01 14:59:55 -0800468class ORMQuerySet(list):
469 """ Makes lists look like django querysets """
Zack Williams045b63d2019-01-22 16:30:57 -0700470
Scott Bakerb05393b2017-03-01 14:59:55 -0800471 def first(self):
Zack Williams045b63d2019-01-22 16:30:57 -0700472 if len(self) > 0:
Scott Bakerb05393b2017-03-01 14:59:55 -0800473 return self[0]
474 else:
475 return None
476
Scott Baker8c7267d2017-03-14 19:34:13 -0700477 def exists(self):
Zack Williams045b63d2019-01-22 16:30:57 -0700478 return len(self) > 0
479
Scott Baker8c7267d2017-03-14 19:34:13 -0700480
Scott Baker96b995a2017-02-15 16:21:12 -0800481class ORMLocalObjectManager(object):
482 """ Manages a local list of objects """
483
Scott Bakerc59f1bc2017-12-04 16:55:05 -0800484 def __init__(self, stub, modelName, idList, writeable):
Scott Baker96b995a2017-02-15 16:21:12 -0800485 self._stub = stub
486 self._modelName = modelName
487 self._idList = idList
Scott Bakerc59f1bc2017-12-04 16:55:05 -0800488 self._writeable = writeable
Scott Baker96b995a2017-02-15 16:21:12 -0800489 self._cache = None
490
491 def resolve_queryset(self):
492 if self._cache is not None:
493 return self._cache
494
495 models = []
496 for id in self._idList:
Zack Williams045b63d2019-01-22 16:30:57 -0700497 models.append(
498 self._stub.invoke("Get%s" % self._modelName, self._stub.make_ID(id=id))
499 )
Scott Baker96b995a2017-02-15 16:21:12 -0800500
501 self._cache = models
502
503 return models
504
505 def all(self):
506 models = self.resolve_queryset()
Zack Williams045b63d2019-01-22 16:30:57 -0700507 return [make_ORMWrapper(x, self._stub) for x in models]
Scott Baker96b995a2017-02-15 16:21:12 -0800508
Scott Baker8c7267d2017-03-14 19:34:13 -0700509 def exists(self):
Zack Williams045b63d2019-01-22 16:30:57 -0700510 return len(self._idList) > 0
Scott Baker8c7267d2017-03-14 19:34:13 -0700511
Scott Bakera1eae7a2017-06-06 09:20:15 -0700512 def count(self):
513 return len(self._idList)
514
Scott Baker8c7267d2017-03-14 19:34:13 -0700515 def first(self):
516 if self._idList:
Zack Williams045b63d2019-01-22 16:30:57 -0700517 model = make_ORMWrapper(
518 self._stub.invoke(
519 "Get%s" % self._modelName, self._stub.make_ID(id=self._idList[0])
520 ),
521 self._stub,
522 )
Scott Baker8c7267d2017-03-14 19:34:13 -0700523 return model
524 else:
525 return None
526
Scott Bakerc59f1bc2017-12-04 16:55:05 -0800527 def add(self, model):
528 if not self._writeable:
529 raise Exception("Only ManyToMany lists are writeable")
530
531 if isinstance(model, int):
532 id = model
533 else:
534 if not model.id:
535 raise Exception("Model %s has no id" % model)
536 id = model.id
537
538 self._idList.append(id)
539
540 def remove(self, model):
541 if not self._writeable:
542 raise Exception("Only ManyToMany lists are writeable")
543
544 if isinstance(model, int):
545 id = model
546 else:
547 if not model.id:
548 raise Exception("Model %s has no id" % model)
549 id = model.id
550
551 self._idList.remove(id)
552
Zack Williams045b63d2019-01-22 16:30:57 -0700553
Scott Baker96b995a2017-02-15 16:21:12 -0800554class ORMObjectManager(object):
555 """ Manages a remote list of objects """
556
Scott Bakerac2f2b52017-02-21 14:53:23 -0800557 # constants better agree with common.proto
Scott Bakerea1f4d02018-12-17 10:21:50 -0800558 DEFAULT = 0
559 ALL = 1
Scott Bakerbae9d842017-03-21 10:44:10 -0700560 SYNCHRONIZER_DIRTY_OBJECTS = 2
561 SYNCHRONIZER_DELETED_OBJECTS = 3
562 SYNCHRONIZER_DIRTY_POLICIES = 4
563 SYNCHRONIZER_DELETED_POLICIES = 5
Scott Bakerac2f2b52017-02-21 14:53:23 -0800564
Scott Bakerea1f4d02018-12-17 10:21:50 -0800565 def __init__(self, stub, modelName, packageName, kind=0):
Scott Baker96b995a2017-02-15 16:21:12 -0800566 self._stub = stub
567 self._modelName = modelName
568 self._packageName = packageName
Scott Bakerea1f4d02018-12-17 10:21:50 -0800569 self._kind = kind
Scott Baker96b995a2017-02-15 16:21:12 -0800570
571 def wrap_single(self, obj):
Scott Baker22796cc2017-02-23 16:53:34 -0800572 return make_ORMWrapper(obj, self._stub)
Scott Baker96b995a2017-02-15 16:21:12 -0800573
574 def wrap_list(self, obj):
Zack Williams045b63d2019-01-22 16:30:57 -0700575 result = []
Scott Baker96b995a2017-02-15 16:21:12 -0800576 for item in obj.items:
Scott Baker22796cc2017-02-23 16:53:34 -0800577 result.append(make_ORMWrapper(item, self._stub))
Scott Bakerb05393b2017-03-01 14:59:55 -0800578 return ORMQuerySet(result)
Scott Baker96b995a2017-02-15 16:21:12 -0800579
580 def all(self):
Zack Williams045b63d2019-01-22 16:30:57 -0700581 if self._kind == self.DEFAULT:
582 return self.wrap_list(
583 self._stub.invoke("List%s" % self._modelName, self._stub.make_empty())
584 )
Scott Bakerea1f4d02018-12-17 10:21:50 -0800585 else:
586 return self.filter()
Scott Baker96b995a2017-02-15 16:21:12 -0800587
Scott Baker22796cc2017-02-23 16:53:34 -0800588 def first(self):
Scott Bakerea1f4d02018-12-17 10:21:50 -0800589 objs = self.all()
Scott Baker22796cc2017-02-23 16:53:34 -0800590 if not objs:
591 return None
592 return objs[0]
593
Scott Bakerac2f2b52017-02-21 14:53:23 -0800594 def filter(self, **kwargs):
595 q = self._stub.make_Query()
Scott Bakerea1f4d02018-12-17 10:21:50 -0800596 q.kind = self._kind
Scott Bakerac2f2b52017-02-21 14:53:23 -0800597
598 for (name, val) in kwargs.items():
599 el = q.elements.add()
600
601 if name.endswith("__gt"):
602 name = name[:-4]
603 el.operator = el.GREATER_THAN
604 elif name.endswith("__gte"):
605 name = name[:-5]
606 el.operator = el.GREATER_THAN_OR_EQUAL
607 elif name.endswith("__lt"):
608 name = name[:-4]
609 el.operator = el.LESS_THAN
610 elif name.endswith("__lte"):
611 name = name[:-5]
612 el.operator = el.LESS_THAN_OR_EQUAL
Scott Bakere1607b82018-09-20 14:10:59 -0700613 elif name.endswith("__iexact"):
614 name = name[:-8]
615 el.operator = el.IEXACT
Scott Bakerac2f2b52017-02-21 14:53:23 -0800616 else:
617 el.operator = el.EQUAL
618
619 el.name = name
620 if isinstance(val, int):
621 el.iValue = val
622 else:
623 el.sValue = val
624
625 return self.wrap_list(self._stub.invoke("Filter%s" % self._modelName, q))
626
627 def filter_special(self, kind):
628 q = self._stub.make_Query()
629 q.kind = kind
630 return self.wrap_list(self._stub.invoke("Filter%s" % self._modelName, q))
631
Scott Baker22796cc2017-02-23 16:53:34 -0800632 def get(self, **kwargs):
633 if kwargs.keys() == ["id"]:
634 # the fast and easy case, look it up by id
Zack Williams045b63d2019-01-22 16:30:57 -0700635 return self.wrap_single(
636 self._stub.invoke(
637 "Get%s" % self._modelName, self._stub.make_ID(id=kwargs["id"])
638 )
639 )
Scott Baker22796cc2017-02-23 16:53:34 -0800640 else:
641 # the slightly more difficult case, filter and return the first item
642 objs = self.filter(**kwargs)
643 return objs[0]
Scott Baker96b995a2017-02-15 16:21:12 -0800644
645 def new(self, **kwargs):
Zack Williams045b63d2019-01-22 16:30:57 -0700646 if self._kind != ORMObjectManager.DEFAULT:
647 raise Exception(
648 "Creating objects is only supported by the DEFAULT object manager"
649 )
Scott Bakerea1f4d02018-12-17 10:21:50 -0800650
Scott Bakeraa556b02017-03-07 16:07:34 -0800651 cls = self._stub.all_grpc_classes[self._modelName]
Scott Bakerfe42a6f2017-03-18 09:11:31 -0700652 o = make_ORMWrapper(cls(), self._stub, is_new=True)
Zack Williams045b63d2019-01-22 16:30:57 -0700653 for (k, v) in kwargs.items():
Scott Bakerfe42a6f2017-03-18 09:11:31 -0700654 setattr(o, k, v)
Scott Baker5b7fba02018-10-17 08:46:46 -0700655 o.recompute_initial()
Scott Bakerfe42a6f2017-03-18 09:11:31 -0700656 return o
Scott Baker96b995a2017-02-15 16:21:12 -0800657
Zack Williams045b63d2019-01-22 16:30:57 -0700658
Scott Baker96b995a2017-02-15 16:21:12 -0800659class ORMModelClass(object):
660 def __init__(self, stub, model_name, package_name):
Scott Baker22796cc2017-02-23 16:53:34 -0800661 self.model_name = model_name
Scott Bakeraa556b02017-03-07 16:07:34 -0800662 self._stub = stub
Scott Baker96b995a2017-02-15 16:21:12 -0800663 self.objects = ORMObjectManager(stub, model_name, package_name)
Zack Williams045b63d2019-01-22 16:30:57 -0700664 self.deleted_objects = ORMObjectManager(
665 stub,
666 model_name,
667 package_name,
668 ORMObjectManager.SYNCHRONIZER_DELETED_OBJECTS,
669 )
Scott Baker96b995a2017-02-15 16:21:12 -0800670
Scott Bakerbb81e152017-03-02 15:28:36 -0800671 @property
Scott Baker22796cc2017-02-23 16:53:34 -0800672 def __name__(self):
673 return self.model_name
674
Scott Bakeraa556b02017-03-07 16:07:34 -0800675 @property
676 def content_type_id(self):
677 return self._stub.reverse_content_type_map[self.model_name]
678
Scott Baker8a6d91f2017-03-22 11:23:11 -0700679 def __call__(self, *args, **kwargs):
680 return self.objects.new(*args, **kwargs)
681
Zack Williams045b63d2019-01-22 16:30:57 -0700682
Scott Baker96b995a2017-02-15 16:21:12 -0800683class ORMStub(object):
Zack Williams045b63d2019-01-22 16:30:57 -0700684 def __init__(
685 self,
686 stub,
687 protos,
688 package_name,
689 invoker=None,
690 caller_kind="grpcapi",
691 empty=None,
692 enable_backoff=True,
693 restart_on_disconnect=False,
694 ):
Scott Baker96b995a2017-02-15 16:21:12 -0800695 self.grpc_stub = stub
Scott Bakerb96ba432018-02-26 09:53:48 -0800696 self.protos = protos
697 self.common_protos = protos.common__pb2
Scott Baker96b995a2017-02-15 16:21:12 -0800698 self.all_model_names = []
Scott Bakeraa556b02017-03-07 16:07:34 -0800699 self.all_grpc_classes = {}
700 self.content_type_map = {}
701 self.reverse_content_type_map = {}
Scott Bakeref8d85d2017-02-21 16:44:28 -0800702 self.invoker = invoker
Scott Baker22796cc2017-02-23 16:53:34 -0800703 self.caller_kind = caller_kind
Scott Baker500f8c72017-05-19 09:41:50 -0700704 self.enable_backoff = enable_backoff
Scott Bakerb06e3e02017-12-12 11:05:53 -0800705 self.restart_on_disconnect = restart_on_disconnect
Scott Baker96b995a2017-02-15 16:21:12 -0800706
Scott Bakerf0ee0dc2017-05-15 10:10:05 -0700707 if not empty:
Scott Bakerb96ba432018-02-26 09:53:48 -0800708 empty = self.protos.google_dot_protobuf_dot_empty__pb2.Empty
Scott Bakerf0ee0dc2017-05-15 10:10:05 -0700709 self._empty = empty
710
Scott Baker96b995a2017-02-15 16:21:12 -0800711 for name in dir(stub):
Zack Williams045b63d2019-01-22 16:30:57 -0700712 if name.startswith("Get"):
713 model_name = name[3:]
714 setattr(self, model_name, ORMModelClass(self, model_name, package_name))
Scott Baker96b995a2017-02-15 16:21:12 -0800715
Zack Williams045b63d2019-01-22 16:30:57 -0700716 self.all_model_names.append(model_name)
Scott Baker96b995a2017-02-15 16:21:12 -0800717
Zack Williams045b63d2019-01-22 16:30:57 -0700718 grpc_class = getattr(self.protos, model_name)
719 self.all_grpc_classes[model_name] = grpc_class
Scott Bakeraa556b02017-03-07 16:07:34 -0800720
Zack Williams045b63d2019-01-22 16:30:57 -0700721 ct = grpc_class.DESCRIPTOR.GetOptions().Extensions._FindExtensionByName(
722 "xos.contentTypeId"
723 )
724 if ct:
725 ct = grpc_class.DESCRIPTOR.GetOptions().Extensions[ct]
726 if ct:
727 self.content_type_map[ct] = model_name
728 self.reverse_content_type_map[model_name] = ct
Scott Bakeraa556b02017-03-07 16:07:34 -0800729
730 def genericForeignKeyResolve(self, content_type_id, id):
Scott Bakerd0f1dc12018-04-23 12:05:32 -0700731 if content_type_id.endswith("_decl"):
732 content_type_id = content_type_id[:-5]
733
734 if content_type_id not in self.content_type_map:
Zack Williams045b63d2019-01-22 16:30:57 -0700735 raise ORMGenericContentNotFoundException(
736 "Content_type %s not found in self.content_type_map" % content_type_id
737 )
Scott Bakerd0f1dc12018-04-23 12:05:32 -0700738
Scott Bakeraa556b02017-03-07 16:07:34 -0800739 model_name = self.content_type_map[content_type_id]
Scott Bakerd0f1dc12018-04-23 12:05:32 -0700740
Scott Bakeraa556b02017-03-07 16:07:34 -0800741 model = getattr(self, model_name)
Scott Bakerd0f1dc12018-04-23 12:05:32 -0700742 objs = model.objects.filter(id=id)
743 if not objs:
Zack Williams045b63d2019-01-22 16:30:57 -0700744 raise ORMGenericObjectNotFoundException(
745 "Object %s of model %s was not found" % (id, model_name)
746 )
Scott Bakerd0f1dc12018-04-23 12:05:32 -0700747
Scott Bakeraa556b02017-03-07 16:07:34 -0800748 return model.objects.get(id=id)
749
Scott Baker22796cc2017-02-23 16:53:34 -0800750 def add_default_metadata(self, metadata):
Zack Williams045b63d2019-01-22 16:30:57 -0700751 default_metadata = [("caller_kind", self.caller_kind)]
Scott Baker22796cc2017-02-23 16:53:34 -0800752
Scott Bakerd8246712018-07-12 18:08:31 -0700753 # introspect to see if we're running from a synchronizer thread
754 if getattr(threading.current_thread(), "is_sync_thread", False):
Zack Williams045b63d2019-01-22 16:30:57 -0700755 default_metadata.append(("is_sync_save", "1"))
Scott Bakerd8246712018-07-12 18:08:31 -0700756
757 # introspect to see if we're running from a model_policy thread
758 if getattr(threading.current_thread(), "is_policy_thread", False):
Zack Williams045b63d2019-01-22 16:30:57 -0700759 default_metadata.append(("is_policy_save", "1"))
Scott Bakerd8246712018-07-12 18:08:31 -0700760
Scott Baker22796cc2017-02-23 16:53:34 -0800761 # build up a list of metadata keys we already have
Zack Williams045b63d2019-01-22 16:30:57 -0700762 md_keys = [x[0] for x in metadata]
Scott Baker22796cc2017-02-23 16:53:34 -0800763
764 # add any defaults that we don't already have
765 for md in default_metadata:
766 if md[0] not in md_keys:
Zack Williams045b63d2019-01-22 16:30:57 -0700767 metadata.append((md[0], md[1]))
Scott Baker22796cc2017-02-23 16:53:34 -0800768
Scott Baker57c74822017-02-23 11:13:04 -0800769 def invoke(self, name, request, metadata=[]):
Scott Baker22796cc2017-02-23 16:53:34 -0800770 self.add_default_metadata(metadata)
771
Scott Bakeref8d85d2017-02-21 16:44:28 -0800772 if self.invoker:
773 # Hook in place to call Chameleon's invoke method, as soon as we
774 # have rewritten the synchronizer to use reactor.
Zack Williams045b63d2019-01-22 16:30:57 -0700775 return self.invoker.invoke(
776 self.grpc_stub.__class__, name, request, metadata={}
777 ).result[0]
Scott Baker500f8c72017-05-19 09:41:50 -0700778 elif self.enable_backoff:
Scott Bakeref8d85d2017-02-21 16:44:28 -0800779 # Our own retry mechanism. This works fine if there is a temporary
780 # failure in connectivity, but does not re-download gRPC schema.
Scott Baker500f8c72017-05-19 09:41:50 -0700781 import grpc
Zack Williams045b63d2019-01-22 16:30:57 -0700782
Scott Bakerb06e3e02017-12-12 11:05:53 -0800783 backoff = [0.5, 1, 2, 4, 8]
Scott Bakeref8d85d2017-02-21 16:44:28 -0800784 while True:
Scott Bakeref8d85d2017-02-21 16:44:28 -0800785 try:
786 method = getattr(self.grpc_stub, name)
Scott Baker57c74822017-02-23 11:13:04 -0800787 return method(request, metadata=metadata)
Zack Williams045b63d2019-01-22 16:30:57 -0700788 except grpc._channel._Rendezvous as e:
Scott Bakeref8d85d2017-02-21 16:44:28 -0800789 code = e.code()
790 if code == grpc.StatusCode.UNAVAILABLE:
Scott Bakerb06e3e02017-12-12 11:05:53 -0800791 if self.restart_on_disconnect:
792 # This is a blunt technique... We lost connectivity to the core, and we don't know that
793 # the core is still serving up the same models it was when we established connectivity,
794 # so restart the synchronizer.
795 # TODO: Hash check on the core models to tell if something changed would be better.
Zack Williams045b63d2019-01-22 16:30:57 -0700796 os.execv(sys.executable, ["python"] + sys.argv)
Scott Bakeref8d85d2017-02-21 16:44:28 -0800797 if not backoff:
798 raise Exception("No more retries on %s" % name)
799 time.sleep(backoff.pop(0))
800 else:
801 raise
Zack Williams045b63d2019-01-22 16:30:57 -0700802 except BaseException:
Scott Bakeref8d85d2017-02-21 16:44:28 -0800803 raise
Scott Baker500f8c72017-05-19 09:41:50 -0700804 else:
805 method = getattr(self.grpc_stub, name)
806 return method(request, metadata=metadata)
807
Scott Baker96b995a2017-02-15 16:21:12 -0800808 def make_ID(self, id):
Scott Bakerb96ba432018-02-26 09:53:48 -0800809 return getattr(self.common_protos, "ID")(id=id)
Scott Bakerf0ee0dc2017-05-15 10:10:05 -0700810
811 def make_empty(self):
812 return self._empty()
Scott Baker96b995a2017-02-15 16:21:12 -0800813
Scott Bakerac2f2b52017-02-21 14:53:23 -0800814 def make_Query(self):
Scott Bakerb96ba432018-02-26 09:53:48 -0800815 return getattr(self.common_protos, "Query")()
Scott Bakerac2f2b52017-02-21 14:53:23 -0800816
Scott Bakerf6145a22017-03-29 14:50:25 -0700817 def listObjects(self):
818 return self.all_model_names
819
Zack Williams045b63d2019-01-22 16:30:57 -0700820
Scott Baker22796cc2017-02-23 16:53:34 -0800821def register_convenience_wrapper(class_name, wrapper):
822 global convenience_wrappers
Scott Baker96b995a2017-02-15 16:21:12 -0800823
Scott Baker22796cc2017-02-23 16:53:34 -0800824 convenience_wrappers[class_name] = wrapper
825
Zack Williams045b63d2019-01-22 16:30:57 -0700826
Scott Baker22796cc2017-02-23 16:53:34 -0800827def make_ORMWrapper(wrapped_class, *args, **kwargs):
Scott Baker2f314d52018-08-24 08:31:19 -0700828 cls = None
829
830 if (not cls) and wrapped_class.__class__.__name__ in convenience_wrappers:
Scott Baker22796cc2017-02-23 16:53:34 -0800831 cls = convenience_wrappers[wrapped_class.__class__.__name__]
Scott Baker2f314d52018-08-24 08:31:19 -0700832
Zack Williams045b63d2019-01-22 16:30:57 -0700833 if not cls:
Scott Baker2f314d52018-08-24 08:31:19 -0700834 # Search the list of class names for this model to see if we have any applicable wrappers. The list is always
835 # sorted from most specific to least specific, so the first one we find will automatically be the most relevant
836 # one. If we don't find any, then default to ORMWrapper
837
838 # Note: Only works on objects that have been fetched from the server, not objects that are created on the
839 # client. This is because wrapped_class.class_names is filled in by the server.
840
841 # TODO(smbaker): Ought to be able to make this work with newly created objects after they are saved.
842
843 for name in wrapped_class.class_names.split(","):
844 if name in convenience_wrappers:
845 cls = convenience_wrappers[name]
846
Zack Williams045b63d2019-01-22 16:30:57 -0700847 if not cls:
Scott Baker22796cc2017-02-23 16:53:34 -0800848 cls = ORMWrapper
849
850 return cls(wrapped_class, *args, **kwargs)
851
Zack Williams045b63d2019-01-22 16:30:57 -0700852
Matteo Scandolo10a2f3c2018-04-20 16:59:38 +0200853def import_convenience_methods():
854
855 log.info("Loading convenience methods")
856
857 cwd = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
858 api_convenience_dir = os.path.join(cwd, "convenience")
859 for file in os.listdir(api_convenience_dir):
Zack Williams045b63d2019-01-22 16:30:57 -0700860 if file.endswith(".py") and "test" not in file:
Matteo Scandolo10a2f3c2018-04-20 16:59:38 +0200861 pathname = os.path.join(api_convenience_dir, file)
862 try:
863 log.debug("Loading: %s" % file)
864 imp.load_source(file[:-3], pathname)
Zack Williams045b63d2019-01-22 16:30:57 -0700865 except Exception:
866 log.exception(
867 "Cannot import api convenience method for: %s, %s"
868 % (file[:-3], pathname)
869 )