CORD-763 add reverse foreign key support
Change-Id: I7310ef9169deb4fcb7d6dfab7f70ba9cb68e3913
diff --git a/xos/grpc/apihelper.py b/xos/grpc/apihelper.py
index b757ae7..e6d3a40 100644
--- a/xos/grpc/apihelper.py
+++ b/xos/grpc/apihelper.py
@@ -61,6 +61,19 @@
setattr(p_obj, field.name, float(getattr(obj, field.name)))
elif (ftype == "GenericIPAddressField"):
setattr(p_obj, field.name, str(getattr(obj, field.name)))
+
+ for field in obj._meta.related_objects:
+ related_name = field.related_name
+ if not related_name:
+ continue
+ if "+" in related_name:
+ continue
+ rel_objs = getattr(obj, related_name)
+ for rel_obj in rel_objs.all():
+ if not hasattr(p_obj,related_name+"_ids"):
+ continue
+ x=getattr(p_obj,related_name+"_ids").append(rel_obj.id)
+
return p_obj
def protoToArgs(self, djangoClass, message):
diff --git a/xos/grpc/grpc_client.py b/xos/grpc/grpc_client.py
index cd561c7..f4f1c6c 100644
--- a/xos/grpc/grpc_client.py
+++ b/xos/grpc/grpc_client.py
@@ -1,5 +1,6 @@
import base64
import grpc
+import orm
from protos.common_pb2 import *
from protos.xos_pb2 import *
from protos.utility_pb2 import *
@@ -40,6 +41,8 @@
self.modeldefs = modeldefs_pb2_grpc.modeldefsStub(self.channel)
self.utility = utility_pb2_grpc.utilityStub(self.channel)
+ self.xos_orm = orm.ORMStub(self.stub, "xos")
+
class SecureClient(XOSClient):
def __init__(self, hostname, port=50051, cacert=SERVER_CA, username=None, password=None, sessionid=None):
super(SecureClient,self).__init__(hostname, port)
@@ -57,6 +60,8 @@
self.modeldefs = modeldefs_pb2_grpc.modeldefsStub(self.channel)
self.utility = utility_pb2_grpc.utilityStub(self.channel)
+ self.xos_orm = orm.ORMStub(self.stub, "xos")
+
def main(): # self-test
client = InsecureClient("xos-core.cord.lab")
print client.stub.ListUser(Empty())
diff --git a/xos/grpc/orm.py b/xos/grpc/orm.py
index 380a208..79d43e9 100644
--- a/xos/grpc/orm.py
+++ b/xos/grpc/orm.py
@@ -9,29 +9,37 @@
someSlice = api.Slices.get(id=1) ... get slice #1
someSlice.site ... automatically resolves site_id into a site object
+ someSlice.instances ... automatically resolves instances_ids into instance objects
someSlice.save() ... saves the slice object
"""
"""
import grpc_client, orm
c=grpc_client.SecureClient("xos-core.cord.lab", username="padmin@vicci.org", password="letmein")
-xos_orm=orm.ORMStub(c.stub)
-u=xos_orm.User.objects.get(id=1)
+u=c.xos_orm.User.objects.get(id=1)
"""
import functools
-import grpc_client
from google.protobuf.empty_pb2 import Empty
from protos.common_pb2 import ID
-from protos.xosoptions_pb2 import foreignKey
+from protos.xosoptions_pb2 import foreignKey, reverseForeignKey
+
+from google.protobuf import symbol_database as _symbol_database
+_sym_db = _symbol_database.Default()
class ORMWrapper(object):
- def __init__(self, wrapped_class, stub):
+ """ Wraps a protobuf object to provide ORM features """
+
+ def __init__(self, wrapped_class, stub, is_new=False):
super(ORMWrapper, self).__setattr__("_wrapped_class", wrapped_class)
super(ORMWrapper, self).__setattr__("stub", stub)
super(ORMWrapper, self).__setattr__("cache", {})
+ super(ORMWrapper, self).__setattr__("reverse_cache", {})
+ super(ORMWrapper, self).__setattr__("is_new", is_new)
fkmap=self.gen_fkmap()
super(ORMWrapper, self).__setattr__("_fkmap", fkmap)
+ reverse_fkmap=self.gen_reverse_fkmap()
+ super(ORMWrapper, self).__setattr__("_reverse_fkmap", reverse_fkmap)
def gen_fkmap(self):
fkmap = {}
@@ -44,6 +52,17 @@
return fkmap
+ def gen_reverse_fkmap(self):
+ reverse_fkmap = {}
+
+ for (name, field) in self._wrapped_class.DESCRIPTOR.fields_by_name.items():
+ if name.endswith("_ids"):
+ fk = field.GetOptions().Extensions[reverseForeignKey]
+ if fk:
+ reverse_fkmap[name[:-4]] = {"src_fieldName": name, "modelName": fk.modelName}
+
+ return reverse_fkmap
+
def fk_resolve(self, name):
if name in self.cache:
return ORMWrapper(self.cache[name], self.stub)
@@ -57,6 +76,13 @@
return ORMWrapper(dest_model, self.stub)
+ def reverse_fk_resolve(self, name):
+ if name not in self.reverse_cache:
+ fk_entry = self._reverse_fkmap[name]
+ self.cache[name] = ORMLocalObjectManager(self.stub, fk_entry["modelName"], getattr(self, fk_entry["src_fieldName"]))
+
+ return self.cache[name]
+
def __getattr__(self, name, *args, **kwargs):
# note: getattr is only called for attributes that do not exist in
# self.__dict__
@@ -64,6 +90,9 @@
if name in self._fkmap.keys():
return self.fk_resolve(name)
+ if name in self._reverse_fkmap.keys():
+ return self.reverse_fk_resolve(name)
+
return getattr(self._wrapped_class, name, *args, **kwargs)
def __setattr__(self, name, value):
@@ -76,18 +105,53 @@
return self._wrapped_class.__repr__()
def save(self):
- update_method = getattr(self.stub,"Update%s" % self._wrapped_class.__class__.__name__)
- update_method(self._wrapped_class)
+ if self.is_new:
+ create_method = getattr(self.stub,"Create%s" % self._wrapped_class.__class__.__name__)
+ new_class = create_method(self._wrapped_class)
+ self._wrapped_class = new_class
+ self.is_new = False
+ else:
+ update_method = getattr(self.stub,"Update%s" % self._wrapped_class.__class__.__name__)
+ update_method(self._wrapped_class)
def delete(self):
delete_method = getattr(self.stub,"Delete%s" % self._wrapped_class.__class__.__name__)
id = ID(id=self._wrapped_class.id)
delete_method(id)
-class ORMObjectManager(object):
- def __init__(self, stub, modelName):
+class ORMLocalObjectManager(object):
+ """ Manages a local list of objects """
+
+ def __init__(self, stub, modelName, idList):
self._stub = stub
self._modelName = modelName
+ self._idList = idList
+ self._cache = None
+
+ def resolve_queryset(self):
+ if self._cache is not None:
+ return self._cache
+
+ models = []
+ get_method = getattr(self._stub, "Get%s" % self._modelName)
+ for id in self._idList:
+ models.append(get_method(ID(id=id)))
+
+ self._cache = models
+
+ return models
+
+ def all(self):
+ models = self.resolve_queryset()
+ return [ORMWrapper(x,self._stub) for x in models]
+
+class ORMObjectManager(object):
+ """ Manages a remote list of objects """
+
+ def __init__(self, stub, modelName, packageName):
+ self._stub = stub
+ self._modelName = modelName
+ self._packageName = packageName
def wrap_single(self, obj):
return ORMWrapper(obj, self._stub)
@@ -106,16 +170,21 @@
get_method = getattr(self._stub, "Get%s" % self._modelName)
return self.wrap_single(get_method(ID(id=id)))
+ def new(self, **kwargs):
+ full_model_name = "%s.%s" % (self._packageName, self._modelName)
+ cls = _sym_db._classes[full_model_name]
+ return ORMWrapper(cls(), self._stub, is_new=True)
+
class ORMModelClass(object):
- def __init__(self, stub, model_name):
- self.objects = ORMObjectManager(stub, model_name)
+ def __init__(self, stub, model_name, package_name):
+ self.objects = ORMObjectManager(stub, model_name, package_name)
class ORMStub(object):
- def __init__(self, stub):
+ def __init__(self, stub, package_name):
for name in dir(stub):
if name.startswith("Get"):
model_name = name[3:]
- setattr(self,model_name, ORMModelClass(stub, model_name))
+ setattr(self,model_name, ORMModelClass(stub, model_name, package_name))
#def wrap_get(*args, **kwargs):
# stub=kwargs.pop("stub")
diff --git a/xos/grpc/protos/Makefile b/xos/grpc/protos/Makefile
index a1c82da..df07cfd 100644
--- a/xos/grpc/protos/Makefile
+++ b/xos/grpc/protos/Makefile
@@ -85,4 +85,4 @@
cd ../../tools/apigen && python ./modelgen -a core protobuf.template.txt > /opt/xos/grpc/protos/xos.proto
cd ../../tools/apigen && python ./modelgen -a core grpc_api.template.py > /opt/xos/grpc/xos_grpc_api.py
cd ../../tools/apigen && python ./modelgen -a core grpc_list_test.template.py > /opt/xos/grpc/list_test.py
-
+ cd ../../tools/apigen && python ./modelgen -a core chameleon_list_test.template.sh > /opt/xos/grpc/tests/chameleon_list_test.sh
diff --git a/xos/grpc/protos/xosoptions.proto b/xos/grpc/protos/xosoptions.proto
index d82fe6f..e8f06d2 100644
--- a/xos/grpc/protos/xosoptions.proto
+++ b/xos/grpc/protos/xosoptions.proto
@@ -14,7 +14,12 @@
string modelName = 1;
}
+message ReverseForeignKeyRule {
+ string modelName = 1;
+}
+
extend google.protobuf.FieldOptions {
ValRule val = 1001;
ForeignKeyRule foreignKey = 1002;
+ ReverseForeignKeyRule reverseForeignKey = 1003;
}
diff --git a/xos/grpc/tests/api_user_crud.py b/xos/grpc/tests/api_user_crud.py
new file mode 100644
index 0000000..ee455fa
--- /dev/null
+++ b/xos/grpc/tests/api_user_crud.py
@@ -0,0 +1,37 @@
+import sys
+sys.path.append("..")
+
+import grpc_client
+
+print "api_user_crud"
+
+#c=grpc_client.InsecureClient("localhost")
+c=grpc_client.SecureClient("xos-core.cord.lab", username="padmin@vicci.org", password="letmein")
+u=grpc_client.User()
+import random, string
+u.email=''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10))
+u.site_id=1
+u2=c.stub.CreateUser(u)
+
+# update the user
+u2.password="foobar"
+c.stub.UpdateUser(u2)
+
+# do a listall and make sure user exists
+u_all = c.stub.ListUser(grpc_client.Empty()).items
+u_all = [x for x in u_all if x.email == u.email]
+assert(len(u_all)==1)
+
+u3=c.stub.GetUser(grpc_client.ID(id=u2.id))
+assert(u3.id == u2.id)
+assert(u3.password=="foobar")
+
+c.stub.DeleteUser(grpc_client.ID(id=u3.id))
+
+# make sure it is deleted
+u_all = c.stub.ListUser(grpc_client.Empty()).items
+u_all = [x for x in u_all if x.email == u.email]
+assert(len(u_all)==0)
+
+print " okay"
+
diff --git a/xos/grpc/tests/cham_slice_crud.sh b/xos/grpc/tests/cham_slice_crud.sh
new file mode 100644
index 0000000..bc7259a
--- /dev/null
+++ b/xos/grpc/tests/cham_slice_crud.sh
@@ -0,0 +1,22 @@
+source /opt/xos/grpc/tests/chamconfig.sh
+
+RESPONSE=`curl -X POST -H "Content-Type: application/json" -d '{"username": "padmin@vicci.org", "password": "letmein"}' http://$HOSTNAME:8080/xosapi/v1/utility/login`
+SESSIONID=`echo $RESPONSE | python -c "import json,sys; print json.load(sys.stdin)['sessionid']"`
+echo "sessionid=$SESSIONID"
+
+RS=`cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1`
+SLICENAME="mysite_$RS"
+
+echo "slicename=$SLICENAME"
+
+RESPONSE=`curl -X POST -H "x-xossession: $SESSIONID" -H "Content-Type: application/json" -d "{\"name\": \"$SLICENAME\", \"site_id\": 1}" http://$HOSTNAME:8080/xosapi/v1/slices`
+
+echo "create response: $RESPONSE"
+SLICEID=`echo $RESPONSE | python -c "import json,sys; print json.load(sys.stdin)['id']"`
+
+RESPONSE=`curl -X GET -H "x-xossession: $SESSIONID" http://$HOSTNAME:8080/xosapi/v1/slices/$SLICEID`
+echo "get response: $RESPONSE"
+
+RESPONSE=`curl -X DELETE -H "x-xossession: $SESSIONID" http://$HOSTNAME:8080/xosapi/v1/slices/$SLICEID`
+
+echo "delete response: $RESPONSE"
diff --git a/xos/grpc/tests/chamconfig.sh b/xos/grpc/tests/chamconfig.sh
new file mode 100644
index 0000000..b2eaa31
--- /dev/null
+++ b/xos/grpc/tests/chamconfig.sh
@@ -0,0 +1,3 @@
+# enter your hostname here:
+
+HOSTNAME=my_hostname
\ No newline at end of file
diff --git a/xos/grpc/tests/orm_user_crud.py b/xos/grpc/tests/orm_user_crud.py
new file mode 100644
index 0000000..6eda2b5
--- /dev/null
+++ b/xos/grpc/tests/orm_user_crud.py
@@ -0,0 +1,57 @@
+import sys
+sys.path.append("..")
+
+import grpc_client
+
+print "orm_user_crud"
+
+c=grpc_client.SecureClient("xos-core.cord.lab", username="padmin@vicci.org", password="letmein")
+
+# create a new user and save it
+u=c.xos_orm.User.objects.new()
+assert(u.id==0)
+import random, string
+u.email=''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10))
+u.site_id=1
+u.save()
+
+# when we created the user, he should be assigned an id
+orig_id = u.id
+assert(orig_id!=0)
+
+# site object should be populated
+assert(u.site is not None)
+
+# site object should have a backpointer to user
+u_all = u.site.users.all()
+u_all = [x for x in u_all if x.email == u.email]
+assert(len(u_all)==1)
+
+# update the user
+u.password="foobar"
+u.save()
+
+# update should not have changed it
+assert(u.id==orig_id)
+
+# check a listall and make sure the user is listed
+u_all = c.xos_orm.User.objects.all()
+u_all = [x for x in u_all if x.email == u.email]
+assert(len(u_all)==1)
+u2 = u_all[0]
+assert(u2.id == u.id)
+
+# get and make sure the password was updated
+u3 = c.xos_orm.User.objects.get(id=orig_id)
+assert(u3.password=="foobar")
+
+# delete the user
+u3.delete()
+
+# make sure it is deleted
+u_all = c.xos_orm.User.objects.all()
+u_all = [x for x in u_all if x.email == u.email]
+assert(len(u_all)==0)
+
+print " okay"
+
diff --git a/xos/tools/apigen/chameleon_list_test.template.sh b/xos/tools/apigen/chameleon_list_test.template.sh
new file mode 100644
index 0000000..e46503d
--- /dev/null
+++ b/xos/tools/apigen/chameleon_list_test.template.sh
@@ -0,0 +1,17 @@
+source /opt/xos/grpc/tests/chamconfig.sh
+
+# test modeldefs
+curl -f --silent http://$HOSTNAME:8080/xosapi/v1/modeldefs > /dev/null
+if [[ $? -ne 0 ]]; then
+ echo fail modeldefs
+fi
+
+{% for object in generator.all() %}
+curl -f --silent http://$HOSTNAME:8080/xosapi/v1/{{ object.camel()|lower }}s > /dev/null
+if [[ $? -ne 0 ]]; then
+ echo fail {{ object.camel() }}
+fi
+{%- endfor %}
+
+echo "okay"
+
diff --git a/xos/tools/apigen/grpc_list_test.template.py b/xos/tools/apigen/grpc_list_test.template.py
index 9b7fffb..4b00219 100644
--- a/xos/tools/apigen/grpc_list_test.template.py
+++ b/xos/tools/apigen/grpc_list_test.template.py
@@ -31,3 +31,10 @@
print "Okay"
{%- endfor %}
+c=grpc_client.SecureClient("xos-core.cord.lab", sessionid=session.sessionid)
+{% for object in generator.all() %}
+print "testing session secure xos_orm.{{ object.camel() }}.objects.all() ...",
+c.xos_orm.{{ object.camel() }}.objects.all()
+print "Okay"
+{%- endfor %}
+
diff --git a/xos/tools/apigen/modelgen b/xos/tools/apigen/modelgen
index a34798f..2beb508 100755
--- a/xos/tools/apigen/modelgen
+++ b/xos/tools/apigen/modelgen
@@ -80,6 +80,7 @@
self.all_fields = []
self.field_dict = []
self.refs = []
+ self.reverse_refs = []
self.plural_name = None
def plural(self):
@@ -216,6 +217,20 @@
obj.refs.append(cobj)
+ for obj in self.values():
+ # generate foreign key reverse references
+ for f in obj.model._meta.related_objects:
+ related_model = getattr(f, "related_model", None)
+ if not f.related_name:
+ continue
+ if "+" in f.related_name:
+ continue
+ if related_model and (related_model.__name__.lower() in self.keys()):
+ cobj = copy.deepcopy(self[related_model.__name__.lower()])
+ cobj.related_name = f.related_name
+ obj.reverse_refs.append(cobj)
+
+
diff --git a/xos/tools/apigen/protobuf.template.txt b/xos/tools/apigen/protobuf.template.txt
index 359426a..1c9ce4b 100644
--- a/xos/tools/apigen/protobuf.template.txt
+++ b/xos/tools/apigen/protobuf.template.txt
@@ -56,6 +56,9 @@
UNKNOWN {{ field.get_internal_type() }} {{ field.name }} = {{ loop.index }};
{%- endif %}
}
+ {%- endfor -%}
+ {%- for ref in object.reverse_refs %}
+ repeated int32 {{ ref.related_name }}_ids = {{ loop.index+100 }} [(reverseForeignKey).modelName = "{{ ref.camel() }}"];
{%- endfor %}
}