CORD-762 create xos_client python library
Change-Id: I4b1db15a83c5539952d8577515a626bd0f738d68
diff --git a/containers/xos/Dockerfile.client b/containers/xos/Dockerfile.client
new file mode 100644
index 0000000..2a60577
--- /dev/null
+++ b/containers/xos/Dockerfile.client
@@ -0,0 +1,25 @@
+FROM xosproject/xos-base
+
+ARG XOS_GIT_COMMIT_HASH=unknown
+ARG XOS_GIT_COMMIT_DATE=unknown
+
+LABEL XOS_GIT_COMMIT_HASH=$XOS_GIT_COMMIT_HASH
+LABEL XOS_GIT_COMMIT_DATE=$XOS_GIT_COMMIT_DATE
+
+# Include certificates from Openstack
+ADD containers/xos/local_certs.crt /usr/local/share/ca-certificates/local_certs.crt
+RUN update-ca-certificates
+
+# Install XOS client
+ADD xos/xos_client /tmp/xos_client
+
+# Install chameleon
+ADD containers/xos/tmp.chameleon /tmp/xos_client/xosapi/chameleon
+
+#ENV HOME /root
+#WORKDIR /opt/xos
+
+# install the client library and xossh
+RUN chdir /tmp/xos_client && make
+
+CMD xossh
diff --git a/containers/xos/Makefile b/containers/xos/Makefile
index d7be421..b46c9ca 100644
--- a/containers/xos/Makefile
+++ b/containers/xos/Makefile
@@ -57,6 +57,15 @@
sudo docker build --no-cache=${NO_DOCKER_CACHE} --rm \
-f Dockerfile.test -t xosproject/xos-test ${BUILD_ARGS} ../..
+client:
+ rm -rf tmp.chameleon
+ cp -R /opt/cord/component/chameleon tmp.chameleon
+ sudo docker build --no-cache=${NO_DOCKER_CACHE} --rm \
+ --build-arg XOS_GIT_COMMIT_HASH="${XOS_GIT_COMMIT_HASH}" \
+ --build-arg XOS_GIT_COMMIT_DATE="${XOS_GIT_COMMIT_DATE}" \
+ -f Dockerfile.client -t xosproject/xos-client ${BUILD_ARGS} ../..
+ rm -rf tmp.chameleon
+
run:
sudo docker run -d --name ${CONTAINER_NAME} -p 80:8000 \
${IMAGE_NAME}
diff --git a/xos/xos_client/Makefile b/xos/xos_client/Makefile
new file mode 100644
index 0000000..f018a65
--- /dev/null
+++ b/xos/xos_client/Makefile
@@ -0,0 +1,7 @@
+all: chameleon_protos xosapi_install
+
+chameleon_protos:
+ cd xosapi/chameleon/protos; VOLTHA_BASE=anything make
+
+xosapi_install:
+ python ./setup.py install
diff --git a/xos/xos_client/setup.py b/xos/xos_client/setup.py
new file mode 100644
index 0000000..4d601c2
--- /dev/null
+++ b/xos/xos_client/setup.py
@@ -0,0 +1,52 @@
+#! /usr/bin/env python
+
+import os
+import site
+from distutils.core import setup
+
+CHAMELEON_DIR='xosapi/chameleon'
+
+if not os.path.exists(CHAMELEON_DIR):
+ raise Exception("%s does not exist!" % CHAMELEON_DIR)
+
+if not os.path.exists(os.path.join(CHAMELEON_DIR, "protos/schema_pb2.py")):
+ raise Exception("Please make the chameleon protos")
+
+setup(name='xosapi',
+ description='XOS api client',
+ package_dir= {'xosapi.chameleon': CHAMELEON_DIR},
+ packages=['xosapi.chameleon.grpc_client',
+ 'xosapi.chameleon.protos',
+ 'xosapi.chameleon.protos.third_party',
+ 'xosapi.chameleon.protos.third_party.google',
+ 'xosapi.chameleon.protos.third_party.google.api',
+ 'xosapi.chameleon.utils',
+ 'xosapi.chameleon.protoc_plugins',
+ 'xosapi'],
+ py_modules= ['xosapi.chameleon.__init__'],
+ include_package_data=True,
+ package_data = {'xosapi.chameleon.protos.third_party.google.api': ['*.proto'],
+ 'xosapi.chameleon.protos': ['*.proto'],
+ 'xosapi.chameleon.protoc_plugins': ['*.desc']},
+ scripts = ['xossh'],
+ )
+
+# Chameleon needs the following files set as executable
+for dir in site.getsitepackages():
+ fn = os.path.join(dir, "xosapi/chameleon/protoc_plugins/gw_gen.py")
+ if os.path.exists(fn):
+ os.chmod(fn, 0777)
+ fn = os.path.join(dir, "xosapi/chameleon/protoc_plugins/swagger_gen.py")
+ if os.path.exists(fn):
+ os.chmod(fn, 0777)
+
+
+"""
+from twisted.internet import reactor
+from xosapi.xos_grpc_client import InsecureClient
+client = InsecureClient(endpoint="xos-core.cord.lab:50055")
+client.start()
+reactor.run()
+"""
+
+
diff --git a/xos/xos_client/xosapi/__init__.py b/xos/xos_client/xosapi/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/xos/xos_client/xosapi/__init__.py
diff --git a/xos/xos_client/xosapi/orm.py b/xos/xos_client/xosapi/orm.py
new file mode 100644
index 0000000..f0ef355
--- /dev/null
+++ b/xos/xos_client/xosapi/orm.py
@@ -0,0 +1,208 @@
+"""
+Django-like ORM layer for gRPC
+
+Usage:
+ api = ORMStub(stub)
+
+ api.Slices.all() ... list all slices
+
+ someSlice = api.Slices.get(id=1) ... get slice #1
+
+ someSlice.site ... automatically resolves site_id into a site object
+ someSlice.instances ... automatically resolves instances_ids into instance objects
+ someSlice.save() ... saves the slice object
+"""
+
+"""
+import grpc_client, orm
+c=grpc_client.SecureClient("xos-core.cord.lab", username="padmin@vicci.org", password="letmein")
+u=c.xos_orm.User.objects.get(id=1)
+"""
+
+import functools
+from google.protobuf.empty_pb2 import Empty
+
+from google.protobuf import symbol_database as _symbol_database
+_sym_db = _symbol_database.Default()
+
+class ORMWrapper(object):
+ """ Wraps a protobuf object to provide ORM features """
+
+ def __init__(self, wrapped_class, stub, is_new=False):
+ super(ORMWrapper, self).__setattr__("_wrapped_class", wrapped_class)
+ super(ORMWrapper, self).__setattr__("stub", stub)
+ super(ORMWrapper, self).__setattr__("cache", {})
+ super(ORMWrapper, self).__setattr__("reverse_cache", {})
+ super(ORMWrapper, self).__setattr__("is_new", is_new)
+ fkmap=self.gen_fkmap()
+ super(ORMWrapper, self).__setattr__("_fkmap", fkmap)
+ reverse_fkmap=self.gen_reverse_fkmap()
+ super(ORMWrapper, self).__setattr__("_reverse_fkmap", reverse_fkmap)
+
+ def gen_fkmap(self):
+ fkmap = {}
+
+ for (name, field) in self._wrapped_class.DESCRIPTOR.fields_by_name.items():
+ if name.endswith("_id"):
+ foreignKey = field.GetOptions().Extensions._FindExtensionByName("xos.foreignKey")
+ fk = field.GetOptions().Extensions[foreignKey]
+ if fk:
+ fkmap[name[:-3]] = {"src_fieldName": name, "modelName": fk.modelName}
+
+ return fkmap
+
+ def gen_reverse_fkmap(self):
+ reverse_fkmap = {}
+
+ for (name, field) in self._wrapped_class.DESCRIPTOR.fields_by_name.items():
+ if name.endswith("_ids"):
+ reverseForeignKey = field.GetOptions().Extensions._FindExtensionByName("xos.reverseForeignKey")
+ fk = field.GetOptions().Extensions[reverseForeignKey]
+ if fk:
+ reverse_fkmap[name[:-4]] = {"src_fieldName": name, "modelName": fk.modelName}
+
+ return reverse_fkmap
+
+ def fk_resolve(self, name):
+ if name in self.cache:
+ return ORMWrapper(self.cache[name], self.stub)
+
+ fk_entry = self._fkmap[name]
+ id=self.stub.make_ID(id=getattr(self, fk_entry["src_fieldName"]))
+ dest_model = self.stub.invoke("Get%s" % fk_entry["modelName"], id)
+
+ self.cache[name] = dest_model
+
+ return ORMWrapper(dest_model, self.stub)
+
+ def reverse_fk_resolve(self, name):
+ if name not in self.reverse_cache:
+ fk_entry = self._reverse_fkmap[name]
+ self.cache[name] = ORMLocalObjectManager(self.stub, fk_entry["modelName"], getattr(self, fk_entry["src_fieldName"]))
+
+ return self.cache[name]
+
+ def __getattr__(self, name, *args, **kwargs):
+ # note: getattr is only called for attributes that do not exist in
+ # self.__dict__
+
+ if name in self._fkmap.keys():
+ return self.fk_resolve(name)
+
+ if name in self._reverse_fkmap.keys():
+ return self.reverse_fk_resolve(name)
+
+ return getattr(self._wrapped_class, name, *args, **kwargs)
+
+ def __setattr__(self, name, value):
+ if name in self.__dict__:
+ super(ORMWrapper,self).__setattr__(name, value)
+ else:
+ setattr(self._wrapped_class, name, value)
+
+ def __repr__(self):
+ return self._wrapped_class.__repr__()
+
+ def save(self):
+ if self.is_new:
+ new_class = self.stub.invoke("Create%s" % self._wrapped_class.__class__.__name__, self._wrapped_class)
+ self._wrapped_class = new_class
+ self.is_new = False
+ else:
+ self.stub.invoke("Update%s" % self._wrapped_class.__class__.__name__, self._wrapped_class)
+
+ def delete(self):
+ id = self.stub.make_ID(id=self._wrapped_class.id)
+ self.stub.invoke("Delete%s" % self._wrapped_class.__class__.__name__, id)
+
+class ORMLocalObjectManager(object):
+ """ Manages a local list of objects """
+
+ def __init__(self, stub, modelName, idList):
+ self._stub = stub
+ self._modelName = modelName
+ self._idList = idList
+ self._cache = None
+
+ def resolve_queryset(self):
+ if self._cache is not None:
+ return self._cache
+
+ models = []
+ for id in self._idList:
+ models.append(self._stub.invoke("Get%s" % self._modelName, self._stub.make_ID(id=id)))
+
+ self._cache = models
+
+ return models
+
+ def all(self):
+ models = self.resolve_queryset()
+ return [ORMWrapper(x,self._stub) for x in models]
+
+class ORMObjectManager(object):
+ """ Manages a remote list of objects """
+
+ def __init__(self, stub, modelName, packageName):
+ self._stub = stub
+ self._modelName = modelName
+ self._packageName = packageName
+
+ def wrap_single(self, obj):
+ return ORMWrapper(obj, self._stub)
+
+ def wrap_list(self, obj):
+ result=[]
+ for item in obj.items:
+ result.append(ORMWrapper(item, self._stub))
+ return result
+
+ def all(self):
+ return self.wrap_list(self._stub.invoke("List%s" % self._modelName, Empty()))
+
+ def get(self, id):
+ return self.wrap_single(self._stub.invoke("Get%s" % self._modelName, self._stub.make_ID(id=id)))
+
+ def new(self, **kwargs):
+ full_model_name = "%s.%s" % (self._packageName, self._modelName)
+ cls = _sym_db._classes[full_model_name]
+ return ORMWrapper(cls(), self._stub, is_new=True)
+
+class ORMModelClass(object):
+ def __init__(self, stub, model_name, package_name):
+ self.objects = ORMObjectManager(stub, model_name, package_name)
+
+class ORMStub(object):
+ def __init__(self, stub, package_name):
+ self.grpc_stub = stub
+ self.all_model_names = []
+
+ for name in dir(stub):
+ if name.startswith("Get"):
+ model_name = name[3:]
+ setattr(self,model_name, ORMModelClass(self, model_name, package_name))
+
+ self.all_model_names.append(model_name)
+
+ def listObjects(self):
+ return self.all_model_names
+
+ def invoke(self, name, request):
+ method = getattr(self.grpc_stub, name)
+ return method(request)
+
+ def make_ID(self, id):
+ return _sym_db._classes["xos.ID"](id=id)
+
+
+#def wrap_get(*args, **kwargs):
+# stub=kwargs.pop("stub")
+# getmethod=kwargs.pop("getmethod")
+# result = getmethod(*args, **kwargs)
+# return ORMWrapper(result)
+#
+#def wrap_stub(stub):
+# for name in dir(stub):
+# if name.startswith("Get"):
+# setattr(stub, name, functools.partial(wrap_get, stub=stub, getmethod=getattr(stub,name)))
+
diff --git a/xos/xos_client/xosapi/xos_grpc_client.py b/xos/xos_client/xosapi/xos_grpc_client.py
new file mode 100644
index 0000000..2b32914
--- /dev/null
+++ b/xos/xos_client/xosapi/xos_grpc_client.py
@@ -0,0 +1,132 @@
+import base64
+import functools
+import grpc
+import orm
+import os
+import pdb
+import sys
+from google.protobuf.empty_pb2 import Empty
+from grpc import metadata_call_credentials, ChannelCredentials, composite_channel_credentials, ssl_channel_credentials
+
+# fix up sys.path for chameleon
+import inspect
+currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
+sys.path = [currentdir] + sys.path
+
+import chameleon.grpc_client.grpc_client as chameleon_client
+
+from twisted.internet import reactor
+
+
+SERVER_CA="/usr/local/share/ca-certificates/local_certs.crt"
+
+class UsernamePasswordCallCredentials(grpc.AuthMetadataPlugin):
+ """Metadata wrapper for raw access token credentials."""
+ def __init__(self, username, password):
+ self._username = username
+ self._password = password
+ def __call__(self, context, callback):
+ basic_auth = "Basic %s" % base64.b64encode("%s:%s" % (self._username, self._password))
+ metadata = (('Authorization', basic_auth),)
+ callback(metadata, None)
+
+class SessionIdCallCredentials(grpc.AuthMetadataPlugin):
+ """Metadata wrapper for raw access token credentials."""
+ def __init__(self, sessionid):
+ self._sessionid = sessionid
+ def __call__(self, context, callback):
+ metadata = (('x-xossession', self._sessionid),)
+ callback(metadata, None)
+
+class XOSClient(chameleon_client.GrpcClient):
+ # We layer our own reconnect_callback functionality so we can setup the
+ # ORM before calling reconnect_callback.
+
+ def set_reconnect_callback(self, reconnect_callback):
+ self.reconnect_callback2 = reconnect_callback
+ return self
+
+ def reconnected(self):
+ for api in ['modeldefs', 'utility', 'xos']:
+ pb2_file_name = os.path.join(self.work_dir, api + "_pb2.py")
+ pb2_grpc_file_name = os.path.join(self.work_dir, api + "_pb2_grpc.py")
+
+ if os.path.exists(pb2_file_name) and os.path.exists(pb2_grpc_file_name):
+ orig_sys_path = sys.path
+ try:
+ sys.path.append(self.work_dir)
+ m_protos = __import__(api + "_pb2")
+ m_grpc = __import__(api + "_pb2_grpc")
+ finally:
+ sys.path = orig_sys_path
+
+ stub_class = getattr(m_grpc, api+"Stub")
+
+ setattr(self, api, stub_class(self.channel))
+ else:
+ print >> sys.stderr, "failed to locate api", api
+
+ if hasattr(self, "xos"):
+ self.xos_orm = orm.ORMStub(self.xos, "xos")
+
+ if self.reconnect_callback2:
+ self.reconnect_callback2()
+
+
+class InsecureClient(XOSClient):
+ def __init__(self, consul_endpoint=None, work_dir="/tmp/xos_grpc_protos", endpoint='localhost:50055', reconnect_callback=None):
+ super(InsecureClient,self).__init__(consul_endpoint, work_dir, endpoint, self.reconnected)
+
+ self.reconnect_callback2 = reconnect_callback
+
+class SecureClient(XOSClient):
+ def __init__(self, consul_endpoint=None, work_dir="/tmp/xos_grpc_protos", endpoint='localhost:50055', reconnect_callback=None, cacert=SERVER_CA, username=None, password=None, sessionid=None):
+ server_ca = open(cacert,"r").read()
+ if (sessionid):
+ call_creds = metadata_call_credentials(SessionIdCallCredentials(sessionid))
+ else:
+ call_creds = metadata_call_credentials(UsernamePasswordCallCredentials(username, password))
+ chan_creds = ssl_channel_credentials(server_ca)
+ chan_creds = composite_channel_credentials(chan_creds, call_creds)
+
+ super(SecureClient,self).__init__(consul_endpoint, work_dir, endpoint, self.reconnected, chan_creds)
+
+ self.reconnect_callback2 = reconnect_callback
+
+# -----------------------------------------------------------------------------
+# Self test
+# -----------------------------------------------------------------------------
+
+def insecure_callback(client):
+ print "insecure self_test start"
+ print client.xos_orm.User.objects.all()
+ print "insecure self_test done"
+
+ # now start the next test
+ client.stop()
+ reactor.callLater(0, start_secure_test)
+
+def start_insecure_test():
+ client = InsecureClient(endpoint="xos-core.cord.lab:50055")
+ client.set_reconnect_callback(functools.partial(insecure_callback, client))
+ client.start()
+
+def secure_callback(client):
+ print "secure self_test start"
+ print client.xos_orm.User.objects.all()
+ print "secure self_test done"
+ reactor.stop()
+
+def start_secure_test():
+ client = SecureClient(endpoint="xos-core.cord.lab:50051", username="xosadmin@opencord.org", password="BQSPdpRsR0MrrZ9u7SPe")
+ client.set_reconnect_callback(functools.partial(secure_callback, client))
+ client.start()
+
+def main():
+ reactor.callLater(0, start_insecure_test)
+
+ reactor.run()
+
+if __name__=="__main__":
+ main()
+
diff --git a/xos/xos_client/xossh b/xos/xos_client/xossh
new file mode 100755
index 0000000..4fc5998
--- /dev/null
+++ b/xos/xos_client/xossh
@@ -0,0 +1,92 @@
+#!/usr/bin/python
+#
+# XOSAPI interactive shell
+
+
+import functools
+import os, sys
+import atexit
+import readline
+import rlcompleter
+
+from twisted.internet import reactor
+from xosapi.xos_grpc_client import InsecureClient
+
+coreapi_endpoint = "xos-core.cord.lab:50055"
+
+def start_xossh(client):
+ global coreapi
+ coreapi = client.xos_orm
+
+ print "XOS Core server at %s" % client.endpoint
+
+ print 'Type "coreapi.listObjects()" for a list of all objects'
+
+ # Load command history
+ history_path = os.path.join(os.environ["HOME"], ".xossh_history")
+ try:
+ file(history_path, 'a').close()
+ readline.read_history_file(history_path)
+ atexit.register(readline.write_history_file, history_path)
+ except IOError:
+ pass
+
+ # Enable tab completion
+ readline.parse_and_bind("tab: complete")
+
+ prompt = "xossh "
+
+ try:
+ while True:
+ command = ""
+ while True:
+ # Get line
+ try:
+ if command == "":
+ sep = ">>> "
+ else:
+ sep = "... "
+ line = raw_input(prompt + sep)
+ # Ctrl-C
+ except KeyboardInterrupt:
+ command = ""
+ print
+ break
+
+ # Build up multi-line command
+ command += line
+
+ # Blank line or first line does not end in :
+ if line == "" or (command == line and line[-1] != ':'):
+ break
+
+ command += os.linesep
+
+ # Blank line
+ if command == "":
+ continue
+ # Quit
+ elif command in ["q", "quit", "exit"]:
+ break
+
+ try:
+ # Do it
+ code = compile(command, "<stdin>", "single")
+ exec code
+ except Exception, err:
+ print err
+
+ except EOFError:
+ print
+ pass
+
+ reactor.stop()
+
+def main():
+ client = InsecureClient(endpoint=coreapi_endpoint)
+ client.set_reconnect_callback(functools.partial(start_xossh, client))
+ client.start()
+ reactor.run()
+
+if __name__ == "__main__":
+ main()