CORD-1256 Implement LoadModels
Change-Id: I4546f32bd7272e219f887782d30780e702e10412
diff --git a/lib/xos-config/xosconfig/synchronizer-config-schema.yaml b/lib/xos-config/xosconfig/synchronizer-config-schema.yaml
index bafc8e5..dd25888 100644
--- a/lib/xos-config/xosconfig/synchronizer-config-schema.yaml
+++ b/lib/xos-config/xosconfig/synchronizer-config-schema.yaml
@@ -59,6 +59,8 @@
type: str
sys_dir:
type: str
+ models_dir:
+ type: str
accessor:
type: map
required: True
diff --git a/unittest.cfg b/unittest.cfg
index 7ca4d6b..6050396 100644
--- a/unittest.cfg
+++ b/unittest.cfg
@@ -2,4 +2,4 @@
plugins=nose2-plugins.exclude
code-directories=xos-genx
xos-config
-
+ coreapi
diff --git a/xos/coreapi/Makefile b/xos/coreapi/Makefile
new file mode 100644
index 0000000..9c20c37
--- /dev/null
+++ b/xos/coreapi/Makefile
@@ -0,0 +1,40 @@
+#
+# Copyright 2017 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+all: prep start
+
+start:
+ bash -c "source env.sh && python ./core_main.py"
+
+prep: app_lists rebuild_protos compile_protos try_models makemigrations migrate
+
+app_lists:
+ python app_list_builder.py
+
+makemigrations:
+ bash /opt/xos/tools/xos-manage makemigrations
+
+migrate:
+ python /opt/xos/manage.py migrate
+
+rebuild_protos:
+ cd protos && make rebuild-protos
+
+compile_protos:
+ cd protos && make
+
+try_models:
+ python ./try_models.py
diff --git a/xos/coreapi/app_list_builder.py b/xos/coreapi/app_list_builder.py
new file mode 100644
index 0000000..7bdec45
--- /dev/null
+++ b/xos/coreapi/app_list_builder.py
@@ -0,0 +1,50 @@
+
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+def makedirs_if_noexist(pathname):
+ if not os.path.exists(pathname):
+ os.makedirs(pathname)
+
+class AppListBuilder(object):
+ def __init__(self):
+ self.app_metadata_dir = "/opt/xos/xos"
+ self.services_dest_dir = "/opt/xos/services"
+
+ def generate_app_lists(self):
+ # TODO: Once static onboarding is no more, we will get these from the manifests rather than using listdir
+ app_names = []
+ for fn in os.listdir(self.services_dest_dir):
+ service_dir = os.path.join(self.services_dest_dir, fn)
+ if (not fn.startswith(".")) and os.path.isdir(service_dir):
+ models_fn = os.path.join(service_dir, "models.py")
+ if os.path.exists(models_fn):
+ if fn == "syndicate_storage":
+ continue
+ app_names.append(fn)
+
+ # Generate the migration list
+ mig_list_fn = os.path.join(self.app_metadata_dir, "xosbuilder_migration_list")
+ makedirs_if_noexist(os.path.dirname(mig_list_fn))
+ file(mig_list_fn, "w").write("\n".join(app_names) + "\n")
+
+ # Generate the app list
+ app_list_fn = os.path.join(self.app_metadata_dir, "xosbuilder_app_list")
+ makedirs_if_noexist(os.path.dirname(app_list_fn))
+ file(app_list_fn, "w").write("\n".join(["services.%s" % x for x in app_names]) + "\n")
+
+if __name__ == "__main__":
+ AppListBuilder().generate_app_lists()
diff --git a/xos/coreapi/core_main.py b/xos/coreapi/core_main.py
index 7318159..8c3a3fe 100644
--- a/xos/coreapi/core_main.py
+++ b/xos/coreapi/core_main.py
@@ -18,38 +18,41 @@
import sys
import time
-import django
-xos_path = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + '/..')
-sys.path.append(xos_path)
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "xos.settings")
-
-from reaper import ReaperThread
-from grpc_server import XOSGrpcServer, restart_chameleon
+from grpc_server import XOSGrpcServer, restart_related_containers
from xosconfig import Config
from multistructlog import create_logger
log = create_logger(Config().get('logging'))
+def init_reaper():
+ reaper = None
+ try:
+ from reaper import ReaperThread
+ reaper = ReaperThread()
+ reaper.start()
+ except:
+ logger.log_exception("Failed to initialize reaper")
+
+ return reaper
+
if __name__ == '__main__':
+ server = XOSGrpcServer()
+ server.init_django()
+ server.start()
- django.setup()
+ reaper = init_reaper()
- reaper = ReaperThread()
- reaper.start()
-
- server = XOSGrpcServer().start()
-
- restart_chameleon()
+ restart_related_containers()
log.info("XOS core entering wait loop")
-
_ONE_DAY_IN_SECONDS = 60 * 60 * 24
try:
- while 1:
- time.sleep(_ONE_DAY_IN_SECONDS)
+ while True:
+ if server.exit_event.wait(_ONE_DAY_IN_SECONDS):
+ break
except KeyboardInterrupt:
log.info("XOS core terminated by keyboard interrupt")
- server.stop()
- reaper.stop()
+ server.stop()
+ reaper.stop()
diff --git a/xos/coreapi/dynamicbuild.py b/xos/coreapi/dynamicbuild.py
new file mode 100644
index 0000000..e536e39
--- /dev/null
+++ b/xos/coreapi/dynamicbuild.py
@@ -0,0 +1,212 @@
+
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import hashlib
+import os
+import shutil
+from xosgenx.generator import XOSGenerator
+
+from xosconfig import Config
+from multistructlog import create_logger
+log = create_logger(Config().get('logging'))
+
+DEFAULT_BASE_DIR="/opt/xos"
+
+class DynamicBuilder(object):
+ NOTHING_TO_DO = 0
+ SOMETHING_CHANGED = 1
+
+ def __init__(self, base_dir=DEFAULT_BASE_DIR):
+ self.services_dir = os.path.join(base_dir, "dynamic_services")
+ self.manifest_dir = os.path.join(base_dir, "dynamic_services/manifests")
+ self.services_dest_dir = os.path.join(base_dir, "services")
+ self.coreapi_dir = os.path.join(base_dir, "coreapi")
+ self.protos_dir = os.path.join(base_dir, "coreapi/protos")
+ self.app_metadata_dir = os.path.join(base_dir, "xos")
+
+ def pre_validate_file(self, item):
+ # someone might be trying to trick us into writing files outside the designated directory
+ if "/" in item.filename:
+ raise Exception("illegal character in filename %s" % item.filename)
+
+ def pre_validate_models(self, request):
+ # do whatever validation we can before saving the files
+ for item in request.xprotos:
+ self.pre_validate_file(item)
+
+ for item in request.decls:
+ self.pre_validate_file(item)
+
+ for item in request.attics:
+ self.pre_validate_file(item)
+
+ def handle_loadmodels_request(self, request):
+ manifest_fn = os.path.join(self.manifest_dir, request.name + ".json")
+ if os.path.exists(manifest_fn):
+ try:
+ manifest = json.loads(open(manifest_fn).read())
+ except:
+ log.exception("Error loading old manifest", filename=manifest_fn)
+ manifest = {}
+ else:
+ manifest = {}
+
+ # TODO: Check version number to make sure this is not a downgrade ?
+
+ hash = self.generate_request_hash(request)
+ if hash == manifest.get("hash"):
+ # The hash of the incoming request is identical to the manifest that we have saved, so this request is a
+ # no-op.
+ log.info("Models are already up-to-date; skipping dynamic load.", name=request.name)
+ return self.NOTHING_TO_DO
+
+ self.pre_validate_models(request)
+
+ manifest = self.save_models(request, hash=hash)
+
+ self.run_xosgenx_service(manifest)
+
+ log.debug("Saving service manifest", name=request.name)
+ file(manifest_fn, "w").write(json.dumps(manifest))
+
+ log.info("Finished LoadModels request", name=request.name)
+
+ return self.SOMETHING_CHANGED
+
+ # TODO: schedule a restart
+
+ def generate_request_hash(self, request):
+ # TODO: could we hash the request rather than individually hashing the subcomponents of the request?
+ m = hashlib.sha1()
+ m.update(request.name)
+ m.update(request.version)
+ for item in request.xprotos:
+ m.update(item.filename)
+ m.update(item.contents)
+ for item in request.decls:
+ m.update(item.filename)
+ m.update(item.contents)
+ for item in request.decls:
+ m.update(item.filename)
+ m.update(item.contents)
+ return m.hexdigest()
+
+ def save_models(self, request, hash=None):
+ if not hash:
+ hash = self.generate_request_hash(request)
+
+ service_dir = os.path.join(self.services_dir, request.name)
+ if not os.path.exists(service_dir):
+ os.makedirs(service_dir)
+
+ if not os.path.exists(self.manifest_dir):
+ os.makedirs(self.manifest_dir)
+
+ manifest_fn = os.path.join(self.manifest_dir, request.name + ".json")
+
+ # Invariant is that if a manifest file exists, then it accurately reflects that has been stored to disk. Since
+ # we're about to potentially overwrite files, destroy the old manifest.
+ if os.path.exists(manifest_fn):
+ os.remove(manifest_fn)
+
+ # convert the request to a manifest, so we can save it
+ service_manifest = {"name": request.name,
+ "version": request.version,
+ "hash": hash,
+ "dir": service_dir,
+ "manifest_fn": manifest_fn,
+ "dest_dir": os.path.join(self.services_dest_dir, request.name),
+ "xprotos": [],
+ "decls": [],
+ "attics": []}
+
+ for item in request.xprotos:
+ file(os.path.join(service_dir, item.filename), "w").write(item.contents)
+ service_manifest["xprotos"].append({"filename": item.filename})
+
+ for item in request.decls:
+ file(os.path.join(service_dir, item.filename), "w").write(item.contents)
+ service_manifest["decls"].append({"filename": item.filename})
+
+ if request.attics:
+ attic_dir = os.path.join(service_dir, "attic")
+ service_manifest["attic_dir"] = attic_dir
+ if not os.path.exists(attic_dir):
+ os.makedirs(attic_dir)
+ for item in request.attics:
+ file(os.path.join(attic_dir, item.filename), "w").write(item.contents)
+ service_manifest["attics"].append({"filename": item.filename})
+
+ return service_manifest
+
+ def run_xosgenx_service(self, manifest):
+ if not os.path.exists(manifest["dest_dir"]):
+ os.makedirs(manifest["dest_dir"])
+
+ xproto_filenames = [os.path.join(manifest["dir"], x["filename"]) for x in manifest["xprotos"]]
+
+ class Args:
+ pass
+
+ # Generate models
+ is_service = manifest["name"] != 'core'
+
+ args = Args()
+ args.output = manifest["dest_dir"]
+ args.attic = os.path.join(manifest["dir"], 'attic')
+ args.files = xproto_filenames
+
+ if is_service:
+ args.target = 'service.xtarget'
+ args.write_to_file = 'target'
+ else:
+ args.target = 'django.xtarget'
+ args.dest_extension = 'py'
+ args.write_to_file = 'model'
+
+ XOSGenerator.generate(args)
+
+ # Generate security checks
+ class SecurityArgs:
+ output = manifest["dest_dir"]
+ target = 'django-security.xtarget'
+ dest_file = 'security.py'
+ write_to_file = 'single'
+ files = xproto_filenames
+
+ XOSGenerator.generate(SecurityArgs())
+
+ # Generate __init__.py
+ if manifest["name"] == "core":
+ class InitArgs:
+ output = manifest["dest_dir"]
+ target = 'init.xtarget'
+ dest_file = '__init__.py'
+ write_to_file = 'single'
+ files = xproto_filenames
+
+ XOSGenerator.generate(InitArgs())
+
+ else:
+ init_py_filename = os.path.join(manifest["dest_dir"], "__init__.py")
+ if not os.path.exists(init_py_filename):
+ open(init_py_filename, "w").write("# created by dynamicbuild")
+
+ # the xosgenx templates don't handle copying the models.py file for us, so do it here.
+ for item in manifest["decls"]:
+ src_fn = os.path.join(manifest["dir"], item["filename"])
+ dest_fn = os.path.join(manifest["dest_dir"], item["filename"])
+ shutil.copyfile(src_fn, dest_fn)
diff --git a/xos/coreapi/grpc_server.py b/xos/coreapi/grpc_server.py
index 8edf87f..56811b3 100644
--- a/xos/coreapi/grpc_server.py
+++ b/xos/coreapi/grpc_server.py
@@ -13,26 +13,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
-#
-# Copyright 2017 the original author or authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
"""gRPC server endpoint"""
import os
import sys
+import threading
import uuid
from collections import OrderedDict
from os.path import abspath, basename, dirname, join, walk
@@ -40,33 +24,27 @@
from concurrent import futures
import zlib
+xos_path = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + '/..')
+sys.path.append(xos_path)
+
from xosconfig import Config
from multistructlog import create_logger
+Config.init()
log = create_logger(Config().get('logging'))
-if __name__ == "__main__":
- import django
- sys.path.append('/opt/xos')
- os.environ.setdefault("DJANGO_SETTINGS_MODULE", "xos.settings")
-
-from protos import xos_pb2, schema_pb2, modeldefs_pb2, utility_pb2
-from xos_grpc_api import XosService
-from xos_modeldefs_api import ModelDefsService
-from xos_utility_api import UtilityService
+from protos import schema_pb2, dynamicload_pb2
+#from xos_modeldefs_api import ModelDefsService
+#from xos_utility_api import UtilityService
+from xos_dynamicload_api import DynamicLoadService
+from dynamicbuild import DynamicBuilder
from google.protobuf.empty_pb2 import Empty
-
-
SERVER_KEY="/opt/cord_profile/core_api_key.pem"
SERVER_CERT="/opt/cord_profile/core_api_cert.pem"
SERVER_CA="/usr/local/share/ca-certificates/local_certs.crt"
-#SERVER_KEY="certs/server.key"
-#SERVER_CERT="certs/server.crt"
-#SERVER_CA="certs/ca.crt"
-
class SchemaService(schema_pb2.SchemaServiceServicer):
def __init__(self, thread_pool):
@@ -130,20 +108,48 @@
self.credentials = grpc.ssl_server_credentials([(server_key, server_cert)], server_ca, False)
+ self.delayed_shutdown_timer = None
+ self.exit_event = threading.Event()
+
self.services = []
+ def init_django(self):
+ import django
+ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "xos.settings")
+ django.setup()
+
+ def register_core(self):
+ from xos_grpc_api import XosService
+ from protos import xos_pb2
+
+ self.register("xos", xos_pb2.add_xosServicer_to_server, XosService(self.thread_pool))
+
+ def register_utility(self):
+ from xos_utility_api import UtilityService
+ from protos import utility_pb2
+
+ self.register("utility", utility_pb2.add_utilityServicer_to_server, UtilityService(self.thread_pool))
+
+ def register_modeldefs(self):
+ from xos_modeldefs_api import ModelDefsService
+ from protos import modeldefs_pb2
+
+ self.register("modeldefs", modeldefs_pb2.add_modeldefsServicer_to_server, ModelDefsService(self.thread_pool))
+
def start(self):
log.info('Starting GRPC Server')
- # add each service unit to the server and also to the list
- for activator_func, service_class in (
- (schema_pb2.add_SchemaServiceServicer_to_server, SchemaService),
- (xos_pb2.add_xosServicer_to_server, XosService),
- (modeldefs_pb2.add_modeldefsServicer_to_server, ModelDefsService),
- (utility_pb2.add_utilityServicer_to_server, UtilityService),
- ):
- service = service_class(self.thread_pool)
- self.register(activator_func, service)
+ self.register("schema",
+ schema_pb2.add_SchemaServiceServicer_to_server,
+ SchemaService(self.thread_pool))
+
+ self.register("dynamicload",
+ dynamicload_pb2.add_dynamicloadServicer_to_server,
+ DynamicLoadService(self.thread_pool, self))
+
+ self.register_core()
+ self.register_utility()
+ self.register_modeldefs()
# open port
self.server.add_insecure_port('[::]:%s' % self.port)
@@ -163,7 +169,11 @@
self.server.stop(grace)
log.info('stopped')
- def register(self, activator_func, service):
+ def stop_and_exit(self):
+ log.info("Stop and Exit")
+ self.exit_event.set()
+
+ def register(self, name, activator_func, service):
"""
Allow late registration of gRPC servicers
:param activator_func: The gRPC "add_XYZServicer_to_server method
@@ -174,8 +184,14 @@
self.services.append(service)
activator_func(service, self.server)
+ def delayed_shutdown(self, seconds):
+ log.info("Delayed shutdown", seconds=seconds)
+ if self.delayed_shutdown_timer:
+ self.delayed_shutdown_timer.cancel()
+ self.delayed_shutdown_timer = threading.Timer(seconds, self.stop_and_exit)
+ self.delayed_shutdown_timer.start()
-def restart_chameleon():
+def restart_docker_container(name):
import docker
def find_container(client, search_name):
@@ -186,31 +202,39 @@
return None
client=docker.from_env()
- chameleon_container = find_container(client, "xos_chameleon_1")
- if chameleon_container:
+ container = find_container(client, name)
+ if container:
try:
# the first attempt always fails with 404 error
# docker-py bug?
- client.restart(chameleon_container["Names"][0])
+ client.restart(container["Names"][0])
except:
- client.restart(chameleon_container["Names"][0])
+ client.restart(container["Names"][0])
+
+def restart_related_containers():
+ restart_docker_container("xos_chameleon_1")
+ # TODO: remove once Tosca container is able to react internally
+ restart_docker_container("xos_tosca_1")
# This is to allow running the GRPC server in stand-alone mode
if __name__ == '__main__':
- django.setup()
+ server = XOSGrpcServer()
+ server.init_django()
+ server.start()
- server = XOSGrpcServer().start()
+ restart_related_containers()
- restart_chameleon()
-
- import time
+ log.info("XOS core entering wait loop")
_ONE_DAY_IN_SECONDS = 60 * 60 * 24
try:
- while 1:
- time.sleep(_ONE_DAY_IN_SECONDS)
+ while True:
+ if server.exit_event.wait(_ONE_DAY_IN_SECONDS):
+ break
except KeyboardInterrupt:
- server.stop()
+ log.info("XOS core terminated by keyboard interrupt")
+
+ server.stop()
diff --git a/xos/coreapi/protos/Makefile b/xos/coreapi/protos/Makefile
index cfbaa52..ea86b39 100644
--- a/xos/coreapi/protos/Makefile
+++ b/xos/coreapi/protos/Makefile
@@ -39,7 +39,7 @@
PROTOC_DOWNLOAD_URI := $(PROTOC_DOWNLOAD_PREFIX)/v$(PROTOC_VERSION)/$(PROTOC_TARBALL)
PROTOC_BUILD_TMP_DIR := "/tmp/protobuf-build-$(shell uname -s | tr '[:upper:]' '[:lower:]')"
-XPROTO_FILES := $(wildcard $(XOS_DIR)/core/models/core.xproto $(XOS_DIR)/services/*/*.xproto)
+XPROTO_FILES := $(wildcard $(XOS_DIR)/core/models/core.xproto $(XOS_DIR)/services/*/*.xproto $(XOS_DIR)/dynamic_services/*/*.xproto)
XOSGEN_PATH := $(XOS_DIR)/lib/xos-genx/xosgenx
build: $(PROTOC) $(PROTO_PB2_FILES)
diff --git a/xos/coreapi/protos/dynamicload.proto b/xos/coreapi/protos/dynamicload.proto
new file mode 100644
index 0000000..d4142a1
--- /dev/null
+++ b/xos/coreapi/protos/dynamicload.proto
@@ -0,0 +1,47 @@
+syntax = "proto3";
+
+package xos;
+
+// import "google/protobuf/empty.proto";
+import "google/api/annotations.proto";
+// import "common.proto";
+
+message Xproto {
+ string filename = 1;
+ string contents = 2;
+}
+
+message DeclFile {
+ string filename = 1;
+ string contents = 2;
+};
+
+message AtticFile {
+ string filename = 1;
+ string contents = 2;
+};
+
+message LoadModelsRequest {
+ string name = 1;
+ string version = 2;
+ repeated Xproto xprotos = 3;
+ repeated DeclFile decls = 4;
+ repeated AtticFile attics = 5;
+};
+
+message LoadModelsReply {
+ enum LoadModelsStatus {
+ SUCCESS = 0;
+ ERROR = 1;
+ }
+ LoadModelsStatus status = 1;
+};
+
+service dynamicload {
+ rpc LoadModels(LoadModelsRequest) returns (LoadModelsReply) {
+ option (google.api.http) = {
+ post: "/xosapi/v1/dynamicload/load_models"
+ body: "*"
+ };
+ }
+};
\ No newline at end of file
diff --git a/xos/coreapi/start_coreapi.sh b/xos/coreapi/start_coreapi.sh
index 5b8a78f..9a2b7c7 100755
--- a/xos/coreapi/start_coreapi.sh
+++ b/xos/coreapi/start_coreapi.sh
@@ -14,17 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
-# seed initial data in the db
-bash /opt/xos/tools/xos-manage makemigrations
-python /opt/xos/manage.py migrate;
-
-# build protobuf
-cd protos
-make rebuild-protos
-make
-
-# start the grpc server
-cd ..
-source env.sh
-python ./core_main.py
+while true; do
+ make
+ sleep 1
+done
diff --git a/xos/coreapi/test_config.yaml b/xos/coreapi/test_config.yaml
new file mode 100644
index 0000000..ae389f4
--- /dev/null
+++ b/xos/coreapi/test_config.yaml
@@ -0,0 +1,26 @@
+
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+name: test-coreapi
+logging:
+ version: 1
+ handlers:
+ console:
+ class: logging.StreamHandler
+ loggers:
+ 'multistructlog':
+ handlers:
+ - console
diff --git a/xos/coreapi/test_dynamicbuild.py b/xos/coreapi/test_dynamicbuild.py
new file mode 100644
index 0000000..35359ac
--- /dev/null
+++ b/xos/coreapi/test_dynamicbuild.py
@@ -0,0 +1,145 @@
+
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import shutil
+import sys
+import tempfile
+import unittest
+from mock import patch
+
+from xosconfig import Config
+
+class DynamicLoadItem():
+ def __init__(self, **kwargs):
+ for (k,v) in kwargs.items():
+ setattr(self, k, v)
+
+class DynamicLoadRequest():
+ def __init__(self, **kwargs):
+ self.xprotos = []
+ self.decls = []
+ self.attics = []
+ for (k,v) in kwargs.items():
+ setattr(self, k, v)
+
+class TestDynamicBuild(unittest.TestCase):
+ def setUp(self):
+ global dynamicbuild
+
+ config = basic_conf = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/test_config.yaml")
+ Config.clear() # in case left unclean by a previous test case
+ Config.init(config)
+
+ import dynamicbuild
+
+ self.base_dir = tempfile.mkdtemp()
+ self.example_xproto = """option app_label = "exampleservice";
+option name = "exampleservice";
+
+message ExampleService (Service){
+ option verbose_name = "Example Service";
+ required string service_message = 1 [help_text = "Service Message to Display", max_length = 254, null = False, db_index = False, blank = False];
+}
+
+message Color (XOSBase){
+ option verbose_name = "Color";
+ required string name = 1 [help_text = "Name for this color", db_index = False, max_length = 256, null = False, blank = False];
+ required string html_code = 2 [help_text = "Code for this color", db_index = False, max_length = 256, null = False, blank = False];
+}
+
+message ExampleServiceInstance (TenantWithContainer){
+ option verbose_name = "Example Service Instance";
+ required string tenant_message = 1 [help_text = "Tenant Message to Display", max_length = 254, null = False, db_index = False, blank = False];
+ optional manytoone foreground_color->Color:serviceinstance_foreground_colors = 3 [db_index = True, null = True, blank = True];
+ optional manytoone background_color->Color:serviceinstance_background_colors = 3 [db_index = True, null = True, blank = True];
+}
+
+message EmbeddedImage (XOSBase){
+ option verbose_name = "Embedded Image";
+ required string name = 1 [help_text = "Name for this image", db_index = False, max_length = 256, null = False, blank = False];
+ required string url = 2 [help_text = "URL for this image", db_index = False, max_length = 256, null = False, blank = False];
+ optional manytoone serviceinstance->ExampleServiceInstance:embedded_images = 3 [db_index = True, null = True, blank = True];
+}
+ """
+
+ self.example_xproto_item = DynamicLoadItem(filename = "exampleservice.xproto",
+ contents = self.example_xproto)
+
+ self.example_request = DynamicLoadRequest(name = "exampleservice",
+ version = "1",
+ xprotos = [self.example_xproto_item])
+
+ self.builder = dynamicbuild.DynamicBuilder(base_dir = self.base_dir)
+
+ def tearDown(self):
+ if os.path.abspath(self.base_dir).startswith("/tmp"): # be paranoid about recursive deletes
+ shutil.rmtree(self.base_dir)
+
+ def test_pre_validate_file(self):
+ self.builder.pre_validate_file(self.example_xproto_item)
+
+ def test_pre_validate_models(self):
+ self.builder.pre_validate_models(self.example_request)
+
+ def test_generate_request_hash(self):
+ hash = self.builder.generate_request_hash(self.example_request)
+ self.assertEqual(hash, "162de5012a8399883344085cbc232a2e627c5091")
+
+ def test_handle_loadmodels_request(self):
+ with patch.object(dynamicbuild.DynamicBuilder, "save_models", wraps=self.builder.save_models) as save_models, \
+ patch.object(dynamicbuild.DynamicBuilder, "run_xosgenx_service", wraps=self.builder.run_xosgenx_service) as run_xosgenx_service:
+ result = self.builder.handle_loadmodels_request(self.example_request)
+
+ save_models.assert_called()
+ run_xosgenx_service.assert_called()
+
+ self.assertEqual(result, self.builder.SOMETHING_CHANGED)
+
+ self.assertTrue(os.path.exists(self.builder.manifest_dir))
+ self.assertTrue(os.path.exists(os.path.join(self.builder.manifest_dir, "exampleservice.json")))
+
+ service_dir = os.path.join(self.base_dir, "services", "exampleservice")
+
+ self.assertTrue(os.path.exists(service_dir))
+ self.assertTrue(os.path.exists(os.path.join(service_dir, "__init__.py")))
+ self.assertTrue(os.path.exists(os.path.join(service_dir, "models.py")))
+ self.assertTrue(os.path.exists(os.path.join(service_dir, "security.py")))
+
+ def test_handle_loadmodels_request_twice(self):
+ result = self.builder.handle_loadmodels_request(self.example_request)
+ self.assertEqual(result, self.builder.SOMETHING_CHANGED)
+
+ result = self.builder.handle_loadmodels_request(self.example_request)
+ self.assertEqual(result, self.builder.NOTHING_TO_DO)
+
+ def test_save_models(self):
+ manifest = self.builder.save_models(self.example_request)
+
+ dynamic_dir = os.path.join(self.base_dir, "dynamic_services", "exampleservice")
+ service_dir = os.path.join(self.base_dir, "services", "exampleservice")
+
+ self.assertEqual(manifest["name"], self.example_request.name)
+ self.assertEqual(manifest["version"], self.example_request.version)
+ self.assertEqual(manifest["hash"], "162de5012a8399883344085cbc232a2e627c5091")
+ self.assertEqual(manifest["dir"], dynamic_dir)
+ self.assertEqual(manifest["dest_dir"], service_dir)
+ self.assertEqual(len(manifest["xprotos"]), 1)
+
+def main():
+ unittest.main()
+
+if __name__ == "__main__":
+ main()
diff --git a/xos/coreapi/try_models.py b/xos/coreapi/try_models.py
new file mode 100644
index 0000000..c8675cd
--- /dev/null
+++ b/xos/coreapi/try_models.py
@@ -0,0 +1,25 @@
+
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+
+import django
+
+xos_path = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + '/..')
+sys.path.append(xos_path)
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "xos.settings")
+django.setup()
diff --git a/xos/coreapi/xos_dynamicload_api.py b/xos/coreapi/xos_dynamicload_api.py
new file mode 100644
index 0000000..f7d79ed
--- /dev/null
+++ b/xos/coreapi/xos_dynamicload_api.py
@@ -0,0 +1,57 @@
+
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import base64
+import fnmatch
+import os
+import sys
+import time
+import traceback
+from protos import dynamicload_pb2
+from google.protobuf.empty_pb2 import Empty
+
+from importlib import import_module
+
+from dynamicbuild import DynamicBuilder
+
+class DynamicLoadService(dynamicload_pb2.dynamicloadServicer):
+ def __init__(self, thread_pool, server):
+ self.thread_pool = thread_pool
+ self.server = server
+
+ def stop(self):
+ pass
+
+ #@translate_exceptions
+ def LoadModels(self, request, context):
+ print "load models"
+
+ try:
+ builder = DynamicBuilder()
+ result = builder.handle_loadmodels_request(request)
+
+ if (result == builder.SOMETHING_CHANGED):
+ self.server.delayed_shutdown(5)
+
+ response = dynamicload_pb2.LoadModelsReply()
+ response.status = response.SUCCESS
+
+ return response
+ except Exception, e:
+ import traceback; traceback.print_exc()
+ raise e
+
+
diff --git a/xos/coreapi/xos_utility_api.py b/xos/coreapi/xos_utility_api.py
index bd07fea..6f6ad11 100644
--- a/xos/coreapi/xos_utility_api.py
+++ b/xos/coreapi/xos_utility_api.py
@@ -216,7 +216,10 @@
service_dir = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + '/../services')
services_xprotos = get_xproto(service_dir)
- xprotos = core_xprotos + services_xprotos
+ dynamic_service_dir = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + '/../dynamic_services')
+ dynamic_services_xprotos = get_xproto(dynamic_service_dir)
+
+ xprotos = core_xprotos + services_xprotos + dynamic_services_xprotos
xproto = ""
diff --git a/xos/synchronizers/new_base/loadmodels.py b/xos/synchronizers/new_base/loadmodels.py
new file mode 100644
index 0000000..b5c0a7d
--- /dev/null
+++ b/xos/synchronizers/new_base/loadmodels.py
@@ -0,0 +1,46 @@
+
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+class ModelLoadClient(object):
+ def __init__(self, api):
+ self.api = api
+
+ def upload_models(self, name, dir):
+ request = self.api.dynamicload_pb2.LoadModelsRequest(name=name, version="unknown")
+
+ for fn in os.listdir(dir):
+ if fn.endswith(".xproto"):
+ item = request.xprotos.add()
+ item.filename = fn
+ item.contents = open(os.path.join(dir, fn)).read()
+
+ models_fn = os.path.join(dir, "models.py")
+ if os.path.exists(models_fn):
+ item = request.decls.add()
+ item.filename = "models.py"
+ item.contents = open(models_fn).read()
+
+ attic_dir = os.path.join(dir, "attic")
+ if os.path.exists(attic_dir):
+ for fn in os.listdir(attic_dir):
+ if fn.endswith(".py"):
+ item = request.attics.add()
+ item.filename = fn
+ item.contents = open(os.path.join(attic_dir, fn)).read()
+
+ result = self.api.dynamicload.LoadModels(request)
+
diff --git a/xos/synchronizers/new_base/model_policies/model_policy_tenantwithcontainer.py b/xos/synchronizers/new_base/model_policies/model_policy_tenantwithcontainer.py
index 5190363..2fd891b 100644
--- a/xos/synchronizers/new_base/model_policies/model_policy_tenantwithcontainer.py
+++ b/xos/synchronizers/new_base/model_policies/model_policy_tenantwithcontainer.py
@@ -41,6 +41,7 @@
class LeastLoadedNodeScheduler(Scheduler):
# This scheduler always return the node with the fewest number of
# instances.
+
def pick(self):
set_label = False
diff --git a/xos/synchronizers/new_base/modelaccessor.py b/xos/synchronizers/new_base/modelaccessor.py
index 48026e4..0d1ffd9 100644
--- a/xos/synchronizers/new_base/modelaccessor.py
+++ b/xos/synchronizers/new_base/modelaccessor.py
@@ -28,8 +28,10 @@
import importlib
import os
import signal
+import time
from xosconfig import Config
from diag import update_diag
+from loadmodels import ModelLoadClient
from xosconfig import Config
from multistructlog import create_logger
@@ -136,7 +138,7 @@
# Keep checking the connection to wait for it to become unavailable.
# Then reconnect.
- # logger.info("keep_trying") # message is unneccesarily verbose
+ # log.info("keep_trying") # message is unneccesarily verbose
from xosapi.xos_grpc_client import Empty
@@ -146,7 +148,7 @@
# If we caught an exception, then the API has become unavailable.
# So reconnect.
- log.exception("exception in NoOp", e)
+ log.exception("exception in NoOp", e=e)
client.connected = False
client.connect()
return
@@ -160,9 +162,19 @@
# this will prevent updated timestamps from being automatically updated
client.xos_orm.caller_kind = "synchronizer"
+ client.xos_orm.restart_on_disconnect = True
+
from apiaccessor import CoreApiModelAccessor
model_accessor = CoreApiModelAccessor(orm=client.xos_orm)
+ if Config.get("models_dir"):
+ try:
+ ModelLoadClient(client).upload_models(Config.get("name"), Config.get("models_dir"))
+ except Exception, e: # TODO: narrow exception scope
+ log.exception("failed to onboard models")
+ reactor.callLater(10, functools.partial(grpcapi_reconnect, client, reactor))
+ return
+
# If required_models is set, then check to make sure the required_models
# are present. If not, then the synchronizer needs to go to sleep until
# the models show up.
diff --git a/xos/tools/xos-manage b/xos/tools/xos-manage
index 50b152b..6123dc7 100755
--- a/xos/tools/xos-manage
+++ b/xos/tools/xos-manage
@@ -81,10 +81,6 @@
echo "Syncing XOS services..."
python $XOS_DIR/manage.py syncdb --noinput
python $XOS_DIR/manage.py migrate
- if [[ $DJANGO_17 ]]; then
- echo "Loading initial data from fixture..."
- python $XOS_DIR/manage.py --noobserver loaddata $XOS_DIR/core/fixtures/core_initial_data.json
- fi
}
function evolvedb {
echo "Evolving XOS services..."
diff --git a/xos/xos/settings.py b/xos/xos/settings.py
index c7147c2..b257b34 100644
--- a/xos/xos/settings.py
+++ b/xos/xos/settings.py
@@ -23,7 +23,11 @@
# Initializing xosconfig module
from xosconfig import Config
-Config.init()
+from xosconfig.config import INITIALIZED as CONFIG_INITIALIZED
+
+# this really shouldn't be called from settings.py.
+if not CONFIG_INITIALIZED:
+ Config.init()
GEOIP_PATH = "/usr/share/GeoIP"
XOS_DIR = Config.get('xos_dir')
diff --git a/xos/xos_client/xosapi/orm.py b/xos/xos_client/xosapi/orm.py
index e4be772..adbe58d 100644
--- a/xos/xos_client/xosapi/orm.py
+++ b/xos/xos_client/xosapi/orm.py
@@ -36,6 +36,8 @@
"""
import functools
+import os
+import sys
import time
convenience_wrappers = {}
@@ -487,7 +489,8 @@
return self.objects.new(*args, **kwargs)
class ORMStub(object):
- def __init__(self, stub, package_name, invoker=None, caller_kind="grpcapi", sym_db = None, empty = None, enable_backoff=True):
+ def __init__(self, stub, package_name, invoker=None, caller_kind="grpcapi", sym_db = None, empty = None,
+ enable_backoff=True, restart_on_disconnect=False):
self.grpc_stub = stub
self.all_model_names = []
self.all_grpc_classes = {}
@@ -496,6 +499,7 @@
self.invoker = invoker
self.caller_kind = caller_kind
self.enable_backoff = enable_backoff
+ self.restart_on_disconnect = restart_on_disconnect
if not sym_db:
from google.protobuf import symbol_database as _symbol_database
@@ -552,14 +556,20 @@
# Our own retry mechanism. This works fine if there is a temporary
# failure in connectivity, but does not re-download gRPC schema.
import grpc
+ backoff = [0.5, 1, 2, 4, 8]
while True:
- backoff = [0.5, 1, 2, 4, 8]
try:
method = getattr(self.grpc_stub, name)
return method(request, metadata=metadata)
except grpc._channel._Rendezvous, e:
code = e.code()
if code == grpc.StatusCode.UNAVAILABLE:
+ if self.restart_on_disconnect:
+ # This is a blunt technique... We lost connectivity to the core, and we don't know that
+ # the core is still serving up the same models it was when we established connectivity,
+ # so restart the synchronizer.
+ # TODO: Hash check on the core models to tell if something changed would be better.
+ os.execv(sys.executable, ['python'] + sys.argv)
if not backoff:
raise Exception("No more retries on %s" % name)
time.sleep(backoff.pop(0))
diff --git a/xos/xos_client/xosapi/xos_grpc_client.py b/xos/xos_client/xosapi/xos_grpc_client.py
index f8815ed..a8fa51d 100644
--- a/xos/xos_client/xosapi/xos_grpc_client.py
+++ b/xos/xos_client/xosapi/xos_grpc_client.py
@@ -64,7 +64,7 @@
return self
def reconnected(self):
- for api in ['modeldefs', 'utility', 'xos']:
+ for api in ['modeldefs', 'utility', 'xos', 'dynamicload']:
pb2_file_name = os.path.join(self.work_dir, api + "_pb2.py")
pb2_grpc_file_name = os.path.join(self.work_dir, api + "_pb2_grpc.py")