| # Copyright 2017-present Open Networking Foundation |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| import json |
| import hashlib |
| import os |
| import re |
| import shutil |
| import tempfile |
| from xosgenx.generator import XOSProcessor, XOSProcessorArgs |
| |
| from xosconfig import Config |
| from multistructlog import create_logger |
| |
| log = create_logger(Config().get("logging")) |
| |
| DEFAULT_BASE_DIR = "/opt/xos" |
| |
| |
| class DynamicBuilder(object): |
| # These constants must agree with what is declared in dnyamicload.proto |
| SUCCESS = 0 |
| SUCCESS_NOTHING_CHANGED = 1 |
| ERROR = 101 |
| ERROR_LIVE_MODELS = 102 |
| ERROR_DELETION_IN_PROGRESS = 103 |
| TRYAGAIN = 201 |
| |
| def __init__(self, base_dir=DEFAULT_BASE_DIR): |
| self.services_dir = os.path.join(base_dir, "dynamic_services") |
| self.manifest_dir = os.path.join(base_dir, "dynamic_services/manifests") |
| self.services_dest_dir = os.path.join(base_dir, "services") |
| self.coreapi_dir = os.path.join(base_dir, "coreapi") |
| self.protos_dir = os.path.join(base_dir, "coreapi/protos") |
| self.app_metadata_dir = os.path.join(base_dir, "xos") |
| self.convenience_methods_dir = os.path.join( |
| base_dir, "dynamic_services/convenience_methods" |
| ) |
| |
| def pre_validate_file(self, item): |
| # various filename validations |
| # alphas (upper and lower), digits, underscore, dash, and period. |
| if not re.match("^[\w.-]+$", item.filename): # noqa: W605 |
| raise Exception("illegal character in filename %s" % item.filename) |
| |
| def pre_validate_python(self, item): |
| (handle, fn) = tempfile.mkstemp() |
| try: |
| os.write(handle, item.contents) |
| os.close(handle) |
| if os.system("python -m py_compile %s" % fn) != 0: |
| raise Exception("python file %s failed compile test" % item.filename) |
| finally: |
| os.remove(fn) |
| |
| def pre_validate_models(self, request): |
| # do whatever validation we can before saving the files |
| for item in request.xprotos: |
| self.pre_validate_file(item) |
| |
| for item in request.decls: |
| self.pre_validate_file(item) |
| |
| for item in request.attics: |
| self.pre_validate_file(item) |
| |
| for item in request.convenience_methods: |
| self.pre_validate_file(item) |
| self.pre_validate_python(item) |
| |
| for item in request.migrations: |
| self.pre_validate_file(item) |
| self.pre_validate_python(item) |
| |
| def get_manifests(self): |
| if not os.path.exists(self.manifest_dir): |
| return [] |
| |
| manifests = [] |
| for fn in os.listdir(self.manifest_dir): |
| if fn.endswith(".json"): |
| manifest_fn = os.path.join(self.manifest_dir, fn) |
| try: |
| manifest = json.loads(open(manifest_fn).read()) |
| manifests.append(manifest) |
| except BaseException: |
| log.exception("Error loading manifest", filename=manifest_fn) |
| return manifests |
| |
| def load_manifest_from_request(self, request): |
| manifest_fn = os.path.join(self.manifest_dir, request.name + ".json") |
| if os.path.exists(manifest_fn): |
| try: |
| manifest = json.loads(open(manifest_fn).read()) |
| except BaseException: |
| log.exception("Error loading old manifest", filename=manifest_fn) |
| manifest = {} |
| else: |
| manifest = {} |
| |
| return (manifest, manifest_fn) |
| |
| def handle_loadmodels_request(self, request): |
| (manifest, manifest_fn) = self.load_manifest_from_request(request) |
| |
| # TODO: Check version number to make sure this is not a downgrade ? |
| |
| hash = self.generate_request_hash(request, state="load") |
| if hash == manifest.get("hash"): |
| # The hash of the incoming request is identical to the manifest that we have saved, so this request is a |
| # no-op. |
| log.info( |
| "Models are already up-to-date; skipping dynamic load.", |
| name=request.name, |
| ) |
| return self.SUCCESS_NOTHING_CHANGED |
| |
| self.pre_validate_models(request) |
| |
| manifest = self.save_models(request, state="load", hash=hash) |
| |
| self.run_xosgenx_service(manifest) |
| |
| log.debug("Saving service manifest", name=request.name) |
| file(manifest_fn, "w").write(json.dumps(manifest)) |
| |
| log.info("Finished LoadModels request", name=request.name) |
| |
| return self.SUCCESS |
| |
| def handle_unloadmodels_request(self, request, models): |
| (manifest, manifest_fn) = self.load_manifest_from_request(request) |
| |
| for (name, model) in models.items(): |
| if model.objects.exists(): |
| if request.cleanup_behavior == request.REQUIRE_CLEAN: |
| log.info("UnloadModels: Returning error due to live model existence", model=model) |
| return self.ERROR_LIVE_MODELS |
| elif request.cleanup_behavior == request.AUTOMATICALLY_CLEAN: |
| # Deleting the model will add it to model.deleted_objects automatically, and it will be caught |
| # by the next loop and return a TRYAGAIN as necessary. |
| log.info("UnloadModels: Mass deleting", model=model) |
| # NOTE: model.objects.all().delete() does not behave as expected. |
| # Loop and delete each one instead. |
| for object in model.objects.all(): |
| object.delete() |
| |
| for (name, model) in models.items(): |
| if model.deleted_objects.exists(): |
| if request.cleanup_behavior == request.REQUIRE_CLEAN: |
| log.info("UnloadModels: Returning error due to models in soft-delete", model=model) |
| return self.ERROR_DELETION_IN_PROGRESS |
| elif request.cleanup_behavior == request.AUTOMATICALLY_CLEAN: |
| log.info("UnloadModels: Returning tryagain due to models in soft-delete", model=model) |
| return self.TRYAGAIN |
| |
| hash = self.generate_request_hash(request, state="unload") |
| if hash == manifest.get("hash"): |
| # The hash of the incoming request is identical to the manifest that we have saved, so this request is a |
| # no-op. |
| log.info( |
| "Models are already up-to-date; skipping dynamic unload.", |
| name=request.name, |
| ) |
| return self.SUCCESS_NOTHING_CHANGED |
| |
| manifest = self.save_models(request, state="unload", hash=hash) |
| |
| self.remove_service(manifest) |
| |
| log.debug("Saving service manifest", name=request.name) |
| file(manifest_fn, "w").write(json.dumps(manifest)) |
| |
| log.info("Finished UnloadModels request", name=request.name) |
| |
| return self.SUCCESS |
| |
| def generate_request_hash(self, request, state): |
| # TODO: could we hash the request rather than individually hashing the subcomponents of the request? |
| m = hashlib.sha1() |
| m.update(request.name) |
| m.update(request.version) |
| if state == "load": |
| for item in request.xprotos: |
| m.update(item.filename) |
| m.update(item.contents) |
| for item in request.decls: |
| m.update(item.filename) |
| m.update(item.contents) |
| for item in request.decls: |
| m.update(item.filename) |
| m.update(item.contents) |
| return m.hexdigest() |
| |
| def save_models(self, request, state, hash=None): |
| if not hash: |
| hash = self.generate_request_hash(request, state) |
| |
| service_dir = os.path.join(self.services_dir, request.name) |
| if not os.path.exists(service_dir): |
| os.makedirs(service_dir) |
| |
| if not os.path.exists(self.manifest_dir): |
| os.makedirs(self.manifest_dir) |
| |
| if not os.path.exists(self.convenience_methods_dir): |
| os.makedirs(self.convenience_methods_dir) |
| |
| manifest_fn = os.path.join(self.manifest_dir, request.name + ".json") |
| |
| # Invariant is that if a manifest file exists, then it accurately reflects that has been stored to disk. Since |
| # we're about to potentially overwrite files, destroy the old manifest. |
| if os.path.exists(manifest_fn): |
| os.remove(manifest_fn) |
| |
| # convert the request to a manifest, so we can save it |
| service_manifest = { |
| "name": request.name, |
| "version": request.version, |
| "hash": hash, |
| "state": state, |
| "dir": service_dir, |
| "manifest_fn": manifest_fn, |
| "dest_dir": os.path.join(self.services_dest_dir, request.name), |
| "xprotos": [], |
| "decls": [], |
| "attics": [], |
| "convenience_methods": [], |
| } |
| |
| if state == "load": |
| for item in request.xprotos: |
| file(os.path.join(service_dir, item.filename), "w").write(item.contents) |
| service_manifest["xprotos"].append({"filename": item.filename}) |
| |
| for item in request.decls: |
| file(os.path.join(service_dir, item.filename), "w").write(item.contents) |
| service_manifest["decls"].append({"filename": item.filename}) |
| |
| if request.attics: |
| attic_dir = os.path.join(service_dir, "attic") |
| service_manifest["attic_dir"] = attic_dir |
| if not os.path.exists(attic_dir): |
| os.makedirs(attic_dir) |
| for item in request.attics: |
| file(os.path.join(attic_dir, item.filename), "w").write( |
| item.contents |
| ) |
| service_manifest["attics"].append({"filename": item.filename}) |
| |
| for item in request.convenience_methods: |
| save_path = os.path.join(self.convenience_methods_dir, item.filename) |
| file(save_path, "w").write(item.contents) |
| service_manifest["convenience_methods"].append( |
| {"filename": item.filename, "path": save_path} |
| ) |
| |
| if request.migrations: |
| # These can be saved directly to the service destination directory, since they are not processed by |
| # xosgenx. |
| migrations_dir = os.path.join(service_manifest["dest_dir"], "migrations") |
| service_manifest["migrations_dir"] = migrations_dir |
| service_manifest["migrations"] = [] |
| if not os.path.exists(migrations_dir): |
| os.makedirs(migrations_dir) |
| for item in request.migrations: |
| file(os.path.join(migrations_dir, item.filename), "w").write( |
| item.contents |
| ) |
| service_manifest["migrations"].append({"filename": item.filename}) |
| |
| migrations_init_py_filename = os.path.join(migrations_dir, "__init__.py") |
| if not os.path.exists(migrations_init_py_filename): |
| open(migrations_init_py_filename, "w").write("# created by dynamicbuild") |
| |
| return service_manifest |
| |
| def run_xosgenx_service(self, manifest): |
| if not os.path.exists(manifest["dest_dir"]): |
| os.makedirs(manifest["dest_dir"]) |
| |
| xproto_filenames = [ |
| os.path.join(manifest["dir"], x["filename"]) for x in manifest["xprotos"] |
| ] |
| |
| # Generate models |
| is_service = manifest["name"] != "core" |
| |
| args = XOSProcessorArgs( |
| output=manifest["dest_dir"], |
| attic=os.path.join(manifest["dir"], "attic"), |
| files=xproto_filenames, |
| ) |
| |
| if is_service: |
| args.target = "service.xtarget" |
| args.write_to_file = "target" |
| else: |
| args.target = "django.xtarget" |
| args.dest_extension = "py" |
| args.write_to_file = "model" |
| |
| XOSProcessor.process(args) |
| |
| # Generate security checks |
| security_args = XOSProcessorArgs( |
| output=manifest["dest_dir"], |
| target="django-security.xtarget", |
| dest_file="security.py", |
| write_to_file="single", |
| files=xproto_filenames, |
| ) |
| |
| XOSProcessor.process(security_args) |
| |
| # Generate __init__.py |
| if manifest["name"] == "core": |
| |
| class InitArgs: |
| output = manifest["dest_dir"] |
| target = "init.xtarget" |
| dest_file = "__init__.py" |
| write_to_file = "single" |
| files = xproto_filenames |
| |
| XOSProcessor.process(InitArgs()) |
| |
| else: |
| init_py_filename = os.path.join(manifest["dest_dir"], "__init__.py") |
| if not os.path.exists(init_py_filename): |
| open(init_py_filename, "w").write("# created by dynamicbuild") |
| |
| # the xosgenx templates don't handle copying the models.py file for us, so do it here. |
| for item in manifest["decls"]: |
| src_fn = os.path.join(manifest["dir"], item["filename"]) |
| dest_fn = os.path.join(manifest["dest_dir"], item["filename"]) |
| shutil.copyfile(src_fn, dest_fn) |
| |
| # If the attic has a header.py, make sure it is copied to the right place |
| attic_header_py_src = os.path.join(manifest["dir"], "attic", "header.py") |
| service_header_py_dest = os.path.join(manifest["dest_dir"], "header.py") |
| if os.path.exists(attic_header_py_src): |
| shutil.copyfile(attic_header_py_src, service_header_py_dest) |
| elif os.path.exists(service_header_py_dest): |
| os.remove(service_header_py_dest) |
| |
| def remove_service(self, manifest): |
| # remove any xproto files, otherwise "make rebuild_protos" will pick them up |
| if os.path.exists(manifest["dir"]): |
| for fn in os.listdir(manifest["dir"]): |
| fn = os.path.join(manifest["dir"], fn) |
| if fn.endswith(".xproto"): |
| os.remove(fn) |
| |
| # Rather than trying to unmigrate while the core is running, let's handle unmigrating the service while we're |
| # outside of the core process. We're going to save the manifest file, and the manifest file will have |
| # {"state": "unload"} in it. That can be our external signal to unmigrate. |
| |
| # This is what unmigrate will do, external to this process: |
| # 1) remove the models (./manage.py migrate my_app_name zero) |
| # 2) remove the contenttypes |
| # # does step 1 already do this? |
| # from django.contrib.contenttypes.models import ContentType |
| # for c in ContentType.objects.all(): |
| # if not c.model_class(): |
| # print "deleting %s" % c |
| # c.delete() |
| # 3) Remove the service files |