blob: 37b5af2ec1a312a6158cf12056206d8d27029014 [file] [log] [blame]
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import hashlib
import os
import shutil
import tempfile
from xosgenx.generator import XOSProcessor
from xosconfig import Config
from multistructlog import create_logger
log = create_logger(Config().get('logging'))
DEFAULT_BASE_DIR="/opt/xos"
class DynamicBuilder(object):
NOTHING_TO_DO = 0
SOMETHING_CHANGED = 1
def __init__(self, base_dir=DEFAULT_BASE_DIR):
self.services_dir = os.path.join(base_dir, "dynamic_services")
self.manifest_dir = os.path.join(base_dir, "dynamic_services/manifests")
self.services_dest_dir = os.path.join(base_dir, "services")
self.coreapi_dir = os.path.join(base_dir, "coreapi")
self.protos_dir = os.path.join(base_dir, "coreapi/protos")
self.app_metadata_dir = os.path.join(base_dir, "xos")
self.convenience_methods_dir = os.path.join(base_dir, "xos_client/xosapi/convenience")
def pre_validate_file(self, item):
# someone might be trying to trick us into writing files outside the designated directory
if "/" in item.filename:
raise Exception("illegal character in filename %s" % item.filename)
def pre_validate_python(self, item):
(handle, fn) = tempfile.mkstemp()
try:
os.write(handle, item.contents)
os.close(handle)
if (os.system("python -m py_compile %s" % fn) != 0):
raise Exception("python file %s failed compile test" % item.filename)
finally:
os.remove(fn)
def pre_validate_models(self, request):
# do whatever validation we can before saving the files
for item in request.xprotos:
self.pre_validate_file(item)
for item in request.decls:
self.pre_validate_file(item)
for item in request.attics:
self.pre_validate_file(item)
for item in request.convenience_methods:
self.pre_validate_file(item)
self.pre_validate_python(item)
def get_manifests(self):
if not os.path.exists(self.manifest_dir):
return []
manifests=[]
for fn in os.listdir(self.manifest_dir):
if fn.endswith(".json"):
manifest_fn = os.path.join(self.manifest_dir, fn)
try:
manifest = json.loads(open(manifest_fn).read())
manifests.append(manifest)
except:
log.exception("Error loading manifest", filename=manifest_fn)
return manifests
def load_manifest_from_request(self, request):
manifest_fn = os.path.join(self.manifest_dir, request.name + ".json")
if os.path.exists(manifest_fn):
try:
manifest = json.loads(open(manifest_fn).read())
except:
log.exception("Error loading old manifest", filename=manifest_fn)
manifest = {}
else:
manifest = {}
return (manifest, manifest_fn)
def handle_loadmodels_request(self, request):
(manifest, manifest_fn) = self.load_manifest_from_request(request)
# TODO: Check version number to make sure this is not a downgrade ?
hash = self.generate_request_hash(request, state="load")
if hash == manifest.get("hash"):
# The hash of the incoming request is identical to the manifest that we have saved, so this request is a
# no-op.
log.info("Models are already up-to-date; skipping dynamic load.", name=request.name)
return self.NOTHING_TO_DO
self.pre_validate_models(request)
manifest = self.save_models(request, state="load", hash=hash)
self.run_xosgenx_service(manifest)
log.debug("Saving service manifest", name=request.name)
file(manifest_fn, "w").write(json.dumps(manifest))
log.info("Finished LoadModels request", name=request.name)
return self.SOMETHING_CHANGED
def handle_unloadmodels_request(self, request):
(manifest, manifest_fn) = self.load_manifest_from_request(request)
# TODO: Check version number to make sure this is not a downgrade ?
hash = self.generate_request_hash(request, state="unload")
if hash == manifest.get("hash"):
# The hash of the incoming request is identical to the manifest that we have saved, so this request is a
# no-op.
log.info("Models are already up-to-date; skipping dynamic unload.", name=request.name)
return self.NOTHING_TO_DO
manifest = self.save_models(request, state="unload", hash=hash)
self.remove_service(manifest)
log.debug("Saving service manifest", name=request.name)
file(manifest_fn, "w").write(json.dumps(manifest))
log.info("Finished UnloadModels request", name=request.name)
return self.SOMETHING_CHANGED
def generate_request_hash(self, request, state):
# TODO: could we hash the request rather than individually hashing the subcomponents of the request?
m = hashlib.sha1()
m.update(request.name)
m.update(request.version)
if (state == "load"):
for item in request.xprotos:
m.update(item.filename)
m.update(item.contents)
for item in request.decls:
m.update(item.filename)
m.update(item.contents)
for item in request.decls:
m.update(item.filename)
m.update(item.contents)
return m.hexdigest()
def save_models(self, request, state, hash=None):
if not hash:
hash = self.generate_request_hash(request, state)
service_dir = os.path.join(self.services_dir, request.name)
if not os.path.exists(service_dir):
os.makedirs(service_dir)
if not os.path.exists(self.manifest_dir):
os.makedirs(self.manifest_dir)
manifest_fn = os.path.join(self.manifest_dir, request.name + ".json")
# Invariant is that if a manifest file exists, then it accurately reflects that has been stored to disk. Since
# we're about to potentially overwrite files, destroy the old manifest.
if os.path.exists(manifest_fn):
os.remove(manifest_fn)
# convert the request to a manifest, so we can save it
service_manifest = {"name": request.name,
"version": request.version,
"hash": hash,
"state": state,
"dir": service_dir,
"manifest_fn": manifest_fn,
"dest_dir": os.path.join(self.services_dest_dir, request.name),
"xprotos": [],
"decls": [],
"attics": [],
"convenience_methods": []}
if (state == "load"):
for item in request.xprotos:
file(os.path.join(service_dir, item.filename), "w").write(item.contents)
service_manifest["xprotos"].append({"filename": item.filename})
for item in request.decls:
file(os.path.join(service_dir, item.filename), "w").write(item.contents)
service_manifest["decls"].append({"filename": item.filename})
if request.attics:
attic_dir = os.path.join(service_dir, "attic")
service_manifest["attic_dir"] = attic_dir
if not os.path.exists(attic_dir):
os.makedirs(attic_dir)
for item in request.attics:
file(os.path.join(attic_dir, item.filename), "w").write(item.contents)
service_manifest["attics"].append({"filename": item.filename})
for item in request.convenience_methods:
save_path = os.path.join(self.convenience_methods_dir, item.filename)
file(save_path, "w").write(item.contents)
service_manifest["convenience_methods"].append({
"filename": item.filename,
"path": save_path
})
return service_manifest
def run_xosgenx_service(self, manifest):
if not os.path.exists(manifest["dest_dir"]):
os.makedirs(manifest["dest_dir"])
xproto_filenames = [os.path.join(manifest["dir"], x["filename"]) for x in manifest["xprotos"]]
class Args:
pass
# Generate models
is_service = manifest["name"] != 'core'
args = Args()
args.verbosity = 0
args.output = manifest["dest_dir"]
args.attic = os.path.join(manifest["dir"], 'attic')
args.files = xproto_filenames
if is_service:
args.target = 'service.xtarget'
args.write_to_file = 'target'
else:
args.target = 'django.xtarget'
args.dest_extension = 'py'
args.write_to_file = 'model'
XOSProcessor.process(args)
# Generate security checks
class SecurityArgs:
verbosity = 0
output = manifest["dest_dir"]
target = 'django-security.xtarget'
dest_file = 'security.py'
write_to_file = 'single'
files = xproto_filenames
XOSProcessor.process(SecurityArgs())
# Generate __init__.py
if manifest["name"] == "core":
class InitArgs:
output = manifest["dest_dir"]
target = 'init.xtarget'
dest_file = '__init__.py'
write_to_file = 'single'
files = xproto_filenames
XOSProcessor.process(InitArgs())
else:
init_py_filename = os.path.join(manifest["dest_dir"], "__init__.py")
if not os.path.exists(init_py_filename):
open(init_py_filename, "w").write("# created by dynamicbuild")
# the xosgenx templates don't handle copying the models.py file for us, so do it here.
for item in manifest["decls"]:
src_fn = os.path.join(manifest["dir"], item["filename"])
dest_fn = os.path.join(manifest["dest_dir"], item["filename"])
shutil.copyfile(src_fn, dest_fn)
# If the attic has a header.py, make sure it is copied to the right place
attic_header_py_src = os.path.join(manifest["dir"], "attic", "header.py")
service_header_py_dest = os.path.join(manifest["dest_dir"], "header.py")
if os.path.exists(attic_header_py_src):
shutil.copyfile(attic_header_py_src, service_header_py_dest)
elif os.path.exists(service_header_py_dest):
os.remove(service_header_py_dest)
def remove_service(self, manifest):
# remove any xproto files, otherwise "make rebuild_protos" will pick them up
if os.path.exists(manifest["dir"]):
for fn in os.listdir(manifest["dir"]):
fn = os.path.join(manifest["dir"], fn)
if fn.endswith(".xproto"):
os.remove(fn)
# Rather than trying to unmigrate while the core is running, let's handle unmigrating the service while we're
# outside of the core process. We're going to save the manifest file, and the manifest file will have
# {"state": "unload"} in it. That can be our external signal to unmigrate.
# This is what unmigrate will do, external to this process:
# 1) remove the models (./manage.py migrate my_app_name zero)
# 2) remove the contenttypes
# # does step 1 already do this?
# from django.contrib.contenttypes.models import ContentType
# for c in ContentType.objects.all():
# if not c.model_class():
# print "deleting %s" % c
# c.delete()
# 3) Remove the service files