blob: f8f5d6bc8a397cfa465d81e231e4613d28bc5d0c [file] [log] [blame]
#!/usr/bin/python
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# TO RUN
# source scripts/setup_venv.sh
# xos-migrate [-s <service-name>] [-r ~/cord]
# eg: xos-migrate -r ~/Sites/cord -s core -s fabric
# TODO
# - add support to specify a name to be given to the generated migration (--name parameter in django makemigrations)
# - add support to generate empty migrations (needed for data-only migrations)
import os
import sys
import argparse
import yaml
import shutil
from xosgenx.generator import XOSProcessor, XOSProcessorArgs
from xosconfig import Config
from multistructlog import create_logger
def get_abs_path(dir_):
""" Convert a path specified by the user, which might be relative or based on
home directory location, into an absolute path.
"""
if os.path.isabs(dir_):
return os.path.realpath(dir_)
if dir_[0] == "~" and not os.path.exists(dir_):
dir_ = os.path.expanduser(dir_)
return os.path.abspath(dir_)
return os.path.abspath(os.path.join(os.getcwd(), dir_))
def get_migration_library_path(dir_):
""" Return a directory relative to the location of the migration library """
return os.path.dirname(os.path.realpath(__file__)) + "/" + dir_
def print_banner(root):
log.info(r"---------------------------------------------------------------")
log.info(r" _ __ ")
log.info(r" _ ______ _____ ____ ___ (_)___ __________ _/ /____ ")
log.info(r" | |/_/ __ \/ ___/_____/ __ `__ \/ / __ `/ ___/ __ `/ __/ _ \ ")
log.info(r" _> </ /_/ (__ )_____/ / / / / / / /_/ / / / /_/ / /_/ __/ ")
log.info(r"/_/|_|\____/____/ /_/ /_/ /_/_/\__, /_/ \__,_/\__/\___/ ")
log.info(r" /____/ ")
log.info(r"---------------------------------------------------------------")
log.debug("CORD repo root", root=root)
log.debug("Storing logs in: %s" % os.environ["LOG_FILE"])
log.debug(r"---------------------------------------------------------------")
def generate_core_models(core_dir):
core_xproto = os.path.join(core_dir, "core.xproto")
args = XOSProcessorArgs(
output=core_dir,
target="django.xtarget",
dest_extension="py",
write_to_file="model",
files=[core_xproto],
)
XOSProcessor.process(args)
security_args = XOSProcessorArgs(
output=core_dir,
target="django-security.xtarget",
dest_file="security.py",
write_to_file="single",
files=[core_xproto],
)
XOSProcessor.process(security_args)
init_args = XOSProcessorArgs(
output=core_dir,
target="init.xtarget",
dest_file="__init__.py",
write_to_file="single",
files=[core_xproto],
)
XOSProcessor.process(init_args)
def find_xproto_in_folder(path):
"""
Recursively iterate a folder tree to look for any xProto file.
We use this function in case that the name of the xProto is different from the name of the folder (eg: olt-service)
:param path: the root folder to start the search
:return: [string]
"""
xprotos = []
for fn in os.listdir(path):
# skip hidden files and folders. plus other useless things
if fn.startswith(".") or fn == "venv-xos" or fn == "htmlcov":
continue
full_path = os.path.join(path, fn)
if fn.endswith(".xproto"):
xprotos.append(full_path)
elif os.path.isdir(full_path):
xprotos = xprotos + find_xproto_in_folder(full_path)
return xprotos
def find_decls_models(path):
"""
Recursively iterate a folder tree to look for any models.py file.
This files contain the base model for _decl generated models.
:param path: the root folder to start the search
:return: [string]
"""
decls = []
for fn in os.listdir(path):
# skip hidden files and folders. plus other useless things
if fn.startswith(".") or fn == "venv-xos" or fn == "htmlcov":
continue
full_path = os.path.join(path, fn)
if fn == "models.py":
decls.append(full_path)
elif os.path.isdir(full_path):
decls = decls + find_decls_models(full_path)
return decls
def get_service_name_from_config(path):
"""
Given a service folder look for the config.yaml file and find the name
:param path: the root folder to start the search
:return: string
"""
config = os.path.join(path, "xos/synchronizer/config.yaml")
if not os.path.isfile(config):
raise Exception("Config file not found at: %s" % config)
cfg_file = open(config)
cfg = yaml.safe_load(cfg_file)
return cfg["name"]
def generate_service_models(service_dir, service_dest_dir, service_name):
"""
Generate the django code starting from xProto for a given service.
:param service_dir: string (path to the folder)
:param service_name: string (name of the service)
:return: void
"""
xprotos = find_xproto_in_folder(service_dir)
decls = find_decls_models(service_dir)
log.debug("Generating models for %s from files %s" % (service_name, ", ".join(xprotos)))
out_dir = os.path.join(service_dest_dir, service_name)
if not os.path.isdir(out_dir):
os.mkdir(out_dir)
args = XOSProcessorArgs(
output=out_dir,
files=xprotos,
target="service.xtarget",
write_to_file="target",
)
XOSProcessor.process(args)
security_args = XOSProcessorArgs(
output=out_dir,
target="django-security.xtarget",
dest_file="security.py",
write_to_file="single",
files=xprotos,
)
XOSProcessor.process(security_args)
init_py_filename = os.path.join(out_dir, "__init__.py")
if not os.path.exists(init_py_filename):
open(init_py_filename, "w").write("# created by dynamicbuild")
# copy over models.py files from the service
if len(decls) > 0:
for file in decls:
fn = os.path.basename(file)
src_fn = file
dest_fn = os.path.join(out_dir, fn)
log.debug("Copying models.py from %s to %s" % (src_fn, dest_fn))
shutil.copyfile(src_fn, dest_fn)
# copy existing migrations from the service, otherwise they won't be incremental
src_dir = os.path.join(service_dir, "xos", "synchronizer", "migrations")
if os.path.isdir(src_dir):
dest_dir = os.path.join(out_dir, "migrations")
if os.path.isdir(dest_dir):
shutil.rmtree(dest_dir) # empty the folder, we'll copy everything again
shutil.copytree(src_dir, dest_dir)
def copy_service_migrations(service_dir, service_dest_dir, service_name):
"""
Once the migrations are generated, copy them in the correct location
:param service_dir: string (path to the folder)
:param service_name: string (name of the service)
:return: void
"""
log.debug("Copying %s migrations to %s" % (service_name, service_dir))
migration_dir = os.path.join(service_dest_dir, service_name, "migrations")
dest_dir = os.path.join(service_dir, "xos", "synchronizer", "migrations")
if os.path.isdir(dest_dir):
shutil.rmtree(dest_dir) # empty the folder, we'll copy everything again
shutil.copytree(migration_dir, dest_dir)
# clean after the tool, generated migrations has been moved in the service repo
shutil.rmtree(get_abs_path(os.path.join(migration_dir, "../")))
def monkey_patch_migration_template():
import django
django.setup()
import django.db.migrations.writer as dj
dj.MIGRATION_TEMPLATE = """\
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# -*- coding: utf-8 -*-
# Generated by Django %(version)s on %(timestamp)s
from __future__ import unicode_literals
%(imports)s
class Migration(migrations.Migration):
%(replaces_str)s%(initial_str)s
dependencies = [
%(dependencies)s\
]
operations = [
%(operations)s\
]
"""
def configure_logging(verbose):
global log
# INITIALIZING LOGGER
Config.init()
cfg = Config().get("logging")
if verbose:
cfg["handlers"]["console"]["level"] = "DEBUG"
log = create_logger(cfg)
# SETTING ENV
os.environ["LOG_FILE"] = get_migration_library_path("django.log")
os.environ["XOS_CONFIG_SCHEMA"] = get_migration_library_path("migration_cfg_schema.yaml")
os.environ["XOS_CONFIG_FILE"] = get_migration_library_path("migration_cfg.yaml")
os.environ["MIGRATIONS"] = "true"
# this is populated in case we generate migrations for services and it's used in settings.py
os.environ["INSTALLED_APPS"] = ""
# PARAMS
parser = argparse.ArgumentParser(description="XOS Migrations")
required = parser.add_argument_group("required arguments")
required.add_argument(
"-s",
"--service",
action="append",
required=True,
dest="service_names",
help="The name of the folder containing the service in cord/orchestration/xos_services"
)
parser.add_argument(
"-r",
"--repo",
default=get_abs_path("~/cord"),
dest="repo_root",
help="The location of the folder containing the CORD repo root (default to ~/cord)"
)
parser.add_argument(
"--check",
default=False,
action="store_true",
dest="check",
help="Check if the migrations are generated for a given service. Does not apply any change."
)
parser.add_argument(
"-v",
"--verbose",
help="increase log verbosity",
dest="verbose",
action="store_true"
)
def run():
# cleaning up from possible incorrect states
if "INSTALLED_APPS" in os.environ:
del os.environ["INSTALLED_APPS"]
args = parser.parse_args()
configure_logging(args.verbose)
print_banner(args.repo_root)
# find absolute path to the code
xos_path = get_abs_path(os.path.join(args.repo_root, "orchestration/xos/xos/"))
core_dir = get_abs_path(os.path.join(xos_path, "core/models/"))
service_base_dir = get_abs_path(os.path.join(xos_path, "../../xos_services/"))
service_dest_dir = get_abs_path(os.path.join(xos_path, "services/"))
# we need to append the xos folder to sys.path
original_sys_path = sys.path
sys.path.append(xos_path)
log.info("Services: %s" % ", ".join(args.service_names))
django_cli_args = ['xos-migrate.py', "makemigrations"]
# generate the code for each service and create a list of parameters to pass to django
app_list = []
for service in args.service_names:
# NOTE we need core models to be there as all the services depend on them
generate_core_models(core_dir)
if service == "core":
django_cli_args.append("core")
else:
service_dir = os.path.join(service_base_dir, service)
service_name = get_service_name_from_config(service_dir)
generate_service_models(service_dir, service_dest_dir, service_name)
app_list.append("services.%s" % service_name)
django_cli_args.append(service_name)
if len(app_list) > 0:
os.environ["INSTALLED_APPS"] = ",".join(app_list)
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "xos.settings")
monkey_patch_migration_template()
if args.check:
django_cli_args.append("--check")
django_cli_args.append("--dry-run")
from django.core.management import execute_from_command_line
try:
log.debug("Django CLI Args", args=django_cli_args)
execute_from_command_line(django_cli_args)
returncode = 0
except SystemExit as e:
returncode = e.message
if returncode != 0:
if args.check:
log.error("Migrations are not up to date with the service changes!")
else:
log.error("An error occurred")
sys.exit(returncode)
# copying migrations back to the service
for service in args.service_names:
if service == "core":
# we don't need to copy migrations for the core
continue
else:
service_dir = os.path.join(service_base_dir, service)
service_name = get_service_name_from_config(service_dir)
copy_service_migrations(service_dir, service_dest_dir, service_name)
# restore orginal sys.path
sys.path = original_sys_path