[SEBA-461][WIP] Adding xos-migrate to the XOS Toolchain
Change-Id: I3a6e2a86b804efe207e7a71109763b11ba9acdaa
diff --git a/lib/xos-migrate/.gitignore b/lib/xos-migrate/.gitignore
new file mode 100644
index 0000000..ed8b98a
--- /dev/null
+++ b/lib/xos-migrate/.gitignore
@@ -0,0 +1,6 @@
+build
+dist
+XosMigrate.egg-info
+.coverage
+coverage.xml
+cover
diff --git a/lib/xos-migrate/MANIFEST.in b/lib/xos-migrate/MANIFEST.in
new file mode 100644
index 0000000..a38c6ba
--- /dev/null
+++ b/lib/xos-migrate/MANIFEST.in
@@ -0,0 +1,2 @@
+include xosmigrate/migration_cfg_schema.yaml
+include xosmigrate/migration_cfg.yaml
diff --git a/lib/xos-migrate/bin/xos-migrate b/lib/xos-migrate/bin/xos-migrate
new file mode 100644
index 0000000..574bc40
--- /dev/null
+++ b/lib/xos-migrate/bin/xos-migrate
@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+
+# from xosmigrate import run
+from xosmigrate.main import run
+run()
diff --git a/lib/xos-migrate/setup.py b/lib/xos-migrate/setup.py
new file mode 100644
index 0000000..fe795de
--- /dev/null
+++ b/lib/xos-migrate/setup.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+try:
+ from xosutil.autoversion_setup import setup_with_auto_version as setup
+except ImportError:
+ # xosutil is not installed. Expect this to happen when we build an egg, in which case xosgenx.version will
+ # automatically have the right version.
+ from setuptools import setup
+
+from xosgenx.version import __version__
+
+setup(
+ name="XosMigrate",
+ version=__version__,
+ description="XOS Migrations Toolkit",
+ author="Matteo Scandolo",
+ author_email="teo@opennetworking.org",
+ packages=["xosmigrate"],
+ scripts=["bin/xos-migrate"],
+ include_package_data=True,
+ # TODO add all deps to the install_requires section
+ install_requires=[],
+)
diff --git a/lib/xos-migrate/xosmigrate/__init__.py b/lib/xos-migrate/xosmigrate/__init__.py
new file mode 100644
index 0000000..0612f4f
--- /dev/null
+++ b/lib/xos-migrate/xosmigrate/__init__.py
@@ -0,0 +1,14 @@
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import main
diff --git a/lib/xos-migrate/xosmigrate/main.py b/lib/xos-migrate/xosmigrate/main.py
new file mode 100644
index 0000000..e6901e2
--- /dev/null
+++ b/lib/xos-migrate/xosmigrate/main.py
@@ -0,0 +1,354 @@
+#!/usr/bin/python
+
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# TO RUN
+# source scripts/setup_venv.sh
+# xos-migrate [-s <service-name>] [-r ~/cord]
+# eg: xos-migrate -r ~/Sites/cord -s core -s fabric
+
+# TODO
+# - add support for services that are in the cord/orchestration/profiles folders
+# - add support to specify a name to be given to the generated migration (--name parameter in django makemigrations)
+# - add support to generate empty migrations (needed for data-only migrations)
+
+import os
+import sys
+import argparse
+import yaml
+import shutil
+from xosgenx.generator import XOSProcessor, XOSProcessorArgs
+from xosconfig import Config
+from multistructlog import create_logger
+
+
+def get_abs_path(dir_):
+ if os.path.isabs(dir_):
+ return dir_
+ if dir_[0] == '~' and not os.path.exists(dir_):
+ dir_ = os.path.expanduser(dir_)
+ return os.path.abspath(dir_)
+ return os.path.dirname(os.path.realpath(__file__)) + "/" + dir_
+
+
+def print_banner(root):
+ log.info(r"---------------------------------------------------------------")
+ log.info(r" _ __ ")
+ log.info(r" _ ______ _____ ____ ___ (_)___ __________ _/ /____ ")
+ log.info(r" | |/_/ __ \/ ___/_____/ __ `__ \/ / __ `/ ___/ __ `/ __/ _ \ ")
+ log.info(r" _> </ /_/ (__ )_____/ / / / / / / /_/ / / / /_/ / /_/ __/ ")
+ log.info(r"/_/|_|\____/____/ /_/ /_/ /_/_/\__, /_/ \__,_/\__/\___/ ")
+ log.info(r" /____/ ")
+ log.info(r"---------------------------------------------------------------")
+ log.debug("CORD repo root", root=root)
+ log.debug("Storing logs in: %s" % os.environ["LOG_FILE"])
+ # log.debug("Config schema: %s" % os.environ['XOS_CONFIG_SCHEMA'])
+ # log.debug("Config: %s" % os.environ['XOS_CONFIG_FILE'])
+ log.debug(r"---------------------------------------------------------------")
+
+
+def generate_core_models(core_dir):
+ core_xproto = os.path.join(core_dir, "core.xproto")
+
+ args = XOSProcessorArgs(
+ output=core_dir,
+ target="django.xtarget",
+ dest_extension="py",
+ write_to_file="model",
+ files=[core_xproto],
+ )
+ XOSProcessor.process(args)
+
+ security_args = XOSProcessorArgs(
+ output=core_dir,
+ target="django-security.xtarget",
+ dest_file="security.py",
+ write_to_file="single",
+ files=[core_xproto],
+ )
+
+ XOSProcessor.process(security_args)
+
+ init_args = XOSProcessorArgs(
+ output=core_dir,
+ target="init.xtarget",
+ dest_file="__init__.py",
+ write_to_file="single",
+ files=[core_xproto],
+ )
+ XOSProcessor.process(init_args)
+
+
+def find_xproto_in_folder(path):
+ """
+ Recursively iterate a folder tree to look for any xProto file.
+ We use this function in case that the name of the xProto is different from the name of the folder (eg: olt-service)
+ :param path: the root folder to start the search
+ :return: [string]
+ """
+ xprotos = []
+ for fn in os.listdir(path):
+ # skip hidden files and folders. plus other useless things
+ if fn.startswith(".") or fn == "venv-xos" or fn == "htmlcov":
+ continue
+ full_path = os.path.join(path, fn)
+ if fn.endswith(".xproto"):
+ xprotos.append(full_path)
+ elif os.path.isdir(full_path):
+ xprotos = xprotos + find_xproto_in_folder(full_path)
+ return xprotos
+
+
+def find_decls_models(path):
+ """
+ Recursively iterate a folder tree to look for any models.py file.
+ This files contain the base model for _decl generated models.
+ :param path: the root folder to start the search
+ :return: [string]
+ """
+ decls = []
+ for fn in os.listdir(path):
+ # skip hidden files and folders. plus other useless things
+ if fn.startswith(".") or fn == "venv-xos" or fn == "htmlcov":
+ continue
+ full_path = os.path.join(path, fn)
+ if fn == "models.py":
+ decls.append(full_path)
+ elif os.path.isdir(full_path):
+ decls = decls + find_decls_models(full_path)
+ return decls
+
+
+def get_service_name_from_config(path):
+ """
+ Given a service folder look for the config.yaml file and find the name
+ :param path: the root folder to start the search
+ :return: string
+ """
+ config = os.path.join(path, "xos/synchronizer/config.yaml")
+ if not os.path.isfile(config):
+ raise Exception("Config file not found at: %s" % config)
+
+ cfg_file = open(config)
+ cfg = yaml.load(cfg_file)
+ return cfg['name']
+
+
+def generate_service_models(service_dir, service_dest_dir, service_name):
+ """
+ Generate the django code starting from xProto for a given service.
+ :param service_dir: string (path to the folder)
+ :param service_name: string (name of the service)
+ :return: void
+ """
+ xprotos = find_xproto_in_folder(service_dir)
+ decls = find_decls_models(service_dir)
+ log.debug("Generating models for %s from files %s" % (service_name, ", ".join(xprotos)))
+ out_dir = os.path.join(service_dest_dir, service_name)
+ if not os.path.isdir(out_dir):
+ os.mkdir(out_dir)
+
+ args = XOSProcessorArgs(
+ output=out_dir,
+ files=xprotos,
+ target="service.xtarget",
+ write_to_file="target",
+ )
+ XOSProcessor.process(args)
+
+ security_args = XOSProcessorArgs(
+ output=out_dir,
+ target="django-security.xtarget",
+ dest_file="security.py",
+ write_to_file="single",
+ files=xprotos,
+ )
+
+ XOSProcessor.process(security_args)
+
+ init_py_filename = os.path.join(out_dir, "__init__.py")
+ if not os.path.exists(init_py_filename):
+ open(init_py_filename, "w").write("# created by dynamicbuild")
+
+ # copy over models.py files from the service
+ if len(decls) > 0:
+ for file in decls:
+ fn = os.path.basename(file)
+ src_fn = file
+ dest_fn = os.path.join(out_dir, fn)
+ log.debug("Copying models.py from %s to %s" % (src_fn, dest_fn))
+ shutil.copyfile(src_fn, dest_fn)
+
+ # copy existing migrations from the service, otherwise they won't be incremental
+ src_dir = os.path.join(service_dir, "xos", "synchronizer", "migrations")
+ if os.path.isdir(src_dir):
+ dest_dir = os.path.join(out_dir, "migrations")
+ if os.path.isdir(dest_dir):
+ shutil.rmtree(dest_dir) # empty the folder, we'll copy everything again
+ shutil.copytree(src_dir, dest_dir)
+
+
+def copy_service_migrations(service_dir, service_dest_dir, service_name):
+ """
+ Once the migrations are generated, copy them in the correct location
+ :param service_dir: string (path to the folder)
+ :param service_name: string (name of the service)
+ :return: void
+ """
+ # FIXME Django eats this message
+ log.debug("Copying %s migrations to %s" % (service_name, service_dir))
+ migration_dir = os.path.join(service_dest_dir, service_name, "migrations")
+ dest_dir = os.path.join(service_dir, "xos", "synchronizer", "migrations")
+ if os.path.isdir(dest_dir):
+ shutil.rmtree(dest_dir) # empty the folder, we'll copy everything again
+ shutil.copytree(migration_dir, dest_dir)
+
+
+def monkey_patch_migration_template():
+ import django
+ django.setup()
+
+ import django.db.migrations.writer as dj
+ dj.MIGRATION_TEMPLATE = """\
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# -*- coding: utf-8 -*-
+# Generated by Django %(version)s on %(timestamp)s
+from __future__ import unicode_literals
+
+%(imports)s
+
+class Migration(migrations.Migration):
+%(replaces_str)s%(initial_str)s
+ dependencies = [
+%(dependencies)s\
+ ]
+
+ operations = [
+%(operations)s\
+ ]
+"""
+
+# SETTING ENV
+os.environ['LOG_FILE'] = get_abs_path("django.log")
+os.environ['XOS_CONFIG_SCHEMA'] = get_abs_path("migration_cfg_schema.yaml")
+os.environ['XOS_CONFIG_FILE'] = get_abs_path("migration_cfg.yaml")
+os.environ["MIGRATIONS"] = "true"
+# this is populated in case we generate migrations for services and it's used in settings.py
+os.environ["INSTALLED_APPS"] = ""
+
+# PARAMS
+parser = argparse.ArgumentParser(description="XOS Migrations")
+required = parser.add_argument_group('required arguments')
+
+required.add_argument(
+ '-s',
+ '--service',
+ action='append',
+ required=True,
+ dest="service_names",
+ help='The name of the folder containing the service in cord/orchestration/xos_services'
+)
+
+parser.add_argument(
+ '-r',
+ '--repo',
+ default=get_abs_path("~/cord"),
+ dest="repo_root",
+ help='The location of the folder containing the CORD repo root (default to ~/cord)'
+)
+
+# FIXME this is not working with multistructlog
+parser.add_argument(
+ "-v",
+ "--verbose",
+ help="increase log verbosity",
+ action="store_true"
+)
+
+# INITIALIZING LOGGER
+Config.init()
+log = create_logger(Config().get('logging'))
+
+
+def run():
+
+ args = parser.parse_args()
+
+ print_banner(args.repo_root)
+
+ # find absolute path to the code
+ xos_path = get_abs_path(os.path.join(args.repo_root, "orchestration/xos/xos/"))
+ core_dir = get_abs_path(os.path.join(xos_path, "core/models/"))
+ service_base_dir = get_abs_path(os.path.join(xos_path, "../../xos_services/"))
+ service_dest_dir = get_abs_path(os.path.join(xos_path, "services/"))
+
+ # we need to append the xos folder to sys.path
+ original_sys_path = sys.path
+ sys.path.append(xos_path)
+
+ log.info("Services: %s" % ", ".join(args.service_names))
+
+ django_cli_args = ['xos-migrate.py', 'makemigrations']
+
+ # generate the code for each service and create a list of parameters to pass to django
+ app_list = []
+ for service in args.service_names:
+ if service == "core":
+
+ generate_core_models(core_dir)
+ django_cli_args.append("core")
+ else:
+ service_dir = os.path.join(service_base_dir, service)
+ service_name = get_service_name_from_config(service_dir)
+ generate_service_models(service_dir, service_dest_dir, service_name)
+ app_list.append("services.%s" % service_name)
+
+ django_cli_args.append(service_name)
+
+ os.environ["INSTALLED_APPS"] = ",".join(app_list)
+
+ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "xos.settings")
+
+ monkey_patch_migration_template()
+
+ from django.core.management import execute_from_command_line
+
+ execute_from_command_line(django_cli_args)
+
+ # copying migrations back to the service
+ for service in args.service_names:
+ if service == "core":
+ # we don't need to copy migrations for the core
+ continue
+ else:
+ service_dir = os.path.join(service_base_dir, service)
+ service_name = get_service_name_from_config(service_dir)
+ copy_service_migrations(service_dir, service_dest_dir, service_name)
+
+ # restore orginal sys.path
+ sys.path = original_sys_path
diff --git a/lib/xos-migrate/xosmigrate/migration_cfg.yaml b/lib/xos-migrate/xosmigrate/migration_cfg.yaml
new file mode 100644
index 0000000..b3d6e58
--- /dev/null
+++ b/lib/xos-migrate/xosmigrate/migration_cfg.yaml
@@ -0,0 +1,26 @@
+
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+name: migrations
+logging:
+ version: 1
+ handlers:
+ console:
+ class: logging.StreamHandler
+ loggers:
+ 'multistructlog':
+ handlers:
+ - console
diff --git a/lib/xos-migrate/xosmigrate/migration_cfg_schema.yaml b/lib/xos-migrate/xosmigrate/migration_cfg_schema.yaml
new file mode 100644
index 0000000..3a28d64
--- /dev/null
+++ b/lib/xos-migrate/xosmigrate/migration_cfg_schema.yaml
@@ -0,0 +1,16 @@
+
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+type: any
\ No newline at end of file
diff --git a/lib/xos-synchronizer/.gitignore b/lib/xos-synchronizer/.gitignore
new file mode 100644
index 0000000..5be50bc
--- /dev/null
+++ b/lib/xos-synchronizer/.gitignore
@@ -0,0 +1,7 @@
+.noseids
+build
+XosSynchronizer.egg-info
+dist
+.coverage
+coverage.xml
+cover
\ No newline at end of file