CORD-1081 corebuilder tool
Change-Id: I4bf06242d9a88584eb3f84b7931ec0f06071d78b
diff --git a/containers/xos/Dockerfile.UI b/containers/xos/Dockerfile.UI
new file mode 100644
index 0000000..cd20156
--- /dev/null
+++ b/containers/xos/Dockerfile.UI
@@ -0,0 +1,10 @@
+FROM xosproject/xos:candidate
+
+ARG XOS_GIT_COMMIT_HASH=unknown
+ARG XOS_GIT_COMMIT_DATE=unknown
+
+LABEL XOS_GIT_COMMIT_HASH=$XOS_GIT_COMMIT_HASH
+LABEL XOS_GIT_COMMIT_DATE=$XOS_GIT_COMMIT_DATE
+
+# Install file from corebuilder
+ADD containers/xos/BUILD /
diff --git a/containers/xos/Dockerfile.corebuilder b/containers/xos/Dockerfile.corebuilder
new file mode 100644
index 0000000..bf0a508
--- /dev/null
+++ b/containers/xos/Dockerfile.corebuilder
@@ -0,0 +1,17 @@
+FROM xosproject/xos-base:candidate
+
+ARG XOS_GIT_COMMIT_HASH=unknown
+ARG XOS_GIT_COMMIT_DATE=unknown
+
+LABEL XOS_GIT_COMMIT_HASH=$XOS_GIT_COMMIT_HASH
+LABEL XOS_GIT_COMMIT_DATE=$XOS_GIT_COMMIT_DATE
+
+# Install the corebuilder tools and the tosca custom_types that it needs
+ADD xos/tools/corebuilder /opt/xos/tools/corebuilder
+ADD xos/tosca/custom_types /opt/xos/tools/corebuilder/custom_types
+
+ENV HOME /root
+WORKDIR /opt/xos/tools/corebuilder
+
+ENTRYPOINT ["/usr/bin/python", "corebuilder.py"]
+
diff --git a/containers/xos/Makefile b/containers/xos/Makefile
index e096612..f1713d7 100644
--- a/containers/xos/Makefile
+++ b/containers/xos/Makefile
@@ -65,10 +65,26 @@
--build-arg XOS_GIT_COMMIT_DATE="${XOS_GIT_COMMIT_DATE}" \
-f Dockerfile.client -t xosproject/xos-client ${BUILD_ARGS} ../..
rm -rf tmp.chameleon
+ docker tag xosproject/xos-client:latest xosproject/xos-client:candidate
+
+corebuilder:
+ sudo docker build --no-cache=${NO_DOCKER_CACHE} --rm \
+ --build-arg XOS_GIT_COMMIT_HASH="${XOS_GIT_COMMIT_HASH}" \
+ --build-arg XOS_GIT_COMMIT_DATE="${XOS_GIT_COMMIT_DATE}" \
+ -f Dockerfile.corebuilder -t xosproject/xos-corebuilder ${BUILD_ARGS} ../..
+ docker tag xosproject/xos-corebuilder:latest xosproject/xos-corebuilder:candidate
+
+ui:
+ sudo docker build --no-cache=${NO_DOCKER_CACHE} --rm \
+ --build-arg XOS_GIT_COMMIT_HASH="${XOS_GIT_COMMIT_HASH}" \
+ --build-arg XOS_GIT_COMMIT_DATE="${XOS_GIT_COMMIT_DATE}" \
+ -f Dockerfile.UI -t xosproject/xos-ui ${BUILD_ARGS} ../..
+ docker tag xosproject/xos-ui:latest xosproject/xos-ui:candidate
synchronizer-base:
sudo docker build --no-cache=${NO_DOCKER_CACHE} --rm \
-f Dockerfile.synchronizer-base -t xosproject/xos-synchronizer-base ${BUILD_ARGS} ../..
+ docker tag xosproject/xos-synchronizer-base:latest xosproject/xos-synchronizer-base:candidate
run:
sudo docker run -d --name ${CONTAINER_NAME} -p 80:8000 \
diff --git a/group_vars/all b/group_vars/all
index 876bee0..e4d2f61 100644
--- a/group_vars/all
+++ b/group_vars/all
@@ -51,6 +51,11 @@
dockerfile: "Dockerfile.xos-gui-extension-builder"
pull: False
publish: False
+ - name: "xosproject/xos-corebuilder"
+ path: "{{ cord_dir }}/orchestration/xos"
+ dockerfile: "containers/xos/Dockerfile.corebuilder"
+ pull: False
+ publish: True
build_optional_images: False
diff --git a/xos/core/admin.py b/xos/core/admin.py
index 0cf0be5..9e7a86d 100644
--- a/xos/core/admin.py
+++ b/xos/core/admin.py
@@ -1148,7 +1148,7 @@
list_display = ("backend_status_icon", "name",)
list_display_links = ('backend_status_icon', 'name',)
fieldList = ["name", "ui_port", "bootstrap_ui_port", "docker_project_name", "db_container_name", "redis_container_name", "enable_build", "frontend_only",
- "source_ui_image", "extra_hosts", "no_start"]
+ "source_ui_image", "dest_ui_image", "cert_chain_name", "extra_hosts", "no_start", "no_build"]
fieldsets = [
(None, {'fields': fieldList, 'classes': ['suit-tab suit-tab-general']})]
inlines = [XOSVolumeInline]
diff --git a/xos/core/models/xosmodel.py b/xos/core/models/xosmodel.py
index d5c2f5e..d6c5541 100644
--- a/xos/core/models/xosmodel.py
+++ b/xos/core/models/xosmodel.py
@@ -17,8 +17,11 @@
enable_build = models.BooleanField(help_text="True if Onboarding Synchronizer should build XOS as necessary", default=True)
frontend_only = models.BooleanField(help_text="If True, XOS will not start synchronizer containers", default=False)
source_ui_image = StrippedCharField(max_length=200, default="xosproject/xos")
+ dest_ui_image = StrippedCharField(max_length=200, default="xosproject/xos-ui")
+ cert_chain_name = StrippedCharField(max_length=200, default="/opt/cord_profile/im_cert_chain.pem")
extra_hosts = StrippedCharField(max_length=1024, help_text="list of hostname mappings that will be passed to docker-compose", null=True, blank=True)
no_start = models.BooleanField(help_text="Do not start the XOS UI inside of the UI docker container", default=False)
+ no_build = models.BooleanField(help_text="Do not build the XOS UI container image", default=False)
def __unicode__(self): return u'%s' % (self.name)
diff --git a/xos/synchronizers/onboarding/steps/sync_xos.py b/xos/synchronizers/onboarding/steps/sync_xos.py
index 60f5d74..b1ecf98 100644
--- a/xos/synchronizers/onboarding/steps/sync_xos.py
+++ b/xos/synchronizers/onboarding/steps/sync_xos.py
@@ -52,7 +52,11 @@
self.create_docker_compose()
- dockerfiles = [self.create_ui_dockerfile()]
+ if xos.no_build:
+ dockerfiles = []
+ else:
+ dockerfiles = [self.create_ui_dockerfile()]
+
tenant_fields = {"dockerfiles": dockerfiles,
"build_dir": self.build_dir,
"docker_project_name": xos.docker_project_name,
diff --git a/xos/synchronizers/onboarding/xosbuilder.py b/xos/synchronizers/onboarding/xosbuilder.py
index ad7efd4..400fe7c 100644
--- a/xos/synchronizers/onboarding/xosbuilder.py
+++ b/xos/synchronizers/onboarding/xosbuilder.py
@@ -338,7 +338,7 @@
# eventually xos_ui will go away, and only xos_core shall remain.
containers["xos_ui"] = {
- "image": "xosproject/xos-ui",
+ "image": xos.dest_ui_image,
"command": "python /opt/xos/manage.py runserver 0.0.0.0:%d --insecure --makemigrations" % xos.ui_port,
"networks": networks,
"ports": {"%d" % xos.ui_port: "%d" % xos.ui_port},
@@ -350,7 +350,7 @@
core_volume_list = volume_list + [{"host_path": "/var/run/docker.sock", "container_path": "/var/run/docker.sock", "read_only": False}]
containers["xos_core"] = {
- "image": "xosproject/xos-ui",
+ "image": xos.dest_ui_image,
"command": 'bash -c "cd coreapi; bash ./start_coreapi.sh"',
"networks": networks,
"ports": {"50055": "50055", "50051" : "50051"},
diff --git a/xos/tools/corebuilder/corebuilder.py b/xos/tools/corebuilder/corebuilder.py
new file mode 100644
index 0000000..c2a584a
--- /dev/null
+++ b/xos/tools/corebuilder/corebuilder.py
@@ -0,0 +1,285 @@
+""" CoreBuilder
+
+ Read XOS Tosca Onboarding Recipes and generate a BUILD directory.
+
+ Arguments:
+ A list of onboarding recipes. Except this list to originate from
+ platform-install's service inventory in the profile manifest.
+
+ Output:
+ /opt/xos_corebuilder/BUILD, populated with files from services
+
+ Example:
+ # for testing, run from inside a UI container
+ python ./corebuilder.py \
+ /opt/xos_services/olt/xos/volt-onboard.yaml \
+ /opt/xos_services/vtn/xos/vtn-onboard.yaml \
+ /opt/xos_services/openstack/xos/openstack-onboard.yaml \
+ /opt/xos_services/onos-service/xos/onos-onboard.yaml \
+ /opt/xos_services/vrouter/xos/vrouter-onboard.yaml \
+ /opt/xos_services/vsg/xos/vsg-onboard.yaml \
+ /opt/xos_services/vtr/xos/vtr-onboard.yaml \
+ /opt/xos_services/fabric/xos/fabric-onboard.yaml \
+ /opt/xos_services/exampleservice/xos/exampleservice-onboard.yaml \
+ /opt/xos_services/monitoring/xos/monitoring-onboard.yaml \
+ /opt/xos_libraries/ng-xos-lib/ng-xos-lib-onboard.yaml
+
+ # (hypothetical) run from build container
+ python ./corebuilder.py \
+ /opt/cord/onos-apps/apps/olt/xos/volt-onboard.yaml \
+ /opt/cord/onos-apps/apps/vtn/xos/vtn-onboard.yaml \
+ /opt/cord/orchestration/xos_services/openstack/xos/openstack-onboard.yaml \
+ /opt/cord/orchestration/xos_services/onos-service/xos/onos-onboard.yaml \
+ /opt/cord/orchestration/xos_services/vrouter/xos/vrouter-onboard.yaml \
+ /opt/cord/orchestration/xos_services/vsg/xos/vsg-onboard.yaml \
+ /opt/cord/orchestration/xos_services/vtr/xos/vtr-onboard.yaml \
+ /opt/cord/orchestration/xos_services/fabric/xos/fabric-onboard.yaml \
+ /opt/cord/orchestration/xos_services/exampleservice/xos/exampleservice-onboard.yaml \
+ /opt/cord/orchestration/xos_services/monitoring/xos/monitoring-onboard.yaml \
+ /opt/cord/orchestration/xos_libraries/ng-xos-lib/ng-xos-lib-onboard.yaml
+"""
+
+import os
+import pdb
+import shutil
+import sys
+import tempfile
+import traceback
+import urlparse
+
+from toscaparser.tosca_template import ToscaTemplate
+
+BUILD_DIR = "/opt/xos_corebuilder/BUILD"
+
+def makedirs_if_noexist(pathname):
+ if not os.path.exists(pathname):
+ os.makedirs(pathname)
+
+class XOSCoreBuilder(object):
+ def __init__(self, recipe_list, parent_dir=None):
+ # TOSCA will look for imports using a relative path from where the
+ # template file is located, so we have to put the template file
+ # in a specific place.
+ if not parent_dir:
+ parent_dir = os.getcwd()
+
+ self.parent_dir = parent_dir
+
+ # list of resources in the form (src_fn, dest_fn)
+ self.resources = []
+
+ # list of __init__.py files that should be ensured
+ self.inits = []
+
+ self.app_names = []
+
+ for recipe in recipe_list:
+ tosca_yaml = open(recipe).read()
+ self.execute_recipe(tosca_yaml)
+
+ def get_property_default(self, nodetemplate, name, default=None):
+ props = nodetemplate.get_properties()
+ if props and name in props.keys():
+ return props[name].value
+ return default
+
+ def get_dest_dir(self, kind, service_name):
+ xos_base = "opt/xos"
+ base_dirs = {"models": "%s/services/%s/" % (xos_base, service_name),
+ "xproto": "%s/services/%s/xproto/" % (xos_base, service_name),
+ "admin": "%s/services/%s/" % (xos_base, service_name),
+ "admin_template": "%s/services/%s/templates/" % (xos_base, service_name),
+ "django_library": "%s/services/%s/" % (xos_base, service_name),
+ "synchronizer": "%s/synchronizers/%s/" % (xos_base, service_name),
+ "tosca_custom_types": "%s/tosca/custom_types/" % (xos_base),
+ "tosca_resource": "%s/tosca/resources/" % (xos_base),
+ "rest_service": "%s/api/service/" % (xos_base),
+ "rest_tenant": "%s/api/tenant/" % (xos_base),
+ "private_key": "%s/services/%s/keys/" % (xos_base, service_name),
+ "public_key": "%s/services/%s/keys/" % (xos_base, service_name),
+ "vendor_js": "%s/core/xoslib/static/vendor/" % (xos_base)}
+ dest_dir = base_dirs[kind]
+
+ return dest_dir
+
+ def fixup_path(self, fn):
+ """ This is to maintain compatibility with the legacy Onboarding
+ synchronizer and recipes, which has some oddly-named directories
+ """
+
+# if fn.startswith("/opt/xos/key_import"):
+# fn = "/opt/cord_profile/key_import" + fn[19:]
+
+ fixups = ( ("/opt/xos_services/olt/", "/opt/cord/onos-apps/apps/olt/"),
+ ("/opt/xos_services/vtn/", "/opt/cord/onos-apps/apps/vtn/"),
+ ("/opt/xos_services/", "/opt/cord/orchestration/xos_services/"),
+ ("/opt/xos_libraries/", "/opt/cord/orchestration/xos_libraries/") )
+
+ for (pattern, replace) in fixups:
+ if fn.startswith(pattern):
+ fn = replace + fn[len(pattern):]
+
+ return fn
+
+ def execute_recipe(self, tosca_yaml):
+ tmp_pathname = None
+ try:
+ (tmp_handle, tmp_pathname) = tempfile.mkstemp(dir=self.parent_dir, suffix=".yaml")
+ os.write(tmp_handle, tosca_yaml)
+ os.close(tmp_handle)
+
+ template = ToscaTemplate(tmp_pathname)
+ except:
+ traceback.print_exc()
+ raise
+ finally:
+ if tmp_pathname:
+ os.remove(tmp_pathname)
+
+ # Only one model (ServiceController aka Library), so no need to sort
+ # dependencies...
+
+ for nodetemplate in template.nodetemplates:
+ self.execute_nodetemplate(nodetemplate)
+
+ def execute_nodetemplate(self, nodetemplate):
+ if nodetemplate.type == "tosca.nodes.ServiceController":
+ self.execute_servicecontroller(nodetemplate)
+ elif nodetemplate.type == "tosca.nodes.Library":
+ # Library works just like ServiceController
+ self.execute_servicecontroller(nodetemplate)
+ else:
+ raise Exception("Nodetemplate %s's type %s is not a known resource" % (nodetemplate.name, nodetemplate.type))
+
+ def execute_servicecontroller(self, nodetemplate):
+ service_name = nodetemplate.name
+ if "#" in service_name:
+ service_name = service_name.split("#")[1]
+
+ base = self.get_property_default(nodetemplate, "base_url", None)
+
+ copyin_resources = ("xproto", "models", "admin", "admin_template", "django_library", "tosca_custom_types", "tosca_resource",
+ "rest_service", "rest_tenant", "private_key", "public_key", "vendor_js")
+
+ for k in copyin_resources:
+ v = self.get_property_default(nodetemplate, k, None)
+ if not v:
+ continue
+
+ # Private keys should not be installed to core, only synchronizers
+ if (k=="private_key"):
+ continue
+
+ # Public keys should be volume mounted in /opt/cord_profile
+ if (k=="public_key"):
+ continue
+
+ # If the ServiceController has models, then add it to the list of
+ # django apps.
+ if (k=="models"):
+ self.app_names.append(service_name)
+
+ # filenames can be comma-separated
+ for src_fn in v.split(","):
+ src_fn = src_fn.strip()
+
+ # parse the "subdirectory:name" syntax
+ subdirectory = ""
+ if (" " in src_fn):
+ parts=src_fn.split()
+ for part in parts[:-1]:
+ if ":" in part:
+ (lhs, rhs) = part.split(":", 1)
+ if lhs=="subdirectory":
+ subdirectory=rhs
+ else:
+ raise Exception("Malformed value %s" % value)
+ else:
+ raise Exception("Malformed value %s" % value)
+ src_fn = parts[-1]
+
+ # apply base_url to src_fn
+ if base:
+ src_fn = urlparse.urljoin(base, src_fn)
+
+ # ensure that it's a file:// url
+ if not src_fn.startswith("file://"):
+ raise Exception("%s does not start with file://" % src_fn)
+ src_fn = src_fn[7:]
+
+ src_fn = self.fixup_path(src_fn)
+
+ if not os.path.exists(src_fn):
+ raise Exception("%s does not exist" % src_fn)
+
+ dest_dir = self.get_dest_dir(k, service_name)
+ dest_fn = os.path.join(dest_dir, subdirectory, os.path.basename(src_fn))
+
+ self.resources.append( (k, src_fn, dest_fn) )
+
+ # add __init__.py files anywhere that we created a new
+ # directory.
+ if k in ["admin", "models", "rest_service", "rest_tenant"]:
+ if dest_dir not in self.inits:
+ self.inits.append(dest_dir)
+
+ if subdirectory:
+ dir = dest_dir
+ for part in subdirectory.split("/"):
+ dir = os.path.join(dir, part)
+ if dir not in self.inits:
+ self.inits.append(dir)
+
+ def build(self):
+ # Destroy anything in the old build directory
+ if os.path.exists(BUILD_DIR):
+ for dir in os.listdir(BUILD_DIR):
+ shutil.rmtree(os.path.join(BUILD_DIR, dir))
+
+ # Copy all of the resources into the build directory
+ for (kind, src_fn, dest_fn) in self.resources:
+# if (kind == "xproto"):
+# build_dest_dir = os.path.join(BUILD_DIR, os.path.dirname(dest_fn))
+
+ # TODO: If we wanted to statically compile xproto files, then
+ # this is where we could do it. src_fn would be the name of
+ # the xproto file, and build_dest_dir would be the place
+ # to store the generated files.
+
+ build_dest_fn = os.path.join(BUILD_DIR, dest_fn)
+ makedirs_if_noexist(os.path.dirname(build_dest_fn))
+ shutil.copyfile(src_fn, build_dest_fn)
+
+ # Create the __init__.py files
+ for fn in self.inits:
+ build_dest_fn = os.path.join(BUILD_DIR, fn, "__init__.py")
+ makedirs_if_noexist(os.path.dirname(build_dest_fn))
+ file(build_dest_fn, "w").write("")
+
+ # Generate the migration list
+ mig_list_fn = os.path.join(BUILD_DIR, "opt/xos/xos", "xosbuilder_migration_list")
+ makedirs_if_noexist(os.path.dirname(mig_list_fn))
+ file(mig_list_fn, "w").write("\n".join(self.app_names)+"\n")
+
+ # Generate the app list
+ app_list_fn = os.path.join(BUILD_DIR, "opt/xos/xos", "xosbuilder_app_list")
+ makedirs_if_noexist(os.path.dirname(app_list_fn))
+ file(app_list_fn, "w").write("\n".join(["services.%s" % x for x in self.app_names])+"\n")
+
+def main():
+ if len(sys.argv)<=1:
+ print >> sys.stderr, "Syntax: corebuilder.py [recipe1, recipe2, ...]"
+
+ builder = XOSCoreBuilder(sys.argv[1:])
+ builder.build()
+
+if __name__ == "__main__":
+ main()
+
+
+
+
+
+
+
+
diff --git a/xos/tools/corebuilder/headnode_corebuilder b/xos/tools/corebuilder/headnode_corebuilder
new file mode 100644
index 0000000..f3ffecf
--- /dev/null
+++ b/xos/tools/corebuilder/headnode_corebuilder
@@ -0,0 +1,6 @@
+#! /bin/bash
+
+# Runs the corebuilder tools from inside the xosproject/corebuilder container
+# This script is setup to be run from the headnode, using paths based in /opt/cord
+
+docker run -it -v /opt/cord:/opt/cord:ro -v /opt/cord/orchestration/xos/containers/xos/BUILD:/opt/xos_corebuilder/BUILD xosproject/xos-corebuilder:candidate $@
diff --git a/xos/tools/xossh b/xos/tools/xossh
index e71b5f7..fa61ebe 100755
--- a/xos/tools/xossh
+++ b/xos/tools/xossh
@@ -2,4 +2,4 @@
# This is a stub for launching xossh in the xosproject/xos-client container
-docker run -it -v /opt/cord_profile/im_cert_chain.pem:/usr/local/share/ca-certificates/local_certs.crt xosproject/xos-client:candidate -u xosadmin@opencord.org -p `cat /opt/cord/build/platform-install/credentials/xosadmin@opencord.org`
\ No newline at end of file
+docker run --rm -it -v /opt/cord_profile/im_cert_chain.pem:/usr/local/share/ca-certificates/local_certs.crt xosproject/xos-client:candidate -u xosadmin@opencord.org -p `cat /opt/cord/build/platform-install/credentials/xosadmin@opencord.org`
\ No newline at end of file
diff --git a/xos/tosca/custom_types/xos.m4 b/xos/tosca/custom_types/xos.m4
index 03f16bd..8d22c7d 100644
--- a/xos/tosca/custom_types/xos.m4
+++ b/xos/tosca/custom_types/xos.m4
@@ -35,6 +35,14 @@
type: string
required: false
description: Source UI docker image name
+ dest_ui_image:
+ type: string
+ required: false
+ description: Dest UI docker image name
+ cert_chain_name:
+ type: string
+ required: false
+ description: Name of file that holds certificate chain
enable_build:
type: boolean
required: false
@@ -50,7 +58,11 @@
no_start:
type: boolean
required: false
- description: Wheter to start the Django server or not
+ description: Whether to start the Django server or not
+ no_build:
+ type: boolean
+ required: false
+ description: Whether to build the core UI image
tosca.nodes.XOSVolume:
derived_from: tosca.nodes.Root
diff --git a/xos/tosca/custom_types/xos.yaml b/xos/tosca/custom_types/xos.yaml
index 8439771..93b13db 100644
--- a/xos/tosca/custom_types/xos.yaml
+++ b/xos/tosca/custom_types/xos.yaml
@@ -65,6 +65,14 @@
type: string
required: false
description: Source UI docker image name
+ dest_ui_image:
+ type: string
+ required: false
+ description: Dest UI docker image name
+ cert_chain_name:
+ type: string
+ required: false
+ description: Name of file that holds certificate chain
enable_build:
type: boolean
required: false
@@ -80,7 +88,11 @@
no_start:
type: boolean
required: false
- description: Wheter to start the Django server or not
+ description: Whether to start the Django server or not
+ no_build:
+ type: boolean
+ required: false
+ description: Whether to build the core UI image
tosca.nodes.XOSVolume:
derived_from: tosca.nodes.Root
diff --git a/xos/tosca/resources/xosmodel.py b/xos/tosca/resources/xosmodel.py
index a935b65..37a60ad 100644
--- a/xos/tosca/resources/xosmodel.py
+++ b/xos/tosca/resources/xosmodel.py
@@ -7,7 +7,8 @@
xos_model = XOS
copyin_props = [
"ui_port", "bootstrap_ui_port", "docker_project_name", "db_container_name", "redis_container_name",
- "enable_build", "frontend_only", "source_ui_image", "extra_hosts", "no_start"
+ "enable_build", "frontend_only", "source_ui_image", "extra_hosts", "no_start", "no_build",
+ "dest_ui_image", "cert_chain_name",
]