now building dockerfiles
diff --git a/xos/core/models/service.py b/xos/core/models/service.py
index 1b800ab..a27199f 100644
--- a/xos/core/models/service.py
+++ b/xos/core/models/service.py
@@ -5,6 +5,7 @@
from core.models.plcorebase import StrippedCharField
from django.db import models
from xos.exceptions import *
+import urlparse
COARSE_KIND = "coarse"
@@ -65,7 +66,7 @@
attrname))
class ServiceController(PlCoreBase):
- xos = models.ForeignKey(XOS, related_name='servicecontrolers', help_text="Pointer to XOS", default=get_xos)
+ xos = models.ForeignKey(XOS, related_name='service_controllers', help_text="Pointer to XOS", default=get_xos)
name = StrippedCharField(max_length=30, help_text="Service Name")
base_url = StrippedCharField(max_length=1024, help_text="Base URL, allows use of relative URLs for resources", null=True, blank=True)
@@ -85,7 +86,7 @@
('docker', 'Docker Container'),
('yaml', 'YAML'))
- service_controller = models.ForeignKey(ServiceController, related_name='servicecontrolerresources',
+ service_controller = models.ForeignKey(ServiceController, related_name='service_controller_resources',
help_text="The Service Controller this resource is associated with")
name = StrippedCharField(max_length=30, help_text="Object Name")
@@ -95,6 +96,13 @@
def __unicode__(self): return u'%s' % (self.name)
+ @property
+ def full_url(self):
+ if self.service_controller and self.service_controller.base_url:
+ return urlparse.urljoin(self.service_controller.base_url, self.url)
+ else:
+ return self.url
+
class Service(PlCoreBase, AttributeMixin):
# when subclassing a service, redefine KIND to describe the new service
KIND = "generic"
diff --git a/xos/onboard/exampleservice/exampleservice-onboard.yaml b/xos/onboard/exampleservice/exampleservice-onboard.yaml
index 67413d0..5948edc 100644
--- a/xos/onboard/exampleservice/exampleservice-onboard.yaml
+++ b/xos/onboard/exampleservice/exampleservice-onboard.yaml
@@ -10,7 +10,7 @@
exampleservice:
type: tosca.nodes.ServiceController
properties:
- base_url: file:/opt/xos/onboard/exampleservice/
+ base_url: file:///opt/xos/onboard/exampleservice/
# The following will concatenate with base_url automatically, if
# base_url is non-null.
models: models.py
diff --git a/xos/synchronizers/onboarding/model-deps b/xos/synchronizers/onboarding/model-deps
index 0a12cd8..4aa86c7 100644
--- a/xos/synchronizers/onboarding/model-deps
+++ b/xos/synchronizers/onboarding/model-deps
@@ -4,5 +4,5 @@
],
"ServiceController": [
"ServiceControllerResource"
- ],
+ ]
}
diff --git a/xos/synchronizers/onboarding/onboarding_synchronizer_config b/xos/synchronizers/onboarding/onboarding_synchronizer_config
index 8ded6ae..d80ff2d 100644
--- a/xos/synchronizers/onboarding/onboarding_synchronizer_config
+++ b/xos/synchronizers/onboarding/onboarding_synchronizer_config
@@ -28,6 +28,7 @@
deleters_dir=/opt/xos/synchronizers/onboarding/deleters
log_file=console
driver=None
+backoff_disabled=True
[feefie]
client_id='vicci_dev_central'
diff --git a/xos/synchronizers/onboarding/steps/sync_servicecontrollerresource.py b/xos/synchronizers/onboarding/steps/sync_servicecontrollerresource.py
index 1d444cb..59ae93f 100644
--- a/xos/synchronizers/onboarding/steps/sync_servicecontrollerresource.py
+++ b/xos/synchronizers/onboarding/steps/sync_servicecontrollerresource.py
@@ -24,9 +24,14 @@
SyncStep.__init__(self, **args)
XOSBuilder.__init__(self)
- def sync_record(self, cp):
- pass
+ def sync_record(self, scr):
+ logger.info("Sync'ing ServiceControllerResource %s" % scr)
+ self.download_resource(scr)
def delete_record(self, m):
pass
+ def fetch_pending(self, deleted=False):
+ pend = super(SyncServiceControllerResource, self).fetch_pending(deleted)
+ return pend
+
diff --git a/xos/synchronizers/onboarding/steps/sync_xos.py b/xos/synchronizers/onboarding/steps/sync_xos.py
new file mode 100644
index 0000000..4c0fe7b
--- /dev/null
+++ b/xos/synchronizers/onboarding/steps/sync_xos.py
@@ -0,0 +1,37 @@
+import os
+import sys
+import base64
+from django.db.models import F, Q
+from xos.config import Config
+from synchronizers.base.syncstep import SyncStep
+from core.models import XOS
+from xos.logger import Logger, logging
+
+# xosbuilder will be in steps/..
+parentdir = os.path.join(os.path.dirname(__file__),"..")
+sys.path.insert(0,parentdir)
+
+from xosbuilder import XOSBuilder
+
+logger = Logger(level=logging.INFO)
+
+class SyncXOS(SyncStep, XOSBuilder):
+ provides=[XOS]
+ observes=XOS
+ requested_interval=0
+
+ def __init__(self, **args):
+ SyncStep.__init__(self, **args)
+ XOSBuilder.__init__(self)
+
+ def sync_record(self, scr):
+ logger.info("Sync'ing XOS %s" % scr)
+ self.build_xos()
+
+ def delete_record(self, m):
+ pass
+
+ def fetch_pending(self, deleted=False):
+ pend = super(SyncXOS, self).fetch_pending(deleted)
+ return pend
+
diff --git a/xos/synchronizers/onboarding/xosbuilder.py b/xos/synchronizers/onboarding/xosbuilder.py
index 4000219..7d4c7e5 100644
--- a/xos/synchronizers/onboarding/xosbuilder.py
+++ b/xos/synchronizers/onboarding/xosbuilder.py
@@ -2,12 +2,10 @@
import base64
import string
import sys
+import urllib2
+import urlparse
import xmlrpclib
-if __name__ == '__main__':
- sys.path.append("/opt/xos")
- os.environ.setdefault("DJANGO_SETTINGS_MODULE", "xos.settings")
-
from xos.config import Config
from core.models import Service, ServiceController, ServiceControllerResource
from xos.logger import Logger, logging
@@ -15,5 +13,125 @@
logger = Logger(level=logging.INFO)
class XOSBuilder(object):
+ UI_KINDS=["models", "admin", "django_library", "rest", "tosca_custom_types", "tosca_resource"]
+ SYNC_KINDS=["synchronizer"]
+
def __init__(self):
- self.source_image = "xosproject/xos"
+ self.source_ui_image = "xosproject/xos"
+ self.source_sync_image = "xosproject/xos-synchronizer-openstack"
+ self.build_dir = "/opt/xos/BUILD/"
+
+ # stuff that has to do with downloading
+
+ def get_dest_dir(self, scr):
+ xos_base = "opt/xos"
+ service_name = scr.service_controller.name
+ base_dirs = {"models": "%s/services/%s/" % (xos_base, service_name),
+ "admin": "%s/services/%s/" % (xos_base, service_name),
+ "django_library": "%s/services/%s/" % (xos_base, service_name),
+ "synchronizer": "%s/synchronizers/%s/" % (xos_base, service_name),
+ "tosca_custom_types": "%s/tosca/custom_types/" % (xos_base),
+ "tosca_resource": "%s/tosca/resources/" % (xos_base)}
+ return base_dirs[scr.kind]
+
+ def get_build_fn(self, scr):
+ dest_dir = self.get_dest_dir(scr)
+ dest_fn = os.path.split(urlparse.urlsplit(scr.full_url).path)[-1]
+ return os.path.join(dest_dir, dest_fn)
+
+ def get_download_fn(self, scr):
+ dest_fn = self.get_build_fn(scr)
+ return os.path.join(self.build_dir, dest_fn)
+
+ def read_manifest(self, scr, fn):
+ manifest = []
+ manifest_lines = file(fn).readlines()
+ manifest_lines = [x.strip() for x in manifest_lines]
+ manifest_lines = [x for x in manifest.lines if x]
+ for line in manifest_lines:
+ url_parts = urlsplit(scr.full_url)
+ url_parts.path = os.path.join(os.path.join(*os.path.split(url_parts).path[:-1]),line)
+ url = urlparse.urlunsplit(url_parts)
+
+ build_fn = os.path.join(self.get_dest_dir(scr), line)
+
+ manifest.append( (url, download_fn, build_fn) )
+ return manifest
+
+ def download_file(self, url, dest_fn):
+ logger.info("Download %s to %s" % (url, dest_fn))
+ if not os.path.exists(os.path.dirname(dest_fn)):
+ os.makedirs(os.path.dirname(dest_fn))
+ obj = urllib2.urlopen(url)
+ file(dest_fn,"w").write(obj.read())
+
+ def download_resource(self, scr):
+ if scr.format == "manifest":
+ manifest_fn = self.get_download_fn(scr)
+ self.download_file(scr.full_url, manifest_fn)
+ mainfest = self.read_manifest(scr, manifest_fn)
+ for (url, download_fn, build_fn) in manifest:
+ self.download_file(url, download_fn)
+ else:
+ self.download_file(scr.full_url, self.get_download_fn(scr))
+
+ def get_docker_lines(self, scr):
+ if scr.format == "manifest":
+ manifest_fn = self.get_download_fn(scr)
+ manifest = self.read_manifest(scr, manifest_fn)
+ lines = []
+ for (url, download_fn, build_fn) in manifest:
+ lines.append("ADD %s /%s" % (build_fn, build_fn))
+ return lines
+ else:
+ build_fn = self.get_build_fn(scr)
+ return ["ADD %s /%s" % (build_fn, build_fn)]
+
+ def get_controller_docker_lines(self, controller, kinds):
+ dockerfile=[]
+ for scr in controller.service_controller_resources.all():
+ if scr.kind in kinds:
+ lines = self.get_docker_lines(scr)
+ dockerfile = dockerfile + lines
+ return dockerfile
+
+ # stuff that has to do with building
+
+ def create_ui_dockerfile(self):
+ dockerfile_fn = "Dockerfile.UI"
+
+ dockerfile = ["FROM %s" % self.source_ui_image]
+ for controller in ServiceController.objects.all():
+ dockerfile = dockerfile + self.get_controller_docker_lines(controller, self.UI_KINDS)
+
+ file(os.path.join(self.build_dir, dockerfile_fn), "w").write("\n".join(dockerfile)+"\n")
+
+ return [dockerfile_fn]
+
+ def create_synchronizer_dockerfile(self, controller):
+ lines = self.get_controller_docker_lines(controller, self.SYNC_KINDS)
+ if not lines:
+ return []
+
+ dockerfile_fn = "Dockerfile.%s" % controller.name
+ dockerfile = ["FROM %s" % self.source_sync_image]
+ dockerfile = dockerfile + lines
+ file(os.path.join(self.build_dir, dockerfile_fn), "w").writelines(dockerfile)
+
+ return [dockerfile_fn]
+
+ def run_docker(self, dockerfile_fn):
+ pass
+
+ def build_xos(self):
+ dockerfiles=[]
+ dockerfiles = dockerfiles + self.create_ui_dockerfile()
+
+ for controller in ServiceController.objects.all():
+ dockerfiles = dockerfiles + self.create_synchronizer_dockerfile(controller)
+
+ for dockerfile in dockerfiles:
+ self.run_docker(dockerfile)
+
+
+