dependency checking in service controllers
break out loadablemodule from servicecontroller
Change-Id: I9d2d8f6d1ee14de9976354714ea68e8e412de7c5
diff --git a/xos/api/utility/onboarding.py b/xos/api/utility/onboarding.py
index 1144387..c04be52 100644
--- a/xos/api/utility/onboarding.py
+++ b/xos/api/utility/onboarding.py
@@ -69,7 +69,7 @@
result.append( ("XOS", self.is_ready(xos)) )
- for sc in xos.service_controllers.all():
+ for sc in xos.loadable_modules.all():
result.append( (sc.name, self.is_ready(sc)) )
result = "\n".join( ["%s: %s" % (x[0], x[1]) for x in result] )
@@ -86,7 +86,7 @@
xos=xos[0]
result = []
- for sc in xos.service_controllers.all():
+ for sc in xos.loadable_modules.all():
result.append(sc.name)
return HttpResponse( json.dumps(result), content_type="application/javascript")
@@ -98,7 +98,7 @@
xos=xos[0]
- sc=xos.service_controllers.filter(name=service)
+ sc=xos.loadable_modules.filter(name=service)
if not sc:
return HttpResponse("Not Found", status_code=404)
diff --git a/xos/core/admin.py b/xos/core/admin.py
index fa28734..83492ab 100644
--- a/xos/core/admin.py
+++ b/xos/core/admin.py
@@ -1032,10 +1032,9 @@
class ServiceAdmin(XOSBaseAdmin):
- list_display = ("backend_status_icon", "name", "kind",
- "versionNumber", "enabled", "published")
+ list_display = ("backend_status_icon", "name", "kind", "enabled", "published")
list_display_links = ('backend_status_icon', 'name', )
- fieldList = ["backend_status_text", "name", "kind", "description", "controller", "versionNumber", "enabled", "published",
+ fieldList = ["backend_status_text", "name", "kind", "description", "controller", "enabled", "published",
"view_url", "icon_url", "public_key", "private_key_fn", "service_specific_attribute", "service_specific_id"]
fieldsets = [
(None, {'fields': fieldList, 'classes': ['suit-tab suit-tab-general']})]
@@ -1061,7 +1060,7 @@
class ServiceControllerAdmin(XOSBaseAdmin):
list_display = ("backend_status_icon", "name",)
list_display_links = ('backend_status_icon', 'name',)
- fieldList = ["backend_status_text", "name", "xos", "base_url", "synchronizer_run", "synchronizer_config", "no_start"]
+ fieldList = ["backend_status_text", "name", "xos", "version", "provides", "requires", "base_url", "synchronizer_run", "synchronizer_config", "no_start"]
fieldsets = [
(None, {'fields': fieldList, 'classes': ['suit-tab suit-tab-general']})]
inlines = [ServiceControllerResourceInline]
diff --git a/xos/core/models/__init__.py b/xos/core/models/__init__.py
index 3630d9d..abf8da2 100644
--- a/xos/core/models/__init__.py
+++ b/xos/core/models/__init__.py
@@ -4,7 +4,7 @@
from .xosmodel import XOS, XOSVolume
from .service import Service, Tenant, TenantWithContainer, CoarseTenant, ServicePrivilege, TenantRoot, TenantRootPrivilege, TenantRootRole, TenantPrivilege, TenantRole, Subscriber, Provider
from .service import ServiceAttribute, TenantAttribute, ServiceRole
-from .service import ServiceController, ServiceControllerResource
+from .service import ServiceController, ServiceControllerResource, LoadableModule, LoadableModuleResource
from .tag import Tag
from .role import Role
from .site import Site, Deployment, DeploymentRole, DeploymentPrivilege, Controller, ControllerRole, ControllerSite, SiteDeployment,Diag
diff --git a/xos/core/models/service.py b/xos/core/models/service.py
index a9b3aea..2685fe4 100644
--- a/xos/core/models/service.py
+++ b/xos/core/models/service.py
@@ -1,5 +1,8 @@
import json
+import operator
+
from operator import attrgetter
+from distutils.version import LooseVersion
from core.models import PlCoreBase, PlCoreBaseManager, SingletonModel, XOS
from core.models.plcorebase import StrippedCharField
@@ -16,7 +19,6 @@
return xos[0]
else:
return None
-
class AttributeMixin(object):
# helper for extracting things from a json-encoded
@@ -65,27 +67,90 @@
None,
attrname))
-class ServiceController(PlCoreBase):
- xos = models.ForeignKey(XOS, related_name='service_controllers', help_text="Pointer to XOS", default=get_xos)
+class LoadableModule(PlCoreBase):
+ xos = models.ForeignKey(XOS, related_name='loadable_modules', help_text="Pointer to XOS", default=get_xos)
name = StrippedCharField(max_length=30, help_text="Service Name")
base_url = StrippedCharField(max_length=1024, help_text="Base URL, allows use of relative URLs for resources", null=True, blank=True)
- synchronizer_run = StrippedCharField(max_length=1024, help_text="synchronizer run command", null=True, blank=True)
- synchronizer_config = StrippedCharField(max_length=1024, help_text="synchronizer config file", null=True, blank=True)
-
- no_start = models.BooleanField(help_text="Do not start the XOS UI inside of the UI docker container", default=False)
+ version = StrippedCharField(blank=True, null=True,
+ max_length=30, help_text="Version of Service Controller", default = "1.0.0")
+ provides = StrippedCharField(blank=True, null=True,
+ max_length=254, help_text="Comma-separated list of things provided")
+ requires = StrippedCharField(blank=True, null=True,
+ max_length=254, help_text="Comma-separated list of required Service Controllers")
def __unicode__(self): return u'%s' % (self.name)
def save(self, *args, **kwargs):
- super(ServiceController, self).save(*args, **kwargs)
+ super(LoadableModule, self).save(*args, **kwargs)
if self.xos:
# force XOS to rebuild
# XXX somewhat hackish XXX
self.xos.save(update_fields=["updated"])
-class ServiceControllerResource(PlCoreBase):
+ def get_provides_list(self):
+ prov_list = []
+ if self.provides and self.provides.strip():
+ for prov in self.provides.split(","):
+ prov=prov.strip()
+ if "=" in prov:
+ (name, version) = prov.split("=",1)
+ name = name.strip()
+ version = version.strip()
+ else:
+ name = prov
+ version = "1.0.0"
+ prov_list.append( {"name": name, "version": version} )
+
+ # every controller provides itself
+ prov_list.append( {"name": self.name, "version": self.version} )
+
+ return prov_list
+
+
+ @classmethod
+ def dependency_check(cls, dep_list):
+ missing = []
+ satisfied = []
+ operators = {">=": operator.ge,
+ "<=": operator.le,
+ ">": operator.gt,
+ "<": operator.lt,
+ "!=": operator.ne,
+ "=": operator.eq}
+ for dep in dep_list:
+ dep = dep.strip()
+ name = dep
+ version = None
+ this_op = None
+ for op in operators.keys():
+ if op in dep:
+ (name, version) = dep.split(op,1)
+ name = name.strip()
+ version = version.strip()
+ this_op = operators[op]
+ break
+ found=False
+ scs = ServiceController.objects.all()
+ for sc in scs:
+ for provide in sc.get_provides_list():
+ if (provide["name"] != name):
+ continue
+ if not this_op:
+ satisfied.append(sc)
+ found=True
+ break
+ elif this_op(LooseVersion(provide["version"]), LooseVersion(version)):
+ satisfied.append(sc)
+ found=True
+ break
+ if not found:
+ missing.append(dep)
+
+ return (satisfied, missing)
+
+class LoadableModuleResource(PlCoreBase):
KIND_CHOICES = (('models', 'Models'),
('admin', 'Admin'),
('admin_template', 'Admin Template'),
@@ -104,8 +169,8 @@
('yaml', 'YAML'),
('raw', 'raw'))
- service_controller = models.ForeignKey(ServiceController, related_name='service_controller_resources',
- help_text="The Service Controller this resource is associated with")
+ loadable_module = models.ForeignKey(LoadableModule, related_name='loadable_module_resources',
+ help_text="The Loadable Module this resource is associated with")
name = StrippedCharField(max_length=30, help_text="Object Name")
subdirectory = StrippedCharField(max_length=1024, help_text="optional subdirectory", null=True, blank=True)
@@ -117,11 +182,21 @@
@property
def full_url(self):
- if self.service_controller and self.service_controller.base_url:
- return urlparse.urljoin(self.service_controller.base_url, self.url)
+ if self.loadable_module and self.loadable_module.base_url:
+ return urlparse.urljoin(self.loadable_module.base_url, self.url)
else:
return self.url
+class ServiceController(LoadableModule):
+ synchronizer_run = StrippedCharField(max_length=1024, help_text="synchronizer run command", null=True, blank=True)
+ synchronizer_config = StrippedCharField(max_length=1024, help_text="synchronizer config file", null=True, blank=True)
+
+ no_start = models.BooleanField(help_text="Do not start the XOS UI inside of the UI docker container", default=False)
+
+class ServiceControllerResource(LoadableModuleResource):
+ class Meta:
+ proxy = True
+
class Service(PlCoreBase, AttributeMixin):
# when subclassing a service, redefine KIND to describe the new service
KIND = "generic"
@@ -133,7 +208,7 @@
max_length=30, help_text="Kind of service", default=KIND)
name = StrippedCharField(max_length=30, help_text="Service Name")
versionNumber = StrippedCharField(blank=True, null=True,
- max_length=30, help_text="Version of Service Definition")
+ max_length=30, help_text="Version of Service Definition") # deprecated
published = models.BooleanField(default=True)
view_url = StrippedCharField(blank=True, null=True, max_length=1024)
icon_url = StrippedCharField(blank=True, null=True, max_length=1024)
diff --git a/xos/core/models/xosmodel.py b/xos/core/models/xosmodel.py
index 361dd63..ab7edd6 100644
--- a/xos/core/models/xosmodel.py
+++ b/xos/core/models/xosmodel.py
@@ -35,12 +35,12 @@
# If `services` is empty, then only rebuild the UI
# Otherwise, only rebuild the services listed in `services`
with transaction.atomic():
- for service_controller in self.service_controllers.all():
- if (services) and (service_controller.name not in services):
+ for loadable_module in self.loadable_modules.all():
+ if (services) and (loadable_module.name not in services):
continue
- for scr in service_controller.service_controller_resources.all():
- scr.save()
- service_controller.save()
+ for lmr in loadable_module.loadable_module_resources.all():
+ lmr.save()
+ loadable_module.save()
self.save()
class XOSVolume(PlCoreBase):
diff --git a/xos/synchronizers/onboarding/steps/sync_xos.py b/xos/synchronizers/onboarding/steps/sync_xos.py
index 80753e4..a9d2089 100644
--- a/xos/synchronizers/onboarding/steps/sync_xos.py
+++ b/xos/synchronizers/onboarding/steps/sync_xos.py
@@ -46,9 +46,9 @@
# is in error state, but it is important that a single broken service does
# not takedown the entirety of XOS.
- for scr in xos.service_controllers.all():
+ for scr in xos.loadable_modules.all():
if (scr.backend_status is not None) and (scr.backend_status.startswith("0")):
- raise DeferredException("Detected unsynced service controller. Deferring.")
+ raise DeferredException("Detected unsynced loadable module. Deferring.")
self.create_docker_compose()
diff --git a/xos/synchronizers/onboarding/xosbuilder.py b/xos/synchronizers/onboarding/xosbuilder.py
index fb71a28..5289fc7 100644
--- a/xos/synchronizers/onboarding/xosbuilder.py
+++ b/xos/synchronizers/onboarding/xosbuilder.py
@@ -8,7 +8,7 @@
import xmlrpclib
from xos.config import Config
-from core.models import Service, ServiceController, ServiceControllerResource, XOS
+from core.models import Service, ServiceController, ServiceControllerResource, LoadableModule, LoadableModuleResource, XOS
from xos.logger import Logger, logging
from django.utils import timezone
@@ -33,7 +33,7 @@
def get_base_dest_dir(self, scr):
xos_base = "opt/xos"
- service_name = scr.service_controller.name
+ service_name = scr.loadable_module.name
base_dirs = {"models": "%s/services/%s/" % (xos_base, service_name),
"admin": "%s/services/%s/" % (xos_base, service_name),
"admin_template": "%s/services/%s/templates/" % (xos_base, service_name),
@@ -125,7 +125,7 @@
# def get_controller_docker_lines(self, controller, kinds):
# need_service_init_py = False
# dockerfile=[]
-# for scr in controller.service_controller_resources.all():
+# for scr in controller.loadable_module_resources.all():
# if scr.kind in kinds:
# lines = self.get_docker_lines(scr)
# dockerfile = dockerfile + lines
@@ -156,7 +156,7 @@
need_service_init_py = False
script=[]
inits=[]
- for scr in list(controller.service_controller_resources.all()):
+ for scr in list(controller.loadable_module_resources.all()):
if not (scr.kind in kinds):
continue
@@ -196,7 +196,7 @@
def check_controller_unready(self, controller):
unready_resources=[]
- for scr in controller.service_controller_resources.all():
+ for scr in controller.loadable_module_resources.all():
if (not scr.backend_status) or (not scr.backend_status.startswith("1")):
unready_resources.append(scr)
@@ -230,14 +230,14 @@
dockerfile = ["FROM %s" % xos.source_ui_image]
script = []
- for controller in ServiceController.objects.all():
+ for controller in LoadableModule.objects.all():
if self.check_controller_unready(controller):
- logger.warning("Controller %s has unready resources" % str(controller))
+ logger.warning("Loadable Module %s has unready resources" % str(controller))
continue
#dockerfile = dockerfile + self.get_controller_docker_lines(controller, self.UI_KINDS)
script = script + self.get_controller_script_lines(controller, self.UI_KINDS)
- if controller.service_controller_resources.filter(kind="models").exists():
+ if controller.loadable_module_resources.filter(kind="models").exists():
app_list.append("services." + controller.name)
migration_list.append(controller.name)
@@ -258,6 +258,10 @@
def create_synchronizer_dockerfile(self, controller):
self.build_tainted = False
+ if not controller.loadable_module_resources.filter(kind="synchronizer").exists():
+ # it doesn't have a synchronizer, therefore it doesn't need a dockerfile
+ return None
+
# bake in the synchronizer from this controller
sync_lines = self.get_controller_script_lines(controller, self.SYNC_CONTROLLER_KINDS)
@@ -277,10 +281,9 @@
# It's important to bake all services in, because some services'
# synchronizers may depend on models from another service.
app_list = []
- for c in ServiceController.objects.all():
- #dockerfile = dockerfile + self.get_controller_docker_lines(c, self.SYNC_ALLCONTROLLER_KINDS)
+ for c in LoadableModule.objects.all():
script = script + self.get_controller_script_lines(c, self.SYNC_ALLCONTROLLER_KINDS)
- if c.service_controller_resources.filter(kind="models").exists():
+ if c.loadable_module_resources.filter(kind="models").exists():
app_list.append("services." + c.name)
self.create_xos_app_data(controller.name, script, app_list, None)
@@ -344,7 +347,7 @@
logger.warning("Controller %s has unready resources" % str(c))
continue
- if c.service_controller_resources.filter(kind="synchronizer").exists():
+ if c.loadable_module_resources.filter(kind="synchronizer").exists():
if c.synchronizer_run and c.synchronizer_config:
command = 'bash -c "sleep 120; cd /opt/xos/synchronizers/%s; python ./%s -C %s"' % (c.name, c.synchronizer_run, c.synchronizer_config)
else:
diff --git a/xos/tosca/custom_types/xos.m4 b/xos/tosca/custom_types/xos.m4
index c935eef..6372f9c 100644
--- a/xos/tosca/custom_types/xos.m4
+++ b/xos/tosca/custom_types/xos.m4
@@ -80,6 +80,18 @@
type: string
required: false
description: Base url, to allow resources to use relative URLs
+ version:
+ type: string
+ required: false
+ description: Version number of this Service Controller
+ provides:
+ type: string
+ required: false
+ description: Comma-separated list of things provided
+ requires:
+ type: string
+ required: false
+ description: Comma-separated list of requirements
models:
type: string
required: false
diff --git a/xos/tosca/custom_types/xos.yaml b/xos/tosca/custom_types/xos.yaml
index a7ab219..44ade1f 100644
--- a/xos/tosca/custom_types/xos.yaml
+++ b/xos/tosca/custom_types/xos.yaml
@@ -188,6 +188,18 @@
type: string
required: false
description: Base url, to allow resources to use relative URLs
+ version:
+ type: string
+ required: false
+ description: Version number of this Service Controller
+ provides:
+ type: string
+ required: false
+ description: Comma-separated list of things provided
+ requires:
+ type: string
+ required: false
+ description: Comma-separated list of requirements
models:
type: string
required: false
diff --git a/xos/tosca/resources/servicecontroller.py b/xos/tosca/resources/servicecontroller.py
index 1ad4a17..e14c604 100644
--- a/xos/tosca/resources/servicecontroller.py
+++ b/xos/tosca/resources/servicecontroller.py
@@ -5,14 +5,14 @@
sys.path.append("/opt/tosca")
from translator.toscalib.tosca_template import ToscaTemplate
-from core.models import ServiceController, ServiceControllerResource
+from core.models import ServiceController, ServiceControllerResource, LoadableModule, LoadableModuleResource
from xosresource import XOSResource
class XOSServiceController(XOSResource):
provides = "tosca.nodes.ServiceController"
xos_model = ServiceController
- copyin_props = ["base_url", "synchronizer_run", "synchronizer_config"]
+ copyin_props = ["version", "provides", "requires", "base_url", "synchronizer_run", "synchronizer_config"]
def postprocess_resource_prop(self, obj, kind, format):
values = self.get_property(kind)
@@ -39,7 +39,7 @@
value = parts[-1]
- scr = ServiceControllerResource.objects.filter(service_controller=obj, name=name, kind=kind, format=format)
+ scr = LoadableModuleResource.objects.filter(loadable_module=obj, name=name, kind=kind, format=format)
if scr:
scr=scr[0]
if (scr.url != value) or (scr.subdirectory!=subdirectory):
@@ -49,7 +49,7 @@
scr.save()
else:
self.info("adding resource %s" % kind)
- scr = ServiceControllerResource(service_controller=obj, name=name, kind=kind, format=format, url=value, subdirectory=subdirectory)
+ scr = LoadableModuleResource(loadable_module=obj, name=name, kind=kind, format=format, url=value, subdirectory=subdirectory)
scr.save()
def postprocess(self, obj):
@@ -66,3 +66,12 @@
self.postprocess_resource_prop(obj, "rest_service", "python")
self.postprocess_resource_prop(obj, "rest_tenant", "python")
+ def save_created_obj(self, xos_obj):
+ if xos_obj.requires and xos_obj.requires.strip():
+ (satisfied, missing) = ServiceController.dependency_check([x.strip() for x in xos_obj.requires.split(",")])
+ if missing:
+ raise Exception("missing dependencies for ServiceController %s: %s" % (xos_obj.name, ", ".join(missing)))
+
+ super(XOSServiceController, self).save_created_obj(xos_obj)
+
+
diff --git a/xos/tosca/resources/servicecontrollerresource.py b/xos/tosca/resources/servicecontrollerresource.py
index 96ea83d..91e61dd 100644
--- a/xos/tosca/resources/servicecontrollerresource.py
+++ b/xos/tosca/resources/servicecontrollerresource.py
@@ -5,7 +5,7 @@
sys.path.append("/opt/tosca")
from translator.toscalib.tosca_template import ToscaTemplate
-from core.models import ServiceControllerResource, ServiceController
+from core.models import ServiceControllerResource, ServiceController, LoadableModuleResource, LoadableModule
from xosresource import XOSResource
@@ -19,7 +19,7 @@
controller_name = self.get_requirement("tosca.relationships.UsedByController", throw_exception=throw_exception)
if controller_name:
- args["service_controller"] = self.get_xos_object(ServiceController, throw_exception=throw_exception, name=controller_name)
+ args["loadable_module"] = self.get_xos_object(ServiceController, throw_exception=throw_exception, name=controller_name)
return args
diff --git a/xos/tosca/resources/xosresource.py b/xos/tosca/resources/xosresource.py
index f65a231..427b5eb 100644
--- a/xos/tosca/resources/xosresource.py
+++ b/xos/tosca/resources/xosresource.py
@@ -238,12 +238,16 @@
return args
+ def save_created_obj(self, xos_obj):
+ xos_obj.save()
+
def create(self):
xos_args = self.get_xos_args()
xos_obj = self.xos_model(**xos_args)
if self.user:
xos_obj.caller = self.user
- xos_obj.save()
+
+ self.save_created_obj(xos_obj)
self.info("Created %s '%s'" % (self.xos_model.__name__,str(xos_obj)))