Merge branch 'master' of github.com:open-cloud/xos
diff --git a/upgrade-container.sh b/upgrade-container.sh
new file mode 100755
index 0000000..09c639d
--- /dev/null
+++ b/upgrade-container.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+TMPDIR="/tmp/initdata"
+XOSDIR="/home/ubuntu/xos"
+
+mkdir -p $TMPDIR
+rm -f $TMPDIR/*.json
+
+XOS=$( docker ps|grep "xos:latest"|awk '{print $NF}' )
+docker exec $XOS /opt/xos/scripts/opencloud dumpdata
+docker cp $XOS:/opt/xos_backups/dumpdata-latest.json $TMPDIR
+docker cp $XOS:/opt/xos/xos_config $TMPDIR
+cp $TMPDIR/*.json $XOSDIR/xos/core/fixtures/initial_data.json
+cp $TMPDIR/xos_config $XOSDIR/xos/
+
+git pull
+
+if [[ $? != 0 ]]; then
+ echo "git pull" failed
+ exit
+fi
+
+docker build -t xos .
+
+docker stop $XOS
+docker run -p 8000:8000 xos
diff --git a/xos.spec b/xos.spec
index 435285c..c585e7e 100644
--- a/xos.spec
+++ b/xos.spec
@@ -1,7 +1,7 @@
Summary: OpenCloud core services
Name: xos
Version: 1.2.0
-Release: 4
+Release: 5
License: GPL+
Group: Development/Tools
Source0: %{_tmppath}/%{name}-%{version}.tar.gz
diff --git a/xos/cord/models.py b/xos/cord/models.py
index d9cc2df..346fcc9 100644
--- a/xos/cord/models.py
+++ b/xos/cord/models.py
@@ -24,24 +24,24 @@
t.caller = User.objects.all()[0]
t.save()
-for v in VOLTTenant.objects.all():
+for v in VOLTTenant.get_tenant_objects().all():
v.caller = User.objects.all()[0]
v.delete()
-for v in VCPETenant.objects.all():
+for v in VCPETenant.get_tenant_objects().all():
v.caller = User.objects.all()[0]
v.delete()
-for v in VOLTTenant.objects.all():
+for v in VOLTTenant.get_tenant_objects().all():
v.caller = User.objects.all()[0]
v.delete()
-for v in VOLTTenant.objects.all():
+for v in VOLTTenant.get_tenant_objects().all():
if not v.creator:
v.creator= User.objects.all()[0]
v.save()
-for v in VCPETenant.objects.all():
+for v in VCPETenant.get_tenant_objects().all():
if not v.creator:
v.creator= User.objects.all()[0]
v.save()
diff --git a/xos/core/admin.py b/xos/core/admin.py
index 69cae61..1ded815 100644
--- a/xos/core/admin.py
+++ b/xos/core/admin.py
@@ -57,6 +57,31 @@
flatatt(final_attrs),
force_text(value))
+class SliderWidget(forms.HiddenInput):
+ def render(self, name, value, attrs=None):
+ if value is None:
+ value = '0'
+ final_attrs = self.build_attrs(attrs, name=name)
+ attrs = attrs or attrs[:]
+ attrs["name"] = name
+ attrs["value"] = value
+ html = """<div style="width:640px"><span id="%(id)s_label">%(value)s</span><div id="%(id)s_slider" style="float:right;width:610px;margin-top:5px"></div></div>
+ <script>
+ $(function() {
+ $("#%(id)s_slider").slider({
+ value: %(value)s,
+ slide: function(event, ui) { $("#%(id)s").val( ui.value ); $("#%(id)s_label").html(ui.value); },
+ });
+ });
+ </script>
+ <input type="hidden" id="%(id)s" name="%(name)s" value="%(value)s"></input>
+ """ % attrs
+ html = html.replace("{","{{").replace("}","}}")
+ return format_html(html,
+ flatatt(final_attrs),
+ force_text(value))
+
+
class PlainTextWidget(forms.HiddenInput):
input_type = 'hidden'
diff --git a/xos/core/models/service.py b/xos/core/models/service.py
index 19bcda9..8a10f37 100644
--- a/xos/core/models/service.py
+++ b/xos/core/models/service.py
@@ -29,6 +29,79 @@
def __unicode__(self): return u'%s' % (self.name)
+ def get_scalable_nodes(self, slice, max_per_node=None, exclusive_slices=[]):
+ """
+ Get a list of nodes that can be used to scale up a slice.
+
+ slice - slice to scale up
+ max_per_node - maximum numbers of slivers that 'slice' can have on a single node
+ exclusive_slices - list of slices that must have no nodes in common with 'slice'.
+ """
+
+ from core.models import Node, Sliver # late import to get around order-of-imports constraint in __init__.py
+
+ nodes = list(Node.objects.all())
+
+ conflicting_slivers = Sliver.objects.filter(slice__in = exclusive_slices)
+ conflicting_nodes = Node.objects.filter(slivers__in = conflicting_slivers)
+
+ nodes = [x for x in nodes if x not in conflicting_nodes]
+
+ # If max_per_node is set, then limit the number of slivers this slice
+ # can have on a single node.
+ if max_per_node:
+ acceptable_nodes = []
+ for node in nodes:
+ existing_count = node.slivers.filter(slice=slice).count()
+ if existing_count < max_per_node:
+ acceptable_nodes.append(node)
+ nodes = acceptable_nodes
+
+ return nodes
+
+ def pick_node(self, slice, max_per_node=None, exclusive_slices=[]):
+ # Pick the best node to scale up a slice.
+
+ nodes = self.get_scalable_nodes(slice, max_per_node, exclusive_slices)
+ nodes = sorted(nodes, key=lambda node: node.slivers.all().count())
+ if not nodes:
+ return None
+ return nodes[0]
+
+ def adjust_scale(self, slice_hint, scale, max_per_node=None, exclusive_slices=[]):
+ from core.models import Sliver # late import to get around order-of-imports constraint in __init__.py
+
+ slices = [x for x in self.slices.all() if slice_hint in x.name]
+ for slice in slices:
+ while slice.slivers.all().count() > scale:
+ s = slice.slivers.all()[0]
+ # print "drop sliver", s
+ s.delete()
+
+ while slice.slivers.all().count() < scale:
+ node = self.pick_node(slice, max_per_node, exclusive_slices)
+ if not node:
+ # no more available nodes
+ break
+
+ image = slice.default_image
+ if not image:
+ raise XOSConfigurationError("No default_image for slice %s" % slice.name)
+
+ flavor = slice.default_flavor
+ if not flavor:
+ raise XOSConfigurationError("No default_flavor for slice %s" % slice.name)
+
+ s = Sliver(slice=slice,
+ node=node,
+ creator=slice.creator,
+ image=image,
+ flavor=flavor,
+ deployment=node.site_deployment.deployment)
+ s.save()
+
+ # print "add sliver", s
+
class ServiceAttribute(PlCoreBase):
name = models.SlugField(help_text="Attribute Name", max_length=128)
value = StrippedCharField(help_text="Attribute Value", max_length=1024)
diff --git a/xos/core/static/xos.css b/xos/core/static/xos.css
index 513dc06..282c9f4 100644
--- a/xos/core/static/xos.css
+++ b/xos/core/static/xos.css
@@ -182,7 +182,7 @@
float: right;
border: 2px darkGrey;
}
-.ui-state-default, .ui-widget-content .ui-state-default, .ui-widget-header .ui-state-default{
+.ui-state-default #hometabs, .ui-widget-content .ui-state-default #hometabs, .ui-widget-header .ui-state-default {
background: none !important;
border-top: 0px !important;
border-left: 0px !important;
diff --git a/xos/hpc/admin.py b/xos/hpc/admin.py
index 20364b6..08a1cdb 100644
--- a/xos/hpc/admin.py
+++ b/xos/hpc/admin.py
@@ -10,7 +10,7 @@
from django.utils import timezone
from django.contrib.contenttypes import generic
from suit.widgets import LinkedSelect
-from core.admin import ServiceAppAdmin,SliceInline,ServiceAttrAsTabInline, ReadOnlyAwareAdmin, XOSTabularInline
+from core.admin import ServiceAppAdmin,SliceInline,ServiceAttrAsTabInline, ReadOnlyAwareAdmin, XOSTabularInline, SliderWidget
from functools import update_wrapper
from django.contrib.admin.views.main import ChangeList
@@ -106,15 +106,30 @@
# filtered_change_view rather than the default change_view.
return FilteredChangeList
+class HpcServiceForm(forms.ModelForm):
+ scale = forms.IntegerField(widget = SliderWidget, required=False)
+
+ def __init__(self, *args, **kwargs):
+ super(HpcServiceForm, self).__init__(*args, **kwargs)
+ self.fields['scale'].initial = kwargs["instance"].scale
+
+ def save(self, *args, **kwargs):
+ if self.cleaned_data['scale']:
+ self.instance.scale = self.cleaned_data['scale']
+
+ return super(HpcServiceForm, self).save(*args, **kwargs)
+
+
class HpcServiceAdmin(ReadOnlyAwareAdmin):
model = HpcService
verbose_name = "HPC Service"
verbose_name_plural = "HPC Service"
list_display = ("backend_status_icon", "name","enabled")
list_display_links = ('backend_status_icon', 'name', )
- fieldsets = [(None, {'fields': ['backend_status_text', 'name','enabled','versionNumber', 'description', "cmi_hostname"], 'classes':['suit-tab suit-tab-general']})]
+ fieldsets = [(None, {'fields': ['backend_status_text', 'name','scale','enabled','versionNumber', 'description', "cmi_hostname"], 'classes':['suit-tab suit-tab-general']})]
readonly_fields = ('backend_status_text', )
inlines = [SliceInline,ServiceAttrAsTabInline]
+ form = HpcServiceForm
extracontext_registered_admins = True
diff --git a/xos/hpc/models.py b/xos/hpc/models.py
index e915fbc..1cd51ce 100644
--- a/xos/hpc/models.py
+++ b/xos/hpc/models.py
@@ -17,6 +17,26 @@
cmi_hostname = StrippedCharField(max_length=254, null=True, blank=True)
+ @property
+ def scale(self):
+ hpc_slices = [x for x in self.slices.all() if "hpc" in x.name]
+ if not hpc_slices:
+ return 0
+ return hpc_slices[0].slivers.count()
+
+ @scale.setter
+ def scale(self, value):
+ self.set_scale = value
+
+ def save(self, *args, **kwargs):
+ super(HpcService, self).save(*args, **kwargs)
+
+ # scale up/down
+ scale = getattr(self, "set_scale", None)
+ if scale is not None:
+ exclude_slices = [x for x in self.slices.all() if "cmi" in x.name]
+ self.adjust_scale(slice_hint="hpc", scale=scale, exclusive_slices = exclude_slices, max_per_node=1)
+
class ServiceProvider(PlCoreBase):
class Meta:
app_label = "hpc"
diff --git a/xos/hpc_observer/run.sh b/xos/hpc_observer/run.sh
deleted file mode 100644
index abf620a..0000000
--- a/xos/hpc_observer/run.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-if [[ ! -e ./hpc-backend.py ]]; then
- ln -s ../xos-observer.py hpc-backend.py
-fi
-
-export XOS_DIR=/opt/xos
-python hpc-backend.py -C $XOS_DIR/hpc_observer/hpc_observer_config
diff --git a/xos/hpc_observer/start.sh b/xos/hpc_observer/start.sh
deleted file mode 100644
index 07a1663..0000000
--- a/xos/hpc_observer/start.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-if [[ ! -e ./hpc-backend.py ]]; then
- ln -s ../xos-observer.py hpc-backend.py
-fi
-
-export XOS_DIR=/opt/xos
-nohup python hpc-backend.py -C $XOS_DIR/hpc_observer/hpc_observer_config > /dev/null 2>&1 &
diff --git a/xos/hpc_observer/stop.sh b/xos/hpc_observer/stop.sh
deleted file mode 100644
index 632f7a6..0000000
--- a/xos/hpc_observer/stop.sh
+++ /dev/null
@@ -1 +0,0 @@
-pkill -9 -f hpc-backend.py
diff --git a/xos/hpc_observer/Makefile b/xos/observers/hpc/Makefile
similarity index 100%
rename from xos/hpc_observer/Makefile
rename to xos/observers/hpc/Makefile
diff --git a/xos/hpc_observer/fsck.py b/xos/observers/hpc/fsck.py
similarity index 100%
rename from xos/hpc_observer/fsck.py
rename to xos/observers/hpc/fsck.py
diff --git a/xos/observers/hpc/hpc-observer.py b/xos/observers/hpc/hpc-observer.py
new file mode 100755
index 0000000..d6a71ff
--- /dev/null
+++ b/xos/observers/hpc/hpc-observer.py
@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+
+# This imports and runs ../../xos-observer.py
+
+import importlib
+import os
+import sys
+observer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),"../..")
+sys.path.append(observer_path)
+mod = importlib.import_module("xos-observer")
+mod.main()
diff --git a/xos/hpc_observer/hpc_observer_config b/xos/observers/hpc/hpc_observer_config
similarity index 74%
rename from xos/hpc_observer/hpc_observer_config
rename to xos/observers/hpc/hpc_observer_config
index 0e7e53a..16d9077 100644
--- a/xos/hpc_observer/hpc_observer_config
+++ b/xos/observers/hpc/hpc_observer_config
@@ -23,9 +23,9 @@
[observer]
name=hpc
-dependency_graph=/opt/xos/hpc_observer/model-deps
-steps_dir=/opt/xos/hpc_observer/steps
-deleters_dir=/opt/xos/hpc_observer/deleters
+dependency_graph=/opt/xos/observers/hpc/model-deps
+steps_dir=/opt/xos/observers/hpc/steps
+deleters_dir=/opt/xos/observers/hpc/deleters
log_file=console
#/var/log/hpc.log
driver=None
diff --git a/xos/hpc_observer/hpc_watcher.py b/xos/observers/hpc/hpc_watcher.py
similarity index 100%
rename from xos/hpc_observer/hpc_watcher.py
rename to xos/observers/hpc/hpc_watcher.py
diff --git a/xos/hpc_observer/hpclib.py b/xos/observers/hpc/hpclib.py
similarity index 100%
rename from xos/hpc_observer/hpclib.py
rename to xos/observers/hpc/hpclib.py
diff --git a/xos/hpc_observer/model-deps b/xos/observers/hpc/model-deps
similarity index 100%
rename from xos/hpc_observer/model-deps
rename to xos/observers/hpc/model-deps
diff --git a/xos/observers/hpc/run.sh b/xos/observers/hpc/run.sh
new file mode 100755
index 0000000..f77d751
--- /dev/null
+++ b/xos/observers/hpc/run.sh
@@ -0,0 +1,6 @@
+#if [[ ! -e ./hpc-backend.py ]]; then
+# ln -s ../xos-observer.py hpc-backend.py
+#fi
+
+export XOS_DIR=/opt/xos
+python hpc-observer.py -C $XOS_DIR/observers/hpc/hpc_observer_config
diff --git a/xos/observers/hpc/start.sh b/xos/observers/hpc/start.sh
new file mode 100755
index 0000000..305c07f
--- /dev/null
+++ b/xos/observers/hpc/start.sh
@@ -0,0 +1,6 @@
+#if [[ ! -e ./hpc-backend.py ]]; then
+# ln -s ../xos-observer.py hpc-backend.py
+#fi
+
+export XOS_DIR=/opt/xos
+nohup python hpc-observer.py -C $XOS_DIR/observers/hpc/hpc_observer_config > /dev/null 2>&1 &
diff --git a/xos/hpc_observer/steps/garbage_collector.py b/xos/observers/hpc/steps/garbage_collector.py
similarity index 100%
rename from xos/hpc_observer/steps/garbage_collector.py
rename to xos/observers/hpc/steps/garbage_collector.py
diff --git a/xos/hpc_observer/steps/sync_cdnprefix.py b/xos/observers/hpc/steps/sync_cdnprefix.py
similarity index 100%
rename from xos/hpc_observer/steps/sync_cdnprefix.py
rename to xos/observers/hpc/steps/sync_cdnprefix.py
diff --git a/xos/hpc_observer/steps/sync_contentprovider.py b/xos/observers/hpc/steps/sync_contentprovider.py
similarity index 100%
rename from xos/hpc_observer/steps/sync_contentprovider.py
rename to xos/observers/hpc/steps/sync_contentprovider.py
diff --git a/xos/hpc_observer/steps/sync_hpcservices.py b/xos/observers/hpc/steps/sync_hpcservices.py
similarity index 100%
rename from xos/hpc_observer/steps/sync_hpcservices.py
rename to xos/observers/hpc/steps/sync_hpcservices.py
diff --git a/xos/hpc_observer/steps/sync_originserver.py b/xos/observers/hpc/steps/sync_originserver.py
similarity index 100%
rename from xos/hpc_observer/steps/sync_originserver.py
rename to xos/observers/hpc/steps/sync_originserver.py
diff --git a/xos/hpc_observer/steps/sync_serviceprovider.py b/xos/observers/hpc/steps/sync_serviceprovider.py
similarity index 100%
rename from xos/hpc_observer/steps/sync_serviceprovider.py
rename to xos/observers/hpc/steps/sync_serviceprovider.py
diff --git a/xos/hpc_observer/steps/sync_sitemap.py b/xos/observers/hpc/steps/sync_sitemap.py
similarity index 100%
rename from xos/hpc_observer/steps/sync_sitemap.py
rename to xos/observers/hpc/steps/sync_sitemap.py
diff --git a/xos/observers/hpc/stop.sh b/xos/observers/hpc/stop.sh
new file mode 100755
index 0000000..a0b4a8e
--- /dev/null
+++ b/xos/observers/hpc/stop.sh
@@ -0,0 +1 @@
+pkill -9 -f hpc-observer.py
diff --git a/xos/observers/hpc/supervisor/hpc-observer.conf b/xos/observers/hpc/supervisor/hpc-observer.conf
new file mode 100644
index 0000000..f2c79d4
--- /dev/null
+++ b/xos/observers/hpc/supervisor/hpc-observer.conf
@@ -0,0 +1,2 @@
+[program:hpc-observer]
+command=python /opt/xos/observers/hpc/hpc-observer.py -C /opt/xos/observers/hpc/hpc_observer_config
diff --git a/xos/observers/hpc/supervisor/hpc-watcher.conf b/xos/observers/hpc/supervisor/hpc-watcher.conf
new file mode 100644
index 0000000..e0f4eb1
--- /dev/null
+++ b/xos/observers/hpc/supervisor/hpc-watcher.conf
@@ -0,0 +1,2 @@
+[program:hpc-watcher]
+command=python /opt/xos/observers/hpc/hpc_watcher.py
diff --git a/xos/observers/vbng/run.sh b/xos/observers/vbng/run.sh
index 7829ac0..efb586f 100755
--- a/xos/observers/vbng/run.sh
+++ b/xos/observers/vbng/run.sh
@@ -1,6 +1,6 @@
-if [[ ! -e ./vbng-observer.py ]]; then
- ln -s ../../xos-observer.py vbng-observer.py
-fi
+#if [[ ! -e ./vbng-observer.py ]]; then
+# ln -s ../../xos-observer.py vbng-observer.py
+#fi
export XOS_DIR=/opt/xos
python vbng-observer.py -C $XOS_DIR/observers/vbng/vbng_observer_config
diff --git a/xos/observers/vbng/start.sh b/xos/observers/vbng/start.sh
index 5ceff27..98008f4 100755
--- a/xos/observers/vbng/start.sh
+++ b/xos/observers/vbng/start.sh
@@ -1,6 +1,6 @@
-if [[ ! -e ./vbng-observer.py ]]; then
- ln -s ../../xos-observer.py vbng-observer.py
-fi
+#if [[ ! -e ./vbng-observer.py ]]; then
+# ln -s ../../xos-observer.py vbng-observer.py
+#fi
export XOS_DIR=/opt/xos
nohup python vbng-observer.py -C $XOS_DIR/observers/vbng/vbng_observer_config > /dev/null 2>&1 &
diff --git a/xos/observers/vbng/supervisor/vbng-observer.conf b/xos/observers/vbng/supervisor/vbng-observer.conf
new file mode 100644
index 0000000..cff77b8
--- /dev/null
+++ b/xos/observers/vbng/supervisor/vbng-observer.conf
@@ -0,0 +1,2 @@
+[program:vbng-observer]
+command=python /opt/xos/observers/vbng/vbng-observer.py -C /opt/xos/observers/vbng/vbng_observer_config
diff --git a/xos/observers/vbng/vbng-observer.py b/xos/observers/vbng/vbng-observer.py
new file mode 100755
index 0000000..d6a71ff
--- /dev/null
+++ b/xos/observers/vbng/vbng-observer.py
@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+
+# This imports and runs ../../xos-observer.py
+
+import importlib
+import os
+import sys
+observer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),"../..")
+sys.path.append(observer_path)
+mod = importlib.import_module("xos-observer")
+mod.main()
diff --git a/xos/observers/vbng/vbng_observer_config b/xos/observers/vbng/vbng_observer_config
index 217ce71..9094bb1 100644
--- a/xos/observers/vbng/vbng_observer_config
+++ b/xos/observers/vbng/vbng_observer_config
@@ -25,7 +25,7 @@
name=vbng
dependency_graph=/opt/xos/observers/vbng/model-deps
steps_dir=/opt/xos/observers/vbng/steps
-sys_dir=/opt/xos/observer/vbng/sys
+sys_dir=/opt/xos/observers/vbng/sys
deleters_dir=/opt/xos/observers/vbng/deleters
log_file=console
#/var/log/hpc.log
diff --git a/xos/observers/vcpe/run.sh b/xos/observers/vcpe/run.sh
index c582842..f180e66 100755
--- a/xos/observers/vcpe/run.sh
+++ b/xos/observers/vcpe/run.sh
@@ -1,6 +1,6 @@
-if [[ ! -e ./vcpe-observer.py ]]; then
- ln -s ../../xos-observer.py vcpe-observer.py
-fi
+#if [[ ! -e ./vcpe-observer.py ]]; then
+# ln -s ../../xos-observer.py vcpe-observer.py
+#fi
export XOS_DIR=/opt/xos
python vcpe-observer.py -C $XOS_DIR/observers/vcpe/vcpe_observer_config
diff --git a/xos/observers/vcpe/start.sh b/xos/observers/vcpe/start.sh
index 96cb6e4..b402e5d 100755
--- a/xos/observers/vcpe/start.sh
+++ b/xos/observers/vcpe/start.sh
@@ -1,6 +1,6 @@
-if [[ ! -e ./vcpe-observer.py ]]; then
- ln -s ../../xos-observer.py vcpe-observer.py
-fi
+#if [[ ! -e ./vcpe-observer.py ]]; then
+# ln -s ../../xos-observer.py vcpe-observer.py
+#fi
export XOS_DIR=/opt/xos
nohup python vcpe-observer.py -C $XOS_DIR/observers/vcpe/vcpe_observer_config > /dev/null 2>&1 &
diff --git a/xos/observers/vcpe/steps/sync_vcpetenant.py b/xos/observers/vcpe/steps/sync_vcpetenant.py
index 3fc9310..8cda4ef 100644
--- a/xos/observers/vcpe/steps/sync_vcpetenant.py
+++ b/xos/observers/vcpe/steps/sync_vcpetenant.py
@@ -97,6 +97,7 @@
"hostname": sliver.node.name,
"instance_id": sliver.instance_id,
"private_key": service_key,
+ "ansible_tag": "vcpe_tenant_" + str(o.id)
}
if hasattr(o, "sync_attributes"):
diff --git a/xos/observers/vcpe/supervisor/vcpe-observer.conf b/xos/observers/vcpe/supervisor/vcpe-observer.conf
new file mode 100644
index 0000000..27d2796
--- /dev/null
+++ b/xos/observers/vcpe/supervisor/vcpe-observer.conf
@@ -0,0 +1,2 @@
+[program:vcpe-observer]
+command=python /opt/xos/observers/vcpe/vcpe-observer.py -C /opt/xos/observers/vcpe/vcpe_observer_config
diff --git a/xos/observers/vcpe/vcpe-observer.py b/xos/observers/vcpe/vcpe-observer.py
new file mode 100755
index 0000000..d6a71ff
--- /dev/null
+++ b/xos/observers/vcpe/vcpe-observer.py
@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+
+# This imports and runs ../../xos-observer.py
+
+import importlib
+import os
+import sys
+observer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),"../..")
+sys.path.append(observer_path)
+mod = importlib.import_module("xos-observer")
+mod.main()
diff --git a/xos/observers/vcpe/vcpe_observer_config b/xos/observers/vcpe/vcpe_observer_config
index c6500ae..6d58340 100644
--- a/xos/observers/vcpe/vcpe_observer_config
+++ b/xos/observers/vcpe/vcpe_observer_config
@@ -25,12 +25,14 @@
name=vcpe
dependency_graph=/opt/xos/observers/vcpe/model-deps
steps_dir=/opt/xos/observers/vcpe/steps
-sys_dir=/opt/xos/observer/vcpe/sys
+sys_dir=/opt/xos/observers/vcpe/sys
deleters_dir=/opt/xos/observers/vcpe/deleters
log_file=console
#/var/log/hpc.log
driver=None
pretend=False
+backoff_disabled=True
+save_ansible_output=True
[feefie]
client_id='vicci_dev_central'
diff --git a/xos/openstack_observer/ansible.py b/xos/openstack_observer/ansible.py
index 17504b3..5b6bd1d 100644
--- a/xos/openstack_observer/ansible.py
+++ b/xos/openstack_observer/ansible.py
@@ -43,24 +43,44 @@
return results
+def parse_unreachable(msg):
+ total_unreachable=0
+ for l in msg.splitlines():
+ x = re.findall('ok=([0-9]+).*changed=([0-9]+).*unreachable=([0-9]+).*failed=([0-9]+)', l)
+ if x:
+ (ok, changed, unreachable, failed) = x[0]
+ ok=int(ok)
+ changed=int(changed)
+ unreachable=int(unreachable)
+ failed=int(failed)
+
+ total_unreachable += unreachable
+ return total_unreachable
+
+
def id_generator(size=6, chars=string.ascii_uppercase + string.digits):
return ''.join(random.choice(chars) for _ in range(size))
def shellquote(s):
return "'" + s.replace("'", "'\\''") + "'"
-def run_template(name, opts,path='', expected_num=None, ansible_config=None, ansible_hosts=None, run_ansible_script=None):
+def get_playbook_fn(opts, path):
+ if not opts.get("ansible_tag", None):
+ # if no ansible_tag is in the options, then generate a unique one
+ objname= id_generator()
+ opts = opts.copy()
+ opts["ansible_tag"] = objname
+
+ objname = opts["ansible_tag"]
+
+ os.system('mkdir -p %s' % os.path.join(sys_dir, path))
+ return (opts, os.path.join(sys_dir,path,objname))
+
+def run_template(name, opts, path='', expected_num=None, ansible_config=None, ansible_hosts=None, run_ansible_script=None):
template = os_template_env.get_template(name)
buffer = template.render(opts)
- try:
- objname = opts['ansible_tag']
- except:
- objname= id_generator()
-
- os.system('mkdir -p %s'%'/'.join([sys_dir,path]))
- fqp = '/'.join([sys_dir,path,objname])
-
+ (opts, fqp) = get_playbook_fn(opts, path)
f = open(fqp,'w')
f.write(buffer)
@@ -78,11 +98,17 @@
if not run_ansible_script:
run_ansible_script = os.path.join(XOS_DIR, "observer/run_ansible")
- #run = os.popen(XOS_DIR + '/observer/run_ansible %s'%shellquote(fqp), env=env)
- run = subprocess.Popen("%s %s" % (run_ansible_script, shellquote(fqp)), shell=True, stdout=subprocess.PIPE, env=env).stdout
- msg = run.read()
- status = run.close()
+ process = subprocess.Popen("%s %s" % (run_ansible_script, shellquote(fqp)), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
+ msg = process.stdout.read()
+ err_msg = process.stderr.read()
+ if getattr(Config(), "observer_save_ansible_output", False):
+ try:
+ open(fqp+".out","w").write(msg)
+ open(fqp+".err","w").write(err_msg)
+ except:
+ # fail silently
+ pass
else:
msg = open(fqp+'.out').read()
@@ -91,6 +117,10 @@
ok_results = parse_output(msg)
if (expected_num is not None) and (len(ok_results) != expected_num):
raise ValueError('Unexpected num %s!=%d' % (str(expected_num), len(ok_results)) )
+
+ total_unreachable = parse_unreachable(msg)
+ if (total_unreachable > 0):
+ raise ValueError("Unreachable results in ansible recipe")
except ValueError,e:
all_fatal = [e.message] + re.findall(r'^msg: (.*)',msg,re.MULTILINE)
all_fatal2 = re.findall(r'^ERROR: (.*)',msg,re.MULTILINE)
@@ -110,35 +140,35 @@
hostname = opts["hostname"]
private_key = opts["private_key"]
- (private_key_handle, private_key_pathname) = tempfile.mkstemp()
- (config_handle, config_pathname) = tempfile.mkstemp()
- (hosts_handle, hosts_pathname) = tempfile.mkstemp()
+ (opts, fqp) = get_playbook_fn(opts, path)
+ private_key_pathname = fqp + ".key"
+ config_pathname = fqp + ".config"
+ hosts_pathname = fqp + ".hosts"
- try:
- proxy_command = "ProxyCommand ssh -q -i %s -o StrictHostKeyChecking=no %s@%s" % (private_key_pathname, instance_id, hostname)
+ proxy_command = "ProxyCommand ssh -q -i %s -o StrictHostKeyChecking=no %s@%s" % (private_key_pathname, instance_id, hostname)
- os.write(private_key_handle, private_key)
- os.close(private_key_handle)
+ f = open(private_key_pathname, "w")
+ f.write(private_key)
+ f.close()
- os.write(config_handle, "[ssh_connection]\n")
- os.write(config_handle, 'ssh_args = -o "%s" -o StrictHostKeyChecking=no\n' % proxy_command)
- os.write(config_handle, 'scp_if_ssh = True\n')
- os.close(config_handle)
+ f = open(config_pathname, "w")
+ f.write("[ssh_connection]\n")
+ f.write('ssh_args = -o "%s" -o StrictHostKeyChecking=no\n' % proxy_command)
+ f.write('scp_if_ssh = True\n')
+ f.close()
- os.write(hosts_handle, "[%s]\n" % sliver_name)
- os.write(hosts_handle, "%s ansible_ssh_private_key_file=%s\n" % (hostname, private_key_pathname))
- os.close(hosts_handle)
+ f = open(hosts_pathname, "w")
+ f.write("[%s]\n" % sliver_name)
+ f.write("%s ansible_ssh_private_key_file=%s\n" % (hostname, private_key_pathname))
+ f.close()
- print "ANSIBLE_CONFIG=%s" % config_pathname
- print "ANSIBLE_HOSTS=%s" % hosts_pathname
+ # SSH will complain if private key is world or group readable
+ os.chmod(private_key_pathname, 0600)
- return run_template(name, opts, path, expected_num, ansible_config = config_pathname, ansible_hosts = hosts_pathname, run_ansible_script="/opt/xos/observer/run_ansible_verbose")
+ print "ANSIBLE_CONFIG=%s" % config_pathname
+ print "ANSIBLE_HOSTS=%s" % hosts_pathname
- finally:
- #os.remove(private_key_pathname)
- #os.remove(config_pathname)
- #os.remove(hosts_pathname)
- pass
+ return run_template(name, opts, path, expected_num, ansible_config = config_pathname, ansible_hosts = hosts_pathname, run_ansible_script="/opt/xos/observer/run_ansible_verbose")
diff --git a/xos/tools/cleanup_unique.py b/xos/tools/cleanup_unique.py
new file mode 100644
index 0000000..29fb047
--- /dev/null
+++ b/xos/tools/cleanup_unique.py
@@ -0,0 +1,101 @@
+import os
+import sys
+sys.path.append("/opt/xos")
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "xos.settings")
+import django
+from core.models import *
+from hpc.models import *
+from cord.models import *
+django.setup()
+
+for obj in ControllerNetwork.deleted_objects.all():
+ print "Purging deleted object", obj
+ obj.delete(purge=True)
+
+for obj in ControllerSite.deleted_objects.all():
+ print "Purging deleted object", obj
+ obj.delete(purge=True)
+
+for obj in ControllerSlice.deleted_objects.all():
+ print "Purging deleted object", obj
+ obj.delete(purge=True)
+
+for obj in NetworkSlice.deleted_objects.all():
+ print "Purging deleted object", obj
+ obj.delete(purge=True)
+
+for obj in NetworkSliver.deleted_objects.all():
+ print "Purging deleted object", obj
+ obj.delete(purge=True)
+
+for obj in DeploymentPrivilege.deleted_objects.all():
+ print "Purging deleted object", obj
+ obj.delete(purge=True)
+
+for obj in SiteDeployment.deleted_objects.all():
+ print "Purging deleted object", obj
+ obj.delete(purge=True)
+
+seen=[]
+for obj in ControllerNetwork.objects.all():
+ seen.append(obj.id)
+ conflicts = ControllerNetwork.objects.filter(network=obj.network, controller=obj.controller)
+ for conflict in conflicts:
+ if conflict.id not in seen:
+ print "Purging", conflict, conflict.id, "due to duplicate of", obj.id
+ conflict.delete(purge=True)
+
+seen=[]
+for obj in NetworkSlice.objects.all():
+ seen.append(obj.id)
+ conflicts = NetworkSlice.objects.filter(network=obj.network, slice=obj.slice)
+ for conflict in conflicts:
+ if conflict.id not in seen:
+ print "Purging", conflict, conflict.id, "due to duplicate of", obj.id
+ conflict.delete(purge=True)
+
+seen=[]
+for obj in NetworkSliver.objects.all():
+ seen.append(obj.id)
+ conflicts = NetworkSliver.objects.filter(network=obj.network, sliver=obj.sliver)
+ for conflict in conflicts:
+ if conflict.id not in seen:
+ print "Purging", conflict, conflict.id, "due to duplicate of", obj.id
+ conflict.delete(purge=True)
+
+seen=[]
+for obj in DeploymentPrivilege.objects.all():
+ seen.append(obj.id)
+ conflicts = DeploymentPrivilege.objects.filter(user=obj.user, deployment=obj.deployment, role=obj.role)
+ for conflict in conflicts:
+ if conflict.id not in seen:
+ print "Purging", conflict, conflict.id, "due to duplicate of", obj.id
+ conflict.delete(purge=True)
+
+seen=[]
+for obj in SiteDeployment.objects.all():
+ seen.append(obj.id)
+ conflicts = SiteDeployment.objects.filter(site=obj.site, deployment=obj.deployment, controller=obj.controller)
+ for conflict in conflicts:
+ if conflict.id not in seen:
+ print "Purging", conflict, conflict.id, "due to duplicate of", obj.id
+ conflict.delete(purge=True)
+
+seen=[]
+for obj in ControllerSite.objects.all():
+ seen.append(obj.id)
+ conflicts = ControllerSite.objects.filter(site=obj.site, controller=obj.controller)
+ for conflict in conflicts:
+ if conflict.id not in seen:
+ print "Purging", conflict, conflict.id, "due to duplicate of", obj.id
+ conflict.delete(purge=True)
+
+seen=[]
+for obj in ControllerSlice.objects.all():
+ seen.append(obj.id)
+ conflicts = ControllerSlice.objects.filter(slice=obj.slice, controller=obj.controller)
+ for conflict in conflicts:
+ if conflict.id not in seen:
+ print "Purging", conflict, conflict.id, "due to duplicate of", obj.id
+ conflict.delete(purge=True)
+