Merge pull request #85 from open-cloud/service_permissions
Service permissions
diff --git a/upgrade-container.sh b/upgrade-container.sh
new file mode 100755
index 0000000..09c639d
--- /dev/null
+++ b/upgrade-container.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+TMPDIR="/tmp/initdata"
+XOSDIR="/home/ubuntu/xos"
+
+mkdir -p $TMPDIR
+rm -f $TMPDIR/*.json
+
+XOS=$( docker ps|grep "xos:latest"|awk '{print $NF}' )
+docker exec $XOS /opt/xos/scripts/opencloud dumpdata
+docker cp $XOS:/opt/xos_backups/dumpdata-latest.json $TMPDIR
+docker cp $XOS:/opt/xos/xos_config $TMPDIR
+cp $TMPDIR/*.json $XOSDIR/xos/core/fixtures/initial_data.json
+cp $TMPDIR/xos_config $XOSDIR/xos/
+
+git pull
+
+if [[ $? != 0 ]]; then
+ echo "git pull" failed
+ exit
+fi
+
+docker build -t xos .
+
+docker stop $XOS
+docker run -p 8000:8000 xos
diff --git a/xos/cord/models.py b/xos/cord/models.py
index 346fcc9..17dd816 100644
--- a/xos/cord/models.py
+++ b/xos/cord/models.py
@@ -1,5 +1,5 @@
from django.db import models
-from core.models import Service, PlCoreBase, Slice, Sliver, Tenant, Node, Image, User
+from core.models import Service, PlCoreBase, Slice, Sliver, Tenant, Node, Image, User, Flavor
from core.models.plcorebase import StrippedCharField
import os
from django.db import models
@@ -203,6 +203,7 @@
"firewall_rules": "accept all anywhere anywhere",
"url_filter_enable": False,
"url_filter_rules": "allow all",
+ "url_filter_level": "PG",
"cdn_enable": False,
"sliver_id": None}
@@ -316,6 +317,14 @@
self.set_attribute("url_filter_enable", value)
@property
+ def url_filter_level(self):
+ return self.get_attribute("url_filter_level", self.default_attributes["url_filter_level"])
+
+ @url_filter_level.setter
+ def url_filter_level(self, value):
+ self.set_attribute("url_filter_level", value)
+
+ @property
def url_filter_rules(self):
return self.get_attribute("url_filter_rules", self.default_attributes["url_filter_rules"])
@@ -331,6 +340,27 @@
def cdn_enable(self, value):
self.set_attribute("cdn_enable", value)
+ @property
+ def users(self):
+ return [ {"name": "mom", "id": 1, "role": "admin"},
+ {"name": "dad", "id": 2, "role": "admin"},
+ {"name": "kid1", "id": 3, "role": "user"},
+ {"name": "kid2", "id": 4, "role": "user"} ]
+
+ @users.setter
+ def users(self, value):
+ pass
+
+ @property
+ def services(self):
+ return [ {"cdn": self.cdn_enable,
+ "url_filter": self.url_filter_enable,
+ "firewall": self.firewall_enable} ]
+
+ @services.setter
+ def services(self, value):
+ pass
+
def pick_node(self):
nodes = list(Node.objects.all())
# TODO: logic to filter nodes by which nodes are up, and which
@@ -352,12 +382,17 @@
if not self.provider_service.slices.count():
raise XOSConfigurationError("The VCPE service has no slices")
+ flavors = Flavor.objects.filter(name="m1.small")
+ if not flavors:
+ raise XOSConfigurationError("No m1.small flavor")
+
node =self.pick_node()
sliver = Sliver(slice = self.provider_service.slices.all()[0],
node = node,
image = self.image,
creator = self.creator,
- deployment = node.site_deployment.deployment)
+ deployment = node.site_deployment.deployment,
+ flavor = flavors[0])
sliver.save()
try:
diff --git a/xos/core/models/sliver.py b/xos/core/models/sliver.py
index ff1e9b4..6c102d5 100644
--- a/xos/core/models/sliver.py
+++ b/xos/core/models/sliver.py
@@ -185,3 +185,11 @@
pubkeys.add(self.slice.service.public_key)
return pubkeys
+
+def controller_setter(instance, **kwargs):
+ try:
+ instance.controller = instance.node.site_deployment.controller
+ except:
+ instance.controller = None
+
+models.signals.post_init.connect(controller_setter, Sliver)
diff --git a/xos/core/xoslib/methods/__init__.py b/xos/core/xoslib/methods/__init__.py
index 0b891eb..c777668 100644
--- a/xos/core/xoslib/methods/__init__.py
+++ b/xos/core/xoslib/methods/__init__.py
@@ -1,5 +1,6 @@
from django.views.generic import View
from django.conf.urls import patterns, url
+from rest_framework.routers import DefaultRouter
import os, sys
import inspect
import importlib
@@ -40,6 +41,19 @@
urlpatterns.append(url(r'^' + view_url[1] + '/$', view_url[3].as_view(), name=view_url[1]+'list'))
elif view_url[0] == "detail":
urlpatterns.append(url(r'^' + view_url[1] + '/(?P<pk>[a-zA-Z0-9\-]+)/$', view_url[3].as_view(), name=view_url[1]+'detail'))
+ elif view_url[0] == "viewset":
+ viewset = view_url[3]
+
+ urlpatterns.extend(viewset.get_urlpatterns())
+
+ #urlpatterns.append(url(r'^' + view_url[1] + '/$', viewset.as_view({'get': 'list'}), name=view_url[1]+'list'))
+ #urlpatterns.append(url(r'^' + view_url[1] + '/(?P<pk>[a-zA-Z0-9\-]+)/$', viewset.as_view({'get': 'retrieve', 'put': 'update', 'post': 'create', 'delete': 'destroy', 'patch': 'partial_update'}), name=view_url[1]+'detail'))
+ #urlpatterns.extend(
+
+ #router = DefaultRouter()
+ #router.register(r'^' + view_url[1], view_url[3], base_name="foo")
+ #urlpatterns.extend(router.urls)
+ #urlpatterns.append(url(r'^' + view_url[1], view_url[3]))
finally:
sys.path = sys_path_save
diff --git a/xos/core/xoslib/methods/cordsubscriber.py b/xos/core/xoslib/methods/cordsubscriber.py
index 08aa9d9..a6c9a63 100644
--- a/xos/core/xoslib/methods/cordsubscriber.py
+++ b/xos/core/xoslib/methods/cordsubscriber.py
@@ -3,12 +3,17 @@
from rest_framework.reverse import reverse
from rest_framework import serializers
from rest_framework import generics
+from rest_framework import viewsets
+from rest_framework.decorators import detail_route, list_route
from core.models import *
from django.forms import widgets
+from django.conf.urls import patterns, url
from cord.models import VOLTTenant
from core.xoslib.objects.cordsubscriber import CordSubscriber
from plus import PlusSerializerMixin
+from django.shortcuts import get_object_or_404
from xos.apibase import XOSListCreateAPIView, XOSRetrieveUpdateDestroyAPIView, XOSPermissionDenied
+import json
if hasattr(serializers, "ReadOnlyField"):
# rest_framework 3.x
@@ -29,6 +34,7 @@
firewall_rules = serializers.CharField()
url_filter_enable = serializers.BooleanField()
url_filter_rules = serializers.CharField()
+ url_filter_level = serializers.CharField(required=False)
cdn_enable = serializers.BooleanField()
sliver_name = ReadOnlyField()
image_name = ReadOnlyField()
@@ -40,7 +46,7 @@
model = CordSubscriber
fields = ('humanReadableName', 'id',
'service_specific_id', 'vlan_id',
- 'vcpe_id', 'sliver', 'sliver_name', 'image', 'image_name', 'firewall_enable', 'firewall_rules', 'url_filter_enable', 'url_filter_rules', 'cdn_enable', 'vbng_id', 'routeable_subnet',)
+ 'vcpe_id', 'sliver', 'sliver_name', 'image', 'image_name', 'firewall_enable', 'firewall_rules', 'url_filter_enable', 'url_filter_rules', 'url_filter_level', 'cdn_enable', 'vbng_id', 'routeable_subnet',)
def getHumanReadableName(self, obj):
@@ -60,4 +66,83 @@
method_kind = "detail"
method_name = "cordsubscriber"
+# this may be moved into plus.py...
+
+class XOSViewSet(viewsets.ModelViewSet):
+ @classmethod
+ def detail_url(self, pattern, viewdict, name):
+ return url(r'^' + self.method_name + r'/(?P<pk>[a-zA-Z0-9\-]+)/' + pattern,
+ self.as_view(viewdict),
+ name=self.base_name+"_"+name)
+
+ @classmethod
+ def get_urlpatterns(self):
+ patterns = []
+
+ patterns.append(url(r'^' + self.method_name + '/$', self.as_view({'get': 'list'}), name=self.base_name+'_list'))
+ patterns.append(url(r'^' + self.method_name + '/(?P<pk>[a-zA-Z0-9\-]+)/$', self.as_view({'get': 'retrieve', 'put': 'update', 'post': 'update', 'delete': 'destroy', 'patch': 'partial_update'}), name=self.base_name+'_detail'))
+
+ return patterns
+
+# the "new" API with many more REST endpoints.
+
+class CordSubscriberViewSet(XOSViewSet):
+ base_name = "subscriber"
+ method_name = "rs/subscriber"
+ method_kind = "viewset"
+ queryset = CordSubscriber.get_tenant_objects().select_related().all()
+ serializer_class = CordSubscriberIdSerializer
+
+ @classmethod
+ def get_urlpatterns(self):
+ patterns = super(CordSubscriberViewSet, self).get_urlpatterns()
+ patterns.append( self.detail_url("url_filtering/$", {"get": "get_url_filtering"}, "url_filtering") )
+ patterns.append( self.detail_url("url_filtering/(?P<level>[a-zA-Z0-9\-]+)/$", {"get": "set_url_filtering"}, "url_filtering") )
+ patterns.append( self.detail_url("users/$", {"get": "get_users"}, "users") )
+ patterns.append( self.detail_url("services/$", {"get": "get_services"}, "services") )
+ patterns.append( self.detail_url("services/(?P<service>[a-zA-Z0-9\-]+)/$", {"get": "get_service"}, "get_service") )
+ patterns.append( self.detail_url("services/(?P<service>[a-zA-Z0-9\-]+)/true/$", {"get": "enable_service"}, "enable_service") )
+ patterns.append( self.detail_url("services/(?P<service>[a-zA-Z0-9\-]+)/false/$", {"get": "disable_service"}, "disable_service") )
+
+ return patterns
+
+ def get_url_filtering(self, request, pk=None):
+ subscriber = self.get_object()
+ return Response(subscriber.url_filter_level)
+
+ def set_url_filtering(self, request, pk=None, level=None):
+ subscriber = self.get_object()
+ subscriber.url_filter_level = level
+ subscriber.save()
+ return Response(subscriber.url_filter_level)
+
+ def get_users(self, request, pk=None):
+ subscriber = self.get_object()
+ return Response(subscriber.users)
+
+ def get_services(self, request, pk=None):
+ subscriber = self.get_object()
+ return Response(subscriber.services)
+
+ def get_service(self, request, pk=None, service=None):
+ service_attr = service+"_enable"
+ subscriber = self.get_object()
+ return Response(getattr(subscriber, service_attr))
+
+ def enable_service(self, request, pk=None, service=None):
+ service_attr = service+"_enable"
+ subscriber = self.get_object()
+ setattr(subscriber, service_attr, True)
+ subscriber.save()
+ return Response(getattr(subscriber, service_attr))
+
+ def disable_service(self, request, pk=None, service=None):
+ service_attr = service+"_enable"
+ subscriber = self.get_object()
+ setattr(subscriber, service_attr, False)
+ subscriber.save()
+ return Response(getattr(subscriber, service_attr))
+
+
+
diff --git a/xos/core/xoslib/objects/cordsubscriber.py b/xos/core/xoslib/objects/cordsubscriber.py
index f4cd8e6..72a9984 100644
--- a/xos/core/xoslib/objects/cordsubscriber.py
+++ b/xos/core/xoslib/objects/cordsubscriber.py
@@ -29,6 +29,9 @@
("firewall_rules", "vcpe.firewall_rules"),
("url_filter_enable", "vcpe.url_filter_enable"),
("url_filter_rules", "vcpe.url_filter_rules"),
+ ("url_filter_level", "vcpe.url_filter_level"),
+ ("users", "vcpe.users"),
+ ("services", "vcpe.services"),
("cdn_enable", "vcpe.cdn_enable"),
("image", "vcpe.image.id"),
("image_name", "vcpe.image.name"),
diff --git a/xos/hpc/models.py b/xos/hpc/models.py
index a3b7c90..1cd51ce 100644
--- a/xos/hpc/models.py
+++ b/xos/hpc/models.py
@@ -35,7 +35,7 @@
scale = getattr(self, "set_scale", None)
if scale is not None:
exclude_slices = [x for x in self.slices.all() if "cmi" in x.name]
- self.adjust_scale(slice_hint="hpc", scale=scale, exclusive_slices = exclude_slices)
+ self.adjust_scale(slice_hint="hpc", scale=scale, exclusive_slices = exclude_slices, max_per_node=1)
class ServiceProvider(PlCoreBase):
class Meta:
diff --git a/xos/hpc_observer/run.sh b/xos/hpc_observer/run.sh
deleted file mode 100644
index abf620a..0000000
--- a/xos/hpc_observer/run.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-if [[ ! -e ./hpc-backend.py ]]; then
- ln -s ../xos-observer.py hpc-backend.py
-fi
-
-export XOS_DIR=/opt/xos
-python hpc-backend.py -C $XOS_DIR/hpc_observer/hpc_observer_config
diff --git a/xos/hpc_observer/start.sh b/xos/hpc_observer/start.sh
deleted file mode 100644
index 07a1663..0000000
--- a/xos/hpc_observer/start.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-if [[ ! -e ./hpc-backend.py ]]; then
- ln -s ../xos-observer.py hpc-backend.py
-fi
-
-export XOS_DIR=/opt/xos
-nohup python hpc-backend.py -C $XOS_DIR/hpc_observer/hpc_observer_config > /dev/null 2>&1 &
diff --git a/xos/hpc_observer/stop.sh b/xos/hpc_observer/stop.sh
deleted file mode 100644
index 632f7a6..0000000
--- a/xos/hpc_observer/stop.sh
+++ /dev/null
@@ -1 +0,0 @@
-pkill -9 -f hpc-backend.py
diff --git a/xos/model_policies/model_policy_Controller.py b/xos/model_policies/model_policy_Controller.py
index fa84be7..2db7a63 100644
--- a/xos/model_policies/model_policy_Controller.py
+++ b/xos/model_policies/model_policy_Controller.py
@@ -22,7 +22,7 @@
slices = Slice.objects.all()
for slice in slices:
if slice not in ctrls_by_slice or \
- controller not in ctrls_by_slice:
+ controller not in ctrls_by_slice[slice]:
controller_slice = ControllerSlice(controller=controller, slice=slice)
controller_slice.save()
# relations for all users
diff --git a/xos/hpc_observer/Makefile b/xos/observers/hpc/Makefile
similarity index 100%
rename from xos/hpc_observer/Makefile
rename to xos/observers/hpc/Makefile
diff --git a/xos/hpc_observer/fsck.py b/xos/observers/hpc/fsck.py
similarity index 100%
rename from xos/hpc_observer/fsck.py
rename to xos/observers/hpc/fsck.py
diff --git a/xos/observers/hpc/hpc-observer.py b/xos/observers/hpc/hpc-observer.py
new file mode 100755
index 0000000..d6a71ff
--- /dev/null
+++ b/xos/observers/hpc/hpc-observer.py
@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+
+# This imports and runs ../../xos-observer.py
+
+import importlib
+import os
+import sys
+observer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),"../..")
+sys.path.append(observer_path)
+mod = importlib.import_module("xos-observer")
+mod.main()
diff --git a/xos/hpc_observer/hpc_observer_config b/xos/observers/hpc/hpc_observer_config
similarity index 74%
rename from xos/hpc_observer/hpc_observer_config
rename to xos/observers/hpc/hpc_observer_config
index 0e7e53a..16d9077 100644
--- a/xos/hpc_observer/hpc_observer_config
+++ b/xos/observers/hpc/hpc_observer_config
@@ -23,9 +23,9 @@
[observer]
name=hpc
-dependency_graph=/opt/xos/hpc_observer/model-deps
-steps_dir=/opt/xos/hpc_observer/steps
-deleters_dir=/opt/xos/hpc_observer/deleters
+dependency_graph=/opt/xos/observers/hpc/model-deps
+steps_dir=/opt/xos/observers/hpc/steps
+deleters_dir=/opt/xos/observers/hpc/deleters
log_file=console
#/var/log/hpc.log
driver=None
diff --git a/xos/hpc_observer/hpc_watcher.py b/xos/observers/hpc/hpc_watcher.py
similarity index 100%
rename from xos/hpc_observer/hpc_watcher.py
rename to xos/observers/hpc/hpc_watcher.py
diff --git a/xos/hpc_observer/hpclib.py b/xos/observers/hpc/hpclib.py
similarity index 100%
rename from xos/hpc_observer/hpclib.py
rename to xos/observers/hpc/hpclib.py
diff --git a/xos/hpc_observer/model-deps b/xos/observers/hpc/model-deps
similarity index 100%
rename from xos/hpc_observer/model-deps
rename to xos/observers/hpc/model-deps
diff --git a/xos/observers/hpc/run.sh b/xos/observers/hpc/run.sh
new file mode 100755
index 0000000..f77d751
--- /dev/null
+++ b/xos/observers/hpc/run.sh
@@ -0,0 +1,6 @@
+#if [[ ! -e ./hpc-backend.py ]]; then
+# ln -s ../xos-observer.py hpc-backend.py
+#fi
+
+export XOS_DIR=/opt/xos
+python hpc-observer.py -C $XOS_DIR/observers/hpc/hpc_observer_config
diff --git a/xos/observers/hpc/start.sh b/xos/observers/hpc/start.sh
new file mode 100755
index 0000000..305c07f
--- /dev/null
+++ b/xos/observers/hpc/start.sh
@@ -0,0 +1,6 @@
+#if [[ ! -e ./hpc-backend.py ]]; then
+# ln -s ../xos-observer.py hpc-backend.py
+#fi
+
+export XOS_DIR=/opt/xos
+nohup python hpc-observer.py -C $XOS_DIR/observers/hpc/hpc_observer_config > /dev/null 2>&1 &
diff --git a/xos/hpc_observer/steps/garbage_collector.py b/xos/observers/hpc/steps/garbage_collector.py
similarity index 100%
rename from xos/hpc_observer/steps/garbage_collector.py
rename to xos/observers/hpc/steps/garbage_collector.py
diff --git a/xos/hpc_observer/steps/sync_cdnprefix.py b/xos/observers/hpc/steps/sync_cdnprefix.py
similarity index 100%
rename from xos/hpc_observer/steps/sync_cdnprefix.py
rename to xos/observers/hpc/steps/sync_cdnprefix.py
diff --git a/xos/hpc_observer/steps/sync_contentprovider.py b/xos/observers/hpc/steps/sync_contentprovider.py
similarity index 100%
rename from xos/hpc_observer/steps/sync_contentprovider.py
rename to xos/observers/hpc/steps/sync_contentprovider.py
diff --git a/xos/hpc_observer/steps/sync_hpcservices.py b/xos/observers/hpc/steps/sync_hpcservices.py
similarity index 100%
rename from xos/hpc_observer/steps/sync_hpcservices.py
rename to xos/observers/hpc/steps/sync_hpcservices.py
diff --git a/xos/hpc_observer/steps/sync_originserver.py b/xos/observers/hpc/steps/sync_originserver.py
similarity index 100%
rename from xos/hpc_observer/steps/sync_originserver.py
rename to xos/observers/hpc/steps/sync_originserver.py
diff --git a/xos/hpc_observer/steps/sync_serviceprovider.py b/xos/observers/hpc/steps/sync_serviceprovider.py
similarity index 100%
rename from xos/hpc_observer/steps/sync_serviceprovider.py
rename to xos/observers/hpc/steps/sync_serviceprovider.py
diff --git a/xos/hpc_observer/steps/sync_sitemap.py b/xos/observers/hpc/steps/sync_sitemap.py
similarity index 100%
rename from xos/hpc_observer/steps/sync_sitemap.py
rename to xos/observers/hpc/steps/sync_sitemap.py
diff --git a/xos/observers/hpc/stop.sh b/xos/observers/hpc/stop.sh
new file mode 100755
index 0000000..a0b4a8e
--- /dev/null
+++ b/xos/observers/hpc/stop.sh
@@ -0,0 +1 @@
+pkill -9 -f hpc-observer.py
diff --git a/xos/observers/hpc/supervisor/hpc-observer.conf b/xos/observers/hpc/supervisor/hpc-observer.conf
new file mode 100644
index 0000000..f2c79d4
--- /dev/null
+++ b/xos/observers/hpc/supervisor/hpc-observer.conf
@@ -0,0 +1,2 @@
+[program:hpc-observer]
+command=python /opt/xos/observers/hpc/hpc-observer.py -C /opt/xos/observers/hpc/hpc_observer_config
diff --git a/xos/observers/hpc/supervisor/hpc-watcher.conf b/xos/observers/hpc/supervisor/hpc-watcher.conf
new file mode 100644
index 0000000..e0f4eb1
--- /dev/null
+++ b/xos/observers/hpc/supervisor/hpc-watcher.conf
@@ -0,0 +1,2 @@
+[program:hpc-watcher]
+command=python /opt/xos/observers/hpc/hpc_watcher.py
diff --git a/xos/observers/vbng/run.sh b/xos/observers/vbng/run.sh
index 7829ac0..efb586f 100755
--- a/xos/observers/vbng/run.sh
+++ b/xos/observers/vbng/run.sh
@@ -1,6 +1,6 @@
-if [[ ! -e ./vbng-observer.py ]]; then
- ln -s ../../xos-observer.py vbng-observer.py
-fi
+#if [[ ! -e ./vbng-observer.py ]]; then
+# ln -s ../../xos-observer.py vbng-observer.py
+#fi
export XOS_DIR=/opt/xos
python vbng-observer.py -C $XOS_DIR/observers/vbng/vbng_observer_config
diff --git a/xos/observers/vbng/start.sh b/xos/observers/vbng/start.sh
index 5ceff27..98008f4 100755
--- a/xos/observers/vbng/start.sh
+++ b/xos/observers/vbng/start.sh
@@ -1,6 +1,6 @@
-if [[ ! -e ./vbng-observer.py ]]; then
- ln -s ../../xos-observer.py vbng-observer.py
-fi
+#if [[ ! -e ./vbng-observer.py ]]; then
+# ln -s ../../xos-observer.py vbng-observer.py
+#fi
export XOS_DIR=/opt/xos
nohup python vbng-observer.py -C $XOS_DIR/observers/vbng/vbng_observer_config > /dev/null 2>&1 &
diff --git a/xos/observers/vbng/supervisor/vbng-observer.conf b/xos/observers/vbng/supervisor/vbng-observer.conf
new file mode 100644
index 0000000..cff77b8
--- /dev/null
+++ b/xos/observers/vbng/supervisor/vbng-observer.conf
@@ -0,0 +1,2 @@
+[program:vbng-observer]
+command=python /opt/xos/observers/vbng/vbng-observer.py -C /opt/xos/observers/vbng/vbng_observer_config
diff --git a/xos/observers/vbng/vbng-observer.py b/xos/observers/vbng/vbng-observer.py
new file mode 100755
index 0000000..d6a71ff
--- /dev/null
+++ b/xos/observers/vbng/vbng-observer.py
@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+
+# This imports and runs ../../xos-observer.py
+
+import importlib
+import os
+import sys
+observer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),"../..")
+sys.path.append(observer_path)
+mod = importlib.import_module("xos-observer")
+mod.main()
diff --git a/xos/observers/vbng/vbng_observer_config b/xos/observers/vbng/vbng_observer_config
index 217ce71..9094bb1 100644
--- a/xos/observers/vbng/vbng_observer_config
+++ b/xos/observers/vbng/vbng_observer_config
@@ -25,7 +25,7 @@
name=vbng
dependency_graph=/opt/xos/observers/vbng/model-deps
steps_dir=/opt/xos/observers/vbng/steps
-sys_dir=/opt/xos/observer/vbng/sys
+sys_dir=/opt/xos/observers/vbng/sys
deleters_dir=/opt/xos/observers/vbng/deleters
log_file=console
#/var/log/hpc.log
diff --git a/xos/observers/vcpe/files/docker.list b/xos/observers/vcpe/files/docker.list
new file mode 100644
index 0000000..0ee9ae0
--- /dev/null
+++ b/xos/observers/vcpe/files/docker.list
@@ -0,0 +1 @@
+deb https://get.docker.com/ubuntu docker main
diff --git a/xos/observers/vcpe/run.sh b/xos/observers/vcpe/run.sh
index c582842..f180e66 100755
--- a/xos/observers/vcpe/run.sh
+++ b/xos/observers/vcpe/run.sh
@@ -1,6 +1,6 @@
-if [[ ! -e ./vcpe-observer.py ]]; then
- ln -s ../../xos-observer.py vcpe-observer.py
-fi
+#if [[ ! -e ./vcpe-observer.py ]]; then
+# ln -s ../../xos-observer.py vcpe-observer.py
+#fi
export XOS_DIR=/opt/xos
python vcpe-observer.py -C $XOS_DIR/observers/vcpe/vcpe_observer_config
diff --git a/xos/observers/vcpe/start.sh b/xos/observers/vcpe/start.sh
index 96cb6e4..b402e5d 100755
--- a/xos/observers/vcpe/start.sh
+++ b/xos/observers/vcpe/start.sh
@@ -1,6 +1,6 @@
-if [[ ! -e ./vcpe-observer.py ]]; then
- ln -s ../../xos-observer.py vcpe-observer.py
-fi
+#if [[ ! -e ./vcpe-observer.py ]]; then
+# ln -s ../../xos-observer.py vcpe-observer.py
+#fi
export XOS_DIR=/opt/xos
nohup python vcpe-observer.py -C $XOS_DIR/observers/vcpe/vcpe_observer_config > /dev/null 2>&1 &
diff --git a/xos/observers/vcpe/steps/sync_vcpetenant.py b/xos/observers/vcpe/steps/sync_vcpetenant.py
index 3fc9310..8cda4ef 100644
--- a/xos/observers/vcpe/steps/sync_vcpetenant.py
+++ b/xos/observers/vcpe/steps/sync_vcpetenant.py
@@ -97,6 +97,7 @@
"hostname": sliver.node.name,
"instance_id": sliver.instance_id,
"private_key": service_key,
+ "ansible_tag": "vcpe_tenant_" + str(o.id)
}
if hasattr(o, "sync_attributes"):
diff --git a/xos/observers/vcpe/steps/sync_vcpetenant.yaml b/xos/observers/vcpe/steps/sync_vcpetenant.yaml
index 6c2bad4..92da2cc 100644
--- a/xos/observers/vcpe/steps/sync_vcpetenant.yaml
+++ b/xos/observers/vcpe/steps/sync_vcpetenant.yaml
@@ -22,14 +22,71 @@
{% endfor %}
tasks:
- - name: make sure /etc/dnsmasq.d exists
- file: path=/etc/dnsmasq.d state=directory owner=root group=root
+ - name: Docker repository
+ copy: src=/opt/xos/observers/vcpe/files/docker.list
+ dest=/etc/apt/sources.list.d/docker.list
+
+ - name: Import the repository key
+ apt_key: keyserver=keyserver.ubuntu.com id=36A1D7869245C8950F966E92D8576A8BA88D21E9
+
+ - name: install Docker
+ apt: name=lxc-docker-1.5.0 state=present update_cache=yes
+
+ - name: install python-setuptools
+ apt: name=python-setuptools state=present
+
+ - name: install pip
+ easy_install: name=pip
+
+ - name: install docker-py
+ pip: name=docker-py version=0.5.3
+
+ - name: install Pipework
+ get_url: url=https://raw.githubusercontent.com/jpetazzo/pipework/master/pipework
+ dest=/usr/local/bin/pipework
+ mode=0755
+
+ - name: bring up LAN interface
+ shell: ifconfig eth2 up
+
+ - name: bring up WAN interface
+ shell: ifconfig eth3 up
+
+ - name: make sure /etc/dnsmasq.d exists
+ file: path=/etc/dnsmasq.d state=directory owner=root group=root
- - name: configure dnsmasq servers
- template: src=/opt/xos/observers/vcpe/templates/dnsmasq_servers.j2 dest=/etc/dnsmasq.d/servers.conf owner=root group=root
+ - name: dnsmasq config
+ template: src=/opt/xos/observers/vcpe/templates/dnsmasq_servers.j2 dest=/etc/dnsmasq.d/servers.conf owner=root group=root
- - name: setup networking
- template: src=/opt/xos/observers/vcpe/templates/vlan_sample.j2 dest=/etc/vlan_sample owner=root group=root
+ - name: networking info
+ template: src=/opt/xos/observers/vcpe/templates/vlan_sample.j2 dest=/etc/vlan_sample owner=root group=root
- - name: setup firewall
- template: src=/opt/xos/observers/vcpe/templates/firewall_sample.j2 dest=/etc/firewall_sample owner=root group=root
+ - name: firewall info
+ template: src=/opt/xos/observers/vcpe/templates/firewall_sample.j2 dest=/etc/firewall_sample owner=root group=root
+
+ - name: Make sure iptables module loaded
+ shell: "iptables -L > /dev/null"
+
+ - name: Make sure ip6tables module loaded
+ shell: "ip6tables -L > /dev/null"
+
+ - name: Disable resolvconf updates (to avoid overwriting /etc/resolv.conf on host)
+ shell: service resolvconf disable-updates
+
+ - name: spin up container
+ docker: name=vcpe
+ image=andybavier/docker-vcpe
+ net=none
+ privileged=yes
+
+ # This needs attention once ONOS is integrated
+ - name: Connect container to WAN network
+ shell: docker exec vcpe ifconfig eth0 >> /dev/null || pipework eth3 -i eth0 vcpe 10.1.2.3/24
+
+ # Only uses the first vlan_id
+ - name: Connect container to LAN network, VLAN {{ vlan_ids[0] }}
+ shell: docker exec vcpe ifconfig eth1 >> /dev/null || pipework eth2 -i eth1 vcpe 192.168.0.1/24 @{{ vlan_ids[0] }}
+
+ - name: Start container services
+ shell: docker exec vcpe service dnsmasq start
+
diff --git a/xos/observers/vcpe/supervisor/vcpe-observer.conf b/xos/observers/vcpe/supervisor/vcpe-observer.conf
new file mode 100644
index 0000000..27d2796
--- /dev/null
+++ b/xos/observers/vcpe/supervisor/vcpe-observer.conf
@@ -0,0 +1,2 @@
+[program:vcpe-observer]
+command=python /opt/xos/observers/vcpe/vcpe-observer.py -C /opt/xos/observers/vcpe/vcpe_observer_config
diff --git a/xos/observers/vcpe/templates/dnsmasq_servers.j2 b/xos/observers/vcpe/templates/dnsmasq_servers.j2
index 359070a..76b8f1c 100644
--- a/xos/observers/vcpe/templates/dnsmasq_servers.j2
+++ b/xos/observers/vcpe/templates/dnsmasq_servers.j2
@@ -9,8 +9,13 @@
{% endif %}
{% if url_filter_enable %}
-# placeholder; figure out what to really use...
-server=dns.xerocole.com
+# Point to BroadbandShield service, disable cache, and enable eDNS MAC passing
+add-mac
+cache-size=0
+server=198.105.255.10
+server=198.105.255.11
+server=198.105.255.12
+server=198.105.255.13
{% else %}
# use google's DNS service
server=8.8.8.8
diff --git a/xos/observers/vcpe/vcpe-observer.py b/xos/observers/vcpe/vcpe-observer.py
new file mode 100755
index 0000000..d6a71ff
--- /dev/null
+++ b/xos/observers/vcpe/vcpe-observer.py
@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+
+# This imports and runs ../../xos-observer.py
+
+import importlib
+import os
+import sys
+observer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),"../..")
+sys.path.append(observer_path)
+mod = importlib.import_module("xos-observer")
+mod.main()
diff --git a/xos/observers/vcpe/vcpe_observer_config b/xos/observers/vcpe/vcpe_observer_config
index c6500ae..6d58340 100644
--- a/xos/observers/vcpe/vcpe_observer_config
+++ b/xos/observers/vcpe/vcpe_observer_config
@@ -25,12 +25,14 @@
name=vcpe
dependency_graph=/opt/xos/observers/vcpe/model-deps
steps_dir=/opt/xos/observers/vcpe/steps
-sys_dir=/opt/xos/observer/vcpe/sys
+sys_dir=/opt/xos/observers/vcpe/sys
deleters_dir=/opt/xos/observers/vcpe/deleters
log_file=console
#/var/log/hpc.log
driver=None
pretend=False
+backoff_disabled=True
+save_ansible_output=True
[feefie]
client_id='vicci_dev_central'
diff --git a/xos/openstack_observer/ansible.py b/xos/openstack_observer/ansible.py
index 17504b3..5b6bd1d 100644
--- a/xos/openstack_observer/ansible.py
+++ b/xos/openstack_observer/ansible.py
@@ -43,24 +43,44 @@
return results
+def parse_unreachable(msg):
+ total_unreachable=0
+ for l in msg.splitlines():
+ x = re.findall('ok=([0-9]+).*changed=([0-9]+).*unreachable=([0-9]+).*failed=([0-9]+)', l)
+ if x:
+ (ok, changed, unreachable, failed) = x[0]
+ ok=int(ok)
+ changed=int(changed)
+ unreachable=int(unreachable)
+ failed=int(failed)
+
+ total_unreachable += unreachable
+ return total_unreachable
+
+
def id_generator(size=6, chars=string.ascii_uppercase + string.digits):
return ''.join(random.choice(chars) for _ in range(size))
def shellquote(s):
return "'" + s.replace("'", "'\\''") + "'"
-def run_template(name, opts,path='', expected_num=None, ansible_config=None, ansible_hosts=None, run_ansible_script=None):
+def get_playbook_fn(opts, path):
+ if not opts.get("ansible_tag", None):
+ # if no ansible_tag is in the options, then generate a unique one
+ objname= id_generator()
+ opts = opts.copy()
+ opts["ansible_tag"] = objname
+
+ objname = opts["ansible_tag"]
+
+ os.system('mkdir -p %s' % os.path.join(sys_dir, path))
+ return (opts, os.path.join(sys_dir,path,objname))
+
+def run_template(name, opts, path='', expected_num=None, ansible_config=None, ansible_hosts=None, run_ansible_script=None):
template = os_template_env.get_template(name)
buffer = template.render(opts)
- try:
- objname = opts['ansible_tag']
- except:
- objname= id_generator()
-
- os.system('mkdir -p %s'%'/'.join([sys_dir,path]))
- fqp = '/'.join([sys_dir,path,objname])
-
+ (opts, fqp) = get_playbook_fn(opts, path)
f = open(fqp,'w')
f.write(buffer)
@@ -78,11 +98,17 @@
if not run_ansible_script:
run_ansible_script = os.path.join(XOS_DIR, "observer/run_ansible")
- #run = os.popen(XOS_DIR + '/observer/run_ansible %s'%shellquote(fqp), env=env)
- run = subprocess.Popen("%s %s" % (run_ansible_script, shellquote(fqp)), shell=True, stdout=subprocess.PIPE, env=env).stdout
- msg = run.read()
- status = run.close()
+ process = subprocess.Popen("%s %s" % (run_ansible_script, shellquote(fqp)), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
+ msg = process.stdout.read()
+ err_msg = process.stderr.read()
+ if getattr(Config(), "observer_save_ansible_output", False):
+ try:
+ open(fqp+".out","w").write(msg)
+ open(fqp+".err","w").write(err_msg)
+ except:
+ # fail silently
+ pass
else:
msg = open(fqp+'.out').read()
@@ -91,6 +117,10 @@
ok_results = parse_output(msg)
if (expected_num is not None) and (len(ok_results) != expected_num):
raise ValueError('Unexpected num %s!=%d' % (str(expected_num), len(ok_results)) )
+
+ total_unreachable = parse_unreachable(msg)
+ if (total_unreachable > 0):
+ raise ValueError("Unreachable results in ansible recipe")
except ValueError,e:
all_fatal = [e.message] + re.findall(r'^msg: (.*)',msg,re.MULTILINE)
all_fatal2 = re.findall(r'^ERROR: (.*)',msg,re.MULTILINE)
@@ -110,35 +140,35 @@
hostname = opts["hostname"]
private_key = opts["private_key"]
- (private_key_handle, private_key_pathname) = tempfile.mkstemp()
- (config_handle, config_pathname) = tempfile.mkstemp()
- (hosts_handle, hosts_pathname) = tempfile.mkstemp()
+ (opts, fqp) = get_playbook_fn(opts, path)
+ private_key_pathname = fqp + ".key"
+ config_pathname = fqp + ".config"
+ hosts_pathname = fqp + ".hosts"
- try:
- proxy_command = "ProxyCommand ssh -q -i %s -o StrictHostKeyChecking=no %s@%s" % (private_key_pathname, instance_id, hostname)
+ proxy_command = "ProxyCommand ssh -q -i %s -o StrictHostKeyChecking=no %s@%s" % (private_key_pathname, instance_id, hostname)
- os.write(private_key_handle, private_key)
- os.close(private_key_handle)
+ f = open(private_key_pathname, "w")
+ f.write(private_key)
+ f.close()
- os.write(config_handle, "[ssh_connection]\n")
- os.write(config_handle, 'ssh_args = -o "%s" -o StrictHostKeyChecking=no\n' % proxy_command)
- os.write(config_handle, 'scp_if_ssh = True\n')
- os.close(config_handle)
+ f = open(config_pathname, "w")
+ f.write("[ssh_connection]\n")
+ f.write('ssh_args = -o "%s" -o StrictHostKeyChecking=no\n' % proxy_command)
+ f.write('scp_if_ssh = True\n')
+ f.close()
- os.write(hosts_handle, "[%s]\n" % sliver_name)
- os.write(hosts_handle, "%s ansible_ssh_private_key_file=%s\n" % (hostname, private_key_pathname))
- os.close(hosts_handle)
+ f = open(hosts_pathname, "w")
+ f.write("[%s]\n" % sliver_name)
+ f.write("%s ansible_ssh_private_key_file=%s\n" % (hostname, private_key_pathname))
+ f.close()
- print "ANSIBLE_CONFIG=%s" % config_pathname
- print "ANSIBLE_HOSTS=%s" % hosts_pathname
+ # SSH will complain if private key is world or group readable
+ os.chmod(private_key_pathname, 0600)
- return run_template(name, opts, path, expected_num, ansible_config = config_pathname, ansible_hosts = hosts_pathname, run_ansible_script="/opt/xos/observer/run_ansible_verbose")
+ print "ANSIBLE_CONFIG=%s" % config_pathname
+ print "ANSIBLE_HOSTS=%s" % hosts_pathname
- finally:
- #os.remove(private_key_pathname)
- #os.remove(config_pathname)
- #os.remove(hosts_pathname)
- pass
+ return run_template(name, opts, path, expected_num, ansible_config = config_pathname, ansible_hosts = hosts_pathname, run_ansible_script="/opt/xos/observer/run_ansible_verbose")
diff --git a/xos/openstack_observer/steps/sync_controller_images.py b/xos/openstack_observer/steps/sync_controller_images.py
index 94b18a0..948fcea 100644
--- a/xos/openstack_observer/steps/sync_controller_images.py
+++ b/xos/openstack_observer/steps/sync_controller_images.py
@@ -4,6 +4,7 @@
from django.db.models import F, Q
from xos.config import Config
from observer.openstacksyncstep import OpenStackSyncStep
+from observer.syncstep import *
from core.models import Controller
from core.models import Image, ControllerImages
from util.logger import observer_logger as logger
@@ -27,7 +28,7 @@
controller_register = json.loads(controller_image.controller.backend_register)
if (controller_register.get('disabled',False)):
- raise Exception('Controller %s is disabled'%controller_image.controller.name)
+ raise InnocuousException('Controller %s is disabled'%controller_image.controller.name)
image_fields = {'endpoint':controller_image.controller.auth_url,
'admin_user':controller_image.controller.admin_user,
diff --git a/xos/openstack_observer/steps/sync_controller_networks.py b/xos/openstack_observer/steps/sync_controller_networks.py
index d327b7b..278d018 100644
--- a/xos/openstack_observer/steps/sync_controller_networks.py
+++ b/xos/openstack_observer/steps/sync_controller_networks.py
@@ -5,6 +5,7 @@
from django.db.models import F, Q
from xos.config import Config
from observer.openstacksyncstep import OpenStackSyncStep
+from observer.syncstep import *
from core.models.network import *
from core.models.slice import *
from core.models.sliver import Sliver
@@ -65,7 +66,7 @@
controller_register = json.loads(controller_network.controller.backend_register)
if (controller_register.get('disabled',False)):
- raise Exception('Controller %s is disabled'%controller_network.controller.name)
+ raise InnocuousException('Controller %s is disabled'%controller_network.controller.name)
if not controller_network.controller.admin_user:
logger.info("controller %r has no admin_user, skipping" % controller_network.controller)
@@ -78,7 +79,7 @@
def delete_record(self, controller_network):
controller_register = json.loads(controller_network.controller.backend_register)
if (controller_register.get('disabled',False)):
- raise Exception('Controller %s is disabled'%controller_network.controller.name)
+ raise InnocuousException('Controller %s is disabled'%controller_network.controller.name)
try:
slice = controller_network.network.owner # XXX: FIXME!!
diff --git a/xos/openstack_observer/steps/sync_controller_site_privileges.py b/xos/openstack_observer/steps/sync_controller_site_privileges.py
index 6a13736..a2c40ef 100644
--- a/xos/openstack_observer/steps/sync_controller_site_privileges.py
+++ b/xos/openstack_observer/steps/sync_controller_site_privileges.py
@@ -4,6 +4,7 @@
from django.db.models import F, Q
from xos.config import Config
from observer.openstacksyncstep import OpenStackSyncStep
+from observer.syncstep import *
from core.models.site import Controller, SitePrivilege
from core.models.user import User
from core.models.controlleruser import ControllerUser, ControllerSitePrivilege
@@ -28,7 +29,7 @@
controller_register = json.loads(controller_site_privilege.controller.backend_register)
if (controller_register.get('disabled',False)):
- raise Exception('Controller %s is disabled'%controller_site_privilege.controller.name)
+ raise InnocuousException('Controller %s is disabled'%controller_site_privilege.controller.name)
if not controller_site_privilege.controller.admin_user:
@@ -76,7 +77,7 @@
def delete_record(self, controller_site_privilege):
controller_register = json.loads(controller_site_privilege.controller.backend_register)
if (controller_register.get('disabled',False)):
- raise Exception('Controller %s is disabled'%controller_site_privilege.controller.name)
+ raise InnocuousException('Controller %s is disabled'%controller_site_privilege.controller.name)
if controller_site_privilege.role_id:
driver = self.driver.admin_driver(controller=controller_site_privilege.controller)
diff --git a/xos/openstack_observer/steps/sync_controller_sites.py b/xos/openstack_observer/steps/sync_controller_sites.py
index f101315..670f09c 100644
--- a/xos/openstack_observer/steps/sync_controller_sites.py
+++ b/xos/openstack_observer/steps/sync_controller_sites.py
@@ -4,6 +4,7 @@
from xos.config import Config
from openstack_observer.openstacksyncstep import OpenStackSyncStep
from core.models.site import *
+from observer.syncstep import *
from observer.ansible import *
from util.logger import observer_logger as logger
import json
@@ -20,7 +21,7 @@
def sync_record(self, controller_site):
controller_register = json.loads(controller_site.controller.backend_register)
if (controller_register.get('disabled',False)):
- raise Exception('Controller %s is disabled'%controller_site.controller.name)
+ raise InnocuousException('Controller %s is disabled'%controller_site.controller.name)
template = os_template_env.get_template('sync_controller_sites.yaml')
tenant_fields = {'endpoint':controller_site.controller.auth_url,
@@ -41,7 +42,7 @@
def delete_record(self, controller_site):
controller_register = json.loads(controller_site.controller.backend_register)
if (controller_register.get('disabled',False)):
- raise Exception('Controller %s is disabled'%controller_site.controller.name)
+ raise InnocuousException('Controller %s is disabled'%controller_site.controller.name)
if controller_site.tenant_id:
driver = self.driver.admin_driver(controller=controller_site.controller)
diff --git a/xos/openstack_observer/steps/sync_controller_slice_privileges.py b/xos/openstack_observer/steps/sync_controller_slice_privileges.py
index 38f23c2..2e2e63c 100644
--- a/xos/openstack_observer/steps/sync_controller_slice_privileges.py
+++ b/xos/openstack_observer/steps/sync_controller_slice_privileges.py
@@ -4,6 +4,7 @@
from django.db.models import F, Q
from xos.config import Config
from observer.openstacksyncstep import OpenStackSyncStep
+from observer.syncstep import *
from core.models.slice import Controller, SlicePrivilege
from core.models.user import User
from core.models.controlleruser import ControllerUser, ControllerSlicePrivilege
@@ -28,7 +29,7 @@
controller_register = json.loads(controller_slice_privilege.controller.backend_register)
if (controller_register.get('disabled',False)):
- raise Exception('Controller %s is disabled'%controller_slice_privilege.controller.name)
+ raise InnocuousException('Controller %s is disabled'%controller_slice_privilege.controller.name)
if not controller_slice_privilege.controller.admin_user:
logger.info("controller %r has no admin_user, skipping" % controller_slice_privilege.controller)
@@ -75,7 +76,7 @@
def delete_record(self, controller_slice_privilege):
controller_register = json.loads(controller_slice_privilege.controller.backend_register)
if (controller_register.get('disabled',False)):
- raise Exception('Controller %s is disabled'%controller_slice_privilege.controller.name)
+ raise InnocuousException('Controller %s is disabled'%controller_slice_privilege.controller.name)
if controller_slice_privilege.role_id:
driver = self.driver.admin_driver(controller=controller_slice_privilege.controller)
diff --git a/xos/openstack_observer/steps/sync_controller_slices.py b/xos/openstack_observer/steps/sync_controller_slices.py
index 8d4a5e0..f64f9a8 100644
--- a/xos/openstack_observer/steps/sync_controller_slices.py
+++ b/xos/openstack_observer/steps/sync_controller_slices.py
@@ -5,6 +5,7 @@
from django.db.models import F, Q
from xos.config import Config
from observer.openstacksyncstep import OpenStackSyncStep
+from observer.syncstep import *
from core.models import *
from observer.ansible import *
from openstack.driver import OpenStackDriver
@@ -27,7 +28,7 @@
controller_register = json.loads(controller_slice.controller.backend_register)
if (controller_register.get('disabled',False)):
- raise Exception('Controller %s is disabled'%controller_slice.controller.name)
+ raise InnocuousException('Controller %s is disabled'%controller_slice.controller.name)
if not controller_slice.controller.admin_user:
logger.info("controller %r has no admin_user, skipping" % controller_slice.controller)
@@ -72,7 +73,7 @@
def delete_record(self, controller_slice):
controller_register = json.loads(controller_slice.controller.backend_register)
if (controller_register.get('disabled',False)):
- raise Exception('Controller %s is disabled'%controller_slice.controller.name)
+ raise InnocuousException('Controller %s is disabled'%controller_slice.controller.name)
controller_users = ControllerUser.objects.filter(user=controller_slice.slice.creator,
controller=controller_slice.controller)
diff --git a/xos/openstack_observer/steps/sync_controller_slices.yaml b/xos/openstack_observer/steps/sync_controller_slices.yaml
index 380f001..28c406d 100644
--- a/xos/openstack_observer/steps/sync_controller_slices.yaml
+++ b/xos/openstack_observer/steps/sync_controller_slices.yaml
@@ -2,9 +2,9 @@
- hosts: 127.0.0.1
connection: local
tasks:
- {% if delete %}
+ {% if delete -%}
- keystone_user: endpoint={{ endpoint }} login_user={{ admin_user }} login_password={{ admin_password }} login_tenant_name={{ admin_tenant }} tenant={{ tenant }} tenant_description="{{ tenant_description }}" state=absent
- {% else %}
+ {% else -%}
- keystone_user: endpoint={{ endpoint }} login_user={{ admin_user }} login_password={{ admin_password }} login_tenant_name={{ admin_tenant }} tenant={{ tenant }} tenant_description="{{ tenant_description }}"
{% for role in roles %}
- keystone_user: endpoint={{ endpoint}} login_user={{ admin_user }} login_password={{ admin_password }} login_tenant_name={{ admin_tenant }} user="{{ name }}" role={{ role }} tenant={{ tenant }}
diff --git a/xos/openstack_observer/steps/sync_controller_users.py b/xos/openstack_observer/steps/sync_controller_users.py
index 47d1096..7979d82 100644
--- a/xos/openstack_observer/steps/sync_controller_users.py
+++ b/xos/openstack_observer/steps/sync_controller_users.py
@@ -4,6 +4,7 @@
from django.db.models import F, Q
from xos.config import Config
from observer.openstacksyncstep import OpenStackSyncStep
+from observer.syncstep import *
from core.models.site import Controller, SiteDeployment, SiteDeployment
from core.models.user import User
from core.models.controlleruser import ControllerUser
@@ -28,7 +29,7 @@
controller_register = json.loads(controller_user.controller.backend_register)
if (controller_register.get('disabled',False)):
- raise Exception('Controller %s is disabled'%controller_user.controller.name)
+ raise InnocuousException('Controller %s is disabled'%controller_user.controller.name)
if not controller_user.controller.admin_user:
logger.info("controller %r has no admin_user, skipping" % controller_user.controller)
@@ -79,7 +80,7 @@
def delete_record(self, controller_user):
controller_register = json.loads(controller_user.controller.backend_register)
if (controller_register.get('disabled',False)):
- raise Exception('Controller %s is disabled'%controller_user.controller.name)
+ raise InnocuousException('Controller %s is disabled'%controller_user.controller.name)
if controller_user.kuser_id:
driver = self.driver.admin_driver(controller=controller_user.controller)
diff --git a/xos/openstack_observer/steps/sync_slivers.py b/xos/openstack_observer/steps/sync_slivers.py
index 9b5dd99..48f5c25 100644
--- a/xos/openstack_observer/steps/sync_slivers.py
+++ b/xos/openstack_observer/steps/sync_slivers.py
@@ -9,6 +9,7 @@
from core.models.slice import Slice, SlicePrivilege, ControllerSlice
from core.models.network import Network, NetworkSlice, ControllerNetwork
from observer.ansible import *
+from observer.syncstep import *
from util.logger import observer_logger as logger
def escape(s):
@@ -32,7 +33,7 @@
controller_register = json.loads(sliver.node.site_deployment.controller.backend_register)
if (controller_register.get('disabled',False)):
- raise Exception('Controller %s is disabled'%sliver.node.site_deployment.controller.name)
+ raise InnocuousException('Controller %s is disabled'%sliver.node.site_deployment.controller.name)
metadata_update = {}
if (sliver.numberCores):
@@ -82,20 +83,19 @@
if net['name']=='public':
nics.append(net['id'])
- image_id = None
+ image_name = None
controller_images = sliver.image.controllerimages.filter(controller=sliver.node.site_deployment.controller)
if controller_images:
- image_id = controller_images[0].glance_image_id
- logger.info("using image_id from ControllerImage object: " + str(image_id))
+ image_name = controller_images[0].image.name
+ logger.info("using image from ControllerImage object: " + str(image_name))
- if image_id is None:
+ if image_name is None:
controller_driver = self.driver.admin_driver(controller=sliver.node.site_deployment.controller)
- image_id = None
images = controller_driver.shell.glanceclient.images.list()
for image in images:
- if image.name == sliver.image.name or not image_id:
- image_id = image.id
- logger.info("using image_id from glance: " + str(image_id))
+ if image.name == sliver.image.name or not image_name:
+ image_name = image.name
+ logger.info("using image from glance: " + str(image_name))
try:
legacy = Config().observer_legacy
@@ -124,8 +124,8 @@
'name':sliver_name,
'ansible_tag':sliver_name,
'availability_zone': availability_zone_filter,
- 'image_id':image_id,
- 'flavor_id':sliver.flavor.id,
+ 'image_name':image_name,
+ 'flavor_name':sliver.flavor.name,
'nics':nics,
'meta':metadata_update,
'user_data':r'%s'%escape(userData)}
@@ -150,7 +150,7 @@
controller_register = json.loads(sliver.node.site_deployment.controller.backend_register)
if (controller_register.get('disabled',False)):
- raise Exception('Controller %s is disabled'%sliver.node.site_deployment.controller.name)
+ raise InnocuousException('Controller %s is disabled'%sliver.node.site_deployment.controller.name)
sliver_name = '%s-%d'%(sliver.slice.name,sliver.id)
controller = sliver.node.site_deployment.controller
diff --git a/xos/openstack_observer/steps/sync_slivers.yaml b/xos/openstack_observer/steps/sync_slivers.yaml
index 6ac995f..c543227 100644
--- a/xos/openstack_observer/steps/sync_slivers.yaml
+++ b/xos/openstack_observer/steps/sync_slivers.yaml
@@ -8,14 +8,14 @@
login_password: {{ admin_password }}
login_tenant_name: {{ admin_tenant }}
name: {{ name }}
- {% if delete %}
+ {% if delete -%}
state: absent
- {% else %}
+ {% else -%}
state: present
availability_zone: {{ availability_zone }}
- image_id: {{ image_id }}
+ image_name: {{ image_name }}
wait_for: 200
- flavor_id: {{ flavor_id }}
+ flavor_name: {{ flavor_name }}
user_data: "{{ user_data }}"
nics:
{% for net in nics %}
diff --git a/xos/openstack_observer/syncstep.py b/xos/openstack_observer/syncstep.py
index b752760..fb06bdd 100644
--- a/xos/openstack_observer/syncstep.py
+++ b/xos/openstack_observer/syncstep.py
@@ -21,20 +21,24 @@
strs = backend_str.split(' // ')
strs2 = f7(strs)
return ' // '.join(strs2)
-
+
def deepgetattr(obj, attr):
return reduce(getattr, attr.split('.'), obj)
+
+class InnocuousException(Exception):
+ pass
+
class FailedDependency(Exception):
pass
class SyncStep(object):
- """ An XOS Sync step.
+ """ An XOS Sync step.
Attributes:
- psmodel Model name the step synchronizes
+ psmodel Model name the step synchronizes
dependencies list of names of models that must be synchronized first if the current model depends on them
- """
+ """
slow=False
def get_prop(self, prop):
try:
@@ -73,20 +77,26 @@
return objs
#return Sliver.objects.filter(ip=None)
-
+
def check_dependencies(self, obj, failed):
for dep in self.dependencies:
peer_name = dep[0].lower() + dep[1:] # django names are camelCased with the first letter lower
-
+
try:
peer_object = deepgetattr(obj, peer_name)
- try:
- peer_objects = peer_object.all()
+ try:
+ peer_objects = peer_object.all()
except AttributeError:
- peer_objects = [peer_object]
+ peer_objects = [peer_object]
except:
peer_objects = []
+ if (hasattr(obj,'controller')):
+ try:
+ peer_objects = filter(lambda o:o.controller==obj.controller, peer_objects)
+ except AttributeError:
+ pass
+
if (failed in peer_objects):
if (obj.backend_status!=failed.backend_status):
obj.backend_status = failed.backend_status
@@ -109,6 +119,7 @@
if (not backoff_disabled and next_run>time.time()):
sync_failed = True
except:
+ logger.log_exc("Exception while loading scratchpad")
pass
if (not sync_failed):
@@ -125,26 +136,32 @@
o.backend_register = json.dumps(scratchpad)
o.backend_status = "1 - OK"
o.save(update_fields=['enacted','backend_status','backend_register'])
- except Exception,e:
+ except (InnocuousException,Exception) as e:
logger.log_exc("sync step failed!")
try:
if (o.backend_status.startswith('2 - ')):
str_e = '%s // %r'%(o.backend_status[4:],e)
- str_e = elim_dups(str_e)
+ str_e = elim_dups(str_e)
else:
str_e = '%r'%e
except:
str_e = '%r'%e
try:
- o.backend_status = '2 - %s'%self.error_map.map(str_e)
+ error = self.error_map.map(str_e)
except:
- o.backend_status = '2 - %s'%str_e
+ error = '2 - %s'%str_e
+
+ if isinstance(e, InnocuousException) and not force_error:
+ o.backend_status = '1 - %s'%error
+ else:
+ o.backend_status = '2 - %s'%error
try:
scratchpad = json.loads(o.backend_register)
scratchpad['exponent']
except:
+ logger.log_exc("Exception while updating scratchpad")
scratchpad = {'next_run':0, 'exponent':0}
# Second failure
@@ -163,7 +180,7 @@
if (o.pk):
try:
o.backend_status = o.backend_status[:1024]
- o.save(update_fields=['backend_status','backend_register'])
+ o.save(update_fields=['backend_status','backend_register','updated'])
except:
print "Could not update backend status field!"
pass
diff --git a/xos/tools/cleanup_unique.py b/xos/tools/cleanup_unique.py
index 0ee0c7a..29fb047 100644
--- a/xos/tools/cleanup_unique.py
+++ b/xos/tools/cleanup_unique.py
@@ -8,38 +8,94 @@
from cord.models import *
django.setup()
+for obj in ControllerNetwork.deleted_objects.all():
+ print "Purging deleted object", obj
+ obj.delete(purge=True)
+for obj in ControllerSite.deleted_objects.all():
+ print "Purging deleted object", obj
+ obj.delete(purge=True)
+
+for obj in ControllerSlice.deleted_objects.all():
+ print "Purging deleted object", obj
+ obj.delete(purge=True)
+
+for obj in NetworkSlice.deleted_objects.all():
+ print "Purging deleted object", obj
+ obj.delete(purge=True)
+
+for obj in NetworkSliver.deleted_objects.all():
+ print "Purging deleted object", obj
+ obj.delete(purge=True)
+
+for obj in DeploymentPrivilege.deleted_objects.all():
+ print "Purging deleted object", obj
+ obj.delete(purge=True)
+
+for obj in SiteDeployment.deleted_objects.all():
+ print "Purging deleted object", obj
+ obj.delete(purge=True)
+
+seen=[]
for obj in ControllerNetwork.objects.all():
+ seen.append(obj.id)
conflicts = ControllerNetwork.objects.filter(network=obj.network, controller=obj.controller)
for conflict in conflicts:
- if conflict.id != obj.id:
- print "Purging", conflict
+ if conflict.id not in seen:
+ print "Purging", conflict, conflict.id, "due to duplicate of", obj.id
conflict.delete(purge=True)
+seen=[]
for obj in NetworkSlice.objects.all():
+ seen.append(obj.id)
conflicts = NetworkSlice.objects.filter(network=obj.network, slice=obj.slice)
for conflict in conflicts:
- if conflict.id != obj.id:
- print "Purging", conflict
+ if conflict.id not in seen:
+ print "Purging", conflict, conflict.id, "due to duplicate of", obj.id
conflict.delete(purge=True)
+seen=[]
for obj in NetworkSliver.objects.all():
+ seen.append(obj.id)
conflicts = NetworkSliver.objects.filter(network=obj.network, sliver=obj.sliver)
for conflict in conflicts:
- if conflict.id != obj.id:
- print "Purging", conflict
+ if conflict.id not in seen:
+ print "Purging", conflict, conflict.id, "due to duplicate of", obj.id
conflict.delete(purge=True)
+seen=[]
for obj in DeploymentPrivilege.objects.all():
+ seen.append(obj.id)
conflicts = DeploymentPrivilege.objects.filter(user=obj.user, deployment=obj.deployment, role=obj.role)
for conflict in conflicts:
- if conflict.id != obj.id:
- print "Purging", conflict
+ if conflict.id not in seen:
+ print "Purging", conflict, conflict.id, "due to duplicate of", obj.id
conflict.delete(purge=True)
+seen=[]
for obj in SiteDeployment.objects.all():
+ seen.append(obj.id)
conflicts = SiteDeployment.objects.filter(site=obj.site, deployment=obj.deployment, controller=obj.controller)
for conflict in conflicts:
- if conflict.id != obj.id:
- print "Purging", conflict
+ if conflict.id not in seen:
+ print "Purging", conflict, conflict.id, "due to duplicate of", obj.id
conflict.delete(purge=True)
+
+seen=[]
+for obj in ControllerSite.objects.all():
+ seen.append(obj.id)
+ conflicts = ControllerSite.objects.filter(site=obj.site, controller=obj.controller)
+ for conflict in conflicts:
+ if conflict.id not in seen:
+ print "Purging", conflict, conflict.id, "due to duplicate of", obj.id
+ conflict.delete(purge=True)
+
+seen=[]
+for obj in ControllerSlice.objects.all():
+ seen.append(obj.id)
+ conflicts = ControllerSlice.objects.filter(slice=obj.slice, controller=obj.controller)
+ for conflict in conflicts:
+ if conflict.id not in seen:
+ print "Purging", conflict, conflict.id, "due to duplicate of", obj.id
+ conflict.delete(purge=True)
+