diff --git a/upgrade-container.sh b/upgrade-container.sh
new file mode 100755
index 0000000..09c639d
--- /dev/null
+++ b/upgrade-container.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+TMPDIR="/tmp/initdata"
+XOSDIR="/home/ubuntu/xos"
+
+mkdir -p $TMPDIR
+rm -f $TMPDIR/*.json
+
+XOS=$( docker ps|grep "xos:latest"|awk '{print $NF}' )
+docker exec $XOS /opt/xos/scripts/opencloud dumpdata
+docker cp $XOS:/opt/xos_backups/dumpdata-latest.json $TMPDIR
+docker cp $XOS:/opt/xos/xos_config $TMPDIR
+cp $TMPDIR/*.json $XOSDIR/xos/core/fixtures/initial_data.json
+cp $TMPDIR/xos_config $XOSDIR/xos/
+
+git pull
+
+if [[ $? != 0 ]]; then
+    echo "git pull" failed
+    exit
+fi
+
+docker build -t xos .
+
+docker stop $XOS
+docker run -p 8000:8000 xos 
diff --git a/xos/cord/models.py b/xos/cord/models.py
index 346fcc9..17dd816 100644
--- a/xos/cord/models.py
+++ b/xos/cord/models.py
@@ -1,5 +1,5 @@
 from django.db import models
-from core.models import Service, PlCoreBase, Slice, Sliver, Tenant, Node, Image, User
+from core.models import Service, PlCoreBase, Slice, Sliver, Tenant, Node, Image, User, Flavor
 from core.models.plcorebase import StrippedCharField
 import os
 from django.db import models
@@ -203,6 +203,7 @@
                           "firewall_rules": "accept all anywhere anywhere",
                           "url_filter_enable": False,
                           "url_filter_rules": "allow all",
+                          "url_filter_level": "PG",
                           "cdn_enable": False,
                           "sliver_id": None}
 
@@ -316,6 +317,14 @@
         self.set_attribute("url_filter_enable", value)
 
     @property
+    def url_filter_level(self):
+        return self.get_attribute("url_filter_level", self.default_attributes["url_filter_level"])
+
+    @url_filter_level.setter
+    def url_filter_level(self, value):
+        self.set_attribute("url_filter_level", value)
+
+    @property
     def url_filter_rules(self):
         return self.get_attribute("url_filter_rules", self.default_attributes["url_filter_rules"])
 
@@ -331,6 +340,27 @@
     def cdn_enable(self, value):
         self.set_attribute("cdn_enable", value)
 
+    @property
+    def users(self):
+        return [ {"name": "mom", "id": 1, "role": "admin"},
+                 {"name": "dad", "id": 2, "role": "admin"},
+                 {"name": "kid1", "id": 3, "role": "user"},
+                 {"name": "kid2", "id": 4, "role": "user"} ]
+
+    @users.setter
+    def users(self, value):
+        pass
+
+    @property
+    def services(self):
+        return [ {"cdn": self.cdn_enable,
+                  "url_filter": self.url_filter_enable,
+                  "firewall": self.firewall_enable} ]
+
+    @services.setter
+    def services(self, value):
+        pass
+
     def pick_node(self):
         nodes = list(Node.objects.all())
         # TODO: logic to filter nodes by which nodes are up, and which
@@ -352,12 +382,17 @@
             if not self.provider_service.slices.count():
                 raise XOSConfigurationError("The VCPE service has no slices")
 
+            flavors = Flavor.objects.filter(name="m1.small")
+            if not flavors:
+                raise XOSConfigurationError("No m1.small flavor")
+
             node =self.pick_node()
             sliver = Sliver(slice = self.provider_service.slices.all()[0],
                             node = node,
                             image = self.image,
                             creator = self.creator,
-                            deployment = node.site_deployment.deployment)
+                            deployment = node.site_deployment.deployment,
+                            flavor = flavors[0])
             sliver.save()
 
             try:
diff --git a/xos/core/models/sliver.py b/xos/core/models/sliver.py
index ff1e9b4..6c102d5 100644
--- a/xos/core/models/sliver.py
+++ b/xos/core/models/sliver.py
@@ -185,3 +185,11 @@
             pubkeys.add(self.slice.service.public_key)
 
         return pubkeys
+
+def controller_setter(instance, **kwargs):
+    try:
+        instance.controller = instance.node.site_deployment.controller
+    except:
+        instance.controller = None
+
+models.signals.post_init.connect(controller_setter, Sliver)
diff --git a/xos/core/xoslib/methods/__init__.py b/xos/core/xoslib/methods/__init__.py
index 0b891eb..c777668 100644
--- a/xos/core/xoslib/methods/__init__.py
+++ b/xos/core/xoslib/methods/__init__.py
@@ -1,5 +1,6 @@
 from django.views.generic import View
 from django.conf.urls import patterns, url
+from rest_framework.routers import DefaultRouter
 import os, sys
 import inspect
 import importlib
@@ -40,6 +41,19 @@
            urlpatterns.append(url(r'^' + view_url[1] + '/$',  view_url[3].as_view(), name=view_url[1]+'list'))
         elif view_url[0] == "detail":
            urlpatterns.append(url(r'^' + view_url[1] + '/(?P<pk>[a-zA-Z0-9\-]+)/$',  view_url[3].as_view(), name=view_url[1]+'detail'))
+        elif view_url[0] == "viewset":
+           viewset = view_url[3]
+
+           urlpatterns.extend(viewset.get_urlpatterns())
+
+           #urlpatterns.append(url(r'^' + view_url[1] + '/$', viewset.as_view({'get': 'list'}), name=view_url[1]+'list'))
+           #urlpatterns.append(url(r'^' + view_url[1] + '/(?P<pk>[a-zA-Z0-9\-]+)/$', viewset.as_view({'get': 'retrieve', 'put': 'update', 'post': 'create', 'delete': 'destroy', 'patch': 'partial_update'}), name=view_url[1]+'detail'))
+           #urlpatterns.extend(
+
+           #router = DefaultRouter()
+           #router.register(r'^' + view_url[1], view_url[3], base_name="foo")
+           #urlpatterns.extend(router.urls)
+           #urlpatterns.append(url(r'^' + view_url[1], view_url[3]))
 
 finally:
     sys.path = sys_path_save
diff --git a/xos/core/xoslib/methods/cordsubscriber.py b/xos/core/xoslib/methods/cordsubscriber.py
index 08aa9d9..a6c9a63 100644
--- a/xos/core/xoslib/methods/cordsubscriber.py
+++ b/xos/core/xoslib/methods/cordsubscriber.py
@@ -3,12 +3,17 @@
 from rest_framework.reverse import reverse
 from rest_framework import serializers
 from rest_framework import generics
+from rest_framework import viewsets
+from rest_framework.decorators import detail_route, list_route
 from core.models import *
 from django.forms import widgets
+from django.conf.urls import patterns, url
 from cord.models import VOLTTenant
 from core.xoslib.objects.cordsubscriber import CordSubscriber
 from plus import PlusSerializerMixin
+from django.shortcuts import get_object_or_404
 from xos.apibase import XOSListCreateAPIView, XOSRetrieveUpdateDestroyAPIView, XOSPermissionDenied
+import json
 
 if hasattr(serializers, "ReadOnlyField"):
     # rest_framework 3.x
@@ -29,6 +34,7 @@
         firewall_rules = serializers.CharField()
         url_filter_enable = serializers.BooleanField()
         url_filter_rules = serializers.CharField()
+        url_filter_level = serializers.CharField(required=False)
         cdn_enable = serializers.BooleanField()
         sliver_name = ReadOnlyField()
         image_name = ReadOnlyField()
@@ -40,7 +46,7 @@
             model = CordSubscriber
             fields = ('humanReadableName', 'id',
                       'service_specific_id', 'vlan_id',
-                      'vcpe_id', 'sliver', 'sliver_name', 'image', 'image_name', 'firewall_enable', 'firewall_rules', 'url_filter_enable', 'url_filter_rules', 'cdn_enable', 'vbng_id', 'routeable_subnet',)
+                      'vcpe_id', 'sliver', 'sliver_name', 'image', 'image_name', 'firewall_enable', 'firewall_rules', 'url_filter_enable', 'url_filter_rules', 'url_filter_level', 'cdn_enable', 'vbng_id', 'routeable_subnet',)
 
 
         def getHumanReadableName(self, obj):
@@ -60,4 +66,83 @@
     method_kind = "detail"
     method_name = "cordsubscriber"
 
+# this may be moved into plus.py...
+
+class XOSViewSet(viewsets.ModelViewSet):
+    @classmethod
+    def detail_url(self, pattern, viewdict, name):
+        return url(r'^' + self.method_name + r'/(?P<pk>[a-zA-Z0-9\-]+)/' + pattern,
+                   self.as_view(viewdict),
+                   name=self.base_name+"_"+name)
+
+    @classmethod
+    def get_urlpatterns(self):
+        patterns = []
+
+        patterns.append(url(r'^' + self.method_name + '/$', self.as_view({'get': 'list'}), name=self.base_name+'_list'))
+        patterns.append(url(r'^' + self.method_name + '/(?P<pk>[a-zA-Z0-9\-]+)/$', self.as_view({'get': 'retrieve', 'put': 'update', 'post': 'update', 'delete': 'destroy', 'patch': 'partial_update'}), name=self.base_name+'_detail'))
+
+        return patterns
+
+# the "new" API with many more REST endpoints.
+
+class CordSubscriberViewSet(XOSViewSet):
+    base_name = "subscriber"
+    method_name = "rs/subscriber"
+    method_kind = "viewset"
+    queryset = CordSubscriber.get_tenant_objects().select_related().all()
+    serializer_class = CordSubscriberIdSerializer
+
+    @classmethod
+    def get_urlpatterns(self):
+        patterns = super(CordSubscriberViewSet, self).get_urlpatterns()
+        patterns.append( self.detail_url("url_filtering/$", {"get": "get_url_filtering"}, "url_filtering") )
+        patterns.append( self.detail_url("url_filtering/(?P<level>[a-zA-Z0-9\-]+)/$", {"get": "set_url_filtering"}, "url_filtering") )
+        patterns.append( self.detail_url("users/$", {"get": "get_users"}, "users") )
+        patterns.append( self.detail_url("services/$", {"get": "get_services"}, "services") )
+        patterns.append( self.detail_url("services/(?P<service>[a-zA-Z0-9\-]+)/$", {"get": "get_service"}, "get_service") )
+        patterns.append( self.detail_url("services/(?P<service>[a-zA-Z0-9\-]+)/true/$", {"get": "enable_service"}, "enable_service") )
+        patterns.append( self.detail_url("services/(?P<service>[a-zA-Z0-9\-]+)/false/$", {"get": "disable_service"}, "disable_service") )
+
+        return patterns
+
+    def get_url_filtering(self, request, pk=None):
+        subscriber = self.get_object()
+        return Response(subscriber.url_filter_level)
+
+    def set_url_filtering(self, request, pk=None, level=None):
+        subscriber = self.get_object()
+        subscriber.url_filter_level = level
+        subscriber.save()
+        return Response(subscriber.url_filter_level)
+
+    def get_users(self, request, pk=None):
+        subscriber = self.get_object()
+        return Response(subscriber.users)
+
+    def get_services(self, request, pk=None):
+        subscriber = self.get_object()
+        return Response(subscriber.services)
+
+    def get_service(self, request, pk=None, service=None):
+        service_attr = service+"_enable"
+        subscriber = self.get_object()
+        return Response(getattr(subscriber, service_attr))
+
+    def enable_service(self, request, pk=None, service=None):
+        service_attr = service+"_enable"
+        subscriber = self.get_object()
+        setattr(subscriber, service_attr, True)
+        subscriber.save()
+        return Response(getattr(subscriber, service_attr))
+
+    def disable_service(self, request, pk=None, service=None):
+        service_attr = service+"_enable"
+        subscriber = self.get_object()
+        setattr(subscriber, service_attr, False)
+        subscriber.save()
+        return Response(getattr(subscriber, service_attr))
+
+
+
 
diff --git a/xos/core/xoslib/objects/cordsubscriber.py b/xos/core/xoslib/objects/cordsubscriber.py
index f4cd8e6..72a9984 100644
--- a/xos/core/xoslib/objects/cordsubscriber.py
+++ b/xos/core/xoslib/objects/cordsubscriber.py
@@ -29,6 +29,9 @@
                      ("firewall_rules", "vcpe.firewall_rules"),
                      ("url_filter_enable", "vcpe.url_filter_enable"),
                      ("url_filter_rules", "vcpe.url_filter_rules"),
+                     ("url_filter_level", "vcpe.url_filter_level"),
+                     ("users", "vcpe.users"),
+                     ("services", "vcpe.services"),
                      ("cdn_enable", "vcpe.cdn_enable"),
                      ("image", "vcpe.image.id"),
                      ("image_name", "vcpe.image.name"),
diff --git a/xos/hpc/models.py b/xos/hpc/models.py
index a3b7c90..1cd51ce 100644
--- a/xos/hpc/models.py
+++ b/xos/hpc/models.py
@@ -35,7 +35,7 @@
         scale = getattr(self, "set_scale", None)
         if scale is not None:
             exclude_slices = [x for x in self.slices.all() if "cmi" in x.name]
-            self.adjust_scale(slice_hint="hpc", scale=scale, exclusive_slices = exclude_slices)
+            self.adjust_scale(slice_hint="hpc", scale=scale, exclusive_slices = exclude_slices, max_per_node=1)
 
 class ServiceProvider(PlCoreBase):
     class Meta:
diff --git a/xos/hpc_observer/run.sh b/xos/hpc_observer/run.sh
deleted file mode 100644
index abf620a..0000000
--- a/xos/hpc_observer/run.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-if [[ ! -e ./hpc-backend.py ]]; then
-    ln -s ../xos-observer.py hpc-backend.py
-fi
-
-export XOS_DIR=/opt/xos
-python hpc-backend.py  -C $XOS_DIR/hpc_observer/hpc_observer_config
diff --git a/xos/hpc_observer/start.sh b/xos/hpc_observer/start.sh
deleted file mode 100644
index 07a1663..0000000
--- a/xos/hpc_observer/start.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-if [[ ! -e ./hpc-backend.py ]]; then
-    ln -s ../xos-observer.py hpc-backend.py
-fi
-
-export XOS_DIR=/opt/xos
-nohup python hpc-backend.py  -C $XOS_DIR/hpc_observer/hpc_observer_config > /dev/null 2>&1 &
diff --git a/xos/hpc_observer/stop.sh b/xos/hpc_observer/stop.sh
deleted file mode 100644
index 632f7a6..0000000
--- a/xos/hpc_observer/stop.sh
+++ /dev/null
@@ -1 +0,0 @@
-pkill -9 -f hpc-backend.py
diff --git a/xos/model_policies/model_policy_Controller.py b/xos/model_policies/model_policy_Controller.py
index fa84be7..2db7a63 100644
--- a/xos/model_policies/model_policy_Controller.py
+++ b/xos/model_policies/model_policy_Controller.py
@@ -22,7 +22,7 @@
     slices = Slice.objects.all()
     for slice in slices:
         if slice not in ctrls_by_slice or \
-            controller not in ctrls_by_slice:
+            controller not in ctrls_by_slice[slice]:
             controller_slice = ControllerSlice(controller=controller, slice=slice)
             controller_slice.save()
     # relations for all users
diff --git a/xos/hpc_observer/Makefile b/xos/observers/hpc/Makefile
similarity index 100%
rename from xos/hpc_observer/Makefile
rename to xos/observers/hpc/Makefile
diff --git a/xos/hpc_observer/fsck.py b/xos/observers/hpc/fsck.py
similarity index 100%
rename from xos/hpc_observer/fsck.py
rename to xos/observers/hpc/fsck.py
diff --git a/xos/observers/hpc/hpc-observer.py b/xos/observers/hpc/hpc-observer.py
new file mode 100755
index 0000000..d6a71ff
--- /dev/null
+++ b/xos/observers/hpc/hpc-observer.py
@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+
+# This imports and runs ../../xos-observer.py
+
+import importlib
+import os
+import sys
+observer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),"../..")
+sys.path.append(observer_path)
+mod = importlib.import_module("xos-observer")
+mod.main()
diff --git a/xos/hpc_observer/hpc_observer_config b/xos/observers/hpc/hpc_observer_config
similarity index 74%
rename from xos/hpc_observer/hpc_observer_config
rename to xos/observers/hpc/hpc_observer_config
index 0e7e53a..16d9077 100644
--- a/xos/hpc_observer/hpc_observer_config
+++ b/xos/observers/hpc/hpc_observer_config
@@ -23,9 +23,9 @@
 
 [observer]
 name=hpc
-dependency_graph=/opt/xos/hpc_observer/model-deps
-steps_dir=/opt/xos/hpc_observer/steps
-deleters_dir=/opt/xos/hpc_observer/deleters
+dependency_graph=/opt/xos/observers/hpc/model-deps
+steps_dir=/opt/xos/observers/hpc/steps
+deleters_dir=/opt/xos/observers/hpc/deleters
 log_file=console
 #/var/log/hpc.log
 driver=None
diff --git a/xos/hpc_observer/hpc_watcher.py b/xos/observers/hpc/hpc_watcher.py
similarity index 100%
rename from xos/hpc_observer/hpc_watcher.py
rename to xos/observers/hpc/hpc_watcher.py
diff --git a/xos/hpc_observer/hpclib.py b/xos/observers/hpc/hpclib.py
similarity index 100%
rename from xos/hpc_observer/hpclib.py
rename to xos/observers/hpc/hpclib.py
diff --git a/xos/hpc_observer/model-deps b/xos/observers/hpc/model-deps
similarity index 100%
rename from xos/hpc_observer/model-deps
rename to xos/observers/hpc/model-deps
diff --git a/xos/observers/hpc/run.sh b/xos/observers/hpc/run.sh
new file mode 100755
index 0000000..f77d751
--- /dev/null
+++ b/xos/observers/hpc/run.sh
@@ -0,0 +1,6 @@
+#if [[ ! -e ./hpc-backend.py ]]; then
+#    ln -s ../xos-observer.py hpc-backend.py
+#fi
+
+export XOS_DIR=/opt/xos
+python hpc-observer.py  -C $XOS_DIR/observers/hpc/hpc_observer_config
diff --git a/xos/observers/hpc/start.sh b/xos/observers/hpc/start.sh
new file mode 100755
index 0000000..305c07f
--- /dev/null
+++ b/xos/observers/hpc/start.sh
@@ -0,0 +1,6 @@
+#if [[ ! -e ./hpc-backend.py ]]; then
+#    ln -s ../xos-observer.py hpc-backend.py
+#fi
+
+export XOS_DIR=/opt/xos
+nohup python hpc-observer.py  -C $XOS_DIR/observers/hpc/hpc_observer_config > /dev/null 2>&1 &
diff --git a/xos/hpc_observer/steps/garbage_collector.py b/xos/observers/hpc/steps/garbage_collector.py
similarity index 100%
rename from xos/hpc_observer/steps/garbage_collector.py
rename to xos/observers/hpc/steps/garbage_collector.py
diff --git a/xos/hpc_observer/steps/sync_cdnprefix.py b/xos/observers/hpc/steps/sync_cdnprefix.py
similarity index 100%
rename from xos/hpc_observer/steps/sync_cdnprefix.py
rename to xos/observers/hpc/steps/sync_cdnprefix.py
diff --git a/xos/hpc_observer/steps/sync_contentprovider.py b/xos/observers/hpc/steps/sync_contentprovider.py
similarity index 100%
rename from xos/hpc_observer/steps/sync_contentprovider.py
rename to xos/observers/hpc/steps/sync_contentprovider.py
diff --git a/xos/hpc_observer/steps/sync_hpcservices.py b/xos/observers/hpc/steps/sync_hpcservices.py
similarity index 100%
rename from xos/hpc_observer/steps/sync_hpcservices.py
rename to xos/observers/hpc/steps/sync_hpcservices.py
diff --git a/xos/hpc_observer/steps/sync_originserver.py b/xos/observers/hpc/steps/sync_originserver.py
similarity index 100%
rename from xos/hpc_observer/steps/sync_originserver.py
rename to xos/observers/hpc/steps/sync_originserver.py
diff --git a/xos/hpc_observer/steps/sync_serviceprovider.py b/xos/observers/hpc/steps/sync_serviceprovider.py
similarity index 100%
rename from xos/hpc_observer/steps/sync_serviceprovider.py
rename to xos/observers/hpc/steps/sync_serviceprovider.py
diff --git a/xos/hpc_observer/steps/sync_sitemap.py b/xos/observers/hpc/steps/sync_sitemap.py
similarity index 100%
rename from xos/hpc_observer/steps/sync_sitemap.py
rename to xos/observers/hpc/steps/sync_sitemap.py
diff --git a/xos/observers/hpc/stop.sh b/xos/observers/hpc/stop.sh
new file mode 100755
index 0000000..a0b4a8e
--- /dev/null
+++ b/xos/observers/hpc/stop.sh
@@ -0,0 +1 @@
+pkill -9 -f hpc-observer.py
diff --git a/xos/observers/hpc/supervisor/hpc-observer.conf b/xos/observers/hpc/supervisor/hpc-observer.conf
new file mode 100644
index 0000000..f2c79d4
--- /dev/null
+++ b/xos/observers/hpc/supervisor/hpc-observer.conf
@@ -0,0 +1,2 @@
+[program:hpc-observer]
+command=python /opt/xos/observers/hpc/hpc-observer.py -C /opt/xos/observers/hpc/hpc_observer_config
diff --git a/xos/observers/hpc/supervisor/hpc-watcher.conf b/xos/observers/hpc/supervisor/hpc-watcher.conf
new file mode 100644
index 0000000..e0f4eb1
--- /dev/null
+++ b/xos/observers/hpc/supervisor/hpc-watcher.conf
@@ -0,0 +1,2 @@
+[program:hpc-watcher]
+command=python /opt/xos/observers/hpc/hpc_watcher.py
diff --git a/xos/observers/vbng/run.sh b/xos/observers/vbng/run.sh
index 7829ac0..efb586f 100755
--- a/xos/observers/vbng/run.sh
+++ b/xos/observers/vbng/run.sh
@@ -1,6 +1,6 @@
-if [[ ! -e ./vbng-observer.py ]]; then
-    ln -s ../../xos-observer.py vbng-observer.py
-fi
+#if [[ ! -e ./vbng-observer.py ]]; then
+#    ln -s ../../xos-observer.py vbng-observer.py
+#fi
 
 export XOS_DIR=/opt/xos
 python vbng-observer.py  -C $XOS_DIR/observers/vbng/vbng_observer_config
diff --git a/xos/observers/vbng/start.sh b/xos/observers/vbng/start.sh
index 5ceff27..98008f4 100755
--- a/xos/observers/vbng/start.sh
+++ b/xos/observers/vbng/start.sh
@@ -1,6 +1,6 @@
-if [[ ! -e ./vbng-observer.py ]]; then
-    ln -s ../../xos-observer.py vbng-observer.py
-fi
+#if [[ ! -e ./vbng-observer.py ]]; then
+#    ln -s ../../xos-observer.py vbng-observer.py
+#fi
 
 export XOS_DIR=/opt/xos
 nohup python vbng-observer.py  -C $XOS_DIR/observers/vbng/vbng_observer_config > /dev/null 2>&1 &
diff --git a/xos/observers/vbng/supervisor/vbng-observer.conf b/xos/observers/vbng/supervisor/vbng-observer.conf
new file mode 100644
index 0000000..cff77b8
--- /dev/null
+++ b/xos/observers/vbng/supervisor/vbng-observer.conf
@@ -0,0 +1,2 @@
+[program:vbng-observer]
+command=python /opt/xos/observers/vbng/vbng-observer.py -C /opt/xos/observers/vbng/vbng_observer_config
diff --git a/xos/observers/vbng/vbng-observer.py b/xos/observers/vbng/vbng-observer.py
new file mode 100755
index 0000000..d6a71ff
--- /dev/null
+++ b/xos/observers/vbng/vbng-observer.py
@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+
+# This imports and runs ../../xos-observer.py
+
+import importlib
+import os
+import sys
+observer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),"../..")
+sys.path.append(observer_path)
+mod = importlib.import_module("xos-observer")
+mod.main()
diff --git a/xos/observers/vbng/vbng_observer_config b/xos/observers/vbng/vbng_observer_config
index 217ce71..9094bb1 100644
--- a/xos/observers/vbng/vbng_observer_config
+++ b/xos/observers/vbng/vbng_observer_config
@@ -25,7 +25,7 @@
 name=vbng
 dependency_graph=/opt/xos/observers/vbng/model-deps
 steps_dir=/opt/xos/observers/vbng/steps
-sys_dir=/opt/xos/observer/vbng/sys
+sys_dir=/opt/xos/observers/vbng/sys
 deleters_dir=/opt/xos/observers/vbng/deleters
 log_file=console
 #/var/log/hpc.log
diff --git a/xos/observers/vcpe/files/docker.list b/xos/observers/vcpe/files/docker.list
new file mode 100644
index 0000000..0ee9ae0
--- /dev/null
+++ b/xos/observers/vcpe/files/docker.list
@@ -0,0 +1 @@
+deb https://get.docker.com/ubuntu docker main
diff --git a/xos/observers/vcpe/run.sh b/xos/observers/vcpe/run.sh
index c582842..f180e66 100755
--- a/xos/observers/vcpe/run.sh
+++ b/xos/observers/vcpe/run.sh
@@ -1,6 +1,6 @@
-if [[ ! -e ./vcpe-observer.py ]]; then
-    ln -s ../../xos-observer.py vcpe-observer.py
-fi
+#if [[ ! -e ./vcpe-observer.py ]]; then
+#    ln -s ../../xos-observer.py vcpe-observer.py
+#fi
 
 export XOS_DIR=/opt/xos
 python vcpe-observer.py  -C $XOS_DIR/observers/vcpe/vcpe_observer_config
diff --git a/xos/observers/vcpe/start.sh b/xos/observers/vcpe/start.sh
index 96cb6e4..b402e5d 100755
--- a/xos/observers/vcpe/start.sh
+++ b/xos/observers/vcpe/start.sh
@@ -1,6 +1,6 @@
-if [[ ! -e ./vcpe-observer.py ]]; then
-    ln -s ../../xos-observer.py vcpe-observer.py
-fi
+#if [[ ! -e ./vcpe-observer.py ]]; then
+#    ln -s ../../xos-observer.py vcpe-observer.py
+#fi
 
 export XOS_DIR=/opt/xos
 nohup python vcpe-observer.py  -C $XOS_DIR/observers/vcpe/vcpe_observer_config > /dev/null 2>&1 &
diff --git a/xos/observers/vcpe/steps/sync_vcpetenant.py b/xos/observers/vcpe/steps/sync_vcpetenant.py
index 3fc9310..8cda4ef 100644
--- a/xos/observers/vcpe/steps/sync_vcpetenant.py
+++ b/xos/observers/vcpe/steps/sync_vcpetenant.py
@@ -97,6 +97,7 @@
                    "hostname": sliver.node.name,
                    "instance_id": sliver.instance_id,
                    "private_key": service_key,
+                   "ansible_tag": "vcpe_tenant_" + str(o.id)
                  }
 
         if hasattr(o, "sync_attributes"):
diff --git a/xos/observers/vcpe/steps/sync_vcpetenant.yaml b/xos/observers/vcpe/steps/sync_vcpetenant.yaml
index 6c2bad4..92da2cc 100644
--- a/xos/observers/vcpe/steps/sync_vcpetenant.yaml
+++ b/xos/observers/vcpe/steps/sync_vcpetenant.yaml
@@ -22,14 +22,71 @@
         {% endfor %}
 
   tasks:
-    - name: make sure /etc/dnsmasq.d exists
-      file: path=/etc/dnsmasq.d state=directory owner=root group=root
+  - name: Docker repository
+    copy: src=/opt/xos/observers/vcpe/files/docker.list
+      dest=/etc/apt/sources.list.d/docker.list
+
+  - name: Import the repository key
+    apt_key: keyserver=keyserver.ubuntu.com id=36A1D7869245C8950F966E92D8576A8BA88D21E9
+
+  - name: install Docker
+    apt: name=lxc-docker-1.5.0 state=present update_cache=yes
+
+  - name: install python-setuptools
+    apt: name=python-setuptools state=present
+
+  - name: install pip
+    easy_install: name=pip
+
+  - name: install docker-py
+    pip: name=docker-py version=0.5.3
+
+  - name: install Pipework
+    get_url: url=https://raw.githubusercontent.com/jpetazzo/pipework/master/pipework
+       dest=/usr/local/bin/pipework
+       mode=0755
+
+  - name: bring up LAN interface
+    shell: ifconfig eth2 up
+
+  - name: bring up WAN interface
+    shell: ifconfig eth3 up
+
+  - name: make sure /etc/dnsmasq.d exists
+    file: path=/etc/dnsmasq.d state=directory owner=root group=root
    
-    - name: configure dnsmasq servers
-      template: src=/opt/xos/observers/vcpe/templates/dnsmasq_servers.j2 dest=/etc/dnsmasq.d/servers.conf owner=root group=root
+  - name: dnsmasq config
+    template: src=/opt/xos/observers/vcpe/templates/dnsmasq_servers.j2 dest=/etc/dnsmasq.d/servers.conf owner=root group=root
 
-    - name: setup networking
-      template: src=/opt/xos/observers/vcpe/templates/vlan_sample.j2 dest=/etc/vlan_sample owner=root group=root
+  - name: networking info
+    template: src=/opt/xos/observers/vcpe/templates/vlan_sample.j2 dest=/etc/vlan_sample owner=root group=root
 
-    - name: setup firewall
-      template: src=/opt/xos/observers/vcpe/templates/firewall_sample.j2 dest=/etc/firewall_sample owner=root group=root
+  - name: firewall info
+    template: src=/opt/xos/observers/vcpe/templates/firewall_sample.j2 dest=/etc/firewall_sample owner=root group=root
+
+  - name: Make sure iptables module loaded
+    shell: "iptables -L > /dev/null"
+
+  - name: Make sure ip6tables module loaded
+    shell: "ip6tables -L > /dev/null"
+
+  - name: Disable resolvconf updates (to avoid overwriting /etc/resolv.conf on host)
+    shell: service resolvconf disable-updates
+
+  - name: spin up container
+    docker: name=vcpe
+      image=andybavier/docker-vcpe
+      net=none
+      privileged=yes
+
+  # This needs attention once ONOS is integrated
+  - name: Connect container to WAN network
+    shell: docker exec vcpe ifconfig eth0 >> /dev/null || pipework eth3 -i eth0 vcpe 10.1.2.3/24
+
+  # Only uses the first vlan_id
+  - name: Connect container to LAN network, VLAN {{ vlan_ids[0] }}
+    shell: docker exec vcpe ifconfig eth1 >> /dev/null || pipework eth2 -i eth1 vcpe 192.168.0.1/24 @{{ vlan_ids[0] }}
+
+  - name: Start container services
+    shell: docker exec vcpe service dnsmasq start
+
diff --git a/xos/observers/vcpe/supervisor/vcpe-observer.conf b/xos/observers/vcpe/supervisor/vcpe-observer.conf
new file mode 100644
index 0000000..27d2796
--- /dev/null
+++ b/xos/observers/vcpe/supervisor/vcpe-observer.conf
@@ -0,0 +1,2 @@
+[program:vcpe-observer]
+command=python /opt/xos/observers/vcpe/vcpe-observer.py -C /opt/xos/observers/vcpe/vcpe_observer_config
diff --git a/xos/observers/vcpe/templates/dnsmasq_servers.j2 b/xos/observers/vcpe/templates/dnsmasq_servers.j2
index 359070a..76b8f1c 100644
--- a/xos/observers/vcpe/templates/dnsmasq_servers.j2
+++ b/xos/observers/vcpe/templates/dnsmasq_servers.j2
@@ -9,8 +9,13 @@
 {% endif %}
 
 {% if url_filter_enable %}
-# placeholder; figure out what to really use...
-server=dns.xerocole.com
+# Point to BroadbandShield service, disable cache, and enable eDNS MAC passing
+add-mac
+cache-size=0
+server=198.105.255.10
+server=198.105.255.11
+server=198.105.255.12
+server=198.105.255.13
 {% else %}
 # use google's DNS service
 server=8.8.8.8
diff --git a/xos/observers/vcpe/vcpe-observer.py b/xos/observers/vcpe/vcpe-observer.py
new file mode 100755
index 0000000..d6a71ff
--- /dev/null
+++ b/xos/observers/vcpe/vcpe-observer.py
@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+
+# This imports and runs ../../xos-observer.py
+
+import importlib
+import os
+import sys
+observer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),"../..")
+sys.path.append(observer_path)
+mod = importlib.import_module("xos-observer")
+mod.main()
diff --git a/xos/observers/vcpe/vcpe_observer_config b/xos/observers/vcpe/vcpe_observer_config
index c6500ae..6d58340 100644
--- a/xos/observers/vcpe/vcpe_observer_config
+++ b/xos/observers/vcpe/vcpe_observer_config
@@ -25,12 +25,14 @@
 name=vcpe
 dependency_graph=/opt/xos/observers/vcpe/model-deps
 steps_dir=/opt/xos/observers/vcpe/steps
-sys_dir=/opt/xos/observer/vcpe/sys
+sys_dir=/opt/xos/observers/vcpe/sys
 deleters_dir=/opt/xos/observers/vcpe/deleters
 log_file=console
 #/var/log/hpc.log
 driver=None
 pretend=False
+backoff_disabled=True
+save_ansible_output=True
 
 [feefie]
 client_id='vicci_dev_central'
diff --git a/xos/openstack_observer/ansible.py b/xos/openstack_observer/ansible.py
index 17504b3..5b6bd1d 100644
--- a/xos/openstack_observer/ansible.py
+++ b/xos/openstack_observer/ansible.py
@@ -43,24 +43,44 @@
 
     return results
 
+def parse_unreachable(msg):
+    total_unreachable=0
+    for l in msg.splitlines():
+        x = re.findall('ok=([0-9]+).*changed=([0-9]+).*unreachable=([0-9]+).*failed=([0-9]+)', l)
+        if x:
+            (ok, changed, unreachable, failed) = x[0]
+            ok=int(ok)
+            changed=int(changed)
+            unreachable=int(unreachable)
+            failed=int(failed)
+
+            total_unreachable += unreachable
+    return total_unreachable
+
+
 def id_generator(size=6, chars=string.ascii_uppercase + string.digits):
     return ''.join(random.choice(chars) for _ in range(size))
 
 def shellquote(s):
     return "'" + s.replace("'", "'\\''") + "'"
 
-def run_template(name, opts,path='', expected_num=None, ansible_config=None, ansible_hosts=None, run_ansible_script=None):
+def get_playbook_fn(opts, path):
+    if not opts.get("ansible_tag", None):
+        # if no ansible_tag is in the options, then generate a unique one
+        objname= id_generator()
+        opts = opts.copy()
+        opts["ansible_tag"] = objname
+
+    objname = opts["ansible_tag"]
+
+    os.system('mkdir -p %s' % os.path.join(sys_dir, path))
+    return (opts, os.path.join(sys_dir,path,objname))
+
+def run_template(name, opts, path='', expected_num=None, ansible_config=None, ansible_hosts=None, run_ansible_script=None):
     template = os_template_env.get_template(name)
     buffer = template.render(opts)
 
-    try:
-        objname = opts['ansible_tag']
-    except:
-        objname= id_generator()
-
-    os.system('mkdir -p %s'%'/'.join([sys_dir,path]))
-    fqp = '/'.join([sys_dir,path,objname])
-
+    (opts, fqp) = get_playbook_fn(opts, path)
 
     f = open(fqp,'w')
     f.write(buffer)
@@ -78,11 +98,17 @@
         if not run_ansible_script:
             run_ansible_script = os.path.join(XOS_DIR, "observer/run_ansible")
 
-        #run = os.popen(XOS_DIR + '/observer/run_ansible %s'%shellquote(fqp), env=env)
-        run = subprocess.Popen("%s %s" % (run_ansible_script, shellquote(fqp)), shell=True, stdout=subprocess.PIPE, env=env).stdout
-        msg = run.read()
-        status = run.close()
+        process = subprocess.Popen("%s %s" % (run_ansible_script, shellquote(fqp)), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
+        msg = process.stdout.read()
+        err_msg = process.stderr.read()
 
+        if getattr(Config(), "observer_save_ansible_output", False):
+            try:
+                open(fqp+".out","w").write(msg)
+                open(fqp+".err","w").write(err_msg)
+            except:
+                # fail silently
+                pass
         
     else:
         msg = open(fqp+'.out').read()
@@ -91,6 +117,10 @@
         ok_results = parse_output(msg)
         if (expected_num is not None) and (len(ok_results) != expected_num):
             raise ValueError('Unexpected num %s!=%d' % (str(expected_num), len(ok_results)) )
+
+        total_unreachable = parse_unreachable(msg)
+        if (total_unreachable > 0):
+            raise ValueError("Unreachable results in ansible recipe")
     except ValueError,e:
         all_fatal = [e.message] + re.findall(r'^msg: (.*)',msg,re.MULTILINE)
         all_fatal2 = re.findall(r'^ERROR: (.*)',msg,re.MULTILINE)
@@ -110,35 +140,35 @@
     hostname = opts["hostname"]
     private_key = opts["private_key"]
 
-    (private_key_handle, private_key_pathname) = tempfile.mkstemp()
-    (config_handle, config_pathname) = tempfile.mkstemp()
-    (hosts_handle, hosts_pathname) = tempfile.mkstemp()
+    (opts, fqp) = get_playbook_fn(opts, path)
+    private_key_pathname = fqp + ".key"
+    config_pathname = fqp + ".config"
+    hosts_pathname = fqp + ".hosts"
 
-    try:
-        proxy_command = "ProxyCommand ssh -q -i %s -o StrictHostKeyChecking=no %s@%s" % (private_key_pathname, instance_id, hostname)
+    proxy_command = "ProxyCommand ssh -q -i %s -o StrictHostKeyChecking=no %s@%s" % (private_key_pathname, instance_id, hostname)
 
-        os.write(private_key_handle, private_key)
-        os.close(private_key_handle)
+    f = open(private_key_pathname, "w")
+    f.write(private_key)
+    f.close()
 
-        os.write(config_handle, "[ssh_connection]\n")
-        os.write(config_handle, 'ssh_args = -o "%s" -o StrictHostKeyChecking=no\n' % proxy_command)
-        os.write(config_handle, 'scp_if_ssh = True\n')
-        os.close(config_handle)
+    f = open(config_pathname, "w")
+    f.write("[ssh_connection]\n")
+    f.write('ssh_args = -o "%s" -o StrictHostKeyChecking=no\n' % proxy_command)
+    f.write('scp_if_ssh = True\n')
+    f.close()
 
-        os.write(hosts_handle, "[%s]\n" % sliver_name)
-        os.write(hosts_handle, "%s ansible_ssh_private_key_file=%s\n" % (hostname, private_key_pathname))
-        os.close(hosts_handle)
+    f = open(hosts_pathname, "w")
+    f.write("[%s]\n" % sliver_name)
+    f.write("%s ansible_ssh_private_key_file=%s\n" % (hostname, private_key_pathname))
+    f.close()
 
-        print "ANSIBLE_CONFIG=%s" % config_pathname
-        print "ANSIBLE_HOSTS=%s" % hosts_pathname
+    # SSH will complain if private key is world or group readable
+    os.chmod(private_key_pathname, 0600)
 
-        return run_template(name, opts, path, expected_num, ansible_config = config_pathname, ansible_hosts = hosts_pathname, run_ansible_script="/opt/xos/observer/run_ansible_verbose")
+    print "ANSIBLE_CONFIG=%s" % config_pathname
+    print "ANSIBLE_HOSTS=%s" % hosts_pathname
 
-    finally:
-        #os.remove(private_key_pathname)
-        #os.remove(config_pathname)
-        #os.remove(hosts_pathname)
-        pass
+    return run_template(name, opts, path, expected_num, ansible_config = config_pathname, ansible_hosts = hosts_pathname, run_ansible_script="/opt/xos/observer/run_ansible_verbose")
 
 
 
diff --git a/xos/openstack_observer/steps/sync_controller_images.py b/xos/openstack_observer/steps/sync_controller_images.py
index 94b18a0..948fcea 100644
--- a/xos/openstack_observer/steps/sync_controller_images.py
+++ b/xos/openstack_observer/steps/sync_controller_images.py
@@ -4,6 +4,7 @@
 from django.db.models import F, Q
 from xos.config import Config
 from observer.openstacksyncstep import OpenStackSyncStep
+from observer.syncstep import *
 from core.models import Controller
 from core.models import Image, ControllerImages
 from util.logger import observer_logger as logger 
@@ -27,7 +28,7 @@
 
 	controller_register = json.loads(controller_image.controller.backend_register)
         if (controller_register.get('disabled',False)):
-                raise Exception('Controller %s is disabled'%controller_image.controller.name)
+                raise InnocuousException('Controller %s is disabled'%controller_image.controller.name)
 
         image_fields = {'endpoint':controller_image.controller.auth_url,
                         'admin_user':controller_image.controller.admin_user,
diff --git a/xos/openstack_observer/steps/sync_controller_networks.py b/xos/openstack_observer/steps/sync_controller_networks.py
index d327b7b..278d018 100644
--- a/xos/openstack_observer/steps/sync_controller_networks.py
+++ b/xos/openstack_observer/steps/sync_controller_networks.py
@@ -5,6 +5,7 @@
 from django.db.models import F, Q
 from xos.config import Config
 from observer.openstacksyncstep import OpenStackSyncStep
+from observer.syncstep import *
 from core.models.network import *
 from core.models.slice import *
 from core.models.sliver import Sliver
@@ -65,7 +66,7 @@
 
 	controller_register = json.loads(controller_network.controller.backend_register)
         if (controller_register.get('disabled',False)):
-                raise Exception('Controller %s is disabled'%controller_network.controller.name)
+                raise InnocuousException('Controller %s is disabled'%controller_network.controller.name)
 
         if not controller_network.controller.admin_user:
             logger.info("controller %r has no admin_user, skipping" % controller_network.controller)
@@ -78,7 +79,7 @@
     def delete_record(self, controller_network):
 	controller_register = json.loads(controller_network.controller.backend_register)
         if (controller_register.get('disabled',False)):
-                raise Exception('Controller %s is disabled'%controller_network.controller.name)
+                raise InnocuousException('Controller %s is disabled'%controller_network.controller.name)
 
 	try:
         	slice = controller_network.network.owner # XXX: FIXME!!
diff --git a/xos/openstack_observer/steps/sync_controller_site_privileges.py b/xos/openstack_observer/steps/sync_controller_site_privileges.py
index 6a13736..a2c40ef 100644
--- a/xos/openstack_observer/steps/sync_controller_site_privileges.py
+++ b/xos/openstack_observer/steps/sync_controller_site_privileges.py
@@ -4,6 +4,7 @@
 from django.db.models import F, Q
 from xos.config import Config
 from observer.openstacksyncstep import OpenStackSyncStep
+from observer.syncstep import *
 from core.models.site import Controller, SitePrivilege 
 from core.models.user import User
 from core.models.controlleruser import ControllerUser, ControllerSitePrivilege
@@ -28,7 +29,7 @@
 
 	controller_register = json.loads(controller_site_privilege.controller.backend_register)
         if (controller_register.get('disabled',False)):
-                raise Exception('Controller %s is disabled'%controller_site_privilege.controller.name)
+                raise InnocuousException('Controller %s is disabled'%controller_site_privilege.controller.name)
 
 
         if not controller_site_privilege.controller.admin_user:
@@ -76,7 +77,7 @@
     def delete_record(self, controller_site_privilege):
 	controller_register = json.loads(controller_site_privilege.controller.backend_register)
         if (controller_register.get('disabled',False)):
-                raise Exception('Controller %s is disabled'%controller_site_privilege.controller.name)
+                raise InnocuousException('Controller %s is disabled'%controller_site_privilege.controller.name)
 
         if controller_site_privilege.role_id:
             driver = self.driver.admin_driver(controller=controller_site_privilege.controller)
diff --git a/xos/openstack_observer/steps/sync_controller_sites.py b/xos/openstack_observer/steps/sync_controller_sites.py
index f101315..670f09c 100644
--- a/xos/openstack_observer/steps/sync_controller_sites.py
+++ b/xos/openstack_observer/steps/sync_controller_sites.py
@@ -4,6 +4,7 @@
 from xos.config import Config
 from openstack_observer.openstacksyncstep import OpenStackSyncStep
 from core.models.site import *
+from observer.syncstep import *
 from observer.ansible import *
 from util.logger import observer_logger as logger
 import json
@@ -20,7 +21,7 @@
     def sync_record(self, controller_site):
 	controller_register = json.loads(controller_site.controller.backend_register)
         if (controller_register.get('disabled',False)):
-                raise Exception('Controller %s is disabled'%controller_site.controller.name)
+                raise InnocuousException('Controller %s is disabled'%controller_site.controller.name)
 
 	template = os_template_env.get_template('sync_controller_sites.yaml')
 	tenant_fields = {'endpoint':controller_site.controller.auth_url,
@@ -41,7 +42,7 @@
     def delete_record(self, controller_site):
 	controller_register = json.loads(controller_site.controller.backend_register)
         if (controller_register.get('disabled',False)):
-                raise Exception('Controller %s is disabled'%controller_site.controller.name)
+                raise InnocuousException('Controller %s is disabled'%controller_site.controller.name)
 
 	if controller_site.tenant_id:
             driver = self.driver.admin_driver(controller=controller_site.controller)
diff --git a/xos/openstack_observer/steps/sync_controller_slice_privileges.py b/xos/openstack_observer/steps/sync_controller_slice_privileges.py
index 38f23c2..2e2e63c 100644
--- a/xos/openstack_observer/steps/sync_controller_slice_privileges.py
+++ b/xos/openstack_observer/steps/sync_controller_slice_privileges.py
@@ -4,6 +4,7 @@
 from django.db.models import F, Q
 from xos.config import Config
 from observer.openstacksyncstep import OpenStackSyncStep
+from observer.syncstep import *
 from core.models.slice import Controller, SlicePrivilege 
 from core.models.user import User
 from core.models.controlleruser import ControllerUser, ControllerSlicePrivilege
@@ -28,7 +29,7 @@
 
 	controller_register = json.loads(controller_slice_privilege.controller.backend_register)
         if (controller_register.get('disabled',False)):
-                raise Exception('Controller %s is disabled'%controller_slice_privilege.controller.name)
+                raise InnocuousException('Controller %s is disabled'%controller_slice_privilege.controller.name)
 
         if not controller_slice_privilege.controller.admin_user:
             logger.info("controller %r has no admin_user, skipping" % controller_slice_privilege.controller)
@@ -75,7 +76,7 @@
     def delete_record(self, controller_slice_privilege):
 	controller_register = json.loads(controller_slice_privilege.controller.backend_register)
         if (controller_register.get('disabled',False)):
-                raise Exception('Controller %s is disabled'%controller_slice_privilege.controller.name)
+                raise InnocuousException('Controller %s is disabled'%controller_slice_privilege.controller.name)
 
         if controller_slice_privilege.role_id:
             driver = self.driver.admin_driver(controller=controller_slice_privilege.controller)
diff --git a/xos/openstack_observer/steps/sync_controller_slices.py b/xos/openstack_observer/steps/sync_controller_slices.py
index 8d4a5e0..f64f9a8 100644
--- a/xos/openstack_observer/steps/sync_controller_slices.py
+++ b/xos/openstack_observer/steps/sync_controller_slices.py
@@ -5,6 +5,7 @@
 from django.db.models import F, Q
 from xos.config import Config
 from observer.openstacksyncstep import OpenStackSyncStep
+from observer.syncstep import *
 from core.models import *
 from observer.ansible import *
 from openstack.driver import OpenStackDriver
@@ -27,7 +28,7 @@
 
         controller_register = json.loads(controller_slice.controller.backend_register)
         if (controller_register.get('disabled',False)):
-            raise Exception('Controller %s is disabled'%controller_slice.controller.name)
+            raise InnocuousException('Controller %s is disabled'%controller_slice.controller.name)
 
         if not controller_slice.controller.admin_user:
             logger.info("controller %r has no admin_user, skipping" % controller_slice.controller)
@@ -72,7 +73,7 @@
     def delete_record(self, controller_slice):
         controller_register = json.loads(controller_slice.controller.backend_register)
         if (controller_register.get('disabled',False)):
-            raise Exception('Controller %s is disabled'%controller_slice.controller.name)
+            raise InnocuousException('Controller %s is disabled'%controller_slice.controller.name)
 
         controller_users = ControllerUser.objects.filter(user=controller_slice.slice.creator,
                                                               controller=controller_slice.controller)
diff --git a/xos/openstack_observer/steps/sync_controller_slices.yaml b/xos/openstack_observer/steps/sync_controller_slices.yaml
index 380f001..28c406d 100644
--- a/xos/openstack_observer/steps/sync_controller_slices.yaml
+++ b/xos/openstack_observer/steps/sync_controller_slices.yaml
@@ -2,9 +2,9 @@
 - hosts: 127.0.0.1
   connection: local
   tasks:
-  {% if delete %}
+  {% if delete -%}
   - keystone_user: endpoint={{ endpoint }} login_user={{ admin_user }} login_password={{ admin_password }} login_tenant_name={{ admin_tenant }} tenant={{ tenant }} tenant_description="{{ tenant_description }}" state=absent
-  {% else %}	
+  {% else -%}	
   - keystone_user: endpoint={{ endpoint }} login_user={{ admin_user }} login_password={{ admin_password }} login_tenant_name={{ admin_tenant }} tenant={{ tenant }} tenant_description="{{ tenant_description }}"
   {% for role in roles %}
   - keystone_user: endpoint={{ endpoint}} login_user={{ admin_user }} login_password={{ admin_password }} login_tenant_name={{ admin_tenant }} user="{{ name }}" role={{ role }} tenant={{ tenant }}
diff --git a/xos/openstack_observer/steps/sync_controller_users.py b/xos/openstack_observer/steps/sync_controller_users.py
index 47d1096..7979d82 100644
--- a/xos/openstack_observer/steps/sync_controller_users.py
+++ b/xos/openstack_observer/steps/sync_controller_users.py
@@ -4,6 +4,7 @@
 from django.db.models import F, Q
 from xos.config import Config
 from observer.openstacksyncstep import OpenStackSyncStep
+from observer.syncstep import *
 from core.models.site import Controller, SiteDeployment, SiteDeployment
 from core.models.user import User
 from core.models.controlleruser import ControllerUser
@@ -28,7 +29,7 @@
 
 	controller_register = json.loads(controller_user.controller.backend_register)
         if (controller_register.get('disabled',False)):
-                raise Exception('Controller %s is disabled'%controller_user.controller.name)
+                raise InnocuousException('Controller %s is disabled'%controller_user.controller.name)
 
         if not controller_user.controller.admin_user:
             logger.info("controller %r has no admin_user, skipping" % controller_user.controller)
@@ -79,7 +80,7 @@
     def delete_record(self, controller_user):
 	controller_register = json.loads(controller_user.controller.backend_register)
         if (controller_register.get('disabled',False)):
-                raise Exception('Controller %s is disabled'%controller_user.controller.name)
+                raise InnocuousException('Controller %s is disabled'%controller_user.controller.name)
 
         if controller_user.kuser_id:
             driver = self.driver.admin_driver(controller=controller_user.controller)
diff --git a/xos/openstack_observer/steps/sync_slivers.py b/xos/openstack_observer/steps/sync_slivers.py
index 9b5dd99..48f5c25 100644
--- a/xos/openstack_observer/steps/sync_slivers.py
+++ b/xos/openstack_observer/steps/sync_slivers.py
@@ -9,6 +9,7 @@
 from core.models.slice import Slice, SlicePrivilege, ControllerSlice
 from core.models.network import Network, NetworkSlice, ControllerNetwork
 from observer.ansible import *
+from observer.syncstep import *
 from util.logger import observer_logger as logger
 
 def escape(s):
@@ -32,7 +33,7 @@
         controller_register = json.loads(sliver.node.site_deployment.controller.backend_register)
 
         if (controller_register.get('disabled',False)):
-            raise Exception('Controller %s is disabled'%sliver.node.site_deployment.controller.name)
+            raise InnocuousException('Controller %s is disabled'%sliver.node.site_deployment.controller.name)
 
         metadata_update = {}
         if (sliver.numberCores):
@@ -82,20 +83,19 @@
                 if net['name']=='public':
                     nics.append(net['id'])
 
-        image_id = None
+        image_name = None
         controller_images = sliver.image.controllerimages.filter(controller=sliver.node.site_deployment.controller)
         if controller_images:
-            image_id = controller_images[0].glance_image_id
-            logger.info("using image_id from ControllerImage object: " + str(image_id))
+            image_name = controller_images[0].image.name
+            logger.info("using image from ControllerImage object: " + str(image_name))
 
-        if image_id is None:
+        if image_name is None:
             controller_driver = self.driver.admin_driver(controller=sliver.node.site_deployment.controller)
-            image_id = None
             images = controller_driver.shell.glanceclient.images.list()
             for image in images:
-                if image.name == sliver.image.name or not image_id:
-                    image_id = image.id
-                    logger.info("using image_id from glance: " + str(image_id))
+                if image.name == sliver.image.name or not image_name:
+                    image_name = image.name
+                    logger.info("using image from glance: " + str(image_name))
 
         try:
             legacy = Config().observer_legacy
@@ -124,8 +124,8 @@
                      'name':sliver_name,
                      'ansible_tag':sliver_name,
                      'availability_zone': availability_zone_filter,
-                     'image_id':image_id,
-                     'flavor_id':sliver.flavor.id,
+                     'image_name':image_name,
+                     'flavor_name':sliver.flavor.name,
                      'nics':nics,
                      'meta':metadata_update,
                      'user_data':r'%s'%escape(userData)}
@@ -150,7 +150,7 @@
         controller_register = json.loads(sliver.node.site_deployment.controller.backend_register)
 
         if (controller_register.get('disabled',False)):
-            raise Exception('Controller %s is disabled'%sliver.node.site_deployment.controller.name)
+            raise InnocuousException('Controller %s is disabled'%sliver.node.site_deployment.controller.name)
 
         sliver_name = '%s-%d'%(sliver.slice.name,sliver.id)
         controller = sliver.node.site_deployment.controller
diff --git a/xos/openstack_observer/steps/sync_slivers.yaml b/xos/openstack_observer/steps/sync_slivers.yaml
index 6ac995f..c543227 100644
--- a/xos/openstack_observer/steps/sync_slivers.yaml
+++ b/xos/openstack_observer/steps/sync_slivers.yaml
@@ -8,14 +8,14 @@
       login_password: {{ admin_password }}
       login_tenant_name: {{ admin_tenant }}
       name: {{ name }}
-      {% if delete %}
+      {% if delete -%}
       state: absent
-      {% else %}
+      {% else -%}
       state: present
       availability_zone: {{ availability_zone }}
-      image_id: {{ image_id }}
+      image_name: {{ image_name }}
       wait_for: 200
-      flavor_id: {{ flavor_id }}
+      flavor_name: {{ flavor_name }}
       user_data: "{{ user_data }}"
       nics:
       {% for net in nics %}
diff --git a/xos/openstack_observer/syncstep.py b/xos/openstack_observer/syncstep.py
index b752760..fb06bdd 100644
--- a/xos/openstack_observer/syncstep.py
+++ b/xos/openstack_observer/syncstep.py
@@ -21,20 +21,24 @@
     strs = backend_str.split(' // ')
     strs2 = f7(strs)
     return ' // '.join(strs2)
-    
+
 def deepgetattr(obj, attr):
     return reduce(getattr, attr.split('.'), obj)
 
+
+class InnocuousException(Exception):
+    pass
+
 class FailedDependency(Exception):
     pass
 
 class SyncStep(object):
-    """ An XOS Sync step. 
+    """ An XOS Sync step.
 
     Attributes:
-        psmodel        Model name the step synchronizes 
+        psmodel        Model name the step synchronizes
         dependencies    list of names of models that must be synchronized first if the current model depends on them
-    """ 
+    """
     slow=False
     def get_prop(self, prop):
         try:
@@ -73,20 +77,26 @@
 
         return objs
         #return Sliver.objects.filter(ip=None)
-    
+
     def check_dependencies(self, obj, failed):
         for dep in self.dependencies:
             peer_name = dep[0].lower() + dep[1:]    # django names are camelCased with the first letter lower
- 
+
             try:
                 peer_object = deepgetattr(obj, peer_name)
-                try: 
-                    peer_objects = peer_object.all() 
+                try:
+                    peer_objects = peer_object.all()
                 except AttributeError:
-                    peer_objects = [peer_object] 
+                    peer_objects = [peer_object]
             except:
                 peer_objects = []
 
+            if (hasattr(obj,'controller')):
+                try:
+                    peer_objects = filter(lambda o:o.controller==obj.controller, peer_objects)
+                except AttributeError:
+                    pass
+
             if (failed in peer_objects):
                 if (obj.backend_status!=failed.backend_status):
                     obj.backend_status = failed.backend_status
@@ -109,6 +119,7 @@
                     if (not backoff_disabled and next_run>time.time()):
                         sync_failed = True
             except:
+                logger.log_exc("Exception while loading scratchpad")
                 pass
 
             if (not sync_failed):
@@ -125,26 +136,32 @@
                         o.backend_register = json.dumps(scratchpad)
                         o.backend_status = "1 - OK"
                         o.save(update_fields=['enacted','backend_status','backend_register'])
-                except Exception,e:
+                except (InnocuousException,Exception) as e:
                     logger.log_exc("sync step failed!")
                     try:
                         if (o.backend_status.startswith('2 - ')):
                             str_e = '%s // %r'%(o.backend_status[4:],e)
-			    str_e = elim_dups(str_e)
+                            str_e = elim_dups(str_e)
                         else:
                             str_e = '%r'%e
                     except:
                         str_e = '%r'%e
 
                     try:
-                        o.backend_status = '2 - %s'%self.error_map.map(str_e)
+                        error = self.error_map.map(str_e)
                     except:
-                        o.backend_status = '2 - %s'%str_e
+                        error = '2 - %s'%str_e
+
+                    if isinstance(e, InnocuousException) and not force_error:
+                        o.backend_status = '1 - %s'%error
+                    else:
+                        o.backend_status = '2 - %s'%error
 
                     try:
                         scratchpad = json.loads(o.backend_register)
                         scratchpad['exponent']
                     except:
+                        logger.log_exc("Exception while updating scratchpad")
                         scratchpad = {'next_run':0, 'exponent':0}
 
                     # Second failure
@@ -163,7 +180,7 @@
                     if (o.pk):
                         try:
                             o.backend_status = o.backend_status[:1024]
-                            o.save(update_fields=['backend_status','backend_register'])
+                            o.save(update_fields=['backend_status','backend_register','updated'])
                         except:
                             print "Could not update backend status field!"
                             pass
diff --git a/xos/tools/cleanup_unique.py b/xos/tools/cleanup_unique.py
index 0ee0c7a..29fb047 100644
--- a/xos/tools/cleanup_unique.py
+++ b/xos/tools/cleanup_unique.py
@@ -8,38 +8,94 @@
 from cord.models import *
 django.setup()
 
+for obj in ControllerNetwork.deleted_objects.all():
+    print "Purging deleted object", obj
+    obj.delete(purge=True)
 
+for obj in ControllerSite.deleted_objects.all():
+    print "Purging deleted object", obj
+    obj.delete(purge=True)
+
+for obj in ControllerSlice.deleted_objects.all():
+    print "Purging deleted object", obj
+    obj.delete(purge=True)
+
+for obj in NetworkSlice.deleted_objects.all():
+    print "Purging deleted object", obj
+    obj.delete(purge=True)
+
+for obj in NetworkSliver.deleted_objects.all():
+    print "Purging deleted object", obj
+    obj.delete(purge=True)
+
+for obj in DeploymentPrivilege.deleted_objects.all():
+    print "Purging deleted object", obj
+    obj.delete(purge=True)
+
+for obj in SiteDeployment.deleted_objects.all():
+    print "Purging deleted object", obj
+    obj.delete(purge=True)
+
+seen=[]
 for obj in ControllerNetwork.objects.all():
+     seen.append(obj.id)
      conflicts = ControllerNetwork.objects.filter(network=obj.network, controller=obj.controller)
      for conflict in conflicts:
-         if conflict.id != obj.id:
-             print "Purging", conflict
+         if conflict.id not in seen:
+             print "Purging", conflict, conflict.id, "due to duplicate of", obj.id
              conflict.delete(purge=True)
 
+seen=[]
 for obj in NetworkSlice.objects.all():
+     seen.append(obj.id)
      conflicts = NetworkSlice.objects.filter(network=obj.network, slice=obj.slice)
      for conflict in conflicts:
-         if conflict.id != obj.id:
-             print "Purging", conflict        
+         if conflict.id not in seen:
+             print "Purging", conflict, conflict.id, "due to duplicate of", obj.id
              conflict.delete(purge=True)
 
+seen=[]
 for obj in NetworkSliver.objects.all():
+     seen.append(obj.id)
      conflicts = NetworkSliver.objects.filter(network=obj.network, sliver=obj.sliver)
      for conflict in conflicts:
-         if conflict.id != obj.id:
-             print "Purging", conflict 
+         if conflict.id not in seen:
+             print "Purging", conflict, conflict.id, "due to duplicate of", obj.id
              conflict.delete(purge=True)
 
+seen=[]
 for obj in DeploymentPrivilege.objects.all():
+     seen.append(obj.id)
      conflicts = DeploymentPrivilege.objects.filter(user=obj.user, deployment=obj.deployment, role=obj.role)
      for conflict in conflicts:
-         if conflict.id != obj.id:
-             print "Purging", conflict 
+         if conflict.id not in seen:
+             print "Purging", conflict, conflict.id, "due to duplicate of", obj.id
              conflict.delete(purge=True)
 
+seen=[]
 for obj in SiteDeployment.objects.all():
+     seen.append(obj.id)
      conflicts = SiteDeployment.objects.filter(site=obj.site, deployment=obj.deployment, controller=obj.controller)
      for conflict in conflicts:
-         if conflict.id != obj.id:
-             print "Purging", conflict 
+         if conflict.id not in seen:
+             print "Purging", conflict, conflict.id, "due to duplicate of", obj.id
              conflict.delete(purge=True)
+
+seen=[]
+for obj in ControllerSite.objects.all():
+     seen.append(obj.id)
+     conflicts = ControllerSite.objects.filter(site=obj.site, controller=obj.controller)
+     for conflict in conflicts:
+         if conflict.id not in seen:
+             print "Purging", conflict, conflict.id, "due to duplicate of", obj.id
+             conflict.delete(purge=True)
+
+seen=[]
+for obj in ControllerSlice.objects.all():
+     seen.append(obj.id)
+     conflicts = ControllerSlice.objects.filter(slice=obj.slice, controller=obj.controller)
+     for conflict in conflicts:
+         if conflict.id not in seen:
+             print "Purging", conflict, conflict.id, "due to duplicate of", obj.id
+             conflict.delete(purge=True)
+