Merge branch 'master' of github.com:open-cloud/xos
diff --git a/xos/core/models/sliver.py b/xos/core/models/sliver.py
index ff1e9b4..6c102d5 100644
--- a/xos/core/models/sliver.py
+++ b/xos/core/models/sliver.py
@@ -185,3 +185,11 @@
             pubkeys.add(self.slice.service.public_key)
 
         return pubkeys
+
+def controller_setter(instance, **kwargs):
+    try:
+        instance.controller = instance.node.site_deployment.controller
+    except:
+        instance.controller = None
+
+models.signals.post_init.connect(controller_setter, Sliver)
diff --git a/xos/model_policies/model_policy_Controller.py b/xos/model_policies/model_policy_Controller.py
index fa84be7..2db7a63 100644
--- a/xos/model_policies/model_policy_Controller.py
+++ b/xos/model_policies/model_policy_Controller.py
@@ -22,7 +22,7 @@
     slices = Slice.objects.all()
     for slice in slices:
         if slice not in ctrls_by_slice or \
-            controller not in ctrls_by_slice:
+            controller not in ctrls_by_slice[slice]:
             controller_slice = ControllerSlice(controller=controller, slice=slice)
             controller_slice.save()
     # relations for all users
diff --git a/xos/openstack_observer/steps/sync_controller_images.py b/xos/openstack_observer/steps/sync_controller_images.py
index 94b18a0..948fcea 100644
--- a/xos/openstack_observer/steps/sync_controller_images.py
+++ b/xos/openstack_observer/steps/sync_controller_images.py
@@ -4,6 +4,7 @@
 from django.db.models import F, Q
 from xos.config import Config
 from observer.openstacksyncstep import OpenStackSyncStep
+from observer.syncstep import *
 from core.models import Controller
 from core.models import Image, ControllerImages
 from util.logger import observer_logger as logger 
@@ -27,7 +28,7 @@
 
 	controller_register = json.loads(controller_image.controller.backend_register)
         if (controller_register.get('disabled',False)):
-                raise Exception('Controller %s is disabled'%controller_image.controller.name)
+                raise InnocuousException('Controller %s is disabled'%controller_image.controller.name)
 
         image_fields = {'endpoint':controller_image.controller.auth_url,
                         'admin_user':controller_image.controller.admin_user,
diff --git a/xos/openstack_observer/steps/sync_controller_networks.py b/xos/openstack_observer/steps/sync_controller_networks.py
index d327b7b..278d018 100644
--- a/xos/openstack_observer/steps/sync_controller_networks.py
+++ b/xos/openstack_observer/steps/sync_controller_networks.py
@@ -5,6 +5,7 @@
 from django.db.models import F, Q
 from xos.config import Config
 from observer.openstacksyncstep import OpenStackSyncStep
+from observer.syncstep import *
 from core.models.network import *
 from core.models.slice import *
 from core.models.sliver import Sliver
@@ -65,7 +66,7 @@
 
 	controller_register = json.loads(controller_network.controller.backend_register)
         if (controller_register.get('disabled',False)):
-                raise Exception('Controller %s is disabled'%controller_network.controller.name)
+                raise InnocuousException('Controller %s is disabled'%controller_network.controller.name)
 
         if not controller_network.controller.admin_user:
             logger.info("controller %r has no admin_user, skipping" % controller_network.controller)
@@ -78,7 +79,7 @@
     def delete_record(self, controller_network):
 	controller_register = json.loads(controller_network.controller.backend_register)
         if (controller_register.get('disabled',False)):
-                raise Exception('Controller %s is disabled'%controller_network.controller.name)
+                raise InnocuousException('Controller %s is disabled'%controller_network.controller.name)
 
 	try:
         	slice = controller_network.network.owner # XXX: FIXME!!
diff --git a/xos/openstack_observer/steps/sync_controller_site_privileges.py b/xos/openstack_observer/steps/sync_controller_site_privileges.py
index 6a13736..a2c40ef 100644
--- a/xos/openstack_observer/steps/sync_controller_site_privileges.py
+++ b/xos/openstack_observer/steps/sync_controller_site_privileges.py
@@ -4,6 +4,7 @@
 from django.db.models import F, Q
 from xos.config import Config
 from observer.openstacksyncstep import OpenStackSyncStep
+from observer.syncstep import *
 from core.models.site import Controller, SitePrivilege 
 from core.models.user import User
 from core.models.controlleruser import ControllerUser, ControllerSitePrivilege
@@ -28,7 +29,7 @@
 
 	controller_register = json.loads(controller_site_privilege.controller.backend_register)
         if (controller_register.get('disabled',False)):
-                raise Exception('Controller %s is disabled'%controller_site_privilege.controller.name)
+                raise InnocuousException('Controller %s is disabled'%controller_site_privilege.controller.name)
 
 
         if not controller_site_privilege.controller.admin_user:
@@ -76,7 +77,7 @@
     def delete_record(self, controller_site_privilege):
 	controller_register = json.loads(controller_site_privilege.controller.backend_register)
         if (controller_register.get('disabled',False)):
-                raise Exception('Controller %s is disabled'%controller_site_privilege.controller.name)
+                raise InnocuousException('Controller %s is disabled'%controller_site_privilege.controller.name)
 
         if controller_site_privilege.role_id:
             driver = self.driver.admin_driver(controller=controller_site_privilege.controller)
diff --git a/xos/openstack_observer/steps/sync_controller_sites.py b/xos/openstack_observer/steps/sync_controller_sites.py
index f101315..670f09c 100644
--- a/xos/openstack_observer/steps/sync_controller_sites.py
+++ b/xos/openstack_observer/steps/sync_controller_sites.py
@@ -4,6 +4,7 @@
 from xos.config import Config
 from openstack_observer.openstacksyncstep import OpenStackSyncStep
 from core.models.site import *
+from observer.syncstep import *
 from observer.ansible import *
 from util.logger import observer_logger as logger
 import json
@@ -20,7 +21,7 @@
     def sync_record(self, controller_site):
 	controller_register = json.loads(controller_site.controller.backend_register)
         if (controller_register.get('disabled',False)):
-                raise Exception('Controller %s is disabled'%controller_site.controller.name)
+                raise InnocuousException('Controller %s is disabled'%controller_site.controller.name)
 
 	template = os_template_env.get_template('sync_controller_sites.yaml')
 	tenant_fields = {'endpoint':controller_site.controller.auth_url,
@@ -41,7 +42,7 @@
     def delete_record(self, controller_site):
 	controller_register = json.loads(controller_site.controller.backend_register)
         if (controller_register.get('disabled',False)):
-                raise Exception('Controller %s is disabled'%controller_site.controller.name)
+                raise InnocuousException('Controller %s is disabled'%controller_site.controller.name)
 
 	if controller_site.tenant_id:
             driver = self.driver.admin_driver(controller=controller_site.controller)
diff --git a/xos/openstack_observer/steps/sync_controller_slice_privileges.py b/xos/openstack_observer/steps/sync_controller_slice_privileges.py
index 38f23c2..2e2e63c 100644
--- a/xos/openstack_observer/steps/sync_controller_slice_privileges.py
+++ b/xos/openstack_observer/steps/sync_controller_slice_privileges.py
@@ -4,6 +4,7 @@
 from django.db.models import F, Q
 from xos.config import Config
 from observer.openstacksyncstep import OpenStackSyncStep
+from observer.syncstep import *
 from core.models.slice import Controller, SlicePrivilege 
 from core.models.user import User
 from core.models.controlleruser import ControllerUser, ControllerSlicePrivilege
@@ -28,7 +29,7 @@
 
 	controller_register = json.loads(controller_slice_privilege.controller.backend_register)
         if (controller_register.get('disabled',False)):
-                raise Exception('Controller %s is disabled'%controller_slice_privilege.controller.name)
+                raise InnocuousException('Controller %s is disabled'%controller_slice_privilege.controller.name)
 
         if not controller_slice_privilege.controller.admin_user:
             logger.info("controller %r has no admin_user, skipping" % controller_slice_privilege.controller)
@@ -75,7 +76,7 @@
     def delete_record(self, controller_slice_privilege):
 	controller_register = json.loads(controller_slice_privilege.controller.backend_register)
         if (controller_register.get('disabled',False)):
-                raise Exception('Controller %s is disabled'%controller_slice_privilege.controller.name)
+                raise InnocuousException('Controller %s is disabled'%controller_slice_privilege.controller.name)
 
         if controller_slice_privilege.role_id:
             driver = self.driver.admin_driver(controller=controller_slice_privilege.controller)
diff --git a/xos/openstack_observer/steps/sync_controller_slices.py b/xos/openstack_observer/steps/sync_controller_slices.py
index 8d4a5e0..f64f9a8 100644
--- a/xos/openstack_observer/steps/sync_controller_slices.py
+++ b/xos/openstack_observer/steps/sync_controller_slices.py
@@ -5,6 +5,7 @@
 from django.db.models import F, Q
 from xos.config import Config
 from observer.openstacksyncstep import OpenStackSyncStep
+from observer.syncstep import *
 from core.models import *
 from observer.ansible import *
 from openstack.driver import OpenStackDriver
@@ -27,7 +28,7 @@
 
         controller_register = json.loads(controller_slice.controller.backend_register)
         if (controller_register.get('disabled',False)):
-            raise Exception('Controller %s is disabled'%controller_slice.controller.name)
+            raise InnocuousException('Controller %s is disabled'%controller_slice.controller.name)
 
         if not controller_slice.controller.admin_user:
             logger.info("controller %r has no admin_user, skipping" % controller_slice.controller)
@@ -72,7 +73,7 @@
     def delete_record(self, controller_slice):
         controller_register = json.loads(controller_slice.controller.backend_register)
         if (controller_register.get('disabled',False)):
-            raise Exception('Controller %s is disabled'%controller_slice.controller.name)
+            raise InnocuousException('Controller %s is disabled'%controller_slice.controller.name)
 
         controller_users = ControllerUser.objects.filter(user=controller_slice.slice.creator,
                                                               controller=controller_slice.controller)
diff --git a/xos/openstack_observer/steps/sync_controller_slices.yaml b/xos/openstack_observer/steps/sync_controller_slices.yaml
index 380f001..28c406d 100644
--- a/xos/openstack_observer/steps/sync_controller_slices.yaml
+++ b/xos/openstack_observer/steps/sync_controller_slices.yaml
@@ -2,9 +2,9 @@
 - hosts: 127.0.0.1
   connection: local
   tasks:
-  {% if delete %}
+  {% if delete -%}
   - keystone_user: endpoint={{ endpoint }} login_user={{ admin_user }} login_password={{ admin_password }} login_tenant_name={{ admin_tenant }} tenant={{ tenant }} tenant_description="{{ tenant_description }}" state=absent
-  {% else %}	
+  {% else -%}	
   - keystone_user: endpoint={{ endpoint }} login_user={{ admin_user }} login_password={{ admin_password }} login_tenant_name={{ admin_tenant }} tenant={{ tenant }} tenant_description="{{ tenant_description }}"
   {% for role in roles %}
   - keystone_user: endpoint={{ endpoint}} login_user={{ admin_user }} login_password={{ admin_password }} login_tenant_name={{ admin_tenant }} user="{{ name }}" role={{ role }} tenant={{ tenant }}
diff --git a/xos/openstack_observer/steps/sync_controller_users.py b/xos/openstack_observer/steps/sync_controller_users.py
index 47d1096..7979d82 100644
--- a/xos/openstack_observer/steps/sync_controller_users.py
+++ b/xos/openstack_observer/steps/sync_controller_users.py
@@ -4,6 +4,7 @@
 from django.db.models import F, Q
 from xos.config import Config
 from observer.openstacksyncstep import OpenStackSyncStep
+from observer.syncstep import *
 from core.models.site import Controller, SiteDeployment, SiteDeployment
 from core.models.user import User
 from core.models.controlleruser import ControllerUser
@@ -28,7 +29,7 @@
 
 	controller_register = json.loads(controller_user.controller.backend_register)
         if (controller_register.get('disabled',False)):
-                raise Exception('Controller %s is disabled'%controller_user.controller.name)
+                raise InnocuousException('Controller %s is disabled'%controller_user.controller.name)
 
         if not controller_user.controller.admin_user:
             logger.info("controller %r has no admin_user, skipping" % controller_user.controller)
@@ -79,7 +80,7 @@
     def delete_record(self, controller_user):
 	controller_register = json.loads(controller_user.controller.backend_register)
         if (controller_register.get('disabled',False)):
-                raise Exception('Controller %s is disabled'%controller_user.controller.name)
+                raise InnocuousException('Controller %s is disabled'%controller_user.controller.name)
 
         if controller_user.kuser_id:
             driver = self.driver.admin_driver(controller=controller_user.controller)
diff --git a/xos/openstack_observer/steps/sync_slivers.py b/xos/openstack_observer/steps/sync_slivers.py
index 9b5dd99..d3df266 100644
--- a/xos/openstack_observer/steps/sync_slivers.py
+++ b/xos/openstack_observer/steps/sync_slivers.py
@@ -9,6 +9,7 @@
 from core.models.slice import Slice, SlicePrivilege, ControllerSlice
 from core.models.network import Network, NetworkSlice, ControllerNetwork
 from observer.ansible import *
+from observer.syncstep import *
 from util.logger import observer_logger as logger
 
 def escape(s):
@@ -32,7 +33,7 @@
         controller_register = json.loads(sliver.node.site_deployment.controller.backend_register)
 
         if (controller_register.get('disabled',False)):
-            raise Exception('Controller %s is disabled'%sliver.node.site_deployment.controller.name)
+            raise InnocuousException('Controller %s is disabled'%sliver.node.site_deployment.controller.name)
 
         metadata_update = {}
         if (sliver.numberCores):
@@ -150,7 +151,7 @@
         controller_register = json.loads(sliver.node.site_deployment.controller.backend_register)
 
         if (controller_register.get('disabled',False)):
-            raise Exception('Controller %s is disabled'%sliver.node.site_deployment.controller.name)
+            raise InnocuousException('Controller %s is disabled'%sliver.node.site_deployment.controller.name)
 
         sliver_name = '%s-%d'%(sliver.slice.name,sliver.id)
         controller = sliver.node.site_deployment.controller
diff --git a/xos/openstack_observer/syncstep.py b/xos/openstack_observer/syncstep.py
index b752760..fb06bdd 100644
--- a/xos/openstack_observer/syncstep.py
+++ b/xos/openstack_observer/syncstep.py
@@ -21,20 +21,24 @@
     strs = backend_str.split(' // ')
     strs2 = f7(strs)
     return ' // '.join(strs2)
-    
+
 def deepgetattr(obj, attr):
     return reduce(getattr, attr.split('.'), obj)
 
+
+class InnocuousException(Exception):
+    pass
+
 class FailedDependency(Exception):
     pass
 
 class SyncStep(object):
-    """ An XOS Sync step. 
+    """ An XOS Sync step.
 
     Attributes:
-        psmodel        Model name the step synchronizes 
+        psmodel        Model name the step synchronizes
         dependencies    list of names of models that must be synchronized first if the current model depends on them
-    """ 
+    """
     slow=False
     def get_prop(self, prop):
         try:
@@ -73,20 +77,26 @@
 
         return objs
         #return Sliver.objects.filter(ip=None)
-    
+
     def check_dependencies(self, obj, failed):
         for dep in self.dependencies:
             peer_name = dep[0].lower() + dep[1:]    # django names are camelCased with the first letter lower
- 
+
             try:
                 peer_object = deepgetattr(obj, peer_name)
-                try: 
-                    peer_objects = peer_object.all() 
+                try:
+                    peer_objects = peer_object.all()
                 except AttributeError:
-                    peer_objects = [peer_object] 
+                    peer_objects = [peer_object]
             except:
                 peer_objects = []
 
+            if (hasattr(obj,'controller')):
+                try:
+                    peer_objects = filter(lambda o:o.controller==obj.controller, peer_objects)
+                except AttributeError:
+                    pass
+
             if (failed in peer_objects):
                 if (obj.backend_status!=failed.backend_status):
                     obj.backend_status = failed.backend_status
@@ -109,6 +119,7 @@
                     if (not backoff_disabled and next_run>time.time()):
                         sync_failed = True
             except:
+                logger.log_exc("Exception while loading scratchpad")
                 pass
 
             if (not sync_failed):
@@ -125,26 +136,32 @@
                         o.backend_register = json.dumps(scratchpad)
                         o.backend_status = "1 - OK"
                         o.save(update_fields=['enacted','backend_status','backend_register'])
-                except Exception,e:
+                except (InnocuousException,Exception) as e:
                     logger.log_exc("sync step failed!")
                     try:
                         if (o.backend_status.startswith('2 - ')):
                             str_e = '%s // %r'%(o.backend_status[4:],e)
-			    str_e = elim_dups(str_e)
+                            str_e = elim_dups(str_e)
                         else:
                             str_e = '%r'%e
                     except:
                         str_e = '%r'%e
 
                     try:
-                        o.backend_status = '2 - %s'%self.error_map.map(str_e)
+                        error = self.error_map.map(str_e)
                     except:
-                        o.backend_status = '2 - %s'%str_e
+                        error = '2 - %s'%str_e
+
+                    if isinstance(e, InnocuousException) and not force_error:
+                        o.backend_status = '1 - %s'%error
+                    else:
+                        o.backend_status = '2 - %s'%error
 
                     try:
                         scratchpad = json.loads(o.backend_register)
                         scratchpad['exponent']
                     except:
+                        logger.log_exc("Exception while updating scratchpad")
                         scratchpad = {'next_run':0, 'exponent':0}
 
                     # Second failure
@@ -163,7 +180,7 @@
                     if (o.pk):
                         try:
                             o.backend_status = o.backend_status[:1024]
-                            o.save(update_fields=['backend_status','backend_register'])
+                            o.save(update_fields=['backend_status','backend_register','updated'])
                         except:
                             print "Could not update backend status field!"
                             pass