Support for innocuous exceptions. Fail and retry, but do not report as error in GUI.
diff --git a/xos/openstack_observer/steps/sync_controller_images.py b/xos/openstack_observer/steps/sync_controller_images.py
index 94b18a0..948fcea 100644
--- a/xos/openstack_observer/steps/sync_controller_images.py
+++ b/xos/openstack_observer/steps/sync_controller_images.py
@@ -4,6 +4,7 @@
from django.db.models import F, Q
from xos.config import Config
from observer.openstacksyncstep import OpenStackSyncStep
+from observer.syncstep import *
from core.models import Controller
from core.models import Image, ControllerImages
from util.logger import observer_logger as logger
@@ -27,7 +28,7 @@
controller_register = json.loads(controller_image.controller.backend_register)
if (controller_register.get('disabled',False)):
- raise Exception('Controller %s is disabled'%controller_image.controller.name)
+ raise InnocuousException('Controller %s is disabled'%controller_image.controller.name)
image_fields = {'endpoint':controller_image.controller.auth_url,
'admin_user':controller_image.controller.admin_user,
diff --git a/xos/openstack_observer/steps/sync_controller_networks.py b/xos/openstack_observer/steps/sync_controller_networks.py
index d327b7b..278d018 100644
--- a/xos/openstack_observer/steps/sync_controller_networks.py
+++ b/xos/openstack_observer/steps/sync_controller_networks.py
@@ -5,6 +5,7 @@
from django.db.models import F, Q
from xos.config import Config
from observer.openstacksyncstep import OpenStackSyncStep
+from observer.syncstep import *
from core.models.network import *
from core.models.slice import *
from core.models.sliver import Sliver
@@ -65,7 +66,7 @@
controller_register = json.loads(controller_network.controller.backend_register)
if (controller_register.get('disabled',False)):
- raise Exception('Controller %s is disabled'%controller_network.controller.name)
+ raise InnocuousException('Controller %s is disabled'%controller_network.controller.name)
if not controller_network.controller.admin_user:
logger.info("controller %r has no admin_user, skipping" % controller_network.controller)
@@ -78,7 +79,7 @@
def delete_record(self, controller_network):
controller_register = json.loads(controller_network.controller.backend_register)
if (controller_register.get('disabled',False)):
- raise Exception('Controller %s is disabled'%controller_network.controller.name)
+ raise InnocuousException('Controller %s is disabled'%controller_network.controller.name)
try:
slice = controller_network.network.owner # XXX: FIXME!!
diff --git a/xos/openstack_observer/steps/sync_controller_site_privileges.py b/xos/openstack_observer/steps/sync_controller_site_privileges.py
index 6a13736..a2c40ef 100644
--- a/xos/openstack_observer/steps/sync_controller_site_privileges.py
+++ b/xos/openstack_observer/steps/sync_controller_site_privileges.py
@@ -4,6 +4,7 @@
from django.db.models import F, Q
from xos.config import Config
from observer.openstacksyncstep import OpenStackSyncStep
+from observer.syncstep import *
from core.models.site import Controller, SitePrivilege
from core.models.user import User
from core.models.controlleruser import ControllerUser, ControllerSitePrivilege
@@ -28,7 +29,7 @@
controller_register = json.loads(controller_site_privilege.controller.backend_register)
if (controller_register.get('disabled',False)):
- raise Exception('Controller %s is disabled'%controller_site_privilege.controller.name)
+ raise InnocuousException('Controller %s is disabled'%controller_site_privilege.controller.name)
if not controller_site_privilege.controller.admin_user:
@@ -76,7 +77,7 @@
def delete_record(self, controller_site_privilege):
controller_register = json.loads(controller_site_privilege.controller.backend_register)
if (controller_register.get('disabled',False)):
- raise Exception('Controller %s is disabled'%controller_site_privilege.controller.name)
+ raise InnocuousException('Controller %s is disabled'%controller_site_privilege.controller.name)
if controller_site_privilege.role_id:
driver = self.driver.admin_driver(controller=controller_site_privilege.controller)
diff --git a/xos/openstack_observer/steps/sync_controller_sites.py b/xos/openstack_observer/steps/sync_controller_sites.py
index f101315..670f09c 100644
--- a/xos/openstack_observer/steps/sync_controller_sites.py
+++ b/xos/openstack_observer/steps/sync_controller_sites.py
@@ -4,6 +4,7 @@
from xos.config import Config
from openstack_observer.openstacksyncstep import OpenStackSyncStep
from core.models.site import *
+from observer.syncstep import *
from observer.ansible import *
from util.logger import observer_logger as logger
import json
@@ -20,7 +21,7 @@
def sync_record(self, controller_site):
controller_register = json.loads(controller_site.controller.backend_register)
if (controller_register.get('disabled',False)):
- raise Exception('Controller %s is disabled'%controller_site.controller.name)
+ raise InnocuousException('Controller %s is disabled'%controller_site.controller.name)
template = os_template_env.get_template('sync_controller_sites.yaml')
tenant_fields = {'endpoint':controller_site.controller.auth_url,
@@ -41,7 +42,7 @@
def delete_record(self, controller_site):
controller_register = json.loads(controller_site.controller.backend_register)
if (controller_register.get('disabled',False)):
- raise Exception('Controller %s is disabled'%controller_site.controller.name)
+ raise InnocuousException('Controller %s is disabled'%controller_site.controller.name)
if controller_site.tenant_id:
driver = self.driver.admin_driver(controller=controller_site.controller)
diff --git a/xos/openstack_observer/steps/sync_controller_slice_privileges.py b/xos/openstack_observer/steps/sync_controller_slice_privileges.py
index 38f23c2..2e2e63c 100644
--- a/xos/openstack_observer/steps/sync_controller_slice_privileges.py
+++ b/xos/openstack_observer/steps/sync_controller_slice_privileges.py
@@ -4,6 +4,7 @@
from django.db.models import F, Q
from xos.config import Config
from observer.openstacksyncstep import OpenStackSyncStep
+from observer.syncstep import *
from core.models.slice import Controller, SlicePrivilege
from core.models.user import User
from core.models.controlleruser import ControllerUser, ControllerSlicePrivilege
@@ -28,7 +29,7 @@
controller_register = json.loads(controller_slice_privilege.controller.backend_register)
if (controller_register.get('disabled',False)):
- raise Exception('Controller %s is disabled'%controller_slice_privilege.controller.name)
+ raise InnocuousException('Controller %s is disabled'%controller_slice_privilege.controller.name)
if not controller_slice_privilege.controller.admin_user:
logger.info("controller %r has no admin_user, skipping" % controller_slice_privilege.controller)
@@ -75,7 +76,7 @@
def delete_record(self, controller_slice_privilege):
controller_register = json.loads(controller_slice_privilege.controller.backend_register)
if (controller_register.get('disabled',False)):
- raise Exception('Controller %s is disabled'%controller_slice_privilege.controller.name)
+ raise InnocuousException('Controller %s is disabled'%controller_slice_privilege.controller.name)
if controller_slice_privilege.role_id:
driver = self.driver.admin_driver(controller=controller_slice_privilege.controller)
diff --git a/xos/openstack_observer/steps/sync_controller_slices.py b/xos/openstack_observer/steps/sync_controller_slices.py
index 8d4a5e0..f64f9a8 100644
--- a/xos/openstack_observer/steps/sync_controller_slices.py
+++ b/xos/openstack_observer/steps/sync_controller_slices.py
@@ -5,6 +5,7 @@
from django.db.models import F, Q
from xos.config import Config
from observer.openstacksyncstep import OpenStackSyncStep
+from observer.syncstep import *
from core.models import *
from observer.ansible import *
from openstack.driver import OpenStackDriver
@@ -27,7 +28,7 @@
controller_register = json.loads(controller_slice.controller.backend_register)
if (controller_register.get('disabled',False)):
- raise Exception('Controller %s is disabled'%controller_slice.controller.name)
+ raise InnocuousException('Controller %s is disabled'%controller_slice.controller.name)
if not controller_slice.controller.admin_user:
logger.info("controller %r has no admin_user, skipping" % controller_slice.controller)
@@ -72,7 +73,7 @@
def delete_record(self, controller_slice):
controller_register = json.loads(controller_slice.controller.backend_register)
if (controller_register.get('disabled',False)):
- raise Exception('Controller %s is disabled'%controller_slice.controller.name)
+ raise InnocuousException('Controller %s is disabled'%controller_slice.controller.name)
controller_users = ControllerUser.objects.filter(user=controller_slice.slice.creator,
controller=controller_slice.controller)
diff --git a/xos/openstack_observer/steps/sync_controller_users.py b/xos/openstack_observer/steps/sync_controller_users.py
index 47d1096..7979d82 100644
--- a/xos/openstack_observer/steps/sync_controller_users.py
+++ b/xos/openstack_observer/steps/sync_controller_users.py
@@ -4,6 +4,7 @@
from django.db.models import F, Q
from xos.config import Config
from observer.openstacksyncstep import OpenStackSyncStep
+from observer.syncstep import *
from core.models.site import Controller, SiteDeployment, SiteDeployment
from core.models.user import User
from core.models.controlleruser import ControllerUser
@@ -28,7 +29,7 @@
controller_register = json.loads(controller_user.controller.backend_register)
if (controller_register.get('disabled',False)):
- raise Exception('Controller %s is disabled'%controller_user.controller.name)
+ raise InnocuousException('Controller %s is disabled'%controller_user.controller.name)
if not controller_user.controller.admin_user:
logger.info("controller %r has no admin_user, skipping" % controller_user.controller)
@@ -79,7 +80,7 @@
def delete_record(self, controller_user):
controller_register = json.loads(controller_user.controller.backend_register)
if (controller_register.get('disabled',False)):
- raise Exception('Controller %s is disabled'%controller_user.controller.name)
+ raise InnocuousException('Controller %s is disabled'%controller_user.controller.name)
if controller_user.kuser_id:
driver = self.driver.admin_driver(controller=controller_user.controller)
diff --git a/xos/openstack_observer/steps/sync_slivers.py b/xos/openstack_observer/steps/sync_slivers.py
index 9b5dd99..d3df266 100644
--- a/xos/openstack_observer/steps/sync_slivers.py
+++ b/xos/openstack_observer/steps/sync_slivers.py
@@ -9,6 +9,7 @@
from core.models.slice import Slice, SlicePrivilege, ControllerSlice
from core.models.network import Network, NetworkSlice, ControllerNetwork
from observer.ansible import *
+from observer.syncstep import *
from util.logger import observer_logger as logger
def escape(s):
@@ -32,7 +33,7 @@
controller_register = json.loads(sliver.node.site_deployment.controller.backend_register)
if (controller_register.get('disabled',False)):
- raise Exception('Controller %s is disabled'%sliver.node.site_deployment.controller.name)
+ raise InnocuousException('Controller %s is disabled'%sliver.node.site_deployment.controller.name)
metadata_update = {}
if (sliver.numberCores):
@@ -150,7 +151,7 @@
controller_register = json.loads(sliver.node.site_deployment.controller.backend_register)
if (controller_register.get('disabled',False)):
- raise Exception('Controller %s is disabled'%sliver.node.site_deployment.controller.name)
+ raise InnocuousException('Controller %s is disabled'%sliver.node.site_deployment.controller.name)
sliver_name = '%s-%d'%(sliver.slice.name,sliver.id)
controller = sliver.node.site_deployment.controller
diff --git a/xos/openstack_observer/syncstep.py b/xos/openstack_observer/syncstep.py
index b752760..b3fdde0 100644
--- a/xos/openstack_observer/syncstep.py
+++ b/xos/openstack_observer/syncstep.py
@@ -21,20 +21,24 @@
strs = backend_str.split(' // ')
strs2 = f7(strs)
return ' // '.join(strs2)
-
+
def deepgetattr(obj, attr):
return reduce(getattr, attr.split('.'), obj)
+
+class InnocuousException(Exception):
+ pass
+
class FailedDependency(Exception):
pass
class SyncStep(object):
- """ An XOS Sync step.
+ """ An XOS Sync step.
Attributes:
- psmodel Model name the step synchronizes
+ psmodel Model name the step synchronizes
dependencies list of names of models that must be synchronized first if the current model depends on them
- """
+ """
slow=False
def get_prop(self, prop):
try:
@@ -73,20 +77,26 @@
return objs
#return Sliver.objects.filter(ip=None)
-
+
def check_dependencies(self, obj, failed):
for dep in self.dependencies:
peer_name = dep[0].lower() + dep[1:] # django names are camelCased with the first letter lower
-
+
try:
peer_object = deepgetattr(obj, peer_name)
- try:
- peer_objects = peer_object.all()
+ try:
+ peer_objects = peer_object.all()
except AttributeError:
- peer_objects = [peer_object]
+ peer_objects = [peer_object]
except:
peer_objects = []
+ if (hasattr(obj,'controller')):
+ try:
+ peer_objects = filter(lambda o:o.controller==obj.controller, peer_objects)
+ except AttributeError:
+ pass
+
if (failed in peer_objects):
if (obj.backend_status!=failed.backend_status):
obj.backend_status = failed.backend_status
@@ -109,6 +119,7 @@
if (not backoff_disabled and next_run>time.time()):
sync_failed = True
except:
+ logger.log_exc("Exception while loading scratchpad")
pass
if (not sync_failed):
@@ -125,26 +136,32 @@
o.backend_register = json.dumps(scratchpad)
o.backend_status = "1 - OK"
o.save(update_fields=['enacted','backend_status','backend_register'])
- except Exception,e:
+ except (InnocuousException,Exception) as e:
logger.log_exc("sync step failed!")
try:
if (o.backend_status.startswith('2 - ')):
str_e = '%s // %r'%(o.backend_status[4:],e)
- str_e = elim_dups(str_e)
+ str_e = elim_dups(str_e)
else:
str_e = '%r'%e
except:
str_e = '%r'%e
try:
- o.backend_status = '2 - %s'%self.error_map.map(str_e)
+ error = self.error_map.map(str_e)
except:
- o.backend_status = '2 - %s'%str_e
+ error = '2 - %s'%str_e
+
+ if isinstance(e, InnocuousException) and not force_error:
+ o.backend_status = '1 - %s'%error
+ else:
+ o.backend_status = '3 - %s'%error
try:
scratchpad = json.loads(o.backend_register)
scratchpad['exponent']
except:
+ logger.log_exc("Exception while updating scratchpad")
scratchpad = {'next_run':0, 'exponent':0}
# Second failure