add DeferredException for steps that want to wait only a short while and try again
diff --git a/xos/openstack_observer/steps/sync_slivers.py b/xos/openstack_observer/steps/sync_slivers.py
index 3cb3347..4c29e3d 100644
--- a/xos/openstack_observer/steps/sync_slivers.py
+++ b/xos/openstack_observer/steps/sync_slivers.py
@@ -64,7 +64,7 @@
exclude_templates = set()
for ns in sliver.networkslivers.all():
if not ns.port_id:
- raise Exception("Port %s on sliver %s has no id; Try again later" % (str(ns), str(sliver)) )
+ raise DeferredException("Port %s on sliver %s has no id; Try again later" % (str(ns), str(sliver)) )
ports.append(ns.port_id)
exclude_networks.add(ns.network)
exclude_templates.add(ns.network.template)
@@ -79,7 +79,7 @@
if controller_network.network.template.visibility == 'private' and \
controller_network.network.template.translation == 'none':
if not controller_network.net_id:
- raise Exception("Private Network %s has no id; Try again later" % controller_network.network.name)
+ raise DeferredException("Private Network %s has no id; Try again later" % controller_network.network.name)
nics.append(controller_network.net_id)
# Now include network templates, for those networks that use a
diff --git a/xos/openstack_observer/syncstep.py b/xos/openstack_observer/syncstep.py
index 165e3d6..03f6d16 100644
--- a/xos/openstack_observer/syncstep.py
+++ b/xos/openstack_observer/syncstep.py
@@ -30,6 +30,9 @@
class InnocuousException(Exception):
pass
+class DeferredException(Exception):
+ pass
+
class FailedDependency(Exception):
pass
@@ -144,7 +147,7 @@
o.backend_register = json.dumps(scratchpad)
o.backend_status = "1 - OK"
o.save(update_fields=['enacted','backend_status','backend_register'])
- except (InnocuousException,Exception) as e:
+ except (InnocuousException,Exception,DeferredException) as e:
logger.log_exc("sync step failed!")
try:
if (o.backend_status.startswith('2 - ')):
@@ -174,9 +177,13 @@
# Second failure
if (scratchpad['exponent']):
- delay = scratchpad['exponent'] * 600 # 10 minutes
- if (delay<1440):
- delay = 1440
+ if isinstance(e,DeferredException):
+ delay = scratchpad['exponent'] * 60 # 1 minute
+ else:
+ delay = scratchpad['exponent'] * 600 # 10 minutes
+ # cap delays at 8 hours
+ if (delay>8*60*60):
+ delay=8*60*60
scratchpad['next_run'] = time.time() + delay
scratchpad['exponent']+=1