Merge branch 'master' of github.com:open-cloud/xos
diff --git a/planetstack/core/models/plcorebase.py b/planetstack/core/models/plcorebase.py
index 4b63062..6b0a66b 100644
--- a/planetstack/core/models/plcorebase.py
+++ b/planetstack/core/models/plcorebase.py
@@ -159,6 +159,11 @@
updated = models.DateTimeField(auto_now=True, default=timezone.now)
enacted = models.DateTimeField(null=True, blank=True, default=None)
policed = models.DateTimeField(null=True, blank=True, default=None)
+
+ # This is a scratchpad used by the Observer
+ backend_register = models.CharField(max_length=140,
+ default="{}", null=True)
+
backend_status = models.CharField(max_length=140,
default="0 - Provisioning in progress")
deleted = models.BooleanField(default=False)
diff --git a/planetstack/core/views/observer.py b/planetstack/core/views/observer.py
new file mode 100644
index 0000000..67b2fde
--- /dev/null
+++ b/planetstack/core/views/observer.py
@@ -0,0 +1,19 @@
+from django.http import HttpResponse
+from monitor import driver
+from core.models import *
+import json
+import time
+
+def Observer(request):
+ t = time.time()
+ status_str = open('/tmp/observer_last_run','r').read()
+ d = json.loads(status_str)
+ comp = d['last_run'] + d['last_duration']*2 + 300
+ if comp>t:
+ d['health'] = ':-)'
+ else:
+ d['health'] = ':-X'
+ d['time'] = t
+ d['comp'] = comp
+
+ return HttpResponse(json.dumps(d))
diff --git a/planetstack/dmdot b/planetstack/dmdot
index 9e13df2..4bcf7e1 100644
--- a/planetstack/dmdot
+++ b/planetstack/dmdot
@@ -34,40 +34,52 @@
PlCoreBase = getattr(models_module,"PlCoreBase")
+synonyms = {
+ 'user':'creator'
+}
+
model_classes = []
class_names = []
lower_class_names = {}
for classname in dir(models_module):
c = getattr(models_module, classname, None)
- if type(c)==type(PlCoreBase):
- model_classes.append(c)
- class_names.append(c.__name__)
+ if type(c)==type(PlCoreBase):
+ model_classes.append(c)
+ class_names.append(c.__name__)
lower_class_names[c.__name__.lower()] = c
+ try:
+ synonym = synonyms[c.__name__.lower()]
+ lower_class_names[synonym] = c
+ except:
+ pass
+
# django doesn't use the correct case in field.name.title() for objects that
# have CamelCased class names. So, compare everything in lower case.
if (output=='-dot'):
- print "digraph plstack {";
- for c in model_classes:
- fields = c._meta.fields
- for f in fields:
- if type(f)==ForeignKey and f.name.lower() in lower_class_names:
+ print "digraph plstack {";
+ for c in model_classes:
+ fields = c._meta.fields
+
+ for f in fields:
+ if type(f)==ForeignKey and f.name.lower() in lower_class_names:
linked_class = lower_class_names[f.name.lower()]
- print '\t"%s"->"%s";'%(c.__name__,linked_class.__name__)
- print "}\n";
+ print '\t"%s"->"%s";'%(c.__name__,linked_class.__name__)
+ print "}\n";
elif (output=='-json'):
- d = {}
- for c in model_classes:
- fields = c._meta.fields
- for f in fields:
- if type(f)==ForeignKey and f.name.lower() in lower_class_names:
+ d = {}
+ for c in model_classes:
+ fields = c._meta.fields
+
+ for f in fields:
+ if type(f)==ForeignKey and f.name.lower() in lower_class_names:
linked_class = lower_class_names[f.name.lower()]
- try:
- d[c.__name__].append(linked_class.__name__)
- except KeyError:
- d[c.__name__]=[linked_class.__name__]
- d['NetworkDeployments'].append('SliceDeployments')
- print json.dumps(d,indent=4)
-
-
+ try:
+ d[c.__name__].append(linked_class.__name__)
+ except KeyError:
+ d[c.__name__]=[linked_class.__name__]
+ #d['ControllerNetwork'].append('SliceDeployments')
+ print json.dumps(d,indent=4)
+
+
diff --git a/planetstack/model-deps b/planetstack/model-deps
index 6647031..4d46d31 100644
--- a/planetstack/model-deps
+++ b/planetstack/model-deps
@@ -2,7 +2,8 @@
"Slice": [
"Site",
"Service",
- "ServiceClass"
+ "ServiceClass",
+ "User"
],
"ImageDeployments": [
"Image",
@@ -15,6 +16,10 @@
"ReservedResource": [
"Sliver"
],
+ "ControllerNetwork": [
+ "Network",
+ "Controller"
+ ],
"NetworkSlice": [
"Network",
"Slice"
@@ -24,19 +29,22 @@
"Slice",
"Invoice"
],
- "ControllerPrivilege": [
- "User",
+ "ControllerSite": [
+ "Site",
+ "Controller"
+ ],
+ "Node": [
+ "Site"
+ ],
+ "ControllerSlice": [
"Controller",
- "Role"
+ "Slice"
],
- "Sliver": [
- "Image",
- "Slice",
- "Node",
- "Flavor"
+ "ControllerSitePrivilege": [
+ "Controller"
],
- "ControllerNetwork": [
- "Network",
+ "ControllerUser": [
+ "User",
"Controller"
],
"UserCredential": [
@@ -61,6 +69,15 @@
"ServiceResource": [
"ServiceClass"
],
+ "Sliver": [
+ "Image",
+ "User",
+ "Slice",
+ "Deployment",
+ "Node",
+ "Flavor",
+ "Network"
+ ],
"Payment": [
"Account"
],
@@ -70,7 +87,12 @@
"ServiceAttribute": [
"Service"
],
- "ControllerSites": [
+ "ControllerSlicePrivilege": [
+ "Controller"
+ ],
+ "SiteDeployment": [
+ "Site",
+ "Deployment",
"Controller"
],
"SlicePrivilege": [
@@ -87,22 +109,17 @@
"Site",
"Role"
],
- "ControllerUser": [
- "User",
- "Controller"
- ],
"SiteCredential": [
"Site"
],
- "ControllerSlice": [
- "Controller",
- "Slice",
- "ControllerUser"
- ],
- "SiteDeployments": [
- "Site",
+ "DeploymentPrivilege": [
+ "User",
"Deployment",
- "Controller"
+ "Role"
+ ],
+ "ControllerDashboardView": [
+ "Controller",
+ "DashboardView"
],
"PlanetStackPrivilege": [
"User",
@@ -112,11 +129,6 @@
"Invoice": [
"Account"
],
- "DeploymentPrivilege": [
- "User",
- "Deployment",
- "Role"
- ],
"SliceCredential": [
"Slice"
]
diff --git a/planetstack/model_policies/__init__.py b/planetstack/model_policies/__init__.py
index cfcbe8f..8285cb2 100644
--- a/planetstack/model_policies/__init__.py
+++ b/planetstack/model_policies/__init__.py
@@ -5,5 +5,7 @@
from .model_policy_SitePrivilege import *
from .model_policy_SlicePrivilege import *
from .model_policy_ControllerSlice import *
+from .model_policy_ControllerSite import *
+from .model_policy_ControllerUser import *
from .model_policy_Controller import *
from .model_policy_Image import *
diff --git a/planetstack/model_policies/model_policy_ControllerSite.py b/planetstack/model_policies/model_policy_ControllerSite.py
new file mode 100644
index 0000000..4b76080
--- /dev/null
+++ b/planetstack/model_policies/model_policy_ControllerSite.py
@@ -0,0 +1,16 @@
+def handle(controller_site):
+ from core.models import ControllerSite, Site
+
+ try:
+ my_status_code = int(controller_site.backend_status[0])
+ try:
+ his_status_code = int(controller_site.site.backend_status[0])
+ except:
+ his_status_code = 0
+
+ if (my_status_code not in [0,his_status_code]):
+ controller_site.site.backend_status = controller_site.backend_status
+ controller_site.site.save(update_fields = ['backend_status'])
+ except Exception,e:
+ print str(e)
+ pass
diff --git a/planetstack/model_policies/model_policy_ControllerSlice.py b/planetstack/model_policies/model_policy_ControllerSlice.py
index a7f6c9a..1861c66 100644
--- a/planetstack/model_policies/model_policy_ControllerSlice.py
+++ b/planetstack/model_policies/model_policy_ControllerSlice.py
@@ -8,9 +8,9 @@
except:
his_status_code = 0
- print "My: %d His: %d"%(my_status_code, his_status_code)
if (my_status_code not in [0,his_status_code]):
controller_slice.slice.backend_status = controller_slice.backend_status
+ controller_slice.slice.save(update_fields = ['backend_status'])
except Exception,e:
print str(e)
pass
diff --git a/planetstack/model_policies/model_policy_ControllerUser.py b/planetstack/model_policies/model_policy_ControllerUser.py
new file mode 100644
index 0000000..b69c9b8
--- /dev/null
+++ b/planetstack/model_policies/model_policy_ControllerUser.py
@@ -0,0 +1,16 @@
+def handle(controller_user):
+ from core.models import ControllerUser, User
+
+ try:
+ my_status_code = int(controller_user.backend_status[0])
+ try:
+ his_status_code = int(controller_user.user.backend_status[0])
+ except:
+ his_status_code = 0
+
+ if (my_status_code not in [0,his_status_code]):
+ controller_user.user.backend_status = controller_user.backend_status
+ controller_user.user.save(update_fields = ['backend_status'])
+ except Exception,e:
+ print str(e)
+ pass
diff --git a/planetstack/model_policy.py b/planetstack/model_policy.py
index 8ddb82e..dc14766 100644
--- a/planetstack/model_policy.py
+++ b/planetstack/model_policy.py
@@ -53,10 +53,10 @@
instance.save(update_fields=['policed'])
def run_policy():
- from core.models import Slice,Controller,Network,User,SlicePrivilege,Site,SitePrivilege,Image,ControllerSlice
+ from core.models import Slice,Controller,Network,User,SlicePrivilege,Site,SitePrivilege,Image,ControllerSlice,ControllerUser,ControllerSite
while (True):
start = time.time()
- models = [Slice, Controller, Network, User, SlicePrivilege, Site, SitePrivilege, Image, ControllerSlice]
+ models = [Slice, Controller, Network, User, SlicePrivilege, Site, SitePrivilege, Image, ControllerSlice, ControllerSite, ControllerUser]
objects = []
for m in models:
diff --git a/planetstack/openstack_observer/event_loop.py b/planetstack/openstack_observer/event_loop.py
index 51be5d3..bc0b226 100644
--- a/planetstack/openstack_observer/event_loop.py
+++ b/planetstack/openstack_observer/event_loop.py
@@ -136,6 +136,12 @@
logger.info('Loading backend dependency graph from %s' % backend_path)
# This contains dependencies between backend records
self.backend_dependency_graph = json.loads(open(backend_path).read())
+ for k,v in self.backend_dependency_graph.items():
+ try:
+ self.model_dependency_graph[k].extend(v)
+ except KeyError:
+ self.model_dependency_graphp[k] = v
+
except Exception,e:
logger.info('Backend dependency graph not loaded')
# We can work without a backend graph
@@ -173,28 +179,6 @@
pass
# no dependencies, pass
- #import pdb
- #pdb.set_trace()
- if (self.backend_dependency_graph):
- backend_dict = {}
- for s in self.sync_steps:
- for m in s.serves:
- backend_dict[m]=s.__name__
-
- for k,v in backend_dependency_graph.iteritems():
- try:
- source = backend_dict[k]
- for m in v:
- try:
- dest = backend_dict[m]
- except KeyError:
- # no deps, pass
- pass
- step_graph[source]=dest
-
- except KeyError:
- pass
- # no dependencies, pass
self.dependency_graph = step_graph
self.deletion_dependency_graph = invert_graph(step_graph)
@@ -395,6 +379,7 @@
while True:
try:
+ loop_start = time.time()
error_map_file = getattr(Config(), "error_map_path", "/opt/planetstack/error_map.txt")
self.error_mapper = ErrorMapper(error_map_file)
@@ -446,6 +431,8 @@
t.join()
self.save_run_times()
+ loop_end = time.time()
+ open('/tmp/observer_last_run','w').write(json.dumps({'last_run': loop_end, 'last_duration':loop_end - loop_start}))
except Exception, e:
logger.error('Core error. This seems like a misconfiguration or bug: %r. This error will not be relayed to the user!' % e)
logger.log_exc("Exception in observer run loop")
diff --git a/planetstack/openstack_observer/steps/sync_controller_images.py b/planetstack/openstack_observer/steps/sync_controller_images.py
index fa0cad1..e076d81 100644
--- a/planetstack/openstack_observer/steps/sync_controller_images.py
+++ b/planetstack/openstack_observer/steps/sync_controller_images.py
@@ -13,6 +13,7 @@
class SyncControllerImages(OpenStackSyncStep):
provides=[ControllerImages]
+ observes = ControllerImages
requested_interval=0
def fetch_pending(self, deleted):
diff --git a/planetstack/openstack_observer/steps/sync_controller_networks.py b/planetstack/openstack_observer/steps/sync_controller_networks.py
index 1071ef7..83cf8ea 100644
--- a/planetstack/openstack_observer/steps/sync_controller_networks.py
+++ b/planetstack/openstack_observer/steps/sync_controller_networks.py
@@ -11,11 +11,14 @@
from util.logger import Logger, logging
from observer.ansible import *
+import pdb
+
logger = Logger(level=logging.INFO)
class SyncControllerNetworks(OpenStackSyncStep):
requested_interval = 0
provides=[Network]
+ observes=ControllerNetwork
def alloc_subnet(self, uuid):
# 16 bits only
diff --git a/planetstack/openstack_observer/steps/sync_controller_site_privileges.py b/planetstack/openstack_observer/steps/sync_controller_site_privileges.py
index 89fa27c..0ea86e1 100644
--- a/planetstack/openstack_observer/steps/sync_controller_site_privileges.py
+++ b/planetstack/openstack_observer/steps/sync_controller_site_privileges.py
@@ -16,6 +16,7 @@
class SyncControllerSitePrivileges(OpenStackSyncStep):
provides=[SitePrivilege]
requested_interval=0
+ observes=ControllerSitePrivilege
def fetch_pending(self, deleted):
diff --git a/planetstack/openstack_observer/steps/sync_controller_sites.py b/planetstack/openstack_observer/steps/sync_controller_sites.py
index 2dc13da..f7905c9 100644
--- a/planetstack/openstack_observer/steps/sync_controller_sites.py
+++ b/planetstack/openstack_observer/steps/sync_controller_sites.py
@@ -9,13 +9,13 @@
class SyncControllerSites(OpenStackSyncStep):
requested_interval=0
provides=[Site]
+ observes=ControllerSite
def fetch_pending(self, deleted=False):
pending = super(OpenStackSyncStep, self).fetch_pending(deleted)
return pending.filter(controller__isnull=False)
def sync_record(self, controller_site):
-
template = os_template_env.get_template('sync_controller_sites.yaml')
tenant_fields = {'endpoint':controller_site.controller.auth_url,
'admin_user': controller_site.controller.admin_user,
diff --git a/planetstack/openstack_observer/steps/sync_controller_slice_privileges.py b/planetstack/openstack_observer/steps/sync_controller_slice_privileges.py
index 35d1b99..99000ea 100644
--- a/planetstack/openstack_observer/steps/sync_controller_slice_privileges.py
+++ b/planetstack/openstack_observer/steps/sync_controller_slice_privileges.py
@@ -16,6 +16,7 @@
class SyncControllerSlicePrivileges(OpenStackSyncStep):
provides=[SlicePrivilege]
requested_interval=0
+ observes=ControllerSlicePrivilege
def fetch_pending(self, deleted):
diff --git a/planetstack/openstack_observer/steps/sync_controller_slices.py b/planetstack/openstack_observer/steps/sync_controller_slices.py
index 9e47669..0c24ae1 100644
--- a/planetstack/openstack_observer/steps/sync_controller_slices.py
+++ b/planetstack/openstack_observer/steps/sync_controller_slices.py
@@ -16,6 +16,7 @@
class SyncControllerSlices(OpenStackSyncStep):
provides=[Slice]
requested_interval=0
+ observes=ControllerSlice
def fetch_pending(self, deleted):
if (deleted):
diff --git a/planetstack/openstack_observer/steps/sync_controller_users.py b/planetstack/openstack_observer/steps/sync_controller_users.py
index d7d79f7..08123fe 100644
--- a/planetstack/openstack_observer/steps/sync_controller_users.py
+++ b/planetstack/openstack_observer/steps/sync_controller_users.py
@@ -16,6 +16,7 @@
class SyncControllerUsers(OpenStackSyncStep):
provides=[User]
requested_interval=0
+ observes=ControllerUser
def fetch_pending(self, deleted):
diff --git a/planetstack/openstack_observer/steps/sync_slivers.py b/planetstack/openstack_observer/steps/sync_slivers.py
index 3989c16..9a25c19 100644
--- a/planetstack/openstack_observer/steps/sync_slivers.py
+++ b/planetstack/openstack_observer/steps/sync_slivers.py
@@ -19,6 +19,7 @@
class SyncSlivers(OpenStackSyncStep):
provides=[Sliver]
requested_interval=0
+ observes=Sliver
def get_userdata(self, sliver):
userdata = 'opencloud:\n slicename: "%s"\n hostname: "%s"\n' % (sliver.slice.name, sliver.node.name)
diff --git a/planetstack/openstack_observer/syncstep.py b/planetstack/openstack_observer/syncstep.py
index fd34f55..247d4c9 100644
--- a/planetstack/openstack_observer/syncstep.py
+++ b/planetstack/openstack_observer/syncstep.py
@@ -5,6 +5,9 @@
from util.logger import Logger, logging
from observer.steps import *
from django.db.models import F, Q
+import json
+import time
+import pdb
logger = Logger(level=logging.INFO)
@@ -48,7 +51,7 @@
# This is the most common implementation of fetch_pending
# Steps should override it if they have their own logic
# for figuring out what objects are outstanding.
- main_obj = self.provides[0]
+ main_obj = self.observes
if (not deletion):
objs = main_obj.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
else:
@@ -74,34 +77,69 @@
def call(self, failed=[], deletion=False):
pending = self.fetch_pending(deletion)
for o in pending:
+ sync_failed = False
try:
- for f in failed:
- self.check_dependencies(o,f) # Raises exception if failed
- if (deletion):
- self.delete_record(o)
- o.delete(purge=True)
- else:
- self.sync_record(o)
- o.enacted = datetime.now() # Is this the same timezone? XXX
- o.backend_status = "1 - OK"
- o.save(update_fields=['enacted'])
- except Exception,e:
- logger.log_exc("sync step failed!")
- str_e = '%r'%e
+ scratchpad = json.loads(o.backend_register)
+ if (scratchpad):
+ next_run = scratchpad['next_run']
+ if (next_run>time.time()):
+ sync_failed = True
+ print "BACKING OFF, exponent = %d"%scratchpad['exponent']
+ except:
+ pass
+
+ if (not sync_failed):
try:
- o.backend_status = '2 - %s'%self.error_map.map(str_e)
- except:
- o.backend_status = '2 - %s'%str_e
-
- # TOFIX:
- # DatabaseError: value too long for type character varying(140)
- if (o.pk):
+ for f in failed:
+ self.check_dependencies(o,f) # Raises exception if failed
+ if (deletion):
+ self.delete_record(o)
+ o.delete(purge=True)
+ else:
+ self.sync_record(o)
+ o.enacted = datetime.now() # Is this the same timezone? XXX
+ scratchpad = {'next_run':0, 'exponent':0}
+ o.backend_register = json.dumps(scratchpad)
+ o.backend_status = "1 - OK"
+ o.save(update_fields=['enacted','backend_status','backend_register'])
+ except Exception,e:
+ logger.log_exc("sync step failed!")
+ str_e = '%r'%e
try:
- o.save(update_fields=['backend_status'])
+ o.backend_status = '2 - %s'%self.error_map.map(str_e)
except:
- print "Could not update backend status field!"
- pass
+ o.backend_status = '2 - %s'%str_e
+ try:
+ scratchpad = json.loads(o.backend_register)
+ scratchpad['exponent']
+ except:
+ scratchpad = {'next_run':0, 'exponent':0}
+
+ # Second failure
+ if (scratchpad['exponent']):
+ delay = scratchpad['exponent'] * 600 # 10 minutes
+ if (delay<1440):
+ delay = 1440
+ scratchpad['next_run'] = time.time() + delay
+
+ scratchpad['exponent']+=1
+
+ o.backend_register = json.dumps(scratchpad)
+
+ # TOFIX:
+ # DatabaseError: value too long for type character varying(140)
+ if (o.pk):
+ try:
+ o.backend_status = o.backend_status[:140]
+ o.save(update_fields=['backend_status','backend_register'])
+ except:
+ print "Could not update backend status field!"
+ pass
+ sync_failed = True
+
+
+ if (sync_failed):
failed.append(o)
return failed
diff --git a/planetstack/planetstack/urls.py b/planetstack/planetstack/urls.py
index 30ad665..7e780fb 100644
--- a/planetstack/planetstack/urls.py
+++ b/planetstack/planetstack/urls.py
@@ -26,8 +26,9 @@
# Examples:
# url(r'^$', 'planetstack.views.home', name='home'),
# url(r'^planetstack/', include('planetstack.foo.urls')),
-
url(r'^stats', 'core.views.stats.Stats', name='stats'),
+ url(r'^observer', 'core.views.observer.Observer', name='observer'),
+
# Uncomment the admin/doc line below to enable admin documentation:
url(r'^admin/doc/', include('django.contrib.admindocs.urls')),