update diag more frequently
diff --git a/xos/core/admin.py b/xos/core/admin.py
index 6d16fc9..200bb5c 100644
--- a/xos/core/admin.py
+++ b/xos/core/admin.py
@@ -2392,6 +2392,15 @@
def has_add_permission(self, request):
return False
+class DiagAdmin(XOSBaseAdmin):
+ list_display = ("name", "backend_status", "backend_register")
+ list_display_links = ('name',)
+
+ fieldsets = [
+ (None, {'fields': ['name', 'backend_status', 'backend_register'],
+ 'classes':['suit-tab suit-tab-general']}),
+ ]
+
# Now register the new UserAdmin...
admin.site.register(User, UserAdmin)
# ... and, since we're not using Django's builtin permissions,
@@ -2436,3 +2445,4 @@
admin.site.register(TenantRole, TenantRoleAdmin)
admin.site.register(TenantAttribute, TenantAttributeAdmin)
admin.site.register(AddressPool, AddressPoolAdmin)
+ admin.site.register(Diag, DiagAdmin)
diff --git a/xos/core/models/plcorebase.py b/xos/core/models/plcorebase.py
index 6be949d..b84b526 100644
--- a/xos/core/models/plcorebase.py
+++ b/xos/core/models/plcorebase.py
@@ -204,7 +204,7 @@
policed = models.DateTimeField(null=True, blank=True, default=None)
# This is a scratchpad used by the Observer
- backend_register = models.CharField(max_length=140,
+ backend_register = models.CharField(max_length=1024,
default="{}", null=True)
backend_status = models.CharField(max_length=1024,
diff --git a/xos/synchronizers/base/backend.py b/xos/synchronizers/base/backend.py
index d7307fa..af51e73 100644
--- a/xos/synchronizers/base/backend.py
+++ b/xos/synchronizers/base/backend.py
@@ -7,12 +7,16 @@
from xos.logger import Logger, logging
from synchronizers.model_policy import run_policy
from xos.config import Config
+from django.utils import timezone
+from diag import update_diag
logger = Logger(level=logging.INFO)
class Backend:
def run(self):
+ update_diag(sync_start=time.time(), backend_status="0 - Synchronizer Start")
+
# start the openstack observer
observer = XOSObserver()
observer_thread = threading.Thread(target=observer.run,name='synchronizer')
diff --git a/xos/synchronizers/base/diag.py b/xos/synchronizers/base/diag.py
new file mode 100644
index 0000000..170daad
--- /dev/null
+++ b/xos/synchronizers/base/diag.py
@@ -0,0 +1,40 @@
+import os
+import time
+import sys
+import traceback
+import json
+
+from core.models import Diag
+from xos.config import Config, XOS_DIR
+from xos.logger import Logger, logging, logger
+
+logger = Logger(level=logging.INFO)
+
+def update_diag(loop_end=None, loop_start=None, syncrecord_start=None, sync_start=None, backend_status=None):
+ try:
+ observer_name = Config().observer_name
+ except:
+ observer_name = 'global'
+
+ try:
+ diag = Diag.objects.filter(name=observer_name).first()
+ if (not diag):
+ diag = Diag(name=observer_name)
+ br_str = diag.backend_register
+ br = json.loads(br_str)
+ if loop_end:
+ br['last_run'] = loop_end
+ if loop_end and loop_start:
+ br['last_duration'] = loop_end - loop_start
+ if syncrecord_start:
+ br['last_syncrecord_start'] = syncrecord_start
+ if sync_start:
+ br['last_synchronizer_start'] = sync_start
+ if backend_status:
+ diag.backend_status = backend_status
+ diag.backend_register = json.dumps(br)
+ diag.save()
+ except:
+ logger.log_exc("Exception in update_diag")
+ traceback.print_exc()
+
diff --git a/xos/synchronizers/base/event_loop.py b/xos/synchronizers/base/event_loop.py
index c006c8a..5f31c96 100644
--- a/xos/synchronizers/base/event_loop.py
+++ b/xos/synchronizers/base/event_loop.py
@@ -28,6 +28,8 @@
from synchronizers.base.error_mapper import *
from synchronizers.openstack.openstacksyncstep import OpenStackSyncStep
from synchronizers.base.steps.sync_object import SyncObject
+from django.utils import timezone
+from diag import update_diag
# Load app models
@@ -521,22 +523,9 @@
loop_end = time.time()
- try:
- observer_name = Config().observer_name
- except:
- observer_name = 'global'
-
- diag = Diag.objects.filter(name=observer_name).first()
- if (not diag):
- diag = Diag(name=observer_name)
- diag.humanReadableName = observer_name
- br_str = diag.backend_register
- br = json.loads(br_str)
- br['last_run'] = loop_end
- br['last_duration'] = loop_end - loop_start
- diag.backend_register = json.dumps(br)
- diag.save()
+ update_diag(loop_end=loop_end, loop_start=loop_start, backend_status="1 - Bottom Of Loop")
except Exception, e:
logger.error('Core error. This seems like a misconfiguration or bug: %r. This error will not be relayed to the user!' % e)
logger.log_exc("Exception in observer run loop")
traceback.print_exc()
+ update_diag(backend_status="2 - Exception in Event Loop")
diff --git a/xos/synchronizers/base/syncstep.py b/xos/synchronizers/base/syncstep.py
index 42a9db4..27bb358 100644
--- a/xos/synchronizers/base/syncstep.py
+++ b/xos/synchronizers/base/syncstep.py
@@ -10,8 +10,8 @@
from django.db import reset_queries
from synchronizers.base.ansible import *
from generate.dependency_walker import *
+from diag import update_diag
-from time import time
import json
import time
import pdb
@@ -230,6 +230,7 @@
else:
new_enacted = timezone.now()
self.sync_record(o)
+ update_diag(syncrecord_start = time.time(), backend_status="1 - Synced Record")
o.enacted = new_enacted
scratchpad = {'next_run':0, 'exponent':0, 'last_success':time.time()}
o.backend_register = json.dumps(scratchpad)