add reset_queries() in hopes of preventing memory exhaustion if DEBUG=True
diff --git a/xos/model_policy.py b/xos/model_policy.py
index 19c281a..2813416 100644
--- a/xos/model_policy.py
+++ b/xos/model_policy.py
@@ -7,6 +7,7 @@
from datetime import datetime
import time
from core.models import *
+from django.db import reset_queries
from django.db.transaction import atomic
from django.db.models import F, Q
@@ -39,6 +40,8 @@
d.save(update_fields=save_fields)
except AttributeError,e:
raise e
+ except Exception,e:
+ logger.info('Could not save %r. Exception: %r'%(d,e))
def delete_if_inactive(d, o):
try:
@@ -124,5 +127,11 @@
print 'Purging object %r'%d
d.delete(purge=True)
+ try:
+ reset_queries()
+ except:
+ # this shouldn't happen, but in case it does, catch it...
+ logger.log_exc("exception in reset_queries")
+
if (time.time()-start<1):
time.sleep(1)
diff --git a/xos/openstack_observer/event_loop.py b/xos/openstack_observer/event_loop.py
index a63ff3c..57d6a31 100644
--- a/xos/openstack_observer/event_loop.py
+++ b/xos/openstack_observer/event_loop.py
@@ -16,6 +16,7 @@
from core.models import *
from django.db.models import F, Q
from django.db import connection
+from django.db import reset_queries
#from openstack.manager import OpenStackManager
from openstack.driver import OpenStackDriver
from util.logger import Logger, logging, logger
@@ -382,6 +383,12 @@
logger.info('Step %r is a leaf' % step)
pass
finally:
+ try:
+ reset_queries()
+ except:
+ # this shouldn't happen, but in case it does, catch it...
+ logger.log_exc("exception in reset_queries")
+
connection.close()
def run(self):
@@ -442,11 +449,19 @@
for t in threads:
t.start()
+ # another spot to clean up debug state
+ try:
+ reset_queries()
+ except:
+ # this shouldn't happen, but in case it does, catch it...
+ logger.log_exc("exception in reset_queries")
+
# Wait for all threads to finish before continuing with the run loop
for t in threads:
t.join()
self.save_run_times()
+
loop_end = time.time()
open('/tmp/%sobserver_last_run'%self.observer_name,'w').write(json.dumps({'last_run': loop_end, 'last_duration':loop_end - loop_start}))
except Exception, e:
diff --git a/xos/openstack_observer/syncstep.py b/xos/openstack_observer/syncstep.py
index 3dadbf4..9ec79cc 100644
--- a/xos/openstack_observer/syncstep.py
+++ b/xos/openstack_observer/syncstep.py
@@ -5,7 +5,8 @@
from util.logger import Logger, logging
from observer.steps import *
from django.db.models import F, Q
-from core.models import *
+from core.models import *
+from django.db import reset_queries
import json
import time
import pdb
@@ -106,6 +107,13 @@
def call(self, failed=[], deletion=False):
pending = self.fetch_pending(deletion)
for o in pending:
+ # another spot to clean up debug state
+ try:
+ reset_queries()
+ except:
+ # this shouldn't happen, but in case it does, catch it...
+ logger.log_exc("exception in reset_queries")
+
sync_failed = False
try:
backoff_disabled = Config().observer_backoff_disabled