CORD-1009 move reaper from model_policy to core
Change-Id: Id2d527b2b677bec214510e39f03d3ec629699387
diff --git a/xos/grpc/README.md b/xos/coreapi/README.md
similarity index 100%
rename from xos/grpc/README.md
rename to xos/coreapi/README.md
diff --git a/xos/grpc/apihelper.py b/xos/coreapi/apihelper.py
similarity index 100%
rename from xos/grpc/apihelper.py
rename to xos/coreapi/apihelper.py
diff --git a/xos/grpc/certs/Makefile b/xos/coreapi/certs/Makefile
similarity index 100%
rename from xos/grpc/certs/Makefile
rename to xos/coreapi/certs/Makefile
diff --git a/xos/coreapi/core_main.py b/xos/coreapi/core_main.py
new file mode 100644
index 0000000..6e25fcd
--- /dev/null
+++ b/xos/coreapi/core_main.py
@@ -0,0 +1,34 @@
+import os
+import sys
+import time
+
+import django
+sys.path.append('/opt/xos')
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "xos.settings")
+
+from reaper import ReaperThread
+from grpc_server import XOSGrpcServer, restart_chameleon
+
+from xos.logger import Logger, logging
+logger = Logger(level=logging.DEBUG)
+
+if __name__ == '__main__':
+ django.setup()
+
+ reaper = ReaperThread()
+ reaper.start()
+
+ server = XOSGrpcServer().start()
+
+ restart_chameleon()
+
+ logger.info("Core_main entering wait loop")
+
+ _ONE_DAY_IN_SECONDS = 60 * 60 * 24
+ try:
+ while 1:
+ time.sleep(_ONE_DAY_IN_SECONDS)
+ except KeyboardInterrupt:
+ server.stop()
+ reaper.stop()
+
diff --git a/xos/grpc/env.sh b/xos/coreapi/env.sh
similarity index 100%
rename from xos/grpc/env.sh
rename to xos/coreapi/env.sh
diff --git a/xos/grpc/grpc_client.py b/xos/coreapi/grpc_client.py
similarity index 100%
rename from xos/grpc/grpc_client.py
rename to xos/coreapi/grpc_client.py
diff --git a/xos/grpc/grpc_server.py b/xos/coreapi/grpc_server.py
similarity index 97%
rename from xos/grpc/grpc_server.py
rename to xos/coreapi/grpc_server.py
index cb5b2a7..aacc649 100644
--- a/xos/grpc/grpc_server.py
+++ b/xos/coreapi/grpc_server.py
@@ -18,16 +18,16 @@
import os
import sys
import uuid
-from Queue import Queue
from collections import OrderedDict
from os.path import abspath, basename, dirname, join, walk
import grpc
from concurrent import futures
import zlib
-import django
-sys.path.append('/opt/xos')
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "xos.settings")
+if __name__ == "__main__":
+ import django
+ sys.path.append('/opt/xos')
+ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "xos.settings")
from protos import xos_pb2, schema_pb2, modeldefs_pb2, utility_pb2
from xos_grpc_api import XosService
diff --git a/xos/grpc/list_test.py b/xos/coreapi/list_test.py
similarity index 100%
rename from xos/grpc/list_test.py
rename to xos/coreapi/list_test.py
diff --git a/xos/grpc/orm.py b/xos/coreapi/orm.py
similarity index 100%
rename from xos/grpc/orm.py
rename to xos/coreapi/orm.py
diff --git a/xos/grpc/protos/Makefile b/xos/coreapi/protos/Makefile
similarity index 93%
rename from xos/grpc/protos/Makefile
rename to xos/coreapi/protos/Makefile
index 771a4bc..dbe1fed 100644
--- a/xos/grpc/protos/Makefile
+++ b/xos/coreapi/protos/Makefile
@@ -82,7 +82,7 @@
sudo make uninstall
rebuild-protos:
- cd ../../tools/apigen && python ./modelgen -a "*" protobuf.template.txt > /opt/xos/grpc/protos/xos.proto
- cd ../../tools/apigen && python ./modelgen -a "*" grpc_api.template.py > /opt/xos/grpc/xos_grpc_api.py
- cd ../../tools/apigen && python ./modelgen -a "*" grpc_list_test.template.py > /opt/xos/grpc/tests/list_test.py
- cd ../../tools/apigen && python ./modelgen -a "*" chameleon_list_test.template.sh > /opt/xos/grpc/tests/chameleon_list_test.sh
+ cd ../../tools/apigen && python ./modelgen -a "*" protobuf.template.txt > /opt/xos/coreapi/protos/xos.proto
+ cd ../../tools/apigen && python ./modelgen -a "*" grpc_api.template.py > /opt/xos/coreapi/xos_grpc_api.py
+ cd ../../tools/apigen && python ./modelgen -a "*" grpc_list_test.template.py > /opt/xos/coreapi/tests/list_test.py
+ cd ../../tools/apigen && python ./modelgen -a "*" chameleon_list_test.template.sh > /opt/xos/coreapi/tests/chameleon_list_test.sh
diff --git a/xos/grpc/protos/__init__.py b/xos/coreapi/protos/__init__.py
similarity index 100%
rename from xos/grpc/protos/__init__.py
rename to xos/coreapi/protos/__init__.py
diff --git a/xos/grpc/protos/common.proto b/xos/coreapi/protos/common.proto
similarity index 100%
rename from xos/grpc/protos/common.proto
rename to xos/coreapi/protos/common.proto
diff --git a/xos/grpc/protos/modeldefs.proto b/xos/coreapi/protos/modeldefs.proto
similarity index 100%
rename from xos/grpc/protos/modeldefs.proto
rename to xos/coreapi/protos/modeldefs.proto
diff --git a/xos/grpc/protos/schema.proto b/xos/coreapi/protos/schema.proto
similarity index 100%
rename from xos/grpc/protos/schema.proto
rename to xos/coreapi/protos/schema.proto
diff --git a/xos/grpc/protos/third_party/__init__.py b/xos/coreapi/protos/third_party/__init__.py
similarity index 100%
rename from xos/grpc/protos/third_party/__init__.py
rename to xos/coreapi/protos/third_party/__init__.py
diff --git a/xos/grpc/protos/third_party/google/LICENSE b/xos/coreapi/protos/third_party/google/LICENSE
similarity index 100%
rename from xos/grpc/protos/third_party/google/LICENSE
rename to xos/coreapi/protos/third_party/google/LICENSE
diff --git a/xos/grpc/protos/third_party/google/__init__.py b/xos/coreapi/protos/third_party/google/__init__.py
similarity index 100%
rename from xos/grpc/protos/third_party/google/__init__.py
rename to xos/coreapi/protos/third_party/google/__init__.py
diff --git a/xos/grpc/protos/third_party/google/api/__init__.py b/xos/coreapi/protos/third_party/google/api/__init__.py
similarity index 100%
rename from xos/grpc/protos/third_party/google/api/__init__.py
rename to xos/coreapi/protos/third_party/google/api/__init__.py
diff --git a/xos/grpc/protos/third_party/google/api/annotations.proto b/xos/coreapi/protos/third_party/google/api/annotations.proto
similarity index 100%
rename from xos/grpc/protos/third_party/google/api/annotations.proto
rename to xos/coreapi/protos/third_party/google/api/annotations.proto
diff --git a/xos/grpc/protos/third_party/google/api/http.proto b/xos/coreapi/protos/third_party/google/api/http.proto
similarity index 100%
rename from xos/grpc/protos/third_party/google/api/http.proto
rename to xos/coreapi/protos/third_party/google/api/http.proto
diff --git a/xos/grpc/protos/utility.proto b/xos/coreapi/protos/utility.proto
similarity index 100%
rename from xos/grpc/protos/utility.proto
rename to xos/coreapi/protos/utility.proto
diff --git a/xos/grpc/protos/xosoptions.proto b/xos/coreapi/protos/xosoptions.proto
similarity index 100%
rename from xos/grpc/protos/xosoptions.proto
rename to xos/coreapi/protos/xosoptions.proto
diff --git a/xos/coreapi/reaper.py b/xos/coreapi/reaper.py
new file mode 100644
index 0000000..f7adb06
--- /dev/null
+++ b/xos/coreapi/reaper.py
@@ -0,0 +1,175 @@
+""" Reaper
+
+ The reaper implements permanent deletion of soft-deleted objects.
+
+ It does this by polling for soft-deleted objects. For each object, the
+ reaper checks to see if its cascade set is empty. If so, the object will
+ be purged. If it is non-empty, then the reaper will skip the object under
+ the assumption that it will eventually become empty.
+"""
+
+import os
+import sys
+import threading
+
+if __name__ == "__main__":
+ import django
+ sys.path.append('/opt/xos')
+ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "xos.settings")
+
+from datetime import datetime
+from django.db import reset_queries
+from django.db.models import F, Q
+from django.db.models.signals import post_save
+from django.db.transaction import atomic
+from django.dispatch import receiver
+from django.utils import timezone
+from django.db import models as django_models
+from core.models.plcorebase import XOSCollector
+from django.db import router
+from xos.logger import Logger, logging
+
+import pdb
+import time
+import traceback
+
+logger = Logger(level=logging.DEBUG)
+
+class ReaperThread(threading.Thread):
+ daemon = True
+ interval = 5
+
+ def __init__(self, *args, **kwargs):
+ self.terminate_signal = False
+ super(ReaperThread, self).__init__(*args, **kwargs)
+
+ def check_db_connection_okay(self):
+ # django implodes if the database connection is closed by docker-compose
+ from django import db
+ try:
+ db.connection.cursor()
+ #diag = Diag.objects.filter(name="foo").first()
+ except Exception, e:
+ if "connection already closed" in traceback.format_exc():
+ logger.error("XXX connection already closed")
+ try:
+ # if db.connection:
+ # db.connection.close()
+ db.close_old_connections()
+ except:
+ logger.log_exc("XXX we failed to fix the failure")
+ else:
+ logger.log_exc("XXX some other error")
+
+ def journal_object(self, o, operation, msg=None, timestamp=None):
+ # not implemented at this time
+ pass
+
+ def get_cascade_set(self, m):
+ """ Get the set of objects that would cascade if this object was
+ deleted.
+ """
+ collector = XOSCollector(using=router.db_for_write(m.__class__, instance=m))
+ collector.collect([m])
+ deps=[]
+ for (k, models) in collector.data.items():
+ for model in models:
+ if model==m:
+ # collector will return ourself; ignore it.
+ continue
+ if issubclass(m.__class__, model.__class__):
+ # collector will return our parent classes; ignore them.
+ continue
+ # We don't actually need this check, as with multiple passes the reaper can
+ # clean up a hierarchy of objects.
+ # if getattr(model, "backend_need_reap", False):
+ # # model is already marked for reaping; ignore it.
+ # continue
+ deps.append(model)
+ return deps
+
+ def run_reaper_once(self):
+ objects = []
+ deleted_objects = []
+
+ # logger.debug("REAPER: run_reaper_once()")
+
+ self.check_db_connection_okay()
+
+ # Reap non-sync'd models here
+ # models_to_reap = [Slice,Network,NetworkSlice]
+
+ models_to_reap = django_models.get_models(include_auto_created=False)
+ for m in models_to_reap:
+ if not hasattr(m, "deleted_objects"):
+ continue
+
+ dobjs = m.deleted_objects.all()
+ for d in dobjs:
+ if hasattr(d,"_meta") and hasattr(d._meta,"proxy") and d._meta.proxy:
+ # skip proxy objects; we'll get the base instead
+ continue
+
+ if (not getattr(d, "backend_need_reap", False)) and getattr(d, "backend_need_delete", False):
+ self.journal_object(d, "reaper.need_delete")
+ logger.info("REAPER: skipping %r because it has need_delete set" % d)
+ continue
+
+ cascade_set = self.get_cascade_set(d)
+ if cascade_set:
+ self.journal_object(d, "reaper.cascade_set", msg=",".join([str(m) for m in cascade_set]))
+ logger.info('REAPER: cannot purge object %r because its cascade_set is nonempty: %s' % (d, ",".join([str(m) for m in cascade_set])))
+ continue
+
+# XXX I don't think we need dependency_walker here anymore,
+# XXX since the cascade set would include any inverse
+# XXX dependencies automatically.
+# deps = walk_inv_deps(noop, d)
+# if (not deps):
+
+ if (True):
+ self.journal_object(d, "reaper.purge")
+ logger.info('REAPER: purging object %r'%d)
+ try:
+ d.delete(purge=True)
+ except:
+ self.journal_object(d, "reaper.purge.exception")
+ logger.error('REAPER: exception purging object %r'%d)
+ traceback.print_exc()
+ try:
+ reset_queries()
+ except:
+ # this shouldn't happen, but in case it does, catch it...
+ logger.log_exc("REAPER: exception in reset_queries")
+
+ # logger.debug("REAPER: finished run_reaper_once()")
+
+ def run(self):
+ while (not self.terminate_signal):
+ start = time.time()
+ try:
+ self.run_reaper_once()
+ except:
+ logger.log_exc("REAPER: Exception in run loop")
+
+ telap = time.time()-start
+ if telap<self.interval:
+ time.sleep(self.interval - telap)
+
+ def stop(self):
+ self.terminate_signal = True
+
+if __name__ == '__main__':
+ django.setup()
+
+ reaper = ReaperThread()
+ reaper.start()
+
+ import time
+ _ONE_DAY_IN_SECONDS = 60 * 60 * 24
+ try:
+ while 1:
+ time.sleep(_ONE_DAY_IN_SECONDS)
+ except KeyboardInterrupt:
+ reaper.stop()
+
diff --git a/xos/coreapi/start_coreapi.sh b/xos/coreapi/start_coreapi.sh
new file mode 100755
index 0000000..077b76a
--- /dev/null
+++ b/xos/coreapi/start_coreapi.sh
@@ -0,0 +1,6 @@
+cd protos
+make rebuild-protos
+make
+cd ..
+source env.sh
+python ./core_main.py
diff --git a/xos/grpc/start_grpc_server.sh b/xos/coreapi/start_grpc_server.sh
similarity index 100%
rename from xos/grpc/start_grpc_server.sh
rename to xos/coreapi/start_grpc_server.sh
diff --git a/xos/grpc/tests/api_user_crud.py b/xos/coreapi/tests/api_user_crud.py
similarity index 100%
rename from xos/grpc/tests/api_user_crud.py
rename to xos/coreapi/tests/api_user_crud.py
diff --git a/xos/grpc/tests/cham_slice_crud.sh b/xos/coreapi/tests/cham_slice_crud.sh
similarity index 100%
rename from xos/grpc/tests/cham_slice_crud.sh
rename to xos/coreapi/tests/cham_slice_crud.sh
diff --git a/xos/grpc/tests/orm_user_crud.py b/xos/coreapi/tests/orm_user_crud.py
similarity index 100%
rename from xos/grpc/tests/orm_user_crud.py
rename to xos/coreapi/tests/orm_user_crud.py
diff --git a/xos/grpc/tests/testconfig-chameleon.sh b/xos/coreapi/tests/testconfig-chameleon.sh
similarity index 100%
rename from xos/grpc/tests/testconfig-chameleon.sh
rename to xos/coreapi/tests/testconfig-chameleon.sh
diff --git a/xos/grpc/tests/testconfig.py b/xos/coreapi/tests/testconfig.py
similarity index 100%
rename from xos/grpc/tests/testconfig.py
rename to xos/coreapi/tests/testconfig.py
diff --git a/xos/grpc/tests/tosca.py b/xos/coreapi/tests/tosca.py
similarity index 100%
rename from xos/grpc/tests/tosca.py
rename to xos/coreapi/tests/tosca.py
diff --git a/xos/grpc/xos_grpc_api.py b/xos/coreapi/xos_grpc_api.py
similarity index 100%
rename from xos/grpc/xos_grpc_api.py
rename to xos/coreapi/xos_grpc_api.py
diff --git a/xos/grpc/xos_modeldefs_api.py b/xos/coreapi/xos_modeldefs_api.py
similarity index 100%
rename from xos/grpc/xos_modeldefs_api.py
rename to xos/coreapi/xos_modeldefs_api.py
diff --git a/xos/grpc/xos_utility_api.py b/xos/coreapi/xos_utility_api.py
similarity index 100%
rename from xos/grpc/xos_utility_api.py
rename to xos/coreapi/xos_utility_api.py
diff --git a/xos/synchronizers/model_policy.py b/xos/synchronizers/model_policy.py
index d6145da..d79ed81 100644
--- a/xos/synchronizers/model_policy.py
+++ b/xos/synchronizers/model_policy.py
@@ -145,30 +145,6 @@
if (time.time()-start<1):
time.sleep(1)
-from core.models.plcorebase import XOSCollector
-from django.db import router
-def has_deleted_dependencies(m):
- # Check to see if 'm' would cascade to any objects that have the 'deleted'
- # field set in them.
- collector = XOSCollector(using=router.db_for_write(m.__class__, instance=m))
- collector.collect([m])
- deps=[]
- for (k, models) in collector.data.items():
- for model in models:
- if model==m:
- # collector will return ourself; ignore it.
- continue
- if issubclass(m.__class__, model.__class__):
- # collector will return our parent classes; ignore them.
- continue
-# We don't actually need this check, as with multiple passes the reaper can
-# clean up a hierarchy of objects.
-# if getattr(model, "backend_need_reap", False):
-# # model is already marked for reaping; ignore it.
-# continue
- deps.append(model)
- return deps
-
def run_policy_once():
from core.models import Instance,Slice,Controller,Network,User,SlicePrivilege,Site,SitePrivilege,Image,ControllerSlice,ControllerUser,ControllerSite
models = [Controller, Site, SitePrivilege, Image, ControllerSlice, ControllerSite, ControllerUser, User, Slice, Network, Instance, SlicePrivilege]
@@ -191,40 +167,6 @@
for o in deleted_objects:
execute_model_policy(o, True)
- # Reap non-sync'd models here
- # models_to_reap = [Slice,Network,NetworkSlice]
-
- models_to_reap = django_models.get_models(include_auto_created=False)
- for m in models_to_reap:
- if not hasattr(m, "deleted_objects"):
- continue
-
- dobjs = m.deleted_objects.all()
- for d in dobjs:
- if hasattr(d,"_meta") and hasattr(d._meta,"proxy") and d._meta.proxy:
- # skip proxy objects; we'll get the base instead
- continue
- if (not getattr(d, "backend_need_reap", False)) and getattr(d, "backend_need_delete", False):
- journal_object(d, "reaper.need_delete")
- print "Reaper: skipping %r because it has need_delete set" % d
- continue
- deleted_deps = has_deleted_dependencies(d)
- if deleted_deps:
- journal_object(d, "reaper.has_deleted_dependencies", msg=",".join([str(m) for m in deleted_deps]))
- print 'Reaper: cannot purge object %r because it has deleted dependencies: %s' % (d, ",".join([str(m) for m in deleted_deps]))
- continue
- deps = walk_inv_deps(noop, d)
- if (not deps):
- journal_object(d, "reaper.purge")
- print 'Reaper: purging object %r'%d
- try:
- d.delete(purge=True)
- except:
- journal_object(d, "reaper.purge.exception")
- print 'Reaper: exception purging object %r'%d
- traceback.print_exc()
-
-
try:
reset_queries()
except:
diff --git a/xos/synchronizers/new_base/modelaccessor.py b/xos/synchronizers/new_base/modelaccessor.py
index 62475bc..d4b9ace 100644
--- a/xos/synchronizers/new_base/modelaccessor.py
+++ b/xos/synchronizers/new_base/modelaccessor.py
@@ -73,7 +73,7 @@
""" returns True if obj is of model type "name" or is a descendant """
raise Exception("Not Implemented")
- def journal_object(o, operation, msg=None, timestamp=None):
+ def journal_object(self, o, operation, msg=None, timestamp=None):
pass
def import_models_to_globals():
diff --git a/xos/synchronizers/onboarding/xosbuilder.py b/xos/synchronizers/onboarding/xosbuilder.py
index 31d9a5f..2a34f27 100644
--- a/xos/synchronizers/onboarding/xosbuilder.py
+++ b/xos/synchronizers/onboarding/xosbuilder.py
@@ -349,7 +349,7 @@
containers["xos_core"] = {
"image": "xosproject/xos-ui",
- "command": 'bash -c "cd grpc; bash ./start_grpc_server.sh"',
+ "command": 'bash -c "cd coreapi; bash ./start_coreapi.sh"',
"networks": networks,
"ports": {"50055": "50055", "50051" : "50051"},
"external_links": external_links,