Merge branch 'master' of ssh://git.planet-lab.org/git/plstackapi
diff --git a/planetstack/hpc_observer/fsck.py b/planetstack/hpc_observer/fsck.py
new file mode 100644
index 0000000..82258a7
--- /dev/null
+++ b/planetstack/hpc_observer/fsck.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+import argparse
+import imp
+import inspect
+import os
+import sys
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "planetstack.settings")
+sys.path.append("/opt/planetstack")
+from planetstack.config import Config
+from util.logger import Logger, logging
+from observer.syncstep import SyncStep
+
+try:
+    from django import setup as django_setup # django 1.7
+except:
+    django_setup = False
+
+logger = Logger(level=logging.INFO)
+
+class PlanetStackConsistencyCheck:
+	def __init__(self):
+                self.sync_steps = []
+		self.load_sync_step_modules()
+
+	def load_sync_step_modules(self, step_dir=None):
+		if step_dir is None:
+			if hasattr(Config(), "observer_steps_dir"):
+				step_dir = Config().observer_steps_dir
+			else:
+				step_dir = "/opt/planetstack/observer/steps"
+
+		for fn in os.listdir(step_dir):
+			pathname = os.path.join(step_dir,fn)
+			if os.path.isfile(pathname) and fn.endswith(".py") and (fn!="__init__.py"):
+				module = imp.load_source(fn[:-3],pathname)
+				for classname in dir(module):
+					c = getattr(module, classname, None)
+
+					# make sure 'c' is a descendent of SyncStep and has a
+					# provides field (this eliminates the abstract base classes
+					# since they don't have a provides)
+
+					if inspect.isclass(c) and issubclass(c, SyncStep) and hasattr(c,"provides") and (c not in self.sync_steps):
+						self.sync_steps.append(c)
+		logger.info('loaded sync steps: %s' % ",".join([x.__name__ for x in self.sync_steps]))
+
+        def run(self):
+            updated = True
+            while updated:
+                updated = False
+
+                for step in self.sync_steps:
+                    if hasattr(step, "consistency_check"):
+                        updated = updated or step(driver=None).consistency_check()
+
+                if updated:
+                    logger.info('re-running consistency checks because something changed')
+
+def main():
+    if not "-C" in sys.argv:
+        print >> sys.stderr, "You probably wanted to use -C /opt/planetstack/hpc_observer/hpc_observer_config"
+
+    # Generate command line parser
+    parser = argparse.ArgumentParser(usage='%(prog)s [options]')
+    # smbaker: util/config.py parses sys.argv[] directly to get config file name; include the option here to avoid
+    #   throwing unrecognized argument exceptions
+    parser.add_argument('-C', '--config', dest='config_file', action='store', default="/opt/planetstack/plstackapi_config",
+                        help='Name of config file.')
+    args = parser.parse_args()
+
+    if django_setup: # 1.7
+        django_setup()
+
+    cc = PlanetStackConsistencyCheck()
+    cc.run()
+
+if __name__ == '__main__':
+    main()
+
diff --git a/planetstack/hpc_observer/steps/sync_cdnprefix.py b/planetstack/hpc_observer/steps/sync_cdnprefix.py
index 55b2326..dfa6ef8 100644
--- a/planetstack/hpc_observer/steps/sync_cdnprefix.py
+++ b/planetstack/hpc_observer/steps/sync_cdnprefix.py
@@ -25,11 +25,14 @@
         HpcLibrary.__init__(self)
 
     def fetch_pending(self, deleted):
-        self.sanity_check()
+        #self.consistency_check()
 
         return SyncStep.fetch_pending(self, deleted)
 
-    def sanity_check(self):
+    def consistency_check(self):
+        # set to true if something changed
+        result=False
+
         # sanity check to make sure our PS objects have CMI objects behind them
         all_p_ids = [x["cdn_prefix_id"] for x in self.client.onev.ListAll("CDNPrefix")]
 
@@ -48,10 +51,14 @@
                 logger.info("CDN Prefix %s was not found on CMI" % p.cdn_prefix_id)
                 p.cdn_prefix_id=None
                 p.save()
+                result = True
 
             if (p.defaultOriginServer!=None) and (all_origins.get(p.cdn_prefix_id,None) != p.defaultOriginServer.url):
                 logger.info("CDN Prefix %s does not have default origin server on CMI" % str(p))
-                p.save() # this will set updated>enacted and force run
+                p.save() # this will set updated>enacted and force observer to re-sync
+                result = True
+
+        return result
 
     def sync_record(self, cp):
         logger.info("sync'ing cdn prefix %s" % str(cp))
diff --git a/planetstack/hpc_observer/steps/sync_contentprovider.py b/planetstack/hpc_observer/steps/sync_contentprovider.py
index cebe186..b3c2f66 100644
--- a/planetstack/hpc_observer/steps/sync_contentprovider.py
+++ b/planetstack/hpc_observer/steps/sync_contentprovider.py
@@ -25,11 +25,14 @@
         HpcLibrary.__init__(self)
 
     def fetch_pending(self, deleted):
-        self.sanity_check()
+        #self.consistency_check()
 
         return SyncStep.fetch_pending(self, deleted)
 
-    def sanity_check(self):
+    def consistency_check(self):
+        # set to true if something changed
+        result=False
+
         # sanity check to make sure our PS objects have CMI objects behind them
         all_cp_ids = [x["content_provider_id"] for x in self.client.onev.ListAll("ContentProvider")]
         for cp in ContentProvider.objects.all():
@@ -37,11 +40,13 @@
                 logger.info("Content provider %s was not found on CMI" % cp.content_provider_id)
                 cp.content_provider_id=None
                 cp.save()
+                result = True
+
+        return result
 
     def sync_record(self, cp):
         logger.info("sync'ing content provider %s" % str(cp))
         account_name = self.make_account_name(cp.name)
-        print "XXX", cp.name, account_name
 
         if (not cp.serviceProvider) or (not cp.serviceProvider.service_provider_id):
             return
diff --git a/planetstack/hpc_observer/steps/sync_originserver.py b/planetstack/hpc_observer/steps/sync_originserver.py
index 92ac0f4..d18c672 100644
--- a/planetstack/hpc_observer/steps/sync_originserver.py
+++ b/planetstack/hpc_observer/steps/sync_originserver.py
@@ -1,6 +1,7 @@
 import os
 import sys
 import base64
+
 from django.db.models import F, Q
 from planetstack.config import Config
 from observer.syncstep import SyncStep
@@ -25,11 +26,14 @@
         HpcLibrary.__init__(self)
 
     def fetch_pending(self, deleted):
-        self.sanity_check()
+        #self.consistency_check()
 
         return SyncStep.fetch_pending(self, deleted)
 
-    def sanity_check(self):
+    def consistency_check(self):
+        # set to true if something changed
+        result=False
+
         # sanity check to make sure our PS objects have CMI objects behind them
         all_ors_ids = [x["origin_server_id"] for x in self.client.onev.ListAll("OriginServer")]
         for ors in OriginServer.objects.all():
@@ -40,6 +44,9 @@
                 logger.info("origin server %s was not found on CMI" % ors.origin_server_id)
                 ors.origin_server_id=None
                 ors.save()
+                result = True
+
+        return result
 
     def sync_record(self, ors):
         logger.info("sync'ing origin server %s" % str(ors))
diff --git a/planetstack/hpc_observer/steps/sync_serviceprovider.py b/planetstack/hpc_observer/steps/sync_serviceprovider.py
index 78fd196..ca0ba6a 100644
--- a/planetstack/hpc_observer/steps/sync_serviceprovider.py
+++ b/planetstack/hpc_observer/steps/sync_serviceprovider.py
@@ -25,11 +25,14 @@
         HpcLibrary.__init__(self)
 
     def fetch_pending(self, deleted):
-        self.sanity_check()
+        #self.consistency_check()
 
         return SyncStep.fetch_pending(self, deleted)
 
-    def sanity_check(self):
+    def consistency_check(self):
+        # set to true if something changed
+        result=False
+
         # sanity check to make sure our PS objects have CMI objects behind them
         all_sp_ids = [x["service_provider_id"] for x in self.client.onev.ListAll("ServiceProvider")]
         for sp in ServiceProvider.objects.all():
@@ -37,11 +40,13 @@
                 logger.info("Service provider %s was not found on CMI" % sp.service_provider_id)
                 sp.service_provider_id=None
                 sp.save()
+                result = True
+
+        return result
 
     def sync_record(self, sp):
         logger.info("sync'ing service provider %s" % str(sp))
         account_name = self.make_account_name(sp.name)
-        print "XXX", sp.name, account_name
         sp_dict = {"account": account_name, "name": sp.name, "enabled": sp.enabled}
         if not sp.service_provider_id:
             id = self.client.onev.Create("ServiceProvider", sp_dict)