initial code migration from xos repo
Change-Id: I8c848929ec4583a7a18ba9da44095f8f688f96c0
diff --git a/xos/synchronizer/steps/garbage_collector.py b/xos/synchronizer/steps/garbage_collector.py
new file mode 100644
index 0000000..658f7a1
--- /dev/null
+++ b/xos/synchronizer/steps/garbage_collector.py
@@ -0,0 +1,67 @@
+import os
+import sys
+import base64
+import traceback
+from collections import defaultdict
+from django.db.models import F, Q
+from xos.config import Config
+from xos.logger import Logger, logging
+from synchronizers.base.syncstep import SyncStep
+from services.hpc.models import ServiceProvider, ContentProvider, CDNPrefix, OriginServer
+from core.models import *
+
+# hpclibrary will be in steps/..
+parentdir = os.path.join(os.path.dirname(__file__),"..")
+sys.path.insert(0,parentdir)
+
+from hpclib import HpcLibrary
+
+logger = Logger(level=logging.INFO)
+
+class GarbageCollector(SyncStep, HpcLibrary):
+# requested_interval = 86400
+ requested_interval = 0
+ provides=[]
+
+ def __init__(self, **args):
+ SyncStep.__init__(self, **args)
+ HpcLibrary.__init__(self)
+
+ def call(self, **args):
+ logger.info("running garbage collector")
+ try:
+ self.gc_originservers()
+ self.gc_cdnprefixes()
+ self.gc_contentproviders()
+ self.gc_serviceproviders()
+ except:
+ traceback.print_exc()
+
+ def gc_onev(self, ps_class, ps_idField, onev_className, onev_idField):
+ # get the CMI's objects
+ onev_objs = self.client.onev.ListAll(onev_className)
+
+ # get the data model's objects,
+ ps_objs = ps_class.objects.filter(enacted__isnull=False)
+ ps_ids = [str(getattr(x,ps_idField,None)) for x in ps_objs]
+
+ # for each onev object, if it's id does not exist in a data model
+ # object, then delete it.
+ for onev_obj in onev_objs:
+ onev_id = onev_obj[onev_idField]
+ if str(onev_id) not in ps_ids:
+ logger.info("garbage collecting %s %s" % (onev_className, str(onev_id)))
+ self.client.onev.Delete(onev_className, onev_id)
+
+ def gc_originservers(self):
+ self.gc_onev(OriginServer, "origin_server_id", "OriginServer", "origin_server_id")
+
+ def gc_cdnprefixes(self):
+ self.gc_onev(CDNPrefix, "cdn_prefix_id", "CDNPrefix", "cdn_prefix_id")
+
+ def gc_contentproviders(self):
+ self.gc_onev(ContentProvider, "content_provider_id", "ContentProvider", "content_provider_id")
+
+ def gc_serviceproviders(self):
+ self.gc_onev(ServiceProvider, "service_provider_id", "ServiceProvider", "service_provider_id")
+
diff --git a/xos/synchronizer/steps/sync_cdnprefix.py b/xos/synchronizer/steps/sync_cdnprefix.py
new file mode 100644
index 0000000..eff3b5d
--- /dev/null
+++ b/xos/synchronizer/steps/sync_cdnprefix.py
@@ -0,0 +1,101 @@
+import os
+import sys
+import base64
+from django.db.models import F, Q
+from xos.config import Config
+from synchronizers.base.syncstep import SyncStep
+from core.models import Service
+from services.hpc.models import ServiceProvider, ContentProvider, CDNPrefix
+from xos.logger import Logger, logging
+
+# hpclibrary will be in steps/..
+parentdir = os.path.join(os.path.dirname(__file__),"..")
+sys.path.insert(0,parentdir)
+
+from hpclib import HpcLibrary
+
+logger = Logger(level=logging.INFO)
+
+class SyncCDNPrefix(SyncStep, HpcLibrary):
+ provides=[CDNPrefix]
+ observes=CDNPrefix
+ requested_interval=0
+
+ def __init__(self, **args):
+ SyncStep.__init__(self, **args)
+ HpcLibrary.__init__(self)
+
+ def filter_hpc_service(self, objs):
+ hpcService = self.get_hpc_service()
+
+ return [x for x in objs if x.contentProvider.serviceProvider.hpcService == hpcService]
+
+ def fetch_pending(self, deleted):
+ #self.consistency_check()
+
+ return self.filter_hpc_service(SyncStep.fetch_pending(self, deleted))
+
+ def consistency_check(self):
+ # set to true if something changed
+ result=False
+
+ # sanity check to make sure our PS objects have CMI objects behind them
+ all_p_ids = [x["cdn_prefix_id"] for x in self.client.onev.ListAll("CDNPrefix")]
+
+ all_p_ids = []
+ all_origins = {}
+ for x in self.client.onev.ListAll("CDNPrefix"):
+ id = x["cdn_prefix_id"]
+ all_p_ids.append(id)
+ all_origins[id] = x.get("default_origin_server", None)
+
+ for p in CDNPrefix.objects.all():
+ if (p.cdn_prefix_id is None):
+ continue
+
+ if (p.cdn_prefix_id not in all_p_ids):
+ logger.info("CDN Prefix %s was not found on CMI" % p.cdn_prefix_id)
+ p.cdn_prefix_id=None
+ p.save()
+ result = True
+
+ if (p.defaultOriginServer!=None) and (all_origins.get(p.cdn_prefix_id,None) != p.defaultOriginServer.url):
+ logger.info("CDN Prefix %s does not have default origin server on CMI" % str(p))
+ p.save() # this will set updated>enacted and force observer to re-sync
+ result = True
+
+ return result
+
+ def sync_record(self, cp):
+ logger.info("sync'ing cdn prefix %s" % str(cp),extra=cp.tologdict())
+
+ if (not cp.contentProvider) or (not cp.contentProvider.content_provider_id):
+ raise Exception("CDN Prefix %s is linked to a contentProvider without an id" % str(cp))
+
+ cpid = cp.contentProvider.content_provider_id
+
+ cp_dict = {"service": "HyperCache", "enabled": cp.enabled, "content_provider_id": cpid, "cdn_prefix": cp.prefix}
+
+ if cp.defaultOriginServer and cp.defaultOriginServer.url:
+ if (not cp.defaultOriginServer.origin_server_id):
+ # It's probably a bad idea to try to set defaultOriginServer before
+ # we've crated defaultOriginServer.
+ raise Exception("cdn prefix %s is waiting for it's default origin server to get an id" % str(cp))
+
+ cp_dict["default_origin_server"] = cp.defaultOriginServer.url
+
+ #print cp_dict
+
+ if not cp.cdn_prefix_id:
+ id = self.client.onev.Create("CDNPrefix", cp_dict)
+ cp.cdn_prefix_id = id
+ else:
+ del cp_dict["content_provider_id"] # this can't be updated
+ del cp_dict["cdn_prefix"] # this can't be updated either
+ self.client.onev.Update("CDNPrefix", cp.cdn_prefix_id, cp_dict)
+
+ cp.save()
+
+ def delete_record(self, m):
+ if m.cdn_prefix_id is not None:
+ self.client.onev.Delete("CDNPrefix", m.cdn_prefix_id)
diff --git a/xos/synchronizer/steps/sync_contentprovider.py b/xos/synchronizer/steps/sync_contentprovider.py
new file mode 100644
index 0000000..3e30ed3
--- /dev/null
+++ b/xos/synchronizer/steps/sync_contentprovider.py
@@ -0,0 +1,78 @@
+import os
+import sys
+import base64
+from django.db.models import F, Q
+from xos.config import Config
+from synchronizers.base.syncstep import SyncStep
+from core.models import Service
+from services.hpc.models import ServiceProvider, ContentProvider
+from xos.logger import Logger, logging
+
+# hpclibrary will be in steps/..
+parentdir = os.path.join(os.path.dirname(__file__),"..")
+sys.path.insert(0,parentdir)
+
+from hpclib import HpcLibrary
+
+logger = Logger(level=logging.INFO)
+
+class SyncContentProvider(SyncStep, HpcLibrary):
+ provides=[ContentProvider]
+ observes=ContentProvider
+ requested_interval=0
+
+ def __init__(self, **args):
+ SyncStep.__init__(self, **args)
+ HpcLibrary.__init__(self)
+
+ def filter_hpc_service(self, objs):
+ hpcService = self.get_hpc_service()
+
+ return [x for x in objs if x.serviceProvider.hpcService == hpcService]
+
+ def fetch_pending(self, deleted):
+ #self.consistency_check()
+
+ return self.filter_hpc_service(SyncStep.fetch_pending(self, deleted))
+
+ def consistency_check(self):
+ # set to true if something changed
+ result=False
+
+ # sanity check to make sure our PS objects have CMI objects behind them
+ all_cp_ids = [x["content_provider_id"] for x in self.client.onev.ListAll("ContentProvider")]
+ for cp in ContentProvider.objects.all():
+ if (cp.content_provider_id is not None) and (cp.content_provider_id not in all_cp_ids):
+ logger.info("Content provider %s was not found on CMI" % cp.content_provider_id)
+ cp.content_provider_id=None
+ cp.save()
+ result = True
+
+ return result
+
+ def sync_record(self, cp):
+ logger.info("sync'ing content provider %s" % str(cp), extra=cp.tologdict())
+ account_name = self.make_account_name(cp.name)
+
+ if (not cp.serviceProvider) or (not cp.serviceProvider.service_provider_id):
+ raise Exception("ContentProvider %s is linked to a serviceProvider with no id" % str(cp))
+
+ spid = cp.serviceProvider.service_provider_id
+
+ cp_dict = {"account": account_name, "name": cp.name, "enabled": cp.enabled}
+
+ #print cp_dict
+
+ if not cp.content_provider_id:
+ cp_dict["service_provider_id"] = spid
+ id = self.client.onev.Create("ContentProvider", cp_dict)
+ cp.content_provider_id = id
+ else:
+ self.client.onev.Update("ContentProvider", cp.content_provider_id, cp_dict)
+
+ cp.save()
+
+ def delete_record(self, m):
+ if m.content_provider_id is not None:
+ self.client.onev.Delete("ContentProvider", m.content_provider_id)
+
diff --git a/xos/synchronizer/steps/sync_hpcservices.py b/xos/synchronizer/steps/sync_hpcservices.py
new file mode 100644
index 0000000..63bf19b
--- /dev/null
+++ b/xos/synchronizer/steps/sync_hpcservices.py
@@ -0,0 +1,43 @@
+import os
+import sys
+import base64
+from django.db.models import F, Q
+from xos.config import Config
+from synchronizers.base.syncstep import SyncStep
+from core.models import Service
+from services.hpc.models import HpcService
+from services.requestrouter.models import RequestRouterService
+from xos.logger import Logger, logging
+
+# hpclibrary will be in steps/..
+parentdir = os.path.join(os.path.dirname(__file__),"..")
+sys.path.insert(0,parentdir)
+
+from hpclib import HpcLibrary
+
+logger = Logger(level=logging.INFO)
+
+class SyncHpcService(SyncStep, HpcLibrary):
+ provides=[HpcService]
+ observes=HpcService
+ requested_interval=0
+
+ def __init__(self, **args):
+ SyncStep.__init__(self, **args)
+ HpcLibrary.__init__(self)
+
+ def filter_hpc_service(self, objs):
+ hpcService = self.get_hpc_service()
+
+ return [x for x in objs if x == hpcService]
+
+ def fetch_pending(self, deleted):
+ # Looks like deletion is not supported for this object - Sapan
+ if (deleted):
+ return []
+ else:
+ return self.filter_hpc_service(HpcService.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None)))
+
+ def sync_record(self, hpc_service):
+ logger.info("sync'ing hpc_service %s" % str(hpc_service),extra=hpc_service.tologdict())
+ hpc_service.save()
diff --git a/xos/synchronizer/steps/sync_originserver.py b/xos/synchronizer/steps/sync_originserver.py
new file mode 100644
index 0000000..bd5b227
--- /dev/null
+++ b/xos/synchronizer/steps/sync_originserver.py
@@ -0,0 +1,92 @@
+import os
+import sys
+import base64
+
+from django.db.models import F, Q
+from xos.config import Config
+from synchronizers.base.syncstep import SyncStep
+from core.models import Service
+from services.hpc.models import ServiceProvider, ContentProvider, CDNPrefix, OriginServer
+from xos.logger import Logger, logging
+
+# hpclibrary will be in steps/..
+parentdir = os.path.join(os.path.dirname(__file__),"..")
+sys.path.insert(0,parentdir)
+
+from hpclib import HpcLibrary
+
+logger = Logger(level=logging.INFO)
+
+class SyncOriginServer(SyncStep, HpcLibrary):
+ provides=[OriginServer]
+ observes=OriginServer
+ requested_interval=0
+
+ def __init__(self, **args):
+ SyncStep.__init__(self, **args)
+ HpcLibrary.__init__(self)
+
+ def filter_hpc_service(self, objs):
+ hpcService = self.get_hpc_service()
+
+ return [x for x in objs if x.contentProvider.serviceProvider.hpcService == hpcService]
+
+ def fetch_pending(self, deleted):
+ #self.consistency_check()
+
+ return self.filter_hpc_service(SyncStep.fetch_pending(self, deleted))
+
+ def consistency_check(self):
+ # set to true if something changed
+ result=False
+
+ # sanity check to make sure our PS objects have CMI objects behind them
+ all_ors_ids = [x["origin_server_id"] for x in self.client.onev.ListAll("OriginServer")]
+ for ors in OriginServer.objects.all():
+ if (ors.origin_server_id is not None) and (ors.origin_server_id not in all_ors_ids):
+ # we have an origin server ID, but it doesn't exist in the CMI
+ # something went wrong
+ # start over
+ logger.info("origin server %s was not found on CMI" % ors.origin_server_id)
+ ors.origin_server_id=None
+ ors.save()
+ result = True
+
+ return result
+
+ def sync_record(self, ors):
+ logger.info("sync'ing origin server %s" % str(ors),extra=ors.tologdict())
+
+ if (not ors.contentProvider) or (not ors.contentProvider.content_provider_id):
+ raise Exception("Origin Server %s is linked to a contentProvider with no id" % str(ors))
+
+ cpid = ors.contentProvider.content_provider_id
+
+ # validation requires URL start with http://
+ url = ors.url
+ if not url.startswith("http://"):
+ url = "http://" + url
+
+ ors_dict = {"authenticated_content": ors.authenticated, "zone_redirects": ors.redirects, "content_provider_id": cpid, "url": url, "service_type": "HyperCache", "caching_type": "Optimistic", "description": ors.description}
+ if not ors_dict["description"]:
+ ors_dict["description"] = "blank"
+
+ #print os_dict
+
+ if not ors.origin_server_id:
+ id = self.client.onev.Create("OriginServer", ors_dict)
+ ors.origin_server_id = id
+ else:
+ self.client.onev.Update("OriginServer", ors.origin_server_id, ors_dict)
+
+ # ... something breaks (analytics) if the URL starts with http://, so we
+ # change it in cob after we added it via onev.
+ url = url[7:]
+ self.client.cob.UpdateContent(ors.origin_server_id, {"url": url})
+
+ ors.silent = True
+ ors.save()
+
+ def delete_record(self, m):
+ if m.origin_server_id is not None:
+ self.client.onev.Delete("OriginServer", m.origin_server_id)
diff --git a/xos/synchronizer/steps/sync_serviceprovider.py b/xos/synchronizer/steps/sync_serviceprovider.py
new file mode 100644
index 0000000..af6d685
--- /dev/null
+++ b/xos/synchronizer/steps/sync_serviceprovider.py
@@ -0,0 +1,67 @@
+import os
+import sys
+import base64
+from django.db.models import F, Q
+from xos.config import Config
+from synchronizers.base.syncstep import SyncStep
+from core.models import Service
+from services.hpc.models import ServiceProvider
+from xos.logger import Logger, logging
+
+# hpclibrary will be in steps/..
+parentdir = os.path.join(os.path.dirname(__file__),"..")
+sys.path.insert(0,parentdir)
+
+from hpclib import HpcLibrary
+
+logger = Logger(level=logging.INFO)
+
+class SyncServiceProvider(SyncStep, HpcLibrary):
+ provides=[ServiceProvider]
+ observes=ServiceProvider
+ requested_interval=0
+
+ def __init__(self, **args):
+ SyncStep.__init__(self, **args)
+ HpcLibrary.__init__(self)
+
+ def filter_hpc_service(self, objs):
+ hpcService = self.get_hpc_service()
+
+ return [x for x in objs if x.hpcService == hpcService]
+
+ def fetch_pending(self, deleted):
+ #self.consistency_check()
+
+ return self.filter_hpc_service(SyncStep.fetch_pending(self, deleted))
+
+ def consistency_check(self):
+ # set to true if something changed
+ result=False
+
+ # sanity check to make sure our PS objects have CMI objects behind them
+ all_sp_ids = [x["service_provider_id"] for x in self.client.onev.ListAll("ServiceProvider")]
+ for sp in ServiceProvider.objects.all():
+ if (sp.service_provider_id is not None) and (sp.service_provider_id not in all_sp_ids):
+ logger.info("Service provider %s was not found on CMI" % sp.service_provider_id)
+ sp.service_provider_id=None
+ sp.save()
+ result = True
+
+ return result
+
+ def sync_record(self, sp):
+ logger.info("sync'ing service provider %s" % str(sp),extra=sp.tologdict())
+ account_name = self.make_account_name(sp.name)
+ sp_dict = {"account": account_name, "name": sp.name, "enabled": sp.enabled}
+ if not sp.service_provider_id:
+ id = self.client.onev.Create("ServiceProvider", sp_dict)
+ sp.service_provider_id = id
+ else:
+ self.client.onev.Update("ServiceProvider", sp.service_provider_id, sp_dict)
+
+ sp.save()
+
+ def delete_record(self, m):
+ if m.service_provider_id is not None:
+ self.client.onev.Delete("ServiceProvider", m.service_provider_id)
diff --git a/xos/synchronizer/steps/sync_sitemap.py b/xos/synchronizer/steps/sync_sitemap.py
new file mode 100644
index 0000000..a1d177b
--- /dev/null
+++ b/xos/synchronizer/steps/sync_sitemap.py
@@ -0,0 +1,118 @@
+import os
+import sys
+import base64
+from django.db.models import F, Q
+from xos.config import Config
+from synchronizers.base.syncstep import SyncStep
+from core.models import Service
+from services.hpc.models import ServiceProvider, ContentProvider, CDNPrefix, SiteMap
+from xos.logger import Logger, logging
+
+# hpclibrary will be in steps/..
+parentdir = os.path.join(os.path.dirname(__file__),"..")
+sys.path.insert(0,parentdir)
+
+from hpclib import HpcLibrary
+
+logger = Logger(level=logging.INFO)
+
+class SyncSiteMap(SyncStep, HpcLibrary):
+ provides=[SiteMap]
+ observes=SiteMap
+ requested_interval=0
+
+ def __init__(self, **args):
+ SyncStep.__init__(self, **args)
+ HpcLibrary.__init__(self)
+
+ def filter_hpc_service(self, objs):
+ hpcService = self.get_hpc_service()
+
+ filtered_objs = []
+ for x in objs:
+ if ((x.hpcService == hpcService) or
+ ((x.serviceProvider != None) and (x.serviceProvider.hpcService == hpcService)) or
+ ((x.contentProvider != None) and (x.contentProvider.serviceProvider.hpcService == hpcService)) or
+ ((x.cdnPrefix!= None) and (x.cdnPrefix.contentProvider.serviceProvider.hpcService == hpcService))):
+ filtered_objs.append(x)
+
+ return filtered_objs
+
+ def fetch_pending(self, deleted):
+ return self.filter_hpc_service(SyncStep.fetch_pending(self, deleted))
+
+ def consistency_check(self):
+ # set to true if something changed
+ result=False
+
+ # sanity check to make sure our PS objects have CMI objects behind them
+ all_map_ids = [x["map_id"] for x in self.client.onev.ListAll("Map")]
+ for map in SiteMap.objects.all():
+ if (map.map_id is not None) and (map.map_id not in all_map_ids):
+ logger.info("Map %s was not found on CMI" % map.map_id,extra=map.tologdict())
+ map.map_id=None
+ map.save()
+ result = True
+
+ return result
+
+ def update_bind(self, map, map_dict, field_name, to_name, ids):
+ for id in ids:
+ if (not id in map_dict.get(field_name, [])):
+ print "Bind Map", map.map_id, "to", to_name, id
+ self.client.onev.Bind("Map", map.map_id, to_name, id)
+
+ for id in map_dict.get(field_name, []):
+ if (not id in ids):
+ print "Unbind Map", map.map_id, "from", to_name, id
+ self.client.onev.UnBind("map", map.map_id, to_name, id)
+
+ def sync_record(self, map):
+ logger.info("sync'ing SiteMap %s" % str(map),extra=map.tologdict())
+
+ if not map.map:
+ # no contents
+ return
+
+ content = map.map.read()
+
+ map_dict = {"name": map.name, "type": "site", "content": content}
+
+ cdn_prefix_ids=[]
+ service_provider_ids=[]
+ content_provider_ids=[]
+
+ if (map.contentProvider):
+ if not map.contentProvider.content_provider_id:
+ raise Exception("Map %s links to a contentProvider with no id" % map.name)
+ conent_provider_ids = [map.contentProvider.content_provider_id]
+
+ if (map.serviceProvider):
+ if not map.serviceProvider.service_provider_id:
+ raise Exception("Map %s links to a serviceProvider with no id" % map.name)
+ service_provider_ids = [map.serviceProvider.service_provider_id]
+
+ if (map.cdnPrefix):
+ if not map.cdnPrefix.cdn_prefix_id:
+ raise Exception("Map %s links to a cdnPrefix with no id" % map.name)
+ cdn_prefix_ids = [map.cdnPrefix.cdn_prefix_id]
+
+ if not map.map_id:
+ print "Create Map", map_dict
+ id = self.client.onev.Create("Map", map_dict)
+ map.map_id = id
+ else:
+ print "Update Map", map_dict
+ # these things we probably cannot update
+ del map_dict["name"]
+ self.client.onev.Update("Map", map.map_id, map_dict)
+
+ cmi_map_dict = self.client.onev.Read("Map", map.map_id)
+
+ self.update_bind(map, cmi_map_dict, "cdn_prefix_ids", "CDNPrefix", cdn_prefix_ids)
+
+ map.save()
+
+ def delete_record(self, m):
+ if m.map_id is not None:
+ self.client.onev.Delete("Map", m.map_id)