Merge branch 'master' of git.planet-lab.org:/git/plstackapi
diff --git a/planetstack/hpc_observer/hpc_observer_config b/planetstack/hpc_observer/hpc_observer_config
index 13bfaaf..30c9a8d 100644
--- a/planetstack/hpc_observer/hpc_observer_config
+++ b/planetstack/hpc_observer/hpc_observer_config
@@ -25,7 +25,8 @@
 dependency_graph=/opt/planetstack/hpc_observer/model-deps
 steps_dir=/opt/planetstack/hpc_observer/steps
 deleters_dir=/opt/planetstack/hpc_ovserver/deleters
-log_file=/var/log/hpc.log
+log_file=console
+#/var/log/hpc.log
 driver=None
 
 [feefie]
diff --git a/planetstack/hpc_observer/hpclib.py b/planetstack/hpc_observer/hpclib.py
index e91a468..92e87fe 100644
--- a/planetstack/hpc_observer/hpclib.py
+++ b/planetstack/hpc_observer/hpclib.py
@@ -1,6 +1,8 @@
 import os
 import base64
+import string
 import sys
+import xmlrpclib
 
 if __name__ == '__main__':
     sys.path.append("/opt/planetstack")
@@ -14,7 +16,55 @@
 
 logger = Logger(level=logging.INFO)
 
+class APIHelper:
+    def __init__(self, proxy, auth, method=None):
+        self.proxy = proxy
+        self.auth = auth
+        self.method = method
+
+    def __getattr__(self, name):
+        if name.startswith("_"):
+            return getattr(self, name)
+        else:
+            return APIHelper(self.proxy, self.auth, name)
+
+    def __call__(self, *args):
+        method = getattr(self.proxy, self.method)
+        return method(self.auth, *args)
+
+class CmiClient:
+    def __init__(self, hostname, port=8003, username="apiuser", password="apiuser"):
+        self.connect_api(hostname, port, username, password)
+
+    def connect_api(self, hostname, port=8003, username="apiuser", password="apiuser"):
+        #print "https://%s:%d/COAPI/" % (hostname, port)
+        cob = xmlrpclib.ServerProxy("https://%s:%d/COAPI/" % (hostname, port), allow_none=True)
+        cob_auth = {}
+        cob_auth["Username"] = username
+        cob_auth["AuthString"] = password
+        cob_auth["AuthMethod"] = "password"
+
+        onev = xmlrpclib.ServerProxy("https://%s:%d/ONEV_API/" % (hostname, port), allow_none=True)
+        onev_auth = {}
+        onev_auth["Username"] = username
+        onev_auth["AuthString"] = password
+        onev_auth["AuthMethod"] = "password"
+
+        self.cob = APIHelper(cob, cob_auth)
+        self.onev = APIHelper(onev, onev_auth)
+
 class HpcLibrary:
+    def __init__(self):
+        self._client = None
+
+    def make_account_name(self, x):
+        x=x.lower()
+        y = ""
+        for c in x:
+            if (c in (string.lowercase + string.digits)):
+                y = y + c
+        return y[:20]
+
     def extract_slice_info(self, service):
         """ Produce a dict that describes the slices for the CMI
 
@@ -62,14 +112,25 @@
 
         return mapping
 
-    def write_slices_file(self, hpc_service, rr_service):
+    def get_cmi_hostname(self, hpc_service=None):
+        if (hpc_service is None):
+            hpc_service = HpcService.objects.get()
+
+        slice_info = self.extract_slice_info(hpc_service)
+        return slice_info["hostname_cmi"]
+
+    def write_slices_file(self, hpc_service=None, rr_service=None):
+        if (hpc_service is None):
+            hpc_service = HpcService.objects.get()
+
+        if (rr_service is None):
+            rr_service = RequestRouterService.objects.get()
+
         mapping = self.extract_slice_info(hpc_service)
         rr_mapping = self.extract_slice_info(rr_service)
 
         mapping.update(rr_mapping)
 
-        print mapping
-
         fn = "/tmp/slices"
 
         f = open(fn, "w")
@@ -94,10 +155,21 @@
 PUPPET_MASTER_PORT="8140"
 """ % mapping)
 
+    @property
+    def client(self):
+        if self._client is None:
+            self._client = CmiClient(self.get_cmi_hostname())
+        return self._client
+
 if __name__ == '__main__':
-    hpc_service = HpcService.objects.get()
-    rr_service = RequestRouterService.objects.get()
+    print "testing write_slices_file"
     lib = HpcLibrary()
-    lib.write_slices_file(hpc_service, rr_service)
+    lib.write_slices_file()
+
+    print "testing API connection"
+    lib.client.cob.GetNewObjects()
+    lib.client.onev.ListAll("CDN")
+
+
 
 
diff --git a/planetstack/hpc_observer/steps/sync_cdnprefix.py b/planetstack/hpc_observer/steps/sync_cdnprefix.py
new file mode 100644
index 0000000..01bb687
--- /dev/null
+++ b/planetstack/hpc_observer/steps/sync_cdnprefix.py
@@ -0,0 +1,48 @@
+import os
+import sys
+import base64
+from django.db.models import F, Q
+from planetstack.config import Config
+from observer.syncstep import SyncStep
+from core.models import Service
+from hpc.models import ServiceProvider, ContentProvider, CDNPrefix
+from util.logger import Logger, logging
+
+# hpclibrary will be in steps/..
+parentdir = os.path.join(os.path.dirname(__file__),"..")
+sys.path.insert(0,parentdir)

+
+from hpclib import HpcLibrary
+
+logger = Logger(level=logging.INFO)
+
+class SyncCDNPrefix(SyncStep, HpcLibrary):
+    provides=[CDNPrefix]
+    requested_interval=0
+
+    def __init__(self, **args):
+        SyncStep.__init__(self, **args)
+        HpcLibrary.__init__(self)
+
+    def fetch_pending(self):
+        return CDNPrefix.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
+
+    def sync_record(self, cp):
+        logger.info("sync'ing cdn prefix %s" % str(cp))
+
+        if (not cp.contentProvider) or (not cp.contentProvider.content_provider_id):
+            return
+
+        cpid = cp.contentProvider.content_provider_id
+
+        cp_dict = {"service": "HyperCache", "enabled": cp.enabled, "content_provider_id": cpid, "cdn_prefix": cp.prefix}
+
+        #print cp_dict
+
+        if not cp.cdn_prefix_id:
+            id = self.client.onev.Create("CDNPrefix", cp_dict)
+            cp.cdn_prefix_id = id
+        else:
+            self.client.onev.Update("CDNPrefix", cp.cdn_prefix_id, cp_dict)
+
+        cp.save()
diff --git a/planetstack/hpc_observer/steps/sync_contentprovider.py b/planetstack/hpc_observer/steps/sync_contentprovider.py
new file mode 100644
index 0000000..00117b8
--- /dev/null
+++ b/planetstack/hpc_observer/steps/sync_contentprovider.py
@@ -0,0 +1,50 @@
+import os
+import sys
+import base64
+from django.db.models import F, Q
+from planetstack.config import Config
+from observer.syncstep import SyncStep
+from core.models import Service
+from hpc.models import ServiceProvider, ContentProvider
+from util.logger import Logger, logging
+
+# hpclibrary will be in steps/..
+parentdir = os.path.join(os.path.dirname(__file__),"..")
+sys.path.insert(0,parentdir)

+
+from hpclib import HpcLibrary
+
+logger = Logger(level=logging.INFO)
+
+class SyncContentProvider(SyncStep, HpcLibrary):
+    provides=[ContentProvider]
+    requested_interval=0
+
+    def __init__(self, **args):
+        SyncStep.__init__(self, **args)
+        HpcLibrary.__init__(self)
+
+    def fetch_pending(self):
+        return ContentProvider.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
+
+    def sync_record(self, cp):
+        logger.info("sync'ing service provider %s" % str(cp))
+        account_name = self.make_account_name(cp.name)
+        print "XXX", cp.name, account_name
+
+        if (not cp.serviceProvider) or (not cp.serviceProvider.service_provider_id):
+            return
+
+        spid = cp.serviceProvider.service_provider_id
+
+        cp_dict = {"account": account_name, "name": cp.name, "enabled": cp.enabled, "service_provider_id": spid}
+
+        #print cp_dict
+
+        if not cp.content_provider_id:
+            id = self.client.onev.Create("ContentProvider", cp_dict)
+            cp.content_provider_id = id
+        else:
+            self.client.onev.Update("ContentProvider", cp.content_provider_id, cp_dict)
+
+        cp.save()
diff --git a/planetstack/hpc_observer/steps/sync_hpcservices.py b/planetstack/hpc_observer/steps/sync_hpcservices.py
index 049b316..a4c9e77 100644
--- a/planetstack/hpc_observer/steps/sync_hpcservices.py
+++ b/planetstack/hpc_observer/steps/sync_hpcservices.py
@@ -1,89 +1,34 @@
 import os
+import sys
 import base64
 from django.db.models import F, Q
 from planetstack.config import Config
 from observer.syncstep import SyncStep
 from core.models import Service
 from hpc.models import HpcService
+from requestrouter.models import RequestRouterService
 from util.logger import Logger, logging
 
+# hpclibrary will be in steps/..
+parentdir = os.path.join(os.path.dirname(__file__),"..")
+sys.path.insert(0,parentdir)

+
+from hpclib import HpcLibrary
+
 logger = Logger(level=logging.INFO)
 
-class SyncHpcService(SyncStep):
+class SyncHpcService(SyncStep, HpcLibrary):
     provides=[HpcService]
     requested_interval=0
 
+    def __init__(self, **args):
+        SyncStep.__init__(self, **args)
+        HpcLibrary.__init__(self)
+
     def fetch_pending(self):
         return HpcService.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
 
-    def extract_slice_info(hpc_service):
-        """ Produce a dict that describes the slices for the CMI
-
-            slice_coblitz = <name of coblitz slice>
-            service_coblitz = <name of coblitz service>
-            hostname_coblitz = <name of first coblitz slice>
-            hostnames_coblitz = <name_of_first_cob_slice>,<name_of_second_cob_slice>,...
-
-            slice_cmi = <name of cmi slice>
-            ...
-        """
-
-        slicenames = {}
-        slicehosts = {}
-        for slice in hpc_service.service.all():
-            name = slice.name
-            if not ("_" in name):
-                continue
-
-            if "coblitz" in name:
-                slicenames["coblitz"] = name
-                slicehosts["coblitz"] = [sliver.node.name for sliver in slice.slivers.all()]
-            elif "cmi" in name:
-                slicenames["cmi"] = name
-                slicehosts["cmi"] = [sliver.node.name for sliver in slice.slivers.all()]
-
-        base_hrn = None
-        if "coblitz" in slicenames:
-            base_hrn = slicenames["coblitz"].split("_")[0]
-
-        mapping = {}
-        mapping["base_hrn"] = base_hrn
-        for (k,v) in slicenames.items():
-            mapping["slice_" + k] = v
-            mapping["service_" + k] = v.split("_",1)[1]
-        for (k,v) in slicehosts.items()
-            mapping["hostname_" + k] = v[0]
-            mapping["hostnames_" + k] = ",".join(v)
-
-        return mapping
-
-    def write_slices_file(self, hpc_service):
-        mapping = self.extract_slicenames(hpc_service)
-
-        fn = "/tmp/slices"
-
-        f = open(fn, "w")
-        f.write("""
-ENABLE_PLC=True
-ENABLE_PS=False
-BASE_HRN="%(base_hrn)"
-RELEVANT_SERVICE_NAMES=['%(service_coblitz)', '%(service_dnsredir)', '%(service_dnsdemux)']
-COBLITZ_SLICE_NAME="%(slice_coblitz)"
-COBLITZ_SLICE_ID=1
-COBLITZ_PS_SLICE_NAME="%(slice_coblitz)"
-DNSREDIR_SLICE_NAME="%(slice_dnsredir)"
-DNSREDIR_SLICE_ID=2
-DNSREDIR_PS_SLICE_NAME="%(slice_dnsredir)"
-DNSDEMUX_SLICE_NAME="%(slice_dnsdemux)"
-DNSDEMUX_SLICE_ID=3
-DNSDEMUX_PS_SLICE_NAME="%(slice_dnsdemux)"
-CMI_URL="http://%(hostname_cmi)"
-CMI_HTTP_PORT="8004"
-CMI_HTTPS_PORT="8003"
-PUPPET_MASTER_HOSTNAME="%(hostname_cmi)"
-PUPPET_MASTER_PORT="8140"
-""")
-
     def sync_record(self, hpc_service):
         logger.info("sync'ing hpc_service %s" % str(hpc_service))
+        self.write_slices_file(hpc_service, None)
         hpc_service.save()
diff --git a/planetstack/hpc_observer/steps/sync_originserver.py b/planetstack/hpc_observer/steps/sync_originserver.py
new file mode 100644
index 0000000..e57c282
--- /dev/null
+++ b/planetstack/hpc_observer/steps/sync_originserver.py
@@ -0,0 +1,58 @@
+import os
+import sys
+import base64
+from django.db.models import F, Q
+from planetstack.config import Config
+from observer.syncstep import SyncStep
+from core.models import Service
+from hpc.models import ServiceProvider, ContentProvider, CDNPrefix, OriginServer
+from util.logger import Logger, logging
+
+# hpclibrary will be in steps/..
+parentdir = os.path.join(os.path.dirname(__file__),"..")
+sys.path.insert(0,parentdir)

+
+from hpclib import HpcLibrary
+
+logger = Logger(level=logging.INFO)
+
+class SyncOriginServer(SyncStep, HpcLibrary):
+    provides=[OriginServer]
+    requested_interval=0
+
+    def __init__(self, **args):
+        SyncStep.__init__(self, **args)
+        HpcLibrary.__init__(self)
+
+    def fetch_pending(self):
+        return OriginServer.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
+
+    def sync_record(self, ors):
+        logger.info("sync'ing origin server %s" % str(ors))
+
+        if (not ors.contentProvider) or (not ors.contentProvider.content_provider_id):
+            return
+
+        cpid = ors.contentProvider.content_provider_id
+
+        # validation requires URL start with http://
+        url = ors.url
+        if not url.startswith("http://"):
+            url = "http://" + url
+
+        ors_dict = {"authenticated_content": ors.authenticated, "zone_redirects": ors.redirects, "content_provider_id": cpid, "url": url, "service_type": "HyperCache", "caching_type": "Optimistic", "description": ors.description}
+
+        #print os_dict
+
+        if not ors.origin_server_id:
+            id = self.client.onev.Create("OriginServer", ors_dict)
+            ors.origin_server_id = id
+        else:
+            self.client.onev.Update("OriginServer", ors.origin_server_id, ors_dict)
+
+        # ... something breaks (analytics) if the URL starts with http://, so we
+        # change it in cob after we added it via onev.
+        url = url[7:]
+        self.client.cob.UpdateContent(ors.origin_server_id, {"url": url})
+
+        ors.save()
diff --git a/planetstack/hpc_observer/steps/sync_serviceprovider.py b/planetstack/hpc_observer/steps/sync_serviceprovider.py
new file mode 100644
index 0000000..5eb991a
--- /dev/null
+++ b/planetstack/hpc_observer/steps/sync_serviceprovider.py
@@ -0,0 +1,41 @@
+import os
+import sys
+import base64
+from django.db.models import F, Q
+from planetstack.config import Config
+from observer.syncstep import SyncStep
+from core.models import Service
+from hpc.models import ServiceProvider
+from util.logger import Logger, logging
+
+# hpclibrary will be in steps/..
+parentdir = os.path.join(os.path.dirname(__file__),"..")
+sys.path.insert(0,parentdir)

+
+from hpclib import HpcLibrary
+
+logger = Logger(level=logging.INFO)
+
+class SyncServiceProvider(SyncStep, HpcLibrary):
+    provides=[ServiceProvider]
+    requested_interval=0
+
+    def __init__(self, **args):
+        SyncStep.__init__(self, **args)
+        HpcLibrary.__init__(self)
+
+    def fetch_pending(self):
+        return ServiceProvider.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
+
+    def sync_record(self, sp):
+        logger.info("sync'ing service provider %s" % str(sp))
+        account_name = self.make_account_name(sp.name)
+        print "XXX", sp.name, account_name
+        sp_dict = {"account": account_name, "name": sp.name, "enabled": sp.enabled}
+        if not sp.service_provider_id:
+            id = self.client.onev.Create("ServiceProvider", sp_dict)
+            sp.service_provider_id = id
+        else:
+            self.client.onev.Update("ServiceProvider", sp.service_provider_id, sp_dict)
+
+        sp.save()