Merge branch 'master' of github.com:open-cloud/xos
diff --git a/xos/api/import_methods.py b/xos/api/import_methods.py
index 3702f8a..fbcd990 100644
--- a/xos/api/import_methods.py
+++ b/xos/api/import_methods.py
@@ -88,8 +88,10 @@
viewset = view_url[3]
urlpatterns.extend(viewset.get_urlpatterns(api_path="^"+api_path+"/"))
- if not has_index_view:
- urlpatterns.append(url('^' + api_path + '/$', XOSIndexViewSet.as_view({'get': 'list'}, view_urls=view_urls, subdirs=subdirs), name="api_path"+"_index"))
+ # Only add an index_view if 1) the is not already an index view, and
+ # 2) we have found some methods in this directory.
+ if (not has_index_view) and (urlpatterns):
+ urlpatterns.append(url('^' + api_path + '/$', XOSIndexViewSet.as_view({'get': 'list'}, view_urls=view_urls, subdirs=subdirs, api_path=api_path), name=api_path+"_index"))
return urlpatterns
diff --git a/xos/api/service/hpc/__init__.py b/xos/api/service/hpc/__init__.py
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/xos/api/service/hpc/__init__.py
@@ -0,0 +1 @@
+
diff --git a/xos/api/service/hpc/hpcview.py b/xos/api/service/hpc/hpcview.py
new file mode 100644
index 0000000..3e31e73
--- /dev/null
+++ b/xos/api/service/hpc/hpcview.py
@@ -0,0 +1,210 @@
+from rest_framework.decorators import api_view
+from rest_framework.response import Response
+from rest_framework.reverse import reverse
+from rest_framework import serializers
+from rest_framework import generics
+from rest_framework.views import APIView
+from core.models import *
+from services.hpc.models import *
+from services.requestrouter.models import *
+from django.forms import widgets
+from django.core.exceptions import PermissionDenied
+from django.contrib.contenttypes.models import ContentType
+import json
+import socket
+import time
+
+def lookup_tag(service, instance, name, default=None):
+ instance_type = ContentType.objects.get_for_model(instance)
+ t = Tag.objects.filter(service=service, name=name, content_type__pk=instance_type.id, object_id=instance.id)
+ if t:
+ return t[0].value
+ else:
+ return default
+
+def lookup_time(service, instance, name):
+ v = lookup_tag(service, instance, name)
+ if v:
+ return str(time.time() - float(v))
+ else:
+ return None
+
+def json_default(d, default):
+ if not d:
+ return default
+ return json.loads(d)
+
+def compute_config_run(d):
+ if not d:
+ return "null"
+
+ try:
+ d = json.loads(d)
+ except:
+ return "error decoding json '%s'" % str(d)
+
+ status = d.get("status", "null")
+ if status!="success":
+ return status
+
+ config_run = d.get("config.run")
+ if not config_run:
+ return "null"
+
+ try:
+ config_run = max(0, int(time.time()) - int(float(config_run)))
+ except:
+ pass
+
+ return config_run
+
+# from hpc_watcher.py
+def get_public_ip(service, instance):
+ network_name = None
+ if "hpc" in instance.slice.name:
+ network_name = getattr(service, "watcher_hpc_network", None)
+ elif "demux" in instance.slice.name:
+ network_name = getattr(service, "watcher_dnsdemux_network", None)
+ elif "redir" in instance.slice.name:
+ network_name = getattr(service, "watcher_dnsredir_network", None)
+
+ if network_name and network_name.lower()=="nat":
+ return None
+
+ if (network_name is None) or (network_name=="") or (network_name.lower()=="public"):
+ return instance.get_public_ip()
+
+ for ns in instance.ports.all():
+ if (ns.ip) and (ns.network.name==network_name):
+ return ns.ip
+
+ raise ValueError("Couldn't find network %s" % str(network_name))
+
+def getHpcDict(user, pk):
+ hpc = HpcService.objects.get(pk=pk)
+ slices = hpc.slices.all()
+
+ dnsdemux_slice = None
+ dnsredir_slice = None
+ hpc_slice = None
+ for slice in slices:
+ if "dnsdemux" in slice.name:
+ dnsdemux_service = hpc
+ dnsdemux_slice = slice
+ if "dnsredir" in slice.name:
+ dnsredir_service = hpc
+ dnsredir_slice = slice
+ if "hpc" in slice.name:
+ hpc_service = hpc
+ hpc_slice = slice
+
+ if not dnsdemux_slice:
+ rr = RequestRouterService.objects.all()
+ if rr:
+ rr=rr[0]
+ slices = rr.slices.all()
+ for slice in slices:
+ if "dnsdemux" in slice.name:
+ dnsdemux_service = rr
+ dnsdemux_slice = slice
+ if "dnsredir" in slice.name:
+ dnsredir_service = rr
+ dnsredir_slice = slice
+
+ if not dnsredir_slice:
+ print "no dnsredir slice"
+ return
+
+ if not dnsdemux_slice:
+ print "no dnsdemux slice"
+ return
+
+ #dnsdemux_has_public_network = False
+ #for network in dnsdemux_slice.networks.all():
+ # if (network.template) and (network.template.visibility=="public") and (network.template.translation=="none"):
+ # dnsdemux_has_public_network = True
+
+ nameservers = {}
+ for nshc in hpc.hpchealthcheck_set.filter(kind="nameserver"):
+ nameserver = nshc.resource_name
+ try:
+ nameservers[nameserver] = {"name": nameserver, "ip": socket.gethostbyname(nameserver), "hit": False}
+ except:
+ nameservers[nameserver] = {"name": nameserver, "ip": "exception", "hit": False}
+
+ dnsdemux=[]
+ for instance in dnsdemux_slice.instances.all():
+ ip=None
+ try:
+ ip = get_public_ip(dnsdemux_service, instance)
+ except Exception, e:
+ ip = "Exception: " + str(e)
+ if not ip:
+ try:
+ ip = socket.gethostbyname(instance.node.name)
+ except:
+ ip = "??? " + instance.node.name
+
+ instance_nameservers = []
+ for ns in nameservers.values():
+ if ns["ip"]==ip:
+ instance_nameservers.append(ns["name"])
+ ns["hit"]=True
+
+ # now find the dnsredir instance that is also on this node
+ watcherd_dnsredir = "no-redir-instance"
+ for dnsredir_instance in dnsredir_slice.instances.all():
+ if dnsredir_instance.node == instance.node:
+ watcherd_dnsredir = lookup_tag(dnsredir_service, dnsredir_instance, "watcher.watcher.msg")
+
+ watcherd_dnsdemux = lookup_tag(dnsdemux_service, instance, "watcher.watcher.msg")
+
+ dnsdemux.append( {"name": instance.node.name,
+ "watcher.DNS.msg": lookup_tag(dnsdemux_service, instance, "watcher.DNS.msg"),
+ "watcher.DNS.time": lookup_time(dnsdemux_service, instance, "watcher.DNS.time"),
+ "ip": ip,
+ "nameservers": instance_nameservers,
+ "dnsdemux_config_age": compute_config_run(watcherd_dnsdemux),
+ "dnsredir_config_age": compute_config_run(watcherd_dnsredir) })
+
+ hpc=[]
+ for instance in hpc_slice.instances.all():
+ watcherd_hpc = lookup_tag(hpc_service, instance, "watcher.watcher.msg")
+
+ hpc.append( {"name": instance.node.name,
+ "watcher.HPC-hb.msg": lookup_tag(hpc_service, instance, "watcher.HPC-hb.msg"),
+ "watcher.HPC-hb.time": lookup_time(hpc_service, instance, "watcher.HPC-hb.time"),
+ "watcher.HPC-fetch.msg": lookup_tag(hpc_service, instance, "watcher.HPC-fetch.msg"),
+ "watcher.HPC-fetch.time": lookup_time(hpc_service, instance, "watcher.HPC-fetch.time"),
+ "watcher.HPC-fetch.urls": json_default(lookup_tag(hpc_service, instance, "watcher.HPC-fetch-urls.msg"), []),
+ "config_age": compute_config_run(watcherd_hpc),
+
+ })
+
+ return { "id": pk,
+ "dnsdemux": dnsdemux,
+ "hpc": hpc,
+ "nameservers": nameservers,}
+
+
+class HpcList(APIView):
+ method_kind = "list"
+ method_name = "hpcview"
+
+ def get(self, request, format=None):
+ if (not request.user.is_authenticated()):
+ raise PermissionDenied("You must be authenticated in order to use this API")
+ results = []
+ for hpc in HpcService.objects.all():
+ results.append(getHpcDict(request.user, hpc.pk))
+ return Response( results )
+
+class HpcDetail(APIView):
+ method_kind = "detail"
+ method_name = "hpcview"
+
+ def get(self, request, format=None, pk=0):
+ if (not request.user.is_authenticated()):
+ raise PermissionDenied("You must be authenticated in order to use this API")
+ return Response( [getHpcDict(request.user, pk)] )
+
diff --git a/xos/api/xosapi_helpers.py b/xos/api/xosapi_helpers.py
index 8c737cb..aa5c770 100644
--- a/xos/api/xosapi_helpers.py
+++ b/xos/api/xosapi_helpers.py
@@ -6,6 +6,8 @@
from rest_framework import viewsets
from django.conf.urls import patterns, url
from xos.exceptions import *
+from rest_framework.reverse import reverse
+from django.core.urlresolvers import get_script_prefix, resolve, Resolver404
if hasattr(serializers, "ReadOnlyField"):
# rest_framework 3.x
@@ -133,20 +135,32 @@
class XOSIndexViewSet(viewsets.ViewSet):
view_urls=[]
subdirs=[]
+ api_path = None
- def __init__(self, view_urls, subdirs):
+ def __init__(self, view_urls, subdirs, api_path):
self.view_urls = view_urls
self.subdirs = subdirs
+ self.api_path = api_path
super(XOSIndexViewSet, self).__init__()
def list(self, request):
- endpoints = []
+ endpoints = {}
for view_url in self.view_urls:
method_name = view_url[1].split("/")[-1]
- endpoints.append(method_name)
+ method_url = "http://" + request.get_host() + get_script_prefix() + self.api_path + "/" + method_name
+ endpoints[method_name] = method_url
for subdir in self.subdirs:
- endpoints.append(subdir)
+ method_name = subdir
+ method_url = get_script_prefix() + self.api_path + "/" + subdir + "/"
+ # Check to make sure that an endpoint exists at this method_url. This
+ # prunes out subdirs that don't have any methods (like examples/)
+ try:
+ resolve(method_url)
+ except Resolver404:
+ continue
+ method_url = "http://" + request.get_host() + method_url
+ endpoints[method_name] = method_url
- return Response({"endpoints": endpoints})
+ return Response(endpoints)
diff --git a/xos/core/models/plcorebase.py b/xos/core/models/plcorebase.py
index 060570d..76c2e54 100644
--- a/xos/core/models/plcorebase.py
+++ b/xos/core/models/plcorebase.py
@@ -198,8 +198,8 @@
# default values for created and updated are only there to keep evolution
# from failing.
- created = models.DateTimeField(auto_now_add=True, default=timezone.now)
- updated = models.DateTimeField(default=timezone.now)
+ created = models.DateTimeField(auto_now_add=True)
+ updated = models.DateTimeField(auto_now=True)
enacted = models.DateTimeField(null=True, blank=True, default=None)
policed = models.DateTimeField(null=True, blank=True, default=None)
diff --git a/xos/synchronizers/base/syncstep.py b/xos/synchronizers/base/syncstep.py
index 0e34010..42a9db4 100644
--- a/xos/synchronizers/base/syncstep.py
+++ b/xos/synchronizers/base/syncstep.py
@@ -1,6 +1,7 @@
import os
import base64
from datetime import datetime
+from django.utils import timezone
from xos.config import Config
from xos.logger import Logger, logging
from synchronizers.base.steps import *
@@ -227,7 +228,7 @@
self.delete_record(o)
o.delete(purge=True)
else:
- new_enacted = datetime.now() # Is this the same timezone? XXX
+ new_enacted = timezone.now()
self.sync_record(o)
o.enacted = new_enacted
scratchpad = {'next_run':0, 'exponent':0, 'last_success':time.time()}
diff --git a/xos/xos/settings.py b/xos/xos/settings.py
index dad7888..417be81 100644
--- a/xos/xos/settings.py
+++ b/xos/xos/settings.py
@@ -2,6 +2,7 @@
from django import VERSION as DJANGO_VERSION
import socket
import os
+import warnings
from urlparse import urlparse
# Django settings for XOS.
@@ -67,6 +68,12 @@
# In a Windows environment this must be set to your system time zone.
TIME_ZONE = 'America/New_York'
+# Verbose warnings when a naive datetime is used, gives a traceback
+# from: https://docs.djangoproject.com/en/1.9/topics/i18n/timezones/#code
+warnings.filterwarnings(
+ 'error', r"DateTimeField .* received a naive datetime",
+ RuntimeWarning, r'django\.db\.models\.fields')
+
# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'