blob: 11cc4f5142c88a1ea2164390a2d272f86d63e3ff [file] [log] [blame]
Siobhan Tullycf04fb62014-01-11 11:25:57 -05001#views.py
Scott Bakerbd57a432014-04-14 16:12:15 -07002import functools
3import math
Scott Baker771819b2014-03-19 22:10:17 -07004import os
5import sys
Siobhan Tully06d2c032014-02-23 23:48:52 -05006from django.views.generic import TemplateView, View
7import datetime
Scott Baker771819b2014-03-19 22:10:17 -07008from pprint import pprint
9import json
Scott Baker168b7412014-05-16 14:25:41 -070010from syndicate.models import *
Scott Baker537d8232014-04-08 23:40:56 -070011from core.models import *
Scott Bakere9f4ec92014-04-29 11:29:45 -070012from hpc.models import ContentProvider
Scott Bakerbd57a432014-04-14 16:12:15 -070013from operator import attrgetter
Scott Baker186ae612014-05-19 10:42:51 -070014from django import template
Scott Bakerbd57a432014-04-14 16:12:15 -070015from django.views.decorators.csrf import csrf_exempt
16from django.http import HttpResponse, HttpResponseServerError
Scott Baker537d8232014-04-08 23:40:56 -070017from django.core import urlresolvers
Scott Bakerbd57a432014-04-14 16:12:15 -070018from django.contrib.gis.geoip import GeoIP
19from ipware.ip import get_ip
Scott Baker2c3cb642014-05-19 17:55:56 -070020from operator import itemgetter, attrgetter
Scott Baker771819b2014-03-19 22:10:17 -070021import traceback
Scott Baker6253dfc2014-04-14 09:29:30 -070022import socket
Siobhan Tully06d2c032014-02-23 23:48:52 -050023
Scott Bakera880bb52014-04-14 23:52:48 -070024BLESSED_SITES = ["Stanford", "Washington", "Princeton", "GeorgiaTech", "MaxPlanck"]
25
Scott Baker771819b2014-03-19 22:10:17 -070026if os.path.exists("/home/smbaker/projects/vicci/cdn/bigquery"):
27 sys.path.append("/home/smbaker/projects/vicci/cdn/bigquery")
28else:
29 sys.path.append("/opt/planetstack/hpc_wizard")
30import hpc_wizard
Scott Baker75095b62014-04-21 17:32:09 -070031from planetstack_analytics import DoPlanetStackAnalytics, PlanetStackAnalytics, RED_LOAD, BLUE_LOAD
Siobhan Tullycf04fb62014-01-11 11:25:57 -050032
33class DashboardWelcomeView(TemplateView):
34 template_name = 'admin/dashboard/welcome.html'
35
36 def get(self, request, *args, **kwargs):
37 context = self.get_context_data(**kwargs)
Scott Baker2c3cb642014-05-19 17:55:56 -070038 context = getDashboardContext(request.user, context)
Siobhan Tullycf04fb62014-01-11 11:25:57 -050039 return self.render_to_response(context=context)
Siobhan Tullye18b3442014-02-23 14:23:34 -050040
Scott Baker2c3cb642014-05-19 17:55:56 -070041class DashboardDynamicView(TemplateView):
Scott Baker186ae612014-05-19 10:42:51 -070042 head_template = r"""{% extends "admin/dashboard/dashboard_base.html" %}
43 {% load admin_static %}
44 {% block content %}
45 """
46
47 tail_template = r"{% endblock %}"
48
Scott Baker2c3cb642014-05-19 17:55:56 -070049 def get(self, request, name="root", *args, **kwargs):
Scott Baker186ae612014-05-19 10:42:51 -070050 context = self.get_context_data(**kwargs)
Scott Baker2c3cb642014-05-19 17:55:56 -070051 context = getDashboardContext(request.user, context)
Scott Baker186ae612014-05-19 10:42:51 -070052
Scott Baker2c3cb642014-05-19 17:55:56 -070053 if name=="root":
54 return self.multiDashboardView(request, context)
55 else:
56 return self.singleDashboardView(request, name, context)
57
58 def readDashboard(self, fn):
59 try:
60 template= open("/opt/planetstack/templates/admin/dashboard/%s.html" % fn, "r").read()
61 if (fn=="tenant"):
62 template = '<div id="tabs-5"></div>' + template
63 return template
64 except:
65 return "failed to open %s" % fn
66
67 def multiDashboardView(self, request, context):
Scott Bakerfc06e782014-05-19 10:52:10 -070068 head_template = self.head_template
69 tail_template = self.tail_template
70
Scott Baker2c3cb642014-05-19 17:55:56 -070071 body = """
72 <div id="hometabs" >
73 <ul id="suit_form_tabs" class="nav nav-tabs nav-tabs-suit" data-tab-prefix="suit-tab">
74 """
Scott Bakerfc06e782014-05-19 10:52:10 -070075
Scott Baker2c3cb642014-05-19 17:55:56 -070076 dashboards = request.user.get_dashboards()
Scott Bakerfc06e782014-05-19 10:52:10 -070077
Scott Baker2c3cb642014-05-19 17:55:56 -070078 # customize is a special dashboard they always get
79 customize = DashboardView.objects.filter(name="Customize")
80 if customize:
81 dashboards.append(customize[0])
Scott Baker186ae612014-05-19 10:42:51 -070082
Scott Baker2c3cb642014-05-19 17:55:56 -070083 for i,view in enumerate(dashboards):
84 body = body + '<li><a href="#dashtab-%d">%s</a></li>\n' % (i, view.name)
Scott Baker186ae612014-05-19 10:42:51 -070085
Scott Baker2c3cb642014-05-19 17:55:56 -070086 body = body + "</ul>\n"
87
88 for i,view in enumerate(dashboards):
89 url = view.url
90 body = body + '<div id="dashtab-%d">\n' % i
91 if url.startswith("template:"):
92 fn = url[9:]
93 body = body + self.readDashboard(fn)
94 body = body + '</div>\n'
95
96 body=body+"</div>\n"
97
98 t = template.Template(head_template + body + self.tail_template)
Scott Baker186ae612014-05-19 10:42:51 -070099
100 response_kwargs = {}
101 response_kwargs.setdefault('content_type', self.content_type)
102 return self.response_class(
103 request = request,
104 template = t,
105 context = context,
106 **response_kwargs)
107
Scott Baker2c3cb642014-05-19 17:55:56 -0700108 def singleDashboardView(self, request, name, context):
109 head_template = self.head_template
110 tail_template = self.tail_template
111
112 t = template.Template(head_template + self.readDashboard(fn) + self.tail_template)
113
114 response_kwargs = {}
115 response_kwargs.setdefault('content_type', self.content_type)
116 return self.response_class(
117 request = request,
118 template = t,
119 context = context,
120 **response_kwargs)
121
122def getDashboardContext(user, context={}, tableFormat = False):
123 context = {}
Scott Baker771819b2014-03-19 22:10:17 -0700124
125 userSliceData = getSliceInfo(user)
126 if (tableFormat):
Scott Baker2c3cb642014-05-19 17:55:56 -0700127 context['userSliceInfo'] = userSliceTableFormatter(userSliceData)
Scott Baker771819b2014-03-19 22:10:17 -0700128 else:
Scott Baker2c3cb642014-05-19 17:55:56 -0700129 context['userSliceInfo'] = userSliceData
130 context['cdnData'] = getCDNOperatorData(wait=False)
131 context['cdnContentProviders'] = getCDNContentProviderData()
132
133 (dashboards, unusedDashboards)= getDashboards(user)
134 unusedDashboards=[x for x in unusedDashboards if x!="Customize"]
135 context['dashboards'] = dashboards
136 context['unusedDashboards'] = unusedDashboards
137
138 return context
139
140def getDashboards(user):
141 #dashboards = sorted(list(user.dashboardViews.all()), key=attrgetter('order'))
142 dashboards = user.get_dashboards()
143
144 dashboard_names = [d.name for d in dashboards]
145
146 unused_dashboard_names = []
147 for dashboardView in DashboardView.objects.all():
148 if not dashboardView.name in dashboard_names:
149 unused_dashboard_names.append(dashboardView.name)
150
151 return (dashboard_names, unused_dashboard_names)
Scott Baker771819b2014-03-19 22:10:17 -0700152
Scott Baker537d8232014-04-08 23:40:56 -0700153class TenantCreateSlice(View):
154 def post(self, request, *args, **kwargs):
155 sliceName = request.POST.get("sliceName", "0")
156 serviceClass = request.POST.get("serviceClass", "0")
157 imageName = request.POST.get("imageName", "0")
158 actionToDo = request.POST.get("actionToDo", "0")
Scott Baker168b7412014-05-16 14:25:41 -0700159 #network = request.POST.get("network","0")
Scott Bakeredf2f672014-04-22 10:17:37 -0700160 mountDataSets = request.POST.get("mountDataSets","0")
Scott Baker537d8232014-04-08 23:40:56 -0700161 if (actionToDo == "add"):
162 serviceClass = ServiceClass.objects.get(name=serviceClass)
163 site = request.user.site
Scott Baker4b0cedd2014-04-22 21:01:21 -0700164 image = Image.objects.get(name=imageName)
Scott Baker168b7412014-05-16 14:25:41 -0700165 newSlice = Slice(name=sliceName,serviceClass=serviceClass,site=site,imagePreference=image,mountDataSets=mountDataSets)
Scott Baker537d8232014-04-08 23:40:56 -0700166 newSlice.save()
Scott Baker4b0cedd2014-04-22 21:01:21 -0700167 return HttpResponse("Slice created")
Scott Baker537d8232014-04-08 23:40:56 -0700168
Scott Bakeredf2f672014-04-22 10:17:37 -0700169class TenantUpdateSlice(View):
170 def post(self, request, *args, **kwargs):
171 sliceName = request.POST.get("sliceName", "0")
172 serviceClass = request.POST.get("serviceClass", "0")
173 imageName = request.POST.get("imageName", "0")
174 actionToDo = request.POST.get("actionToDo", "0")
Scott Baker168b7412014-05-16 14:25:41 -0700175 #network = request.POST.get("network","0")
Scott Baker4b0cedd2014-04-22 21:01:21 -0700176 dataSet = request.POST.get("dataSet","0")
177 slice = Slice.objects.all()
178 for entry in slice:
179 serviceClass = ServiceClass.objects.get(name=serviceClass)
180 if(entry.name==sliceName):
181 if (actionToDo == "update"):
182 setattr(entry,'serviceClass',serviceClass)
183 setattr(entry,'imagePreference',imageName)
Scott Baker168b7412014-05-16 14:25:41 -0700184 #setattr(entry,'network',network)
Scott Baker4b0cedd2014-04-22 21:01:21 -0700185 setattr(entry,'mountDataSets',dataSet)
186 entry.save()
187 break
188 return HttpResponse("Slice updated")
189
Scott Baker537d8232014-04-08 23:40:56 -0700190def getTenantSliceInfo(user, tableFormat = False):
191 tenantSliceDetails = {}
192 tenantSliceData = getTenantInfo(user)
193 tenantServiceClassData = getServiceClassInfo(user)
194 if (tableFormat):
195 tenantSliceDetails['userSliceInfo'] = userSliceTableFormatter(tenantSliceData)
196 tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData)
197 else:
198 tenantSliceDetails['userSliceInfo'] = tenantSliceData
199 tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData)
200 tenantSliceDetails['image']=userSliceTableFormatter(getImageInfo(user))
Scott Baker168b7412014-05-16 14:25:41 -0700201 #tenantSliceDetails['network']=userSliceTableFormatter(getNetworkInfo(user))
Scott Baker537d8232014-04-08 23:40:56 -0700202 tenantSliceDetails['deploymentSites']=userSliceTableFormatter(getDeploymentSites())
Scott Bakerb85c26b2014-04-18 11:25:32 -0700203 tenantSliceDetails['sites'] = userSliceTableFormatter(getTenantSitesInfo())
204 tenantSliceDetails['mountDataSets'] = userSliceTableFormatter(getMountDataSets())
Scott Baker168b7412014-05-16 14:25:41 -0700205 tenantSliceDetails['publicKey'] = getPublicKey(user)
Scott Baker537d8232014-04-08 23:40:56 -0700206 return tenantSliceDetails
207
Scott Baker537d8232014-04-08 23:40:56 -0700208def getTenantInfo(user):
209 slices =Slice.objects.all()
210 userSliceInfo = []
211 for entry in slices:
212 sliceName = Slice.objects.get(id=entry.id).name
213 slice = Slice.objects.get(name=Slice.objects.get(id=entry.id).name)
214 sliceServiceClass = entry.serviceClass.name
215 preferredImage = entry.imagePreference
Scott Bakeredf2f672014-04-22 10:17:37 -0700216 sliceDataSet = entry.mountDataSets
Scott Baker168b7412014-05-16 14:25:41 -0700217 #sliceNetwork = entry.network
Scott Baker537d8232014-04-08 23:40:56 -0700218 numSliver = 0
219 sliceImage=""
220 sliceSite = {}
Scott Bakeredf2f672014-04-22 10:17:37 -0700221 sliceNode = {}
222 sliceInstance= {}
Scott Baker537d8232014-04-08 23:40:56 -0700223 for sliver in slice.slivers.all():
Scott Bakera880bb52014-04-14 23:52:48 -0700224 if sliver.node.site.name in BLESSED_SITES:
Scott Bakera880bb52014-04-14 23:52:48 -0700225 sliceSite[sliver.node.site.name] = sliceSite.get(sliver.node.site.name,0) + 1
226 sliceImage = sliver.image.name
Scott Bakerb37ce332014-04-22 22:57:40 -0700227 sliceNode[str(sliver)] = sliver.node.name
Scott Bakeredf2f672014-04-22 10:17:37 -0700228 numSliver = sum(sliceSite.values())
Scott Bakerb85c26b2014-04-18 11:25:32 -0700229 numSites = len(sliceSite)
Scott Baker168b7412014-05-16 14:25:41 -0700230 userSliceInfo.append({'sliceName': sliceName,'sliceServiceClass': sliceServiceClass,'preferredImage':preferredImage,'numOfSites':numSites, 'sliceSite':sliceSite,'sliceImage':sliceImage,'numOfSlivers':numSliver,'sliceDataSet':sliceDataSet,'instanceNodePair':sliceNode})
Scott Baker537d8232014-04-08 23:40:56 -0700231 return userSliceInfo
232
233def getTenantSitesInfo():
234 tenantSiteInfo=[]
235 for entry in Site.objects.all():
Scott Bakera880bb52014-04-14 23:52:48 -0700236 if entry.name in BLESSED_SITES:
237 tenantSiteInfo.append({'siteName':entry.name})
Scott Baker537d8232014-04-08 23:40:56 -0700238 return tenantSiteInfo
239
Scott Baker771819b2014-03-19 22:10:17 -0700240def userSliceTableFormatter(data):
241# pprint(data)
242 formattedData = {
243 'rows' : data
244 }
245 return formattedData
246
Scott Baker168b7412014-05-16 14:25:41 -0700247def getPublicKey(user):
248 users=User.objects.all()
249 for key in users:
250 if (str(key.email)==str(user)):
251 sshKey = key.public_key
252 return sshKey
253
Scott Baker537d8232014-04-08 23:40:56 -0700254def getServiceClassInfo(user):
255 serviceClassList = ServiceClass.objects.all()
256 sliceInfo = []
257 for entry in serviceClassList:
258 sliceInfo.append({'serviceClass':entry.name})
259 return sliceInfo
260
261def getImageInfo(user):
Scott Baker4b0cedd2014-04-22 21:01:21 -0700262 imageList = Image.objects.all()
263 #imageList = ['Fedora 16 LXC rev 1.3','Hadoop','MPI']
Scott Baker537d8232014-04-08 23:40:56 -0700264 imageInfo = []
265 for imageEntry in imageList:
Scott Baker4b0cedd2014-04-22 21:01:21 -0700266 imageInfo.append({'Image':imageEntry.name})
267 #imageInfo.append({'Image':imageEntry})
Scott Baker537d8232014-04-08 23:40:56 -0700268 return imageInfo
269
Scott Baker168b7412014-05-16 14:25:41 -0700270def createPrivateVolume(user, sliceName):
271 caps = Volume.CAP_READ_DATA | Volume.CAP_WRITE_DATA | Volume.CAP_HOST_DATA
272 getattr(Volume.default_gateway_caps,"read data") | \
273 getattr(Volume.default_gateway_caps,"write data") | \
274 getattr(Volume.default_gateway_caps,"host files")
275 v = Volume(name="private_" + sliceName, owner_id=user, description="private volume for %s" % sliceName, blocksize=61440, private=True, archive=False, default_gateway_caps = caps)
276 v.save()
277
278SYNDICATE_REPLICATE_PORTNUM = 1025
279
280def get_free_port():
281 inuse={}
282 inuse[SYNDICATE_REPLICATE_PORTNUM] = True
283 for vs in VolumeSlice.objects.all():
284 inuse[vs.peer_portnum]=True
285 inuse[vs.replicate_portnum]=True
286 for network in Network.objects.all():
287 network_ports = [x.strip() for x in network.ports.split(",")]
288 for network_port in network_ports:
289 try:
290 inuse[int(network_port)] = True
291 except:
292 # in case someone has put a malformed port number in the list
293 pass
294 for i in range(1025, 65535):
295 if not inuse.get(i,False):
296 return i
297 return False
298
299def mountVolume(sliceName, volumeName, readWrite):
300 slice = Slice.objects.get(name=sliceName)
301 volume = Volume.objects.get(name=volumeName)
302 # choose some unused port numbers
303 flags = Volume.CAP_READ_DATA
304 if readWrite:
305 flags = flags | Volume.CAP_WRITE_DATA
306 vs = VolumeSlice(volume_id = volume, slice_id = slice, gateway_caps=flags, peer_portnum = get_free_port(), replicate_portnum = SYNDICATE_REPLICATE_PORTNUM)
307 vs.save()
308
309def hasPrivateVolume(sliceName):
310 slice = Slice.objects.get(name=sliceName)
311 for vs in VolumeSlice.objects.filter(slice_id=slice):
312 if vs.volume_id.private:
313 return True
314 return False
315
Scott Bakerb85c26b2014-04-18 11:25:32 -0700316def getMountDataSets():
Scott Baker168b7412014-05-16 14:25:41 -0700317 dataSetInfo=[]
318 for volume in Volume.objects.all():
319 if not volume.private:
320 dataSetInfo.append({'DataSet': volume.name})
321
Scott Bakerb85c26b2014-04-18 11:25:32 -0700322 return dataSetInfo
323
Scott Baker537d8232014-04-08 23:40:56 -0700324def getNetworkInfo(user):
325 #networkList = Network.objects.all()
326 networkList = ['Private Only','Private and Publicly Routable']
327 networkInfo = []
328 for networkEntry in networkList:
329 #networkInfo.append({'Network':networkEntry.name})
330 networkInfo.append({'Network':networkEntry})
331 return networkInfo
332
333def getDeploymentSites():
334 deploymentList = Deployment.objects.all()
335 deploymentInfo = []
336 for entry in deploymentList:
337 deploymentInfo.append({'DeploymentSite':entry.name})
338 return deploymentInfo
339
Scott Baker771819b2014-03-19 22:10:17 -0700340def getSliceInfo(user):
Siobhan Tully06d2c032014-02-23 23:48:52 -0500341 sliceList = Slice.objects.all()
Scott Baker771819b2014-03-19 22:10:17 -0700342 slicePrivs = SlicePrivilege.objects.filter(user=user)
Siobhan Tully06d2c032014-02-23 23:48:52 -0500343 userSliceInfo = []
344 for entry in slicePrivs:
Siobhan Tullye18b3442014-02-23 14:23:34 -0500345
Scott Baker771819b2014-03-19 22:10:17 -0700346 slicename = Slice.objects.get(id=entry.slice.id).name
Scott Baker537d8232014-04-08 23:40:56 -0700347 slice = Slice.objects.get(name=Slice.objects.get(id=entry.slice.id).name)
348 sliverList=Sliver.objects.all()
349 sites_used = {}
350 for sliver in slice.slivers.all():
351 #sites_used['deploymentSites'] = sliver.node.deployment.name
352 # sites_used[sliver.image.name] = sliver.image.name
353 sites_used[sliver.node.site.name] = sliver.numberCores
Scott Baker771819b2014-03-19 22:10:17 -0700354 sliceid = Slice.objects.get(id=entry.slice.id).id
Siobhan Tully06d2c032014-02-23 23:48:52 -0500355 try:
Scott Baker771819b2014-03-19 22:10:17 -0700356 sliverList = Sliver.objects.filter(slice=entry.slice.id)
357 siteList = {}
358 for x in sliverList:
359 if x.node.site not in siteList:
360 siteList[x.node.site] = 1
361 slivercount = len(sliverList)
362 sitecount = len(siteList)
Siobhan Tully06d2c032014-02-23 23:48:52 -0500363 except:
Scott Baker771819b2014-03-19 22:10:17 -0700364 traceback.print_exc()
365 slivercount = 0
366 sitecount = 0
Siobhan Tully06d2c032014-02-23 23:48:52 -0500367
Scott Baker771819b2014-03-19 22:10:17 -0700368 userSliceInfo.append({'slicename': slicename, 'sliceid':sliceid,
Scott Baker537d8232014-04-08 23:40:56 -0700369 'sitesUsed':sites_used,
370 'role': SliceRole.objects.get(id=entry.role.id).role,
371 'slivercount': slivercount,
372 'sitecount':sitecount})
Scott Baker771819b2014-03-19 22:10:17 -0700373
Siobhan Tully06d2c032014-02-23 23:48:52 -0500374 return userSliceInfo
375
Scott Bakere9f4ec92014-04-29 11:29:45 -0700376def getCDNContentProviderData():
377 cps = []
378 for dm_cp in ContentProvider.objects.all():
379 cp = {"name": dm_cp.name,
380 "account": dm_cp.account}
381 cps.append(cp)
382
383 return cps
384
Scott Baker75095b62014-04-21 17:32:09 -0700385def getCDNOperatorData(randomizeData = False, wait=True):
386 HPC_SLICE_NAME = "HyperCache"
387
Scott Baker0017c102014-04-18 10:48:31 -0700388 bq = PlanetStackAnalytics()
Siobhan Tully06d2c032014-02-23 23:48:52 -0500389
Scott Baker4a4472d2014-05-13 17:07:46 -0700390 rows = bq.get_cached_query_results(bq.compose_cached_query(), wait)
Siobhan Tully06d2c032014-02-23 23:48:52 -0500391
Scott Bakeredf2f672014-04-22 10:17:37 -0700392 # wait=False on the first time the Dashboard is opened. This means we might
393 # not have any rows yet. The dashboard code polls every 30 seconds, so it
394 # will eventually pick them up.
Scott Baker0017c102014-04-18 10:48:31 -0700395
Scott Baker75095b62014-04-21 17:32:09 -0700396 if rows:
Scott Baker696e2662014-04-24 17:03:29 -0700397 rows = bq.postprocess_results(rows, filter={"event": "hpc_heartbeat"}, maxi=["cpu"], count=["hostname"], computed=["bytes_sent/elapsed"], groupBy=["Time","site"], maxDeltaTime=80)
Scott Baker0017c102014-04-18 10:48:31 -0700398
Scott Baker75095b62014-04-21 17:32:09 -0700399 # dictionaryize the statistics rows by site name
400 stats_rows = {}
401 for row in rows:
402 stats_rows[row["site"]] = row
Scott Bakeredf2f672014-04-22 10:17:37 -0700403 else:
404 stats_rows = {}
Scott Baker0017c102014-04-18 10:48:31 -0700405
Scott Baker09fad022014-04-30 09:50:01 -0700406 slice = Slice.objects.filter(name=HPC_SLICE_NAME)
407 if slice:
408 slice_slivers = list(slice[0].slivers.all())
409 else:
410 slice_slivers = []
Scott Baker0017c102014-04-18 10:48:31 -0700411
412 new_rows = {}
Scott Baker75095b62014-04-21 17:32:09 -0700413 for site in Site.objects.all():
414 # compute number of slivers allocated in the data model
415 allocated_slivers = 0
416 for sliver in slice_slivers:
417 if sliver.node.site == site:
418 allocated_slivers = allocated_slivers + 1
419
420 stats_row = stats_rows.get(site.name,{})
421
422 max_cpu = stats_row.get("max_avg_cpu", stats_row.get("max_cpu",0))
423 cpu=float(max_cpu)/100.0
424 hotness = max(0.0, ((cpu*RED_LOAD) - BLUE_LOAD)/(RED_LOAD-BLUE_LOAD))
425
426 # format it to what that CDN Operations View is expecting
427 new_row = {"lat": float(site.location.longitude),
428 "long": float(site.location.longitude),
429 "lat": float(site.location.latitude),
Scott Baker0017c102014-04-18 10:48:31 -0700430 "health": 0,
Scott Baker75095b62014-04-21 17:32:09 -0700431 "numNodes": int(site.nodes.count()),
Scott Bakere9f4ec92014-04-29 11:29:45 -0700432 "activeHPCSlivers": int(stats_row.get("count_hostname", 0)), # measured number of slivers, from bigquery statistics
Scott Baker75095b62014-04-21 17:32:09 -0700433 "numHPCSlivers": allocated_slivers, # allocated number of slivers, from data model
434 "siteUrl": str(site.site_url),
435 "bandwidth": stats_row.get("sum_computed_bytes_sent_div_elapsed",0),
436 "load": max_cpu,
437 "hot": float(hotness)}
438 new_rows[str(site.name)] = new_row
439
440 # get rid of sites with 0 slivers that overlap other sites with >0 slivers
441 for (k,v) in new_rows.items():
442 bad=False
443 if v["numHPCSlivers"]==0:
444 for v2 in new_rows.values():
445 if (v!=v2) and (v2["numHPCSlivers"]>=0):
446 d = haversine(v["lat"],v["long"],v2["lat"],v2["long"])
447 if d<100:
448 bad=True
449 if bad:
450 del new_rows[k]
Scott Baker0017c102014-04-18 10:48:31 -0700451
452 return new_rows
Siobhan Tully06d2c032014-02-23 23:48:52 -0500453
Scott Baker168b7412014-05-16 14:25:41 -0700454def getPageSummary(request):
455 slice = request.GET.get('slice', None)
456 site = request.GET.get('site', None)
457 node = request.GET.get('node', None)
458
Scott Baker771819b2014-03-19 22:10:17 -0700459class SimulatorView(View):
460 def get(self, request, **kwargs):
461 sim = json.loads(file("/tmp/simulator.json","r").read())
462 text = "<html><head></head><body>"
463 text += "Iteration: %d<br>" % sim["iteration"]
464 text += "Elapsed since report %d<br><br>" % sim["elapsed_since_report"]
465 text += "<table border=1>"
466 text += "<tr><th>site</th><th>trend</th><th>weight</th><th>bytes_sent</th><th>hot</th></tr>"
467 for site in sim["site_load"].values():
468 text += "<tr>"
469 text += "<td>%s</td><td>%0.2f</td><td>%0.2f</td><td>%d</td><td>%0.2f</td>" % \
470 (site["name"], site["trend"], site["weight"], site["bytes_sent"], site["load_frac"])
471 text += "</tr>"
472 text += "</table>"
473 text += "</body></html>"
474 return HttpResponse(text)
475
476class DashboardUserSiteView(View):
477 def get(self, request, **kwargs):
Scott Baker2c3cb642014-05-19 17:55:56 -0700478 return HttpResponse(json.dumps(getDashboardContext(request.user, tableFormat=True)), mimetype='application/javascript')
Scott Baker771819b2014-03-19 22:10:17 -0700479
Scott Baker537d8232014-04-08 23:40:56 -0700480class TenantViewData(View):
481 def get(self, request, **kwargs):
482 return HttpResponse(json.dumps(getTenantSliceInfo(request.user, True)), mimetype='application/javascript')
483
Scott Baker75095b62014-04-21 17:32:09 -0700484def haversine(site_lat, site_lon, lat, lon):
Scott Baker75095b62014-04-21 17:32:09 -0700485 d=0
486 if lat and lon and site_lat and site_lon:
Scott Baker4b0cedd2014-04-22 21:01:21 -0700487 site_lat = float(site_lat)
488 site_lon = float(site_lon)
489 lat = float(lat)
490 lon = float(lon)
Scott Baker75095b62014-04-21 17:32:09 -0700491 R = 6378.1
492 a = math.sin( math.radians((lat - site_lat)/2.0) )**2 + math.cos( math.radians(lat) )*math.cos( math.radians(site_lat) )*(math.sin( math.radians((lon - site_lon)/2.0 ) )**2)
493 c = 2 * math.atan2( math.sqrt(a), math.sqrt(1 - a) )
494 d = R * c
495
496 return d
497
Scott Bakerbd57a432014-04-14 16:12:15 -0700498def siteSortKey(site, slice=None, count=None, lat=None, lon=None):
499 # try to pick a site we're already using
500 has_slivers_here=False
501 if slice:
502 for sliver in slice.slivers.all():
503 if sliver.node.site.name == site.name:
504 has_slivers_here=True
505
506 # Haversine method
Scott Baker75095b62014-04-21 17:32:09 -0700507 d = haversine(site.location.latitude, site.location.longitude, lat, lon)
Scott Bakerbd57a432014-04-14 16:12:15 -0700508
509 return (-has_slivers_here, d)
510
511def tenant_pick_sites(user, user_ip=None, slice=None, count=None):
512 """ Returns list of sites, sorted from most favorable to least favorable """
513 lat=None
514 lon=None
515 try:
516 client_geo = GeoIP().city(user_ip)
517 if client_geo:
518 lat=float(client_geo["latitude"])
519 lon=float(client_geo["longitude"])
520 except:
521 print "exception in geo code"
522 traceback.print_exc()
523
524 sites = Site.objects.all()
Scott Bakera880bb52014-04-14 23:52:48 -0700525 sites = [x for x in sites if x.name in BLESSED_SITES]
Scott Bakerbd57a432014-04-14 16:12:15 -0700526 sites = sorted(sites, key=functools.partial(siteSortKey, slice=slice, count=count, lat=lat, lon=lon))
527
528 return sites
529
Scott Bakerfe6a7e12014-04-15 10:58:31 -0700530def slice_increase_slivers(user, user_ip, siteList, slice, count, noAct=False):
Scott Bakere4deddd2014-04-14 16:48:05 -0700531 sitesChanged = {}
532
Scott Bakerbd57a432014-04-14 16:12:15 -0700533 # let's compute how many slivers are in use in each node of each site
534 for site in siteList:
535 site.nodeList = list(site.nodes.all())
536 for node in site.nodeList:
537 node.sliverCount = 0
Scott Baker6253dfc2014-04-14 09:29:30 -0700538 for sliver in node.slivers.all():
Scott Baker21511942014-04-15 17:44:27 -0700539 if sliver.slice.id == slice.id:
Scott Baker0017c102014-04-18 10:48:31 -0700540 node.sliverCount = node.sliverCount + 1
Scott Baker6253dfc2014-04-14 09:29:30 -0700541
Scott Bakerbd57a432014-04-14 16:12:15 -0700542 # Allocate slivers to nodes
543 # for now, assume we want to allocate all slivers from the same site
544 nodes = siteList[0].nodeList
545 while (count>0):
546 # Sort the node list by number of slivers per node, then pick the
547 # node with the least number of slivers.
548 nodes = sorted(nodes, key=attrgetter("sliverCount"))
549 node = nodes[0]
Scott Baker6253dfc2014-04-14 09:29:30 -0700550
Scott Bakerbd57a432014-04-14 16:12:15 -0700551 print "adding sliver at node", node.name, "of site", node.site.name
Scott Baker6253dfc2014-04-14 09:29:30 -0700552
Scott Bakere4deddd2014-04-14 16:48:05 -0700553 if not noAct:
554 sliver = Sliver(name=node.name,
555 slice=slice,
556 node=node,
557 image = Image.objects.all()[0],
558 creator = User.objects.get(email=user),
559 deploymentNetwork=node.deployment,
560 numberCores =1 )
561 sliver.save()
Scott Bakerbd57a432014-04-14 16:12:15 -0700562
563 node.sliverCount = node.sliverCount + 1
564
565 count = count - 1
566
Scott Bakere4deddd2014-04-14 16:48:05 -0700567 sitesChanged[node.site.name] = sitesChanged.get(node.site.name,0) + 1
568
569 return sitesChanged
570
Scott Bakerfe6a7e12014-04-15 10:58:31 -0700571def slice_decrease_slivers(user, siteList, slice, count, noAct=False):
Scott Bakere4deddd2014-04-14 16:48:05 -0700572 sitesChanged = {}
Scott Bakerb85c26b2014-04-18 11:25:32 -0700573 sliverList ={}
Scott Bakerbd57a432014-04-14 16:12:15 -0700574 if siteList:
575 siteNames = [site.name for site in siteList]
576 else:
577 siteNames = None
578
579 for sliver in slice.slivers.all():
Scott Bakerb85c26b2014-04-18 11:25:32 -0700580 if(not siteNames) or (sliver.node.site.name in siteNames):
581 node = sliver.node
582 sliverList[sliver.name]=node.name
Scott Bakerbd57a432014-04-14 16:12:15 -0700583
Scott Bakerb85c26b2014-04-18 11:25:32 -0700584 for key in sliverList:
585 if count>0:
Scott Bakerf1b65c02014-04-21 01:03:17 -0700586 sliver = Sliver.objects.filter(name=key)[0]
Scott Bakerb85c26b2014-04-18 11:25:32 -0700587 sliver.delete()
588 print "deleting sliver",sliverList[key],"at node",sliver.node.name
589 count=count-1
Scott Bakerf1b65c02014-04-21 01:03:17 -0700590 sitesChanged[sliver.node.site.name] = sitesChanged.get(sliver.node.site.name,0) - 1
Scott Bakerb85c26b2014-04-18 11:25:32 -0700591
Scott Bakere4deddd2014-04-14 16:48:05 -0700592 return sitesChanged
593
Scott Bakerb85c26b2014-04-18 11:25:32 -0700594class TenantDeleteSliceView(View):
595 def post(self,request):
596 sliceName = request.POST.get("sliceName",None)
597 slice = Slice.objects.get(name=sliceName)
Scott Baker168b7412014-05-16 14:25:41 -0700598 #print slice, slice.id
Scott Bakerb85c26b2014-04-18 11:25:32 -0700599 sliceToDel=Slice(name=sliceName, id=slice.id)
600 sliceToDel.delete()
Scott Baker4b0cedd2014-04-22 21:01:21 -0700601 return HttpResponse("Slice deleted")
Scott Bakerb85c26b2014-04-18 11:25:32 -0700602
Scott Baker6253dfc2014-04-14 09:29:30 -0700603class TenantAddOrRemoveSliverView(View):
Scott Bakere4deddd2014-04-14 16:48:05 -0700604 """ Add or remove slivers from a Slice
605
606 Arguments:
607 siteName - name of site. If not specified, PlanetStack will pick the
608 best site.,
609 actionToDo - [add | rem]
610 count - number of slivers to add or remove
611 sliceName - name of slice
Scott Bakera880bb52014-04-14 23:52:48 -0700612 noAct - if set, no changes will be made to db, but result will still
613 show which sites would have been modified.
Scott Bakere4deddd2014-04-14 16:48:05 -0700614
615 Returns:
616 Dictionary of sites that were modified, and the count of nodes
617 that were added or removed at each site.
618 """
Scott Baker6253dfc2014-04-14 09:29:30 -0700619 def post(self, request, *args, **kwargs):
Scott Bakerbd57a432014-04-14 16:12:15 -0700620 siteName = request.POST.get("siteName", None)
621 actionToDo = request.POST.get("actionToDo", None)
622 count = int(request.POST.get("count","0"))
623 sliceName = request.POST.get("slice", None)
Scott Bakere4deddd2014-04-14 16:48:05 -0700624 noAct = request.POST.get("noAct", False)
Scott Bakerbd57a432014-04-14 16:12:15 -0700625
626 if not sliceName:
627 return HttpResponseServerError("No slice name given")
628
629 slice = Slice.objects.get(name=sliceName)
630
631 if siteName:
632 siteList = [Site.objects.get(name=siteName)]
633 else:
634 siteList = None
Scott Baker6253dfc2014-04-14 09:29:30 -0700635
636 if (actionToDo == "add"):
Scott Bakerbd57a432014-04-14 16:12:15 -0700637 user_ip = request.GET.get("ip", get_ip(request))
638 if (siteList is None):
639 siteList = tenant_pick_sites(user, user_ip, slice, count)
640
Scott Bakerfe6a7e12014-04-15 10:58:31 -0700641 sitesChanged = slice_increase_slivers(request.user, user_ip, siteList, slice, count, noAct)
Scott Baker6253dfc2014-04-14 09:29:30 -0700642 elif (actionToDo == "rem"):
Scott Bakerfe6a7e12014-04-15 10:58:31 -0700643 sitesChanged = slice_decrease_slivers(request.user, siteList, slice, count, noAct)
Scott Bakerbd57a432014-04-14 16:12:15 -0700644 else:
645 return HttpResponseServerError("Unknown actionToDo %s" % actionToDo)
646
Scott Bakere4deddd2014-04-14 16:48:05 -0700647 return HttpResponse(json.dumps(sitesChanged), mimetype='application/javascript')
Scott Bakerbd57a432014-04-14 16:12:15 -0700648
649 def get(self, request, *args, **kwargs):
650 request.POST = request.GET
651 return self.post(request, *args, **kwargs) # for testing REST in browser
652 #return HttpResponseServerError("GET is not supported")
653
654class TenantPickSitesView(View):
655 """ primarily just for testing purposes """
656 def get(self, request, *args, **kwargs):
657 count = request.GET.get("count","0")
658 slice = request.GET.get("slice",None)
659 if slice:
660 slice = Slice.objects.get(name=slice)
661 ip = request.GET.get("ip", get_ip(request))
662 sites = tenant_pick_sites(request.user, user_ip=ip, count=0, slice=slice)
663 sites = [x.name for x in sites]
664 return HttpResponse(json.dumps(sites), mimetype='application/javascript')
Scott Baker6253dfc2014-04-14 09:29:30 -0700665
Scott Baker771819b2014-03-19 22:10:17 -0700666class DashboardSummaryAjaxView(View):
667 def get(self, request, **kwargs):
Scott Baker0017c102014-04-18 10:48:31 -0700668 def avg(x):
Scott Baker111702b2014-05-15 16:17:16 -0700669 if len(x)==0:
670 return 0
Scott Baker0017c102014-04-18 10:48:31 -0700671 return float(sum(x))/len(x)
672
673 sites = getCDNOperatorData().values()
674
675 sites = [site for site in sites if site["numHPCSlivers"]>0]
676
677 total_slivers = sum( [site["numHPCSlivers"] for site in sites] )
678 total_bandwidth = sum( [site["bandwidth"] for site in sites] )
679 average_cpu = int(avg( [site["load"] for site in sites] ))
680
681 result= {"total_slivers": total_slivers,
682 "total_bandwidth": total_bandwidth,
683 "average_cpu": average_cpu}
684
685 return HttpResponse(json.dumps(result), mimetype='application/javascript')
Scott Baker771819b2014-03-19 22:10:17 -0700686
687class DashboardAddOrRemoveSliverView(View):
Scott Bakerfe6a7e12014-04-15 10:58:31 -0700688 # TODO: deprecate this view in favor of using TenantAddOrRemoveSliverView
689
Scott Baker771819b2014-03-19 22:10:17 -0700690 def post(self, request, *args, **kwargs):
Scott Bakerfe6a7e12014-04-15 10:58:31 -0700691 siteName = request.POST.get("site", None)
Scott Baker771819b2014-03-19 22:10:17 -0700692 actionToDo = request.POST.get("actionToDo", "0")
693
Scott Bakerfe6a7e12014-04-15 10:58:31 -0700694 siteList = [Site.objects.get(name=siteName)]
695 slice = Slice.objects.get(name="HyperCache")
696
Scott Baker771819b2014-03-19 22:10:17 -0700697 if (actionToDo == "add"):
Scott Bakerfe6a7e12014-04-15 10:58:31 -0700698 user_ip = request.GET.get("ip", get_ip(request))
699 slice_increase_slivers(request.user, user_ip, siteList, slice, 1)
Scott Baker771819b2014-03-19 22:10:17 -0700700 elif (actionToDo == "rem"):
Scott Bakerfe6a7e12014-04-15 10:58:31 -0700701 slice_decrease_slivers(request.user, siteList, slice, 1)
Scott Baker771819b2014-03-19 22:10:17 -0700702
703 print '*' * 50
704 print 'Ask for site: ' + siteName + ' to ' + actionToDo + ' another HPC Sliver'
705 return HttpResponse('This is POST request ')
Siobhan Tully06d2c032014-02-23 23:48:52 -0500706
707class DashboardAjaxView(View):
708 def get(self, request, **kwargs):
709 return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript')
Scott Baker771819b2014-03-19 22:10:17 -0700710
711class DashboardAnalyticsAjaxView(View):
712 def get(self, request, name="hello_world", **kwargs):
713 if (name == "hpcSummary"):
714 return HttpResponse(json.dumps(hpc_wizard.get_hpc_wizard().get_summary_for_view()), mimetype='application/javascript')
715 elif (name == "hpcUserSite"):
Scott Baker2c3cb642014-05-19 17:55:56 -0700716 return HttpResponse(json.dumps(getDashboardContext(request.user, tableFormat=True)), mimetype='application/javascript')
Scott Baker771819b2014-03-19 22:10:17 -0700717 elif (name == "hpcMap"):
718 return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript')
719 elif (name == "bigquery"):
720 (mimetype, data) = DoPlanetStackAnalytics(request)
721 return HttpResponse(data, mimetype=mimetype)
722 else:
723 return HttpResponse(json.dumps("Unknown"), mimetype='application/javascript')
724
Scott Baker2c3cb642014-05-19 17:55:56 -0700725class DashboardCustomize(View):
726 def post(self, request, *args, **kwargs):
727 dashboards = request.POST.get("dashboards", None)
728 if not dashboards:
Scott Baker2870e272014-05-19 18:14:54 -0700729 dashboards=[]
730 else:
731 dashboards = [x.strip() for x in dashboards.split(",")]
732 dashboards = [DashboardView.objects.get(name=x) for x in dashboards]
Scott Baker2c3cb642014-05-19 17:55:56 -0700733
734 request.user.dashboardViews.all().delete()
735
736 for i,dashboard in enumerate(dashboards):
737 udbv = UserDashboardView(user=request.user, dashboardView=dashboard, order=i)
738 udbv.save()
739
740 return HttpResponse("updated")
741