blob: 4dcca7968c8cd1a324bda0055d3f219c002123eb [file] [log] [blame]
Siobhan Tullycf04fb62014-01-11 11:25:57 -05001#views.py
Scott Bakerbd57a432014-04-14 16:12:15 -07002import functools
3import math
Scott Baker771819b2014-03-19 22:10:17 -07004import os
5import sys
Siobhan Tully06d2c032014-02-23 23:48:52 -05006from django.views.generic import TemplateView, View
7import datetime
Scott Baker771819b2014-03-19 22:10:17 -07008from pprint import pprint
9import json
Scott Baker168b7412014-05-16 14:25:41 -070010from syndicate.models import *
Scott Baker537d8232014-04-08 23:40:56 -070011from core.models import *
Scott Bakere9f4ec92014-04-29 11:29:45 -070012from hpc.models import ContentProvider
Scott Bakerbd57a432014-04-14 16:12:15 -070013from operator import attrgetter
Scott Baker186ae612014-05-19 10:42:51 -070014from django import template
Scott Bakerbd57a432014-04-14 16:12:15 -070015from django.views.decorators.csrf import csrf_exempt
Scott Baker60fd8cd2014-05-21 18:02:44 -070016from django.http import HttpResponse, HttpResponseServerError, HttpResponseForbidden
Scott Baker537d8232014-04-08 23:40:56 -070017from django.core import urlresolvers
Scott Bakerbd57a432014-04-14 16:12:15 -070018from django.contrib.gis.geoip import GeoIP
19from ipware.ip import get_ip
Scott Baker2c3cb642014-05-19 17:55:56 -070020from operator import itemgetter, attrgetter
Scott Baker771819b2014-03-19 22:10:17 -070021import traceback
Scott Baker6253dfc2014-04-14 09:29:30 -070022import socket
Siobhan Tully06d2c032014-02-23 23:48:52 -050023
Scott Bakera880bb52014-04-14 23:52:48 -070024BLESSED_SITES = ["Stanford", "Washington", "Princeton", "GeorgiaTech", "MaxPlanck"]
25
Scott Baker771819b2014-03-19 22:10:17 -070026if os.path.exists("/home/smbaker/projects/vicci/cdn/bigquery"):
27 sys.path.append("/home/smbaker/projects/vicci/cdn/bigquery")
28else:
29 sys.path.append("/opt/planetstack/hpc_wizard")
30import hpc_wizard
Scott Baker75095b62014-04-21 17:32:09 -070031from planetstack_analytics import DoPlanetStackAnalytics, PlanetStackAnalytics, RED_LOAD, BLUE_LOAD
Siobhan Tullycf04fb62014-01-11 11:25:57 -050032
33class DashboardWelcomeView(TemplateView):
34 template_name = 'admin/dashboard/welcome.html'
35
36 def get(self, request, *args, **kwargs):
37 context = self.get_context_data(**kwargs)
Scott Baker2c3cb642014-05-19 17:55:56 -070038 context = getDashboardContext(request.user, context)
Siobhan Tullycf04fb62014-01-11 11:25:57 -050039 return self.render_to_response(context=context)
Siobhan Tullye18b3442014-02-23 14:23:34 -050040
Scott Baker2c3cb642014-05-19 17:55:56 -070041class DashboardDynamicView(TemplateView):
Scott Baker186ae612014-05-19 10:42:51 -070042 head_template = r"""{% extends "admin/dashboard/dashboard_base.html" %}
43 {% load admin_static %}
44 {% block content %}
45 """
46
47 tail_template = r"{% endblock %}"
48
Scott Baker2c3cb642014-05-19 17:55:56 -070049 def get(self, request, name="root", *args, **kwargs):
Scott Baker186ae612014-05-19 10:42:51 -070050 context = self.get_context_data(**kwargs)
Scott Baker2c3cb642014-05-19 17:55:56 -070051 context = getDashboardContext(request.user, context)
Scott Baker186ae612014-05-19 10:42:51 -070052
Scott Baker2c3cb642014-05-19 17:55:56 -070053 if name=="root":
54 return self.multiDashboardView(request, context)
55 else:
56 return self.singleDashboardView(request, name, context)
57
58 def readDashboard(self, fn):
59 try:
60 template= open("/opt/planetstack/templates/admin/dashboard/%s.html" % fn, "r").read()
61 if (fn=="tenant"):
Scott Baker7237cdc2014-05-21 16:55:18 -070062 # fix for tenant view - it writes html to a div called tabs-5
Scott Baker2c3cb642014-05-19 17:55:56 -070063 template = '<div id="tabs-5"></div>' + template
Scott Baker7237cdc2014-05-21 16:55:18 -070064 if (fn=="slice_interactions"):
65 # fix for slice_interactions - it gives its container div a 40px
66 # margin, and then positions it's header at -40px
Scott Baker2de3f8f2014-05-21 16:56:58 -070067 template = '<div id="tabs-4">' + template + '</div>'
Scott Baker2c3cb642014-05-19 17:55:56 -070068 return template
69 except:
70 return "failed to open %s" % fn
71
72 def multiDashboardView(self, request, context):
Scott Bakerfc06e782014-05-19 10:52:10 -070073 head_template = self.head_template
74 tail_template = self.tail_template
75
Scott Baker2c3cb642014-05-19 17:55:56 -070076 body = """
77 <div id="hometabs" >
78 <ul id="suit_form_tabs" class="nav nav-tabs nav-tabs-suit" data-tab-prefix="suit-tab">
79 """
Scott Bakerfc06e782014-05-19 10:52:10 -070080
Scott Baker2c3cb642014-05-19 17:55:56 -070081 dashboards = request.user.get_dashboards()
Scott Bakerfc06e782014-05-19 10:52:10 -070082
Scott Baker2c3cb642014-05-19 17:55:56 -070083 # customize is a special dashboard they always get
84 customize = DashboardView.objects.filter(name="Customize")
85 if customize:
86 dashboards.append(customize[0])
Scott Baker186ae612014-05-19 10:42:51 -070087
Scott Baker2c3cb642014-05-19 17:55:56 -070088 for i,view in enumerate(dashboards):
89 body = body + '<li><a href="#dashtab-%d">%s</a></li>\n' % (i, view.name)
Scott Baker186ae612014-05-19 10:42:51 -070090
Scott Baker2c3cb642014-05-19 17:55:56 -070091 body = body + "</ul>\n"
92
93 for i,view in enumerate(dashboards):
94 url = view.url
95 body = body + '<div id="dashtab-%d">\n' % i
96 if url.startswith("template:"):
97 fn = url[9:]
98 body = body + self.readDashboard(fn)
99 body = body + '</div>\n'
100
101 body=body+"</div>\n"
102
103 t = template.Template(head_template + body + self.tail_template)
Scott Baker186ae612014-05-19 10:42:51 -0700104
105 response_kwargs = {}
106 response_kwargs.setdefault('content_type', self.content_type)
107 return self.response_class(
108 request = request,
109 template = t,
110 context = context,
111 **response_kwargs)
112
Scott Baker2c3cb642014-05-19 17:55:56 -0700113 def singleDashboardView(self, request, name, context):
114 head_template = self.head_template
115 tail_template = self.tail_template
116
117 t = template.Template(head_template + self.readDashboard(fn) + self.tail_template)
118
119 response_kwargs = {}
120 response_kwargs.setdefault('content_type', self.content_type)
121 return self.response_class(
122 request = request,
123 template = t,
124 context = context,
125 **response_kwargs)
126
127def getDashboardContext(user, context={}, tableFormat = False):
128 context = {}
Scott Baker771819b2014-03-19 22:10:17 -0700129
130 userSliceData = getSliceInfo(user)
131 if (tableFormat):
Scott Baker2c3cb642014-05-19 17:55:56 -0700132 context['userSliceInfo'] = userSliceTableFormatter(userSliceData)
Scott Baker771819b2014-03-19 22:10:17 -0700133 else:
Scott Baker2c3cb642014-05-19 17:55:56 -0700134 context['userSliceInfo'] = userSliceData
135 context['cdnData'] = getCDNOperatorData(wait=False)
136 context['cdnContentProviders'] = getCDNContentProviderData()
137
138 (dashboards, unusedDashboards)= getDashboards(user)
139 unusedDashboards=[x for x in unusedDashboards if x!="Customize"]
140 context['dashboards'] = dashboards
141 context['unusedDashboards'] = unusedDashboards
142
143 return context
144
145def getDashboards(user):
146 #dashboards = sorted(list(user.dashboardViews.all()), key=attrgetter('order'))
147 dashboards = user.get_dashboards()
148
149 dashboard_names = [d.name for d in dashboards]
150
151 unused_dashboard_names = []
152 for dashboardView in DashboardView.objects.all():
153 if not dashboardView.name in dashboard_names:
154 unused_dashboard_names.append(dashboardView.name)
155
156 return (dashboard_names, unused_dashboard_names)
Scott Baker771819b2014-03-19 22:10:17 -0700157
Scott Baker537d8232014-04-08 23:40:56 -0700158class TenantCreateSlice(View):
159 def post(self, request, *args, **kwargs):
160 sliceName = request.POST.get("sliceName", "0")
161 serviceClass = request.POST.get("serviceClass", "0")
162 imageName = request.POST.get("imageName", "0")
163 actionToDo = request.POST.get("actionToDo", "0")
Scott Baker168b7412014-05-16 14:25:41 -0700164 #network = request.POST.get("network","0")
Scott Bakeredf2f672014-04-22 10:17:37 -0700165 mountDataSets = request.POST.get("mountDataSets","0")
Scott Baker537d8232014-04-08 23:40:56 -0700166 if (actionToDo == "add"):
167 serviceClass = ServiceClass.objects.get(name=serviceClass)
168 site = request.user.site
Scott Baker4b0cedd2014-04-22 21:01:21 -0700169 image = Image.objects.get(name=imageName)
Scott Baker168b7412014-05-16 14:25:41 -0700170 newSlice = Slice(name=sliceName,serviceClass=serviceClass,site=site,imagePreference=image,mountDataSets=mountDataSets)
Scott Baker537d8232014-04-08 23:40:56 -0700171 newSlice.save()
Scott Baker4b0cedd2014-04-22 21:01:21 -0700172 return HttpResponse("Slice created")
Scott Baker537d8232014-04-08 23:40:56 -0700173
Scott Bakeredf2f672014-04-22 10:17:37 -0700174class TenantUpdateSlice(View):
175 def post(self, request, *args, **kwargs):
176 sliceName = request.POST.get("sliceName", "0")
177 serviceClass = request.POST.get("serviceClass", "0")
178 imageName = request.POST.get("imageName", "0")
179 actionToDo = request.POST.get("actionToDo", "0")
Scott Baker168b7412014-05-16 14:25:41 -0700180 #network = request.POST.get("network","0")
Scott Baker4b0cedd2014-04-22 21:01:21 -0700181 dataSet = request.POST.get("dataSet","0")
182 slice = Slice.objects.all()
183 for entry in slice:
184 serviceClass = ServiceClass.objects.get(name=serviceClass)
185 if(entry.name==sliceName):
186 if (actionToDo == "update"):
187 setattr(entry,'serviceClass',serviceClass)
188 setattr(entry,'imagePreference',imageName)
Scott Baker168b7412014-05-16 14:25:41 -0700189 #setattr(entry,'network',network)
Scott Baker4b0cedd2014-04-22 21:01:21 -0700190 setattr(entry,'mountDataSets',dataSet)
191 entry.save()
192 break
193 return HttpResponse("Slice updated")
194
Scott Baker537d8232014-04-08 23:40:56 -0700195def getTenantSliceInfo(user, tableFormat = False):
196 tenantSliceDetails = {}
197 tenantSliceData = getTenantInfo(user)
198 tenantServiceClassData = getServiceClassInfo(user)
199 if (tableFormat):
200 tenantSliceDetails['userSliceInfo'] = userSliceTableFormatter(tenantSliceData)
201 tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData)
202 else:
203 tenantSliceDetails['userSliceInfo'] = tenantSliceData
204 tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData)
205 tenantSliceDetails['image']=userSliceTableFormatter(getImageInfo(user))
Scott Baker168b7412014-05-16 14:25:41 -0700206 #tenantSliceDetails['network']=userSliceTableFormatter(getNetworkInfo(user))
Scott Baker537d8232014-04-08 23:40:56 -0700207 tenantSliceDetails['deploymentSites']=userSliceTableFormatter(getDeploymentSites())
Scott Bakerb85c26b2014-04-18 11:25:32 -0700208 tenantSliceDetails['sites'] = userSliceTableFormatter(getTenantSitesInfo())
209 tenantSliceDetails['mountDataSets'] = userSliceTableFormatter(getMountDataSets())
Scott Baker168b7412014-05-16 14:25:41 -0700210 tenantSliceDetails['publicKey'] = getPublicKey(user)
Scott Baker537d8232014-04-08 23:40:56 -0700211 return tenantSliceDetails
212
Scott Baker537d8232014-04-08 23:40:56 -0700213def getTenantInfo(user):
214 slices =Slice.objects.all()
215 userSliceInfo = []
216 for entry in slices:
217 sliceName = Slice.objects.get(id=entry.id).name
218 slice = Slice.objects.get(name=Slice.objects.get(id=entry.id).name)
219 sliceServiceClass = entry.serviceClass.name
220 preferredImage = entry.imagePreference
Scott Bakeredf2f672014-04-22 10:17:37 -0700221 sliceDataSet = entry.mountDataSets
Scott Baker168b7412014-05-16 14:25:41 -0700222 #sliceNetwork = entry.network
Scott Baker537d8232014-04-08 23:40:56 -0700223 numSliver = 0
224 sliceImage=""
225 sliceSite = {}
Scott Bakeredf2f672014-04-22 10:17:37 -0700226 sliceNode = {}
227 sliceInstance= {}
Scott Baker537d8232014-04-08 23:40:56 -0700228 for sliver in slice.slivers.all():
Scott Bakera880bb52014-04-14 23:52:48 -0700229 if sliver.node.site.name in BLESSED_SITES:
Scott Bakera880bb52014-04-14 23:52:48 -0700230 sliceSite[sliver.node.site.name] = sliceSite.get(sliver.node.site.name,0) + 1
231 sliceImage = sliver.image.name
Scott Bakerb37ce332014-04-22 22:57:40 -0700232 sliceNode[str(sliver)] = sliver.node.name
Scott Bakeredf2f672014-04-22 10:17:37 -0700233 numSliver = sum(sliceSite.values())
Scott Bakerb85c26b2014-04-18 11:25:32 -0700234 numSites = len(sliceSite)
Scott Baker168b7412014-05-16 14:25:41 -0700235 userSliceInfo.append({'sliceName': sliceName,'sliceServiceClass': sliceServiceClass,'preferredImage':preferredImage,'numOfSites':numSites, 'sliceSite':sliceSite,'sliceImage':sliceImage,'numOfSlivers':numSliver,'sliceDataSet':sliceDataSet,'instanceNodePair':sliceNode})
Scott Baker537d8232014-04-08 23:40:56 -0700236 return userSliceInfo
237
238def getTenantSitesInfo():
239 tenantSiteInfo=[]
240 for entry in Site.objects.all():
Scott Bakera880bb52014-04-14 23:52:48 -0700241 if entry.name in BLESSED_SITES:
242 tenantSiteInfo.append({'siteName':entry.name})
Scott Baker537d8232014-04-08 23:40:56 -0700243 return tenantSiteInfo
244
Scott Baker771819b2014-03-19 22:10:17 -0700245def userSliceTableFormatter(data):
246# pprint(data)
247 formattedData = {
248 'rows' : data
249 }
250 return formattedData
251
Scott Baker168b7412014-05-16 14:25:41 -0700252def getPublicKey(user):
253 users=User.objects.all()
254 for key in users:
255 if (str(key.email)==str(user)):
256 sshKey = key.public_key
257 return sshKey
258
Scott Baker537d8232014-04-08 23:40:56 -0700259def getServiceClassInfo(user):
260 serviceClassList = ServiceClass.objects.all()
261 sliceInfo = []
262 for entry in serviceClassList:
263 sliceInfo.append({'serviceClass':entry.name})
264 return sliceInfo
265
266def getImageInfo(user):
Scott Baker4b0cedd2014-04-22 21:01:21 -0700267 imageList = Image.objects.all()
268 #imageList = ['Fedora 16 LXC rev 1.3','Hadoop','MPI']
Scott Baker537d8232014-04-08 23:40:56 -0700269 imageInfo = []
270 for imageEntry in imageList:
Scott Baker4b0cedd2014-04-22 21:01:21 -0700271 imageInfo.append({'Image':imageEntry.name})
272 #imageInfo.append({'Image':imageEntry})
Scott Baker537d8232014-04-08 23:40:56 -0700273 return imageInfo
274
Scott Baker168b7412014-05-16 14:25:41 -0700275def createPrivateVolume(user, sliceName):
276 caps = Volume.CAP_READ_DATA | Volume.CAP_WRITE_DATA | Volume.CAP_HOST_DATA
277 getattr(Volume.default_gateway_caps,"read data") | \
278 getattr(Volume.default_gateway_caps,"write data") | \
279 getattr(Volume.default_gateway_caps,"host files")
280 v = Volume(name="private_" + sliceName, owner_id=user, description="private volume for %s" % sliceName, blocksize=61440, private=True, archive=False, default_gateway_caps = caps)
281 v.save()
282
283SYNDICATE_REPLICATE_PORTNUM = 1025
284
285def get_free_port():
286 inuse={}
287 inuse[SYNDICATE_REPLICATE_PORTNUM] = True
288 for vs in VolumeSlice.objects.all():
289 inuse[vs.peer_portnum]=True
290 inuse[vs.replicate_portnum]=True
291 for network in Network.objects.all():
292 network_ports = [x.strip() for x in network.ports.split(",")]
293 for network_port in network_ports:
294 try:
295 inuse[int(network_port)] = True
296 except:
297 # in case someone has put a malformed port number in the list
298 pass
299 for i in range(1025, 65535):
300 if not inuse.get(i,False):
301 return i
302 return False
303
304def mountVolume(sliceName, volumeName, readWrite):
305 slice = Slice.objects.get(name=sliceName)
306 volume = Volume.objects.get(name=volumeName)
307 # choose some unused port numbers
308 flags = Volume.CAP_READ_DATA
309 if readWrite:
310 flags = flags | Volume.CAP_WRITE_DATA
311 vs = VolumeSlice(volume_id = volume, slice_id = slice, gateway_caps=flags, peer_portnum = get_free_port(), replicate_portnum = SYNDICATE_REPLICATE_PORTNUM)
312 vs.save()
313
314def hasPrivateVolume(sliceName):
315 slice = Slice.objects.get(name=sliceName)
316 for vs in VolumeSlice.objects.filter(slice_id=slice):
317 if vs.volume_id.private:
318 return True
319 return False
320
Scott Bakerb85c26b2014-04-18 11:25:32 -0700321def getMountDataSets():
Scott Baker168b7412014-05-16 14:25:41 -0700322 dataSetInfo=[]
323 for volume in Volume.objects.all():
324 if not volume.private:
325 dataSetInfo.append({'DataSet': volume.name})
326
Scott Bakerb85c26b2014-04-18 11:25:32 -0700327 return dataSetInfo
328
Scott Baker537d8232014-04-08 23:40:56 -0700329def getNetworkInfo(user):
330 #networkList = Network.objects.all()
331 networkList = ['Private Only','Private and Publicly Routable']
332 networkInfo = []
333 for networkEntry in networkList:
334 #networkInfo.append({'Network':networkEntry.name})
335 networkInfo.append({'Network':networkEntry})
336 return networkInfo
337
338def getDeploymentSites():
339 deploymentList = Deployment.objects.all()
340 deploymentInfo = []
341 for entry in deploymentList:
342 deploymentInfo.append({'DeploymentSite':entry.name})
343 return deploymentInfo
344
Scott Baker771819b2014-03-19 22:10:17 -0700345def getSliceInfo(user):
Siobhan Tully06d2c032014-02-23 23:48:52 -0500346 sliceList = Slice.objects.all()
Scott Baker771819b2014-03-19 22:10:17 -0700347 slicePrivs = SlicePrivilege.objects.filter(user=user)
Siobhan Tully06d2c032014-02-23 23:48:52 -0500348 userSliceInfo = []
349 for entry in slicePrivs:
Siobhan Tullye18b3442014-02-23 14:23:34 -0500350
Scott Baker771819b2014-03-19 22:10:17 -0700351 slicename = Slice.objects.get(id=entry.slice.id).name
Scott Baker537d8232014-04-08 23:40:56 -0700352 slice = Slice.objects.get(name=Slice.objects.get(id=entry.slice.id).name)
353 sliverList=Sliver.objects.all()
354 sites_used = {}
355 for sliver in slice.slivers.all():
356 #sites_used['deploymentSites'] = sliver.node.deployment.name
357 # sites_used[sliver.image.name] = sliver.image.name
358 sites_used[sliver.node.site.name] = sliver.numberCores
Scott Baker771819b2014-03-19 22:10:17 -0700359 sliceid = Slice.objects.get(id=entry.slice.id).id
Siobhan Tully06d2c032014-02-23 23:48:52 -0500360 try:
Scott Baker771819b2014-03-19 22:10:17 -0700361 sliverList = Sliver.objects.filter(slice=entry.slice.id)
362 siteList = {}
363 for x in sliverList:
364 if x.node.site not in siteList:
365 siteList[x.node.site] = 1
366 slivercount = len(sliverList)
367 sitecount = len(siteList)
Siobhan Tully06d2c032014-02-23 23:48:52 -0500368 except:
Scott Baker771819b2014-03-19 22:10:17 -0700369 traceback.print_exc()
370 slivercount = 0
371 sitecount = 0
Siobhan Tully06d2c032014-02-23 23:48:52 -0500372
Scott Baker771819b2014-03-19 22:10:17 -0700373 userSliceInfo.append({'slicename': slicename, 'sliceid':sliceid,
Scott Baker537d8232014-04-08 23:40:56 -0700374 'sitesUsed':sites_used,
375 'role': SliceRole.objects.get(id=entry.role.id).role,
376 'slivercount': slivercount,
377 'sitecount':sitecount})
Scott Baker771819b2014-03-19 22:10:17 -0700378
Siobhan Tully06d2c032014-02-23 23:48:52 -0500379 return userSliceInfo
380
Scott Bakere9f4ec92014-04-29 11:29:45 -0700381def getCDNContentProviderData():
382 cps = []
383 for dm_cp in ContentProvider.objects.all():
384 cp = {"name": dm_cp.name,
385 "account": dm_cp.account}
386 cps.append(cp)
387
388 return cps
389
Scott Baker75095b62014-04-21 17:32:09 -0700390def getCDNOperatorData(randomizeData = False, wait=True):
391 HPC_SLICE_NAME = "HyperCache"
392
Scott Baker0017c102014-04-18 10:48:31 -0700393 bq = PlanetStackAnalytics()
Siobhan Tully06d2c032014-02-23 23:48:52 -0500394
Scott Baker4a4472d2014-05-13 17:07:46 -0700395 rows = bq.get_cached_query_results(bq.compose_cached_query(), wait)
Siobhan Tully06d2c032014-02-23 23:48:52 -0500396
Scott Bakeredf2f672014-04-22 10:17:37 -0700397 # wait=False on the first time the Dashboard is opened. This means we might
398 # not have any rows yet. The dashboard code polls every 30 seconds, so it
399 # will eventually pick them up.
Scott Baker0017c102014-04-18 10:48:31 -0700400
Scott Baker75095b62014-04-21 17:32:09 -0700401 if rows:
Scott Baker696e2662014-04-24 17:03:29 -0700402 rows = bq.postprocess_results(rows, filter={"event": "hpc_heartbeat"}, maxi=["cpu"], count=["hostname"], computed=["bytes_sent/elapsed"], groupBy=["Time","site"], maxDeltaTime=80)
Scott Baker0017c102014-04-18 10:48:31 -0700403
Scott Baker75095b62014-04-21 17:32:09 -0700404 # dictionaryize the statistics rows by site name
405 stats_rows = {}
406 for row in rows:
407 stats_rows[row["site"]] = row
Scott Bakeredf2f672014-04-22 10:17:37 -0700408 else:
409 stats_rows = {}
Scott Baker0017c102014-04-18 10:48:31 -0700410
Scott Baker09fad022014-04-30 09:50:01 -0700411 slice = Slice.objects.filter(name=HPC_SLICE_NAME)
412 if slice:
413 slice_slivers = list(slice[0].slivers.all())
414 else:
415 slice_slivers = []
Scott Baker0017c102014-04-18 10:48:31 -0700416
417 new_rows = {}
Scott Baker75095b62014-04-21 17:32:09 -0700418 for site in Site.objects.all():
419 # compute number of slivers allocated in the data model
420 allocated_slivers = 0
421 for sliver in slice_slivers:
422 if sliver.node.site == site:
423 allocated_slivers = allocated_slivers + 1
424
425 stats_row = stats_rows.get(site.name,{})
426
427 max_cpu = stats_row.get("max_avg_cpu", stats_row.get("max_cpu",0))
428 cpu=float(max_cpu)/100.0
429 hotness = max(0.0, ((cpu*RED_LOAD) - BLUE_LOAD)/(RED_LOAD-BLUE_LOAD))
430
431 # format it to what that CDN Operations View is expecting
432 new_row = {"lat": float(site.location.longitude),
433 "long": float(site.location.longitude),
434 "lat": float(site.location.latitude),
Scott Baker0017c102014-04-18 10:48:31 -0700435 "health": 0,
Scott Baker75095b62014-04-21 17:32:09 -0700436 "numNodes": int(site.nodes.count()),
Scott Bakere9f4ec92014-04-29 11:29:45 -0700437 "activeHPCSlivers": int(stats_row.get("count_hostname", 0)), # measured number of slivers, from bigquery statistics
Scott Baker75095b62014-04-21 17:32:09 -0700438 "numHPCSlivers": allocated_slivers, # allocated number of slivers, from data model
439 "siteUrl": str(site.site_url),
440 "bandwidth": stats_row.get("sum_computed_bytes_sent_div_elapsed",0),
441 "load": max_cpu,
442 "hot": float(hotness)}
443 new_rows[str(site.name)] = new_row
444
445 # get rid of sites with 0 slivers that overlap other sites with >0 slivers
446 for (k,v) in new_rows.items():
447 bad=False
448 if v["numHPCSlivers"]==0:
449 for v2 in new_rows.values():
450 if (v!=v2) and (v2["numHPCSlivers"]>=0):
451 d = haversine(v["lat"],v["long"],v2["lat"],v2["long"])
452 if d<100:
453 bad=True
454 if bad:
455 del new_rows[k]
Scott Baker0017c102014-04-18 10:48:31 -0700456
457 return new_rows
Siobhan Tully06d2c032014-02-23 23:48:52 -0500458
Scott Baker168b7412014-05-16 14:25:41 -0700459def getPageSummary(request):
460 slice = request.GET.get('slice', None)
461 site = request.GET.get('site', None)
462 node = request.GET.get('node', None)
463
Scott Baker771819b2014-03-19 22:10:17 -0700464class SimulatorView(View):
465 def get(self, request, **kwargs):
466 sim = json.loads(file("/tmp/simulator.json","r").read())
467 text = "<html><head></head><body>"
468 text += "Iteration: %d<br>" % sim["iteration"]
469 text += "Elapsed since report %d<br><br>" % sim["elapsed_since_report"]
470 text += "<table border=1>"
471 text += "<tr><th>site</th><th>trend</th><th>weight</th><th>bytes_sent</th><th>hot</th></tr>"
472 for site in sim["site_load"].values():
473 text += "<tr>"
474 text += "<td>%s</td><td>%0.2f</td><td>%0.2f</td><td>%d</td><td>%0.2f</td>" % \
475 (site["name"], site["trend"], site["weight"], site["bytes_sent"], site["load_frac"])
476 text += "</tr>"
477 text += "</table>"
478 text += "</body></html>"
479 return HttpResponse(text)
480
481class DashboardUserSiteView(View):
482 def get(self, request, **kwargs):
Scott Baker2c3cb642014-05-19 17:55:56 -0700483 return HttpResponse(json.dumps(getDashboardContext(request.user, tableFormat=True)), mimetype='application/javascript')
Scott Baker771819b2014-03-19 22:10:17 -0700484
Scott Baker537d8232014-04-08 23:40:56 -0700485class TenantViewData(View):
486 def get(self, request, **kwargs):
487 return HttpResponse(json.dumps(getTenantSliceInfo(request.user, True)), mimetype='application/javascript')
488
Scott Baker75095b62014-04-21 17:32:09 -0700489def haversine(site_lat, site_lon, lat, lon):
Scott Baker75095b62014-04-21 17:32:09 -0700490 d=0
491 if lat and lon and site_lat and site_lon:
Scott Baker4b0cedd2014-04-22 21:01:21 -0700492 site_lat = float(site_lat)
493 site_lon = float(site_lon)
494 lat = float(lat)
495 lon = float(lon)
Scott Baker75095b62014-04-21 17:32:09 -0700496 R = 6378.1
497 a = math.sin( math.radians((lat - site_lat)/2.0) )**2 + math.cos( math.radians(lat) )*math.cos( math.radians(site_lat) )*(math.sin( math.radians((lon - site_lon)/2.0 ) )**2)
498 c = 2 * math.atan2( math.sqrt(a), math.sqrt(1 - a) )
499 d = R * c
500
501 return d
502
Scott Bakerbd57a432014-04-14 16:12:15 -0700503def siteSortKey(site, slice=None, count=None, lat=None, lon=None):
504 # try to pick a site we're already using
505 has_slivers_here=False
506 if slice:
507 for sliver in slice.slivers.all():
508 if sliver.node.site.name == site.name:
509 has_slivers_here=True
510
511 # Haversine method
Scott Baker75095b62014-04-21 17:32:09 -0700512 d = haversine(site.location.latitude, site.location.longitude, lat, lon)
Scott Bakerbd57a432014-04-14 16:12:15 -0700513
514 return (-has_slivers_here, d)
515
516def tenant_pick_sites(user, user_ip=None, slice=None, count=None):
517 """ Returns list of sites, sorted from most favorable to least favorable """
518 lat=None
519 lon=None
520 try:
521 client_geo = GeoIP().city(user_ip)
522 if client_geo:
523 lat=float(client_geo["latitude"])
524 lon=float(client_geo["longitude"])
525 except:
526 print "exception in geo code"
527 traceback.print_exc()
528
529 sites = Site.objects.all()
Scott Bakera880bb52014-04-14 23:52:48 -0700530 sites = [x for x in sites if x.name in BLESSED_SITES]
Scott Bakerbd57a432014-04-14 16:12:15 -0700531 sites = sorted(sites, key=functools.partial(siteSortKey, slice=slice, count=count, lat=lat, lon=lon))
532
533 return sites
534
Scott Bakerfe6a7e12014-04-15 10:58:31 -0700535def slice_increase_slivers(user, user_ip, siteList, slice, count, noAct=False):
Scott Bakere4deddd2014-04-14 16:48:05 -0700536 sitesChanged = {}
537
Scott Bakerbd57a432014-04-14 16:12:15 -0700538 # let's compute how many slivers are in use in each node of each site
539 for site in siteList:
540 site.nodeList = list(site.nodes.all())
541 for node in site.nodeList:
542 node.sliverCount = 0
Scott Baker6253dfc2014-04-14 09:29:30 -0700543 for sliver in node.slivers.all():
Scott Baker21511942014-04-15 17:44:27 -0700544 if sliver.slice.id == slice.id:
Scott Baker0017c102014-04-18 10:48:31 -0700545 node.sliverCount = node.sliverCount + 1
Scott Baker6253dfc2014-04-14 09:29:30 -0700546
Scott Bakerbd57a432014-04-14 16:12:15 -0700547 # Allocate slivers to nodes
548 # for now, assume we want to allocate all slivers from the same site
549 nodes = siteList[0].nodeList
550 while (count>0):
551 # Sort the node list by number of slivers per node, then pick the
552 # node with the least number of slivers.
553 nodes = sorted(nodes, key=attrgetter("sliverCount"))
554 node = nodes[0]
Scott Baker6253dfc2014-04-14 09:29:30 -0700555
Scott Bakerbd57a432014-04-14 16:12:15 -0700556 print "adding sliver at node", node.name, "of site", node.site.name
Scott Baker6253dfc2014-04-14 09:29:30 -0700557
Scott Bakere4deddd2014-04-14 16:48:05 -0700558 if not noAct:
559 sliver = Sliver(name=node.name,
560 slice=slice,
561 node=node,
562 image = Image.objects.all()[0],
563 creator = User.objects.get(email=user),
564 deploymentNetwork=node.deployment,
565 numberCores =1 )
566 sliver.save()
Scott Bakerbd57a432014-04-14 16:12:15 -0700567
568 node.sliverCount = node.sliverCount + 1
569
570 count = count - 1
571
Scott Bakere4deddd2014-04-14 16:48:05 -0700572 sitesChanged[node.site.name] = sitesChanged.get(node.site.name,0) + 1
573
574 return sitesChanged
575
Scott Bakerfe6a7e12014-04-15 10:58:31 -0700576def slice_decrease_slivers(user, siteList, slice, count, noAct=False):
Scott Bakere4deddd2014-04-14 16:48:05 -0700577 sitesChanged = {}
Scott Bakerb85c26b2014-04-18 11:25:32 -0700578 sliverList ={}
Scott Bakerbd57a432014-04-14 16:12:15 -0700579 if siteList:
580 siteNames = [site.name for site in siteList]
581 else:
582 siteNames = None
583
584 for sliver in slice.slivers.all():
Scott Bakerb85c26b2014-04-18 11:25:32 -0700585 if(not siteNames) or (sliver.node.site.name in siteNames):
586 node = sliver.node
587 sliverList[sliver.name]=node.name
Scott Bakerbd57a432014-04-14 16:12:15 -0700588
Scott Bakerb85c26b2014-04-18 11:25:32 -0700589 for key in sliverList:
590 if count>0:
Scott Bakerf1b65c02014-04-21 01:03:17 -0700591 sliver = Sliver.objects.filter(name=key)[0]
Scott Bakerb85c26b2014-04-18 11:25:32 -0700592 sliver.delete()
593 print "deleting sliver",sliverList[key],"at node",sliver.node.name
594 count=count-1
Scott Bakerf1b65c02014-04-21 01:03:17 -0700595 sitesChanged[sliver.node.site.name] = sitesChanged.get(sliver.node.site.name,0) - 1
Scott Bakerb85c26b2014-04-18 11:25:32 -0700596
Scott Bakere4deddd2014-04-14 16:48:05 -0700597 return sitesChanged
598
Scott Bakerb85c26b2014-04-18 11:25:32 -0700599class TenantDeleteSliceView(View):
600 def post(self,request):
601 sliceName = request.POST.get("sliceName",None)
602 slice = Slice.objects.get(name=sliceName)
Scott Baker168b7412014-05-16 14:25:41 -0700603 #print slice, slice.id
Scott Bakerb85c26b2014-04-18 11:25:32 -0700604 sliceToDel=Slice(name=sliceName, id=slice.id)
605 sliceToDel.delete()
Scott Baker4b0cedd2014-04-22 21:01:21 -0700606 return HttpResponse("Slice deleted")
Scott Bakerb85c26b2014-04-18 11:25:32 -0700607
Scott Baker6253dfc2014-04-14 09:29:30 -0700608class TenantAddOrRemoveSliverView(View):
Scott Bakere4deddd2014-04-14 16:48:05 -0700609 """ Add or remove slivers from a Slice
610
611 Arguments:
612 siteName - name of site. If not specified, PlanetStack will pick the
613 best site.,
614 actionToDo - [add | rem]
615 count - number of slivers to add or remove
616 sliceName - name of slice
Scott Bakera880bb52014-04-14 23:52:48 -0700617 noAct - if set, no changes will be made to db, but result will still
618 show which sites would have been modified.
Scott Bakere4deddd2014-04-14 16:48:05 -0700619
620 Returns:
621 Dictionary of sites that were modified, and the count of nodes
622 that were added or removed at each site.
623 """
Scott Baker6253dfc2014-04-14 09:29:30 -0700624 def post(self, request, *args, **kwargs):
Scott Bakerbd57a432014-04-14 16:12:15 -0700625 siteName = request.POST.get("siteName", None)
626 actionToDo = request.POST.get("actionToDo", None)
627 count = int(request.POST.get("count","0"))
628 sliceName = request.POST.get("slice", None)
Scott Bakere4deddd2014-04-14 16:48:05 -0700629 noAct = request.POST.get("noAct", False)
Scott Bakerbd57a432014-04-14 16:12:15 -0700630
631 if not sliceName:
632 return HttpResponseServerError("No slice name given")
633
634 slice = Slice.objects.get(name=sliceName)
635
636 if siteName:
637 siteList = [Site.objects.get(name=siteName)]
638 else:
639 siteList = None
Scott Baker6253dfc2014-04-14 09:29:30 -0700640
641 if (actionToDo == "add"):
Scott Bakerbd57a432014-04-14 16:12:15 -0700642 user_ip = request.GET.get("ip", get_ip(request))
643 if (siteList is None):
644 siteList = tenant_pick_sites(user, user_ip, slice, count)
645
Scott Bakerfe6a7e12014-04-15 10:58:31 -0700646 sitesChanged = slice_increase_slivers(request.user, user_ip, siteList, slice, count, noAct)
Scott Baker6253dfc2014-04-14 09:29:30 -0700647 elif (actionToDo == "rem"):
Scott Bakerfe6a7e12014-04-15 10:58:31 -0700648 sitesChanged = slice_decrease_slivers(request.user, siteList, slice, count, noAct)
Scott Bakerbd57a432014-04-14 16:12:15 -0700649 else:
650 return HttpResponseServerError("Unknown actionToDo %s" % actionToDo)
651
Scott Bakere4deddd2014-04-14 16:48:05 -0700652 return HttpResponse(json.dumps(sitesChanged), mimetype='application/javascript')
Scott Bakerbd57a432014-04-14 16:12:15 -0700653
654 def get(self, request, *args, **kwargs):
655 request.POST = request.GET
656 return self.post(request, *args, **kwargs) # for testing REST in browser
657 #return HttpResponseServerError("GET is not supported")
658
659class TenantPickSitesView(View):
660 """ primarily just for testing purposes """
661 def get(self, request, *args, **kwargs):
662 count = request.GET.get("count","0")
663 slice = request.GET.get("slice",None)
664 if slice:
665 slice = Slice.objects.get(name=slice)
666 ip = request.GET.get("ip", get_ip(request))
667 sites = tenant_pick_sites(request.user, user_ip=ip, count=0, slice=slice)
668 sites = [x.name for x in sites]
669 return HttpResponse(json.dumps(sites), mimetype='application/javascript')
Scott Baker6253dfc2014-04-14 09:29:30 -0700670
Scott Baker771819b2014-03-19 22:10:17 -0700671class DashboardSummaryAjaxView(View):
672 def get(self, request, **kwargs):
Scott Baker0017c102014-04-18 10:48:31 -0700673 def avg(x):
Scott Baker111702b2014-05-15 16:17:16 -0700674 if len(x)==0:
675 return 0
Scott Baker0017c102014-04-18 10:48:31 -0700676 return float(sum(x))/len(x)
677
678 sites = getCDNOperatorData().values()
679
680 sites = [site for site in sites if site["numHPCSlivers"]>0]
681
682 total_slivers = sum( [site["numHPCSlivers"] for site in sites] )
683 total_bandwidth = sum( [site["bandwidth"] for site in sites] )
684 average_cpu = int(avg( [site["load"] for site in sites] ))
685
686 result= {"total_slivers": total_slivers,
687 "total_bandwidth": total_bandwidth,
688 "average_cpu": average_cpu}
689
690 return HttpResponse(json.dumps(result), mimetype='application/javascript')
Scott Baker771819b2014-03-19 22:10:17 -0700691
692class DashboardAddOrRemoveSliverView(View):
Scott Bakerfe6a7e12014-04-15 10:58:31 -0700693 # TODO: deprecate this view in favor of using TenantAddOrRemoveSliverView
694
Scott Baker771819b2014-03-19 22:10:17 -0700695 def post(self, request, *args, **kwargs):
Scott Bakerfe6a7e12014-04-15 10:58:31 -0700696 siteName = request.POST.get("site", None)
Scott Baker771819b2014-03-19 22:10:17 -0700697 actionToDo = request.POST.get("actionToDo", "0")
698
Scott Bakerfe6a7e12014-04-15 10:58:31 -0700699 siteList = [Site.objects.get(name=siteName)]
700 slice = Slice.objects.get(name="HyperCache")
701
Scott Baker771819b2014-03-19 22:10:17 -0700702 if (actionToDo == "add"):
Scott Bakerfe6a7e12014-04-15 10:58:31 -0700703 user_ip = request.GET.get("ip", get_ip(request))
704 slice_increase_slivers(request.user, user_ip, siteList, slice, 1)
Scott Baker771819b2014-03-19 22:10:17 -0700705 elif (actionToDo == "rem"):
Scott Bakerfe6a7e12014-04-15 10:58:31 -0700706 slice_decrease_slivers(request.user, siteList, slice, 1)
Scott Baker771819b2014-03-19 22:10:17 -0700707
708 print '*' * 50
709 print 'Ask for site: ' + siteName + ' to ' + actionToDo + ' another HPC Sliver'
710 return HttpResponse('This is POST request ')
Siobhan Tully06d2c032014-02-23 23:48:52 -0500711
712class DashboardAjaxView(View):
713 def get(self, request, **kwargs):
714 return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript')
Scott Baker771819b2014-03-19 22:10:17 -0700715
716class DashboardAnalyticsAjaxView(View):
717 def get(self, request, name="hello_world", **kwargs):
718 if (name == "hpcSummary"):
719 return HttpResponse(json.dumps(hpc_wizard.get_hpc_wizard().get_summary_for_view()), mimetype='application/javascript')
720 elif (name == "hpcUserSite"):
Scott Baker2c3cb642014-05-19 17:55:56 -0700721 return HttpResponse(json.dumps(getDashboardContext(request.user, tableFormat=True)), mimetype='application/javascript')
Scott Baker771819b2014-03-19 22:10:17 -0700722 elif (name == "hpcMap"):
723 return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript')
724 elif (name == "bigquery"):
725 (mimetype, data) = DoPlanetStackAnalytics(request)
726 return HttpResponse(data, mimetype=mimetype)
727 else:
728 return HttpResponse(json.dumps("Unknown"), mimetype='application/javascript')
729
Scott Baker2c3cb642014-05-19 17:55:56 -0700730class DashboardCustomize(View):
731 def post(self, request, *args, **kwargs):
Scott Baker60fd8cd2014-05-21 18:02:44 -0700732 if request.user.isReadOnlyUser():
733 return HttpResponseForbidden("User is in read-only mode")
734
Scott Baker2c3cb642014-05-19 17:55:56 -0700735 dashboards = request.POST.get("dashboards", None)
736 if not dashboards:
Scott Baker2870e272014-05-19 18:14:54 -0700737 dashboards=[]
738 else:
739 dashboards = [x.strip() for x in dashboards.split(",")]
740 dashboards = [DashboardView.objects.get(name=x) for x in dashboards]
Scott Baker2c3cb642014-05-19 17:55:56 -0700741
742 request.user.dashboardViews.all().delete()
743
744 for i,dashboard in enumerate(dashboards):
745 udbv = UserDashboardView(user=request.user, dashboardView=dashboard, order=i)
746 udbv.save()
747
Scott Baker60fd8cd2014-05-21 18:02:44 -0700748 return HttpResponse(json.dumps("Success"), mimetype='application/javascript')
Scott Baker2c3cb642014-05-19 17:55:56 -0700749