Siobhan Tully | cf04fb6 | 2014-01-11 11:25:57 -0500 | [diff] [blame] | 1 | #views.py |
Scott Baker | bd57a43 | 2014-04-14 16:12:15 -0700 | [diff] [blame] | 2 | import functools |
| 3 | import math |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 4 | import os |
| 5 | import sys |
Siobhan Tully | 06d2c03 | 2014-02-23 23:48:52 -0500 | [diff] [blame] | 6 | from django.views.generic import TemplateView, View |
| 7 | import datetime |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 8 | from pprint import pprint |
| 9 | import json |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 10 | from core.models import * |
Scott Baker | e9f4ec9 | 2014-04-29 11:29:45 -0700 | [diff] [blame] | 11 | from hpc.models import ContentProvider |
Scott Baker | bd57a43 | 2014-04-14 16:12:15 -0700 | [diff] [blame] | 12 | from operator import attrgetter |
| 13 | from django.views.decorators.csrf import csrf_exempt |
| 14 | from django.http import HttpResponse, HttpResponseServerError |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 15 | from django.core import urlresolvers |
Scott Baker | bd57a43 | 2014-04-14 16:12:15 -0700 | [diff] [blame] | 16 | from django.contrib.gis.geoip import GeoIP |
| 17 | from ipware.ip import get_ip |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 18 | import traceback |
Scott Baker | 6253dfc | 2014-04-14 09:29:30 -0700 | [diff] [blame] | 19 | import socket |
Siobhan Tully | 06d2c03 | 2014-02-23 23:48:52 -0500 | [diff] [blame] | 20 | |
Scott Baker | a880bb5 | 2014-04-14 23:52:48 -0700 | [diff] [blame] | 21 | BLESSED_SITES = ["Stanford", "Washington", "Princeton", "GeorgiaTech", "MaxPlanck"] |
| 22 | |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 23 | if os.path.exists("/home/smbaker/projects/vicci/cdn/bigquery"): |
| 24 | sys.path.append("/home/smbaker/projects/vicci/cdn/bigquery") |
| 25 | else: |
| 26 | sys.path.append("/opt/planetstack/hpc_wizard") |
| 27 | import hpc_wizard |
Scott Baker | 75095b6 | 2014-04-21 17:32:09 -0700 | [diff] [blame] | 28 | from planetstack_analytics import DoPlanetStackAnalytics, PlanetStackAnalytics, RED_LOAD, BLUE_LOAD |
Siobhan Tully | cf04fb6 | 2014-01-11 11:25:57 -0500 | [diff] [blame] | 29 | |
| 30 | class DashboardWelcomeView(TemplateView): |
| 31 | template_name = 'admin/dashboard/welcome.html' |
| 32 | |
| 33 | def get(self, request, *args, **kwargs): |
| 34 | context = self.get_context_data(**kwargs) |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 35 | userDetails = getUserSliceInfo(request.user) |
| 36 | #context['site'] = userDetails['site'] |
Siobhan Tully | cf04fb6 | 2014-01-11 11:25:57 -0500 | [diff] [blame] | 37 | |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 38 | context['userSliceInfo'] = userDetails['userSliceInfo'] |
| 39 | context['cdnData'] = userDetails['cdnData'] |
Scott Baker | e9f4ec9 | 2014-04-29 11:29:45 -0700 | [diff] [blame] | 40 | context['cdnContentProviders'] = userDetails['cdnContentProviders'] |
Siobhan Tully | cf04fb6 | 2014-01-11 11:25:57 -0500 | [diff] [blame] | 41 | return self.render_to_response(context=context) |
Siobhan Tully | e18b344 | 2014-02-23 14:23:34 -0500 | [diff] [blame] | 42 | |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 43 | def getUserSliceInfo(user, tableFormat = False): |
| 44 | userDetails = {} |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 45 | |
| 46 | userSliceData = getSliceInfo(user) |
| 47 | if (tableFormat): |
| 48 | # pprint("******* GET USER SLICE INFO") |
| 49 | userDetails['userSliceInfo'] = userSliceTableFormatter(userSliceData) |
| 50 | else: |
| 51 | userDetails['userSliceInfo'] = userSliceData |
Scott Baker | e9f4ec9 | 2014-04-29 11:29:45 -0700 | [diff] [blame] | 52 | userDetails['cdnData'] = getCDNOperatorData(wait=False) |
| 53 | userDetails['cdnContentProviders'] = getCDNContentProviderData() |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 54 | return userDetails |
| 55 | |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 56 | class TenantCreateSlice(View): |
| 57 | def post(self, request, *args, **kwargs): |
| 58 | sliceName = request.POST.get("sliceName", "0") |
| 59 | serviceClass = request.POST.get("serviceClass", "0") |
| 60 | imageName = request.POST.get("imageName", "0") |
| 61 | actionToDo = request.POST.get("actionToDo", "0") |
Scott Baker | edf2f67 | 2014-04-22 10:17:37 -0700 | [diff] [blame] | 62 | network = request.POST.get("network","0") |
| 63 | mountDataSets = request.POST.get("mountDataSets","0") |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 64 | if (actionToDo == "add"): |
| 65 | serviceClass = ServiceClass.objects.get(name=serviceClass) |
| 66 | site = request.user.site |
Scott Baker | 4b0cedd | 2014-04-22 21:01:21 -0700 | [diff] [blame] | 67 | image = Image.objects.get(name=imageName) |
| 68 | newSlice = Slice(name=sliceName,serviceClass=serviceClass,site=site,imagePreference=image,mountDataSets=mountDataSets,network=network) |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 69 | newSlice.save() |
Scott Baker | 4b0cedd | 2014-04-22 21:01:21 -0700 | [diff] [blame] | 70 | return HttpResponse("Slice created") |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 71 | |
Scott Baker | edf2f67 | 2014-04-22 10:17:37 -0700 | [diff] [blame] | 72 | class TenantUpdateSlice(View): |
| 73 | def post(self, request, *args, **kwargs):
|
| 74 | sliceName = request.POST.get("sliceName", "0")
|
| 75 | serviceClass = request.POST.get("serviceClass", "0")
|
| 76 | imageName = request.POST.get("imageName", "0")
|
| 77 | actionToDo = request.POST.get("actionToDo", "0")
|
| 78 | network = request.POST.get("network","0")
|
Scott Baker | 4b0cedd | 2014-04-22 21:01:21 -0700 | [diff] [blame] | 79 | dataSet = request.POST.get("dataSet","0")
|
| 80 | slice = Slice.objects.all()
|
| 81 | for entry in slice:
|
| 82 | serviceClass = ServiceClass.objects.get(name=serviceClass)
|
| 83 | if(entry.name==sliceName):
|
| 84 | if (actionToDo == "update"):
|
| 85 | setattr(entry,'serviceClass',serviceClass)
|
| 86 | setattr(entry,'imagePreference',imageName)
|
| 87 | setattr(entry,'network',network)
|
| 88 | setattr(entry,'mountDataSets',dataSet)
|
| 89 | entry.save()
|
| 90 | break
|
| 91 | return HttpResponse("Slice updated")
|
| 92 |
|
Scott Baker | edf2f67 | 2014-04-22 10:17:37 -0700 | [diff] [blame] | 93 | def update_slice(sliceName,**fields): |
| 94 | slice = Slice.objects.filter(name = sliceName)
|
| 95 | for (k,v) in fields.items():
|
| 96 | setattr(slice, k, v)
|
| 97 | slice.save()
|
| 98 | return slice |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 99 | |
| 100 | def getTenantSliceInfo(user, tableFormat = False): |
| 101 | tenantSliceDetails = {} |
| 102 | tenantSliceData = getTenantInfo(user) |
| 103 | tenantServiceClassData = getServiceClassInfo(user) |
| 104 | if (tableFormat): |
| 105 | tenantSliceDetails['userSliceInfo'] = userSliceTableFormatter(tenantSliceData) |
| 106 | tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData) |
| 107 | else: |
| 108 | tenantSliceDetails['userSliceInfo'] = tenantSliceData |
| 109 | tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData) |
| 110 | tenantSliceDetails['image']=userSliceTableFormatter(getImageInfo(user)) |
| 111 | tenantSliceDetails['network']=userSliceTableFormatter(getNetworkInfo(user)) |
| 112 | tenantSliceDetails['deploymentSites']=userSliceTableFormatter(getDeploymentSites()) |
Scott Baker | b85c26b | 2014-04-18 11:25:32 -0700 | [diff] [blame] | 113 | tenantSliceDetails['sites'] = userSliceTableFormatter(getTenantSitesInfo()) |
| 114 | tenantSliceDetails['mountDataSets'] = userSliceTableFormatter(getMountDataSets()) |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 115 | return tenantSliceDetails |
| 116 | |
| 117 | |
| 118 | def getTenantInfo(user): |
| 119 | slices =Slice.objects.all() |
| 120 | userSliceInfo = [] |
| 121 | for entry in slices: |
| 122 | sliceName = Slice.objects.get(id=entry.id).name |
| 123 | slice = Slice.objects.get(name=Slice.objects.get(id=entry.id).name) |
| 124 | sliceServiceClass = entry.serviceClass.name |
| 125 | preferredImage = entry.imagePreference |
Scott Baker | edf2f67 | 2014-04-22 10:17:37 -0700 | [diff] [blame] | 126 | sliceDataSet = entry.mountDataSets |
| 127 | sliceNetwork = entry.network |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 128 | numSliver = 0 |
| 129 | sliceImage="" |
| 130 | sliceSite = {} |
Scott Baker | edf2f67 | 2014-04-22 10:17:37 -0700 | [diff] [blame] | 131 | sliceNode = {} |
| 132 | sliceInstance= {} |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 133 | for sliver in slice.slivers.all(): |
Scott Baker | a880bb5 | 2014-04-14 23:52:48 -0700 | [diff] [blame] | 134 | if sliver.node.site.name in BLESSED_SITES: |
Scott Baker | a880bb5 | 2014-04-14 23:52:48 -0700 | [diff] [blame] | 135 | sliceSite[sliver.node.site.name] = sliceSite.get(sliver.node.site.name,0) + 1 |
| 136 | sliceImage = sliver.image.name |
Scott Baker | b37ce33 | 2014-04-22 22:57:40 -0700 | [diff] [blame] | 137 | sliceNode[str(sliver)] = sliver.node.name |
Scott Baker | edf2f67 | 2014-04-22 10:17:37 -0700 | [diff] [blame] | 138 | numSliver = sum(sliceSite.values()) |
Scott Baker | b85c26b | 2014-04-18 11:25:32 -0700 | [diff] [blame] | 139 | numSites = len(sliceSite) |
Scott Baker | edf2f67 | 2014-04-22 10:17:37 -0700 | [diff] [blame] | 140 | userSliceInfo.append({'sliceName': sliceName,'sliceServiceClass': sliceServiceClass,'preferredImage':preferredImage,'numOfSites':numSites, 'sliceSite':sliceSite,'sliceImage':sliceImage,'numOfSlivers':numSliver,'sliceDataSet':sliceDataSet,'sliceNetwork':sliceNetwork, 'instanceNodePair':sliceNode}) |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 141 | return userSliceInfo |
| 142 | |
| 143 | def getTenantSitesInfo(): |
| 144 | tenantSiteInfo=[] |
| 145 | for entry in Site.objects.all(): |
Scott Baker | a880bb5 | 2014-04-14 23:52:48 -0700 | [diff] [blame] | 146 | if entry.name in BLESSED_SITES: |
| 147 | tenantSiteInfo.append({'siteName':entry.name}) |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 148 | return tenantSiteInfo |
| 149 | |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 150 | def userSliceTableFormatter(data): |
| 151 | # pprint(data) |
| 152 | formattedData = { |
| 153 | 'rows' : data |
| 154 | } |
| 155 | return formattedData |
| 156 | |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 157 | def getServiceClassInfo(user): |
| 158 | serviceClassList = ServiceClass.objects.all() |
| 159 | sliceInfo = [] |
| 160 | for entry in serviceClassList: |
| 161 | sliceInfo.append({'serviceClass':entry.name}) |
| 162 | return sliceInfo |
| 163 | |
| 164 | def getImageInfo(user): |
Scott Baker | 4b0cedd | 2014-04-22 21:01:21 -0700 | [diff] [blame] | 165 | imageList = Image.objects.all() |
| 166 | #imageList = ['Fedora 16 LXC rev 1.3','Hadoop','MPI'] |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 167 | imageInfo = [] |
| 168 | for imageEntry in imageList: |
Scott Baker | 4b0cedd | 2014-04-22 21:01:21 -0700 | [diff] [blame] | 169 | imageInfo.append({'Image':imageEntry.name}) |
| 170 | #imageInfo.append({'Image':imageEntry}) |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 171 | return imageInfo |
| 172 | |
Scott Baker | b85c26b | 2014-04-18 11:25:32 -0700 | [diff] [blame] | 173 | def getMountDataSets(): |
Scott Baker | edf2f67 | 2014-04-22 10:17:37 -0700 | [diff] [blame] | 174 | dataSetList = ['------','GenBank','LSST','LHC','NOAA','Measurement Lab','Common Crawl']
|
Scott Baker | b85c26b | 2014-04-18 11:25:32 -0700 | [diff] [blame] | 175 | dataSetInfo = []
|
| 176 | for entry in dataSetList:
|
| 177 | dataSetInfo.append({'DataSet':entry})
|
| 178 | return dataSetInfo |
| 179 | |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 180 | def getNetworkInfo(user): |
| 181 | #networkList = Network.objects.all() |
| 182 | networkList = ['Private Only','Private and Publicly Routable'] |
| 183 | networkInfo = [] |
| 184 | for networkEntry in networkList: |
| 185 | #networkInfo.append({'Network':networkEntry.name}) |
| 186 | networkInfo.append({'Network':networkEntry}) |
| 187 | return networkInfo |
| 188 | |
| 189 | def getDeploymentSites(): |
| 190 | deploymentList = Deployment.objects.all() |
| 191 | deploymentInfo = [] |
| 192 | for entry in deploymentList: |
| 193 | deploymentInfo.append({'DeploymentSite':entry.name}) |
| 194 | return deploymentInfo |
| 195 | |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 196 | def getSliceInfo(user): |
Siobhan Tully | 06d2c03 | 2014-02-23 23:48:52 -0500 | [diff] [blame] | 197 | sliceList = Slice.objects.all() |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 198 | slicePrivs = SlicePrivilege.objects.filter(user=user) |
Siobhan Tully | 06d2c03 | 2014-02-23 23:48:52 -0500 | [diff] [blame] | 199 | userSliceInfo = [] |
| 200 | for entry in slicePrivs: |
Siobhan Tully | e18b344 | 2014-02-23 14:23:34 -0500 | [diff] [blame] | 201 | |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 202 | slicename = Slice.objects.get(id=entry.slice.id).name |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 203 | slice = Slice.objects.get(name=Slice.objects.get(id=entry.slice.id).name) |
| 204 | sliverList=Sliver.objects.all() |
| 205 | sites_used = {} |
| 206 | for sliver in slice.slivers.all(): |
| 207 | #sites_used['deploymentSites'] = sliver.node.deployment.name |
| 208 | # sites_used[sliver.image.name] = sliver.image.name |
| 209 | sites_used[sliver.node.site.name] = sliver.numberCores |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 210 | sliceid = Slice.objects.get(id=entry.slice.id).id |
Siobhan Tully | 06d2c03 | 2014-02-23 23:48:52 -0500 | [diff] [blame] | 211 | try: |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 212 | sliverList = Sliver.objects.filter(slice=entry.slice.id) |
| 213 | siteList = {} |
| 214 | for x in sliverList: |
| 215 | if x.node.site not in siteList: |
| 216 | siteList[x.node.site] = 1 |
| 217 | slivercount = len(sliverList) |
| 218 | sitecount = len(siteList) |
Siobhan Tully | 06d2c03 | 2014-02-23 23:48:52 -0500 | [diff] [blame] | 219 | except: |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 220 | traceback.print_exc() |
| 221 | slivercount = 0 |
| 222 | sitecount = 0 |
Siobhan Tully | 06d2c03 | 2014-02-23 23:48:52 -0500 | [diff] [blame] | 223 | |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 224 | userSliceInfo.append({'slicename': slicename, 'sliceid':sliceid, |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 225 | 'sitesUsed':sites_used, |
| 226 | 'role': SliceRole.objects.get(id=entry.role.id).role, |
| 227 | 'slivercount': slivercount, |
| 228 | 'sitecount':sitecount}) |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 229 | |
Siobhan Tully | 06d2c03 | 2014-02-23 23:48:52 -0500 | [diff] [blame] | 230 | return userSliceInfo |
| 231 | |
Scott Baker | e9f4ec9 | 2014-04-29 11:29:45 -0700 | [diff] [blame] | 232 | def getCDNContentProviderData(): |
| 233 | cps = [] |
| 234 | for dm_cp in ContentProvider.objects.all(): |
| 235 | cp = {"name": dm_cp.name, |
| 236 | "account": dm_cp.account} |
| 237 | cps.append(cp) |
| 238 | |
| 239 | return cps |
| 240 | |
Scott Baker | 75095b6 | 2014-04-21 17:32:09 -0700 | [diff] [blame] | 241 | def getCDNOperatorData(randomizeData = False, wait=True): |
| 242 | HPC_SLICE_NAME = "HyperCache" |
| 243 | |
Scott Baker | 0017c10 | 2014-04-18 10:48:31 -0700 | [diff] [blame] | 244 | bq = PlanetStackAnalytics() |
Siobhan Tully | 06d2c03 | 2014-02-23 23:48:52 -0500 | [diff] [blame] | 245 | |
Scott Baker | 75095b6 | 2014-04-21 17:32:09 -0700 | [diff] [blame] | 246 | rows = bq.get_cached_query_results(bq.compose_latest_query(groupByFields=["%hostname", "event", "%slice"]), wait) |
Siobhan Tully | 06d2c03 | 2014-02-23 23:48:52 -0500 | [diff] [blame] | 247 | |
Scott Baker | edf2f67 | 2014-04-22 10:17:37 -0700 | [diff] [blame] | 248 | # wait=False on the first time the Dashboard is opened. This means we might |
| 249 | # not have any rows yet. The dashboard code polls every 30 seconds, so it |
| 250 | # will eventually pick them up. |
Scott Baker | 0017c10 | 2014-04-18 10:48:31 -0700 | [diff] [blame] | 251 | |
Scott Baker | 75095b6 | 2014-04-21 17:32:09 -0700 | [diff] [blame] | 252 | if rows: |
Scott Baker | 696e266 | 2014-04-24 17:03:29 -0700 | [diff] [blame] | 253 | rows = bq.postprocess_results(rows, filter={"event": "hpc_heartbeat"}, maxi=["cpu"], count=["hostname"], computed=["bytes_sent/elapsed"], groupBy=["Time","site"], maxDeltaTime=80) |
Scott Baker | 0017c10 | 2014-04-18 10:48:31 -0700 | [diff] [blame] | 254 | |
Scott Baker | 75095b6 | 2014-04-21 17:32:09 -0700 | [diff] [blame] | 255 | # dictionaryize the statistics rows by site name |
| 256 | stats_rows = {} |
| 257 | for row in rows: |
| 258 | stats_rows[row["site"]] = row |
Scott Baker | edf2f67 | 2014-04-22 10:17:37 -0700 | [diff] [blame] | 259 | else: |
| 260 | stats_rows = {} |
Scott Baker | 0017c10 | 2014-04-18 10:48:31 -0700 | [diff] [blame] | 261 | |
Scott Baker | 75095b6 | 2014-04-21 17:32:09 -0700 | [diff] [blame] | 262 | slice = Slice.objects.get(name=HPC_SLICE_NAME) |
| 263 | slice_slivers = list(slice.slivers.all()) |
Scott Baker | 0017c10 | 2014-04-18 10:48:31 -0700 | [diff] [blame] | 264 | |
| 265 | new_rows = {} |
Scott Baker | 75095b6 | 2014-04-21 17:32:09 -0700 | [diff] [blame] | 266 | for site in Site.objects.all(): |
| 267 | # compute number of slivers allocated in the data model |
| 268 | allocated_slivers = 0 |
| 269 | for sliver in slice_slivers: |
| 270 | if sliver.node.site == site: |
| 271 | allocated_slivers = allocated_slivers + 1 |
| 272 | |
| 273 | stats_row = stats_rows.get(site.name,{}) |
| 274 | |
| 275 | max_cpu = stats_row.get("max_avg_cpu", stats_row.get("max_cpu",0)) |
| 276 | cpu=float(max_cpu)/100.0 |
| 277 | hotness = max(0.0, ((cpu*RED_LOAD) - BLUE_LOAD)/(RED_LOAD-BLUE_LOAD)) |
| 278 | |
| 279 | # format it to what that CDN Operations View is expecting |
| 280 | new_row = {"lat": float(site.location.longitude), |
| 281 | "long": float(site.location.longitude), |
| 282 | "lat": float(site.location.latitude), |
Scott Baker | 0017c10 | 2014-04-18 10:48:31 -0700 | [diff] [blame] | 283 | "health": 0, |
Scott Baker | 75095b6 | 2014-04-21 17:32:09 -0700 | [diff] [blame] | 284 | "numNodes": int(site.nodes.count()), |
Scott Baker | e9f4ec9 | 2014-04-29 11:29:45 -0700 | [diff] [blame] | 285 | "activeHPCSlivers": int(stats_row.get("count_hostname", 0)), # measured number of slivers, from bigquery statistics |
Scott Baker | 75095b6 | 2014-04-21 17:32:09 -0700 | [diff] [blame] | 286 | "numHPCSlivers": allocated_slivers, # allocated number of slivers, from data model |
| 287 | "siteUrl": str(site.site_url), |
| 288 | "bandwidth": stats_row.get("sum_computed_bytes_sent_div_elapsed",0), |
| 289 | "load": max_cpu, |
| 290 | "hot": float(hotness)} |
| 291 | new_rows[str(site.name)] = new_row |
| 292 | |
| 293 | # get rid of sites with 0 slivers that overlap other sites with >0 slivers |
| 294 | for (k,v) in new_rows.items(): |
| 295 | bad=False |
| 296 | if v["numHPCSlivers"]==0: |
| 297 | for v2 in new_rows.values(): |
| 298 | if (v!=v2) and (v2["numHPCSlivers"]>=0): |
| 299 | d = haversine(v["lat"],v["long"],v2["lat"],v2["long"]) |
| 300 | if d<100: |
| 301 | bad=True |
| 302 | if bad: |
| 303 | del new_rows[k] |
Scott Baker | 0017c10 | 2014-04-18 10:48:31 -0700 | [diff] [blame] | 304 | |
| 305 | return new_rows |
Siobhan Tully | 06d2c03 | 2014-02-23 23:48:52 -0500 | [diff] [blame] | 306 | |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 307 | class SimulatorView(View): |
| 308 | def get(self, request, **kwargs): |
| 309 | sim = json.loads(file("/tmp/simulator.json","r").read()) |
| 310 | text = "<html><head></head><body>" |
| 311 | text += "Iteration: %d<br>" % sim["iteration"] |
| 312 | text += "Elapsed since report %d<br><br>" % sim["elapsed_since_report"] |
| 313 | text += "<table border=1>" |
| 314 | text += "<tr><th>site</th><th>trend</th><th>weight</th><th>bytes_sent</th><th>hot</th></tr>" |
| 315 | for site in sim["site_load"].values(): |
| 316 | text += "<tr>" |
| 317 | text += "<td>%s</td><td>%0.2f</td><td>%0.2f</td><td>%d</td><td>%0.2f</td>" % \ |
| 318 | (site["name"], site["trend"], site["weight"], site["bytes_sent"], site["load_frac"]) |
| 319 | text += "</tr>" |
| 320 | text += "</table>" |
| 321 | text += "</body></html>" |
| 322 | return HttpResponse(text) |
| 323 | |
| 324 | class DashboardUserSiteView(View): |
| 325 | def get(self, request, **kwargs): |
| 326 | return HttpResponse(json.dumps(getUserSliceInfo(request.user, True)), mimetype='application/javascript') |
| 327 | |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 328 | class TenantViewData(View): |
| 329 | def get(self, request, **kwargs): |
| 330 | return HttpResponse(json.dumps(getTenantSliceInfo(request.user, True)), mimetype='application/javascript') |
| 331 | |
Scott Baker | 75095b6 | 2014-04-21 17:32:09 -0700 | [diff] [blame] | 332 | def haversine(site_lat, site_lon, lat, lon): |
Scott Baker | 75095b6 | 2014-04-21 17:32:09 -0700 | [diff] [blame] | 333 | d=0 |
| 334 | if lat and lon and site_lat and site_lon: |
Scott Baker | 4b0cedd | 2014-04-22 21:01:21 -0700 | [diff] [blame] | 335 | site_lat = float(site_lat) |
| 336 | site_lon = float(site_lon) |
| 337 | lat = float(lat) |
| 338 | lon = float(lon) |
Scott Baker | 75095b6 | 2014-04-21 17:32:09 -0700 | [diff] [blame] | 339 | R = 6378.1 |
| 340 | a = math.sin( math.radians((lat - site_lat)/2.0) )**2 + math.cos( math.radians(lat) )*math.cos( math.radians(site_lat) )*(math.sin( math.radians((lon - site_lon)/2.0 ) )**2) |
| 341 | c = 2 * math.atan2( math.sqrt(a), math.sqrt(1 - a) ) |
| 342 | d = R * c |
| 343 | |
| 344 | return d |
| 345 | |
Scott Baker | bd57a43 | 2014-04-14 16:12:15 -0700 | [diff] [blame] | 346 | def siteSortKey(site, slice=None, count=None, lat=None, lon=None): |
| 347 | # try to pick a site we're already using |
| 348 | has_slivers_here=False |
| 349 | if slice: |
| 350 | for sliver in slice.slivers.all(): |
| 351 | if sliver.node.site.name == site.name: |
| 352 | has_slivers_here=True |
| 353 | |
| 354 | # Haversine method |
Scott Baker | 75095b6 | 2014-04-21 17:32:09 -0700 | [diff] [blame] | 355 | d = haversine(site.location.latitude, site.location.longitude, lat, lon) |
Scott Baker | bd57a43 | 2014-04-14 16:12:15 -0700 | [diff] [blame] | 356 | |
| 357 | return (-has_slivers_here, d) |
| 358 | |
| 359 | def tenant_pick_sites(user, user_ip=None, slice=None, count=None): |
| 360 | """ Returns list of sites, sorted from most favorable to least favorable """ |
| 361 | lat=None |
| 362 | lon=None |
| 363 | try: |
| 364 | client_geo = GeoIP().city(user_ip) |
| 365 | if client_geo: |
| 366 | lat=float(client_geo["latitude"]) |
| 367 | lon=float(client_geo["longitude"]) |
| 368 | except: |
| 369 | print "exception in geo code" |
| 370 | traceback.print_exc() |
| 371 | |
| 372 | sites = Site.objects.all() |
Scott Baker | a880bb5 | 2014-04-14 23:52:48 -0700 | [diff] [blame] | 373 | sites = [x for x in sites if x.name in BLESSED_SITES] |
Scott Baker | bd57a43 | 2014-04-14 16:12:15 -0700 | [diff] [blame] | 374 | sites = sorted(sites, key=functools.partial(siteSortKey, slice=slice, count=count, lat=lat, lon=lon)) |
| 375 | |
| 376 | return sites |
| 377 | |
Scott Baker | fe6a7e1 | 2014-04-15 10:58:31 -0700 | [diff] [blame] | 378 | def slice_increase_slivers(user, user_ip, siteList, slice, count, noAct=False): |
Scott Baker | e4deddd | 2014-04-14 16:48:05 -0700 | [diff] [blame] | 379 | sitesChanged = {} |
| 380 | |
Scott Baker | bd57a43 | 2014-04-14 16:12:15 -0700 | [diff] [blame] | 381 | # let's compute how many slivers are in use in each node of each site |
| 382 | for site in siteList: |
| 383 | site.nodeList = list(site.nodes.all()) |
| 384 | for node in site.nodeList: |
| 385 | node.sliverCount = 0 |
Scott Baker | 6253dfc | 2014-04-14 09:29:30 -0700 | [diff] [blame] | 386 | for sliver in node.slivers.all(): |
Scott Baker | 2151194 | 2014-04-15 17:44:27 -0700 | [diff] [blame] | 387 | if sliver.slice.id == slice.id: |
Scott Baker | 0017c10 | 2014-04-18 10:48:31 -0700 | [diff] [blame] | 388 | node.sliverCount = node.sliverCount + 1 |
Scott Baker | 6253dfc | 2014-04-14 09:29:30 -0700 | [diff] [blame] | 389 | |
Scott Baker | bd57a43 | 2014-04-14 16:12:15 -0700 | [diff] [blame] | 390 | # Allocate slivers to nodes |
| 391 | # for now, assume we want to allocate all slivers from the same site |
| 392 | nodes = siteList[0].nodeList |
| 393 | while (count>0): |
| 394 | # Sort the node list by number of slivers per node, then pick the |
| 395 | # node with the least number of slivers. |
| 396 | nodes = sorted(nodes, key=attrgetter("sliverCount")) |
| 397 | node = nodes[0] |
Scott Baker | 6253dfc | 2014-04-14 09:29:30 -0700 | [diff] [blame] | 398 | |
Scott Baker | bd57a43 | 2014-04-14 16:12:15 -0700 | [diff] [blame] | 399 | print "adding sliver at node", node.name, "of site", node.site.name |
Scott Baker | 6253dfc | 2014-04-14 09:29:30 -0700 | [diff] [blame] | 400 | |
Scott Baker | e4deddd | 2014-04-14 16:48:05 -0700 | [diff] [blame] | 401 | if not noAct: |
| 402 | sliver = Sliver(name=node.name, |
| 403 | slice=slice, |
| 404 | node=node, |
| 405 | image = Image.objects.all()[0], |
| 406 | creator = User.objects.get(email=user), |
| 407 | deploymentNetwork=node.deployment, |
| 408 | numberCores =1 ) |
| 409 | sliver.save() |
Scott Baker | bd57a43 | 2014-04-14 16:12:15 -0700 | [diff] [blame] | 410 | |
| 411 | node.sliverCount = node.sliverCount + 1 |
| 412 | |
| 413 | count = count - 1 |
| 414 | |
Scott Baker | e4deddd | 2014-04-14 16:48:05 -0700 | [diff] [blame] | 415 | sitesChanged[node.site.name] = sitesChanged.get(node.site.name,0) + 1 |
| 416 | |
| 417 | return sitesChanged |
| 418 | |
Scott Baker | fe6a7e1 | 2014-04-15 10:58:31 -0700 | [diff] [blame] | 419 | def slice_decrease_slivers(user, siteList, slice, count, noAct=False): |
Scott Baker | e4deddd | 2014-04-14 16:48:05 -0700 | [diff] [blame] | 420 | sitesChanged = {} |
Scott Baker | b85c26b | 2014-04-18 11:25:32 -0700 | [diff] [blame] | 421 | sliverList ={} |
Scott Baker | bd57a43 | 2014-04-14 16:12:15 -0700 | [diff] [blame] | 422 | if siteList: |
| 423 | siteNames = [site.name for site in siteList] |
| 424 | else: |
| 425 | siteNames = None |
| 426 | |
| 427 | for sliver in slice.slivers.all(): |
Scott Baker | b85c26b | 2014-04-18 11:25:32 -0700 | [diff] [blame] | 428 | if(not siteNames) or (sliver.node.site.name in siteNames):
|
| 429 | node = sliver.node
|
| 430 | sliverList[sliver.name]=node.name |
Scott Baker | bd57a43 | 2014-04-14 16:12:15 -0700 | [diff] [blame] | 431 | |
Scott Baker | b85c26b | 2014-04-18 11:25:32 -0700 | [diff] [blame] | 432 | for key in sliverList: |
| 433 | if count>0: |
Scott Baker | f1b65c0 | 2014-04-21 01:03:17 -0700 | [diff] [blame] | 434 | sliver = Sliver.objects.filter(name=key)[0]
|
Scott Baker | b85c26b | 2014-04-18 11:25:32 -0700 | [diff] [blame] | 435 | sliver.delete()
|
| 436 | print "deleting sliver",sliverList[key],"at node",sliver.node.name
|
| 437 | count=count-1
|
Scott Baker | f1b65c0 | 2014-04-21 01:03:17 -0700 | [diff] [blame] | 438 | sitesChanged[sliver.node.site.name] = sitesChanged.get(sliver.node.site.name,0) - 1
|
Scott Baker | b85c26b | 2014-04-18 11:25:32 -0700 | [diff] [blame] | 439 |
|
Scott Baker | e4deddd | 2014-04-14 16:48:05 -0700 | [diff] [blame] | 440 | return sitesChanged |
| 441 | |
Scott Baker | b85c26b | 2014-04-18 11:25:32 -0700 | [diff] [blame] | 442 | class TenantDeleteSliceView(View): |
| 443 | def post(self,request):
|
| 444 | sliceName = request.POST.get("sliceName",None)
|
| 445 | slice = Slice.objects.get(name=sliceName)
|
| 446 | print slice, slice.id
|
| 447 | sliceToDel=Slice(name=sliceName, id=slice.id)
|
| 448 | sliceToDel.delete() |
Scott Baker | 4b0cedd | 2014-04-22 21:01:21 -0700 | [diff] [blame] | 449 | return HttpResponse("Slice deleted") |
Scott Baker | b85c26b | 2014-04-18 11:25:32 -0700 | [diff] [blame] | 450 | |
Scott Baker | 6253dfc | 2014-04-14 09:29:30 -0700 | [diff] [blame] | 451 | class TenantAddOrRemoveSliverView(View): |
Scott Baker | e4deddd | 2014-04-14 16:48:05 -0700 | [diff] [blame] | 452 | """ Add or remove slivers from a Slice |
| 453 | |
| 454 | Arguments: |
| 455 | siteName - name of site. If not specified, PlanetStack will pick the |
| 456 | best site., |
| 457 | actionToDo - [add | rem] |
| 458 | count - number of slivers to add or remove |
| 459 | sliceName - name of slice |
Scott Baker | a880bb5 | 2014-04-14 23:52:48 -0700 | [diff] [blame] | 460 | noAct - if set, no changes will be made to db, but result will still |
| 461 | show which sites would have been modified. |
Scott Baker | e4deddd | 2014-04-14 16:48:05 -0700 | [diff] [blame] | 462 | |
| 463 | Returns: |
| 464 | Dictionary of sites that were modified, and the count of nodes |
| 465 | that were added or removed at each site. |
| 466 | """ |
Scott Baker | 6253dfc | 2014-04-14 09:29:30 -0700 | [diff] [blame] | 467 | def post(self, request, *args, **kwargs): |
Scott Baker | bd57a43 | 2014-04-14 16:12:15 -0700 | [diff] [blame] | 468 | siteName = request.POST.get("siteName", None) |
| 469 | actionToDo = request.POST.get("actionToDo", None) |
| 470 | count = int(request.POST.get("count","0")) |
| 471 | sliceName = request.POST.get("slice", None) |
Scott Baker | e4deddd | 2014-04-14 16:48:05 -0700 | [diff] [blame] | 472 | noAct = request.POST.get("noAct", False) |
Scott Baker | bd57a43 | 2014-04-14 16:12:15 -0700 | [diff] [blame] | 473 | |
| 474 | if not sliceName: |
| 475 | return HttpResponseServerError("No slice name given") |
| 476 | |
| 477 | slice = Slice.objects.get(name=sliceName) |
| 478 | |
| 479 | if siteName: |
| 480 | siteList = [Site.objects.get(name=siteName)] |
| 481 | else: |
| 482 | siteList = None |
Scott Baker | 6253dfc | 2014-04-14 09:29:30 -0700 | [diff] [blame] | 483 | |
| 484 | if (actionToDo == "add"): |
Scott Baker | bd57a43 | 2014-04-14 16:12:15 -0700 | [diff] [blame] | 485 | user_ip = request.GET.get("ip", get_ip(request)) |
| 486 | if (siteList is None): |
| 487 | siteList = tenant_pick_sites(user, user_ip, slice, count) |
| 488 | |
Scott Baker | fe6a7e1 | 2014-04-15 10:58:31 -0700 | [diff] [blame] | 489 | sitesChanged = slice_increase_slivers(request.user, user_ip, siteList, slice, count, noAct) |
Scott Baker | 6253dfc | 2014-04-14 09:29:30 -0700 | [diff] [blame] | 490 | elif (actionToDo == "rem"): |
Scott Baker | fe6a7e1 | 2014-04-15 10:58:31 -0700 | [diff] [blame] | 491 | sitesChanged = slice_decrease_slivers(request.user, siteList, slice, count, noAct) |
Scott Baker | bd57a43 | 2014-04-14 16:12:15 -0700 | [diff] [blame] | 492 | else: |
| 493 | return HttpResponseServerError("Unknown actionToDo %s" % actionToDo) |
| 494 | |
Scott Baker | e4deddd | 2014-04-14 16:48:05 -0700 | [diff] [blame] | 495 | return HttpResponse(json.dumps(sitesChanged), mimetype='application/javascript') |
Scott Baker | bd57a43 | 2014-04-14 16:12:15 -0700 | [diff] [blame] | 496 | |
| 497 | def get(self, request, *args, **kwargs): |
| 498 | request.POST = request.GET |
| 499 | return self.post(request, *args, **kwargs) # for testing REST in browser |
| 500 | #return HttpResponseServerError("GET is not supported") |
| 501 | |
| 502 | class TenantPickSitesView(View): |
| 503 | """ primarily just for testing purposes """ |
| 504 | def get(self, request, *args, **kwargs): |
| 505 | count = request.GET.get("count","0") |
| 506 | slice = request.GET.get("slice",None) |
| 507 | if slice: |
| 508 | slice = Slice.objects.get(name=slice) |
| 509 | ip = request.GET.get("ip", get_ip(request)) |
| 510 | sites = tenant_pick_sites(request.user, user_ip=ip, count=0, slice=slice) |
| 511 | sites = [x.name for x in sites] |
| 512 | return HttpResponse(json.dumps(sites), mimetype='application/javascript') |
Scott Baker | 6253dfc | 2014-04-14 09:29:30 -0700 | [diff] [blame] | 513 | |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 514 | class DashboardSummaryAjaxView(View): |
| 515 | def get(self, request, **kwargs): |
Scott Baker | 0017c10 | 2014-04-18 10:48:31 -0700 | [diff] [blame] | 516 | def avg(x): |
| 517 | return float(sum(x))/len(x) |
| 518 | |
| 519 | sites = getCDNOperatorData().values() |
| 520 | |
| 521 | sites = [site for site in sites if site["numHPCSlivers"]>0] |
| 522 | |
| 523 | total_slivers = sum( [site["numHPCSlivers"] for site in sites] ) |
| 524 | total_bandwidth = sum( [site["bandwidth"] for site in sites] ) |
| 525 | average_cpu = int(avg( [site["load"] for site in sites] )) |
| 526 | |
| 527 | result= {"total_slivers": total_slivers, |
| 528 | "total_bandwidth": total_bandwidth, |
| 529 | "average_cpu": average_cpu} |
| 530 | |
| 531 | return HttpResponse(json.dumps(result), mimetype='application/javascript') |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 532 | |
| 533 | class DashboardAddOrRemoveSliverView(View): |
Scott Baker | fe6a7e1 | 2014-04-15 10:58:31 -0700 | [diff] [blame] | 534 | # TODO: deprecate this view in favor of using TenantAddOrRemoveSliverView |
| 535 | |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 536 | def post(self, request, *args, **kwargs): |
Scott Baker | fe6a7e1 | 2014-04-15 10:58:31 -0700 | [diff] [blame] | 537 | siteName = request.POST.get("site", None) |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 538 | actionToDo = request.POST.get("actionToDo", "0") |
| 539 | |
Scott Baker | fe6a7e1 | 2014-04-15 10:58:31 -0700 | [diff] [blame] | 540 | siteList = [Site.objects.get(name=siteName)] |
| 541 | slice = Slice.objects.get(name="HyperCache") |
| 542 | |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 543 | if (actionToDo == "add"): |
Scott Baker | fe6a7e1 | 2014-04-15 10:58:31 -0700 | [diff] [blame] | 544 | user_ip = request.GET.get("ip", get_ip(request)) |
| 545 | slice_increase_slivers(request.user, user_ip, siteList, slice, 1) |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 546 | elif (actionToDo == "rem"): |
Scott Baker | fe6a7e1 | 2014-04-15 10:58:31 -0700 | [diff] [blame] | 547 | slice_decrease_slivers(request.user, siteList, slice, 1) |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 548 | |
| 549 | print '*' * 50 |
| 550 | print 'Ask for site: ' + siteName + ' to ' + actionToDo + ' another HPC Sliver' |
| 551 | return HttpResponse('This is POST request ') |
Siobhan Tully | 06d2c03 | 2014-02-23 23:48:52 -0500 | [diff] [blame] | 552 | |
| 553 | class DashboardAjaxView(View): |
| 554 | def get(self, request, **kwargs): |
| 555 | return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript') |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 556 | |
| 557 | class DashboardAnalyticsAjaxView(View): |
| 558 | def get(self, request, name="hello_world", **kwargs): |
| 559 | if (name == "hpcSummary"): |
| 560 | return HttpResponse(json.dumps(hpc_wizard.get_hpc_wizard().get_summary_for_view()), mimetype='application/javascript') |
| 561 | elif (name == "hpcUserSite"): |
| 562 | return HttpResponse(json.dumps(getUserSliceInfo(request.user, True)), mimetype='application/javascript') |
| 563 | elif (name == "hpcMap"): |
| 564 | return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript') |
| 565 | elif (name == "bigquery"): |
| 566 | (mimetype, data) = DoPlanetStackAnalytics(request) |
| 567 | return HttpResponse(data, mimetype=mimetype) |
| 568 | else: |
| 569 | return HttpResponse(json.dumps("Unknown"), mimetype='application/javascript') |
| 570 | |