Siobhan Tully | cf04fb6 | 2014-01-11 11:25:57 -0500 | [diff] [blame] | 1 | #views.py |
Scott Baker | bd57a43 | 2014-04-14 16:12:15 -0700 | [diff] [blame] | 2 | import functools |
| 3 | import math |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 4 | import os |
| 5 | import sys |
Siobhan Tully | 06d2c03 | 2014-02-23 23:48:52 -0500 | [diff] [blame] | 6 | from django.views.generic import TemplateView, View |
| 7 | import datetime |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 8 | from pprint import pprint |
| 9 | import json |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 10 | from core.models import * |
Scott Baker | bd57a43 | 2014-04-14 16:12:15 -0700 | [diff] [blame] | 11 | from operator import attrgetter |
| 12 | from django.views.decorators.csrf import csrf_exempt |
| 13 | from django.http import HttpResponse, HttpResponseServerError |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 14 | from django.core import urlresolvers |
Scott Baker | bd57a43 | 2014-04-14 16:12:15 -0700 | [diff] [blame] | 15 | from django.contrib.gis.geoip import GeoIP |
| 16 | from ipware.ip import get_ip |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 17 | import traceback |
Scott Baker | 6253dfc | 2014-04-14 09:29:30 -0700 | [diff] [blame] | 18 | import socket |
Siobhan Tully | 06d2c03 | 2014-02-23 23:48:52 -0500 | [diff] [blame] | 19 | |
Scott Baker | a880bb5 | 2014-04-14 23:52:48 -0700 | [diff] [blame] | 20 | BLESSED_SITES = ["Stanford", "Washington", "Princeton", "GeorgiaTech", "MaxPlanck"] |
| 21 | |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 22 | if os.path.exists("/home/smbaker/projects/vicci/cdn/bigquery"): |
| 23 | sys.path.append("/home/smbaker/projects/vicci/cdn/bigquery") |
| 24 | else: |
| 25 | sys.path.append("/opt/planetstack/hpc_wizard") |
| 26 | import hpc_wizard |
Scott Baker | 75095b6 | 2014-04-21 17:32:09 -0700 | [diff] [blame] | 27 | from planetstack_analytics import DoPlanetStackAnalytics, PlanetStackAnalytics, RED_LOAD, BLUE_LOAD |
Siobhan Tully | cf04fb6 | 2014-01-11 11:25:57 -0500 | [diff] [blame] | 28 | |
| 29 | class DashboardWelcomeView(TemplateView): |
| 30 | template_name = 'admin/dashboard/welcome.html' |
| 31 | |
| 32 | def get(self, request, *args, **kwargs): |
| 33 | context = self.get_context_data(**kwargs) |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 34 | userDetails = getUserSliceInfo(request.user) |
| 35 | #context['site'] = userDetails['site'] |
Siobhan Tully | cf04fb6 | 2014-01-11 11:25:57 -0500 | [diff] [blame] | 36 | |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 37 | context['userSliceInfo'] = userDetails['userSliceInfo'] |
| 38 | context['cdnData'] = userDetails['cdnData'] |
Siobhan Tully | cf04fb6 | 2014-01-11 11:25:57 -0500 | [diff] [blame] | 39 | return self.render_to_response(context=context) |
Siobhan Tully | e18b344 | 2014-02-23 14:23:34 -0500 | [diff] [blame] | 40 | |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 41 | def getUserSliceInfo(user, tableFormat = False): |
| 42 | userDetails = {} |
| 43 | # try: |
| 44 | # // site = Site.objects.filter(id=user.site.id) |
| 45 | # // except: |
| 46 | # // site = Site.objects.filter(name="Princeton") |
| 47 | # // userDetails['sitename'] = site[0].name |
| 48 | # // userDetails['siteid'] = site[0].id |
| 49 | |
| 50 | userSliceData = getSliceInfo(user) |
| 51 | if (tableFormat): |
| 52 | # pprint("******* GET USER SLICE INFO") |
| 53 | userDetails['userSliceInfo'] = userSliceTableFormatter(userSliceData) |
| 54 | else: |
| 55 | userDetails['userSliceInfo'] = userSliceData |
Scott Baker | 75095b6 | 2014-04-21 17:32:09 -0700 | [diff] [blame] | 56 | userDetails['cdnData'] = getCDNOperatorData(wait=False); |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 57 | # pprint( userDetails) |
| 58 | return userDetails |
| 59 | |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 60 | class TenantCreateSlice(View): |
| 61 | def post(self, request, *args, **kwargs): |
| 62 | sliceName = request.POST.get("sliceName", "0") |
| 63 | serviceClass = request.POST.get("serviceClass", "0") |
| 64 | imageName = request.POST.get("imageName", "0") |
| 65 | actionToDo = request.POST.get("actionToDo", "0") |
Scott Baker | edf2f67 | 2014-04-22 10:17:37 -0700 | [diff] [blame] | 66 | network = request.POST.get("network","0") |
| 67 | mountDataSets = request.POST.get("mountDataSets","0") |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 68 | if (actionToDo == "add"): |
| 69 | serviceClass = ServiceClass.objects.get(name=serviceClass) |
| 70 | site = request.user.site |
Scott Baker | 4b0cedd | 2014-04-22 21:01:21 -0700 | [diff] [blame] | 71 | image = Image.objects.get(name=imageName) |
| 72 | newSlice = Slice(name=sliceName,serviceClass=serviceClass,site=site,imagePreference=image,mountDataSets=mountDataSets,network=network) |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 73 | newSlice.save() |
Scott Baker | 4b0cedd | 2014-04-22 21:01:21 -0700 | [diff] [blame] | 74 | return HttpResponse("Slice created") |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 75 | |
Scott Baker | edf2f67 | 2014-04-22 10:17:37 -0700 | [diff] [blame] | 76 | class TenantUpdateSlice(View): |
| 77 | def post(self, request, *args, **kwargs):
|
| 78 | sliceName = request.POST.get("sliceName", "0")
|
| 79 | serviceClass = request.POST.get("serviceClass", "0")
|
| 80 | imageName = request.POST.get("imageName", "0")
|
| 81 | actionToDo = request.POST.get("actionToDo", "0")
|
| 82 | network = request.POST.get("network","0")
|
Scott Baker | 4b0cedd | 2014-04-22 21:01:21 -0700 | [diff] [blame] | 83 | dataSet = request.POST.get("dataSet","0")
|
| 84 | slice = Slice.objects.all()
|
| 85 | for entry in slice:
|
| 86 | serviceClass = ServiceClass.objects.get(name=serviceClass)
|
| 87 | if(entry.name==sliceName):
|
| 88 | if (actionToDo == "update"):
|
| 89 | setattr(entry,'serviceClass',serviceClass)
|
| 90 | setattr(entry,'imagePreference',imageName)
|
| 91 | setattr(entry,'network',network)
|
| 92 | setattr(entry,'mountDataSets',dataSet)
|
| 93 | entry.save()
|
| 94 | break
|
| 95 | return HttpResponse("Slice updated")
|
| 96 |
|
Scott Baker | edf2f67 | 2014-04-22 10:17:37 -0700 | [diff] [blame] | 97 | def update_slice(sliceName,**fields): |
| 98 | slice = Slice.objects.filter(name = sliceName)
|
| 99 | for (k,v) in fields.items():
|
| 100 | setattr(slice, k, v)
|
| 101 | slice.save()
|
| 102 | return slice |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 103 | |
| 104 | def getTenantSliceInfo(user, tableFormat = False): |
| 105 | tenantSliceDetails = {} |
| 106 | tenantSliceData = getTenantInfo(user) |
| 107 | tenantServiceClassData = getServiceClassInfo(user) |
| 108 | if (tableFormat): |
| 109 | tenantSliceDetails['userSliceInfo'] = userSliceTableFormatter(tenantSliceData) |
| 110 | tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData) |
| 111 | else: |
| 112 | tenantSliceDetails['userSliceInfo'] = tenantSliceData |
| 113 | tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData) |
| 114 | tenantSliceDetails['image']=userSliceTableFormatter(getImageInfo(user)) |
| 115 | tenantSliceDetails['network']=userSliceTableFormatter(getNetworkInfo(user)) |
| 116 | tenantSliceDetails['deploymentSites']=userSliceTableFormatter(getDeploymentSites()) |
Scott Baker | b85c26b | 2014-04-18 11:25:32 -0700 | [diff] [blame] | 117 | tenantSliceDetails['sites'] = userSliceTableFormatter(getTenantSitesInfo()) |
| 118 | tenantSliceDetails['mountDataSets'] = userSliceTableFormatter(getMountDataSets()) |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 119 | return tenantSliceDetails |
| 120 | |
| 121 | |
| 122 | def getTenantInfo(user): |
| 123 | slices =Slice.objects.all() |
| 124 | userSliceInfo = [] |
| 125 | for entry in slices: |
| 126 | sliceName = Slice.objects.get(id=entry.id).name |
| 127 | slice = Slice.objects.get(name=Slice.objects.get(id=entry.id).name) |
| 128 | sliceServiceClass = entry.serviceClass.name |
| 129 | preferredImage = entry.imagePreference |
Scott Baker | edf2f67 | 2014-04-22 10:17:37 -0700 | [diff] [blame] | 130 | sliceDataSet = entry.mountDataSets |
| 131 | sliceNetwork = entry.network |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 132 | numSliver = 0 |
| 133 | sliceImage="" |
| 134 | sliceSite = {} |
Scott Baker | edf2f67 | 2014-04-22 10:17:37 -0700 | [diff] [blame] | 135 | sliceNode = {} |
| 136 | sliceInstance= {} |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 137 | for sliver in slice.slivers.all(): |
Scott Baker | a880bb5 | 2014-04-14 23:52:48 -0700 | [diff] [blame] | 138 | if sliver.node.site.name in BLESSED_SITES: |
Scott Baker | a880bb5 | 2014-04-14 23:52:48 -0700 | [diff] [blame] | 139 | sliceSite[sliver.node.site.name] = sliceSite.get(sliver.node.site.name,0) + 1 |
| 140 | sliceImage = sliver.image.name |
Scott Baker | 9532628 | 2014-04-22 22:33:34 -0700 | [diff] [blame^] | 141 | sliceNode[str(sliver)] = sliver.name |
Scott Baker | edf2f67 | 2014-04-22 10:17:37 -0700 | [diff] [blame] | 142 | numSliver = sum(sliceSite.values()) |
Scott Baker | b85c26b | 2014-04-18 11:25:32 -0700 | [diff] [blame] | 143 | numSites = len(sliceSite) |
Scott Baker | edf2f67 | 2014-04-22 10:17:37 -0700 | [diff] [blame] | 144 | userSliceInfo.append({'sliceName': sliceName,'sliceServiceClass': sliceServiceClass,'preferredImage':preferredImage,'numOfSites':numSites, 'sliceSite':sliceSite,'sliceImage':sliceImage,'numOfSlivers':numSliver,'sliceDataSet':sliceDataSet,'sliceNetwork':sliceNetwork, 'instanceNodePair':sliceNode}) |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 145 | return userSliceInfo |
| 146 | |
| 147 | def getTenantSitesInfo(): |
| 148 | tenantSiteInfo=[] |
| 149 | for entry in Site.objects.all(): |
Scott Baker | a880bb5 | 2014-04-14 23:52:48 -0700 | [diff] [blame] | 150 | if entry.name in BLESSED_SITES: |
| 151 | tenantSiteInfo.append({'siteName':entry.name}) |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 152 | return tenantSiteInfo |
| 153 | |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 154 | def userSliceTableFormatter(data): |
| 155 | # pprint(data) |
| 156 | formattedData = { |
| 157 | 'rows' : data |
| 158 | } |
| 159 | return formattedData |
| 160 | |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 161 | def getServiceClassInfo(user): |
| 162 | serviceClassList = ServiceClass.objects.all() |
| 163 | sliceInfo = [] |
| 164 | for entry in serviceClassList: |
| 165 | sliceInfo.append({'serviceClass':entry.name}) |
| 166 | return sliceInfo |
| 167 | |
| 168 | def getImageInfo(user): |
Scott Baker | 4b0cedd | 2014-04-22 21:01:21 -0700 | [diff] [blame] | 169 | imageList = Image.objects.all() |
| 170 | #imageList = ['Fedora 16 LXC rev 1.3','Hadoop','MPI'] |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 171 | imageInfo = [] |
| 172 | for imageEntry in imageList: |
Scott Baker | 4b0cedd | 2014-04-22 21:01:21 -0700 | [diff] [blame] | 173 | imageInfo.append({'Image':imageEntry.name}) |
| 174 | #imageInfo.append({'Image':imageEntry}) |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 175 | return imageInfo |
| 176 | |
Scott Baker | b85c26b | 2014-04-18 11:25:32 -0700 | [diff] [blame] | 177 | def getMountDataSets(): |
Scott Baker | edf2f67 | 2014-04-22 10:17:37 -0700 | [diff] [blame] | 178 | dataSetList = ['------','GenBank','LSST','LHC','NOAA','Measurement Lab','Common Crawl']
|
Scott Baker | b85c26b | 2014-04-18 11:25:32 -0700 | [diff] [blame] | 179 | dataSetInfo = []
|
| 180 | for entry in dataSetList:
|
| 181 | dataSetInfo.append({'DataSet':entry})
|
| 182 | return dataSetInfo |
| 183 | |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 184 | def getNetworkInfo(user): |
| 185 | #networkList = Network.objects.all() |
| 186 | networkList = ['Private Only','Private and Publicly Routable'] |
| 187 | networkInfo = [] |
| 188 | for networkEntry in networkList: |
| 189 | #networkInfo.append({'Network':networkEntry.name}) |
| 190 | networkInfo.append({'Network':networkEntry}) |
| 191 | return networkInfo |
| 192 | |
| 193 | def getDeploymentSites(): |
| 194 | deploymentList = Deployment.objects.all() |
| 195 | deploymentInfo = [] |
| 196 | for entry in deploymentList: |
| 197 | deploymentInfo.append({'DeploymentSite':entry.name}) |
| 198 | return deploymentInfo |
| 199 | |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 200 | def getSliceInfo(user): |
Siobhan Tully | 06d2c03 | 2014-02-23 23:48:52 -0500 | [diff] [blame] | 201 | sliceList = Slice.objects.all() |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 202 | slicePrivs = SlicePrivilege.objects.filter(user=user) |
Siobhan Tully | 06d2c03 | 2014-02-23 23:48:52 -0500 | [diff] [blame] | 203 | userSliceInfo = [] |
| 204 | for entry in slicePrivs: |
Siobhan Tully | e18b344 | 2014-02-23 14:23:34 -0500 | [diff] [blame] | 205 | |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 206 | slicename = Slice.objects.get(id=entry.slice.id).name |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 207 | slice = Slice.objects.get(name=Slice.objects.get(id=entry.slice.id).name) |
| 208 | sliverList=Sliver.objects.all() |
| 209 | sites_used = {} |
| 210 | for sliver in slice.slivers.all(): |
| 211 | #sites_used['deploymentSites'] = sliver.node.deployment.name |
| 212 | # sites_used[sliver.image.name] = sliver.image.name |
| 213 | sites_used[sliver.node.site.name] = sliver.numberCores |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 214 | sliceid = Slice.objects.get(id=entry.slice.id).id |
Siobhan Tully | 06d2c03 | 2014-02-23 23:48:52 -0500 | [diff] [blame] | 215 | try: |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 216 | sliverList = Sliver.objects.filter(slice=entry.slice.id) |
| 217 | siteList = {} |
| 218 | for x in sliverList: |
| 219 | if x.node.site not in siteList: |
| 220 | siteList[x.node.site] = 1 |
| 221 | slivercount = len(sliverList) |
| 222 | sitecount = len(siteList) |
Siobhan Tully | 06d2c03 | 2014-02-23 23:48:52 -0500 | [diff] [blame] | 223 | except: |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 224 | traceback.print_exc() |
| 225 | slivercount = 0 |
| 226 | sitecount = 0 |
Siobhan Tully | 06d2c03 | 2014-02-23 23:48:52 -0500 | [diff] [blame] | 227 | |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 228 | userSliceInfo.append({'slicename': slicename, 'sliceid':sliceid, |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 229 | 'sitesUsed':sites_used, |
| 230 | 'role': SliceRole.objects.get(id=entry.role.id).role, |
| 231 | 'slivercount': slivercount, |
| 232 | 'sitecount':sitecount}) |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 233 | |
Siobhan Tully | 06d2c03 | 2014-02-23 23:48:52 -0500 | [diff] [blame] | 234 | return userSliceInfo |
| 235 | |
Scott Baker | 75095b6 | 2014-04-21 17:32:09 -0700 | [diff] [blame] | 236 | def getCDNOperatorData(randomizeData = False, wait=True): |
| 237 | HPC_SLICE_NAME = "HyperCache" |
| 238 | |
Scott Baker | 0017c10 | 2014-04-18 10:48:31 -0700 | [diff] [blame] | 239 | bq = PlanetStackAnalytics() |
Siobhan Tully | 06d2c03 | 2014-02-23 23:48:52 -0500 | [diff] [blame] | 240 | |
Scott Baker | 75095b6 | 2014-04-21 17:32:09 -0700 | [diff] [blame] | 241 | rows = bq.get_cached_query_results(bq.compose_latest_query(groupByFields=["%hostname", "event", "%slice"]), wait) |
Siobhan Tully | 06d2c03 | 2014-02-23 23:48:52 -0500 | [diff] [blame] | 242 | |
Scott Baker | edf2f67 | 2014-04-22 10:17:37 -0700 | [diff] [blame] | 243 | # wait=False on the first time the Dashboard is opened. This means we might |
| 244 | # not have any rows yet. The dashboard code polls every 30 seconds, so it |
| 245 | # will eventually pick them up. |
Scott Baker | 0017c10 | 2014-04-18 10:48:31 -0700 | [diff] [blame] | 246 | |
Scott Baker | 75095b6 | 2014-04-21 17:32:09 -0700 | [diff] [blame] | 247 | if rows: |
| 248 | rows = bq.postprocess_results(rows, filter={"slice": HPC_SLICE_NAME}, maxi=["cpu"], count=["hostname"], computed=["bytes_sent/elapsed"], groupBy=["Time","site"], maxDeltaTime=80) |
Scott Baker | 0017c10 | 2014-04-18 10:48:31 -0700 | [diff] [blame] | 249 | |
Scott Baker | 75095b6 | 2014-04-21 17:32:09 -0700 | [diff] [blame] | 250 | # dictionaryize the statistics rows by site name |
| 251 | stats_rows = {} |
| 252 | for row in rows: |
| 253 | stats_rows[row["site"]] = row |
Scott Baker | edf2f67 | 2014-04-22 10:17:37 -0700 | [diff] [blame] | 254 | else: |
| 255 | stats_rows = {} |
Scott Baker | 0017c10 | 2014-04-18 10:48:31 -0700 | [diff] [blame] | 256 | |
Scott Baker | 75095b6 | 2014-04-21 17:32:09 -0700 | [diff] [blame] | 257 | slice = Slice.objects.get(name=HPC_SLICE_NAME) |
| 258 | slice_slivers = list(slice.slivers.all()) |
Scott Baker | 0017c10 | 2014-04-18 10:48:31 -0700 | [diff] [blame] | 259 | |
| 260 | new_rows = {} |
Scott Baker | 75095b6 | 2014-04-21 17:32:09 -0700 | [diff] [blame] | 261 | for site in Site.objects.all(): |
| 262 | # compute number of slivers allocated in the data model |
| 263 | allocated_slivers = 0 |
| 264 | for sliver in slice_slivers: |
| 265 | if sliver.node.site == site: |
| 266 | allocated_slivers = allocated_slivers + 1 |
| 267 | |
| 268 | stats_row = stats_rows.get(site.name,{}) |
| 269 | |
| 270 | max_cpu = stats_row.get("max_avg_cpu", stats_row.get("max_cpu",0)) |
| 271 | cpu=float(max_cpu)/100.0 |
| 272 | hotness = max(0.0, ((cpu*RED_LOAD) - BLUE_LOAD)/(RED_LOAD-BLUE_LOAD)) |
| 273 | |
| 274 | # format it to what that CDN Operations View is expecting |
| 275 | new_row = {"lat": float(site.location.longitude), |
| 276 | "long": float(site.location.longitude), |
| 277 | "lat": float(site.location.latitude), |
Scott Baker | 0017c10 | 2014-04-18 10:48:31 -0700 | [diff] [blame] | 278 | "health": 0, |
Scott Baker | 75095b6 | 2014-04-21 17:32:09 -0700 | [diff] [blame] | 279 | "numNodes": int(site.nodes.count()), |
| 280 | "activeHPCSlivers": int(stats_row.get("count_hostname", 0)), # measured number of slivers, from bigquery statistics |
| 281 | "numHPCSlivers": allocated_slivers, # allocated number of slivers, from data model |
| 282 | "siteUrl": str(site.site_url), |
| 283 | "bandwidth": stats_row.get("sum_computed_bytes_sent_div_elapsed",0), |
| 284 | "load": max_cpu, |
| 285 | "hot": float(hotness)} |
| 286 | new_rows[str(site.name)] = new_row |
| 287 | |
| 288 | # get rid of sites with 0 slivers that overlap other sites with >0 slivers |
| 289 | for (k,v) in new_rows.items(): |
| 290 | bad=False |
| 291 | if v["numHPCSlivers"]==0: |
| 292 | for v2 in new_rows.values(): |
| 293 | if (v!=v2) and (v2["numHPCSlivers"]>=0): |
| 294 | d = haversine(v["lat"],v["long"],v2["lat"],v2["long"]) |
| 295 | if d<100: |
| 296 | bad=True |
| 297 | if bad: |
| 298 | del new_rows[k] |
Scott Baker | 0017c10 | 2014-04-18 10:48:31 -0700 | [diff] [blame] | 299 | |
| 300 | return new_rows |
Siobhan Tully | 06d2c03 | 2014-02-23 23:48:52 -0500 | [diff] [blame] | 301 | |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 302 | class SimulatorView(View): |
| 303 | def get(self, request, **kwargs): |
| 304 | sim = json.loads(file("/tmp/simulator.json","r").read()) |
| 305 | text = "<html><head></head><body>" |
| 306 | text += "Iteration: %d<br>" % sim["iteration"] |
| 307 | text += "Elapsed since report %d<br><br>" % sim["elapsed_since_report"] |
| 308 | text += "<table border=1>" |
| 309 | text += "<tr><th>site</th><th>trend</th><th>weight</th><th>bytes_sent</th><th>hot</th></tr>" |
| 310 | for site in sim["site_load"].values(): |
| 311 | text += "<tr>" |
| 312 | text += "<td>%s</td><td>%0.2f</td><td>%0.2f</td><td>%d</td><td>%0.2f</td>" % \ |
| 313 | (site["name"], site["trend"], site["weight"], site["bytes_sent"], site["load_frac"]) |
| 314 | text += "</tr>" |
| 315 | text += "</table>" |
| 316 | text += "</body></html>" |
| 317 | return HttpResponse(text) |
| 318 | |
| 319 | class DashboardUserSiteView(View): |
| 320 | def get(self, request, **kwargs): |
| 321 | return HttpResponse(json.dumps(getUserSliceInfo(request.user, True)), mimetype='application/javascript') |
| 322 | |
Scott Baker | 537d823 | 2014-04-08 23:40:56 -0700 | [diff] [blame] | 323 | class TenantViewData(View): |
| 324 | def get(self, request, **kwargs): |
| 325 | return HttpResponse(json.dumps(getTenantSliceInfo(request.user, True)), mimetype='application/javascript') |
| 326 | |
Scott Baker | 75095b6 | 2014-04-21 17:32:09 -0700 | [diff] [blame] | 327 | def haversine(site_lat, site_lon, lat, lon): |
Scott Baker | 75095b6 | 2014-04-21 17:32:09 -0700 | [diff] [blame] | 328 | d=0 |
| 329 | if lat and lon and site_lat and site_lon: |
Scott Baker | 4b0cedd | 2014-04-22 21:01:21 -0700 | [diff] [blame] | 330 | site_lat = float(site_lat) |
| 331 | site_lon = float(site_lon) |
| 332 | lat = float(lat) |
| 333 | lon = float(lon) |
Scott Baker | 75095b6 | 2014-04-21 17:32:09 -0700 | [diff] [blame] | 334 | R = 6378.1 |
| 335 | a = math.sin( math.radians((lat - site_lat)/2.0) )**2 + math.cos( math.radians(lat) )*math.cos( math.radians(site_lat) )*(math.sin( math.radians((lon - site_lon)/2.0 ) )**2) |
| 336 | c = 2 * math.atan2( math.sqrt(a), math.sqrt(1 - a) ) |
| 337 | d = R * c |
| 338 | |
| 339 | return d |
| 340 | |
Scott Baker | bd57a43 | 2014-04-14 16:12:15 -0700 | [diff] [blame] | 341 | def siteSortKey(site, slice=None, count=None, lat=None, lon=None): |
| 342 | # try to pick a site we're already using |
| 343 | has_slivers_here=False |
| 344 | if slice: |
| 345 | for sliver in slice.slivers.all(): |
| 346 | if sliver.node.site.name == site.name: |
| 347 | has_slivers_here=True |
| 348 | |
| 349 | # Haversine method |
Scott Baker | 75095b6 | 2014-04-21 17:32:09 -0700 | [diff] [blame] | 350 | d = haversine(site.location.latitude, site.location.longitude, lat, lon) |
Scott Baker | bd57a43 | 2014-04-14 16:12:15 -0700 | [diff] [blame] | 351 | |
| 352 | return (-has_slivers_here, d) |
| 353 | |
| 354 | def tenant_pick_sites(user, user_ip=None, slice=None, count=None): |
| 355 | """ Returns list of sites, sorted from most favorable to least favorable """ |
| 356 | lat=None |
| 357 | lon=None |
| 358 | try: |
| 359 | client_geo = GeoIP().city(user_ip) |
| 360 | if client_geo: |
| 361 | lat=float(client_geo["latitude"]) |
| 362 | lon=float(client_geo["longitude"]) |
| 363 | except: |
| 364 | print "exception in geo code" |
| 365 | traceback.print_exc() |
| 366 | |
| 367 | sites = Site.objects.all() |
Scott Baker | a880bb5 | 2014-04-14 23:52:48 -0700 | [diff] [blame] | 368 | sites = [x for x in sites if x.name in BLESSED_SITES] |
Scott Baker | bd57a43 | 2014-04-14 16:12:15 -0700 | [diff] [blame] | 369 | sites = sorted(sites, key=functools.partial(siteSortKey, slice=slice, count=count, lat=lat, lon=lon)) |
| 370 | |
| 371 | return sites |
| 372 | |
Scott Baker | fe6a7e1 | 2014-04-15 10:58:31 -0700 | [diff] [blame] | 373 | def slice_increase_slivers(user, user_ip, siteList, slice, count, noAct=False): |
Scott Baker | e4deddd | 2014-04-14 16:48:05 -0700 | [diff] [blame] | 374 | sitesChanged = {} |
| 375 | |
Scott Baker | bd57a43 | 2014-04-14 16:12:15 -0700 | [diff] [blame] | 376 | # let's compute how many slivers are in use in each node of each site |
| 377 | for site in siteList: |
| 378 | site.nodeList = list(site.nodes.all()) |
| 379 | for node in site.nodeList: |
| 380 | node.sliverCount = 0 |
Scott Baker | 6253dfc | 2014-04-14 09:29:30 -0700 | [diff] [blame] | 381 | for sliver in node.slivers.all(): |
Scott Baker | 2151194 | 2014-04-15 17:44:27 -0700 | [diff] [blame] | 382 | if sliver.slice.id == slice.id: |
Scott Baker | 0017c10 | 2014-04-18 10:48:31 -0700 | [diff] [blame] | 383 | node.sliverCount = node.sliverCount + 1 |
Scott Baker | 6253dfc | 2014-04-14 09:29:30 -0700 | [diff] [blame] | 384 | |
Scott Baker | bd57a43 | 2014-04-14 16:12:15 -0700 | [diff] [blame] | 385 | # Allocate slivers to nodes |
| 386 | # for now, assume we want to allocate all slivers from the same site |
| 387 | nodes = siteList[0].nodeList |
| 388 | while (count>0): |
| 389 | # Sort the node list by number of slivers per node, then pick the |
| 390 | # node with the least number of slivers. |
| 391 | nodes = sorted(nodes, key=attrgetter("sliverCount")) |
| 392 | node = nodes[0] |
Scott Baker | 6253dfc | 2014-04-14 09:29:30 -0700 | [diff] [blame] | 393 | |
Scott Baker | bd57a43 | 2014-04-14 16:12:15 -0700 | [diff] [blame] | 394 | print "adding sliver at node", node.name, "of site", node.site.name |
Scott Baker | 6253dfc | 2014-04-14 09:29:30 -0700 | [diff] [blame] | 395 | |
Scott Baker | e4deddd | 2014-04-14 16:48:05 -0700 | [diff] [blame] | 396 | if not noAct: |
| 397 | sliver = Sliver(name=node.name, |
| 398 | slice=slice, |
| 399 | node=node, |
| 400 | image = Image.objects.all()[0], |
| 401 | creator = User.objects.get(email=user), |
| 402 | deploymentNetwork=node.deployment, |
| 403 | numberCores =1 ) |
| 404 | sliver.save() |
Scott Baker | bd57a43 | 2014-04-14 16:12:15 -0700 | [diff] [blame] | 405 | |
| 406 | node.sliverCount = node.sliverCount + 1 |
| 407 | |
| 408 | count = count - 1 |
| 409 | |
Scott Baker | e4deddd | 2014-04-14 16:48:05 -0700 | [diff] [blame] | 410 | sitesChanged[node.site.name] = sitesChanged.get(node.site.name,0) + 1 |
| 411 | |
| 412 | return sitesChanged |
| 413 | |
Scott Baker | fe6a7e1 | 2014-04-15 10:58:31 -0700 | [diff] [blame] | 414 | def slice_decrease_slivers(user, siteList, slice, count, noAct=False): |
Scott Baker | e4deddd | 2014-04-14 16:48:05 -0700 | [diff] [blame] | 415 | sitesChanged = {} |
Scott Baker | b85c26b | 2014-04-18 11:25:32 -0700 | [diff] [blame] | 416 | sliverList ={} |
Scott Baker | bd57a43 | 2014-04-14 16:12:15 -0700 | [diff] [blame] | 417 | if siteList: |
| 418 | siteNames = [site.name for site in siteList] |
| 419 | else: |
| 420 | siteNames = None |
| 421 | |
| 422 | for sliver in slice.slivers.all(): |
Scott Baker | b85c26b | 2014-04-18 11:25:32 -0700 | [diff] [blame] | 423 | if(not siteNames) or (sliver.node.site.name in siteNames):
|
| 424 | node = sliver.node
|
| 425 | sliverList[sliver.name]=node.name |
Scott Baker | bd57a43 | 2014-04-14 16:12:15 -0700 | [diff] [blame] | 426 | |
Scott Baker | b85c26b | 2014-04-18 11:25:32 -0700 | [diff] [blame] | 427 | for key in sliverList: |
| 428 | if count>0: |
Scott Baker | f1b65c0 | 2014-04-21 01:03:17 -0700 | [diff] [blame] | 429 | sliver = Sliver.objects.filter(name=key)[0]
|
Scott Baker | b85c26b | 2014-04-18 11:25:32 -0700 | [diff] [blame] | 430 | sliver.delete()
|
| 431 | print "deleting sliver",sliverList[key],"at node",sliver.node.name
|
| 432 | count=count-1
|
Scott Baker | f1b65c0 | 2014-04-21 01:03:17 -0700 | [diff] [blame] | 433 | sitesChanged[sliver.node.site.name] = sitesChanged.get(sliver.node.site.name,0) - 1
|
Scott Baker | b85c26b | 2014-04-18 11:25:32 -0700 | [diff] [blame] | 434 |
|
Scott Baker | e4deddd | 2014-04-14 16:48:05 -0700 | [diff] [blame] | 435 | return sitesChanged |
| 436 | |
Scott Baker | b85c26b | 2014-04-18 11:25:32 -0700 | [diff] [blame] | 437 | class TenantDeleteSliceView(View): |
| 438 | def post(self,request):
|
| 439 | sliceName = request.POST.get("sliceName",None)
|
| 440 | slice = Slice.objects.get(name=sliceName)
|
| 441 | print slice, slice.id
|
| 442 | sliceToDel=Slice(name=sliceName, id=slice.id)
|
| 443 | sliceToDel.delete() |
Scott Baker | 4b0cedd | 2014-04-22 21:01:21 -0700 | [diff] [blame] | 444 | return HttpResponse("Slice deleted") |
Scott Baker | b85c26b | 2014-04-18 11:25:32 -0700 | [diff] [blame] | 445 | |
Scott Baker | 6253dfc | 2014-04-14 09:29:30 -0700 | [diff] [blame] | 446 | class TenantAddOrRemoveSliverView(View): |
Scott Baker | e4deddd | 2014-04-14 16:48:05 -0700 | [diff] [blame] | 447 | """ Add or remove slivers from a Slice |
| 448 | |
| 449 | Arguments: |
| 450 | siteName - name of site. If not specified, PlanetStack will pick the |
| 451 | best site., |
| 452 | actionToDo - [add | rem] |
| 453 | count - number of slivers to add or remove |
| 454 | sliceName - name of slice |
Scott Baker | a880bb5 | 2014-04-14 23:52:48 -0700 | [diff] [blame] | 455 | noAct - if set, no changes will be made to db, but result will still |
| 456 | show which sites would have been modified. |
Scott Baker | e4deddd | 2014-04-14 16:48:05 -0700 | [diff] [blame] | 457 | |
| 458 | Returns: |
| 459 | Dictionary of sites that were modified, and the count of nodes |
| 460 | that were added or removed at each site. |
| 461 | """ |
Scott Baker | 6253dfc | 2014-04-14 09:29:30 -0700 | [diff] [blame] | 462 | def post(self, request, *args, **kwargs): |
Scott Baker | bd57a43 | 2014-04-14 16:12:15 -0700 | [diff] [blame] | 463 | siteName = request.POST.get("siteName", None) |
| 464 | actionToDo = request.POST.get("actionToDo", None) |
| 465 | count = int(request.POST.get("count","0")) |
| 466 | sliceName = request.POST.get("slice", None) |
Scott Baker | e4deddd | 2014-04-14 16:48:05 -0700 | [diff] [blame] | 467 | noAct = request.POST.get("noAct", False) |
Scott Baker | bd57a43 | 2014-04-14 16:12:15 -0700 | [diff] [blame] | 468 | |
| 469 | if not sliceName: |
| 470 | return HttpResponseServerError("No slice name given") |
| 471 | |
| 472 | slice = Slice.objects.get(name=sliceName) |
| 473 | |
| 474 | if siteName: |
| 475 | siteList = [Site.objects.get(name=siteName)] |
| 476 | else: |
| 477 | siteList = None |
Scott Baker | 6253dfc | 2014-04-14 09:29:30 -0700 | [diff] [blame] | 478 | |
| 479 | if (actionToDo == "add"): |
Scott Baker | bd57a43 | 2014-04-14 16:12:15 -0700 | [diff] [blame] | 480 | user_ip = request.GET.get("ip", get_ip(request)) |
| 481 | if (siteList is None): |
| 482 | siteList = tenant_pick_sites(user, user_ip, slice, count) |
| 483 | |
Scott Baker | fe6a7e1 | 2014-04-15 10:58:31 -0700 | [diff] [blame] | 484 | sitesChanged = slice_increase_slivers(request.user, user_ip, siteList, slice, count, noAct) |
Scott Baker | 6253dfc | 2014-04-14 09:29:30 -0700 | [diff] [blame] | 485 | elif (actionToDo == "rem"): |
Scott Baker | fe6a7e1 | 2014-04-15 10:58:31 -0700 | [diff] [blame] | 486 | sitesChanged = slice_decrease_slivers(request.user, siteList, slice, count, noAct) |
Scott Baker | bd57a43 | 2014-04-14 16:12:15 -0700 | [diff] [blame] | 487 | else: |
| 488 | return HttpResponseServerError("Unknown actionToDo %s" % actionToDo) |
| 489 | |
Scott Baker | e4deddd | 2014-04-14 16:48:05 -0700 | [diff] [blame] | 490 | return HttpResponse(json.dumps(sitesChanged), mimetype='application/javascript') |
Scott Baker | bd57a43 | 2014-04-14 16:12:15 -0700 | [diff] [blame] | 491 | |
| 492 | def get(self, request, *args, **kwargs): |
| 493 | request.POST = request.GET |
| 494 | return self.post(request, *args, **kwargs) # for testing REST in browser |
| 495 | #return HttpResponseServerError("GET is not supported") |
| 496 | |
| 497 | class TenantPickSitesView(View): |
| 498 | """ primarily just for testing purposes """ |
| 499 | def get(self, request, *args, **kwargs): |
| 500 | count = request.GET.get("count","0") |
| 501 | slice = request.GET.get("slice",None) |
| 502 | if slice: |
| 503 | slice = Slice.objects.get(name=slice) |
| 504 | ip = request.GET.get("ip", get_ip(request)) |
| 505 | sites = tenant_pick_sites(request.user, user_ip=ip, count=0, slice=slice) |
| 506 | sites = [x.name for x in sites] |
| 507 | return HttpResponse(json.dumps(sites), mimetype='application/javascript') |
Scott Baker | 6253dfc | 2014-04-14 09:29:30 -0700 | [diff] [blame] | 508 | |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 509 | class DashboardSummaryAjaxView(View): |
| 510 | def get(self, request, **kwargs): |
Scott Baker | 0017c10 | 2014-04-18 10:48:31 -0700 | [diff] [blame] | 511 | def avg(x): |
| 512 | return float(sum(x))/len(x) |
| 513 | |
| 514 | sites = getCDNOperatorData().values() |
| 515 | |
| 516 | sites = [site for site in sites if site["numHPCSlivers"]>0] |
| 517 | |
| 518 | total_slivers = sum( [site["numHPCSlivers"] for site in sites] ) |
| 519 | total_bandwidth = sum( [site["bandwidth"] for site in sites] ) |
| 520 | average_cpu = int(avg( [site["load"] for site in sites] )) |
| 521 | |
| 522 | result= {"total_slivers": total_slivers, |
| 523 | "total_bandwidth": total_bandwidth, |
| 524 | "average_cpu": average_cpu} |
| 525 | |
| 526 | return HttpResponse(json.dumps(result), mimetype='application/javascript') |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 527 | |
| 528 | class DashboardAddOrRemoveSliverView(View): |
Scott Baker | fe6a7e1 | 2014-04-15 10:58:31 -0700 | [diff] [blame] | 529 | # TODO: deprecate this view in favor of using TenantAddOrRemoveSliverView |
| 530 | |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 531 | def post(self, request, *args, **kwargs): |
Scott Baker | fe6a7e1 | 2014-04-15 10:58:31 -0700 | [diff] [blame] | 532 | siteName = request.POST.get("site", None) |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 533 | actionToDo = request.POST.get("actionToDo", "0") |
| 534 | |
Scott Baker | fe6a7e1 | 2014-04-15 10:58:31 -0700 | [diff] [blame] | 535 | siteList = [Site.objects.get(name=siteName)] |
| 536 | slice = Slice.objects.get(name="HyperCache") |
| 537 | |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 538 | if (actionToDo == "add"): |
Scott Baker | fe6a7e1 | 2014-04-15 10:58:31 -0700 | [diff] [blame] | 539 | user_ip = request.GET.get("ip", get_ip(request)) |
| 540 | slice_increase_slivers(request.user, user_ip, siteList, slice, 1) |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 541 | elif (actionToDo == "rem"): |
Scott Baker | fe6a7e1 | 2014-04-15 10:58:31 -0700 | [diff] [blame] | 542 | slice_decrease_slivers(request.user, siteList, slice, 1) |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 543 | |
| 544 | print '*' * 50 |
| 545 | print 'Ask for site: ' + siteName + ' to ' + actionToDo + ' another HPC Sliver' |
| 546 | return HttpResponse('This is POST request ') |
Siobhan Tully | 06d2c03 | 2014-02-23 23:48:52 -0500 | [diff] [blame] | 547 | |
| 548 | class DashboardAjaxView(View): |
| 549 | def get(self, request, **kwargs): |
| 550 | return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript') |
Scott Baker | 771819b | 2014-03-19 22:10:17 -0700 | [diff] [blame] | 551 | |
| 552 | class DashboardAnalyticsAjaxView(View): |
| 553 | def get(self, request, name="hello_world", **kwargs): |
| 554 | if (name == "hpcSummary"): |
| 555 | return HttpResponse(json.dumps(hpc_wizard.get_hpc_wizard().get_summary_for_view()), mimetype='application/javascript') |
| 556 | elif (name == "hpcUserSite"): |
| 557 | return HttpResponse(json.dumps(getUserSliceInfo(request.user, True)), mimetype='application/javascript') |
| 558 | elif (name == "hpcMap"): |
| 559 | return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript') |
| 560 | elif (name == "bigquery"): |
| 561 | (mimetype, data) = DoPlanetStackAnalytics(request) |
| 562 | return HttpResponse(data, mimetype=mimetype) |
| 563 | else: |
| 564 | return HttpResponse(json.dumps("Unknown"), mimetype='application/javascript') |
| 565 | |