blob: ae6cab96e81278729223c580adc07eef598d0c41 [file] [log] [blame]
Scott Bakerb63ea792016-08-11 10:24:48 -07001import os
2#os.environ.setdefault("DJANGO_SETTINGS_MODULE", "xos.settings")
3import string
4import random
5import hashlib
6from datetime import datetime
7from django.utils import timezone
8
9from netaddr import IPAddress, IPNetwork
10from xos import settings
11from django.core import management
12from core.models import *
13from xos.config import Config
14try:
Scott Baker04a37f52016-08-11 10:52:21 -070015 from synchronizers.openstack.client import OpenStackClient
16 from synchronizers.openstack.driver import OpenStackDriver
Scott Bakerb63ea792016-08-11 10:24:48 -070017 has_openstack = True
18except:
19 has_openstack = False
20
21manager_enabled = Config().api_nova_enabled
22
23
24def random_string(size=6):
25 return ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(size))
26
27def require_enabled(callable):
28 def wrapper(*args, **kwds):
29 if manager_enabled and has_openstack:
30 return callable(*args, **kwds)
31 else:
32 return None
33 return wrapper
34
35
36class OpenStackManager:
37
38 def __init__(self, auth={}, caller=None):
39 self.client = None
40 self.driver = None
41 self.caller = None
42 self.has_openstack = has_openstack
43 self.enabled = manager_enabled
44
45 if has_openstack and manager_enabled:
46 if auth:
47 try:
48 self.init_user(auth, caller)
49 except:
50 # if this fails then it meanse the caller doesn't have a
51 # role at the slice's tenant. if the caller is an admin
52 # just use the admin client/manager.
53 if caller and caller.is_admin:
54 self.init_admin()
55 else: raise
56 else:
57 self.init_admin()
58
59 @require_enabled
60 def init_caller(self, caller, tenant):
61 auth = {'username': caller.email,
62 'password': hashlib.md5(caller.password).hexdigest()[:6],
63 'tenant': tenant}
64 self.client = OpenStackClient(**auth)
65 self.driver = OpenStackDriver(client=self.client)
66 self.caller = caller
67
68 @require_enabled
69 def init_admin(self, tenant=None):
70 # use the admin credentials
71 self.client = OpenStackClient(tenant=tenant)
72 self.driver = OpenStackDriver(client=self.client)
73 self.caller = self.driver.admin_user
74 self.caller.kuser_id = self.caller.id
75
76 @require_enabled
77 def save_role(self, role):
78 if not role.role:
79 keystone_role = self.driver.create_role(role.role_type)
80 role.role = keystone_role.id
81
82 @require_enabled
83 def delete_role(self, role):
84 if role.role:
85 self.driver.delete_role({'id': role.role})
86
87 @require_enabled
88 def save_key(self, key, name):
89 key_fields = {'name': name,
90 'public_key': key}
91 nova_key = self.driver.create_keypair(**key_fields)
92
93 @require_enabled
94 def delete_key(self, key):
95 if key.nkey_id:
96 self.driver.delete_keypair(key.nkey_id)
97
98 @require_enabled
99 def save_user(self, user):
100 name = user.email[:user.email.find('@')]
101 user_fields = {'name': name,
102 'email': user.email,
103 'password': hashlib.md5(user.password).hexdigest()[:6],
104 'enabled': True}
105 if not user.kuser_id:
106 keystone_user = self.driver.create_user(**user_fields)
107 user.kuser_id = keystone_user.id
108 else:
109 self.driver.update_user(user.kuser_id, user_fields)
110
111 if user.site:
112 self.driver.add_user_role(user.kuser_id, user.site.tenant_id, 'user')
113 if user.is_admin:
114 self.driver.add_user_role(user.kuser_id, user.site.tenant_id, 'admin')
115 else:
116 # may have admin role so attempt to remove it
117 self.driver.delete_user_role(user.kuser_id, user.site.tenant_id, 'admin')
118
119 if user.public_key:
120 self.init_caller(user, user.site.login_base)
121 self.save_key(user.public_key, user.keyname)
122 self.init_admin()
123
124 user.save()
125 user.enacted = timezone.now()
126 user.save(update_fields=['enacted'])
127
128 @require_enabled
129 def delete_user(self, user):
130 if user.kuser_id:
131 self.driver.delete_user(user.kuser_id)
132
133 @require_enabled
134 def save_site(self, site, add_role=True):
135 if not site.tenant_id:
136 tenant = self.driver.create_tenant(tenant_name=site.login_base,
137 description=site.name,
138 enabled=site.enabled)
139 site.tenant_id = tenant.id
140 # give caller an admin role at the tenant they've created
141 self.driver.add_user_role(self.caller.kuser_id, tenant.id, 'admin')
142
143 # update the record
144 if site.id and site.tenant_id:
145 self.driver.update_tenant(site.tenant_id,
146 description=site.name,
147 enabled=site.enabled)
148
149 # commit the updated record
150 site.save()
151 site.enacted = timezone.now()
152 site.save(update_fields=['enacted']) # enusre enacted > updated
153
154
155 @require_enabled
156 def delete_site(self, site):
157 if site.tenant_id:
158 self.driver.delete_tenant(site.tenant_id)
159
160 @require_enabled
161 def save_site_privilege(self, site_priv):
162 if site_priv.user.kuser_id and site_priv.site.tenant_id:
163 self.driver.add_user_role(site_priv.user.kuser_id,
164 site_priv.site.tenant_id,
165 site_priv.role.role_type)
166 site_priv.enacted = timezone.now()
167 site_priv.save(update_fields=['enacted'])
168
169
170 @require_enabled
171 def delete_site_privilege(self, site_priv):
172 self.driver.delete_user_role(site_priv.user.kuser_id,
173 site_priv.site.tenant_id,
174 site_priv.role.role_type)
175
176 @require_enabled
177 def save_slice(self, slice):
178 if not slice.tenant_id:
179 nova_fields = {'tenant_name': slice.name,
180 'description': slice.description,
181 'enabled': slice.enabled}
182 tenant = self.driver.create_tenant(**nova_fields)
183 slice.tenant_id = tenant.id
184
185 # give caller an admin role at the tenant they've created
186 self.driver.add_user_role(self.caller.kuser_id, tenant.id, 'admin')
187
188 # refresh credentials using this tenant
189 self.driver.shell.connect(username=self.driver.shell.keystone.username,
190 password=self.driver.shell.keystone.password,
191 tenant=tenant.name)
192
193 # create network
194 network = self.driver.create_network(slice.name)
195 slice.network_id = network['id']
196
197 # create router
198 router = self.driver.create_router(slice.name)
199 slice.router_id = router['id']
200
201 # create subnet
202 next_subnet = self.get_next_subnet()
203 cidr = str(next_subnet.cidr)
204 ip_version = next_subnet.version
205 start = str(next_subnet[2])
206 end = str(next_subnet[-2])
207 subnet = self.driver.create_subnet(name=slice.name,
208 network_id = network['id'],
209 cidr_ip = cidr,
210 ip_version = ip_version,
211 start = start,
212 end = end)
213 slice.subnet_id = subnet['id']
214 # add subnet as interface to slice's router
215 self.driver.add_router_interface(router['id'], subnet['id'])
216 # add external route
217 self.driver.add_external_route(subnet)
218
219
220 if slice.id and slice.tenant_id:
221 self.driver.update_tenant(slice.tenant_id,
222 description=slice.description,
223 enabled=slice.enabled)
224
225 slice.save()
226 slice.enacted = timezone.now()
227 slice.save(update_fields=['enacted'])
228
229 @require_enabled
230 def delete_slice(self, slice):
231 if slice.tenant_id:
232 self._delete_slice(slice.tenant_id, slice.network_id,
233 slice.router_id, slice.subnet_id)
234 @require_enabled
235 def _delete_slice(self, tenant_id, network_id, router_id, subnet_id):
236 self.driver.delete_router_interface(slice.router_id, slice.subnet_id)
237 self.driver.delete_subnet(slice.subnet_id)
238 self.driver.delete_router(slice.router_id)
239 self.driver.delete_network(slice.network_id)
240 self.driver.delete_tenant(slice.tenant_id)
241 # delete external route
242 subnet = None
243 subnets = self.driver.shell.neutron.list_subnets()['subnets']
244 for snet in subnets:
245 if snet['id'] == slice.subnet_id:
246 subnet = snet
247 if subnet:
248 self.driver.delete_external_route(subnet)
249
250
251 @require_enabled
252 def save_slice_membership(self, slice_memb):
253 if slice_memb.user.kuser_id and slice_memb.slice.tenant_id:
254 self.driver.add_user_role(slice_memb.user.kuser_id,
255 slice_memb.slice.tenant_id,
256 slice_memb.role.role_type)
257 slice_memb.enacted = timezone.now()
258 slice_memb.save(update_fields=['enacted'])
259
260
261 @require_enabled
262 def delete_slice_membership(self, slice_memb):
263 self.driver.delete_user_role(slice_memb.user.kuser_id,
264 slice_memb.slice.tenant_id,
265 slice_memb.role.role_type)
266
267
268 @require_enabled
269 def get_next_subnet(self):
270 # limit ourself to 10.0.x.x for now
271 valid_subnet = lambda net: net.startswith('10.0')
272 subnets = self.driver.shell.neutron.list_subnets()['subnets']
273 ints = [int(IPNetwork(subnet['cidr']).ip) for subnet in subnets \
274 if valid_subnet(subnet['cidr'])]
275 ints.sort()
276 last_ip = IPAddress(ints[-1])
277 last_network = IPNetwork(str(last_ip) + "/24")
278 next_network = IPNetwork(str(IPAddress(last_network) + last_network.size) + "/24")
279 return next_network
280
281 @require_enabled
282 def save_subnet(self, subnet):
283 if not subnet.subnet_id:
284 neutron_subnet = self.driver.create_subnet(name= subnet.slice.name,
285 network_id=subnet.slice.network_id,
286 cidr_ip = subnet.cidr,
287 ip_version=subnet.ip_version,
288 start = subnet.start,
289 end = subnet.end)
290 subnet.subnet_id = neutron_subnet['id']
291 # add subnet as interface to slice's router
292 self.driver.add_router_interface(subnet.slice.router_id, subnet.subnet_id)
293 #add_route = 'route add -net %s dev br-ex gw 10.100.0.5' % self.cidr
294 #commands.getstatusoutput(add_route)
295
296
297 @require_enabled
298 def delete_subnet(self, subnet):
299 if subnet.subnet_id:
300 self.driver.delete_router_interface(subnet.slice.router_id, subnet.subnet_id)
301 self.driver.delete_subnet(subnet.subnet_id)
302 #del_route = 'route del -net %s' % self.cidr
303 #commands.getstatusoutput(del_route)
304
305 def get_requested_networks(self, slice):
306 network_ids = [x.network_id for x in slice.networks.all()]
307
308 if slice.network_id is not None:
309 network_ids.append(slice.network_id)
310
311 networks = []
312 for network_id in network_ids:
313 networks.append({"net-id": network_id})
314
315 return networks
316
317 @require_enabled
318 def save_instance(self, instance):
319 metadata_update = {}
320 if ("numberCores" in instance.changed_fields):
321 metadata_update["cpu_cores"] = str(instance.numberCores)
322
323 for tag in instance.slice.tags.all():
324 if tag.name.startswith("sysctl-"):
325 metadata_update[tag.name] = tag.value
326
327 if not instance.instance_id:
328 nics = self.get_requested_networks(instance.slice)
329 for nic in nics:
330 # If a network hasn't been instantiated yet, then we'll fail
331 # during slice creation. Defer saving the instance for now.
332 if not nic.get("net-id", None):
333 instance.save() # in case it hasn't been saved yet
334 return
335 slice_memberships = SliceMembership.objects.filter(slice=instance.slice)
336 pubkeys = [sm.user.public_key for sm in slice_memberships if sm.user.public_key]
337 pubkeys.append(instance.creator.public_key)
338 instance = self.driver.spawn_instance(name=instance.name,
339 key_name = instance.creator.keyname,
340 image_id = instance.image.image_id,
341 hostname = instance.node.name,
342 pubkeys = pubkeys,
343 nics = nics,
344 metadata = metadata_update )
345 instance.instance_id = instance.id
346 instance.instance_name = getattr(instance, 'OS-EXT-SRV-ATTR:instance_name')
347 else:
348 if metadata_update:
349 self.driver.update_instance_metadata(instance.instance_id, metadata_update)
350
351 instance.save()
352 instance.enacted = timezone.now()
353 instance.save(update_fields=['enacted'])
354
355 @require_enabled
356 def delete_instance(self, instance):
357 if instance.instance_id:
358 self.driver.destroy_instance(instance.instance_id)
359
360
361 def refresh_nodes(self):
362 # collect local nodes
363 nodes = Node.objects.all()
364 nodes_dict = {}
365 for node in nodes:
366 if 'viccidev10' not in node.name:
367 nodes_dict[node.name] = node
368
369 deployment = Deployment.objects.filter(name='VICCI')[0]
370 login_bases = ['princeton', 'stanford', 'gt', 'uw', 'mpisws']
371 sites = Site.objects.filter(login_base__in=login_bases)
372 # collect nova nodes:
373 compute_nodes = self.client.nova.hypervisors.list()
374
375 compute_nodes_dict = {}
376 for compute_node in compute_nodes:
377 compute_nodes_dict[compute_node.hypervisor_hostname] = compute_node
378
379 # add new nodes:
380 new_node_names = set(compute_nodes_dict.keys()).difference(nodes_dict.keys())
381 i = 0
382 max = len(sites)
383 for name in new_node_names:
384 if i == max:
385 i = 0
386 site = sites[i]
387 node = Node(name=compute_nodes_dict[name].hypervisor_hostname,
388 site=site,
389 deployment=deployment)
390 node.save()
391 i+=1
392
393 # remove old nodes
394 old_node_names = set(nodes_dict.keys()).difference(compute_nodes_dict.keys())
395 Node.objects.filter(name__in=old_node_names).delete()
396
397 def refresh_images(self):
398 from core.models.image import Image
399 # collect local images
400 images = Image.objects.all()
401 images_dict = {}
402 for image in images:
403 images_dict[image.name] = image
404
405 # collect glance images
406 glance_images = self.client.glance.get_images()
407 glance_images_dict = {}
408 for glance_image in glance_images:
409 glance_images_dict[glance_image['name']] = glance_image
410
411 # add new images
412 new_image_names = set(glance_images_dict.keys()).difference(images_dict.keys())
413 for name in new_image_names:
414 image = Image(image_id=glance_images_dict[name]['id'],
415 name=glance_images_dict[name]['name'],
416 disk_format=glance_images_dict[name]['disk_format'],
417 container_format=glance_images_dict[name]['container_format'])
418 image.save()
419
420 # remove old images
421 old_image_names = set(images_dict.keys()).difference(glance_images_dict.keys())
422 Image.objects.filter(name__in=old_image_names).delete()
423
424 @require_enabled
425 def save_network(self, network):
426 if not network.network_id:
427 if network.template.shared_network_name:
428 network.network_id = network.template.shared_network_id
429 (network.subnet_id, network.subnet) = self.driver.get_network_subnet(network.network_id)
430 else:
431 network_name = network.name
432
433 # create network
434 os_network = self.driver.create_network(network_name, shared=True)
435 network.network_id = os_network['id']
436
437 # create router
438 router = self.driver.create_router(network_name)
439 network.router_id = router['id']
440
441 # create subnet
442 next_subnet = self.get_next_subnet()
443 cidr = str(next_subnet.cidr)
444 ip_version = next_subnet.version
445 start = str(next_subnet[2])
446 end = str(next_subnet[-2])
447 subnet = self.driver.create_subnet(name=network_name,
448 network_id = network.network_id,
449 cidr_ip = cidr,
450 ip_version = ip_version,
451 start = start,
452 end = end)
453 network.subnet = cidr
454 network.subnet_id = subnet['id']
455 # add subnet as interface to slice's router
456 self.driver.add_router_interface(router['id'], subnet['id'])
457 # add external route
458 self.driver.add_external_route(subnet)
459
460 network.save()
461 network.enacted = timezone.now()
462 network.save(update_fields=['enacted'])
463
464 def delete_network(self, network):
465 if (network.router_id) and (network.subnet_id):
466 self.driver.delete_router_interface(network.router_id, network.subnet_id)
467 if network.subnet_id:
468 self.driver.delete_subnet(network.subnet_id)
469 if network.router_id:
470 self.driver.delete_router(network.router_id)
471 if network.network_id:
472 self.driver.delete_network(network.network_id)
473
474 def save_network_template(self, template):
475 if (template.shared_network_name) and (not template.shared_network_id):
476 os_networks = self.driver.shell.neutron.list_networks(name=template.shared_network_name)['networks']
477 if os_networks:
478 template.shared_network_id = os_networks[0]["id"]
479
480 template.save()
481 template.enacted = timezone.now()
482 template.save(update_fields=['enacted'])
483
484 def find_or_make_template_for_network(self, name):
485 """ Given a network name, try to guess the right template for it """
486
487 # templates for networks we may encounter
488 if name=='nat-net':
489 template_dict = None # {"name": "private-nat", "visibility": "private", "translation": "nat"}
490 elif name=='sharednet1':
491 template_dict = {"name": "dedicated-public", "visibility": "public", "translation": "none"}
492 else:
493 template_dict = {"name": "private", "visibility": "private", "translation": "none"}
494
495 # if we have an existing template return it
496 templates = NetworkTemplate.objects.filter(name=template_dict["name"])
497 if templates:
498 return templates[0]
499
500 if template_dict == None:
501 return None
502
503 template = NetworkTemplate(**template_dict)
504 template.save()
505 return template
506
507 def refresh_network_templates(self):
508 for template in NetworkTemplate.objects.all():
509 if (template.shared_network_name) and (not template.shared_network_id):
510 # this will cause us to try to fill in the shared_network_id
511 self.save_network_template(template)
512
513 def refresh_networks(self):
514 # get a list of all networks in the model
515
516 networks = Network.objects.all()
517 networks_by_name = {}
518 networks_by_id = {}
519 for network in networks:
520 networks_by_name[network.name] = network
521 networks_by_id[network.network_id] = network
522
523 # Get a list of all shared networks in OS
524
525 os_networks = self.driver.shell.neutron.list_networks()['networks']
526 os_networks_by_name = {}
527 os_networks_by_id = {}
528 for os_network in os_networks:
529 os_networks_by_name[os_network['name']] = os_network
530 os_networks_by_id[os_network['id']] = os_network
531
532 for (uuid, os_network) in os_networks_by_id.items():
533 #print "checking OS network", os_network['name']
534 if (os_network['shared']) and (uuid not in networks_by_id):
535 # Only automatically create shared networks. This is for Andy's
536 # nat-net and sharednet1.
537
538 owner_slice = Slice.objects.get(tenant_id = os_network['tenant_id'])
539 template = self.find_or_make_template_for_network(os_network['name'])
540
541 if (template is None):
542 # This is our way of saying we don't want to auto-instantiate
543 # this network type.
544 continue
545
546 (subnet_id, subnet) = self.driver.get_network_subnet(os_network['id'])
547
548 if owner_slice:
549 #print "creating model object for OS network", os_network['name']
550 new_network = Network(name = os_network['name'],
551 template = template,
552 owner = owner_slice,
553 network_id = uuid,
554 subnet_id = subnet_id)
555 new_network.save()
556
557 for (network_id, network) in networks_by_id.items():
558 # If the network disappeared from OS, then reset its network_id to None
559 if (network.network_id is not None) and (network.network_id not in os_networks_by_id):
560 network.network_id = None
561
562 # If no OS object exists, then saving the network will create one
563 if (network.network_id is None):
564 #print "creating OS network for", network.name
565 self.save_network(network)
566 else:
567 pass #print "network", network.name, "has its OS object"
568
569