| import os |
| #os.environ.setdefault("DJANGO_SETTINGS_MODULE", "xos.settings") |
| import string |
| import random |
| import hashlib |
| from datetime import datetime |
| from django.utils import timezone |
| |
| from netaddr import IPAddress, IPNetwork |
| from xos import settings |
| from django.core import management |
| from core.models import * |
| from xos.config import Config |
| try: |
| from synchronizers.openstack.client import OpenStackClient |
| from synchronizers.openstack.driver import OpenStackDriver |
| has_openstack = True |
| except: |
| has_openstack = False |
| |
| manager_enabled = Config().api_nova_enabled |
| |
| |
| def random_string(size=6): |
| return ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(size)) |
| |
| def require_enabled(callable): |
| def wrapper(*args, **kwds): |
| if manager_enabled and has_openstack: |
| return callable(*args, **kwds) |
| else: |
| return None |
| return wrapper |
| |
| |
| class OpenStackManager: |
| |
| def __init__(self, auth={}, caller=None): |
| self.client = None |
| self.driver = None |
| self.caller = None |
| self.has_openstack = has_openstack |
| self.enabled = manager_enabled |
| |
| if has_openstack and manager_enabled: |
| if auth: |
| try: |
| self.init_user(auth, caller) |
| except: |
| # if this fails then it meanse the caller doesn't have a |
| # role at the slice's tenant. if the caller is an admin |
| # just use the admin client/manager. |
| if caller and caller.is_admin: |
| self.init_admin() |
| else: raise |
| else: |
| self.init_admin() |
| |
| @require_enabled |
| def init_caller(self, caller, tenant): |
| auth = {'username': caller.email, |
| 'password': hashlib.md5(caller.password).hexdigest()[:6], |
| 'tenant': tenant} |
| self.client = OpenStackClient(**auth) |
| self.driver = OpenStackDriver(client=self.client) |
| self.caller = caller |
| |
| @require_enabled |
| def init_admin(self, tenant=None): |
| # use the admin credentials |
| self.client = OpenStackClient(tenant=tenant) |
| self.driver = OpenStackDriver(client=self.client) |
| self.caller = self.driver.admin_user |
| self.caller.kuser_id = self.caller.id |
| |
| @require_enabled |
| def save_role(self, role): |
| if not role.role: |
| keystone_role = self.driver.create_role(role.role_type) |
| role.role = keystone_role.id |
| |
| @require_enabled |
| def delete_role(self, role): |
| if role.role: |
| self.driver.delete_role({'id': role.role}) |
| |
| @require_enabled |
| def save_key(self, key, name): |
| key_fields = {'name': name, |
| 'public_key': key} |
| nova_key = self.driver.create_keypair(**key_fields) |
| |
| @require_enabled |
| def delete_key(self, key): |
| if key.nkey_id: |
| self.driver.delete_keypair(key.nkey_id) |
| |
| @require_enabled |
| def save_user(self, user): |
| name = user.email[:user.email.find('@')] |
| user_fields = {'name': name, |
| 'email': user.email, |
| 'password': hashlib.md5(user.password).hexdigest()[:6], |
| 'enabled': True} |
| if not user.kuser_id: |
| keystone_user = self.driver.create_user(**user_fields) |
| user.kuser_id = keystone_user.id |
| else: |
| self.driver.update_user(user.kuser_id, user_fields) |
| |
| if user.site: |
| self.driver.add_user_role(user.kuser_id, user.site.tenant_id, 'user') |
| if user.is_admin: |
| self.driver.add_user_role(user.kuser_id, user.site.tenant_id, 'admin') |
| else: |
| # may have admin role so attempt to remove it |
| self.driver.delete_user_role(user.kuser_id, user.site.tenant_id, 'admin') |
| |
| if user.public_key: |
| self.init_caller(user, user.site.login_base) |
| self.save_key(user.public_key, user.keyname) |
| self.init_admin() |
| |
| user.save() |
| user.enacted = timezone.now() |
| user.save(update_fields=['enacted']) |
| |
| @require_enabled |
| def delete_user(self, user): |
| if user.kuser_id: |
| self.driver.delete_user(user.kuser_id) |
| |
| @require_enabled |
| def save_site(self, site, add_role=True): |
| if not site.tenant_id: |
| tenant = self.driver.create_tenant(tenant_name=site.login_base, |
| description=site.name, |
| enabled=site.enabled) |
| site.tenant_id = tenant.id |
| # give caller an admin role at the tenant they've created |
| self.driver.add_user_role(self.caller.kuser_id, tenant.id, 'admin') |
| |
| # update the record |
| if site.id and site.tenant_id: |
| self.driver.update_tenant(site.tenant_id, |
| description=site.name, |
| enabled=site.enabled) |
| |
| # commit the updated record |
| site.save() |
| site.enacted = timezone.now() |
| site.save(update_fields=['enacted']) # enusre enacted > updated |
| |
| |
| @require_enabled |
| def delete_site(self, site): |
| if site.tenant_id: |
| self.driver.delete_tenant(site.tenant_id) |
| |
| @require_enabled |
| def save_site_privilege(self, site_priv): |
| if site_priv.user.kuser_id and site_priv.site.tenant_id: |
| self.driver.add_user_role(site_priv.user.kuser_id, |
| site_priv.site.tenant_id, |
| site_priv.role.role_type) |
| site_priv.enacted = timezone.now() |
| site_priv.save(update_fields=['enacted']) |
| |
| |
| @require_enabled |
| def delete_site_privilege(self, site_priv): |
| self.driver.delete_user_role(site_priv.user.kuser_id, |
| site_priv.site.tenant_id, |
| site_priv.role.role_type) |
| |
| @require_enabled |
| def save_slice(self, slice): |
| if not slice.tenant_id: |
| nova_fields = {'tenant_name': slice.name, |
| 'description': slice.description, |
| 'enabled': slice.enabled} |
| tenant = self.driver.create_tenant(**nova_fields) |
| slice.tenant_id = tenant.id |
| |
| # give caller an admin role at the tenant they've created |
| self.driver.add_user_role(self.caller.kuser_id, tenant.id, 'admin') |
| |
| # refresh credentials using this tenant |
| self.driver.shell.connect(username=self.driver.shell.keystone.username, |
| password=self.driver.shell.keystone.password, |
| tenant=tenant.name) |
| |
| # create network |
| network = self.driver.create_network(slice.name) |
| slice.network_id = network['id'] |
| |
| # create router |
| router = self.driver.create_router(slice.name) |
| slice.router_id = router['id'] |
| |
| # create subnet |
| next_subnet = self.get_next_subnet() |
| cidr = str(next_subnet.cidr) |
| ip_version = next_subnet.version |
| start = str(next_subnet[2]) |
| end = str(next_subnet[-2]) |
| subnet = self.driver.create_subnet(name=slice.name, |
| network_id = network['id'], |
| cidr_ip = cidr, |
| ip_version = ip_version, |
| start = start, |
| end = end) |
| slice.subnet_id = subnet['id'] |
| # add subnet as interface to slice's router |
| self.driver.add_router_interface(router['id'], subnet['id']) |
| # add external route |
| self.driver.add_external_route(subnet) |
| |
| |
| if slice.id and slice.tenant_id: |
| self.driver.update_tenant(slice.tenant_id, |
| description=slice.description, |
| enabled=slice.enabled) |
| |
| slice.save() |
| slice.enacted = timezone.now() |
| slice.save(update_fields=['enacted']) |
| |
| @require_enabled |
| def delete_slice(self, slice): |
| if slice.tenant_id: |
| self._delete_slice(slice.tenant_id, slice.network_id, |
| slice.router_id, slice.subnet_id) |
| @require_enabled |
| def _delete_slice(self, tenant_id, network_id, router_id, subnet_id): |
| self.driver.delete_router_interface(slice.router_id, slice.subnet_id) |
| self.driver.delete_subnet(slice.subnet_id) |
| self.driver.delete_router(slice.router_id) |
| self.driver.delete_network(slice.network_id) |
| self.driver.delete_tenant(slice.tenant_id) |
| # delete external route |
| subnet = None |
| subnets = self.driver.shell.neutron.list_subnets()['subnets'] |
| for snet in subnets: |
| if snet['id'] == slice.subnet_id: |
| subnet = snet |
| if subnet: |
| self.driver.delete_external_route(subnet) |
| |
| |
| @require_enabled |
| def save_slice_membership(self, slice_memb): |
| if slice_memb.user.kuser_id and slice_memb.slice.tenant_id: |
| self.driver.add_user_role(slice_memb.user.kuser_id, |
| slice_memb.slice.tenant_id, |
| slice_memb.role.role_type) |
| slice_memb.enacted = timezone.now() |
| slice_memb.save(update_fields=['enacted']) |
| |
| |
| @require_enabled |
| def delete_slice_membership(self, slice_memb): |
| self.driver.delete_user_role(slice_memb.user.kuser_id, |
| slice_memb.slice.tenant_id, |
| slice_memb.role.role_type) |
| |
| |
| @require_enabled |
| def get_next_subnet(self): |
| # limit ourself to 10.0.x.x for now |
| valid_subnet = lambda net: net.startswith('10.0') |
| subnets = self.driver.shell.neutron.list_subnets()['subnets'] |
| ints = [int(IPNetwork(subnet['cidr']).ip) for subnet in subnets \ |
| if valid_subnet(subnet['cidr'])] |
| ints.sort() |
| last_ip = IPAddress(ints[-1]) |
| last_network = IPNetwork(str(last_ip) + "/24") |
| next_network = IPNetwork(str(IPAddress(last_network) + last_network.size) + "/24") |
| return next_network |
| |
| @require_enabled |
| def save_subnet(self, subnet): |
| if not subnet.subnet_id: |
| neutron_subnet = self.driver.create_subnet(name= subnet.slice.name, |
| network_id=subnet.slice.network_id, |
| cidr_ip = subnet.cidr, |
| ip_version=subnet.ip_version, |
| start = subnet.start, |
| end = subnet.end) |
| subnet.subnet_id = neutron_subnet['id'] |
| # add subnet as interface to slice's router |
| self.driver.add_router_interface(subnet.slice.router_id, subnet.subnet_id) |
| #add_route = 'route add -net %s dev br-ex gw 10.100.0.5' % self.cidr |
| #commands.getstatusoutput(add_route) |
| |
| |
| @require_enabled |
| def delete_subnet(self, subnet): |
| if subnet.subnet_id: |
| self.driver.delete_router_interface(subnet.slice.router_id, subnet.subnet_id) |
| self.driver.delete_subnet(subnet.subnet_id) |
| #del_route = 'route del -net %s' % self.cidr |
| #commands.getstatusoutput(del_route) |
| |
| def get_requested_networks(self, slice): |
| network_ids = [x.network_id for x in slice.networks.all()] |
| |
| if slice.network_id is not None: |
| network_ids.append(slice.network_id) |
| |
| networks = [] |
| for network_id in network_ids: |
| networks.append({"net-id": network_id}) |
| |
| return networks |
| |
| @require_enabled |
| def save_instance(self, instance): |
| metadata_update = {} |
| if ("numberCores" in instance.changed_fields): |
| metadata_update["cpu_cores"] = str(instance.numberCores) |
| |
| for tag in instance.slice.tags.all(): |
| if tag.name.startswith("sysctl-"): |
| metadata_update[tag.name] = tag.value |
| |
| if not instance.instance_id: |
| nics = self.get_requested_networks(instance.slice) |
| for nic in nics: |
| # If a network hasn't been instantiated yet, then we'll fail |
| # during slice creation. Defer saving the instance for now. |
| if not nic.get("net-id", None): |
| instance.save() # in case it hasn't been saved yet |
| return |
| slice_memberships = SliceMembership.objects.filter(slice=instance.slice) |
| pubkeys = [sm.user.public_key for sm in slice_memberships if sm.user.public_key] |
| pubkeys.append(instance.creator.public_key) |
| instance = self.driver.spawn_instance(name=instance.name, |
| key_name = instance.creator.keyname, |
| image_id = instance.image.image_id, |
| hostname = instance.node.name, |
| pubkeys = pubkeys, |
| nics = nics, |
| metadata = metadata_update ) |
| instance.instance_id = instance.id |
| instance.instance_name = getattr(instance, 'OS-EXT-SRV-ATTR:instance_name') |
| else: |
| if metadata_update: |
| self.driver.update_instance_metadata(instance.instance_id, metadata_update) |
| |
| instance.save() |
| instance.enacted = timezone.now() |
| instance.save(update_fields=['enacted']) |
| |
| @require_enabled |
| def delete_instance(self, instance): |
| if instance.instance_id: |
| self.driver.destroy_instance(instance.instance_id) |
| |
| |
| def refresh_nodes(self): |
| # collect local nodes |
| nodes = Node.objects.all() |
| nodes_dict = {} |
| for node in nodes: |
| if 'viccidev10' not in node.name: |
| nodes_dict[node.name] = node |
| |
| deployment = Deployment.objects.filter(name='VICCI')[0] |
| login_bases = ['princeton', 'stanford', 'gt', 'uw', 'mpisws'] |
| sites = Site.objects.filter(login_base__in=login_bases) |
| # collect nova nodes: |
| compute_nodes = self.client.nova.hypervisors.list() |
| |
| compute_nodes_dict = {} |
| for compute_node in compute_nodes: |
| compute_nodes_dict[compute_node.hypervisor_hostname] = compute_node |
| |
| # add new nodes: |
| new_node_names = set(compute_nodes_dict.keys()).difference(nodes_dict.keys()) |
| i = 0 |
| max = len(sites) |
| for name in new_node_names: |
| if i == max: |
| i = 0 |
| site = sites[i] |
| node = Node(name=compute_nodes_dict[name].hypervisor_hostname, |
| site=site, |
| deployment=deployment) |
| node.save() |
| i+=1 |
| |
| # remove old nodes |
| old_node_names = set(nodes_dict.keys()).difference(compute_nodes_dict.keys()) |
| Node.objects.filter(name__in=old_node_names).delete() |
| |
| def refresh_images(self): |
| from core.models.image import Image |
| # collect local images |
| images = Image.objects.all() |
| images_dict = {} |
| for image in images: |
| images_dict[image.name] = image |
| |
| # collect glance images |
| glance_images = self.client.glance.get_images() |
| glance_images_dict = {} |
| for glance_image in glance_images: |
| glance_images_dict[glance_image['name']] = glance_image |
| |
| # add new images |
| new_image_names = set(glance_images_dict.keys()).difference(images_dict.keys()) |
| for name in new_image_names: |
| image = Image(image_id=glance_images_dict[name]['id'], |
| name=glance_images_dict[name]['name'], |
| disk_format=glance_images_dict[name]['disk_format'], |
| container_format=glance_images_dict[name]['container_format']) |
| image.save() |
| |
| # remove old images |
| old_image_names = set(images_dict.keys()).difference(glance_images_dict.keys()) |
| Image.objects.filter(name__in=old_image_names).delete() |
| |
| @require_enabled |
| def save_network(self, network): |
| if not network.network_id: |
| if network.template.shared_network_name: |
| network.network_id = network.template.shared_network_id |
| (network.subnet_id, network.subnet) = self.driver.get_network_subnet(network.network_id) |
| else: |
| network_name = network.name |
| |
| # create network |
| os_network = self.driver.create_network(network_name, shared=True) |
| network.network_id = os_network['id'] |
| |
| # create router |
| router = self.driver.create_router(network_name) |
| network.router_id = router['id'] |
| |
| # create subnet |
| next_subnet = self.get_next_subnet() |
| cidr = str(next_subnet.cidr) |
| ip_version = next_subnet.version |
| start = str(next_subnet[2]) |
| end = str(next_subnet[-2]) |
| subnet = self.driver.create_subnet(name=network_name, |
| network_id = network.network_id, |
| cidr_ip = cidr, |
| ip_version = ip_version, |
| start = start, |
| end = end) |
| network.subnet = cidr |
| network.subnet_id = subnet['id'] |
| # add subnet as interface to slice's router |
| self.driver.add_router_interface(router['id'], subnet['id']) |
| # add external route |
| self.driver.add_external_route(subnet) |
| |
| network.save() |
| network.enacted = timezone.now() |
| network.save(update_fields=['enacted']) |
| |
| def delete_network(self, network): |
| if (network.router_id) and (network.subnet_id): |
| self.driver.delete_router_interface(network.router_id, network.subnet_id) |
| if network.subnet_id: |
| self.driver.delete_subnet(network.subnet_id) |
| if network.router_id: |
| self.driver.delete_router(network.router_id) |
| if network.network_id: |
| self.driver.delete_network(network.network_id) |
| |
| def save_network_template(self, template): |
| if (template.shared_network_name) and (not template.shared_network_id): |
| os_networks = self.driver.shell.neutron.list_networks(name=template.shared_network_name)['networks'] |
| if os_networks: |
| template.shared_network_id = os_networks[0]["id"] |
| |
| template.save() |
| template.enacted = timezone.now() |
| template.save(update_fields=['enacted']) |
| |
| def find_or_make_template_for_network(self, name): |
| """ Given a network name, try to guess the right template for it """ |
| |
| # templates for networks we may encounter |
| if name=='nat-net': |
| template_dict = None # {"name": "private-nat", "visibility": "private", "translation": "nat"} |
| elif name=='sharednet1': |
| template_dict = {"name": "dedicated-public", "visibility": "public", "translation": "none"} |
| else: |
| template_dict = {"name": "private", "visibility": "private", "translation": "none"} |
| |
| # if we have an existing template return it |
| templates = NetworkTemplate.objects.filter(name=template_dict["name"]) |
| if templates: |
| return templates[0] |
| |
| if template_dict == None: |
| return None |
| |
| template = NetworkTemplate(**template_dict) |
| template.save() |
| return template |
| |
| def refresh_network_templates(self): |
| for template in NetworkTemplate.objects.all(): |
| if (template.shared_network_name) and (not template.shared_network_id): |
| # this will cause us to try to fill in the shared_network_id |
| self.save_network_template(template) |
| |
| def refresh_networks(self): |
| # get a list of all networks in the model |
| |
| networks = Network.objects.all() |
| networks_by_name = {} |
| networks_by_id = {} |
| for network in networks: |
| networks_by_name[network.name] = network |
| networks_by_id[network.network_id] = network |
| |
| # Get a list of all shared networks in OS |
| |
| os_networks = self.driver.shell.neutron.list_networks()['networks'] |
| os_networks_by_name = {} |
| os_networks_by_id = {} |
| for os_network in os_networks: |
| os_networks_by_name[os_network['name']] = os_network |
| os_networks_by_id[os_network['id']] = os_network |
| |
| for (uuid, os_network) in os_networks_by_id.items(): |
| #print "checking OS network", os_network['name'] |
| if (os_network['shared']) and (uuid not in networks_by_id): |
| # Only automatically create shared networks. This is for Andy's |
| # nat-net and sharednet1. |
| |
| owner_slice = Slice.objects.get(tenant_id = os_network['tenant_id']) |
| template = self.find_or_make_template_for_network(os_network['name']) |
| |
| if (template is None): |
| # This is our way of saying we don't want to auto-instantiate |
| # this network type. |
| continue |
| |
| (subnet_id, subnet) = self.driver.get_network_subnet(os_network['id']) |
| |
| if owner_slice: |
| #print "creating model object for OS network", os_network['name'] |
| new_network = Network(name = os_network['name'], |
| template = template, |
| owner = owner_slice, |
| network_id = uuid, |
| subnet_id = subnet_id) |
| new_network.save() |
| |
| for (network_id, network) in networks_by_id.items(): |
| # If the network disappeared from OS, then reset its network_id to None |
| if (network.network_id is not None) and (network.network_id not in os_networks_by_id): |
| network.network_id = None |
| |
| # If no OS object exists, then saving the network will create one |
| if (network.network_id is None): |
| #print "creating OS network for", network.name |
| self.save_network(network) |
| else: |
| pass #print "network", network.name, "has its OS object" |
| |
| |