This tree was mixed up, with an old version of the EC2 Observer. This
change copies across the version demo'd the OpenCloud developer meeting.
diff --git a/planetstack/ec2_observer/steps/__init__.py b/planetstack/ec2_observer/steps/__init__.py
index eabf46c..e56c4b1 100644
--- a/planetstack/ec2_observer/steps/__init__.py
+++ b/planetstack/ec2_observer/steps/__init__.py
@@ -1,15 +1,15 @@
#from .sync_external_routes import SyncExternalRoutes
-from .sync_network_slivers import SyncNetworkSlivers
-from .sync_networks import SyncNetworks
-from .sync_network_deployments import SyncNetworkDeployments
-from .sync_site_privileges import SyncSitePrivileges
-from .sync_sites import SyncSites
-from .sync_slice_memberships import SyncSliceMemberships
-from .sync_slices import SyncSlices
+#from .sync_network_slivers import SyncNetworkSlivers
+#from .sync_networks import SyncNetworks
+#from .sync_network_deployments import SyncNetworkDeployments
+#from .sync_site_privileges import SyncSitePrivileges
+#from .sync_slice_memberships import SyncSliceMemberships
+#from .sync_slices import SyncSlices
#from .sync_sliver_ips import SyncSliverIps
-from .sync_slivers import SyncSlivers
-from .sync_users import SyncUsers
-from .sync_roles import SyncRoles
-from .sync_nodes import SyncNodes
-from .sync_images import SyncImages
-from .garbage_collector import GarbageCollector
+#from .sync_slivers import SyncSlivers
+#from .sync_users import SyncUsers
+#from .sync_roles import SyncRoles
+#from .sync_nodes import SyncNodes
+#from .sync_images import SyncImages
+#from .garbage_collector import GarbageCollector
+
diff --git a/planetstack/ec2_observer/steps/sync_deployments.py b/planetstack/ec2_observer/steps/sync_deployments.py
new file mode 100644
index 0000000..8a258f1
--- /dev/null
+++ b/planetstack/ec2_observer/steps/sync_deployments.py
@@ -0,0 +1,22 @@
+import os
+import base64
+from django.db.models import F, Q
+from planetstack.config import Config
+from ec2_observer.syncstep import SyncStep
+from core.models.site import *
+
+class SyncDeployments(SyncStep):
+ requested_interval=86400
+ provides=[Deployment]
+
+ def fetch_pending(self,deletion):
+ deployments = Deployment.objects.filter(Q(name="Amazon EC2"))
+ if (not deployments):
+ deployments = [Deployment(name="Amazon EC2")]
+ else:
+ deployments = []
+
+ return deployments
+
+ def sync_record(self, deployment):
+ deployment.save()
diff --git a/planetstack/ec2_observer/steps/sync_images.py b/planetstack/ec2_observer/steps/sync_images.py
index 32b3363..3a15b33 100644
--- a/planetstack/ec2_observer/steps/sync_images.py
+++ b/planetstack/ec2_observer/steps/sync_images.py
@@ -2,30 +2,59 @@
import base64
from django.db.models import F, Q
from planetstack.config import Config
-from observer.syncstep import SyncStep
+from ec2_observer.syncstep import SyncStep
from core.models.image import Image
-from awslib import *
+from ec2_observer.awslib import *
-class SyncImages(OpenStackSyncStep):
+
+class SyncImages(SyncStep):
provides=[Image]
requested_interval=3600
- def fetch_pending(self):
+ def fetch_pending(self,deletion):
images = Image.objects.all()
image_names = [image.name for image in images]
new_images = []
- aws_images = aws_run('ec2 describe-images')
+ try:
+ aws_images = json.loads(open('/opt/planetstack/aws-images').read())
+ except:
+ aws_images = aws_run('ec2 describe-images --owner 099720109477')
+ open('/opt/planetstack/aws-images','w').write(json.dumps(aws_images))
+
+
+ aws_images=aws_images['Images']
+ aws_images=filter(lambda x:x['ImageType']=='machine',aws_images)[:50]
+
+ names = set([])
for aws_image in aws_images:
- if aws_image not in image_names:
- image = Image(image_id=image_id,
- name=aws_image['name'],
- disk_format='XXX'
- container_format='XXX'
- new_images.append(image)
-
+ desc_ok = True
+
+ try:
+ desc = aws_image['Description']
+ except:
+ try:
+ desc = aws_image['ImageLocation']
+ except:
+ desc_ok = False
+
+ if (desc_ok):
+ try:
+ desc_ok = desc and desc not in names and desc not in image_names and '14.04' in desc
+ except KeyError:
+ desc_ok = False
+
+ if desc_ok and aws_image['ImageType']=='machine':
+ image = Image(disk_format=aws_image['ImageLocation'],
+ name=desc,
+ container_format=aws_image['Hypervisor'],
+ path=aws_image['ImageId'])
+ new_images.append(image)
+ names.add(image.name)
+
+ #print new_images
return new_images
def sync_record(self, image):
diff --git a/planetstack/ec2_observer/steps/sync_nodes.py b/planetstack/ec2_observer/steps/sync_nodes.py
new file mode 100644
index 0000000..b66eb2e
--- /dev/null
+++ b/planetstack/ec2_observer/steps/sync_nodes.py
@@ -0,0 +1,48 @@
+import os
+import base64
+import random
+import time
+from datetime import datetime
+from django.db.models import F, Q
+from planetstack.config import Config
+from ec2_observer.syncstep import SyncStep
+from core.models.node import Node
+from core.models.site import *
+from ec2_observer.awslib import *
+import pdb
+
+class SyncNodes(SyncStep):
+ provides=[Node]
+ requested_interval=0
+
+ def fetch_pending(self, deletion):
+ deployment = Deployment.objects.filter(Q(name="Amazon EC2"))[0]
+ current_site_deployments = SiteDeployments.objects.filter(Q(deployment=deployment))
+
+ zone_ret = aws_run('ec2 describe-availability-zones')
+ zones = zone_ret['AvailabilityZones']
+
+ # collect local nodes
+ instance_types = 'm1.small | m1.medium | m1.large | m1.xlarge | m3.medium | m3.large | m3.xlarge | m3.2xlarge'.split(' | ')
+
+ all_new_nodes = []
+ for sd in current_site_deployments:
+ s = sd.site
+ current_fqns = [n.name for n in s.nodes.all()]
+ all_fqns = ['.'.join([n,s.name]) for n in instance_types]
+ new_node_names = list(set(all_fqns) - set(current_fqns))
+
+ new_nodes = []
+ for node_name in new_node_names:
+ node = Node(name=node_name,
+ site=s,deployment=deployment)
+ new_nodes.append(node)
+
+ all_new_nodes.extend(new_nodes)
+
+ return all_new_nodes
+
+
+ def sync_record(self, node):
+ node.save()
+
diff --git a/planetstack/ec2_observer/steps/sync_site_deployments.py b/planetstack/ec2_observer/steps/sync_site_deployments.py
index a996c85..30fdf97 100644
--- a/planetstack/ec2_observer/steps/sync_site_deployments.py
+++ b/planetstack/ec2_observer/steps/sync_site_deployments.py
@@ -2,27 +2,46 @@
import base64
from django.db.models import F, Q
from planetstack.config import Config
-from observer.openstacksyncstep import OpenStackSyncStep
+from ec2_observer.syncstep import SyncStep
from core.models.site import *
+from ec2_observer.awslib import *
+import pdb
-class SyncSiteDeployments(OpenStackSyncStep):
- requested_interval=0
- provides=[Site, SiteDeployments]
+class SyncSiteDeployments(SyncStep):
+ requested_interval=86400
+ provides=[SiteDeployments]
- def fetch_pending(self):
- return SiteDeployments.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
+ def fetch_pending(self, deletion):
+ zone_ret = aws_run('ec2 describe-availability-zones')
+ zones = zone_ret['AvailabilityZones']
+ available_sites = [zone['ZoneName'] for zone in zones]
- def sync_record(self, site_deployment):
- if not site_deployment.tenant_id:
- driver = self.driver.admin_driver(deployment=site_deployment.deployment.name)
- tenant = driver.create_tenant(tenant_name=site_deployment.site.login_base,
- description=site_deployment.site.name,
- enabled=site_deployment.site.enabled)
- site_deployment.tenant_id = tenant.id
- site_deployment.save()
- elif site_deployment.site.id and site_deployment.tenant_id:
- driver = self.driver.admin_driver(deployment=site_deployment.name)
- driver.update_tenant(site_deployment.tenant_id,
- description=site_deployment.site.name,
- enabled=site_deployment.site.enabled)
-
+ current_sites = []
+ for s in available_sites:
+ site = Site.objects.filter(Q(name=s))
+ if (site):
+ current_sites.append(site[0])
+
+ # OK not to catch exception
+ # The syncstep should catch it
+ # At any rate, we should not run if there are no deployments
+ deployment = Deployment.objects.filter(Q(name="Amazon EC2"))[0]
+ current_site_deployments = SiteDeployments.objects.filter(Q(deployment=deployment))
+ site_dict = {}
+
+ for sd in current_site_deployments:
+ site_dict[sd.site]=sd
+
+ updated_site_deployments = []
+ for site in current_sites:
+ try:
+ site_record = site_dict[site]
+ except KeyError:
+ sd = SiteDeployments(site=site,deployment=deployment,tenant_id=base64.urlsafe_b64encode(os.urandom(12)))
+ updated_site_deployments.append(sd)
+
+ return updated_site_deployments
+
+
+ def sync_record(self, site_deployment):
+ site_deployment.save()
diff --git a/planetstack/ec2_observer/steps/sync_sites.py b/planetstack/ec2_observer/steps/sync_sites.py
index 5771aef..7a5c0dc 100644
--- a/planetstack/ec2_observer/steps/sync_sites.py
+++ b/planetstack/ec2_observer/steps/sync_sites.py
@@ -2,20 +2,27 @@
import base64
from django.db.models import F, Q
from planetstack.config import Config
-from observer.syncstep import SyncStep
-from core.models.site import Site
+from ec2_observer.syncstep import SyncStep
+from core.models.site import *
from ec2_observer.awslib import *
+import pdb
class SyncSites(SyncStep):
- provides=[Site]
- requested_interval=3600
+ provides=[Site]
+ requested_interval=3600
- def fetch_pending(self):
- current_sites = Site.objects.all()
- zones = aws_run('ec2 describe-availability-zones')
+ def fetch_pending(self, deletion):
+
+ deployment = Deployment.objects.filter(Q(name="Amazon EC2"))[0]
+ current_site_deployments = SiteDeployments.objects.filter(Q(deployment=deployment))
+
+ zone_ret = aws_run('ec2 describe-availability-zones')
+ zones = zone_ret['AvailabilityZones']
+
available_sites = [zone['ZoneName'] for zone in zones]
+ site_names = [sd.site.name for sd in current_site_deployments]
- new_site_names = list(set(available_sites) - set(zones))
+ new_site_names = list(set(available_sites) - set(site_names))
new_sites = []
for s in new_site_names:
@@ -27,8 +34,8 @@
abbreviated_name=s)
new_sites.append(site)
- return new_sites
+ return new_sites
- def sync_record(self, site):
- site.save()
+ def sync_record(self, site):
+ site.save()
diff --git a/planetstack/ec2_observer/steps/sync_slivers.py b/planetstack/ec2_observer/steps/sync_slivers.py
index b576bbc..2079859 100644
--- a/planetstack/ec2_observer/steps/sync_slivers.py
+++ b/planetstack/ec2_observer/steps/sync_slivers.py
@@ -1,91 +1,103 @@
import os
+import json
import base64
from django.db.models import F, Q
from planetstack.config import Config
-from observer.openstacksyncstep import OpenStackSyncStep
+from ec2_observer.syncstep import SyncStep
from core.models.sliver import Sliver
from core.models.slice import SlicePrivilege, SliceDeployments
from core.models.network import Network, NetworkSlice, NetworkDeployments
from util.logger import Logger, logging
+from ec2_observer.awslib import *
+from core.models.site import *
+from core.models.slice import *
+import pdb
logger = Logger(level=logging.INFO)
-class SyncSlivers(OpenStackSyncStep):
- provides=[Sliver]
- requested_interval=0
+class SyncSlivers(SyncStep):
+ provides=[Sliver]
+ requested_interval=0
- def fetch_pending(self):
- return Sliver.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
+ def fetch_pending(self, deletion):
+ all_slivers = Sliver.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
+ my_slivers = []
- def sync_record(self, sliver):
- logger.info("sync'ing sliver:%s deployment:%s " % (sliver, sliver.node.deployment))
- metadata_update = {}
- if ("numberCores" in sliver.changed_fields):
- metadata_update["cpu_cores"] = str(sliver.numberCores)
+ for sliver in all_slivers:
+ sd = SliceDeployments.objects.filter(Q(slice=sliver.slice))
+ if (sd):
+ if (sd.deployment.name=='Amazon EC2'):
+ my_slivers.append(sliver)
+ if (sliver.node.deployment.name=='Amazon EC2'):
+ my_slivers.append(sliver)
+ return my_slivers
- for tag in sliver.slice.tags.all():
- if tag.name.startswith("sysctl-"):
- metadata_update[tag.name] = tag.value
+ def sync_record(self, sliver):
+ logger.info("sync'ing sliver:%s deployment:%s " % (sliver, sliver.node.deployment))
- if not sliver.instance_id:
- driver = self.driver.client_driver(caller=sliver.creator, tenant=sliver.slice.name, deployment=sliver.deploymentNetwork.name)
- # public keys
- slice_memberships = SlicePrivilege.objects.filter(slice=sliver.slice)
- pubkeys = [sm.user.public_key for sm in slice_memberships if sm.user.public_key]
- if sliver.creator.public_key:
- pubkeys.append(sliver.creator.public_key)
- if sliver.slice.creator.public_key:
- pubkeys.append(sliver.slice.creator.public_key)
- # netowrks
- # include all networks available to the slice and/or associated network templates
- nics = []
- networks = [ns.network for ns in NetworkSlice.objects.filter(slice=sliver.slice)]
- network_deployments = NetworkDeployments.objects.filter(network__in=networks,
- deployment=sliver.node.deployment)
- # Gather private networks first. This includes networks with a template that has
- # visibility = private and translation = none
- for network_deployment in network_deployments:
- if network_deployment.network.template.visibility == 'private' and \
- network_deployment.network.template.translation == 'none':
- nics.append({'net-id': network_deployment.net_id})
-
- # now include network template
- network_templates = [network.template.sharedNetworkName for network in networks \
- if network.template.sharedNetworkName]
- for net in driver.shell.quantum.list_networks()['networks']:
- if net['name'] in network_templates:
- nics.append({'net-id': net['id']})
+ if not sliver.instance_id:
+ # public keys
+ slice_memberships = SlicePrivilege.objects.filter(slice=sliver.slice)
+ pubkeys = [sm.user.public_key for sm in slice_memberships if sm.user.public_key]
- file("/tmp/scott-manager","a").write("slice: %s\nreq: %s\n" % (str(sliver.slice.name), str(nics)))
-
- # look up image id
- deployment_driver = self.driver.admin_driver(deployment=sliver.deploymentNetwork.name)
- image_id = None
- images = deployment_driver.shell.glance.get_images()
- for image in images:
- if image['name'] == sliver.image.name:
- image_id = image['id']
-
- # look up key name at the deployment
- # create/fetch keypair
- keyname = None
- if sliver.creator.public_key:
- keyname = sliver.creator.email.lower().replace('@', 'AT').replace('.', '') +\
- sliver.slice.name
- key_fields = {'name': keyname,
- 'public_key': sliver.creator.public_key}
- driver.create_keypair(**key_fields)
-
- instance = driver.spawn_instance(name=sliver.name,
- key_name = keyname,
- image_id = image_id,
- hostname = sliver.node.name,
- pubkeys = pubkeys,
- nics = nics )
- sliver.instance_id = instance.id
- sliver.instance_name = getattr(instance, 'OS-EXT-SRV-ATTR:instance_name')
- sliver.save()
+ if sliver.creator.public_key:
+ pubkeys.append(sliver.creator.public_key)
- if sliver.instance_id and metadata_update:
- driver.update_instance_metadata(sliver.instance_id, metadata_update)
+ if sliver.slice.creator.public_key:
+ pubkeys.append(sliver.slice.creator.public_key)
+
+ # netowrks
+ # include all networks available to the slice and/or associated network templates
+ #nics = []
+ #networks = [ns.network for ns in NetworkSlice.objects.filter(slice=sliver.slice)]
+ #network_deployments = NetworkDeployments.objects.filter(network__in=networks,
+ #deployment=sliver.node.deployment)
+ # Gather private networks first. This includes networks with a template that has
+ # visibility = private and translation = none
+ #for network_deployment in network_deployments:
+ # if network_deployment.network.template.visibility == 'private' and \
+ # network_deployment.network.template.translation == 'none':
+ # nics.append({'net-id': network_deployment.net_id})
+
+ # now include network template
+ #network_templates = [network.template.sharedNetworkName for network in networks \
+ # if network.template.sharedNetworkName]
+ #for net in driver.shell.quantum.list_networks()['networks']:
+ # if net['name'] in network_templates:
+ # nics.append({'net-id': net['id']})
+ # look up image id
+
+ instance_type = sliver.node.name.rsplit('.',1)[0]
+
+ # Bail out of we don't have a key
+ key_name = sliver.creator.email.lower().replace('@', 'AT').replace('.', '')
+ key_sig = aws_run('ec2 describe-key-pairs')
+ ec2_keys = key_sig['KeyPairs']
+ key_found = False
+ for key in ec2_keys:
+ if (key['KeyName']==key_name):
+ key_found = True
+ break
+
+ if (not key_found):
+ # set backend_status
+ raise Exception('Will not sync sliver without key')
+
+ image_id = sliver.image.path
+ instance_sig = aws_run('ec2 run-instances --image-id %s --instance-type %s --count 1 --key-name %s --placement AvailabilityZone=%s'%(image_id,instance_type,key_name,sliver.node.site.name))
+ sliver.instance_id = instance_sig['Instances'][0]['InstanceId']
+ sliver.save()
+ state = instance_sig['Instances'][0]['State']['Code']
+ if (state==16):
+ sliver.ip = instance_sig['Instances'][0]['PublicIpAddress']
+ sliver.save()
+ else:
+ # This status message should go into backend_status
+ raise Exception('Waiting for instance to start')
+ else:
+ ret = aws_run('ec2 describe-instances --instance-ids %s'%sliver.instance_id)
+ state = ret['Reservations'][0]['Instances'][0]['State']['Code']
+ if (state==16):
+ sliver.ip = ret['Reservations'][0]['Instances'][0]['PublicIpAddress']
+ sliver.save()
diff --git a/planetstack/ec2_observer/steps/sync_users.py b/planetstack/ec2_observer/steps/sync_users.py
index 71f9c0f..d742186 100644
--- a/planetstack/ec2_observer/steps/sync_users.py
+++ b/planetstack/ec2_observer/steps/sync_users.py
@@ -1,20 +1,43 @@
import os
import base64
-import hashlib
+import random
+import time
+from datetime import datetime
from django.db.models import F, Q
from planetstack.config import Config
-from observer.openstacksyncstep import OpenStackSyncStep
-from core.models.user import User, UserDeployments
+from ec2_observer.syncstep import SyncStep
+from core.models.user import User
+from core.models.site import *
+from ec2_observer.awslib import *
+import pdb
-class SyncUsers(OpenStackSyncStep):
- provides=[User]
- requested_interval=0
+class SyncUsers(SyncStep):
+ provides=[User]
+ requested_interval=0
- def fetch_pending(self):
- return User.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
+ def fetch_pending(self, deletion):
+ users = User.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
+ if (users):
+ key_sig = aws_run('ec2 describe-key-pairs')
+ ec2_keys = key_sig['KeyPairs']
+ else:
+ ec2_keys = []
- def sync_record(self, user):
- for user_deployment in UserDeployments.objects.filter(user=user):
- # bump the 'updated' field so user account are updated across
- # deployments.
- user_deployment.save()
+ for user in users:
+ if (user.public_key):
+ key_name = user.email.lower().replace('@', 'AT').replace('.', '')
+ key_found = False
+
+ for key in ec2_keys:
+ if (key['KeyName']==key_name):
+ key_found = True
+ break
+
+ if (not key_found):
+ aws_run('ec2 import-key-pair --key-name %s --public-key-material "%s"'%(key_name, user.public_key))
+
+ return users
+
+ def sync_record(self, node):
+ node.save()
+