Merge branch 'master' of ssh://git.planet-lab.org/git/plstackapi
diff --git a/planetstack/core/admin.py b/planetstack/core/admin.py
index 07029a2..4e19c36 100644
--- a/planetstack/core/admin.py
+++ b/planetstack/core/admin.py
@@ -718,7 +718,7 @@
class ImageAdmin(PlanetStackBaseAdmin):
fieldsets = [('Image Details',
- {'fields': ['image_id', 'name', 'disk_format', 'container_format'],
+ {'fields': ['name', 'disk_format', 'container_format'],
'classes': ['suit-tab suit-tab-general']})
]
diff --git a/planetstack/core/fixtures/initial_data.json b/planetstack/core/fixtures/initial_data.json
index 3657b6b..c0c6f38 100644
--- a/planetstack/core/fixtures/initial_data.json
+++ b/planetstack/core/fixtures/initial_data.json
@@ -62,7 +62,6 @@
"updated": "2013-12-17T18:00:47.910Z",
"name": "Stanford",
"created": "2013-04-03T23:14:11.072Z",
- "tenant_id": "",
"enabled": true,
"longitude": null,
"site_url": "http://www.stanford.edu/",
@@ -84,7 +83,6 @@
"updated": "2013-12-17T18:00:38.431Z",
"name": "Washington",
"created": "2013-04-03T23:14:11.072Z",
- "tenant_id": "",
"enabled": true,
"longitude": null,
"site_url": "https://www.washington.edu/",
@@ -106,7 +104,6 @@
"updated": "2013-12-17T18:00:28.495Z",
"name": "Princeton",
"created": "2013-04-03T23:14:11.072Z",
- "tenant_id": "",
"enabled": true,
"longitude": null,
"site_url": "http://princeton.edu/",
@@ -128,7 +125,6 @@
"updated": "2013-12-17T18:00:18.964Z",
"name": "GeorgiaTech",
"created": "2013-04-03T23:14:11.072Z",
- "tenant_id": "",
"enabled": true,
"longitude": null,
"site_url": "http://www.gatech.edu/",
@@ -150,7 +146,6 @@
"updated": "2013-12-17T18:00:07.704Z",
"name": "MaxPlanck",
"created": "2013-04-03T23:14:11.072Z",
- "tenant_id": "",
"enabled": true,
"longitude": null,
"site_url": "http://www.mpi-sws.mpg.de/",
@@ -172,7 +167,6 @@
"updated": "2013-06-21T21:17:13.982Z",
"name": "I2 Atlanta",
"created": "2013-04-03T23:14:11.072Z",
- "tenant_id": "",
"enabled": true,
"longitude": null,
"site_url": "http://www.internet2.edu",
@@ -194,7 +188,6 @@
"updated": "2013-06-21T21:17:13.982Z",
"name": "I2 Chicago",
"created": "2013-04-03T23:14:11.072Z",
- "tenant_id": "",
"enabled": true,
"longitude": null,
"site_url": "http://www.internet2.edu",
@@ -216,7 +209,6 @@
"updated": "2013-06-21T21:17:13.982Z",
"name": "I2 Houston",
"created": "2013-04-03T23:14:11.072Z",
- "tenant_id": "",
"enabled": true,
"longitude": null,
"site_url": "http://www.internet2.edu",
@@ -238,7 +230,6 @@
"updated": "2013-06-21T21:17:13.982Z",
"name": "I2 Kansas City",
"created": "2013-04-03T23:14:11.072Z",
- "tenant_id": "",
"enabled": true,
"longitude": null,
"site_url": "http://www.internet2.edu",
@@ -260,7 +251,6 @@
"updated": "2013-06-21T21:17:13.982Z",
"name": "I2 Los Angeles",
"created": "2013-04-03T23:14:11.072Z",
- "tenant_id": "",
"enabled": true,
"longitude": null,
"site_url": "http://www.internet2.edu",
@@ -282,7 +272,6 @@
"updated": "2013-06-21T21:17:13.982Z",
"name": "I2 New York",
"created": "2013-04-03T23:14:11.072Z",
- "tenant_id": "",
"enabled": true,
"longitude": null,
"site_url": "http://www.internet2.edu",
@@ -304,7 +293,6 @@
"updated": "2013-06-21T21:17:13.982Z",
"name": "I2 Salt Lake City",
"created": "2013-04-03T23:14:11.072Z",
- "tenant_id": "",
"enabled": true,
"longitude": null,
"site_url": "http://www.internet2.edu",
@@ -326,7 +314,6 @@
"updated": "2013-06-21T21:17:13.982Z",
"name": "I2 Seattle",
"created": "2013-04-03T23:14:11.072Z",
- "tenant_id": "",
"enabled": true,
"longitude": null,
"site_url": "http://www.internet2.edu",
@@ -348,7 +335,6 @@
"updated": "2013-06-21T21:17:13.982Z",
"name": "I2 Washington DC",
"created": "2013-04-03T23:14:11.072Z",
- "tenant_id": "",
"enabled": true,
"longitude": null,
"site_url": "http://www.internet2.edu",
@@ -370,7 +356,6 @@
"updated": "2013-12-17T17:30:14.491Z",
"name": "ON.Lab",
"created": "2013-04-03T23:14:11.072Z",
- "tenant_id": "",
"enabled": true,
"longitude": null,
"site_url": "http://www.onlab.us/",
@@ -397,7 +382,6 @@
"updated": "2013-12-17T18:21:43.870Z",
"name": "I2 Singapore",
"created": "2013-12-17T17:08:49.669Z",
- "tenant_id": null,
"enabled": true,
"longitude": null,
"site_url": "http://www.internet2.edu/",
@@ -419,7 +403,6 @@
"updated": "2013-12-17T18:08:01.373Z",
"name": "Arizona",
"created": "2013-12-17T18:07:14.190Z",
- "tenant_id": null,
"enabled": true,
"longitude": null,
"site_url": "http://www.cs.arizona.edu/",
diff --git a/planetstack/core/models/image.py b/planetstack/core/models/image.py
index b4803e2..db01e1b 100644
--- a/planetstack/core/models/image.py
+++ b/planetstack/core/models/image.py
@@ -1,13 +1,23 @@
import os
from django.db import models
from core.models import PlCoreBase
+from core.models import Deployment
# Create your models here.
class Image(PlCoreBase):
- image_id = models.CharField(max_length=256, unique=True)
name = models.CharField(max_length=256, unique=True)
disk_format = models.CharField(max_length=256)
container_format = models.CharField(max_length=256)
+ path = models.CharField(max_length=256, null=True, blank=True, help_text="Path to image on local disk")
def __unicode__(self): return u'%s' % (self.name)
+
+class ImageDeployments(PlCoreBase):
+ image = models.ForeignKey(Image)
+ deployment = models.ForeignKey(Deployment)
+ glance_image_id = models.CharField(null=True, blank=True, max_length=200, help_text="Glance image id")
+
+ def __unicode__(self): return u'%s %s' % (self.image, self.deployment)
+
+
diff --git a/planetstack/observer/steps/__init__.py b/planetstack/observer/steps/__init__.py
index eabf46c..2ef6922 100644
--- a/planetstack/observer/steps/__init__.py
+++ b/planetstack/observer/steps/__init__.py
@@ -12,4 +12,5 @@
from .sync_roles import SyncRoles
from .sync_nodes import SyncNodes
from .sync_images import SyncImages
+from .sync_image_deployments import SyncImageDeployments
from .garbage_collector import GarbageCollector
diff --git a/planetstack/observer/steps/sync_image_deployments.py b/planetstack/observer/steps/sync_image_deployments.py
new file mode 100644
index 0000000..31556c3
--- /dev/null
+++ b/planetstack/observer/steps/sync_image_deployments.py
@@ -0,0 +1,50 @@
+import os
+import base64
+from collections import defaultdict
+from django.db.models import F, Q
+from planetstack.config import Config
+from observer.openstacksyncstep import OpenStackSyncStep
+from core.models.deployment import Deployment
+from core.models.image import Image, ImageDeployments
+
+class SyncImageDeployments(OpenStackSyncStep):
+ provides=[ImageDeployments]
+ requested_interval=0
+
+ def fetch_pending(self):
+ # ensure images are available across all deployments
+ image_deployments = ImageDeployments.objects.all()
+ image_deploy_lookup = defaultdict(list)
+ for image_deployment in image_deployments:
+ image_deploy_lookup[image_deployment.image].append(image_deployment.deployment)
+
+ all_deployments = Deployment.objects.all()
+ for image in Image.objects.all():
+ expected_deployments = all_deployments
+ for expected_deployment in expected_deployments:
+ if image not in image_deploy_lookup or \
+ expected_deployment not in image_deploy_lookup[image]:
+ id = ImageDeployments(image=image, deployment=expected_deployment)
+ id.save()
+
+ # now we return all images that need to be enacted
+ return ImageDeployments.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
+
+ def sync_record(self, image_deployment):
+ driver = self.driver.admin_driver(deployment=image_deployment.deployment.name)
+ images = driver.shell.glance.get_images()
+ glance_image = None
+ for image in images:
+ if image['name'] == image_deployment.image.name:
+ glance_image = image
+ break
+ if glance_image:
+ image_deployment.glance_image_id = glance_image['id']
+ elif image_deployment.image.path:
+ glance_image = driver.shell.glanceclient.images.create(name=image_deployment.image.name,
+ is_public=True,
+ disk_format='raw',
+ container_format='bare')
+ glance_image.update(data=open(image_deployment.image.path, 'rb'))
+ image_deployment.glance_image_id = glance_image.id
+ image_deployment.save()
diff --git a/planetstack/observer/steps/sync_images.py b/planetstack/observer/steps/sync_images.py
index 2dbd74d..6ee53fe 100644
--- a/planetstack/observer/steps/sync_images.py
+++ b/planetstack/observer/steps/sync_images.py
@@ -10,20 +10,28 @@
requested_interval=0
def fetch_pending(self):
+ # get list of images on disk
+ images_path = Config().observer_images_directory
+ available_images = {}
+ for f in os.listdir(images_path):
+ if os.path.isfile(os.path.join(images_path ,f)):
+ available_images[f] = os.path.join(images_path ,f)
+
images = Image.objects.all()
image_names = [image.name for image in images]
+
+ for image_name in available_images:
+ #remove file extension
+ clean_name = ".".join(image_name.split('.')[:-1])
+ if image_name not in image_names:
+ image = Image(name=clean_name,
+ disk_format='raw',
+ container_format='bare',
+ path = available_images[image_name])
+ image.save()
- new_images = []
- glance_images = self.driver.shell.glance.get_images()
- for glance_image in glance_images:
- if glance_image['name'] not in image_names:
- image = Image(image_id=glance_image['id'],
- name=glance_image['name'],
- disk_format=glance_image['disk_format'],
- container_format=glance_image['container_format'])
- new_images.append(image)
-
- return new_images
+
+ return Image.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
def sync_record(self, image):
image.save()
diff --git a/planetstack/openstack/client.py b/planetstack/openstack/client.py
index c543947..465425d 100644
--- a/planetstack/openstack/client.py
+++ b/planetstack/openstack/client.py
@@ -1,6 +1,8 @@
+import urlparse
try:
from keystoneclient.v2_0 import client as keystone_client
from glance import client as glance_client
+ import glanceclient
from novaclient.v1_1 import client as nova_client
from quantumclient.v2_0 import client as quantum_client
from nova.db.sqlalchemy import api as nova_db_api
@@ -123,6 +125,16 @@
def __getattr__(self, name):
return getattr(self.client, name)
+class GlanceClientNew(Client):
+ def __init__(self, version, endpoint, token, *args, **kwds):
+ Client.__init__(self, *args, **kwds)
+ if has_openstack:
+ self.client = glanceclient.Client(version, endpoint=endpoint, token=token)
+
+ @require_enabled
+ def __getattr__(self, name):
+ return getattr(self.client, name)
+
class NovaClient(Client):
def __init__(self, *args, **kwds):
Client.__init__(self, *args, **kwds)
@@ -187,11 +199,17 @@
def __init__ ( self, *args, **kwds) :
# instantiate managers
self.keystone = KeystoneClient(*args, **kwds)
+ url_parsed = urlparse.urlparse(self.keystone.url)
+ hostname = url_parsed.netloc.split(':')[0]
+ token = self.keystone.client.tokens.authenticate(username=self.keystone.username, password=self.keystone.password, tenant_name=self.keystone.tenant)
self.keystone_db = KeystoneDB()
self.glance = GlanceClient(*args, **kwds)
+
+ self.glanceclient = GlanceClientNew('1', endpoint='http://%s:9292' % hostname, token=token.id)
self.nova = NovaClient(*args, **kwds)
self.nova_db = NovaDB(*args, **kwds)
self.quantum = QuantumClient(*args, **kwds)
+
@require_enabled
def connect(self, *args, **kwds):
diff --git a/planetstack/plstackapi_config b/planetstack/plstackapi_config
index 5d95231..a61e7ed 100644
--- a/planetstack/plstackapi_config
+++ b/planetstack/plstackapi_config
@@ -31,5 +31,6 @@
default_security_group=default
[observer]
+images_directory=/opt/planetstack/images
dependency_graph=/opt/planetstack/model-deps
logfile=/var/log/planetstack_backend.log