Merge branch 'master' of github.com:open-cloud/xos
diff --git a/xos/dmdot b/xos/dmdot
old mode 100644
new mode 100755
index d1b11c1..8570c4d
--- a/xos/dmdot
+++ b/xos/dmdot
@@ -12,7 +12,7 @@
from django.db.models.fields.related import ForeignKey
# defaults
-app = "core"
+apps = ["core", "hpc", "cord", "requestrouter"]
output = "-json"
# syntax: dmdot [-json | -dot] [app_name]
@@ -24,35 +24,36 @@
else:
app = arg
-app = app + ".models"
-#models_module = imp.load_source(app, ".")
-models_module = __import__(app)
-for part in app.split(".")[1:]:
- if hasattr(models_module, "PlCoreBase"):
- break
- models_module = getattr(models_module,part)
-
-PlCoreBase = getattr(models_module,"PlCoreBase")
-
-synonyms = {
- 'user':'creator'
-}
-
model_classes = []
class_names = []
lower_class_names = {}
-for classname in dir(models_module):
- c = getattr(models_module, classname, None)
- if type(c)==type(PlCoreBase):
- model_classes.append(c)
- class_names.append(c.__name__)
- lower_class_names[c.__name__.lower()] = c
- try:
- synonym = synonyms[c.__name__.lower()]
- lower_class_names[synonym] = c
- except:
- pass
-
+synonyms = {
+ 'user':'creator'
+}
+
+for app in apps:
+ app = app + ".models"
+ #models_module = imp.load_source(app, ".")
+ models_module = __import__(app)
+ for part in app.split(".")[1:]:
+ if hasattr(models_module, "PlCoreBase"):
+ break
+ models_module = getattr(models_module,part)
+
+ PlCoreBase = getattr(models_module,"PlCoreBase")
+
+ for classname in dir(models_module):
+ c = getattr(models_module, classname, None)
+ if type(c)==type(PlCoreBase):
+ model_classes.append(c)
+ class_names.append(c.__name__)
+ lower_class_names[c.__name__.lower()] = c
+ try:
+ synonym = synonyms[c.__name__.lower()]
+ lower_class_names[synonym] = c
+ except:
+ pass
+
# django doesn't use the correct case in field.name.title() for objects that
# have CamelCased class names. So, compare everything in lower case.
@@ -63,20 +64,32 @@
fields = c._meta.fields
for f in fields:
- if type(f)==ForeignKey and f.name.lower() in lower_class_names:
+ if type(f)==ForeignKey and f.name.lower().split('_') in lower_class_names:
linked_class = lower_class_names[f.name.lower()]
- print '\t"%s"->"%s";'%(c.__name__,linked_class.__name__)
+ if ('backref' in f.name):
+ print '\t"%s"->"%s";'%(linked_class.__name__,c.__name__)
+ else:
+ print '\t"%s"->"%s";'%(c.__name__,linked_class.__name__)
print "}\n";
elif (output=='-json'):
d = {}
for c in model_classes:
fields = c._meta.fields
-
+
for f in fields:
- if type(f)==ForeignKey and f.name.lower() in lower_class_names:
- linked_class = lower_class_names[f.name.lower()]
+ field_type = f.name.lower().split('_')[0]
+ if type(f)==ForeignKey and field_type in lower_class_names:
+ linked_class = lower_class_names[field_type]
+ if ('backref' in f.name.lower()):
+ a = linked_class.__name__
+ b = c.__name__
+ else:
+ b = linked_class.__name__
+ a = c.__name__
+
try:
- d[c.__name__].append(linked_class.__name__)
+ if (b not in d[a]):
+ d[a].append(b)
except KeyError:
d[c.__name__]=[linked_class.__name__]
#d['ControllerNetwork'].append('SliceDeployments')
diff --git a/xos/helloworld/__init__.py b/xos/helloworld/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/xos/helloworld/__init__.py
diff --git a/xos/helloworld/models.py b/xos/helloworld/models.py
new file mode 100644
index 0000000..37f6751
--- /dev/null
+++ b/xos/helloworld/models.py
@@ -0,0 +1,18 @@
+from django.db import models
+from core.models import User, Service, SingletonModel, PlCoreBase, Instance
+from core.models.plcorebase import StrippedCharField
+import os
+from django.db import models
+from django.forms.models import model_to_dict
+from django.db.models import Q
+
+
+# Create your models here.
+
+class Hello(PlCoreBase):
+ name = models.CharField(max_length=254,help_text="Salutation e.g. Hello or Bonjour")
+ sliver_backref = models.ForeignKey(Instance)
+
+class World(PlCoreBase):
+ name = models.CharField(max_length=254,help_text="Name of planet")
+ hello = models.ForeignKey(Hello)
diff --git a/xos/model-deps b/xos/model-deps
index 0218517..ea32eb9 100644
--- a/xos/model-deps
+++ b/xos/model-deps
@@ -10,8 +10,7 @@
"Deployment"
],
"ControllerImages": [
- "Image",
- "Controller"
+ "Image"
],
"ReservedResource": [
"Instance"
@@ -72,10 +71,7 @@
"Instance": [
"Image",
"User",
- "Slice",
- "Deployment",
- "Node",
- "Flavor"
+ "Slice"
],
"Payment": [
"Account"
diff --git a/xos/observers/base/SyncInstanceUsingAnsible.py b/xos/observers/base/SyncInstanceUsingAnsible.py
new file mode 100644
index 0000000..9455780
--- /dev/null
+++ b/xos/observers/base/SyncInstanceUsingAnsible.py
@@ -0,0 +1,95 @@
+import hashlib
+import os
+import socket
+import sys
+import base64
+import time
+from django.db.models import F, Q
+from xos.config import Config
+from observer.syncstep import SyncStep
+from observer.ansible import run_template_ssh
+from core.models import Service, Slice
+from util.logger import Logger, logging
+
+logger = Logger(level=logging.INFO)
+
+class SyncInstanceUsingAnsible(SyncStep):
+ # All of the following should be defined for classes derived from this
+ # base class. Examples below use VCPETenant.
+
+ # provides=[VCPETenant]
+ # observes=VCPETenant
+ # requested_interval=0
+ # template_name = "sync_vcpetenant.yaml"
+ # service_key_name = "/opt/xos/observers/vcpe/vcpe_private_key"
+
+ def __init__(self, **args):
+ SyncStep.__init__(self, **args)
+
+ def defer_sync(self, o, reason):
+ logger.info("defer object %s due to %s" % (str(o), reason))
+ raise Exception("defer object %s due to %s" % (str(o), reason))
+
+ def get_extra_attributes(self, o):
+ # This is a place to include extra attributes that aren't part of the
+ # object itself.
+
+ return {}
+
+ def get_instance(self, o):
+ # We need to know what instance is associated with the object. Let's
+ # assume 'o' has a field called 'instance'. If the field is called
+ # something else, or if custom logic is needed, then override this
+ # method.
+
+ return o.instance
+
+ def run_playbook(self, o, fields):
+ tStart = time.time()
+ run_template_ssh(self.template_name, fields)
+ logger.info("playbook execution time %d" % int(time.time()-tStart))
+
+ def pre_sync_hook(self, o, fields):
+ pass
+
+ def post_sync_hook(self, o, fields):
+ pass
+
+ def sync_fields(self, o, fields):
+ self.run_playbook(o, fields)
+
+ def sync_record(self, o):
+ logger.info("sync'ing object %s" % str(o))
+
+ instance = self.get_instance(o)
+ if not instance:
+ self.defer_sync(o, "waiting on instance")
+ return
+
+ if not os.path.exists(self.service_key_name):
+ raise Exception("Service key %s does not exist" % self.service_key_name)
+
+ service_key = file(self.service_key_name).read()
+
+ fields = { "instance_name": instance.name,
+ "hostname": instance.node.name,
+ "instance_id": instance.instance_id,
+ "private_key": service_key,
+ "ansible_tag": "vcpe_tenant_" + str(o.id)
+ }
+
+ # If 'o' defines a 'sync_attributes' list, then we'll copy those
+ # attributes into the Ansible recipe's field list automatically.
+ if hasattr(o, "sync_attributes"):
+ for attribute_name in o.sync_attributes:
+ fields[attribute_name] = getattr(o, attribute_name)
+
+ fields.update(self.get_extra_attributes(o))
+
+ self.sync_fields(o, fields)
+
+ o.save()
+
+ def delete_record(self, m):
+ pass
+
diff --git a/xos/observers/hello_world/helloworld-observer.py b/xos/observers/hello_world/helloworld-observer.py
new file mode 100755
index 0000000..d6a71ff
--- /dev/null
+++ b/xos/observers/hello_world/helloworld-observer.py
@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+
+# This imports and runs ../../xos-observer.py
+
+import importlib
+import os
+import sys
+observer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),"../..")
+sys.path.append(observer_path)
+mod = importlib.import_module("xos-observer")
+mod.main()
diff --git a/xos/observers/hello_world/helloworld_config b/xos/observers/hello_world/helloworld_config
new file mode 100644
index 0000000..97248ae
--- /dev/null
+++ b/xos/observers/hello_world/helloworld_config
@@ -0,0 +1,46 @@
+[plc]
+name=plc
+deployment=plc
+
+[db]
+name=xos
+user=postgres
+password=password
+host=localhost
+port=5432
+
+[api]
+host=localhost
+port=8000
+ssl_key=None
+ssl_cert=None
+ca_ssl_cert=None
+ratelimit_enabled=0
+omf_enabled=0
+mail_support_address=support@localhost
+nova_enabled=True
+logfile=/var/log/xos.log
+
+[nova]
+admin_user=admin@domain.com
+admin_password=admin
+admin_tenant=admin
+url=http://localhost:5000/v2.0/
+default_image=None
+default_flavor=m1.small
+default_security_group=default
+ca_ssl_cert=/etc/ssl/certs/ca-certificates.crt
+
+[observer]
+pretend=False
+backoff_disabled=False
+images_directory=/opt/xos/images
+dependency_graph=/opt/xos/model-deps
+logfile=/var/log/xos_backend.log
+steps_dir=/opt/xos/observers/hello_world/steps
+
+[gui]
+disable_minidashboard=True
+#branding_name=CORD
+#branding_css=/static/cord.css
+#branding_icon=/static/onos-logo.png
diff --git a/xos/observers/hello_world/model-deps b/xos/observers/hello_world/model-deps
new file mode 100644
index 0000000..63188f0
--- /dev/null
+++ b/xos/observers/hello_world/model-deps
@@ -0,0 +1,19 @@
+{
+ "OriginServer": [
+ "ContentProvider"
+ ],
+ "ContentProvider": [
+ "ServiceProvider"
+ ],
+ "CDNPrefix": [
+ "ContentProvider"
+ ],
+ "AccessMap": [
+ "ContentProvider"
+ ],
+ "SiteMap": [
+ "ContentProvider",
+ "ServiceProvider",
+ "CDNPrefix"
+ ]
+}
diff --git a/xos/observers/hello_world/nohup.out b/xos/observers/hello_world/nohup.out
new file mode 100644
index 0000000..74072c6
--- /dev/null
+++ b/xos/observers/hello_world/nohup.out
@@ -0,0 +1 @@
+python: can't open file 'helloworld-observer.py': [Errno 2] No such file or directory
diff --git a/xos/observers/hello_world/run.sh b/xos/observers/hello_world/run.sh
new file mode 100755
index 0000000..f77d751
--- /dev/null
+++ b/xos/observers/hello_world/run.sh
@@ -0,0 +1,6 @@
+#if [[ ! -e ./hpc-backend.py ]]; then
+# ln -s ../xos-observer.py hpc-backend.py
+#fi
+
+export XOS_DIR=/opt/xos
+python hpc-observer.py -C $XOS_DIR/observers/hpc/hpc_observer_config
diff --git a/xos/observers/hello_world/start.sh b/xos/observers/hello_world/start.sh
new file mode 100755
index 0000000..89240cd
--- /dev/null
+++ b/xos/observers/hello_world/start.sh
@@ -0,0 +1,4 @@
+export XOS_DIR=/opt/xos
+
+echo $XOS_DIR/observers/helloworld/helloworld_config
+python helloworld-observer.py -C $XOS_DIR/observers/helloworld/helloworld_config
diff --git a/xos/observers/hello_world/steps/sync_hello.py b/xos/observers/hello_world/steps/sync_hello.py
new file mode 100644
index 0000000..f59ec5c
--- /dev/null
+++ b/xos/observers/hello_world/steps/sync_hello.py
@@ -0,0 +1,24 @@
+import os
+import sys
+import base64
+from django.db.models import F, Q
+from xos.config import Config
+from observer.syncstep import SyncStep
+from helloworld.models import Hello,World
+from util.logger import Logger, logging
+
+parentdir = os.path.join(os.path.dirname(__file__),"..")
+sys.path.insert(0,parentdir)
+
+logger = Logger(level=logging.INFO)
+
+class SyncHello(SyncStep):
+ provides=[Hello]
+ observes=Hello
+ requested_interval=0
+
+ def sync_record(self, record):
+ open('/tmp/hello-synchronizer','w').write(record.name)
+
+ def delete_record(self, m):
+ return
diff --git a/xos/observers/hello_world/steps/sync_world.py b/xos/observers/hello_world/steps/sync_world.py
new file mode 100644
index 0000000..01e285f
--- /dev/null
+++ b/xos/observers/hello_world/steps/sync_world.py
@@ -0,0 +1,25 @@
+import os
+import sys
+import base64
+from django.db.models import F, Q
+from xos.config import Config
+from observer.syncstep import SyncStep
+from helloworld.models import Hello,World
+from util.logger import Logger, logging
+
+# hpclibrary will be in steps/..
+parentdir = os.path.join(os.path.dirname(__file__),"..")
+sys.path.insert(0,parentdir)
+
+logger = Logger(level=logging.INFO)
+
+class SyncWorld(SyncStep):
+ provides=[World]
+ observes=World
+ requested_interval=0
+
+ def sync_record(self, record):
+ open('/tmp/hello-synchronizer','w').write(record.name)
+
+ def delete_record(self, m):
+ return
diff --git a/xos/observers/hello_world/stop.sh b/xos/observers/hello_world/stop.sh
new file mode 100755
index 0000000..a0b4a8e
--- /dev/null
+++ b/xos/observers/hello_world/stop.sh
@@ -0,0 +1 @@
+pkill -9 -f hpc-observer.py
diff --git a/xos/openstack_observer/ansible.py b/xos/openstack_observer/ansible.py
index fad7610..cb512ce 100755
--- a/xos/openstack_observer/ansible.py
+++ b/xos/openstack_observer/ansible.py
@@ -45,6 +45,7 @@
def parse_unreachable(msg):
total_unreachable=0
+ total_failed=0
for l in msg.splitlines():
x = re.findall('ok=([0-9]+).*changed=([0-9]+).*unreachable=([0-9]+).*failed=([0-9]+)', l)
if x:
@@ -55,8 +56,9 @@
failed=int(failed)
total_unreachable += unreachable
- return total_unreachable
-
+ total_failed += failed
+ return {'unreachable':total_unreachable,'failed':total_failed}
+
def id_generator(size=6, chars=string.ascii_uppercase + string.digits):
return ''.join(random.choice(chars) for _ in range(size))
@@ -118,7 +120,12 @@
if (expected_num is not None) and (len(ok_results) != expected_num):
raise ValueError('Unexpected num %s!=%d' % (str(expected_num), len(ok_results)) )
- total_unreachable = parse_unreachable(msg)
+ parsed = parse_unreachable(msg)
+ total_unreachable = parsed['unreachable']
+ failed = parsed['failed']
+ if (failed):
+ raise ValueError('Ansible playbook failed.')
+
if (total_unreachable > 0):
raise ValueError("Unreachable results in ansible recipe")
except ValueError,e:
@@ -181,7 +188,7 @@
print "ANSIBLE_CONFIG=%s" % config_pathname
print "ANSIBLE_HOSTS=%s" % hosts_pathname
- return run_template(name, opts, path, expected_num, ansible_config = config_pathname, ansible_hosts = hosts_pathname, run_ansible_script="/opt/xos/observer/run_ansible_verbose")
+ return run_template(name, opts, path, ansible_config = config_pathname, ansible_hosts = hosts_pathname, run_ansible_script="/opt/xos/observer/run_ansible_verbose")
diff --git a/xos/openstack_observer/event_loop.py b/xos/openstack_observer/event_loop.py
index fc07e7d..31db875 100644
--- a/xos/openstack_observer/event_loop.py
+++ b/xos/openstack_observer/event_loop.py
@@ -179,6 +179,8 @@
except:
step_graph[source]=[dest]
except KeyError:
+ if (not provides_dict.has_key(m)):
+ step_graph[source]='#%s'%m
pass
except KeyError:
@@ -244,7 +246,17 @@
for e in self.ordered_steps:
self.last_deletion_run_times[e]=0
-
+ def lookup_step(self,s):
+ if ('#' in s):
+ objname = s[1:]
+ so = SyncObject()
+ so.provides=[globals()[objname]]
+ so.observes=globals()[objname]
+ step = so
+ else:
+ step = self.step_lookup[s]
+ return step
+
def save_run_times(self):
run_times = json.dumps(self.last_run_times)
open('/tmp/%sobserver_run_times'%self.observer_name,'w').write(run_times)
@@ -264,7 +276,7 @@
def sync(self, S, deletion):
try:
- step = self.step_lookup[S]
+ step = self.lookup_step(S)
start_time=time.time()
logger.info("Starting to work on step %s, deletion=%s" % (step.__name__, str(deletion)))
diff --git a/xos/openstack_observer/steps/purge_disabled_users.py b/xos/openstack_observer/steps/purge_disabled_users.py
index 9e30708..b5168e3 100644
--- a/xos/openstack_observer/steps/purge_disabled_users.py
+++ b/xos/openstack_observer/steps/purge_disabled_users.py
@@ -20,5 +20,6 @@
# disabled users that haven't been updated in over a week
one_week_ago = datetime.datetime.now() - datetime.timedelta(days=7)
return User.objects.filter(is_active=False, updated__gt=one_week_ago)
+
def sync_record(self, user):
user.delete()
diff --git a/xos/openstack_observer/steps/sync_controller_images.py b/xos/openstack_observer/steps/sync_controller_images.py
index 948fcea..3434f01 100644
--- a/xos/openstack_observer/steps/sync_controller_images.py
+++ b/xos/openstack_observer/steps/sync_controller_images.py
@@ -15,6 +15,7 @@
provides=[ControllerImages]
observes = ControllerImages
requested_interval=0
+ playbook='sync_controller_images.yaml'
def fetch_pending(self, deleted):
if (deleted):
@@ -23,13 +24,7 @@
# now we return all images that need to be enacted
return ControllerImages.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
- def sync_record(self, controller_image):
- logger.info("Working on image %s on controller %s" % (controller_image.image.name, controller_image.controller))
-
- controller_register = json.loads(controller_image.controller.backend_register)
- if (controller_register.get('disabled',False)):
- raise InnocuousException('Controller %s is disabled'%controller_image.controller.name)
-
+ def map_inputs(self, controller_image):
image_fields = {'endpoint':controller_image.controller.auth_url,
'admin_user':controller_image.controller.admin_user,
'admin_password':controller_image.controller.admin_password,
@@ -38,9 +33,9 @@
'ansible_tag': '%s@%s'%(controller_image.image.name,controller_image.controller.name), # name of ansible playbook
}
+ return image_fields
- res = run_template('sync_controller_images.yaml', image_fields, path='controller_images', expected_num=1)
-
+ def map_outputs(self, controller_image):
image_id = res[0]['id']
controller_image.glance_image_id = image_id
controller_image.backend_status = '1 - OK'
diff --git a/xos/openstack_observer/steps/sync_controller_networks.py b/xos/openstack_observer/steps/sync_controller_networks.py
index 54f2466..0b636e2 100644
--- a/xos/openstack_observer/steps/sync_controller_networks.py
+++ b/xos/openstack_observer/steps/sync_controller_networks.py
@@ -20,6 +20,7 @@
requested_interval = 0
provides=[Network]
observes=ControllerNetwork
+ playbook='sync_controller_networks.yaml'
def alloc_subnet(self, uuid):
# 16 bits only
@@ -37,6 +38,7 @@
network_name = controller_network.network.name
subnet_name = '%s-%d'%(network_name,controller_network.pk)
cidr = self.alloc_subnet(controller_network.pk)
+ self.cidr=cidr
slice = controller_network.network.owner
network_fields = {'endpoint':controller_network.controller.auth_url,
@@ -49,46 +51,37 @@
'cidr':cidr,
'delete':False
}
+ return network_fields
- res = run_template('sync_controller_networks.yaml', network_fields, path = 'controller_networks',expected_num=2)
-
+ def map_sync_outputs(self, controller_network,res):
network_id = res[0]['id']
subnet_id = res[1]['id']
controller_network.net_id = network_id
- controller_network.subnet = cidr
+ controller_network.subnet = self.cidr
controller_network.subnet_id = subnet_id
controller_network.backend_status = '1 - OK'
controller_network.save()
- def sync_record(self, controller_network):
+ def map_sync_inputs(self, controller_network):
if (controller_network.network.template.name!='Private'):
logger.info("skipping network controller %s because it is not private" % controller_network)
# We only sync private networks
return
- logger.info("sync'ing network controller %s for network %s slice %s controller %s" % (controller_network, controller_network.network, str(controller_network.network.owner), controller_network.controller))
-
- controller_register = json.loads(controller_network.controller.backend_register)
- if (controller_register.get('disabled',False)):
- raise InnocuousException('Controller %s is disabled'%controller_network.controller.name)
-
if not controller_network.controller.admin_user:
logger.info("controller %r has no admin_user, skipping" % controller_network.controller)
return
if controller_network.network.owner and controller_network.network.owner.creator:
- self.save_controller_network(controller_network)
- logger.info("saved network controller: %s" % (controller_network))
+ return self.save_controller_network(controller_network)
+ else:
+ raise Exception('Could not save network controller %s'%controller_network)
- def delete_record(self, controller_network):
+ def map_delete_inputs(self, controller_network):
if (controller_network.network.template.name!='Private'):
# We only sync private networks
return
- controller_register = json.loads(controller_network.controller.backend_register)
- if (controller_register.get('disabled',False)):
- raise InnocuousException('Controller %s is disabled'%controller_network.controller.name)
-
try:
slice = controller_network.network.owner # XXX: FIXME!!
except:
@@ -108,7 +101,7 @@
'delete':True
}
- res = run_template('sync_controller_networks.yaml', network_fields, path = 'controller_networks',expected_num=1)
+ return network_fields
"""
driver = OpenStackDriver().client_driver(caller=controller_network.network.owner.creator,
diff --git a/xos/openstack_observer/steps/sync_controller_site_privileges.py b/xos/openstack_observer/steps/sync_controller_site_privileges.py
index a2c40ef..d52c999 100644
--- a/xos/openstack_observer/steps/sync_controller_site_privileges.py
+++ b/xos/openstack_observer/steps/sync_controller_site_privileges.py
@@ -16,27 +16,14 @@
provides=[SitePrivilege]
requested_interval=0
observes=ControllerSitePrivilege
+ playbook='sync_controller_users.yaml'
- def fetch_pending(self, deleted):
-
- if (deleted):
- return ControllerSitePrivilege.deleted_objects.all()
- else:
- return ControllerSitePrivilege.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
-
- def sync_record(self, controller_site_privilege):
- logger.info("sync'ing controler_site_privilege %s at controller %s" % (controller_site_privilege, controller_site_privilege.controller))
-
+ def map_sync_inputs(self, controller_site_privilege):
controller_register = json.loads(controller_site_privilege.controller.backend_register)
- if (controller_register.get('disabled',False)):
- raise InnocuousException('Controller %s is disabled'%controller_site_privilege.controller.name)
-
-
if not controller_site_privilege.controller.admin_user:
logger.info("controller %r has no admin_user, skipping" % controller_site_privilege.controller)
return
- template = os_template_env.get_template('sync_controller_users.yaml')
roles = [controller_site_privilege.site_privilege.role.role]
# setup user home site roles at controller
if not controller_site_privilege.site_privilege.user.site:
@@ -63,10 +50,9 @@
'roles':roles,
'tenant':controller_site_privilege.site_privilege.site.login_base}
- rendered = template.render(user_fields)
- expected_length = len(roles) + 1
- res = run_template('sync_controller_users.yaml', user_fields,path='controller_site_privileges', expected_num=expected_length)
+ return user_fields
+ def map_sync_outputs(self, controller_site_privilege, res):
# results is an array in which each element corresponds to an
# "ok" string received per operation. If we get as many oks as
# the number of operations we issued, that means a grand success.
diff --git a/xos/openstack_observer/steps/sync_controller_sites.py b/xos/openstack_observer/steps/sync_controller_sites.py
index 670f09c..2f1680c 100644
--- a/xos/openstack_observer/steps/sync_controller_sites.py
+++ b/xos/openstack_observer/steps/sync_controller_sites.py
@@ -13,17 +13,13 @@
requested_interval=0
provides=[Site]
observes=ControllerSite
+ playbook = 'sync_controller_sites.yaml'
def fetch_pending(self, deleted=False):
- pending = super(OpenStackSyncStep, self).fetch_pending(deleted)
- return pending.filter(controller__isnull=False)
+ lobjs = ControllerSite.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None),Q(lazy_blocked=False),Q(controller__isnull=False))
+ return lobjs
- def sync_record(self, controller_site):
- controller_register = json.loads(controller_site.controller.backend_register)
- if (controller_register.get('disabled',False)):
- raise InnocuousException('Controller %s is disabled'%controller_site.controller.name)
-
- template = os_template_env.get_template('sync_controller_sites.yaml')
+ def map_sync_inputs(self, controller_site):
tenant_fields = {'endpoint':controller_site.controller.auth_url,
'admin_user': controller_site.controller.admin_user,
'admin_password': controller_site.controller.admin_password,
@@ -31,10 +27,9 @@
'ansible_tag': '%s@%s'%(controller_site.site.login_base,controller_site.controller.name), # name of ansible playbook
'tenant': controller_site.site.login_base,
'tenant_description': controller_site.site.name}
+ return tenant_fields
- rendered = template.render(tenant_fields)
- res = run_template('sync_controller_sites.yaml', tenant_fields, path='controller_sites', expected_num=1)
-
+ def map_sync_outputs(self, controller_site, res):
controller_site.tenant_id = res[0]['id']
controller_site.backend_status = '1 - OK'
controller_site.save()
diff --git a/xos/openstack_observer/steps/sync_controller_slice_privileges.py b/xos/openstack_observer/steps/sync_controller_slice_privileges.py
index 2e2e63c..a998460 100644
--- a/xos/openstack_observer/steps/sync_controller_slice_privileges.py
+++ b/xos/openstack_observer/steps/sync_controller_slice_privileges.py
@@ -16,21 +16,9 @@
provides=[SlicePrivilege]
requested_interval=0
observes=ControllerSlicePrivilege
+ playbook = 'sync_controller_users.yaml'
- def fetch_pending(self, deleted):
-
- if (deleted):
- return ControllerSlicePrivilege.deleted_objects.all()
- else:
- return ControllerSlicePrivilege.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
-
- def sync_record(self, controller_slice_privilege):
- logger.info("sync'ing controler_slice_privilege %s at controller %s" % (controller_slice_privilege, controller_slice_privilege.controller))
-
- controller_register = json.loads(controller_slice_privilege.controller.backend_register)
- if (controller_register.get('disabled',False)):
- raise InnocuousException('Controller %s is disabled'%controller_slice_privilege.controller.name)
-
+ def map_inputs(self, controller_slice_privilege):
if not controller_slice_privilege.controller.admin_user:
logger.info("controller %r has no admin_user, skipping" % controller_slice_privilege.controller)
return
@@ -61,17 +49,11 @@
'admin_tenant': controller_slice_privilege.controller.admin_tenant,
'roles':roles,
'tenant':controller_slice_privilege.slice_privilege.slice.name}
+ return user_fields
- rendered = template.render(user_fields)
- expected_length = len(roles) + 1
- res = run_template('sync_controller_users.yaml', user_fields, path='controller_slice_privileges', expected_num=expected_length)
-
- # results is an array in which each element corresponds to an
- # "ok" string received per operation. If we get as many oks as
- # the number of operations we issued, that means a grand success.
- # Otherwise, the number of oks tell us which operation failed.
- controller_slice_privilege.role_id = res[0]['id']
- controller_slice_privilege.save()
+ def map_sync_outputs(self, controller_slice_privilege, res):
+ controller_slice_privilege.role_id = res[0]['id']
+ controller_slice_privilege.save()
def delete_record(self, controller_slice_privilege):
controller_register = json.loads(controller_slice_privilege.controller.backend_register)
diff --git a/xos/openstack_observer/steps/sync_controller_slices.py b/xos/openstack_observer/steps/sync_controller_slices.py
index c456a2f..a790a67 100644
--- a/xos/openstack_observer/steps/sync_controller_slices.py
+++ b/xos/openstack_observer/steps/sync_controller_slices.py
@@ -16,20 +16,11 @@
provides=[Slice]
requested_interval=0
observes=ControllerSlice
+ playbook='sync_controller_slices.yaml'
- def fetch_pending(self, deleted):
- if (deleted):
- return ControllerSlice.deleted_objects.all()
- else:
- return ControllerSlice.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
-
- def sync_record(self, controller_slice):
+ def map_sync_inputs(self, controller_slice):
logger.info("sync'ing slice controller %s" % controller_slice)
- controller_register = json.loads(controller_slice.controller.backend_register)
- if (controller_register.get('disabled',False)):
- raise InnocuousException('Controller %s is disabled'%controller_slice.controller.name)
-
if not controller_slice.controller.admin_user:
logger.info("controller %r has no admin_user, skipping" % controller_slice.controller)
return
@@ -54,8 +45,9 @@
'ansible_tag':'%s@%s'%(controller_slice.slice.name,controller_slice.controller.name),
'max_instances':max_instances}
- expected_num = len(roles)+1
- res = run_template('sync_controller_slices.yaml', tenant_fields, path='controller_slices', expected_num=expected_num)
+ return tenant_fields
+
+ def map_sync_outputs(self, controller_slice, res):
tenant_id = res[0]['id']
if (not controller_slice.tenant_id):
try:
@@ -70,11 +62,7 @@
controller_slice.save()
- def delete_record(self, controller_slice):
- controller_register = json.loads(controller_slice.controller.backend_register)
- if (controller_register.get('disabled',False)):
- raise InnocuousException('Controller %s is disabled'%controller_slice.controller.name)
-
+ def map_delete_inputs(self, controller_slice):
controller_users = ControllerUser.objects.filter(user=controller_slice.slice.creator,
controller=controller_slice.controller)
if not controller_users:
@@ -91,6 +79,4 @@
'name':controller_user.user.email,
'ansible_tag':'%s@%s'%(controller_slice.slice.name,controller_slice.controller.name),
'delete': True}
-
- expected_num = 1
- run_template('sync_controller_slices.yaml', tenant_fields, path='controller_slices', expected_num=expected_num)
+ return tenant_fields
diff --git a/xos/openstack_observer/steps/sync_controller_users.py b/xos/openstack_observer/steps/sync_controller_users.py
index d30d0ff..ae04460 100644
--- a/xos/openstack_observer/steps/sync_controller_users.py
+++ b/xos/openstack_observer/steps/sync_controller_users.py
@@ -16,27 +16,13 @@
provides=[User]
requested_interval=0
observes=ControllerUser
+ playbook='sync_controller_users.yaml'
- def fetch_pending(self, deleted):
-
- if (deleted):
- return ControllerUser.deleted_objects.all()
- else:
- return ControllerUser.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
-
- def sync_record(self, controller_user):
- logger.info("sync'ing user %s at controller %s" % (controller_user.user, controller_user.controller))
-
- controller_register = json.loads(controller_user.controller.backend_register)
- if (controller_register.get('disabled',False)):
- raise InnocuousException('Controller %s is disabled'%controller_user.controller.name)
-
+ def map_sync_inputs(self, controller_user):
if not controller_user.controller.admin_user:
logger.info("controller %r has no admin_user, skipping" % controller_user.controller)
return
- template = os_template_env.get_template('sync_controller_users.yaml')
-
# All users will have at least the 'user' role at their home site/tenant.
# We must also check if the user should have the admin role
roles = ['user']
@@ -68,21 +54,14 @@
'roles':roles,
'tenant':controller_user.user.site.login_base
}
+ return user_fields
- rendered = template.render(user_fields)
- expected_length = len(roles) + 1
-
- res = run_template('sync_controller_users.yaml', user_fields,path='controller_users', expected_num=expected_length)
-
- controller_user.kuser_id = res[0]['id']
- controller_user.backend_status = '1 - OK'
- controller_user.save()
+ def map_sync_outputs(self, controller_user, res):
+ controller_user.kuser_id = res[0]['id']
+ controller_user.backend_status = '1 - OK'
+ controller_user.save()
def delete_record(self, controller_user):
- controller_register = json.loads(controller_user.controller.backend_register)
- if (controller_register.get('disabled',False)):
- raise InnocuousException('Controller %s is disabled'%controller_user.controller.name)
-
if controller_user.kuser_id:
driver = self.driver.admin_driver(controller=controller_user.controller)
driver.delete_user(controller_user.kuser_id)
diff --git a/xos/openstack_observer/steps/sync_instances.py b/xos/openstack_observer/steps/sync_instances.py
index 3f61d35..963d859 100644
--- a/xos/openstack_observer/steps/sync_instances.py
+++ b/xos/openstack_observer/steps/sync_instances.py
@@ -20,6 +20,7 @@
provides=[Instance]
requested_interval=0
observes=Instance
+ playbook='sync_instances.yaml'
def get_userdata(self, instance, pubkeys):
userdata = '#cloud-config\n\nopencloud:\n slicename: "%s"\n hostname: "%s"\n restapi_hostname: "%s"\n restapi_port: "%s"\n' % (instance.slice.name, instance.node.name, RESTAPI_HOSTNAME, str(RESTAPI_PORT))
@@ -28,14 +29,9 @@
userdata += ' - %s\n' % key
return userdata
- def sync_record(self, instance):
- logger.info("sync'ing instance:%s slice:%s controller:%s " % (instance, instance.slice.name, instance.node.site_deployment.controller))
- controller_register = json.loads(instance.node.site_deployment.controller.backend_register)
-
- if (controller_register.get('disabled',False)):
- raise InnocuousException('Controller %s is disabled'%instance.node.site_deployment.controller.name)
-
- metadata_update = {}
+ def map_sync_inputs(self, instance):
+ inputs = {}
+ metadata_update = {}
if (instance.numberCores):
metadata_update["cpu_cores"] = str(instance.numberCores)
@@ -43,8 +39,7 @@
if tag.name.startswith("sysctl-"):
metadata_update[tag.name] = tag.value
- # public keys
- slice_memberships = SlicePrivilege.objects.filter(slice=instance.slice)
+ slice_memberships = SlicePrivilege.objects.filter(slice=instance.slice)
pubkeys = set([sm.user.public_key for sm in slice_memberships if sm.user.public_key])
if instance.creator.public_key:
pubkeys.add(instance.creator.public_key)
@@ -55,23 +50,8 @@
if instance.slice.service and instance.slice.service.public_key:
pubkeys.add(instance.slice.service.public_key)
- # Handle any ports that are already created and attached to the instance.
- # If we do have a port for a network, then add that network to an
- # exclude list so we won't try to auto-create ports on that network
- # when instantiating.
- ports = []
- exclude_networks = set()
- exclude_templates = set()
- for ns in instance.ports.all():
- if not ns.port_id:
- raise DeferredException("Port %s on instance %s has no id; Try again later" % (str(ns), str(instance)) )
- ports.append(ns.port_id)
- exclude_networks.add(ns.network)
- exclude_templates.add(ns.network.template)
-
nics = []
networks = [ns.network for ns in NetworkSlice.objects.filter(slice=instance.slice)]
- networks = [n for n in networks if (n not in exclude_networks)]
controller_networks = ControllerNetwork.objects.filter(network__in=networks,
controller=instance.node.site_deployment.controller)
@@ -79,14 +59,12 @@
if controller_network.network.template.visibility == 'private' and \
controller_network.network.template.translation == 'none':
if not controller_network.net_id:
- raise DeferredException("Private Network %s has no id; Try again later" % controller_network.network.name)
+ raise Exception("Private Network %s has no id; Try again later" % controller_network.network.name)
nics.append(controller_network.net_id)
- # Now include network templates, for those networks that use a
- # shared_network_name.
+ # now include network template
network_templates = [network.template.shared_network_name for network in networks \
if network.template.shared_network_name]
- network_templates = [nt for nt in network_templates if (nt not in exclude_templates)]
#driver = self.driver.client_driver(caller=instance.creator, tenant=instance.slice.name, controller=instance.controllerNetwork)
driver = self.driver.admin_driver(tenant='admin', controller=instance.node.site_deployment.controller)
@@ -95,9 +73,7 @@
if net['name'] in network_templates:
nics.append(net['id'])
- # If the slice isn't connected to anything, then at least put it on
- # the public network.
- if (not nics) and (not ports):
+ if (not nics):
for net in nets:
if net['name']=='public':
nics.append(net['id'])
@@ -116,7 +92,7 @@
image_name = image.name
logger.info("using image from glance: " + str(image_name))
- try:
+ try:
legacy = Config().observer_legacy
except:
legacy = False
@@ -128,13 +104,14 @@
availability_zone_filter = 'nova:%s'%host_filter
instance_name = '%s-%d'%(instance.slice.name,instance.id)
+ self.instance_name = instance_name
userData = self.get_userdata(instance, pubkeys)
if instance.userData:
userData = instance.userData
controller = instance.node.site_deployment.controller
- tenant_fields = {'endpoint':controller.auth_url,
+ fields = {'endpoint':controller.auth_url,
'admin_user': instance.creator.email,
'admin_password': instance.creator.remote_password,
'admin_tenant': instance.slice.name,
@@ -146,15 +123,16 @@
'image_name':image_name,
'flavor_name':instance.flavor.name,
'nics':nics,
- 'ports':ports,
'meta':metadata_update,
'user_data':r'%s'%escape(userData)}
+ return fields
- res = run_template('sync_instances.yaml', tenant_fields,path='instances', expected_num=1)
- instance_id = res[0]['info']['OS-EXT-SRV-ATTR:instance_name']
+
+ def map_sync_outputs(self, instance, res):
+ instance_id = res[0]['info']['OS-EXT-SRV-ATTR:instance_name']
instance_uuid = res[0]['id']
- try:
+ try:
hostname = res[0]['info']['OS-EXT-SRV-ATTR:hypervisor_hostname']
ip = socket.gethostbyname(hostname)
instance.ip = ip
@@ -163,10 +141,11 @@
instance.instance_id = instance_id
instance.instance_uuid = instance_uuid
- instance.instance_name = instance_name
+ instance.instance_name = self.instance_name
instance.save()
-
- def delete_record(self, instance):
+
+
+ def map_delete_inputs(self, instance):
controller_register = json.loads(instance.node.site_deployment.controller.backend_register)
if (controller_register.get('disabled',False)):
@@ -174,7 +153,7 @@
instance_name = '%s-%d'%(instance.slice.name,instance.id)
controller = instance.node.site_deployment.controller
- tenant_fields = {'endpoint':controller.auth_url,
+ input = {'endpoint':controller.auth_url,
'admin_user': instance.creator.email,
'admin_password': instance.creator.remote_password,
'admin_tenant': instance.slice.name,
@@ -183,14 +162,4 @@
'name':instance_name,
'ansible_tag':instance_name,
'delete': True}
-
- try:
- res = run_template('sync_instances.yaml', tenant_fields,path='instances', expected_num=1)
- except Exception,e:
- print "Could not sync %s"%instance_name
- #import traceback
- #traceback.print_exc()
- raise e
-
- if (len(res)!=1):
- raise Exception('Could not delete instance %s'%instance.slice.name)
+ return input
diff --git a/xos/openstack_observer/steps/sync_object.py b/xos/openstack_observer/steps/sync_object.py
new file mode 100644
index 0000000..5e70464
--- /dev/null
+++ b/xos/openstack_observer/steps/sync_object.py
@@ -0,0 +1,20 @@
+import os
+import base64
+from collections import defaultdict
+from django.db.models import F, Q
+from xos.config import Config
+from observer.openstacksyncstep import OpenStackSyncStep
+from observer.syncstep import *
+from core.models import *
+from observer.ansible import *
+from openstack.driver import OpenStackDriver
+from util.logger import observer_logger as logger
+import json
+
+class SyncObject(OpenStackSyncStep):
+ provides=[] # Caller fills this in
+ requested_interval=0
+ observes=[] # Caller fills this in
+
+ def sync_record(self, r):
+ raise Exception('Waiting for Service dependency')
diff --git a/xos/openstack_observer/steps/sync_roles.py b/xos/openstack_observer/steps/sync_roles.py
index 42a36c6..c4bbe3f 100644
--- a/xos/openstack_observer/steps/sync_roles.py
+++ b/xos/openstack_observer/steps/sync_roles.py
@@ -11,27 +11,7 @@
class SyncRoles(OpenStackSyncStep):
provides=[Role]
requested_interval=0
- observes=Role
-
- def fetch_pending(self, deleted):
- # Deleting roles is not supported yet
- if (deleted):
- return []
-
- site_roles = SiteRole.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
- slice_roles = SliceRole.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
- controller_roles = ControllerRole.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
-
- roles = []
- for site_role in site_roles:
- roles.append(site_role)
- for slice_role in slice_roles:
- roles.append(slice_role)
- for controller_role in controller_roles:
- roles.append(controller_role)
-
- return roles
-
+ observes=[SiteRole,SliceRole,ControllerRole]
def sync_record(self, role):
if not role.enacted:
diff --git a/xos/openstack_observer/syncstep.py b/xos/openstack_observer/syncstep.py
index 791a89a..7954bb7 100644
--- a/xos/openstack_observer/syncstep.py
+++ b/xos/openstack_observer/syncstep.py
@@ -7,6 +7,8 @@
from django.db.models import F, Q
from core.models import *
from django.db import reset_queries
+from observer.ansible import *
+
import json
import time
import pdb
@@ -73,11 +75,18 @@
# This is the most common implementation of fetch_pending
# Steps should override it if they have their own logic
# for figuring out what objects are outstanding.
- main_obj = self.observes
- if (not deletion):
- objs = main_obj.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None), Q(lazy_blocked=False), Q(no_sync=False))
- else:
- objs = main_obj.deleted_objects.all()
+
+ main_objs = self.observes
+ if (type(main_objs) is not list):
+ main_objs=[main_objs]
+
+ objs = []
+ for main_obj in main_objs:
+ if (not deletion):
+ lobjs = main_obj.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None),Q(lazy_blocked=False),Q(no_sync=False))
+ else:
+ lobjs = main_obj.deleted_objects.all()
+ objs.extend(lobjs)
return objs
#return Instance.objects.filter(ip=None)
@@ -107,8 +116,58 @@
obj.save(update_fields=['backend_status'])
raise FailedDependency("Failed dependency for %s:%s peer %s:%s failed %s:%s" % (obj.__class__.__name__, str(getattr(obj,"pk","no_pk")), peer_object.__class__.__name__, str(getattr(peer_object,"pk","no_pk")), failed.__class__.__name__, str(getattr(failed,"pk","no_pk"))))
+
+ def sync_record(self, o):
+ try:
+ controller = o.get_controller()
+ controller_register = json.loads(o.node.site_deployment.controller.backend_register)
+
+ if (controller_register.get('disabled',False)):
+ raise InnocuousException('Controller %s is disabled'%sliver.node.site_deployment.controller.name)
+ except AttributeError:
+ pass
+
+ tenant_fields = self.map_sync_inputs(o)
+ main_objs=self.observes
+ if (type(main_objs) is list):
+ main_objs=main_objs[0]
+
+ path = ''.join(main_objs.__name__).lower()
+ res = run_template(self.playbook,tenant_fields,path=path)
+
+ try:
+ self.map_sync_outputs(o,res)
+ except AttributeError:
+ pass
+
+ def delete_record(self, o):
+ try:
+ controller = o.get_controller()
+ controller_register = json.loads(o.node.site_deployment.controller.backend_register)
+
+ if (controller_register.get('disabled',False)):
+ raise InnocuousException('Controller %s is disabled'%sliver.node.site_deployment.controller.name)
+ except AttributeError:
+ pass
+
+ tenant_fields = self.map_delete_outputs(o)
+
+ main_objs=self.observes
+ if (type(main_objs) is list):
+ main_objs=main_objs[0]
+
+ path = ''.join(main_objs.__name__).lower()
+
+ tenant_fields['delete']=True
+ res = run_template(self.playbook,tenant_fields,path=path)
+ try:
+ self.map_delete_outputs(o,res)
+ except AttributeError:
+ pass
+
def call(self, failed=[], deletion=False):
pending = self.fetch_pending(deletion)
+
for o in pending:
# another spot to clean up debug state
try:
@@ -207,11 +266,5 @@
return failed
- def sync_record(self, o):
- return
-
- def delete_record(self, o):
- return
-
def __call__(self, **args):
return self.call(**args)