Refactored sync steps, making ansible explicit
diff --git a/xos/openstack_observer/steps/purge_disabled_users.py b/xos/openstack_observer/steps/purge_disabled_users.py
index 9e30708..b5168e3 100644
--- a/xos/openstack_observer/steps/purge_disabled_users.py
+++ b/xos/openstack_observer/steps/purge_disabled_users.py
@@ -20,5 +20,6 @@
# disabled users that haven't been updated in over a week
one_week_ago = datetime.datetime.now() - datetime.timedelta(days=7)
return User.objects.filter(is_active=False, updated__gt=one_week_ago)
+
def sync_record(self, user):
user.delete()
diff --git a/xos/openstack_observer/steps/sync_controller_images.py b/xos/openstack_observer/steps/sync_controller_images.py
index 948fcea..3434f01 100644
--- a/xos/openstack_observer/steps/sync_controller_images.py
+++ b/xos/openstack_observer/steps/sync_controller_images.py
@@ -15,6 +15,7 @@
provides=[ControllerImages]
observes = ControllerImages
requested_interval=0
+ playbook='sync_controller_images.yaml'
def fetch_pending(self, deleted):
if (deleted):
@@ -23,13 +24,7 @@
# now we return all images that need to be enacted
return ControllerImages.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
- def sync_record(self, controller_image):
- logger.info("Working on image %s on controller %s" % (controller_image.image.name, controller_image.controller))
-
- controller_register = json.loads(controller_image.controller.backend_register)
- if (controller_register.get('disabled',False)):
- raise InnocuousException('Controller %s is disabled'%controller_image.controller.name)
-
+ def map_inputs(self, controller_image):
image_fields = {'endpoint':controller_image.controller.auth_url,
'admin_user':controller_image.controller.admin_user,
'admin_password':controller_image.controller.admin_password,
@@ -38,9 +33,9 @@
'ansible_tag': '%s@%s'%(controller_image.image.name,controller_image.controller.name), # name of ansible playbook
}
+ return image_fields
- res = run_template('sync_controller_images.yaml', image_fields, path='controller_images', expected_num=1)
-
+ def map_outputs(self, controller_image):
image_id = res[0]['id']
controller_image.glance_image_id = image_id
controller_image.backend_status = '1 - OK'
diff --git a/xos/openstack_observer/steps/sync_controller_networks.py b/xos/openstack_observer/steps/sync_controller_networks.py
index 54f2466..da9c46b 100644
--- a/xos/openstack_observer/steps/sync_controller_networks.py
+++ b/xos/openstack_observer/steps/sync_controller_networks.py
@@ -20,6 +20,7 @@
requested_interval = 0
provides=[Network]
observes=ControllerNetwork
+ playbook='sync_controller_networks.yaml'
def alloc_subnet(self, uuid):
# 16 bits only
@@ -49,9 +50,9 @@
'cidr':cidr,
'delete':False
}
+ return network_fields
- res = run_template('sync_controller_networks.yaml', network_fields, path = 'controller_networks',expected_num=2)
-
+ def map_sync_outputs(self, controller_network,res):
network_id = res[0]['id']
subnet_id = res[1]['id']
controller_network.net_id = network_id
@@ -61,34 +62,25 @@
controller_network.save()
- def sync_record(self, controller_network):
+ def map_sync_inputs(self, controller_network):
if (controller_network.network.template.name!='Private'):
logger.info("skipping network controller %s because it is not private" % controller_network)
# We only sync private networks
return
- logger.info("sync'ing network controller %s for network %s slice %s controller %s" % (controller_network, controller_network.network, str(controller_network.network.owner), controller_network.controller))
-
- controller_register = json.loads(controller_network.controller.backend_register)
- if (controller_register.get('disabled',False)):
- raise InnocuousException('Controller %s is disabled'%controller_network.controller.name)
-
if not controller_network.controller.admin_user:
logger.info("controller %r has no admin_user, skipping" % controller_network.controller)
return
if controller_network.network.owner and controller_network.network.owner.creator:
- self.save_controller_network(controller_network)
- logger.info("saved network controller: %s" % (controller_network))
+ return self.save_controller_network(controller_network)
+ else:
+ raise Exception('Could not save network controller %s'%controller_network)
- def delete_record(self, controller_network):
+ def map_delete_inputs(self, controller_network):
if (controller_network.network.template.name!='Private'):
# We only sync private networks
return
- controller_register = json.loads(controller_network.controller.backend_register)
- if (controller_register.get('disabled',False)):
- raise InnocuousException('Controller %s is disabled'%controller_network.controller.name)
-
try:
slice = controller_network.network.owner # XXX: FIXME!!
except:
@@ -108,7 +100,7 @@
'delete':True
}
- res = run_template('sync_controller_networks.yaml', network_fields, path = 'controller_networks',expected_num=1)
+ return network_fields
"""
driver = OpenStackDriver().client_driver(caller=controller_network.network.owner.creator,
diff --git a/xos/openstack_observer/steps/sync_controller_site_privileges.py b/xos/openstack_observer/steps/sync_controller_site_privileges.py
index a2c40ef..d52c999 100644
--- a/xos/openstack_observer/steps/sync_controller_site_privileges.py
+++ b/xos/openstack_observer/steps/sync_controller_site_privileges.py
@@ -16,27 +16,14 @@
provides=[SitePrivilege]
requested_interval=0
observes=ControllerSitePrivilege
+ playbook='sync_controller_users.yaml'
- def fetch_pending(self, deleted):
-
- if (deleted):
- return ControllerSitePrivilege.deleted_objects.all()
- else:
- return ControllerSitePrivilege.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
-
- def sync_record(self, controller_site_privilege):
- logger.info("sync'ing controler_site_privilege %s at controller %s" % (controller_site_privilege, controller_site_privilege.controller))
-
+ def map_sync_inputs(self, controller_site_privilege):
controller_register = json.loads(controller_site_privilege.controller.backend_register)
- if (controller_register.get('disabled',False)):
- raise InnocuousException('Controller %s is disabled'%controller_site_privilege.controller.name)
-
-
if not controller_site_privilege.controller.admin_user:
logger.info("controller %r has no admin_user, skipping" % controller_site_privilege.controller)
return
- template = os_template_env.get_template('sync_controller_users.yaml')
roles = [controller_site_privilege.site_privilege.role.role]
# setup user home site roles at controller
if not controller_site_privilege.site_privilege.user.site:
@@ -63,10 +50,9 @@
'roles':roles,
'tenant':controller_site_privilege.site_privilege.site.login_base}
- rendered = template.render(user_fields)
- expected_length = len(roles) + 1
- res = run_template('sync_controller_users.yaml', user_fields,path='controller_site_privileges', expected_num=expected_length)
+ return user_fields
+ def map_sync_outputs(self, controller_site_privilege, res):
# results is an array in which each element corresponds to an
# "ok" string received per operation. If we get as many oks as
# the number of operations we issued, that means a grand success.
diff --git a/xos/openstack_observer/steps/sync_controller_sites.py b/xos/openstack_observer/steps/sync_controller_sites.py
index 670f09c..07b0091 100644
--- a/xos/openstack_observer/steps/sync_controller_sites.py
+++ b/xos/openstack_observer/steps/sync_controller_sites.py
@@ -13,17 +13,13 @@
requested_interval=0
provides=[Site]
observes=ControllerSite
+ playbook = 'sync_controller_sites.yaml'
def fetch_pending(self, deleted=False):
pending = super(OpenStackSyncStep, self).fetch_pending(deleted)
return pending.filter(controller__isnull=False)
- def sync_record(self, controller_site):
- controller_register = json.loads(controller_site.controller.backend_register)
- if (controller_register.get('disabled',False)):
- raise InnocuousException('Controller %s is disabled'%controller_site.controller.name)
-
- template = os_template_env.get_template('sync_controller_sites.yaml')
+ def map_sync_inputs(self, controller_site):
tenant_fields = {'endpoint':controller_site.controller.auth_url,
'admin_user': controller_site.controller.admin_user,
'admin_password': controller_site.controller.admin_password,
@@ -31,7 +27,9 @@
'ansible_tag': '%s@%s'%(controller_site.site.login_base,controller_site.controller.name), # name of ansible playbook
'tenant': controller_site.site.login_base,
'tenant_description': controller_site.site.name}
+ return tenant_fields
+ def map_sync_outputs(self, controller_site, res):
rendered = template.render(tenant_fields)
res = run_template('sync_controller_sites.yaml', tenant_fields, path='controller_sites', expected_num=1)
diff --git a/xos/openstack_observer/steps/sync_controller_slice_privileges.py b/xos/openstack_observer/steps/sync_controller_slice_privileges.py
index 2e2e63c..a998460 100644
--- a/xos/openstack_observer/steps/sync_controller_slice_privileges.py
+++ b/xos/openstack_observer/steps/sync_controller_slice_privileges.py
@@ -16,21 +16,9 @@
provides=[SlicePrivilege]
requested_interval=0
observes=ControllerSlicePrivilege
+ playbook = 'sync_controller_users.yaml'
- def fetch_pending(self, deleted):
-
- if (deleted):
- return ControllerSlicePrivilege.deleted_objects.all()
- else:
- return ControllerSlicePrivilege.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
-
- def sync_record(self, controller_slice_privilege):
- logger.info("sync'ing controler_slice_privilege %s at controller %s" % (controller_slice_privilege, controller_slice_privilege.controller))
-
- controller_register = json.loads(controller_slice_privilege.controller.backend_register)
- if (controller_register.get('disabled',False)):
- raise InnocuousException('Controller %s is disabled'%controller_slice_privilege.controller.name)
-
+ def map_inputs(self, controller_slice_privilege):
if not controller_slice_privilege.controller.admin_user:
logger.info("controller %r has no admin_user, skipping" % controller_slice_privilege.controller)
return
@@ -61,17 +49,11 @@
'admin_tenant': controller_slice_privilege.controller.admin_tenant,
'roles':roles,
'tenant':controller_slice_privilege.slice_privilege.slice.name}
+ return user_fields
- rendered = template.render(user_fields)
- expected_length = len(roles) + 1
- res = run_template('sync_controller_users.yaml', user_fields, path='controller_slice_privileges', expected_num=expected_length)
-
- # results is an array in which each element corresponds to an
- # "ok" string received per operation. If we get as many oks as
- # the number of operations we issued, that means a grand success.
- # Otherwise, the number of oks tell us which operation failed.
- controller_slice_privilege.role_id = res[0]['id']
- controller_slice_privilege.save()
+ def map_sync_outputs(self, controller_slice_privilege, res):
+ controller_slice_privilege.role_id = res[0]['id']
+ controller_slice_privilege.save()
def delete_record(self, controller_slice_privilege):
controller_register = json.loads(controller_slice_privilege.controller.backend_register)
diff --git a/xos/openstack_observer/steps/sync_controller_slices.py b/xos/openstack_observer/steps/sync_controller_slices.py
index c456a2f..ba3e955 100644
--- a/xos/openstack_observer/steps/sync_controller_slices.py
+++ b/xos/openstack_observer/steps/sync_controller_slices.py
@@ -17,19 +17,9 @@
requested_interval=0
observes=ControllerSlice
- def fetch_pending(self, deleted):
- if (deleted):
- return ControllerSlice.deleted_objects.all()
- else:
- return ControllerSlice.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
-
- def sync_record(self, controller_slice):
+ def map_sync_inputs(self, controller_slice):
logger.info("sync'ing slice controller %s" % controller_slice)
- controller_register = json.loads(controller_slice.controller.backend_register)
- if (controller_register.get('disabled',False)):
- raise InnocuousException('Controller %s is disabled'%controller_slice.controller.name)
-
if not controller_slice.controller.admin_user:
logger.info("controller %r has no admin_user, skipping" % controller_slice.controller)
return
@@ -54,8 +44,9 @@
'ansible_tag':'%s@%s'%(controller_slice.slice.name,controller_slice.controller.name),
'max_instances':max_instances}
- expected_num = len(roles)+1
- res = run_template('sync_controller_slices.yaml', tenant_fields, path='controller_slices', expected_num=expected_num)
+ return tenant_fields
+
+ def map_sync_outputs(self, controller_slice, res):
tenant_id = res[0]['id']
if (not controller_slice.tenant_id):
try:
@@ -70,11 +61,7 @@
controller_slice.save()
- def delete_record(self, controller_slice):
- controller_register = json.loads(controller_slice.controller.backend_register)
- if (controller_register.get('disabled',False)):
- raise InnocuousException('Controller %s is disabled'%controller_slice.controller.name)
-
+ def map_delete_inputs(self, controller_slice):
controller_users = ControllerUser.objects.filter(user=controller_slice.slice.creator,
controller=controller_slice.controller)
if not controller_users:
@@ -91,6 +78,4 @@
'name':controller_user.user.email,
'ansible_tag':'%s@%s'%(controller_slice.slice.name,controller_slice.controller.name),
'delete': True}
-
- expected_num = 1
- run_template('sync_controller_slices.yaml', tenant_fields, path='controller_slices', expected_num=expected_num)
+ return tenant_fields
diff --git a/xos/openstack_observer/steps/sync_controller_users.py b/xos/openstack_observer/steps/sync_controller_users.py
index d30d0ff..ae04460 100644
--- a/xos/openstack_observer/steps/sync_controller_users.py
+++ b/xos/openstack_observer/steps/sync_controller_users.py
@@ -16,27 +16,13 @@
provides=[User]
requested_interval=0
observes=ControllerUser
+ playbook='sync_controller_users.yaml'
- def fetch_pending(self, deleted):
-
- if (deleted):
- return ControllerUser.deleted_objects.all()
- else:
- return ControllerUser.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
-
- def sync_record(self, controller_user):
- logger.info("sync'ing user %s at controller %s" % (controller_user.user, controller_user.controller))
-
- controller_register = json.loads(controller_user.controller.backend_register)
- if (controller_register.get('disabled',False)):
- raise InnocuousException('Controller %s is disabled'%controller_user.controller.name)
-
+ def map_sync_inputs(self, controller_user):
if not controller_user.controller.admin_user:
logger.info("controller %r has no admin_user, skipping" % controller_user.controller)
return
- template = os_template_env.get_template('sync_controller_users.yaml')
-
# All users will have at least the 'user' role at their home site/tenant.
# We must also check if the user should have the admin role
roles = ['user']
@@ -68,21 +54,14 @@
'roles':roles,
'tenant':controller_user.user.site.login_base
}
+ return user_fields
- rendered = template.render(user_fields)
- expected_length = len(roles) + 1
-
- res = run_template('sync_controller_users.yaml', user_fields,path='controller_users', expected_num=expected_length)
-
- controller_user.kuser_id = res[0]['id']
- controller_user.backend_status = '1 - OK'
- controller_user.save()
+ def map_sync_outputs(self, controller_user, res):
+ controller_user.kuser_id = res[0]['id']
+ controller_user.backend_status = '1 - OK'
+ controller_user.save()
def delete_record(self, controller_user):
- controller_register = json.loads(controller_user.controller.backend_register)
- if (controller_register.get('disabled',False)):
- raise InnocuousException('Controller %s is disabled'%controller_user.controller.name)
-
if controller_user.kuser_id:
driver = self.driver.admin_driver(controller=controller_user.controller)
driver.delete_user(controller_user.kuser_id)
diff --git a/xos/openstack_observer/steps/sync_instances.py b/xos/openstack_observer/steps/sync_instances.py
index 3f61d35..fa76228 100644
--- a/xos/openstack_observer/steps/sync_instances.py
+++ b/xos/openstack_observer/steps/sync_instances.py
@@ -20,6 +20,7 @@
provides=[Instance]
requested_interval=0
observes=Instance
+ playbook='sync_instances.yaml'
def get_userdata(self, instance, pubkeys):
userdata = '#cloud-config\n\nopencloud:\n slicename: "%s"\n hostname: "%s"\n restapi_hostname: "%s"\n restapi_port: "%s"\n' % (instance.slice.name, instance.node.name, RESTAPI_HOSTNAME, str(RESTAPI_PORT))
@@ -28,14 +29,9 @@
userdata += ' - %s\n' % key
return userdata
- def sync_record(self, instance):
- logger.info("sync'ing instance:%s slice:%s controller:%s " % (instance, instance.slice.name, instance.node.site_deployment.controller))
- controller_register = json.loads(instance.node.site_deployment.controller.backend_register)
-
- if (controller_register.get('disabled',False)):
- raise InnocuousException('Controller %s is disabled'%instance.node.site_deployment.controller.name)
-
- metadata_update = {}
+ def map_sync_inputs(self, instance):
+ inputs = {}
+ metadata_update = {}
if (instance.numberCores):
metadata_update["cpu_cores"] = str(instance.numberCores)
@@ -43,8 +39,7 @@
if tag.name.startswith("sysctl-"):
metadata_update[tag.name] = tag.value
- # public keys
- slice_memberships = SlicePrivilege.objects.filter(slice=instance.slice)
+ slice_memberships = SlicePrivilege.objects.filter(slice=instance.slice)
pubkeys = set([sm.user.public_key for sm in slice_memberships if sm.user.public_key])
if instance.creator.public_key:
pubkeys.add(instance.creator.public_key)
@@ -55,23 +50,8 @@
if instance.slice.service and instance.slice.service.public_key:
pubkeys.add(instance.slice.service.public_key)
- # Handle any ports that are already created and attached to the instance.
- # If we do have a port for a network, then add that network to an
- # exclude list so we won't try to auto-create ports on that network
- # when instantiating.
- ports = []
- exclude_networks = set()
- exclude_templates = set()
- for ns in instance.ports.all():
- if not ns.port_id:
- raise DeferredException("Port %s on instance %s has no id; Try again later" % (str(ns), str(instance)) )
- ports.append(ns.port_id)
- exclude_networks.add(ns.network)
- exclude_templates.add(ns.network.template)
-
nics = []
networks = [ns.network for ns in NetworkSlice.objects.filter(slice=instance.slice)]
- networks = [n for n in networks if (n not in exclude_networks)]
controller_networks = ControllerNetwork.objects.filter(network__in=networks,
controller=instance.node.site_deployment.controller)
@@ -79,14 +59,12 @@
if controller_network.network.template.visibility == 'private' and \
controller_network.network.template.translation == 'none':
if not controller_network.net_id:
- raise DeferredException("Private Network %s has no id; Try again later" % controller_network.network.name)
+ raise Exception("Private Network %s has no id; Try again later" % controller_network.network.name)
nics.append(controller_network.net_id)
- # Now include network templates, for those networks that use a
- # shared_network_name.
+ # now include network template
network_templates = [network.template.shared_network_name for network in networks \
if network.template.shared_network_name]
- network_templates = [nt for nt in network_templates if (nt not in exclude_templates)]
#driver = self.driver.client_driver(caller=instance.creator, tenant=instance.slice.name, controller=instance.controllerNetwork)
driver = self.driver.admin_driver(tenant='admin', controller=instance.node.site_deployment.controller)
@@ -95,9 +73,7 @@
if net['name'] in network_templates:
nics.append(net['id'])
- # If the slice isn't connected to anything, then at least put it on
- # the public network.
- if (not nics) and (not ports):
+ if (not nics):
for net in nets:
if net['name']=='public':
nics.append(net['id'])
@@ -116,7 +92,7 @@
image_name = image.name
logger.info("using image from glance: " + str(image_name))
- try:
+ try:
legacy = Config().observer_legacy
except:
legacy = False
@@ -134,7 +110,7 @@
userData = instance.userData
controller = instance.node.site_deployment.controller
- tenant_fields = {'endpoint':controller.auth_url,
+ fields = {'endpoint':controller.auth_url,
'admin_user': instance.creator.email,
'admin_password': instance.creator.remote_password,
'admin_tenant': instance.slice.name,
@@ -146,15 +122,16 @@
'image_name':image_name,
'flavor_name':instance.flavor.name,
'nics':nics,
- 'ports':ports,
'meta':metadata_update,
'user_data':r'%s'%escape(userData)}
+ return fields
- res = run_template('sync_instances.yaml', tenant_fields,path='instances', expected_num=1)
- instance_id = res[0]['info']['OS-EXT-SRV-ATTR:instance_name']
+
+ def map_sync_outputs(self, instance, res):
+ instance_id = res[0]['info']['OS-EXT-SRV-ATTR:instance_name']
instance_uuid = res[0]['id']
- try:
+ try:
hostname = res[0]['info']['OS-EXT-SRV-ATTR:hypervisor_hostname']
ip = socket.gethostbyname(hostname)
instance.ip = ip
@@ -165,8 +142,9 @@
instance.instance_uuid = instance_uuid
instance.instance_name = instance_name
instance.save()
-
- def delete_record(self, instance):
+
+
+ def map_delete_inputs(self, instance):
controller_register = json.loads(instance.node.site_deployment.controller.backend_register)
if (controller_register.get('disabled',False)):
@@ -174,7 +152,7 @@
instance_name = '%s-%d'%(instance.slice.name,instance.id)
controller = instance.node.site_deployment.controller
- tenant_fields = {'endpoint':controller.auth_url,
+ input = {'endpoint':controller.auth_url,
'admin_user': instance.creator.email,
'admin_password': instance.creator.remote_password,
'admin_tenant': instance.slice.name,
@@ -183,14 +161,4 @@
'name':instance_name,
'ansible_tag':instance_name,
'delete': True}
-
- try:
- res = run_template('sync_instances.yaml', tenant_fields,path='instances', expected_num=1)
- except Exception,e:
- print "Could not sync %s"%instance_name
- #import traceback
- #traceback.print_exc()
- raise e
-
- if (len(res)!=1):
- raise Exception('Could not delete instance %s'%instance.slice.name)
+ return input
diff --git a/xos/openstack_observer/steps/sync_instances.yaml b/xos/openstack_observer/steps/sync_instances.yaml
index 803a294..83f8f3c 100644
--- a/xos/openstack_observer/steps/sync_instances.yaml
+++ b/xos/openstack_observer/steps/sync_instances.yaml
@@ -3,6 +3,8 @@
connection: local
tasks:
- nova_compute:
+ register: instance
+ ignore_errors: yes
auth_url: {{ endpoint }}
login_username: {{ admin_user }}
login_password: {{ admin_password }}
@@ -32,3 +34,5 @@
{% endfor %}
{% endif %}
{% endif %}
+
+ - command: "dig +short -x {{ instance.info['OS-EXT-SRV-ATTR:hypervisor_hostname'] }}"
diff --git a/xos/openstack_observer/steps/sync_roles.py b/xos/openstack_observer/steps/sync_roles.py
index 42a36c6..c4bbe3f 100644
--- a/xos/openstack_observer/steps/sync_roles.py
+++ b/xos/openstack_observer/steps/sync_roles.py
@@ -11,27 +11,7 @@
class SyncRoles(OpenStackSyncStep):
provides=[Role]
requested_interval=0
- observes=Role
-
- def fetch_pending(self, deleted):
- # Deleting roles is not supported yet
- if (deleted):
- return []
-
- site_roles = SiteRole.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
- slice_roles = SliceRole.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
- controller_roles = ControllerRole.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
-
- roles = []
- for site_role in site_roles:
- roles.append(site_role)
- for slice_role in slice_roles:
- roles.append(slice_role)
- for controller_role in controller_roles:
- roles.append(controller_role)
-
- return roles
-
+ observes=[SiteRole,SliceRole,ControllerRole]
def sync_record(self, role):
if not role.enacted: