CORD-1010 refactor openstack synchronizer to use new_base
Change-Id: I4eb4e5131275faf69e1da05efc81a3d8d0d12296
diff --git a/xos/synchronizer/steps/purge_disabled_users.py b/xos/synchronizer/steps/purge_disabled_users.py
index 5cb7b06..425d4e4 100644
--- a/xos/synchronizer/steps/purge_disabled_users.py
+++ b/xos/synchronizer/steps/purge_disabled_users.py
@@ -1,11 +1,10 @@
import os
import base64
import datetime
-from django.db.models import F, Q
from xos.config import Config
from synchronizers.openstack.openstacksyncstep import OpenStackSyncStep
-from core.models.user import User
from xos.logger import observer_logger as logger
+from synchronizers.new_base.modelaccessor import *
#class SyncRoles(OpenStackSyncStep):
# provides=[User]
diff --git a/xos/synchronizer/steps/sync_container.py b/xos/synchronizer/steps/sync_container.py
index 9449af1..7b69fa5 100644
--- a/xos/synchronizer/steps/sync_container.py
+++ b/xos/synchronizer/steps/sync_container.py
@@ -4,13 +4,12 @@
import sys
import base64
import time
-from django.db.models import F, Q
from xos.config import Config
-from synchronizers.base.SyncInstanceUsingAnsible import SyncInstanceUsingAnsible
-from synchronizers.base.syncstep import SyncStep, DeferredException
-from synchronizers.base.ansible_helper import run_template_ssh
-from core.models import Service, Slice, Instance
+from synchronizers.new_base.SyncInstanceUsingAnsible import SyncInstanceUsingAnsible
+from synchronizers.new_base.syncstep import DeferredException
+from synchronizers.new_base.ansible_helper import run_template_ssh
from xos.logger import Logger, logging
+from synchronizers.new_base.modelaccessor import *
# hpclibrary will be in steps/..
parentdir = os.path.join(os.path.dirname(__file__),"..")
@@ -34,7 +33,7 @@
def get_instance_port(self, container_port):
for p in container_port.network.links.all():
- if (p.instance) and (p.instance.isolation=="vm") and (p.instance.node == container_port.instance.node) and (p.mac):
+ if (p.instance) and (p.instance.isolation=="vm") and (p.instance.node.id == container_port.instance.node.id) and (p.mac):
return p
return None
diff --git a/xos/synchronizer/steps/sync_controller_images.py b/xos/synchronizer/steps/sync_controller_images.py
index b944dca..2241fec 100644
--- a/xos/synchronizer/steps/sync_controller_images.py
+++ b/xos/synchronizer/steps/sync_controller_images.py
@@ -1,15 +1,11 @@
import os
import base64
-from collections import defaultdict
-from django.db.models import F, Q
from xos.config import Config
from synchronizers.openstack.openstacksyncstep import OpenStackSyncStep
-from synchronizers.base.syncstep import *
-from core.models import Controller
-from core.models import Image, ControllerImages
-from xos.logger import observer_logger as logger
-from synchronizers.base.ansible_helper import *
-import json
+from synchronizers.new_base.syncstep import *
+from xos.logger import observer_logger as logger
+from synchronizers.new_base.ansible_helper import *
+from synchronizers.new_base.modelaccessor import *
class SyncControllerImages(OpenStackSyncStep):
provides=[ControllerImages]
@@ -21,8 +17,7 @@
if (deleted):
return []
- # now we return all images that need to be enacted
- return ControllerImages.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
+ return super(SyncControllerImages, self).fetch_pending(deleted)
def map_sync_inputs(self, controller_image):
image_fields = {'endpoint':controller_image.controller.auth_url,
diff --git a/xos/synchronizer/steps/sync_controller_networks.py b/xos/synchronizer/steps/sync_controller_networks.py
index 357b084..8187e84 100644
--- a/xos/synchronizer/steps/sync_controller_networks.py
+++ b/xos/synchronizer/steps/sync_controller_networks.py
@@ -2,21 +2,14 @@
import base64
import struct
import socket
-from collections import defaultdict
from netaddr import IPAddress, IPNetwork
-from django.db.models import F, Q
from xos.config import Config
from synchronizers.openstack.openstacksyncstep import OpenStackSyncStep
-from synchronizers.base.syncstep import *
-from core.models.network import *
-from core.models.slice import *
-from core.models.instance import Instance
+from synchronizers.new_base.syncstep import *
from xos.logger import observer_logger as logger
-from synchronizers.base.ansible_helper import *
+from synchronizers.new_base.ansible_helper import *
from xos.config import Config
-import json
-
-import pdb
+from synchronizers.new_base.modelaccessor import *
class SyncControllerNetworks(OpenStackSyncStep):
requested_interval = 0
diff --git a/xos/synchronizer/steps/sync_controller_site_privileges.py b/xos/synchronizer/steps/sync_controller_site_privileges.py
index 805f446..2768090 100644
--- a/xos/synchronizer/steps/sync_controller_site_privileges.py
+++ b/xos/synchronizer/steps/sync_controller_site_privileges.py
@@ -1,16 +1,12 @@
import os
import base64
-from collections import defaultdict
-from django.db.models import F, Q
+import json
from xos.config import Config
from synchronizers.openstack.openstacksyncstep import OpenStackSyncStep
-from synchronizers.base.syncstep import *
-from core.models.site import Controller, SitePrivilege
-from core.models.user import User
-from core.models.controlleruser import ControllerUser, ControllerSitePrivilege
+from synchronizers.new_base.syncstep import *
from xos.logger import observer_logger as logger
-from synchronizers.base.ansible_helper import *
-import json
+from synchronizers.new_base.ansible_helper import *
+from synchronizers.new_base.modelaccessor import *
class SyncControllerSitePrivileges(OpenStackSyncStep):
provides=[SitePrivilege]
diff --git a/xos/synchronizer/steps/sync_controller_sites.py b/xos/synchronizer/steps/sync_controller_sites.py
index 1a1ce41..ba489aa 100644
--- a/xos/synchronizer/steps/sync_controller_sites.py
+++ b/xos/synchronizer/steps/sync_controller_sites.py
@@ -1,13 +1,12 @@
import os
import base64
-from django.db.models import F, Q
from xos.config import Config
from synchronizers.openstack.openstacksyncstep import OpenStackSyncStep
-from core.models.site import *
-from synchronizers.base.syncstep import *
-from synchronizers.base.ansible_helper import *
+from synchronizers.new_base.syncstep import *
+from synchronizers.new_base.ansible_helper import *
from xos.logger import observer_logger as logger
import json
+from synchronizers.new_base.modelaccessor import *
class SyncControllerSites(OpenStackSyncStep):
requested_interval=0
@@ -16,7 +15,12 @@
playbook = 'sync_controller_sites.yaml'
def fetch_pending(self, deleted=False):
- lobjs = ControllerSite.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None),Q(lazy_blocked=False),Q(controller__isnull=False))
+ lobjs = super(SyncControllerSites, self).fetch_pending(deleted)
+
+ if not deleted:
+ # filter out objects with null controllers
+ lobjs = [x for x in lobjs if x.controller]
+
return lobjs
def map_sync_inputs(self, controller_site):
diff --git a/xos/synchronizer/steps/sync_controller_slice_privileges.py b/xos/synchronizer/steps/sync_controller_slice_privileges.py
index b7c0487..3a2d167 100644
--- a/xos/synchronizer/steps/sync_controller_slice_privileges.py
+++ b/xos/synchronizer/steps/sync_controller_slice_privileges.py
@@ -1,16 +1,12 @@
import os
import base64
-from collections import defaultdict
-from django.db.models import F, Q
+import json
from xos.config import Config
from synchronizers.openstack.openstacksyncstep import OpenStackSyncStep
-from synchronizers.base.syncstep import *
-from core.models.slice import Controller, SlicePrivilege
-from core.models.user import User
-from core.models.controlleruser import ControllerUser, ControllerSlicePrivilege
-from synchronizers.base.ansible_helper import *
+from synchronizers.new_base.syncstep import *
+from synchronizers.new_base.ansible_helper import *
from xos.logger import observer_logger as logger
-import json
+from synchronizers.new_base.modelaccessor import *
class SyncControllerSlicePrivileges(OpenStackSyncStep):
provides=[SlicePrivilege]
@@ -29,15 +25,6 @@
if not controller_slice_privilege.slice_privilege.user.site:
raise Exception('Sliceless user %s'%controller_slice_privilege.slice_privilege.user.email)
else:
- # look up tenant id for the user's slice at the controller
- #ctrl_slice_deployments = SliceDeployment.objects.filter(
- # slice_deployment__slice=controller_slice_privilege.user.slice,
- # controller=controller_slice_privilege.controller)
-
- #if ctrl_slice_deployments:
- # # need the correct tenant id for slice at the controller
- # tenant_id = ctrl_slice_deployments[0].tenant_id
- # tenant_name = ctrl_slice_deployments[0].slice_deployment.slice.login_base
user_fields = {
'endpoint':controller_slice_privilege.controller.auth_url,
'endpoint_v3': controller_slice_privilege.controller.auth_url_v3,
@@ -64,14 +51,16 @@
if controller_slice_privilege.role_id:
driver = self.driver.admin_driver(controller=controller_slice_privilege.controller)
- user = ControllerUser.objects.get(
- controller=controller_slice_privilege.controller,
- user=controller_slice_privilege.slice_privilege.user
+ user = ControllerUser.objects.filter(
+ controller_id=controller_slice_privilege.controller.id,
+ user_id=controller_slice_privilege.slice_privilege.user.id
)
- slice = ControllerSlice.objects.get(
- controller=controller_slice_privilege.controller,
- user=controller_slice_privilege.slice_privilege.user
+ user = user[0]
+ slice = ControllerSlice.objects.filter(
+ controller_id=controller_slice_privilege.controller.id,
+ user_id=controller_slice_privilege.slice_privilege.user.id
)
+ slice = slice[0]
driver.delete_user_role(
user.kuser_id,
slice.tenant_id,
diff --git a/xos/synchronizer/steps/sync_controller_slices.py b/xos/synchronizer/steps/sync_controller_slices.py
index 2bc0555..a01c2b2 100644
--- a/xos/synchronizer/steps/sync_controller_slices.py
+++ b/xos/synchronizer/steps/sync_controller_slices.py
@@ -1,15 +1,11 @@
import os
import base64
-from collections import defaultdict
-from netaddr import IPAddress, IPNetwork
-from django.db.models import F, Q
from xos.config import Config
from synchronizers.openstack.openstacksyncstep import OpenStackSyncStep
-from synchronizers.base.syncstep import *
-from core.models import *
-from synchronizers.base.ansible_helper import *
+from synchronizers.new_base.syncstep import *
+from synchronizers.new_base.ansible_helper import *
from xos.logger import observer_logger as logger
-import json
+from synchronizers.new_base.modelaccessor import *
class SyncControllerSlices(OpenStackSyncStep):
provides=[Slice]
@@ -24,8 +20,8 @@
logger.info("controller %r has no admin_user, skipping" % controller_slice.controller)
return
- controller_users = ControllerUser.objects.filter(user=controller_slice.slice.creator,
- controller=controller_slice.controller)
+ controller_users = ControllerUser.objects.filter(user_id=controller_slice.slice.creator.id,
+ controller_id=controller_slice.controller.id)
if not controller_users:
raise Exception("slice createor %s has not accout at controller %s" % (controller_slice.slice.creator, controller_slice.controller.name))
else:
@@ -65,8 +61,8 @@
def map_delete_inputs(self, controller_slice):
- controller_users = ControllerUser.objects.filter(user=controller_slice.slice.creator,
- controller=controller_slice.controller)
+ controller_users = ControllerUser.objects.filter(user_id=controller_slice.slice.creator.id,
+ controller_id=controller_slice.controller.id)
if not controller_users:
raise Exception("slice createor %s has not accout at controller %s" % (controller_slice.slice.creator, controller_slice.controller.name))
else:
diff --git a/xos/synchronizer/steps/sync_controller_users.py b/xos/synchronizer/steps/sync_controller_users.py
index 189703a..449dc11 100644
--- a/xos/synchronizer/steps/sync_controller_users.py
+++ b/xos/synchronizer/steps/sync_controller_users.py
@@ -1,16 +1,11 @@
import os
import base64
-from collections import defaultdict
-from django.db.models import F, Q
from xos.config import Config
from synchronizers.openstack.openstacksyncstep import OpenStackSyncStep
-from synchronizers.base.syncstep import *
-from core.models.site import Controller, SiteDeployment, SiteDeployment
-from core.models.user import User
-from core.models.controlleruser import ControllerUser
-from synchronizers.base.ansible_helper import *
+from synchronizers.new_base.syncstep import *
+from synchronizers.new_base.ansible_helper import *
from xos.logger import observer_logger as logger
-import json
+from synchronizers.new_base.modelaccessor import *
class SyncControllerUsers(OpenStackSyncStep):
provides=[User]
@@ -35,15 +30,6 @@
if not controller_user.user.site:
raise Exception('Siteless user %s'%controller_user.user.email)
else:
- # look up tenant id for the user's site at the controller
- #ctrl_site_deployments = SiteDeployment.objects.filter(
- # site_deployment__site=controller_user.user.site,
- # controller=controller_user.controller)
-
- #if ctrl_site_deployments:
- # # need the correct tenant id for site at the controller
- # tenant_id = ctrl_site_deployments[0].tenant_id
- # tenant_name = ctrl_site_deployments[0].site_deployment.site.login_base
user_fields = {
'endpoint':controller_user.controller.auth_url,
'endpoint_v3': controller_user.controller.auth_url_v3,
diff --git a/xos/synchronizer/steps/sync_images.py b/xos/synchronizer/steps/sync_images.py
index d4ef430..61e8f18 100644
--- a/xos/synchronizer/steps/sync_images.py
+++ b/xos/synchronizer/steps/sync_images.py
@@ -1,52 +1,15 @@
import os
import base64
-from django.db.models import F, Q
from xos.config import Config
from synchronizers.openstack.openstacksyncstep import OpenStackSyncStep
-from core.models.image import Image
from xos.logger import observer_logger as logger
+from synchronizers.new_base.modelaccessor import *
class SyncImages(OpenStackSyncStep):
provides=[Image]
requested_interval=0
- observes=Image
+ observes=[Image]
- def fetch_pending(self, deleted):
- # Images come from the back end
- # You can't delete them
- if (deleted):
- logger.info("SyncImages: returning because deleted=True")
- return []
-
- # get list of images on disk
- images_path = Config().observer_images_directory
-
- logger.info("SyncImages: deleted=False, images_path=%s" % images_path)
-
- available_images = {}
- if os.path.exists(images_path):
- for f in os.listdir(images_path):
- filename = os.path.join(images_path, f)
- if os.path.isfile(filename) and filename.endswith(".img"):
- available_images[f] = filename
-
- logger.info("SyncImages: available_images = %s" % str(available_images))
-
- images = Image.objects.all()
- image_names = [image.name for image in images]
-
- for image_name in available_images:
- #remove file extension
- clean_name = ".".join(image_name.split('.')[:-1])
- if clean_name not in image_names:
- logger.info("SyncImages: adding %s" % clean_name)
- image = Image(name=clean_name,
- disk_format='raw',
- container_format='bare',
- path = available_images[image_name])
- image.save()
-
- return Image.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
-
- def sync_record(self, image):
- image.save()
+ def sync_record(self, role):
+ # do nothing
+ pass
diff --git a/xos/synchronizer/steps/sync_instances.py b/xos/synchronizer/steps/sync_instances.py
index 395b9b1..3682fd3 100644
--- a/xos/synchronizer/steps/sync_instances.py
+++ b/xos/synchronizer/steps/sync_instances.py
@@ -1,16 +1,13 @@
import os
import base64
import socket
-from django.db.models import F, Q
from xos.config import Config
from xos.settings import RESTAPI_HOSTNAME, RESTAPI_PORT
from synchronizers.openstack.openstacksyncstep import OpenStackSyncStep
-from core.models.instance import Instance
-from core.models.slice import Slice, SlicePrivilege, ControllerSlice
-from core.models.network import Network, NetworkSlice, ControllerNetwork
-from synchronizers.base.ansible_helper import *
-from synchronizers.base.syncstep import *
+from synchronizers.new_base.ansible_helper import *
+from synchronizers.new_base.syncstep import *
from xos.logger import observer_logger as logger
+from synchronizers.new_base.modelaccessor import *
def escape(s):
s = s.replace('\n',r'\n').replace('"',r'\"')
@@ -105,7 +102,7 @@
if tag.name.startswith("sysctl-"):
metadata_update[tag.name] = tag.value
- slice_memberships = SlicePrivilege.objects.filter(slice=instance.slice)
+ slice_memberships = SlicePrivilege.objects.filter(slice_id=instance.slice.id)
pubkeys = set([sm.user.public_key for sm in slice_memberships if sm.user.public_key])
if instance.creator.public_key:
pubkeys.add(instance.creator.public_key)
@@ -120,23 +117,25 @@
# handle ports the were created by the user
port_ids=[]
- for port in Port.objects.filter(instance=instance):
+ for port in Port.objects.filter(instance_id=instance.id):
if not port.port_id:
raise DeferredException("Instance %s waiting on port %s" % (instance, port))
nics.append({"kind": "port", "value": port.port_id, "network": port.network})
# we want to exclude from 'nics' any network that already has a Port
- existing_port_networks = [port.network for port in Port.objects.filter(instance=instance)]
+ existing_port_networks = [port.network for port in Port.objects.filter(instance_id=instance.id)]
+ existing_port_network_ids = [x.id for x in existing_port_networks]
- networks = [ns.network for ns in NetworkSlice.objects.filter(slice=instance.slice) if ns.network not in existing_port_networks]
- controller_networks = ControllerNetwork.objects.filter(network__in=networks,
- controller=instance.node.site_deployment.controller)
+ networks = [ns.network for ns in NetworkSlice.objects.filter(slice_id=instance.slice.id) if ns.network.id not in existing_port_network_ids]
+ networks_ids = [x.id for x in networks]
+ controller_networks = ControllerNetwork.objects.filter(controller_id=instance.node.site_deployment.controller.id)
+ controller_networks = [x for x in controller_networks if x.id in networks_ids]
+
for network in networks:
- if not ControllerNetwork.objects.filter(network=network, controller=instance.node.site_deployment.controller).exists():
+ if not ControllerNetwork.objects.filter(network_id=network.id, controller_id=instance.node.site_deployment.controller.id).exists():
raise DeferredException("Instance %s Private Network %s lacks ControllerNetwork object" % (instance, network.name))
- #controller_networks = self.sort_controller_networks(controller_networks)
for controller_network in controller_networks:
# Lenient exception - causes slow backoff
if controller_network.network.template.translation == 'none':
@@ -148,7 +147,6 @@
network_templates = [network.template.shared_network_name for network in networks \
if network.template.shared_network_name]
- #driver = self.driver.client_driver(caller=instance.creator, tenant=instance.slice.name, controller=instance.controllerNetwork)
driver = self.driver.admin_driver(tenant='admin', controller=instance.node.site_deployment.controller)
nets = driver.shell.neutron.list_networks()['networks']
for net in nets:
@@ -163,7 +161,8 @@
nics = self.sort_nics(nics)
image_name = None
- controller_images = instance.image.controllerimages.filter(controller=instance.node.site_deployment.controller)
+ controller_images = instance.image.controllerimages.all()
+ controller_images = [x for x in controller_images if x.controller_id==instance.node.site_deployment.controller.id]
if controller_images:
image_name = controller_images[0].image.name
logger.info("using image from ControllerImage object: " + str(image_name))
diff --git a/xos/synchronizer/steps/sync_object.py b/xos/synchronizer/steps/sync_object.py
deleted file mode 100644
index e03bf92..0000000
--- a/xos/synchronizer/steps/sync_object.py
+++ /dev/null
@@ -1,19 +0,0 @@
-import os
-import base64
-from collections import defaultdict
-from django.db.models import F, Q
-from xos.config import Config
-from synchronizers.openstack.openstacksyncstep import OpenStackSyncStep
-from synchronizers.base.syncstep import *
-from core.models import *
-from synchronizers.base.ansible_helper import *
-from xos.logger import observer_logger as logger
-import json
-
-class SyncObject(OpenStackSyncStep):
- provides=[] # Caller fills this in
- requested_interval=0
- observes=[] # Caller fills this in
-
- def sync_record(self, r):
- raise DeferredException('Waiting for Service dependency: %r'%r)
diff --git a/xos/synchronizer/steps/sync_ports.py b/xos/synchronizer/steps/sync_ports.py
index b0f1113..d6f1303 100644
--- a/xos/synchronizer/steps/sync_ports.py
+++ b/xos/synchronizer/steps/sync_ports.py
@@ -1,11 +1,9 @@
import os
import base64
-from django.db.models import F, Q
from xos.config import Config
from synchronizers.openstack.openstacksyncstep import OpenStackSyncStep
-from core.models import Controller
-from core.models.network import *
from xos.logger import observer_logger as logger
+from synchronizers.new_base.modelaccessor import *
class SyncPorts(OpenStackSyncStep):
requested_interval = 0 # 3600
@@ -170,14 +168,17 @@
continue
# For ports that were created by the user, find that ones
- # that don't have neutron ports, and create them.
- for port in Port.objects.filter(Q(port_id__isnull=True), Q(instance__isnull=False) ):
+ # that don't have neutron ports, and create them. These are ports
+ # with a null port_id and a non-null instance_id.
+ ports = Port.objects.all()
+ ports = [x for x in ports if ((not x.port_id) and (x.instance_id))]
+ for port in ports:
logger.info("XXX working on port %s" % port)
controller = port.instance.node.site_deployment.controller
slice = port.instance.slice
if controller:
- cn=port.network.controllernetworks.filter(controller=controller)
+ cn=[x for x in port.network.controllernetworks.all() if x.controller_id==controller.id]
if not cn:
logger.log_exc("no controllernetwork for %s" % port)
continue
diff --git a/xos/synchronizer/steps/sync_roles.py b/xos/synchronizer/steps/sync_roles.py
index 418b4ea..c202f74 100644
--- a/xos/synchronizer/steps/sync_roles.py
+++ b/xos/synchronizer/steps/sync_roles.py
@@ -1,12 +1,9 @@
import os
import base64
-from django.db.models import F, Q
from xos.config import Config
from synchronizers.openstack.openstacksyncstep import OpenStackSyncStep
-from core.models.role import Role
-from core.models.site import SiteRole, Controller, ControllerRole
-from core.models.slice import SliceRole
from xos.logger import observer_logger as logger
+from synchronizers.new_base.modelaccessor import *
class SyncRoles(OpenStackSyncStep):
provides=[Role]