SEBA-405 Update openstack synchronizer to use synchronizer library
Change-Id: I9adae42fa7eec94ee496b728c32a7ce5db3c39e0
diff --git a/xos/synchronizer/steps/newopenstacksyncstep.py b/xos/synchronizer/steps/newopenstacksyncstep.py
index 1e16b49..c80ffe7 100644
--- a/xos/synchronizer/steps/newopenstacksyncstep.py
+++ b/xos/synchronizer/steps/newopenstacksyncstep.py
@@ -15,7 +15,7 @@
from distutils.version import LooseVersion
-from synchronizers.new_base.syncstep import SyncStep
+from xossynchronizer.steps.syncstep import SyncStep
class NewOpenStackSyncStep(SyncStep):
""" XOS Sync step for copying data to OpenStack
diff --git a/xos/synchronizer/steps/sync_container.py b/xos/synchronizer/steps/sync_container.py
index 476877d..9da1ce5 100644
--- a/xos/synchronizer/steps/sync_container.py
+++ b/xos/synchronizer/steps/sync_container.py
@@ -20,18 +20,19 @@
import sys
import base64
import time
-from synchronizers.new_base.SyncInstanceUsingAnsible import SyncInstanceUsingAnsible
-from synchronizers.new_base.syncstep import DeferredException
-from synchronizers.new_base.ansible_helper import run_template_ssh
-from xos.logger import Logger, logging
-from synchronizers.new_base.modelaccessor import *
+from xossynchronizer.steps.SyncInstanceUsingAnsible import SyncInstanceUsingAnsible
+from xossynchronizer.steps.syncstep import DeferredException
+from xossynchronizer.ansible_helper import run_template_ssh
+from xossynchronizer.modelaccessor import *
+from xosconfig import Config
+from multistructlog import create_logger
+
+log = create_logger(Config().get('logging'))
# hpclibrary will be in steps/..
parentdir = os.path.join(os.path.dirname(__file__),"..")
sys.path.insert(0,parentdir)
-logger = Logger(level=logging.INFO)
-
class SyncContainer(SyncInstanceUsingAnsible):
provides=[Instance]
observes=Instance
@@ -132,7 +133,7 @@
return fields
def sync_record(self, o):
- logger.info("sync'ing object %s" % str(o),extra=o.tologdict())
+ log.info("sync'ing object %s" % str(o),extra=o.tologdict())
fields = self.get_ansible_fields(o)
@@ -152,7 +153,7 @@
o.save()
def delete_record(self, o):
- logger.info("delete'ing object %s" % str(o),extra=o.tologdict())
+ log.info("delete'ing object %s" % str(o),extra=o.tologdict())
fields = self.get_ansible_fields(o)
@@ -171,6 +172,6 @@
template_name = self.template_name
tStart = time.time()
run_template_ssh(template_name, fields, path="container", object=o)
- logger.info("playbook execution time %d" % int(time.time()-tStart),extra=o.tologdict())
+ log.info("playbook execution time %d" % int(time.time()-tStart),extra=o.tologdict())
diff --git a/xos/synchronizer/steps/sync_controller_images.py b/xos/synchronizer/steps/sync_controller_images.py
index 67c5a94..e85a94d 100644
--- a/xos/synchronizer/steps/sync_controller_images.py
+++ b/xos/synchronizer/steps/sync_controller_images.py
@@ -12,13 +12,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import os
import urlparse
-import base64
-from synchronizers.openstack.openstacksyncstep import OpenStackSyncStep
-from synchronizers.new_base.syncstep import *
-from synchronizers.new_base.ansible_helper import *
-from synchronizers.new_base.modelaccessor import *
+from openstacksyncstep import OpenStackSyncStep
+from xossynchronizer.modelaccessor import *
+from xosconfig import Config
+from multistructlog import create_logger
+
+log = create_logger(Config().get('logging'))
class SyncControllerImages(OpenStackSyncStep):
provides=[ControllerImages]
diff --git a/xos/synchronizer/steps/sync_controller_networks.py b/xos/synchronizer/steps/sync_controller_networks.py
index d33962a..e5c080e 100644
--- a/xos/synchronizer/steps/sync_controller_networks.py
+++ b/xos/synchronizer/steps/sync_controller_networks.py
@@ -19,11 +19,12 @@
import struct
import socket
from netaddr import IPAddress, IPNetwork
-from synchronizers.openstack.openstacksyncstep import OpenStackSyncStep
-from synchronizers.new_base.syncstep import *
-from xos.logger import observer_logger as logger
-from synchronizers.new_base.ansible_helper import *
-from synchronizers.new_base.modelaccessor import *
+from openstacksyncstep import OpenStackSyncStep
+from xossynchronizer.modelaccessor import *
+from xosconfig import Config
+from multistructlog import create_logger
+
+log = create_logger(Config().get('logging'))
class SyncControllerNetworks(OpenStackSyncStep):
requested_interval = 0
@@ -139,7 +140,7 @@
return SyncStep.SYNC_WITHOUT_RUNNING
if not controller_network.controller.admin_user:
- logger.info("controller %r has no admin_user, skipping" % controller_network.controller)
+ log.info("controller %r has no admin_user, skipping" % controller_network.controller)
return
if controller_network.network.owner and controller_network.network.owner.creator:
diff --git a/xos/synchronizer/steps/sync_controller_site_privileges.py b/xos/synchronizer/steps/sync_controller_site_privileges.py
index 72e5b04..bfe436d 100644
--- a/xos/synchronizer/steps/sync_controller_site_privileges.py
+++ b/xos/synchronizer/steps/sync_controller_site_privileges.py
@@ -17,11 +17,12 @@
import os
import base64
import json
-from synchronizers.openstack.openstacksyncstep import OpenStackSyncStep
-from synchronizers.new_base.syncstep import *
-from xos.logger import observer_logger as logger
-from synchronizers.new_base.ansible_helper import *
-from synchronizers.new_base.modelaccessor import *
+from openstacksyncstep import OpenStackSyncStep
+from xossynchronizer.modelaccessor import *
+from xosconfig import Config
+from multistructlog import create_logger
+
+log = create_logger(Config().get('logging'))
class SyncControllerSitePrivileges(OpenStackSyncStep):
provides=[SitePrivilege]
diff --git a/xos/synchronizer/steps/sync_controller_sites.py b/xos/synchronizer/steps/sync_controller_sites.py
index f12f477..a4c56a5 100644
--- a/xos/synchronizer/steps/sync_controller_sites.py
+++ b/xos/synchronizer/steps/sync_controller_sites.py
@@ -16,11 +16,14 @@
import os
import base64
-from synchronizers.openstack.openstacksyncstep import OpenStackSyncStep
-from synchronizers.new_base.syncstep import *
-from synchronizers.new_base.ansible_helper import *
import json
-from synchronizers.new_base.modelaccessor import *
+from openstacksyncstep import OpenStackSyncStep
+from xossynchronizer.modelaccessor import *
+from xosconfig import Config
+from multistructlog import create_logger
+
+log = create_logger(Config().get('logging'))
+
class SyncControllerSites(OpenStackSyncStep):
requested_interval=0
diff --git a/xos/synchronizer/steps/sync_controller_slice_privileges.py b/xos/synchronizer/steps/sync_controller_slice_privileges.py
index e598e1c..5e8466e 100644
--- a/xos/synchronizer/steps/sync_controller_slice_privileges.py
+++ b/xos/synchronizer/steps/sync_controller_slice_privileges.py
@@ -17,11 +17,13 @@
import os
import base64
import json
-from synchronizers.openstack.openstacksyncstep import OpenStackSyncStep
-from synchronizers.new_base.syncstep import *
-from synchronizers.new_base.ansible_helper import *
-from xos.logger import observer_logger as logger
-from synchronizers.new_base.modelaccessor import *
+from openstacksyncstep import OpenStackSyncStep
+from xossynchronizer.modelaccessor import *
+from xosconfig import Config
+from multistructlog import create_logger
+
+log = create_logger(Config().get('logging'))
+
class SyncControllerSlicePrivileges(OpenStackSyncStep):
provides=[SlicePrivilege]
@@ -31,7 +33,7 @@
def map_sync_inputs(self, controller_slice_privilege):
if not controller_slice_privilege.controller.admin_user:
- logger.info("controller %r has no admin_user, skipping" % controller_slice_privilege.controller)
+ log.info("controller %r has no admin_user, skipping" % controller_slice_privilege.controller)
return
template = os_template_env.get_template('sync_controller_users.yaml')
diff --git a/xos/synchronizer/steps/sync_controller_slices.py b/xos/synchronizer/steps/sync_controller_slices.py
index 15a92ed..1481fb0 100644
--- a/xos/synchronizer/steps/sync_controller_slices.py
+++ b/xos/synchronizer/steps/sync_controller_slices.py
@@ -16,11 +16,13 @@
import os
import base64
-from synchronizers.openstack.openstacksyncstep import OpenStackSyncStep
-from synchronizers.new_base.syncstep import *
-from synchronizers.new_base.ansible_helper import *
-from xos.logger import observer_logger as logger
-from synchronizers.new_base.modelaccessor import *
+from openstacksyncstep import OpenStackSyncStep
+from xossynchronizer.modelaccessor import *
+from xosconfig import Config
+from multistructlog import create_logger
+
+log = create_logger(Config().get('logging'))
+
class SyncControllerSlices(OpenStackSyncStep):
provides=[Slice]
@@ -29,10 +31,10 @@
playbook='sync_controller_slices.yaml'
def map_sync_inputs(self, controller_slice):
- logger.info("sync'ing slice controller %s" % controller_slice)
+ log.info("sync'ing slice controller %s" % controller_slice)
if not controller_slice.controller.admin_user:
- logger.info("controller %r has no admin_user, skipping" % controller_slice.controller)
+ log.info("controller %r has no admin_user, skipping" % controller_slice.controller)
return
controller_users = ControllerUser.objects.filter(user_id=controller_slice.slice.creator.id,
@@ -67,7 +69,7 @@
driver = self.driver.admin_driver(controller=controller_slice.controller)
driver.shell.nova.quotas.update(tenant_id=tenant_id, instances=int(controller_slice.slice.max_instances))
except:
- logger.log_exc('Could not update quota for %s'%controller_slice.slice.name)
+ log.exception('Could not update quota for %s'%controller_slice.slice.name)
raise Exception('Could not update quota for %s'%controller_slice.slice.name)
controller_slice.tenant_id = tenant_id
diff --git a/xos/synchronizer/steps/sync_controller_users.py b/xos/synchronizer/steps/sync_controller_users.py
index 72f67da..78f2d80 100644
--- a/xos/synchronizer/steps/sync_controller_users.py
+++ b/xos/synchronizer/steps/sync_controller_users.py
@@ -16,11 +16,12 @@
import os
import base64
-from synchronizers.openstack.openstacksyncstep import OpenStackSyncStep
-from synchronizers.new_base.syncstep import *
-from synchronizers.new_base.ansible_helper import *
-from xos.logger import observer_logger as logger
-from synchronizers.new_base.modelaccessor import *
+from openstacksyncstep import OpenStackSyncStep
+from xossynchronizer.modelaccessor import *
+from xosconfig import Config
+from multistructlog import create_logger
+
+log = create_logger(Config().get('logging'))
class SyncControllerUsers(OpenStackSyncStep):
provides=[User]
@@ -30,7 +31,7 @@
def map_sync_inputs(self, controller_user):
if not controller_user.controller.admin_user:
- logger.info("controller %r has no admin_user, skipping" % controller_user.controller)
+ log.info("controller %r has no admin_user, skipping" % controller_user.controller)
return
# All users will have at least the 'user' role at their home site/tenant.
diff --git a/xos/synchronizer/steps/sync_images.py b/xos/synchronizer/steps/sync_images.py
index ffda11d..ad6c6ef 100644
--- a/xos/synchronizer/steps/sync_images.py
+++ b/xos/synchronizer/steps/sync_images.py
@@ -14,11 +14,12 @@
# limitations under the License.
-import os
-import base64
-from synchronizers.openstack.openstacksyncstep import OpenStackSyncStep
-from xos.logger import observer_logger as logger
-from synchronizers.new_base.modelaccessor import *
+from openstacksyncstep import OpenStackSyncStep
+from xossynchronizer.modelaccessor import *
+from xosconfig import Config
+from multistructlog import create_logger
+
+log = create_logger(Config().get('logging'))
class SyncImages(OpenStackSyncStep):
provides=[Image]
diff --git a/xos/synchronizer/steps/sync_instances.py b/xos/synchronizer/steps/sync_instances.py
index 4194f89..ccbcb2c 100644
--- a/xos/synchronizer/steps/sync_instances.py
+++ b/xos/synchronizer/steps/sync_instances.py
@@ -14,14 +14,13 @@
# limitations under the License.
-import os
-import base64
import socket
-from synchronizers.openstack.openstacksyncstep import OpenStackSyncStep
-from synchronizers.new_base.ansible_helper import *
-from synchronizers.new_base.syncstep import *
-from xos.logger import observer_logger as logger
-from synchronizers.new_base.modelaccessor import *
+from openstacksyncstep import OpenStackSyncStep
+from xossynchronizer.modelaccessor import *
+from xosconfig import Config
+from multistructlog import create_logger
+
+log = create_logger(Config().get('logging'))
RESTAPI_HOSTNAME = socket.gethostname()
RESTAPI_PORT = "8000"
@@ -193,7 +192,7 @@
x.controller_id == instance.node.site_deployment.controller.id]
if controller_images:
image_name = controller_images[0].image.name
- logger.info("using image from ControllerImage object: " + str(image_name))
+ log.info("using image from ControllerImage object: " + str(image_name))
if image_name is None:
controller_driver = self.driver.admin_driver(controller=instance.node.site_deployment.controller)
@@ -201,7 +200,7 @@
for image in images:
if image.name == instance.image.name or not image_name:
image_name = image.name
- logger.info("using image from glance: " + str(image_name))
+ log.info("using image from glance: " + str(image_name))
host_filter = instance.node.name.strip()
diff --git a/xos/synchronizer/steps/sync_openstack_service.py b/xos/synchronizer/steps/sync_openstack_service.py
index 79e2df8..829abaa 100644
--- a/xos/synchronizer/steps/sync_openstack_service.py
+++ b/xos/synchronizer/steps/sync_openstack_service.py
@@ -14,7 +14,7 @@
# limitations under the License.
-from synchronizers.new_base.modelaccessor import OpenStackService
+from xossynchronizer.modelaccessor import OpenStackService
from newopenstacksyncstep import NewOpenStackSyncStep
class SyncOpenStackService(NewOpenStackSyncStep):
diff --git a/xos/synchronizer/steps/sync_openstackserviceinstance.py b/xos/synchronizer/steps/sync_openstackserviceinstance.py
index 6f32d38..746cc7f 100644
--- a/xos/synchronizer/steps/sync_openstackserviceinstance.py
+++ b/xos/synchronizer/steps/sync_openstackserviceinstance.py
@@ -17,7 +17,7 @@
import random
import string
-from synchronizers.new_base.modelaccessor import OpenStackServiceInstance, Node, NetworkSlice, Flavor
+from xossynchronizer.modelaccessor import OpenStackServiceInstance, Node, NetworkSlice, Flavor
from newopenstacksyncstep import NewOpenStackSyncStep
from xosconfig import Config
diff --git a/xos/synchronizer/steps/sync_ports.py b/xos/synchronizer/steps/sync_ports.py
index 27ad67e..03b0133 100644
--- a/xos/synchronizer/steps/sync_ports.py
+++ b/xos/synchronizer/steps/sync_ports.py
@@ -13,12 +13,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from openstacksyncstep import OpenStackSyncStep
+from xossynchronizer.modelaccessor import *
+from xosconfig import Config
+from multistructlog import create_logger
-import os
-import base64
-from synchronizers.openstack.openstacksyncstep import OpenStackSyncStep
-from xos.logger import observer_logger as logger
-from synchronizers.new_base.modelaccessor import *
+log = create_logger(Config().get('logging'))
class SyncPorts(OpenStackSyncStep):
requested_interval = 0 # 3600
@@ -57,7 +57,7 @@
return driver
def sync_ports(self):
- logger.info("sync'ing Ports [delete=False]")
+ log.info("sync'ing Ports [delete=False]")
ports = Port.objects.all()
ports_by_id = {}
@@ -72,10 +72,6 @@
for nd in network.controllernetworks.all():
networks_by_id[nd.net_id] = network
- #logger.info("networks_by_id = ")
- #for (network_id, network) in networks_by_id.items():
- # logger.info(" %s: %s" % (network_id, network.name))
-
instances = Instance.objects.all()
instances_by_instance_uuid = {}
for instance in instances:
@@ -87,13 +83,13 @@
templates_by_id = {}
for controller in Controller.objects.all():
if not controller.admin_tenant:
- logger.info("controller %s has no admin_tenant" % controller)
+ log.info("controller %s has no admin_tenant" % controller)
continue
try:
driver = self.driver.admin_driver(controller = controller)
ports = driver.shell.neutron.list_ports()["ports"]
except:
- logger.log_exc("failed to get ports from controller %s" % controller)
+ log.exception("failed to get ports from controller %s" % controller)
continue
for port in ports:
@@ -113,7 +109,6 @@
templates_by_id[network["id"]] = template
for port in ports_by_id.values():
- #logger.info("port %s" % str(port))
if port["id"] in ports_by_neutron_port:
# we already have it
#logger.info("already accounted for port %s" % port["id"])
@@ -126,7 +121,7 @@
instance = instances_by_instance_uuid.get(port['device_id'], None)
if not instance:
- logger.info("no instance for port %s device_id %s" % (port["id"], port['device_id']))
+ log.info("no instance for port %s device_id %s" % (port["id"], port['device_id']))
continue
network = networks_by_id.get(port['network_id'], None)
@@ -140,7 +135,7 @@
if candidate_network.template == template:
network=candidate_network
if not network:
- logger.info("no network for port %s network %s" % (port["id"], port["network_id"]))
+ log.info("no network for port %s network %s" % (port["id"], port["network_id"]))
# we know it's associated with a instance, but we don't know
# which network it is part of.
@@ -155,20 +150,20 @@
network = None
for candidate_network in networks:
if (candidate_network.owner == instance.slice):
- logger.info("found network %s" % candidate_network)
+ log.info("found network %s" % candidate_network)
network = candidate_network
if not network:
- logger.info("failed to find the correct network for a shared template for port %s network %s" % (port["id"], port["network_id"]))
+ log.info("failed to find the correct network for a shared template for port %s network %s" % (port["id"], port["network_id"]))
continue
if not port["fixed_ips"]:
- logger.info("port %s has no fixed_ips" % port["id"])
+ log.info("port %s has no fixed_ips" % port["id"])
continue
ip=port["fixed_ips"][0]["ip_address"]
mac=port["mac_address"]
- logger.info("creating Port (%s, %s, %s, %s)" % (str(network), str(instance), ip, str(port["id"])))
+ log.info("creating Port (%s, %s, %s, %s)" % (str(network), str(instance), ip, str(port["id"])))
ns = Port(network=network,
instance=instance,
@@ -179,7 +174,7 @@
try:
ns.save()
except:
- logger.log_exc("failed to save port %s" % str(ns))
+ log.exception("failed to save port %s" % str(ns))
continue
# For ports that were created by the user, find that ones
@@ -188,23 +183,23 @@
ports = Port.objects.all()
ports = [x for x in ports if ((not x.port_id) and (x.instance_id))]
for port in ports:
- logger.info("XXX working on port %s" % port)
+ log.info("XXX working on port %s" % port)
controller = port.instance.node.site_deployment.controller
slice = port.instance.slice
if controller:
cn=[x for x in port.network.controllernetworks.all() if x.controller_id==controller.id]
if not cn:
- logger.log_exc("no controllernetwork for %s" % port)
+ log.exception("no controllernetwork for %s" % port)
continue
cn=cn[0]
if cn.lazy_blocked:
cn.lazy_blocked=False
cn.save()
- logger.info("deferring port %s because controllerNetwork was lazy-blocked" % port)
+ log.info("deferring port %s because controllerNetwork was lazy-blocked" % port)
continue
if not cn.net_id:
- logger.info("deferring port %s because controllerNetwork does not have a port-id yet" % port)
+ log.info("deferring port %s because controllerNetwork does not have a port-id yet" % port)
continue
try:
driver = self.get_driver(port)
@@ -223,28 +218,28 @@
port.ip = neutron_port["fixed_ips"][0]["ip_address"]
port.mac = neutron_port["mac_address"]
port.xos_created = True
- logger.info("created neutron port %s for %s" % (port.port_id, port))
+ log.info("created neutron port %s for %s" % (port.port_id, port))
except:
- logger.log_exc("failed to create neutron port for %s" % port)
+ log.exception("failed to create neutron port for %s" % port)
continue
port.save()
def delete_ports(self):
- logger.info("sync'ing Ports [delete=True]")
+ log.info("sync'ing Ports [delete=True]")
ports = self.fetch_pending(deletion=True)
for port in ports:
self.delete_record(port)
def delete_record(self, port):
if port.xos_created and port.port_id:
- logger.info("calling openstack to destroy port %s" % port.port_id)
+ log.info("calling openstack to destroy port %s" % port.port_id)
try:
driver = self.get_driver(port)
driver.shell.neutron.delete_port(port.port_id)
except:
- logger.log_exc("failed to delete port %s from neutron" % port.port_id)
+ log.exception("failed to delete port %s from neutron" % port.port_id)
return
- logger.info("Purging port %s" % port)
+ log.info("Purging port %s" % port)
port.delete(purge=True)
diff --git a/xos/synchronizer/steps/sync_principal.py b/xos/synchronizer/steps/sync_principal.py
index 9b401a9..b78923d 100644
--- a/xos/synchronizer/steps/sync_principal.py
+++ b/xos/synchronizer/steps/sync_principal.py
@@ -14,7 +14,7 @@
# limitations under the License.
-from synchronizers.new_base.modelaccessor import TrustDomain, Principal
+from xossynchronizer.modelaccessor import TrustDomain, Principal
from newopenstacksyncstep import NewOpenStackSyncStep
from xosconfig import Config
diff --git a/xos/synchronizer/steps/sync_roles.py b/xos/synchronizer/steps/sync_roles.py
index 8f2d09e..c197159 100644
--- a/xos/synchronizer/steps/sync_roles.py
+++ b/xos/synchronizer/steps/sync_roles.py
@@ -13,12 +13,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from openstacksyncstep import OpenStackSyncStep
+from xossynchronizer.modelaccessor import *
+from xosconfig import Config
+from multistructlog import create_logger
-import os
-import base64
-from synchronizers.openstack.openstacksyncstep import OpenStackSyncStep
-from xos.logger import observer_logger as logger
-from synchronizers.new_base.modelaccessor import *
+log = create_logger(Config().get('logging'))
class SyncRoles(OpenStackSyncStep):
provides=[Role]
@@ -28,7 +28,7 @@
def sync_record(self, role):
if not role.enacted:
controllers = Controller.objects.all()
- for controller in controllers:
+ for controller in controllers:
driver = self.driver.admin_driver(controller=controller)
driver.create_role(role.role)
role.save()
diff --git a/xos/synchronizer/steps/sync_slice.py b/xos/synchronizer/steps/sync_slice.py
index 81ca6dc..ad8a253 100644
--- a/xos/synchronizer/steps/sync_slice.py
+++ b/xos/synchronizer/steps/sync_slice.py
@@ -14,8 +14,7 @@
# limitations under the License.
-from synchronizers.new_base.ansible_helper import *
-from synchronizers.new_base.modelaccessor import Slice
+from xossynchronizer.modelaccessor import Slice
from newopenstacksyncstep import NewOpenStackSyncStep
from xosconfig import Config
diff --git a/xos/synchronizer/steps/sync_trustdomain.py b/xos/synchronizer/steps/sync_trustdomain.py
index 67bf752..1fe66ce 100644
--- a/xos/synchronizer/steps/sync_trustdomain.py
+++ b/xos/synchronizer/steps/sync_trustdomain.py
@@ -14,8 +14,7 @@
# limitations under the License.
-from synchronizers.new_base.ansible_helper import *
-from synchronizers.new_base.modelaccessor import TrustDomain
+from xossynchronizer.modelaccessor import TrustDomain
from newopenstacksyncstep import NewOpenStackSyncStep
from xosconfig import Config