CORD-3048 Unit Tests for Kubernetes Synchronizer

Change-Id: I0ff9146d544a2e0a212264b7d366500d6a51ff1c
diff --git a/xos/synchronizer/steps/sync_configmap.py b/xos/synchronizer/steps/sync_configmap.py
index e69a392..ea5c8e2 100644
--- a/xos/synchronizer/steps/sync_configmap.py
+++ b/xos/synchronizer/steps/sync_configmap.py
@@ -26,9 +26,6 @@
 from xosconfig import Config
 from multistructlog import create_logger
 
-from kubernetes.client.rest import ApiException
-from kubernetes import client as kubernetes_client, config as kubernetes_config
-
 log = create_logger(Config().get('logging'))
 
 class SyncKubernetesConfigMap(SyncStep):
@@ -45,16 +42,23 @@
 
     def __init__(self, *args, **kwargs):
         super(SyncKubernetesConfigMap, self).__init__(*args, **kwargs)
+        self.init_kubernetes_client()
+
+    def init_kubernetes_client(self):
+        from kubernetes import client as kubernetes_client, config as kubernetes_config
+        from kubernetes.client.rest import ApiException
         kubernetes_config.load_incluster_config()
-        self.v1 = kubernetes_client.CoreV1Api()
+        self.kubernetes_client = kubernetes_client
+        self.v1core = kubernetes_client.CoreV1Api()
+        self.ApiException = ApiException
 
     def get_config_map(self, o):
         """ Given an XOS KubernetesConfigMap object, read the corresponding ConfigMap from Kubernetes.
             return None if no ConfigMap exists.
         """
         try:
-            config_map = self.v1.read_namespaced_config_map(o.name, o.trust_domain.name)
-        except ApiException, e:
+            config_map = self.v1core.read_namespaced_config_map(o.name, o.trust_domain.name)
+        except self.ApiException, e:
             if e.status == 404:
                 return None
             raise
@@ -63,14 +67,14 @@
     def sync_record(self, o):
             config_map = self.get_config_map(o)
             if not config_map:
-                config_map = kubernetes_client.V1ConfigMap()
+                config_map = self.kubernetes_client.V1ConfigMap()
                 config_map.data = json.loads(o.data)
-                config_map.metadata = kubernetes_client.V1ObjectMeta(name=o.name)
+                config_map.metadata = self.kubernetes_client.V1ObjectMeta(name=o.name)
 
-                config_map = self.v1.create_namespaced_config_map(o.trust_domain.name, config_map)
+                config_map = self.v1core.create_namespaced_config_map(o.trust_domain.name, config_map)
             else:
                 config_map.data = json.loads(o.data)
-                self.v1.patch_namespaced_config_map(o.name, o.trust_domain.name, config_map)
+                self.v1core.patch_namespaced_config_map(o.name, o.trust_domain.name, config_map)
 
             if (not o.backend_handle):
                 o.backend_handle = config_map.metadata.self_link
diff --git a/xos/synchronizer/steps/sync_kubernetesserviceinstance.py b/xos/synchronizer/steps/sync_kubernetesserviceinstance.py
index 0980896..4f1088a 100644
--- a/xos/synchronizer/steps/sync_kubernetesserviceinstance.py
+++ b/xos/synchronizer/steps/sync_kubernetesserviceinstance.py
@@ -29,9 +29,6 @@
 from xosconfig import Config
 from multistructlog import create_logger
 
-from kubernetes.client.rest import ApiException
-from kubernetes import client as kubernetes_client, config as kubernetes_config
-
 log = create_logger(Config().get('logging'))
 
 class SyncKubernetesServiceInstance(SyncStep):
@@ -48,24 +45,31 @@
 
     def __init__(self, *args, **kwargs):
         super(SyncKubernetesServiceInstance, self).__init__(*args, **kwargs)
+        self.init_kubernetes_client()
+
+    def init_kubernetes_client(self):
+        from kubernetes.client.rest import ApiException
+        from kubernetes import client as kubernetes_client, config as kubernetes_config
         kubernetes_config.load_incluster_config()
-        self.v1 = kubernetes_client.CoreV1Api()
+        self.kubernetes_client = kubernetes_client
+        self.v1core = kubernetes_client.CoreV1Api()
+        self.ApiException = ApiException
 
     def get_pod(self, o):
         """ Given a KubernetesServiceInstance, read the pod from Kubernetes.
             Return None if the pod does not exist.
         """
         try:
-            pod = self.v1.read_namespaced_pod(o.name, o.slice.trust_domain.name)
-        except ApiException, e:
+            pod = self.v1core.read_namespaced_pod(o.name, o.slice.trust_domain.name)
+        except self.ApiException, e:
             if e.status == 404:
                 return None
             raise
         return pod
 
     def generate_pod_spec(self, o):
-        pod = kubernetes_client.V1Pod()
-        pod.metadata = kubernetes_client.V1ObjectMeta(name=o.name)
+        pod = self.kubernetes_client.V1Pod()
+        pod.metadata = self.kubernetes_client.V1ObjectMeta(name=o.name)
 
         if o.slice.trust_domain:
             pod.metadata.namespace = o.slice.trust_domain.name
@@ -81,31 +85,31 @@
 
         # Attach and mount the configmaps
         for xos_vol in o.kubernetes_config_volume_mounts.all():
-            k8s_vol = kubernetes_client.V1Volume(name=xos_vol.config.name)
-            k8s_vol.config_map = kubernetes_client.V1ConfigMapVolumeSource(name=xos_vol.config.name)
+            k8s_vol = self.kubernetes_client.V1Volume(name=xos_vol.config.name)
+            k8s_vol.config_map = self.kubernetes_client.V1ConfigMapVolumeSource(name=xos_vol.config.name)
             volumes.append(k8s_vol)
 
-            k8s_vol_m = kubernetes_client.V1VolumeMount(name=xos_vol.config.name,
+            k8s_vol_m = self.kubernetes_client.V1VolumeMount(name=xos_vol.config.name,
                                                         mount_path=xos_vol.mount_path,
                                                         sub_path=xos_vol.sub_path)
             volume_mounts.append(k8s_vol_m)
 
         # Attach and mount the secrets
         for xos_vol in o.kubernetes_secret_volume_mounts.all():
-            k8s_vol = kubernetes_client.V1Volume(name=xos_vol.secret.name)
-            k8s_vol.secret = kubernetes_client.V1SecretVolumeSource(secret_name=xos_vol.secret.name)
+            k8s_vol = self.kubernetes_client.V1Volume(name=xos_vol.secret.name)
+            k8s_vol.secret = self.kubernetes_client.V1SecretVolumeSource(secret_name=xos_vol.secret.name)
             volumes.append(k8s_vol)
 
-            k8s_vol_m = kubernetes_client.V1VolumeMount(name=xos_vol.secret.name,
+            k8s_vol_m = self.kubernetes_client.V1VolumeMount(name=xos_vol.secret.name,
                                                         mount_path=xos_vol.mount_path,
                                                         sub_path=xos_vol.sub_path)
             volume_mounts.append(k8s_vol_m)
 
-        container = kubernetes_client.V1Container(name=o.name,
+        container = self.kubernetes_client.V1Container(name=o.name,
                                                   image=imageName,
                                                   volume_mounts=volume_mounts)
 
-        spec = kubernetes_client.V1PodSpec(containers=[container], volumes=volumes)
+        spec = self.kubernetes_client.V1PodSpec(containers=[container], volumes=volumes)
         pod.spec = spec
 
         if o.slice.principal:
@@ -127,7 +131,7 @@
 
                 log.info("Creating pod", o=o, pod=pod)
 
-                pod = self.v1.create_namespaced_pod(o.slice.trust_domain.name, pod)
+                pod = self.v1core.create_namespaced_pod(o.slice.trust_domain.name, pod)
             else:
                 log.info("Replacing pod", o=o, pod=pod)
 
@@ -137,7 +141,7 @@
                 # If we don't apply any changes to the pod, it's still the case that Kubernetes will pull in new
                 # mounts of existing configmaps during the replace operation, if the configmap contents have changed.
 
-                pod = self.v1.replace_namespaced_pod(o.name, o.slice.trust_domain.name, pod)
+                pod = self.v1core.replace_namespaced_pod(o.name, o.slice.trust_domain.name, pod)
 
             if (not o.backend_handle):
                 o.backend_handle = pod.metadata.self_link
diff --git a/xos/synchronizer/steps/sync_principal.py b/xos/synchronizer/steps/sync_principal.py
index d3e61af..3806888 100644
--- a/xos/synchronizer/steps/sync_principal.py
+++ b/xos/synchronizer/steps/sync_principal.py
@@ -25,9 +25,6 @@
 from xosconfig import Config
 from multistructlog import create_logger
 
-from kubernetes.client.rest import ApiException
-from kubernetes import client as kubernetes_client, config as kubernetes_config
-
 log = create_logger(Config().get('logging'))
 
 class SyncPrincipal(SyncStep):
@@ -44,16 +41,23 @@
 
     def __init__(self, *args, **kwargs):
         super(SyncPrincipal, self).__init__(*args, **kwargs)
+        self.init_kubernetes_client()
+
+    def init_kubernetes_client(self):
+        from kubernetes.client.rest import ApiException
+        from kubernetes import client as kubernetes_client, config as kubernetes_config
         kubernetes_config.load_incluster_config()
-        self.v1 = kubernetes_client.CoreV1Api()
+        self.kubernetes_client = kubernetes_client
+        self.v1core = kubernetes_client.CoreV1Api()
+        self.ApiException = ApiException
 
     def get_service_account(self, o):
         """ Given an XOS Principal object, read the corresponding ServiceAccount from Kubernetes.
             return None if no ServiceAccount exists.
         """
         try:
-            service_account = self.v1.read_namespaced_service_account(o.name, o.trust_domain.name)
-        except ApiException, e:
+            service_account = self.v1core.read_namespaced_service_account(o.name, o.trust_domain.name)
+        except self.ApiException, e:
             if e.status == 404:
                 return None
             raise
@@ -79,10 +83,10 @@
     def sync_record(self, o):
             service_account = self.get_service_account(o)
             if not service_account:
-                service_account = kubernetes_client.V1ServiceAccount()
-                service_account.metadata = kubernetes_client.V1ObjectMeta(name=o.name)
+                service_account = self.kubernetes_client.V1ServiceAccount()
+                service_account.metadata = self.kubernetes_client.V1ObjectMeta(name=o.name)
 
-                service_account = self.v1.create_namespaced_service_account(o.trust_domain.name, service_account)
+                service_account = self.v1core.create_namespaced_service_account(o.trust_domain.name, service_account)
 
             if (not o.backend_handle):
                 o.backend_handle = service_account.metadata.self_link
diff --git a/xos/synchronizer/steps/sync_secret.py b/xos/synchronizer/steps/sync_secret.py
index 903be4f..a020b43 100644
--- a/xos/synchronizer/steps/sync_secret.py
+++ b/xos/synchronizer/steps/sync_secret.py
@@ -26,9 +26,6 @@
 from xosconfig import Config
 from multistructlog import create_logger
 
-from kubernetes.client.rest import ApiException
-from kubernetes import client as kubernetes_client, config as kubernetes_config
-
 log = create_logger(Config().get('logging'))
 
 class SyncKubernetesSecret(SyncStep):
@@ -45,16 +42,23 @@
 
     def __init__(self, *args, **kwargs):
         super(SyncKubernetesSecret, self).__init__(*args, **kwargs)
+        self.init_kubernetes_client()
+
+    def init_kubernetes_client(self):
+        from kubernetes.client.rest import ApiException
+        from kubernetes import client as kubernetes_client, config as kubernetes_config
         kubernetes_config.load_incluster_config()
-        self.v1 = kubernetes_client.CoreV1Api()
+        self.kubernetes_client = kubernetes_client
+        self.v1core = kubernetes_client.CoreV1Api()
+        self.ApiException = ApiException
 
     def get_secret(self, o):
         """ Given an XOS KubernetesSecret object, read the corresponding Secret from Kubernetes.
             return None if no Secret exists.
         """
         try:
-            secret = self.v1.read_namespaced_secret(o.name, o.trust_domain.name)
-        except ApiException, e:
+            secret = self.v1core.read_namespaced_secret(o.name, o.trust_domain.name)
+        except self.ApiException, e:
             if e.status == 404:
                 return None
             raise
@@ -63,14 +67,14 @@
     def sync_record(self, o):
             secret = self.get_secret(o)
             if not secret:
-                secret = kubernetes_client.V1Secret()
+                secret = self.kubernetes_client.V1Secret()
                 secret.data = json.loads(o.data)
-                secret.metadata = kubernetes_client.V1ObjectMeta(name=o.name)
+                secret.metadata = self.kubernetes_client.V1ObjectMeta(name=o.name)
 
-                secret = self.v1.create_namespaced_secret(o.trust_domain.name, secret)
+                secret = self.v1core.create_namespaced_secret(o.trust_domain.name, secret)
             else:
                 secret.data = json.loads(o.data)
-                self.v1.patch_namespaced_secret(o.name, o.trust_domain.name, secret)
+                self.v1core.patch_namespaced_secret(o.name, o.trust_domain.name, secret)
 
             if (not o.backend_handle):
                 o.backend_handle = secret.metadata.self_link
diff --git a/xos/synchronizer/steps/sync_service.py b/xos/synchronizer/steps/sync_service.py
index 333f675..2fe88f4 100644
--- a/xos/synchronizer/steps/sync_service.py
+++ b/xos/synchronizer/steps/sync_service.py
@@ -26,9 +26,6 @@
 from xosconfig import Config
 from multistructlog import create_logger
 
-from kubernetes.client.rest import ApiException
-from kubernetes import client as kubernetes_client, config as kubernetes_config
-
 log = create_logger(Config().get('logging'))
 
 class SyncService(SyncStep):
@@ -45,8 +42,15 @@
 
     def __init__(self, *args, **kwargs):
         super(SyncService, self).__init__(*args, **kwargs)
+        self.init_kubernetes_client()
+
+    def init_kubernetes_client(self):
+        from kubernetes.client.rest import ApiException
+        from kubernetes import client as kubernetes_client, config as kubernetes_config
         kubernetes_config.load_incluster_config()
-        self.v1 = kubernetes_client.CoreV1Api()
+        self.kubernetes_client = kubernetes_client
+        self.v1core = kubernetes_client.CoreV1Api()
+        self.ApiException = ApiException
 
     def fetch_pending(self, deletion=False):
         """ Filter the set of pending objects.
@@ -95,8 +99,8 @@
             If no Kubernetes service exists, return None
         """
         try:
-            k8s_service = self.v1.read_namespaced_service(o.name, trust_domain.name)
-        except ApiException, e:
+            k8s_service = self.v1core.read_namespaced_service(o.name, trust_domain.name)
+        except self.ApiException, e:
             if e.status == 404:
                 return None
             raise
@@ -107,22 +111,26 @@
         k8s_service = self.get_service(o,trust_domain)
 
         if not k8s_service:
-            k8s_service = kubernetes_client.V1Service()
-            k8s_service.metadata = kubernetes_client.V1ObjectMeta(name=o.name)
+            k8s_service = self.kubernetes_client.V1Service()
+            k8s_service.metadata = self.kubernetes_client.V1ObjectMeta(name=o.name)
 
             ports=[]
             for service_port in o.serviceports.all():
-                port=kubernetes_client.V1ServicePort(name = service_port.name,
+                port=self.kubernetes_client.V1ServicePort(name = service_port.name,
                                                   node_port = service_port.external_port,
                                                   port = service_port.internal_port,
                                                   target_port = service_port.internal_port,
                                                   protocol = service_port.protocol)
                 ports.append(port)
 
-            k8s_service.spec = kubernetes_client.V1ServiceSpec(ports=ports,
+            k8s_service.spec = self.kubernetes_client.V1ServiceSpec(ports=ports,
                                                                type="NodePort")
 
-            self.v1.create_namespaced_service(trust_domain.name, k8s_service)
+            k8s_service = self.v1core.create_namespaced_service(trust_domain.name, k8s_service)
+
+        if (not o.backend_handle):
+            o.backend_handle = k8s_service.metadata.self_link
+            o.save(update_fields=["backend_handle"])
 
     def delete_record(self, o):
         # TODO(smbaker): Implement delete step
diff --git a/xos/synchronizer/steps/sync_trustdomain.py b/xos/synchronizer/steps/sync_trustdomain.py
index e2d0a1d..90b6afd 100644
--- a/xos/synchronizer/steps/sync_trustdomain.py
+++ b/xos/synchronizer/steps/sync_trustdomain.py
@@ -25,9 +25,6 @@
 from xosconfig import Config
 from multistructlog import create_logger
 
-from kubernetes.client.rest import ApiException
-from kubernetes import client as kubernetes_client, config as kubernetes_config
-
 log = create_logger(Config().get('logging'))
 
 class SyncTrustDomain(SyncStep):
@@ -44,8 +41,15 @@
 
     def __init__(self, *args, **kwargs):
         super(SyncTrustDomain, self).__init__(*args, **kwargs)
+        self.init_kubernetes_client()
+
+    def init_kubernetes_client(self):
+        from kubernetes.client.rest import ApiException
+        from kubernetes import client as kubernetes_client, config as kubernetes_config
         kubernetes_config.load_incluster_config()
-        self.v1 = kubernetes_client.CoreV1Api()
+        self.kubernetes_client = kubernetes_client
+        self.v1core = kubernetes_client.CoreV1Api()
+        self.ApiException = ApiException
 
     def fetch_pending(self, deleted):
         """ Figure out which TrustDomains are interesting to the K8s synchronizer. It's necessary to filter as we're
@@ -64,8 +68,8 @@
             Return None if no namespace exists.
         """
         try:
-            ns = self.v1.read_namespace(o.name)
-        except ApiException, e:
+            ns = self.v1core.read_namespace(o.name)
+        except self.ApiException, e:
             if e.status == 404:
                 return None
             raise
@@ -74,11 +78,11 @@
     def sync_record(self, o):
             ns = self.get_namespace(o)
             if not ns:
-                ns = kubernetes_client.V1Namespace()
-                ns.metadata = kubernetes_client.V1ObjectMeta(name=o.name)
+                ns = self.kubernetes_client.V1Namespace()
+                ns.metadata = self.kubernetes_client.V1ObjectMeta(name=o.name)
 
                 log.info("creating namespace %s" % o.name)
-                ns=self.v1.create_namespace(ns)
+                ns=self.v1core.create_namespace(ns)
 
             if (not o.backend_handle):
                 o.backend_handle = ns.metadata.self_link