[CORD-2516] Implement HSS-DB service and its synchronizer

Change-Id: I3b5185669bf8a4506d4d60b8bc858537f41c640a
diff --git a/xos/hssdb-onboard.yaml b/xos/hssdb-onboard.yaml
new file mode 100644
index 0000000..4f50131
--- /dev/null
+++ b/xos/hssdb-onboard.yaml
@@ -0,0 +1,34 @@
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+tosca_definitions_version: tosca_simple_yaml_1_0
+
+description: Onboard the HSS DB
+
+imports:
+   - custom_types/xos.yaml
+
+topology_template:
+  node_templates:
+    servicecontroller#hssdb:
+      type: tosca.nodes.ServiceController
+      properties:
+          base_url: file:///opt/xos_services/hss_db/xos/
+          # The following will concatenate with base_url automatically, if
+          # base_url is non-null.
+          xproto: ./
+          synchronizer: synchronizer/manifest
+          synchronizer_run: hssdb-synchronizer.py
+          private_key: file:///opt/xos/key_import/mcord_rsa
+          public_key: file:///opt/xos/key_import/mcord_rsa.pub
\ No newline at end of file
diff --git a/xos/hssdb.xproto b/xos/hssdb.xproto
new file mode 100644
index 0000000..f6a4429
--- /dev/null
+++ b/xos/hssdb.xproto
@@ -0,0 +1,18 @@
+option name = "hssdb";
+option app_label = "hssdb";
+
+message HSSDBService (Service){
+    option verbose_name = "HSS Database Service";
+}
+
+message HSSDBVendor (XOSBase){
+    option verbose_name = "HSS Database Vendor";
+    required string name = 1 [help_text = "vendor name", max_length = 32, null = False, db_index = False, blank = False];
+    required manytoone image->Image:+ = 2 [help_text = "select image for this vendor", db_index = True, null = False, blank = False];
+    required manytoone flavor->Flavor:+ = 3 [help_text = "select openstack flavor for vendor image", db_index = True, null = False, blank = False];
+}
+
+message HSSDBServiceInstance (TenantWithContainer){
+    option verbose_name = "HSS Database ServiceInstance";
+    optional manytoone hssdb_vendor->HSSDBVendor:vendor_tenants = 1 [help_text = "select vendor of choice, leave blank for slice default", db_index = True, null = True, blank = True];
+}
\ No newline at end of file
diff --git a/xos/make_synchronizer_manifest.sh b/xos/make_synchronizer_manifest.sh
new file mode 100644
index 0000000..6e04ec2
--- /dev/null
+++ b/xos/make_synchronizer_manifest.sh
@@ -0,0 +1,16 @@
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#! /bin/bash
+find synchronizer -type f | cut -b 14- > synchronizer/manifest
\ No newline at end of file
diff --git a/xos/synchronizer/Dockerfile.synchronizer b/xos/synchronizer/Dockerfile.synchronizer
new file mode 100644
index 0000000..66c99ce
--- /dev/null
+++ b/xos/synchronizer/Dockerfile.synchronizer
@@ -0,0 +1,53 @@
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# xosproject/hssdb-synchronizer
+FROM xosproject/xos-synchronizer-base:candidate
+
+COPY . /opt/xos/synchronizers/hssdb
+
+ENTRYPOINT []
+
+WORKDIR "/opt/xos/synchronizers/hssdb"
+
+# Label image
+ARG org_label_schema_schema_version=1.0
+ARG org_label_schema_name=hssdb-synchronizer
+ARG org_label_schema_version=unknown
+ARG org_label_schema_vcs_url=unknown
+ARG org_label_schema_vcs_ref=unknown
+ARG org_label_schema_build_date=unknown
+ARG org_opencord_vcs_commit_date=unknown
+ARG org_opencord_component_chameleon_version=unknown
+ARG org_opencord_component_chameleon_vcs_url=unknown
+ARG org_opencord_component_chameleon_vcs_ref=unknown
+ARG org_opencord_component_xos_version=unknown
+ARG org_opencord_component_xos_vcs_url=unknown
+ARG org_opencord_component_xos_vcs_ref=unknown
+
+LABEL org.label-schema.schema-version=$org_label_schema_schema_version \
+      org.label-schema.name=$org_label_schema_name \
+      org.label-schema.version=$org_label_schema_version \
+      org.label-schema.vcs-url=$org_label_schema_vcs_url \
+      org.label-schema.vcs-ref=$org_label_schema_vcs_ref \
+      org.label-schema.build-date=$org_label_schema_build_date \
+      org.opencord.vcs-commit-date=$org_opencord_vcs_commit_date \
+      org.opencord.component.chameleon.version=$org_opencord_component_chameleon_version \
+      org.opencord.component.chameleon.vcs-url=$org_opencord_component_chameleon_vcs_url \
+      org.opencord.component.chameleon.vcs-ref=$org_opencord_component_chameleon_vcs_ref \
+      org.opencord.component.xos.version=$org_opencord_component_xos_version \
+      org.opencord.component.xos.vcs-url=$org_opencord_component_xos_vcs_url \
+      org.opencord.component.xos.vcs-ref=$org_opencord_component_xos_vcs_ref
+
+CMD bash -c "cd /opt/xos/synchronizers/hssdb; ./run-from-api.sh"
\ No newline at end of file
diff --git a/xos/synchronizer/hssdb-synchronizer.py b/xos/synchronizer/hssdb-synchronizer.py
new file mode 100644
index 0000000..f98bd92
--- /dev/null
+++ b/xos/synchronizer/hssdb-synchronizer.py
@@ -0,0 +1,31 @@
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/usr/bin/env python
+
+# This imports and runs ../../xos-observer.py
+
+import importlib
+import os
+import sys
+
+from xosconfig import Config
+config_file = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + '/hssdb_config.yaml')
+
+Config.init(config_file, 'synchronizer-config-schema.yaml')
+observer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../synchronizers/new_base")
+
+sys.path.append(observer_path)
+mod = importlib.import_module("xos-synchronizer")
+mod.main()
diff --git a/xos/synchronizer/hssdb_config.yaml b/xos/synchronizer/hssdb_config.yaml
new file mode 100644
index 0000000..a672282
--- /dev/null
+++ b/xos/synchronizer/hssdb_config.yaml
@@ -0,0 +1,22 @@
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: hssdb-synchronizer
+accessor:
+  username: xosadmin@opencord.org
+  password: "@/opt/xos/services/hssdb/credentials/xosadmin@opencord.org"
+dependency_graph: "/opt/xos/synchronizers/hssdb/model-deps"
+steps_dir: "/opt/xos/synchronizers/hssdb/steps"
+sys_dir: "/opt/xos/synchronizers/hssdb/sys"
+model_policies_dir: "/opt/xos/synchronizers/hssdb/model_policies"
diff --git a/xos/synchronizer/model-deps b/xos/synchronizer/model-deps
new file mode 100644
index 0000000..9e26dfe
--- /dev/null
+++ b/xos/synchronizer/model-deps
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/xos/synchronizer/model_policies/model_policy_hssdbserviceinstance.py b/xos/synchronizer/model_policies/model_policy_hssdbserviceinstance.py
new file mode 100644
index 0000000..de459ac
--- /dev/null
+++ b/xos/synchronizer/model_policies/model_policy_hssdbserviceinstance.py
@@ -0,0 +1,156 @@
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from synchronizers.new_base.modelaccessor import *
+from synchronizers.new_base.model_policies.model_policy_tenantwithcontainer import TenantWithContainerPolicy, LeastLoadedNodeScheduler
+from synchronizers.new_base.exceptions import *
+
+class HSSDBServiceInstancePolicy(TenantWithContainerPolicy):
+    model_name = "HSSDBServiceInstance"
+
+    def handle_create(self, service_instance):
+        return self.handle_update(service_instance)
+
+    def handle_update(self, service_instance):
+        if (service_instance.link_deleted_count>0) and (not service_instance.provided_links.exists()):
+            self.logger.info("The last provided link has been deleted -- self-destructing.")
+            self.handle_delete(service_instance)
+            if HSSDBServiceInstance.objects.filter(id=service_instance.id).exists():
+                service_instance.delete()
+            else:
+                self.logger.info("Tenant %s is already deleted" % service_instance)
+            return
+
+        self.manage_container(service_instance)
+
+    def handle_delete(self, service_instance):
+        if service_instance.instance and (not service_instance.instance.deleted):
+            all_service_instances_this_instance = HSSDBServiceInstance.objects.filter(instance_id=service_instance.instance.id)
+            other_service_instances_this_instance = [x for x in all_service_instances_this_instance if x.id != service_instance.id]
+            if (not other_service_instances_this_instance):
+                self.logger.info("HSSDBServiceInstance %s is now unused -- deleting" % service_instance.instance)
+                self.delete_instance(service_instance, service_instance.instance)
+            else:
+                self.logger.info("HSSDBServiceInstance %s has %d other service instances attached" % (service_instance.instance, len(other_service_instances_this_instance)))
+
+    def get_service(self, service_instance):
+        service_name = service_instance.owner.leaf_model_name
+        service_class = globals()[service_name]
+        return service_class.objects.get(id=service_instance.owner.id)
+
+    def find_instance_for_instance_tag(self, instance_tag):
+        tags = Tag.objects.filter(name="instance_tag", value=instance_tag)
+        if tags:
+            return tags[0].content_object
+        return None
+
+    def find_or_make_instance_for_instance_tag(self, service_instance):
+        instance_tag = self.get_instance_tag(service_instance)
+        instance = self.find_instance_for_instance_tag(instance_tag)
+        if instance:
+            if instance.no_sync:
+                # if no_sync is still set, then perhaps we failed while saving it and need to retry.
+                self.save_instance(service_instance, instance)
+            return instance
+
+        desired_image = self.get_image(service_instance)
+        desired_flavor = self.get_flavor(service_instance)
+
+        slice = service_instance.owner.slices.first()
+
+        (node, parent) = LeastLoadedNodeScheduler(slice, label=None).pick()
+
+        assert (slice is not None)
+        assert (node is not None)
+        assert (desired_image is not None)
+        assert (service_instance.creator is not None)
+        assert (node.site_deployment.deployment is not None)
+        assert (desired_image is not None)
+
+        instance = Instance(slice=slice,
+                            node=node,
+                            image=desired_image,
+                            creator=service_instance.creator,
+                            deployment=node.site_deployment.deployment,
+                            flavor=desired_flavor,
+                            isolation=slice.default_isolation,
+                            parent=parent)
+
+        self.save_instance(service_instance, instance)
+
+        return instance
+
+    def manage_container(self, service_instance):
+        if service_instance.deleted:
+            return
+
+        if service_instance.instance:
+            # We're good.
+            return
+
+        instance = self.find_or_make_instance_for_instance_tag(service_instance)
+        service_instance.instance = instance
+        # TODO: possible for partial failure here?
+        service_instance.save()
+
+    def delete_instance(self, service_instance, instance):
+        # delete the `instance_tag` tags
+        tags = Tag.objects.filter(service_id=service_instance.owner.id, content_type=instance.self_content_type_id,
+                                  object_id=instance.id, name="instance_tag")
+        for tag in tags:
+            tag.delete()
+
+        tags = Tag.objects.filter(content_type=instance.self_content_type_id, object_id=instance.id,
+                                  name="vm_vrouter_tenant")
+        for tag in tags:
+            address_manager_instances = list(ServiceInstance.objects.filter(id=tag.value))
+            tag.delete()
+
+            # TODO: Potential partial failure
+
+            for address_manager_instance in address_manager_instances:
+                self.logger.info("Deleting address_manager_instance %s" % address_manager_instance)
+                address_manager_instance.delete()
+
+        instance.delete()
+
+    def save_instance(self, service_instance, instance):
+        instance.no_sync = True   # prevent instance from being synced until we're done with it
+        super(HSSDBServiceInstancePolicy, self).save_instance(instance)
+
+        try:
+            if instance.isolation in ["container", "container_vm"]:
+                raise Exception("Not supported")
+
+            instance_tag = self.get_instance_tag(service_instance)
+            if instance_tag:
+                tags = Tag.objects.filter(name="instance_tag", value=instance_tag)
+                if not tags:
+                    tag = Tag(service=service_instance.owner, content_type=instance.self_content_type_id, object_id=instance.id, name="instance_tag", value=str(instance_tag))
+                    tag.save()
+
+            instance.no_sync = False   # allow the synchronizer to run now
+            super(HSSDBServiceInstancePolicy, self).save_instance(instance)
+        except:
+            # need to clean up any failures here
+            raise
+
+    def get_instance_tag(self, service_instance):
+        return '%d'%service_instance.id
+
+    def get_image(self, service_instance):
+        return service_instance.hssdb_vendor.image
+
+    def get_flavor(self, service_vendor):
+        return service_vendor.hssdb_vendor.flavor
\ No newline at end of file
diff --git a/xos/synchronizer/run-from-api.sh b/xos/synchronizer/run-from-api.sh
new file mode 100755
index 0000000..820eefa
--- /dev/null
+++ b/xos/synchronizer/run-from-api.sh
@@ -0,0 +1,15 @@
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+python hssdb-synchronizer.py
\ No newline at end of file
diff --git a/xos/synchronizer/steps/hssdbserviceinstance_playbook.yaml b/xos/synchronizer/steps/hssdbserviceinstance_playbook.yaml
new file mode 100644
index 0000000..35148b5
--- /dev/null
+++ b/xos/synchronizer/steps/hssdbserviceinstance_playbook.yaml
@@ -0,0 +1,23 @@
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- hosts: {{ instance_name }}
+  gather_facts: False
+  connection: ssh
+  user: ubuntu
+  sudo: yes
+  tasks:
+  vars:
+
diff --git a/xos/synchronizer/steps/sync_hssdbserviceinstance.py b/xos/synchronizer/steps/sync_hssdbserviceinstance.py
new file mode 100644
index 0000000..4cb1379
--- /dev/null
+++ b/xos/synchronizer/steps/sync_hssdbserviceinstance.py
@@ -0,0 +1,33 @@
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+from django.db.models import Q, F
+from synchronizers.new_base.modelaccessor import *
+from synchronizers.new_base.SyncInstanceUsingAnsible import SyncInstanceUsingAnsible
+
+parentdir = os.path.join(os.path.dirname(__file__), "..")
+sys.path.insert(0, parentdir)
+
+class SyncHSSDBServiceInstance(SyncInstanceUsingAnsible):
+
+    observes = HSSDBServiceInstance
+    template_name = "hssdbserviceinstance_playbook.yaml"
+    service_key_name = "/opt/xos/configurations/mcord/mcord_private_key"
+
+    def __init__(self, *args, **kwargs):
+        super(SyncHSSDBServiceInstance, self).__init__(*args, **kwargs)
+
+