Starting point

Change-Id: I86dab2674688f6d818ce2fb0861ea893d6841df3
diff --git a/src/grpc_client/__init__.py b/src/grpc_client/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/grpc_client/__init__.py
diff --git a/src/grpc_client/main.py b/src/grpc_client/main.py
new file mode 100644
index 0000000..b5bfa4f
--- /dev/null
+++ b/src/grpc_client/main.py
@@ -0,0 +1,43 @@
+import functools
+from xosapi.xos_grpc_client import SecureClient, InsecureClient
+from twisted.internet import reactor, defer
+from tosca.resources import RESOURCES
+from xosconfig import Config
+
+LOCAL_CERT = '/Users/teone/Sites/opencord/orchestration/xos-tosca/local_certs.crt'
+
+class GRPC_Client:
+    def __init__(self):
+        self.client = None
+
+        self.grpc_secure_endpoint = Config.get('grpc.secure_endpoint')
+        self.grpc_insecure_endpoint = Config.get('grpc.insecure_endpoint')
+        self.username = Config.get('grpc.admin_username')
+        self.password = Config.get('grpc.admin_password')
+
+    def setup_resources(self, client):
+        print "[XOS-TOSCA] Loading resources"
+        for k in client.xos_orm.all_model_names:
+            RESOURCES[k] = getattr(client.xos_orm, k)
+
+    def start(self):
+        print "[XOS-TOSCA] Connecting to xos-core"
+
+        deferred = defer.Deferred()
+
+        if self.client:
+            self.client.stop()
+            self.client.session_change = True
+
+        if self.username and self.password:
+            # NOTE if we authenticate users given the parameters in the rest request, do we need this?
+            self.client = SecureClient(endpoint=self.grpc_secure_endpoint, username=self.username, password=self.password, cacert=LOCAL_CERT)
+        else:
+            self.client = InsecureClient(endpoint=self.grpc_insecure_endpoint)
+
+        self.client.set_reconnect_callback(functools.partial(self.setup_resources, self.client))
+        self.client.start()
+
+        # TODO can we call this once the client is setted up?
+        reactor.callLater(12, deferred.callback, self.client)
+        return deferred
\ No newline at end of file
diff --git a/src/main.py b/src/main.py
new file mode 100644
index 0000000..e9e8c10
--- /dev/null
+++ b/src/main.py
@@ -0,0 +1,38 @@
+import os
+from grpc_client.main import GRPC_Client
+from tosca.generator import TOSCA_Generator
+from web_server.main import TOSCA_WebServer
+from twisted.internet import reactor, defer
+from xosconfig import Config
+
+current_dir = os.path.dirname(os.path.realpath(__file__))
+config_file = os.path.join(current_dir, './xos-tosca-config.yaml')
+config_schema = os.path.join(current_dir, './xos-tosca-config-schema.yaml')
+Config.init(config_file, config_schema)
+
+class Main:
+
+    def __init__(self):
+        self.grpc_client = None
+
+    def generate_tosca(self, client):
+
+        deferred = defer.Deferred()
+
+        TOSCA_Generator().generate(client)
+
+        reactor.callLater(0, TOSCA_WebServer)
+
+        return deferred
+
+    def start(self):
+        print "[XOS-TOSCA] Starting"
+
+        grpc_setup = GRPC_Client().start()
+        grpc_setup.addCallback(self.generate_tosca)
+
+        reactor.run()
+
+
+if __name__ == '__main__':
+    Main().start()
\ No newline at end of file
diff --git a/src/tosca/__init__.py b/src/tosca/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/tosca/__init__.py
diff --git a/src/tosca/custom_types/.gitignore b/src/tosca/custom_types/.gitignore
new file mode 100644
index 0000000..c96a04f
--- /dev/null
+++ b/src/tosca/custom_types/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
\ No newline at end of file
diff --git a/src/tosca/default.py b/src/tosca/default.py
new file mode 100644
index 0000000..1353075
--- /dev/null
+++ b/src/tosca/default.py
@@ -0,0 +1,4 @@
+import os
+
+TOSCA_DEFS_DIR = os.path.dirname(os.path.realpath(__file__)) + "/custom_types"
+TOSCA_RECIPES_DIR = os.path.dirname(os.path.realpath(__file__)) + "/"
\ No newline at end of file
diff --git a/src/tosca/generator.py b/src/tosca/generator.py
new file mode 100644
index 0000000..9e02011
--- /dev/null
+++ b/src/tosca/generator.py
@@ -0,0 +1,36 @@
+import os
+from default import TOSCA_DEFS_DIR
+from xosgenx.generator import XOSGenerator
+from xosapi.xos_grpc_client import Empty
+
+class Args:
+    pass
+
+current_dir = os.path.dirname(os.path.realpath(__file__))
+
+class TOSCA_Generator:
+
+    def _clean(self, dir=TOSCA_DEFS_DIR):
+        filesToRemove = [f for f in os.listdir(dir)]
+        for f in filesToRemove:
+            if not f.startswith('.'):
+                os.remove(dir + '/' + f)
+
+    def generate(self, client):
+        print "[XOS-TOSCA] Generating TOSCA"
+        self._clean()
+        xproto = client.utility.GetXproto(Empty())
+
+        try:
+            args = Args()
+            args.output = TOSCA_DEFS_DIR
+            args.inputs = str(xproto.xproto)
+            args.target = os.path.join(current_dir, 'xtarget/tosca.xtarget')
+            args.write_to_file = 'model'
+            args.dest_extension = 'yaml'
+            XOSGenerator.generate(args)
+            print "[XOS-TOSCA] Recipes generated in %s" % args.output
+        except Exception as e:
+            print "[XOS-TOSCA] Failed to generate TOSCA"
+            print e
+
diff --git a/src/tosca/parser.py b/src/tosca/parser.py
new file mode 100644
index 0000000..7eb3381
--- /dev/null
+++ b/src/tosca/parser.py
@@ -0,0 +1,176 @@
+from toscaparser.tosca_template import ToscaTemplate
+from default import TOSCA_RECIPES_DIR
+from resources import RESOURCES
+from grpc._channel import _Rendezvous
+
+class TOSCA_Parser:
+
+    def compute_dependencies(self):
+        nodetemplates_by_name = {}
+        for nodetemplate in self.template.nodetemplates:
+            nodetemplates_by_name[nodetemplate.name] = nodetemplate
+
+        self.nodetemplates_by_name = nodetemplates_by_name
+
+        for nodetemplate in self.template.nodetemplates:
+            nodetemplate.dependencies = []
+            nodetemplate.dependencies_names = []
+            for reqs in nodetemplate.requirements:
+                for (k,v) in reqs.items():
+                    name = v["node"]
+                    if (name in nodetemplates_by_name):
+                        nodetemplate.dependencies.append(nodetemplates_by_name[name])
+                        nodetemplate.dependencies_names.append(name)
+
+                    # go another level deep, as our requirements can have requirements...
+                    for sd_req in v.get("requirements",[]):
+                        for (sd_req_k, sd_req_v) in sd_req.items():
+                            name = sd_req_v["node"]
+                            if (name in nodetemplates_by_name):
+                                nodetemplate.dependencies.append(nodetemplates_by_name[name])
+                                nodetemplate.dependencies_names.append(name)
+
+    def topsort_dependencies(self):
+        # stolen from observer
+        g = self.nodetemplates_by_name
+
+        # Get set of all nodes, including those without outgoing edges
+        keys = set(g.keys())
+        values = set({})
+        for v in g.values():
+            values = values | set(v.dependencies_names)
+
+        all_nodes = list(keys | values)
+        steps = all_nodes
+
+
+        # Final order
+        order = []
+
+        # DFS stack, not using recursion
+        stack = []
+
+        # Unmarked set
+        unmarked = all_nodes
+
+        # visiting = [] - skip, don't expect 1000s of nodes, |E|/|V| is small
+
+        while unmarked:
+            stack.insert(0, unmarked[0])  # push first unmarked
+
+            while (stack):
+                n = stack[0]
+                add = True
+                try:
+                    for m in g[n].dependencies_names:
+                        if (m in unmarked):
+                            add = False
+                            stack.insert(0, m)
+                except KeyError:
+                    pass
+                if (add):
+                    if (n in steps and n not in order):
+                        order.append(n)
+                    item = stack.pop(0)
+                    try:
+                        unmarked.remove(item)
+                    except ValueError:
+                        pass
+
+        noorder = list(set(steps) - set(order))
+        return order + noorder
+
+    def execute(self):
+        for nodetemplate in self.ordered_nodetemplates:
+            self.execute_nodetemplate(nodetemplate)
+
+    def execute_nodetemplate(self, nodetemplate):
+        node_class = nodetemplate.type.replace("tosca.nodes.", "")
+        if node_class not in RESOURCES:
+            raise Exception("Nodetemplate %s's type %s is not a known resource" % (nodetemplate.name, node_class))
+
+        # find the class corresponding to a node
+        cls = RESOURCES[node_class]
+
+
+        # read properties from TOSCA
+        data = nodetemplate.templates[nodetemplate.name]['properties']
+
+        TOSCA_Parser.creat_or_update(cls, data)
+
+    @staticmethod
+    def populate_model(model, data):
+        for k,v in data.iteritems():
+            setattr(model, k, v)
+        return model
+
+    @staticmethod
+    def creat_or_update(cls, data):
+
+        # default case
+        if data.get('name'):
+            used_key = 'name'
+        else:
+            used_key = data.keys()[0]
+
+        models = cls.objects.filter(**{used_key: data[used_key]})
+
+        if len(models) == 1:
+            print "[XOS-Tosca] Model %s already exist, updating..." % data[used_key]
+            model = models[0]
+        elif len(models) == 0:
+            model = cls.objects.new()
+            print "[XOS-Tosca] Model %s is new, creating..." % data[used_key]
+        else:
+            raise Exception("[XOS-Tosca] Model %s has multiple instances, I can't handle it" % data[used_key])
+
+        model = TOSCA_Parser.populate_model(model, data)
+        model.save()
+
+
+    @staticmethod
+    def _translate_exception(msg):
+        readable = []
+        for line in msg.splitlines():
+            print line
+            if line.strip().startswith('MissingRequiredFieldError'):
+                readable.append(line)
+
+        if len(readable) > 0:
+            return '/n'.join(readable)
+        else:
+            return msg
+
+    def __init__(self, recipe):
+
+        self.template = None
+        self.nodetemplates_by_name = None
+        self.ordered_nodetemplates = []
+        self.ordered_names = None
+
+        tmp_file_path = TOSCA_RECIPES_DIR + '/tmp.yaml'
+
+        # write the receive recipe in a tmp file
+        tmp_file = open(tmp_file_path, 'w')
+        tmp_file.write(recipe)
+        tmp_file.close()
+
+        try:
+            self.template = ToscaTemplate(tmp_file_path)
+            self.compute_dependencies()
+            self.ordered_names = self.topsort_dependencies()
+            for name in self.ordered_names:
+                if name in self.nodetemplates_by_name:
+                    self.ordered_nodetemplates.append(self.nodetemplates_by_name[name])
+
+            self.execute()
+
+        except Exception as e:
+            print e
+            import sys, os
+            exc_type, exc_obj, exc_tb = sys.exc_info()
+            fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
+            print(exc_type, fname, exc_tb.tb_lineno)
+            raise Exception(self._translate_exception(e.message))
+
+
diff --git a/src/tosca/resources.py b/src/tosca/resources.py
new file mode 100644
index 0000000..06db4c4
--- /dev/null
+++ b/src/tosca/resources.py
@@ -0,0 +1,2 @@
+# NOTE will add all the resources in this dictionary
+RESOURCES = {}
\ No newline at end of file
diff --git a/src/tosca/tmp.yaml b/src/tosca/tmp.yaml
new file mode 100644
index 0000000..5f5d6e6
--- /dev/null
+++ b/src/tosca/tmp.yaml
@@ -0,0 +1,29 @@
+tosca_definitions_version: tosca_simple_yaml_1_0
+
+description: Persist xos-sample-gui-extension
+
+imports:
+   - custom_types/user.yaml
+   - custom_types/xosguiextension.yaml
+
+topology_template:
+  node_templates:
+
+    # UI Extension
+    test:
+      type: tosca.nodes.XOSGuiExtension
+      properties:
+        name: test
+        files: /spa/extensions/test/vendor.js, /spa/extensions/test/app.js
+
+    xosadmin@opencord.org:
+      type: tosca.nodes.User
+      properties:
+        password: rk1UYDHZXbu6KVCMkhmV
+        firstname: XOS
+        lastname: Admin
+#        is_admin: True
+#      requirements:
+#        - site:
+#            node: some
+#            relationship: tosca.relationships.MemberOfSite
\ No newline at end of file
diff --git a/src/tosca/xtarget/tosca.xtarget b/src/tosca/xtarget/tosca.xtarget
new file mode 100644
index 0000000..fc1256d
--- /dev/null
+++ b/src/tosca/xtarget/tosca.xtarget
@@ -0,0 +1,31 @@
+tosca_definitions_version: tosca_simple_yaml_1_0
+
+node_types:
+{% for m in proto.messages %}
+    tosca.nodes.{{ m.name }}:
+        derived_from: tosca.nodes.Root
+        description: {{ m.name }}
+        properties:
+            no-delete:
+                type: boolean
+                default: false
+                description: Do not allow Tosca to delete this object
+            no-create:
+                type: boolean
+                default: false
+                description: Do not allow Tosca to create this object
+            no-update:
+                type: boolean
+                default: false
+                description: Do not allow Tosca to update this object
+            replaces:
+                type: string
+                required: false
+                descrption: Replaces/renames this object
+            {%- for f in m.fields %}
+            {{ f.name }}:
+                type: {{ f.type }}
+                required: {{ xproto_tosca_required(f.options.null, f.options.blank, f.options.default) }}
+                description: {{ f.options.help_text }}
+            {%- endfor %}
+{%- endfor %}
\ No newline at end of file
diff --git a/src/web_server/__init__.py b/src/web_server/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/web_server/__init__.py
diff --git a/src/web_server/main.py b/src/web_server/main.py
new file mode 100644
index 0000000..b1e2758
--- /dev/null
+++ b/src/web_server/main.py
@@ -0,0 +1,30 @@
+from flask import Flask, make_response, request
+from tosca.parser import TOSCA_Parser
+
+BANNER = """
+   _  ______  _____    __________  _____ _________ 
+  | |/ / __ \/ ___/   /_  __/ __ \/ ___// ____/   |
+  |   / / / /\__ \     / / / / / /\__ \/ /   / /| |
+ /   / /_/ /___/ /    / / / /_/ /___/ / /___/ ___ |
+/_/|_\____//____/    /_/  \____//____/\____/_/  |_|
+"""
+
+class TOSCA_WebServer:
+    app = Flask('TOSCA-Web-Server')
+
+    @app.route("/", methods=['GET', 'POST'])
+    def home():
+        if request.method == 'GET':
+            response =  make_response(BANNER)
+            response.headers["content-type"] = "text/plain"
+            return response
+        else:
+            try:
+                # print request.headers['xos-password']
+                parsed = TOSCA_Parser(request.get_data())
+                return make_response(str(parsed.ordered_names), 201)
+            except Exception, e:
+                return make_response(e.message, 400)
+
+    def __init__(self):
+        self.app.run(host='localhost', port='9200')
\ No newline at end of file
diff --git a/src/xos-tosca-config-schema.yaml b/src/xos-tosca-config-schema.yaml
new file mode 100644
index 0000000..5eb221f
--- /dev/null
+++ b/src/xos-tosca-config-schema.yaml
@@ -0,0 +1,31 @@
+map:
+  name:
+    type: str
+    required: True
+  grpc:
+    type: map
+    map:
+      admin_username:
+        type: str
+        required: True
+      admin_password:
+        type: str
+        required: True
+      secure_endpoint:
+        type: str
+        required: True
+      insecure_endpoint:
+        type: str
+        required: True
+  logging:
+    type: map
+    map:
+      logstash_hostport:
+        type: str
+      level:
+        type: str
+      channels:
+        type: seq
+        sequence:
+          - type: str
+            enum: ['file', 'console', 'elkstack']
diff --git a/src/xos-tosca-config.yaml b/src/xos-tosca-config.yaml
new file mode 100644
index 0000000..2555112
--- /dev/null
+++ b/src/xos-tosca-config.yaml
@@ -0,0 +1,6 @@
+name: xos-tosca
+grpc:
+  admin_username: "xosadmin@opencord.org"
+  admin_password: "rk1UYDHZXbu6KVCMkhmV"
+  secure_endpoint: "xos-core.opencord.org:50051"
+  insecure_endpoint: "xos-core.opencord.org:50055"