Starting point
Change-Id: I86dab2674688f6d818ce2fb0861ea893d6841df3
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..35f596d
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,4 @@
+.idea
+*.pyc
+.noseids
+local_certs.crt
\ No newline at end of file
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..fe7ab6d
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,2 @@
+# xosproject/xos-tosca
+FROM xosproject/xos-client:candidate
\ No newline at end of file
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..5025e0f
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,14 @@
+help:
+ @echo "tests: Run unit tests (if you're running local, you'll need to have virtual-env activated)"
+
+tests:
+ nosetests -s -v --with-id
+
+build:
+ @echo 'TBD'
+
+clean:
+ @echo 'TBD'
+
+test-call:
+ curl -H "xos-username: xosadmin@opencord.org" -H "xos-password: rk1UYDHZXbu6KVCMkhmV" -X POST --data-binary @test/tosca/test.yaml 127.0.0.1:9200
\ No newline at end of file
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..4812ea6
--- /dev/null
+++ b/README.md
@@ -0,0 +1,22 @@
+# XOS TOSCA
+
+Welcome to the XOS TOSCA.
+
+## Documentation
+You can find the documentation in the `docs` folder. It has been created using `gitbook` and can be consumed as a local website.
+To bring it up, just open a termina pointing to this folder and execute: `gitbook serve`, then open a browser at `http://localhost:4000`
+
+## Support
+
+For support please refer to:
+
+**Slack**<br/>
+[slackin.opencord.org](https://slackin.opencord.org/)
+
+**Mailing List**<br/>
+[CORD Discuss](https://groups.google.com/a/opencord.org/forum/#!forum/cord-discuss)<br/>
+[CORD Developers](https://groups.google.com/a/opencord.org/forum/#!forum/cord-dev)
+
+**Wiki**<br/>
+[wiki.opencord.org](https://wiki.opencord.org/)
+
diff --git a/book.json b/book.json
new file mode 100644
index 0000000..abbf32f
--- /dev/null
+++ b/book.json
@@ -0,0 +1,4 @@
+{
+ "title": "XOS-TOSCA User Documentation",
+ "root": "./docs"
+}
diff --git a/docs/GLOSSARY.md b/docs/GLOSSARY.md
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/docs/GLOSSARY.md
diff --git a/docs/README.md b/docs/README.md
new file mode 100644
index 0000000..018227d
--- /dev/null
+++ b/docs/README.md
@@ -0,0 +1,23 @@
+# XOS TOSCA User Manual
+
+## Development
+
+You'll need to have:
+- an XOS configuration running in the frontend vm
+- source the xos virtual env (from `xos` root: `source scripts/setup_venv.sh`)
+- install `xos-tosca` specific dependencies: `pip install -r requirements`
+
+### Run the xos-tosca process
+
+You can run this either from an IDE or:
+```bash
+python scr/main.py
+```
+
+### Sample call
+
+To send an example request to `xos-tosca`:
+```bash
+curl -X POST --data-binary @test.yaml 127.0.0.1:9200
+```
+
diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md
new file mode 100644
index 0000000..263112f
--- /dev/null
+++ b/docs/SUMMARY.md
@@ -0,0 +1,4 @@
+# Summary
+
+* [Introduction](README.md)
+
diff --git a/pip_requiremts.txt b/pip_requiremts.txt
new file mode 100644
index 0000000..43e625b
--- /dev/null
+++ b/pip_requiremts.txt
@@ -0,0 +1 @@
+Flask==0.12.2
\ No newline at end of file
diff --git a/src/grpc_client/__init__.py b/src/grpc_client/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/grpc_client/__init__.py
diff --git a/src/grpc_client/main.py b/src/grpc_client/main.py
new file mode 100644
index 0000000..b5bfa4f
--- /dev/null
+++ b/src/grpc_client/main.py
@@ -0,0 +1,43 @@
+import functools
+from xosapi.xos_grpc_client import SecureClient, InsecureClient
+from twisted.internet import reactor, defer
+from tosca.resources import RESOURCES
+from xosconfig import Config
+
+LOCAL_CERT = '/Users/teone/Sites/opencord/orchestration/xos-tosca/local_certs.crt'
+
+class GRPC_Client:
+ def __init__(self):
+ self.client = None
+
+ self.grpc_secure_endpoint = Config.get('grpc.secure_endpoint')
+ self.grpc_insecure_endpoint = Config.get('grpc.insecure_endpoint')
+ self.username = Config.get('grpc.admin_username')
+ self.password = Config.get('grpc.admin_password')
+
+ def setup_resources(self, client):
+ print "[XOS-TOSCA] Loading resources"
+ for k in client.xos_orm.all_model_names:
+ RESOURCES[k] = getattr(client.xos_orm, k)
+
+ def start(self):
+ print "[XOS-TOSCA] Connecting to xos-core"
+
+ deferred = defer.Deferred()
+
+ if self.client:
+ self.client.stop()
+ self.client.session_change = True
+
+ if self.username and self.password:
+ # NOTE if we authenticate users given the parameters in the rest request, do we need this?
+ self.client = SecureClient(endpoint=self.grpc_secure_endpoint, username=self.username, password=self.password, cacert=LOCAL_CERT)
+ else:
+ self.client = InsecureClient(endpoint=self.grpc_insecure_endpoint)
+
+ self.client.set_reconnect_callback(functools.partial(self.setup_resources, self.client))
+ self.client.start()
+
+ # TODO can we call this once the client is setted up?
+ reactor.callLater(12, deferred.callback, self.client)
+ return deferred
\ No newline at end of file
diff --git a/src/main.py b/src/main.py
new file mode 100644
index 0000000..e9e8c10
--- /dev/null
+++ b/src/main.py
@@ -0,0 +1,38 @@
+import os
+from grpc_client.main import GRPC_Client
+from tosca.generator import TOSCA_Generator
+from web_server.main import TOSCA_WebServer
+from twisted.internet import reactor, defer
+from xosconfig import Config
+
+current_dir = os.path.dirname(os.path.realpath(__file__))
+config_file = os.path.join(current_dir, './xos-tosca-config.yaml')
+config_schema = os.path.join(current_dir, './xos-tosca-config-schema.yaml')
+Config.init(config_file, config_schema)
+
+class Main:
+
+ def __init__(self):
+ self.grpc_client = None
+
+ def generate_tosca(self, client):
+
+ deferred = defer.Deferred()
+
+ TOSCA_Generator().generate(client)
+
+ reactor.callLater(0, TOSCA_WebServer)
+
+ return deferred
+
+ def start(self):
+ print "[XOS-TOSCA] Starting"
+
+ grpc_setup = GRPC_Client().start()
+ grpc_setup.addCallback(self.generate_tosca)
+
+ reactor.run()
+
+
+if __name__ == '__main__':
+ Main().start()
\ No newline at end of file
diff --git a/src/tosca/__init__.py b/src/tosca/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/tosca/__init__.py
diff --git a/src/tosca/custom_types/.gitignore b/src/tosca/custom_types/.gitignore
new file mode 100644
index 0000000..c96a04f
--- /dev/null
+++ b/src/tosca/custom_types/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
\ No newline at end of file
diff --git a/src/tosca/default.py b/src/tosca/default.py
new file mode 100644
index 0000000..1353075
--- /dev/null
+++ b/src/tosca/default.py
@@ -0,0 +1,4 @@
+import os
+
+TOSCA_DEFS_DIR = os.path.dirname(os.path.realpath(__file__)) + "/custom_types"
+TOSCA_RECIPES_DIR = os.path.dirname(os.path.realpath(__file__)) + "/"
\ No newline at end of file
diff --git a/src/tosca/generator.py b/src/tosca/generator.py
new file mode 100644
index 0000000..9e02011
--- /dev/null
+++ b/src/tosca/generator.py
@@ -0,0 +1,36 @@
+import os
+from default import TOSCA_DEFS_DIR
+from xosgenx.generator import XOSGenerator
+from xosapi.xos_grpc_client import Empty
+
+class Args:
+ pass
+
+current_dir = os.path.dirname(os.path.realpath(__file__))
+
+class TOSCA_Generator:
+
+ def _clean(self, dir=TOSCA_DEFS_DIR):
+ filesToRemove = [f for f in os.listdir(dir)]
+ for f in filesToRemove:
+ if not f.startswith('.'):
+ os.remove(dir + '/' + f)
+
+ def generate(self, client):
+ print "[XOS-TOSCA] Generating TOSCA"
+ self._clean()
+ xproto = client.utility.GetXproto(Empty())
+
+ try:
+ args = Args()
+ args.output = TOSCA_DEFS_DIR
+ args.inputs = str(xproto.xproto)
+ args.target = os.path.join(current_dir, 'xtarget/tosca.xtarget')
+ args.write_to_file = 'model'
+ args.dest_extension = 'yaml'
+ XOSGenerator.generate(args)
+ print "[XOS-TOSCA] Recipes generated in %s" % args.output
+ except Exception as e:
+ print "[XOS-TOSCA] Failed to generate TOSCA"
+ print e
+
diff --git a/src/tosca/parser.py b/src/tosca/parser.py
new file mode 100644
index 0000000..7eb3381
--- /dev/null
+++ b/src/tosca/parser.py
@@ -0,0 +1,176 @@
+from toscaparser.tosca_template import ToscaTemplate
+from default import TOSCA_RECIPES_DIR
+from resources import RESOURCES
+from grpc._channel import _Rendezvous
+
+class TOSCA_Parser:
+
+ def compute_dependencies(self):
+ nodetemplates_by_name = {}
+ for nodetemplate in self.template.nodetemplates:
+ nodetemplates_by_name[nodetemplate.name] = nodetemplate
+
+ self.nodetemplates_by_name = nodetemplates_by_name
+
+ for nodetemplate in self.template.nodetemplates:
+ nodetemplate.dependencies = []
+ nodetemplate.dependencies_names = []
+ for reqs in nodetemplate.requirements:
+ for (k,v) in reqs.items():
+ name = v["node"]
+ if (name in nodetemplates_by_name):
+ nodetemplate.dependencies.append(nodetemplates_by_name[name])
+ nodetemplate.dependencies_names.append(name)
+
+ # go another level deep, as our requirements can have requirements...
+ for sd_req in v.get("requirements",[]):
+ for (sd_req_k, sd_req_v) in sd_req.items():
+ name = sd_req_v["node"]
+ if (name in nodetemplates_by_name):
+ nodetemplate.dependencies.append(nodetemplates_by_name[name])
+ nodetemplate.dependencies_names.append(name)
+
+ def topsort_dependencies(self):
+ # stolen from observer
+ g = self.nodetemplates_by_name
+
+ # Get set of all nodes, including those without outgoing edges
+ keys = set(g.keys())
+ values = set({})
+ for v in g.values():
+ values = values | set(v.dependencies_names)
+
+ all_nodes = list(keys | values)
+ steps = all_nodes
+
+
+ # Final order
+ order = []
+
+ # DFS stack, not using recursion
+ stack = []
+
+ # Unmarked set
+ unmarked = all_nodes
+
+ # visiting = [] - skip, don't expect 1000s of nodes, |E|/|V| is small
+
+ while unmarked:
+ stack.insert(0, unmarked[0]) # push first unmarked
+
+ while (stack):
+ n = stack[0]
+ add = True
+ try:
+ for m in g[n].dependencies_names:
+ if (m in unmarked):
+ add = False
+ stack.insert(0, m)
+ except KeyError:
+ pass
+ if (add):
+ if (n in steps and n not in order):
+ order.append(n)
+ item = stack.pop(0)
+ try:
+ unmarked.remove(item)
+ except ValueError:
+ pass
+
+ noorder = list(set(steps) - set(order))
+ return order + noorder
+
+ def execute(self):
+ for nodetemplate in self.ordered_nodetemplates:
+ self.execute_nodetemplate(nodetemplate)
+
+ def execute_nodetemplate(self, nodetemplate):
+ node_class = nodetemplate.type.replace("tosca.nodes.", "")
+ if node_class not in RESOURCES:
+ raise Exception("Nodetemplate %s's type %s is not a known resource" % (nodetemplate.name, node_class))
+
+ # find the class corresponding to a node
+ cls = RESOURCES[node_class]
+
+
+ # read properties from TOSCA
+ data = nodetemplate.templates[nodetemplate.name]['properties']
+
+ TOSCA_Parser.creat_or_update(cls, data)
+
+ @staticmethod
+ def populate_model(model, data):
+ for k,v in data.iteritems():
+ setattr(model, k, v)
+ return model
+
+ @staticmethod
+ def creat_or_update(cls, data):
+
+ # default case
+ if data.get('name'):
+ used_key = 'name'
+ else:
+ used_key = data.keys()[0]
+
+ models = cls.objects.filter(**{used_key: data[used_key]})
+
+ if len(models) == 1:
+ print "[XOS-Tosca] Model %s already exist, updating..." % data[used_key]
+ model = models[0]
+ elif len(models) == 0:
+ model = cls.objects.new()
+ print "[XOS-Tosca] Model %s is new, creating..." % data[used_key]
+ else:
+ raise Exception("[XOS-Tosca] Model %s has multiple instances, I can't handle it" % data[used_key])
+
+ model = TOSCA_Parser.populate_model(model, data)
+ model.save()
+
+
+ @staticmethod
+ def _translate_exception(msg):
+ readable = []
+ for line in msg.splitlines():
+ print line
+ if line.strip().startswith('MissingRequiredFieldError'):
+ readable.append(line)
+
+ if len(readable) > 0:
+ return '/n'.join(readable)
+ else:
+ return msg
+
+ def __init__(self, recipe):
+
+ self.template = None
+ self.nodetemplates_by_name = None
+ self.ordered_nodetemplates = []
+ self.ordered_names = None
+
+ tmp_file_path = TOSCA_RECIPES_DIR + '/tmp.yaml'
+
+ # write the receive recipe in a tmp file
+ tmp_file = open(tmp_file_path, 'w')
+ tmp_file.write(recipe)
+ tmp_file.close()
+
+ try:
+ self.template = ToscaTemplate(tmp_file_path)
+ self.compute_dependencies()
+ self.ordered_names = self.topsort_dependencies()
+ for name in self.ordered_names:
+ if name in self.nodetemplates_by_name:
+ self.ordered_nodetemplates.append(self.nodetemplates_by_name[name])
+
+ self.execute()
+
+ except Exception as e:
+ print e
+ import sys, os
+ exc_type, exc_obj, exc_tb = sys.exc_info()
+ fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
+ print(exc_type, fname, exc_tb.tb_lineno)
+ raise Exception(self._translate_exception(e.message))
+
+
diff --git a/src/tosca/resources.py b/src/tosca/resources.py
new file mode 100644
index 0000000..06db4c4
--- /dev/null
+++ b/src/tosca/resources.py
@@ -0,0 +1,2 @@
+# NOTE will add all the resources in this dictionary
+RESOURCES = {}
\ No newline at end of file
diff --git a/src/tosca/tmp.yaml b/src/tosca/tmp.yaml
new file mode 100644
index 0000000..5f5d6e6
--- /dev/null
+++ b/src/tosca/tmp.yaml
@@ -0,0 +1,29 @@
+tosca_definitions_version: tosca_simple_yaml_1_0
+
+description: Persist xos-sample-gui-extension
+
+imports:
+ - custom_types/user.yaml
+ - custom_types/xosguiextension.yaml
+
+topology_template:
+ node_templates:
+
+ # UI Extension
+ test:
+ type: tosca.nodes.XOSGuiExtension
+ properties:
+ name: test
+ files: /spa/extensions/test/vendor.js, /spa/extensions/test/app.js
+
+ xosadmin@opencord.org:
+ type: tosca.nodes.User
+ properties:
+ password: rk1UYDHZXbu6KVCMkhmV
+ firstname: XOS
+ lastname: Admin
+# is_admin: True
+# requirements:
+# - site:
+# node: some
+# relationship: tosca.relationships.MemberOfSite
\ No newline at end of file
diff --git a/src/tosca/xtarget/tosca.xtarget b/src/tosca/xtarget/tosca.xtarget
new file mode 100644
index 0000000..fc1256d
--- /dev/null
+++ b/src/tosca/xtarget/tosca.xtarget
@@ -0,0 +1,31 @@
+tosca_definitions_version: tosca_simple_yaml_1_0
+
+node_types:
+{% for m in proto.messages %}
+ tosca.nodes.{{ m.name }}:
+ derived_from: tosca.nodes.Root
+ description: {{ m.name }}
+ properties:
+ no-delete:
+ type: boolean
+ default: false
+ description: Do not allow Tosca to delete this object
+ no-create:
+ type: boolean
+ default: false
+ description: Do not allow Tosca to create this object
+ no-update:
+ type: boolean
+ default: false
+ description: Do not allow Tosca to update this object
+ replaces:
+ type: string
+ required: false
+ descrption: Replaces/renames this object
+ {%- for f in m.fields %}
+ {{ f.name }}:
+ type: {{ f.type }}
+ required: {{ xproto_tosca_required(f.options.null, f.options.blank, f.options.default) }}
+ description: {{ f.options.help_text }}
+ {%- endfor %}
+{%- endfor %}
\ No newline at end of file
diff --git a/src/web_server/__init__.py b/src/web_server/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/web_server/__init__.py
diff --git a/src/web_server/main.py b/src/web_server/main.py
new file mode 100644
index 0000000..b1e2758
--- /dev/null
+++ b/src/web_server/main.py
@@ -0,0 +1,30 @@
+from flask import Flask, make_response, request
+from tosca.parser import TOSCA_Parser
+
+BANNER = """
+ _ ______ _____ __________ _____ _________
+ | |/ / __ \/ ___/ /_ __/ __ \/ ___// ____/ |
+ | / / / /\__ \ / / / / / /\__ \/ / / /| |
+ / / /_/ /___/ / / / / /_/ /___/ / /___/ ___ |
+/_/|_\____//____/ /_/ \____//____/\____/_/ |_|
+"""
+
+class TOSCA_WebServer:
+ app = Flask('TOSCA-Web-Server')
+
+ @app.route("/", methods=['GET', 'POST'])
+ def home():
+ if request.method == 'GET':
+ response = make_response(BANNER)
+ response.headers["content-type"] = "text/plain"
+ return response
+ else:
+ try:
+ # print request.headers['xos-password']
+ parsed = TOSCA_Parser(request.get_data())
+ return make_response(str(parsed.ordered_names), 201)
+ except Exception, e:
+ return make_response(e.message, 400)
+
+ def __init__(self):
+ self.app.run(host='localhost', port='9200')
\ No newline at end of file
diff --git a/src/xos-tosca-config-schema.yaml b/src/xos-tosca-config-schema.yaml
new file mode 100644
index 0000000..5eb221f
--- /dev/null
+++ b/src/xos-tosca-config-schema.yaml
@@ -0,0 +1,31 @@
+map:
+ name:
+ type: str
+ required: True
+ grpc:
+ type: map
+ map:
+ admin_username:
+ type: str
+ required: True
+ admin_password:
+ type: str
+ required: True
+ secure_endpoint:
+ type: str
+ required: True
+ insecure_endpoint:
+ type: str
+ required: True
+ logging:
+ type: map
+ map:
+ logstash_hostport:
+ type: str
+ level:
+ type: str
+ channels:
+ type: seq
+ sequence:
+ - type: str
+ enum: ['file', 'console', 'elkstack']
diff --git a/src/xos-tosca-config.yaml b/src/xos-tosca-config.yaml
new file mode 100644
index 0000000..2555112
--- /dev/null
+++ b/src/xos-tosca-config.yaml
@@ -0,0 +1,6 @@
+name: xos-tosca
+grpc:
+ admin_username: "xosadmin@opencord.org"
+ admin_password: "rk1UYDHZXbu6KVCMkhmV"
+ secure_endpoint: "xos-core.opencord.org:50051"
+ insecure_endpoint: "xos-core.opencord.org:50055"
diff --git a/test/test_tosca_parser.py b/test/test_tosca_parser.py
new file mode 100644
index 0000000..eb31ed1
--- /dev/null
+++ b/test/test_tosca_parser.py
@@ -0,0 +1,74 @@
+import sys, os
+import unittest
+from mock import patch, MagicMock
+from tosca.parser import TOSCA_Parser
+
+class FakeObj:
+ new = None
+ filter = None
+
+class FakeResource:
+ objects = FakeObj
+
+class FakeModel:
+ save = None
+
+class TOSCA_Parser_Create_or_update_Test(unittest.TestCase):
+
+ @patch.object(FakeResource.objects, "filter")
+ @patch.object(FakeResource.objects, "new", MagicMock(return_value=FakeModel))
+ @patch.object(FakeModel, "save")
+ def test_new_model(self, mock_save, mock_filter):
+ """
+ [TOSCA_Parser] create_or_update: should create a new model
+ """
+ data = {
+ "name": "test"
+ }
+
+ TOSCA_Parser.creat_or_update(FakeResource, data)
+ mock_filter.assert_called_with(name="test")
+ mock_save.assert_called_once()
+
+ @patch.object(FakeResource.objects, "filter", MagicMock(return_value=[FakeModel]))
+ @patch.object(FakeModel, "save")
+ def test_existing_model(self, mock_save):
+ """
+ [TOSCA_Parser] create_or_update: should update an existing model
+ """
+ data = {
+ "name": "test"
+ }
+
+ TOSCA_Parser.creat_or_update(FakeResource, data)
+ mock_save.assert_called_once()
+
+ @patch.object(FakeResource.objects, "filter", MagicMock(return_value=['a', 'b']))
+ def test_multiple_models(self):
+ """
+ [TOSCA_Parser] create_or_update: should raise an exception if multiple instances are found
+ """
+ data = {
+ "name": "test"
+ }
+ with self.assertRaises(Exception) as e:
+ TOSCA_Parser.creat_or_update(FakeResource, data)
+ self.assertEqual(e.exception.message, "[XOS-Tosca] Model test has multiple instances, I can't handle it")
+
+ @patch.object(FakeResource.objects, "new", MagicMock(return_value=FakeModel))
+ @patch.object(FakeResource.objects, "filter")
+ @patch.object(FakeModel, "save")
+ def test_find_model_without_name_property(self, mock_save, mock_filter):
+ """
+ [TOSCA_Parser] create_or_update: should lookup a model by the first property
+ """
+ data = {
+ 'foo': 'bar',
+ 'something': 'else'
+ }
+ TOSCA_Parser.creat_or_update(FakeResource, data)
+ mock_filter.assert_called_with(foo="bar")
+ mock_save.assert_called_once()
+
+if __name__ == '__main__':
+ unittest.main()
\ No newline at end of file
diff --git a/test/tosca/test.yaml b/test/tosca/test.yaml
new file mode 100644
index 0000000..5f5d6e6
--- /dev/null
+++ b/test/tosca/test.yaml
@@ -0,0 +1,29 @@
+tosca_definitions_version: tosca_simple_yaml_1_0
+
+description: Persist xos-sample-gui-extension
+
+imports:
+ - custom_types/user.yaml
+ - custom_types/xosguiextension.yaml
+
+topology_template:
+ node_templates:
+
+ # UI Extension
+ test:
+ type: tosca.nodes.XOSGuiExtension
+ properties:
+ name: test
+ files: /spa/extensions/test/vendor.js, /spa/extensions/test/app.js
+
+ xosadmin@opencord.org:
+ type: tosca.nodes.User
+ properties:
+ password: rk1UYDHZXbu6KVCMkhmV
+ firstname: XOS
+ lastname: Admin
+# is_admin: True
+# requirements:
+# - site:
+# node: some
+# relationship: tosca.relationships.MemberOfSite
\ No newline at end of file