refactor
Change-Id: I833f751ab3f307f6996d0822bef7a749dc6f6d2f
diff --git a/.gitignore b/.gitignore
index 35f596d..635880e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,9 @@
.idea
*.pyc
.noseids
-local_certs.crt
\ No newline at end of file
+local_certs.crt
+
+# Test Coverage
+.coverage
+cover
+coverage.xml
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 5025e0f..190d0e1 100644
--- a/Makefile
+++ b/Makefile
@@ -1,8 +1,10 @@
help:
@echo "tests: Run unit tests (if you're running local, you'll need to have virtual-env activated)"
+ @echo "test-call: Send a sample tosca recipe"
+ @echo "sample-tosca: Generate tosca definition from core.xproto"
tests:
- nosetests -s -v --with-id
+ nosetests -s -v --with-id --with-coverage --cover-html --cover-erase --cover-xml --cover-package="grpc_client, tosca"
build:
@echo 'TBD'
@@ -11,4 +13,7 @@
@echo 'TBD'
test-call:
- curl -H "xos-username: xosadmin@opencord.org" -H "xos-password: rk1UYDHZXbu6KVCMkhmV" -X POST --data-binary @test/tosca/test.yaml 127.0.0.1:9200
\ No newline at end of file
+ curl -H "xos-username: xosadmin@opencord.org" -H "xos-password: rk1UYDHZXbu6KVCMkhmV" -X POST --data-binary @test/tosca/test.yaml 127.0.0.1:9200
+
+sample-tosca:
+ xosgenx --target=src/tosca/xtarget/tosca.xtarget --output=src/tosca/custom_types --write-to-file=model --dest-extension=yaml ../xos/xos/core/models/core.xproto
\ No newline at end of file
diff --git a/pip_requirements.txt b/pip_requirements.txt
index 43e625b..6a26d82 100644
--- a/pip_requirements.txt
+++ b/pip_requirements.txt
@@ -1 +1,2 @@
-Flask==0.12.2
\ No newline at end of file
+Flask==0.12.2
+coverage==4.4.1
\ No newline at end of file
diff --git a/src/grpc_client/main.py b/src/grpc_client/main.py
index b5bfa4f..5612d76 100644
--- a/src/grpc_client/main.py
+++ b/src/grpc_client/main.py
@@ -1,7 +1,7 @@
import functools
from xosapi.xos_grpc_client import SecureClient, InsecureClient
from twisted.internet import reactor, defer
-from tosca.resources import RESOURCES
+from resources import RESOURCES
from xosconfig import Config
LOCAL_CERT = '/Users/teone/Sites/opencord/orchestration/xos-tosca/local_certs.crt'
diff --git a/src/grpc_client/models_accessor.py b/src/grpc_client/models_accessor.py
new file mode 100644
index 0000000..29c2f11
--- /dev/null
+++ b/src/grpc_client/models_accessor.py
@@ -0,0 +1,34 @@
+from resources import RESOURCES
+
+class GRPCModelsAccessor:
+ """
+ This class provide the glue between the models managed by TOSCA and the ones living in xos-core
+ """
+
+ @staticmethod
+ def get_model_from_classname(class_name, data):
+ """
+ Give a Model Class Name and some data, check if that exits or instantiate a new one
+ """
+
+
+ if data.get('name'):
+ used_key = 'name'
+ else:
+ used_key = data.keys()[0]
+
+ if class_name not in RESOURCES:
+ raise Exception('[XOS-TOSCA] The model your tring to create (%s: %s, class: %s) is not know by xos-core' % (used_key, data[used_key], class_name))
+
+ cls = RESOURCES[class_name]
+ models = cls.objects.filter(**{used_key: data[used_key]})
+
+ if len(models) == 1:
+ print "[XOS-Tosca] Model %s already exist, retrieving instance..." % data[used_key]
+ model = models[0]
+ elif len(models) == 0:
+ model = cls.objects.new()
+ print "[XOS-Tosca] Model %s is new, creating new instance..." % data[used_key]
+ else:
+ raise Exception("[XOS-Tosca] Model %s has multiple instances, I can't handle it" % data[used_key])
+ return model
\ No newline at end of file
diff --git a/src/tosca/resources.py b/src/grpc_client/resources.py
similarity index 100%
rename from src/tosca/resources.py
rename to src/grpc_client/resources.py
diff --git a/src/tosca/parser.py b/src/tosca/parser.py
index 7eb3381..af70d27 100644
--- a/src/tosca/parser.py
+++ b/src/tosca/parser.py
@@ -1,38 +1,34 @@
from toscaparser.tosca_template import ToscaTemplate
from default import TOSCA_RECIPES_DIR
-from resources import RESOURCES
-from grpc._channel import _Rendezvous
+from grpc_client.resources import RESOURCES
+from grpc_client.models_accessor import GRPCModelsAccessor
class TOSCA_Parser:
- def compute_dependencies(self):
- nodetemplates_by_name = {}
- for nodetemplate in self.template.nodetemplates:
- nodetemplates_by_name[nodetemplate.name] = nodetemplate
-
- self.nodetemplates_by_name = nodetemplates_by_name
-
- for nodetemplate in self.template.nodetemplates:
+ def compute_dependencies(self, template, models_by_name):
+ """
+ NOTE this method is augmenting self.template, isn't there a more explicit way to achieve it?
+ """
+ for nodetemplate in template.nodetemplates:
nodetemplate.dependencies = []
nodetemplate.dependencies_names = []
for reqs in nodetemplate.requirements:
for (k,v) in reqs.items():
name = v["node"]
- if (name in nodetemplates_by_name):
- nodetemplate.dependencies.append(nodetemplates_by_name[name])
+ if (name in models_by_name):
+ nodetemplate.dependencies.append(models_by_name[name])
nodetemplate.dependencies_names.append(name)
# go another level deep, as our requirements can have requirements...
for sd_req in v.get("requirements",[]):
for (sd_req_k, sd_req_v) in sd_req.items():
name = sd_req_v["node"]
- if (name in nodetemplates_by_name):
- nodetemplate.dependencies.append(nodetemplates_by_name[name])
+ if (name in models_by_name):
+ nodetemplate.dependencies.append(models_by_name[name])
nodetemplate.dependencies_names.append(name)
- def topsort_dependencies(self):
- # stolen from observer
- g = self.nodetemplates_by_name
+ @staticmethod
+ def topsort_dependencies(g):
# Get set of all nodes, including those without outgoing edges
keys = set(g.keys())
@@ -80,24 +76,6 @@
noorder = list(set(steps) - set(order))
return order + noorder
- def execute(self):
- for nodetemplate in self.ordered_nodetemplates:
- self.execute_nodetemplate(nodetemplate)
-
- def execute_nodetemplate(self, nodetemplate):
- node_class = nodetemplate.type.replace("tosca.nodes.", "")
- if node_class not in RESOURCES:
- raise Exception("Nodetemplate %s's type %s is not a known resource" % (nodetemplate.name, node_class))
-
- # find the class corresponding to a node
- cls = RESOURCES[node_class]
-
-
- # read properties from TOSCA
- data = nodetemplate.templates[nodetemplate.name]['properties']
-
- TOSCA_Parser.creat_or_update(cls, data)
-
@staticmethod
def populate_model(model, data):
for k,v in data.iteritems():
@@ -105,30 +83,6 @@
return model
@staticmethod
- def creat_or_update(cls, data):
-
- # default case
- if data.get('name'):
- used_key = 'name'
- else:
- used_key = data.keys()[0]
-
- models = cls.objects.filter(**{used_key: data[used_key]})
-
- if len(models) == 1:
- print "[XOS-Tosca] Model %s already exist, updating..." % data[used_key]
- model = models[0]
- elif len(models) == 0:
- model = cls.objects.new()
- print "[XOS-Tosca] Model %s is new, creating..." % data[used_key]
- else:
- raise Exception("[XOS-Tosca] Model %s has multiple instances, I can't handle it" % data[used_key])
-
- model = TOSCA_Parser.populate_model(model, data)
- model.save()
-
-
- @staticmethod
def _translate_exception(msg):
readable = []
for line in msg.splitlines():
@@ -141,36 +95,74 @@
else:
return msg
- def __init__(self, recipe):
-
- self.template = None
- self.nodetemplates_by_name = None
- self.ordered_nodetemplates = []
- self.ordered_names = None
-
- tmp_file_path = TOSCA_RECIPES_DIR + '/tmp.yaml'
-
- # write the receive recipe in a tmp file
- tmp_file = open(tmp_file_path, 'w')
+ def save_recipe_to_tmp_file(self, recipe):
+ tmp_file = open(self.recipe_file, 'w')
tmp_file.write(recipe)
tmp_file.close()
- try:
- self.template = ToscaTemplate(tmp_file_path)
- self.compute_dependencies()
- self.ordered_names = self.topsort_dependencies()
- for name in self.ordered_names:
- if name in self.nodetemplates_by_name:
- self.ordered_nodetemplates.append(self.nodetemplates_by_name[name])
+ @staticmethod
+ def get_tosca_models_by_name(template):
+ models_by_name = {}
+ for node in template.nodetemplates:
+ models_by_name[node.name] = node
+ return models_by_name
- self.execute()
+ @staticmethod
+ def get_ordered_models_template(ordered_models_name, templates_by_model_name):
+ ordered_models_templates = []
+ for name in ordered_models_name:
+ if name in templates_by_model_name:
+ ordered_models_templates.append(templates_by_model_name[name])
+ return ordered_models_templates
+
+
+ def __init__(self, recipe):
+
+ # the template returned by TOSCA-Parser
+ self.template = None
+ # dictionary containing the models in the recipe and their template
+ self.templates_by_model_name = None
+ # list of models ordered by requirements
+ self.ordered_models_name = None
+
+ self.ordered_models_template = []
+ self.recipe_file = TOSCA_RECIPES_DIR + '/tmp.yaml'
+
+ try:
+ # [] save the recipe to a tmp file
+ self.save_recipe_to_tmp_file(recipe)
+ # [] parse the recipe with TOSCA Parse
+ self.template = ToscaTemplate(self.recipe_file)
+ # [] get all models in the recipe
+ self.templates_by_model_name = self.get_tosca_models_by_name(self.template)
+ # [] compute requirements
+ self.compute_dependencies(self.template, self.templates_by_model_name)
+ # [] topsort requirements
+ self.ordered_models_name = self.topsort_dependencies(self.templates_by_model_name)
+ # [] topsort templates
+ self.ordered_models_template = self.get_ordered_models_template(self.ordered_models_name, self.templates_by_model_name)
+
+ for recipe in self.ordered_models_template:
+ # get properties from tosca
+ data = recipe.templates[recipe.name]['properties']
+ # [] get model by class name
+ class_name = recipe.type.replace("tosca.nodes.", "")
+ if class_name not in RESOURCES:
+ raise Exception("Nodetemplate %s's type %s is not a known resource" % (recipe.name, class_name))
+ model = GRPCModelsAccessor.get_model_from_classname(class_name, data)
+ # [] populate model with data
+ model = self.populate_model(model, data)
+ # [] check if the model has requirements
+ # [] if it has populate them
+ # [] save or update
+ model.save()
except Exception as e:
print e
- import sys, os
- exc_type, exc_obj, exc_tb = sys.exc_info()
- fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
- print(exc_type, fname, exc_tb.tb_lineno)
- raise Exception(self._translate_exception(e.message))
+ if e.message:
+ exception_msg = e.message
+ else:
+ exception_msg = str(e)
+ raise Exception(exception_msg)
diff --git a/src/tosca/tmp.yaml b/src/tosca/tmp.yaml
index 5f5d6e6..e5b9fbd 100644
--- a/src/tosca/tmp.yaml
+++ b/src/tosca/tmp.yaml
@@ -4,6 +4,7 @@
imports:
- custom_types/user.yaml
+ - custom_types/site.yaml
- custom_types/xosguiextension.yaml
topology_template:
@@ -16,14 +17,23 @@
name: test
files: /spa/extensions/test/vendor.js, /spa/extensions/test/app.js
- xosadmin@opencord.org:
+ # Site
+ onlab:
+ type: tosca.nodes.Site
+ properties:
+ name: Open Networking Lab
+ site_url: http://onlab.us/
+ hosts_nodes: True
+
+ # User
+ test@opencord.org:
type: tosca.nodes.User
properties:
- password: rk1UYDHZXbu6KVCMkhmV
- firstname: XOS
- lastname: Admin
-# is_admin: True
-# requirements:
-# - site:
-# node: some
-# relationship: tosca.relationships.MemberOfSite
\ No newline at end of file
+ password: mypwd
+ firstname: User
+ lastname: Test
+ is_admin: True
+ requirements:
+ - site:
+ node: onlab
+ relationship: tosca.relationships.BelongsToOne
\ No newline at end of file
diff --git a/src/tosca/xtarget/tosca.xtarget b/src/tosca/xtarget/tosca.xtarget
index fc1256d..313874b 100644
--- a/src/tosca/xtarget/tosca.xtarget
+++ b/src/tosca/xtarget/tosca.xtarget
@@ -4,7 +4,10 @@
{% for m in proto.messages %}
tosca.nodes.{{ m.name }}:
derived_from: tosca.nodes.Root
- description: {{ m.name }}
+ description: {% if m.options.tosca_description -%}{{ m.options.tosca_description}}{% else%}"An XOS {{ m.name }}"{%- endif %}
+ capabilities:
+ {{ m.name|lower }}:
+ type: tosca.capabilities.xos.{{ m.name }}
properties:
no-delete:
type: boolean
@@ -22,10 +25,29 @@
type: string
required: false
descrption: Replaces/renames this object
- {%- for f in m.fields %}
+ {% for f in m.fields %}
+ {%- if not f.link -%}
{{ f.name }}:
- type: {{ f.type }}
+ type: {{ xproto_tosca_field_type(f.type) }}
required: {{ xproto_tosca_required(f.options.null, f.options.blank, f.options.default) }}
description: {{ f.options.help_text }}
+ {% endif %}
{%- endfor %}
+
+ {% for l in m.links %}
+ {%- if l.link_type == "manytoone" -%}
+ tosca.relationships.BelongsToOne:
+ derived_from: tosca.relationships.Root
+ valid_target_types: [ tosca.capabilities.xos.{{ l.peer.name }} ]
+ {%- endif%}
+ {%- if l.link_type == "manytomany" -%}
+ tosca.relationships.BelongsToMany:
+ derived_from: tosca.relationships.Root
+ valid_target_types: [ tosca.capabilities.xos.{{ l.peer.name }} ]
+ {%- endif%}
+ {% endfor %}
+
+ tosca.capabilities.xos.{{ m.name }}:
+ derived_from: tosca.capabilities.Root
+ description: {{ m.name }}
{%- endfor %}
\ No newline at end of file
diff --git a/src/web_server/main.py b/src/web_server/main.py
index b1e2758..6995b12 100644
--- a/src/web_server/main.py
+++ b/src/web_server/main.py
@@ -22,7 +22,8 @@
try:
# print request.headers['xos-password']
parsed = TOSCA_Parser(request.get_data())
- return make_response(str(parsed.ordered_names), 201)
+ response_text = "Created models: %s" % str(parsed.ordered_models_name)
+ return make_response(response_text, 201)
except Exception, e:
return make_response(e.message, 400)
diff --git a/test/test_grpc_models_accessor.py b/test/test_grpc_models_accessor.py
new file mode 100644
index 0000000..707b5ef
--- /dev/null
+++ b/test/test_grpc_models_accessor.py
@@ -0,0 +1,89 @@
+import unittest
+from mock import patch, MagicMock
+from grpc_client.models_accessor import GRPCModelsAccessor
+from grpc_client.resources import RESOURCES
+
+class FakeObj:
+ new = None
+ filter = None
+
+class FakeResource:
+ objects = FakeObj
+
+class FakeModel:
+ pass
+
+class FakeExistingModel:
+ pass
+
+mock_resources = {
+ 'test-model': FakeResource
+}
+
+class GRPCModelsAccessor_Create_or_update_Test(unittest.TestCase):
+
+ def test_unkown_module(self):
+ """
+ [GRPCModelsAccessor] get_model_from_classname: If a model is not know by the grpc api, raise
+ """
+ data = {
+ "name": "test"
+ }
+ with self.assertRaises(Exception) as e:
+ GRPCModelsAccessor.get_model_from_classname('i-do-not-exists', data)
+ self.assertEqual(e.exception.message, "[XOS-TOSCA] The model your tring to create (name: test, class: i-do-not-exists) is not know by xos-core")
+
+ @patch.object(FakeResource.objects, "filter")
+ @patch.object(FakeResource.objects, "new", MagicMock(return_value=FakeModel))
+ def test_new_model(self, mock_filter):
+ """
+ [GRPCModelsAccessor] get_model_from_classname: should create a new model
+ """
+ data = {
+ "name": "test"
+ }
+ with patch.dict(RESOURCES, mock_resources, clear=True):
+ model = GRPCModelsAccessor.get_model_from_classname('test-model', data)
+ mock_filter.assert_called_with(name="test")
+ self.assertEqual(model, FakeModel)
+
+ @patch.object(FakeResource.objects, "filter", MagicMock(return_value=[FakeExistingModel]))
+ def test_existing_model(self):
+ """
+ [GRPCModelsAccessor] get_model_from_classname: should update an existing model
+ """
+ data = {
+ "name": "test"
+ }
+ with patch.dict(RESOURCES, mock_resources, clear=True):
+ model = GRPCModelsAccessor.get_model_from_classname('test-model', data)
+ self.assertEqual(model, FakeExistingModel)
+
+ @patch.object(FakeResource.objects, "filter", MagicMock(return_value=['a', 'b']))
+ def test_multiple_models(self):
+ """
+ [GRPCModelsAccessor] get_model_from_classname: should raise an exception if multiple instances are found
+ """
+ data = {
+ "name": "test"
+ }
+ with patch.dict(RESOURCES, mock_resources, clear=True):
+ with self.assertRaises(Exception) as e:
+ GRPCModelsAccessor.get_model_from_classname('test-model', data)
+ self.assertEqual(e.exception.message, "[XOS-Tosca] Model test has multiple instances, I can't handle it")
+
+ @patch.dict(RESOURCES, mock_resources, clear=True)
+ @patch.object(FakeResource.objects, "filter")
+ def _test_find_model_without_name_property(self, mock_filter):
+ """
+ [GRPCModelsAccessor] get_model_from_classname: should lookup a model by the first property
+ """
+ data = {
+ 'foo': 'bar',
+ 'something': 'else'
+ }
+ GRPCModelsAccessor.get_model_from_classname('test-model', data)
+ mock_filter.assert_called_with(foo="bar")
+
+if __name__ == '__main__':
+ unittest.main()
\ No newline at end of file
diff --git a/test/test_tosca_parser.py b/test/test_tosca_parser.py
index eb31ed1..9a369fe 100644
--- a/test/test_tosca_parser.py
+++ b/test/test_tosca_parser.py
@@ -1,74 +1,24 @@
-import sys, os
import unittest
-from mock import patch, MagicMock
from tosca.parser import TOSCA_Parser
-class FakeObj:
- new = None
- filter = None
+class TOSCA_Parser_Test(unittest.TestCase):
-class FakeResource:
- objects = FakeObj
-
-class FakeModel:
- save = None
-
-class TOSCA_Parser_Create_or_update_Test(unittest.TestCase):
-
- @patch.object(FakeResource.objects, "filter")
- @patch.object(FakeResource.objects, "new", MagicMock(return_value=FakeModel))
- @patch.object(FakeModel, "save")
- def test_new_model(self, mock_save, mock_filter):
+ def test_get_tosca_models_by_name(self):
"""
- [TOSCA_Parser] create_or_update: should create a new model
+ [TOSCA_Parser] get_tosca_models_by_name: should extract models from the TOSCA recipe and store them in a dict
"""
- data = {
- "name": "test"
- }
+ class FakeNode:
+ def __init__(self, name):
+ self.name = name
- TOSCA_Parser.creat_or_update(FakeResource, data)
- mock_filter.assert_called_with(name="test")
- mock_save.assert_called_once()
+ class FakeTemplate:
+ nodetemplates = [
+ FakeNode('model1'),
+ FakeNode('model2')
+ ]
+ pass
- @patch.object(FakeResource.objects, "filter", MagicMock(return_value=[FakeModel]))
- @patch.object(FakeModel, "save")
- def test_existing_model(self, mock_save):
- """
- [TOSCA_Parser] create_or_update: should update an existing model
- """
- data = {
- "name": "test"
- }
- TOSCA_Parser.creat_or_update(FakeResource, data)
- mock_save.assert_called_once()
-
- @patch.object(FakeResource.objects, "filter", MagicMock(return_value=['a', 'b']))
- def test_multiple_models(self):
- """
- [TOSCA_Parser] create_or_update: should raise an exception if multiple instances are found
- """
- data = {
- "name": "test"
- }
- with self.assertRaises(Exception) as e:
- TOSCA_Parser.creat_or_update(FakeResource, data)
- self.assertEqual(e.exception.message, "[XOS-Tosca] Model test has multiple instances, I can't handle it")
-
- @patch.object(FakeResource.objects, "new", MagicMock(return_value=FakeModel))
- @patch.object(FakeResource.objects, "filter")
- @patch.object(FakeModel, "save")
- def test_find_model_without_name_property(self, mock_save, mock_filter):
- """
- [TOSCA_Parser] create_or_update: should lookup a model by the first property
- """
- data = {
- 'foo': 'bar',
- 'something': 'else'
- }
- TOSCA_Parser.creat_or_update(FakeResource, data)
- mock_filter.assert_called_with(foo="bar")
- mock_save.assert_called_once()
-
-if __name__ == '__main__':
- unittest.main()
\ No newline at end of file
+ res = TOSCA_Parser.get_tosca_models_by_name(FakeTemplate)
+ self.assertIsInstance(res['model1'], FakeNode)
+ self.assertIsInstance(res['model2'], FakeNode)
\ No newline at end of file
diff --git a/test/tosca/test.yaml b/test/tosca/test.yaml
index 5f5d6e6..e5b9fbd 100644
--- a/test/tosca/test.yaml
+++ b/test/tosca/test.yaml
@@ -4,6 +4,7 @@
imports:
- custom_types/user.yaml
+ - custom_types/site.yaml
- custom_types/xosguiextension.yaml
topology_template:
@@ -16,14 +17,23 @@
name: test
files: /spa/extensions/test/vendor.js, /spa/extensions/test/app.js
- xosadmin@opencord.org:
+ # Site
+ onlab:
+ type: tosca.nodes.Site
+ properties:
+ name: Open Networking Lab
+ site_url: http://onlab.us/
+ hosts_nodes: True
+
+ # User
+ test@opencord.org:
type: tosca.nodes.User
properties:
- password: rk1UYDHZXbu6KVCMkhmV
- firstname: XOS
- lastname: Admin
-# is_admin: True
-# requirements:
-# - site:
-# node: some
-# relationship: tosca.relationships.MemberOfSite
\ No newline at end of file
+ password: mypwd
+ firstname: User
+ lastname: Test
+ is_admin: True
+ requirements:
+ - site:
+ node: onlab
+ relationship: tosca.relationships.BelongsToOne
\ No newline at end of file