src/tosca/parser.py - xos-tosca - Gitiles

 # Copyright 2017-present Open Networking Foundation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 # http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.

 from __future__ import absolute_import

 import json
 import os
 from tempfile import NamedTemporaryFile

 from grpc._channel import _Rendezvous
 from grpc_client.models_accessor import GRPCModelsAccessor, GRPCModelsException
 from multistructlog import create_logger
 from toscaparser.tosca_template import ToscaTemplate, ValidationError
 from xosconfig import Config

 from .default import TOSCA_RECIPES_DIR

 log = create_logger(Config().get("logging"))


 class TOSCA_Parser:
     def compute_dependencies(self, template, models_by_name):
         """
         NOTE this method is augmenting self.template, isn't there a more explicit way to achieve it?
         """
         for nodetemplate in template.nodetemplates:
             nodetemplate.dependencies = []
             nodetemplate.dependencies_names = []
             for reqs in nodetemplate.requirements:
                 for (k, v) in reqs.items():
                     name = v["node"]
                     if name in models_by_name:
                         nodetemplate.dependencies.append(models_by_name[name])
                         nodetemplate.dependencies_names.append(name)

                     # go another level deep, as our requirements can have requirements...
                     # NOTE do we still need to go deep?
                     for sd_req in v.get("requirements", []):
                         for (sd_req_k, sd_req_v) in sd_req.items():
                             name = sd_req_v["node"]
                             if name in models_by_name:
                                 nodetemplate.dependencies.append(models_by_name[name])
                                 nodetemplate.dependencies_names.append(name)

     @staticmethod
     def topsort_dependencies(g):

         # Get set of all nodes, including those without outgoing edges
         keys = set(g.keys())
         values = set({})
         for v in g.values():
             values = values | set(v.dependencies_names)

         all_nodes = list(keys | values)

         steps = all_nodes

         # Final order
         order = []

         # DFS stack, not using recursion
         stack = []

         # Unmarked set, alpha sorted to provide determinism not present in dicts
         unmarked = sorted(all_nodes)

         # visiting = [] - skip, don't expect 1000s of nodes, |E|/|V| is small

         while unmarked:
             stack.insert(0, unmarked[0])  # push first unmarked

             while stack:
                 n = stack[0]
                 add = True
                 try:
                     for m in g[n].dependencies_names:
                         if m in unmarked:
                             add = False
                             stack.insert(0, m)
                 except KeyError:
                     pass
                 if add:
                     if n in steps and n not in order:
                         order.append(n)
                     item = stack.pop(0)
                     try:
                         unmarked.remove(item)
                     except ValueError:
                         pass

         # remove ordred items from steps, then alpha sort unordered items (determinism)
         noorder = sorted(list(set(steps) - set(order)))
         return order + noorder

     @staticmethod
     def populate_model(model, data):
         for key in sorted(data):
             # NOTE must-exists is a TOSCA implementation choice, remove it before saving the model
             if key != "must-exist":
                 try:
                     setattr(model, key, data[key])
                 except TypeError as e:
                     raise Exception(
                         'Failed to set %s on field %s for class %s, Exception was: "%s"'
                         % (data[key], key, model.model_name, e)
                     )
         return model

     @staticmethod
     def _translate_exception(msg):
         readable = []
         for line in msg.splitlines():
             if (
                 line.strip().startswith("MissingRequiredFieldError")
                 or line.strip().startswith("UnknownFieldError")
                 or line.strip().startswith("ImportError")
                 or line.strip().startswith("InvalidTypeError")
                 or line.strip().startswith("TypeMismatchError")
             ):
                 readable.append(line)

         if len(readable) > 0:
             return "\n".join(readable) + "\n"
         else:
             return msg

     @staticmethod
     def get_tosca_models_by_name(template):
         models_by_name = {}
         for node in template.nodetemplates:
             models_by_name[node.name] = node
         return models_by_name

     @staticmethod
     def get_ordered_models_template(ordered_models_name, templates_by_model_name):
         ordered_models_templates = []
         for name in ordered_models_name:
             if name in templates_by_model_name:
                 ordered_models_templates.append(templates_by_model_name[name])
         return ordered_models_templates

     @staticmethod
     def populate_dependencies(model, requirements, saved_models):
         for dep in requirements:
             class_name = list(dep.keys())[0]
             related_model = saved_models[dep[class_name]["node"]]
             setattr(model, "%s_id" % class_name, related_model.id)
         return model

     @staticmethod
     def add_dependencies(data, requirements, saved_models):
         for dep in requirements:
             class_name = list(dep.keys())[0]
             related_model = saved_models[dep[class_name]["node"]]
             data["%s_id" % class_name] = related_model.id
         return data

     def __init__(self, recipe, username, password, **kwargs):

         self.delete = False
         if "delete" in kwargs:
             self.delete = kwargs["delete"]

         # store username/password combination to read resources
         self.username = username
         self.password = password

         # the template returned by TOSCA-Parser
         self.template = None
         # dictionary containing the models in the recipe and their template
         self.templates_by_model_name = None
         # list of models ordered by requirements
         self.ordered_models_name = []
         # dictionary containing the saved model
         self.saved_model_by_name = {}

         self.ordered_models_template = []

         self.recipe = recipe

     def execute(self):

         try:
             # [] save the recipe to a tmp file
             with NamedTemporaryFile(
                 mode="w", delete=False, suffix=".yaml", dir=TOSCA_RECIPES_DIR
             ) as recipe_file:
                 try:
                     recipe_file.write(self.recipe)
                     recipe_file.close()

                     # [] parse the recipe with TOSCA Parse
                     self.template = ToscaTemplate(recipe_file.name)
                 finally:
                     # [] Make sure the temporary file is cleaned up
                     os.remove(recipe_file.name)

             # [] get all models in the recipe
             self.templates_by_model_name = self.get_tosca_models_by_name(self.template)
             # [] compute requirements
             self.compute_dependencies(self.template, self.templates_by_model_name)
             # [] topsort requirements
             self.ordered_models_name = self.topsort_dependencies(
                 self.templates_by_model_name
             )
             # [] topsort templates
             self.ordered_models_template = self.get_ordered_models_template(
                 self.ordered_models_name, self.templates_by_model_name
             )

             for recipe in self.ordered_models_template:
                 try:
                     # get properties from tosca
                     if "properties" not in recipe.templates[recipe.name]:
                         data = {}
                     else:
                         data = recipe.templates[recipe.name]["properties"]
                         if data is None:
                             data = {}
                     # [] get model by class name
                     class_name = recipe.type.replace("tosca.nodes.", "")

                     # augemnt data with relations
                     data = self.add_dependencies(
                         data, recipe.requirements, self.saved_model_by_name
                     )

                     model = GRPCModelsAccessor.get_model_from_classname(
                         class_name, data, self.username, self.password
                     )
                     # [] populate model with data
                     model = self.populate_model(model, data)
                     # [] check if the model has requirements
                     # [] if it has populate them
                     model = self.populate_dependencies(
                         model, recipe.requirements, self.saved_model_by_name
                     )
                     # [] save, update or delete

                     reference_only = False
                     if "must-exist" in data:
                         reference_only = True

                     if self.delete and not model.is_new and not reference_only:
                         log.info(
                             "[XOS-Tosca] Deleting model %s[%s]" % (class_name, model.id)
                         )
                         model.delete()
                     elif not self.delete:
                         log.info(
                             "[XOS-Tosca] Saving model %s[%s]" % (class_name, model.id)
                         )
                         model.save()

                     self.saved_model_by_name[recipe.name] = model

                 except GRPCModelsException as e:
                     raise Exception(
                         "[XOS-TOSCA] Failed to save or delete model %s [%s]: %s"
                         % (class_name, recipe.name, str(e))
                     )

         except ValidationError as e:
             if e.message:
                 exception_msg = TOSCA_Parser._translate_exception(e.message)
             else:
                 exception_msg = TOSCA_Parser._translate_exception(str(e))
             raise Exception(exception_msg)

         except _Rendezvous as e:
             try:
                 details = json.loads(e._state.details)
                 exception_msg = details["error"]
                 if "specific_error" in details:
                     exception_msg = "%s: %s" % (
                         exception_msg,
                         details["specific_error"],
                     )
             except Exception:
                 exception_msg = e._state.details
             raise Exception(exception_msg)
	# Copyright 2017-present Open Networking Foundation
	#
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.

	from __future__ import absolute_import

	import json
	import os
	from tempfile import NamedTemporaryFile

	from grpc._channel import _Rendezvous
	from grpc_client.models_accessor import GRPCModelsAccessor, GRPCModelsException
	from multistructlog import create_logger
	from toscaparser.tosca_template import ToscaTemplate, ValidationError
	from xosconfig import Config

	from .default import TOSCA_RECIPES_DIR

	log = create_logger(Config().get("logging"))


	class TOSCA_Parser:
	def compute_dependencies(self, template, models_by_name):
	"""
	NOTE this method is augmenting self.template, isn't there a more explicit way to achieve it?
	"""
	for nodetemplate in template.nodetemplates:
	nodetemplate.dependencies = []
	nodetemplate.dependencies_names = []
	for reqs in nodetemplate.requirements:
	for (k, v) in reqs.items():
	name = v["node"]
	if name in models_by_name:
	nodetemplate.dependencies.append(models_by_name[name])
	nodetemplate.dependencies_names.append(name)

	# go another level deep, as our requirements can have requirements...
	# NOTE do we still need to go deep?
	for sd_req in v.get("requirements", []):
	for (sd_req_k, sd_req_v) in sd_req.items():
	name = sd_req_v["node"]
	if name in models_by_name:
	nodetemplate.dependencies.append(models_by_name[name])
	nodetemplate.dependencies_names.append(name)

	@staticmethod
	def topsort_dependencies(g):

	# Get set of all nodes, including those without outgoing edges
	keys = set(g.keys())
	values = set({})
	for v in g.values():
	values = values \| set(v.dependencies_names)

	all_nodes = list(keys \| values)

	steps = all_nodes

	# Final order
	order = []

	# DFS stack, not using recursion
	stack = []

	# Unmarked set, alpha sorted to provide determinism not present in dicts
	unmarked = sorted(all_nodes)

	# visiting = [] - skip, don't expect 1000s of nodes, \|E\|/\|V\| is small

	while unmarked:
	stack.insert(0, unmarked[0]) # push first unmarked

	while stack:
	n = stack[0]
	add = True
	try:
	for m in g[n].dependencies_names:
	if m in unmarked:
	add = False
	stack.insert(0, m)
	except KeyError:
	pass
	if add:
	if n in steps and n not in order:
	order.append(n)
	item = stack.pop(0)
	try:
	unmarked.remove(item)
	except ValueError:
	pass

	# remove ordred items from steps, then alpha sort unordered items (determinism)
	noorder = sorted(list(set(steps) - set(order)))
	return order + noorder

	@staticmethod
	def populate_model(model, data):
	for key in sorted(data):
	# NOTE must-exists is a TOSCA implementation choice, remove it before saving the model
	if key != "must-exist":
	try:
	setattr(model, key, data[key])
	except TypeError as e:
	raise Exception(
	'Failed to set %s on field %s for class %s, Exception was: "%s"'
	% (data[key], key, model.model_name, e)
	)
	return model

	@staticmethod
	def _translate_exception(msg):
	readable = []
	for line in msg.splitlines():
	if (
	line.strip().startswith("MissingRequiredFieldError")
	or line.strip().startswith("UnknownFieldError")
	or line.strip().startswith("ImportError")
	or line.strip().startswith("InvalidTypeError")
	or line.strip().startswith("TypeMismatchError")
	):
	readable.append(line)

	if len(readable) > 0:
	return "\n".join(readable) + "\n"
	else:
	return msg

	@staticmethod
	def get_tosca_models_by_name(template):
	models_by_name = {}
	for node in template.nodetemplates:
	models_by_name[node.name] = node
	return models_by_name

	@staticmethod
	def get_ordered_models_template(ordered_models_name, templates_by_model_name):
	ordered_models_templates = []
	for name in ordered_models_name:
	if name in templates_by_model_name:
	ordered_models_templates.append(templates_by_model_name[name])
	return ordered_models_templates

	@staticmethod
	def populate_dependencies(model, requirements, saved_models):
	for dep in requirements:
	class_name = list(dep.keys())[0]
	related_model = saved_models[dep[class_name]["node"]]
	setattr(model, "%s_id" % class_name, related_model.id)
	return model

	@staticmethod
	def add_dependencies(data, requirements, saved_models):
	for dep in requirements:
	class_name = list(dep.keys())[0]
	related_model = saved_models[dep[class_name]["node"]]
	data["%s_id" % class_name] = related_model.id
	return data

	def __init__(self, recipe, username, password, **kwargs):

	self.delete = False
	if "delete" in kwargs:
	self.delete = kwargs["delete"]

	# store username/password combination to read resources
	self.username = username
	self.password = password

	# the template returned by TOSCA-Parser
	self.template = None
	# dictionary containing the models in the recipe and their template
	self.templates_by_model_name = None
	# list of models ordered by requirements
	self.ordered_models_name = []
	# dictionary containing the saved model
	self.saved_model_by_name = {}

	self.ordered_models_template = []

	self.recipe = recipe

	def execute(self):

	try:
	# [] save the recipe to a tmp file
	with NamedTemporaryFile(
	mode="w", delete=False, suffix=".yaml", dir=TOSCA_RECIPES_DIR
	) as recipe_file:
	try:
	recipe_file.write(self.recipe)
	recipe_file.close()

	# [] parse the recipe with TOSCA Parse
	self.template = ToscaTemplate(recipe_file.name)
	finally:
	# [] Make sure the temporary file is cleaned up
	os.remove(recipe_file.name)

	# [] get all models in the recipe
	self.templates_by_model_name = self.get_tosca_models_by_name(self.template)
	# [] compute requirements
	self.compute_dependencies(self.template, self.templates_by_model_name)
	# [] topsort requirements
	self.ordered_models_name = self.topsort_dependencies(
	self.templates_by_model_name
	)
	# [] topsort templates
	self.ordered_models_template = self.get_ordered_models_template(
	self.ordered_models_name, self.templates_by_model_name
	)

	for recipe in self.ordered_models_template:
	try:
	# get properties from tosca
	if "properties" not in recipe.templates[recipe.name]:
	data = {}
	else:
	data = recipe.templates[recipe.name]["properties"]
	if data is None:
	data = {}
	# [] get model by class name
	class_name = recipe.type.replace("tosca.nodes.", "")

	# augemnt data with relations
	data = self.add_dependencies(
	data, recipe.requirements, self.saved_model_by_name
	)

	model = GRPCModelsAccessor.get_model_from_classname(
	class_name, data, self.username, self.password
	)
	# [] populate model with data
	model = self.populate_model(model, data)
	# [] check if the model has requirements
	# [] if it has populate them
	model = self.populate_dependencies(
	model, recipe.requirements, self.saved_model_by_name
	)
	# [] save, update or delete

	reference_only = False
	if "must-exist" in data:
	reference_only = True

	if self.delete and not model.is_new and not reference_only:
	log.info(
	"[XOS-Tosca] Deleting model %s[%s]" % (class_name, model.id)
	)
	model.delete()
	elif not self.delete:
	log.info(
	"[XOS-Tosca] Saving model %s[%s]" % (class_name, model.id)
	)
	model.save()

	self.saved_model_by_name[recipe.name] = model

	except GRPCModelsException as e:
	raise Exception(
	"[XOS-TOSCA] Failed to save or delete model %s [%s]: %s"
	% (class_name, recipe.name, str(e))
	)

	except ValidationError as e:
	if e.message:
	exception_msg = TOSCA_Parser._translate_exception(e.message)
	else:
	exception_msg = TOSCA_Parser._translate_exception(str(e))
	raise Exception(exception_msg)

	except _Rendezvous as e:
	try:
	details = json.loads(e._state.details)
	exception_msg = details["error"]
	if "specific_error" in details:
	exception_msg = "%s: %s" % (
	exception_msg,
	details["specific_error"],
	)
	except Exception:
	exception_msg = e._state.details
	raise Exception(exception_msg)