blob: 6643ac8e28d3a9d4869b0ee1438b95e020a86f95 [file] [log] [blame]
Matteo Scandolo920e8fd2017-08-08 13:05:24 -07001# Copyright 2017-present Open Networking Foundation
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
Zack Williams6bb2cfe2019-03-27 15:01:45 -070015from __future__ import absolute_import
Matteo Scandolo920e8fd2017-08-08 13:05:24 -070016
Zack Williams6bb2cfe2019-03-27 15:01:45 -070017import json
Test Userfb023562018-10-18 10:56:45 -070018import os
19from tempfile import NamedTemporaryFile
Matteo Scandolo5a07a2c2018-06-05 18:04:00 -070020
Matteo Scandolo21dde412017-07-11 18:54:12 -070021from grpc._channel import _Rendezvous
Zack Williams6bb2cfe2019-03-27 15:01:45 -070022from grpc_client.models_accessor import GRPCModelsAccessor, GRPCModelsException
23from multistructlog import create_logger
24from toscaparser.tosca_template import ToscaTemplate, ValidationError
25from xosconfig import Config
26
27from .default import TOSCA_RECIPES_DIR
28
29log = create_logger(Config().get("logging"))
30
Matteo Scandolo9ce18252017-06-22 10:48:25 -070031
32class TOSCA_Parser:
Matteo Scandolo485b7132017-06-30 11:46:47 -070033 def compute_dependencies(self, template, models_by_name):
34 """
35 NOTE this method is augmenting self.template, isn't there a more explicit way to achieve it?
36 """
37 for nodetemplate in template.nodetemplates:
Matteo Scandolo9ce18252017-06-22 10:48:25 -070038 nodetemplate.dependencies = []
39 nodetemplate.dependencies_names = []
40 for reqs in nodetemplate.requirements:
Zack Williams6bb2cfe2019-03-27 15:01:45 -070041 for (k, v) in reqs.items():
Matteo Scandolo9ce18252017-06-22 10:48:25 -070042 name = v["node"]
Zack Williams6bb2cfe2019-03-27 15:01:45 -070043 if name in models_by_name:
Matteo Scandolo485b7132017-06-30 11:46:47 -070044 nodetemplate.dependencies.append(models_by_name[name])
Matteo Scandolo9ce18252017-06-22 10:48:25 -070045 nodetemplate.dependencies_names.append(name)
46
47 # go another level deep, as our requirements can have requirements...
Matteo Scandolodf2600b2017-07-05 17:01:29 -070048 # NOTE do we still need to go deep?
Zack Williams6bb2cfe2019-03-27 15:01:45 -070049 for sd_req in v.get("requirements", []):
Matteo Scandolo9ce18252017-06-22 10:48:25 -070050 for (sd_req_k, sd_req_v) in sd_req.items():
51 name = sd_req_v["node"]
Zack Williams6bb2cfe2019-03-27 15:01:45 -070052 if name in models_by_name:
Matteo Scandolo485b7132017-06-30 11:46:47 -070053 nodetemplate.dependencies.append(models_by_name[name])
Matteo Scandolo9ce18252017-06-22 10:48:25 -070054 nodetemplate.dependencies_names.append(name)
55
Matteo Scandolo485b7132017-06-30 11:46:47 -070056 @staticmethod
57 def topsort_dependencies(g):
Matteo Scandolo9ce18252017-06-22 10:48:25 -070058
59 # Get set of all nodes, including those without outgoing edges
60 keys = set(g.keys())
61 values = set({})
62 for v in g.values():
63 values = values | set(v.dependencies_names)
64
65 all_nodes = list(keys | values)
Matteo Scandolo9ce18252017-06-22 10:48:25 -070066
Zack Williams6bb2cfe2019-03-27 15:01:45 -070067 steps = all_nodes
Matteo Scandolo9ce18252017-06-22 10:48:25 -070068
69 # Final order
70 order = []
71
72 # DFS stack, not using recursion
73 stack = []
74
Zack Williams6bb2cfe2019-03-27 15:01:45 -070075 # Unmarked set, alpha sorted to provide determinism not present in dicts
76 unmarked = sorted(all_nodes)
Matteo Scandolo9ce18252017-06-22 10:48:25 -070077
78 # visiting = [] - skip, don't expect 1000s of nodes, |E|/|V| is small
79
80 while unmarked:
81 stack.insert(0, unmarked[0]) # push first unmarked
82
Zack Williams6bb2cfe2019-03-27 15:01:45 -070083 while stack:
Matteo Scandolo9ce18252017-06-22 10:48:25 -070084 n = stack[0]
85 add = True
86 try:
87 for m in g[n].dependencies_names:
Zack Williams6bb2cfe2019-03-27 15:01:45 -070088 if m in unmarked:
Matteo Scandolo9ce18252017-06-22 10:48:25 -070089 add = False
90 stack.insert(0, m)
91 except KeyError:
92 pass
Zack Williams6bb2cfe2019-03-27 15:01:45 -070093 if add:
94 if n in steps and n not in order:
Matteo Scandolo9ce18252017-06-22 10:48:25 -070095 order.append(n)
96 item = stack.pop(0)
97 try:
98 unmarked.remove(item)
99 except ValueError:
100 pass
101
Zack Williams6bb2cfe2019-03-27 15:01:45 -0700102 # remove ordred items from steps, then alpha sort unordered items (determinism)
103 noorder = sorted(list(set(steps) - set(order)))
Matteo Scandolo9ce18252017-06-22 10:48:25 -0700104 return order + noorder
105
Matteo Scandolo9ce18252017-06-22 10:48:25 -0700106 @staticmethod
107 def populate_model(model, data):
Zack Williams6bb2cfe2019-03-27 15:01:45 -0700108 for key in sorted(data):
Matteo Scandolo1fedfae2017-10-09 13:57:00 -0700109 # NOTE must-exists is a TOSCA implementation choice, remove it before saving the model
Zack Williams6bb2cfe2019-03-27 15:01:45 -0700110 if key != "must-exist":
Matteo Scandolo5ccdb132018-01-17 14:02:12 -0800111 try:
Zack Williams6bb2cfe2019-03-27 15:01:45 -0700112 setattr(model, key, data[key])
113 except TypeError as e:
114 raise Exception(
115 'Failed to set %s on field %s for class %s, Exception was: "%s"'
116 % (data[key], key, model.model_name, e)
117 )
Matteo Scandolo9ce18252017-06-22 10:48:25 -0700118 return model
119
120 @staticmethod
Matteo Scandolo9ce18252017-06-22 10:48:25 -0700121 def _translate_exception(msg):
122 readable = []
123 for line in msg.splitlines():
Zack Williams6bb2cfe2019-03-27 15:01:45 -0700124 if (
125 line.strip().startswith("MissingRequiredFieldError")
126 or line.strip().startswith("UnknownFieldError")
127 or line.strip().startswith("ImportError")
128 or line.strip().startswith("InvalidTypeError")
129 or line.strip().startswith("TypeMismatchError")
130 ):
Matteo Scandolod12be212017-07-07 10:44:34 -0700131 readable.append(line)
Matteo Scandolo9ce18252017-06-22 10:48:25 -0700132
133 if len(readable) > 0:
Zack Williams6bb2cfe2019-03-27 15:01:45 -0700134 return "\n".join(readable) + "\n"
Matteo Scandolo9ce18252017-06-22 10:48:25 -0700135 else:
136 return msg
137
Matteo Scandolo485b7132017-06-30 11:46:47 -0700138 @staticmethod
139 def get_tosca_models_by_name(template):
140 models_by_name = {}
141 for node in template.nodetemplates:
142 models_by_name[node.name] = node
143 return models_by_name
Matteo Scandolo9ce18252017-06-22 10:48:25 -0700144
Matteo Scandolo485b7132017-06-30 11:46:47 -0700145 @staticmethod
146 def get_ordered_models_template(ordered_models_name, templates_by_model_name):
147 ordered_models_templates = []
148 for name in ordered_models_name:
149 if name in templates_by_model_name:
150 ordered_models_templates.append(templates_by_model_name[name])
151 return ordered_models_templates
152
Matteo Scandolo8bbb03a2017-07-05 14:03:33 -0700153 @staticmethod
154 def populate_dependencies(model, requirements, saved_models):
155 for dep in requirements:
Zack Williams6bb2cfe2019-03-27 15:01:45 -0700156 class_name = list(dep.keys())[0]
157 related_model = saved_models[dep[class_name]["node"]]
Matteo Scandolo8bbb03a2017-07-05 14:03:33 -0700158 setattr(model, "%s_id" % class_name, related_model.id)
159 return model
Matteo Scandolo485b7132017-06-30 11:46:47 -0700160
Matteo Scandolo1bd10762017-10-18 09:53:14 +0200161 @staticmethod
162 def add_dependencies(data, requirements, saved_models):
163 for dep in requirements:
Zack Williams6bb2cfe2019-03-27 15:01:45 -0700164 class_name = list(dep.keys())[0]
165 related_model = saved_models[dep[class_name]["node"]]
Matteo Scandolo1bd10762017-10-18 09:53:14 +0200166 data["%s_id" % class_name] = related_model.id
167 return data
168
Matteo Scandolo78ca3eb2017-07-13 16:58:22 -0700169 def __init__(self, recipe, username, password, **kwargs):
170
171 self.delete = False
Zack Williams6bb2cfe2019-03-27 15:01:45 -0700172 if "delete" in kwargs:
173 self.delete = kwargs["delete"]
Matteo Scandolo21dde412017-07-11 18:54:12 -0700174
175 # store username/password combination to read resources
176 self.username = username
177 self.password = password
Matteo Scandolo485b7132017-06-30 11:46:47 -0700178
179 # the template returned by TOSCA-Parser
180 self.template = None
181 # dictionary containing the models in the recipe and their template
182 self.templates_by_model_name = None
183 # list of models ordered by requirements
Matteo Scandolodf2600b2017-07-05 17:01:29 -0700184 self.ordered_models_name = []
Matteo Scandolo8bbb03a2017-07-05 14:03:33 -0700185 # dictionary containing the saved model
186 self.saved_model_by_name = {}
Matteo Scandolo485b7132017-06-30 11:46:47 -0700187
188 self.ordered_models_template = []
Matteo Scandolo485b7132017-06-30 11:46:47 -0700189
Matteo Scandolodf2600b2017-07-05 17:01:29 -0700190 self.recipe = recipe
191
192 def execute(self):
Matteo Scandolo78ca3eb2017-07-13 16:58:22 -0700193
Matteo Scandolo485b7132017-06-30 11:46:47 -0700194 try:
195 # [] save the recipe to a tmp file
Zack Williams6bb2cfe2019-03-27 15:01:45 -0700196 with NamedTemporaryFile(
197 mode="w", delete=False, suffix=".yaml", dir=TOSCA_RECIPES_DIR
198 ) as recipe_file:
Test Userfb023562018-10-18 10:56:45 -0700199 try:
200 recipe_file.write(self.recipe)
201 recipe_file.close()
202
203 # [] parse the recipe with TOSCA Parse
204 self.template = ToscaTemplate(recipe_file.name)
205 finally:
206 # [] Make sure the temporary file is cleaned up
207 os.remove(recipe_file.name)
208
Matteo Scandolo485b7132017-06-30 11:46:47 -0700209 # [] get all models in the recipe
210 self.templates_by_model_name = self.get_tosca_models_by_name(self.template)
211 # [] compute requirements
212 self.compute_dependencies(self.template, self.templates_by_model_name)
213 # [] topsort requirements
Zack Williams6bb2cfe2019-03-27 15:01:45 -0700214 self.ordered_models_name = self.topsort_dependencies(
215 self.templates_by_model_name
216 )
Matteo Scandolo485b7132017-06-30 11:46:47 -0700217 # [] topsort templates
Zack Williams6bb2cfe2019-03-27 15:01:45 -0700218 self.ordered_models_template = self.get_ordered_models_template(
219 self.ordered_models_name, self.templates_by_model_name
220 )
Matteo Scandolo485b7132017-06-30 11:46:47 -0700221
222 for recipe in self.ordered_models_template:
Matteo Scandolo1fedfae2017-10-09 13:57:00 -0700223 try:
224 # get properties from tosca
Zack Williams6bb2cfe2019-03-27 15:01:45 -0700225 if "properties" not in recipe.templates[recipe.name]:
Matteo Scandolo1fedfae2017-10-09 13:57:00 -0700226 data = {}
227 else:
Zack Williams6bb2cfe2019-03-27 15:01:45 -0700228 data = recipe.templates[recipe.name]["properties"]
229 if data is None:
Matteo Scandolo1fedfae2017-10-09 13:57:00 -0700230 data = {}
231 # [] get model by class name
232 class_name = recipe.type.replace("tosca.nodes.", "")
Matteo Scandolo1bd10762017-10-18 09:53:14 +0200233
234 # augemnt data with relations
Zack Williams6bb2cfe2019-03-27 15:01:45 -0700235 data = self.add_dependencies(
236 data, recipe.requirements, self.saved_model_by_name
237 )
Matteo Scandolo1bd10762017-10-18 09:53:14 +0200238
Zack Williams6bb2cfe2019-03-27 15:01:45 -0700239 model = GRPCModelsAccessor.get_model_from_classname(
240 class_name, data, self.username, self.password
241 )
Matteo Scandolo1fedfae2017-10-09 13:57:00 -0700242 # [] populate model with data
243 model = self.populate_model(model, data)
244 # [] check if the model has requirements
245 # [] if it has populate them
Zack Williams6bb2cfe2019-03-27 15:01:45 -0700246 model = self.populate_dependencies(
247 model, recipe.requirements, self.saved_model_by_name
248 )
Matteo Scandolo1fedfae2017-10-09 13:57:00 -0700249 # [] save, update or delete
Matteo Scandolo9ce18252017-06-22 10:48:25 -0700250
Matteo Scandolo91d7aeb2018-02-09 14:54:09 -0800251 reference_only = False
Zack Williams6bb2cfe2019-03-27 15:01:45 -0700252 if "must-exist" in data:
Matteo Scandolo91d7aeb2018-02-09 14:54:09 -0800253 reference_only = True
254
255 if self.delete and not model.is_new and not reference_only:
Zack Williams6bb2cfe2019-03-27 15:01:45 -0700256 log.info(
257 "[XOS-Tosca] Deleting model %s[%s]" % (class_name, model.id)
258 )
Matteo Scandolo1fedfae2017-10-09 13:57:00 -0700259 model.delete()
260 elif not self.delete:
Zack Williams6bb2cfe2019-03-27 15:01:45 -0700261 log.info(
262 "[XOS-Tosca] Saving model %s[%s]" % (class_name, model.id)
263 )
Matteo Scandolo1fedfae2017-10-09 13:57:00 -0700264 model.save()
265
266 self.saved_model_by_name[recipe.name] = model
Zack Williams6bb2cfe2019-03-27 15:01:45 -0700267
268 except GRPCModelsException as e:
269 raise Exception(
270 "[XOS-TOSCA] Failed to save or delete model %s [%s]: %s"
271 % (class_name, recipe.name, str(e))
272 )
Matteo Scandolo8bbb03a2017-07-05 14:03:33 -0700273
Matteo Scandolod12be212017-07-07 10:44:34 -0700274 except ValidationError as e:
Matteo Scandolo485b7132017-06-30 11:46:47 -0700275 if e.message:
Matteo Scandolod12be212017-07-07 10:44:34 -0700276 exception_msg = TOSCA_Parser._translate_exception(e.message)
Matteo Scandolo485b7132017-06-30 11:46:47 -0700277 else:
Matteo Scandolod12be212017-07-07 10:44:34 -0700278 exception_msg = TOSCA_Parser._translate_exception(str(e))
Matteo Scandolo485b7132017-06-30 11:46:47 -0700279 raise Exception(exception_msg)
Matteo Scandolo9ce18252017-06-22 10:48:25 -0700280
Zack Williams6bb2cfe2019-03-27 15:01:45 -0700281 except _Rendezvous as e:
Matteo Scandolo21dde412017-07-11 18:54:12 -0700282 try:
Matteo Scandoloc4680752018-05-17 12:16:30 -0700283 details = json.loads(e._state.details)
284 exception_msg = details["error"]
285 if "specific_error" in details:
Zack Williams6bb2cfe2019-03-27 15:01:45 -0700286 exception_msg = "%s: %s" % (
287 exception_msg,
288 details["specific_error"],
289 )
Matteo Scandolo21dde412017-07-11 18:54:12 -0700290 except Exception:
291 exception_msg = e._state.details
292 raise Exception(exception_msg)