[SEBA-497]
Change to using alpine-grpc-base
Fix issue with nested exceptions causing an error in structlog
Reformat and python3 fixes, v3.5 mock support
Record execution times in the loader
Change-Id: I6d7923818d57012fca32ce44668820de422206d6
diff --git a/src/__init__.py b/src/__init__.py
new file mode 100644
index 0000000..19d1424
--- /dev/null
+++ b/src/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2019-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/src/grpc_client/KEYS.reference.py b/src/grpc_client/KEYS.reference.py
index fd8c776..6453117 100644
--- a/src/grpc_client/KEYS.reference.py
+++ b/src/grpc_client/KEYS.reference.py
@@ -19,57 +19,57 @@
#########################################################################
TOSCA_KEYS = {
- 'XOSBase': [],
- 'User': ['email'],
- 'Privilege': [],
- 'AddressPool': ['name'],
- 'Controller': ['name'],
- 'ControllerImages': [],
- 'ControllerNetwork': [],
- 'ControllerRole': [],
- 'ControllerSite': [],
- 'ControllerPrivilege': [],
- 'ControllerSitePrivilege': [],
- 'ControllerSlice': [],
- 'ControllerSlicePrivilege': [],
- 'ControllerUser': [],
- 'Deployment': ['name'],
- 'DeploymentPrivilege': [],
- 'DeploymentRole': [],
- 'Diag': ['name'],
- 'Flavor': ['name'],
- 'Image': ['name'],
- 'ImageDeployments': [],
- 'Instance': ['name'],
- 'Network': ['name'],
- 'NetworkParameter': [],
- 'NetworkParameterType': ['name'],
- 'NetworkSlice': ['network', 'slice'],
- 'NetworkTemplate': ['name'],
- 'Node': ['name'],
- 'NodeLabel': ['name'],
- 'Port': [],
- 'Role': [],
- 'Service': ['name'],
- 'ServiceAttribute': ['name'],
- 'ServiceDependency': ['provider_service'],
- 'ServiceMonitoringAgentInfo': ['name'],
- 'ServicePrivilege': [],
- 'ServiceRole': [],
- 'Site': ['name'],
- 'SiteDeployment': ['site', 'deployment'],
- 'SitePrivilege': ['site', 'role'],
- 'SiteRole': [],
- 'Slice': ['name'],
- 'SlicePrivilege': [],
- 'SliceRole': [],
- 'Tag': ['name'],
- 'InterfaceType': ['name'],
- 'ServiceInterface': ['service', 'interface_type'],
- 'ServiceInstance': ['name'],
- 'ServiceInstanceLink': ['provider_service_instance'],
- 'ServiceInstanceAttribute': ['name'],
- 'TenantWithContainer': ['name'],
- 'XOS': ['name'],
- 'XOSGuiExtension': ['name'],
-}
\ No newline at end of file
+ "XOSBase": [],
+ "User": ["email"],
+ "Privilege": [],
+ "AddressPool": ["name"],
+ "Controller": ["name"],
+ "ControllerImages": [],
+ "ControllerNetwork": [],
+ "ControllerRole": [],
+ "ControllerSite": [],
+ "ControllerPrivilege": [],
+ "ControllerSitePrivilege": [],
+ "ControllerSlice": [],
+ "ControllerSlicePrivilege": [],
+ "ControllerUser": [],
+ "Deployment": ["name"],
+ "DeploymentPrivilege": [],
+ "DeploymentRole": [],
+ "Diag": ["name"],
+ "Flavor": ["name"],
+ "Image": ["name"],
+ "ImageDeployments": [],
+ "Instance": ["name"],
+ "Network": ["name"],
+ "NetworkParameter": [],
+ "NetworkParameterType": ["name"],
+ "NetworkSlice": ["network", "slice"],
+ "NetworkTemplate": ["name"],
+ "Node": ["name"],
+ "NodeLabel": ["name"],
+ "Port": [],
+ "Role": [],
+ "Service": ["name"],
+ "ServiceAttribute": ["name"],
+ "ServiceDependency": ["provider_service"],
+ "ServiceMonitoringAgentInfo": ["name"],
+ "ServicePrivilege": [],
+ "ServiceRole": [],
+ "Site": ["name"],
+ "SiteDeployment": ["site", "deployment"],
+ "SitePrivilege": ["site", "role"],
+ "SiteRole": [],
+ "Slice": ["name"],
+ "SlicePrivilege": [],
+ "SliceRole": [],
+ "Tag": ["name"],
+ "InterfaceType": ["name"],
+ "ServiceInterface": ["service", "interface_type"],
+ "ServiceInstance": ["name"],
+ "ServiceInstanceLink": ["provider_service_instance"],
+ "ServiceInstanceAttribute": ["name"],
+ "TenantWithContainer": ["name"],
+ "XOS": ["name"],
+ "XOSGuiExtension": ["name"],
+}
diff --git a/src/grpc_client/__init__.py b/src/grpc_client/__init__.py
index d4e8062..c7879fb 100644
--- a/src/grpc_client/__init__.py
+++ b/src/grpc_client/__init__.py
@@ -1,4 +1,3 @@
-
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,4 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
+from .main import GRPC_Client
+
+__all__ = [
+ 'GRPC_Client',
+]
diff --git a/src/grpc_client/main.py b/src/grpc_client/main.py
index 619b43b..314081f 100644
--- a/src/grpc_client/main.py
+++ b/src/grpc_client/main.py
@@ -1,4 +1,3 @@
-
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,23 +13,27 @@
# limitations under the License.
-import functools
-from xosapi.xos_grpc_client import SecureClient, InsecureClient
-from twisted.internet import defer
-from resources import RESOURCES
-from xosconfig import Config
-from twisted.internet import reactor
+from __future__ import absolute_import
-from xosconfig import Config
+import functools
+
+from xosapi.xos_grpc_client import InsecureClient, SecureClient
+
from multistructlog import create_logger
-log = create_logger(Config().get('logging'))
+from twisted.internet import defer, reactor
+from xosconfig import Config
+
+from .resources import RESOURCES
+
+log = create_logger(Config().get("logging"))
+
class GRPC_Client:
def __init__(self):
self.client = None
- insecure = Config.get('gprc_endpoint')
- secure = Config.get('gprc_endpoint')
+ insecure = Config.get("gprc_endpoint")
+ secure = Config.get("gprc_endpoint")
self.grpc_secure_endpoint = secure + ":50051"
self.grpc_insecure_endpoint = insecure + ":50055"
@@ -55,26 +58,36 @@
self.client = InsecureClient(endpoint=self.grpc_insecure_endpoint)
self.client.restart_on_disconnect = True
- self.client.set_reconnect_callback(functools.partial(deferred.callback, self.client))
+ self.client.set_reconnect_callback(
+ functools.partial(deferred.callback, self.client)
+ )
self.client.start()
return deferred
def create_secure_client(self, username, password, arg):
"""
- This method will check if this combination of username/password already has stored orm classes in RESOURCES, otherwise create them
+ This method will check if this combination of username/password already
+ has stored orm classes in RESOURCES, otherwise create them
"""
deferred = defer.Deferred()
key = "%s~%s" % (username, password)
if key in RESOURCES:
reactor.callLater(0, deferred.callback, arg)
else:
- local_cert = Config.get('local_cert')
- client = SecureClient(endpoint=self.grpc_secure_endpoint, username=username, password=password, cacert=local_cert)
+ local_cert = Config.get("local_cert")
+ client = SecureClient(
+ endpoint=self.grpc_secure_endpoint,
+ username=username,
+ password=password,
+ cacert=local_cert,
+ )
client.restart_on_disconnect = True
# SecureClient is preceeded by an insecure client, so treat all secure clients as previously connected
# See CORD-3152
client.was_connected = True
- client.set_reconnect_callback(functools.partial(self.setup_resources, client, key, deferred, arg))
+ client.set_reconnect_callback(
+ functools.partial(self.setup_resources, client, key, deferred, arg)
+ )
client.start()
return deferred
diff --git a/src/grpc_client/models_accessor.py b/src/grpc_client/models_accessor.py
index 2c96c48..445918a 100644
--- a/src/grpc_client/models_accessor.py
+++ b/src/grpc_client/models_accessor.py
@@ -1,4 +1,3 @@
-
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,11 +12,22 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from xosconfig import Config
-from multistructlog import create_logger
-log = create_logger(Config().get('logging'))
+from __future__ import absolute_import
-from resources import RESOURCES
+from multistructlog import create_logger
+from xosconfig import Config
+
+from .resources import RESOURCES
+
+log = create_logger(Config().get("logging"))
+
+
+class GRPCModelsException(Exception):
+ """
+ Differentiates between exceptions created by GRPCModelsAccessor and other exceptions
+ """
+ pass
+
class GRPCModelsAccessor:
"""
@@ -31,24 +41,31 @@
"""
# NOTE: we need to import this later as it's generated by main.py
- from KEYS import TOSCA_KEYS
+ from .KEYS import TOSCA_KEYS
# get the key for this model
try:
filter_keys = TOSCA_KEYS[class_name]
- except KeyError, e:
- raise Exception("[XOS-TOSCA] Model %s doesn't have a tosca_key specified" % (class_name))
+ except KeyError:
+ raise GRPCModelsException(
+ "[XOS-TOSCA] Model %s doesn't have a tosca_key specified" % (class_name)
+ )
if len(filter_keys) == 0:
- raise Exception("[XOS-TOSCA] Model %s doesn't have a tosca_key specified" % (class_name))
+ raise GRPCModelsException(
+ "[XOS-TOSCA] Model %s doesn't have a tosca_key specified" % (class_name)
+ )
filter = {}
for k in filter_keys:
if isinstance(k, str):
try:
filter[k] = data[k]
- except KeyError, e:
- raise Exception("[XOS-TOSCA] Model %s doesn't have a property for the specified tosca_key (%s)" % (class_name, e))
+ except KeyError as e:
+ raise GRPCModelsException(
+ "[XOS-TOSCA] Model %s doesn't have a property for the specified tosca_key (%s)"
+ % (class_name, e)
+ )
elif isinstance(k, list):
# one of they keys in this list has to be set
one_of_key = None
@@ -57,15 +74,23 @@
one_of_key = i
one_of_key_val = data[i]
if not one_of_key:
- raise Exception("[XOS-TOSCA] Model %s doesn't have a property for the specified tosca_key_one_of (%s)" % (class_name, k))
+ raise GRPCModelsException(
+ "[XOS-TOSCA] Model %s doesn't have a property for the specified tosca_key_one_of (%s)"
+ % (class_name, k)
+ )
else:
filter[one_of_key] = one_of_key_val
key = "%s~%s" % (username, password)
- if not key in RESOURCES:
- raise Exception("[XOS-TOSCA] User '%s' does not have ready resources" % username)
+ if key not in RESOURCES:
+ raise GRPCModelsException(
+ "[XOS-TOSCA] User '%s' does not have ready resources" % username
+ )
if class_name not in RESOURCES[key]:
- raise Exception('[XOS-TOSCA] The model you are trying to create (class: %s, properties, %s) is not know by xos-core' % (class_name, str(filter)))
+ raise GRPCModelsException(
+ "[XOS-TOSCA] The model you are trying to create (class: %s, properties, %s) is not know by xos-core"
+ % (class_name, str(filter))
+ )
cls = RESOURCES[key][class_name]
@@ -73,15 +98,27 @@
if len(models) == 1:
model = models[0]
- log.info("[XOS-Tosca] Model of class %s and properties %s already exist, retrieving instance..." % (class_name, str(filter)), model=model)
+ log.info(
+ "[XOS-TOSCA] Model of class %s and properties %s already exist, retrieving instance..."
+ % (class_name, str(filter)),
+ model=model,
+ )
elif len(models) == 0:
- if 'must-exist' in data and data['must-exist']:
- raise Exception("[XOS-TOSCA] Model of class %s and properties %s has property 'must-exist' but cannot be found" % (class_name, str(filter)))
+ if "must-exist" in data and data["must-exist"]:
+ raise GRPCModelsException(
+ "[XOS-TOSCA] Model of class %s and properties %s has property 'must-exist' but cannot be found"
+ % (class_name, str(filter))
+ )
model = cls.objects.new()
- log.info("[XOS-Tosca] Model (%s) is new, creating new instance..." % str(filter))
+ log.info(
+ "[XOS-TOSCA] Model (%s) is new, creating new instance..." % str(filter)
+ )
else:
- raise Exception("[XOS-Tosca] Model of class %s and properties %s has multiple instances, I can't handle it" % (class_name, str(filter)))
+ raise GRPCModelsException(
+ "[XOS-TOSCA] Model of class %s and properties %s has multiple instances, I can't handle it"
+ % (class_name, str(filter))
+ )
- return model
\ No newline at end of file
+ return model
diff --git a/src/grpc_client/resources.py b/src/grpc_client/resources.py
index cfe3e2e..88ce137 100644
--- a/src/grpc_client/resources.py
+++ b/src/grpc_client/resources.py
@@ -1,4 +1,3 @@
-
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,4 +14,4 @@
# NOTE will add all the resources in this dictionary
-RESOURCES = {}
\ No newline at end of file
+RESOURCES = {}
diff --git a/src/main.py b/src/main.py
index 6b0359d..94955b0 100644
--- a/src/main.py
+++ b/src/main.py
@@ -1,4 +1,3 @@
-
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,26 +12,28 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
import os
+
+from twisted.internet import defer
from xosconfig import Config
from multistructlog import create_logger
current_dir = os.path.dirname(os.path.realpath(__file__))
-config_file = os.path.join(current_dir, 'xos-tosca.config.yaml')
-config_schema = os.path.join(current_dir, 'xos-tosca-config-schema.yaml')
+config_file = os.path.join(current_dir, "xos-tosca.config.yaml")
+config_schema = os.path.join(current_dir, "xos-tosca-config-schema.yaml")
Config.init(config_file, config_schema)
-log = create_logger(Config().get('logging'))
+log = create_logger(Config().get("logging"))
-from grpc_client.main import GRPC_Client
-from tosca.generator import TOSCA_Generator
-from web_server.main import TOSCA_WebServer
-from twisted.internet import defer
+# config needs to be init before these imports, thus E402
+from grpc_client import GRPC_Client # noqa: E402
+from tosca import TOSCA_Generator # noqa: E402
+from web_server import TOSCA_WebServer # noqa: E402
class Main:
-
def __init__(self):
self.grpc_client = None
@@ -47,9 +48,11 @@
def start(self):
log.info("[XOS-TOSCA] Starting")
- # Remove generated TOSCA and KEYS that may have been downloaded by a previous session. This is done here, rather
- # than in the generator, to cover the case where the TOSCA engine is restarted and a web request is received
- # and processed before generate_tosca() has completed.
+ # Remove generated TOSCA and KEYS that may have been downloaded by a
+ # previous session. This is done here, rather than in the generator, to
+ # cover the case where the TOSCA engine is restarted and a web request
+ # is received and processed before generate_tosca() has completed.
+
TOSCA_Generator().clean()
TOSCA_Generator().clean_keys()
@@ -60,5 +63,5 @@
TOSCA_WebServer()
-if __name__ == '__main__':
- Main().start()
\ No newline at end of file
+if __name__ == "__main__":
+ Main().start()
diff --git a/src/tosca/__init__.py b/src/tosca/__init__.py
index d4e8062..cbdb48b 100644
--- a/src/tosca/__init__.py
+++ b/src/tosca/__init__.py
@@ -1,4 +1,3 @@
-
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,4 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
+from .generator import TOSCA_Generator
+
+__all__ = [
+ 'TOSCA_Generator',
+]
diff --git a/src/tosca/default.py b/src/tosca/default.py
index 4de0975..02d7de4 100644
--- a/src/tosca/default.py
+++ b/src/tosca/default.py
@@ -1,4 +1,3 @@
-
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,8 +13,12 @@
# limitations under the License.
+from __future__ import absolute_import
+
import os
TOSCA_DEFS_DIR = os.path.dirname(os.path.realpath(__file__)) + "/custom_types"
TOSCA_RECIPES_DIR = os.path.dirname(os.path.realpath(__file__)) + "/"
-TOSCA_KEYS_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/../grpc_client")
\ No newline at end of file
+TOSCA_KEYS_DIR = os.path.abspath(
+ os.path.dirname(os.path.realpath(__file__)) + "/../grpc_client"
+)
diff --git a/src/tosca/generator.py b/src/tosca/generator.py
index 1c2dccb..fbaf4e5 100644
--- a/src/tosca/generator.py
+++ b/src/tosca/generator.py
@@ -1,4 +1,3 @@
-
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,24 +12,30 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from xosconfig import Config
-from multistructlog import create_logger
-log = create_logger(Config().get('logging'))
+from __future__ import absolute_import
import os
-from default import TOSCA_DEFS_DIR, TOSCA_KEYS_DIR
-from xosgenx.generator import XOSProcessor, XOSProcessorArgs
+
from xosapi.xos_grpc_client import Empty
+from xosgenx.generator import XOSProcessor, XOSProcessorArgs
+
+from multistructlog import create_logger
+from xosconfig import Config
+
+from .default import TOSCA_DEFS_DIR, TOSCA_KEYS_DIR
+
+log = create_logger(Config().get("logging"))
+
current_dir = os.path.dirname(os.path.realpath(__file__))
-class TOSCA_Generator:
+class TOSCA_Generator:
def clean(self, dir=TOSCA_DEFS_DIR):
filesToRemove = [f for f in os.listdir(dir)]
for f in filesToRemove:
- if not f.startswith('.'):
- os.remove(dir + '/' + f)
+ if not f.startswith("."):
+ os.remove(dir + "/" + f)
def clean_keys(self, dir=TOSCA_KEYS_DIR):
keys_fn = os.path.join(dir, "KEYS.py")
@@ -42,23 +47,27 @@
try:
xproto = client.utility.GetXproto(Empty())
- args = XOSProcessorArgs(output = TOSCA_DEFS_DIR,
- inputs = str(xproto.xproto),
- target = os.path.join(current_dir, 'xtarget/tosca.xtarget'),
- write_to_file = 'target')
+ args = XOSProcessorArgs(
+ output=TOSCA_DEFS_DIR,
+ inputs=str(xproto.xproto),
+ target=os.path.join(current_dir, "xtarget/tosca.xtarget"),
+ write_to_file="target",
+ )
XOSProcessor.process(args)
log.info("[XOS-TOSCA] Recipes generated in %s" % args.output)
- except Exception as e:
+ except Exception:
log.exception("[XOS-TOSCA] Failed to generate TOSCA")
try:
xproto = client.utility.GetXproto(Empty())
- args = XOSProcessorArgs(output = TOSCA_KEYS_DIR,
- inputs = str(xproto.xproto),
- target = os.path.join(current_dir, 'xtarget/tosca_keys.xtarget'),
- write_to_file = 'single',
- dest_file = 'KEYS.py')
+ args = XOSProcessorArgs(
+ output=TOSCA_KEYS_DIR,
+ inputs=str(xproto.xproto),
+ target=os.path.join(current_dir, "xtarget/tosca_keys.xtarget"),
+ write_to_file="single",
+ dest_file="KEYS.py",
+ )
XOSProcessor.process(args)
log.info("[XOS-TOSCA] TOSCA Keys generated in %s" % args.output)
- except Exception as e:
+ except Exception:
log.exception("[XOS-TOSCA] Failed to generate TOSCA Keys")
diff --git a/src/tosca/parser.py b/src/tosca/parser.py
index 3f44c50..6643ac8 100644
--- a/src/tosca/parser.py
+++ b/src/tosca/parser.py
@@ -1,4 +1,3 @@
-
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,23 +12,24 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
+import json
import os
from tempfile import NamedTemporaryFile
-from xosconfig import Config
-from multistructlog import create_logger
-log = create_logger(Config().get('logging'))
-from toscaparser.tosca_template import ToscaTemplate, ValidationError
-from default import TOSCA_RECIPES_DIR
-from grpc_client.resources import RESOURCES
-from grpc_client.models_accessor import GRPCModelsAccessor
from grpc._channel import _Rendezvous
-import json
-import traceback
+from grpc_client.models_accessor import GRPCModelsAccessor, GRPCModelsException
+from multistructlog import create_logger
+from toscaparser.tosca_template import ToscaTemplate, ValidationError
+from xosconfig import Config
+
+from .default import TOSCA_RECIPES_DIR
+
+log = create_logger(Config().get("logging"))
+
class TOSCA_Parser:
-
def compute_dependencies(self, template, models_by_name):
"""
NOTE this method is augmenting self.template, isn't there a more explicit way to achieve it?
@@ -38,18 +38,18 @@
nodetemplate.dependencies = []
nodetemplate.dependencies_names = []
for reqs in nodetemplate.requirements:
- for (k,v) in reqs.items():
+ for (k, v) in reqs.items():
name = v["node"]
- if (name in models_by_name):
+ if name in models_by_name:
nodetemplate.dependencies.append(models_by_name[name])
nodetemplate.dependencies_names.append(name)
# go another level deep, as our requirements can have requirements...
# NOTE do we still need to go deep?
- for sd_req in v.get("requirements",[]):
+ for sd_req in v.get("requirements", []):
for (sd_req_k, sd_req_v) in sd_req.items():
name = sd_req_v["node"]
- if (name in models_by_name):
+ if name in models_by_name:
nodetemplate.dependencies.append(models_by_name[name])
nodetemplate.dependencies_names.append(name)
@@ -63,8 +63,8 @@
values = values | set(v.dependencies_names)
all_nodes = list(keys | values)
- steps = all_nodes
+ steps = all_nodes
# Final order
order = []
@@ -72,26 +72,26 @@
# DFS stack, not using recursion
stack = []
- # Unmarked set
- unmarked = all_nodes
+ # Unmarked set, alpha sorted to provide determinism not present in dicts
+ unmarked = sorted(all_nodes)
# visiting = [] - skip, don't expect 1000s of nodes, |E|/|V| is small
while unmarked:
stack.insert(0, unmarked[0]) # push first unmarked
- while (stack):
+ while stack:
n = stack[0]
add = True
try:
for m in g[n].dependencies_names:
- if (m in unmarked):
+ if m in unmarked:
add = False
stack.insert(0, m)
except KeyError:
pass
- if (add):
- if (n in steps and n not in order):
+ if add:
+ if n in steps and n not in order:
order.append(n)
item = stack.pop(0)
try:
@@ -99,33 +99,39 @@
except ValueError:
pass
- noorder = list(set(steps) - set(order))
+ # remove ordred items from steps, then alpha sort unordered items (determinism)
+ noorder = sorted(list(set(steps) - set(order)))
return order + noorder
@staticmethod
def populate_model(model, data):
- for k,v in data.iteritems():
+ for key in sorted(data):
# NOTE must-exists is a TOSCA implementation choice, remove it before saving the model
- if k != "must-exist":
+ if key != "must-exist":
try:
- setattr(model, k, v)
- except TypeError, e:
- raise Exception('Failed to set %s on field %s for class %s, Exception was: "%s"' % (v, k, model.model_name, e))
+ setattr(model, key, data[key])
+ except TypeError as e:
+ raise Exception(
+ 'Failed to set %s on field %s for class %s, Exception was: "%s"'
+ % (data[key], key, model.model_name, e)
+ )
return model
@staticmethod
def _translate_exception(msg):
readable = []
for line in msg.splitlines():
- if line.strip().startswith('MissingRequiredFieldError') or \
- line.strip().startswith('UnknownFieldError') or \
- line.strip().startswith('ImportError') or \
- line.strip().startswith('InvalidTypeError') or \
- line.strip().startswith('TypeMismatchError'):
+ if (
+ line.strip().startswith("MissingRequiredFieldError")
+ or line.strip().startswith("UnknownFieldError")
+ or line.strip().startswith("ImportError")
+ or line.strip().startswith("InvalidTypeError")
+ or line.strip().startswith("TypeMismatchError")
+ ):
readable.append(line)
if len(readable) > 0:
- return '\n'.join(readable) + '\n'
+ return "\n".join(readable) + "\n"
else:
return msg
@@ -147,24 +153,24 @@
@staticmethod
def populate_dependencies(model, requirements, saved_models):
for dep in requirements:
- class_name = dep.keys()[0]
- related_model = saved_models[dep[class_name]['node']]
+ class_name = list(dep.keys())[0]
+ related_model = saved_models[dep[class_name]["node"]]
setattr(model, "%s_id" % class_name, related_model.id)
return model
@staticmethod
def add_dependencies(data, requirements, saved_models):
for dep in requirements:
- class_name = dep.keys()[0]
- related_model = saved_models[dep[class_name]['node']]
+ class_name = list(dep.keys())[0]
+ related_model = saved_models[dep[class_name]["node"]]
data["%s_id" % class_name] = related_model.id
return data
def __init__(self, recipe, username, password, **kwargs):
self.delete = False
- if 'delete' in kwargs:
- self.delete = True
+ if "delete" in kwargs:
+ self.delete = kwargs["delete"]
# store username/password combination to read resources
self.username = username
@@ -187,7 +193,9 @@
try:
# [] save the recipe to a tmp file
- with NamedTemporaryFile(delete=False, suffix=".yaml", dir=TOSCA_RECIPES_DIR) as recipe_file:
+ with NamedTemporaryFile(
+ mode="w", delete=False, suffix=".yaml", dir=TOSCA_RECIPES_DIR
+ ) as recipe_file:
try:
recipe_file.write(self.recipe)
recipe_file.close()
@@ -203,49 +211,65 @@
# [] compute requirements
self.compute_dependencies(self.template, self.templates_by_model_name)
# [] topsort requirements
- self.ordered_models_name = self.topsort_dependencies(self.templates_by_model_name)
+ self.ordered_models_name = self.topsort_dependencies(
+ self.templates_by_model_name
+ )
# [] topsort templates
- self.ordered_models_template = self.get_ordered_models_template(self.ordered_models_name, self.templates_by_model_name)
+ self.ordered_models_template = self.get_ordered_models_template(
+ self.ordered_models_name, self.templates_by_model_name
+ )
for recipe in self.ordered_models_template:
try:
# get properties from tosca
- if not 'properties' in recipe.templates[recipe.name]:
+ if "properties" not in recipe.templates[recipe.name]:
data = {}
else:
- data = recipe.templates[recipe.name]['properties']
- if data == None:
+ data = recipe.templates[recipe.name]["properties"]
+ if data is None:
data = {}
# [] get model by class name
class_name = recipe.type.replace("tosca.nodes.", "")
# augemnt data with relations
- data = self.add_dependencies(data, recipe.requirements, self.saved_model_by_name)
+ data = self.add_dependencies(
+ data, recipe.requirements, self.saved_model_by_name
+ )
- model = GRPCModelsAccessor.get_model_from_classname(class_name, data, self.username, self.password)
+ model = GRPCModelsAccessor.get_model_from_classname(
+ class_name, data, self.username, self.password
+ )
# [] populate model with data
model = self.populate_model(model, data)
# [] check if the model has requirements
# [] if it has populate them
- model = self.populate_dependencies(model, recipe.requirements, self.saved_model_by_name)
+ model = self.populate_dependencies(
+ model, recipe.requirements, self.saved_model_by_name
+ )
# [] save, update or delete
reference_only = False
- if 'must-exist' in data:
+ if "must-exist" in data:
reference_only = True
if self.delete and not model.is_new and not reference_only:
- log.info("[XOS-Tosca] Deleting model %s[%s]" % (class_name, model.id))
+ log.info(
+ "[XOS-Tosca] Deleting model %s[%s]" % (class_name, model.id)
+ )
model.delete()
elif not self.delete:
- log.info("[XOS-Tosca] Saving model %s[%s]" % (class_name, model.id))
+ log.info(
+ "[XOS-Tosca] Saving model %s[%s]" % (class_name, model.id)
+ )
model.save()
-
self.saved_model_by_name[recipe.name] = model
- except Exception, e:
- log.exception("[XOS-TOSCA] Failed to save model: %s [%s]" % (class_name, recipe.name))
- raise e
+
+ except GRPCModelsException as e:
+ raise Exception(
+ "[XOS-TOSCA] Failed to save or delete model %s [%s]: %s"
+ % (class_name, recipe.name, str(e))
+ )
except ValidationError as e:
if e.message:
@@ -254,17 +278,15 @@
exception_msg = TOSCA_Parser._translate_exception(str(e))
raise Exception(exception_msg)
- except _Rendezvous, e:
+ except _Rendezvous as e:
try:
details = json.loads(e._state.details)
exception_msg = details["error"]
if "specific_error" in details:
- exception_msg = "%s: %s" % (exception_msg, details["specific_error"])
+ exception_msg = "%s: %s" % (
+ exception_msg,
+ details["specific_error"],
+ )
except Exception:
exception_msg = e._state.details
raise Exception(exception_msg)
- except Exception, e:
- log.exception(e)
- raise Exception(e)
-
-
diff --git a/src/tosca/xtarget/tosca_keys.xtarget b/src/tosca/xtarget/tosca_keys.xtarget
index 9569061..6b73cb3 100644
--- a/src/tosca/xtarget/tosca_keys.xtarget
+++ b/src/tosca/xtarget/tosca_keys.xtarget
@@ -2,4 +2,5 @@
{%- for m in proto.messages %}
'{{ m.name }}': {{ xproto_fields_to_tosca_keys(m.fields + xproto_base_fields(m, proto.message_table), m) }},
{%- endfor %}
-}
\ No newline at end of file
+}
+
diff --git a/src/web_server/__init__.py b/src/web_server/__init__.py
index d4e8062..ae374a7 100644
--- a/src/web_server/__init__.py
+++ b/src/web_server/__init__.py
@@ -1,4 +1,3 @@
-
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,4 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
+from .main import TOSCA_WebServer
+
+__all__ = [
+ 'TOSCA_WebServer',
+]
diff --git a/src/web_server/main.py b/src/web_server/main.py
index 271b1b2..34a83f5 100644
--- a/src/web_server/main.py
+++ b/src/web_server/main.py
@@ -1,4 +1,3 @@
-
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,29 +12,34 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from xosconfig import Config
-from multistructlog import create_logger
-log = create_logger(Config().get('logging'))
+from __future__ import absolute_import, print_function
-from grpc_client.main import GRPC_Client
-from klein import Klein
-import os
-from tosca.parser import TOSCA_Parser
-from tosca.default import TOSCA_DEFS_DIR
import json
+import os
-BANNER = """
- _ ______ _____ __________ _____ _________
+from grpc_client import GRPC_Client
+from klein import Klein
+from multistructlog import create_logger
+from tosca.default import TOSCA_DEFS_DIR
+from tosca.parser import TOSCA_Parser
+from xosconfig import Config
+
+log = create_logger(Config().get("logging"))
+
+
+BANNER = r"""
+ _ ______ _____ __________ _____ _________
| |/ / __ \/ ___/ /_ __/ __ \/ ___// ____/ |
| / / / /\__ \ / / / / / /\__ \/ / / /| |
/ / /_/ /___/ / / / / /_/ /___/ / /___/ ___ |
/_/|_\____//____/ /_/ \____//____/\____/_/ |_|
"""
+
class TOSCA_WebServer:
current_dir = os.path.dirname(os.path.realpath(__file__))
- template_dir = os.path.join(current_dir, 'templates/')
+ template_dir = os.path.join(current_dir, "templates/")
app = Klein()
@@ -59,48 +63,43 @@
log.info("[XOS-TOSCA] Fatal Error: \n\n", failure=failure)
return "Internal server error, please report this along with the failed recipe."
- @app.route('/', methods=['GET'])
+ @app.route("/", methods=["GET"])
def index(self, request):
request.responseHeaders.addRawHeader(b"content-type", b"application/json")
- tosca_defs = [f for f in os.listdir(TOSCA_DEFS_DIR) if not f.startswith('.')]
+ tosca_defs = [f for f in os.listdir(TOSCA_DEFS_DIR) if not f.startswith(".")]
response = {}
for d in tosca_defs:
- name = d.replace('.yaml', '')
+ name = d.replace(".yaml", "")
response[name] = "/custom_type/%s" % name
return json.dumps(response)
@app.route("/custom_type/<name>")
def custom_type(self, request, name):
request.responseHeaders.addRawHeader(b"content-type", b"text/plain")
- custom_type = open(TOSCA_DEFS_DIR + '/' + name + '.yaml').read()
+ custom_type = open(TOSCA_DEFS_DIR + "/" + name + ".yaml").read()
return custom_type
- @app.route('/run', methods=['POST'])
+ @app.route("/run", methods=["POST"])
def run(self, request):
- recipe = request.content.read()
- headers = request.getAllHeaders()
- username = headers['xos-username']
- password = headers['xos-password']
+ return self._handle_post(request, delete=False)
- parser = TOSCA_Parser(recipe, username, password)
- d = GRPC_Client().create_secure_client(username, password, parser)
- tosca_execution = d.addCallback(self.execute_tosca)
- tosca_execution.addErrback(self.errorCallback, request)
- return d
-
- @app.route('/delete', methods=['POST'])
+ @app.route("/delete", methods=["POST"])
def delete(self, request):
- recipe = request.content.read()
- headers = request.getAllHeaders()
- username = headers['xos-username']
- password = headers['xos-password']
+ return self._handle_post(request, delete=True)
- parser = TOSCA_Parser(recipe, username, password, delete=True)
+ def _handle_post(self, request, delete=False):
+
+ headers = request.getAllHeaders()
+ username = headers["xos-username"]
+ password = headers["xos-password"]
+ recipe = request.content.read()
+
+ parser = TOSCA_Parser(recipe, username, password, delete=delete)
d = GRPC_Client().create_secure_client(username, password, parser)
tosca_execution = d.addCallback(self.execute_tosca)
tosca_execution.addErrback(self.errorCallback, request)
return d
def __init__(self):
- self.app.run('0.0.0.0', '9102')
\ No newline at end of file
+ self.app.run("0.0.0.0", "9102")
diff --git a/src/xos-tosca-config-schema.yaml b/src/xos-tosca-config-schema.yaml
index db68a43..281bf77 100644
--- a/src/xos-tosca-config-schema.yaml
+++ b/src/xos-tosca-config-schema.yaml
@@ -1,4 +1,4 @@
-
+---
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,16 +13,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
map:
name:
type: str
- required: True
+ required: true
gprc_endpoint:
type: str
- required: True
+ required: true
local_cert:
type: str
- required: True
+ required: true
logging:
type: any
diff --git a/src/xos-tosca.config.yaml b/src/xos-tosca.config.yaml
new file mode 100644
index 0000000..4231b3b
--- /dev/null
+++ b/src/xos-tosca.config.yaml
@@ -0,0 +1,14 @@
+---
+name: "xos-tosca"
+gprc_endpoint: "xos-core:30001"
+local_cert: ""
+logging:
+ version: 1
+ handlers:
+ console:
+ class: logging.StreamHandler
+ loggers:
+ 'multistructlog':
+ handlers:
+ - console
+ level: ERROR