[SEBA-412] Automated reformat of Python code
Passes of modernize, autopep8, black, then check with flake8
flake8 + manual fixes:
lib/xos-config
lib/xos-kafka
lib/xos-util
xos/coreapi
xos/api
xos/xos_client
Change-Id: Ib23cf84cb13beb3c6381fa0d79594dc9131dc815
diff --git a/lib/xos-config/setup.py b/lib/xos-config/setup.py
index 777a7a1..1e683f1 100644
--- a/lib/xos-config/setup.py
+++ b/lib/xos-config/setup.py
@@ -14,8 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from setuptools import setup
-
try:
from xosutil.autoversion_setup import setup_with_auto_version as setup
except ImportError:
@@ -25,15 +23,14 @@
from xosconfig.version import __version__
-setup(name='XosConfig',
- version=__version__,
- description='XOS Config Library',
- author='Matteo Scandolo',
- author_email='teo@onlab.us',
- packages=['xosconfig'],
- include_package_data=True,
- # TODO add all deps to the install_requires section
- install_requires=[
- 'pykwalify>=1.6.0'
- ]
- )
+setup(
+ name="XosConfig",
+ version=__version__,
+ description="XOS Config Library",
+ author="Matteo Scandolo",
+ author_email="teo@onlab.us",
+ packages=["xosconfig"],
+ include_package_data=True,
+ # TODO add all deps to the install_requires section
+ install_requires=["pykwalify>=1.6.0"],
+)
diff --git a/lib/xos-config/tests/__init__.py b/lib/xos-config/tests/__init__.py
index d4e8062..b0fb0b2 100644
--- a/lib/xos-config/tests/__init__.py
+++ b/lib/xos-config/tests/__init__.py
@@ -1,4 +1,3 @@
-
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,5 +11,3 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
-
diff --git a/lib/xos-config/tests/test_config.py b/lib/xos-config/tests/test_config.py
index ee80b52..5eb86af 100644
--- a/lib/xos-config/tests/test_config.py
+++ b/lib/xos-config/tests/test_config.py
@@ -1,4 +1,3 @@
-
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,39 +14,50 @@
import unittest
-from mock import patch
import os
from xosconfig import Config
from xosconfig import Config as Config2
-basic_conf = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/confs/basic_conf.yaml")
-yaml_not_valid = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/confs/yaml_not_valid.yaml")
-invalid_format = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/confs/invalid_format.yaml")
-sample_conf = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/confs/sample_conf.yaml")
-override_conf = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/confs/override_conf.yaml")
-extend_conf = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/confs/extend_conf.yaml")
+basic_conf = os.path.abspath(
+ os.path.dirname(os.path.realpath(__file__)) + "/confs/basic_conf.yaml"
+)
+yaml_not_valid = os.path.abspath(
+ os.path.dirname(os.path.realpath(__file__)) + "/confs/yaml_not_valid.yaml"
+)
+invalid_format = os.path.abspath(
+ os.path.dirname(os.path.realpath(__file__)) + "/confs/invalid_format.yaml"
+)
+sample_conf = os.path.abspath(
+ os.path.dirname(os.path.realpath(__file__)) + "/confs/sample_conf.yaml"
+)
+override_conf = os.path.abspath(
+ os.path.dirname(os.path.realpath(__file__)) + "/confs/override_conf.yaml"
+)
+extend_conf = os.path.abspath(
+ os.path.dirname(os.path.realpath(__file__)) + "/confs/extend_conf.yaml"
+)
-small_schema = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/schemas/small_schema.yaml")
+small_schema = os.path.abspath(
+ os.path.dirname(os.path.realpath(__file__)) + "/schemas/small_schema.yaml"
+)
-services_list = {
- "xos-ws": [],
- "xos-db": [],
-}
+services_list = {"xos-ws": [], "xos-db": []}
db_service = [
- {
- "ModifyIndex": 6,
- "CreateIndex": 6,
- "Node": "0152982c3159",
- "Address": "172.19.0.2",
- "ServiceID": "0d53ce210785:frontend_xos_db_1:5432",
- "ServiceName": "xos-db",
- "ServiceTags": [],
- "ServiceAddress": "172.18.0.4",
- "ServicePort": 5432,
- "ServiceEnableTagOverride": "false"
- }
- ]
+ {
+ "ModifyIndex": 6,
+ "CreateIndex": 6,
+ "Node": "0152982c3159",
+ "Address": "172.19.0.2",
+ "ServiceID": "0d53ce210785:frontend_xos_db_1:5432",
+ "ServiceName": "xos-db",
+ "ServiceTags": [],
+ "ServiceAddress": "172.18.0.4",
+ "ServicePort": 5432,
+ "ServiceEnableTagOverride": "false",
+ }
+]
+
class XOSConfigTest(unittest.TestCase):
"""
@@ -77,15 +87,19 @@
"""
with self.assertRaises(Exception) as e:
Config.get("database")
- self.assertEqual(e.exception.message, "[XOS-Config] Module has not been initialized")
+ self.assertEqual(
+ e.exception.message, "[XOS-Config] Module has not been initialized"
+ )
def test_missing_file_exception(self):
"""
- [XOS-Config] Raise if file not found
+ [XOS-Config] Raise if file not found
"""
with self.assertRaises(Exception) as e:
Config.init("missing_conf")
- self.assertEqual(e.exception.message, "[XOS-Config] Config file not found at: missing_conf")
+ self.assertEqual(
+ e.exception.message, "[XOS-Config] Config file not found at: missing_conf"
+ )
def test_yaml_not_valid(self):
"""
@@ -93,7 +107,9 @@
"""
with self.assertRaises(Exception) as e:
Config.init(yaml_not_valid)
- self.assertTrue(e.exception.message.startswith("[XOS-Config] The config format is wrong:"))
+ self.assertTrue(
+ e.exception.message.startswith("[XOS-Config] The config format is wrong:")
+ )
def test_invalid_format(self):
"""
@@ -101,7 +117,13 @@
"""
with self.assertRaises(Exception) as e:
Config.init(invalid_format)
- self.assertEqual(e.exception.message, "[XOS-Config] The config format is wrong: Schema validation failed:\n - Value '['I am', 'a yaml', 'but the', 'format is not', 'correct']' is not a dict. Value path: ''.")
+ self.assertEqual(
+ e.exception.message,
+ (
+ "[XOS-Config] The config format is wrong: Schema validation failed:\n"
+ " - Value '['I am', 'a yaml', 'but the', 'format is not', 'correct']' is not a dict. Value path: ''."
+ ),
+ )
def test_env_override(self):
"""
@@ -110,7 +132,9 @@
os.environ["XOS_CONFIG_FILE"] = "env.yaml"
with self.assertRaises(Exception) as e:
Config.init("missing_conf")
- self.assertEqual(e.exception.message, "[XOS-Config] Config file not found at: env.yaml")
+ self.assertEqual(
+ e.exception.message, "[XOS-Config] Config file not found at: env.yaml"
+ )
del os.environ["XOS_CONFIG_FILE"]
def test_schema_override(self):
@@ -120,7 +144,10 @@
os.environ["XOS_CONFIG_SCHEMA"] = "env-schema.yaml"
with self.assertRaises(Exception) as e:
Config.init(basic_conf)
- self.assertRegexpMatches(e.exception.message, '\[XOS\-Config\] Config schema not found at: (.+)env-schema\.yaml')
+ self.assertRegexpMatches(
+ e.exception.message,
+ r"\[XOS\-Config\] Config schema not found at: (.+)env-schema\.yaml",
+ )
# self.assertEqual(e.exception.message, "[XOS-Config] Config schema not found at: env-schema.yaml")
del os.environ["XOS_CONFIG_SCHEMA"]
@@ -131,7 +158,13 @@
os.environ["XOS_CONFIG_SCHEMA"] = small_schema
with self.assertRaises(Exception) as e:
Config.init(basic_conf)
- self.assertEqual(e.exception.message, "[XOS-Config] The config format is wrong: Schema validation failed:\n - Key 'database' was not defined. Path: ''.")
+ self.assertEqual(
+ e.exception.message,
+ (
+ "[XOS-Config] The config format is wrong: Schema validation failed:\n"
+ " - Key 'database' was not defined. Path: ''."
+ ),
+ )
del os.environ["XOS_CONFIG_SCHEMA"]
def test_get_cli_param(self):
@@ -174,11 +207,7 @@
# NOTE we are using Config2 here to be sure that the configuration is readable from any import,
# not only from the one that has been used to initialize it
res = Config2.get("database")
- self.assertEqual(res, {
- "name": "xos",
- "username": "test",
- "password": "safe"
- })
+ self.assertEqual(res, {"name": "xos", "username": "test", "password": "safe"})
def test_get_child_level(self):
"""
@@ -192,7 +221,7 @@
"""
[XOS-Config] If an override is provided for the config, it should return the overridden value
"""
- Config.init(sample_conf, 'xos-config-schema.yaml', override_conf)
+ Config.init(sample_conf, "xos-config-schema.yaml", override_conf)
res = Config.get("logging.level")
self.assertEqual(res, "info")
res = Config.get("database.password")
@@ -200,13 +229,16 @@
def test_config_extend(self):
"""
- [XOS-Config] If an override is provided for the config, it should return the overridden value (also if not defined in the base one)
+ [XOS-Config] If an override is provided for the config, it should
+ return the overridden value (also if not defined in the base one)
"""
- Config.init(sample_conf, 'xos-config-schema.yaml', extend_conf)
+
+ Config.init(sample_conf, "xos-config-schema.yaml", extend_conf)
res = Config.get("xos_dir")
self.assertEqual(res, "/opt/xos")
res = Config.get("database.password")
self.assertEqual(res, "safe")
-if __name__ == '__main__':
+
+if __name__ == "__main__":
unittest.main()
diff --git a/lib/xos-config/xosconfig/__init__.py b/lib/xos-config/xosconfig/__init__.py
index d4e8305..9a0b30c 100644
--- a/lib/xos-config/xosconfig/__init__.py
+++ b/lib/xos-config/xosconfig/__init__.py
@@ -13,3 +13,5 @@
# limitations under the License.
from .config import Config
+
+__all__ = ["Config"]
diff --git a/lib/xos-config/xosconfig/config.py b/lib/xos-config/xosconfig/config.py
index ebc696f..aac6ffb 100644
--- a/lib/xos-config/xosconfig/config.py
+++ b/lib/xos-config/xosconfig/config.py
@@ -1,4 +1,3 @@
-
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,7 +16,6 @@
import os
import sys
import yaml
-import requests
import default
from pykwalify.core import Core as PyKwalify
import pykwalify
@@ -25,20 +23,25 @@
pykwalify.init_logging(1)
DEFAULT_CONFIG_FILE = "/opt/xos/xos_config.yaml"
-DEFAULT_CONFIG_SCHEMA = 'xos-config-schema.yaml'
+DEFAULT_CONFIG_SCHEMA = "xos-config-schema.yaml"
INITIALIZED = False
CONFIG_FILE = None
CONFIG = {}
OVERRIDE_CONFIG = {}
+
class Config:
"""
XOS Configuration APIs
"""
@staticmethod
- def init(config_file=DEFAULT_CONFIG_FILE, config_schema=DEFAULT_CONFIG_SCHEMA, override_config_file=None):
+ def init(
+ config_file=DEFAULT_CONFIG_FILE,
+ config_schema=DEFAULT_CONFIG_SCHEMA,
+ override_config_file=None,
+ ):
# make schema relative to this directory
# TODO give the possibility to specify an absolute path
@@ -58,37 +61,40 @@
# the config module can be initialized only one
if INITIALIZED:
- raise Exception('[XOS-Config] Module already initialized')
+ raise Exception("[XOS-Config] Module already initialized")
INITIALIZED = True
# if XOS_CONFIG_FILE is defined override the config_file
# FIXME shouldn't this stay in whatever module call this one? and then just pass the file to the init method
- if os.environ.get('XOS_CONFIG_FILE'):
- config_file = os.environ['XOS_CONFIG_FILE']
+ if os.environ.get("XOS_CONFIG_FILE"):
+ config_file = os.environ["XOS_CONFIG_FILE"]
# if XOS_CONFIG_SCHEMA is defined override the config_schema
# FIXME shouldn't this stay in whatever module call this one? and then just pass the file to the init method
- if os.environ.get('XOS_CONFIG_SCHEMA'):
- config_schema = Config.get_abs_path(os.environ['XOS_CONFIG_SCHEMA'])
+ if os.environ.get("XOS_CONFIG_SCHEMA"):
+ config_schema = Config.get_abs_path(os.environ["XOS_CONFIG_SCHEMA"])
# allow OVERRIDE_CONFIG_* to be overridden by env vars
- if os.environ.get('XOS_OVERRIDE_CONFIG_FILE'):
- OVERRIDE_CONFIG_FILE = os.environ['XOS_OVERRIDE_CONFIG_FILE']
- if os.environ.get('XOS_OVERRIDE_CONFIG_SCHEMA'):
- OVERRIDE_CONFIG_SCHEMA = Config.get_abs_path(os.environ['XOS_OVERRIDE_CONFIG_SCHEMA'])
+ if os.environ.get("XOS_OVERRIDE_CONFIG_FILE"):
+ OVERRIDE_CONFIG_FILE = os.environ["XOS_OVERRIDE_CONFIG_FILE"]
+ if os.environ.get("XOS_OVERRIDE_CONFIG_SCHEMA"):
+ OVERRIDE_CONFIG_SCHEMA = Config.get_abs_path(
+ os.environ["XOS_OVERRIDE_CONFIG_SCHEMA"]
+ )
# if a -C parameter is set in the cli override the config_file
# FIXME shouldn't this stay in whatever module call this one? and then just pass the file to the init method
if Config.get_cli_param(sys.argv):
config_schema = Config.get_cli_param(sys.argv)
-
CONFIG_FILE = config_file
CONFIG = Config.read_config(config_file, config_schema)
# if an override is set
if OVERRIDE_CONFIG_FILE is not None:
- OVERRIDE_CONFIG = Config.read_config(OVERRIDE_CONFIG_FILE, OVERRIDE_CONFIG_SCHEMA, True)
+ OVERRIDE_CONFIG = Config.read_config(
+ OVERRIDE_CONFIG_FILE, OVERRIDE_CONFIG_SCHEMA, True
+ )
@staticmethod
def get_config_file():
@@ -103,7 +109,7 @@
def get_abs_path(path):
if os.path.isabs(path):
return path
- return os.path.dirname(os.path.realpath(__file__)) + '/' + path
+ return os.path.dirname(os.path.realpath(__file__)) + "/" + path
@staticmethod
def validate_config_format(config_file, config_schema):
@@ -115,7 +121,7 @@
def get_cli_param(args):
last = None
for arg in args:
- if last == '-C':
+ if last == "-C":
return arg
last = arg
@@ -127,25 +133,27 @@
:return: dict
"""
- if(not os.path.exists(config_file) and ignore_if_not_found):
+ if not os.path.exists(config_file) and ignore_if_not_found:
return {}
if not os.path.exists(config_file):
- raise Exception('[XOS-Config] Config file not found at: %s' % config_file)
+ raise Exception("[XOS-Config] Config file not found at: %s" % config_file)
if not os.path.exists(config_schema):
- raise Exception('[XOS-Config] Config schema not found at: %s' % config_schema)
+ raise Exception(
+ "[XOS-Config] Config schema not found at: %s" % config_schema
+ )
try:
Config.validate_config_format(config_file, config_schema)
- except Exception, e:
+ except Exception as e:
try:
error_msg = e.msg
except AttributeError:
error_msg = str(e)
- raise Exception('[XOS-Config] The config format is wrong: %s' % error_msg)
+ raise Exception("[XOS-Config] The config format is wrong: %s" % error_msg)
- with open(config_file, 'r') as stream:
+ with open(config_file, "r") as stream:
return yaml.safe_load(stream)
@staticmethod
@@ -161,7 +169,7 @@
global OVERRIDE_CONFIG_FILE
if not INITIALIZED:
- raise Exception('[XOS-Config] Module has not been initialized')
+ raise Exception("[XOS-Config] Module has not been initialized")
val = Config.get_param(query, CONFIG)
if OVERRIDE_CONFIG_FILE or not val:
@@ -186,10 +194,10 @@
:param config: the config source to read from (can be the config file or the defaults)
:return: the requested parameter in any format the parameter is specified
"""
- keys = query.split('.')
+ keys = query.split(".")
if len(keys) == 1:
key = keys[0]
- if not config.has_key(key):
+ if key not in config:
return None
return config[key]
else:
@@ -205,10 +213,11 @@
"""
param = config
for k in keys:
- if not param.has_key(k):
+ if k not in param:
return None
param = param[k]
return param
-if __name__ == '__main__':
+
+if __name__ == "__main__":
Config.init()
diff --git a/lib/xos-config/xosconfig/default.py b/lib/xos-config/xosconfig/default.py
index 2c73b26..afed387 100644
--- a/lib/xos-config/xosconfig/default.py
+++ b/lib/xos-config/xosconfig/default.py
@@ -13,52 +13,34 @@
# limitations under the License.
DEFAULT_VALUES = {
- 'xos_dir': '/opt/xos',
-
+ "xos_dir": "/opt/xos",
# The configuration below inherits from the standard config of the Python logging module
# See: https://docs.python.org/2/library/logging.config.html
# multistructlog supports this config in all of its generality
# So for instance, you can add new handlers. Note that all handlers will
# receive logs simultaneously.
-
- 'blueprints': {},
- 'logging': {
- 'version': 1,
- 'handlers': {
- 'console': {
- 'class': 'logging.StreamHandler',
+ "blueprints": {},
+ "logging": {
+ "version": 1,
+ "handlers": {
+ "console": {"class": "logging.StreamHandler"},
+ "file": {
+ "class": "logging.handlers.RotatingFileHandler",
+ "filename": "/var/log/xos.log",
+ "maxBytes": 10485760,
+ "backupCount": 5,
},
- 'file': {
- 'class': 'logging.handlers.RotatingFileHandler',
- 'filename': '/var/log/xos.log',
- 'maxBytes': 10485760,
- 'backupCount': 5
- }
},
- 'loggers': {
- '': {
- 'handlers': ['console', 'file'],
- 'level': 'DEBUG'
- }
- }
+ "loggers": {"": {"handlers": ["console", "file"], "level": "DEBUG"}},
},
- 'accessor': {
- 'endpoint': 'xos-core.cord.lab:50051',
- 'kind': 'grpcapi',
- },
- 'keep_temp_files': False,
- 'dependency_graph': None,
- 'error_map_path': '/opt/xos/error_map.txt',
- 'feefie': {
- 'client_user': 'pl'
- },
- 'proxy_ssh': {
- 'enabled': True,
- 'key': '/opt/cord_profile/node_key',
- 'user': 'root'
- },
- 'node_key': '/opt/cord_profile/node_key',
- 'config_dir': '/etc/xos/sync',
- 'backoff_disabled': True,
- 'kafka_bootstrap_servers': ['cord-kafka:9092'],
+ "accessor": {"endpoint": "xos-core.cord.lab:50051", "kind": "grpcapi"},
+ "keep_temp_files": False,
+ "dependency_graph": None,
+ "error_map_path": "/opt/xos/error_map.txt",
+ "feefie": {"client_user": "pl"},
+ "proxy_ssh": {"enabled": True, "key": "/opt/cord_profile/node_key", "user": "root"},
+ "node_key": "/opt/cord_profile/node_key",
+ "config_dir": "/etc/xos/sync",
+ "backoff_disabled": True,
+ "kafka_bootstrap_servers": ["cord-kafka:9092"],
}
diff --git a/lib/xos-config/xosconfig/version.py b/lib/xos-config/xosconfig/version.py
index a118c43..2c84950 100644
--- a/lib/xos-config/xosconfig/version.py
+++ b/lib/xos-config/xosconfig/version.py
@@ -1,4 +1,3 @@
-
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,7 +12,5 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import os
-
# This file will be replaced by setup.py
__version__ = "unknown"