[CORD-1273] Base structure for XOS-Config
Change-Id: I27f0b6664127ae2b26a1e7f489c70d5d32b73c3c
diff --git a/lib/xos-config/.gitignore b/lib/xos-config/.gitignore
new file mode 100644
index 0000000..161b154
--- /dev/null
+++ b/lib/xos-config/.gitignore
@@ -0,0 +1,2 @@
+.noseids
+build
\ No newline at end of file
diff --git a/lib/xos-config/Makefile b/lib/xos-config/Makefile
new file mode 100644
index 0000000..70ae716
--- /dev/null
+++ b/lib/xos-config/Makefile
@@ -0,0 +1,2 @@
+test:
+ nosetests -s -v --with-id
\ No newline at end of file
diff --git a/lib/xos-config/setup.py b/lib/xos-config/setup.py
new file mode 100644
index 0000000..2e29a40
--- /dev/null
+++ b/lib/xos-config/setup.py
@@ -0,0 +1,14 @@
+#!/usr/bin/env python
+
+from distutils.core import setup
+
+setup(name='XosConfig',
+ version='1.0',
+ description='XOS Config Library',
+ author='Matteo Scandolo',
+ author_email='teo@onlab.us',
+ packages=['xosconfig'],
+ data_files=[
+ ('.', ['xosconfig/config-schema.yaml'])
+ ]
+ )
\ No newline at end of file
diff --git a/lib/xos-config/tests/__init__.py b/lib/xos-config/tests/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/lib/xos-config/tests/__init__.py
diff --git a/lib/xos-config/tests/config_test.py b/lib/xos-config/tests/config_test.py
new file mode 100644
index 0000000..c89ed47
--- /dev/null
+++ b/lib/xos-config/tests/config_test.py
@@ -0,0 +1,194 @@
+import unittest
+from mock import patch
+import os
+from xosconfig import Config
+from xosconfig import Config as Config2
+
+basic_conf = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/confs/basic_conf.yaml")
+yaml_not_valid = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/confs/yaml_not_valid.yaml")
+invalid_format = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/confs/invalid_format.yaml")
+sample_conf = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/confs/sample_conf.yaml")
+
+services_list = {
+ "xos-ws": [],
+ "xos-db": [],
+}
+
+db_service = [
+ {
+ "ModifyIndex": 6,
+ "CreateIndex": 6,
+ "Node": "0152982c3159",
+ "Address": "172.19.0.2",
+ "ServiceID": "0d53ce210785:frontend_xos_db_1:5432",
+ "ServiceName": "xos-db",
+ "ServiceTags": [],
+ "ServiceAddress": "172.18.0.4",
+ "ServicePort": 5432,
+ "ServiceEnableTagOverride": "false"
+ }
+ ]
+
+class XOSConfigTest(unittest.TestCase):
+ """
+ Testing the XOS Config Module
+ """
+
+ def tearDown(self):
+ # NOTE clear the config after each test
+ Config.clear()
+
+ def test_initialize_only_once(self):
+ """
+ [XOS-Config] Raise if initialized twice
+ """
+ with self.assertRaises(Exception) as e:
+ Config.init(sample_conf)
+ Config2.init(sample_conf)
+ self.assertEqual(e.exception.message, "[XOS-Config] Module already initialized")
+
+ def test_config_not_initialized(self):
+ """
+ [XOS-Config] Raise if accessing properties without initialization
+ """
+ with self.assertRaises(Exception) as e:
+ Config.get("database")
+ self.assertEqual(e.exception.message, "[XOS-Config] Module has not been initialized")
+
+ def test_missing_file_exception(self):
+ """
+ [XOS-Config] Raise if file not found
+ """
+ with self.assertRaises(Exception) as e:
+ Config.init("missing_conf")
+ self.assertEqual(e.exception.message, "[XOS-Config] Config file not found at: missing_conf")
+
+ def test_yaml_not_valid(self):
+ """
+ [XOS-Config] Raise if yaml is not valid
+ """
+ with self.assertRaises(Exception) as e:
+ Config.init(yaml_not_valid)
+ self.assertEqual(e.exception.message, "[XOS-Config] The config format is wrong: Unable to load any data from source yaml file")
+
+ def test_invalid_format(self):
+ """
+ [XOS-Config] Raise if format is not valid (we expect a dictionary)
+ """
+ with self.assertRaises(Exception) as e:
+ Config.init(invalid_format)
+ self.assertEqual(e.exception.message, "[XOS-Config] The config format is wrong: Schema validation failed:\n - Value '['I am', 'a yaml', 'but the', 'format is not', 'correct']' is not a dict. Value path: ''.")
+
+ def test_env_override(self):
+ """
+ [XOS-Config] the XOS-CONFIG environment variable should override the config_file
+ """
+ os.environ["XOS-CONFIG"] = "env.yaml"
+ with self.assertRaises(Exception) as e:
+ Config.init("missing_conf")
+ self.assertEqual(e.exception.message, "[XOS-Config] Config file not found at: env.yaml")
+ del os.environ["XOS-CONFIG"]
+
+ def test_get_cli_param(self):
+ """
+ [XOS-Config] Should read CLI -C param
+ """
+ args = ["-A", "Foo", "-c", "Bar", "-C", "config.yaml"]
+ res = Config.get_cli_param(args)
+ self.assertEqual(res, "config.yaml")
+
+ def test_get_default_val_for_missing_param(self):
+ """
+ [XOS-Config] Should raise reading a missing param
+ """
+ Config.init(basic_conf)
+ log = Config.get("logging")
+ self.assertEqual(log, {
+ "level": "info",
+ "channels": ["file", "console"]
+ })
+
+ def _test_get_missing_param(self):
+ """
+ [XOS-Config] Should raise reading a missing param
+ """
+ Config.init(sample_conf)
+ with self.assertRaises(Exception) as e:
+ Config.get("foo")
+ self.assertEqual(e.exception.message, "[XOS-Config] Config does not have a value (or a default) parameter foo")
+
+ def test_get_first_level(self):
+ """
+ [XOS-Config] Should return a first level param
+ """
+ Config.init(sample_conf)
+ # NOTE we are using Config2 here to be sure that the configuration is readable from any import,
+ # not only from the one that has been used to initialize it
+ res = Config2.get("database")
+ self.assertEqual(res, {
+ "username": "test",
+ "password": "safe"
+ })
+
+ def _test_get_child_level(self):
+ """
+ [XOS-Config] Should return a child level param
+ """
+ Config.init(sample_conf)
+ res = Config.get("nested.parameter.for")
+ self.assertEqual(res, "testing")
+
+ def test_get_service_list(self):
+ """
+ [XOS-Config] Should query registrator and return a list of services
+ """
+ with patch("xosconfig.config.requests.get") as mock_get:
+ mock_get.return_value.json.return_value = services_list
+ res = Config.get_service_list()
+ self.assertEqual(res, [
+ "xos-ws",
+ "xos-db",
+ ])
+
+ def test_get_service_info(self):
+ """
+ [XOS-Config] Should query registrator and return service info
+ """
+ with patch("xosconfig.config.requests.get") as mock_get:
+ mock_get.return_value.json.return_value = db_service
+ info = Config.get_service_info("xos-db")
+ self.assertEqual(info, {
+ "name": "xos-db",
+ "url": "172.18.0.4",
+ "port": 5432
+ })
+
+ def test_fail_get_service_info(self):
+ """
+ [XOS-Config] Should query registrator and return an exception if it"s down
+ """
+ with patch("xosconfig.config.requests.get") as mock_get:
+ mock_get.return_value.ok = False
+ with self.assertRaises(Exception) as e:
+ Config.get_service_info("missing-service")
+ self.assertEqual(e.exception.message, "[XOS-Config] Registrator is down")
+
+ def test_missing_get_service_info(self):
+ """
+ [XOS-Config] Should query registrator and return an exception if service is not there
+ """
+ with patch("xosconfig.config.requests.get") as mock_get:
+ mock_get.return_value.json.return_value = []
+ with self.assertRaises(Exception) as e:
+ Config.get_service_info("missing-service")
+ self.assertEqual(e.exception.message, "[XOS-Config] The service missing-service looking for does not exist")
+
+
+ def test_get_service_endpoint(self):
+ """
+ [XOS-Config] Should query registrator and return service endpoint
+ """
+ with patch("xosconfig.config.requests.get") as mock_get:
+ mock_get.return_value.json.return_value = db_service
+ endpoint = Config.get_service_endpoint("xos-db")
+ self.assertEqual(endpoint, "http://172.18.0.4:5432")
\ No newline at end of file
diff --git a/lib/xos-config/tests/confs/basic_conf.yaml b/lib/xos-config/tests/confs/basic_conf.yaml
new file mode 100644
index 0000000..0f1d0b0
--- /dev/null
+++ b/lib/xos-config/tests/confs/basic_conf.yaml
@@ -0,0 +1,4 @@
+name: test
+database:
+ username: test
+ password: safe
\ No newline at end of file
diff --git a/lib/xos-config/tests/confs/invalid_format.yaml b/lib/xos-config/tests/confs/invalid_format.yaml
new file mode 100644
index 0000000..1972c5b
--- /dev/null
+++ b/lib/xos-config/tests/confs/invalid_format.yaml
@@ -0,0 +1,5 @@
+- I am
+- a yaml
+- but the
+- format is not
+- correct
\ No newline at end of file
diff --git a/lib/xos-config/tests/confs/sample_conf.yaml b/lib/xos-config/tests/confs/sample_conf.yaml
new file mode 100644
index 0000000..b398b40
--- /dev/null
+++ b/lib/xos-config/tests/confs/sample_conf.yaml
@@ -0,0 +1,9 @@
+name: xos-core
+database:
+ username: test
+ password: safe
+logging:
+ level: info
+ channels:
+ - file
+ - console
\ No newline at end of file
diff --git a/lib/xos-config/tests/confs/yaml_not_valid.yaml b/lib/xos-config/tests/confs/yaml_not_valid.yaml
new file mode 100644
index 0000000..a06cd5b
--- /dev/null
+++ b/lib/xos-config/tests/confs/yaml_not_valid.yaml
@@ -0,0 +1,4 @@
+i am:
+ not:
+ - a valid
+ yaml
\ No newline at end of file
diff --git a/lib/xos-config/xosconfig/__init__.py b/lib/xos-config/xosconfig/__init__.py
new file mode 100644
index 0000000..3558f42
--- /dev/null
+++ b/lib/xos-config/xosconfig/__init__.py
@@ -0,0 +1 @@
+from .config import Config
\ No newline at end of file
diff --git a/lib/xos-config/xosconfig/config-schema.yaml b/lib/xos-config/xosconfig/config-schema.yaml
new file mode 100644
index 0000000..9a6f000
--- /dev/null
+++ b/lib/xos-config/xosconfig/config-schema.yaml
@@ -0,0 +1,21 @@
+map:
+ name:
+ type: str
+ required: True
+ database:
+ type: map
+ map:
+ username:
+ type: str
+ password:
+ type: str
+ logging:
+ type: map
+ map:
+ level:
+ type: str
+ channels:
+ type: seq
+ sequence:
+ - type: str
+ enum: ['file', 'console', 'elkstack']
\ No newline at end of file
diff --git a/lib/xos-config/xosconfig/config.py b/lib/xos-config/xosconfig/config.py
new file mode 100644
index 0000000..e58477c
--- /dev/null
+++ b/lib/xos-config/xosconfig/config.py
@@ -0,0 +1,172 @@
+import os
+import sys
+import yaml
+import requests
+import default
+from pykwalify.core import Core as PyKwalify
+
+DEFAULT_CONFIG_FILE = "/opt/xos/config.yaml"
+INITIALIZED = False
+CONFIG = {}
+
+class Config:
+ """
+ XOS Configuration APIs
+ """
+
+ @staticmethod
+ def init(config_file=DEFAULT_CONFIG_FILE):
+ global INITIALIZED
+ global CONFIG
+ # the config module can be initialized only one
+ if INITIALIZED:
+ raise Exception('[XOS-Config] Module already initialized')
+ INITIALIZED = True
+
+ # if XOS-CONFIG is defined override the config_file
+ if os.environ.get('XOS-CONFIG'):
+ config_file = os.environ['XOS-CONFIG']
+
+ # if a -C parameter is set in the cli override the config_file
+ # FIXME shouldn't this stay in whatever module call this one? and then just pass the file to the init method
+ if Config.get_cli_param(sys.argv):
+ config_file = Config.get_cli_param(sys.argv)
+
+ CONFIG = Config.read_config(config_file)
+
+ @staticmethod
+ def clear():
+ global INITIALIZED
+ INITIALIZED = False
+
+ @staticmethod
+ def validate_config_format(config_file):
+ schema = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + '/config-schema.yaml')
+ c = PyKwalify(source_file=config_file, schema_files=[schema])
+ c.validate(raise_exception=True)
+
+ @staticmethod
+ def get_cli_param(args):
+ last = None
+ for arg in args:
+ if last == '-C':
+ return arg
+ last = arg
+
+ @staticmethod
+ def read_config(config_file):
+ """
+ Read the configuration file and return a dictionary
+ :param config_file: string
+ :return: dict
+ """
+ if not os.path.exists(config_file):
+ raise Exception('[XOS-Config] Config file not found at: %s' % config_file)
+
+ try:
+ Config.validate_config_format(config_file)
+ except Exception, e:
+ raise Exception('[XOS-Config] The config format is wrong: %s' % e.msg)
+
+ with open(config_file, 'r') as stream:
+ return yaml.safe_load(stream)
+
+ @staticmethod
+ def get(query):
+ """
+ Read a parameter from the config
+ :param query: a dot separated selector for configuration options (eg: database.username)
+ :return: the requested parameter in any format the parameter is specified
+ """
+ global INITIALIZED
+ global CONFIG
+
+ if not INITIALIZED:
+ raise Exception('[XOS-Config] Module has not been initialized')
+
+ val = Config.get_param(query, CONFIG)
+ if not val:
+ val = Config.get_param(query, default.DEFAULT_VALUES)
+ if not val:
+ raise Exception('[XOS-Config] Config does not have a value (or a default) parameter %s' % query)
+ return val
+
+ @staticmethod
+ def get_param(query, config):
+ """
+ Search for a parameter in config's first level, other call get_nested_param
+ :param query: a dot separated selector for configuration options (eg: database.username)
+ :param config: the config source to read from (can be the config file or the defaults)
+ :return: the requested parameter in any format the parameter is specified
+ """
+ keys = query.split('.')
+ if len(keys) == 1:
+ key = keys[0]
+ if not config.has_key(key):
+ return None
+ return config[key]
+ else:
+ return Config.get_nested_param(keys, config)
+
+ @staticmethod
+ def get_nested_param(keys, config):
+ """
+
+ :param keys: a list of descending selector
+ :param config: the config source to read from (can be the config file or the defaults)
+ :return: the requested parameter in any format the parameter is specified
+ """
+ param = config
+ for k in keys:
+ if not param.has_key(k):
+ return None
+ param = param[k]
+ return param
+
+ @staticmethod
+ def get_service_list():
+ """
+ Query registrator to get the list of services
+ NOTE: we assume that consul is a valid URL
+ :return: a list of service names
+ """
+ service_dict = requests.get('http://consul:8500/v1/catalog/services').json()
+ service_list = []
+ for s in service_dict:
+ service_list.append(s)
+ return service_list
+
+ @staticmethod
+ def get_service_info(service_name):
+ """
+ Query registrator to get the details about a service
+ NOTE: we assume that consul is a valid URL
+ :param service_name: the name of the service, can be retrieved from get_service_list
+ :return: the informations about a service
+ """
+ response = requests.get('http://consul:8500/v1/catalog/service/%s' % service_name)
+ if not response.ok:
+ raise Exception('[XOS-Config] Registrator is down')
+ service = response.json()
+ if not service or len(service) == 0:
+ raise Exception('[XOS-Config] The service missing-service looking for does not exist')
+ return {
+ 'name': service[0]['ServiceName'],
+ 'url': service[0]['ServiceAddress'],
+ 'port': service[0]['ServicePort']
+ }
+
+ @staticmethod
+ def get_service_endpoint(service_name):
+ """
+ Query registrator to get the details about a service and return the endpoint in for of a string
+ :param service_name: the name of the service, can be retrieved from get_service_list
+ :return: the endpoint of the service
+ """
+ service = Config.get_service_info(service_name)
+ return 'http://%s:%s' % (service['url'], service['port'])
+
+# NOTE is this needed if this package is not meant to be execute from the CLI?
+if __name__ == '__main__':
+ config = Config()
+ config.init()
\ No newline at end of file
diff --git a/lib/xos-config/xosconfig/default.py b/lib/xos-config/xosconfig/default.py
new file mode 100644
index 0000000..221e0e8
--- /dev/null
+++ b/lib/xos-config/xosconfig/default.py
@@ -0,0 +1,6 @@
+DEFAULT_VALUES = {
+ 'logging': {
+ 'level': 'info',
+ 'channels': ['file', 'console']
+ }
+}
\ No newline at end of file