[CORD-1273] Base structure for XOS-Config
Change-Id: I27f0b6664127ae2b26a1e7f489c70d5d32b73c3c
diff --git a/containers/xos/Dockerfile.xos b/containers/xos/Dockerfile.xos
index 7b2f9dd..ab42758 100644
--- a/containers/xos/Dockerfile.xos
+++ b/containers/xos/Dockerfile.xos
@@ -3,6 +3,7 @@
# Install XOS
ADD xos /opt/xos
+ADD lib /opt/xos/lib
RUN chmod +x /opt/xos/tools/xos-manage \
&& mkdir -p /opt/cord_profile /opt/xos_services /opt/xos_libraries \
@@ -15,6 +16,10 @@
# Define working directory
WORKDIR /opt/xos
+# Install the config module
+RUN cd /opt/xos/lib/xos-config/; \
+ python setup.py install
+
# Label image
ARG org_label_schema_schema_version=1.0
ARG org_label_schema_name=xos
diff --git a/containers/xos/pip_requirements.txt b/containers/xos/pip_requirements.txt
index 720ba8d..00be225 100644
--- a/containers/xos/pip_requirements.txt
+++ b/containers/xos/pip_requirements.txt
@@ -71,6 +71,7 @@
kombu==3.0.35
lxml==3.6.4
meld3==0.6.10
+mock==2.0.0
monotonic==1.2
msgpack-python==0.4.8
munch==2.0.4
@@ -103,13 +104,14 @@
prettytable==0.7.2
protobuf==3.2.0
psycopg2==2.6.2
-pyOpenSSL==16.1.0
+pyOpenSSL==16.2.0
pyasn1==0.1.9
pyasn1-modules==0.0.8
pycparser==2.14
pycrypto==2.6.1
pycurl==7.43.0
pygraphviz==1.3.1
+pykwalify==1.6.0
pyparsing==2.1.8
python-cinderclient==1.8.0
python-consul==0.7.0
@@ -135,6 +137,8 @@
retrying==1.3.3
rfc3986==0.4.1
rsa==3.4.2
+ruamel.ordereddict==0.4.9
+ruamel.yaml==0.11.15
shade==1.9.0
simplejson==3.8.2
six==1.10.0
diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md
index f770070..a5819ca 100644
--- a/docs/SUMMARY.md
+++ b/docs/SUMMARY.md
@@ -2,5 +2,8 @@
* [Introduction](README.md)
- [Contribute to the documentation](contribute_docs.md)
-* [Developer HowTos](developer.md)
+* Modules
+ - [Xos Config](modules/xosconfig.md)
+* Developer How Tos
- [xproto](xproto.md)
+
diff --git a/docs/modules/xosconfig.md b/docs/modules/xosconfig.md
new file mode 100644
index 0000000..094d93f
--- /dev/null
+++ b/docs/modules/xosconfig.md
@@ -0,0 +1,102 @@
+# Xos Config
+
+This module is responsible to read, validate and distribute the configuration for all
+the XOS based components.
+
+_The code for this module can be found in lib/xos-config_
+
+The `xosconfig` module use a combination of parameters provided via a `.yaml` file and a service discovery mechanism.
+
+## How to use this module
+
+This module needs to be initialized once (and only once) when you application start, you can do it with:
+```python
+from xosconfig import Config
+Config.init()
+```
+
+By default the `xosconfig` module will look for a configuration file in `/opt/xos/config.yaml`, if for any reason you need to pass a different config file it can be done with:
+```python
+from xosconfig import Config
+Config.init("/path/to/my/config.yaml")
+```
+
+### Reading data from the configuration file
+
+To access static information defined in the `config.yaml` file you can use this api:
+```python
+from xosconfig import Config
+res = Config.get('database')
+```
+this call will return something like:
+```python
+{
+ 'username': 'test',
+ 'password': 'safe'
+}
+```
+Since the configuration support nested dictionary is possible to query directly nested values using a `dot` notation, for example:
+```python
+from xosconfig import Config
+res = Config.get('database.username')
+```
+will return:
+```python
+"test"
+```
+**The configuration schema is defined in `/lib/xos-config/config-schema.yaml`**
+
+### Reading service information
+
+XOS is composed by a plethora of services, to discover them and their address we are using
+a tool called [registrator](https://github.com/gliderlabs/registrator).
+
+#### Retrieve a list of services:
+```python
+from xosconfig import Config
+Config.get_service_list()
+```
+this call will return an array of available services, by default:
+```python
+[
+ "xos-ws",
+ "xos-ui-deprecated",
+ "xos-rest",
+ "xos-gui",
+ "xos-db",
+ "consul-rest",
+ "consul",
+]
+```
+>You can get the same information on the `head node` using:
+>```bash
+> curl consul:8500/v1/catalog/services
+>```
+
+#### Retrieve information for a single service:
+```python
+from xosconfig import Config
+Config.get_service_info('xos-db')
+```
+that will return:
+```python
+{
+ 'name': 'xos-db',
+ 'url': '172.18.0.4',
+ 'port': 5432
+}
+```
+>You can get the same information on the `head node` using:
+>```bash
+> curl consul:8500/v1/catalog/service/xos-db
+>```
+
+#### Retrieve endpoint for a single service:
+```python
+from xosconfig import Config
+Config.get_service_endpoint('xos-db')
+```
+that will return:
+```python
+"http://172.18.0.4:5432"
+```
\ No newline at end of file
diff --git a/group_vars/all b/group_vars/all
index f832a21..8471ce6 100644
--- a/group_vars/all
+++ b/group_vars/all
@@ -67,6 +67,16 @@
dockerfile: ""
pull: True
publish: False
+ - name: "gliderlabs/consul-server:latest"
+ path: ""
+ dockerfile: ""
+ pull: True
+ publish: False
+ - name: "gliderlabs/registrator:latest"
+ path: ""
+ dockerfile: ""
+ pull: True
+ publish: False
build_optional_images: False
diff --git a/lib/xos-config/.gitignore b/lib/xos-config/.gitignore
new file mode 100644
index 0000000..161b154
--- /dev/null
+++ b/lib/xos-config/.gitignore
@@ -0,0 +1,2 @@
+.noseids
+build
\ No newline at end of file
diff --git a/lib/xos-config/Makefile b/lib/xos-config/Makefile
new file mode 100644
index 0000000..70ae716
--- /dev/null
+++ b/lib/xos-config/Makefile
@@ -0,0 +1,2 @@
+test:
+ nosetests -s -v --with-id
\ No newline at end of file
diff --git a/lib/xos-config/setup.py b/lib/xos-config/setup.py
new file mode 100644
index 0000000..2e29a40
--- /dev/null
+++ b/lib/xos-config/setup.py
@@ -0,0 +1,14 @@
+#!/usr/bin/env python
+
+from distutils.core import setup
+
+setup(name='XosConfig',
+ version='1.0',
+ description='XOS Config Library',
+ author='Matteo Scandolo',
+ author_email='teo@onlab.us',
+ packages=['xosconfig'],
+ data_files=[
+ ('.', ['xosconfig/config-schema.yaml'])
+ ]
+ )
\ No newline at end of file
diff --git a/lib/xos-config/tests/__init__.py b/lib/xos-config/tests/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/lib/xos-config/tests/__init__.py
diff --git a/lib/xos-config/tests/config_test.py b/lib/xos-config/tests/config_test.py
new file mode 100644
index 0000000..c89ed47
--- /dev/null
+++ b/lib/xos-config/tests/config_test.py
@@ -0,0 +1,194 @@
+import unittest
+from mock import patch
+import os
+from xosconfig import Config
+from xosconfig import Config as Config2
+
+basic_conf = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/confs/basic_conf.yaml")
+yaml_not_valid = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/confs/yaml_not_valid.yaml")
+invalid_format = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/confs/invalid_format.yaml")
+sample_conf = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/confs/sample_conf.yaml")
+
+services_list = {
+ "xos-ws": [],
+ "xos-db": [],
+}
+
+db_service = [
+ {
+ "ModifyIndex": 6,
+ "CreateIndex": 6,
+ "Node": "0152982c3159",
+ "Address": "172.19.0.2",
+ "ServiceID": "0d53ce210785:frontend_xos_db_1:5432",
+ "ServiceName": "xos-db",
+ "ServiceTags": [],
+ "ServiceAddress": "172.18.0.4",
+ "ServicePort": 5432,
+ "ServiceEnableTagOverride": "false"
+ }
+ ]
+
+class XOSConfigTest(unittest.TestCase):
+ """
+ Testing the XOS Config Module
+ """
+
+ def tearDown(self):
+ # NOTE clear the config after each test
+ Config.clear()
+
+ def test_initialize_only_once(self):
+ """
+ [XOS-Config] Raise if initialized twice
+ """
+ with self.assertRaises(Exception) as e:
+ Config.init(sample_conf)
+ Config2.init(sample_conf)
+ self.assertEqual(e.exception.message, "[XOS-Config] Module already initialized")
+
+ def test_config_not_initialized(self):
+ """
+ [XOS-Config] Raise if accessing properties without initialization
+ """
+ with self.assertRaises(Exception) as e:
+ Config.get("database")
+ self.assertEqual(e.exception.message, "[XOS-Config] Module has not been initialized")
+
+ def test_missing_file_exception(self):
+ """
+ [XOS-Config] Raise if file not found
+ """
+ with self.assertRaises(Exception) as e:
+ Config.init("missing_conf")
+ self.assertEqual(e.exception.message, "[XOS-Config] Config file not found at: missing_conf")
+
+ def test_yaml_not_valid(self):
+ """
+ [XOS-Config] Raise if yaml is not valid
+ """
+ with self.assertRaises(Exception) as e:
+ Config.init(yaml_not_valid)
+ self.assertEqual(e.exception.message, "[XOS-Config] The config format is wrong: Unable to load any data from source yaml file")
+
+ def test_invalid_format(self):
+ """
+ [XOS-Config] Raise if format is not valid (we expect a dictionary)
+ """
+ with self.assertRaises(Exception) as e:
+ Config.init(invalid_format)
+ self.assertEqual(e.exception.message, "[XOS-Config] The config format is wrong: Schema validation failed:\n - Value '['I am', 'a yaml', 'but the', 'format is not', 'correct']' is not a dict. Value path: ''.")
+
+ def test_env_override(self):
+ """
+ [XOS-Config] the XOS-CONFIG environment variable should override the config_file
+ """
+ os.environ["XOS-CONFIG"] = "env.yaml"
+ with self.assertRaises(Exception) as e:
+ Config.init("missing_conf")
+ self.assertEqual(e.exception.message, "[XOS-Config] Config file not found at: env.yaml")
+ del os.environ["XOS-CONFIG"]
+
+ def test_get_cli_param(self):
+ """
+ [XOS-Config] Should read CLI -C param
+ """
+ args = ["-A", "Foo", "-c", "Bar", "-C", "config.yaml"]
+ res = Config.get_cli_param(args)
+ self.assertEqual(res, "config.yaml")
+
+ def test_get_default_val_for_missing_param(self):
+ """
+ [XOS-Config] Should raise reading a missing param
+ """
+ Config.init(basic_conf)
+ log = Config.get("logging")
+ self.assertEqual(log, {
+ "level": "info",
+ "channels": ["file", "console"]
+ })
+
+ def _test_get_missing_param(self):
+ """
+ [XOS-Config] Should raise reading a missing param
+ """
+ Config.init(sample_conf)
+ with self.assertRaises(Exception) as e:
+ Config.get("foo")
+ self.assertEqual(e.exception.message, "[XOS-Config] Config does not have a value (or a default) parameter foo")
+
+ def test_get_first_level(self):
+ """
+ [XOS-Config] Should return a first level param
+ """
+ Config.init(sample_conf)
+ # NOTE we are using Config2 here to be sure that the configuration is readable from any import,
+ # not only from the one that has been used to initialize it
+ res = Config2.get("database")
+ self.assertEqual(res, {
+ "username": "test",
+ "password": "safe"
+ })
+
+ def _test_get_child_level(self):
+ """
+ [XOS-Config] Should return a child level param
+ """
+ Config.init(sample_conf)
+ res = Config.get("nested.parameter.for")
+ self.assertEqual(res, "testing")
+
+ def test_get_service_list(self):
+ """
+ [XOS-Config] Should query registrator and return a list of services
+ """
+ with patch("xosconfig.config.requests.get") as mock_get:
+ mock_get.return_value.json.return_value = services_list
+ res = Config.get_service_list()
+ self.assertEqual(res, [
+ "xos-ws",
+ "xos-db",
+ ])
+
+ def test_get_service_info(self):
+ """
+ [XOS-Config] Should query registrator and return service info
+ """
+ with patch("xosconfig.config.requests.get") as mock_get:
+ mock_get.return_value.json.return_value = db_service
+ info = Config.get_service_info("xos-db")
+ self.assertEqual(info, {
+ "name": "xos-db",
+ "url": "172.18.0.4",
+ "port": 5432
+ })
+
+ def test_fail_get_service_info(self):
+ """
+ [XOS-Config] Should query registrator and return an exception if it"s down
+ """
+ with patch("xosconfig.config.requests.get") as mock_get:
+ mock_get.return_value.ok = False
+ with self.assertRaises(Exception) as e:
+ Config.get_service_info("missing-service")
+ self.assertEqual(e.exception.message, "[XOS-Config] Registrator is down")
+
+ def test_missing_get_service_info(self):
+ """
+ [XOS-Config] Should query registrator and return an exception if service is not there
+ """
+ with patch("xosconfig.config.requests.get") as mock_get:
+ mock_get.return_value.json.return_value = []
+ with self.assertRaises(Exception) as e:
+ Config.get_service_info("missing-service")
+ self.assertEqual(e.exception.message, "[XOS-Config] The service missing-service looking for does not exist")
+
+
+ def test_get_service_endpoint(self):
+ """
+ [XOS-Config] Should query registrator and return service endpoint
+ """
+ with patch("xosconfig.config.requests.get") as mock_get:
+ mock_get.return_value.json.return_value = db_service
+ endpoint = Config.get_service_endpoint("xos-db")
+ self.assertEqual(endpoint, "http://172.18.0.4:5432")
\ No newline at end of file
diff --git a/lib/xos-config/tests/confs/basic_conf.yaml b/lib/xos-config/tests/confs/basic_conf.yaml
new file mode 100644
index 0000000..0f1d0b0
--- /dev/null
+++ b/lib/xos-config/tests/confs/basic_conf.yaml
@@ -0,0 +1,4 @@
+name: test
+database:
+ username: test
+ password: safe
\ No newline at end of file
diff --git a/lib/xos-config/tests/confs/invalid_format.yaml b/lib/xos-config/tests/confs/invalid_format.yaml
new file mode 100644
index 0000000..1972c5b
--- /dev/null
+++ b/lib/xos-config/tests/confs/invalid_format.yaml
@@ -0,0 +1,5 @@
+- I am
+- a yaml
+- but the
+- format is not
+- correct
\ No newline at end of file
diff --git a/lib/xos-config/tests/confs/sample_conf.yaml b/lib/xos-config/tests/confs/sample_conf.yaml
new file mode 100644
index 0000000..b398b40
--- /dev/null
+++ b/lib/xos-config/tests/confs/sample_conf.yaml
@@ -0,0 +1,9 @@
+name: xos-core
+database:
+ username: test
+ password: safe
+logging:
+ level: info
+ channels:
+ - file
+ - console
\ No newline at end of file
diff --git a/lib/xos-config/tests/confs/yaml_not_valid.yaml b/lib/xos-config/tests/confs/yaml_not_valid.yaml
new file mode 100644
index 0000000..a06cd5b
--- /dev/null
+++ b/lib/xos-config/tests/confs/yaml_not_valid.yaml
@@ -0,0 +1,4 @@
+i am:
+ not:
+ - a valid
+ yaml
\ No newline at end of file
diff --git a/lib/xos-config/xosconfig/__init__.py b/lib/xos-config/xosconfig/__init__.py
new file mode 100644
index 0000000..3558f42
--- /dev/null
+++ b/lib/xos-config/xosconfig/__init__.py
@@ -0,0 +1 @@
+from .config import Config
\ No newline at end of file
diff --git a/lib/xos-config/xosconfig/config-schema.yaml b/lib/xos-config/xosconfig/config-schema.yaml
new file mode 100644
index 0000000..9a6f000
--- /dev/null
+++ b/lib/xos-config/xosconfig/config-schema.yaml
@@ -0,0 +1,21 @@
+map:
+ name:
+ type: str
+ required: True
+ database:
+ type: map
+ map:
+ username:
+ type: str
+ password:
+ type: str
+ logging:
+ type: map
+ map:
+ level:
+ type: str
+ channels:
+ type: seq
+ sequence:
+ - type: str
+ enum: ['file', 'console', 'elkstack']
\ No newline at end of file
diff --git a/lib/xos-config/xosconfig/config.py b/lib/xos-config/xosconfig/config.py
new file mode 100644
index 0000000..e58477c
--- /dev/null
+++ b/lib/xos-config/xosconfig/config.py
@@ -0,0 +1,172 @@
+import os
+import sys
+import yaml
+import requests
+import default
+from pykwalify.core import Core as PyKwalify
+
+DEFAULT_CONFIG_FILE = "/opt/xos/config.yaml"
+INITIALIZED = False
+CONFIG = {}
+
+class Config:
+ """
+ XOS Configuration APIs
+ """
+
+ @staticmethod
+ def init(config_file=DEFAULT_CONFIG_FILE):
+ global INITIALIZED
+ global CONFIG
+ # the config module can be initialized only one
+ if INITIALIZED:
+ raise Exception('[XOS-Config] Module already initialized')
+ INITIALIZED = True
+
+ # if XOS-CONFIG is defined override the config_file
+ if os.environ.get('XOS-CONFIG'):
+ config_file = os.environ['XOS-CONFIG']
+
+ # if a -C parameter is set in the cli override the config_file
+ # FIXME shouldn't this stay in whatever module call this one? and then just pass the file to the init method
+ if Config.get_cli_param(sys.argv):
+ config_file = Config.get_cli_param(sys.argv)
+
+ CONFIG = Config.read_config(config_file)
+
+ @staticmethod
+ def clear():
+ global INITIALIZED
+ INITIALIZED = False
+
+ @staticmethod
+ def validate_config_format(config_file):
+ schema = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + '/config-schema.yaml')
+ c = PyKwalify(source_file=config_file, schema_files=[schema])
+ c.validate(raise_exception=True)
+
+ @staticmethod
+ def get_cli_param(args):
+ last = None
+ for arg in args:
+ if last == '-C':
+ return arg
+ last = arg
+
+ @staticmethod
+ def read_config(config_file):
+ """
+ Read the configuration file and return a dictionary
+ :param config_file: string
+ :return: dict
+ """
+ if not os.path.exists(config_file):
+ raise Exception('[XOS-Config] Config file not found at: %s' % config_file)
+
+ try:
+ Config.validate_config_format(config_file)
+ except Exception, e:
+ raise Exception('[XOS-Config] The config format is wrong: %s' % e.msg)
+
+ with open(config_file, 'r') as stream:
+ return yaml.safe_load(stream)
+
+ @staticmethod
+ def get(query):
+ """
+ Read a parameter from the config
+ :param query: a dot separated selector for configuration options (eg: database.username)
+ :return: the requested parameter in any format the parameter is specified
+ """
+ global INITIALIZED
+ global CONFIG
+
+ if not INITIALIZED:
+ raise Exception('[XOS-Config] Module has not been initialized')
+
+ val = Config.get_param(query, CONFIG)
+ if not val:
+ val = Config.get_param(query, default.DEFAULT_VALUES)
+ if not val:
+ raise Exception('[XOS-Config] Config does not have a value (or a default) parameter %s' % query)
+ return val
+
+ @staticmethod
+ def get_param(query, config):
+ """
+ Search for a parameter in config's first level, other call get_nested_param
+ :param query: a dot separated selector for configuration options (eg: database.username)
+ :param config: the config source to read from (can be the config file or the defaults)
+ :return: the requested parameter in any format the parameter is specified
+ """
+ keys = query.split('.')
+ if len(keys) == 1:
+ key = keys[0]
+ if not config.has_key(key):
+ return None
+ return config[key]
+ else:
+ return Config.get_nested_param(keys, config)
+
+ @staticmethod
+ def get_nested_param(keys, config):
+ """
+
+ :param keys: a list of descending selector
+ :param config: the config source to read from (can be the config file or the defaults)
+ :return: the requested parameter in any format the parameter is specified
+ """
+ param = config
+ for k in keys:
+ if not param.has_key(k):
+ return None
+ param = param[k]
+ return param
+
+ @staticmethod
+ def get_service_list():
+ """
+ Query registrator to get the list of services
+ NOTE: we assume that consul is a valid URL
+ :return: a list of service names
+ """
+ service_dict = requests.get('http://consul:8500/v1/catalog/services').json()
+ service_list = []
+ for s in service_dict:
+ service_list.append(s)
+ return service_list
+
+ @staticmethod
+ def get_service_info(service_name):
+ """
+ Query registrator to get the details about a service
+ NOTE: we assume that consul is a valid URL
+ :param service_name: the name of the service, can be retrieved from get_service_list
+ :return: the informations about a service
+ """
+ response = requests.get('http://consul:8500/v1/catalog/service/%s' % service_name)
+ if not response.ok:
+ raise Exception('[XOS-Config] Registrator is down')
+ service = response.json()
+ if not service or len(service) == 0:
+ raise Exception('[XOS-Config] The service missing-service looking for does not exist')
+ return {
+ 'name': service[0]['ServiceName'],
+ 'url': service[0]['ServiceAddress'],
+ 'port': service[0]['ServicePort']
+ }
+
+ @staticmethod
+ def get_service_endpoint(service_name):
+ """
+ Query registrator to get the details about a service and return the endpoint in for of a string
+ :param service_name: the name of the service, can be retrieved from get_service_list
+ :return: the endpoint of the service
+ """
+ service = Config.get_service_info(service_name)
+ return 'http://%s:%s' % (service['url'], service['port'])
+
+# NOTE is this needed if this package is not meant to be execute from the CLI?
+if __name__ == '__main__':
+ config = Config()
+ config.init()
\ No newline at end of file
diff --git a/lib/xos-config/xosconfig/default.py b/lib/xos-config/xosconfig/default.py
new file mode 100644
index 0000000..221e0e8
--- /dev/null
+++ b/lib/xos-config/xosconfig/default.py
@@ -0,0 +1,6 @@
+DEFAULT_VALUES = {
+ 'logging': {
+ 'level': 'info',
+ 'channels': ['file', 'console']
+ }
+}
\ No newline at end of file
diff --git a/scripts/setup_venv b/scripts/setup_venv.sh
similarity index 71%
rename from scripts/setup_venv
rename to scripts/setup_venv.sh
index 8c5f50d..a8d620d 100644
--- a/scripts/setup_venv
+++ b/scripts/setup_venv.sh
@@ -23,10 +23,14 @@
fi
# install pip requirements
-if pip install -r $REQUIREMENTS
+if \
+pip install cryptography --global-option=build_ext --global-option="-L/usr/local/opt/openssl/lib" --global-option="-I/usr/local/opt/openssl/include" && \
+pip install -r $REQUIREMENTS && \
+cd $BASEDIR/lib/xos-config; python setup.py install
then
echo "Requirements installed."
echo "Virtualenv ready"
else
echo "An error occurred"
fi
+cd $BASEDIR