Chameleon outline: a REST-to-GRPC gateway
diff --git a/__init__.py b/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/__init__.py
diff --git a/chameleon.yml b/chameleon.yml
new file mode 100644
index 0000000..2b1d69e
--- /dev/null
+++ b/chameleon.yml
@@ -0,0 +1,44 @@
+logging:
+ version: 1
+
+ formatters:
+ brief:
+ format: '%(message)s'
+ default:
+ format: '%(asctime)s.%(msecs)03d %(levelname)-8s %(module)s.%(funcName)s %(message)s'
+ datefmt: '%Y%m%dT%H%M%S'
+ fluent_fmt:
+ '()': fluent.handler.FluentRecordFormatter
+ format:
+ level: '%(levelname)s'
+ hostname: '%(hostname)s'
+ where: '%(module)s.%(funcName)s'
+
+ handlers:
+ console:
+ class : logging.StreamHandler
+ level: DEBUG
+ formatter: default
+ stream: ext://sys.stdout
+ fluent:
+ class: fluent.handler.FluentHandler
+ host: localhost
+ port: 24224
+ tag: voltha.logging
+ formatter: fluent_fmt
+ level: DEBUG
+ null:
+ class: logging.NullHandler
+
+ loggers:
+ amqp:
+ handlers: [null]
+ propagate: False
+ conf:
+ handlers: [null]
+ propagate: False
+ '': # root logger
+ handlers: [console, fluent]
+ level: INFO # this can be bumped up/down by -q and -v command line
+ # options
+ propagate: False
diff --git a/dockerhelpers.py b/dockerhelpers.py
new file mode 100644
index 0000000..0927711
--- /dev/null
+++ b/dockerhelpers.py
@@ -0,0 +1,50 @@
+#
+# Copyright 2016 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+Some docker related convenience functions
+"""
+
+import os
+from structlog import get_logger
+
+from docker import Client
+
+
+log = get_logger()
+
+
+def get_my_containers_name():
+ """
+ Return the docker containers name in which this process is running.
+ To look up the container name, we use the container ID extracted from the
+ $HOSTNAME environment variable (which is set by docker conventions).
+ :return: String with the docker container name (or None if any issue is
+ encountered)
+ """
+ my_container_id = os.environ.get('HOSTNAME', None)
+
+ try:
+ docker_cli = Client(base_url='unix://tmp/docker.sock')
+ info = docker_cli.inspect_container(my_container_id)
+
+ except Exception, e:
+ log.exception('failed', my_container_id=my_container_id, e=e)
+ raise
+
+ name = info['Name'].lstrip('/')
+
+ return name
diff --git a/main.py b/main.py
new file mode 100755
index 0000000..fa63210
--- /dev/null
+++ b/main.py
@@ -0,0 +1,224 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""A REST protocol gateway to self-describing GRPC end-points"""
+
+import argparse
+import os
+import sys
+import time
+import yaml
+from twisted.internet.defer import inlineCallbacks
+
+base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(base_dir)
+
+from chameleon.structlog_setup import setup_logging
+from chameleon.nethelpers import get_my_primary_local_ipv4
+from chameleon.dockerhelpers import get_my_containers_name
+
+
+defs = dict(
+ #consul=os.environ.get('CONSUL', 'localhost:8500'),
+ instance_id=os.environ.get('INSTANCE_ID', os.environ.get('HOSTNAME', '1')),
+ config=os.environ.get('CONFIG', './chameleon.yml'),
+ internal_host_address=os.environ.get('INTERNAL_HOST_ADDRESS',
+ get_my_primary_local_ipv4()),
+ external_host_address=os.environ.get('EXTERNAL_HOST_ADDRESS',
+ get_my_primary_local_ipv4()),
+ fluentd=os.environ.get('FLUENTD', None),
+ rest_port=os.environ.get('REST_PORT', 8881),
+)
+
+
+def parse_args():
+
+ parser = argparse.ArgumentParser()
+
+ _help = ('Path to chameleon.yml config file (default: %s). '
+ 'If relative, it is relative to main.py of chameleon.'
+ % defs['config'])
+ parser.add_argument('-c', '--config',
+ dest='config',
+ action='store',
+ default=defs['config'],
+ help=_help)
+
+ '''
+ _help = '<hostname>:<port> to consul agent (default: %s)' % defs['consul']
+ parser.add_argument(
+ '-C', '--consul', dest='consul', action='store',
+ default=defs['consul'],
+ help=_help)
+ '''
+
+ _help = ('<hostname> or <ip> at which Chameleon is reachable from outside '
+ 'the cluster (default: %s)' % defs['external_host_address'])
+ parser.add_argument('-E', '--external-host-address',
+ dest='external_host_address',
+ action='store',
+ default=defs['external_host_address'],
+ help=_help)
+
+ _help = ('<hostname>:<port> to fluentd server (default: %s). (If not '
+ 'specified (None), the address from the config file is used'
+ % defs['fluentd'])
+ parser.add_argument('-F', '--fluentd',
+ dest='fluentd',
+ action='store',
+ default=defs['fluentd'],
+ help=_help)
+
+ _help = ('<hostname> or <ip> at which Chameleon is reachable from inside'
+ 'the cluster (default: %s)' % defs['internal_host_address'])
+ parser.add_argument('-H', '--internal-host-address',
+ dest='internal_host_address',
+ action='store',
+ default=defs['internal_host_address'],
+ help=_help)
+
+ _help = ('unique string id of this Chameleon instance (default: %s)'
+ % defs['instance_id'])
+ parser.add_argument('-i', '--instance-id',
+ dest='instance_id',
+ action='store',
+ default=defs['instance_id'],
+ help=_help)
+
+ _help = 'omit startup banner log lines'
+ parser.add_argument('-n', '--no-banner',
+ dest='no_banner',
+ action='store_true',
+ default=False,
+ help=_help)
+
+ _help = ('port number for the rest service (default: %d)'
+ % defs['rest_port'])
+ parser.add_argument('-R', '--rest-port',
+ dest='rest_port',
+ action='store',
+ type=int,
+ default=defs['rest_port'],
+ help=_help)
+
+ _help = "suppress debug and info logs"
+ parser.add_argument('-q', '--quiet',
+ dest='quiet',
+ action='count',
+ help=_help)
+
+ _help = 'enable verbose logging'
+ parser.add_argument('-v', '--verbose',
+ dest='verbose',
+ action='count',
+ help=_help)
+
+ _help = ('use docker container name as Chameleon instance id'
+ ' (overrides -i/--instance-id option)')
+ parser.add_argument('--instance-id-is-container-name',
+ dest='instance_id_is_container_name',
+ action='store_true',
+ default=False,
+ help=_help)
+
+ args = parser.parse_args()
+
+ # post-processing
+
+ if args.instance_id_is_container_name:
+ args.instance_id = get_my_containers_name()
+
+ return args
+
+
+def load_config(args):
+ path = args.config
+ if path.startswith('.'):
+ dir = os.path.dirname(os.path.abspath(__file__))
+ path = os.path.join(dir, path)
+ path = os.path.abspath(path)
+ with open(path) as fd:
+ config = yaml.load(fd)
+ return config
+
+banner = r'''
+ ____ _ _
+ / ___| | |__ __ _ _ __ ___ ___ | | ___ ___ _ __
+ | | | '_ \ / _` | | '_ ` _ \ / _ \ | | / _ \ / _ \ | '_ \
+ | |___ | | | | | (_| | | | | | | | | __/ | | | __/ | (_) | | | | |
+ \____| |_| |_| \__,_| |_| |_| |_| \___| |_| \___| \___/ |_| |_|
+
+'''
+def print_banner(log):
+ for line in banner.strip('\n').splitlines():
+ log.info(line)
+ log.info('(to stop: press Ctrl-C)')
+
+
+class Main(object):
+
+ def __init__(self):
+
+ self.args = args = parse_args()
+ self.config = load_config(args)
+
+ verbosity_adjust = (args.verbose or 0) - (args.quiet or 0)
+ self.log = setup_logging(self.config.get('logging', {}),
+ args.instance_id,
+ verbosity_adjust=verbosity_adjust,
+ fluentd=args.fluentd)
+
+ # components
+ self.rest_server = None
+ self.grpc_client = None
+
+ if not args.no_banner:
+ print_banner(self.log)
+
+ self.startup_components()
+
+ def start(self):
+ self.start_reactor() # will not return except Keyboard interrupt
+
+ def startup_components(self):
+ self.log.info('starting-internal-components')
+
+ # TODO init client
+ # TODO init server
+
+ self.log.info('started-internal-services')
+
+ @inlineCallbacks
+ def shutdown_components(self):
+ """Execute before the reactor is shut down"""
+ self.log.info('exiting-on-keyboard-interrupt')
+ if self.rest_server is not None:
+ yield self.rest_server.shutdown()
+ if self.grpc_client is not None:
+ yield self.grpc_client.shutdown()
+
+ def start_reactor(self):
+ from twisted.internet import reactor
+ reactor.callWhenRunning(
+ lambda: self.log.info('twisted-reactor-started'))
+ reactor.addSystemEventTrigger('before', 'shutdown',
+ self.shutdown_components)
+ reactor.run()
+
+
+if __name__ == '__main__':
+ Main().start()
diff --git a/nethelpers.py b/nethelpers.py
new file mode 100644
index 0000000..52f7f4c
--- /dev/null
+++ b/nethelpers.py
@@ -0,0 +1,46 @@
+#
+# Copyright 2016 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+Some network related convenience functions
+"""
+
+from netifaces import AF_INET
+
+import netifaces as ni
+
+
+def get_my_primary_interface():
+ gateways = ni.gateways()
+ assert 'default' in gateways, \
+ ("No default gateway on host/container, "
+ "cannot determine primary interface")
+ default_gw_index = gateways['default'].keys()[0]
+ # gateways[default_gw_index] has the format (example):
+ # [('10.15.32.1', 'en0', True)]
+ interface_name = gateways[default_gw_index][0][1]
+ return interface_name
+
+
+def get_my_primary_local_ipv4(ifname=None):
+ ifname = get_my_primary_interface() if ifname is None else ifname
+ addresses = ni.ifaddresses(ifname)
+ ipv4 = addresses[AF_INET][0]['addr']
+ return ipv4
+
+
+if __name__ == '__main__':
+ print get_my_primary_local_ipv4()
diff --git a/structlog_setup.py b/structlog_setup.py
new file mode 100644
index 0000000..d4b2e8e
--- /dev/null
+++ b/structlog_setup.py
@@ -0,0 +1,113 @@
+#
+# Copyright 2016 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Setting up proper logging for Voltha"""
+
+import logging
+import logging.config
+from collections import OrderedDict
+
+import structlog
+from structlog.stdlib import BoundLogger, INFO
+
+try:
+ from thread import get_ident as _get_ident
+except ImportError:
+ from dummy_thread import get_ident as _get_ident
+
+
+class FluentRenderer(object):
+ def __call__(self, logger, name, event_dict):
+ # in order to keep structured log data in event_dict to be forwarded as
+ # is to the fluent logger, we need to pass it into the logger framework
+ # as the first positional argument.
+ args = (event_dict, )
+ kwargs = {}
+ return args, kwargs
+
+
+class PlainRenderedOrderedDict(OrderedDict):
+ """Our special version of OrderedDict that renders into string as a dict,
+ to make the log stream output cleaner.
+ """
+ def __repr__(self, _repr_running={}):
+ 'od.__repr__() <==> repr(od)'
+ call_key = id(self), _get_ident()
+ if call_key in _repr_running:
+ return '...'
+ _repr_running[call_key] = 1
+ try:
+ if not self:
+ return '{}'
+ return '{%s}' % ", ".join("%s: %s" % (k, v)
+ for k, v in self.items())
+ finally:
+ del _repr_running[call_key]
+
+
+def setup_logging(log_config, instance_id, verbosity_adjust=0, fluentd=None):
+ """
+ Set up logging such that:
+ - The primary logging entry method is structlog
+ (see http://structlog.readthedocs.io/en/stable/index.html)
+ - By default, the logging backend is Python standard lib logger
+ - Alternatively, fluentd can be configured with to be the backend,
+ providing direct bridge to a fluent logging agent.
+ """
+
+ def add_exc_info_flag_for_exception(_, name, event_dict):
+ if name == 'exception':
+ event_dict['exc_info'] = True
+ return event_dict
+
+ def add_instance_id(_, __, event_dict):
+ event_dict['instance_id'] = instance_id
+ return event_dict
+
+ # if fluentd is specified, we need to override the config data with
+ # its host and port info
+ if fluentd is not None:
+ fluentd_host = fluentd.split(':')[0].strip()
+ fluentd_port = int(fluentd.split(':')[1].strip())
+
+ handlers = log_config.get('handlers', None)
+ if isinstance(handlers, dict):
+ for _, defs in handlers.iteritems():
+ if isinstance(defs, dict):
+ if defs.get('class', '').endswith('FluentHandler'):
+ defs['host'] = fluentd_host
+ defs['port'] = fluentd_port
+
+ # Configure standard logging
+ logging.config.dictConfig(log_config)
+ logging.root.level -= 10 * verbosity_adjust
+
+ processors = [
+ add_exc_info_flag_for_exception,
+ structlog.processors.StackInfoRenderer(),
+ structlog.processors.format_exc_info,
+ add_instance_id,
+ FluentRenderer(),
+ ]
+ structlog.configure(logger_factory=structlog.stdlib.LoggerFactory(),
+ context_class=PlainRenderedOrderedDict,
+ wrapper_class=BoundLogger,
+ processors=processors)
+
+ # Mark first line of log
+ log = structlog.get_logger()
+ log.info("first-line")
+ return log