Async/streaming gRPC client/server proto
This experiment was to fine-tune how we can implement
async gRPC client and server code inside a Twisted
python app.
Change-Id: I945014e27f4b9d6ed624666e0284cc298548adb3
Major cleanup of openflow_13.proto
Change-Id: I4e54eaf87b682124ec518a0ade1a6050a6ec6da8
Relocated openflow_13.proto to voltha
Change-Id: I66ae45a9142d180c2c6651e75c7a1ee08aef7ef8
Removed forced utest from make build
Change-Id: If0da58e9d135ebde6ca68c3316688a03a7b10f2f
twisted openflow agent first pass
Change-Id: Ibe5b4727ccfe92e6fd464ccd3baf6275569ef5d3
store openflow derived files
Change-Id: Ib3e1384bb2ca2a9c0872767f7b793f96b0a154e2
Minor cleanup
Change-Id: I1280ed3acb606121b616a0efd573f5f59d010dca
Factored out common utils
Change-Id: Icd86fcd50f60d0900924674cbcd65e13e47782a1
Refactored twisted agent
Change-Id: I71f26ce5357a4f98477df60b8c5ddc068cf75d43
Relocated openflow agent to ofagent
... and preserved obsolete working (non-twisted) agent under
~/obsolete, so we can still run the olt-oftest and pass tests,
unit the new twisted based agent reaches that maturity point.
Change-Id: I727f8d7144b1291a40276dad2966b7643bd7bc4b
olt-oftest in fake mode works with new agent
Change-Id: I43b4f5812e8dfaa9f45e4a77fdcf6c30ac520f8d
Initial ofagent/voltha operation
Change-Id: Ia8104f1285a6b1c51635d36d7d78fc113f800e79
Additional callouts to Voltha
Change-Id: If8f483d5140d3c9d45f22b480b8d33249a29cd4e
More gRPC calls
Change-Id: I7d24fadf9425217fb26ffe18f25359d072ef38fa
Flow add/list now works
Change-Id: Ie3e3e73108645b47891cef798fc61372a022fd93
Missed some files
Change-Id: I29e81238ff1a26c095c0c73e521579edf7092e21
diff --git a/common/utils/__init__.py b/common/utils/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/common/utils/__init__.py
diff --git a/common/utils/asleep.py b/common/utils/asleep.py
new file mode 100644
index 0000000..e27d313
--- /dev/null
+++ b/common/utils/asleep.py
@@ -0,0 +1,31 @@
+#
+# Copyright 2016 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+""" Async sleep (asleep) method and other twisted goodies """
+
+from twisted.internet import reactor
+from twisted.internet.defer import Deferred
+
+
+def asleep(dt):
+ """
+ Async (event driven) wait for given time period (in seconds)
+ :param dt: Delay in seconds
+ :return: Deferred to be fired with value None when time expires.
+ """
+ d = Deferred()
+ reactor.callLater(dt, lambda: d.callback(None))
+ return d
diff --git a/common/utils/consulhelpers.py b/common/utils/consulhelpers.py
new file mode 100644
index 0000000..91acfc6
--- /dev/null
+++ b/common/utils/consulhelpers.py
@@ -0,0 +1,54 @@
+#
+# Copyright 2016 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+Some consul related convenience functions
+"""
+
+from structlog import get_logger
+from consul import Consul
+from random import randint
+
+log = get_logger()
+
+
+def get_endpoint_from_consul(consul_endpoint, service_name):
+ """Look up, from consul, the service name specified by service-name
+ """
+ log.debug('Retrieving endpoint {} from consul {}'.format(service_name,
+ consul_endpoint))
+ host = consul_endpoint.split(':')[0].strip()
+ port = int(consul_endpoint.split(':')[1].strip())
+
+ consul = Consul(host=host, port=port)
+ _, services = consul.catalog.service(service_name)
+
+ if len(services) == 0:
+ raise Exception(
+ 'Cannot find service {} in consul'.format(service_name))
+
+ # pick a random entry
+ # TODO should we prefer local IP addresses? Probably.
+
+ service = services[randint(0, len(services) - 1)]
+ endpoint = '{}:{}'.format(service['ServiceAddress'],
+ service['ServicePort'])
+
+ return endpoint
+
+
+if __name__ == '__main__':
+ get_endpoint_from_consul('10.100.198.220:8500', 'kafka')
diff --git a/common/utils/dockerhelpers.py b/common/utils/dockerhelpers.py
new file mode 100644
index 0000000..0927711
--- /dev/null
+++ b/common/utils/dockerhelpers.py
@@ -0,0 +1,50 @@
+#
+# Copyright 2016 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+Some docker related convenience functions
+"""
+
+import os
+from structlog import get_logger
+
+from docker import Client
+
+
+log = get_logger()
+
+
+def get_my_containers_name():
+ """
+ Return the docker containers name in which this process is running.
+ To look up the container name, we use the container ID extracted from the
+ $HOSTNAME environment variable (which is set by docker conventions).
+ :return: String with the docker container name (or None if any issue is
+ encountered)
+ """
+ my_container_id = os.environ.get('HOSTNAME', None)
+
+ try:
+ docker_cli = Client(base_url='unix://tmp/docker.sock')
+ info = docker_cli.inspect_container(my_container_id)
+
+ except Exception, e:
+ log.exception('failed', my_container_id=my_container_id, e=e)
+ raise
+
+ name = info['Name'].lstrip('/')
+
+ return name
diff --git a/common/utils/message_queue.py b/common/utils/message_queue.py
new file mode 100644
index 0000000..74b763f
--- /dev/null
+++ b/common/utils/message_queue.py
@@ -0,0 +1,89 @@
+#
+# Copyright 2016 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from twisted.internet.defer import Deferred
+from twisted.internet.defer import succeed
+
+
+class MessageQueue(object):
+ """
+ An event driven queue, similar to twisted.internet.defer.DeferredQueue
+ but which allows selective dequeing based on a predicate function.
+ Unlike DeferredQueue, there is no limit on backlog, and there is no queue
+ limit.
+ """
+
+ def __init__(self):
+ self.waiting = [] # tuples of (d, predicate)
+ self.queue = [] # messages piling up here if no one is waiting
+
+ def reset(self):
+ """
+ Purge all content as well as waiters (by errback-ing their entries).
+ :return: None
+ """
+ for d, _ in self.waiting:
+ d.errback(Exception('mesage queue reset() was called'))
+ self.waiting = []
+ self.queue = []
+
+ def _cancelGet(self, d):
+ """
+ Remove a deferred from our waiting list.
+ :param d: The deferred that was been canceled.
+ :return: None
+ """
+ for i in range(len(self.waiting)):
+ if self.waiting[i][0] is d:
+ self.waiting.pop(i)
+
+ def put(self, obj):
+ """
+ Add an object to this queue
+ :param obj: arbitrary object that will be added to the queue
+ :return:
+ """
+
+ # if someone is waiting for this, return right away
+ for i in range(len(self.waiting)):
+ d, predicate = self.waiting[i]
+ if predicate is None or predicate(obj):
+ self.waiting.pop(i)
+ d.callback(obj)
+ return
+
+ # otherwise...
+ self.queue.append(obj)
+
+ def get(self, predicate=None):
+ """
+ Attempt to retrieve and remove an object from the queue that
+ matches the optional predicate.
+ :return: Deferred which fires with the next object available.
+ If predicate was provided, only objects for which
+ predicate(obj) is True will be considered.
+ """
+ for i in range(len(self.queue)):
+ msg = self.queue[i]
+ if predicate is None or predicate(msg):
+ self.queue.pop(i)
+ return succeed(msg)
+
+ # there were no matching entries if we got here, so we wait
+ d = Deferred(canceller=self._cancelGet)
+ self.waiting.append((d, predicate))
+ return d
+
+
diff --git a/common/utils/nethelpers.py b/common/utils/nethelpers.py
new file mode 100644
index 0000000..52f7f4c
--- /dev/null
+++ b/common/utils/nethelpers.py
@@ -0,0 +1,46 @@
+#
+# Copyright 2016 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+Some network related convenience functions
+"""
+
+from netifaces import AF_INET
+
+import netifaces as ni
+
+
+def get_my_primary_interface():
+ gateways = ni.gateways()
+ assert 'default' in gateways, \
+ ("No default gateway on host/container, "
+ "cannot determine primary interface")
+ default_gw_index = gateways['default'].keys()[0]
+ # gateways[default_gw_index] has the format (example):
+ # [('10.15.32.1', 'en0', True)]
+ interface_name = gateways[default_gw_index][0][1]
+ return interface_name
+
+
+def get_my_primary_local_ipv4(ifname=None):
+ ifname = get_my_primary_interface() if ifname is None else ifname
+ addresses = ni.ifaddresses(ifname)
+ ipv4 = addresses[AF_INET][0]['addr']
+ return ipv4
+
+
+if __name__ == '__main__':
+ print get_my_primary_local_ipv4()
diff --git a/common/utils/structlog_setup.py b/common/utils/structlog_setup.py
new file mode 100644
index 0000000..d4b2e8e
--- /dev/null
+++ b/common/utils/structlog_setup.py
@@ -0,0 +1,113 @@
+#
+# Copyright 2016 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Setting up proper logging for Voltha"""
+
+import logging
+import logging.config
+from collections import OrderedDict
+
+import structlog
+from structlog.stdlib import BoundLogger, INFO
+
+try:
+ from thread import get_ident as _get_ident
+except ImportError:
+ from dummy_thread import get_ident as _get_ident
+
+
+class FluentRenderer(object):
+ def __call__(self, logger, name, event_dict):
+ # in order to keep structured log data in event_dict to be forwarded as
+ # is to the fluent logger, we need to pass it into the logger framework
+ # as the first positional argument.
+ args = (event_dict, )
+ kwargs = {}
+ return args, kwargs
+
+
+class PlainRenderedOrderedDict(OrderedDict):
+ """Our special version of OrderedDict that renders into string as a dict,
+ to make the log stream output cleaner.
+ """
+ def __repr__(self, _repr_running={}):
+ 'od.__repr__() <==> repr(od)'
+ call_key = id(self), _get_ident()
+ if call_key in _repr_running:
+ return '...'
+ _repr_running[call_key] = 1
+ try:
+ if not self:
+ return '{}'
+ return '{%s}' % ", ".join("%s: %s" % (k, v)
+ for k, v in self.items())
+ finally:
+ del _repr_running[call_key]
+
+
+def setup_logging(log_config, instance_id, verbosity_adjust=0, fluentd=None):
+ """
+ Set up logging such that:
+ - The primary logging entry method is structlog
+ (see http://structlog.readthedocs.io/en/stable/index.html)
+ - By default, the logging backend is Python standard lib logger
+ - Alternatively, fluentd can be configured with to be the backend,
+ providing direct bridge to a fluent logging agent.
+ """
+
+ def add_exc_info_flag_for_exception(_, name, event_dict):
+ if name == 'exception':
+ event_dict['exc_info'] = True
+ return event_dict
+
+ def add_instance_id(_, __, event_dict):
+ event_dict['instance_id'] = instance_id
+ return event_dict
+
+ # if fluentd is specified, we need to override the config data with
+ # its host and port info
+ if fluentd is not None:
+ fluentd_host = fluentd.split(':')[0].strip()
+ fluentd_port = int(fluentd.split(':')[1].strip())
+
+ handlers = log_config.get('handlers', None)
+ if isinstance(handlers, dict):
+ for _, defs in handlers.iteritems():
+ if isinstance(defs, dict):
+ if defs.get('class', '').endswith('FluentHandler'):
+ defs['host'] = fluentd_host
+ defs['port'] = fluentd_port
+
+ # Configure standard logging
+ logging.config.dictConfig(log_config)
+ logging.root.level -= 10 * verbosity_adjust
+
+ processors = [
+ add_exc_info_flag_for_exception,
+ structlog.processors.StackInfoRenderer(),
+ structlog.processors.format_exc_info,
+ add_instance_id,
+ FluentRenderer(),
+ ]
+ structlog.configure(logger_factory=structlog.stdlib.LoggerFactory(),
+ context_class=PlainRenderedOrderedDict,
+ wrapper_class=BoundLogger,
+ processors=processors)
+
+ # Mark first line of log
+ log = structlog.get_logger()
+ log.info("first-line")
+ return log