[SEBA-315] Make kafkaloghandler standalone
[SEBA-314] Update multistructlog version
Change-Id: If013b45f8dfb12a9319fbae600d56250965608b5
diff --git a/VERSION b/VERSION
index 0ddcadb..a62db96 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.1.21
+2.1.22
diff --git a/containers/chameleon/Dockerfile.chameleon b/containers/chameleon/Dockerfile.chameleon
index a6f38b6..67dd70f 100644
--- a/containers/chameleon/Dockerfile.chameleon
+++ b/containers/chameleon/Dockerfile.chameleon
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/chameleon
-FROM xosproject/xos-base:2.1.21
+FROM xosproject/xos-base:2.1.22
# xos-base already has protoc and dependencies installed
diff --git a/containers/xos/Dockerfile.client b/containers/xos/Dockerfile.client
index 89b578b..f2fd717 100644
--- a/containers/xos/Dockerfile.client
+++ b/containers/xos/Dockerfile.client
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/xos-client
-FROM xosproject/xos-libraries:2.1.21
+FROM xosproject/xos-libraries:2.1.22
# Install XOS client
COPY xos/xos_client /tmp/xos_client
diff --git a/containers/xos/Dockerfile.libraries b/containers/xos/Dockerfile.libraries
index 4513a2d..801c93e 100644
--- a/containers/xos/Dockerfile.libraries
+++ b/containers/xos/Dockerfile.libraries
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/xos-libraries
-FROM xosproject/xos-base:2.1.21
+FROM xosproject/xos-base:2.1.22
# Add libraries
COPY lib /opt/xos/lib
@@ -23,8 +23,7 @@
RUN cd /opt/xos/lib/xos-util && python setup.py install \
&& cd /opt/xos/lib/xos-config && python setup.py install \
&& cd /opt/xos/lib/xos-genx && python setup.py install \
- && cd /opt/xos/lib/xos-kafka && python setup.py install \
- && cd /opt/xos/lib/kafkaloghandler && python setup.py install
+ && cd /opt/xos/lib/xos-kafka && python setup.py install
# Label image
ARG org_label_schema_schema_version=1.0
diff --git a/containers/xos/Dockerfile.synchronizer-base b/containers/xos/Dockerfile.synchronizer-base
index 35590ab..82cbd49 100644
--- a/containers/xos/Dockerfile.synchronizer-base
+++ b/containers/xos/Dockerfile.synchronizer-base
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/xos-synchronizer-base
-FROM xosproject/xos-client:2.1.21
+FROM xosproject/xos-client:2.1.22
COPY xos/synchronizers/new_base /opt/xos/synchronizers/new_base
COPY xos/xos/logger.py /opt/xos/xos/logger.py
diff --git a/containers/xos/Dockerfile.xos-core b/containers/xos/Dockerfile.xos-core
index b291bc6..bd677ba 100644
--- a/containers/xos/Dockerfile.xos-core
+++ b/containers/xos/Dockerfile.xos-core
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/xos-core
-FROM xosproject/xos-libraries:2.1.21
+FROM xosproject/xos-libraries:2.1.22
# Install XOS
ADD xos /opt/xos
diff --git a/containers/xos/pip_requested.txt b/containers/xos/pip_requested.txt
index a4a7825..34be5b3 100644
--- a/containers/xos/pip_requested.txt
+++ b/containers/xos/pip_requested.txt
@@ -16,10 +16,10 @@
google-api-python-client==1.6.5
grpcio-tools==1.12.0
grpcio==1.12.0
-kafka==1.3.5
+kafkaloghandler==0.8.0
keystoneauth1==3.4.0
mock==2.0.0
-multistructlog==1.5
+multistructlog==2.0.0
netaddr==0.7.19
networkx==1.11
nose==1.3.7
diff --git a/containers/xos/pip_requirements.txt b/containers/xos/pip_requirements.txt
index 49cc324..18666ab 100644
--- a/containers/xos/pip_requirements.txt
+++ b/containers/xos/pip_requirements.txt
@@ -49,14 +49,14 @@
jsonpatch==1.23
jsonpointer==2.0
jsonschema==2.6.0
-kafka==1.3.5
+kafkaloghandler==0.8.0
keystoneauth1==3.4.0
kombu==4.2.1
MarkupSafe==1.0
mock==2.0.0
monotonic==1.5
msgpack==0.5.6
-multistructlog==1.5
+multistructlog==2.0.0
munch==2.3.2
netaddr==0.7.19
netifaces==0.10.7
diff --git a/lib/kafkaloghandler/MANIFEST.in b/lib/kafkaloghandler/MANIFEST.in
deleted file mode 100644
index 9561fb1..0000000
--- a/lib/kafkaloghandler/MANIFEST.in
+++ /dev/null
@@ -1 +0,0 @@
-include README.rst
diff --git a/lib/kafkaloghandler/README.rst b/lib/kafkaloghandler/README.rst
deleted file mode 100644
index 8dd8fd0..0000000
--- a/lib/kafkaloghandler/README.rst
+++ /dev/null
@@ -1,4 +0,0 @@
-KafkaLogHandler
-===============
-
-Provides a logging handler that sends messages to a Kafka message bus.
diff --git a/lib/kafkaloghandler/kafkaloghandler/__init__.py b/lib/kafkaloghandler/kafkaloghandler/__init__.py
deleted file mode 100644
index 4a82628..0000000
--- a/lib/kafkaloghandler/kafkaloghandler/__init__.py
+++ /dev/null
@@ -1,13 +0,0 @@
-# Copyright 2018-present Open Networking Foundation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
diff --git a/lib/kafkaloghandler/kafkaloghandler/kafkaloghandler.py b/lib/kafkaloghandler/kafkaloghandler/kafkaloghandler.py
deleted file mode 100644
index b5a2a06..0000000
--- a/lib/kafkaloghandler/kafkaloghandler/kafkaloghandler.py
+++ /dev/null
@@ -1,150 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright 2018-present Open Networking Foundation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# kafkaloghandler - logging handler that sends to Kafka
-
-import json
-import confluent_kafka
-import logging
-import sys
-import time
-
-
-class KafkaLogHandler(logging.Handler):
-
- def __init__(self,
- bootstrap_servers=["localhost:9092"],
- key="kh", # kafka key
- topic="kafkaloghandler", # kafka topic
- timeout=10.0, # kafka connection timeout
- flatten=3, # maximum depth of dict flattening
- blacklist=["_logger"], # keys excluded from messages
- ):
-
- logging.Handler.__init__(self)
-
- self.bootstrap_servers = bootstrap_servers
- self.topic = topic
- self.key = key
- self.flatten = flatten
- self.blacklist = blacklist
- self.timeout = timeout
- self.producer = None
-
- def _connect(self):
-
- try:
- producer_config = {
- 'bootstrap.servers': ','.join(self.bootstrap_servers),
- }
-
- self.producer = confluent_kafka.Producer(**producer_config)
-
- except confluent_kafka.KafkaError, e:
- print "Kafka Error: %s" % e
- # die if there's an error
- sys.exit(1)
-
- def _flatten(self, ns, toflatten, maxdepth):
- """ flatten dicts creating a key.subkey.subsubkey... hierarchy """
-
- # avoid recursivly flattening forever
- if maxdepth < 1:
- return toflatten
-
- flattened = {}
-
- for k, v in toflatten.iteritems():
-
- prefix = "%s.%s" % (ns, k)
-
- if isinstance(v, dict):
- flattened.update(self._flatten(prefix, v, maxdepth-1))
- else:
- flattened[prefix] = v
-
- return flattened
-
- def emit(self, record):
-
- recvars = {}
-
- for k, v in vars(record).iteritems():
- # skip items in blacklist
- if k in self.blacklist:
- continue
-
- # flatten any sub-dicts down
- if self.flatten and isinstance(v, dict):
- recvars.update(self._flatten(k, v, self.flatten))
- continue
-
- recvars[k] = v
-
- # Replace unserializable items with repr version.
- # Otherwise, the log message may be discarded if it contains any
- # unserializable fields
- json_recvars = json.dumps(
- recvars,
- separators=(',', ':'),
- default=lambda o: repr(o),
- )
-
- if self.producer is None:
- self._connect()
-
- try:
- self.producer.produce(self.topic, json_recvars, self.key)
-
- except confluent_kafka.KafkaError, e:
- print "Kafka Error: %s" % e
- # currently don't do anything on failure...
- pass
-
- def flush(self):
-
- if self.producer:
- self.producer.flush(self.timeout)
-
-
-if __name__ == '__main__':
-
- logger = logging.getLogger(__name__)
- logger.handlers = []
-
- logger.setLevel(logging.INFO)
-
- kh = KafkaLogHandler(
- bootstrap_servers=["test-kafka:9092"],
- topic="testtopic",
- )
-
- logger.addHandler(kh)
-
- logger.error('Error message')
-
- extra_data = {
- "key1": "value1",
- "key2": "value2",
- }
-
- logger.info('Info message with extra data', extra=extra_data)
-
- index = 0
- while True:
- logger.info('Info message - loop count: %s' % index)
- index += 1
- time.sleep(10)
diff --git a/lib/kafkaloghandler/multistructlogtest.py b/lib/kafkaloghandler/multistructlogtest.py
deleted file mode 100644
index c65165f..0000000
--- a/lib/kafkaloghandler/multistructlogtest.py
+++ /dev/null
@@ -1,57 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright 2018-present Open Networking Foundation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import multistructlog
-import time
-
-# config of logging
-logconfig = {
- "version": 1,
- "handlers": {
- "console": {
- "class": "logging.StreamHandler"
- },
- "kafka": {
- "class": "kafkaloghandler.kafkaloghandler.KafkaLogHandler",
- "bootstrap_servers": ["test-kafka:9092"],
- "topic": "testtopic"
- },
- },
-
- "loggers": {
- "multistructlog": {
- "handlers": ["console", "kafka"],
- "level": "DEBUG"
- }
- },
-}
-
-logger = multistructlog.create_logger(logconfig)
-
-logger.error('Test error message')
-
-extra_data = {
- "key1": "value1",
- "key2": "value2",
-}
-
-logger.info('Test info message with extra data', extra=extra_data)
-
-index = 0
-while True:
- logger.info('Info message - loop count: %s' % index)
- index += 1
- time.sleep(10)
diff --git a/lib/kafkaloghandler/setup.py b/lib/kafkaloghandler/setup.py
deleted file mode 100644
index cba89ff..0000000
--- a/lib/kafkaloghandler/setup.py
+++ /dev/null
@@ -1,45 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright 2018-present Open Networking Foundation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from xosutil.autoversion_setup import setup_with_auto_version
-from xosutil.version import __version__
-
-
-def readme():
- with open('README.rst') as f:
- return f.read()
-
-
-setup_with_auto_version(
- name='kafkaloghandler',
- version=__version__,
- description='Kafka Logging Handler',
- long_description=readme(),
- classifiers=[
- 'Topic :: System :: Logging',
- 'Topic :: Internet :: Log Analysis',
- 'License :: OSI Approved :: Apache Software License',
- ],
- author='Zack Williams',
- author_email='zdw@opennetworking.org',
- packages=['kafkaloghandler'],
- license='Apache v2',
- install_requires=[
- 'confluent-kafka>=0.11.5',
- ],
- include_package_data=True,
- zip_safe=False,
- )
diff --git a/lib/xos-config/xosconfig/__init__.py b/lib/xos-config/xosconfig/__init__.py
index a557ff4..d4e8305 100644
--- a/lib/xos-config/xosconfig/__init__.py
+++ b/lib/xos-config/xosconfig/__init__.py
@@ -13,22 +13,3 @@
# limitations under the License.
from .config import Config
-
-# Custom TRACE logging level
-# ref: https://stackoverflow.com/questions/2183233/how-to-add-a-custom-loglevel-to-pythons-logging-facility/13638084#13638084
-
-import logging
-
-# Logging levels: https://docs.python.org/2/library/logging.html#logging-levels
-# Add a sub-DEBUG Trace level
-TRACE_LOGLVL = 5
-
-logging.addLevelName(TRACE_LOGLVL, "TRACE")
-
-
-def trace_loglevel(self, message, *args, **kws):
- if self.isEnabledFor(TRACE_LOGLVL):
- self._log(TRACE_LOGLVL, message, args, **kws)
-
-
-logging.Logger.trace = trace_loglevel
diff --git a/scripts/xos_dev_reqs.txt b/scripts/xos_dev_reqs.txt
index 983e1c3..1b83b3a 100644
--- a/scripts/xos_dev_reqs.txt
+++ b/scripts/xos_dev_reqs.txt
@@ -8,7 +8,7 @@
grpcio-tools==1.12.0
grpcio==1.12.0
ipaddress==1.0.19
-multistructlog==1.5
+multistructlog==2.0.0
netaddr==0.7.19
networkx==1.11
nose2==0.7.4