[VOL-634] Remove fluentd logging reference from voltha services
This update removes the "hardcoded" logging framework (fluentd)
from the voltha, ofagent and netconf services. The logging is
now determined at deployment time. Testing was done in kubernetes
environment and minimally in docker swarm mode environment (docker
swarm mode is not required for voltha 1.3). A separate Jira will
be raised to adjust the docker swarm deployment files, i.e. direct
the logs to fluentd/fluentd-agg listen ports instead of writing locally.
Change-Id: Ic780fc9633f0f833f961c97dc59d89eed2112a53
diff --git a/Jenkinsfile b/Jenkinsfile
index b4acd75..50c55f0 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -13,9 +13,6 @@
stage 'Remove the pre-created venv-linux'
sh 'vagrant ssh -c "rm -rf /cord/incubator/voltha/venv-linux"'
- stage 'Use the jenkins voltha.yml file'
- sh 'vagrant ssh -c "cp /cord/incubator/voltha/voltha/voltha.jenkins.yml /cord/incubator/voltha/voltha/voltha.yml"'
-
stage 'Build voltha'
sh 'vagrant ssh -c "cd /cord/incubator/voltha && source env.sh && make fetch && make build" voltha'
diff --git a/common/structlog_setup.py b/common/structlog_setup.py
index be04ded..cbbda89 100644
--- a/common/structlog_setup.py
+++ b/common/structlog_setup.py
@@ -29,12 +29,12 @@
from dummy_thread import get_ident as _get_ident
-class FluentRenderer(object):
+class StructuredLogRenderer(object):
def __call__(self, logger, name, event_dict):
# in order to keep structured log data in event_dict to be forwarded as
- # is to the fluent logger, we need to pass it into the logger framework
- # as the first positional argument.
- args = (event_dict, )
+ # is, we need to pass it into the logger framework as the first
+ # positional argument.
+ args = (event_dict,)
kwargs = {}
return args, kwargs
@@ -58,14 +58,12 @@
del _repr_running[call_key]
-def setup_logging(log_config, instance_id, verbosity_adjust=0, fluentd=None):
+def setup_logging(log_config, instance_id, verbosity_adjust=0):
"""
Set up logging such that:
- The primary logging entry method is structlog
(see http://structlog.readthedocs.io/en/stable/index.html)
- By default, the logging backend is Python standard lib logger
- - Alternatively, fluentd can be configured with to be the backend,
- providing direct bridge to a fluent logging agent.
"""
def add_exc_info_flag_for_exception(_, name, event_dict):
@@ -77,20 +75,6 @@
event_dict['instance_id'] = instance_id
return event_dict
- # if fluentd is specified, we need to override the config data with
- # its host and port info
- if fluentd is not None:
- fluentd_host = fluentd.split(':')[0].strip()
- fluentd_port = int(fluentd.split(':')[1].strip())
-
- handlers = log_config.get('handlers', None)
- if isinstance(handlers, dict):
- for _, defs in handlers.iteritems():
- if isinstance(defs, dict):
- if defs.get('class', '').endswith('FluentHandler'):
- defs['host'] = fluentd_host
- defs['port'] = fluentd_port
-
# Configure standard logging
logging.config.dictConfig(log_config)
logging.root.level -= 10 * verbosity_adjust
@@ -100,7 +84,7 @@
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
add_instance_id,
- FluentRenderer(),
+ StructuredLogRenderer(),
]
structlog.configure(logger_factory=structlog.stdlib.LoggerFactory(),
context_class=PlainRenderedOrderedDict,
@@ -138,7 +122,7 @@
structlog.processors.format_exc_info,
add_instance_id,
add_vcore_id,
- FluentRenderer(),
+ StructuredLogRenderer(),
]
structlog.configure(processors=processors)
diff --git a/compose/docker-compose-all.yml.j2 b/compose/docker-compose-all.yml.j2
index cdf829b..46cccfe 100755
--- a/compose/docker-compose-all.yml.j2
+++ b/compose/docker-compose-all.yml.j2
@@ -130,7 +130,6 @@
- voltha/voltha/main.py
- -v
- --consul=vconsul:8500
- - --fluentd=fluentd:24224
- --kafka=kafka
- --rest-port=8880
- --grpc-port=50556
@@ -162,7 +161,6 @@
- /ofagent/ofagent/main.py
- -v
- --consul=vconsul:8500
- - --fluentd=fluentd:24224
- --controller=onos:6653
- --grpc-endpoint=vcore:50556
- --instance-id-is-container-name
@@ -242,7 +240,6 @@
- /netconf/netconf/main.py
- -v
- --consul=vconsul:8500
- - --fluentd=fluentd:24224
- --grpc-endpoint=voltha:50555
- --instance-id-is-container-name
networks:
diff --git a/compose/docker-compose-docutests.yml b/compose/docker-compose-docutests.yml
index ba6e94b..aa94006 100644
--- a/compose/docker-compose-docutests.yml
+++ b/compose/docker-compose-docutests.yml
@@ -76,11 +76,15 @@
#
voltha:
image: "${REGISTRY}${REPOSITORY}voltha-voltha:latest"
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "10m"
+ max-file: "3"
command: [
"/voltha/voltha/main.py",
"-v",
"--consul=${DOCKER_HOST_IP}:8500",
- "--fluentd=fluentd:24224",
"--rest-port=8880",
"--grpc-port=50555",
"--kafka=@kafka",
@@ -97,7 +101,6 @@
- vconsul
links:
- vconsul
- - fluentd
environment:
SERVICE_8880_NAME: "voltha-health"
SERVICE_8880_CHECK_HTTP: "/health"
@@ -116,11 +119,15 @@
#
ofagent:
image: "${REGISTRY}${REPOSITORY}voltha-ofagent:latest"
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "10m"
+ max-file: "3"
command: [
"/ofagent/ofagent/main.py",
"-v",
"--consul=${DOCKER_HOST_IP}:8500",
- "--fluentd=fluentd:24224",
"--controller=${DOCKER_HOST_IP}:6653",
"--grpc-endpoint=@voltha-grpc",
"--instance-id-is-container-name",
@@ -131,7 +138,6 @@
- voltha
links:
- vconsul
- - fluentd
volumes:
- "/var/run/docker.sock:/tmp/docker.sock"
restart: unless-stopped
@@ -141,12 +147,16 @@
#
netconf:
image: "${REGISTRY}${REPOSITORY}voltha-netconf:latest"
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "10m"
+ max-file: "3"
privileged: true
command: [
"/netconf/netconf/main.py",
"-v",
"--consul=${DOCKER_HOST_IP}:8500",
- "--fluentd=fluentd:24224",
"--grpc-endpoint=@voltha-grpc",
"--instance-id-is-container-name",
"-v"
@@ -158,7 +168,6 @@
- voltha
links:
- vconsul
- - fluentd
environment:
SERVICE_1830_NAME: "netconf-server"
volumes:
diff --git a/compose/docker-compose-fixed-port.yml b/compose/docker-compose-fixed-port.yml
index c810940..1f9aeb4 100644
--- a/compose/docker-compose-fixed-port.yml
+++ b/compose/docker-compose-fixed-port.yml
@@ -116,7 +116,6 @@
"/voltha/voltha/main.py",
"-v",
"--consul=${DOCKER_HOST_IP}:8500",
- "--fluentd=fluentd:24224",
"--rest-port=8880",
"--grpc-port=50555",
"--kafka=@kafka",
@@ -132,7 +131,6 @@
- vconsul
links:
- vconsul
- - fluentd
environment:
SERVICE_8880_NAME: "voltha-health"
SERVICE_8880_CHECK_HTTP: "/health"
@@ -159,7 +157,6 @@
"/ofagent/ofagent/main.py",
"-v",
"--consul=${DOCKER_HOST_IP}:8500",
- "--fluentd=fluentd:24224",
"--controller=${DOCKER_HOST_IP}:6653",
"--grpc-endpoint=@voltha-grpc",
"--instance-id-is-container-name",
@@ -170,7 +167,6 @@
- voltha
links:
- vconsul
- - fluentd
volumes:
- "/var/run/docker.sock:/tmp/docker.sock"
restart: unless-stopped
@@ -185,7 +181,6 @@
"/netconf/netconf/main.py",
"-v",
"--consul=${DOCKER_HOST_IP}:8500",
- "--fluentd=fluentd:24224",
"--grpc-endpoint=@voltha-grpc",
"--instance-id-is-container-name",
"-v"
@@ -197,7 +192,6 @@
- voltha
links:
- vconsul
- - fluentd
environment:
SERVICE_1830_NAME: "netconf-server"
volumes:
diff --git a/compose/docker-compose-netconf-swarm.yml b/compose/docker-compose-netconf-swarm.yml
index 091ea79..7dcbebf 100644
--- a/compose/docker-compose-netconf-swarm.yml
+++ b/compose/docker-compose-netconf-swarm.yml
@@ -27,7 +27,6 @@
- /netconf/netconf/main.py
- -v
- --consul=vconsul:8500
- - --fluentd=fluentd:24224
- --grpc-endpoint=voltha:50555
- --instance-id-is-container-name
networks:
diff --git a/compose/docker-compose-ofagent-swarm.yml b/compose/docker-compose-ofagent-swarm.yml
index 8fc5ea2..8e766f5 100644
--- a/compose/docker-compose-ofagent-swarm.yml
+++ b/compose/docker-compose-ofagent-swarm.yml
@@ -21,7 +21,6 @@
- /ofagent/ofagent/main.py
- -v
- --consul=vconsul:8500
- - --fluentd=fluentd:24224
- --controller=onos:6653
- --grpc-endpoint=vcore:50556
- --instance-id-is-container-name
diff --git a/compose/docker-compose-ofagent-test.yml b/compose/docker-compose-ofagent-test.yml
index 516196f..2f801c2 100644
--- a/compose/docker-compose-ofagent-test.yml
+++ b/compose/docker-compose-ofagent-test.yml
@@ -55,7 +55,6 @@
"/voltha/voltha/main.py",
"-v",
"--consul=${DOCKER_HOST_IP}:8500",
- "--fluentd=fluentd:24224",
"--rest-port=8880",
"--grpc-port=50556",
"--kafka=@kafka",
@@ -73,7 +72,6 @@
- vconsul
links:
- vconsul
- - fluentd
environment:
SERVICE_8880_NAME: "voltha-health"
SERVICE_8880_CHECK_HTTP: "/health"
@@ -182,13 +180,12 @@
#
ofagent:
image: "${REGISTRY}${REPOSITORY}voltha-ofagent:${TAG:-latest}"
- command: /ofagent/ofagent/main.py -v --consul=${DOCKER_HOST_IP}:8500 --fluentd=fluentd:24224 --controller ${DOCKER_HOST_IP}:6633 ${DOCKER_HOST_IP}:6644 ${DOCKER_HOST_IP}:6655 --grpc-endpoint=@voltha-grpc --instance-id-is-container-name --enable-tls --key-file=/ofagent/pki/voltha.key --cert-file=/ofagent/pki/voltha.crt
+ command: /ofagent/ofagent/main.py -v --consul=${DOCKER_HOST_IP}:8500 --controller ${DOCKER_HOST_IP}:6633 ${DOCKER_HOST_IP}:6644 ${DOCKER_HOST_IP}:6655 --grpc-endpoint=@voltha-grpc --instance-id-is-container-name --enable-tls --key-file=/ofagent/pki/voltha.key --cert-file=/ofagent/pki/voltha.crt
depends_on:
- vconsul
- voltha
links:
- vconsul
- - fluentd
volumes:
- "/var/run/docker.sock:/tmp/docker.sock"
restart: unless-stopped
diff --git a/compose/docker-compose-system-test-dispatcher.yml b/compose/docker-compose-system-test-dispatcher.yml
index 36ae7d6..530c772 100644
--- a/compose/docker-compose-system-test-dispatcher.yml
+++ b/compose/docker-compose-system-test-dispatcher.yml
@@ -114,11 +114,15 @@
#
voltha:
image: "${REGISTRY}${REPOSITORY}voltha-voltha${TAG}"
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "10m"
+ max-file: "3"
command: [
"/voltha/voltha/main.py",
"-v",
"--consul=${DOCKER_HOST_IP}:8500",
- "--fluentd=fluentd:24224",
"--rest-port=8880",
"--grpc-port=50556",
"--kafka=@kafka",
@@ -135,7 +139,6 @@
- vconsul
links:
- vconsul
- - fluentd
environment:
SERVICE_8880_NAME: "voltha-health"
SERVICE_8880_CHECK_HTTP: "/health"
@@ -202,11 +205,15 @@
#
ofagent:
image: "${REGISTRY}${REPOSITORY}voltha-ofagent${TAG}"
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "10m"
+ max-file: "3"
command: [
"/ofagent/ofagent/main.py",
"-v",
"--consul=${DOCKER_HOST_IP}:8500",
- "--fluentd=fluentd:24224",
"--controller=${DOCKER_HOST_IP}:6653",
"--grpc-endpoint=@voltha-grpc",
"--instance-id-is-container-name",
@@ -217,7 +224,6 @@
- voltha
links:
- vconsul
- - fluentd
volumes:
- "/var/run/docker.sock:/tmp/docker.sock"
restart: unless-stopped
@@ -227,12 +233,16 @@
#
netconf:
image: "${REGISTRY}${REPOSITORY}voltha-netconf${TAG}"
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "10m"
+ max-file: "3"
privileged: true
command: [
"/netconf/netconf/main.py",
"-v",
"--consul=${DOCKER_HOST_IP}:8500",
- "--fluentd=fluentd:24224",
"--grpc-endpoint=@voltha-grpc",
"--instance-id-is-container-name",
"-v"
@@ -244,7 +254,6 @@
- voltha
links:
- vconsul
- - fluentd
environment:
SERVICE_1830_NAME: "netconf-server"
volumes:
diff --git a/compose/docker-compose-system-test-encrypted.yml b/compose/docker-compose-system-test-encrypted.yml
index 9c0f9d1..d5c466e 100644
--- a/compose/docker-compose-system-test-encrypted.yml
+++ b/compose/docker-compose-system-test-encrypted.yml
@@ -117,11 +117,15 @@
#
voltha:
image: "${REGISTRY}${REPOSITORY}voltha-voltha${TAG}"
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "10m"
+ max-file: "3"
command: [
"/voltha/voltha/main.py",
"-v",
"--consul=${DOCKER_HOST_IP}:8500",
- "--fluentd=fluentd:24224",
"--rest-port=8880",
"--grpc-port=50555",
"--kafka=@kafka",
@@ -137,7 +141,6 @@
- vconsul
links:
- vconsul
- - fluentd
environment:
SERVICE_8880_NAME: "voltha-health"
SERVICE_8880_CHECK_HTTP: "/health"
@@ -176,11 +179,15 @@
#
ofagent:
image: "${REGISTRY}${REPOSITORY}voltha-ofagent${TAG}"
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "10m"
+ max-file: "3"
command: [
"/ofagent/ofagent/main.py",
"-v",
"--consul=${DOCKER_HOST_IP}:8500",
- "--fluentd=fluentd:24224",
"--controller=${DOCKER_HOST_IP}:6653",
"--grpc-endpoint=@voltha-grpc",
"--instance-id-is-container-name",
@@ -191,7 +198,6 @@
- voltha
links:
- vconsul
- - fluentd
volumes:
- "/var/run/docker.sock:/tmp/docker.sock"
restart: unless-stopped
@@ -201,12 +207,16 @@
#
netconf:
image: "${REGISTRY}${REPOSITORY}voltha-netconf${TAG}"
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "10m"
+ max-file: "3"
privileged: true
command: [
"/netconf/netconf/main.py",
"-v",
"--consul=${DOCKER_HOST_IP}:8500",
- "--fluentd=fluentd:24224",
"--grpc-endpoint=@voltha-grpc",
"--instance-id-is-container-name",
"-v"
@@ -218,7 +228,6 @@
- voltha
links:
- vconsul
- - fluentd
environment:
SERVICE_1830_NAME: "netconf-server"
volumes:
diff --git a/compose/docker-compose-system-test-persistence.yml b/compose/docker-compose-system-test-persistence.yml
index 99f42eb..57d0006 100644
--- a/compose/docker-compose-system-test-persistence.yml
+++ b/compose/docker-compose-system-test-persistence.yml
@@ -114,11 +114,15 @@
#
voltha:
image: "${REGISTRY}${REPOSITORY}voltha-voltha${TAG}"
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "10m"
+ max-file: "3"
command: [
"/voltha/voltha/main.py",
"-v",
"--consul=${DOCKER_HOST_IP}:8500",
- "--fluentd=fluentd:24224",
"--rest-port=8880",
"--grpc-port=50556",
"--kafka=@kafka",
@@ -135,7 +139,6 @@
- vconsul
links:
- vconsul
- - fluentd
environment:
SERVICE_8880_NAME: "voltha-health"
SERVICE_8880_CHECK_HTTP: "/health"
@@ -202,11 +205,15 @@
#
ofagent:
image: "${REGISTRY}${REPOSITORY}voltha-ofagent${TAG}"
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "10m"
+ max-file: "3"
command: [
"/ofagent/ofagent/main.py",
"-v",
"--consul=${DOCKER_HOST_IP}:8500",
- "--fluentd=fluentd:24224",
"--controller=${DOCKER_HOST_IP}:6653",
"--grpc-endpoint=@voltha-grpc",
"--instance-id-is-container-name",
@@ -217,7 +224,6 @@
- voltha
links:
- vconsul
- - fluentd
volumes:
- "/var/run/docker.sock:/tmp/docker.sock"
restart: unless-stopped
@@ -227,12 +233,16 @@
#
netconf:
image: "${REGISTRY}${REPOSITORY}voltha-netconf${TAG}"
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "10m"
+ max-file: "3"
privileged: true
command: [
"/netconf/netconf/main.py",
"-v",
"--consul=${DOCKER_HOST_IP}:8500",
- "--fluentd=fluentd:24224",
"--grpc-endpoint=@voltha-grpc",
"--instance-id-is-container-name",
"-v"
@@ -244,7 +254,6 @@
- voltha
links:
- vconsul
- - fluentd
environment:
SERVICE_1830_NAME: "netconf-server"
volumes:
diff --git a/compose/docker-compose-system-test-with-etcd.yml b/compose/docker-compose-system-test-with-etcd.yml
index 8dd257f..37f8237 100644
--- a/compose/docker-compose-system-test-with-etcd.yml
+++ b/compose/docker-compose-system-test-with-etcd.yml
@@ -134,12 +134,16 @@
#
voltha:
image: "${REGISTRY}${REPOSITORY}voltha-voltha${TAG}"
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "10m"
+ max-file: "3"
command: [
"/voltha/voltha/main.py",
"-v",
"--consul=${DOCKER_HOST_IP}:8500",
"--etcd=${DOCKER_HOST_IP}:2379",
- "--fluentd=fluentd:24224",
"--rest-port=8880",
"--grpc-port=50556",
"--kafka=@kafka",
@@ -159,7 +163,6 @@
links:
- vconsul
- etcd
- - fluentd
environment:
SERVICE_8880_NAME: "voltha-health"
SERVICE_8880_CHECK_HTTP: "/health"
@@ -229,11 +232,15 @@
#
ofagent:
image: "${REGISTRY}${REPOSITORY}voltha-ofagent${TAG}"
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "10m"
+ max-file: "3"
command: [
"/ofagent/ofagent/main.py",
"-v",
"--consul=${DOCKER_HOST_IP}:8500",
- "--fluentd=fluentd:24224",
"--controller=${DOCKER_HOST_IP}:6653",
"--grpc-endpoint=@voltha-grpc",
"--instance-id-is-container-name",
@@ -247,7 +254,6 @@
- voltha
links:
- vconsul
- - fluentd
volumes:
- "/var/run/docker.sock:/tmp/docker.sock"
restart: unless-stopped
@@ -257,12 +263,16 @@
#
netconf:
image: "${REGISTRY}${REPOSITORY}voltha-netconf${TAG}"
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "10m"
+ max-file: "3"
privileged: true
command: [
"/netconf/netconf/main.py",
"-v",
"--consul=${DOCKER_HOST_IP}:8500",
- "--fluentd=fluentd:24224",
"--grpc-endpoint=@voltha-grpc",
"--instance-id-is-container-name",
"-v"
@@ -274,7 +284,6 @@
- voltha
links:
- vconsul
- - fluentd
environment:
SERVICE_1830_NAME: "netconf-server"
volumes:
diff --git a/compose/docker-compose-system-test.yml b/compose/docker-compose-system-test.yml
index 52e3cf7..0ca70fd 100644
--- a/compose/docker-compose-system-test.yml
+++ b/compose/docker-compose-system-test.yml
@@ -114,11 +114,19 @@
#
voltha:
image: "${REGISTRY}${REPOSITORY}voltha-voltha${TAG}"
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "10m"
+ max-file: "3"
+# Use the fluentd driver to push logs to fluentd instead
+# driver: "fluentd"
+# options:
+# fluentd-address: ${DOCKER_HOST_IP}:24224
command: [
"/voltha/voltha/main.py",
"-v",
"--consul=${DOCKER_HOST_IP}:8500",
- "--fluentd=fluentd:24224",
"--rest-port=8880",
"--grpc-port=50556",
"--kafka=@kafka",
@@ -136,7 +144,6 @@
- vconsul
links:
- vconsul
- - fluentd
environment:
SERVICE_8880_NAME: "voltha-health"
SERVICE_8880_CHECK_HTTP: "/health"
@@ -203,11 +210,19 @@
#
ofagent:
image: "${REGISTRY}${REPOSITORY}voltha-ofagent${TAG}"
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "10m"
+ max-file: "3"
+# Use the fluentd driver to push logs to fluentd instead
+# driver: "fluentd"
+# options:
+# fluentd-address: ${DOCKER_HOST_IP}:24224
command: [
"/ofagent/ofagent/main.py",
"-v",
"--consul=${DOCKER_HOST_IP}:8500",
- "--fluentd=fluentd:24224",
"--controller=${DOCKER_HOST_IP}:6653",
"--grpc-endpoint=@voltha-grpc",
"--instance-id-is-container-name",
@@ -221,7 +236,6 @@
- voltha
links:
- vconsul
- - fluentd
volumes:
- "/var/run/docker.sock:/tmp/docker.sock"
restart: unless-stopped
@@ -231,12 +245,20 @@
#
netconf:
image: "${REGISTRY}${REPOSITORY}voltha-netconf${TAG}"
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "10m"
+ max-file: "3"
+# Use the fluentd driver to push logs to fluentd instead
+# driver: "fluentd"
+# options:
+# fluentd-address: ${DOCKER_HOST_IP}:24224
privileged: true
command: [
"/netconf/netconf/main.py",
"-v",
"--consul=${DOCKER_HOST_IP}:8500",
- "--fluentd=fluentd:24224",
"--grpc-endpoint=@voltha-grpc",
"--instance-id-is-container-name",
"-v"
@@ -248,7 +270,6 @@
- voltha
links:
- vconsul
- - fluentd
environment:
SERVICE_1830_NAME: "netconf-server"
volumes:
diff --git a/compose/docker-compose-voltha-swarm.yml b/compose/docker-compose-voltha-swarm.yml
index 00c5b73..3e30236 100644
--- a/compose/docker-compose-voltha-swarm.yml
+++ b/compose/docker-compose-voltha-swarm.yml
@@ -13,7 +13,6 @@
- voltha/voltha/main.py
- -v
- --consul=vconsul:8500
- - --fluentd=fluentd:24224
- --kafka=kafka
- --rest-port=8880
- --grpc-port=50556
diff --git a/compose/voltha-stack.yml.j2 b/compose/voltha-stack.yml.j2
index 728ad5a..8afa37d 100644
--- a/compose/voltha-stack.yml.j2
+++ b/compose/voltha-stack.yml.j2
@@ -339,7 +339,6 @@
- voltha/voltha/main.py
- -v
- --consul=vconsul:8500
- - --fluentd=fluentd:24224
- --kafka=kafka
- --rest-port=8880
- --grpc-port=50556
@@ -371,7 +370,6 @@
- /ofagent/ofagent/main.py
- -v
- --consul=vconsul:8500
- - --fluentd=fluentd:24224
- --controller=onos:6653
- --grpc-endpoint=vcore:50556
- --instance-id-is-container-name
@@ -458,7 +456,6 @@
- /netconf/netconf/main.py
- -v
- --consul=vconsul:8500
- - --fluentd=fluentd:24224
- --grpc-endpoint=voltha:50555
- --instance-id-is-container-name
networks:
diff --git a/install/BuildVoltha.sh b/install/BuildVoltha.sh
index a3bec02..4d589c9 100755
--- a/install/BuildVoltha.sh
+++ b/install/BuildVoltha.sh
@@ -36,10 +36,14 @@
echo 'testMode: "false"' >> settings.vagrant.yaml
fi
-# Special actions that differentiate a cluster build from a singel instance build
-cp voltha/voltha.production.yml voltha/voltha.yml
-cp ofagent/ofagent.production.yml ofagent/ofagent.yml
-cp netconf/netconf.production.yml netconf/netconf.yml
+# This was required before as logging was different in production vs development. The
+# logging decision was made at compile time.
+# By using the docker logging option (docker swarm mode only) in the deployment
+# files, now the logging decision is made at deployment time, hence the same voltha,
+# netconf and ofagent images can be used both in development and production.
+#cp voltha/voltha.production.yml voltha/voltha.yml
+#cp ofagent/ofagent.production.yml ofagent/ofagent.yml
+#cp netconf/netconf.production.yml netconf/netconf.yml
# Destroy the VM if it's running
vagrant destroy voltha${uId}
diff --git a/netconf/main.py b/netconf/main.py
index c0f95fc..6f096a8 100755
--- a/netconf/main.py
+++ b/netconf/main.py
@@ -46,7 +46,6 @@
client_passwords_file=os.environ.get('CLIENT_PASSWORD_FILE',
'client_passwords'),
grpc_endpoint=os.environ.get('GRPC_ENDPOINT', 'localhost:50055'),
- fluentd=os.environ.get('FLUENTD', None),
instance_id=os.environ.get('INSTANCE_ID', os.environ.get('HOSTNAME', '1')),
internal_host_address=os.environ.get('INTERNAL_HOST_ADDRESS',
get_my_primary_local_ipv4()),
@@ -130,15 +129,6 @@
default=defs['client_passwords_file'],
help=_help)
- _help = ('<hostname>:<port> to fluentd server (default: %s). (If not '
- 'specified (None), the address from the config file is used'
- % defs['fluentd'])
- parser.add_argument('-F', '--fluentd',
- dest='fluentd',
- action='store',
- default=defs['fluentd'],
- help=_help)
-
_help = ('gRPC end-point to connect to. It can either be a direct'
'definition in the form of <hostname>:<port>, or it can be an'
'indirect definition in the form of @<service-name> where'
@@ -248,8 +238,7 @@
verbosity_adjust = (args.verbose or 0) - (args.quiet or 0)
self.log = setup_logging(self.config.get('logging', {}),
args.instance_id,
- verbosity_adjust=verbosity_adjust,
- fluentd=args.fluentd)
+ verbosity_adjust=verbosity_adjust)
# components
self.nc_server = None
diff --git a/netconf/netconf.production.yml b/netconf/netconf.production.yml
deleted file mode 100644
index 32fc912..0000000
--- a/netconf/netconf.production.yml
+++ /dev/null
@@ -1,45 +0,0 @@
-logging:
- version: 1
-
- formatters:
- brief:
- format: '%(message)s'
- default:
- format: '%(asctime)s.%(msecs)03d %(levelname)-8s %(module)s.%(funcName)s %(message)s'
- datefmt: '%Y%m%dT%H%M%S'
- fluent_fmt:
- '()': fluent.handler.FluentRecordFormatter
- format:
- level: '%(levelname)s'
- hostname: '%(hostname)s'
- where: '%(module)s.%(funcName)s'
- fluent_default:
- format: '%(asctime)s.%(msecs)03d %(levelname)-8s %(threadName)s %(module)s.%(funcName)s %(message)s'
- datefmt: '%Y%m%dT%H%M%S'
-
- handlers:
- fluent:
- class: fluent.handler.FluentHandler
- host: localhost
- port: 24224
- tag: netconf.logging
- # Since fluent formatter output an exception with grpc data, use
- # a basic formatter for now.
- # formatter: fluent_fmt
- formatter: fluent_default
- level: INFO
- null:
- class: logging.NullHandler
-
- loggers:
- amqp:
- handlers: [null]
- propagate: False
- conf:
- handlers: [null]
- propagate: False
- '': # root logger
- handlers: [fluent]
- level: INFO # this can be bumped up/down by -q and -v command line
- # options
- propagate: False
diff --git a/netconf/netconf.yml b/netconf/netconf.yml
index d0867b6..defc286 100644
--- a/netconf/netconf.yml
+++ b/netconf/netconf.yml
@@ -7,15 +7,6 @@
default:
format: '%(asctime)s.%(msecs)03d %(levelname)-8s %(module)s.%(funcName)s %(message)s'
datefmt: '%Y%m%dT%H%M%S'
- fluent_fmt:
- '()': fluent.handler.FluentRecordFormatter
- format:
- level: '%(levelname)s'
- hostname: '%(hostname)s'
- where: '%(module)s.%(funcName)s'
- fluent_default:
- format: '%(asctime)s.%(msecs)03d %(levelname)-8s %(threadName)s %(module)s.%(funcName)s %(message)s'
- datefmt: '%Y%m%dT%H%M%S'
handlers:
console:
@@ -23,16 +14,6 @@
level: DEBUG
formatter: default
stream: ext://sys.stdout
- fluent:
- class: fluent.handler.FluentHandler
- host: localhost
- port: 24224
- tag: netconf.logging
- # Since fluent formatter output an exception with grpc data, use
- # a basic formatter for now.
- # formatter: fluent_fmt
- formatter: fluent_default
- level: DEBUG
localRotatingFile:
class: logging.handlers.RotatingFileHandler
filename: netconf.log
@@ -48,10 +29,9 @@
handlers: [null]
propagate: False
conf:
- handlers: [null]
propagate: False
'': # root logger
- handlers: [console, fluent, localRotatingFile]
+ handlers: [console, localRotatingFile]
level: INFO # this can be bumped up/down by -q and -v command line
# options
propagate: False
diff --git a/ofagent/main.py b/ofagent/main.py
index 5887b75..23335bb 100755
--- a/ofagent/main.py
+++ b/ofagent/main.py
@@ -33,7 +33,6 @@
external_host_address=os.environ.get('EXTERNAL_HOST_ADDRESS',
get_my_primary_local_ipv4()),
grpc_endpoint=os.environ.get('GRPC_ENDPOINT', 'localhost:50055'),
- fluentd=os.environ.get('FLUENTD', None),
instance_id=os.environ.get('INSTANCE_ID', os.environ.get('HOSTNAME', '1')),
internal_host_address=os.environ.get('INTERNAL_HOST_ADDRESS',
get_my_primary_local_ipv4()),
@@ -77,15 +76,6 @@
default=defs['external_host_address'],
help=_help)
- _help = ('<hostname>:<port> to fluentd server (default: %s). (If not '
- 'specified (None), the address from the config file is used'
- % defs['fluentd'])
- parser.add_argument('-F', '--fluentd',
- dest='fluentd',
- action='store',
- default=defs['fluentd'],
- help=_help)
-
_help = ('gRPC end-point to connect to. It can either be a direct'
'definition in the form of <hostname>:<port>, or it can be an'
'indirect definition in the form of @<service-name> where'
@@ -217,8 +207,7 @@
verbosity_adjust = (args.verbose or 0) - (args.quiet or 0)
self.log = setup_logging(self.config.get('logging', {}),
args.instance_id,
- verbosity_adjust=verbosity_adjust,
- fluentd=args.fluentd)
+ verbosity_adjust=verbosity_adjust)
# components
self.connection_manager = None
diff --git a/ofagent/ofagent.production.yml b/ofagent/ofagent.production.yml
deleted file mode 100644
index e812cf4..0000000
--- a/ofagent/ofagent.production.yml
+++ /dev/null
@@ -1,45 +0,0 @@
-logging:
- version: 1
-
- formatters:
- brief:
- format: '%(message)s'
- default:
- format: '%(asctime)s.%(msecs)03d %(levelname)-8s %(module)s.%(funcName)s %(message)s'
- datefmt: '%Y%m%dT%H%M%S'
- fluent_fmt:
- '()': fluent.handler.FluentRecordFormatter
- format:
- level: '%(levelname)s'
- hostname: '%(hostname)s'
- where: '%(module)s.%(funcName)s'
- fluent_default:
- format: '%(asctime)s.%(msecs)03d %(levelname)-8s %(threadName)s %(module)s.%(funcName)s %(message)s'
- datefmt: '%Y%m%dT%H%M%S'
-
- handlers:
- fluent:
- class: fluent.handler.FluentHandler
- host: localhost
- port: 24224
- tag: ofagent.logging
- # Since fluent formatter output an exception with grpc data, use
- # a basic formatter for now.
- # formatter: fluent_fmt
- formatter: fluent_default
- level: INFO
- null:
- class: logging.NullHandler
-
- loggers:
- amqp:
- handlers: [null]
- propagate: False
- conf:
- handlers: [null]
- propagate: False
- '': # root logger
- handlers: [fluent]
- level: INFO # this can be bumped up/down by -q and -v command line
- # options
- propagate: False
diff --git a/ofagent/ofagent.yml b/ofagent/ofagent.yml
index 25b06a4..92c7787 100644
--- a/ofagent/ofagent.yml
+++ b/ofagent/ofagent.yml
@@ -7,15 +7,6 @@
default:
format: '%(asctime)s.%(msecs)03d %(levelname)-8s %(module)s.%(funcName)s %(message)s'
datefmt: '%Y%m%dT%H%M%S'
- fluent_fmt:
- '()': fluent.handler.FluentRecordFormatter
- format:
- level: '%(levelname)s'
- hostname: '%(hostname)s'
- where: '%(module)s.%(funcName)s'
- fluent_default:
- format: '%(asctime)s.%(msecs)03d %(levelname)-8s %(threadName)s %(module)s.%(funcName)s %(message)s'
- datefmt: '%Y%m%dT%H%M%S'
handlers:
console:
@@ -23,16 +14,6 @@
level: DEBUG
formatter: default
stream: ext://sys.stdout
- fluent:
- class: fluent.handler.FluentHandler
- host: localhost
- port: 24224
- tag: ofagent.logging
- # Since fluent formatter output an exception with grpc data, use
- # a basic formatter for now.
- # formatter: fluent_fmt
- formatter: fluent_default
- level: DEBUG
localRotatingFile:
class: logging.handlers.RotatingFileHandler
filename: ofagent.log
@@ -48,10 +29,9 @@
handlers: [null]
propagate: False
conf:
- handlers: [null]
propagate: False
'': # root logger
- handlers: [console, fluent, localRotatingFile]
+ handlers: [console, localRotatingFile]
level: INFO # this can be bumped up/down by -q and -v command line
# options
propagate: False
diff --git a/voltha/main.py b/voltha/main.py
index 5001aa1..29585b4 100755
--- a/voltha/main.py
+++ b/voltha/main.py
@@ -58,7 +58,6 @@
pon_subnet=os.environ.get('PON_SUBNET', None),
external_host_address=os.environ.get('EXTERNAL_HOST_ADDRESS',
get_my_primary_local_ipv4()),
- fluentd=os.environ.get('FLUENTD', None),
grpc_port=os.environ.get('GRPC_PORT', 50055),
instance_id=os.environ.get('INSTANCE_ID', os.environ.get('HOSTNAME', '1')),
internal_host_address=os.environ.get('INTERNAL_HOST_ADDRESS',
@@ -128,14 +127,6 @@
default=defs['grpc_port'],
help=_help)
- _help = ('<hostname>:<port> to fluentd server (default: %s). (If not '
- 'specified (None), the address from the config file is used'
- % defs['fluentd'])
- parser.add_argument('-F', '--fluentd',
- dest='fluentd',
- action='store',
- default=defs['fluentd'],
- help=_help)
_help = ('<hostname> or <ip> at which Voltha is reachable from inside the'
'cluster (default: %s)' % defs['internal_host_address'])
@@ -289,8 +280,7 @@
verbosity_adjust = (args.verbose or 0) - (args.quiet or 0)
self.log = setup_logging(self.config.get('logging', {}),
args.instance_id,
- verbosity_adjust=verbosity_adjust,
- fluentd=args.fluentd)
+ verbosity_adjust=verbosity_adjust)
# configurable variables from voltha.yml file
#self.configurable_vars = self.config.get('Constants', {})
diff --git a/voltha/voltha.jenkins.yml b/voltha/voltha.jenkins.yml
deleted file mode 100644
index 98c80e3..0000000
--- a/voltha/voltha.jenkins.yml
+++ /dev/null
@@ -1,90 +0,0 @@
-logging:
- version: 1
-
- formatters:
- brief:
- format: '%(message)s'
- default:
- format: '%(asctime)s.%(msecs)03d %(levelname)-8s %(threadName)s %(module)s.%(funcName)s %(message)s'
- datefmt: '%Y%m%dT%H%M%S'
- fluent_fmt:
- '()': fluent.handler.FluentRecordFormatter
- format:
- level: '%(levelname)s'
- hostname: '%(hostname)s'
- where: '%(module)s.%(funcName)s'
-
- handlers:
- console:
- class : logging.StreamHandler
- level: DEBUG
- formatter: default
- stream: ext://sys.stdout
- fluent:
- class: fluent.handler.FluentHandler
- host: localhost
- port: 24224
- tag: voltha.logging
- formatter: fluent_fmt
- level: DEBUG
- localRotatingFile:
- class: logging.handlers.RotatingFileHandler
- filename: voltha.log
- formatter: default
- maxBytes: 2097152
- backupCount: 10
- level: DEBUG
- null:
- class: logging.NullHandler
-
- loggers:
- amqp:
- handlers: [null]
- propagate: False
- conf:
- handlers: [null]
- propagate: False
- '': # root logger
- handlers: [console, fluent, localRotatingFile]
- level: INFO # this can be bumped up/down by -q and -v command line
- # options
- propagate: False
-
-core:
- management_vlan: 4091
-
-coordinator:
- voltha_kv_prefix: 'service/voltha'
- core_store_key: 'data/core'
- leader_key: 'leader'
- membership_key: 'members'
- assignment_key: 'assignments'
- workload_key: 'work'
- membership_watch_relatch_delay: 0.1
- membership_maintenance_loop_delay: 5
- tracking_loop_delay: 1
- session_time_to_live: 60
- session_renewal_loop_delay: 10
- session_renewal_timeout: 10
-
-worker:
- time_to_let_leader_update: 5
- assignments_track_error_to_avoid_flood: 1
-
-leader:
- workload_track_error_to_prevent_flood: 1
- members_track_error_to_prevent_flood: 1
-
-kafka-proxy:
- event_bus_publisher:
- topic_mappings:
- 'model-change-events':
- kafka_topic: 'voltha.events'
- filters: [null]
- 'alarms':
- kafka_topic: 'voltha.alarms'
- filters: [null]
- 'kpis':
- kafka_topic: 'voltha.kpis'
- filters: [null]
-
diff --git a/voltha/voltha.production.yml b/voltha/voltha.production.yml
deleted file mode 100644
index 3df271f..0000000
--- a/voltha/voltha.production.yml
+++ /dev/null
@@ -1,84 +0,0 @@
-logging:
- version: 1
-
- formatters:
- brief:
- format: '%(message)s'
- default:
- format: '%(asctime)s.%(msecs)03d %(levelname)-8s %(threadName)s %(module)s.%(funcName)s %(message)s'
- datefmt: '%Y%m%dT%H%M%S'
- fluent_fmt:
- '()': fluent.handler.FluentRecordFormatter
- format:
- level: '%(levelname)s'
- hostname: '%(hostname)s'
- where: '%(module)s.%(funcName)s'
- fluent_default:
- format: '%(asctime)s.%(msecs)03d %(levelname)-8s %(threadName)s %(module)s.%(funcName)s %(message)s'
- datefmt: '%Y%m%dT%H%M%S'
-
- handlers:
- fluent:
- class: fluent.handler.FluentHandler
- host: localhost
- port: 24224
- tag: voltha.logging
- # Since fluent formatter output an exception with grpc data, use
- # a basic formatter for now.
- # formatter: fluent_fmt
- formatter: fluent_default
- level: INFO
- null:
- class: logging.NullHandler
-
- loggers:
- amqp:
- handlers: [null]
- propagate: False
- conf:
- handlers: [null]
- propagate: False
- '': # root logger
- handlers: [fluent]
- level: INFO # this can be bumped up/down by -q and -v command line
- # options
- propagate: False
-
-core:
- management_vlan: 4091
-
-coordinator:
- voltha_kv_prefix: 'service/voltha'
- core_store_key: 'data/core'
- leader_key: 'leader'
- membership_key: 'members'
- assignment_key: 'assignments'
- workload_key: 'work'
- membership_watch_relatch_delay: 0.1
- membership_maintenance_loop_delay: 5
- tracking_loop_delay: 1
- session_time_to_live: 60
- session_renewal_loop_delay: 10
- session_renewal_timeout: 10
-
-worker:
- time_to_let_leader_update: 5
- assignments_track_error_to_avoid_flood: 1
-
-leader:
- workload_track_error_to_prevent_flood: 1
- members_track_error_to_prevent_flood: 1
-
-kafka-proxy:
- event_bus_publisher:
- topic_mappings:
- 'model-change-events':
- kafka_topic: 'voltha.events'
- filters: [null]
- 'alarms':
- kafka_topic: 'voltha.alarms'
- filters: [null]
- 'kpis':
- kafka_topic: 'voltha.kpis'
- filters: [null]
-
diff --git a/voltha/voltha.yml b/voltha/voltha.yml
index ede51d3..330fe52 100644
--- a/voltha/voltha.yml
+++ b/voltha/voltha.yml
@@ -7,31 +7,13 @@
default:
format: '%(asctime)s.%(msecs)03d %(levelname)-8s %(threadName)s %(module)s.%(funcName)s %(message)s'
datefmt: '%Y%m%dT%H%M%S'
- fluent_fmt:
- '()': fluent.handler.FluentRecordFormatter
- format:
- level: '%(levelname)s'
- hostname: '%(hostname)s'
- where: '%(module)s.%(funcName)s'
- fluent_default:
- format: '%(asctime)s.%(msecs)03d %(levelname)-8s %(threadName)s %(module)s.%(funcName)s %(message)s'
- datefmt: '%Y%m%dT%H%M%S'
+
handlers:
console:
class : logging.StreamHandler
level: DEBUG
formatter: default
stream: ext://sys.stdout
- fluent:
- class: fluent.handler.FluentHandler
- host: localhost
- port: 24224
- tag: voltha.logging
- # Since fluent formatter output an exception with grpc data, use
- # a basic formatter for now.
- # formatter: fluent_fmt
- formatter: fluent_default
- level: DEBUG
localRotatingFile:
class: logging.handlers.RotatingFileHandler
filename: voltha.log
@@ -47,10 +29,9 @@
handlers: [null]
propagate: False
conf:
- handlers: [null]
propagate: False
'': # root logger
- handlers: [console, fluent, localRotatingFile]
+ handlers: [console, localRotatingFile]
level: INFO # this can be bumped up/down by -q and -v command line
# options
propagate: False