Resolving Merge Conflicts.
Included Log Aggregators and Log Forwarders as per review comments.
Spins two fluentd forwarder and one each fluentd aggregators-active, fluentd aggregator-standby
Commit to enable Fluentd Clustering in Voltha
https://jira.opencord.org/browse/VOL-214
Resolved merge conflicts .
Change-Id: I9d45f751b4297c707284582e8050a168229f2e5a
diff --git a/Makefile b/Makefile
index 91d7711..f7135bf 100644
--- a/Makefile
+++ b/Makefile
@@ -102,6 +102,9 @@
prod-containers: docker-base voltha chameleon ofagent netconf shovel dashd vcli grafana consul registrator golang envoyd envoy
containers: docker-base voltha chameleon ofagent podder netconf shovel onos tester config-push dashd vcli portainer grafana nginx consul registrator tools golang envoyd envoy
+prod-containers: docker-base voltha chameleon ofagent netconf shovel dashd vcli grafana consul registrator envoy registry fluentd
+
+containers: docker-base voltha chameleon ofagent podder netconf shovel onos tester config-push dashd vcli portainer grafana nginx consul registrator tools envoy fluentd
docker-base:
docker build -t cord/voltha-base -f docker/Dockerfile.base .
@@ -127,6 +130,9 @@
tools:
docker build -t voltha/tools -f docker/Dockerfile.tools .
+fluentd:
+ docker build -t cord/fluentd -f docker/Dockerfile.fluentd .
+
envoy:
docker build -t voltha/envoy -f docker/Dockerfile.envoy .
diff --git a/compose/docker-compose-fluentd-cluster.yml b/compose/docker-compose-fluentd-cluster.yml
new file mode 100644
index 0000000..c21f37d
--- /dev/null
+++ b/compose/docker-compose-fluentd-cluster.yml
@@ -0,0 +1,60 @@
+version: "3"
+services:
+
+#
+# Deploying a Fluentd cluster using this file assumes that overlay network 'voltha_net'
+# has already been created. To deploy the fluentd cluster, issue the command:
+#
+# docker stack deploy -c docker-compose-fluentd-cluster.yml fluentd
+#
+# This spins up two Fluentd Forwarders and one each Fluentd Aggregator-Active, Fluentd Aggregator-Standby.
+# The forwards listens for voltha components logs on port 23224 and sends them towards Fluentd Aggregator.
+# Fluentd Aggregator writes them to their host mounted disk
+#
+
+ fluentdactv:
+ image: fluent/fluentd
+ deploy:
+ replicas: 1
+ restart_policy:
+ condition: on-failure
+ networks:
+ - voltha-net
+ ports:
+ - "24224"
+ volumes:
+ - "/tmp/fluentd:/fluentd/log"
+
+ fluentdstby:
+ image: fluent/fluentd
+ deploy:
+ replicas: 1
+ restart_policy:
+ condition: on-failure
+ networks:
+ - voltha-net
+ ports:
+ - "24224"
+ volumes:
+ - "/tmp/fluentd:/fluentd/log"
+
+ fluentd:
+ image: cord/fluentd
+ deploy:
+ mode: replicated
+ replicas: 2
+ restart_policy:
+ condition: on-failure
+ environment:
+ SERVICE_24224_NAME: "fluentd-intake"
+ networks:
+ - voltha-net
+ ports:
+ - "24224:24224"
+
+
+networks:
+ voltha-net:
+ external:
+ name: voltha_net
+
diff --git a/docker/Dockerfile.fluentd b/docker/Dockerfile.fluentd
new file mode 100644
index 0000000..560e005
--- /dev/null
+++ b/docker/Dockerfile.fluentd
@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#DockerFile to Create Fluentd Forwards inside cord-voltha
+FROM fluent/fluentd
+COPY fluentd_config/fluent.conf /fluentd/etc/
diff --git a/fluentd_config/fluent.conf b/fluentd_config/fluent.conf
new file mode 100644
index 0000000..0beab19
--- /dev/null
+++ b/fluentd_config/fluent.conf
@@ -0,0 +1,24 @@
+# Configuration for the Fluentd Forwarders, it collects logs from voltha components and sends to Fluentd Log Aggregrators.
+# TCP input
+<source>
+ @type forward
+ port 24224
+</source>
+<match **>
+ @type forward
+
+ # primary host
+ <server>
+ host fluentdactv
+ port 24224
+ </server>
+ # use secondary host
+ <server>
+ host fluentdstby
+ port 24224
+ standby
+ </server>
+ # use longer flush_interval to reduce CPU usage.
+ # note that this is a trade-off against latency.
+ flush_interval 60s
+</match>
diff --git a/install/ansible/roles/cluster-host/tasks/main.yml b/install/ansible/roles/cluster-host/tasks/main.yml
index e3115ef..09e42e4 100644
--- a/install/ansible/roles/cluster-host/tasks/main.yml
+++ b/install/ansible/roles/cluster-host/tasks/main.yml
@@ -86,6 +86,12 @@
- snapd
tags: [cluster_host]
+- name: A fluentd directory under tmp for voltha logs exists
+ file:
+ path: "/tmp/fluentd"
+ state: directory
+ tags: [cluster_host]
+
- name: Dependent software is installed (this can take about 10 Min, DONT'T PANIC, go for coffee instead)
command: dpkg -R -i "{{ target_voltha_home }}/deb_files"
# ignore_errors: true
diff --git a/install/containers.cfg b/install/containers.cfg
index df0d264..a097e12 100644
--- a/install/containers.cfg
+++ b/install/containers.cfg
@@ -15,3 +15,4 @@
- gliderlabs/registrator:master
- voltha/envoy:latest
- registry:2
+ - cord/fluentd
diff --git a/install/voltha-swarm-start.sh b/install/voltha-swarm-start.sh
index be078e3..cd402c8 100755
--- a/install/voltha-swarm-start.sh
+++ b/install/voltha-swarm-start.sh
@@ -10,5 +10,6 @@
docker stack deploy -c ${voltha_base_dir}/compose/docker-compose-vcli.yml cli
docker stack deploy -c ${voltha_base_dir}/compose/docker-compose-chameleon-swarm.yml chameleon
docker stack deploy -c ${voltha_base_dir}/compose/docker-compose-netconf-swarm.yml netconf
+docker stack deploy -c ${voltha_base_dir}/compose/docker-compose-fluentd-cluster.yml fluentd
docker service create -d --name tools --network voltha_net --network kafka_net --publish "4022:22" voltha/tools
diff --git a/install/voltha-swarm-stop.sh b/install/voltha-swarm-stop.sh
index 6b761af..9ea36e6 100644
--- a/install/voltha-swarm-stop.sh
+++ b/install/voltha-swarm-stop.sh
@@ -8,4 +8,6 @@
docker service rm tools
docker stack rm consul
docker stack rm kafka
+docker stack rm fluentd
+docker service rm tools
docker network rm voltha_net