logConf/td-agent.conf

# Copyright (c) 2020 Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
#     http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# input plugin to collect openolt logs
# formating the input logs using regex and creating a feilds such as level, instanceId, caller and msg.
<source>
  @id openolt.log
  @type tail
  path /var/log/openolt.log
  pos_file /var/log/td-agent/openolt.log.pos
  read_from_head true
  tag openolt
  <parse>
    @type multiline
    format_firstline /\[(\d+)(.\s+)(?<level>\w+)(?<instanceId>[\s]\w+)(\s+)\](?<caller>.\w+...\s\d+)(.\s)/
    format1 /\[(\d+)(.\s+)(?<level>\w+)(?<instanceId>[\s]\w+)(\s+)\](?<caller>.\w+...\s\d+)(.\s)(?<msg>.*)/
  </parse>
</source>

# Formating the `instanceId` field by concating with the device `Ip Address`.
# For example: If the extracted value for the field 'instanceId' is `OPENOLT` and `Ip` of the device is `192.x.x.x` after concating the
# `instanceId` will be  "OPENOLT-192.x.x.x"
# The log levels which are coming in the logs are like 'I', 'E' etc so replacing the log level 'I' to 'info', 'W' to 'warn' and 'E' to 'error'
<filter openolt.**>
  @type record_transformer
  enable_ruby true
  <record>
    instanceId ${record["instanceId"]}-${"#{(Socket.ip_address_list.detect do |intf| intf.ipv4_private? end).ip_address}"}
    level ${if record['level'] == 'I' then 'info' elsif record['level'] == 'W' then 'warn' else 'error'; end}
  </record>
</filter>

# input plugin to collect dev_mgmt_daemon logs
# formating the input logs using regex and creating a feilds such as level, instanceId, caller and msg.
<source>
  @id dev_mgmt_daemon.log
  @type tail
  path /var/log/dev_mgmt_daemon.log
  pos_file /var/log/td-agent/dev_mgmt_daemon.log.pos
  tag dev_mgmt
  read_from_head true
  <parse>
    @type multiline
    format_firstline /\[(\d+)(.\s+)(?<level>\w+)(?<instanceId>[\s]\w+)(\s+)\](?<caller>.\w+...\s\d+)(.\s)/
    format1 /\[(\d+)(.\s+)(?<level>\w+)(?<instanceId>[\s]\w+)(\s+)\](?<caller>.\w+...\s\d+)(.\s)(?<msg>.*)/
  </parse>
</source>

# Formating the `instanceId` field by concating with the device `Ip Address`.
# For example: If the extracted value for the field 'instanceId' is `SW_UTIL` and `Ip` of the device is `192.x.x.x` after concating the
# `instanceId` will be  "SW_UTIL-192.x.x.x"
# The log levels which are coming in the logs are like 'I', 'E' etc so replacing the log level 'I' to 'info', 'W' to 'warn' and 'E' to 'error'
<filter dev_mgmt.**>
  @type record_transformer
  enable_ruby true
  <record>
    instanceId ${record["instanceId"]}-${"#{(Socket.ip_address_list.detect do |intf| intf.ipv4_private? end).ip_address}"}
    level ${if record['level'] == 'I' then 'info' elsif record['level'] == 'W' then 'warn' else 'error'; end}
  </record>
</filter>

# input plugin to collect system logs
# formating the input logs using regex and creating a feilds such as host, caller and msg.
<source>
  @type syslog
  port 42185
  tag system
  format /(.*\S \d{1,2}:\d{1,2}:\d{1,2}.)(?<host>[^\s]\w+) (?<caller>[^\s][^:]+)(.)(?<msg>.*)/
</source>

<filter **>
  @type stdout
</filter>

# Output elasticsearch plugin
# provide host and port of elasticsearch
<match **>
  @id elasticsearch
  @type elasticsearch
  @log_level info
  include_tag_key true
  host localhost
  port 9200
  scheme http
  ssl_verify false
  logstash_format true
  reconnect_on_error true
  reload_on_failure false
  reload_connections false
  <buffer>
    @type file
    path /var/log/td-agent/buffer/elasticsearch
    flush_mode interval
    retry_type exponential_backoff
    flush_thread_count 8
    flush_interval 5s
    retry_max_interval 30
    chunk_limit_size 32MB
    queue_limit_length 8
    overflow_action block
  </buffer>
</match>