[SEBA-156] ONOS logs to kafka -> elasticstack
Updated versions of logging components, push to stable
Update ONOS to v1.13.5
Change-Id: Ie0a442cca805dc7804a84aa4c7bbbd03acf4bb60
diff --git a/logging/requirements.yaml b/logging/requirements.yaml
index 7918b0c..7352088 100644
--- a/logging/requirements.yaml
+++ b/logging/requirements.yaml
@@ -16,20 +16,20 @@
dependencies:
- name: elasticsearch
- version: 1.4.3
- repository: https://kubernetes-charts-incubator.storage.googleapis.com/
+ version: 1.11.0
+ repository: https://kubernetes-charts.storage.googleapis.com/
- name: kibana
condition: kibana.enabled
- version: 0.11.0
+ version: 0.16.1
repository: https://kubernetes-charts.storage.googleapis.com/
- name: fluentd-elasticsearch
condition: fluentd-elasticsearch.enabled
- version: 1.0.0
+ version: 1.0.3
repository: https://kubernetes-charts.storage.googleapis.com/
- name: logstash
condition: logstash.enabled
- version: 0.8.2
- repository: https://kubernetes-charts-incubator.storage.googleapis.com/
+ version: 1.0.0
+ repository: https://kubernetes-charts.storage.googleapis.com/
diff --git a/logging/values.yaml b/logging/values.yaml
index b330443..4188756 100644
--- a/logging/values.yaml
+++ b/logging/values.yaml
@@ -45,6 +45,25 @@
env:
ELASTICSEARCH_URL: "http://logging-elasticsearch-client:9200"
+ dashboardImport:
+ dashboards:
+ 1_create_index: |-
+ {
+ "version": "6.4.2",
+ "objects": [
+ {
+ "id": "749ea2f0-d15c-11e8-b4aa-8f5499fd5eba",
+ "type": "index-pattern",
+ "version": 2,
+ "attributes": {
+ "title": "logstash-*",
+ "timeFieldName": "@timestamp",
+ "fields": "[{\"name\":\"@timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"@version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"bundle.id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"bundle.id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"bundle.name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"bundle.name.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"bundle.version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"bundle.version.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"category\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"category.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"error.message\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"error.message.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"error.type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"error.type.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"detail\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"geoip.ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geoip.latitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geoip.location\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geoip.longitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host.name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"host.name.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"input.type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"input.type.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"kafka_key\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"kafka_key.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"kafka_timestamp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"kafka_timestamp.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"kafka_topic\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"kafka_topic.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"levelname\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"levelname.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"levelno\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"lineno\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"message\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"offset\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"msg.model\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"msg.version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"thread\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threadName\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"type.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]"
+ }
+ }
+ ]
+ }
+
# logstash config
# ref: https://github.com/helm/charts/tree/master/incubator/logstash
logstash:
@@ -75,13 +94,16 @@
main: |-
input {
kafka {
+ auto_offset_reset => "earliest" # get all previous items from new topics
bootstrap_servers => "cord-kafka:9092"
client_id => "logstash_ck"
codec => json { charset => "UTF-8" }
consumer_threads => 1
decorate_events => true
+ group_id => "logstash_ck"
metadata_max_age_ms => 60000 # recheck for new topics every minute
- topics_pattern => 'authentication.*|dhcp.*|onos.*|onu.*|voltha.*|xos.*'
+ # other topics that are not indexed: xos.gui_events, voltha.kpis, voltha.heartbeat
+ topics_pattern => '.*\.events|dhcp.*|onos.*|voltha.log.*|xos.log.*'
type => "cord-kafka"
}
}
@@ -97,12 +119,6 @@
}
}
- date {
- match => [ "ts", "UNIX" ]
- target => "voltha_ts"
- tag_on_failure => []
- }
-
json {
source => "data"
target => "data"
diff --git a/onos/templates/_helpers.tpl b/onos/templates/_helpers.tpl
index 23e4c6a..1dab05a 100644
--- a/onos/templates/_helpers.tpl
+++ b/onos/templates/_helpers.tpl
@@ -67,7 +67,7 @@
################################################################################
# Root logger
-log4j.rootLogger=INFO, out, osgi:*, stdout
+log4j.rootLogger=INFO, out, json, osgi:*, stdout
log4j.throwableRenderer=org.apache.log4j.OsgiThrowableRenderer
# CONSOLE appender not used by default
@@ -84,7 +84,17 @@
log4j.appender.out.maxFileSize=10MB
log4j.appender.out.maxBackupIndex=10
-# Sift appender
+# JSON-ish appender (doesn't handle quotes in fields correctly)
+# docs: https://logging.apache.org/log4j/1.2/apidocs/org/apache/log4j/PatternLayout.html
+log4j.appender.json=org.apache.log4j.RollingFileAppender
+log4j.appender.json.layout=org.apache.log4j.PatternLayout
+log4j.appender.json.layout.ConversionPattern={"@timestamp":"%d{yyyy-MM-dd'T'HH:mm:ss.SSS'Z'}","levelname":"%p","threadName":"%t","category":"%c{1}","bundle.id":"%X{bundle.id}","bundle.name":"%X{bundle.name}","bundle.version":"%X{bundle.version}","message":"%m"}%n
+log4j.appender.json.file=${karaf.data}/log/karaf_json.log
+log4j.appender.json.append=true
+log4j.appender.json.maxFileSize=10MB
+log4j.appender.json.maxBackupIndex=10
+
+# Sift appender - one logfile per bundle ID
log4j.appender.sift=org.apache.log4j.sift.MDCSiftingAppender
log4j.appender.sift.key=bundle.name
log4j.appender.sift.default=karaf
@@ -97,4 +107,4 @@
# Application logs
{{ .Values.application_logs }}
-{{- end -}}
\ No newline at end of file
+{{- end -}}
diff --git a/onos/templates/deployment.yaml b/onos/templates/deployment.yaml
index ce4c963..415ecff 100644
--- a/onos/templates/deployment.yaml
+++ b/onos/templates/deployment.yaml
@@ -1,5 +1,4 @@
---
-
# Copyright 2018-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -67,12 +66,34 @@
resources:
{{ toYaml .Values.resources | indent 12 }}
volumeMounts:
- - name: onos-logs-cfg
- mountPath: /root/onos/apache-karaf-3.0.8/etc/org.ops4j.pax.logging.cfg
- subPath: org.ops4j.pax.logging.cfg
- name: node-key
mountPath: /root/vtn
readOnly: true
+ - name: onos-logs-cfg
+ mountPath: /root/onos/apache-karaf-3.0.8/etc/org.ops4j.pax.logging.cfg
+ subPath: org.ops4j.pax.logging.cfg
+ - name: onos-logs
+ mountPath: /root/onos/apache-karaf-3.0.8/data/log
+{{- if .Values.log_agent.enabled }}
+ - name: {{ .Chart.Name }}-log-agent
+ image: "{{ .Values.global.registry }}{{ .Values.images.log_agent.repository }}:{{ .Values.images.log_agent.tag }}"
+ imagePullPolicy: {{ .Values.images.log_agent.pullPolicy }}
+ args: [ "-c", "/conf/filebeat.yml", "-e" ]
+ env:
+ - name: POD_NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ - name: NODE_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: spec.nodeName
+ volumeMounts:
+ - name: onos-logs
+ mountPath: /onos_logs
+ - name: log-agent-configmap-volume
+ mountPath: /conf
+{{- end }}
volumes:
- name: node-key
secret:
@@ -85,7 +106,17 @@
items:
- key: logCfg
path: org.ops4j.pax.logging.cfg
- {{- with .Values.nodeSelector }}
+ - name: onos-logs
+ emptyDir: {}
+{{- if .Values.log_agent.enabled }}
+ - name: log-agent-configmap-volume
+ configMap:
+ name: log-agent-configmap
+ items:
+ - key: config
+ path: filebeat.yml
+{{- end }}
+{{- with .Values.nodeSelector }}
nodeSelector:
{{ toYaml . | indent 8 }}
{{- end }}
@@ -97,3 +128,6 @@
tolerations:
{{ toYaml . | indent 8 }}
{{- end }}
+{{- if .Values.log_agent.enabled }}
+{{- include "onos.log-agent-configmap" . }}
+{{- end }}
diff --git a/onos/templates/log-agent-configmap.yaml b/onos/templates/log-agent-configmap.yaml
new file mode 100644
index 0000000..021ab63
--- /dev/null
+++ b/onos/templates/log-agent-configmap.yaml
@@ -0,0 +1,48 @@
+{{- /*
+Copyright 2017-present Open Networking Foundation
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/ -}}
+{{- define "onos.log-agent-configmap" }}
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+ name: log-agent-configmap
+data:
+ config: |
+ ---
+ # filebeat configuration for ONOS
+ filebeat.inputs:
+ - type: log
+ paths:
+ - "/onos_logs/karaf_json.log"
+
+ fields_under_root: true
+
+ json.keys_under_root: true
+ json.add_error_key: true
+ json.overwrite_keys: true
+ json.message_key: "message"
+
+ multiline.pattern: '^[[:space:]]'
+ multiline.negate: false
+ multiline.match: after
+
+ output.kafka:
+ hosts: {{ .Values.log_agent.kafka_brokers | toJson }}
+ topic: 'onos.log'
+ key: '%{[bundle.name]}'
+
+{{- end }}
+
diff --git a/onos/values.yaml b/onos/values.yaml
index d649c55..b9e912e 100644
--- a/onos/values.yaml
+++ b/onos/values.yaml
@@ -21,7 +21,22 @@
replicaCount: 1
imagePullPolicy: Always
-onosImage: 'onosproject/onos:1.13.4'
+onosImage: 'onosproject/onos:1.13.5'
+
+images:
+
+ # keep in sync with: https://github.com/helm/charts/blob/master/stable/filebeat/values.yaml
+ log_agent:
+ repository: docker.elastic.co/beats/filebeat-oss
+ tag: 6.4.2
+ pullPolicy: IfNotPresent
+
+global:
+ registry: ''
+
+log_agent:
+ enabled: True
+ kafka_brokers: ['cord-kafka:9092']
nameOverride: ""
fullnameOverride: ""