[SEBA-156] ONOS logs to kafka -> elasticstack
Updated versions of logging components, push to stable
Update ONOS to v1.13.5
Change-Id: Ie0a442cca805dc7804a84aa4c7bbbd03acf4bb60
diff --git a/onos/templates/_helpers.tpl b/onos/templates/_helpers.tpl
index 23e4c6a..1dab05a 100644
--- a/onos/templates/_helpers.tpl
+++ b/onos/templates/_helpers.tpl
@@ -67,7 +67,7 @@
################################################################################
# Root logger
-log4j.rootLogger=INFO, out, osgi:*, stdout
+log4j.rootLogger=INFO, out, json, osgi:*, stdout
log4j.throwableRenderer=org.apache.log4j.OsgiThrowableRenderer
# CONSOLE appender not used by default
@@ -84,7 +84,17 @@
log4j.appender.out.maxFileSize=10MB
log4j.appender.out.maxBackupIndex=10
-# Sift appender
+# JSON-ish appender (doesn't handle quotes in fields correctly)
+# docs: https://logging.apache.org/log4j/1.2/apidocs/org/apache/log4j/PatternLayout.html
+log4j.appender.json=org.apache.log4j.RollingFileAppender
+log4j.appender.json.layout=org.apache.log4j.PatternLayout
+log4j.appender.json.layout.ConversionPattern={"@timestamp":"%d{yyyy-MM-dd'T'HH:mm:ss.SSS'Z'}","levelname":"%p","threadName":"%t","category":"%c{1}","bundle.id":"%X{bundle.id}","bundle.name":"%X{bundle.name}","bundle.version":"%X{bundle.version}","message":"%m"}%n
+log4j.appender.json.file=${karaf.data}/log/karaf_json.log
+log4j.appender.json.append=true
+log4j.appender.json.maxFileSize=10MB
+log4j.appender.json.maxBackupIndex=10
+
+# Sift appender - one logfile per bundle ID
log4j.appender.sift=org.apache.log4j.sift.MDCSiftingAppender
log4j.appender.sift.key=bundle.name
log4j.appender.sift.default=karaf
@@ -97,4 +107,4 @@
# Application logs
{{ .Values.application_logs }}
-{{- end -}}
\ No newline at end of file
+{{- end -}}
diff --git a/onos/templates/deployment.yaml b/onos/templates/deployment.yaml
index ce4c963..415ecff 100644
--- a/onos/templates/deployment.yaml
+++ b/onos/templates/deployment.yaml
@@ -1,5 +1,4 @@
---
-
# Copyright 2018-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -67,12 +66,34 @@
resources:
{{ toYaml .Values.resources | indent 12 }}
volumeMounts:
- - name: onos-logs-cfg
- mountPath: /root/onos/apache-karaf-3.0.8/etc/org.ops4j.pax.logging.cfg
- subPath: org.ops4j.pax.logging.cfg
- name: node-key
mountPath: /root/vtn
readOnly: true
+ - name: onos-logs-cfg
+ mountPath: /root/onos/apache-karaf-3.0.8/etc/org.ops4j.pax.logging.cfg
+ subPath: org.ops4j.pax.logging.cfg
+ - name: onos-logs
+ mountPath: /root/onos/apache-karaf-3.0.8/data/log
+{{- if .Values.log_agent.enabled }}
+ - name: {{ .Chart.Name }}-log-agent
+ image: "{{ .Values.global.registry }}{{ .Values.images.log_agent.repository }}:{{ .Values.images.log_agent.tag }}"
+ imagePullPolicy: {{ .Values.images.log_agent.pullPolicy }}
+ args: [ "-c", "/conf/filebeat.yml", "-e" ]
+ env:
+ - name: POD_NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ - name: NODE_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: spec.nodeName
+ volumeMounts:
+ - name: onos-logs
+ mountPath: /onos_logs
+ - name: log-agent-configmap-volume
+ mountPath: /conf
+{{- end }}
volumes:
- name: node-key
secret:
@@ -85,7 +106,17 @@
items:
- key: logCfg
path: org.ops4j.pax.logging.cfg
- {{- with .Values.nodeSelector }}
+ - name: onos-logs
+ emptyDir: {}
+{{- if .Values.log_agent.enabled }}
+ - name: log-agent-configmap-volume
+ configMap:
+ name: log-agent-configmap
+ items:
+ - key: config
+ path: filebeat.yml
+{{- end }}
+{{- with .Values.nodeSelector }}
nodeSelector:
{{ toYaml . | indent 8 }}
{{- end }}
@@ -97,3 +128,6 @@
tolerations:
{{ toYaml . | indent 8 }}
{{- end }}
+{{- if .Values.log_agent.enabled }}
+{{- include "onos.log-agent-configmap" . }}
+{{- end }}
diff --git a/onos/templates/log-agent-configmap.yaml b/onos/templates/log-agent-configmap.yaml
new file mode 100644
index 0000000..021ab63
--- /dev/null
+++ b/onos/templates/log-agent-configmap.yaml
@@ -0,0 +1,48 @@
+{{- /*
+Copyright 2017-present Open Networking Foundation
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/ -}}
+{{- define "onos.log-agent-configmap" }}
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+ name: log-agent-configmap
+data:
+ config: |
+ ---
+ # filebeat configuration for ONOS
+ filebeat.inputs:
+ - type: log
+ paths:
+ - "/onos_logs/karaf_json.log"
+
+ fields_under_root: true
+
+ json.keys_under_root: true
+ json.add_error_key: true
+ json.overwrite_keys: true
+ json.message_key: "message"
+
+ multiline.pattern: '^[[:space:]]'
+ multiline.negate: false
+ multiline.match: after
+
+ output.kafka:
+ hosts: {{ .Values.log_agent.kafka_brokers | toJson }}
+ topic: 'onos.log'
+ key: '%{[bundle.name]}'
+
+{{- end }}
+