[SEBA-154]
Enable KafkaLogHandler for XOS core/services
Reduce restarts of logstash
Values file for kafka on persistent storage
Change-Id: I631472ba1433ba195d1e2fec304a3193c29a81a9
diff --git a/examples/kafka-production.yaml b/examples/kafka-production.yaml
new file mode 100644
index 0000000..135130b
--- /dev/null
+++ b/examples/kafka-production.yaml
@@ -0,0 +1,41 @@
+---
+# Copyright 2018-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Deploy a production ready Kafka with persistent storage
+#
+# Tested with v0.8.8 of the chart from the helm incubator
+#
+# ex:
+# helm install -f examples/kafka-production.yaml --version 0.8.8 -n cord-kafka incubator/kafka
+
+# ref: https://kafka.apache.org/documentation/#configuration
+
+configurationOverrides:
+ "offsets.topic.replication.factor": 3
+ "log.retention.hours": 4
+
+replicas: 3
+
+persistence:
+ enabled: true
+ storageClass: "local-ssd"
+ size: "10Gi"
+
+zookeeper:
+ replicaCount: 3
+ persistence:
+ enabled: true
+ storageClass: "local-hdd"
+ size: "10Gi"
diff --git a/examples/kafka-single.yaml b/examples/kafka-single.yaml
index dc6653b..210dfd0 100644
--- a/examples/kafka-single.yaml
+++ b/examples/kafka-single.yaml
@@ -20,10 +20,12 @@
# ex:
# helm install -f examples/kafka-single.yaml --version 0.8.8 -n cord-kafka incubator/kafka
-replicas: 1
-
+# ref: https://kafka.apache.org/documentation/#configuration
configurationOverrides:
"offsets.topic.replication.factor": 1
+ "log.retention.hours": 4
+
+replicas: 1
persistence:
enabled: false
diff --git a/examples/kafka-topic.yaml b/examples/kafka-topic.yaml
deleted file mode 100644
index 4bb9b3a..0000000
--- a/examples/kafka-topic.yaml
+++ /dev/null
@@ -1,19 +0,0 @@
----
-
-# Copyright 2018-present Open Networking Foundation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-topics:
- - name: "onu.events"
- - name: "authentication.events"
\ No newline at end of file
diff --git a/logging/values.yaml b/logging/values.yaml
index 1d67294..a3c0690 100644
--- a/logging/values.yaml
+++ b/logging/values.yaml
@@ -57,6 +57,13 @@
persistence:
enabled: false
+ # Probe default delays are too short, which puts logstash into a reboot loop
+ livenessProbe:
+ initialDelaySeconds: 120
+
+ readinessProbe:
+ initialDelaySeconds: 120
+
# 'config' k/v are turned into env vars for logstash container
# ref: https://www.elastic.co/guide/en/logstash/current/docker-config.html#docker-env-config
# config:
diff --git a/xos-core/templates/_core.tpl b/xos-core/templates/_core.tpl
index a766082..d83dfe7 100644
--- a/xos-core/templates/_core.tpl
+++ b/xos-core/templates/_core.tpl
@@ -31,11 +31,17 @@
filename: /var/log/xos.log
maxBytes: 10485760
backupCount: 5
+ kafka:
+ class: kafkaloghandler.kafkaloghandler.KafkaLogHandler
+ bootstrap_servers:
+ - "cord-kafka:9092"
+ topic: xos.log.core
loggers:
'multistructlog':
handlers:
- console
- file
+ - kafka
level: DEBUG
{{- end }}
diff --git a/xos-core/templates/_tosca.tpl b/xos-core/templates/_tosca.tpl
index 3292f4b..fe2ebf1 100644
--- a/xos-core/templates/_tosca.tpl
+++ b/xos-core/templates/_tosca.tpl
@@ -28,10 +28,16 @@
filename: /var/log/xos.log
maxBytes: 10485760
backupCount: 5
+ kafka:
+ class: kafkaloghandler.kafkaloghandler.KafkaLogHandler
+ bootstrap_servers:
+ - "cord-kafka:9092"
+ topic: xos.log.tosca
loggers:
'multistructlog':
handlers:
- console
- file
+ - kafka
level: DEBUG
{{- end }}
diff --git a/xos-services/addressmanager/templates/_helpers.tpl b/xos-services/addressmanager/templates/_helpers.tpl
index e80e8e3..b85862a 100644
--- a/xos-services/addressmanager/templates/_helpers.tpl
+++ b/xos-services/addressmanager/templates/_helpers.tpl
@@ -67,10 +67,16 @@
filename: /var/log/xos.log
maxBytes: 10485760
backupCount: 5
+ kafka:
+ class: kafkaloghandler.kafkaloghandler.KafkaLogHandler
+ bootstrap_servers:
+ - "cord-kafka:9092"
+ topic: xos.log.addressmanager
loggers:
'multistructlog':
handlers:
- - console
- - file
+ - console
+ - file
+ - kafka
level: DEBUG
{{- end -}}
diff --git a/xos-services/att-workflow-driver/templates/_helpers.tpl b/xos-services/att-workflow-driver/templates/_helpers.tpl
index 920bcf7..e490c81 100644
--- a/xos-services/att-workflow-driver/templates/_helpers.tpl
+++ b/xos-services/att-workflow-driver/templates/_helpers.tpl
@@ -65,10 +65,16 @@
filename: /var/log/xos.log
maxBytes: 10485760
backupCount: 5
+ kafka:
+ class: kafkaloghandler.kafkaloghandler.KafkaLogHandler
+ bootstrap_servers:
+ - "cord-kafka:9092"
+ topic: xos.log.att-workflow-driver
loggers:
'multistructlog':
handlers:
- - console
- - file
+ - console
+ - file
+ - kafka
level: DEBUG
{{- end -}}
diff --git a/xos-services/epc-service/templates/_helpers.tpl b/xos-services/epc-service/templates/_helpers.tpl
index f5ea4c8..531bb40 100644
--- a/xos-services/epc-service/templates/_helpers.tpl
+++ b/xos-services/epc-service/templates/_helpers.tpl
@@ -62,11 +62,17 @@
filename: /var/log/xos.log
maxBytes: 10485760
backupCount: 5
+ kafka:
+ class: kafkaloghandler.kafkaloghandler.KafkaLogHandler
+ bootstrap_servers:
+ - "cord-kafka:9092"
+ topic: xos.log.epc-service
loggers:
'multistructlog':
handlers:
- - console
- - file
+ - console
+ - file
+ - kafka
level: DEBUG
{{- end -}}
diff --git a/xos-services/exampleservice/templates/_helpers.tpl b/xos-services/exampleservice/templates/_helpers.tpl
index 5087738..c33f782 100644
--- a/xos-services/exampleservice/templates/_helpers.tpl
+++ b/xos-services/exampleservice/templates/_helpers.tpl
@@ -58,11 +58,17 @@
filename: /var/log/xos.log
maxBytes: 10485760
backupCount: 5
+ kafka:
+ class: kafkaloghandler.kafkaloghandler.KafkaLogHandler
+ bootstrap_servers:
+ - "cord-kafka:9092"
+ topic: xos.log.exampleservice
loggers:
'multistructlog':
handlers:
- - console
- - file
+ - console
+ - file
+ - kafka
level: DEBUG
proxy_ssh:
enabled: {{ .Values.global.proxySshEnabled }}
diff --git a/xos-services/fabric-crossconnect/templates/_helpers.tpl b/xos-services/fabric-crossconnect/templates/_helpers.tpl
index 4d4c3b8..1e1b73e 100644
--- a/xos-services/fabric-crossconnect/templates/_helpers.tpl
+++ b/xos-services/fabric-crossconnect/templates/_helpers.tpl
@@ -65,10 +65,16 @@
filename: /var/log/xos.log
maxBytes: 10485760
backupCount: 5
+ kafka:
+ class: kafkaloghandler.kafkaloghandler.KafkaLogHandler
+ bootstrap_servers:
+ - "cord-kafka:9092"
+ topic: xos.log.fabric-crossconnect
loggers:
'multistructlog':
handlers:
- console
- file
+ - kafka
level: DEBUG
{{- end -}}
diff --git a/xos-services/fabric/templates/_helpers.tpl b/xos-services/fabric/templates/_helpers.tpl
index 254524b..808e8dd 100644
--- a/xos-services/fabric/templates/_helpers.tpl
+++ b/xos-services/fabric/templates/_helpers.tpl
@@ -65,10 +65,16 @@
filename: /var/log/xos.log
maxBytes: 10485760
backupCount: 5
+ kafka:
+ class: kafkaloghandler.kafkaloghandler.KafkaLogHandler
+ bootstrap_servers:
+ - "cord-kafka:9092"
+ topic: xos.log.fabric
loggers:
'multistructlog':
handlers:
- console
- file
+ - kafka
level: DEBUG
{{- end -}}
diff --git a/xos-services/hippie-oss/templates/_helpers.tpl b/xos-services/hippie-oss/templates/_helpers.tpl
index 800aa20..5f5aa5d 100644
--- a/xos-services/hippie-oss/templates/_helpers.tpl
+++ b/xos-services/hippie-oss/templates/_helpers.tpl
@@ -65,10 +65,16 @@
filename: /var/log/xos.log
maxBytes: 10485760
backupCount: 5
+ kafka:
+ class: kafkaloghandler.kafkaloghandler.KafkaLogHandler
+ bootstrap_servers:
+ - "cord-kafka:9092"
+ topic: xos.log.hippie-oss
loggers:
'multistructlog':
handlers:
- - console
- - file
+ - console
+ - file
+ - kafka
level: DEBUG
{{- end -}}
diff --git a/xos-services/hssdb/templates/_helpers.tpl b/xos-services/hssdb/templates/_helpers.tpl
index ba1517e..9cadd45 100644
--- a/xos-services/hssdb/templates/_helpers.tpl
+++ b/xos-services/hssdb/templates/_helpers.tpl
@@ -62,11 +62,17 @@
filename: /var/log/xos.log
maxBytes: 10485760
backupCount: 5
+ kafka:
+ class: kafkaloghandler.kafkaloghandler.KafkaLogHandler
+ bootstrap_servers:
+ - "cord-kafka:9092"
+ topic: xos.log.hssdb
loggers:
'multistructlog':
handlers:
- - console
- - file
+ - console
+ - file
+ - kafka
level: DEBUG
proxy_ssh:
enabled: {{ .Values.global.proxySshEnabled }}
diff --git a/xos-services/internetemulator/templates/_helpers.tpl b/xos-services/internetemulator/templates/_helpers.tpl
index d6e6310..13a5e2c 100644
--- a/xos-services/internetemulator/templates/_helpers.tpl
+++ b/xos-services/internetemulator/templates/_helpers.tpl
@@ -62,11 +62,17 @@
filename: /var/log/xos.log
maxBytes: 10485760
backupCount: 5
+ kafka:
+ class: kafkaloghandler.kafkaloghandler.KafkaLogHandler
+ bootstrap_servers:
+ - "cord-kafka:9092"
+ topic: xos.log.internetemulator
loggers:
'multistructlog':
handlers:
- - console
- - file
+ - console
+ - file
+ - kafka
level: DEBUG
proxy_ssh:
enabled: {{ .Values.global.proxySshEnabled }}
diff --git a/xos-services/kubernetes/templates/_helpers.tpl b/xos-services/kubernetes/templates/_helpers.tpl
index 594d897..35d4b4d 100644
--- a/xos-services/kubernetes/templates/_helpers.tpl
+++ b/xos-services/kubernetes/templates/_helpers.tpl
@@ -61,10 +61,16 @@
filename: /var/log/xos.log
maxBytes: 10485760
backupCount: 5
+ kafka:
+ class: kafkaloghandler.kafkaloghandler.KafkaLogHandler
+ bootstrap_servers:
+ - "cord-kafka:9092"
+ topic: xos.log.kubernetes
loggers:
'multistructlog':
handlers:
- console
- file
+ - kafka
level: DEBUG
{{- end -}}
diff --git a/xos-services/mcord-subscriber/templates/_helpers.tpl b/xos-services/mcord-subscriber/templates/_helpers.tpl
index 456e27f..cf10579 100644
--- a/xos-services/mcord-subscriber/templates/_helpers.tpl
+++ b/xos-services/mcord-subscriber/templates/_helpers.tpl
@@ -62,10 +62,16 @@
filename: /var/log/xos.log
maxBytes: 10485760
backupCount: 5
+ kafka:
+ class: kafkaloghandler.kafkaloghandler.KafkaLogHandler
+ bootstrap_servers:
+ - "cord-kafka:9092"
+ topic: xos.log.mcord-subscriber
loggers:
'multistructlog':
handlers:
- - console
- - file
+ - console
+ - file
+ - kafka
level: DEBUG
{{- end -}}
diff --git a/xos-services/onos-service/templates/_helpers.tpl b/xos-services/onos-service/templates/_helpers.tpl
index c28e42f..9eea7d2 100644
--- a/xos-services/onos-service/templates/_helpers.tpl
+++ b/xos-services/onos-service/templates/_helpers.tpl
@@ -66,10 +66,16 @@
filename: /var/log/xos.log
maxBytes: 10485760
backupCount: 5
+ kafka:
+ class: kafkaloghandler.kafkaloghandler.KafkaLogHandler
+ bootstrap_servers:
+ - "cord-kafka:9092"
+ topic: xos.log.onos-service
loggers:
'multistructlog':
handlers:
- - console
- - file
+ - console
+ - file
+ - kafka
level: DEBUG
{{- end -}}
diff --git a/xos-services/openstack/templates/_helpers.tpl b/xos-services/openstack/templates/_helpers.tpl
index e521645..fdca6e8 100644
--- a/xos-services/openstack/templates/_helpers.tpl
+++ b/xos-services/openstack/templates/_helpers.tpl
@@ -57,11 +57,17 @@
filename: /var/log/xos.log
maxBytes: 10485760
backupCount: 5
+ kafka:
+ class: kafkaloghandler.kafkaloghandler.KafkaLogHandler
+ bootstrap_servers:
+ - "cord-kafka:9092"
+ topic: xos.log.openstack
loggers:
'multistructlog':
handlers:
- - console
- - file
+ - console
+ - file
+ - kafka
level: DEBUG
nova:
ca_ssl_cert: "/etc/ssl/certs/ca-certificates.crt"
diff --git a/xos-services/progran/templates/_helpers.tpl b/xos-services/progran/templates/_helpers.tpl
index cd0a765..8c2404d 100644
--- a/xos-services/progran/templates/_helpers.tpl
+++ b/xos-services/progran/templates/_helpers.tpl
@@ -62,10 +62,16 @@
filename: /var/log/xos.log
maxBytes: 10485760
backupCount: 5
+ kafka:
+ class: kafkaloghandler.kafkaloghandler.KafkaLogHandler
+ bootstrap_servers:
+ - "cord-kafka:9092"
+ topic: xos.log.progran
loggers:
'multistructlog':
handlers:
- - console
- - file
+ - console
+ - file
+ - kafka
level: DEBUG
{{- end -}}
diff --git a/xos-services/rcord/templates/_helpers.tpl b/xos-services/rcord/templates/_helpers.tpl
index d9b69b6..aa5ba31 100644
--- a/xos-services/rcord/templates/_helpers.tpl
+++ b/xos-services/rcord/templates/_helpers.tpl
@@ -58,10 +58,16 @@
filename: /var/log/xos.log
maxBytes: 10485760
backupCount: 5
+ kafka:
+ class: kafkaloghandler.kafkaloghandler.KafkaLogHandler
+ bootstrap_servers:
+ - "cord-kafka:9092"
+ topic: xos.log.rcord
loggers:
'multistructlog':
handlers:
- - console
- - file
+ - console
+ - file
+ - kafka
level: DEBUG
{{- end -}}
diff --git a/xos-services/sdncontroller/templates/_helpers.tpl b/xos-services/sdncontroller/templates/_helpers.tpl
index 12048a9..cb055ba 100644
--- a/xos-services/sdncontroller/templates/_helpers.tpl
+++ b/xos-services/sdncontroller/templates/_helpers.tpl
@@ -62,11 +62,17 @@
filename: /var/log/xos.log
maxBytes: 10485760
backupCount: 5
+ kafka:
+ class: kafkaloghandler.kafkaloghandler.KafkaLogHandler
+ bootstrap_servers:
+ - "cord-kafka:9092"
+ topic: xos.log.sdncontroller
loggers:
'multistructlog':
handlers:
- - console
- - file
+ - console
+ - file
+ - kafka
level: DEBUG
proxy_ssh:
enabled: {{ .Values.global.proxySshEnabled }}
diff --git a/xos-services/simpleexampleservice/templates/_helpers.tpl b/xos-services/simpleexampleservice/templates/_helpers.tpl
index 3307f88..58a8fe2 100644
--- a/xos-services/simpleexampleservice/templates/_helpers.tpl
+++ b/xos-services/simpleexampleservice/templates/_helpers.tpl
@@ -81,10 +81,16 @@
filename: /var/log/xos.log
maxBytes: 10485760
backupCount: 5
+ kafka:
+ class: kafkaloghandler.kafkaloghandler.KafkaLogHandler
+ bootstrap_servers:
+ - "cord-kafka:9092"
+ topic: xos.log.simpleexampleservice
loggers:
'multistructlog':
handlers:
- - console
- - file
+ - console
+ - file
+ - kafka
level: DEBUG
{{- end -}}
diff --git a/xos-services/vhss/templates/_helpers.tpl b/xos-services/vhss/templates/_helpers.tpl
index 82ceead..04bc6db 100644
--- a/xos-services/vhss/templates/_helpers.tpl
+++ b/xos-services/vhss/templates/_helpers.tpl
@@ -62,11 +62,17 @@
filename: /var/log/xos.log
maxBytes: 10485760
backupCount: 5
+ kafka:
+ class: kafkaloghandler.kafkaloghandler.KafkaLogHandler
+ bootstrap_servers:
+ - "cord-kafka:9092"
+ topic: xos.log.vhss
loggers:
'multistructlog':
handlers:
- - console
- - file
+ - console
+ - file
+ - kafka
level: DEBUG
proxy_ssh:
enabled: {{ .Values.global.proxySshEnabled }}
diff --git a/xos-services/vmme/templates/_helpers.tpl b/xos-services/vmme/templates/_helpers.tpl
index d0f9f12..0e2c541 100644
--- a/xos-services/vmme/templates/_helpers.tpl
+++ b/xos-services/vmme/templates/_helpers.tpl
@@ -62,11 +62,17 @@
filename: /var/log/xos.log
maxBytes: 10485760
backupCount: 5
+ kafka:
+ class: kafkaloghandler.kafkaloghandler.KafkaLogHandler
+ bootstrap_servers:
+ - "cord-kafka:9092"
+ topic: xos.log.vmme
loggers:
'multistructlog':
handlers:
- - console
- - file
+ - console
+ - file
+ - kafka
level: DEBUG
proxy_ssh:
enabled: {{ .Values.global.proxySshEnabled }}
diff --git a/xos-services/volt/templates/_helpers.tpl b/xos-services/volt/templates/_helpers.tpl
index a5ba343..114b22b 100644
--- a/xos-services/volt/templates/_helpers.tpl
+++ b/xos-services/volt/templates/_helpers.tpl
@@ -65,10 +65,16 @@
filename: /var/log/xos.log
maxBytes: 10485760
backupCount: 5
+ kafka:
+ class: kafkaloghandler.kafkaloghandler.KafkaLogHandler
+ bootstrap_servers:
+ - "cord-kafka:9092"
+ topic: xos.log.volt
loggers:
'multistructlog':
handlers:
- - console
- - file
- level: INFO
+ - console
+ - file
+ - kafka
+ level: DEBUG
{{- end -}}
diff --git a/xos-services/vrouter/templates/_helpers.tpl b/xos-services/vrouter/templates/_helpers.tpl
index 0a26cd1..ef4b394 100644
--- a/xos-services/vrouter/templates/_helpers.tpl
+++ b/xos-services/vrouter/templates/_helpers.tpl
@@ -62,10 +62,16 @@
filename: /var/log/xos.log
maxBytes: 10485760
backupCount: 5
+ kafka:
+ class: kafkaloghandler.kafkaloghandler.KafkaLogHandler
+ bootstrap_servers:
+ - "cord-kafka:9092"
+ topic: xos.log.vrouter
loggers:
'multistructlog':
handlers:
- - console
- - file
+ - console
+ - file
+ - kafka
level: DEBUG
{{- end -}}
diff --git a/xos-services/vsg-hw/templates/_helpers.tpl b/xos-services/vsg-hw/templates/_helpers.tpl
index fcf95f9..facf592 100644
--- a/xos-services/vsg-hw/templates/_helpers.tpl
+++ b/xos-services/vsg-hw/templates/_helpers.tpl
@@ -62,10 +62,16 @@
filename: /var/log/xos.log
maxBytes: 10485760
backupCount: 5
+ kafka:
+ class: kafkaloghandler.kafkaloghandler.KafkaLogHandler
+ bootstrap_servers:
+ - "cord-kafka:9092"
+ topic: xos.log.vsg-hw
loggers:
'multistructlog':
handlers:
- - console
- - file
+ - console
+ - file
+ - kafka
level: DEBUG
{{- end -}}
diff --git a/xos-services/vspgwc/templates/_helpers.tpl b/xos-services/vspgwc/templates/_helpers.tpl
index 6581ccf..8cc7bd0 100644
--- a/xos-services/vspgwc/templates/_helpers.tpl
+++ b/xos-services/vspgwc/templates/_helpers.tpl
@@ -62,11 +62,17 @@
filename: /var/log/xos.log
maxBytes: 10485760
backupCount: 5
+ kafka:
+ class: kafkaloghandler.kafkaloghandler.KafkaLogHandler
+ bootstrap_servers:
+ - "cord-kafka:9092"
+ topic: xos.log.vspgwc
loggers:
'multistructlog':
handlers:
- - console
- - file
+ - console
+ - file
+ - kafka
level: DEBUG
proxy_ssh:
enabled: {{ .Values.global.proxySshEnabled }}
diff --git a/xos-services/vspgwu/templates/_helpers.tpl b/xos-services/vspgwu/templates/_helpers.tpl
index 2932dcb..242c329 100644
--- a/xos-services/vspgwu/templates/_helpers.tpl
+++ b/xos-services/vspgwu/templates/_helpers.tpl
@@ -62,11 +62,17 @@
filename: /var/log/xos.log
maxBytes: 10485760
backupCount: 5
+ kafka:
+ class: kafkaloghandler.kafkaloghandler.KafkaLogHandler
+ bootstrap_servers:
+ - "cord-kafka:9092"
+ topic: xos.log.vspgwu
loggers:
'multistructlog':
handlers:
- - console
- - file
+ - console
+ - file
+ - kafka
level: DEBUG
blueprints:
- name: cord_5_0_blueprint
diff --git a/xos-services/vtn-service/templates/_helpers.tpl b/xos-services/vtn-service/templates/_helpers.tpl
index ef1585d..22e1457 100644
--- a/xos-services/vtn-service/templates/_helpers.tpl
+++ b/xos-services/vtn-service/templates/_helpers.tpl
@@ -62,10 +62,16 @@
filename: /var/log/xos.log
maxBytes: 10485760
backupCount: 5
+ kafka:
+ class: kafkaloghandler.kafkaloghandler.KafkaLogHandler
+ bootstrap_servers:
+ - "cord-kafka:9092"
+ topic: xos.log.vtn-service
loggers:
'multistructlog':
handlers:
- - console
- - file
+ - console
+ - file
+ - kafka
level: DEBUG
{{- end -}}