[SEBA-104]

Log aggregation for SEBA

Required features:
- Elasticsearch for log storage

Optional features:
- Kibana for accessing logs through a web ui
- logstash, configured to read events out of kafka into elasticsearch
- fluentd-elasticsearch for shipping container logs from k8s nodes into
  elasticsearch

Change-Id: Ib81561762a0c2b96613ccbad411b660a403e23de
diff --git a/logging/.helmignore b/logging/.helmignore
new file mode 100644
index 0000000..f0c1319
--- /dev/null
+++ b/logging/.helmignore
@@ -0,0 +1,21 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
diff --git a/logging/Chart.yaml b/logging/Chart.yaml
new file mode 100644
index 0000000..5120ca7
--- /dev/null
+++ b/logging/Chart.yaml
@@ -0,0 +1,18 @@
+---
+# Copyright 2018-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+description: Sets up log aggregation infrastructure in Kubernetes, with elasticstack and kibana
+name: logging
+version: 0.1.0-dev0
diff --git a/logging/README.md b/logging/README.md
new file mode 100644
index 0000000..4cdd5e6
--- /dev/null
+++ b/logging/README.md
@@ -0,0 +1,22 @@
+# Logging
+
+This chart implements a log aggregation framework built on elasticsearch within
+kubernetes.
+
+It requires persistent storage, and currently has default values for the
+`local-provisioner` with storage on each k8s node.
+
+Once these prereqs are satisfied, it can be run with:
+
+    helm install -n logging logging
+
+(NOTE: the name must be `logging` currently, or name lookups within the pod are broken)
+
+## Current log sources
+
+- Container logs from k8s with [fluentd-elasticsearch](https://github.com/helm/charts/tree/master/stable/fluentd-elasticsearch)
+
+## Using Kibana
+
+Visit: http://<k8s_node_hostname>:30601
+
diff --git a/logging/requirements.yaml b/logging/requirements.yaml
new file mode 100644
index 0000000..7918b0c
--- /dev/null
+++ b/logging/requirements.yaml
@@ -0,0 +1,35 @@
+---
+# Copyright 2018-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+dependencies:
+
+- name: elasticsearch
+  version: 1.4.3
+  repository: https://kubernetes-charts-incubator.storage.googleapis.com/
+
+- name: kibana
+  condition: kibana.enabled
+  version: 0.11.0
+  repository: https://kubernetes-charts.storage.googleapis.com/
+
+- name: fluentd-elasticsearch
+  condition: fluentd-elasticsearch.enabled
+  version: 1.0.0
+  repository: https://kubernetes-charts.storage.googleapis.com/
+
+- name: logstash
+  condition: logstash.enabled
+  version: 0.8.2
+  repository: https://kubernetes-charts-incubator.storage.googleapis.com/
diff --git a/logging/templates/NOTES.txt b/logging/templates/NOTES.txt
new file mode 100644
index 0000000..7802534
--- /dev/null
+++ b/logging/templates/NOTES.txt
@@ -0,0 +1,16 @@
+** Log aggregation for CORD **
+
+This chart runs the following services:
+
+- Elasticsearch
+
+and optionally:
+
+- Kibana
+- Logstash
+- fluentd-elasticsearch (container logs from k8s)
+
+To access Kibana, go to:
+
+ http://<any_k8s_node>:{{ .Values.kibana.service.nodePort }}
+
diff --git a/logging/values.yaml b/logging/values.yaml
new file mode 100644
index 0000000..1d67294
--- /dev/null
+++ b/logging/values.yaml
@@ -0,0 +1,130 @@
+---
+# Copyright 2018--present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Values file for SEBA log aggregation infra
+# NOTE: must start this chart with the name "logging":
+#   helm install -n logging logging
+# ortherwise the elasticsearch hostname won't work properly in the other containers
+
+# elasticstack config
+# ref: https://github.com/helm/charts/tree/master/incubator/elasticsearch
+elasticsearch:
+  client:
+    serviceType: "NodePort"
+  master:
+    persistence:
+      storageClass: "local-ssd"
+      size: "5Gi"
+  data:
+    persistence:
+      storageClass: "local-hdd"
+      size: "10Gi"
+
+# kibana config
+# ref: https://github.com/helm/charts/tree/master/stable/kibana
+kibana:
+  enabled: true
+#  dashboardImport:
+#    dashboards: |-
+#
+  service:
+    type: "NodePort"
+    nodePort: 30601
+  env:
+    ELASTICSEARCH_URL: "http://logging-elasticsearch-client:9200"
+
+# logstash config
+# ref: https://github.com/helm/charts/tree/master/incubator/logstash
+logstash:
+  enabled: true
+  elasticsearch:
+    host: "logging-elasticsearch-client"
+
+  # we have kafka, so persistence isn't needed
+  # ref: https://www.elastic.co/guide/en/logstash/current/persistent-queues.html
+  persistence:
+    enabled: false
+
+  # 'config' k/v are turned into env vars for logstash container
+  # ref: https://www.elastic.co/guide/en/logstash/current/docker-config.html#docker-env-config
+  # config:
+  #   LOG_LEVEL: trace
+
+  # Kafka input plugin
+  # ref: https://www.elastic.co/guide/en/logstash/current/plugins-inputs-kafka.html
+  inputs:
+    main: |-
+      input {
+        kafka {
+          bootstrap_servers => "cord-kafka:9092"
+          client_id => "logstash_ck"
+          codec => json { charset => "UTF-8" }
+          consumer_threads => 1
+          decorate_events => true
+          metadata_max_age_ms => 60000 # recheck for new topics every minute
+          topics_pattern => 'authentication.*|dhcp.*|onos.*|onu.*|xos.*'
+          type => "cord-kafka"
+        }
+        kafka {
+          bootstrap_servers => "voltha-kafka:9092"
+          client_id => "logstash_vk"
+          codec => json { charset => "UTF-8" }
+          consumer_threads => 1
+          decorate_events => true
+          metadata_max_age_ms => 60000 # recheck for new topics every minute
+          topics_pattern => 'voltha.*'
+          type => "voltha-kafka"
+        }
+      }
+
+  filters:
+    main: |-
+      filter {
+        mutate {
+          add_field => {
+            "kafka_topic" => "%{[@metadata][kafka][topic]}"
+            "kafka_timestamp" => "%{[@metadata][kafka][timestamp]}"
+          }
+        }
+
+        date {
+          match => [ "ts", "UNIX" ]
+          target => "voltha_ts"
+          tag_on_failure => []
+        }
+
+        json {
+          source => "data"
+          target => "data"
+          skip_on_invalid_json => true
+        }
+      }
+
+  outputs:
+    main: |-
+      output {
+        elasticsearch {
+          hosts => ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}"]
+        }
+      }
+
+# fluentd-elasticsearch config
+# ref: https://github.com/helm/charts/tree/master/stable/fluentd-elasticsearch
+# forwards k8s logs from nodes to elasticsearch
+fluentd-elasticsearch:
+  enabled: false
+  elasticsearch:
+    host: "logging-elasticsearch-client"
+