CORD-2398: Install and configure filebeat in xos containers
Change-Id: I9d1f62d7bfaa777ff500087f0699dd078401ef5f
diff --git a/containers/xos/Dockerfile.base b/containers/xos/Dockerfile.base
index 2d5582e..d94e4db 100644
--- a/containers/xos/Dockerfile.base
+++ b/containers/xos/Dockerfile.base
@@ -49,13 +49,22 @@
&& apt-key adv --keyserver hkp://ha.pool.sks-keyservers.net:80 \
--recv-keys 58118E89F3A912897C070ADBF76221572C52609D \
&& echo "deb https://apt.dockerproject.org/repo ubuntu-trusty main" | \
- sudo tee /etc/apt/sources.list.d/docker.list \
+ tee /etc/apt/sources.list.d/docker.list \
+ && apt-key adv --keyserver hkp://ha.pool.sks-keyservers.net:80 \
+ --recv-keys 46095ACC8548582C1A2699A9D27D666CD88E42B4 \
+ && echo "deb https://artifacts.elastic.co/packages/6.x/apt stable main" | \
+ tee -a /etc/apt/sources.list.d/elastic-6.x.list \
&& apt-get update \
&& apt-get install -y docker-engine \
+ && apt-get install -y filebeat \
+ && update-rc.d filebeat defaults 95 10 \
&& rm -rf /var/lib/apt/lists/* \
&& mkdir /var/xos \
&& pip freeze > /var/xos/pip_freeze_apt_`date -u +%Y%m%dT%H%M%S`
+# Configure filebeat to collect json data
+COPY filebeat.yml /etc/filebeat/filebeat.yml
+
# Install python packages with pip
COPY pip_requirements.txt /tmp/pip_requirements.txt
diff --git a/containers/xos/filebeat.yml b/containers/xos/filebeat.yml
new file mode 100644
index 0000000..7abe958
--- /dev/null
+++ b/containers/xos/filebeat.yml
@@ -0,0 +1,180 @@
+###################### Filebeat Configuration Example #########################
+
+# This file is an example configuration file highlighting only the most common
+# options. The filebeat.reference.yml file from the same directory contains all the
+# supported options with more comments. You can use it as a reference.
+#
+# You can find the full configuration reference here:
+# https://www.elastic.co/guide/en/beats/filebeat/index.html
+
+# For more available modules and options, please see the filebeat.reference.yml sample
+# configuration file.
+
+#=========================== Filebeat prospectors =============================
+
+filebeat.prospectors:
+
+# Each - is a prospector. Most options can be set at the prospector level, so
+# you can use different prospectors for various configurations.
+# Below are the prospector specific configurations.
+
+- type: log
+
+ # Change to true to enable this prospector configuration.
+ enabled: true
+
+ # Paths that should be crawled and fetched. Glob based paths.
+ paths:
+ - /var/log/xos*.log
+
+ json.message_key: event
+ json.keys_under_root: true
+
+ # Exclude lines. A list of regular expressions to match. It drops the lines that are
+ # matching any regular expression from the list.
+ #exclude_lines: ['^DBG']
+
+ # Include lines. A list of regular expressions to match. It exports the lines that are
+ # matching any regular expression from the list.
+ #include_lines: ['^ERR', '^WARN']
+
+ # Exclude files. A list of regular expressions to match. Filebeat drops the files that
+ # are matching any regular expression from the list. By default, no files are dropped.
+ #exclude_files: ['.gz$']
+
+ # Optional additional fields. These fields can be freely picked
+ # to add additional information to the crawled log files for filtering
+ #fields:
+ # level: debug
+ # review: 1
+
+ ### Multiline options
+
+ # Mutiline can be used for log messages spanning multiple lines. This is common
+ # for Java Stack Traces or C-Line Continuation
+
+ # The regexp Pattern that has to be matched. The example pattern matches all lines starting with [
+ #multiline.pattern: ^\[
+
+ # Defines if the pattern set under pattern should be negated or not. Default is false.
+ #multiline.negate: false
+
+ # Match can be set to "after" or "before". It is used to define if lines should be append to a pattern
+ # that was (not) matched before or after or as long as a pattern is not matched based on negate.
+ # Note: After is the equivalent to previous and before is the equivalent to to next in Logstash
+ #multiline.match: after
+
+
+#============================= Filebeat modules ===============================
+
+filebeat.config.modules:
+ # Glob pattern for configuration loading
+ path: ${path.config}/modules.d/*.yml
+
+ # Set to true to enable config reloading
+ reload.enabled: false
+
+ # Period on which files under path should be checked for changes
+ #reload.period: 10s
+
+#==================== Elasticsearch template setting ==========================
+
+setup.template.settings:
+ index.number_of_shards: 3
+ #index.codec: best_compression
+ #_source.enabled: false
+
+#================================ General =====================================
+
+# The name of the shipper that publishes the network data. It can be used to group
+# all the transactions sent by a single shipper in the web interface.
+#name:
+
+# The tags of the shipper are included in their own field with each
+# transaction published.
+#tags: ["service-X", "web-tier"]
+
+# Optional fields that you can specify to add additional information to the
+# output.
+#fields:
+# env: staging
+
+
+#============================== Dashboards =====================================
+# These settings control loading the sample dashboards to the Kibana index. Loading
+# the dashboards is disabled by default and can be enabled either by setting the
+# options here, or by using the `-setup` CLI flag or the `setup` command.
+#setup.dashboards.enabled: false
+
+# The URL from where to download the dashboards archive. By default this URL
+# has a value which is computed based on the Beat name and version. For released
+# versions, this URL points to the dashboard archive on the artifacts.elastic.co
+# website.
+#setup.dashboards.url:
+
+#============================== Kibana =====================================
+
+# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API.
+# This requires a Kibana endpoint configuration.
+setup.kibana:
+
+ # Kibana Host
+ # Scheme and port can be left out and will be set to the default (http and 5601)
+ # In case you specify and additional path, the scheme is required: http://localhost:5601/path
+ # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
+ #host: "localhost:5601"
+
+#============================= Elastic Cloud ==================================
+
+# These settings simplify using filebeat with the Elastic Cloud (https://cloud.elastic.co/).
+
+# The cloud.id setting overwrites the `output.elasticsearch.hosts` and
+# `setup.kibana.host` options.
+# You can find the `cloud.id` in the Elastic Cloud web UI.
+#cloud.id:
+
+# The cloud.auth setting overwrites the `output.elasticsearch.username` and
+# `output.elasticsearch.password` settings. The format is `<user>:<pass>`.
+#cloud.auth:
+
+#================================ Outputs =====================================
+
+# Configure what output to use when sending the data collected by the beat.
+
+#-------------------------- Elasticsearch output ------------------------------
+output.elasticsearch:
+ # Array of hosts to connect to.
+ hosts: ["elk:9200"]
+ template.name: filebeat
+ template.path: filebeat.template.json
+
+ # Optional protocol and basic auth credentials.
+ #protocol: "https"
+ #username: "elastic"
+ #password: "changeme"
+
+#----------------------------- Logstash output --------------------------------
+#output.logstash:
+ # The Logstash hosts
+ #hosts: ["localhost:5044"]
+
+ # Optional SSL. By default is off.
+ # List of root certificates for HTTPS server verifications
+ #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
+
+ # Certificate for SSL client authentication
+ #ssl.certificate: "/etc/pki/client/cert.pem"
+
+ # Client Certificate Key
+ #ssl.key: "/etc/pki/client/cert.key"
+
+#================================ Logging =====================================
+
+# Sets log level. The default log level is info.
+# Available log levels are: critical, error, warning, info, debug
+# logging.level: debug
+
+# At debug level, you can selectively enable logging only for some components.
+# To enable all selectors use ["*"]. Examples of other selectors are "beat",
+# "publish", "service".
+# logging.selectors: ["*"]