blob: b99d7d99b221a8b81f82c5442daf20ad4a7e114c [file] [log] [blame]
David Bainbridgecfd7ca12019-10-06 03:31:41 +00001#!/bin/bash
2# Copyright 2019 Ciena Corporation
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16# This script periodically collects the POD logs and puts them bin a "RAW"
17# location to be procssed into rolling log files
18
19RAW_LOG_DIR=${RAW_LOG_DIR:-./logger/raw}
20PERIOD=${PERIOOD:-15}
21
22# === END OF CONFIGURATION ===
23set -o pipefail
24
25# Ensure raw log area exists
26mkdir -p $RAW_LOG_DIR
27SINCE=
28
29# forever
30while true; do
31 TS=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
32 SINCE_FLAG=
33 SINCE_MSG=
34
35 # On iteraton 2+ we use the --since-time option to minimize the size of
36 # the logs collected as well as minimize the overlap with previous
37 # collection iteration
38 if [ ! -z "$SINCE" ]; then
39 SINCE_FLAG="--since-time=$SINCE"
40 SINCE_MSG="since $SINCE "
41 fi
42
43 # Build up the logs in a temp directory and then move that to the
44 # RAW area when complete
45 WORK=$(mktemp -d)
46
47 # All VOLTHA PODS + ONOS
48 PODS="$(kubectl -n default get pod -o name | grep onos | sed -e 's/^/default:/g') $(kubectl get -n voltha pod -o name | sed -e 's/^/voltha:/g')"
49 if [ $? -ne 0 ]; then
50 echo "Failed to get PODs from Kubernetes, will retry after sleep ..."
51 else
52 echo "Dumping POD logs at $TS $SINCE_MSG..."
53 for POD in $PODS; do
54 NS=$(echo $POD | cut -d: -f1)
55 POD=$(echo $POD | cut -d: -f2 | sed -e 's/^pod\///g')
56 echo " $POD"
57 kubectl logs --timestamps=true $SINCE_FLAG -n $NS --all-containers $LOG_ARGS $POD 2>&1 > $WORK/$POD.log
58 if [ $? -ne 0 ]; then
59 echo " ERROR: Encountered while getting POD log, removing failed entry"
60 rm -f $WORK/$POD.log
61 fi
62 done
63 if [ $(ls -1 $WORK/ | wc -l) -eq 0 ]; then
64 # Work directory is empty, no need to move it to raw area, just
65 # remove it
66 rm -rf $WORK
67 else
68 mv $WORK $RAW_LOG_DIR/$TS
69 fi
70 fi
71
72 # End iteration and sleep until next iteration
73 echo "====="
74 SINCE=$TS
75 echo "Sleep for $PERIOD seconds ..."
76 sleep $PERIOD
77done