blob: 9f6f54a91bff7635c91da5a6ce66f8a7ff465c9d [file] [log] [blame]
#!/bin/sh
# Copyright 2019 Ciena Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This script reads RAW log data collected from the PODS and combines
# this RAW data into a rolloing log file.
RAW_LOG_DIR=${RAW_LOG_DIR:-./logger/raw}
COMBINED_LOG_DIR=${COMBINED_LOG_DIR:-./logger/combined}
ROLL_AT=${ROLL_AT:-100M}
MAX_LOG_COUNT=${MAX_LOG_COUNT:-5}
PERIOD=${PERIOD:-60}
# === END OF CONFIGURATION ===
# Convert the ROLL_AT value to byes
ROLL_AT=$(numfmt --from=iec $ROLL_AT)
# Ensure the combined directory exists
mkdir -p $COMBINED_LOG_DIR
# Get a working area
WORK=$(mktemp)
# forever ...
while true; do
# Iterate over all existing raw entries
for MERGE_DIR in $(ls $RAW_LOG_DIR); do
echo "Merging from $RAW_LOG_DIR/$MERGE_DIR ..."
# Iterate over each file in the RAW data directory
for FILE_PATH in $(ls $RAW_LOG_DIR/$MERGE_DIR/*.log); do
# Get the base of the log file
FILE=$(basename $FILE_PATH)
# Find destination log file with largest index, if none
# exists this will end up with IDX == 1
IDX=2
while [ -f $COMBINED_LOG_DIR/$FILE.$(printf "%04d" $IDX) ]; do
IDX=$(expr $IDX + 1)
done
IDX=$(expr $IDX - 1)
# Get the NAME of the log file to write
NAME=$COMBINED_LOG_DIR/$FILE.$(printf "%04d" $IDX)
# different behavior if the file exists or not
if [ -f $NAME ]; then
# if the file exists, check the size of the file and see
# if we need to move to the next index
SIZE=$(stat -c %s $NAME)
if [ $SIZE -gt $ROLL_AT ]; then
# Combine the exists log file with the new data, this will
# have double entires for the overlap, then run it through
# uniq -D | sort -u to only end up with the overlap entries
cat $NAME $FILE_PATH | sort | uniq -D | sort -u > $WORK
# time to move
IDX=$(expr $IDX + 1)
# If the nex IDX is great than the max log file count thebn
# we shift each log file to index - 1, losing the first (.1)
# forever ...
if [ $IDX -gt $MAX_LOG_COUNT ]; then
echo " Shifting log files for $FILE ..."
I=1
while [ $I -lt $MAX_LOG_COUNT ]; do
rm -f $COMBINED_LOG_DIR/$FILE.$(printf "%04d" $I)
mv $COMBINED_LOG_DIR/$FILE.$(printf "%04d" $(expr $I + 1)) $COMBINED_LOG_DIR/$FILE.$(printf "%04d" $I)
I=$(expr $I + 1)
done
# Reset the IDX to the MAX
IDX=$MAX_LOG_COUNT
fi
#
NAME=$COMBINED_LOG_DIR/$FILE.$(printf "%04d" $IDX)
echo " Creating new log file $NAME ..."
# Combine the list of overlap entries (WORK), i.e. the ones
# that need to be removed from the new data set with the
# new data set so that there will be 2 of each of the overlaps.
# then pipe that to uniq -u to return only the uniq entries.
# this has the affect of removing the overlap from the new
# data set
cat $WORK $FILE_PATH | sort | uniq -u > $NAME
chmod 644 $NAME
else
# Not rolling so a simple combining of the new data set
# with the existing file and sorting to only unique entries
# will do
cat $NAME $FILE_PATH | sort -u > $WORK
rm -f $NAME
mv $WORK $NAME
chmod 644 $NAME
fi
else
# The destination log file does not exist, so just sort the
# the RAW data into the file. This really only happens on the
# first iteration
sort -u $FILE_PATH > $NAME
chmod 644 $NAME
fi
done
# Remove the RAW data directory so we don't try to merge it again
rm -rf $RAW_LOG_DIR/$MERGE_DIR
done
echo "====="
echo "Sleep for $PERIOD seconds ..."
sleep $PERIOD
done