scripts/log-combine.sh

#!/bin/sh
# Copyright 2019 Ciena Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This script reads RAW log data collected from the PODS and combines
# this RAW data into a rolloing log file.

RAW_LOG_DIR=${RAW_LOG_DIR:-./logger/raw}
COMBINED_LOG_DIR=${COMBINED_LOG_DIR:-./logger/combined}
ROLL_AT=${ROLL_AT:-100M}
MAX_LOG_COUNT=${MAX_LOG_COUNT:-5}
PERIOD=${PERIOD:-60}

# === END OF CONFIGURATION ===

# Convert the ROLL_AT value to byes
ROLL_AT=$(numfmt --from=iec $ROLL_AT)

# Ensure the combined directory exists
mkdir -p $COMBINED_LOG_DIR

# Get a working area
WORK=$(mktemp)

# forever ...
while true; do

    # Iterate over all existing raw entries
    for MERGE_DIR in $(ls $RAW_LOG_DIR); do
        echo "Merging from $RAW_LOG_DIR/$MERGE_DIR ..."

        # Iterate over each file in the RAW data directory
        for FILE_PATH in $(ls $RAW_LOG_DIR/$MERGE_DIR/*.log); do

            # Get the base of the log file
            FILE=$(basename $FILE_PATH)

            # Find destination log file with largest index, if none
            # exists this will end up with IDX == 1
            IDX=2
            while [ -f $COMBINED_LOG_DIR/$FILE.$(printf "%04d" $IDX) ]; do
                IDX=$(expr $IDX + 1)
            done
            IDX=$(expr $IDX - 1)

            # Get the NAME of the log file to write
            NAME=$COMBINED_LOG_DIR/$FILE.$(printf "%04d" $IDX)

            # different behavior if the file exists or not
            if [ -f $NAME ]; then

                # if the file exists, check the size of the file and see
                # if we need to move to the next index
                SIZE=$(stat -c %s $NAME)
                if [ $SIZE -gt $ROLL_AT ]; then

                    # Combine the exists log file with the new data, this will
                    # have double entires for the overlap, then run it through
                    # uniq -D | sort -u  to only end up with the overlap entries
                    cat $NAME $FILE_PATH | sort | uniq -D | sort -u > $WORK

                    # time to move
                    IDX=$(expr $IDX + 1)

                    # If the nex IDX is great than the max log file count thebn
                    # we shift each log file to index - 1, losing the first (.1)
                    # forever ... 
                    if [ $IDX -gt $MAX_LOG_COUNT ]; then
                        echo "    Shifting log files for $FILE ..."
                        I=1
                        while [ $I -lt $MAX_LOG_COUNT ]; do
                            rm -f $COMBINED_LOG_DIR/$FILE.$(printf "%04d" $I)
                            mv $COMBINED_LOG_DIR/$FILE.$(printf "%04d" $(expr $I + 1)) $COMBINED_LOG_DIR/$FILE.$(printf "%04d" $I)
                            I=$(expr $I + 1)
                        done

                        # Reset the IDX to the MAX
                        IDX=$MAX_LOG_COUNT
                    fi

                    # 
                    NAME=$COMBINED_LOG_DIR/$FILE.$(printf "%04d" $IDX)
                    echo "    Creating new log file $NAME ..."

                    # Combine the list of overlap entries (WORK), i.e. the ones
                    # that need to be removed from the new data set with the 
                    # new data set so that there will be 2 of each of the overlaps.
                    # then pipe that to uniq -u to return only the uniq entries.
                    # this has the affect of removing the overlap from the new
                    # data set
                    cat $WORK $FILE_PATH | sort | uniq -u > $NAME
                    chmod 644 $NAME
                else
                    # Not rolling so a simple combining of the new data set
                    # with the existing file and sorting to only unique entries
                    # will do
                    cat $NAME $FILE_PATH | sort -u > $WORK
                    rm -f $NAME
                    mv $WORK $NAME
                    chmod 644 $NAME
                fi
            else
                # The destination log file does not exist, so just sort the
                # the RAW data into the file. This really only happens on the
                # first iteration
                sort -u $FILE_PATH > $NAME
                chmod 644 $NAME
            fi
        done

        # Remove the RAW data directory so we don't try to merge it again
        rm -rf $RAW_LOG_DIR/$MERGE_DIR
    done
    echo "====="
    echo "Sleep for $PERIOD seconds ..."
    sleep $PERIOD
done