update to reference latest images
- updated to latest images based on reorg git repos
- adding a bunch of utility scripts
diff --git a/scripts/log-combine.sh b/scripts/log-combine.sh
new file mode 100755
index 0000000..9f6f54a
--- /dev/null
+++ b/scripts/log-combine.sh
@@ -0,0 +1,128 @@
+#!/bin/sh
+# Copyright 2019 Ciena Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script reads RAW log data collected from the PODS and combines
+# this RAW data into a rolloing log file.
+
+RAW_LOG_DIR=${RAW_LOG_DIR:-./logger/raw}
+COMBINED_LOG_DIR=${COMBINED_LOG_DIR:-./logger/combined}
+ROLL_AT=${ROLL_AT:-100M}
+MAX_LOG_COUNT=${MAX_LOG_COUNT:-5}
+PERIOD=${PERIOD:-60}
+
+# === END OF CONFIGURATION ===
+
+# Convert the ROLL_AT value to byes
+ROLL_AT=$(numfmt --from=iec $ROLL_AT)
+
+# Ensure the combined directory exists
+mkdir -p $COMBINED_LOG_DIR
+
+# Get a working area
+WORK=$(mktemp)
+
+# forever ...
+while true; do
+
+ # Iterate over all existing raw entries
+ for MERGE_DIR in $(ls $RAW_LOG_DIR); do
+ echo "Merging from $RAW_LOG_DIR/$MERGE_DIR ..."
+
+ # Iterate over each file in the RAW data directory
+ for FILE_PATH in $(ls $RAW_LOG_DIR/$MERGE_DIR/*.log); do
+
+ # Get the base of the log file
+ FILE=$(basename $FILE_PATH)
+
+ # Find destination log file with largest index, if none
+ # exists this will end up with IDX == 1
+ IDX=2
+ while [ -f $COMBINED_LOG_DIR/$FILE.$(printf "%04d" $IDX) ]; do
+ IDX=$(expr $IDX + 1)
+ done
+ IDX=$(expr $IDX - 1)
+
+ # Get the NAME of the log file to write
+ NAME=$COMBINED_LOG_DIR/$FILE.$(printf "%04d" $IDX)
+
+ # different behavior if the file exists or not
+ if [ -f $NAME ]; then
+
+ # if the file exists, check the size of the file and see
+ # if we need to move to the next index
+ SIZE=$(stat -c %s $NAME)
+ if [ $SIZE -gt $ROLL_AT ]; then
+
+ # Combine the exists log file with the new data, this will
+ # have double entires for the overlap, then run it through
+ # uniq -D | sort -u to only end up with the overlap entries
+ cat $NAME $FILE_PATH | sort | uniq -D | sort -u > $WORK
+
+ # time to move
+ IDX=$(expr $IDX + 1)
+
+ # If the nex IDX is great than the max log file count thebn
+ # we shift each log file to index - 1, losing the first (.1)
+ # forever ...
+ if [ $IDX -gt $MAX_LOG_COUNT ]; then
+ echo " Shifting log files for $FILE ..."
+ I=1
+ while [ $I -lt $MAX_LOG_COUNT ]; do
+ rm -f $COMBINED_LOG_DIR/$FILE.$(printf "%04d" $I)
+ mv $COMBINED_LOG_DIR/$FILE.$(printf "%04d" $(expr $I + 1)) $COMBINED_LOG_DIR/$FILE.$(printf "%04d" $I)
+ I=$(expr $I + 1)
+ done
+
+ # Reset the IDX to the MAX
+ IDX=$MAX_LOG_COUNT
+ fi
+
+ #
+ NAME=$COMBINED_LOG_DIR/$FILE.$(printf "%04d" $IDX)
+ echo " Creating new log file $NAME ..."
+
+ # Combine the list of overlap entries (WORK), i.e. the ones
+ # that need to be removed from the new data set with the
+ # new data set so that there will be 2 of each of the overlaps.
+ # then pipe that to uniq -u to return only the uniq entries.
+ # this has the affect of removing the overlap from the new
+ # data set
+ cat $WORK $FILE_PATH | sort | uniq -u > $NAME
+ chmod 644 $NAME
+ else
+ # Not rolling so a simple combining of the new data set
+ # with the existing file and sorting to only unique entries
+ # will do
+ cat $NAME $FILE_PATH | sort -u > $WORK
+ rm -f $NAME
+ mv $WORK $NAME
+ chmod 644 $NAME
+ fi
+ else
+ # The destination log file does not exist, so just sort the
+ # the RAW data into the file. This really only happens on the
+ # first iteration
+ sort -u $FILE_PATH > $NAME
+ chmod 644 $NAME
+ fi
+ done
+
+ # Remove the RAW data directory so we don't try to merge it again
+ rm -rf $RAW_LOG_DIR/$MERGE_DIR
+ done
+ echo "====="
+ echo "Sleep for $PERIOD seconds ..."
+ sleep $PERIOD
+done