Incorporate the external log files in debug ball
- don't put bash command errors in log files
- add exit on fail option to repeat script
- add ability of repeat script to include external logs
diff --git a/scripts/e2e-repeat.sh b/scripts/e2e-repeat.sh
index 2cd87b7..269b44e 100755
--- a/scripts/e2e-repeat.sh
+++ b/scripts/e2e-repeat.sh
@@ -15,6 +15,12 @@
# this script repeatedly invokes the e2e system test on a VOLTHA instance
+NAME=${NAME:-minimal}
+EXIT_ON_FAIL=${EXIT_ON_FAIL:-no}
+INCLUDE_LOG_DIR=${INCLUDE_LOG_DIR:-}
+
+# === END OF CONFIGURATION ===
+
set -o pipefail
if [ ! -r voltha-system-tests ]; then
@@ -73,7 +79,7 @@
while true; do
RUN_TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)
echo "START RUN $RUN @ $RUN_TS" | tee -a $LOG
- DEPLOY_K8S=y WITH_BBSIM=y ./voltha up
+ DEPLOY_K8S=y WITH_BBSIM=y WITH_SADIS=y WITH_RADIUS=y ./voltha up
# because BBSIM needs time
sleep 60
ETCD=$(kubectl -n voltha get pods | grep etcd-cluster | awk '{print $1}')
@@ -103,7 +109,34 @@
FAILURE_LIST+=($COUNT_SINCE_FAIL)
COUNT_SINCE_FAIL=0
DUMP_FROM=$RUN_TS ./voltha dump
- ./voltha down
+ C_TS=$(echo $RUN_TS | sed -e 's/[:-]//g')
+
+ if [ ! -z "$INCLUDE_LOG_DIR" ]; then
+ # Now that we have the dump compressed tar file, expand it into a temp
+ # area, augment it with the extended (combined) log files and then
+ # repackage it
+ WORK=$(mktemp -d)
+ tar -C $WORK -zxf voltha-debug-dump-$NAME-$C_TS.tgz
+ mkdir $WORK/voltha-debug-dump-$NAME-$C_TS/logs
+ for LOG in $(ls -1 $INCLUDE_LOG_DIR/*.log.[0-9]*); do
+ LOG_NAME=$(basename $LOG)
+ WORK_LOG=$(mktemp)
+ cat $LOG | awk "{if ( \$1 >= \"$RUN_TS\" ) print}" > $WORK_LOG
+ if [ $(stat -c %s $WORK_LOG) -eq 0 ]; then
+ rm -f $WORK_LOG
+ else
+ mv $WORK_LOG $WORK/voltha-debug-dump-$NAME-$C_TS/logs/$LOG_NAME
+ fi
+ done
+
+ tar -C $WORK -czf voltha-debug-dump-$NAME-$C_TS.tgz ./voltha-debug-dump-$NAME-$C_TS
+ fi
+
+ if [ "$EXIT_ON_FAIL" == "yes" ]; then
+ exit
+ fi
+
+ DEPLOY_K8S=n ./voltha down
fi
echo "END RUN: $RUN @ $(date -u +%Y-%m-%dT%H:%M:%SZ)" | tee -a $LOG
COMPLETED=$(expr $COMPLETED + 1)
diff --git a/scripts/log-collector.sh b/scripts/log-collector.sh
index b99d7d9..6577631 100755
--- a/scripts/log-collector.sh
+++ b/scripts/log-collector.sh
@@ -54,10 +54,13 @@
NS=$(echo $POD | cut -d: -f1)
POD=$(echo $POD | cut -d: -f2 | sed -e 's/^pod\///g')
echo " $POD"
- kubectl logs --timestamps=true $SINCE_FLAG -n $NS --all-containers $LOG_ARGS $POD 2>&1 > $WORK/$POD.log
+ kubectl logs --timestamps=true $SINCE_FLAG -n $NS --all-containers $LOG_ARGS $POD 2>/dev/null > $WORK/$POD.log
if [ $? -ne 0 ]; then
echo " ERROR: Encountered while getting POD log, removing failed entry"
rm -f $WORK/$POD.log
+ elif [ $(cat $WORK/$POD.log | wc -l) -eq 0 ]; then
+ # empty
+ rm -f $WORK/$POD.log
fi
done
if [ $(ls -1 $WORK/ | wc -l) -eq 0 ]; then
diff --git a/voltha b/voltha
index c647dc1..68fdab8 100755
--- a/voltha
+++ b/voltha
@@ -576,6 +576,9 @@
if [ "$1" == "dump" ]; then
LOG="dump-$NAME.log"
TS=$(date -u +"%Y%m%dT%H%M%SZ")
+ if [ ! -z "$DUMP_FROM" ]; then
+ TS=$(echo $DUMP_FROM | sed -e 's/[:-]//g')
+ fi
WORK=$(mktemp -u -d)
DATA=$WORK/voltha-debug-dump-$NAME-$TS
mkdir -p $DATA