update to reference latest images
- updated to latest images based on reorg git repos
- adding a bunch of utility scripts
diff --git a/scripts/e2e-repeat.sh b/scripts/e2e-repeat.sh
new file mode 100755
index 0000000..2cd87b7
--- /dev/null
+++ b/scripts/e2e-repeat.sh
@@ -0,0 +1,151 @@
+#!/bin/bash
+# Copyright 2019 Ciena Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# this script repeatedly invokes the e2e system test on a VOLTHA instance
+
+set -o pipefail
+
+if [ ! -r voltha-system-tests ]; then
+ git clone http://gerrit.opencord.org/voltha-system-tests voltha-system-tests
+fi
+
+delta() {
+ local LEFT=$(echo $1 | numfmt --from=iec)
+ local RIGHT=$(echo $2 | numfmt --from=iec)
+ local V=$(expr $LEFT - $RIGHT)
+ echo ${V#-}
+}
+
+average() {
+ local MIN=0
+ local MAX=0
+ local COUNT=0
+ local SUM=0
+ for V in $*; do
+ COUNT=$(expr $COUNT + 1)
+ SUM=$(expr $SUM + $V)
+ if [ $COUNT -eq 1 ]; then
+ MIN=$V
+ MAX=$V
+ else
+ if [ $V -lt $MIN ]; then
+ MIN=$V
+ elif [ $V -gt $MAX ]; then
+ MAX=$V
+ fi
+ fi
+ done
+ if [ $COUNT -gt 3 ]; then
+ SUM=$(expr $SUM - $MIN - $MAX)
+ COUNT=$(expr $COUNT - 2)
+ fi
+ if [ $COUNT -lt 1 ]; then
+ echo 0
+ else
+ echo $(expr $SUM / $COUNT)
+ fi
+}
+
+export WAIT_FOR_DOWN=y
+export TERM=
+
+COMPLETED=0
+COUNT_OK=0
+COUNT_FAIL=0
+COUNT_SINCE_FAIL=0
+FAILURE_LIST=()
+RSS_DIFFS=()
+KEY_DIFFS=()
+SIZE_DIFFS=()
+LOG=$(mktemp)
+while true; do
+ RUN_TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+ echo "START RUN $RUN @ $RUN_TS" | tee -a $LOG
+ DEPLOY_K8S=y WITH_BBSIM=y ./voltha up
+ # because BBSIM needs time
+ sleep 60
+ ETCD=$(kubectl -n voltha get pods | grep etcd-cluster | awk '{print $1}')
+ BEFORE_KEY_COUNT=$(kubectl -n voltha exec -ti $ETCD \
+ -- sh -c 'ETCDCTL_API=3 etcdctl get --command-timeout=60s --from-key=true --keys-only . | sed -e "/^$/d" | wc -l | tr -d "\r\n"')
+ BEFORE_SIZE=$(numfmt --to=iec \
+ $(kubectl -n voltha exec -ti $ETCD \
+ -- sh -c 'ETCDCTL_API=3 etcdctl endpoint status -w json' | tr -d '\r\n' | jq .[].Status.dbSize))
+ BEFORE_RSS=$(ps -eo rss,pid,cmd | grep /usr/local/bin/etcd | grep -v grep | cut -d\ -f1 | numfmt --to=iec)
+ (cd voltha-system-tests; make sanity-kind 2>&1 | tee $LOG)
+ FAIL=$?
+ AFTER_KEY_COUNT=$(kubectl -n voltha exec -ti $ETCD \
+ -- sh -c 'ETCDCTL_API=3 etcdctl get --command-timeout=60s --from-key=true --keys-only . | sed -e "/^$/d" | wc -l | tr -d "\r\n"')
+ AFTER_SIZE=$(numfmt --to=iec \
+ $(kubectl -n voltha exec -ti $ETCD \
+ -- sh -c 'ETCDCTL_API=3 etcdctl endpoint status -w json' | tr -d '\r\n' | jq .[].Status.dbSize))
+ AFTER_RSS=$(ps -eo rss,pid,cmd | grep /usr/local/bin/etcd | grep -v grep | cut -d\ -f1 | numfmt --to=iec)
+ if [ $FAIL -eq 0 ]; then
+ COUNT_OK=$(expr $COUNT_OK + 1)
+ COUNT_SINCE_FAIL=$(expr $COUNT_SINCE_FAIL + 1)
+ helm delete --purge bbsim
+ while [ $(kubectl get --all-namespaces pods,svc 2>&1 | grep -c bbsim) -gt 0 ]; do
+ sleep 3
+ done
+ else
+ COUNT_FAIL=$(expr $COUNT_FAIL + 1)
+ FAILURE_LIST+=($COUNT_SINCE_FAIL)
+ COUNT_SINCE_FAIL=0
+ DUMP_FROM=$RUN_TS ./voltha dump
+ ./voltha down
+ fi
+ echo "END RUN: $RUN @ $(date -u +%Y-%m-%dT%H:%M:%SZ)" | tee -a $LOG
+ COMPLETED=$(expr $COMPLETED + 1)
+ MTBF=0
+ if [ ${#FAILURE_LIST[@]} -gt 0 ]; then
+ for V in ${FAILURE_LIST[@]}; do
+ MTBF=$(expr $MTBF + $V)
+ done
+ MTBF=$(expr $MTBF \/ ${#FAILURE_LIST[@]})
+ fi
+
+ RSS_DIFFS+=($(delta $AFTER_RSS $BEFORE_RSS))
+ KEY_DIFFS+=($(delta $AFTER_KEY_COUNT $BEFORE_KEY_COUNT))
+ SIZE_DIFFS+=($(delta $AFTER_SIZE $BEFORE_SIZE))
+
+ echo "{" | tee -a $LOG
+ echo " NumberOfRuns: $COMPLETED," | tee -a $LOG
+ echo " Success: $COUNT_OK," | tee -a $LOG
+ echo " Failed: $COUNT_FAIL," | tee -a $LOG
+ echo " SinceLastFail: $COUNT_SINCE_FAIL," | tee -a $LOG
+ echo " MTBF: $MTBF," | tee -a $LOG
+ echo " FAILURES: ${FAILURE_LIST[@]}," | tee -a $LOG
+ echo " ETCd: {" | tee -a $LOG
+ echo " KeyCount: {" | tee -a $LOG
+ echo " Before: $BEFORE_KEY_COUNT," | tee -a $LOG
+ echo " After: $AFTER_KEY_COUNT," | tee -a $LOG
+ echo " Average: $(average ${KEY_DIFFS[@]})," | tee -a $LOG
+ echo " }," | tee -a $LOG
+ echo " DbSize: {" | tee -a $LOG
+ echo " Before: $BEFORE_SIZE," | tee -a $LOG
+ echo " After: $AFTER_SIZE," | tee -a $LOG
+ echo " Average: $(numfmt --to=iec $(average ${SIZE_DIFFS[@]}))," | tee -a $LOG
+ echo " }" | tee -a $LOG
+ echo " RSS: {" | tee -a $LOG
+ echo " Before: $BEFORE_RSS," | tee -a $LOG
+ echo " After: $AFTER_RSS," | tee -a $LOG
+ echo " Average: $(numfmt --to=iec $(average ${RSS_DIFFS[@]}))," | tee -a $LOG
+ echo " }" | tee -a $LOG
+ echo " }" | tee -a $LOG
+ echo "}" | tee -a $LOG
+ if [ $FAIL -ne 0 ]; then
+ mkdir -p failures
+ cp $LOG failures/$(date -u +"%Y%m%dT%H%M%SZ")-fail-output.log
+ fi
+done