blob: 2cd87b70625591bf43afba23102badb211a7343c [file] [log] [blame]
#!/bin/bash
# Copyright 2019 Ciena Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# this script repeatedly invokes the e2e system test on a VOLTHA instance
set -o pipefail
if [ ! -r voltha-system-tests ]; then
git clone http://gerrit.opencord.org/voltha-system-tests voltha-system-tests
fi
delta() {
local LEFT=$(echo $1 | numfmt --from=iec)
local RIGHT=$(echo $2 | numfmt --from=iec)
local V=$(expr $LEFT - $RIGHT)
echo ${V#-}
}
average() {
local MIN=0
local MAX=0
local COUNT=0
local SUM=0
for V in $*; do
COUNT=$(expr $COUNT + 1)
SUM=$(expr $SUM + $V)
if [ $COUNT -eq 1 ]; then
MIN=$V
MAX=$V
else
if [ $V -lt $MIN ]; then
MIN=$V
elif [ $V -gt $MAX ]; then
MAX=$V
fi
fi
done
if [ $COUNT -gt 3 ]; then
SUM=$(expr $SUM - $MIN - $MAX)
COUNT=$(expr $COUNT - 2)
fi
if [ $COUNT -lt 1 ]; then
echo 0
else
echo $(expr $SUM / $COUNT)
fi
}
export WAIT_FOR_DOWN=y
export TERM=
COMPLETED=0
COUNT_OK=0
COUNT_FAIL=0
COUNT_SINCE_FAIL=0
FAILURE_LIST=()
RSS_DIFFS=()
KEY_DIFFS=()
SIZE_DIFFS=()
LOG=$(mktemp)
while true; do
RUN_TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)
echo "START RUN $RUN @ $RUN_TS" | tee -a $LOG
DEPLOY_K8S=y WITH_BBSIM=y ./voltha up
# because BBSIM needs time
sleep 60
ETCD=$(kubectl -n voltha get pods | grep etcd-cluster | awk '{print $1}')
BEFORE_KEY_COUNT=$(kubectl -n voltha exec -ti $ETCD \
-- sh -c 'ETCDCTL_API=3 etcdctl get --command-timeout=60s --from-key=true --keys-only . | sed -e "/^$/d" | wc -l | tr -d "\r\n"')
BEFORE_SIZE=$(numfmt --to=iec \
$(kubectl -n voltha exec -ti $ETCD \
-- sh -c 'ETCDCTL_API=3 etcdctl endpoint status -w json' | tr -d '\r\n' | jq .[].Status.dbSize))
BEFORE_RSS=$(ps -eo rss,pid,cmd | grep /usr/local/bin/etcd | grep -v grep | cut -d\ -f1 | numfmt --to=iec)
(cd voltha-system-tests; make sanity-kind 2>&1 | tee $LOG)
FAIL=$?
AFTER_KEY_COUNT=$(kubectl -n voltha exec -ti $ETCD \
-- sh -c 'ETCDCTL_API=3 etcdctl get --command-timeout=60s --from-key=true --keys-only . | sed -e "/^$/d" | wc -l | tr -d "\r\n"')
AFTER_SIZE=$(numfmt --to=iec \
$(kubectl -n voltha exec -ti $ETCD \
-- sh -c 'ETCDCTL_API=3 etcdctl endpoint status -w json' | tr -d '\r\n' | jq .[].Status.dbSize))
AFTER_RSS=$(ps -eo rss,pid,cmd | grep /usr/local/bin/etcd | grep -v grep | cut -d\ -f1 | numfmt --to=iec)
if [ $FAIL -eq 0 ]; then
COUNT_OK=$(expr $COUNT_OK + 1)
COUNT_SINCE_FAIL=$(expr $COUNT_SINCE_FAIL + 1)
helm delete --purge bbsim
while [ $(kubectl get --all-namespaces pods,svc 2>&1 | grep -c bbsim) -gt 0 ]; do
sleep 3
done
else
COUNT_FAIL=$(expr $COUNT_FAIL + 1)
FAILURE_LIST+=($COUNT_SINCE_FAIL)
COUNT_SINCE_FAIL=0
DUMP_FROM=$RUN_TS ./voltha dump
./voltha down
fi
echo "END RUN: $RUN @ $(date -u +%Y-%m-%dT%H:%M:%SZ)" | tee -a $LOG
COMPLETED=$(expr $COMPLETED + 1)
MTBF=0
if [ ${#FAILURE_LIST[@]} -gt 0 ]; then
for V in ${FAILURE_LIST[@]}; do
MTBF=$(expr $MTBF + $V)
done
MTBF=$(expr $MTBF \/ ${#FAILURE_LIST[@]})
fi
RSS_DIFFS+=($(delta $AFTER_RSS $BEFORE_RSS))
KEY_DIFFS+=($(delta $AFTER_KEY_COUNT $BEFORE_KEY_COUNT))
SIZE_DIFFS+=($(delta $AFTER_SIZE $BEFORE_SIZE))
echo "{" | tee -a $LOG
echo " NumberOfRuns: $COMPLETED," | tee -a $LOG
echo " Success: $COUNT_OK," | tee -a $LOG
echo " Failed: $COUNT_FAIL," | tee -a $LOG
echo " SinceLastFail: $COUNT_SINCE_FAIL," | tee -a $LOG
echo " MTBF: $MTBF," | tee -a $LOG
echo " FAILURES: ${FAILURE_LIST[@]}," | tee -a $LOG
echo " ETCd: {" | tee -a $LOG
echo " KeyCount: {" | tee -a $LOG
echo " Before: $BEFORE_KEY_COUNT," | tee -a $LOG
echo " After: $AFTER_KEY_COUNT," | tee -a $LOG
echo " Average: $(average ${KEY_DIFFS[@]})," | tee -a $LOG
echo " }," | tee -a $LOG
echo " DbSize: {" | tee -a $LOG
echo " Before: $BEFORE_SIZE," | tee -a $LOG
echo " After: $AFTER_SIZE," | tee -a $LOG
echo " Average: $(numfmt --to=iec $(average ${SIZE_DIFFS[@]}))," | tee -a $LOG
echo " }" | tee -a $LOG
echo " RSS: {" | tee -a $LOG
echo " Before: $BEFORE_RSS," | tee -a $LOG
echo " After: $AFTER_RSS," | tee -a $LOG
echo " Average: $(numfmt --to=iec $(average ${RSS_DIFFS[@]}))," | tee -a $LOG
echo " }" | tee -a $LOG
echo " }" | tee -a $LOG
echo "}" | tee -a $LOG
if [ $FAIL -ne 0 ]; then
mkdir -p failures
cp $LOG failures/$(date -u +"%Y%m%dT%H%M%SZ")-fail-output.log
fi
done