voltha

#!/bin/bash
# Copyright 2019 Ciena Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# trap ctrl-c and call ctrl_c()
trap ctrl_c INT

function ctrl_c() {
    tput cnorm
    echo ""
    echo "ctrl-c trapped"
    echo "Thank you for trying 'votlha up'"
    exit
}

VOLTCTL_VERSION=${VOLTCTL_VERSION:-0.0.5-dev}
KIND_VERSION=${KIND_VERSION:-v0.4.0}
_VOLTCTL_VERSION=$(echo $VOLTCTL_VERSION | sed -e 's/-/_/g')

BLACK=$(tput setaf 0)
RED=$(tput setaf 1)
GREEN=$(tput setaf 2)
YELLOW=$(tput setaf 3)
BLUE=$(tput setaf 4)
MAGENTA=$(tput setaf 5)
CYAN=$(tput setaf 6)
WHITE=$(tput setaf 7)
BOLD=$(tput bold)
NORMAL=$(tput sgr0)
ERROR="\xe2\x9c\x97\x20"
CEOL=$(tput el)

TYPE=${TYPE:-minimal}
NAME=${NAME:-$TYPE}
WITH_BBSIM=${WITH_BBSIM:-no}
WITH_RADIUS=${WITH_RADIUS:-no}
WITH_ONOS=${WITH_ONOS:-yes}
WITH_TP=${WITH_TP:-yes}
JUST_K8S=${JUST_K8S:-no}
DEPLOY_K8S=${DEPLOY_K8S:-yes}
SKIP_RESTART_API=${SKIP_RESTART_API:-no}
INSTALL_KUBECTL=${INSTALL_KUBECTL:-yes}
INSTALL_HELM=${INSTALL_HELM:-yes}
USE_GO=${USE_GO:-yes}
VOLTHA_LOG_LEVEL=${VOLTHA_LOG_LEVEL:-WARN}
VOLTHA_CHART=${VOLTHA_CHART:=onf/voltha}
VOLTHA_ADAPTER_SIM_CHART=${VOLTHA_ADAPTER_SIM_CHART:-onf/voltha-adapter-simulated}
VOLTHA_ADAPTER_OPEN_OLT_CHART=${VOLTHA_ADAPTER_OPEN_OLT_CHART:-onf/voltha-adapter-openolt}
VOLTHA_ADAPTER_OPEN_ONU_CHART=${VOLTHA_ADAPTER_OPEN_ONU_CHART:-onf/voltha-adapter-openonu}

HAVE_GO=$(which go >/dev/null 2>&1 && echo "yes" || echo "no")
HOSTOS=$(uname -s | tr "[:upper:]" "[:lower:"])
HOSTARCH=$(uname -m | tr "[:upper:]" "[:lower:"])
if [ $HOSTARCH == "x86_64" ]; then
    HOSTARCH="amd64"
fi

# Verify TYPE setting
if [ $(echo ":minimal:full:" | grep -ic ":$TYPE:") -eq 0 ]; then
    >&2 echo -e "${RED}${BOLD}${ERROR}ERROR:${NORMAL}${RED} Invalid \$TYPE value of '$TYPE'. Should be 'minimal' or 'full'${NORMAL}"
    exit 1
fi

# Verify WITH_BBSIM settting and convert uniform value of yes or no
if [ $(echo ":y:yes:true:n:no:false:1:0:" | grep -ic ":$WITH_BBSIM:") -eq 0 ]; then
    >&2 echo -e "${RED}${BOLD}${ERROR}ERROR:${NORMAL}${RED} Invalid \$WITH_BBSIM value of '$WITH_BBSIM'. Should be 'yes' or 'no'${NORMAL}"
    exit 1
fi
if [ $(echo ":y:yes:true:1:" | grep -ic ":$WITH_BBSIM:") -eq 1 ]; then
    WITH_BBSIM=yes
else
    WITH_BBSIM=no
fi

# Verify WITH_RADIUS settting and convert uniform value of yes or no
if [ $(echo ":y:yes:true:n:no:false:1:0:" | grep -ic ":$WITH_RADIUS:") -eq 0 ]; then
    >&2 echo -e "${RED}${BOLD}${ERROR}ERROR:${NORMAL}${RED} Invalid \$WITH_RADIUS value of '$WITH_RADIUS'. Should be 'yes' or 'no'${NORMAL}"
    exit 1
fi
if [ $(echo ":y:yes:true:1:" | grep -ic ":$WITH_RADIUS:") -eq 1 ]; then
    WITH_RADIUS=yes
else
    WITH_RADIUS=no
fi

# Verify WITH_TP settting and convert uniform value of yes or no
if [ $(echo ":y:yes:true:n:no:false:1:0:" | grep -ic ":$WITH_TP:") -eq 0 ]; then
    >&2 echo -e "${RED}${BOLD}${ERROR}ERROR:${NORMAL}${RED} Invalid \$WITH_TP value of '$WITH_TP'. Should be 'yes' or 'no'${NORMAL}"
    exit 1
fi
if [ $(echo ":y:yes:true:1:" | grep -ic ":$WITH_TP:") -eq 1 ]; then
    WITH_TP=yes
else
    WITH_TP=no
fi

# Verify WITH_ONOS settting and convert uniform value of yes or no
if [ $(echo ":y:yes:true:n:no:false:1:0:" | grep -ic ":$WITH_ONOS:") -eq 0 ]; then
    >&2 echo -e "${RED}${BOLD}${ERROR}ERROR:${NORMAL}${RED} Invalid \$WITH_ONOS value of '$WITH_ONOS'. Should be 'yes' or 'no'${NORMAL}"
    exit 1
fi
if [ $(echo ":y:yes:true:1:" | grep -ic ":$WITH_ONOS:") -eq 1 ]; then
    WITH_ONOS=yes
else
    WITH_ONOS=no
fi

# Verify JUST_K8S settting and convert uniform value of yes or no
if [ $(echo ":y:yes:true:n:no:false:1:0:" | grep -ic ":$JUST_K8S:") -eq 0 ]; then
    >&2 echo -e "${RED}${BOLD}${ERROR}ERROR:${NORMAL}${RED} Invalid \$JUST_K8S value of '$JUST_K8S'. Should be 'yes' or 'no'${NORMAL}"
    exit 1
fi
if [ $(echo ":y:yes:true:1:" | grep -ic ":$JUST_K8S:") -eq 1 ]; then
    JUST_K8S=yes
else
    JUST_K8S=no
fi

# Verify DEPLOY_K8S settting and convert uniform value of yes or no
if [ $(echo ":y:yes:true:n:no:false:1:0:" | grep -ic ":$DEPLOY_K8S:") -eq 0 ]; then
    >&2 echo -e "${RED}${BOLD}${ERROR}ERROR:${NORMAL}${RED} Invalid \$DEPLOY_K8S value of '$DEPLOY_K8S'. Should be 'yes' or 'no'${NORMAL}"
    exit 1
fi
if [ $(echo ":y:yes:true:1:" | grep -ic ":$DEPLOY_K8S:") -eq 1 ]; then
    DEPLOY_K8S=yes
else
    DEPLOY_K8S=no
fi

# Verify INSTALL_KUBECTL settting and convert uniform value of yes or no
if [ $(echo ":y:yes:true:n:no:false:1:0:" | grep -ic ":$INSTALL_KUBECTL:") -eq 0 ]; then
    >&2 echo -e "${RED}${BOLD}${ERROR}ERROR:${NORMAL}${RED} Invalid \$INSTALL_KUBECTL value of '$INSTALL_KUBECTL'. Should be 'yes' or 'no'${NORMAL}"
    exit 1
fi

if [ $(echo ":y:yes:true:1:" | grep -ic ":$INSTALL_KUBECTL:") -eq 1 ]; then
    INSTALL_KUBECTL=yes
else
    INSTALL_KUBECTL=no
fi

# Verify SKIP_RESTART_API settting and convert uniform value of yes or no
if [ $(echo ":y:yes:true:n:no:false:1:0:" | grep -ic ":$SKIP_RESTART_API:") -eq 0 ]; then
    >&2 echo -e "${RED}${BOLD}${ERROR}ERROR:${NORMAL}${RED} Invalid \$SKIP_RESTART_API value of '$SKIP_RESTART_API'. Should be 'yes' or 'no'${NORMAL}"
    exit 1
fi
if [ $(echo ":y:yes:true:1:" | grep -ic ":$SKIP_RESTART_API:") -eq 1 ]; then
    SKIP_RESTART_API=yes
else
    SKIP_RESTART_API=no
fi

# Verify USE_GO settting and convert uniform value of yes or no
if [ $(echo ":y:yes:true:n:no:false:1:0:" | grep -ic ":$USE_GO:") -eq 0 ]; then
    >&2 echo -e "${RED}${BOLD}${ERROR}ERROR:${NORMAL}${RED} Invalid \$USE_GO value of '$USE_GO'. Should be 'yes' or 'no'${NORMAL}"
    exit 1
fi
if [ $(echo ":y:yes:true:1:" | grep -ic ":$USE_GO:") -eq 1 ]; then
    USE_GO=true
else
    USE_GO=false
fi

# Verify INSTALL_HELM settting and convert uniform value of yes or no
if [ $(echo ":y:yes:true:n:no:false:1:0:" | grep -ic ":$INSTALL_HELM:") -eq 0 ]; then
    >&2 echo -e "${RED}${BOLD}${ERROR}ERROR:${NORMAL}${RED} Invalid \$INSTALL_HELM value of '$INSTALL_HELM'. Should be 'yes' or 'no'${NORMAL}"
    exit 1
fi

if [ $(echo ":y:yes:true:1:" | grep -ic ":$INSTALL_HELM:") -eq 1 ]; then
    INSTALL_HELM=yes
else
    INSTALL_HELM=no
fi

mkdir -p .voltha
touch .voltha/ports
HAVE=$(grep $NAME .voltha/ports)
if [ "$HAVE X" == " X" ]; then
    # Find free port prefix
    START=81
    while true; do
        if [ $(grep -c $START .voltha/ports) -eq 0 ]; then
            break
        fi
        START=$(expr $START + 1)
    done
    DELTA=$(expr $START - 81)
    ONOS_API_PORT=${START}81
    ONOS_SSH_PORT=${START}01
    VOLTHA_API_PORT=5$(expr 55 + $DELTA)55
    VOLTHA_SSH_PORT=$(expr 50 + $DELTA)22
    VOLTHA_ETCD_PORT=$(expr 23 + $DELTA)79
else
    VALUES=$(echo $HAVE | sed -e 's/\s//g' | cut -d= -f2)
    ONOS_API_PORT=$(echo $VALUES | cut -d, -f1)
    ONOS_SSH_PORT=$(echo $VALUES | cut -d, -f2)
    VOLTHA_API_PORT=$(echo $VALUES | cut -d, -f3)
    VOLTHA_SSH_PORT=$(echo $VALUES | cut -d, -f4)
    VOLTHA_ETCD_PORT=$(echo $VALUES | cut -d, -f5)
fi

PORTTMP=$(mktemp -u)
cat .voltha/ports | grep -v $NAME > $PORTTMP
echo "$NAME=$ONOS_API_PORT,$ONOS_SSH_PORT,$VOLTHA_API_PORT,$VOLTHA_SSH_PORT,$VOLTHA_ETCD_PORT" >> $PORTTMP
cp $PORTTMP .voltha/ports
rm -f $PORTTMP

if [ "$WITH_TP" == "yes" ]; then
    SADIS_VER=3.1.0
    OLT_VER=3.0.1
    AAA_VER=1.9.0
    DHCP_VER=1.6.0
else
    SADIS_VER=2.2.0
    OLT_VER=2.1.0
    AAA_VER=1.8.0
    DHCP_VER=1.5.0
fi
export ONOS_API_PORT ONOS_SSH_PORT SADIS_VER OLT_VER AAA_VER DHCP_VER

spin() {
    PARTS="\
        \xe2\xa2\x8e\xe2\xa1\xb0 \
        \xe2\xa2\x8e\xe2\xa1\xa1 \
        \xe2\xa2\x8e\xe2\xa1\x91 \
        \xe2\xa2\x8e\xe2\xa0\xb1 \
        \xe2\xa0\x8e\xe2\xa1\xb1 \
        \xe2\xa2\x8a\xe2\xa1\xb1 \
        \xe2\xa2\x8c\xe2\xa1\xb1 \
        \xe2\xa2\x86\xe2\xa1\xb1 \
        "
    IDX=1
    tput civis
    while true; do
       C=$(echo $PARTS | cut '-d ' -f $IDX)
       echo -en "$C"
       IDX=$(expr $IDX + 1)
       if [ $IDX -gt 8 ]; then
           IDX=1
       fi
       sleep .15 
       echo -en "\r"
    done
}

SPIN_PARTS="\
        \xe2\xa2\x8e\xe2\xa1\xb0 \
        \xe2\xa2\x8e\xe2\xa1\xa1 \
        \xe2\xa2\x8e\xe2\xa1\x91 \
        \xe2\xa2\x8e\xe2\xa0\xb1 \
        \xe2\xa0\x8e\xe2\xa1\xb1 \
        \xe2\xa2\x8a\xe2\xa1\xb1 \
        \xe2\xa2\x8c\xe2\xa1\xb1 \
        \xe2\xa2\x86\xe2\xa1\xb1 \
        "
IDX=1
NOT_VERIFIED="\xe2\x9c\x97\x20"
VERIFIED="\xe2\x9c\x93\x20"
HELM="\xE2\x8E\x88"
OLD_KEY="\xF0\x9F\x97\x9D"
BIRD="\xF0\x9F\x90\xA6"
HIGH_VOLTAGE="\xE2\x9A\xA1"
PLUG="\xF0\x9F\xa7\xa9"
RESTART="\xf0\x9f\x94\x84"
FORWARD="\xE2\x87\xA8"
INSTALL="\xF0\x9F\x8F\x97"
STOP="\xf0\x9f\x9b\x91"
GO="\xf0\x9f\x9a\x80"
DOWNLOAD="\xf0\x9f\x93\xa5"
GEAR="\xe2\x9a\x99"
NO_ENTRY="\xe2\x9b\x94"
LOCK="\xf0\x9f\x94\x92"

bspin() {
    tput civis
    IDX=1
    local INDENT=
    if [ "$1" == "-" ]; then
        INDENT="  "
        shift
    fi
    echo -en "$INDENT   $*"
}

sspin() {
    local INDENT=
    if [ "$1" == "-" ]; then
        INDENT="  "
        shift
    fi
    C=$(echo $SPIN_PARTS | cut '-d ' -f $IDX)
    echo -en "\r$INDENT$C $*"
    IDX=$(expr $IDX + 1)
    if [ $IDX -gt 8 ]; then
        IDX=1
    fi
}

espin() {
    local INDENT=
    if [ "$1" == "-" ]; then
        INDENT="  "
        shift
    fi
    echo -e "\r$INDENT$*"
    tput cnorm
}

if [ "$1" == "get" -a "$2" == "voltconfig" ]; then
    echo "$HOME/.volt/config-$NAME"
    exit
fi

if [ $# -ne 1 -o $(echo ":up:down:dump:" | grep -c ":$1:") -ne 1 ]; then 
    >&2 echo "What wouild you like to do today:"
    >&2 echo "  up   - bring up voltha"
    >&2 echo "  down - tear down voltha"
    >&2 echo "  dump - create a debug dump of running system"
    exit 1
fi

if [ "$1" == "down" ]; then
    echo "Tearing down voltha cluster $NAME"
    LOG="down-$NAME.log"
    echo $(date -u +"%Y%m%dT%H%M%SZ") >$LOG
    if [ $DEPLOY_K8S == "yes" ]; then
        if [ -x ./bin/kind ]; then
            bspin "Delete Kubernetes Kind Cluster"
            (set -x; ./bin/kind delete cluster --name voltha-$NAME >>$LOG 2>&1) >>$LOG 2>&1
            espin $VERIFIED
        else
            espin "$NO_ENTRY Delete Kubernetes Kind Cluster: kind command not found"
        fi
    else
        EXISTS=$(helm list -q)
        EXPECT="etcd-operator onos open-olt open-onu sim voltha bbsim radius"
        bspin "Remove Helm Deployments"
        for i in $EXISTS; do
            if [ $(echo $EXPECT | grep -c $i) -eq 1 ]; then
                sspin "Remove Helm Deployments: $i$CEOL"
                (set -x; ./bin/helm delete --purge $i >>$LOG 2>&1) >>$LOG 2>&1
            fi
        done
        espin "$VERIFIED Remove Helm Deployments$CEOL"
    fi
    bspin "Remove port-forwards: onos-ui-$NAME"
    for i in $(screen -ls | grep onos-ui-$NAME | awk '{print $1}'); do
        sspin
        (set -x; screen -X -S $i quit >>$LOG 2>&1) >>$LOG 2>&1
    done
    sspin "Remove port-forwards: onos-ssh-$NAME$CEOL"
    for i in $(screen -ls | grep onos-ssh-$NAME | awk '{print $1}'); do
        sspin
        (set -x; screen -X -S $i quit >>$LOG 2>&1) >>$LOG 2>&1
    done
    sspin "Remove port-forwards: voltha-api-$NAME$CEOL"
    for i in $(screen -ls | grep voltha-api-$NAME | awk '{print $1}'); do
        sspin
        (set -x; screen -X -S $i quit >>$LOG 2>&1) >>$LOG 2>&1
    done
    sspin "Remove port-forwards: voltha-ssh-$NAME$CEOL"
    for i in $(screen -ls | grep voltha-ssh-$NAME | awk '{print $1}'); do
        sspin
        (set -x; screen -X -S $i quit >>$LOG 2>&1) >>$LOG 2>&1
    done
    sspin "Remove port-forwards: voltha-etcd-$NAME$CEOL"
    for i in $(screen -ls | grep voltha-etcd-$NAME | awk '{print $1}'); do
        sspin
        (set -x; screen -X -S $i quit >>$LOG 2>&1) >>$LOG 2>&1
    done
    espin "$VERIFIED Remove port-forwards$CEOL"
    exit 
fi

if [ "$1" == "dump" ]; then
    LOG="dump-$NAME.log"
    TS=$(date -u +"%Y%m%dT%H%M%SZ")
    WORK=$(mktemp -u -d)
    DATA=$WORK/voltha-debug-dump-$NAME-$TS
    mkdir -p $DATA
    echo $TS > $LOG
    echo -e "Capturing debug dump to voltha-debug-dump-$NAME-$TS.tgz"
    bspin - "Copy install log"
    if [ -f install-$NAME.log ]; then
        (set -x; cp install-$NAME.log $DATA/install-$NAME.log) >>$LOG 2>&1
        espin - $VERIFIED
    else
        espin - "$NO_ENTRY Copy install log: install-$NAME.log not found"
    fi
    bspin - "Dumping Kubernetes PODs"
    (set -x; kubectl get --all-namespaces pods >> $DATA/all-pods.txt 2>&1) >>$LOG 2>&1
    espin - $VERIFIED
    bspin - "Dumping Kubernetes SERVICEs"
    (set -x; kubectl get --all-namespaces svc >> $DATA/all-services.txt 2>&1) >>$LOG 2>&1
    espin - $VERIFIED
    bspin - "Dumping Kubernetes EVENTs"
    (set -x; kubectl get --all-namespaces events >> $DATA/all-events.txt 2>&1) >>$LOG 2>&1
    espin - $VERIFIED
    bspin - "Dumping VOLTHA POD details"
    PODS=$(kubectl get -n voltha pod -o name)
    for POD in $PODS; do
        sspin - "Dumping VOLTHA POD details: $POD$CEOL"
        mkdir -p $DATA/$POD
        (set -x; kubectl describe -n voltha $POD >> $DATA/$POD/describe.txt 2>&1) >>$LOG 2>&1
        sspin - "Dumping VOLTHA POD details: $POD"
        (set -x; kubectl logs -n voltha --all-containers --previous $LOG_ARGS $POD >> $DATA/$POD/logs-previous.txt 2>&1) >>$LOG 2>&1
        sspin - "Dumping VOLTHA POD details: $POD"
        (set -x; kubectl logs -n voltha --all-containers $LOG_ARGS $POD >> $DATA/$POD/logs-current.txt 2>&1) >>$LOG 2>&1
        sspin - "Dumping VOLTHA POD details: $POD"
    done
    espin - "$VERIFIED Dumping VOLTHA POD details$CEOL"
    bspin - "Dumping ETCD"
    if [ "$(which etcdctl) X" != " X" ]; then
        (set -x; ETCDCTL_API=3 etcdctl --endpoints localhost:$VOLTHA_ETCD_PORT get --prefix service/voltha | hexdump -C >> $DATA/etcd.hex 2>&1) >>$LOG 2>&1
        espin - $VERIFIED
    else
        espin - "$NO_ENTRY Dumping ETCD: etcdctl command not available"
    fi
    bspin - "Creating compressed TAR: voltha-debug-dump-$NAME-$TS.tgz"
    (set -x; tar -C $WORK -zcf voltha-debug-dump-$NAME-$TS.tgz ./voltha-debug-dump-$NAME-$TS) >>$LOG 2>&1
    espin - $VERIFIED
    bspin - "Cleanup"
    (set -x; rm -rf $WORK) >>$LOG 2>&1
    espin - $VERIFIED
    bspin - "$(ls -l voltha-debug-dump-$NAME-$TS.tgz)"
    espin - $VERIFIED
    exit
fi


LOG="install-$NAME.log"
date > $LOG
echo "PORTS=$ONOS_API_PORT,$ONOS_SSH_PORT,$VOLTHA_API_PORT,$VOLTHA_SSH_PORT,$VOLTHA_ETCD_PORT" >> $LOG

# Output install options to log
echo "OPTIONS" >> $LOG
ALL_OPTIONS="NAME TYPE WITH_BBSIM WITH_RADIUS WITH_ONOS WITH_TP JUST_K8S DEPLOY_K8S \
    SKIP_RESTART_API INSTALL_KUBECTL INSTALL_HELM USE_GO VOLTHA_LOG_LEVEL \
    VOLTHA_CHART VOLTHA_ADAPTER_SIM_CHART VOLTHA_ADAPTER_OPEN_OLT_CHART \
    VOLTHA_ADAPTER_OPEN_ONU_CHART SADIS_VER OLT_VER AAA_VER DHCP_VER \
    ONOS_API_PORT ONOS_SSH_PORT VOLTHA_API_PORT VOLTHA_SSH_PORT VOLTHA_ETCD_PORT"
for O in $ALL_OPTIONS; do
    VAL=$(eval echo \$$O)
    if [ $O == "USE_GO" ]; then
        VAL="$(echo $VAL| test $(grep -c true) -eq 1 && echo yes || echo no)"
    fi
    if [ ! -z "$VAL" ]; then
        printf "    %-30s = %s\n" $O $VAL >> $LOG
    fi
done


count_pods() {
    local NAMESPACE=$1; shift
    local PODS=$(kubectl -n $NAMESPACE get pod -o go-template="{{range .items}}{{.metadata.name}}/{{.status.phase}}/_{{range .status.containerStatuses}}{{.ready}}_{{end}} {{end}}")
    local COUNT=0
    local PATTERNS=$*
    for POD in $PODS; do
        local NAME=$(echo $POD | cut -d/ -f 1)
        local STATE=$(echo $POD | cut -d/ -f 2)
        local CONTAINERS=$(echo $POD | cut -d/ -f 3 | sed -e 's/_/ /g')
        if [ "$STATE" == "Running" ]; then
            local TOTAL=$(echo $CONTAINERS | wc -w)
            local FOUND=$(echo $CONTAINERS | grep -o true | wc -l)
            if [ $TOTAL -eq $FOUND ]; then
                for PATTERN in $PATTERNS; do
                    if [[ $NAME =~ $PATTERN ]]; then
                        COUNT=$(expr $COUNT + 1)
                    fi
                done
            fi
        fi
    done
    echo $COUNT
}

wait_for_pods() {
    local INDENT=
    if [ "$1" == "-" ]; then
        INDENT=$1; shift
    fi
    local NAMESPACE=$1; shift
    local EXPECT=$1; shift
    local RETRY=$1; shift
    local MESSAGE=$1; shift
    local PATTERNS=$*
    local HAVE=$(count_pods $NAMESPACE $PATTERNS)
    COUNT=$(expr 300 / 15)
    bspin $INDENT $MESSAGE
    sspin $INDENT
    if [ $HAVE -ne $EXPECT ]; then
        while [ $HAVE -ne $EXPECT ]; do
            sspin $INDENT
            COUNT=$(expr $COUNT - 1)
            if [ $COUNT -eq 0 ]; then
                HAVE=$(count_pods $NAMESPACE $PATTERNS)
                COUNT=$(expr 300 / 15)
            fi
            sleep .15
        done
    fi
    espin $INDENT $VERIFIED
    if [ $HAVE -ne $EXPECT ]; then
        return 1
    fi
    return 0
}

helm_install() { 
    local INDENT=
    if [ "$1" == "-" ]; then
        INDENT=$1; shift
    fi
    local NAMESPACE=$1; shift
    local INAME=$1; shift
    local CHART=$1; shift
    local MESSAGE=$*

    COUNT=$(expr 300 / 15)
    bspin $INDENT $MESSAGE
    (set -x; helm install -f $NAME-values.yaml --set use_go=$USE_GO --set defaults.log_level=$VOLTHA_LOG_LEVEL --namespace $NAMESPACE --name $INAME $EXTRA_HELM_FLAGS $CHART >>$LOG 2>&1) >>$LOG 2>&1
    SUCCESS=$?
    while [ $SUCCESS -ne 0 ]; do
        sspin $INDENT
        COUNT=$(expr $COUNT - 1)
        if [ $COUNT -eq 0 ]; then
            (set -x; helm install -f $NAME-values.yaml --set use_go=$USE_GO --set defaults.log_level=$VOLTHA_LOG_LEVEL --namespace $NAMESPACE --name $INAME $EXTRA_HELM_FLAGS $CHART >>$LOG 2>&1) >>$LOG 2>&1
            COUNT=$(expr 300 / 15)
        fi
        sleep .15
    done
    espin $INDENT $VERIFIED
}

echo "INSTALL TYPE: $TYPE" >> $LOG

bspin "Verify GOPATH"
export GOPATH=$(pwd)
mkdir -p $GOPATH/bin
espin $VERIFIED

if [ "$INSTALL_KUBECTL" == "no" ]; then
    bspin "Skip kubectl install"
    espin $NO_ENTRY
else
    bspin "Verify kubectl $HELM"
    if [ -x $GOPATH/bin/kubectl ]; then
        espin $VERIFIED
    else
        espin $NOT_VERIFIED
        bspin - "Download and install Kubernetes/kubectl $DOWNLOAD"
        (set -x; curl -o $GOPATH/bin/kubectl -sSL https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/$HOSTOS/$HOSTARCH/kubectl >>$LOG 2>&1) >>$LOG 2>&1
        (set -x; chmod 755 $GOPATH/bin/kubectl >>$LOG 2>&1) >>$LOG 2>&1
        espin - $VERIFIED
    fi
fi

if [ "$DEPLOY_K8S" == "no" ]; then
    bspin "Skip Kubernetes/Kind Deployment"
    espin $NO_ENTRY
else
    bspin "Verify Kubernetes/Kind $HELM"
    if [ -x $GOPATH/bin/kind ]; then
        espin $VERIFIED
    else
        espin $NOT_VERIFIED
        bspin - "Download and install Kubernetes/kind $DOWNLOAD"
        (set -x; curl -o $GOPATH/bin/kind -sSL https://github.com/kubernetes-sigs/kind/releases/download/$KIND_VERSION/kind-$HOSTOS-$HOSTARCH >>$LOG 2>&1) >>$LOG 2>&1
        (set -x; chmod 755 $GOPATH/bin/kind >>$LOG 2>&1) >>$LOG 2>&1
        espin - $VERIFIED
    fi
fi

if [ "$INSTALL_HELM" == "no" ]; then
    bspin "Skip Helm Install"
    espin $NO_ENTRY
else
    bspin "Verify Helm $HELM"
    if [ -x $GOPATH/bin/helm ]; then
        espin $VERIFIED
    else
        espin $NOT_VERIFIED
        bspin - "Download and install Helm $DOWNLOAD"
        (set -x; curl -sSL https://git.io/get_helm.sh | USE_SUDO=false HELM_INSTALL_DIR=$GOPATH/bin bash >>$LOG 2>&1) >>$LOG 2>&1
        espin - $VERIFIED
    fi
fi

bspin "Verify voltctl $HIGH_VOLTAGE"
if [ -x $GOPATH/bin/voltctl ]; then
    espin $VERIFIED
else
    espin $NOT_VERIFIED
    bspin - "Download and build voltctl $DOWNLOAD"
    (set -x; curl -o $GOPATH/bin/voltctl -sSL https://github.com/ciena/voltctl/releases/download/$VOLTCTL_VERSION/voltctl-$_VOLTCTL_VERSION-$HOSTOS-$HOSTARCH >>$LOG 2>&1) >>$LOG 2>&1
    (set -x; chmod 755 $GOPATH/bin/voltctl >>$LOG 2>&1) >>$LOG 2>&1
    espin - $VERIFIED
fi

bspin "Verify command PATH"
export PATH=$GOPATH/bin:$PATH
espin $VERIFIED

if [ "$DEPLOY_K8S" == "yes" ]; then
    HAVE=$(kind get clusters | grep -c voltha-$NAME)
    bspin "Verify Kubernetes/Kind Cluster"
    sspin
    if [ $HAVE -eq 0 ]; then
        espin $NOT_VERIFIED
        bspin - "Verify cluster configuration"
        if [ ! -r ./$NAME-cluster.cfg ]; then
            espin - $NOT_VERIFIED
	        bspin - "Download cluster configuration: $TYPE-cluster.cfg to $NAME-cluster.cfg $DOWNLOAD"
            (set -x; curl -o ./$NAME-cluster.cfg -sSL https://raw.githubusercontent.com/ciena/kind-voltha/master/$TYPE-cluster.cfg >>$LOG 2>&1) >>$LOG 2>&1
	        espin - $VERIFIED
	    else
	        espin - $VERIFIED
	    fi
        kind create cluster --name voltha-$NAME --config $NAME-cluster.cfg
     else
        espin $VERIFIED
    fi

    export KUBECONFIG="$(kind get kubeconfig-path --name="voltha-$NAME")"
    P="coredns-.* \
        etcd-voltha-$NAME-control-plane \
        kindnet-.* \
        kube-apiserver-voltha-$NAME-control-plane \
        kube-controller-manager-voltha-$NAME-control-plane \
        kube-proxy-.* \
        kube-scheduler-voltha-$NAME-control-plane"

    EXPECT=$(test "$TYPE" == "minimal" && echo "12" || echo "14")
    wait_for_pods - "kube-system" $EXPECT -1 "Waiting for system PODs to start" $P
fi

COUNT=$(count_pods "kube-system" "tiller-deploy-.*")
bspin "Verify Helm"
if [ $COUNT -ne 1 ]; then
    espin $NOT_VERIFIED
    echo -e "Configuring Helm $GEAR"
    if [ "$INSTALL_HELM" == "no" ]; then
        bspin - "Skip Helm/Tiller Initialization"
        espin - $NO_ENTRY
    else
        bspin - "Initialize Helm"
        (set -x; helm init --upgrade >>$LOG 2>&1) >>$LOG 2>&1
        espin - $VERIFIED
        wait_for_pods - "kube-system" 1 -1 "Waiting for Tiller POD to start" "tiller-deploy-.*"
    fi
    bspin - "Add Google Incubator repository to Helm"
    (set -x; helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com >>$LOG 2>&1) >>$LOG 2>&1
    espin - $VERIFIED
   
    # HACK (sort-of) - the config for tiller is about to be patched, which will
    # cause the tiller pod to be recreated. This can sometimes cause a timing
    # issue with the "wait_for_pods" call on tiller as it may incorrectly
    # identify the running/ready tiller pod that is soon to be terminated as 
    # what it is waiting for. To avoid this issue we do a clean scale down and
    # scale up of the pod so the script controlls when it should be expecting 
    # things
    (set -x; kubectl -n kube-system scale deploy tiller-deploy --replicas=0 >>$LOG 2>&1) >>$LOG 2>&1

    bspin - "Add Google Stable repository to Helm"
    (set -x; helm repo add stable https://kubernetes-charts.storage.googleapis.com >>$LOG 2>&1) >>$LOG 2>&1
    espin - $VERIFIED
    bspin - "Add ONF repository to Helm"
    (set -x; helm repo add onf https://charts.opencord.org >>$LOG 2>&1) >>$LOG 2>&1
    espin - $VERIFIED
    bspin - "Update Helm repository cache"
    (set -x; helm repo update >>$LOG 2>&1) >>$LOG 2>&1
    espin - $VERIFIED
    
    # Create and k8s service account so that Helm can create pods
    bspin - "Create Tiller ServiceAccount"
    (set -x; kubectl create serviceaccount --namespace kube-system tiller >>$LOG 2>&1) >>$LOG 2>&1
    espin - $VERIFIED
    bspin - "Create Tiller ClusterRoleBinding"
    (set -x; kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller >>$LOG 2>&1) >>$LOG 2>&1
    espin - $VERIFIED
    bspin - "Update Tiller Manifest"
    (set -x; kubectl patch deploy --namespace kube-system tiller-deploy -p '{"spec":{"template":{"spec":{"serviceAccount":"tiller"}}}}' >>$LOG 2>&1) >>$LOG 2>&1

    # HACK (sort-of) - part to, spin it back up
    (set -x; kubectl -n kube-system scale deploy tiller-deploy --replicas=1 >>$LOG 2>&1) >>$LOG 2>&1
    espin - $VERIFIED
else
    espin $VERIFIED
fi
wait_for_pods - "kube-system" 1 -1 "Waiting for Tiller POD to start" "tiller-deploy-.*"

bspin "Verify Helm values file: $NAME-values.yaml"
if [ ! -r "./$NAME-values.yaml" ]; then
    espin $NOT_VERIFIED
    bspin - "Download Helm values file: $TYPE-values.yaml to $NAME-values.yaml $DOWNLOAD"
    (set -x; curl -o ./$NAME-values.yaml -sSL https://raw.githubusercontent.com/ciena/kind-voltha/master/$TYPE-values.yaml >>$LOG 2>&1) >>$LOG 2>&1
    espin - $VERIFIED
else
    espin $VERIFIED
fi

if [ "$JUST_K8S" == "yes" ]; then
    echo "Environment deployed, not deploying VOLTHA artifacts as requested. Good bye."
    echo ""
    echo "Please issue the following commands in your terminal to ensure that you" | tee -a $LOG
    echo "are accessing the correct Kubernetes/Kind cluster as well as have the  " | tee -a $LOG
    echo "tools required by VOLTHA in your command path.                         " | tee -a $LOG
    echo "" | tee -a $LOG
    echo -en $BOLD
    if [ $DEPLOY_K8S == "yes" ]; then
        echo "export KUBECONFIG=\"\$(./bin/kind get kubeconfig-path --name=\"voltha-$NAME\")\"" | tee -a $LOG
    fi
    echo "export PATH=$GOPATH/bin:\$PATH" | tee -a $LOG
    echo -en $NORMAL
    echo "" | tee -a $LOG
    echo "Thank you for choosing kind-voltha for you quick cluster needs." | tee -a $LOG
    exit 0
fi

bspin "Verify ETCD Operator $OLD_KEY"
if [ $(helm list --deployed --short --namespace voltha "^etcd-operator\$" | wc -l) -ne 1 ]; then
    espin $NOT_VERIFIED 
    helm_install - voltha etcd-operator stable/etcd-operator "Install ETCD Operator"
else
    espin $VERIFIED
fi
EXPECT=$(test "$TYPE" == "minimal" && echo "1" || echo "3")
wait_for_pods - "voltha" $EXPECT -1 "Waiting for ETCD Operator to start" "etcd-operator-.*"

if [ $WITH_ONOS == "yes" ]; then
    bspin "Verify ONOS installed $BIRD"
    if [ $(helm list --deployed --short --namespace default "^onos\$" | wc -l) -ne 1 ]; then
        espin $NOT_VERIFIED
        helm_install - default onos onf/onos "Install ONOS"
    else
        espin $VERIFIED
    fi
    wait_for_pods - "default" 1 -1 "Waiting for ONOS to start" "onos-.*"
    
    bspin - "Forward ONOS API port $FORWARD"
    for i in $(screen -ls | grep onos-ui-$NAME | awk '{print $1}'); do
        (set -x; screen -X -S $i quit >>$LOG 2>&1) >>$LOG 2>&1
    done
    (set -x; screen -dmS onos-ui-$NAME bash -c "while true; do kubectl port-forward service/onos-ui $ONOS_API_PORT:8181; done" >>$LOG 2>&1) >>$LOG 2>&1
    espin - $VERIFIED
    bspin - "Forward ONOS SSH port $FORWARD"
    for i in $(screen -ls | grep onos-ssh-$NAME | awk '{print $1}'); do
        (set -x; screen -X -S $i quit >>$LOG 2>&1) >>$LOG 2>&1
    done
    (set -x; screen -dmS onos-ssh-$NAME bash -c "while true; do kubectl port-forward service/onos-ssh $ONOS_SSH_PORT:8101; done" >>$LOG 2>&1) >>$LOG 2>&1
    espin - $VERIFIED
    if [ ! -x ./onos-files/install-onos-applications.sh ]; then
        bspin - "Verify or download ONOS configuration support files $DOWNLOAD"
        (set -x; mkdir -p ./onos-files  >>$LOG 2>&1) >>$LOG 2>&1
        for i in dhcp-to-controller-flow.json install-onos-applications.sh olt-onos-enableExtraneousRules.json olt-onos-netcfg.json olt-onos-olt-settings.json radius-config.json; do
            if [ ! -r ./onos-files/$i ]; then
                (set -x; curl -o ./onos-files/$i -sSL https://raw.githubusercontent.com/ciena/kind-voltha/master/onos-files/$i >>$LOG 2>&1) >>$LOG 2>&1
            fi
        done
        (set -x; chmod 755 ./onos-files/install-onos-applications.sh >>$LOG 2>&1) >>$LOG 2>&1
        espin - $VERIFIED
    fi
    
    bspin - "Install required ONOS applications $INSTALL"
    (set -x; ./onos-files/install-onos-applications.sh >>$LOG 2>&1) >>$LOG 2>&1
    espin - $VERIFIED
fi

bspin "Verify VOLTHA installed $HIGH_VOLTAGE"
if [ $(helm list --deployed --short --namespace voltha "^voltha\$" | wc -l) -ne 1 ]; then
    espin $NOT_VERIFIED
    helm_install - voltha voltha $VOLTHA_CHART "Install VOLTHA Core"
else
    espin $VERIFIED
fi

VOLTHA="ofagent-.* \
    ro-core.* \
    rw-core.* \
    voltha-api-server-.* \
    voltha-cli-server-.* \
    voltha-etcd-cluster-.* \
    voltha-kafka-.* \
    voltha-zookeeper-.*"
EXPECT=$(test "$TYPE" == "minimal" && echo "9" || echo "11")
wait_for_pods - "voltha" $EXPECT -1 "Waiting for VOLTHA Core to start" $VOLTHA

echo -e "Verify Adapters $PLUG"
bspin - "Verify Simulated Adapters installed"
if [ $(helm list --deployed --short --namespace voltha "^sim\$" | wc -l) -ne 1 ]; then
    espin - $NOT_VERIFIED
    helm_install - voltha sim $VOLTHA_ADAPTER_SIM_CHART "Install Simulated Adapters"
else
    espin - $VERIFIED
fi

bspin - "Verify OpenOLT Adapter installed"
if [ $(helm list --deployed --short --namespace voltha "^open-olt\$" | wc -l) -ne 1 ]; then
    espin - $NOT_VERIFIED
    helm_install - voltha open-olt $VOLTHA_ADAPTER_OPEN_OLT_CHART "Install OpenOLT Adapter"
else
    espin - $VERIFIED
fi
bspin - "Verify OpenONU Adapter installed"
if [ $(helm list --deployed --short --namespace voltha "^open-onu\$" | wc -l) -ne 1 ]; then
    espin - $NOT_VERIFIED
    helm_install - voltha open-onu $VOLTHA_ADAPTER_OPEN_ONU_CHART "Install OpenONU Adapter"
else
    espin - $VERIFIED
fi

ADAPTERS="adapter-.*"
wait_for_pods - "voltha" 4 -1 "Waiting for adapters to start" $ADAPTERS

if [ $WITH_BBSIM == "yes" ]; then
    echo -e "Verify BBSIM $PLUG"
    bspin - "Verify BBSIM Installed"
    if [ $(helm list --deployed --short --namespace voltha "^bbsim\$" | wc -l) -ne 1 ]; then
        espin - $NOT_VERIFIED
        helm_install - voltha bbsim onf/bbsim "Install BBSIM"
    else
        espin - $VERIFIED
    fi
    wait_for_pods - "voltha" 1 -1 "Waiting for BBSIM to start" "bbsim-.*"
fi

if [ $WITH_RADIUS == "yes" ]; then
    echo -e "Verify RADIUS $LOCK"
    bspin - "Verify RADIUS Installed"
    if [ $(helm list --deployed --short --namespace voltha "^radius\$" | wc -l) -ne 1 ]; then
        espin - $NOT_VERIFIED
        helm_install - voltha radius onf/freeradius "Install RADIUS"
    else
        espin - $VERIFIED
    fi
    wait_for_pods - "voltha" 1 -1 "Waiting for RADIUS to start" "radius-.*"
fi

if [ $SKIP_RESTART_API == "no" ]; then
    echo -e "Restart VOLTHA API $RESTART"
    API="voltha-api-server-.* ofagent-.*"
    (set -x; kubectl scale --replicas=0 deployment -n voltha voltha-api-server ofagent >>$LOG 2>&1) >>$LOG 2>&1
    wait_for_pods - "voltha" 0 -1 "Wait for API to stop $STOP" $API
    (set -x; kubectl scale --replicas=1 deployment -n voltha voltha-api-server ofagent >>$LOG 2>&1) >>$LOG 2>&1
    wait_for_pods - "voltha" 2 -1 "Wait for API to re-start $GO" $API
else
    bspin "Skip VOLTHA API Restart"
    espin $NO_ENTRY
fi

bspin - "Forward VOLTHA API port $FORWARD"
for i in $(screen -ls | grep voltha-api-$NAME | awk '{print $1}'); do
    (set -x; screen -X -S $i quit >>$LOG 2>&1) >>$LOG 2>&1
done
(set -x; screen -dmS voltha-api-$NAME bash -c "while true; do kubectl port-forward -n voltha service/voltha-api $VOLTHA_API_PORT:55555; done" >>$LOG 2>&1) >>$LOG 2>&1
espin - $VERIFIED
bspin - "Forward VOLTHA SSH port $FORWARD"
for i in $(screen -ls | grep voltha-ssh-$NAME | awk '{print $1}'); do
    (set -x; screen -X -S $i quit >>$LOG 2>&1) >>$LOG 2>&1
done
(set -x; screen -dmS voltha-ssh-$NAME bash -c "while true; do kubectl port-forward -n voltha service/voltha-cli $VOLTHA_SSH_PORT:5022; done" >>$LOG 2>&1) >>$LOG 2>&1
espin - $VERIFIED
bspin - "Forward VOLTHA ETCD port $FORWARD"
for i in $(screen -ls | grep voltha-etcd-$NAME | awk '{print $1}'); do
    (set -x; screen -X -S $i quit >>$LOG 2>&1) >>$LOG 2>&1
done
(set -x; screen -dmS voltha-etcd-$NAME bash -c "while true; do kubectl port-forward -n voltha service/voltha-etcd-cluster-client $VOLTHA_ETCD_PORT:2379; done" >>$LOG 2>&1) >>$LOG 2>&1
espin - $VERIFIED

if [ $WITH_ONOS == "yes" -a $WITH_RADIUS == "yes" ]; then
    bspin "Configure ONOS RADIUS Connection $GEAR"
    (set -x; cat onos-files/radius-config.json | sed -e "s/:RADIUS_IP:/$(kubectl -n voltha get service/radius -o jsonpath={.spec.clusterIP})/g" | curl -XPOST -sSL http://karaf:karaf@localhost:8181/onos/v1/network/configuration -H Content-type:application/json -d@- >>$LOG 2>&1) >>$LOG 2>&1
    espin $VERIFIED
fi

bspin "Create voltctl configuration file"
(set -x; mkdir -p $HOME/.volt >>$LOG 2>&1) >>$LOG 2>&1
(set -x; voltctl -a v2 -s localhost:$VOLTHA_API_PORT config > $HOME/.volt/config-$NAME 2>>$LOG) >>$LOG 2>&1
espin $VERIFIED

if [ ! -f "$NAME-env.sh" ]; then
    touch $NAME-env.sh
fi

for O in $ALL_OPTIONS; do
    VAL=$(eval echo \$$O)
    if [ $O == "USE_GO" ]; then
        VAL="$(echo $VAL| test $(grep -c true) -eq 1 && echo yes || echo no)"
    fi
    if [ ! -z "$VAL" -a $(grep -c "^export $O=" $NAME-env.sh) -eq 0 ]; then
        echo "export $O=\"$(eval echo \$$O)\"" >> $NAME-env.sh
    fi
done

if [ $DEPLOY_K8S == "yes" -a $(grep -c "^export KUBECONFIG=" $NAME-env.sh) -eq 0 ]; then
    echo "export KUBECONFIG=\"$(./bin/kind get kubeconfig-path --name=voltha-$NAME)\"" >> $NAME-env.sh
fi

if [ $(grep -c "^export VOLTCONFIG=" $NAME-env.sh) -eq 0 ]; then
    echo "export VOLTCONFIG=\"$HOME/.volt/config-$NAME\"" >> $NAME-env.sh
fi

if [ $(grep -c "^export PATH=" $NAME-env.sh) -eq 0 ]; then
    echo "export PATH=\"$GOPATH/bin:\$PATH\"" >> $NAME-env.sh
fi

echo ""
echo "Please issue the following commands in your terminal to ensure that you" | tee -a $LOG
echo "are accessing the correct Kubernetes/Kind cluster as well as have the  " | tee -a $LOG
echo "tools required by VOLTHA in your command path.                         " | tee -a $LOG
echo "" | tee -a $LOG
echo -en $BOLD
if [ $DEPLOY_K8S == "yes" ]; then
    echo "export KUBECONFIG=\"\$(./bin/kind get kubeconfig-path --name=\"voltha-$NAME\")\"" | tee -a $LOG
fi
echo "export VOLTCONFIG=\"$HOME/.volt/config-$NAME\"" | tee -a $LOG
echo "export PATH=$GOPATH/bin:\$PATH" | tee -a $LOG
echo -en $NORMAL
echo "" | tee -a $LOG
echo "Thank you for choosing kind-voltha for you quick cluster needs." | tee -a $LOG