voltha

#!/bin/bash
# Copyright 2019 Ciena Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -f
unalias -a
shopt -s extglob
TOTAL_START_TIME="$(date +%s)"
mkdir -p "$(pwd)/bin"
PATH="$(pwd)/bin:$PATH"; export PATH
GOPATH="$(pwd)"; export GOPATH

FANCY=${FANCY:-1}
if [ "$TERM X" == " X" ]; then
    FANCY=0
fi

# trap ctrl-c and call ctrl_c()
trap ctrl_c INT

function ctrl_c() {
    echo -en "$CNORM"
    echo ""
    echo "ctrl-c trapped"
    echo "Thank you for trying 'voltha up'"
    exit
}

HELM_VERSION=${HELM_VERSION:-v3.2.4}
VOLTCTL_VERSION=${VOLTCTL_VERSION:-latest}
KIND_VERSION=${KIND_VERSION:-v0.8.1}
VK_RELEASE=${VK_RELEASE:-master}
EXTRA_TOOLS=

RED=
GREEN=
YELLOW=
BLUE=
BOLD=
NORMAL=
ERROR=
CEOL=
CNORM=
CIVIS=
if [ "$FANCY" -eq 1 ]; then
    RED="$(tput setaf 1)"
    GREEN="$(tput setaf 2)"
    YELLOW="$(tput setaf 3)"
    BLUE="$(tput setaf 4)"
    BOLD="$(tput bold)"
    NORMAL="$(tput sgr0)"
    ERROR="\xe2\x9c\x97\x20"
    CEOL="$(tput el)"
    CNORM="$(tput cnorm)"
    CIVIS="$(tput civis)"
fi

NAME=${NAME:-minimal}

ENABLE_ONOS_EXTRANEOUS_RULES=${ENABLE_ONOS_EXTRANEOUS_RULES:-no}
INFRA_NS=${INFRA_NS:-default}
VOLTHA_NS=${VOLTHA_NS:-voltha}
BBSIM_NS=${BBSIM_NS:-voltha}
ADAPTER_NS=${ADAPTER_NS:-voltha}
WITH_TIMINGS=${WITH_TIMINGS:-no}
WITH_BBSIM=${WITH_BBSIM:-no}
WITH_EFK=${WITH_EFK:-no}
WITH_TRACING=${WITH_TRACING:-no}
WITH_RADIUS=${WITH_RADIUS:-no}
WITH_EAPOL=${WITH_EAPOL:-yes}
WITH_DHCP=${WITH_DHCP:-yes}
WITH_IGMP=${WITH_IGMP:-no}
WITH_ONOS=${WITH_ONOS:-yes}
NUM_OF_ONOS=${NUM_OF_ONOS:-1}
NUM_OF_ATOMIX=${NUM_OF_ATOMIX:-0}
WITH_CHAOS=${WITH_CHAOS:-no}
WITH_ADAPTERS=${WITH_ADAPTERS:-yes}
WITH_SIM_ADAPTERS=${WITH_SIM_ADAPTERS:-no}
WITH_OPEN_ADAPTERS=${WITH_OPEN_ADAPTERS:-yes}
WITH_PORT_FORWARDS=${WITH_PORT_FORWARDS:-yes}
WITH_KAFKA=${WITH_KAFKA:-yes}
WITH_ETCD=${WITH_ETCD:-yes}
WITH_PPROF=${WITH_PPROF:-no}
WITH_INCREMENTAL_EVTO_UPDATE=${WITH_INCREMENTAL_EVTO_UPDATE:-no}
SCHEDULE_ON_CONTROL_NODES=${SCHEDULE_ON_CONTROL_NODES:-no}
CONFIG_SADIS=${CONFIG_SADIS:-no} # yes | no | file | bbsim | external | URL
SADIS_CFG=${SADIS_CFG:-onos-files/onos-sadis-sample.json}
BBSIM_CFG=${BBSIM_CFG:-configs/bbsim-sadis-att.yaml}
INSTALL_ONOS_APPS=${INSTALL_ONOS_APPS:-no}
JUST_K8S=${JUST_K8S:-no}
DEPLOY_K8S=${DEPLOY_K8S:-yes}
INSTALL_KUBECTL=${INSTALL_KUBECTL:-yes}
INSTALL_HELM=${INSTALL_HELM:-yes}
HELM_USE_UPGRADE=${HELM_USE_UPGRADE:-no}
UPDATE_HELM_REPOS=${UPDATE_HELM_REPOS:-yes}
WAIT_ON_DOWN=${WAIT_ON_DOWN:-yes}
WAIT_TIMEOUT=${WAIT_TIMEOUT:-30m}
VOLTHA_DOWN_ON_TIMEOUT=${VOLTHA_DOWN_ON_TIMEOUT:-no}
VOLTHA_LOG_LEVEL=${VOLTHA_LOG_LEVEL:-WARN}
VOLTHA_CHART=${VOLTHA_CHART:-onf/voltha}
VOLTHA_CHART_VERSION=${VOLTHA_CHART_VERSION:-latest}
VOLTHA_BBSIM_CHART=${VOLTHA_BBSIM_CHART:-onf/bbsim}
VOLTHA_BBSIM_CHART_VERSION=${VOLTHA_BBSIM_CHART_VERSION:-latest}
ELASTICSEARCH_CHART=${ELASTICSEARCH_CHART:-elastic/elasticsearch}
ELASTICSEARCH_CHART_VERSION=${ELASTICSEARCH_CHART_VERSION:-latest}
KIBANA_CHART=${KIBANA_CHART:-elastic/kibana}
KIBANA_CHART_VERSION=${KIBANA_CHART_VERSION:-latest}
FLUENTD_ELASTICSEARCH_CHART=${FLUENTD_ELASTICSEARCH_CHART:-kiwigrid/fluentd-elasticsearch}
FLUENTD_ELASTICSEARCH_CHART_VERSION=${FLUENTD_ELASTICSEARCH_CHART_VERSION:-latest}
VOLTHA_TRACING_CHART=${VOLTHA_TRACING_CHART:-onf/voltha-tracing}
VOLTHA_TRACING_CHART_VERSION=${VOLTHA_TRACING_CHART_VERSION:-latest}
VOLTHA_ADAPTER_SIM_CHART=${VOLTHA_ADAPTER_SIM_CHART:-onf/voltha-adapter-simulated}
VOLTHA_ADAPTER_SIM_CHART_VERSION=${VOLTHA_ADAPTER_SIM_CHART_VERSION:-latest}
VOLTHA_ADAPTER_OPEN_OLT_CHART=${VOLTHA_ADAPTER_OPEN_OLT_CHART:-onf/voltha-adapter-openolt}
VOLTHA_ADAPTER_OPEN_OLT_CHART_VERSION=${VOLTHA_ADAPTER_OPEN_OLT_CHART_VERSION:-latest}
VOLTHA_ADAPTER_OPEN_ONU_CHART=${VOLTHA_ADAPTER_OPEN_ONU_CHART:-onf/voltha-adapter-openonu}
VOLTHA_ADAPTER_OPEN_ONU_CHART_VERSION=${VOLTHA_ADAPTER_OPEN_ONU_CHART_VERSION:-latest}
ONOS_CHART=${ONOS_CHART:-onf/onos}
ONOS_CHART_VERSION=${ONOS_CHART_VERSION:-latest}
ONOS_CLASSIC_CHART=${ONOS_CLASSIC_CHART:-onos/onos-classic}
ONOS_CLASSIC_CHART_VERSION=${ONOS_CLASSIC_CHART_VERSION:-latest}
KAFKA_CHART=${KAFKA_CHART:-incubator/kafka}
KAFKA_CHART_VERSION=${KAFKA_CHART_VERSION:=latest}
BBSIM_SADIS_SERVER_CHART=${BBSIM_SADIS_SERVER_CHART:-bbsim-sadis/bbsim-sadis-server}
BBSIM_SADIS_SERVER_CHART_VERSION=${BBSIM_SADIS_SERVER_CHART_VERSION:-latest}
ETCD_CHART=${ETCD_CHART:-incubator/etcd}
ETCD_CHART_VERSION=${ETCD_CHART_VERSION:-latest}
RADIUS_CHART=${RADIUS_CHART:-onf/freeradius}
RADIUS_CHART_VERSION=${RADIUS_CHART_VERSION:-latest}
EXTRA_HELM_INSTALL_ARGS=${EXTRA_HELM_INSTALL_ARGS:-}
INTERNAL_EXTRA_HELM_INSTALL_ARGS=
NUM_OF_BBSIM=${NUM_OF_BBSIM:-1}
NUM_OF_WORKER_NODES=${NUM_OF_WORKER_NODES:-2}
NUM_OF_CONTROLLER_NODES=${NUM_OF_CONTROLLER_NODES:-1}
NUM_OF_OPENONU=${NUM_OF_OPENONU:-1}
NUM_OF_KAFKA=${NUM_OF_KAFKA:-1}
NUM_OF_ETCD=${NUM_OF_ETCD:-1}
MAX_NUM_OF_BBSIM=10
MAX_NUM_OF_OPENONU=10
LEGACY_BBSIM_INDEX=${LEGACY_BBSIM_INDEX:-no}
PF_ADDRESS=${PF_ADDRESS:-0.0.0.0}
KIND_CFG_FILE=${KIND_CFG_FILE:-}

HOSTOS="$(uname -s | tr "[:upper:]" "[:lower:"])"
HOSTARCH="$(uname -m | tr "[:upper:]" "[:lower:"])"
if [ "$HOSTARCH" == "x86_64" ]; then
    HOSTARCH="amd64"
fi
BBSIM_LABEL="-l app=bbsim"
NO_LABEL=

# checks to see if a given WORD is in the given LIST of words
function is_in() {
    local WORD LIST
    WORD="$1"; shift
    LIST="$*"
    LIST=${LIST//+([[:space:],|])/:}

    [ "$(echo ":$LIST:" | grep -ic ":$WORD:")" -ne 0 ]
}

function parseDuration() {
    local DUR RESULT TERM TERMS VALUE UNIT
    DUR=$1
    RESULT=0
    TERMS="$(echo "$DUR" | sed -Ee 's/([sSmMhHdD])/\1 /g' -e 's/,$//g')"
    for TERM in $TERMS; do
       VALUE="$(echo "$TERM" | sed -Ee 's/([0-9]+)[sSmMhHdD]/\1/')"
       UNIT="$(echo "$TERM" | sed -Ee 's/[0-9]+([sSmMhHdD])/\1/')"
       case $UNIT in
          s|S)
             RESULT=$((RESULT + VALUE)) ;;
          m|M)
             RESULT=$((RESULT + (VALUE * 60))) ;;
          h|H)
             RESULT=$((RESULT + (VALUE * 3600))) ;;
          d|D)
             RESULT=$((RESULT + (VALUE * 86400))) ;;
          *) ;;
       esac
    done
    echo $RESULT
}

function doTimeout() {
    local MSG=$1
    2>&1 echo -e "\n${RED}${BOLD}${ERROR}TIMEOUT:${NORMAL}${RED} Operation timed out after '$WAIT_TIMEOUT': $MSG"
    exit 125
}

function get_service_ep() {
    local NS=$1
    local NAME=$2
    kubectl -n "$NS" get service "$NAME" -o json | jq -r '.spec.clusterIP + ":" + (.spec.ports[0].port|tostring)'
}

# returns the greater version (0 if equal)
# inspired by https://stackoverflow.com/questions/4023830/how-to-compare-two-strings-in-dot-separated-version-format-in-bash
function semvercompare() {
    if [[ "$1" == "$2" ]]
    then
        echo 0
        return
    fi

    local tmp1 tmp2
    tmp1=$(echo "${1//v/}" | awk -F'-' '{print $1}')
    tmp2=$(echo "${2//v/}" | awk -F'-' '{print $1}')

    local IFS=.
    # shellcheck disable=SC2206
    local i ver1=($tmp1) ver2=($tmp2)

    # fill empty fields in ver1 with zeros
    for ((i=${#ver1[@]}; i<${#ver2[@]}; i++))
    do
        ver1[i]=0
    done
    for ((i=0; i<${#ver1[@]}; i++))
    do
        if [[ -z ${ver2[i]} ]]
        then
            # fill empty fields in ver2 with zeros
            ver2[i]=0
        fi
        if ((10#${ver1[i]} > 10#${ver2[i]}))
        then
            echo 1
            return
        fi
        if ((10#${ver1[i]} < 10#${ver2[i]}))
        then
            echo 2
            return
        fi
    done
    echo 0
    return
}

# returns true if v1 is greater than v2 (false if they are the same)
function semver_greater() {
  local v1
  local v2
  v1="$1" v2="$2"

  # shellcheck disable=SC2128
  if [[ ! "$v1" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
    # if the version is custom, we assume it's newer than master
    echo "true"
    return
  fi

  # shellcheck disable=SC2128
  res=$(semvercompare "$v1" "$v2")
  if [[ $res == 1 ]]; then
    echo "true"
  else
    echo "false"
  fi
}

# returns true if v1 is lesser than v2 (false if they are the same)
function semver_lesser() {
  local v1
  local v2
  v1="$1" v2="$2"

  # shellcheck disable=SC2128
  if [[ ! "$v1" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
    # if the version is custom, we assume it's newer than master
    echo "false"
    return
  fi

  # shellcheck disable=SC2128
  res=$(semvercompare "$v1" "$v2")
  if [[ $res == 2 ]]; then
    echo "true"
  else
    echo "false"
  fi
}

# Used to verify configuration values are set to "yes" or "no" value or convert
# equivalents to "yes" or "no"
YES="y,yes,t,true,1"
NO="n,no,f,false,0"
YES_OR_NO="$YES,$NO"

# If the given VAR matches a truth value then normalize that
# value, else return original value
function normalize_yes_no() {
    local VAR VAL
    VAR=$1
    VAL="$(eval echo "\$$VAR")"
    if ! is_in "$VAL" "$YES_OR_NO"; then
        echo "$VAL"
        return 1
    fi
    if is_in "$VAL" "$YES"; then
        echo "yes"
    else
        echo "no"
    fi
    return 0
}

# If the given VAR matches a truth value then normalize that
# value, else display an error
function verify_yes_no() {
    local VAR VAL
    VAR=$1
    VAL="$(eval echo "\$$VAR")"
    if ! is_in "$VAL" "$YES_OR_NO"; then
        >&2 echo -e "${RED}${BOLD}${ERROR}ERROR:${NORMAL}${RED} Invalid '$VAR' value of '$VAL'. Should be 'yes' or 'no'${NORMAL}"
        echo "INVALID"
        return 1
    fi
    if is_in "$VAL" "$YES"; then
        echo "yes"
    else
        echo "no"
    fi
    return 0
}

ALL_YES_NO="\
    ENABLE_ONOS_EXTRANEOUS_RULES \
    WITH_TIMINGS \
    WITH_BBSIM \
    WITH_EFK \
    WITH_TRACING \
    WITH_EAPOL \
    WITH_DHCP \
    WITH_IGMP \
    WITH_CHAOS \
    WITH_ADAPTERS \
    WITH_SIM_ADAPTERS \
    WITH_OPEN_ADAPTERS \
    WITH_PORT_FORWARDS \
    WITH_PPROF \
    WITH_INCREMENTAL_EVTO_UPDATE \
    JUST_K8S \
    DEPLOY_K8S \
    INSTALL_ONOS_APPS \
    INSTALL_KUBECTL \
    INSTALL_HELM \
    HELM_USE_UPGRADE \
    UPDATE_HELM_REPOS \
    WAIT_ON_DOWN \
    VOLTHA_DOWN_ON_TIMEOUT \
    LEGACY_BBSIM_INDEX \
    SCHEDULE_ON_CONTROL_NODES \
    "

OPT_YES_NO="\
    CONFIG_SADIS \
    WITH_KAFKA \
    WITH_RADIUS \
    WITH_ETCD \
    WITH_ONOS \
    "

ALL_OPTIONS="\
    NAME \
    $ALL_YES_NO \
    $OPT_YES_NO \
    WAIT_TIMEOUT \
    VOLTHA_LOG_LEVEL \
    VOLTHA_CHART \
    VOLTHA_CHART_VERSION \
    VOLTHA_BBSIM_CHART \
    VOLTHA_BBSIM_CHART_VERSION \
    BBSIM_SADIS_SERVER_CHART \
    BBSIM_SADIS_SERVER_CHART_VERSION \
    VOLTHA_TRACING_CHART \
    VOLTHA_TRACING_CHART_VERSION \
    NUM_OF_BBSIM \
    NUM_OF_WORKER_NODES \
    NUM_OF_CONTROLLER_NODES \
    NUM_OF_KAFKA \
    NUM_OF_ETCD \
    ELASTICSEARCH_CHART \
    ELASTICSEARCH_CHART_VERSION \
    KIBANA_CHART \
    KIBANA_CHART_VERSION \
    FLUENTD_ELASTICSEARCH_CHART \
    FLUENTD_ELASTICSEARCH_CHART_VERSION \
    NUM_OF_OPENONU \
    VOLTHA_ADAPTER_SIM_CHART \
    VOLTHA_ADAPTER_SIM_CHART_VERSION \
    VOLTHA_ADAPTER_OPEN_OLT_CHART \
    VOLTHA_ADAPTER_OPEN_OLT_CHART_VERSION \
    VOLTHA_ADAPTER_OPEN_ONU_CHART \
    VOLTHA_ADAPTER_OPEN_ONU_CHART_VERSION \
    ONOS_CHART \
    ONOS_CHART_VERSION \
    ONOS_CLASSIC_CHART \
    ONOS_CLASSIC_CHART_VERSION \
    KAFKA_CHART \
    KAFKA_CHART_VERSION \
    ETCD_CHART \
    ETCD_CHART_VERSION \
    RADIUS_CHART \
    RADIUS_CHART_VERSION \
    ONOS_API_PORT \
    ONOS_SSH_PORT \
    SADIS_CFG \
    BBSIM_CFG \
    VOLTHA_API_PORT \
    VOLTHA_SSH_PORT \
    VOLTHA_ETCD_PORT \
    ELASTICSEARCH_PORT \
    KIBANA_PORT \
    VOLTHA_KAFKA_PORT \
    VOLTHA_PPROF_PORT \
    OPENOLT_PPROF_PORT \
    OFAGENT_PPROF_PORT \
    TRACING_GUI_PORT \
    VK_RELEASE \
    KIND_VERSION \
    VOLTCTL_VERSION \
    HELM_VERSION \
    NUM_OF_ONOS \
    NUM_OF_ATOMIX \
    VOLTHA_NS \
    ADAPTER_NS \
    INFRA_NS \
    BBSIM_NS \
    SADIS_BANDWIDTH_PROFILES \
    SADIS_SUBSCRIBERS \
    PF_ADDRESS \
    KIND_CFG_FILE \
    "

# Iterate over yes/no configuration options and validate
for VAR in $ALL_YES_NO; do
    eval "$VAR"="$(verify_yes_no "$VAR")"
    if [ "$(eval echo "\$$VAR")" == "INVALID" ]; then
        exit 1;
    fi
done

# Iterate over optional yes/no configuration options and
# normalize values if they are truth values
for VAR in $OPT_YES_NO; do
    eval "$VAR"="$(normalize_yes_no "$VAR")"
done

# check number (range) of bbsim, max bbsim must not exceed 10 instances!
# note: instances will be numbered from 0 to 9
if [ "$1" == "up" ]; then
    if [ "$NUM_OF_BBSIM" -lt 1 ]; then
        >&2 echo -e "${RED}${BOLD}${ERROR}ERROR:${NORMAL}${RED} Invalid setting of BBSIM instances. NUM_OF_BBSIM (${NUM_OF_BBSIM}) is less than 1${NORMAL}"
        exit 1
    fi
    if [ "$NUM_OF_BBSIM" -gt "$MAX_NUM_OF_BBSIM" ]; then
        >&2 echo -e "${RED}${BOLD}${ERROR}ERROR:${NORMAL}${RED} Invalid setting of BBSIM instances. NUM_OF_BBSIM (${NUM_OF_BBSIM}) is greater than $MAX_NUM_OF_BBSIM${NORMAL}"
        exit 1
    fi

    if [ "$NUM_OF_OPENONU" -lt 1 ]; then
        >&2 echo -e "${RED}${BOLD}${ERROR}ERROR:${NORMAL}${RED} Invalid setting of OPENONU instances. NUM_OF_OPENONU (${NUM_OF_OPENONU}) is less than 1${NORMAL}"
        exit 1
    fi
    if [ "$NUM_OF_OPENONU" -gt "$MAX_NUM_OF_OPENONU" ]; then
        >&2 echo -e "${RED}${BOLD}${ERROR}ERROR:${NORMAL}${RED} Invalid setting of OPENONU instances. NUM_OF_OPENONU (${NUM_OF_OPENONU}) is greater than $MAX_NUM_OF_OPENONU${NORMAL}"
        exit 1
    fi

    HAVE_CLUSTER="$(kind get clusters 2>/dev/null | grep -c "voltha-$NAME")"
    if [ "$HAVE_CLUSTER" -eq 0 ]; then
        # If the user has specified a kind cluster configuration file,
        # 'KIND_CFG_FILE` then we need to gleem from that file the number
        # of worker and controller nodes as this overrides any setting of
        # these values
        if [ -n "$KIND_CFG_FILE" ]; then
            NUM_OF_WORKER_NODES="$(sed 's/ //g' "$KIND_CFG_FILE" | grep -c "^-role:worker$")"
            NUM_OF_CONTROLLER_NODES="$(sed 's/ //g' "$KIND_CFG_FILE" | grep -c "^-role:control-plane$")"
        fi

        # check that NUM_OF_KAFKA, NUM_OF_ONOS, NUM_OF_ATOMIX, NUM_OF_ETCD is:
        # <= NUM_OF_WORKER_NODES + 1 if SCHEDULE_ON_CONTROL_NODES == y
        # <= NUM_OF_WORKER_NODES if SCHEDULE_ON_CONTROL_NODES == n
        SCHEDULABLE_NODES=$NUM_OF_WORKER_NODES

        if [ "$SCHEDULE_ON_CONTROL_NODES" == "yes" ]; then
          SCHEDULABLE_NODES=$((NUM_OF_CONTROLLER_NODES+NUM_OF_WORKER_NODES))
        fi
    else
        TOTAL_NODES=$(kubectl get --all-namespaces nodes -o name | wc -l)
        NUM_OF_CONTROLLER_NODES=$(kubectl get --all-namespaces nodes -l node-role.kubernetes.io/master -o name | wc -l)
        NUM_OF_WORKER_NODES=$((TOTAL_NODES - NUM_OF_CONTROLLER_NODES))
	# shellcheck disable=SC2016
        SCHEDULABLE_NODES=$(kubectl get no -o 'go-template={{range .items}}{{$taints:=""}}{{range .spec.taints}}{{if eq .effect "NoSchedule"}}{{$taints = print $taints .key ","}}{{end}}{{end}}{{if not $taints}}{{.metadata.name}}{{ "\n"}}{{end}}{{end}}' | wc -l | sed -e 's/ //g')
        rm -f "$TMP_KUBECFG"
    fi
    NODES="SCHEDULE_ON_CONTROL_NODES: $SCHEDULE_ON_CONTROL_NODES, SCHEDULABLE_NODES: $SCHEDULABLE_NODES, NUM_OF_CONTROLLER_NODES: $NUM_OF_CONTROLLER_NODES, NUM_OF_WORKER_NODES: $NUM_OF_WORKER_NODES"

    if is_in "$WITH_KAFKA" "yes,external" && [ ! "$NUM_OF_KAFKA" -le "$SCHEDULABLE_NODES" ]; then
      >&2 echo -e "${RED}${BOLD}${ERROR}ERROR:${NORMAL}${RED} Invalid setting of KAFKA replicas. NUM_OF_KAFKA (${NUM_OF_KAFKA}) is greater than the available nodes ($NODES)${NORMAL}"
      exit 1
    fi

    if is_in "$WITH_ETCD" "yes,external" && [ ! "$NUM_OF_ETCD" -le "$SCHEDULABLE_NODES" ]; then
      >&2 echo -e "${RED}${BOLD}${ERROR}ERROR:${NORMAL}${RED} Invalid setting of ETCD replicas. NUM_OF_ETCD (${NUM_OF_ETCD}) is greater than the available nodes ($NODES)${NORMAL}"
      exit 1
    fi

    if is_in "$WITH_ONOS" "yes,legacy,classic,micro"; then
       if [ ! "$NUM_OF_ATOMIX" -le "$SCHEDULABLE_NODES" ]; then
          >&2 echo -e "${RED}${BOLD}${ERROR}ERROR:${NORMAL}${RED} Invalid setting of ATOMIX replicas. NUM_OF_ATOMIX (${NUM_OF_ATOMIX}) is greater than the available nodes ($NODES)${NORMAL}"
          exit 1
        fi

        if [ ! "$NUM_OF_ONOS" -le "$SCHEDULABLE_NODES" ]; then
          >&2 echo -e "${RED}${BOLD}${ERROR}ERROR:${NORMAL}${RED} Invalid setting of ONOS replicas. NUM_OF_ONOS (${NUM_OF_ONOS}) is greater than the available nodes ($NODES)${NORMAL}"
          exit 1
        fi
    fi
fi

# normalize non-truth value options
if is_in "$WITH_KAFKA" "e,external"; then
    WITH_KAFKA="external"
fi
if is_in "$WITH_ETCD" "e,external"; then
    WITH_ETCD="external"
fi
if is_in "$WITH_ONOS" "l,legacy"; then
    WITH_ONOS="legacy"
elif is_in "$WITH_ONOS" "c,classic"; then
    WITH_ONOS="classic"
elif is_in "$WITH_ONOS" "u,m,micro"; then
    WITH_ONOS="micro"
fi

if [ "$WITH_ONOS" == "micro" ]; then
    >&2 echo -e "${YELLOW}${BOLD}${ERROR}WARNING:${NORMAL}${YELLOW} The value specified as WITH_ONOS, '$WITH_ONOS', is not valid.${NORMAL}"
    exit 1
fi

if is_in "$WITH_ONOS" "yes,classic"; then
    # there is an extra utility required if we are using ONOS CLASSIC as
    # the chart for classic specifies images differently
    EXTRA_TOOLS+=" yq"
fi

if [ "$LEGACY_BBSIM_INDEX" == "no" ]; then
    BBSIM_SADIS_SVC="bbsim0"
else
    BBSIM_SADIS_SVC="bbsim"
fi

SADIS_SUBSCRIBERS=${SADIS_SUBSCRIBERS:-http://$BBSIM_SADIS_SVC.$VOLTHA_NS.svc:50074/v2/subscribers/%s}
SADIS_BANDWIDTH_PROFILES=${SADIS_BANDWIDTH_PROFILES:-http://$BBSIM_SADIS_SVC.$VOLTHA_NS.svc:50074/v2/bandwidthprofiles/%s}

if [ "$HELM_USE_UPGRADE" == "yes" ]; then
    _HELM_DESC="Upgrade/Install"
else
    _HELM_DESC="Install"
fi

# Check for prerequiste tools
TOOLS="curl sed jq $EXTRA_TOOLS"
if [ "$DEPLOY_K8S" == "yes" ]; then
    TOOLS+=" docker"
fi
NOT_FOUND=""
for T in $TOOLS; do
    if [ -z "$(command -v "$T")" ]; then
        NOT_FOUND+=" $T"
    fi
done

if [ -n "$NOT_FOUND" ]; then
    >&2 echo -e "${RED}${BOLD}${ERROR}ERROR:${NORMAL}${RED} The following required tools where not found, please install them:$NOT_FOUND"
    exit 1
fi

TIMEOUT_SECONDS="$(parseDuration "$WAIT_TIMEOUT")"

mkdir -p .voltha
touch .voltha/ports
HAVE="$(grep "$NAME" .voltha/ports)"
if [ "$HAVE X" == " X" ]; then
    # Find free port prefix
    START=81
    while true; do
        if [ "$(grep -c $START .voltha/ports)" -eq 0 ]; then
            break
        fi
        START="$((START + 1))"
    done
    DELTA="$((START - 81))"
    ONOS_API_PORT=${ONOS_API_PORT:-${START}81}
    ONOS_SSH_PORT=${ONOS_SSH_PORT:-${START}01}
    VOLTHA_API_PORT=${VOLTHA_API_PORT:-5$((55 + DELTA))55}
    VOLTHA_SSH_PORT=${VOLTHA_SSH_PORT:-$((50 + DELTA))22}
    VOLTHA_ETCD_PORT=${VOLTHA_ETCD_PORT:-$((23 + DELTA))79}
    VOLTHA_KAFKA_PORT=${VOLTHA_KAFKA_PORT:-$((90 + DELTA))92}
    VOLTHA_PPROF_PORT=${VOLTHA_PPROF_PORT:-$((60 + DELTA))60}
    OPENOLT_PPROF_PORT=${OPENOLT_PPROF_PORT:-$((60 + DELTA))61}
    OFAGENT_PPROF_PORT=${OFAGENT_PPROF_PORT:-$((60 + DELTA))62}
    ELASTICSEARCH_PORT=${ELASTICSEARCH_PORT:-$((92 + DELTA))00}
    KIBANA_PORT=${KIBANA_PORT:-$((56 + DELTA))01}
    TRACING_GUI_PORT=${TRACING_GUI_PORT:-1$((66 + DELTA))86}
else
    VALUES="$(echo "$HAVE" | sed -e 's/\s//g' | cut -d= -f2)"
    ONOS_API_PORT=${ONOS_API_PORT:-$(echo "$VALUES" | cut -d, -f1)}
    ONOS_SSH_PORT=${ONOS_SSH_PORT:-$(echo "$VALUES" | cut -d, -f2)}
    VOLTHA_API_PORT=${VOLTHA_API_PORT:-$(echo "$VALUES" | cut -d, -f3)}
    VOLTHA_SSH_PORT=${VOLTHA_SSH_PORT:-$(echo "$VALUES" | cut -d, -f4)}
    VOLTHA_ETCD_PORT=${VOLTHA_ETCD_PORT:-$(echo "$VALUES" | cut -d, -f5)}
    VOLTHA_KAFKA_PORT=${VOLTHA_KAFKA_PORT:-$(echo "$VALUES" | cut -d, -f6)}
    VOLTHA_PPROF_PORT=${VOLTHA_PPROF_PORT:-$(echo "$VALUES" | cut -d, -f7)}
    OPENOLT_PPROF_PORT=${OPENOLT_PPROF_PORT:-$(echo "$VALUES" | cut -d, -f8)}
    OFAGENT_PPROF_PORT=${OFAGENT_PPROF_PORT:-$(echo "$VALUES" | cut -d, -f9)}
    ELASTICSEARCH_PORT=${ELASTICSEARCH_PORT:-$(echo "$VALUES" | cut -d, -f10)}
    KIBANA_PORT=${KIBANA_PORT:-$(echo "$VALUES" | cut -d, -f11)}
    TRACING_GUI_PORT=${TRACING_GUI_PORT:-$(echo "$VALUES" | cut -d, -f12)}

    # Some ports were added after the .voltha/ports files was created.
    # Calculate the original DELTA from the VOLTHA_SSH_PORT so that it can
    # be used to set up newer port-forwards.
    DELTA="$((VOLTHA_SSH_PORT/100-50))"

    if [ -z "$VOLTHA_KAFKA_PORT" ]; then
        VOLTHA_KAFKA_PORT=${VOLTHA_KAFKA_PORT:-$((90 + DELTA))92}
    fi

    if [ -z "$VOLTHA_PPROF_PORT" ]; then
        VOLTHA_PPROF_PORT=${VOLTHA_PPROF_PORT:-$((60 + DELTA))60}
    fi
    if [ -z "$OPENOLT_PPROF_PORT" ]; then
        OPENOLT_PPROF_PORT=${OPENOLT_PPROF_PORT:-$((60 + DELTA))61}
    fi
    if [ -z "$OFAGENT_PPROF_PORT" ]; then
        OFAGENT_PPROF_PORT=${OFAGENT_PPROF_PORT:-$((60 + DELTA))62}
    fi

    if [ -z "$ELASTICSEARCH_PORT" ]; then
        ELASTICSEARCH_PORT=${ELASTICSEARCH_PORT:-$((92 + DELTA))00}
    fi
    if [ -z "$KIBANA_PORT" ]; then
        KIBANA_PORT=${KIBANA_PORT:-$((56 + DELTA))01}
    fi

    if [ -z "$TRACING_GUI_PORT" ]; then
      TRACING_GUI_PORT=${TRACING_GUI_PORT:-1$((66 + DELTA))86}
    fi
fi

PORTTMP="$(mktemp -u)"
grep -v "$NAME" .voltha/ports > "$PORTTMP"
echo "$NAME=$ONOS_API_PORT,$ONOS_SSH_PORT,$VOLTHA_API_PORT,$VOLTHA_SSH_PORT,$VOLTHA_ETCD_PORT,$VOLTHA_KAFKA_PORT,$VOLTHA_PPROF_PORT,$OPENOLT_PPROF_PORT,$OFAGENT_PPROF_PORT,$ELASTICSEARCH_PORT,$KIBANA_PORT,$TRACING_GUI_PORT" >> "$PORTTMP"
cp "$PORTTMP" .voltha/ports
rm -f "$PORTTMP"

export ONOS_API_PORT ONOS_SSH_PORT

IDX=1
CLOCK="TIME:"
SPIN_PARTS=
NOT_VERIFIED=
THEX=
BUILD=
VERIFIED=
HELM=
OLD_KEY=
BIRD=
HIGH_VOLTAGE=
PLUG=
FORWARD=
GO=
DOWNLOAD=
GEAR=
NO_ENTRY=
LOCK=

if [ "$FANCY" -eq 1 ]; then
    SPIN_PARTS="\xe2\xa2\x8e\xe2\xa1\xb0 \xe2\xa2\x8e\xe2\xa1\xa1 \xe2\xa2\x8e\xe2\xa1\x91 \xe2\xa2\x8e\xe2\xa0\xb1 \xe2\xa0\x8e\xe2\xa1\xb1 \xe2\xa2\x8a\xe2\xa1\xb1 \xe2\xa2\x8c\xe2\xa1\xb1 \xe2\xa2\x86\xe2\xa1\xb1"
    CLOCK="\xe2\x8f\xb1"
    THEX="${RED}${BOLD}\xe2\x9c\x97\x20${NORMAL}"
    BUILD="${YELLOW}${BOLD}\xf0\x9f\x8f\x97${NORMAL}"
    NOT_VERIFIED="$BUILD"
    VERIFIED="${GREEN}${BOLD}\xe2\x9c\x93\x20${NORMAL}"
    HELM="${BLUE}${BOLD}\xE2\x8E\x88${NORMAL}"
    OLD_KEY="\xF0\x9F\x97\x9D"
    BIRD="\xF0\x9F\x90\xA6"
    HIGH_VOLTAGE="\xE2\x9A\xA1"
    PLUG="\xF0\x9F\xa7\xa9"
    FORWARD="\xE2\x87\xA8"
    GO="\xf0\x9f\x9a\x80"
    DOWNLOAD="\xf0\x9f\x93\xa5"
    GEAR="\xe2\x9a\x99"
    NO_ENTRY="\xe2\x9b\x94"
    LOCK="\xf0\x9f\x94\x92"
fi

duration() {
    local h m s t
    h="$(($1 / 3600))"
    m="$(($1 % 3600 / 60))"
    s="$(($1 % 60))"
    t=""

    if [ "$h" -gt 0 ]; then
        t="$t${h}h"
    fi
    if [ "$m" -gt 0 ]; then
        t="$t${m}m"
    fi
    echo "$t${s}s"
}

printtime() {
    local INDENT
    if [ "$1" == "-" ]; then
        INDENT="  "
        shift
    fi
    echo -e "$INDENT  $CLOCK  $(duration "$1")"
}

bspin() {
    IDX=1
    local INDENT
    if [ "$1" == "-" ]; then
        INDENT="  "
        shift
    fi
    if [ "$FANCY" -eq 0 ]; then
        LINE="${*//[[:space:]+-]}"
        if [ "$LINE X" == " X" ]; then
            return
        fi
        echo -e "$CIVIS$INDENT$*"
    else
        echo -en "$CIVIS$INDENT   $*"
    fi
}

sspin() {
    local INDENT
    if [ "$1" == "-" ]; then
        INDENT="  "
        shift
    fi
    if [ "$FANCY" -eq 0 ]; then
        LINE="${*//[[:space:]+-]}"
        if [ "$LINE X" == " X" ]; then
            return
        fi
        echo -e "$INDENT$*"
    else
        C="$(echo "$SPIN_PARTS" | cut '-d ' -f "$IDX")"
        echo -en "\r$INDENT$C $*"
        IDX="$((IDX + 1))"
        if [ "$IDX" -gt 8 ]; then
            IDX=1
        fi
    fi
}

espin() {
    local INDENT
    if [ "$1" == "-" ]; then
        INDENT="  "
        shift
    fi
    if [ "$FANCY" -eq 0 ]; then
        LINE="${*//[[:space:]+-]}"
        if [ "$LINE X" == " X" ]; then
            return
        fi
        echo -e "$INDENT$*"
    else
        echo -e "\r$INDENT$*$CNORM"
    fi
}

if [ "$1" == "get" ] && [ "$2" == "voltconfig" ]; then
    echo "$HOME/.volt/config-$NAME"
    exit
fi

if [ $# -ne 1 ] || ! is_in "$1" "up,down,dump,clean"; then
    >&2 echo "What wouild you like to do today:"
    >&2 echo "  up   - bring up voltha"
    >&2 echo "  down - tear down voltha"
    >&2 echo "  dump - create a debug dump of running system"
    exit 1
fi

do_curl() {
    local DATA_ARGS CREDS CMD_ECHO CMD_OUTPUT SC_OUTPUT WAIT_START INDENTA EXTRAA NOW
    local INDENT=$1
    local OP=$2
    local USER=$3
    local PASS=$4
    local URL=$5
    local DATA_TYPE=$6
    local DATA=$7
    local MSG=$8
    local VALID_RESULTS=$9
    local EXTRA=${10}
    local ICON=${11}

    # Turn into an array for passing
    INDENTA=()
    if [ -n "$INDENT" ]; then
        INDENTA=("$INDENT")
    fi
    IFS=' ' read -r -a EXTRAA <<< "$EXTRA"

    # Thanks to the latest version of ONOS using the return code 207 this gets a
    # whole lot nastier. Can't thank them enough for doing this. So in order to
    # capture the command and the output in the log file as well as capture the
    # status code to verify it is 200 and not 207 mutltiple files and a bit of
    # hackery must be used. Thanks again ONOS.
    CMD_ECHO="$(mktemp -u)"
    CMD_OUTPUT="$(mktemp -u)"
    SC_OUTPUT="$(mktemp -u)"
    WAIT_START="$(date +%s)"

    CREDS=()
    if [ -n "$USER" ]; then
        CREDS=("--user" "$USER:$PASS")
    fi

    DATA_ARGS=()
    if [ -n "$DATA" ]; then
        if [ "$DATA_TYPE" == "file" ]; then
            DATA_ARGS=("--data" "@$DATA")
        elif [ "$DATA_TYPE" == "json" ]; then
            DATA_ARGS=("--data" "$DATA")
        fi
    fi

    bspin "${INDENTA[@]}" "$MSG" "$ICON"
    while true; do
        NOW="$(date +%s)"
        if [ "$((NOW - WAIT_START))" -gt "$TIMEOUT_SECONDS" ]; then
            espin "${INDENTA[@]}" "$THEX"
            rm -f "$CMD_ECHO" "$CMD_OUTPUT" "$SC_OUTPUT"
            doTimeout "$MSG"
        fi
        (set -x; curl -sSL "${CREDS[@]}" -w "%{http_code}" -o "$CMD_OUTPUT" -X "$OP" "${EXTRAA[@]}" "$URL" "${DATA_ARGS[@]}" >"$SC_OUTPUT" 2>/dev/null) >>"$CMD_ECHO" 2>&1
        RESULT=$?
        # Dump everything to the log
        cat "$CMD_ECHO" >> "$LOG"
        test -r "$CMD_OUTPUT" && cat "$CMD_OUTPUT" >> "$LOG"
        SC="$(cat "$SC_OUTPUT")"
        echo "RESPONSE CODE: $SC" >> "$LOG"
        echo "ERROR CODE: $RESULT" >> "$LOG"

        # clean up temp files
        rm -f "$CMD_ECHO" "$CMD_OUTPUT" "$SC_OUTPUT"
        if [ "$RESULT" -eq 0 ] && is_in "$SC" "$VALID_RESULTS"; then
            break
        fi
        sleep 1
        sspin "${INDENTA[@]}"
    done
    espin "${INDENTA[@]}" "$VERIFIED"
}

push_onos_config() {
    local DATA_TYPE=$1
    local MSG=$2
    local RESOURCE=$3
    local DATA=$4

    do_curl "-" "POST" "karaf" "karaf" "http://$_ONOS_API_EP/onos/v1/$RESOURCE" "$DATA_TYPE" "$DATA" "$MSG" "200" "--fail -H Content-Type:application/json" "$GEAR"
}

check_onos_app_active() {
    local APP_ID WAIT_START RESULT NOW
    APP_ID=$1
    WAIT_START="$(date +%s)"

    bspin - "Checking that $APP_ID is active $CLOCK"
    while true; do
        NOW="$(date +%s)"
        if [ "$((NOW - WAIT_START))" -gt "$TIMEOUT_SECONDS" ]; then
            espin - "$THEX"
            doTimeout "waiting for ONOS application activation '$APP_ID'"
        fi
        if (set -x; curl --fail -sSL --user karaf:karaf -X GET "http://$_ONOS_API_EP/onos/v1/applications/$APP_ID" | grep ACTIVE >>"$LOG" 2>&1) >>"$LOG" 2>&1; then
            break
        fi
        sleep 1
        sspin -
    done
    sleep 5 # OSGI components take a little longer that the app to activate
    espin - "$VERIFIED"
}

override_onos_app() {
    local APP NAME WAIT_START NOW
    APP=$1
    NAME="$(basename "$APP" | sed -e 's/^[0-9][0-9]*-//g' -e 's/-.*$//g')"
    WAIT_START="$(date +%s)"

    while true; do
        NOW="$(date +%s)"
        if [ "$((NOW - WAIT_START))" -gt "$TIMEOUT_SECONDS" ]; then
            espin - "$THEX"
            doTimeout "waiting to overwrite ONOS application '$APP'"
        fi
        sspin -
        # Attempt to delete old version (if it exists)
        (set -x; curl --fail -sSL --user karaf:karaf -X DELETE "http://$_ONOS_API_EP/onos/v1/applications/$NAME" >>"$LOG" 2>&1) >>"$LOG" 2>&1
        RESULT=$?
        sspin -
        if [ $RESULT -ne 0 ]; then
            continue
        fi
        if (set -x; curl --fail -sSL --user karaf:karaf -X POST -H Content-Type:application/octet-stream "http://$_ONOS_API_EP/onos/v1/applications?activate=true" --data-binary "@$APP" >>"$LOG" 2>&1) >>"$LOG" 2>&1; then
            break
        fi
        sleep .2
    done
}

activate_onos_app() {
    local MSG APP WAIT_START NOW
    MSG="$1"
    APP=$2
    WAIT_START="$(date +%s)"

    bspin - "$MSG $GO"
    while true; do
        NOW="$(date +%s)"
        if [ "$((NOW - WAIT_START))" -gt "$TIMEOUT_SECONDS" ]; then
            espin - "$THEX"
            doTimeout "waiting to activate ONOS application '$APP'"
        fi
        sspin -
        if (set -x; curl --fail -sSL --user karaf:karaf -X POST "http://$_ONOS_API_EP/onos/v1/applications/$APP/active" >>"$LOG" 2>&1) >>"$LOG" 2>&1; then
            break
        fi
        sleep .2
    done
    espin - "$VERIFIED"
}

count_pods() {
    local NAMESPACE STATES LABELS CMD PODS
    NAMESPACE=$1; shift
    if [ "$NAMESPACE" == "all-namespaces" ]; then
        NAMESPACE="--all-namespaces"
    else
        NAMESPACE="-n $NAMESPACE"
    fi
    STATES=$1; shift
    LABELS=$1; shift
    CMD=("kubectl get $NAMESPACE $LABELS pod")
    PODS=$(${CMD[*]} -o go-template="{{range .items}}{{.metadata.name}}/{{.status.phase}}/_{{range .status.containerStatuses}}{{.ready}}_{{end}} {{end}}")
    local COUNT=0
    local PATTERNS="$*"
    for POD in $PODS; do
        local NAME STATE CONTAINERS TOTAL FOUND
        NAME="$(echo "$POD" | cut -d/ -f 1)"
        STATE="$(echo "$POD" | cut -d/ -f 2)"
        CONTAINERS="$(echo "$POD" | cut -d/ -f 3 | sed -e 's/_/ /g')"
        if [ "$STATES" == "*" ] || [ "$(echo "$STATES" | grep -c ":$STATE:")" -ne 0 ]; then
            TOTAL="$(echo "$CONTAINERS" | wc -w)"
            FOUND="$(echo "$CONTAINERS" | grep -o true | wc -l)"
            if [ "$TOTAL" -eq "$FOUND" ]; then
                for PATTERN in $PATTERNS; do
                    if [[ "$NAME" =~ $PATTERN ]]; then
                        COUNT="$((COUNT + 1))"
                        break
                    fi
                done
            fi
        fi
    done
    echo $COUNT
}

wait_for_pods() {
    local INDENT NAMESPACE EXPECT TYPE MESSAGE LABELS PATTERNS STATES HAVE ALL WAIT_START NOW
    if [ "$1" == "-" ]; then
        INDENT=$1; shift
    fi
    NAMESPACE=$1; shift
    EXPECT=$1; shift
    TYPE=$1; shift
    MESSAGE=$1; shift
    LABELS=$1; shift
    PATTERNS=("$*")
    STATES=":Running:"
    if [ "$TYPE" == "not" ]; then
        STATES="*"
    fi
    HAVE="$(count_pods "$NAMESPACE" "$STATES" "$LABELS" "${PATTERNS[@]}")"
    ALL=$HAVE
    if [ "$TYPE" == "only" ]; then
        ALL="$(count_pods "all-namespaces" "*" ".*")"
    fi
    COUNT="$((300 / 15))"
    WAIT_START="$(date +%s)"
    bspin "$INDENT" "$MESSAGE"
    sspin "$INDENT"
    if [ "$HAVE" -ne "$EXPECT" ] || [ "$ALL" -ne "$HAVE" ]; then
        while [ "$HAVE" -ne "$EXPECT" ] || [ "$ALL" -ne "$HAVE" ]; do
            NOW="$(date +%s)"
            if [ "$((NOW - WAIT_START))" -gt "$TIMEOUT_SECONDS" ]; then
                espin "$INDENT" "$THEX"
                doTimeout "waiting for PODs to start"
            fi
            sspin "$INDENT"
            COUNT="$((COUNT - 1))"
            if [ "$COUNT" -eq 0 ]; then
                HAVE="$(count_pods "$NAMESPACE" "$STATES" "$LABELS" "${PATTERNS[@]}")"
                ALL="$HAVE"
                if [ "$TYPE" == "only" ]; then
                    ALL="$(count_pods "all-namespaces" "*" ".*")"
                fi
                COUNT="$((300 / 15))"
            fi
            sleep .15
        done
    fi
    espin "$INDENT" "$VERIFIED"
    if [ "$HAVE" -ne "$EXPECT" ]; then
        return 1
    fi
    return 0
}

port_forward() {
    local NS=$1; shift
    local SVC=$1; shift
    local PORTS="$*"
    local TAG=$SVC-$NAME

    (set -x; _TAG="$TAG" bash -c "while true; do kubectl port-forward --address $PF_ADDRESS -n $NS service/$SVC $PORTS; done" >>"$PFLOG" 2>&1 &) >>"$PFLOG" 2>&1
}

kill_port_forward() {
    local TAG P_IDS PARENTS KIDS UNKNOWN PP_ID PF

    while [ $# -gt 0 ]; do
        PF=$1; shift
        TAG=
        if [ "$PF" != "__ALL__" ]; then
            TAG="$PF-$NAME"
        fi
        PARENTS=
        KIDS=
        UNKNOWN=
        # shellcheck disable=SC2009
        P_IDS="$(ps e -ww -A | grep "_TAG=$TAG" | grep -v grep | awk '{print $1}')"
        if [ -n "$P_IDS" ]; then
            for P_ID in $P_IDS; do
                PP_ID="$(ps -o ppid "$P_ID" | tail -n +2)"
                if [ -n "$PP_ID" ]; then
                    if [ "$PP_ID" -eq 1 ]; then
                        PARENTS="$PARENTS $P_ID"
                    else
                        KIDS="$KIDS $P_ID"
                    fi
                else
                    UNKNOWN="$UNKNOWN $P_ID"
                fi
            done
            if [ -n "$PARENTS" ]; then
                # shellcheck disable=SC2086
                while ps -h $PARENTS >/dev/null 2>&1; do
                    (set -x; eval "kill -9 $PARENTS" >>"$LOG" 2>&1) >>"$LOG" 2>&1
                done
            fi
            if [ -n "$KIDS" ]; then
                # shellcheck disable=SC2086
                while ps -h $KIDS >/dev/null 2>&1; do
                    (set -x; eval "kill -9 $KIDS" >>"$LOG" 2>&1) >>"$LOG" 2>&1
                done
            fi
            if [ -n "$UNKNOWN" ]; then
                # shellcheck disable=SC2086
                while ps -h $UNKNOWN >/dev/null 2>&1; do
                    (set -x; eval "kill -9 $UNKNOWN" >>"$LOG" 2>&1) >>"$LOG" 2>&1
                done
            fi
        fi
    done
}

resolve_chart_name() {
    helm inspect chart "$1" | grep "^name:" | awk '{print $2}'
}

resolve_chart_version() {
    [ "$2" != "latest" ] && echo "$2" && return
    helm inspect chart "$1" | grep "^version:" | awk '{print $2}'
}

# Compares two semver-formatted version strings (e.g. 2.4.3)
# Returns 0 if $1 == $2, returns 1 if $1 > $2 and returns 2 if $1 < $2
compare_versions() {
    [ "$1" == "$2" ] && return 0
    local lowest
    lowest=$(echo -e "$1\n$2" | sort -V | head -1)
    [ "$lowest" == "$1" ] && return 2
    return 1
}

if [ "$1" == "down" ]; then
    echo "Tearing down voltha cluster $NAME"
    LOG="down-$NAME.log"
    date -u +"%Y%m%dT%H%M%SZ" >"$LOG"
    HELM_MAJOR=$(helm version --client --short | sed -E -e 's/^.*v([0-9]+)\.[0-9]+\.[0-9]+.*$/\1/')
    if is_in "$WITH_ONOS" "yes,legacy,classic"; then
        bspin "Remove port-forwards: onos-ui-$NAME"
        kill_port_forward onos-ui
        sspin "Remove port-forwards: onos-ssh-$NAME$CEOL"
        kill_port_forward onos-ssh
        sspin "Remove port-forwards: onos-onos-classic-hs-$NAME$CEOL"
        kill_port_forward onos-onos-classic-hs
    fi
    sspin "Remove port-forwards: voltha-api-$NAME$CEOL"
    kill_port_forward voltha-api voltha-voltha-api
    if is_in "$WITH_ETCD" "yes,external"; then
        sspin "Remove port-forwards: etcd-$NAME$CEOL"
        kill_port_forward "etcd"
    fi
    if is_in "$WITH_KAFKA" "yes,external"; then
        sspin "Remove port-forwards: kafka-$NAME$CEOL"
        kill_port_forward "kafka"
    fi

    if [ "$WITH_PPROF" == "yes" ]; then
        sspin "Remove port-forwards: *-profiler$CEOL"
        kill_port_forward voltha-rw-core-profiler voltha-voltha-rw-core-profiler
        kill_port_forward voltha-of-agent-profiler voltha-voltha-of-agent-profiler
        if [ "$WITH_OPEN_ADAPTERS" == "yes" ]; then
            kill_port_forward adapter-open-olt-profiler open-olt-adapter-open-olt-profiler
        fi
    fi

    if [ "$WITH_TRACING" == "yes" ]; then
        sspin "Remove port-forwards: tracing-$NAME$CEOL"
        VOLTHA_TRACING_CHART_NAME=$(resolve_chart_name "$VOLTHA_TRACING_CHART")
        kill_port_forward "tracing-${VOLTHA_TRACING_CHART_NAME}-jaeger-gui"
    fi

    espin "$VERIFIED Remove port-forwards$CEOL"
    if [ "$DEPLOY_K8S" == "yes" ]; then
        if [ -x ./bin/kind ]; then
            bspin "Delete Kubernetes Kind Cluster"
            (set -x; ./bin/kind delete cluster --name "voltha-$NAME" >>"$LOG" 2>&1) >>"$LOG" 2>&1
            espin "$VERIFIED"
        else
            espin "$NO_ENTRY Delete Kubernetes Kind Cluster: kind command not found"
        fi
    else
        if [ "$CONFIG_SADIS" == "external" ]; then
            bspin "Remove configmap for BBSIM SADIS server"
            (set -x; kubectl -n "$BBSIM_NS" delete --ignore-not-found configmap kube-config >>"$LOG" 2>&1) >>"$LOG" 2>&1
            espin "$VERIFIED"
        fi

        if [ "$HELM_MAJOR" -le 2 ]; then
            EXISTS=$(helm list --output json 2>/dev/null | jq -r '.Releases | .[] | .Name + "," + .Namespace')
        else
            EXISTS=$(helm list --all-namespaces --output json 2>/dev/null | jq -r '.[] | .name + "," + .namespace')
        fi
        EXPECT="voltha"
        if is_in "$WITH_ONOS" "yes,legacy,classic"; then
            EXPECT+=" onos"
        fi
        if [ "$WITH_RADIUS" == "yes" ]; then
            EXPECT+=" radius"
        fi
        if [ "$CONFIG_SADIS" == "external" ]; then
            EXPECT+=" bbsim-sadis-server"
        fi
        if [ "$WITH_BBSIM" == "yes" ]; then
            EXPECT+=" bbsim"
        fi
        if [ "$WITH_EFK" == "yes" ]; then
            EXPECT+=" elasticsearch kibana fluentd"
        fi
        if [ "$WITH_TRACING" == "yes" ]; then
            EXPECT+=" tracing"
        fi
        if [ "$WITH_OPEN_ADAPTERS" == "yes" ]; then
            EXPECT+=" open-olt open-onu"
        fi
        if [ "$WITH_SIM_ADAPTERS" == "yes" ]; then
            EXPECT+=" sim"
        fi
        if is_in "$WITH_ETCD" "yes,external"; then
            EXPECT+=" etcd"
        fi
        if is_in "$WITH_KAFKA" "yes,external"; then
            EXPECT+=" kafka"
        fi
        bspin "Remove Helm Deployments"
        for i in $EXISTS; do
            for j in $EXPECT; do
                if [[ "$i" =~ $j ]]; then
                    __NAME=$(echo "$i" | cut -d, -f1)
                    __NS=$(echo "$i" | cut -d, -f2)
                    sspin "Remove Helm Deployments: $__NS:$__NAME$CEOL"
                    if [ "$HELM_MAJOR" -le 2 ]; then
                        (set -x; helm delete --no-hooks --purge "$__NAME" >>"$LOG" 2>&1) >>"$LOG" 2>&1
                    else
                        (set -x; helm uninstall --no-hooks --namespace "$__NS" "$__NAME" >>"$LOG" 2>&1) >>"$LOG" 2>&1
                    fi
                fi
            done
        done
        espin "$VERIFIED Remove Helm Deployments$CEOL"
        if [ "$WAIT_ON_DOWN" == "yes" ]; then
            PODS="voltha-ofagent.* voltha-rw-core.*"
            INFRA_PODS=
            ADAPT_PODS=
            SIM_PODS=
            EFK_PODS=
            if [ "$WITH_RADIUS" == "yes" ]; then
                PODS+=" radius.*"
            fi
            if [ "$WITH_BBSIM" == "yes" ]; then
                SIM_PODS+=" bbsim.*"
            fi
            if [ "$WITH_EFK" == "yes" ]; then
                EFK_PODS+=" kibana-* elasticsearch-* fluentd-*"
            fi
            if [ "$WITH_OPEN_ADAPTERS" ] || [ "$WITH_SIM_ADAPTERS" ]; then
                ADAPT_PODS+=" adapter-*"
            fi
            if [ "$WITH_TRACING" == "yes" ]; then
                INFRA_PODS+=" jaeger.*"
            fi
            if is_in "$WITH_ONOS" "yes,legacy,classic"; then
                INFRA_PODS+=" onos-.*"
            fi
            if is_in "$WITH_ETCD" "yes,external"; then
                INFRA_PODS+=" etcd.*"
            fi
            if is_in "$WITH_KAFKA" "yes,external"; then
                INFRA_PODS+=" kafka.*"
            fi
            if [ -n "$SIM_PODS" ]; then
                wait_for_pods "$BBSIM_NS" 0 "not" "Waiting for BBSIM PODs to terminate" "$BBSIM_LABEL" "$SIM_PODS"
            fi
            if [ -n "$INFRA_PODS" ]; then
                wait_for_pods "$INFRA_NS" 0 "not" "Waiting for infrastructure PODs to terminate" "$NO_LABEL" "$INFRA_PODS"
            fi
            if [ -n "$ADAPT_PODS" ]; then
                wait_for_pods "$ADAPTER_NS" 0 "not" "Waiting for adapter  PODs to terminate" "$NO_LABEL" "$ADAPT_PODS"
            fi
            if [ -n "$EFK_PODS" ]; then
                wait_for_pods "$INFRA_NS" 0 "not" "Waiting for EFK PODs to terminate" "$NO_LABEL" "$EFK_PODS"
            fi
            wait_for_pods "$VOLTHA_NS" 0 "not" "Waiting for VOLTHA PODs to terminate" "$NO_LABEL" "$PODS"
        fi
    fi
    exit
fi

if [ "$1" == "dump" ]; then
    LOG="dump-$NAME.log"
    TS="$(date -u +"%Y%m%dT%H%M%SZ")"
    if [ -n "$DUMP_FROM" ]; then
        TS=${DUMP_FROM//[:-]}
    fi
    WORK="$(mktemp -u -d)"
    DATA=$WORK/voltha-debug-dump-$NAME-$TS
    mkdir -p "$DATA"
    echo "$TS" > "$LOG"
    echo -e "Capturing debug dump to voltha-debug-dump-$NAME-$TS.tgz"
    bspin - "Copy install log"
    if [ -f "install-$NAME.log" ]; then
        (set -x; cp "install-$NAME.log" "$DATA/install-$NAME.log") >>"$LOG" 2>&1
        espin - "$VERIFIED"
    else
        espin - "$NO_ENTRY Copy install log: install-$NAME.log not found"
    fi
    bspin - "Dumping Kubernetes PODs"
    (set -x; kubectl get --all-namespaces pods >> "$DATA/all-pods.txt" 2>&1) >>"$LOG" 2>&1
    espin - "$VERIFIED"
    bspin - "Dumping Kubernetes SERVICEs"
    (set -x; kubectl get --all-namespaces svc >> "$DATA/all-services.txt" 2>&1) >>"$LOG" 2>&1
    espin - "$VERIFIED"
    bspin - "Dumping Kubernetes EVENTs"
    (set -x; kubectl get --all-namespaces events >> "$DATA/all-events.txt" 2>&1) >>"$LOG" 2>&1
    espin - "$VERIFIED"
    bspin - "Dumping VOLTHA POD details"
    PODS="$(kubectl -n "$INFRA_NS" get pod -o name | grep onos | sed -e "s/^/$INFRA_NS:/g") $(kubectl get -n "$VOLTHA_NS" pod -o name | sed -e "s/^/$VOLTHA_NS:/g")"
    SINCE=
    if [ -n "$DUMP_FROM" ]; then
        SINCE="--since-time=$DUMP_FROM"
    fi
    for POD in $PODS; do
        NS="$(echo "$POD" | cut -d: -f1)"
        POD="$(echo "$POD" | cut -d: -f2)"
        sspin - "Dumping VOLTHA POD details: $POD$CEOL"
        mkdir -p "$DATA/$POD"
        (set -x; kubectl describe -n "$NS" "$POD" >> "$DATA/$POD/describe.txt" 2>&1) >>"$LOG" 2>&1
        sspin - "Dumping VOLTHA POD details: $POD"
        (set -x; kubectl logs -n "$NS" --all-containers "$SINCE" --previous "$LOG_ARGS" "$POD" >> "$DATA/$POD/logs-previous.txt" 2>&1) >>"$LOG" 2>&1
        sspin - "Dumping VOLTHA POD details: $POD"
        (set -x; kubectl logs -n "$NS" --all-containers "$SINCE" "$LOG_ARGS" "$POD" >> "$DATA/$POD/logs-current.txt" 2>&1) >>"$LOG" 2>&1
        sspin - "Dumping VOLTHA POD details: $POD"
    done
    espin - "$VERIFIED Dumping VOLTHA POD details$CEOL"
    bspin - "Dumping ETCD"
    if [ -n "$(command -v etcdctl)" ]; then
        (set -x; ETCDCTL_API=3 etcdctl --endpoints "localhost:$VOLTHA_ETCD_PORT" get --prefix service/voltha | hexdump -C >> "$DATA/etcd.hex" 2>&1) >>"$LOG" 2>&1
        espin - "$VERIFIED"
    else
        espin - "$NO_ENTRY Dumping ETCD: etcdctl command not available"
    fi
    bspin - "Creating compressed TAR: voltha-debug-dump-$NAME-$TS.tgz"
    (set -x; tar -C "$WORK" -zcf "voltha-debug-dump-$NAME-$TS.tgz" "./voltha-debug-dump-$NAME-$TS") >>"$LOG" 2>&1
    espin - "$VERIFIED"
    bspin - "Cleanup"
    (set -x; rm -rf "$WORK") >>"$LOG" 2>&1
    espin - "$VERIFIED"
    bspin - "$(ls -l "voltha-debug-dump-$NAME-$TS.tgz")"
    espin - "$VERIFIED"
    exit
fi

LOG="install-$NAME.log"
PFLOG="port-forward-$NAME.log"
date > "$LOG"
echo "PORTS=$ONOS_API_PORT,$ONOS_SSH_PORT,$VOLTHA_API_PORT,$VOLTHA_SSH_PORT,$VOLTHA_ETCD_PORT,$VOLTHA_KAFKA_PORT,$VOLTHA_PPROF_PORT,$OPENOLT_PPROF_PORT,$OFAGENT_PPROF_PORT,$ELASTICSEARCH_PORT,$KIBANA_PORT,$TRACING_GUI_PORT" >> "$LOG"

# Output install options to log
echo "OPTIONS" >> "$LOG"
for O in $ALL_OPTIONS; do
    VAL="$(eval echo "\$$O")"
    if [ -n "$VAL" ]; then
        printf "    %-30s = %s\n" "$O" "$VAL" >> "$LOG"
    fi
done

# Force remove any existing k8s port-forward processes. This will
# remove them for all kind clusters.
if [ "$1" == "clean" ]; then
    kill_port_forward __ALL__
    exit
fi

helm_values() {
    local NAMESPACE=$1; shift
    local INAME=$1; shift
    local CHART=$1; shift
    local CHART_VERSION=$1; shift

    if [ "$CHART_VERSION X" != " X" ] && [ "$CHART_VERSION" != "latest" ]; then
        CHART_VERSION="--version $CHART_VERSION"
    else
        CHART_VERSION=
    fi

    local CHART_ARGS=
    if [ -r "${INAME}-values.yaml" ]; then
        CHART_ARGS="-f ${INAME}-values.yaml"
    fi

    if [ "$HELM_USE_UPGRADE" == "yes" ]; then
        _HELM_COMMAND="upgrade --install"
        _HELM_ARGS="xxxdryrunxxx"
    else
        _HELM_COMMAND="install"
        if [ "$HELM_MAJOR" -le 2 ]; then
            _HELM_NAME_ARG="--name xxxdryrunxxx"
            _HELM_ARGS=
        else
            _HELM_NAME_ARG="xxxdryrunxxx"
            _HELM_ARGS="--create-namespace"
        fi
    fi
    CMD=("helm $_HELM_COMMAND --debug --dry-run -f values.yaml $CHART_ARGS $INTERNAL_EXTRA_HELM_INSTALL_ARGS $EXTRA_HELM_INSTALL_ARGS --set defaults.log_level=$VOLTHA_LOG_LEVEL --namespace $NAMESPACE $_HELM_ARGS  $CHART_VERSION $EXTRA_HELM_FLAGS $_HELM_NAME_ARG $CHART")
   ${CMD[*]} 2>/dev/null | awk 'PRINT==1 {print}; /^USER-SUPPLIED VALUES:/ {PRINT = 1}; /^$/ {PRINT = 0}'
}

helm_is_deployed() {
    local NAMESPACE=$1; shift
    local NAME=$1; shift

    if [ "$HELM_MAJOR" -le 2 ]; then
        helm list --deployed --short --namespace "$NAMESPACE" "$NAME" 2>/dev/null | wc -l
    else
        helm list --deployed --short --namespace "$NAMESPACE" --filter "$NAME" 2>/dev/null | wc -l
    fi
}

helm_install() {
    local INDENT NOW VALUES_FILE PPROF_ARG
    if [ "$1" == "-" ]; then
        INDENT=$1; shift
    fi
    local NAMESPACE=$1; shift
    local INAME=$1; shift
    local CHART=$1; shift
    local CHART_VERSION=$1; shift
    local FILTER=$1; shift
    local MESSAGE=$*

    if [ "$CHART_VERSION X" != " X" ] && [ "$CHART_VERSION" != "latest" ]; then
        CHART_VERSION="--version $CHART_VERSION"
    else
        CHART_VERSION=
    fi

    local CHART_ARGS=
    if [ -r "${INAME}-values.yaml" ]; then
        CHART_ARGS="-f ${INAME}-values.yaml"
    fi

    local WAIT_START CMD
    WAIT_START="$(date +%s)"
    COUNT="$((300 / 15))"
    bspin "$INDENT" "$MESSAGE"
    if [ "$HELM_USE_UPGRADE" == "yes" ]; then
        _HELM_COMMAND="upgrade --install"
        _HELM_NAME_ARG="$INAME"
    else
        _HELM_COMMAND="install"
        if [ "$HELM_MAJOR" -le 2 ]; then
            _HELM_NAME_ARG="--name $INAME"
            _HELM_ARGS=
        else
            _HELM_NAME_ARG="$INAME"
            _HELM_ARGS="--create-namespace"
        fi
    fi

    VALUES_FILE="$(mktemp)"
    if [ "$FILTER" == "-" ]; then
        cp "values.yaml" "$VALUES_FILE"
    elif [ "${FILTER:0:1}" == "+" ]; then
        cp "values.yaml" "$VALUES_FILE"
        yq r - "${FILTER:1}" <"values.yaml" >>"$VALUES_FILE"
        cat "$VALUES_FILE" >>"$LOG"
    else
        yq r - "$FILTER" <"values.yaml" >"$VALUES_FILE"
    fi
    if [ "$WITH_PPROF" == "yes" ]; then
        PPROF_ARG="--set profiler.enabled=true"
    fi
    CMD=("helm $_HELM_COMMAND -f $VALUES_FILE $_HELM_ARGS $CHART_ARGS $PPROF_ARG $INTERNAL_EXTRA_HELM_INSTALL_ARGS $EXTRA_HELM_INSTALL_ARGS --set defaults.log_level=$VOLTHA_LOG_LEVEL --namespace $NAMESPACE $CHART_VERSION $EXTRA_HELM_FLAGS $_HELM_NAME_ARG $CHART")
    (set -x; ${CMD[*]} >>"$LOG" 2>&1) >>"$LOG" 2>&1
    SUCCESS=$?
    while [ "$SUCCESS" -ne 0 ]; do
        NOW="$(date +%s)"
        if [ "$((NOW - WAIT_START))" -gt "$TIMEOUT_SECONDS" ]; then
            espin "$THEX"
            rm -f "$VALUES_FILE"
            doTimeout "waiting for helm install $CHART"
        fi
        sspin "$INDENT"
        COUNT="$((COUNT - 1))"
        if [ "$COUNT" -eq 0 ]; then
            if [ "$HELM_USE_UPGRADE" == "no" ]; then
                if [ "$HELM_MAJOR" -le 2 ]; then
                    (set -x; helm delete --purge "$INAME" >>"$LOG" 2>&1) >>"$LOG" 2>&1
                else
                    (set -x; helm uninstall "$INAME" >>"$LOG" 2>&1) >>"$LOG" 2>&1
                fi
            fi
            (set -x; ${CMD[*]} >>"$LOG" 2>&1) >>"$LOG" 2>&1
            SUCCESS=$?
            COUNT="$((300 / 15))"
        fi
        sleep .15
    done
    rm -f "$VALUES_FILE"
    espin "$INDENT" "$VERIFIED"
}

echo "INSTALL NAME: $NAME" >> "$LOG"

STIME="$(date +%s)"
if [ "$INSTALL_KUBECTL" == "no" ]; then
    bspin "Skip kubectl install"
    espin $NO_ENTRY
else
    bspin "Verify kubectl $HELM"
    if [ -x "$GOPATH/bin/kubectl" ]; then
        espin "$VERIFIED"
    else
        ERR_OUT="$(mktemp)"
        espin "$NOT_VERIFIED"
        bspin - "Download and install Kubernetes/kubectl $DOWNLOAD"
        (set -x; curl --fail -o "$GOPATH/bin/kubectl" -sSL "https://storage.googleapis.com/kubernetes-release/release/$(curl --fail -sSL https://storage.googleapis.com/kubernetes-release/release/stable.txt 2>>"$ERR_OUT")/bin/$HOSTOS/$HOSTARCH/kubectl" >>"$LOG" 2>>"$ERR_OUT") >>"$LOG" 2>&1
        RESULT=$?
        if [ "$RESULT" -ne 0 ]; then
            espin - "$THEX"
            >&2 echo -e "${RED}${BOLD}${ERROR}ERROR: unable to download kubectl: $(tail "$ERR_OUT")${NORMAL}"
            cat "$ERR_OUT" >> "$LOG"
            rm -rf "$ERR_OUT" "$GOPATH/bin/kubectl"
            exit 1
        fi
        (set -x; chmod 755 "$GOPATH/bin/kubectl" >>"$LOG" 2>&1) >>"$LOG" 2>&1
        espin - "$VERIFIED"
        rm -rf "$ERR_OUT"
    fi
fi
if [ "$WITH_TIMINGS" == "yes" ]; then
    NOW="$(date +%s)"
    printtime $((NOW - STIME))
fi

STIME="$(date +%s)"
if [ "$DEPLOY_K8S" == "no" ]; then
    bspin "Skip Kubernetes/Kind Deployment"
    espin $NO_ENTRY
else
    bspin "Verify Kubernetes/Kind $HELM"
    IS_INSTALLED=0
    OP_TYPE="install"
    if [ -x "$GOPATH/bin/kind" ]; then
        OP_TYPE="upgrade"
        if [ "$("$GOPATH/bin/kind" version | grep -c "$KIND_VERSION")" -eq 1 ]; then
            IS_INSTALLED=1
            espin "$VERIFIED"
        fi
    fi
    if [ "$IS_INSTALLED" -eq 0 ]; then
        ERR_OUT="$(mktemp)"
        espin "$NOT_VERIFIED"
        bspin - "Download and $OP_TYPE Kubernetes/kind $DOWNLOAD"
        (set -x; curl -o "$GOPATH/bin/kind" --fail -sSL "https://github.com/kubernetes-sigs/kind/releases/download/$KIND_VERSION/kind-$HOSTOS-$HOSTARCH" >>"$LOG" 2>"$ERR_OUT") >>"$LOG" 2>&1
        RESULT=$?
        if [ $RESULT -ne 0 ]; then
            espin - "$THEX"
            >&2 echo -e "${RED}${BOLD}${ERROR}ERROR: unable to download kind: $(tail "$ERR_OUT")${NORMAL}"
            cat "$ERR_OUT" >> "$LOG"
            rm -rf "$ERR_OUT" "$GOPATH/bin/kind"
            exit 1
        fi
        (set -x; chmod 755 "$GOPATH/bin/kind" >>"$LOG" 2>&1) >>"$LOG" 2>&1
        rm -rf "$ERR_OUT"
        espin - "$VERIFIED"
    fi
fi

if [ "$WITH_TIMINGS" == "yes" ]; then
    NOW="$(date +%s)"
    printtime $((NOW - STIME))
fi

STIME="$(date +%s)"
if [ "$INSTALL_HELM" == "no" ]; then
    bspin "Skip Helm Install"
    espin $NO_ENTRY
else
    bspin "Verify Helm $HELM"
    HELM_UPGRADE=0
    HELM_UPGRADE_DESC="install"
    if [ -x "$GOPATH/bin/helm" ]; then
        HAVE_VER=$(helm version --client --short | sed -E -e 's/^.*(v[0-9]+\.[0-9]+\.[0-9]+).*$/\1/')
        HELM_UP_DOWN="$(echo -e "$HAVE_VER\n$HELM_VERSION" | sort -V | head -1)"
        if [ "$HAVE_VER" != "$HELM_VERSION" ]; then
            if [ "$HELM_UP_DOWN" == "$HELM_VERSION" ]; then
                HELM_UPGRADE_DESC="downgrade"
            else
                HELM_UPGRADE_DESC="upgrade"
            fi
            HELM_UPGRADE=1
        else
            espin "$VERIFIED"
        fi
    else
        HELM_UPGRADE=1
    fi

    if [ "$HELM_UPGRADE" -ne 0 ]; then
        ERR_OUT="$(mktemp)"
        INSTALL_HELM_BIN=$(mktemp)
        espin "$NOT_VERIFIED"
        bspin - "Download and $HELM_UPGRADE_DESC Helm $DOWNLOAD"
        (set -x; curl -o "$INSTALL_HELM_BIN" --fail -sSL https://git.io/get_helm.sh >>"$LOG" 2>"$ERR_OUT") >>"$LOG" 2>&1
        RESULT=$?
        if [ $RESULT -ne 0 ]; then
            espin - "$THEX"
            >&2 echo -e "${RED}${BOLD}${ERROR}ERROR: unable to download helm installer: $(tail "$ERR_OUT")${NORMAL}"
            cat "$ERR_OUT" >> "$LOG"
            rm -rf "$INSTALL_HELM_BIN" "$ERR_OUT" "$GOPATH/bin/helm" "$GOPATH/bin/tiller"
            exit 1
        fi
        rm -rf "$ERR_OUT"
        chmod +x "$INSTALL_HELM_BIN"
        (set -x; PATH="$GOPATH/bin:$PATH" DESIRED_VERSION=$HELM_VERSION  USE_SUDO=false HELM_INSTALL_DIR=$GOPATH/bin "$INSTALL_HELM_BIN" >>"$LOG" 2>&1) >>"$LOG" 2>&1
        RESULT=$?
        if [ $RESULT -ne 0 ]; then
            espin - "$THEX"
            >&2 echo -e "${RED}${BOLD}${ERROR}ERROR: unable to install helm, see install log for details${NORMAL}"
            rm -rf "$INSTALL_HELM_BIN" "$ERR_OUT" "$GOPATH/bin/helm" "$GOPATH/bin/tiller"
            exit 1
        fi
        rm -rf "$INSTALL_HELM_BIN" "$ERR_OUT"
        espin - "$VERIFIED"
    fi
fi
if [ "$WITH_TIMINGS" == "yes" ]; then
    NOW="$(date +%s)"
    printtime $((NOW - STIME))
fi
HELM_MAJOR=$(helm version --client --short 2>/dev/null| sed -E -e 's/^.*v([0-9]+)\.[0-9]+\.[0-9]+.*$/\1/')

STIME="$(date +%s)"
bspin "Verify voltctl $HIGH_VOLTAGE"
VOK=0
VMESSAGE="install"
ERR_OUT=$(mktemp)
export VC_VERSION="$VOLTCTL_VERSION"
if [ "$VC_VERSION" == "latest" ]; then
    set -o pipefail # TODO: would be nice to run all in pipefail mode
    VC_VERSION="$(curl --fail -sSL https://api.github.com/repos/opencord/voltctl/releases/latest 2>"$ERR_OUT" | jq -r .tag_name | sed -e 's/^v//g')"
    RESULT=$?
    set +o pipefail
    if [ "$RESULT" -ne 0 ]; then
        espin - "$THEX"
        >&2 echo -e "${RED}${BOLD}${ERROR}ERROR: unable to determine released version of voltctl: $(cat "$ERR_OUT")${NORMAL}"
        rm -rf "$ERR_OUT" "$GOPATH/bin/voltctl"
        exit 1
    fi
    export VC_VERSION
fi

if [ -x "$GOPATH/bin/voltctl" ]; then
    VHAVE="$("$GOPATH/bin/voltctl" version --clientonly -o json 2>/dev/null | jq -r .version)"
    RESULT=$?
    if [ $RESULT -eq 0 ] && [ "$VHAVE" == "$VC_VERSION" ]; then
        VOK=1
        espin "$VERIFIED"
    else
        VCHECK="$(echo -e "$VHAVE\n$VC_VERSION" | sort -V | head -1)"
        if [ "$VCHECK" == "$VHAVE" ]; then
            VMESSAGE="upgrade"
        else
            VMESSAGE="downgrade"
        fi
    fi
fi

if [ "$VOK" -eq 0 ]; then
    espin "$NOT_VERIFIED"
    bspin - "Download and $VMESSAGE voltctl $DOWNLOAD"
    (set -x; curl --fail -o "$GOPATH/bin/voltctl" -sSL "https://github.com/opencord/voltctl/releases/download/v$VC_VERSION/voltctl-$VC_VERSION-$HOSTOS-$HOSTARCH" >>"$LOG" 2>"$ERR_OUT") >>"$LOG" 2>&1
    RESULT=$?
    if [ $RESULT -ne 0 ]; then
        espin - "$THEX"
        >&2 echo -e "${RED}${BOLD}${ERROR}ERROR: unable to download voltctl (version $VC_VERSION): $(cat "$ERR_OUT")${NORMAL}"
        rm -rf "$ERR_OUT" "$GOPATH/bin/voltctl"
        exit 1
    fi
    (set -x; chmod 755 "$GOPATH/bin/voltctl" >>"$LOG" 2>&1) >>"$LOG" 2>&1
    espin - "$VERIFIED"
fi
rm -rf "$ERR_OUT"
if [ "$WITH_TIMINGS" == "yes" ]; then
    NOW="$(date +%s)"
    printtime $((NOW - STIME))
fi

bspin "Verify command PATH"
espin "$VERIFIED"

STIME="$(date +%s)"
if [ "$DEPLOY_K8S" == "yes" ]; then
    HAVE="$(kind get clusters 2>/dev/null | grep -c "voltha-$NAME")"
    bspin "Verify Kubernetes/Kind Cluster"
    sspin
    if [ "$HAVE" -eq 0 ]; then
        espin "$NOT_VERIFIED"
        FILE="$NAME-cluster.cfg"
        if [ -n "$KIND_CFG_FILE" ]; then
            FILE="$KIND_CFG_FILE"
        else
            bspin - "Generating cluster configuration"

            if [ -f "$FILE" ] ; then
              rm "$FILE"
            fi

            touch "$FILE"

            yq w -i "$FILE" kind Cluster
            yq w -i "$FILE" apiVersion "kind.sigs.k8s.io/v1alpha3"

            if [ ! "$NUM_OF_CONTROLLER_NODES" -eq 0 ]; then
              for instance in $(seq 1 "$NUM_OF_CONTROLLER_NODES"); do
                yq w -i "$FILE" "nodes[+].role" "control-plane"
              done
            fi

            if [ ! "$NUM_OF_WORKER_NODES" -eq 0 ]; then
              for instance in $(seq 1 "$NUM_OF_WORKER_NODES"); do
                yq w -i "$FILE" "nodes[+].role" worker
              done
            fi
            espin - "$VERIFIED"
        fi

        cat "$FILE" >> "$LOG" 2>&1

        kind create cluster --name "voltha-$NAME" --config "$FILE"
     else
        espin "$VERIFIED"
    fi

    KUBECONFIG_TMP=$(mktemp)
    kind get kubeconfig --name="voltha-$NAME" >"$KUBECONFIG_TMP"
    mkdir -p "$HOME/.kube"
    KUBECONFIG="$HOME/.kube/kind-config-voltha-$NAME"; export KUBECONFIG
    cp "$KUBECONFIG_TMP" "$KUBECONFIG"
    rm -rf "$KUBECONFIG_TMP"

    P="coredns-.* \
        etcd-voltha-$NAME-control-plane \
        kindnet-.* \
        kube-apiserver-voltha-$NAME-control-plane \
        kube-controller-manager-voltha-$NAME-control-plane \
        kube-proxy-.* \
        kube-scheduler-voltha-$NAME-control-plane"

    EXPECT=2 # Always 2 DNS instances
    EXPECT=$((EXPECT + 4 * NUM_OF_CONTROLLER_NODES)) <