voltha - kind-voltha - Gitiles

 #!/usr/bin/env bash
 # Copyright 2019 Ciena Corporation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.

 # trap ctrl-c and call ctrl_c()
 trap ctrl_c INT

 function ctrl_c() {
     tput cnorm
     echo ""
     echo "ctrl-c trapped"
     echo "Thank you for trying 'votlha up'"
     exit
 }

 TYPE=${TYPE:-minimal}
 # Only minimal cluster works in this script currently
 TYPE=minimal

 if [ $# -ne 1 -o $(echo ":up:down:" | grep -c ":$1:") -ne 1 ]; then
     >&2 echo "up or down?"
     exit 1
 fi

 if [ "$1" == "down" ]; then
     exec ./bin/kind delete cluster --name voltha-$TYPE
     exit
 fi

 LOG="install.log"
 date > $LOG

 spin() {
     PARTS="\
         \xe2\xa2\x8e\xe2\xa1\xb0 \
         \xe2\xa2\x8e\xe2\xa1\xa1 \
         \xe2\xa2\x8e\xe2\xa1\x91 \
         \xe2\xa2\x8e\xe2\xa0\xb1 \
         \xe2\xa0\x8e\xe2\xa1\xb1 \
         \xe2\xa2\x8a\xe2\xa1\xb1 \
         \xe2\xa2\x8c\xe2\xa1\xb1 \
         \xe2\xa2\x86\xe2\xa1\xb1 \
         "
     IDX=1
     tput civis
     while true; do
        C=$(echo $PARTS | cut '-d ' -f $IDX)
        echo -en "$C"
        IDX=$(expr $IDX + 1)
        if [ $IDX -gt 8 ]; then
            IDX=1
        fi
        sleep .15
        echo -en "\r"
     done
 }

 SPIN_PARTS="\
         \xe2\xa2\x8e\xe2\xa1\xb0 \
         \xe2\xa2\x8e\xe2\xa1\xa1 \
         \xe2\xa2\x8e\xe2\xa1\x91 \
         \xe2\xa2\x8e\xe2\xa0\xb1 \
         \xe2\xa0\x8e\xe2\xa1\xb1 \
         \xe2\xa2\x8a\xe2\xa1\xb1 \
         \xe2\xa2\x8c\xe2\xa1\xb1 \
         \xe2\xa2\x86\xe2\xa1\xb1 \
         "
 IDX=1
 NOT_VERIFIED="\xe2\x9c\x97\x20"
 VERIFIED="\xe2\x9c\x93\x20"
 bspin() {
     tput civis
     IDX=1
     local INDENT=
     if [ "$1" == "-" ]; then
         INDENT="  "
         shift
     fi
     echo -en "$INDENT   $*"
 }

 sspin() {
     local INDENT=
     if [ "$1" == "-" ]; then
         INDENT="  "
         shift
     fi
     C=$(echo $SPIN_PARTS | cut '-d ' -f $IDX)
     echo -en "\r$INDENT$C $*"
     IDX=$(expr $IDX + 1)
     if [ $IDX -gt 8 ]; then
         IDX=1
     fi
 }

 espin() {
     local INDENT=
     if [ "$1" == "-" ]; then
         INDENT="  "
         shift
     fi
     echo -e "\r$INDENT$*"
     tput cnorm
 }

 count_pods() {
     local NAMESPACE=$1; shift
     local PODS=$(kubectl -n $NAMESPACE get pod -o go-template="{{range .items}}{{.metadata.name}}/{{.status.phase}}/_{{range .status.containerStatuses}}{{.ready}}_{{end}} {{end}}")
     local COUNT=0
     local PATTERNS=$*
     for POD in $PODS; do
         local NAME=$(echo $POD | cut -d/ -f 1)
         local STATE=$(echo $POD | cut -d/ -f 2)
         local CONTAINERS=$(echo $POD | cut -d/ -f 3 | sed -e 's/_/ /g')
         if [ "$STATE" == "Running" ]; then
             local TOTAL=$(echo $CONTAINERS | wc -w)
             local FOUND=$(echo $CONTAINERS | grep -o true | wc -l)
             if [ $TOTAL -eq $FOUND ]; then
                 for PATTERN in $PATTERNS; do
                     if [[ $NAME =~ $PATTERN ]]; then
                         COUNT=$(expr $COUNT + 1)
                     fi
                 done
             fi
         fi
     done
     echo $COUNT
 }

 wait_for_pods() {
     local INDENT=
     if [ "$1" == "-" ]; then
         INDENT=$1; shift
     fi
     local NAMESPACE=$1; shift
     local EXPECT=$1; shift
     local RETRY=$1; shift
     local MESSAGE=$1; shift
     local PATTERNS=$*
     local HAVE=$(count_pods $NAMESPACE $PATTERNS)
     COUNT=$(expr 300 / 15)
     bspin $INDENT $MESSAGE
     sspin $INDENT
     if [ $HAVE -ne $EXPECT ]; then
         while [ $HAVE -ne $EXPECT ]; do
             sspin $INDENT
             COUNT=$(expr $COUNT - 1)
             if [ $COUNT -eq 0 ]; then
                 HAVE=$(count_pods $NAMESPACE $PATTERNS)
                 COUNT=$(expr 300 / 15)
             fi
             sleep .15
         done
     fi
     espin $INDENT $VERIFIED
     if [ $HAVE -ne $EXPECT ]; then
         return 1
     fi
     return 0
 }

 helm_install() {
     local INDENT=
     if [ "$1" == "-" ]; then
         INDENT=$1; shift
     fi
     local NAMESPACE=$1; shift
     local NAME=$1; shift
     local CHART=$1; shift
     local MESSAGE=$*

     COUNT=$(expr 300 / 15)
     bspin $INDENT $MESSAGE
     (set -x; helm install -f $TYPE-values.yaml --namespace $NAMESPACE --name $NAME $CHART >>$LOG 2>&1) >>$LOG 2>&1
     SUCCESS=$?
     while [ $SUCCESS -ne 0 ]; do
         sspin $INDENT
         COUNT=$(expr $COUNT - 1)
         if [ $COUNT -eq 0 ]; then
             (set -x; helm install -f $TYPE-values.yaml --namespace $NAMESPACE --name $NAME $CHART >>$LOG 2>&1) >>$LOG 2>&1
             COUNT=$(expr 300 / 15)
         fi
         sleep .15
     done
     espin $INDENT $VERIFIED
 }

 echo "INSTALL TYPE: $TYPE" >> $LOG

 bspin "Verify GOPATH"
 export GOPATH=$(pwd)
 mkdir -p $GOPATH/bin
 espin $VERIFIED

 bspin "Verify Kubernetes/Kind"
 if [ -x $GOPATH/bin/kind ]; then
     espin $VERIFIED
 else
     espin $NOT_VERIFIED
     bspin - "Download and build Kubernetes/Kind"
     (set -x; rm -rf $GOPATH/go.mod $GOPATH/go.sum >>$LOG 2>&1) >>$LOG 2>&1
     (set -x; curl -o $GOPATH/bin/kind -sSL https://github.com/kubernetes-sigs/kind/releases/download/v0.4.0/kind-$(go env GOHOSTOS)-$(go env GOARCH) >>$LOG 2>&1) >>$LOG 2>&1
     (set -x; chmod 755 $GOPATH/bin/kind >>$LOG 2>&1) >>$LOG 2>&1
     espin - $VERIFIED
 fi

 bspin "Verify Helm"
 if [ -x $GOPATH/bin/helm ]; then
     espin $VERIFIED
 else
     espin $NOT_VERIFIED
     bspin - "Download and install Helm"
     (set -x; rm -rf $GOPATH/go.mod $GOPATH/go.sum >>$LOG 2>&1) >>$LOG 2>&1
     (set -x; curl -L https://git.io/get_helm.sh | HELM_INSTALL_DIR=$(go env GOPATH)/bin bash >>$LOG 2>&1) >>$LOG 2>&1
     espin - $VERIFIED
 fi

 bspin "Verify voltctl"
 if [ -x $GOPATH/bin/voltctl ]; then
     espin $VERIFIED
 else
     espin $NOT_VERIFIED
     bspin - "Download and build voltctl"
     (set -x; rm -rf $GOPATH/go.mod $GOPATH/go.sum >>$LOG 2>&1) >>$LOG 2>&1
     (set -x; go get github.com/ciena/voltctl/cmd/voltctl >>$LOG 2>&1) >>$LOG 2>&1
     espin - $VERIFIED
 fi

 bspin "Verify command PATH"
 export PATH=$(go env GOPATH)/bin:$PATH
 espin $VERIFIED

 HAVE=$(kind get clusters | grep -c voltha-$TYPE)
 bspin "Verify Kubernetes/Kind Cluster"
 sspin
 if [ $HAVE -eq 0 ]; then
   espin $NOT_VERIFIED
   kind create cluster --name voltha-$TYPE --config $TYPE-cluster.cfg
 else
   espin $VERIFIED
 fi

 export KUBECONFIG="$(kind get kubeconfig-path --name="voltha-$TYPE")"
 P="coredns-.* \
     etcd-voltha-$TYPE-control-plane \
     kindnet-.* \
     kube-apiserver-voltha-$TYPE-control-plane \
     kube-controller-manager-voltha-$TYPE-control-plane \
     kube-proxy-.* \
     kube-scheduler-voltha-$TYPE-control-plane"

 wait_for_pods - "kube-system" 12 -1 "Waiting for system PODs to start..." $P

 COUNT=$(count_pods "kube-system" "tiller-deploy-.*")
 bspin "Verify Helm"
 if [ $(count_pods "kube-system" "tiller-deploy-.*") -ne 1 ]; then
     espin $NOT_VERIFIED
     echo "Configuring Helm ..."
     bspin - "Initialize Helm"
     (set -x; helm init --upgrade >>$LOG 2>&1) >>$LOG 2>&1
     espin - $VERIFIED
     bspin - "Add Google Incubator repository to Helm ..."
     (set -x; helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com >>$LOG 2>&1) >>$LOG 2>&1
     espin - $VERIFIED

     bspin - "Add Google Stable repository to Helm ..."
     (set -x; helm repo add stable https://kubernetes-charts.storage.googleapis.com >>$LOG 2>&1) >>$LOG 2>&1
     espin - $VERIFIED
     bspin - "Add ONF repository to Helm ..."
     (set -x; helm repo add onf https://charts.opencord.org >>$LOG 2>&1) >>$LOG 2>&1
     espin - $VERIFIED
     bspin - "Update Helm repository cache ..."
     (set -x; helm repo update >>$LOG 2>&1) >>$LOG 2>&1
     espin - $VERIFIED

     # Create and k8s service account so that Helm can create pods
     bspin - "Create Tiller ServiceAccount ..."
     (set -x; kubectl create serviceaccount --namespace kube-system tiller >>$LOG 2>&1) >>$LOG 2>&1
     espin - $VERIFIED
     bspin - "Create Tiller ClusterRoleBinding ..."
     (set -x; kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller >>$LOG 2>&1) >>$LOG 2>&1
     espin - $VERIFIED
     bspin - "Update Tiller Manifest ..."
     (set -x; kubectl patch deploy --namespace kube-system tiller-deploy -p '{"spec":{"template":{"spec":{"serviceAccount":"tiller"}}}}' >>$LOG 2>&1) >>$LOG 2>&1
     espin - $VERIFIED
 else
     espin $VERIFIED
 fi
 wait_for_pods - "kube-system" 1 -1 "Waiting for Tiller POD to start..." "tiller-deploy-.*"

 bspin "Verify ETCD Operator"
 if [ $(helm list etcd-operator | grep -c DEPLOYED) -ne 1 ]; then
     espin $NOT_VERIFIED
     helm_install - voltha etcd-operator stable/etcd-operator "Install ETCD Operator"
 else
     espin $VERIFIED
 fi
 wait_for_pods - "voltha" 1 -1 "Waiting for ETCD Operator to start..." "etcd-operator-.*"

 bspin "Verify ONOS installed"
 if [ $(helm list onos | grep -c DEPLOYED) -ne 1 ]; then
     espin $NOT_VERIFIED
     helm_install - default onos onf/onos "Install ONOS"
 else
     espin $VERIFIED
 fi
 wait_for_pods - "default" 1 -1 "Waiting for ONOS to start..." "onos-.*"

 bspin - "Forward ONOS API port"
 (set -x; screen -p 0 -X -S onos-ui  stuff $'\003' >>$LOG 2>&1) >>$LOG 2>&1
 (set -x; screen -dmS onos-ui kubectl port-forward service/onos-ui 8181:8181 >>$LOG 2>&1) >>$LOG 2>&1
 espin - $VERIFIED
 bspin - "Forward ONOS SSH port"
 (set -x; screen -p 0 -X -S onos-ssh  stuff $'\003' >>$LOG 2>&1) >>$LOG 2>&1
 (set -x; screen -dmS onos-ssh kubectl port-forward service/onos-ssh 8101:8101 >>$LOG 2>&1) >>$LOG 2>&1
 espin - $VERIFIED
 bspin - "Install required ONOS applications"
 (set -x; ./onos-files/install-onos-applications.sh >>$LOG 2>&1) >>$LOG 2>&1
 espin - $VERIFIED

 bspin "Verify VOLTHA installed"
 if [ $(helm list voltha | grep -c DEPLOYED) -ne 1 ]; then
     espin $NOT_VERIFIED
     helm_install - voltha voltha onf/voltha "Install VOLTHA Core"
 else
     espin $VERIFIED
 fi
 VOLTHA="ofagent-.* \
     ro-core.* \
     rw-core.* \
     voltha-api-server-.* \
     voltha-cli-server-.* \
     voltha-etcd-cluster-.* \
     voltha-kafka-.* \
     voltha-zookeeper-.*"
 wait_for_pods - "voltha" 9 -1 "Waiting for VOLTHA Core to start..." $VOLTHA

 echo "Verify Adapters"
 bspin - "Verify Simulated Adapters installed"
 if [ $(helm list sim | grep -c DEPLOYED) -ne 1 ]; then
     espin - $NOT_VERIFIED
     helm_install - voltha sim onf/voltha-adapter-simulated "Install Simulated Adapters"
 else
     espin - $VERIFIED
 fi

 bspin - "Verify OpenOLT Adapter installed"
 if [ $(helm list open-olt | grep -c DEPLOYED) -ne 1 ]; then
     espin - $NOT_VERIFIED
     helm_install - voltha open-olt onf/voltha-adapter-openolt "Install OpenOLT Adapter"
 else
     espin - $VERIFIED
 fi
 bspin - "Verify OpenONU Adapter installed"
 if [ $(helm list open-onu | grep -c DEPLOYED) -ne 1 ]; then
     espin - $NOT_VERIFIED
     helm_install - voltha open-onu onf/voltha-adapter-openonu "Install OpenONU Adapter"
 else
     espin - $VERIFIED
 fi

 ADAPTERS="adapter-.*"
 wait_for_pods - "voltha" 4 -1 "Waiting for adapters to start..." $ADAPTERS

 echo "Restart VOLTHA API"
 API="voltha-api-server-.* ofagent-.*"
 (set -x; kubectl scale --replicas=0 deployment -n voltha voltha-api-server ofagent >>$LOG 2>&1) >>$LOG 2>&1
 wait_for_pods - "voltha" 0 -1 "Wait for API to stop" $API
 (set -x; kubectl scale --replicas=1 deployment -n voltha voltha-api-server ofagent >>$LOG 2>&1) >>$LOG 2>&1
 wait_for_pods - "voltha" 2 -1 "Wait for API to re-start ..." $API

 bspin - "Forward VOLTHA API port"
 (set -x; screen -p 0 -X -S voltha-api  stuff $'\003' >>$LOG 2>&1) >>$LOG 2>&1
 (set -x; screen -dmS voltha-api kubectl port-forward -n voltha service/voltha-api 55555:55555 >>$LOG 2>&1) >>$LOG 2>&1
 espin - $VERIFIED
 bspin - "Forward VOLTHA SSH port"
 (set -x; screen -p 0 -X -S voltha-ssh  stuff $'\003' 2>/dev/null >/dev/null >>$LOG 2>&1) >>$LOG 2>&1
 (set -x; screen -dmS voltha-ssh kubectl port-forward -n voltha service/voltha-cli 5022:5022 >>$LOG 2>&1) >>$LOG 2>&1
 espin - $VERIFIED

 bspin - "Create voltctl configuration file"
 (set -x; mkdir -p $HOME/.volt >>$LOG 2>&1) >>$LOG 2>&1
 (set -x; voltctl -a v2 -s localhost:55555 config > $HOME/.volt/config 2>>$LOG) >>$LOG 2>&1
 espin - $VERIFIED
	#!/usr/bin/env bash
	# Copyright 2019 Ciena Corporation
	#
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.

	# trap ctrl-c and call ctrl_c()
	trap ctrl_c INT

	function ctrl_c() {
	tput cnorm
	echo ""
	echo "ctrl-c trapped"
	echo "Thank you for trying 'votlha up'"
	exit
	}

	TYPE=${TYPE:-minimal}
	# Only minimal cluster works in this script currently
	TYPE=minimal

	if [ $# -ne 1 -o $(echo ":up:down:" \| grep -c ":$1:") -ne 1 ]; then
	>&2 echo "up or down?"
	exit 1
	fi

	if [ "$1" == "down" ]; then
	exec ./bin/kind delete cluster --name voltha-$TYPE
	exit
	fi

	LOG="install.log"
	date > $LOG

	spin() {
	PARTS="\
	\xe2\xa2\x8e\xe2\xa1\xb0 \
	\xe2\xa2\x8e\xe2\xa1\xa1 \
	\xe2\xa2\x8e\xe2\xa1\x91 \
	\xe2\xa2\x8e\xe2\xa0\xb1 \
	\xe2\xa0\x8e\xe2\xa1\xb1 \
	\xe2\xa2\x8a\xe2\xa1\xb1 \
	\xe2\xa2\x8c\xe2\xa1\xb1 \
	\xe2\xa2\x86\xe2\xa1\xb1 \
	"
	IDX=1
	tput civis
	while true; do
	C=$(echo $PARTS \| cut '-d ' -f $IDX)
	echo -en "$C"
	IDX=$(expr $IDX + 1)
	if [ $IDX -gt 8 ]; then
	IDX=1
	fi
	sleep .15
	echo -en "\r"
	done
	}

	SPIN_PARTS="\
	\xe2\xa2\x8e\xe2\xa1\xb0 \
	\xe2\xa2\x8e\xe2\xa1\xa1 \
	\xe2\xa2\x8e\xe2\xa1\x91 \
	\xe2\xa2\x8e\xe2\xa0\xb1 \
	\xe2\xa0\x8e\xe2\xa1\xb1 \
	\xe2\xa2\x8a\xe2\xa1\xb1 \
	\xe2\xa2\x8c\xe2\xa1\xb1 \
	\xe2\xa2\x86\xe2\xa1\xb1 \
	"
	IDX=1
	NOT_VERIFIED="\xe2\x9c\x97\x20"
	VERIFIED="\xe2\x9c\x93\x20"
	bspin() {
	tput civis
	IDX=1
	local INDENT=
	if [ "$1" == "-" ]; then
	INDENT=" "
	shift
	fi
	echo -en "$INDENT $*"
	}

	sspin() {
	local INDENT=
	if [ "$1" == "-" ]; then
	INDENT=" "
	shift
	fi
	C=$(echo $SPIN_PARTS \| cut '-d ' -f $IDX)
	echo -en "\r$INDENT$C $*"
	IDX=$(expr $IDX + 1)
	if [ $IDX -gt 8 ]; then
	IDX=1
	fi
	}

	espin() {
	local INDENT=
	if [ "$1" == "-" ]; then
	INDENT=" "
	shift
	fi
	echo -e "\r$INDENT$*"
	tput cnorm
	}

	count_pods() {
	local NAMESPACE=$1; shift
	local PODS=$(kubectl -n $NAMESPACE get pod -o go-template="{{range .items}}{{.metadata.name}}/{{.status.phase}}/_{{range .status.containerStatuses}}{{.ready}}_{{end}} {{end}}")
	local COUNT=0
	local PATTERNS=$*
	for POD in $PODS; do
	local NAME=$(echo $POD \| cut -d/ -f 1)
	local STATE=$(echo $POD \| cut -d/ -f 2)
	local CONTAINERS=$(echo $POD \| cut -d/ -f 3 \| sed -e 's/_/ /g')
	if [ "$STATE" == "Running" ]; then
	local TOTAL=$(echo $CONTAINERS \| wc -w)
	local FOUND=$(echo $CONTAINERS \| grep -o true \| wc -l)
	if [ $TOTAL -eq $FOUND ]; then
	for PATTERN in $PATTERNS; do
	if [[ $NAME =~ $PATTERN ]]; then
	COUNT=$(expr $COUNT + 1)
	fi
	done
	fi
	fi
	done
	echo $COUNT
	}

	wait_for_pods() {
	local INDENT=
	if [ "$1" == "-" ]; then
	INDENT=$1; shift
	fi
	local NAMESPACE=$1; shift
	local EXPECT=$1; shift
	local RETRY=$1; shift
	local MESSAGE=$1; shift
	local PATTERNS=$*
	local HAVE=$(count_pods $NAMESPACE $PATTERNS)
	COUNT=$(expr 300 / 15)
	bspin $INDENT $MESSAGE
	sspin $INDENT
	if [ $HAVE -ne $EXPECT ]; then
	while [ $HAVE -ne $EXPECT ]; do
	sspin $INDENT
	COUNT=$(expr $COUNT - 1)
	if [ $COUNT -eq 0 ]; then
	HAVE=$(count_pods $NAMESPACE $PATTERNS)
	COUNT=$(expr 300 / 15)
	fi
	sleep .15
	done
	fi
	espin $INDENT $VERIFIED
	if [ $HAVE -ne $EXPECT ]; then
	return 1
	fi
	return 0
	}

	helm_install() {
	local INDENT=
	if [ "$1" == "-" ]; then
	INDENT=$1; shift
	fi
	local NAMESPACE=$1; shift
	local NAME=$1; shift
	local CHART=$1; shift
	local MESSAGE=$*

	COUNT=$(expr 300 / 15)
	bspin $INDENT $MESSAGE
	(set -x; helm install -f $TYPE-values.yaml --namespace $NAMESPACE --name $NAME $CHART >>$LOG 2>&1) >>$LOG 2>&1
	SUCCESS=$?
	while [ $SUCCESS -ne 0 ]; do
	sspin $INDENT
	COUNT=$(expr $COUNT - 1)
	if [ $COUNT -eq 0 ]; then
	(set -x; helm install -f $TYPE-values.yaml --namespace $NAMESPACE --name $NAME $CHART >>$LOG 2>&1) >>$LOG 2>&1
	COUNT=$(expr 300 / 15)
	fi
	sleep .15
	done
	espin $INDENT $VERIFIED
	}

	echo "INSTALL TYPE: $TYPE" >> $LOG

	bspin "Verify GOPATH"
	export GOPATH=$(pwd)
	mkdir -p $GOPATH/bin
	espin $VERIFIED

	bspin "Verify Kubernetes/Kind"
	if [ -x $GOPATH/bin/kind ]; then
	espin $VERIFIED
	else
	espin $NOT_VERIFIED
	bspin - "Download and build Kubernetes/Kind"
	(set -x; rm -rf $GOPATH/go.mod $GOPATH/go.sum >>$LOG 2>&1) >>$LOG 2>&1
	(set -x; curl -o $GOPATH/bin/kind -sSL https://github.com/kubernetes-sigs/kind/releases/download/v0.4.0/kind-$(go env GOHOSTOS)-$(go env GOARCH) >>$LOG 2>&1) >>$LOG 2>&1
	(set -x; chmod 755 $GOPATH/bin/kind >>$LOG 2>&1) >>$LOG 2>&1
	espin - $VERIFIED
	fi

	bspin "Verify Helm"
	if [ -x $GOPATH/bin/helm ]; then
	espin $VERIFIED
	else
	espin $NOT_VERIFIED
	bspin - "Download and install Helm"
	(set -x; rm -rf $GOPATH/go.mod $GOPATH/go.sum >>$LOG 2>&1) >>$LOG 2>&1
	(set -x; curl -L https://git.io/get_helm.sh \| HELM_INSTALL_DIR=$(go env GOPATH)/bin bash >>$LOG 2>&1) >>$LOG 2>&1
	espin - $VERIFIED
	fi

	bspin "Verify voltctl"
	if [ -x $GOPATH/bin/voltctl ]; then
	espin $VERIFIED
	else
	espin $NOT_VERIFIED
	bspin - "Download and build voltctl"
	(set -x; rm -rf $GOPATH/go.mod $GOPATH/go.sum >>$LOG 2>&1) >>$LOG 2>&1
	(set -x; go get github.com/ciena/voltctl/cmd/voltctl >>$LOG 2>&1) >>$LOG 2>&1
	espin - $VERIFIED
	fi

	bspin "Verify command PATH"
	export PATH=$(go env GOPATH)/bin:$PATH
	espin $VERIFIED

	HAVE=$(kind get clusters \| grep -c voltha-$TYPE)
	bspin "Verify Kubernetes/Kind Cluster"
	sspin
	if [ $HAVE -eq 0 ]; then
	espin $NOT_VERIFIED
	kind create cluster --name voltha-$TYPE --config $TYPE-cluster.cfg
	else
	espin $VERIFIED
	fi

	export KUBECONFIG="$(kind get kubeconfig-path --name="voltha-$TYPE")"
	P="coredns-.* \
	etcd-voltha-$TYPE-control-plane \
	kindnet-.* \
	kube-apiserver-voltha-$TYPE-control-plane \
	kube-controller-manager-voltha-$TYPE-control-plane \
	kube-proxy-.* \
	kube-scheduler-voltha-$TYPE-control-plane"

	wait_for_pods - "kube-system" 12 -1 "Waiting for system PODs to start..." $P

	COUNT=$(count_pods "kube-system" "tiller-deploy-.*")
	bspin "Verify Helm"
	if [ $(count_pods "kube-system" "tiller-deploy-.*") -ne 1 ]; then
	espin $NOT_VERIFIED
	echo "Configuring Helm ..."
	bspin - "Initialize Helm"
	(set -x; helm init --upgrade >>$LOG 2>&1) >>$LOG 2>&1
	espin - $VERIFIED
	bspin - "Add Google Incubator repository to Helm ..."
	(set -x; helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com >>$LOG 2>&1) >>$LOG 2>&1
	espin - $VERIFIED

	bspin - "Add Google Stable repository to Helm ..."
	(set -x; helm repo add stable https://kubernetes-charts.storage.googleapis.com >>$LOG 2>&1) >>$LOG 2>&1
	espin - $VERIFIED
	bspin - "Add ONF repository to Helm ..."
	(set -x; helm repo add onf https://charts.opencord.org >>$LOG 2>&1) >>$LOG 2>&1
	espin - $VERIFIED
	bspin - "Update Helm repository cache ..."
	(set -x; helm repo update >>$LOG 2>&1) >>$LOG 2>&1
	espin - $VERIFIED

	# Create and k8s service account so that Helm can create pods
	bspin - "Create Tiller ServiceAccount ..."
	(set -x; kubectl create serviceaccount --namespace kube-system tiller >>$LOG 2>&1) >>$LOG 2>&1
	espin - $VERIFIED
	bspin - "Create Tiller ClusterRoleBinding ..."
	(set -x; kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller >>$LOG 2>&1) >>$LOG 2>&1
	espin - $VERIFIED
	bspin - "Update Tiller Manifest ..."
	(set -x; kubectl patch deploy --namespace kube-system tiller-deploy -p '{"spec":{"template":{"spec":{"serviceAccount":"tiller"}}}}' >>$LOG 2>&1) >>$LOG 2>&1
	espin - $VERIFIED
	else
	espin $VERIFIED
	fi
	wait_for_pods - "kube-system" 1 -1 "Waiting for Tiller POD to start..." "tiller-deploy-.*"

	bspin "Verify ETCD Operator"
	if [ $(helm list etcd-operator \| grep -c DEPLOYED) -ne 1 ]; then
	espin $NOT_VERIFIED
	helm_install - voltha etcd-operator stable/etcd-operator "Install ETCD Operator"
	else
	espin $VERIFIED
	fi
	wait_for_pods - "voltha" 1 -1 "Waiting for ETCD Operator to start..." "etcd-operator-.*"

	bspin "Verify ONOS installed"
	if [ $(helm list onos \| grep -c DEPLOYED) -ne 1 ]; then
	espin $NOT_VERIFIED
	helm_install - default onos onf/onos "Install ONOS"
	else
	espin $VERIFIED
	fi
	wait_for_pods - "default" 1 -1 "Waiting for ONOS to start..." "onos-.*"

	bspin - "Forward ONOS API port"
	(set -x; screen -p 0 -X -S onos-ui stuff $'\003' >>$LOG 2>&1) >>$LOG 2>&1
	(set -x; screen -dmS onos-ui kubectl port-forward service/onos-ui 8181:8181 >>$LOG 2>&1) >>$LOG 2>&1
	espin - $VERIFIED
	bspin - "Forward ONOS SSH port"
	(set -x; screen -p 0 -X -S onos-ssh stuff $'\003' >>$LOG 2>&1) >>$LOG 2>&1
	(set -x; screen -dmS onos-ssh kubectl port-forward service/onos-ssh 8101:8101 >>$LOG 2>&1) >>$LOG 2>&1
	espin - $VERIFIED
	bspin - "Install required ONOS applications"
	(set -x; ./onos-files/install-onos-applications.sh >>$LOG 2>&1) >>$LOG 2>&1
	espin - $VERIFIED

	bspin "Verify VOLTHA installed"
	if [ $(helm list voltha \| grep -c DEPLOYED) -ne 1 ]; then
	espin $NOT_VERIFIED
	helm_install - voltha voltha onf/voltha "Install VOLTHA Core"
	else
	espin $VERIFIED
	fi
	VOLTHA="ofagent-.* \
	ro-core.* \
	rw-core.* \
	voltha-api-server-.* \
	voltha-cli-server-.* \
	voltha-etcd-cluster-.* \
	voltha-kafka-.* \
	voltha-zookeeper-.*"
	wait_for_pods - "voltha" 9 -1 "Waiting for VOLTHA Core to start..." $VOLTHA

	echo "Verify Adapters"
	bspin - "Verify Simulated Adapters installed"
	if [ $(helm list sim \| grep -c DEPLOYED) -ne 1 ]; then
	espin - $NOT_VERIFIED
	helm_install - voltha sim onf/voltha-adapter-simulated "Install Simulated Adapters"
	else
	espin - $VERIFIED
	fi

	bspin - "Verify OpenOLT Adapter installed"
	if [ $(helm list open-olt \| grep -c DEPLOYED) -ne 1 ]; then
	espin - $NOT_VERIFIED
	helm_install - voltha open-olt onf/voltha-adapter-openolt "Install OpenOLT Adapter"
	else
	espin - $VERIFIED
	fi
	bspin - "Verify OpenONU Adapter installed"
	if [ $(helm list open-onu \| grep -c DEPLOYED) -ne 1 ]; then
	espin - $NOT_VERIFIED
	helm_install - voltha open-onu onf/voltha-adapter-openonu "Install OpenONU Adapter"
	else
	espin - $VERIFIED
	fi

	ADAPTERS="adapter-.*"
	wait_for_pods - "voltha" 4 -1 "Waiting for adapters to start..." $ADAPTERS

	echo "Restart VOLTHA API"
	API="voltha-api-server-.* ofagent-.*"
	(set -x; kubectl scale --replicas=0 deployment -n voltha voltha-api-server ofagent >>$LOG 2>&1) >>$LOG 2>&1
	wait_for_pods - "voltha" 0 -1 "Wait for API to stop" $API
	(set -x; kubectl scale --replicas=1 deployment -n voltha voltha-api-server ofagent >>$LOG 2>&1) >>$LOG 2>&1
	wait_for_pods - "voltha" 2 -1 "Wait for API to re-start ..." $API

	bspin - "Forward VOLTHA API port"
	(set -x; screen -p 0 -X -S voltha-api stuff $'\003' >>$LOG 2>&1) >>$LOG 2>&1
	(set -x; screen -dmS voltha-api kubectl port-forward -n voltha service/voltha-api 55555:55555 >>$LOG 2>&1) >>$LOG 2>&1
	espin - $VERIFIED
	bspin - "Forward VOLTHA SSH port"
	(set -x; screen -p 0 -X -S voltha-ssh stuff $'\003' 2>/dev/null >/dev/null >>$LOG 2>&1) >>$LOG 2>&1
	(set -x; screen -dmS voltha-ssh kubectl port-forward -n voltha service/voltha-cli 5022:5022 >>$LOG 2>&1) >>$LOG 2>&1
	espin - $VERIFIED

	bspin - "Create voltctl configuration file"
	(set -x; mkdir -p $HOME/.volt >>$LOG 2>&1) >>$LOG 2>&1
	(set -x; voltctl -a v2 -s localhost:55555 config > $HOME/.volt/config 2>>$LOG) >>$LOG 2>&1
	espin - $VERIFIED