Merge pull request #6 from ciena/voltha-onos

Utilize pre-built voltha-onos image
diff --git a/README.md b/README.md
index 28feeee..2784249 100644
--- a/README.md
+++ b/README.md
@@ -62,16 +62,20 @@
 | OPTION                          | DEFAULT                      | DESCRIPTION                                                                         |
 | ------------------------------- | ---------------------------- | ----------------------------------------------------------------------------------- |
 | `TYPE`                          | minimal                      | `minimal` or `full` and determines number of cluster nodes and etcd cluster members |
-| `WITH_TIMINGS`                  | no                           | Outputs duration of various steps of the install                                    |
-| `WITH_BBSIM`                    | no                           | Should the BBSIM POD be deployed?                                                   |
-| `WITH_RADIUS`                   | no                           | Should `freeradius` service be deployed?                                            |
-| `WITH_ONOS`                     | yes                          | Should `ONOS` service be deployed?                                                  |
-| `JUST_K8S`                      | no                           | Should just the KinD Kubernetes cluster be depoyed? (i.e. no VOLTHA)                |
+| `NAME`                          | TYPE                         | Name of the KinD Cluster to be created                                              |
 | `DEPLOY_K8S`                    | yes                          | Should the KinD Kubernetes cluster be deployed?                                     |
+| `JUST_K8S`                      | no                           | Should just the KinD Kubernetes cluster be depoyed? (i.e. no VOLTHA)                |
+| `WITH_BBSIM`                    | no                           | Should the BBSIM POD be deployed?                                                   |
+| `WITH_ONOS`                     | yes                          | Should `ONOS` service be deployed?                                                  |
+| `WITH_RADIUS`                   | no                           | Should `freeradius` service be deployed?                                            |
+| `WITH_TP`                       | yes                          | Install the ONOS image that support Tech Profiles                                   |
+| `WITH_TIMINGS`                  | no                           | Outputs duration of various steps of the install                                    |
+| `INSTALL_ONOS_APPS`             | no                           | Replaces/installs ONOS OAR files in onos-files/onos-apps                            |
 | `SKIP_RESTART_API`              | no                           | Should the VOLTHA API service be restarted after install to avoid known bug?        |
 | `INSTALL_KUBECTL`               | yes                          | Should a copy of `kubectl` be installed locally?                                    |
 | `INSTALL_HELM`                  | yes                          | Should a copy of `helm` be installed locallly?                                      |
 | `USE_GO`                        | yes                          | Should the Go[lang] version of the OpenOLT adapter be used?                         |
+| `ONOS_TAG`                      |                              | Used to override the default image tag for the ONOS docker image                    |
 | `VOLTHA_LOG_LEVEL`              | WARN                         | Log level to set for VOLTHA core processes                                          |
 | `VOLTHA_CHART`                  | onf/voltha                   | Helm chart to used to install voltha                                                |
 | `VOLTHA_ADAPTER_SIM_CHART`      | onf/voltha-adapter-simulated | Helm chart to use to install simulated device adapter                               |
@@ -160,11 +164,24 @@
 screen -dmS onos-ssh kubectl port-forward service/onos-ssh 8101:8101
 ```
 
-#### Installing and Configuring ONOS Applications
-A script has been included, `install-onos-applications.sh`, that can be used
-to download and install the required applications into ONOS.
+#### Configuring ONOS Applications
+Configuration files have been provided to configure aspects of the ONOS deployment. The following
+curl commands push those configurations to the ONOS instance. It is possible (likely) that ONOS
+won't be immediately ready to accept REST requests, so the first `curl` command may need retried
+until ONOS is ready to accept REST connections.
 ```bash
-./onos-files/install-onos-applications.sh
+curl --fail -sSL --user karaf:karaf \
+	-X POST -H Content-Type:application/json \
+	http://127.0.0.1:8181/onos/v1/network/configuration \
+	--data @onos-files/olt-onos-netcfg.json
+curl --fail -sSL --user karaf:karaf \
+	-X POST -H Content-Type:application/json \
+	http://127.0.0.1:8181/onos/v1/configuration/org.opencord.olt.impl.Olt \
+	--data @onos-files/olt-onos-olt-settings.json
+curl --fail -sSL --user karaf:karaf \
+	-X POST -H Content-Type:application/json \
+	http://127.0.0.1:8181/onos/v1/configuration/org.onosproject.net.flow.impl.FlowRuleManager \
+	--data @onos-files/olt-onos-enableExtraneousRules.json
 ```
 
 ## Install VOLTHA Core
@@ -176,7 +193,8 @@
 the `Running` state. The following Helm command installs the core components
 of VOLTHA based on the desired deployment type.
 ```bash
-helm install -f $TYPE-values.yaml --namespace voltha --name voltha onf/voltha
+helm install -f $TYPE-values.yaml --set use_go=true --set defaults.log_level=WARN \
+	--namespace voltha --name voltha onf/voltha
 ```
 
 During the install of the core VOLTHA components some containers may "crash" or
@@ -209,9 +227,12 @@
 The following commands install both the simulated OLT and ONU adapters as well
 as the adapters for an OpenOLT and OpenONU device.
 ```bash
-helm install -f $TYPE-values.yaml --namespace voltha --name sim onf/voltha-adapter-simulated
-helm install -f $TYPE-values.yaml --namespace voltha --name open-olt onf/voltha-adapter-openolt
-helm install -f $TYPE-values.yaml --namespace voltha --name open-onu onf/voltha-adapter-openonu
+helm install -f $TYPE-values.yaml -set use_go=true --set defaults.log_level=WARN \
+	--namespace voltha --name sim onf/voltha-adapter-simulated
+helm install -f $TYPE-values.yaml -set use_go=true --set defaults.log_level=WARN \
+	--namespace voltha --name open-olt onf/voltha-adapter-openolt
+helm install -f $TYPE-values.yaml -set use_go=true --set defaults.log_level=WARN \
+	--namespace voltha --name open-onu onf/voltha-adapter-openonu
 ```
 
 ## Exposing VOLTHA Services
@@ -223,6 +244,17 @@
 screen -dmS voltha-ssh kubectl port-forward -n voltha service/voltha-cli 5022:5022
 ```
 
+## Install BBSIM (Broad Band OLT/ONU Simulator)
+BBSIM provides a simulation of a BB device. It can be useful for testing.
+```bash
+helm install -f minimal-values.yaml --namespace voltha --name bbsim onf/bbsim
+```
+
+## Install FreeRADIUS Service
+```bash
+helm install -f minimal-values.yaml --namespace voltha --name radius onf/freeradius
+```
+
 ## Configure `voltctl` to Connect to VOLTHA
 In order for `voltctl` to connect to the VOLTHA instance deplpoyed in the
 Kubernetes cluster it must know which IP address and port to use. This
@@ -382,14 +414,23 @@
 kind delete cluster --name=voltha-$TYPE
 ```
 
+## Troubleshooting
+There exists a bug in VOLTHA (as of 8/14/2019) where the API server doesn't always
+correctly connect to the back end services. To work around this bug, the `voltha-api-server`
+and `ofagent` can be restarted as described below.
+```bash
+kubectl scale --replicas=0 deployment -n voltha voltha-api-server ofagent
+```
+
+Wait for the POD to be removed, then scale it back up.
+```bash
+kubectl scale --replicas=1 deployment -n voltha voltha-api-server ofagent
+```
+
 ## WIP
 
 ### Create BBSIM Device
 
-#### Install BBSIM Helm Chart
-```bash
-helm install -f $TYPE-values.yaml --namespace voltha --name bbsim onf/bbsim
-```
 
 #### Create BBSIM Device
 ```bash
diff --git a/full-values.yaml b/full-values.yaml
index 00800a6..dbf23d6 100644
--- a/full-values.yaml
+++ b/full-values.yaml
@@ -14,12 +14,19 @@
 
 defaults:
   image_tag: master
+  image_pullPolicy: Always
 
 images:
   adapter_open_olt:
-    tag_go: master-go
+    tag_go: master
+    pullPolicy: Always
+  onos:
+    tag: master
+    repository: voltha/voltha-onos
+    pullPolicy: Always
   bbsim:
     tag: master
+    pullPolicy: Always
 
 onos_env:
 - name: POD_IP
diff --git a/minimal-values.yaml b/minimal-values.yaml
index 0a6a22c..7708ade 100644
--- a/minimal-values.yaml
+++ b/minimal-values.yaml
@@ -14,14 +14,19 @@
 
 defaults:
   image_tag: master
+  image_pullPolicy: Always
 
 images:
   adapter_open_olt:
     tag_go: master
+    pullPolicy: Always
   onos:
-    tag: 1.13.5
+    tag: master
+    repository: voltha/voltha-onos
+    pullPolicy: Always
   bbsim:
     tag: master
+    pullPolicy: Always
 
 deployments:
   etcdOperator: true
diff --git a/onos-files/install-onos-applications.sh b/onos-files/install-onos-applications.sh
deleted file mode 100755
index 5a3d3b9..0000000
--- a/onos-files/install-onos-applications.sh
+++ /dev/null
@@ -1,62 +0,0 @@
-#!/bin/bash
-
-# Copyright 2019 Ciena Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-ONOS_ADDRESS=${ONOS_ADDRESS:-127.0.0.1}
-
-SADIS_VER=${SADIS_VER:-3.1.0}
-OLT_VER=${OLT_VER:-3.0.1}
-AAA_VER=${AAA_VER:-1.9.0}
-DHCP_VER=${DHCP_VER:-1.6.0}
-
-if [ $(expr $AAA_VER \>= 1.9.0) -eq 1 ]; then
-    AAA_NAME=aaa-app
-else
-    AAA_NAME=aaa
-fi
-
-if [ $(expr $DHCP_VER \>= 1.6.0) -eq 1 ]; then
-    DHCP_NAME=dhcpl2relay-app
-else
-    DHCP_NAME=dhcpl2relay
-fi
-
-TYPE=${TYPE:-minimal}
-
-if [ "$TYPE" == "full" ]; then
-    ONOS_API_PORT=${ONOS_API_PORT:-8182}
-    ONOS_SSH_PORT=${ONOS_SSH_PORT:-8102}
-else
-    ONOS_API_PORT=${ONOS_API_PORT:-8181}
-    ONOS_SSH_PORT=${ONOS_SSH_PORT:-8101}
-fi
-
-echo "Target ONOS is ${ONOS_ADDRESS}:${ONOS_API_PORT}"
-
-mkdir -p onos-files/onos-apps
-echo "Downloading ONOS applications"
-curl --fail -sSL https://repo.maven.apache.org/maven2/org/opencord/sadis-app/$SADIS_VER/sadis-app-$SADIS_VER.oar -o ./onos-files/onos-apps/sadis-app-$SADIS_VER.oar
-curl --fail -sSL https://repo.maven.apache.org/maven2/org/opencord/$AAA_NAME/$AAA_VER/$AAA_NAME-$AAA_VER.oar -o ./onos-files/onos-apps/aaa-app-$AAA_VER.oar
-curl --fail -sSL https://repo.maven.apache.org/maven2/org/opencord/olt-app/$OLT_VER/olt-app-$OLT_VER.oar -o ./onos-files/onos-apps/olt-app-$OLT_VER.oar
-curl --fail -sSL https://repo.maven.apache.org/maven2/org/opencord/$DHCP_NAME/$DHCP_VER/$DHCP_NAME-$DHCP_VER.oar -o ./onos-files/onos-apps/dhcpl2relay-app-$DHCP_VER.oar
-
-until test $(curl -w '\n%{http_code}' --fail -sSL --user karaf:karaf -X POST -H Content-Type:application/octet-stream http://$ONOS_ADDRESS:$ONOS_API_PORT/onos/v1/applications?activate=true --data-binary @./onos-files/onos-apps/sadis-app-$SADIS_VER.oar 2>/dev/null | tail -1) -eq 409; do echo "Installing 'SADIS' ONOS application (v${SADIS_VER})..."; sleep 1; done
-until test $(curl -w '\n%{http_code}' --fail -sSL --user karaf:karaf -X POST -H Content-Type:application/octet-stream http://$ONOS_ADDRESS:$ONOS_API_PORT/onos/v1/applications?activate=true --data-binary @./onos-files/onos-apps/olt-app-$OLT_VER.oar 2>/dev/null | tail -1) -eq 409; do echo "Installing 'OLT' ONOS application (v${OLT_VER})..."; sleep 1; done
-until test $(curl -w '\n%{http_code}' --fail -sSL --user karaf:karaf -X POST -H Content-Type:application/octet-stream http://$ONOS_ADDRESS:$ONOS_API_PORT/onos/v1/applications?activate=true --data-binary @./onos-files/onos-apps/aaa-app-$AAA_VER.oar 2>/dev/null | tail -1) -eq 409; do echo "Installing 'AAA' ONOS application (v${AAA_VER})..."; sleep 1; done
-until test $(curl -w '\n%{http_code}' --fail -sSL --user karaf:karaf -X POST -H Content-Type:application/octet-stream http://$ONOS_ADDRESS:$ONOS_API_PORT/onos/v1/applications?activate=true --data-binary @./onos-files/onos-apps/dhcpl2relay-app-$DHCP_VER.oar 2>/dev/null | tail -1) -eq 409; do echo "Installing 'DHCP L2 Relay' ONOS application (v${DHCP_VER})..."; sleep 1; done
-until test $(curl -w '\n%{http_code}' --fail -sSL --user karaf:karaf -X POST -H Content-Type:application/json http://$ONOS_ADDRESS:$ONOS_API_PORT/onos/v1/network/configuration --data @onos-files/olt-onos-netcfg.json 2>/dev/null | tail -1) -eq 200; do echo "Configuring VOLTHA ONOS ..."; sleep 1; done
-until test $(curl -w '\n%{http_code}' --fail -sSL --user karaf:karaf -X POST -H Content-Type:application/json http://$ONOS_ADDRESS:$ONOS_API_PORT/onos/v1/configuration/org.opencord.olt.impl.Olt --data @onos-files/olt-onos-olt-settings.json 2>/dev/null | tail -1) -eq 200; do echo "Enabling VOLTHA ONOS DHCP provisioning..."; sleep 1; done
-until test $(curl -w '\n%{http_code}' --fail -sSL --user karaf:karaf -X POST -H Content-Type:application/json http://$ONOS_ADDRESS:$ONOS_API_PORT/onos/v1/configuration/org.onosproject.net.flow.impl.FlowRuleManager --data @onos-files/olt-onos-enableExtraneousRules.json 2>/dev/null | tail -1) -eq 200; do echo "Enabling extraneous rules for ONOS..."; sleep 1; done
-until test $(curl -w '\n%{http_code}' --fail -sSL --user karaf:karaf -X POST -H Content-Type:application/json "http://$ONOS_ADDRESS:$ONOS_API_PORT/onos/v1/flows/of:0000000000000001?appId=env.voltha" --data @onos-files/dhcp-to-controller-flow.json 2>/dev/null | tail -1) -eq 201; do echo "Establishing DHCP packet-in flow ..."; sleep 1; done
diff --git a/voltha b/voltha
index f5adc4d..0ce23ac 100755
--- a/voltha
+++ b/voltha
@@ -12,6 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+TOTAL_START_TIME=$(date +%s)
 
 FANCY=${FANCY:-1}
 if [ "$TERM X" == " X" ]; then
@@ -70,6 +71,7 @@
 WITH_BBSIM=${WITH_BBSIM:-no}
 WITH_RADIUS=${WITH_RADIUS:-no}
 WITH_ONOS=${WITH_ONOS:-yes}
+INSTALL_ONOS_APPS=${INSTALL_ONOS_APPS:-no}
 WITH_TP=${WITH_TP:-yes}
 JUST_K8S=${JUST_K8S:-no}
 DEPLOY_K8S=${DEPLOY_K8S:-yes}
@@ -220,6 +222,18 @@
     WITH_TIMINGS=no
 fi
 
+# Verify INSTALL_ONOS_APPS settting and convert uniform value of yes or no
+if [ $(echo ":y:yes:true:n:no:false:1:0:" | grep -ic ":$INSTALL_ONOS_APPS:") -eq 0 ]; then
+    >&2 echo -e "${RED}${BOLD}${ERROR}ERROR:${NORMAL}${RED} Invalid \$INSTALL_ONOS_APPS value of '$INSTALL_ONOS_APPS'. Should be 'yes' or 'no'${NORMAL}"
+    exit 1
+fi
+
+if [ $(echo ":y:yes:true:1:" | grep -ic ":$INSTALL_ONOS_APPS:") -eq 1 ]; then
+    INSTALL_ONOS_APPS=yes
+else
+    INSTALL_ONOS_APPS=no
+fi
+
 mkdir -p .voltha
 touch .voltha/ports
 HAVE=$(grep $NAME .voltha/ports)
@@ -253,18 +267,11 @@
 cp $PORTTMP .voltha/ports
 rm -f $PORTTMP
 
-if [ "$WITH_TP" == "yes" ]; then
-    SADIS_VER=3.1.0
-    OLT_VER=3.0.1
-    AAA_VER=1.9.0
-    DHCP_VER=1.6.0
-else
-    SADIS_VER=2.2.0
-    OLT_VER=2.1.0
-    AAA_VER=1.8.0
-    DHCP_VER=1.5.0
+ONOS_TAG=${ONOS_TAG:-}
+if [ "$WITH_TP" == "yes" -a "$ONOS_TAG X" == " X" ]; then
+    ONOS_TAG="voltha-1.7"
 fi
-export ONOS_API_PORT ONOS_SSH_PORT SADIS_VER OLT_VER AAA_VER DHCP_VER
+export ONOS_API_PORT ONOS_SSH_PORT
 
 spin() {
     PARTS="\
@@ -295,6 +302,10 @@
 CLOCK="TIME:"
 SPIN_PARTS=
 NOT_VERIFIED=
+THEX=
+BUILD=
+CROSS=
+ENTER=
 VERIFIED=
 HELM=
 OLD_KEY=
@@ -323,9 +334,13 @@
             \xe2\xa2\x86\xe2\xa1\xb1 \
             "
     CLOCK="\xe2\x8f\xb1"
-    NOT_VERIFIED="\xe2\x9c\x97\x20"
-    VERIFIED="\xe2\x9c\x93\x20"
-    HELM="\xE2\x8E\x88"
+    THEX="${RED}${BOLD}\xe2\x9c\x97\x20${NORMAL}"
+    ENTER="${YELLOW}${BOLD}\xe2\x8e\x86${NORMAL}"
+    CROSS="${YELLOW}${BOLD}\xe2\x9c\x9a${NORMAL}"
+    BUILD="${YELLOW}${BOLD}\xf0\x9f\x8f\x97${NORMAL}"
+    NOT_VERIFIED="$BUILD"
+    VERIFIED="${GREEN}${BOLD}\xe2\x9c\x93\x20${NORMAL}"
+    HELM="${BLUE}${BOLD}\xE2\x8E\x88${NORMAL}"
     OLD_KEY="\xF0\x9F\x97\x9D"
     BIRD="\xF0\x9F\x90\xA6"
     HIGH_VOLTAGE="\xE2\x9A\xA1"
@@ -341,12 +356,7 @@
     LOCK="\xf0\x9f\x94\x92"
 fi
 
-timeout() {
-    local INDENT=
-    if [ "$1" == "-" ]; then
-        INDENT="  "
-        shift
-    fi
+duration() {
     local h=$(expr $1 / 3600)
     local m=$(expr $1 % 3600 / 60)
     local s=$(expr $1 % 60)
@@ -358,8 +368,16 @@
     if [ $m -gt 0 ]; then
         t="$t${m}m"
     fi
-    t="$t${s}s"
-    echo -e "$INDENT  $CLOCK  $t"
+    echo "$t${s}s"
+}
+
+printtime() {
+    local INDENT=
+    if [ "$1" == "-" ]; then
+        INDENT="  "
+        shift
+    fi
+    echo -e "$INDENT  $CLOCK  $(duration $1)"
 }
 
 bspin() {
@@ -548,9 +566,9 @@
 # Output install options to log
 echo "OPTIONS" >> $LOG
 ALL_OPTIONS="NAME TYPE WITH_TIMINGS WITH_BBSIM WITH_RADIUS WITH_ONOS WITH_TP JUST_K8S DEPLOY_K8S \
-    SKIP_RESTART_API INSTALL_KUBECTL INSTALL_HELM USE_GO VOLTHA_LOG_LEVEL \
+    INSTALL_ONOS_APPS SKIP_RESTART_API INSTALL_KUBECTL INSTALL_HELM USE_GO VOLTHA_LOG_LEVEL \
     VOLTHA_CHART VOLTHA_ADAPTER_SIM_CHART VOLTHA_ADAPTER_OPEN_OLT_CHART \
-    VOLTHA_ADAPTER_OPEN_ONU_CHART SADIS_VER OLT_VER AAA_VER DHCP_VER \
+    VOLTHA_ADAPTER_OPEN_ONU_CHART ONOS_TAG \
     ONOS_API_PORT ONOS_SSH_PORT VOLTHA_API_PORT VOLTHA_SSH_PORT VOLTHA_ETCD_PORT"
 for O in $ALL_OPTIONS; do
     VAL=$(eval echo \$$O)
@@ -562,6 +580,38 @@
     fi
 done
 
+push_onos_config() {
+    local MSG=$1
+    local RESOURCE=$2
+    local DATA=$3
+
+    bspin - "$MSG $GEAR"
+    until test; do
+        (set -x; curl --fail -sSL --user karaf:karaf -X POST -H Content-Type:application/json http://127.0.0.1:$ONOS_API_PORT/onos/v1/$RESOURCE --data @$DATA >>$LOG 2>&1) >>$LOG 2>&1
+        if [ $? -eq 0 ]; then
+            break
+        fi
+        sleep .2
+        sspin -
+    done
+    espin - $VERIFIED
+}
+
+override_onos_app() {
+    local APP=$1
+    local NAME=$(basename $APP | sed -e 's/-.*$//g')
+    until test; do
+        sspin -
+        # Attempt to delete old version (if it exists)
+        (set -x; curl --fail -sSL --user karaf:karaf -X DELETE http://127.0.0.1:$ONOS_API_PORT/onos/v1/applications/$NAME >>$LOG 2>&1) >>$LOG 2>&1
+        sspin -
+        (set -x; curl --fail -sSL --user karaf:karaf -X POST -H Content-Type:application/octet-stream http://127.0.0.1:$ONOS_API_PORT/onos/v1/applications?activate=true --data-binary @$APP >>$LOG 2>&1) >>$LOG 2>&1
+        if [ $? -eq 0 ]; then
+            break
+        fi
+        sleep .2
+    done
+}
 
 count_pods() {
     local NAMESPACE=$1; shift
@@ -669,7 +719,7 @@
     fi
 fi
 if [ "$WITH_TIMINGS" == "yes" ]; then
-    timeout $(expr $(date +%s) - $STIME)
+    printtime $(expr $(date +%s) - $STIME)
 fi
 
 STIME=$(date +%s)
@@ -690,7 +740,7 @@
 fi
 
 if [ "$WITH_TIMINGS" == "yes" ]; then
-    timeout $(expr $(date +%s) - $STIME)
+    printtime $(expr $(date +%s) - $STIME)
 fi
 
 STIME=$(date +%s)
@@ -709,7 +759,7 @@
     fi
 fi
 if [ "$WITH_TIMINGS" == "yes" ]; then
-    timeout $(expr $(date +%s) - $STIME)
+    printtime $(expr $(date +%s) - $STIME)
 fi
 
 STIME=$(date +%s)
@@ -724,7 +774,7 @@
     espin - $VERIFIED
 fi
 if [ "$WITH_TIMINGS" == "yes" ]; then
-    timeout $(expr $(date +%s) - $STIME)
+    printtime $(expr $(date +%s) - $STIME)
 fi
 
 bspin "Verify command PATH"
@@ -765,7 +815,7 @@
     wait_for_pods - "kube-system" $EXPECT -1 "Waiting for system PODs to start" $P
 fi
 if [ "$WITH_TIMINGS" == "yes" ]; then
-    timeout $(expr $(date +%s) - $STIME)
+    printtime $(expr $(date +%s) - $STIME)
 fi
 
 STIME=$(date +%s)
@@ -824,7 +874,7 @@
 fi
 wait_for_pods - "kube-system" 1 -1 "Waiting for Tiller POD to start" "tiller-deploy-.*"
 if [ "$WITH_TIMINGS" == "yes" ]; then
-    timeout $(expr $(date +%s) - $STIME)
+    printtime $(expr $(date +%s) - $STIME)
 fi
 
 STIME=$(date +%s)
@@ -838,7 +888,7 @@
     espin $VERIFIED
 fi
 if [ "$WITH_TIMINGS" == "yes" ]; then
-    timeout $(expr $(date +%s) - $STIME)
+    printtime $(expr $(date +%s) - $STIME)
 fi
 
 if [ "$JUST_K8S" == "yes" ]; then
@@ -870,7 +920,7 @@
 EXPECT=$(test "$TYPE" == "minimal" && echo "1" || echo "3")
 wait_for_pods - "voltha" $EXPECT -1 "Waiting for ETCD Operator to start" "etcd-operator-.*"
 if [ "$WITH_TIMINGS" == "yes" ]; then
-    timeout $(expr $(date +%s) - $STIME)
+    printtime $(expr $(date +%s) - $STIME)
 fi
 
 STIME=$(date +%s)
@@ -878,7 +928,11 @@
     bspin "Verify ONOS installed $BIRD"
     if [ $(helm list --deployed --short --namespace default "^onos\$" | wc -l) -ne 1 ]; then
         espin $NOT_VERIFIED
-        helm_install - default onos onf/onos "Install ONOS"
+        SET_TAG=
+        if [ "$ONOS_TAG X" != " X" ]; then
+            SET_TAG="--set images.onos.tag=$ONOS_TAG"
+        fi
+        EXTRA_HELM_FLAGS="$SET_TAG $EXTRA_HELM_FLAGS" helm_install - default onos onf/onos "Install ONOS"
     else
         espin $VERIFIED
     fi
@@ -896,24 +950,34 @@
     done
     (set -x; screen -dmS onos-ssh-$NAME bash -c "while true; do kubectl port-forward service/onos-ssh $ONOS_SSH_PORT:8101; done" >>$LOG 2>&1) >>$LOG 2>&1
     espin - $VERIFIED
-    if [ ! -x ./onos-files/install-onos-applications.sh ]; then
-        bspin - "Verify or download ONOS configuration support files $DOWNLOAD"
-        (set -x; mkdir -p ./onos-files  >>$LOG 2>&1) >>$LOG 2>&1
-        for i in dhcp-to-controller-flow.json install-onos-applications.sh olt-onos-enableExtraneousRules.json olt-onos-netcfg.json olt-onos-olt-settings.json radius-config.json; do
-            if [ ! -r ./onos-files/$i ]; then
-                (set -x; curl -o ./onos-files/$i -sSL https://raw.githubusercontent.com/ciena/kind-voltha/master/onos-files/$i >>$LOG 2>&1) >>$LOG 2>&1
-            fi
-        done
-        (set -x; chmod 755 ./onos-files/install-onos-applications.sh >>$LOG 2>&1) >>$LOG 2>&1
-        espin - $VERIFIED
-    fi
-    
-    bspin - "Install required ONOS applications $INSTALL"
-    (set -x; ./onos-files/install-onos-applications.sh >>$LOG 2>&1) >>$LOG 2>&1
+    bspin - "Verify or download ONOS configuration support files $DOWNLOAD"
+    (set -x; mkdir -p ./onos-files  >>$LOG 2>&1) >>$LOG 2>&1
+    for i in dhcp-to-controller-flow.json olt-onos-enableExtraneousRules.json olt-onos-netcfg.json olt-onos-olt-settings.json radius-config.json; do
+        if [ ! -r ./onos-files/$i ]; then
+            (set -x; curl -o ./onos-files/$i -sSL https://raw.githubusercontent.com/ciena/kind-voltha/master/onos-files/$i >>$LOG 2>&1) >>$LOG 2>&1
+        fi
+    done
     espin - $VERIFIED
+
+    if [ $INSTALL_ONOS_APPS == "yes" ]; then
+        bspin - "Installing custom ONOS applications"
+        if [ -x onos-files/onos-apps -a $(ls -1 onos-files/onos-apps/*.oar 2>/dev/null | wc -l) -gt 0 ];  then
+            for OAR in $(ls -1 onos-files/onos-apps/*.oar); do
+                sspin - "Installing custom ONOS applications - $OAR$CEOL"
+               override_onos_app $OAR
+            done
+            espin - "$VERIFIED Installing custom ONOS applications$CEOL"
+        else
+            espin - "$NOT_VERIFIED Installing custom ONOS applications - None Found"
+        fi
+    fi
+   
+    push_onos_config "Push ONOS Network Configuration" "network/configuration" "onos-files/olt-onos-netcfg.json"
+    push_onos_config "Enable VOLTHA ONOS DHCP provisioning" "configuration/org.opencord.olt.impl.Olt" "onos-files/olt-onos-olt-settings.json"
+    push_onos_config "Enabling extraneous rules for ONOS" "configuration/org.onosproject.net.flow.impl.FlowRuleManager" "onos-files/olt-onos-enableExtraneousRules.json"
 fi
 if [ "$WITH_TIMINGS" == "yes" ]; then
-    timeout $(expr $(date +%s) - $STIME)
+    printtime $(expr $(date +%s) - $STIME)
 fi
 
 STIME=$(date +%s)
@@ -936,7 +1000,7 @@
 EXPECT=$(test "$TYPE" == "minimal" && echo "9" || echo "11")
 wait_for_pods - "voltha" $EXPECT -1 "Waiting for VOLTHA Core to start" $VOLTHA
 if [ "$WITH_TIMINGS" == "yes" ]; then
-    timeout $(expr $(date +%s) - $STIME)
+    printtime $(expr $(date +%s) - $STIME)
 fi
 
 STIME=$(date +%s)
@@ -967,7 +1031,7 @@
 ADAPTERS="adapter-.*"
 wait_for_pods - "voltha" 4 -1 "Waiting for adapters to start" $ADAPTERS
 if [ "$WITH_TIMINGS" == "yes" ]; then
-    timeout $(expr $(date +%s) - $STIME)
+    printtime $(expr $(date +%s) - $STIME)
 fi
 
 if [ $WITH_BBSIM == "yes" ]; then
@@ -982,7 +1046,7 @@
     fi
     wait_for_pods - "voltha" 1 -1 "Waiting for BBSIM to start" "bbsim-.*"
     if [ "$WITH_TIMINGS" == "yes" ]; then
-        timeout $(expr $(date +%s) - $STIME)
+        printtime $(expr $(date +%s) - $STIME)
     fi
 fi
 
@@ -998,7 +1062,7 @@
     fi
     wait_for_pods - "voltha" 1 -1 "Waiting for RADIUS to start" "radius-.*"
     if [ "$WITH_TIMINGS" == "yes" ]; then
-        timeout $(expr $(date +%s) - $STIME)
+        printtime $(expr $(date +%s) - $STIME)
     fi
 fi
 
@@ -1034,7 +1098,7 @@
 (set -x; screen -dmS voltha-etcd-$NAME bash -c "while true; do kubectl port-forward -n voltha service/voltha-etcd-cluster-client $VOLTHA_ETCD_PORT:2379; done" >>$LOG 2>&1) >>$LOG 2>&1
 espin - $VERIFIED
 if [ "$WITH_TIMINGS" == "yes" ]; then
-    timeout $(expr $(date +%s) - $STIME)
+    printtime $(expr $(date +%s) - $STIME)
 fi
 
 if [ $WITH_ONOS == "yes" -a $WITH_RADIUS == "yes" ]; then
@@ -1089,3 +1153,7 @@
 echo "" | tee -a $LOG
 echo "Thank you for choosing kind-voltha for you quick cluster needs." | tee -a $LOG
 
+if [ "$WITH_TIMINGS" == "yes" ]; then
+    echo -e "$CLOCK  ${BOLD}TOTAL: $(duration $(expr $(date +%s) - $TOTAL_START_TIME))${NORMAL}"
+fi
+