[VOL-5177] - verify_voltha-openolt-adapter_sanity-test failure

vars/volthaStackDeploy.groovy
-----------------------------
  o Remove stray 1 from c1fg, gaming keyboard sensitivity & low hovering hands.
  o Move 2.9 ONOS deployment loop into a function with unwound pipelines so
    we can periodically log output.

Change-Id: Id6b946c5976551be7d0a6feae42965ecb5c20fd5
diff --git a/jjb/voltha-e2e/voltha-2.8.yaml b/jjb/voltha-e2e/voltha-2.8.yaml
index 912e530..994a79d 100644
--- a/jjb/voltha-e2e/voltha-2.8.yaml
+++ b/jjb/voltha-e2e/voltha-2.8.yaml
@@ -30,7 +30,7 @@
           code-branch: 'voltha-2.8'
           extraHelmFlags: '--set kafka.externalAccess.enabled=true,kafka.externalAccess.service.type=NodePort,kafka.externalAccess.service.nodePorts[0]=30201,kafka.externalAccess.service.domain=127.0.0.1'
           time-trigger: "H H/23 * * *"
-          logLevel: 'INFO'
+          logLevel: 'DEBUG'
           testTargets: |
             - target: voltha-pm-data-single-kind-att
               workflow: att
diff --git a/vars/volthaStackDeploy.groovy b/vars/volthaStackDeploy.groovy
index 39d5e26..97214b8 100644
--- a/vars/volthaStackDeploy.groovy
+++ b/vars/volthaStackDeploy.groovy
@@ -69,7 +69,7 @@
                script : "rm -f $WORKSPACE/bbsimCfg${cfg.stackId}${i}.yaml",
             )
 
-            if (c1fg.workflow == 'att' || cfg.workflow == 'tt') {
+            if (cfg.workflow == 'att' || cfg.workflow == 'tt') {
                 int startingStag = 900
                 def serviceConfigFile = cfg.workflow
                 if (cfg.withMacLearning && cfg.workflow == 'tt') {
@@ -123,7 +123,7 @@
 set +x #        # Noisy when commented (default: uncommented)
 
 declare -i count=0
-vsd_log='volthaStackDeploy.tmp'
+avsd_log='volthaStackDeploy.tmp'
 touch \$vsd_log
 while true; do
 
@@ -157,8 +157,63 @@
 }
 
 // -----------------------------------------------------------------------
+// Intent: Wait until the pod completed, meaning ONOS fully deployed
 // -----------------------------------------------------------------------
-def process(Map config) {
+void waitForOnosDeploy(Map cfg) {
+    enter('waitForOnosDeploy')
+
+    sh(label  : 'Wait for ONOS full deployment',
+       Scriptx : """
+
+cat <<EOM
+
+** -----------------------------------------------------------------------
+** Wait for ONOS full deployment
+** -----------------------------------------------------------------------
+EOM
+
+# set -euo pipefail
+set +x #        # Noisy when commented (default: uncommented)
+
+declare -i count=0
+avsd_log='volthaStackDeploy.tmp'
+touch \$vsd_log
+while true; do
+
+    ## Exit when the server begins showing signs of life
+    if grep -q '0/' \$vsd_log; then
+        echo 'volthaStackDeploy.groovy: Detected kubectl pods =~ 0/'
+        grep '0/' \$vsd_log
+        break
+    fi
+
+    sleep 5
+    count=\$((\$count - 1))
+
+    if [[ \$count -lt 1 ]]; then # [DEBUG] Display activity every minute or so
+        count=10
+        kubectl get pods -l app=onos-config-loader \
+            -n ${cfg.infraNamespace} --no-headers \
+            --field-selector=status.phase=Running \
+            | tee \$vsd_log
+    else
+        kubectl get pods -l app=onos-config-loader \
+            -n ${cfg.infraNamespace} --no-headers \
+            --field-selector=status.phase=Running \
+            > \$vsd_log
+    fi
+
+done
+rm -f \$vsd_log
+""")
+
+    leave('waitForOnosDeploy')
+    return
+}
+
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+void process(Map config) {
     enter('process')
 
     // note that I can't define this outside the function as there's no global scope in Groovy
@@ -199,6 +254,7 @@
     deployVolthaStack(cfg)
     launchVolthaStack(cfg)
     waitForAdapters(cfg)
+    waitForOnosDeploy(cfg)
     leave('process')
 
     return