VOL-4857 - add debugging

vars/installVoltctl.groovy
--------------------------
   o Display release vars in a single statement to avoid jenkins inserting 'echo' between each.

vars/setOnosLogLevels.groovy
vars/volthaDeploy.groovy
vars/volthaInfraDeploy.groovy
vars/volthaStackDeploy.groovy
-----------------------------
   o Added an ENTER/LEAVE printf to identify running script.
   o Consistent use of a shebang path.
   o Use pgrep to display port forwarding prior to termination
     + command pipeline can be shorted, let pgrep/pkill do heavy lifting.
   o Inlined temporary debugging to view output pipelines are acting on.
     + Debugging etcd:port=null and a few other anomolies.

Change-Id: I40d6b87519cd0fc84406a8a8a76f244da0096496
diff --git a/vars/installVoltctl.groovy b/vars/installVoltctl.groovy
index 6236217..cacf6e8 100644
--- a/vars/installVoltctl.groovy
+++ b/vars/installVoltctl.groovy
@@ -2,7 +2,7 @@
 
 def call(String branch) {
 
-    String iam = 'vars/installVoltha.groovy'
+    String iam = 'vars/installVoltctl.groovy'
     println("** ${iam}: ENTER")
 
     // This logic seems odd given we branch & tag repositories
@@ -19,8 +19,10 @@
     boolean has_binding = binding.hasVariable('WORKSPACE')
 
     // WIP: Probe to find out what is available
-    print(" ** have_released: ${have_released}")
-    print(" ** has_binding: ${has_binding}")
+    print("""
+** have_released: ${have_released}
+**   has_binding: ${has_binding}
+""")
 
     // ---------------------------------------------
     // Sanity check: released version must be frozen
@@ -99,7 +101,7 @@
     voltctl version --clientonly
 
     # Should use diff or md5sum here
-    /bin/ls -l \$(which voltha)
+    /bin/ls -l \$(which voltctl)
   """
 
     println("** ${iam}: LEAVE")
diff --git a/vars/setOnosLogLevels.groovy b/vars/setOnosLogLevels.groovy
index 5b1bd45..5471938 100644
--- a/vars/setOnosLogLevels.groovy
+++ b/vars/setOnosLogLevels.groovy
@@ -1,5 +1,10 @@
+#!/usr/bin/env groovy
+
 def call(Map config) {
 
+    String iam = 'vars/setOnosLogLevels.groovy'
+    println("** ${iam}: ENTER")
+
   def defaultConfig = [
       onosNamespace: "infra",
       apps: ['org.opencord.dhcpl2relay', 'org.opencord.olt', 'org.opencord.aaa'],
@@ -32,8 +37,12 @@
         """
       }
       sh """
+        pgrep --list-all port-forw
+
         ps e -ww -A | grep _TAG="onos-pf" | grep -v grep | awk '{print \$1}' | xargs --no-run-if-empty kill -9
         ps aux | grep port-forward
       """
   }
+
+    println("** ${iam}: LEAVE")
 }
diff --git a/vars/volthaDeploy.groovy b/vars/volthaDeploy.groovy
index 8cb2622..cf76133 100644
--- a/vars/volthaDeploy.groovy
+++ b/vars/volthaDeploy.groovy
@@ -1,8 +1,15 @@
+#!/usr/bin/env groovy
+
 // this keyword is dedicated to deploy a single VOLTHA stack with infra
 // If you need to deploy different configurations you can use the volthaInfraDeploy and volthaStackDeploy keywords
 
 def call(Map config) {
+
+    String iam = 'vars/volthaDeploy.groovy'
+    println("** ${iam}: ENTER")
+
     // note that I can't define this outside the function as there's no global scope in Groovy
+    // [joey] A class method or library call can be used in place of globals, fqdn needed.
     def defaultConfig = [
       onosReplica: 1,
       atomixReplica: 1,
@@ -60,4 +67,6 @@
     if (cfg.withVolthaStack) {
       volthaStackDeploy(cfg)
     }
+
+    println("** ${iam}: LEAVE")
 }
diff --git a/vars/volthaInfraDeploy.groovy b/vars/volthaInfraDeploy.groovy
index a6870d8..189b562 100644
--- a/vars/volthaInfraDeploy.groovy
+++ b/vars/volthaInfraDeploy.groovy
@@ -1,3 +1,5 @@
+#!/usr/bin/env groovy
+
 // usage
 //
 // stage('test stage') {
@@ -8,8 +10,11 @@
 //   }
 // }
 
-
 def call(Map config) {
+
+    String iam = 'vars/volthaInfraDeploy.groovy'
+    println("** ${iam}: ENTER")
+    
     // NOTE use params or directule extraHelmFlags??
     def defaultConfig = [
       onosReplica: 1,
@@ -50,6 +55,13 @@
       kubeconfig = env.KUBECONFIG
     }
 
+    /*
+     [joey] - should pre-existing hint the env is tainted (?)
+     05:24:57  + kubectl create namespace infra
+     05:24:57  Error from server (AlreadyExists): namespaces "infra" already exists
+     05:24:57  error: failed to create configmap: configmaps "kube-config" already exists
+     */
+
     sh """
     kubectl create namespace ${cfg.infraNamespace} || true
     kubectl create configmap -n ${cfg.infraNamespace} kube-config "--from-file=kube_config=${kubeconfig}"  || true
@@ -72,4 +84,6 @@
           --set etcd.replicaCount=${cfg.etcdReplica} \
           -f $WORKSPACE/voltha-helm-charts/examples/${serviceConfigFile}-values.yaml ${cfg.extraHelmFlags}
     """
+
+    println("** ${iam}: LEAVE")    
 }
diff --git a/vars/volthaStackDeploy.groovy b/vars/volthaStackDeploy.groovy
index c63850b..eb6ab0b 100644
--- a/vars/volthaStackDeploy.groovy
+++ b/vars/volthaStackDeploy.groovy
@@ -1,5 +1,10 @@
+#!/usr/bin/env groovy
 
 def call(Map config) {
+
+    String iam = 'vars/volthaStackDeploy.groovy'
+    println("** ${iam}: ENTER")
+
     // note that I can't define this outside the function as there's no global scope in Groovy
     def defaultConfig = [
       bbsimReplica: 1,
@@ -84,6 +89,10 @@
 
     sh """
         set +x
+
+        # [joey]: debug
+        kubectl get pods -n ${cfg.volthaNamespace} -l app.kubernetes.io/part-of=voltha --no-headers
+
         voltha=\$(kubectl get pods -n ${cfg.volthaNamespace} -l app.kubernetes.io/part-of=voltha --no-headers | grep "0/" | wc -l)
         while [[ \$voltha != 0 ]]; do
           sleep 5
@@ -116,4 +125,6 @@
           config=\$(kubectl get pods -l app=onos-config-loader -n ${cfg.infraNamespace} --no-headers --field-selector=status.phase=Running | grep "0/" | wc -l)
         done
     """
+
+    println("** ${iam}: LEAVE")
 }
diff --git a/vars/waitForAdapters.groovy b/vars/waitForAdapters.groovy
index 14b8cdb..e8d0cb7 100644
--- a/vars/waitForAdapters.groovy
+++ b/vars/waitForAdapters.groovy
@@ -1,5 +1,10 @@
+#!/usr/bin/env groovy
+
 def call(Map config) {
 
+    String iam = 'vars/waitForAdapters.groovy'
+    println("** ${iam}: ENTER")
+
     def defaultConfig = [
         volthaNamespace: "voltha",
         stackName: "voltha",
@@ -74,6 +79,10 @@
 
     sh """
       set +x
+      pgrep --list-all port-forw
+
       ps aux | grep port-forw | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9 || true
     """
+
+    println("** ${iam}: LEAVE")
 }