[VOL-5170] - Test debugging openolt-adapter-sanity-test-voltha

vars/createKubernetesCluster.groovy
vars/volthaStackDeploy.groovy
vars/waitForAdapters.groovy
-----------------------------------
  o npm-groovy-lint cleanups
  o Correct indentation
  o emacs untabify (tabs to spaces)
  o Split long shell commands into distinct sh() blocks.
  o Add sh() attribute 'label' so logged commands are self-documenting.
  o Added more banners to label logged output.

vars/volthaStackDeploy.groovy
-----------------------------
  o remove set +x from sh() commands to add log output.
  o re-enable thrown excpetions for process().  This may generate
    duplicate stack traces but at least we see all errors.

vars/pgrep_port_forward.groovy
------------------------------
  o Convenience script, refactor pgrep/pkill logic into a reusable library
    to avoid inlining copy & paste port-forward setup/teardown logic.
  o Shell command must be hardcoded (YUCK!).  Groovy Strings are type
    GString with no native cast to java.lang.string for sh("$cmd") calls.
    String casting works at times due to def/object type but exceptions
    are thrown when incorrect so use a known/reliable value for now.

jjb/verify/voltha-openolt-adapter.yaml
--------------------------------------
  o Jobgen did not run last attempt, include a config file to force it.

Change-Id: I70843625d6a0ae510594a764697e9e0405dfe64b
diff --git a/vars/volthaStackDeploy.groovy b/vars/volthaStackDeploy.groovy
index bb3b1be..fc7bbcd 100644
--- a/vars/volthaStackDeploy.groovy
+++ b/vars/volthaStackDeploy.groovy
@@ -26,89 +26,113 @@
 }
 
 // -----------------------------------------------------------------------
+// Intent: Log progress message
+// -----------------------------------------------------------------------
+void enter(String name) {
+    // Announce ourselves for log usability
+    String iam = getIam(name)
+    println("${iam}: ENTER")
+    return
+}
+
+// -----------------------------------------------------------------------
+// Intent: Log progress message
+// -----------------------------------------------------------------------
+void leave(String name) {
+    // Announce ourselves for log usability
+    String iam = getIam(name)
+    println("${iam}: LEAVE")
+    return
+}
+
+// -----------------------------------------------------------------------
 // -----------------------------------------------------------------------
 def process(Map config)
 {
+    enter('process')
+
     // note that I can't define this outside the function as there's no global scope in Groovy
     def defaultConfig = [
-      bbsimReplica: 1,
-      infraNamespace: "infra",
-      volthaNamespace: "voltha",
-      stackName: "voltha",
-      stackId: 1, // NOTE this is used to differentiate between BBSims across multiple stacks
-      workflow: "att",
-      withMacLearning: false,
-      withFttb: false,
-      extraHelmFlags: "",
-      localCharts: false,
-      onosReplica: 1,
-      adaptersToWait: 2,
+        bbsimReplica: 1,
+        infraNamespace: "infra",
+        volthaNamespace: "voltha",
+        stackName: "voltha",
+        stackId: 1, // NOTE this is used to differentiate between BBSims across multiple stacks
+        workflow: "att",
+        withMacLearning: false,
+        withFttb: false,
+        extraHelmFlags: "",
+        localCharts: false,
+        onosReplica: 1,
+        adaptersToWait: 2,
     ]
-
+    
     def cfg = defaultConfig + config
-
+    
     def volthaStackChart = "onf/voltha-stack"
     def bbsimChart = "onf/bbsim"
-
+    
     if (cfg.localCharts) {
-      volthaStackChart = "$WORKSPACE/voltha-helm-charts/voltha-stack"
-      bbsimChart = "$WORKSPACE/voltha-helm-charts/bbsim"
-
-      sh """
+        volthaStackChart = "$WORKSPACE/voltha-helm-charts/voltha-stack"
+        bbsimChart = "$WORKSPACE/voltha-helm-charts/bbsim"
+        
+        sh """
       pushd $WORKSPACE/voltha-helm-charts/voltha-stack
       helm dep update
       popd
       """
     }
-
+    
     println "Deploying VOLTHA Stack with the following parameters: ${cfg}."
-
-    sh """
+    
+    sh(label  : "Create VOLTHA Stack ${cfg.stackName}, (namespace=${cfg.volthaNamespace})",
+       script : """
     helm upgrade --install --create-namespace -n ${cfg.volthaNamespace} ${cfg.stackName} ${volthaStackChart} \
           --set global.stack_name=${cfg.stackName} \
           --set global.voltha_infra_name=voltha-infra \
           --set voltha.onos_classic.replicas=${cfg.onosReplica} \
           --set global.voltha_infra_namespace=${cfg.infraNamespace} \
           ${cfg.extraHelmFlags}
-    """
+    """)
 
     for(int i = 0;i<cfg.bbsimReplica;i++) {
-      // NOTE we don't need to update the tag for DT
-      script {
-        sh """
-        rm -f $WORKSPACE/bbsimCfg${cfg.stackId}${i}.yaml
-        """
-        if (cfg.workflow == "att" || cfg.workflow == "tt") {
-          def startingStag = 900
-          def serviceConfigFile = cfg.workflow
-          if (cfg.withMacLearning && cfg.workflow == 'tt') {
-            serviceConfigFile = "tt-maclearner"
-          }
-          def bbsimCfg = readYaml file: "$WORKSPACE/voltha-helm-charts/examples/${serviceConfigFile}-values.yaml"
-          // NOTE we assume that the only service that needs a different s_tag is the first one in the list
-          bbsimCfg["servicesConfig"]["services"][0]["s_tag"] = startingStag + i
-          println "Using BBSim Service config ${bbsimCfg['servicesConfig']}"
-          writeYaml file: "$WORKSPACE/bbsimCfg${cfg.stackId}${i}.yaml", data: bbsimCfg
-        } else {
-          // NOTE if it's DT just copy the file over
-          sh """
+        // NOTE we don't need to update the tag for DT
+        script {
+            sh(label  : "Create config[$i]: bbsimCfg${cfg.stackId}${i}.yaml",
+               script : "rm -f $WORKSPACE/bbsimCfg${cfg.stackId}${i}.yaml",
+            )
+            
+            if (cfg.workflow == "att" || cfg.workflow == "tt") {
+                def startingStag = 900
+                def serviceConfigFile = cfg.workflow
+                if (cfg.withMacLearning && cfg.workflow == 'tt') {
+                    serviceConfigFile = "tt-maclearner"
+                }
+                def bbsimCfg = readYaml file: "$WORKSPACE/voltha-helm-charts/examples/${serviceConfigFile}-values.yaml"
+                // NOTE we assume that the only service that needs a different s_tag is the first one in the list
+                bbsimCfg["servicesConfig"]["services"][0]["s_tag"] = startingStag + i
+                println "Using BBSim Service config ${bbsimCfg['servicesConfig']}"
+                writeYaml file: "$WORKSPACE/bbsimCfg${cfg.stackId}${i}.yaml", data: bbsimCfg
+            } else {
+                // NOTE if it's DT just copy the file over
+                sh """
           cp $WORKSPACE/voltha-helm-charts/examples/${cfg.workflow}-values.yaml $WORKSPACE/bbsimCfg${cfg.stackId}${i}.yaml
           """
+            }
         }
-      }
-
-      sh """
+        
+        sh(label  : "HELM: Create namespace=${cfg.volthaNamespace} bbsim${i}",
+           script :  """
         helm upgrade --install --create-namespace -n ${cfg.volthaNamespace} bbsim${i} ${bbsimChart} \
         --set olt_id="${cfg.stackId}${i}" \
         -f $WORKSPACE/bbsimCfg${cfg.stackId}${i}.yaml \
         ${cfg.extraHelmFlags}
-      """
+      """)
     }
-
-    println "Wait for VOLTHA Stack ${cfg.stackName} to start"
-
-    sh """
-        set +x
+    
+    sh(label  : "Wait for VOLTHA Stack ${cfg.stackName} to start",
+       script : """
+#        set +x
 
         # [joey]: debug
         kubectl get pods -n ${cfg.volthaNamespace} -l app.kubernetes.io/part-of=voltha --no-headers
@@ -118,63 +142,48 @@
           sleep 5
           voltha=\$(kubectl get pods -n ${cfg.volthaNamespace} -l app.kubernetes.io/part-of=voltha --no-headers | grep "0/" | wc -l)
         done
-    """
-
+    """)
+    
     waitForAdapters(cfg)
 
     // also make sure that the ONOS config is loaded
     // NOTE that this is only required for VOLTHA-2.8
     println "Wait for ONOS Config loader to complete"
 
-    // NOTE that this is only required for VOLTHA-2.8,
-    // [TODO]: Release cleanup
-    sh """
-        set +x
-        config=\$(kubectl get jobs.batch -n ${cfg.infraNamespace} --no-headers | grep "0/" | wc -l)
-        while [[ \$config != 0 ]]; do
-          sleep 5
-          config=\$(kubectl get jobs.batch -n ${cfg.infraNamespace} --no-headers | grep "0/" | wc -l)
-        done
-    """
-
-    // NOTE that this is only required for VOLTHA-2.9 onwards, to wait until the pod completed,
-    // meaning ONOS fully deployed
-    sh """
-        set +x
+    // Wait until the pod completed, meaning ONOS fully deployed
+    sh(label   : '"Wait for ONOS Config loader to fully deploy',
+        script : """
+#        set +x
         config=\$(kubectl get pods -l app=onos-config-loader -n ${cfg.infraNamespace} --no-headers --field-selector=status.phase=Running | grep "0/" | wc -l)
         while [[ \$config != 0 ]]; do
           sleep 5
           config=\$(kubectl get pods -l app=onos-config-loader -n ${cfg.infraNamespace} --no-headers --field-selector=status.phase=Running | grep "0/" | wc -l)
         done
-    """
+    """)
 
+    leave('process')
+    
     return
 }
 
 // -----------------------------------------------------------------------
 // -----------------------------------------------------------------------
-def call(Map config)
+def call(Map config=[:])
 {
-    String iam = getIam('main')
-    println("** ${iam}: ENTER")
-
-    if (!config) {
-        config = [:]
-    }
-
     try
-    {
-	process(config)
+    {   
+        enter('main')
+        process(config)
     }
-    catch (Exception err)
-    {
-	println("** ${iam}: EXCEPTION ${err}")
-	// Cannot re-throw ATM: too many potential shell errors.
-	// throw err
+    catch (Exception err) {  // groovylint-disable-line CatchException
+        ans = false
+        println("** volthaStackDeploy.groovy: EXCEPTION ${err}")
+        throw err
     }
     finally
     {
-	println("** ${iam}: LEAVE")
+        leave('main')
+        println("** ${iam}: LEAVE")
     }
     return
 }