[VOL-5170] - Test debugging openolt-adapter-sanity-test-voltha

vars/createKubernetesCluster.groovy
vars/volthaStackDeploy.groovy
vars/waitForAdapters.groovy
-----------------------------------
  o npm-groovy-lint cleanups
  o Correct indentation
  o emacs untabify (tabs to spaces)
  o Split long shell commands into distinct sh() blocks.
  o Add sh() attribute 'label' so logged commands are self-documenting.
  o Added more banners to label logged output.

vars/volthaStackDeploy.groovy
-----------------------------
  o remove set +x from sh() commands to add log output.
  o re-enable thrown excpetions for process().  This may generate
    duplicate stack traces but at least we see all errors.

vars/pgrep_port_forward.groovy
------------------------------
  o Convenience script, refactor pgrep/pkill logic into a reusable library
    to avoid inlining copy & paste port-forward setup/teardown logic.
  o Shell command must be hardcoded (YUCK!).  Groovy Strings are type
    GString with no native cast to java.lang.string for sh("$cmd") calls.
    String casting works at times due to def/object type but exceptions
    are thrown when incorrect so use a known/reliable value for now.

jjb/verify/voltha-openolt-adapter.yaml
--------------------------------------
  o Jobgen did not run last attempt, include a config file to force it.

Change-Id: I70843625d6a0ae510594a764697e9e0405dfe64b
diff --git a/jjb/verify/voltha-openolt-adapter.yaml b/jjb/verify/voltha-openolt-adapter.yaml
index 3e9afde..b0ddefe 100644
--- a/jjb/verify/voltha-openolt-adapter.yaml
+++ b/jjb/verify/voltha-openolt-adapter.yaml
@@ -18,10 +18,6 @@
           name-extension: '-voltha-2.11'
           override-branch: 'voltha-2.11'
           branch-regexp: '^voltha-2.11$'
-#      - 'verify-voltha-openolt-adapter-jobs-voltha-2.8':
-#          name-extension: '-voltha-2.8'
-#          override-branch: 'voltha-2.8'
-#          branch-regexp: '^voltha-2.8$'
       - 'publish-voltha-openolt-adapter-jobs':
           branch-regexp: '{all-branches-regexp}'
 
@@ -64,12 +60,6 @@
       - 'voltha-patch-test':
           pipeline-script: 'voltha/voltha-2.11/bbsim-tests.groovy'
 
-#- job-group:
-#    name: 'verify-voltha-openolt-adapter-jobs-voltha-2.8'
-#    jobs:
-#      - 'voltha-patch-test':
-#          pipeline-script: 'voltha/voltha-2.8/bbsim-tests.groovy'
-
 - job-group:
     name: 'publish-voltha-openolt-adapter-jobs'
     jobs:
diff --git a/vars/createKubernetesCluster.groovy b/vars/createKubernetesCluster.groovy
index 9653840..a95d084 100644
--- a/vars/createKubernetesCluster.groovy
+++ b/vars/createKubernetesCluster.groovy
@@ -19,8 +19,7 @@
 
 // -----------------------------------------------------------------------
 // -----------------------------------------------------------------------
-def getIam(String func)
-{
+String getIam(String func) {
     // Cannot rely on a stack trace due to jenkins manipulation
     String src = 'vars/createKubernetesCluster.groovy'
     String iam = [src, func].join('::')
@@ -28,23 +27,39 @@
 }
 
 // -----------------------------------------------------------------------
+// Intent: Log progress message
 // -----------------------------------------------------------------------
-def call(Map config) {
+void enter(String name) {
+    // Announce ourselves for log usability
+    String iam = getIam(name)
+    println("${iam}: ENTER")
+    return
+}
+
+// -----------------------------------------------------------------------
+// Intent: Log progress message
+// -----------------------------------------------------------------------
+void leave(String name) {
+    // Announce ourselves for log usability
+    String iam = getIam(name)
+    println("${iam}: LEAVE")
+    return
+}
+
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+def call(Map config=[:]) {
 
     String iam = getIam('main')
-    println("** ${iam}: ENTER")
+    enter('main')
 
     // note that I can't define this outside the function as there's no global scope in Groovy
     def defaultConfig = [
-      branch: "master", // branch=master ?!?
-      nodes: 1,
-      name: "kind-ci"
+        branch: 'master', // branch=master ?!?
+        nodes: 1,
+        name: 'kind-ci'
     ]
 
-    if (!config) {
-        config = [:]
-    }
-
     def cfg = defaultConfig + config
 
     println "Deploying Kind cluster with the following parameters: ${cfg}."
@@ -79,7 +94,7 @@
 
     // TODO: Skip kind install, make install-kind-command has done it already
     sh """
-      mkdir -p "$WORKSPACE/bin"
+      mkdir -p $WORKSPACE/bin
 
       # download kind (should we add it to the base image?)
       curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.11.0/kind-linux-amd64
@@ -90,10 +105,12 @@
     // install voltctl
     installVoltctl("${cfg.branch}")
 
-    sh """
+    sh(label  : 'Start kind cluster',
+       script : """
 cat <<EOM
 
 ** -----------------------------------------------------------------------
+** IAM: ${iam}
 ** Starting kind cluster
 ** -----------------------------------------------------------------------
 EOM
@@ -104,7 +121,10 @@
       for MNODE in \$(kubectl get node --selector='node-role.kubernetes.io/master' -o json | jq -r '.items[].metadata.name'); do
           kubectl taint node "\$MNODE" node-role.kubernetes.io/master:NoSchedule-
       done
+""")
 
+    sh(label  : 'Normalize config permissions',
+       script : """
       ## ----------------------------------------------------------------------
       ## This logic is problematic, when run on a node processing concurrent
       ## jobs over-write will corrupt config for the other running job.
@@ -116,29 +136,37 @@
       umask 022
 
       echo
-      echo "** Generate ~/.volt/config"
+      echo "** Generate $HOME/.volt/config"
       mkdir -p "$HOME/.volt"
-      chmod -R u+w,go-rwx "$HOME/.volt"
       chmod u=rwx "$HOME/.volt"
       voltctl -s localhost:55555 config > "$HOME/.volt/config"
+      chmod -R u+w,go-rwx "$HOME/.volt"
 
       echo
-      echo "** Generate ~/.kube/config"
+      echo "** Generate $HOME/.kube/config"
       mkdir -p "$HOME/.kube"
-      chmod -R u+w,go-rwx "$HOME/.kube"
       chmod u=rwx "$HOME/.kube"
       kind get kubeconfig --name ${cfg.name} > "$HOME/.kube/config"
+      chmod -R u+w,go-rwx "$HOME/.kube"
 
       echo
-      echo "Display .kube/ and .volt"
+      echo "Display .kube/ and .volt/ configs"
       /bin/ls -l "$HOME/.kube" "$HOME/.volt"
+""")
 
-      echo
-      echo "Install Kail"
+    sh(label  : 'Install kail',
+       script : """
+cat <<EOM
+
+** -----------------------------------------------------------------------
+** IAM: ${iam}
+** Install kail
+** -----------------------------------------------------------------------
+EOM
       make -C "$WORKSPACE/voltha-system-tests" KAIL_PATH="$WORKSPACE/bin" kail
-  """
+""")
 
-    println("** ${iam}: LEAVE")
+    enter('leave')
     return
 }
 
diff --git a/vars/pgrep_port_forward.groovy b/vars/pgrep_port_forward.groovy
new file mode 100644
index 0000000..68c5e5a
--- /dev/null
+++ b/vars/pgrep_port_forward.groovy
@@ -0,0 +1,123 @@
+#!/usr/bin/env groovy
+// -----------------------------------------------------------------------
+// Copyright 2023 Open Networking Foundation (ONF) and the ONF Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// -----------------------------------------------------------------------
+// Install the voltctl command by branch name "voltha-xx"
+// -----------------------------------------------------------------------
+
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+String getIam(String func) {
+    // Cannot rely on a stack trace due to jenkins manipulation
+    String src = 'vars/pgrep_proc.groovy'
+    String iam = [src, func].join('::')
+    return iam
+}
+
+// -----------------------------------------------------------------------
+// Intent: Log progress message
+// -----------------------------------------------------------------------
+void enter(String name) {
+    // Announce ourselves for log usability
+    String iam = getIam(name)
+    println("${iam}: ENTER")
+    return
+}
+
+// -----------------------------------------------------------------------
+// Intent: Log progress message
+// -----------------------------------------------------------------------
+void leave(String name) {
+    // Announce ourselves for log usability
+    String iam = getIam(name)
+    println("${iam}: LEAVE")
+    return
+}
+
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+Boolean process(String proc, Map args) {
+    Boolean ans = true
+    String  iam = getIam('process')
+
+    String cmd = [
+        'pgrep',
+        '--uid', '$(id -u)', // no stray signals
+        '--list-full',
+        '--full',  // hmmm: conditional use (?)
+        "'${proc}",
+    ]
+
+    print("""
+** -----------------------------------------------------------------------
+** Running: $cmd
+** -----------------------------------------------------------------------
+""")
+
+    sh(
+        label  : 'pgrep_proc', // jenkins usability: label log entry 'step'
+        // script : ${cmd}.toString(),
+
+        // Cannot derefence cmd AMT.  Value stored as a grovy String/GString.
+        // No native support for cast to java.lang.String, some objects can
+        // but logic is prone to exception so (YUCK!) hardcode for now.
+
+        script : """
+pgrep --uid \$(uid -u) --list-full --full 'port-forw'
+""",
+    )
+    return(ans)
+}
+
+// -----------------------------------------------------------------------
+// Install: Display a list of port-forwarding processes.
+// -----------------------------------------------------------------------
+// groovylint-disable-next-line None, UnusedMethodParameter
+Boolean call\
+(
+    String  proc,           // name of process or arguments to terminate
+    Map     args=[:],
+    Boolean filler = true     // Groovy, why special case list comma handling (?)
+) {
+    Boolean ans = true
+
+    try {
+        enter('main')
+        process(proc, args)
+    }
+    catch (Exception err) {  // groovylint-disable-line CatchException
+        ans = false
+        println("** ${iam}: EXCEPTION ${err}")
+        throw err
+    }
+    finally {
+        enter('main')
+    }
+
+    return(ans)
+}
+
+// [SEE ALSO]
+// -----------------------------------------------------------------------
+//   o String cmd = [ ... ].join('') -- GString cannot cast to java.String
+//   o https://stackoverflow.com/questions/60304068/artifactory-in-jenkins-pipeline-org-codehaus-groovy-runtime-gstringimpl-cannot
+// -----------------------------------------------------------------------
+// [TODO] - Combine pkill_proc and pgrep_proc
+//    - Usage: do_proc(pkill=true, pgrep=true, args='proc-forward', cmd='kubectl'
+//      o When kill == grep == true: display procs, terminate, recheck: fatal if procs detected
+//      o cmd && args (or command containing args) (or list of patterns passed)
+//        - pass arg --full to match entire command line.
+// -----------------------------------------------------------------------
+// [EOF]
diff --git a/vars/volthaStackDeploy.groovy b/vars/volthaStackDeploy.groovy
index bb3b1be..fc7bbcd 100644
--- a/vars/volthaStackDeploy.groovy
+++ b/vars/volthaStackDeploy.groovy
@@ -26,89 +26,113 @@
 }
 
 // -----------------------------------------------------------------------
+// Intent: Log progress message
+// -----------------------------------------------------------------------
+void enter(String name) {
+    // Announce ourselves for log usability
+    String iam = getIam(name)
+    println("${iam}: ENTER")
+    return
+}
+
+// -----------------------------------------------------------------------
+// Intent: Log progress message
+// -----------------------------------------------------------------------
+void leave(String name) {
+    // Announce ourselves for log usability
+    String iam = getIam(name)
+    println("${iam}: LEAVE")
+    return
+}
+
+// -----------------------------------------------------------------------
 // -----------------------------------------------------------------------
 def process(Map config)
 {
+    enter('process')
+
     // note that I can't define this outside the function as there's no global scope in Groovy
     def defaultConfig = [
-      bbsimReplica: 1,
-      infraNamespace: "infra",
-      volthaNamespace: "voltha",
-      stackName: "voltha",
-      stackId: 1, // NOTE this is used to differentiate between BBSims across multiple stacks
-      workflow: "att",
-      withMacLearning: false,
-      withFttb: false,
-      extraHelmFlags: "",
-      localCharts: false,
-      onosReplica: 1,
-      adaptersToWait: 2,
+        bbsimReplica: 1,
+        infraNamespace: "infra",
+        volthaNamespace: "voltha",
+        stackName: "voltha",
+        stackId: 1, // NOTE this is used to differentiate between BBSims across multiple stacks
+        workflow: "att",
+        withMacLearning: false,
+        withFttb: false,
+        extraHelmFlags: "",
+        localCharts: false,
+        onosReplica: 1,
+        adaptersToWait: 2,
     ]
-
+    
     def cfg = defaultConfig + config
-
+    
     def volthaStackChart = "onf/voltha-stack"
     def bbsimChart = "onf/bbsim"
-
+    
     if (cfg.localCharts) {
-      volthaStackChart = "$WORKSPACE/voltha-helm-charts/voltha-stack"
-      bbsimChart = "$WORKSPACE/voltha-helm-charts/bbsim"
-
-      sh """
+        volthaStackChart = "$WORKSPACE/voltha-helm-charts/voltha-stack"
+        bbsimChart = "$WORKSPACE/voltha-helm-charts/bbsim"
+        
+        sh """
       pushd $WORKSPACE/voltha-helm-charts/voltha-stack
       helm dep update
       popd
       """
     }
-
+    
     println "Deploying VOLTHA Stack with the following parameters: ${cfg}."
-
-    sh """
+    
+    sh(label  : "Create VOLTHA Stack ${cfg.stackName}, (namespace=${cfg.volthaNamespace})",
+       script : """
     helm upgrade --install --create-namespace -n ${cfg.volthaNamespace} ${cfg.stackName} ${volthaStackChart} \
           --set global.stack_name=${cfg.stackName} \
           --set global.voltha_infra_name=voltha-infra \
           --set voltha.onos_classic.replicas=${cfg.onosReplica} \
           --set global.voltha_infra_namespace=${cfg.infraNamespace} \
           ${cfg.extraHelmFlags}
-    """
+    """)
 
     for(int i = 0;i<cfg.bbsimReplica;i++) {
-      // NOTE we don't need to update the tag for DT
-      script {
-        sh """
-        rm -f $WORKSPACE/bbsimCfg${cfg.stackId}${i}.yaml
-        """
-        if (cfg.workflow == "att" || cfg.workflow == "tt") {
-          def startingStag = 900
-          def serviceConfigFile = cfg.workflow
-          if (cfg.withMacLearning && cfg.workflow == 'tt') {
-            serviceConfigFile = "tt-maclearner"
-          }
-          def bbsimCfg = readYaml file: "$WORKSPACE/voltha-helm-charts/examples/${serviceConfigFile}-values.yaml"
-          // NOTE we assume that the only service that needs a different s_tag is the first one in the list
-          bbsimCfg["servicesConfig"]["services"][0]["s_tag"] = startingStag + i
-          println "Using BBSim Service config ${bbsimCfg['servicesConfig']}"
-          writeYaml file: "$WORKSPACE/bbsimCfg${cfg.stackId}${i}.yaml", data: bbsimCfg
-        } else {
-          // NOTE if it's DT just copy the file over
-          sh """
+        // NOTE we don't need to update the tag for DT
+        script {
+            sh(label  : "Create config[$i]: bbsimCfg${cfg.stackId}${i}.yaml",
+               script : "rm -f $WORKSPACE/bbsimCfg${cfg.stackId}${i}.yaml",
+            )
+            
+            if (cfg.workflow == "att" || cfg.workflow == "tt") {
+                def startingStag = 900
+                def serviceConfigFile = cfg.workflow
+                if (cfg.withMacLearning && cfg.workflow == 'tt') {
+                    serviceConfigFile = "tt-maclearner"
+                }
+                def bbsimCfg = readYaml file: "$WORKSPACE/voltha-helm-charts/examples/${serviceConfigFile}-values.yaml"
+                // NOTE we assume that the only service that needs a different s_tag is the first one in the list
+                bbsimCfg["servicesConfig"]["services"][0]["s_tag"] = startingStag + i
+                println "Using BBSim Service config ${bbsimCfg['servicesConfig']}"
+                writeYaml file: "$WORKSPACE/bbsimCfg${cfg.stackId}${i}.yaml", data: bbsimCfg
+            } else {
+                // NOTE if it's DT just copy the file over
+                sh """
           cp $WORKSPACE/voltha-helm-charts/examples/${cfg.workflow}-values.yaml $WORKSPACE/bbsimCfg${cfg.stackId}${i}.yaml
           """
+            }
         }
-      }
-
-      sh """
+        
+        sh(label  : "HELM: Create namespace=${cfg.volthaNamespace} bbsim${i}",
+           script :  """
         helm upgrade --install --create-namespace -n ${cfg.volthaNamespace} bbsim${i} ${bbsimChart} \
         --set olt_id="${cfg.stackId}${i}" \
         -f $WORKSPACE/bbsimCfg${cfg.stackId}${i}.yaml \
         ${cfg.extraHelmFlags}
-      """
+      """)
     }
-
-    println "Wait for VOLTHA Stack ${cfg.stackName} to start"
-
-    sh """
-        set +x
+    
+    sh(label  : "Wait for VOLTHA Stack ${cfg.stackName} to start",
+       script : """
+#        set +x
 
         # [joey]: debug
         kubectl get pods -n ${cfg.volthaNamespace} -l app.kubernetes.io/part-of=voltha --no-headers
@@ -118,63 +142,48 @@
           sleep 5
           voltha=\$(kubectl get pods -n ${cfg.volthaNamespace} -l app.kubernetes.io/part-of=voltha --no-headers | grep "0/" | wc -l)
         done
-    """
-
+    """)
+    
     waitForAdapters(cfg)
 
     // also make sure that the ONOS config is loaded
     // NOTE that this is only required for VOLTHA-2.8
     println "Wait for ONOS Config loader to complete"
 
-    // NOTE that this is only required for VOLTHA-2.8,
-    // [TODO]: Release cleanup
-    sh """
-        set +x
-        config=\$(kubectl get jobs.batch -n ${cfg.infraNamespace} --no-headers | grep "0/" | wc -l)
-        while [[ \$config != 0 ]]; do
-          sleep 5
-          config=\$(kubectl get jobs.batch -n ${cfg.infraNamespace} --no-headers | grep "0/" | wc -l)
-        done
-    """
-
-    // NOTE that this is only required for VOLTHA-2.9 onwards, to wait until the pod completed,
-    // meaning ONOS fully deployed
-    sh """
-        set +x
+    // Wait until the pod completed, meaning ONOS fully deployed
+    sh(label   : '"Wait for ONOS Config loader to fully deploy',
+        script : """
+#        set +x
         config=\$(kubectl get pods -l app=onos-config-loader -n ${cfg.infraNamespace} --no-headers --field-selector=status.phase=Running | grep "0/" | wc -l)
         while [[ \$config != 0 ]]; do
           sleep 5
           config=\$(kubectl get pods -l app=onos-config-loader -n ${cfg.infraNamespace} --no-headers --field-selector=status.phase=Running | grep "0/" | wc -l)
         done
-    """
+    """)
 
+    leave('process')
+    
     return
 }
 
 // -----------------------------------------------------------------------
 // -----------------------------------------------------------------------
-def call(Map config)
+def call(Map config=[:])
 {
-    String iam = getIam('main')
-    println("** ${iam}: ENTER")
-
-    if (!config) {
-        config = [:]
-    }
-
     try
-    {
-	process(config)
+    {   
+        enter('main')
+        process(config)
     }
-    catch (Exception err)
-    {
-	println("** ${iam}: EXCEPTION ${err}")
-	// Cannot re-throw ATM: too many potential shell errors.
-	// throw err
+    catch (Exception err) {  // groovylint-disable-line CatchException
+        ans = false
+        println("** volthaStackDeploy.groovy: EXCEPTION ${err}")
+        throw err
     }
     finally
     {
-	println("** ${iam}: LEAVE")
+        leave('main')
+        println("** ${iam}: LEAVE")
     }
     return
 }
diff --git a/vars/waitForAdapters.groovy b/vars/waitForAdapters.groovy
index dcc7fda..92edf0d 100644
--- a/vars/waitForAdapters.groovy
+++ b/vars/waitForAdapters.groovy
@@ -17,8 +17,7 @@
 
 // -----------------------------------------------------------------------
 // -----------------------------------------------------------------------
-def getIam(String func)
-{
+String getIam(String func) {
     // Cannot rely on a stack trace due to jenkins manipulation
     String src = 'vars/waitForAdapters.groovy'
     String iam = [src, func].join('::')
@@ -26,28 +25,45 @@
 }
 
 // -----------------------------------------------------------------------
+// Intent: Log progress message
 // -----------------------------------------------------------------------
-def getAdapters()
-{
-    String iam = getIam('getAdapters')
+void enter(String name) {
+    // Announce ourselves for log usability
+    String iam = getIam(name)
+    println("${iam}: ENTER")
+    return
+}
 
-    def adapters = ""
-    try
-    {
-        adapters = sh (
+// -----------------------------------------------------------------------
+// Intent: Log progress message
+// -----------------------------------------------------------------------
+void leave(String name) {
+    // Announce ourselves for log usability
+    String iam = getIam(name)
+    println("${iam}: LEAVE")
+    return
+}
+
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+def getAdapters() {
+    String iam = getIam('getAdapters')
+    String adapters = ''
+
+    try {
+        adapters = sh(
             script: 'voltctl adapter list --format "{{gosince .LastCommunication}}"',
             returnStdout: true,
         ).trim()
     }
-    catch (err)
-    {
+    catch (err) {
         // in some older versions of voltctl the command results in
         // ERROR: Unexpected error while attempting to format results
         // as table : template: output:1: function "gosince" not defined
         // if that's the case we won't be able to check the timestamp so
         // it's useless to wait
         println("voltctl can't parse LastCommunication, skip waiting")
-	adapters = 'SKIP' // why not just retry a few times (?)
+        adapters = 'SKIP' // why not just retry a few times (?)
     }
 
     print("** ${iam}: returned $adapters")
@@ -79,113 +95,111 @@
 
     String ans = null
     def found = []
-    for(i=0; i<adapters.size(); i++)
-    {
-	String elapsed = adapters[i]
-	if (debug)
-	{
-	    println("** ${iam} Checking elapsed[$i]: $elapsed")
-	}
+    for(i=0; i<adapters.size(); i++) {
+        String elapsed = adapters[i]
+        if (debug) {
+            println("** ${iam} Checking elapsed[$i]: $elapsed")
+        }
 
-	if (! elapsed) // empty string or null
-	{
-	    ans = 'NULL'
-	    break
-	}
+        if (! elapsed) { // empty string or null
+            ans = 'NULL'
+            break
+        }
 
-	Integer size = elapsed.length()
-	if (size > 2) // 463765h58m52(s)
-	{
-	    // higlander: there can be only one
-	    ans = 'DOUBLE-DIGIT'
-	    break
-	}
+        Integer size = elapsed.length()
+        if (size > 2) { // 463765h58m52(s)
+            // higlander: there can be only one
+            ans = 'DOUBLE-DIGIT'
+            break
+        }
 
-	if (elapsed.endsWith('s'))
-	{
-	    // safer than replaceAll('s'): ssss1s => 1
-	    elapsed = elapsed.substring(0, size-1)
-	}
+        if (elapsed.endsWith('s')) {
+            // safer than replaceAll('s'): ssss1s => 1
+            elapsed = elapsed.substring(0, size-1)
+        }
 
-	// Line noise guard: 'a', 'f', '#', '!'
-	if (! elapsed.isInteger())
-	{
-	    ans = 'NON-NUMERIC'
-	    break
-	}
+        // Line noise guard: 'a', 'f', '#', '!'
+        if (! elapsed.isInteger()) {
+            ans = 'NON-NUMERIC'
+            break
+        }
 
-	// Is value in range:
-	//   discard negative integers as just plain wonky.
-	//   HMMM(?): zero/no latency response is kinda odd to include imho.
-	Integer val = elapsed.toInteger()
-	if (5 >= val && val >= 0)
-	{
-	    found.add(elapsed)
-	}
-	else
-	{
-	    ans = 'SIX-NINE'
-	    break
-	}
+        // Is value in range:
+        //   discard negative integers as just plain wonky.
+        //   HMMM(?): zero/no latency response is kinda odd to include imho.
+        Integer val = elapsed.toInteger()
+        if (5 >= val && val >= 0) {
+            found.add(elapsed)
+        }
+        else {
+            ans = 'SIX-NINE'
+            break
+        }
     } // for()
 
     // Declare success IFF all values are:
     //    o integral
     //    o valid
     //    o within range
-    if (ans == null)
-    {
-	Integer got = found.size()
-	Integer exp = adapters.size()
-	ans = (! exp)      ? 'NO-ADAPTERS'
+    if (ans == null) {
+        Integer got = found.size()
+        Integer exp = adapters.size()
+        ans = (! exp)      ? 'NO-ADAPTERS'
             : (got == exp) ? 'VALID'
             : 'CONTINUE'
     }
 
-    if (debug)
-    {
-	println("** ${iam} return: [$ans]")
+    if (debug) {
+        println("** ${iam} return: [$ans]")
     }
     return ans
 } // getAdaptersState
 
 // -----------------------------------------------------------------------
+// Intent: Poll until adapter status is known.
 // -----------------------------------------------------------------------
-def process(Map config)
-{
+def process(Map config) {
     String iam = getIam('process')
-    println("** ${iam}: ENTER")
+    enter('process')
 
     def defaultConfig = [
-        volthaNamespace: "voltha",
-        stackName: "voltha",
-        adaptersToWait: 2,
+        volthaNamespace : 'voltha',
+        stackName       : 'voltha',
+        adaptersToWait  : 2,
     ]
 
     def cfg = defaultConfig + config
 
-    if (cfg.adaptersToWait == 0){
-       //no need to wait
-       println "No need to wait for adapters to be registered"
-       return
+    if (cfg.adaptersToWait == 0) {
+        //no need to wait
+        println "No need to wait for adapters to be registered"
+        return
     }
 
     println("** ${iam}: Wait for adapters to be registered")
 
     // guarantee that at least the specified number of adapters are registered with VOLTHA before proceeding
-     sh """
-        set +x
-        _TAG="voltha-voltha-api" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${cfg.volthaNamespace} svc/${cfg.stackName}-voltha-api 55555:55555; done"&
-       """
+    sh(label  : 'waitForAdapters: Initiate port forwarding',
+       script : """
+        _TAG="voltha-voltha-api" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${cfg.volthaNamespace} svc/${cfg.stackName}-voltha-api 55555:55555; done" &
+       """)
 
-    sh """
-        set +x
+    sh(label  : 'waitForAdapters: loop until adapter list',
+       script : """
+#        set +x
         adapters=\$(voltctl adapter list -q | wc -l)
         while [[ \$adapters -lt ${cfg.adaptersToWait} ]]; do
           sleep 5
           adapters=\$(voltctl adapter list -q | wc -l)
         done
-    """
+
+        cat << EOM
+** -----------------------------------------------------------------------
+** ${iam}: Available adapters:
+** -----------------------------------------------------------------------
+EOM
+        voltctl adapter list -q
+    """)
 
     // NOTE that we need to wait for LastCommunication to be equal or shorter that 5s
     // as that means the core can talk to the adapters
@@ -193,65 +207,57 @@
 
     println("** ${iam}: Wait for adapter LastCommunication")
     Integer countdown = 60 * 10
-    while (true)
-    {
-	sleep 1
-	def adapters = getAdapters()
-	// Why are we not failing hard on this condition ?
-	// Adapters in an unknown state == testing: roll the dice
-	if (adapters == 'SKIP') { break }
+    while (true) {
+        sleep 1
+        def adapters = getAdapters()
+        // Why are we not failing hard on this condition ?
+        // Adapters in an unknown state == testing: roll the dice
+        if (adapters == 'SKIP') { break }
 
-	def state = getAdaptersState(adapters)
-	if (state == 'VALID') { break }   // IFF
+        def state = getAdaptersState(adapters)
+        if (state == 'VALID') { break }   // IFF
 
-	// ----------------------------------------------------------
-	// Excessive timeout but unsure where startup time boundry is
-	// [TODO] profile for a baseline
-	// [TODO] groovy.transform.TimedInterrupt
-	// ----------------------------------------------------------
-	countdown -= 1
-	if (1 > countdown)
-	{
-	    throw new Exception("ERROR: Timed out waiting on adapter startup")
-	}
+        // ----------------------------------------------------------
+        // Excessive timeout but unsure where startup time boundry is
+        // [TODO] profile for a baseline
+        // [TODO] groovy.transform.TimedInterrupt
+        // ----------------------------------------------------------
+        countdown -= 1
+        if (1 > countdown) {
+            throw new Exception('ERROR: Timed out waiting on adapter startup')
+        }
     }
 
     println("** ${iam}: Tearing down port forwarding")
-    sh("""
-      ps aux \
-          | grep port-forw \
-          | grep -v grep \
-          | awk '{print \$2}' \
-          | xargs --no-run-if-empty kill -9 || true
-    """)
+    // pgrep_port_forward-212
 
-    println("** ${iam}: LEAVE")
+    sh(label  : 'waitForAdapters: Tear down port forwarding',
+       script : """
+if [[ \$(pgrep --count 'port-forw') -gt 0 ]]; then
+    pkill --uid "\$(id -u)" --echo --full 'port-forw'
+fi
+""")
+
+    leave('process')
     return
 }
 
 // -----------------------------------------------------------------------
 // -----------------------------------------------------------------------
-def call(Map config)
+def call(Map config=[:])
 {
     String iam = getIam('main')
-    println("** ${iam}: ENTER")
 
-    if (!config) {
-        config = [:]
+    try {
+        enter('main')
+        process(config)
     }
-
-    try
-    {
-	process(config)
+    catch (Exception err) {  // groovylint-disable-line CatchException
+        println("** ${iam}: EXCEPTION ${err}")
+        throw err
     }
-    catch (Exception err)
-    {
-	println("** ${iam}: EXCEPTION ${err}")
-	throw err
-    }
-    finally
-    {
-	println("** ${iam}: LEAVE")
+    finally {
+        leave('main')
     }
     return
 }