VOL-5155 - triage failure in periodic-voltha-test-bbsim-2.12

vars/pgrep_proc.groovy
vars/pkill_proc.groovy
----------------------
  o Pass arg --full to expand command matching to arguments.
  o Include --uid to limit process consideration.
  o Accept optional (Map args) parameter, will need more later.

jjb/pipeline/voltha/master/bbsim-tests.groovy
jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
--------------------------------------------------
  o pgrep/pkill cleanup -- kubectl is the command to target for cleanup.
  o port-forward is a command parameter and was used to filter tasks.
  o pattern to display/terminate is 'kubectl.*port-forward'

Change-Id: I34b473a4e1c7fcd488910be62f458319d1499254
diff --git a/jjb/pipeline/voltha/master/bbsim-tests.groovy b/jjb/pipeline/voltha/master/bbsim-tests.groovy
index 46cb252..af5c554 100644
--- a/jjb/pipeline/voltha/master/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/master/bbsim-tests.groovy
@@ -130,10 +130,12 @@
                 script {
                     enter('Cleanup')
                     // remove orphaned port-forward from different namespaces
-                    String proc = 'port-forw'
+                    String proc = 'kubectl .*port-forward' // was 'port-forw'
                     pgrep_proc(proc)
                     pkill_proc(proc)
-                    pgrep_proc(proc) // proc count == 0
+
+                    // todo: fatal unless (proc count==0)
+                    pgrep_proc(proc)
                     enter('Cleanup')
                 } // script
             } // timeout
@@ -300,6 +302,7 @@
             // ---------------------------------
             script {
                 enter('port-forward check')
+                // String proc = 'kubectl.*port-forward' // was 'port-forward'
                 String proc = 'port-forward'
                 println("Display spawned ${proc}")
                 pgrep_proc(proc)
diff --git a/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy b/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
index d97877d..a6ce2f2 100644
--- a/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
@@ -130,10 +130,12 @@
                 script {
                     enter('Cleanup')
                     // remove orphaned port-forward from different namespaces
-                    String proc = 'port-forw'
+                    String proc = 'kubectl .*port-forward' // was 'port-forw'
                     pgrep_proc(proc)
                     pkill_proc(proc)
-                    pgrep_proc(proc) // proc count == 0
+
+                    // todo: fatal unless (proc count==0)
+                    pgrep_proc(proc)
                     enter('Cleanup')
                 } // script
             } // timeout
@@ -300,6 +302,7 @@
             // ---------------------------------
             script {
                 enter('port-forward check')
+                // String proc = 'kubectl.*port-forward' // was 'port-forward'
                 String proc = 'port-forward'
                 println("Display spawned ${proc}")
                 pgrep_proc(proc)
@@ -330,7 +333,9 @@
     // -----------------------------------------------------------------------
     stage("Run test ${testTarget} on workflow ${workflow}")
     {
-        sh """
+        sh(
+            label : 'Monitor using mem_consumption.py',
+            script : """
         echo -e "\n** Monitor using mem_consumption.py ?"
 
     if [ ${withMonitoring} = true ] ; then
@@ -352,7 +357,7 @@
     fi
 
     echo -e '** Monitor memory consumption: LEAVE\n'
-    """
+    """)
 
         sh """
         echo -e "\n** make testTarget=[${testTarget}]"
@@ -366,7 +371,9 @@
 
         getPodsInfo("${logsDir}")
 
-        sh """
+        sh(
+            label : 'Gather robot Framework logs',
+            script : """
       echo -e '\n** Gather robot Framework logs: ENTER'
       # set +e
       # collect logs collected in the Robot Framework StartLogging keyword
@@ -375,11 +382,13 @@
       rm -f *-combined.log
 
       echo -e '** Gather robot Framework logs: LEAVE\n'
-    """
+    """)
 
     // -----------------------------------------------------------------------
     // -----------------------------------------------------------------------
-    sh """
+        sh(
+            label  : 'Monitor pod-mem-consumption',
+            script : """
     echo -e '** Monitor pod-mem-consumption: ENTER'
     if [ ${withMonitoring} = true ] ; then
       cat <<EOM
@@ -398,7 +407,7 @@
       python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
     fi
     echo -e '** Monitor pod-mem-consumption: LEAVE\n'
-    """
+    """)
     } // stage
 
     return
@@ -595,7 +604,7 @@
                     // Announce ourselves for log usability
                     enter('Parse and execute tests')
 
-                    def tests = readYaml text: testTargets
+                    def tests = readYaml text: testTargets // typeof == Map (?)
                     println("** [DEBUG]: tests=$tests")
 
                     // Display expected tests for times when output goes dark
diff --git a/vars/pgrep_proc.groovy b/vars/pgrep_proc.groovy
index 1ad5dfc..9559ce6 100644
--- a/vars/pgrep_proc.groovy
+++ b/vars/pgrep_proc.groovy
@@ -28,14 +28,21 @@
 
 // -----------------------------------------------------------------------
 // -----------------------------------------------------------------------
-Boolean process(String proc) {
-    String iam  = getIam('process')
+Boolean process(String proc, Map args) {
     Boolean ans = true
+    String  iam = getIam('process')
 
-    println("** ${iam}: ENTER")
+    if (args.containsKey('debug')) {
+        println("** $iam [DEBUG]: proc=[$proc], args=[$args]")
+    }
 
-    String cmdFull = "pgrep --list-full '${proc}'"
-    String cmd = """if [[ \$(pgrep --count "${proc}") -gt 0 ]]; then ${cmdFull}; fi"""
+    String cmd = [
+        'pgrep',
+        '--uid', '$(id -u)', // no stray signals
+        '--list-full',
+        '--full',  // hmmm: conditional use (?)
+        "'${proc}",
+    ]
 
     print("""
 ** -----------------------------------------------------------------------
@@ -46,8 +53,6 @@
         label  : 'pgrep_proc', // jenkins usability: label log entry 'step'
         script : "${cmd}",
     )
-
-    println("** ${iam}: LEAVE")
     return(ans)
 }
 
@@ -56,22 +61,38 @@
 //    o Paramter branch is passed but not yet used.
 //    o Installer should be release friendly and checkout a frozen version
 // -----------------------------------------------------------------------
-def call(String proc) {
+// groovylint-disable-next-line None, UnusedMethodParameter
+Boolean call\
+(
+    String  proc,           // name of process or arguments to terminate
+    Map     args=[:],
+    Boolean filler=True     // Groovy, why special case list comma handling (?)
+) {
     String iam = getIam('main')
+    Boolean ans = True
 
     println("** ${iam}: ENTER")
 
     try {
-        process(proc)
+        process(proc, args)
     }
-    catch (Exception err) {
+    catch (Exception err) {  // groovylint-disable-line CatchException
+        ans = False
         println("** ${iam}: EXCEPTION ${err}")
         throw err
     }
     finally {
         println("** ${iam}: LEAVE")
     }
-    return
+
+    return(ans)
 }
 
+// -----------------------------------------------------------------------
+// [TODO] - Combine pkill_proc and pgrep_proc
+//    - Usage: do_proc(pkill=true, pgrep=true, args='proc-forward', cmd='kubectl'
+//      o When kill == grep == true: display procs, terminate, recheck: fatal if procs detected
+//      o cmd && args (or command containing args) (or list of patterns passed)
+//        - pass arg --full to match entire command line.
+// -----------------------------------------------------------------------
 // [EOF]
diff --git a/vars/pkill_proc.groovy b/vars/pkill_proc.groovy
index 753a1f1..0166df9 100644
--- a/vars/pkill_proc.groovy
+++ b/vars/pkill_proc.groovy
@@ -28,13 +28,28 @@
 
 // -----------------------------------------------------------------------
 // -----------------------------------------------------------------------
-Boolean process(String proc) {
-    String iam  = getIam('process')
+Boolean process(String proc, Map args) {
     Boolean ans = true
+    String  iam = getIam('process')
 
-    println("** ${iam}: ENTER")
+    if (args.containsKey('debug')) {
+        println("** $iam [DEBUG]: proc=[$proc], args=[$args]")
+    }
 
-    String cmdKill = "pkill --echo '${proc}'"
+    String cmdKill = [
+        'pkill',
+        '--uid', '$(id -u)', // no stray signals
+        '--echo',
+        '--full',  // hmmm: conditional use (?)
+        "'${proc}",
+    ].join(' ')
+
+    /*
+    String cmdKill = (args['command_only')
+        ? "pkill --echo '${proc}'"
+        : "pkill --echo --full '${proc}'"
+     */
+
     String cmd = """if [[ \$(pgrep --count "${proc}") -gt 0 ]]; then ${cmdKill}; fi"""
 
     print("""
@@ -48,7 +63,6 @@
         script : "${cmd}",
     )
 
-    println("** ${iam}: LEAVE")
     return(ans)
 }
 
@@ -57,22 +71,44 @@
 //    o Paramter branch is passed but not yet used.
 //    o Installer should be release friendly and checkout a frozen version
 // -----------------------------------------------------------------------
-def call(String proc) {
+// groovylint-disable-next-line None, UnusedMethodParameter
+void call\
+(
+    String  proc,           // name of process or arguments to terminate
+    Map     args=[:],
+                            // Groovy, why special case list comma handling (?)
+    Boolean filler=True     // groovylint-disable-line UnusedMethodParameter
+) {
+    
     String iam = getIam('main')
+    Boolean ans = True
 
     println("** ${iam}: ENTER")
 
     try {
-        process(proc)
+        // Limit process matching by default
+        if (! mymap.containsKey('command_only')) {
+            mymap['command_only'] = True
+        }
+        process(proc, args)
     }
-    catch (Exception err) {
+    catch (Exception err) { // groovylint-disable-line CatchException
+        ans = False
         println("** ${iam}: EXCEPTION ${err}")
         throw err
     }
     finally {
         println("** ${iam}: LEAVE")
     }
-    return
+
+    return(ans)
 }
 
+// -----------------------------------------------------------------------
+// [TODO] - Combine pkill_proc and pgrep_proc
+//    - Usage: do_proc(pkill=true, pgrep=true, args='proc-forward', cmd='kubectl'
+//      o When kill == grep == true: display procs, terminate, recheck: fatal if procs detected
+//      o cmd && args (or command containing args) (or list of patterns passed)
+//        - pass arg --full to match entire command line.
+// -----------------------------------------------------------------------
 // [EOF]