VOL-5155 - triage failure in periodic-voltha-test-bbsim-2.12

jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
--------------------------------------------------
  o Add more debugging to figure out where the exception
    around port-forwarding cleanup happened.

Change-Id: Iae873416cc5a1c4b3fdfac0d0de7ff26796c9924
diff --git a/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy b/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
index 32065dd..25aaee9 100644
--- a/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
@@ -71,15 +71,27 @@
 // Intent:
 // -----------------------------------------------------------------------
 void pgrep_proc(String proc) {
+
+    println("** RAW PROCESS OUTPUT:")
+    def stream = sh(returnStdout: true, scirpt: 'ps faaux')
+    println(stream)
+    
     String cmd = [
 	'pgrep',
 	// '--older', 5, // switch not supported, nodes using older version
 	'--list-full',
-	proc,
+	"\"${proc}\"",
+	';',
+	'echo', 'DONE',
+	';',
+	'true',
     ].join(' ')
 
     println("** Running: ${cmd}")
-    sh("set +euo pipefail && ${cmd}")
+    sh(
+        returnStdout: true,
+	"set +euo pipefail && ${cmd}"
+    )
     return
 }
 
@@ -93,7 +105,7 @@
 	// good old kill (ps | grep -v -e grep -e '$$-me') }
 	// '--older', 5,
 	'--echo',
-	proc,
+	"\"${proc}\"",
     ].join(' ')
 
     println("** Running: ${cmd}")
@@ -619,13 +631,21 @@
 
                 def tests = readYaml text: testTargets
 
-		println("** $iam: Testing index: tests-to-run")
+		String buffer = []    
 		tests.eachWithIndex { test, idx ->
 		    String  target = test['target']
-		    println("** $idx: $target")
+		    buffer += sprintf("      test[%02d]: %s\n", idx, target)
 	        }
-		println("** NOTE: For odd failures compare tests-to-run with teste output")
 
+		println("** Testing index: tests-to-run")
+		println(buffer)
+		println("""
+** -----------------------------------------------------------------------
+** NOTE: For odd/silent job failures verify a few details
+**   - All tests mentioned in the index have been processed.
+**   - Test suites display ENTER/LEAVE mesasge pairs.
+** -----------------------------------------------------------------------
+""")
 		tests.eachWithIndex { test, idx ->
                     println "** readYaml test suite[$idx]) test=[${test}]"