[VOL-5177] - verify_voltha-openolt-adapter_sanity-test failure

jjb/pipeline/voltha/bbsim-tests.groovy
--------------------------------------
  o https://jenkins.opencord.org/job/verify_voltha-openolt-adapter_sanity-test-voltha-2.12/32/

[DEBUG]
  o More added, last logged output source was 'pkill kail-startup'.
  o killKailStartup:
    - Refactor dup 'pkill kail-startup' logic into a named function.
    - Remove sanity check pgrep --list display, termination verified.
  o Added function findPublishedLogs().  Display logdir contents early
    stage('RobotPublisher') may fail so show what is available.

[COSMETIC]
  o Remove pipelineVer(), function no longer needed with SCM checkouts.
  o Refactor post action {cleanupPortForward() x 3}  into a single post
    actions task 'cleanup' that will run after all others:
    {aborted,failure,always}.

Change-Id: I1535057f696d9d29ba2a79b5bcad825d9b63d155
diff --git a/jjb/pipeline/voltha/bbsim-tests.groovy b/jjb/pipeline/voltha/bbsim-tests.groovy
index 1a7953c..128a20d 100644
--- a/jjb/pipeline/voltha/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/bbsim-tests.groovy
@@ -15,7 +15,7 @@
 // voltha-2.x e2e tests for openonu-go
 // uses bbsim to simulate OLT/ONUs
 
-// [TODO] Update syntax below to the latest supported
+// [TODO] Update library() to the latest DSL syntax supported by jenkins
 library identifier: 'cord-jenkins-libraries@master',
     retriever: modernSCM([
     $class: 'GitSCMSource',
@@ -47,16 +47,6 @@
 }
 
 // -----------------------------------------------------------------------
-// Intent: Difficult at times to determine when pipeline jobs have
-//   regenerated.  Hardcode a version string that can be assigned
-//   per-script to be sure latest repository changes are being used.
-// -----------------------------------------------------------------------
-String pipelineVer() {
-    String version = 'ff337d86399e107cd417793454c4bbd398855d31'
-    return(version)
-}
-
-// -----------------------------------------------------------------------
 // Intent: Due to lack of a reliable stack trace, construct a literal.
 //         Jenkins will re-write the call stack for serialization.S
 // -----------------------------------------------------------------------
@@ -64,7 +54,6 @@
 // -----------------------------------------------------------------------
 String getIam(String func) {
     String branchName = branchName()
-    String version    = pipelineVer()
     String src = [
         'ci-management',
         'jjb',
@@ -74,7 +63,7 @@
         'bbsim-tests.groovy'
     ].join('/')
 
-    String name = [src, version, func].join('::')
+    String name = [src, func].join('::')
     return(name)
 }
 
@@ -127,6 +116,49 @@
     return
 }
 
+// find . \( -name 'log*.html' -o -name 'output*.xml' -o -name 'report*.html' \) -p
+// -----------------------------------------------------------------------
+// Intent: Display contents of the logs directory
+// -----------------------------------------------------------------------
+// [TODO]
+//   o where-4-art-thou logs directory ?
+//   o Replace find {logfile} command with /bin/ls {logdir} when found.
+//     Individual logs may be missing due to failure, show what is available.
+// -----------------------------------------------------------------------
+void findPublishedLogs() {
+    String iam = 'findPublishedLogs'
+
+    enter(iam)
+    sh(label  : iam,
+       script : """
+find . -name 'output.xml' -print
+""")
+    leave(iam)
+    return
+}
+
+// -----------------------------------------------------------------------
+// Intent: Terminate kail-startup process launched earlier
+// -----------------------------------------------------------------------
+// :param caller: Name of parent calling function (debug context)
+// :type caller: String, optional
+// :returns: none
+// :rtype:   void
+// -----------------------------------------------------------------------
+void killKailStartup(String caller='') {
+    String iam = "killKailStartup (caller=$caller)"
+
+    enter(iam)
+    sh(label  : 'Terminate kail-startup',
+       script : """
+if [[ \$(pgrep --count '_TAG=kail-startup') -gt 0 ]]; then
+    pkill --uid \$(id -u) --echo --list-full --full '_TAG=kail-startup'
+fi
+""")
+    leave(iam)
+    return
+}
+
 // -----------------------------------------------------------------------
 // Intent: Iterate over a list of test suites and invoke.
 // -----------------------------------------------------------------------
@@ -274,20 +306,10 @@
                     enter('bbsim-tests::pgrep_port_forward::0')
                     pgrep_port_forward('port-forw')
                     leave('bbsim-tests::pgrep_port_forward::0')
+
+                    killKailStartup('Deploy Voltha')
                 }
 
-                sh(label  : 'Terminate kail-startup',
-                   script : """
-if [[ \$(pgrep --count '_TAG=kail-startup') -gt 0 ]]; then
-    pkill --uid \$(id -u) --echo --list-full --full '_TAG=kail-startup'
-fi
-""")
-
-                sh(label  : 'Lingering kail-startup check',
-                   script : """
-pgrep --uid \$(id -u) --list-full --full 'kail-startup' || true
-""")
-
                 // -----------------------------------------------------------------------
                 // Bundle onos-voltha / kail logs
                 // -----------------------------------------------------------------------
@@ -474,20 +496,8 @@
 
     archiveArtifacts artifacts: '**/*.log,**/*.gz,**/*.txt,**/*.html,**/voltha-pods-mem-consumption-att/*,**/voltha-pods-mem-consumption-dt/*,**/voltha-pods-mem-consumption-tt/*'
 
-    script {
-        enter('pkill _TAG=kail-startup')
-        sh(label  : 'pgrep_proc - kill-pre',
-           script : """
-pgrep --uid "\$(id -u)" --list-full --full 'kail-startup' || true
-""")
-        sh(label  : 'pkill_proc - kail',
-           script : """
-if [[ \$(pgrep --count '_TAG=kail') -gt 0 ]]; then
-    pkill --uid "\$(id -u)" --echo --full 'kail'
-fi
-""")
-        leave('pkill _TAG=kail-startup')
-    }
+    script { killKailStartup('collectArtifacts') }
+    script { findPublishedLogs() }
 
     enter('RobotPublisher')
     step([$class: 'RobotPublisher',
@@ -705,17 +715,17 @@
     } // stages
 
     post
-    {
+    { // https://www.jenkins.io/doc/book/pipeline/syntax/#post
         aborted {
             collectArtifacts('aborted')
-            script { cleanupPortForward() }
         }
         failure {
             collectArtifacts('failed')
-            script { cleanupPortForward() }
         }
         always {
             collectArtifacts('always')
+        }
+        cleanup {
             script { cleanupPortForward() }
         }
     }