VOL-5155 - triage failure in periodic-voltha-test-bbsim-2.12

.gitignore
----------
  o Ignore a.out and archives/ created late by the test target.

config.mk
makefiles/targets/sterile.mk
makefiles/targets/tox.mk
----------------------------
  o Update sterile target to remove the archives directory.
  o tox.mk updated to remove generated .tox/ directory.

jjb/pipeline/voltha/master/bbsim-tests.groovy
jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
--------------------------------------------------
  o Fixed a typo in the 'Combine and compress' banner.
  o In compress combined log block:
    - Display logs to compress.
    - '|| true' removed from gzip.  Wrap call with log collection
      and only invoke when logs are available (allow cmds to fail on err).

Change-Id: I141c0bbd1e8c946c652f00f21401bd897d8a4135
diff --git a/.gitignore b/.gitignore
index 0dc27e0..3ace541 100644
--- a/.gitignore
+++ b/.gitignore
@@ -42,4 +42,8 @@
 # Generated pipeline jobs
 build/
 
+# make test residue
+a.out
+archives/
+
 # [EOF]
diff --git a/a.out b/a.out
deleted file mode 100644
index e69de29..0000000
--- a/a.out
+++ /dev/null
diff --git a/config.mk b/config.mk
index 3cec7fd..849bd73 100644
--- a/config.mk
+++ b/config.mk
@@ -125,6 +125,16 @@
 
 onf-excl-dirs := $(sort $(strip $(onf-excl-dirs)))
 
+# --------------------------------------------------------------------
+# Repository specific values
+# --------------------------------------------------------------------
+sterile-dirs += archives
+
+# submodules
+# sterile-dirs += global-jjb
+# sterile-dirs += lf-ansible
+# sterile-dirs += packer
+
 # [TODO]#
 #  --------------------------------------------------------------------
 #   o two distinct makefiles/ directories are needed, one for onf-make
diff --git a/jjb/pipeline/voltha/master/bbsim-tests.groovy b/jjb/pipeline/voltha/master/bbsim-tests.groovy
index 4ada7c6..887c01b 100644
--- a/jjb/pipeline/voltha/master/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/master/bbsim-tests.groovy
@@ -311,7 +311,7 @@
 cat <<EOM
 
 ** -----------------------------------------------------------------------
-** Combine an compress voltha startup log(s)
+** Combine and compress voltha startup log(s)
 ** -----------------------------------------------------------------------
 EOM
           pushd "${logsDir}" || { echo "ERROR: pushd $logsDir failed"; exit 1; }
@@ -335,13 +335,13 @@
       if [ ${withMonitoring} = true ] ; then
         JENKINS_NODE_COOKIE="dontKillMe" _TAG="nem-monitoring-prometheus-server" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n default svc/nem-monitoring-prometheus-server 31301:80; done"&
       fi
-      ps aux | grep port-forward
+#      ps aux | grep port-forward
       """
             // ---------------------------------
             // Sanity check port-forward spawned
             // ---------------------------------
             script {
-                enter('port-forward check')
+                enter('Display port-forward procs')
                 // String proc = 'kubectl.*port-forward' // was 'port-forward'
                 String proc = 'port-forward'
                 println("Display spawned ${proc}")
@@ -349,7 +349,7 @@
                    script : """
 pgrep --uid "\$(id -u)" --list-full --full "port-forward" || true
 """)
-                leave('port-forward check')
+                leave('Display port-forward procs')
             }
 
             // setting ONOS log level
@@ -418,11 +418,21 @@
             label : 'Gather robot Framework logs',
             script : """
       echo -e '\n** Gather robot Framework logs: ENTER'
+
       # set +e
       # collect logs collected in the Robot Framework StartLogging keyword
-      cd ${logsDir}
-      gzip *-combined.log
-      rm -f *-combined.log
+      cd "${logsDir}"
+
+      echo "** Available logs:"
+      /bin/ls -l "${logsDir}"
+      echo
+
+      readarray -t logs < <(find . -name '*-combined.log' -print)
+      if [[ ${#logs[@]} -gt 0 ]]; then
+          echo '** Bundle combined log'
+          gzip "${logs[@]}"
+          rm -f "${logs[@]}"
+      fi
 
       echo -e '** Gather robot Framework logs: LEAVE\n'
     """)
diff --git a/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy b/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
index 5b765c1..974ad88 100644
--- a/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
@@ -311,7 +311,7 @@
 cat <<EOM
 
 ** -----------------------------------------------------------------------
-** Combine an compress voltha startup log(s)
+** Combine and compress voltha startup log(s)
 ** -----------------------------------------------------------------------
 EOM
           pushd "${logsDir}" || { echo "ERROR: pushd $logsDir failed"; exit 1; }
@@ -335,13 +335,13 @@
       if [ ${withMonitoring} = true ] ; then
         JENKINS_NODE_COOKIE="dontKillMe" _TAG="nem-monitoring-prometheus-server" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n default svc/nem-monitoring-prometheus-server 31301:80; done"&
       fi
-      ps aux | grep port-forward
+#      ps aux | grep port-forward
       """
             // ---------------------------------
             // Sanity check port-forward spawned
             // ---------------------------------
             script {
-                enter('port-forward check')
+                enter('Display port-forward procs')
                 // String proc = 'kubectl.*port-forward' // was 'port-forward'
                 String proc = 'port-forward'
                 println("Display spawned ${proc}")
@@ -349,7 +349,7 @@
                    script : """
 pgrep --uid "\$(id -u)" --list-full --full "port-forward" || true
 """)
-                leave('port-forward check')
+                leave('Display port-forward procs')
             }
 
             // setting ONOS log level
@@ -418,11 +418,21 @@
             label : 'Gather robot Framework logs',
             script : """
       echo -e '\n** Gather robot Framework logs: ENTER'
+
       # set +e
       # collect logs collected in the Robot Framework StartLogging keyword
-      cd ${logsDir}
-      gzip *-combined.log
-      rm -f *-combined.log
+      cd "${logsDir}"
+
+      echo "** Available logs:"
+      /bin/ls -l "${logsDir}"
+      echo
+
+      readarray -t logs < <(find . -name '*-combined.log' -print)
+      if [[ ${#logs[@]} -gt 0 ]]; then
+          echo '** Bundle combined log'
+          gzip "${logs[@]}"
+          rm -f "${logs[@]}"
+      fi
 
       echo -e '** Gather robot Framework logs: LEAVE\n'
     """)
diff --git a/makefiles/targets/sterile.mk b/makefiles/targets/sterile.mk
index c661f53..f5922ee 100644
--- a/makefiles/targets/sterile.mk
+++ b/makefiles/targets/sterile.mk
@@ -31,6 +31,7 @@
 ## -----------------------------------------------------------------------
 .PHONY: sterile
 sterile :: clean
+	$(if $(sterile-dirs),$(RM) -r $(sterile-dirs))
 
 ## -----------------------------------------------------------------------
 ## -----------------------------------------------------------------------
diff --git a/makefiles/targets/tox.mk b/makefiles/targets/tox.mk
index b3a638b..9d50962 100644
--- a/makefiles/targets/tox.mk
+++ b/makefiles/targets/tox.mk
@@ -28,6 +28,9 @@
 lint-tox: lint-jjb
 	tox -e py310
 
+sterile ::
+	$(RM) -r .tox#       # $(TOP)/.tox
+
 ## -----------------------------------------------------------------------
 ## -----------------------------------------------------------------------
 help-verbose += help-tox