VOL-5155 - triage failure in periodic-voltha-test-bbsim-2.12

.gitignore
----------
  o Ignore a.out and archives/ created late by the test target.

config.mk
makefiles/targets/sterile.mk
makefiles/targets/tox.mk
----------------------------
  o Update sterile target to remove the archives directory.
  o tox.mk updated to remove generated .tox/ directory.

jjb/pipeline/voltha/master/bbsim-tests.groovy
jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
--------------------------------------------------
  o Fixed a typo in the 'Combine and compress' banner.
  o In compress combined log block:
    - Display logs to compress.
    - '|| true' removed from gzip.  Wrap call with log collection
      and only invoke when logs are available (allow cmds to fail on err).

Change-Id: I141c0bbd1e8c946c652f00f21401bd897d8a4135
diff --git a/jjb/pipeline/voltha/master/bbsim-tests.groovy b/jjb/pipeline/voltha/master/bbsim-tests.groovy
index 4ada7c6..887c01b 100644
--- a/jjb/pipeline/voltha/master/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/master/bbsim-tests.groovy
@@ -311,7 +311,7 @@
 cat <<EOM
 
 ** -----------------------------------------------------------------------
-** Combine an compress voltha startup log(s)
+** Combine and compress voltha startup log(s)
 ** -----------------------------------------------------------------------
 EOM
           pushd "${logsDir}" || { echo "ERROR: pushd $logsDir failed"; exit 1; }
@@ -335,13 +335,13 @@
       if [ ${withMonitoring} = true ] ; then
         JENKINS_NODE_COOKIE="dontKillMe" _TAG="nem-monitoring-prometheus-server" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n default svc/nem-monitoring-prometheus-server 31301:80; done"&
       fi
-      ps aux | grep port-forward
+#      ps aux | grep port-forward
       """
             // ---------------------------------
             // Sanity check port-forward spawned
             // ---------------------------------
             script {
-                enter('port-forward check')
+                enter('Display port-forward procs')
                 // String proc = 'kubectl.*port-forward' // was 'port-forward'
                 String proc = 'port-forward'
                 println("Display spawned ${proc}")
@@ -349,7 +349,7 @@
                    script : """
 pgrep --uid "\$(id -u)" --list-full --full "port-forward" || true
 """)
-                leave('port-forward check')
+                leave('Display port-forward procs')
             }
 
             // setting ONOS log level
@@ -418,11 +418,21 @@
             label : 'Gather robot Framework logs',
             script : """
       echo -e '\n** Gather robot Framework logs: ENTER'
+
       # set +e
       # collect logs collected in the Robot Framework StartLogging keyword
-      cd ${logsDir}
-      gzip *-combined.log
-      rm -f *-combined.log
+      cd "${logsDir}"
+
+      echo "** Available logs:"
+      /bin/ls -l "${logsDir}"
+      echo
+
+      readarray -t logs < <(find . -name '*-combined.log' -print)
+      if [[ ${#logs[@]} -gt 0 ]]; then
+          echo '** Bundle combined log'
+          gzip "${logs[@]}"
+          rm -f "${logs[@]}"
+      fi
 
       echo -e '** Gather robot Framework logs: LEAVE\n'
     """)
diff --git a/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy b/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
index 5b765c1..974ad88 100644
--- a/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
@@ -311,7 +311,7 @@
 cat <<EOM
 
 ** -----------------------------------------------------------------------
-** Combine an compress voltha startup log(s)
+** Combine and compress voltha startup log(s)
 ** -----------------------------------------------------------------------
 EOM
           pushd "${logsDir}" || { echo "ERROR: pushd $logsDir failed"; exit 1; }
@@ -335,13 +335,13 @@
       if [ ${withMonitoring} = true ] ; then
         JENKINS_NODE_COOKIE="dontKillMe" _TAG="nem-monitoring-prometheus-server" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n default svc/nem-monitoring-prometheus-server 31301:80; done"&
       fi
-      ps aux | grep port-forward
+#      ps aux | grep port-forward
       """
             // ---------------------------------
             // Sanity check port-forward spawned
             // ---------------------------------
             script {
-                enter('port-forward check')
+                enter('Display port-forward procs')
                 // String proc = 'kubectl.*port-forward' // was 'port-forward'
                 String proc = 'port-forward'
                 println("Display spawned ${proc}")
@@ -349,7 +349,7 @@
                    script : """
 pgrep --uid "\$(id -u)" --list-full --full "port-forward" || true
 """)
-                leave('port-forward check')
+                leave('Display port-forward procs')
             }
 
             // setting ONOS log level
@@ -418,11 +418,21 @@
             label : 'Gather robot Framework logs',
             script : """
       echo -e '\n** Gather robot Framework logs: ENTER'
+
       # set +e
       # collect logs collected in the Robot Framework StartLogging keyword
-      cd ${logsDir}
-      gzip *-combined.log
-      rm -f *-combined.log
+      cd "${logsDir}"
+
+      echo "** Available logs:"
+      /bin/ls -l "${logsDir}"
+      echo
+
+      readarray -t logs < <(find . -name '*-combined.log' -print)
+      if [[ ${#logs[@]} -gt 0 ]]; then
+          echo '** Bundle combined log'
+          gzip "${logs[@]}"
+          rm -f "${logs[@]}"
+      fi
 
       echo -e '** Gather robot Framework logs: LEAVE\n'
     """)