Merge "Separating logs by test in VOLTHA Sanity"
diff --git a/jjb/pipeline/voltha-bbsim-tests.groovy b/jjb/pipeline/voltha-bbsim-tests.groovy
index b990a45..3e72f32 100644
--- a/jjb/pipeline/voltha-bbsim-tests.groovy
+++ b/jjb/pipeline/voltha-bbsim-tests.groovy
@@ -272,7 +272,10 @@
            fi
 
            printenv
-           kail -n voltha -n default > $WORKSPACE/onos-voltha-combined.log &
+
+           # start logging
+           mkdir -p $WORKSPACE/att
+           _TAG=kail-att kail -n voltha -n default > $WORKSPACE/att/onos-voltha-combined.log &
 
            cd $WORKSPACE/kind-voltha/
            ./voltha up
@@ -297,6 +300,20 @@
            fi
 
            make -C $WORKSPACE/voltha-system-tests \$TARGET || true
+
+           # stop logging
+           P_IDS="$(ps e -ww -A | grep "_TAG=kail-att" | grep -v grep | awk '{print $1}')"
+           if [ -n "$P_IDS" ]; then
+             echo $P_IDS
+             for P_ID in $P_IDS; do
+               kill -9 $P_ID
+             done
+           fi
+
+           # get pods information
+           kubectl get pods -o wide > $WORKSPACE/att/pods.txt || true
+           kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\n'}" | sort | uniq | tee $WORKSPACE/att/pod-images.txt || true
+           kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.imageID}{'\\n'}" | sort | uniq | tee $WORKSPACE/att/pod-imagesId.txt || true
            '''
       }
     }
@@ -319,6 +336,10 @@
            export CONFIG_SADIS="external"
            export BBSIM_CFG="configs/bbsim-sadis-dt.yaml"
 
+           # start logging
+           mkdir -p $WORKSPACE/dt
+           _TAG=kail-dt kail -n voltha -n default > $WORKSPACE/dt/onos-voltha-combined.log &
+
            DEPLOY_K8S=n ./voltha up
 
            mkdir -p $ROBOT_LOGS_DIR
@@ -337,6 +358,20 @@
            fi
 
            make -C $WORKSPACE/voltha-system-tests \$TARGET || true
+
+           # stop logging
+           P_IDS="$(ps e -ww -A | grep "_TAG=kail-dt" | grep -v grep | awk '{print $1}')"
+           if [ -n "$P_IDS" ]; then
+             echo $P_IDS
+             for P_ID in $P_IDS; do
+               kill -9 $P_ID
+             done
+           fi
+
+           # get pods information
+           kubectl get pods -o wide > $WORKSPACE/dt/pods.txt || true
+           kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\n'}" | sort | uniq | tee $WORKSPACE/dt/pod-images.txt || true
+           kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.imageID}{'\\n'}" | sort | uniq | tee $WORKSPACE/dt/pod-imagesId.txt || true
            '''
       }
     }
@@ -359,6 +394,10 @@
            export CONFIG_SADIS="external"
            export BBSIM_CFG="configs/bbsim-sadis-tt.yaml"
 
+           # start logging
+           mkdir -p $WORKSPACE/tt
+           _TAG=kail-tt kail -n voltha -n default > $WORKSPACE/tt/onos-voltha-combined.log &
+
            DEPLOY_K8S=n ./voltha up
 
            mkdir -p $ROBOT_LOGS_DIR
@@ -377,6 +416,20 @@
            fi
 
            make -C $WORKSPACE/voltha-system-tests \$TARGET || true
+
+           # stop logging
+           P_IDS="$(ps e -ww -A | grep "_TAG=kail-att" | grep -v grep | awk '{print $1}')"
+           if [ -n "$P_IDS" ]; then
+             echo $P_IDS
+             for P_ID in $P_IDS; do
+               kill -9 $P_ID
+             done
+           fi
+
+           # get pods information
+           kubectl get pods -o wide > $WORKSPACE/tt/pods.txt || true
+           kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\n'}" | sort | uniq | tee $WORKSPACE/tt/pod-images.txt || true
+           kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.imageID}{'\\n'}" | sort | uniq | tee $WORKSPACE/tt/pod-imagesId.txt || true
            '''
       }
     }
@@ -387,14 +440,8 @@
       sh '''
          set +e
          cp $WORKSPACE/kind-voltha/install-$NAME.log $WORKSPACE/
-         kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\n'}" | sort | uniq
-         kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.imageID}{'\\n'}" | sort | uniq
-         kubectl get nodes -o wide
-         kubectl get pods -o wide
-         kubectl get pods -n voltha -o wide
 
          sync
-         pkill kail || true
          md5sum $WORKSPACE/kind-voltha/bin/voltctl
 
          ## Pull out errors from log files
@@ -417,7 +464,9 @@
          extract_errors_python adapter-open-onu >> $WORKSPACE/error-report.log
          extract_errors_python voltha-ofagent >> $WORKSPACE/error-report.log
 
-         gzip $WORKSPACE/onos-voltha-combined.log
+         gzip $WORKSPACE/att/onos-voltha-combined.log
+         gzip $WORKSPACE/dt/onos-voltha-combined.log
+         gzip $WORKSPACE/tt/onos-voltha-combined.log
          '''
          step([$class: 'RobotPublisher',
             disableArchiveOutput: false,
@@ -428,7 +477,7 @@
             passThreshold: 100,
             reportFileName: 'RobotLogs/*/report*.html',
             unstableThreshold: 0]);
-         archiveArtifacts artifacts: '*.log,*.gz'
+         archiveArtifacts artifacts: '*.log,**/*.log,**/*.gz,*.gz'
     }
   }
 }