Merge "Checkout working version of cord-tester in siab-test"
diff --git a/jjb/pipeline/voltha-system-test-bbsim.groovy b/jjb/pipeline/voltha-system-test-bbsim.groovy
index ba61757..68f96d7 100644
--- a/jjb/pipeline/voltha-system-test-bbsim.groovy
+++ b/jjb/pipeline/voltha-system-test-bbsim.groovy
@@ -16,6 +16,40 @@
 // uses kind-voltha to deploy voltha-2.X
 // uses bbsim to simulate OLT/ONUs
 
+
+  def logKubernetes(prefix) {
+      sh """
+         set +e
+         cd kind-voltha/
+         cp install-minimal.log $WORKSPACE/${prefix}_instsall-minimal.log
+         kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\t'}{.imageID}{'\\n'}" | sort | uniq -c
+         kubectl get nodes -o wide
+         kubectl get pods -o wide
+         kubectl get pods -n voltha -o wide
+         ## get default pod logs
+         for pod in \$(kubectl get pods --no-headers | awk '{print \$1}');
+         do
+           if [[ \$pod == *"onos"* && \$pod != *"onos-service"* ]]; then
+             kubectl logs \$pod onos> $WORKSPACE/${prefix}_\$pod.log;
+           else
+             kubectl logs \$pod> $WORKSPACE/${prefix}_\$pod.log;
+           fi
+         done
+         ## get voltha pod logs
+         for pod in \$(kubectl get pods --no-headers -n voltha | awk '{print \$1}');
+         do
+           if [[ \$pod == *"-api-"* ]]; then
+             kubectl logs \$pod arouter -n voltha > $WORKSPACE/${prefix}_\$pod.log;
+           elif [[ \$pod == "bbsim-"* ]]; then
+             kubectl logs \$pod -n voltha -p > $WORKSPACE/${prefix}_\$pod.log;
+           else
+             kubectl logs \$pod -n voltha > $WORKSPACE/${prefix}_\$pod.log;
+           fi
+         done
+         """
+  }
+
+
 pipeline {
 
   /* no label, executor is determined by JJB */
@@ -23,7 +57,7 @@
     label "${params.buildNode}"
   }
   options {
-      timeout(time: 40, unit: 'MINUTES')
+      timeout(time: 80, unit: 'MINUTES')
   }
   environment {
     KUBECONFIG="$HOME/.kube/kind-config-voltha-minimal"
@@ -41,7 +75,6 @@
     ROBOT_MISC_ARGS="-d $WORKSPACE/RobotLogs"
   }
   stages {
-
     stage('Download kind-voltha') {
       steps {
         sh """
@@ -65,67 +98,79 @@
         sh '''
            rm -rf $WORKSPACE/RobotLogs; mkdir -p $WORKSPACE/RobotLogs
            git clone https://gerrit.opencord.org/voltha-system-tests
-           make -C $WORKSPACE/voltha-system-tests ${makeTarget} || true
+           make ROBOT_DEBUG_LOG_OPT="-l sanity_log.html -r sanity_result.html -o sanity_result.xml" -C $WORKSPACE/voltha-system-tests ${makeTarget}
            '''
       }
     }
 
+    stage('Log the kubernetes for sanity-test') {
+      steps {
+        logKubernetes('sanity_test')
+      }
+    }
     //Remove this stage once https://jira.opencord.org/browse/VOL-1977 be resolved
-    stage('Deploy Voltha Again') {
+    stage('Deploy Voltha Again for Functional Tests') {
       steps {
         sh """
            pushd kind-voltha/
-           ./voltha down
-           ./voltha up
+           WAIT_ON_DOWN=yes  DEPLOY_K8S=no ./voltha down
+           DEPLOY_K8S=no ./voltha up
            popd
            """
       }
     }
+
     stage('Kubernetes Functional Tests') {
       steps {
         sh '''
-           rm -rf $WORKSPACE/RobotLogs; mkdir -p $WORKSPACE/RobotLogs
-           make -C $WORKSPACE/voltha-system-tests system-scale-test || true
+           make ROBOT_DEBUG_LOG_OPT="-l functional_log.html -r functional_result.html -o functional_output.xml" -C $WORKSPACE/voltha-system-tests system-scale-test
            '''
       }
     }
+
+    stage('Log the kubernetes for functional-test') {
+      steps {
+        logKubernetes('functional')
+      }
+    }
+
+    //Remove this stage once https://jira.opencord.org/browse/VOL-1977 be resolved
+    stage('Deploy Voltha Again for Failure Scenario Tests') {
+      steps {
+        sh """
+           pushd kind-voltha/
+           WAIT_ON_DOWN=yes  DEPLOY_K8S=no ./voltha down
+           DEPLOY_K8S=no ./voltha up
+           popd
+           """
+      }
+    }
+
+    stage('Kubernetes Failure Scenario Tests') {
+      steps {
+        sh '''
+           make ROBOT_DEBUG_LOG_OPT="-l failure_log.html -r failure_result.html -o failure_output.xml"  -C $WORKSPACE/voltha-system-tests failure-test
+           '''
+      }
+    }
+
+    stage('Log the kubernetes for failure scenario test') {
+      steps {
+        logKubernetes('failure')
+      }
+    }
+
   }
 
   post {
+    failure {
+        logKubernetes('last')
+    }
+    aborted {
+        logKubernetes('last')
+    }
     always {
-      sh '''
-         set +e
-         cd kind-voltha/
-         cp install-minimal.log $WORKSPACE/
-         kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\t'}{.imageID}{'\\n'}" | sort | uniq -c
-         kubectl get nodes -o wide
-         kubectl get pods -o wide
-         kubectl get pods -n voltha -o wide
-         ## get default pod logs
-         for pod in \$(kubectl get pods --no-headers | awk '{print \$1}');
-         do
-           if [[ \$pod == *"onos"* && \$pod != *"onos-service"* ]]; then
-             kubectl logs \$pod onos> $WORKSPACE/\$pod.log;
-           else
-             kubectl logs \$pod> $WORKSPACE/\$pod.log;
-           fi
-         done
-         ## get voltha pod logs
-         for pod in \$(kubectl get pods --no-headers -n voltha | awk '{print \$1}');
-         do
-           if [[ \$pod == *"-api-"* ]]; then
-             kubectl logs \$pod arouter -n voltha > $WORKSPACE/\$pod.log;
-           elif [[ \$pod == "bbsim-"* ]]; then
-             kubectl logs \$pod -n voltha -p > $WORKSPACE/\$pod.log;
-           else
-             kubectl logs \$pod -n voltha > $WORKSPACE/\$pod.log;
-           fi
-         done
-         ## clean up node
-         WAIT_ON_DOWN=y ./voltha down
-         cd $WORKSPACE/
-         rm -rf kind-voltha/ voltha-system-tests/ || true
-         '''
+
          step([$class: 'RobotPublisher',
             disableArchiveOutput: false,
             logFileName: 'RobotLogs/*log*.html',
diff --git a/jjb/voltha-e2e.yaml b/jjb/voltha-e2e.yaml
index f7fb5f9..fd85c3b 100644
--- a/jjb/voltha-e2e.yaml
+++ b/jjb/voltha-e2e.yaml
@@ -164,7 +164,7 @@
 - job-template:
     id: 'voltha-patch-test'
     name: 'verify_{project}_sanity-test'
-    extra-helm-flags: '--set defaults.image_tag=$GERRIT_BRANCH'
+    extra-helm-flags: ''
 
     description: |
       <!-- Managed by Jenkins Job Builder -->
@@ -194,7 +194,7 @@
 
       - string:
           name: manifestBranch
-          default: '$GERRIT_BRANCH'
+          default: 'master'
           description: 'Name of the repo branch to use'
 
       - string:
@@ -264,7 +264,7 @@
 
       - string:
           name: manifestBranch
-          default: '$GERRIT_BRANCH'
+          default: master
           description: 'Name of the repo branch to use'
 
       - string:
@@ -406,7 +406,7 @@
 
       - string:
           name: manifestBranch
-          default: '$GERRIT_BRANCH'
+          default: master
           description: 'Name of the repo branch to use'
 
       - string:
@@ -549,7 +549,7 @@
 
       - string:
           name: manifestBranch
-          default: 'master'
+          default: master
           description: 'Name of the repo branch to use (change to $GERRIT_BRANCH if testing a patchset)'
 
       - string: