Merge "[VOL-4072] Fixed Soak job DT Dataplane test pipeline"
diff --git a/jjb/pipeline/voltha/voltha-2.7/voltha-DMI-bbsim-tests.groovy b/jjb/pipeline/voltha/voltha-2.7/voltha-DMI-bbsim-tests.groovy
index cbf04b8..e7dba07 100755
--- a/jjb/pipeline/voltha/voltha-2.7/voltha-DMI-bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/voltha-2.7/voltha-DMI-bbsim-tests.groovy
@@ -23,7 +23,7 @@
     label "${params.buildNode}"
   }
   options {
-      timeout(time: 190, unit: 'MINUTES')
+      timeout(time: 30, unit: 'MINUTES')
   }
   environment {
     KUBECONFIG="$HOME/.kube/kind-config-voltha-minimal"
@@ -63,10 +63,18 @@
     }
     stage('Cleanup') {
       steps {
-        sh """
-        cd $WORKSPACE/kind-voltha/
-        WAIT_ON_DOWN=y DEPLOY_K8S=n ./voltha down || ./voltha down
-        """
+        timeout(time: 10, unit: 'MINUTES') {
+          sh """
+          if [ "${branch}" != "master" ]; then
+            echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
+            source "$WORKSPACE/kind-voltha/releases/${branch}"
+          else
+            echo "on master, using default settings for kind-voltha"
+          fi
+          cd $WORKSPACE/kind-voltha/
+          WAIT_ON_DOWN=y DEPLOY_K8S=n ./voltha down || ./voltha down
+          """
+        }
       }
     }
     stage('Clone voltha-system-tests') {
@@ -89,20 +97,22 @@
 
     stage('Deploy Voltha') {
       steps {
-        sh """
-           export EXTRA_HELM_FLAGS=""
-           if [ "${branch}" != "master" ]; then
-             echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
-             source "$WORKSPACE/kind-voltha/releases/${branch}"
-           else
-             echo "on master, using default settings for kind-voltha"
-           fi
+        timeout(time: 10, unit: 'MINUTES') {
+          sh """
+             export EXTRA_HELM_FLAGS=""
+             if [ "${branch}" != "master" ]; then
+               echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
+               source "$WORKSPACE/kind-voltha/releases/${branch}"
+             else
+               echo "on master, using default settings for kind-voltha"
+             fi
 
-           EXTRA_HELM_FLAGS+="--set log_agent.enabled=False ${params.extraHelmFlags}"
+             EXTRA_HELM_FLAGS+="--set log_agent.enabled=False ${params.extraHelmFlags}"
 
-           cd $WORKSPACE/kind-voltha/
-           ./voltha up
-           """
+             cd $WORKSPACE/kind-voltha/
+             ./voltha up
+             """
+         }
       }
     }
 
@@ -111,6 +121,7 @@
         ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/DMITests"
       }
       steps {
+        timeout(time: 10, unit: 'MINUTES') {
         sh '''
            set +e
            mkdir -p $ROBOT_LOGS_DIR
@@ -121,6 +132,7 @@
            export ROBOT_MISC_ARGS="-d $ROBOT_LOGS_DIR"
            make -C $WORKSPACE/voltha-system-tests ${makeTarget} || true
            '''
+         }
       }
     }
   }
@@ -168,6 +180,12 @@
          gzip *-combined.log || true
 
          ## shut down voltha but leave kind-voltha cluster
+         if [ "${branch}" != "master" ]; then
+           echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
+           source "$WORKSPACE/kind-voltha/releases/${branch}"
+         else
+           echo "on master, using default settings for kind-voltha"
+         fi
          cd $WORKSPACE/kind-voltha/
          DEPLOY_K8S=n WAIT_ON_DOWN=y ./voltha down
          kubectl delete deployment voltctl || true
diff --git a/jjb/pipeline/voltha/voltha-2.7/voltha-bbsim-tests.groovy b/jjb/pipeline/voltha/voltha-2.7/voltha-bbsim-tests.groovy
index 226b7e7..5208368 100644
--- a/jjb/pipeline/voltha/voltha-2.7/voltha-bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/voltha-2.7/voltha-bbsim-tests.groovy
@@ -132,24 +132,27 @@
     }
     stage('Create K8s Cluster') {
       steps {
-        sh """
-           if [ "${branch}" != "master" ]; then
-             echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
-             source "$WORKSPACE/kind-voltha/releases/${branch}"
-           else
-             echo "on master, using default settings for kind-voltha"
-           fi
+        timeout(time: 10, unit: 'MINUTES') {
+          sh """
+             if [ "${branch}" != "master" ]; then
+               echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
+               source "$WORKSPACE/kind-voltha/releases/${branch}"
+             else
+               echo "on master, using default settings for kind-voltha"
+             fi
 
-           cd $WORKSPACE/kind-voltha/
-           JUST_K8S=y ./voltha up
-           bash <( curl -sfL https://raw.githubusercontent.com/boz/kail/master/godownloader.sh) -b "$WORKSPACE/kind-voltha/bin"
-           """
+             cd $WORKSPACE/kind-voltha/
+             JUST_K8S=y ./voltha up
+             bash <( curl -sfL https://raw.githubusercontent.com/boz/kail/master/godownloader.sh) -b "$WORKSPACE/kind-voltha/bin"
+             """
+         }
       }
     }
 
     stage('Build Images') {
       steps {
-        sh """
+        timeout(time: 10, unit: 'MINUTES') {
+          sh """
            make-local () {
              make -C $WORKSPACE/\$1 DOCKER_REGISTRY=mirror.registry.opennetworking.org/ DOCKER_REPOSITORY=voltha/ DOCKER_TAG=citest docker-build
            }
@@ -178,6 +181,7 @@
              make-local ${gerritProject}
            fi
            """
+        }
       }
     }
 
@@ -206,7 +210,8 @@
         ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/ATTWorkflow"
       }
       steps {
-        sh '''
+        timeout(time: 15, unit: 'MINUTES') {
+          sh '''
            if [ "${branch}" != "master" ]; then
              echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
              source "$WORKSPACE/kind-voltha/releases/${branch}"
@@ -321,6 +326,7 @@
            kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\n'}" | sort | uniq | tee $WORKSPACE/att/pod-images.txt || true
            kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.imageID}{'\\n'}" | sort | uniq | tee $WORKSPACE/att/pod-imagesId.txt || true
            '''
+         }
       }
     }
 
@@ -329,9 +335,16 @@
         ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/DTWorkflow"
       }
       steps {
-        sh '''
+        timeout(time: 15, unit: 'MINUTES') {
+          sh '''
            cd $WORKSPACE/kind-voltha/
            source $NAME-env.sh
+           if [ "${branch}" != "master" ]; then
+             echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
+             source "$WORKSPACE/kind-voltha/releases/${branch}"
+           else
+             echo "on master, using default settings for kind-voltha"
+           fi
            WAIT_ON_DOWN=y DEPLOY_K8S=n ./voltha down
 
            # Workflow-specific flags
@@ -383,6 +396,7 @@
            kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\n'}" | sort | uniq | tee $WORKSPACE/dt/pod-images.txt || true
            kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.imageID}{'\\n'}" | sort | uniq | tee $WORKSPACE/dt/pod-imagesId.txt || true
            '''
+         }
       }
     }
 
@@ -391,60 +405,68 @@
         ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/TTWorkflow"
       }
       steps {
-        sh '''
-           cd $WORKSPACE/kind-voltha/
-           source $NAME-env.sh
-           WAIT_ON_DOWN=y DEPLOY_K8S=n ./voltha down
+        timeout(time: 15, unit: 'MINUTES') {
+          sh '''
+             cd $WORKSPACE/kind-voltha/
+             source $NAME-env.sh
+             if [ "${branch}" != "master" ]; then
+               echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
+               source "$WORKSPACE/kind-voltha/releases/${branch}"
+             else
+               echo "on master, using default settings for kind-voltha"
+             fi
+             WAIT_ON_DOWN=y DEPLOY_K8S=n ./voltha down
 
-           # Workflow-specific flags
-           export WITH_RADIUS=no
-           export WITH_EAPOL=no
-           export WITH_DHCP=yes
-           export WITH_IGMP=yes
-           export CONFIG_SADIS="external"
-           export BBSIM_CFG="configs/bbsim-sadis-tt.yaml"
+             # Workflow-specific flags
+             export WITH_RADIUS=no
+             export WITH_EAPOL=no
+             export WITH_DHCP=yes
+             export WITH_IGMP=yes
+             export CONFIG_SADIS="external"
+             export BBSIM_CFG="configs/bbsim-sadis-tt.yaml"
 
-           if [[ "${gerritProject}" == voltha-helm-charts ]]; then
-             export EXTRA_HELM_FLAGS+="--set global.image_tag=null "
-           fi
+             if [[ "${gerritProject}" == voltha-helm-charts ]]; then
+               export EXTRA_HELM_FLAGS+="--set global.image_tag=null "
+             fi
 
-           # start logging
-           mkdir -p $WORKSPACE/tt
-           _TAG=kail-tt kail -n voltha -n default > $WORKSPACE/tt/onos-voltha-combined.log &
+             # start logging
+             mkdir -p $WORKSPACE/tt
+             _TAG=kail-tt kail -n voltha -n default > $WORKSPACE/tt/onos-voltha-combined.log &
 
-           DEPLOY_K8S=n ./voltha up
+             DEPLOY_K8S=n ./voltha up
 
-           mkdir -p $ROBOT_LOGS_DIR
-           export ROBOT_MISC_ARGS="-d $ROBOT_LOGS_DIR -e PowerSwitch"
+             mkdir -p $ROBOT_LOGS_DIR
+             export ROBOT_MISC_ARGS="-d $ROBOT_LOGS_DIR -e PowerSwitch"
 
-           # By default, all tests tagged 'sanityTt' are run.  This covers basic functionality
-           # like running through the TT workflow for a single subscriber.
-           export TARGET=sanity-kind-tt
+             # By default, all tests tagged 'sanityTt' are run.  This covers basic functionality
+             # like running through the TT workflow for a single subscriber.
+             export TARGET=sanity-kind-tt
 
-           # If the Gerrit comment contains a line with "functional tests" then run the full
-           # functional test suite.  This covers tests tagged either 'sanityTt' or 'functionalTt'.
-           # Note: Gerrit comment text will be prefixed by "Patch set n:" and a blank line
-           REGEX="functional tests"
-           if [[ "$GERRIT_EVENT_COMMENT_TEXT" =~ \$REGEX ]]; then
-             TARGET=functional-single-kind-tt
-           fi
+             # If the Gerrit comment contains a line with "functional tests" then run the full
+             # functional test suite.  This covers tests tagged either 'sanityTt' or 'functionalTt'.
+             # Note: Gerrit comment text will be prefixed by "Patch set n:" and a blank line
+             REGEX="functional tests"
+             if [[ "$GERRIT_EVENT_COMMENT_TEXT" =~ \$REGEX ]]; then
+               TARGET=functional-single-kind-tt
+             fi
 
-           make -C $WORKSPACE/voltha-system-tests \$TARGET || true
+             make -C $WORKSPACE/voltha-system-tests \$TARGET || true
 
-           # stop logging
-           P_IDS="$(ps e -ww -A | grep "_TAG=kail-att" | grep -v grep | awk '{print $1}')"
-           if [ -n "$P_IDS" ]; then
-             echo $P_IDS
-             for P_ID in $P_IDS; do
-               kill -9 $P_ID
-             done
-           fi
+             # stop logging
+             P_IDS="$(ps e -ww -A | grep "_TAG=kail-att" | grep -v grep | awk '{print $1}')"
+             if [ -n "$P_IDS" ]; then
+               echo $P_IDS
+               for P_ID in $P_IDS; do
+                 kill -9 $P_ID
+               done
+             fi
 
-           # get pods information
-           kubectl get pods -o wide > $WORKSPACE/tt/pods.txt || true
-           kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\n'}" | sort | uniq | tee $WORKSPACE/tt/pod-images.txt || true
-           kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.imageID}{'\\n'}" | sort | uniq | tee $WORKSPACE/tt/pod-imagesId.txt || true
-           '''
+             # get pods information
+             kubectl get pods -o wide > $WORKSPACE/tt/pods.txt || true
+             kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\n'}" | sort | uniq | tee $WORKSPACE/tt/pod-images.txt || true
+             kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.imageID}{'\\n'}" | sort | uniq | tee $WORKSPACE/tt/pod-imagesId.txt || true
+             '''
+        }
       }
     }
   }
diff --git a/jjb/pipeline/voltha/voltha-2.7/voltha-dt-physical-build-and-tests.groovy b/jjb/pipeline/voltha/voltha-2.7/voltha-dt-physical-build-and-tests.groovy
index 42e253c..2ac34bf 100644
--- a/jjb/pipeline/voltha/voltha-2.7/voltha-dt-physical-build-and-tests.groovy
+++ b/jjb/pipeline/voltha/voltha-2.7/voltha-dt-physical-build-and-tests.groovy
@@ -55,6 +55,12 @@
     stage ('Initialize') {
       steps {
         sh returnStdout: false, script: """
+        if [ "${branch}" != "master" ]; then
+          echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
+          source "$WORKSPACE/kind-voltha/releases/${branch}"
+        else
+          echo "on master, using default settings for kind-voltha"
+        fi
         test -e $WORKSPACE/voltha/kind-voltha/voltha && cd $WORKSPACE/voltha/kind-voltha && ./voltha down
         cd $WORKSPACE
         rm -rf $WORKSPACE/*
diff --git a/jjb/pipeline/voltha/voltha-2.7/voltha-go-multi-tests.groovy b/jjb/pipeline/voltha/voltha-2.7/voltha-go-multi-tests.groovy
index 984d4cb..e7bb4f0 100644
--- a/jjb/pipeline/voltha/voltha-2.7/voltha-go-multi-tests.groovy
+++ b/jjb/pipeline/voltha/voltha-2.7/voltha-go-multi-tests.groovy
@@ -78,42 +78,47 @@
 
     stage('Deploy Voltha') {
       steps {
-        sh """
-           export EXTRA_HELM_FLAGS=""
-           if [ "${branch}" != "master" ]; then
-             echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
-             source "$WORKSPACE/kind-voltha/releases/${branch}"
-           else
-             echo "on master, using default settings for kind-voltha"
-           fi
+        timeout(time: 15, unit: 'MINUTES') {
+          sh """
+             export EXTRA_HELM_FLAGS=""
+             if [ "${branch}" != "master" ]; then
+               echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
+               source "$WORKSPACE/kind-voltha/releases/${branch}"
+             else
+               echo "on master, using default settings for kind-voltha"
+             fi
 
-           EXTRA_HELM_FLAGS+="--set log_agent.enabled=False ${params.extraHelmFlags} "
+             EXTRA_HELM_FLAGS+="--set log_agent.enabled=False ${params.extraHelmFlags} "
 
-           cd $WORKSPACE/kind-voltha/
-           WAIT_ON_DOWN=y DEPLOY_K8S=n ./voltha down || ./voltha down
-           ./voltha up
-           """
+             cd $WORKSPACE/kind-voltha/
+             WAIT_ON_DOWN=y DEPLOY_K8S=n ./voltha down || ./voltha down
+             ./voltha up
+             """
+         }
       }
     }
 
     stage('Run E2E Tests') {
       steps {
-        sh """
-           set +e
-           mkdir -p $WORKSPACE/RobotLogs
+        // this may potentially fail as we don't know how many times we repeat this tests
+        timeout(time: 30, unit: 'MINUTES') {
+          sh """
+             set +e
+             mkdir -p $WORKSPACE/RobotLogs
 
-           cd $WORKSPACE/kind-voltha/scripts
-           ./log-collector.sh > /dev/null &
-           ./log-combine.sh > /dev/null &
+             cd $WORKSPACE/kind-voltha/scripts
+             ./log-collector.sh > /dev/null &
+             ./log-combine.sh > /dev/null &
 
-           for i in \$(seq 1 ${testRuns})
-           do
-             export ROBOT_MISC_ARGS="${params.extraRobotArgs} -d $WORKSPACE/RobotLogs/\$i -e PowerSwitch"
-             make -C $WORKSPACE/voltha-system-tests ${makeTarget}
-             echo "Completed run: \$i"
-             echo ""
-           done
-           """
+             for i in \$(seq 1 ${testRuns})
+             do
+               export ROBOT_MISC_ARGS="${params.extraRobotArgs} -d $WORKSPACE/RobotLogs/\$i -e PowerSwitch"
+               make -C $WORKSPACE/voltha-system-tests ${makeTarget}
+               echo "Completed run: \$i"
+               echo ""
+             done
+             """
+         }
       }
     }
   }
@@ -161,6 +166,12 @@
          gzip *-combined.log || true
 
          ## shut down voltha but leave kind-voltha cluster
+         if [ "${branch}" != "master" ]; then
+           echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
+           source "$WORKSPACE/kind-voltha/releases/${branch}"
+         else
+           echo "on master, using default settings for kind-voltha"
+         fi
          cd $WORKSPACE/kind-voltha/
          DEPLOY_K8S=n WAIT_ON_DOWN=y ./voltha down
          '''
diff --git a/jjb/pipeline/voltha/voltha-2.7/voltha-go-tests.groovy b/jjb/pipeline/voltha/voltha-2.7/voltha-go-tests.groovy
index c8a80fd..61f600c 100644
--- a/jjb/pipeline/voltha/voltha-2.7/voltha-go-tests.groovy
+++ b/jjb/pipeline/voltha/voltha-2.7/voltha-go-tests.groovy
@@ -63,10 +63,18 @@
     }
     stage('Cleanup') {
       steps {
-        sh """
-        cd $WORKSPACE/kind-voltha/
-        WAIT_ON_DOWN=y DEPLOY_K8S=n ./voltha down || true
-        """
+        timeout(time: 10, unit: 'MINUTES') {
+          sh """
+          if [ "${branch}" != "master" ]; then
+            echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
+            source "$WORKSPACE/kind-voltha/releases/${branch}"
+          else
+            echo "on master, using default settings for kind-voltha"
+          fi
+          cd $WORKSPACE/kind-voltha/
+          WAIT_ON_DOWN=y DEPLOY_K8S=n ./voltha down || true
+          """
+        }
       }
     }
     stage('Clone voltha-system-tests') {
@@ -89,44 +97,47 @@
 
     stage('Deploy Voltha') {
       steps {
-        sh """
-           export EXTRA_HELM_FLAGS=""
-           if [ "${branch}" != "master" ]; then
-             echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
-             source "$WORKSPACE/kind-voltha/releases/${branch}"
-           else
-             echo "on master, using default settings for kind-voltha"
-           fi
+        timeout(time: 10, unit: 'MINUTES') {
+          sh """
+             export EXTRA_HELM_FLAGS=""
+             if [ "${branch}" != "master" ]; then
+               echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
+               source "$WORKSPACE/kind-voltha/releases/${branch}"
+             else
+               echo "on master, using default settings for kind-voltha"
+             fi
 
-           if [ "${workFlow}" == "DT" ]; then
-             export WITH_DHCP=no
-             export WITH_IGMP=no
-             export WITH_EAPOL=no
-             export WITH_RADIUS=no
-             export BBSIM_CFG="configs/bbsim-sadis-dt.yaml"
-           fi
+             if [ "${workFlow}" == "DT" ]; then
+               export WITH_DHCP=no
+               export WITH_IGMP=no
+               export WITH_EAPOL=no
+               export WITH_RADIUS=no
+               export BBSIM_CFG="configs/bbsim-sadis-dt.yaml"
+             fi
 
-           EXTRA_HELM_FLAGS+="--set log_agent.enabled=False ${params.extraHelmFlags} "
-
-           cd $WORKSPACE/kind-voltha/
-           WAIT_ON_DOWN=y DEPLOY_K8S=n ./voltha down || ./voltha down
-           ./voltha up
-           """
+             EXTRA_HELM_FLAGS+="--set log_agent.enabled=False ${params.extraHelmFlags} "
+             cd $WORKSPACE/kind-voltha/
+             WAIT_ON_DOWN=y DEPLOY_K8S=n ./voltha down || ./voltha down
+             ./voltha up
+             """
+           }
       }
     }
 
     stage('Run E2E Tests') {
       steps {
-        sh '''
-           set +e
-           mkdir -p $WORKSPACE/RobotLogs
+        timeout(time: 5, unit: 'MINUTES') {
+          sh '''
+             set +e
+             mkdir -p $WORKSPACE/RobotLogs
 
-           cd $WORKSPACE/kind-voltha/scripts
-           ./log-collector.sh > /dev/null &
-           ./log-combine.sh > /dev/null &
+             cd $WORKSPACE/kind-voltha/scripts
+             ./log-collector.sh > /dev/null &
+             ./log-combine.sh > /dev/null &
 
-           make -C $WORKSPACE/voltha-system-tests ${makeTarget} || true
-           '''
+             make -C $WORKSPACE/voltha-system-tests ${makeTarget} || true
+             '''
+         }
       }
     }
   }
@@ -174,6 +185,12 @@
          gzip *-combined.log || true
 
          ## shut down voltha but leave kind-voltha cluster
+         if [ "${branch}" != "master" ]; then
+           echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
+           source "$WORKSPACE/kind-voltha/releases/${branch}"
+         else
+           echo "on master, using default settings for kind-voltha"
+         fi
          cd $WORKSPACE/kind-voltha/
          DEPLOY_K8S=n WAIT_ON_DOWN=y ./voltha down
          kubectl delete deployment voltctl || true
diff --git a/jjb/pipeline/voltha/voltha-2.7/voltha-nightly-tests-bbsim.groovy b/jjb/pipeline/voltha/voltha-2.7/voltha-nightly-tests-bbsim.groovy
index d86aac9..31aa380 100644
--- a/jjb/pipeline/voltha/voltha-2.7/voltha-nightly-tests-bbsim.groovy
+++ b/jjb/pipeline/voltha/voltha-2.7/voltha-nightly-tests-bbsim.groovy
@@ -23,7 +23,7 @@
     label "${params.buildNode}"
   }
   options {
-      timeout(time: 300, unit: 'MINUTES')
+      timeout(time: 100, unit: 'MINUTES')
   }
   environment {
     KUBECONFIG="$HOME/.kube/kind-config-voltha-minimal"
@@ -63,10 +63,18 @@
     }
     stage('Cleanup') {
       steps {
-        sh """
-        cd $WORKSPACE/kind-voltha/
-        WAIT_ON_DOWN=y DEPLOY_K8S=n ./voltha down || ./voltha down
-        """
+        timeout(time: 10, unit: 'MINUTES') {
+          sh """
+          if [ "${branch}" != "master" ]; then
+            echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
+            source "$WORKSPACE/kind-voltha/releases/${branch}"
+          else
+            echo "on master, using default settings for kind-voltha"
+          fi
+          cd $WORKSPACE/kind-voltha/
+          WAIT_ON_DOWN=y DEPLOY_K8S=n ./voltha down || ./voltha down
+          """
+        }
       }
     }
     stage('Clone voltha-system-tests') {
@@ -97,21 +105,23 @@
 
     stage('Deploy Voltha') {
       steps {
-        sh """
-           export EXTRA_HELM_FLAGS=""
-           if [ "${branch}" != "master" ]; then
-             echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
-             source "$WORKSPACE/kind-voltha/releases/${branch}"
-           else
-             export INFRA_NS="infra"
-             echo "on master, using default settings for kind-voltha"
-           fi
+        timeout(time: 10, unit: 'MINUTES') {
+          sh """
+             export EXTRA_HELM_FLAGS=""
+             if [ "${branch}" != "master" ]; then
+               echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
+               source "$WORKSPACE/kind-voltha/releases/${branch}"
+             else
+               export INFRA_NS="infra"
+               echo "on master, using default settings for kind-voltha"
+             fi
 
-           EXTRA_HELM_FLAGS+="--set log_agent.enabled=False ${params.extraHelmFlags} --set defaults.image_registry=mirror.registry.opennetworking.org/ "
+             EXTRA_HELM_FLAGS+="--set log_agent.enabled=False ${params.extraHelmFlags} --set defaults.image_registry=mirror.registry.opennetworking.org/ "
 
-           cd $WORKSPACE/kind-voltha/
-           ./voltha up
-           """
+             cd $WORKSPACE/kind-voltha/
+             ./voltha up
+             """
+           }
       }
     }
 
@@ -120,16 +130,18 @@
         ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/FunctionalTests"
       }
       steps {
-        sh '''
-           set +e
-           mkdir -p $ROBOT_LOGS_DIR
-           cd $WORKSPACE/kind-voltha/scripts
-           ./log-collector.sh > /dev/null &
-           ./log-combine.sh > /dev/null &
+        timeout(time: 20, unit: 'MINUTES') {
+          sh '''
+             set +e
+             mkdir -p $ROBOT_LOGS_DIR
+             cd $WORKSPACE/kind-voltha/scripts
+             ./log-collector.sh > /dev/null &
+             ./log-combine.sh > /dev/null &
 
-           export ROBOT_MISC_ARGS="-d $ROBOT_LOGS_DIR"
-           make -C $WORKSPACE/voltha-system-tests ${makeTarget} || true
-           '''
+             export ROBOT_MISC_ARGS="-d $ROBOT_LOGS_DIR"
+             make -C $WORKSPACE/voltha-system-tests ${makeTarget} || true
+             '''
+         }
       }
     }
 
@@ -143,13 +155,15 @@
         }
       }
       steps {
-        sh '''
-           set +e
-           mkdir -p $WORKSPACE/RobotLogs
+        timeout(time: 5, unit: 'MINUTES') {
+          sh '''
+             set +e
+             mkdir -p $WORKSPACE/RobotLogs
 
-           export ROBOT_MISC_ARGS="-d $ROBOT_LOGS_DIR"
-           make -C $WORKSPACE/voltha-system-tests ${makeAlarmtestTarget} || true
-           '''
+             export ROBOT_MISC_ARGS="-d $ROBOT_LOGS_DIR"
+             make -C $WORKSPACE/voltha-system-tests ${makeAlarmtestTarget} || true
+             '''
+         }
       }
     }
 
@@ -158,13 +172,15 @@
         ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/FailureTests"
       }
       steps {
-        sh '''
-           set +e
-           mkdir -p $WORKSPACE/RobotLogs
+        timeout(time: 30, unit: 'MINUTES') {
+          sh '''
+             set +e
+             mkdir -p $WORKSPACE/RobotLogs
 
-           export ROBOT_MISC_ARGS="-d $ROBOT_LOGS_DIR"
-           make -C $WORKSPACE/voltha-system-tests ${makeFailtestTarget} || true
-           '''
+             export ROBOT_MISC_ARGS="-d $ROBOT_LOGS_DIR"
+             make -C $WORKSPACE/voltha-system-tests ${makeFailtestTarget} || true
+             '''
+         }
       }
     }
     stage('Multiple OLT Tests') {
@@ -172,14 +188,16 @@
         ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/MultipleOLTTests"
       }
       steps {
-        sh '''
-           if [ "${olts}" -gt 1 ]; then
-              set +e
-              mkdir -p $WORKSPACE/RobotLogs
-              export ROBOT_MISC_ARGS="-d $ROBOT_LOGS_DIR"
-              make -C $WORKSPACE/voltha-system-tests ${makeMultiOltTarget} || true
-           fi
-           '''
+        timeout(time: 15, unit: 'MINUTES') {
+          sh '''
+             if [ "${olts}" -gt 1 ]; then
+                set +e
+                mkdir -p $WORKSPACE/RobotLogs
+                export ROBOT_MISC_ARGS="-d $ROBOT_LOGS_DIR"
+                make -C $WORKSPACE/voltha-system-tests ${makeMultiOltTarget} || true
+             fi
+             '''
+         }
       }
     }
 
@@ -188,13 +206,15 @@
         ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/ErrorTests"
       }
       steps {
-        sh '''
-           set +e
-           mkdir -p $WORKSPACE/RobotLogs
+        timeout(time: 10, unit: 'MINUTES') {
+          sh '''
+             set +e
+             mkdir -p $WORKSPACE/RobotLogs
 
-           export ROBOT_MISC_ARGS="-d $ROBOT_LOGS_DIR"
-           make -C $WORKSPACE/voltha-system-tests ${makeErrortestTarget} || true
-           '''
+             export ROBOT_MISC_ARGS="-d $ROBOT_LOGS_DIR"
+             make -C $WORKSPACE/voltha-system-tests ${makeErrortestTarget} || true
+             '''
+        }
       }
     }
   }
@@ -242,6 +262,12 @@
          gzip *-combined.log || true
 
          ## shut down voltha but leave kind-voltha cluster
+         if [ "${branch}" != "master" ]; then
+           echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
+           source "$WORKSPACE/kind-voltha/releases/${branch}"
+         else
+           echo "on master, using default settings for kind-voltha"
+         fi
          cd $WORKSPACE/kind-voltha/
          DEPLOY_K8S=n WAIT_ON_DOWN=y ./voltha down
          kubectl delete deployment voltctl || true
diff --git a/jjb/pipeline/voltha/voltha-2.7/voltha-openonu-go-test-bbsim.groovy b/jjb/pipeline/voltha/voltha-2.7/voltha-openonu-go-test-bbsim.groovy
index a82b9f1..7462c85 100755
--- a/jjb/pipeline/voltha/voltha-2.7/voltha-openonu-go-test-bbsim.groovy
+++ b/jjb/pipeline/voltha/voltha-2.7/voltha-openonu-go-test-bbsim.groovy
@@ -63,10 +63,18 @@
     }
     stage('Cleanup') {
       steps {
-        sh """
-        cd $WORKSPACE/kind-voltha/
-        WAIT_ON_DOWN=y DEPLOY_K8S=n ./voltha down || ./voltha down
-        """
+        timeout(time: 10, unit: 'MINUTES') {
+          sh """
+          if [ "${branch}" != "master" ]; then
+            echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
+            source "$WORKSPACE/kind-voltha/releases/${branch}"
+          else
+            echo "on master, using default settings for kind-voltha"
+          fi
+          cd $WORKSPACE/kind-voltha/
+          WAIT_ON_DOWN=y DEPLOY_K8S=n ./voltha down || ./voltha down
+          """
+        }
       }
     }
     stage('Clone voltha-system-tests') {
@@ -89,21 +97,23 @@
 
     stage('Deploy Voltha') {
       steps {
-        sh """
-           export EXTRA_HELM_FLAGS=""
-           if [ "${branch}" != "master" ]; then
-             echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
-             source "$WORKSPACE/kind-voltha/releases/${branch}"
-           else
-             echo "on master, using default settings for kind-voltha"
-           fi
+        timeout(time: 10, unit: 'MINUTES') {
+          sh """
+             export EXTRA_HELM_FLAGS=""
+             if [ "${branch}" != "master" ]; then
+               echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
+               source "$WORKSPACE/kind-voltha/releases/${branch}"
+             else
+               echo "on master, using default settings for kind-voltha"
+             fi
 
-           EXTRA_HELM_FLAGS+="--set log_agent.enabled=False ${params.extraHelmFlags} --set defaults.image_registry=mirror.registry.opennetworking.org/ "
+             EXTRA_HELM_FLAGS+="--set log_agent.enabled=False ${params.extraHelmFlags} --set defaults.image_registry=mirror.registry.opennetworking.org/ "
 
-           cd $WORKSPACE/kind-voltha/
-           ./voltha up
-           bash <( curl -sfL https://raw.githubusercontent.com/boz/kail/master/godownloader.sh) -b "$WORKSPACE/kind-voltha/bin"
-           """
+             cd $WORKSPACE/kind-voltha/
+             ./voltha up
+             bash <( curl -sfL https://raw.githubusercontent.com/boz/kail/master/godownloader.sh) -b "$WORKSPACE/kind-voltha/bin"
+             """
+         }
       }
     }
 
@@ -112,29 +122,31 @@
         ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/1t1gem"
       }
       steps {
-        sh '''
-          # start logging
-          mkdir -p $WORKSPACE/1t1gem
-          _TAG=kail-1t1gem kail -n voltha -n default > $WORKSPACE/1t1gem/onos-voltha-combined.log &
+        timeout(time: 10, unit: 'MINUTES') {
+          sh '''
+            # start logging
+            mkdir -p $WORKSPACE/1t1gem
+            _TAG=kail-1t1gem kail -n voltha -n default > $WORKSPACE/1t1gem/onos-voltha-combined.log &
 
-          mkdir -p $ROBOT_LOGS_DIR/1t1gem
-          export ROBOT_MISC_ARGS="-d $ROBOT_LOGS_DIR"
-          export KVSTOREPREFIX=voltha_voltha
+            mkdir -p $ROBOT_LOGS_DIR/1t1gem
+            export ROBOT_MISC_ARGS="-d $ROBOT_LOGS_DIR"
+            export KVSTOREPREFIX=voltha_voltha
 
-          make -C $WORKSPACE/voltha-system-tests ${makeTarget} || true
+            make -C $WORKSPACE/voltha-system-tests ${makeTarget} || true
 
-          # stop logging
-          P_IDS="$(ps e -ww -A | grep "_TAG=kail-1t1gem" | grep -v grep | awk '{print $1}')"
-          if [ -n "$P_IDS" ]; then
-            echo $P_IDS
-            for P_ID in $P_IDS; do
-              kill -9 $P_ID
-            done
-          fi
+            # stop logging
+            P_IDS="$(ps e -ww -A | grep "_TAG=kail-1t1gem" | grep -v grep | awk '{print $1}')"
+            if [ -n "$P_IDS" ]; then
+              echo $P_IDS
+              for P_ID in $P_IDS; do
+                kill -9 $P_ID
+              done
+            fi
 
-          # get pods information
-          kubectl get pods -o wide --all-namespaces > $WORKSPACE/1t1gem/pods.txt || true
-        '''
+            # get pods information
+            kubectl get pods -o wide --all-namespaces > $WORKSPACE/1t1gem/pods.txt || true
+          '''
+         }
        }
      }
 
@@ -143,43 +155,51 @@
         ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/1t4gem"
       }
       steps {
-        sh '''
-          cd $WORKSPACE/kind-voltha/
-          WAIT_ON_DOWN=y DEPLOY_K8S=n ./voltha down
+        timeout(time: 10, unit: 'MINUTES') {
+          sh '''
+            if [ "${branch}" != "master" ]; then
+              echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
+              source "$WORKSPACE/kind-voltha/releases/${branch}"
+            else
+              echo "on master, using default settings for kind-voltha"
+            fi
+            cd $WORKSPACE/kind-voltha/
+            WAIT_ON_DOWN=y DEPLOY_K8S=n ./voltha down
 
-          export EXTRA_HELM_FLAGS=""
-          if [ "${branch}" != "master" ]; then
-            echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
-            source "$WORKSPACE/kind-voltha/releases/${branch}"
-          else
-            echo "on master, using default settings for kind-voltha"
-          fi
-          export EXTRA_HELM_FLAGS+="--set log_agent.enabled=False ${extraHelmFlags} "
+            export EXTRA_HELM_FLAGS=""
+            if [ "${branch}" != "master" ]; then
+              echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
+              source "$WORKSPACE/kind-voltha/releases/${branch}"
+            else
+              echo "on master, using default settings for kind-voltha"
+            fi
+            export EXTRA_HELM_FLAGS+="--set log_agent.enabled=False ${extraHelmFlags} "
 
-          # start logging
-          mkdir -p $WORKSPACE/1t4gem
-          _TAG=kail-1t4gem kail -n voltha -n default > $WORKSPACE/1t4gem/onos-voltha-combined.log &
+            # start logging
+            mkdir -p $WORKSPACE/1t4gem
+            _TAG=kail-1t4gem kail -n voltha -n default > $WORKSPACE/1t4gem/onos-voltha-combined.log &
 
-          DEPLOY_K8S=n ./voltha up
+            DEPLOY_K8S=n ./voltha up
 
-          mkdir -p $ROBOT_LOGS_DIR/1t4gem
-          export ROBOT_MISC_ARGS="-d $ROBOT_LOGS_DIR"
-          export KVSTOREPREFIX=voltha_voltha
+            mkdir -p $ROBOT_LOGS_DIR/1t4gem
+            export ROBOT_MISC_ARGS="-d $ROBOT_LOGS_DIR"
+            export KVSTOREPREFIX=voltha_voltha
 
-          make -C $WORKSPACE/voltha-system-tests ${make1t4gemTestTarget} || true
+            make -C $WORKSPACE/voltha-system-tests ${make1t4gemTestTarget} || true
 
-          # stop logging
-          P_IDS="$(ps e -ww -A | grep "_TAG=kail-1t4gem" | grep -v grep | awk '{print $1}')"
-          if [ -n "$P_IDS" ]; then
-            echo $P_IDS
-            for P_ID in $P_IDS; do
-              kill -9 $P_ID
-            done
-          fi
+            # stop logging
+            P_IDS="$(ps e -ww -A | grep "_TAG=kail-1t4gem" | grep -v grep | awk '{print $1}')"
+            if [ -n "$P_IDS" ]; then
+              echo $P_IDS
+              for P_ID in $P_IDS; do
+                kill -9 $P_ID
+              done
+            fi
 
-          # get pods information
-          kubectl get pods -o wide --all-namespaces > $WORKSPACE/1t4gem/pods.txt || true
-        '''
+            # get pods information
+            kubectl get pods -o wide --all-namespaces > $WORKSPACE/1t4gem/pods.txt || true
+          '''
+         }
        }
      }
 
@@ -188,43 +208,51 @@
         ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/1t8gem"
       }
       steps {
-        sh '''
-          cd $WORKSPACE/kind-voltha/
-          WAIT_ON_DOWN=y DEPLOY_K8S=n ./voltha down
+        timeout(time: 10, unit: 'MINUTES') {
+          sh '''
+            if [ "${branch}" != "master" ]; then
+              echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
+              source "$WORKSPACE/kind-voltha/releases/${branch}"
+            else
+              echo "on master, using default settings for kind-voltha"
+            fi
+            cd $WORKSPACE/kind-voltha/
+            WAIT_ON_DOWN=y DEPLOY_K8S=n ./voltha down
 
-          export EXTRA_HELM_FLAGS=""
-          if [ "${branch}" != "master" ]; then
-            echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
-            source "$WORKSPACE/kind-voltha/releases/${branch}"
-          else
-            echo "on master, using default settings for kind-voltha"
-          fi
-          export EXTRA_HELM_FLAGS+="--set log_agent.enabled=False ${extraHelmFlags} "
+            export EXTRA_HELM_FLAGS=""
+            if [ "${branch}" != "master" ]; then
+              echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
+              source "$WORKSPACE/kind-voltha/releases/${branch}"
+            else
+              echo "on master, using default settings for kind-voltha"
+            fi
+            export EXTRA_HELM_FLAGS+="--set log_agent.enabled=False ${extraHelmFlags} "
 
-          # start logging
-          mkdir -p $WORKSPACE/1t8gem
-          _TAG=kail-1t8gem kail -n voltha -n default > $WORKSPACE/1t8gem/onos-voltha-combined.log &
+            # start logging
+            mkdir -p $WORKSPACE/1t8gem
+            _TAG=kail-1t8gem kail -n voltha -n default > $WORKSPACE/1t8gem/onos-voltha-combined.log &
 
-          DEPLOY_K8S=n ./voltha up
+            DEPLOY_K8S=n ./voltha up
 
-          mkdir -p $ROBOT_LOGS_DIR/1t8gem
-          export ROBOT_MISC_ARGS="-d $ROBOT_LOGS_DIR"
-          export KVSTOREPREFIX=voltha_voltha
+            mkdir -p $ROBOT_LOGS_DIR/1t8gem
+            export ROBOT_MISC_ARGS="-d $ROBOT_LOGS_DIR"
+            export KVSTOREPREFIX=voltha_voltha
 
-          make -C $WORKSPACE/voltha-system-tests ${make1t8gemTestTarget} || true
+            make -C $WORKSPACE/voltha-system-tests ${make1t8gemTestTarget} || true
 
-          # stop logging
-          P_IDS="$(ps e -ww -A | grep "_TAG=kail-1t8gem" | grep -v grep | awk '{print $1}')"
-          if [ -n "$P_IDS" ]; then
-            echo $P_IDS
-            for P_ID in $P_IDS; do
-              kill -9 $P_ID
-            done
-          fi
+            # stop logging
+            P_IDS="$(ps e -ww -A | grep "_TAG=kail-1t8gem" | grep -v grep | awk '{print $1}')"
+            if [ -n "$P_IDS" ]; then
+              echo $P_IDS
+              for P_ID in $P_IDS; do
+                kill -9 $P_ID
+              done
+            fi
 
-          # get pods information
-          kubectl get pods -o wide --all-namespaces > $WORKSPACE/1t8gem/pods.txt || true
-        '''
+            # get pods information
+            kubectl get pods -o wide --all-namespaces > $WORKSPACE/1t8gem/pods.txt || true
+          '''
+        }
       }
     }
 
@@ -234,44 +262,52 @@
         ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/openonu-go-MIB"
       }
       steps {
-        sh '''
-           cd $WORKSPACE/kind-voltha/
-           WAIT_ON_DOWN=y DEPLOY_K8S=n ./voltha down
+        timeout(time: 10, unit: 'MINUTES') {
+          sh '''
+             if [ "${branch}" != "master" ]; then
+               echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
+               source "$WORKSPACE/kind-voltha/releases/${branch}"
+             else
+               echo "on master, using default settings for kind-voltha"
+             fi
+             cd $WORKSPACE/kind-voltha/
+             WAIT_ON_DOWN=y DEPLOY_K8S=n ./voltha down
 
-           export EXTRA_HELM_FLAGS=""
-           if [ "${branch}" != "master" ]; then
-             echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
-             source "$WORKSPACE/kind-voltha/releases/${branch}"
-           else
-             echo "on master, using default settings for kind-voltha"
-           fi
-           export EXTRA_HELM_FLAGS+="--set log_agent.enabled=False ${extraHelmFlags} "
-           export EXTRA_HELM_FLAGS+="--set pon=2,onu=2,controlledActivation=only-onu "
+             export EXTRA_HELM_FLAGS=""
+             if [ "${branch}" != "master" ]; then
+               echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
+               source "$WORKSPACE/kind-voltha/releases/${branch}"
+             else
+               echo "on master, using default settings for kind-voltha"
+             fi
+             export EXTRA_HELM_FLAGS+="--set log_agent.enabled=False ${extraHelmFlags} "
+             export EXTRA_HELM_FLAGS+="--set pon=2,onu=2,controlledActivation=only-onu "
 
-           # start logging
-           mkdir -p $WORKSPACE/mib
-           _TAG=kail-mib kail -n voltha -n default > $WORKSPACE/mib/onos-voltha-combined.log &
+             # start logging
+             mkdir -p $WORKSPACE/mib
+             _TAG=kail-mib kail -n voltha -n default > $WORKSPACE/mib/onos-voltha-combined.log &
 
-           DEPLOY_K8S=n ./voltha up
+             DEPLOY_K8S=n ./voltha up
 
-           mkdir -p $ROBOT_LOGS_DIR
-           export ROBOT_MISC_ARGS="-d $ROBOT_LOGS_DIR"
-           export TARGET_DEFAULT=mib-upload-templating-openonu-go-adapter-test
+             mkdir -p $ROBOT_LOGS_DIR
+             export ROBOT_MISC_ARGS="-d $ROBOT_LOGS_DIR"
+             export TARGET_DEFAULT=mib-upload-templating-openonu-go-adapter-test
 
-           make -C $WORKSPACE/voltha-system-tests \$TARGET_DEFAULT || true
+             make -C $WORKSPACE/voltha-system-tests \$TARGET_DEFAULT || true
 
-           # stop logging
-           P_IDS="$(ps e -ww -A | grep "_TAG=kail-mib" | grep -v grep | awk '{print $1}')"
-           if [ -n "$P_IDS" ]; then
-             echo $P_IDS
-             for P_ID in $P_IDS; do
-               kill -9 $P_ID
-             done
-           fi
+             # stop logging
+             P_IDS="$(ps e -ww -A | grep "_TAG=kail-mib" | grep -v grep | awk '{print $1}')"
+             if [ -n "$P_IDS" ]; then
+               echo $P_IDS
+               for P_ID in $P_IDS; do
+                 kill -9 $P_ID
+               done
+             fi
 
-           # get pods information
-           kubectl get pods -o wide --all-namespaces > $WORKSPACE/mib/pods.txt || true
-        '''
+             # get pods information
+             kubectl get pods -o wide --all-namespaces > $WORKSPACE/mib/pods.txt || true
+          '''
+        }
       }
     }
 
@@ -280,50 +316,58 @@
         ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/ReconcileDT"
       }
       steps {
-        sh '''
-           cd $WORKSPACE/kind-voltha/
-           WAIT_ON_DOWN=y DEPLOY_K8S=n ./voltha down
+        timeout(time: 20, unit: 'MINUTES') {
+          sh '''
+             if [ "${branch}" != "master" ]; then
+               echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
+               source "$WORKSPACE/kind-voltha/releases/${branch}"
+             else
+               echo "on master, using default settings for kind-voltha"
+             fi
+             cd $WORKSPACE/kind-voltha/
+             WAIT_ON_DOWN=y DEPLOY_K8S=n ./voltha down
 
-           export EXTRA_HELM_FLAGS=""
-           if [ "${branch}" != "master" ]; then
-             echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
-             source "$WORKSPACE/kind-voltha/releases/${branch}"
-           else
-             echo "on master, using default settings for kind-voltha"
-           fi
-           export EXTRA_HELM_FLAGS+="--set log_agent.enabled=False ${extraHelmFlags} "
+             export EXTRA_HELM_FLAGS=""
+             if [ "${branch}" != "master" ]; then
+               echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
+               source "$WORKSPACE/kind-voltha/releases/${branch}"
+             else
+               echo "on master, using default settings for kind-voltha"
+             fi
+             export EXTRA_HELM_FLAGS+="--set log_agent.enabled=False ${extraHelmFlags} "
 
-           # Workflow-specific flags
-           export WITH_RADIUS=no
-           export WITH_EAPOL=no
-           export WITH_DHCP=no
-           export WITH_IGMP=no
-           export CONFIG_SADIS="external"
-           export BBSIM_CFG="configs/bbsim-sadis-dt.yaml"
+             # Workflow-specific flags
+             export WITH_RADIUS=no
+             export WITH_EAPOL=no
+             export WITH_DHCP=no
+             export WITH_IGMP=no
+             export CONFIG_SADIS="external"
+             export BBSIM_CFG="configs/bbsim-sadis-dt.yaml"
 
-           # start logging
-           mkdir -p $WORKSPACE/reconciledt
-           _TAG=kail-reconcile-dt kail -n voltha -n default > $WORKSPACE/reconciledt/onos-voltha-combined.log &
+             # start logging
+             mkdir -p $WORKSPACE/reconciledt
+             _TAG=kail-reconcile-dt kail -n voltha -n default > $WORKSPACE/reconciledt/onos-voltha-combined.log &
 
-           DEPLOY_K8S=n ./voltha up
+             DEPLOY_K8S=n ./voltha up
 
-           mkdir -p $ROBOT_LOGS_DIR
-           export ROBOT_MISC_ARGS="-d $ROBOT_LOGS_DIR -e PowerSwitch"
+             mkdir -p $ROBOT_LOGS_DIR
+             export ROBOT_MISC_ARGS="-d $ROBOT_LOGS_DIR -e PowerSwitch"
 
-           make -C $WORKSPACE/voltha-system-tests ${makeReconcileDtTestTarget} || true
+             make -C $WORKSPACE/voltha-system-tests ${makeReconcileDtTestTarget} || true
 
-           # stop logging
-           P_IDS="$(ps e -ww -A | grep "_TAG=kail-reconcile-dt" | grep -v grep | awk '{print $1}')"
-           if [ -n "$P_IDS" ]; then
-             echo $P_IDS
-             for P_ID in $P_IDS; do
-               kill -9 $P_ID
-             done
-           fi
+             # stop logging
+             P_IDS="$(ps e -ww -A | grep "_TAG=kail-reconcile-dt" | grep -v grep | awk '{print $1}')"
+             if [ -n "$P_IDS" ]; then
+               echo $P_IDS
+               for P_ID in $P_IDS; do
+                 kill -9 $P_ID
+               done
+             fi
 
-           # get pods information
-           kubectl get pods -o wide --all-namespaces > $WORKSPACE/reconciledt/pods.txt || true
-           '''
+             # get pods information
+             kubectl get pods -o wide --all-namespaces > $WORKSPACE/reconciledt/pods.txt || true
+             '''
+         }
       }
     }
 
@@ -332,56 +376,64 @@
         ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/ReconcileATT"
       }
       steps {
-        sh '''
-           cd $WORKSPACE/kind-voltha/
-           WAIT_ON_DOWN=y DEPLOY_K8S=n ./voltha down
+        timeout(time: 20, unit: 'MINUTES') {
+          sh '''
+             if [ "${branch}" != "master" ]; then
+               echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
+               source "$WORKSPACE/kind-voltha/releases/${branch}"
+             else
+               echo "on master, using default settings for kind-voltha"
+             fi
+             cd $WORKSPACE/kind-voltha/
+             WAIT_ON_DOWN=y DEPLOY_K8S=n ./voltha down
 
-           export EXTRA_HELM_FLAGS=""
-           if [ "${branch}" != "master" ]; then
-             echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
-             source "$WORKSPACE/kind-voltha/releases/${branch}"
-           else
-             echo "on master, using default settings for kind-voltha"
-           fi
-           export EXTRA_HELM_FLAGS+="--set log_agent.enabled=False ${extraHelmFlags} "
+             export EXTRA_HELM_FLAGS=""
+             if [ "${branch}" != "master" ]; then
+               echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
+               source "$WORKSPACE/kind-voltha/releases/${branch}"
+             else
+               echo "on master, using default settings for kind-voltha"
+             fi
+             export EXTRA_HELM_FLAGS+="--set log_agent.enabled=False ${extraHelmFlags} "
 
-           # Workflow-specific flags
-           export WITH_RADIUS=yes
-           export WITH_EAPOL=yes
-           export WITH_BBSIM=yes
-           export DEPLOY_K8S=yes
-           export CONFIG_SADIS="external"
-           export BBSIM_CFG="configs/bbsim-sadis-att.yaml"
+             # Workflow-specific flags
+             export WITH_RADIUS=yes
+             export WITH_EAPOL=yes
+             export WITH_BBSIM=yes
+             export DEPLOY_K8S=yes
+             export CONFIG_SADIS="external"
+             export BBSIM_CFG="configs/bbsim-sadis-att.yaml"
 
-           if [ "${gerritProject}" = "voltctl" ]; then
-             export VOLTCTL_VERSION=$(cat $WORKSPACE/voltctl/VERSION)
-             cp $WORKSPACE/voltctl/voltctl $WORKSPACE/kind-voltha/bin/voltctl
-             md5sum $WORKSPACE/kind-voltha/bin/voltctl
-           fi
+             if [ "${gerritProject}" = "voltctl" ]; then
+               export VOLTCTL_VERSION=$(cat $WORKSPACE/voltctl/VERSION)
+               cp $WORKSPACE/voltctl/voltctl $WORKSPACE/kind-voltha/bin/voltctl
+               md5sum $WORKSPACE/kind-voltha/bin/voltctl
+             fi
 
-           # start logging
-           mkdir -p $WORKSPACE/reconcileatt
-           _TAG=kail-reconcile-att kail -n voltha -n default > $WORKSPACE/reconcileatt/onos-voltha-combined.log &
+             # start logging
+             mkdir -p $WORKSPACE/reconcileatt
+             _TAG=kail-reconcile-att kail -n voltha -n default > $WORKSPACE/reconcileatt/onos-voltha-combined.log &
 
-           DEPLOY_K8S=n ./voltha up
+             DEPLOY_K8S=n ./voltha up
 
-           mkdir -p $ROBOT_LOGS_DIR
-           export ROBOT_MISC_ARGS="-d $ROBOT_LOGS_DIR -e PowerSwitch"
+             mkdir -p $ROBOT_LOGS_DIR
+             export ROBOT_MISC_ARGS="-d $ROBOT_LOGS_DIR -e PowerSwitch"
 
-           make -C $WORKSPACE/voltha-system-tests ${makeReconcileTestTarget} || true
+             make -C $WORKSPACE/voltha-system-tests ${makeReconcileTestTarget} || true
 
-           # stop logging
-           P_IDS="$(ps e -ww -A | grep "_TAG=kail-reconcile-att" | grep -v grep | awk '{print $1}')"
-           if [ -n "$P_IDS" ]; then
-             echo $P_IDS
-             for P_ID in $P_IDS; do
-               kill -9 $P_ID
-             done
-           fi
+             # stop logging
+             P_IDS="$(ps e -ww -A | grep "_TAG=kail-reconcile-att" | grep -v grep | awk '{print $1}')"
+             if [ -n "$P_IDS" ]; then
+               echo $P_IDS
+               for P_ID in $P_IDS; do
+                 kill -9 $P_ID
+               done
+             fi
 
-           # get pods information
-           kubectl get pods -o wide --all-namespaces > $WORKSPACE/reconcileatt/pods.txt || true
-           '''
+             # get pods information
+             kubectl get pods -o wide --all-namespaces > $WORKSPACE/reconcileatt/pods.txt || true
+             '''
+         }
       }
     }
 
@@ -390,50 +442,58 @@
         ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/ReconcileTT"
       }
       steps {
-        sh '''
-           cd $WORKSPACE/kind-voltha/
-           WAIT_ON_DOWN=y DEPLOY_K8S=n ./voltha down
+        timeout(time: 20, unit: 'MINUTES') {
+          sh '''
+             if [ "${branch}" != "master" ]; then
+               echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
+               source "$WORKSPACE/kind-voltha/releases/${branch}"
+             else
+               echo "on master, using default settings for kind-voltha"
+             fi
+             cd $WORKSPACE/kind-voltha/
+             WAIT_ON_DOWN=y DEPLOY_K8S=n ./voltha down
 
-           export EXTRA_HELM_FLAGS=""
-           if [ "${branch}" != "master" ]; then
-             echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
-             source "$WORKSPACE/kind-voltha/releases/${branch}"
-           else
-             echo "on master, using default settings for kind-voltha"
-           fi
-           export EXTRA_HELM_FLAGS+="--set log_agent.enabled=False ${extraHelmFlags} "
+             export EXTRA_HELM_FLAGS=""
+             if [ "${branch}" != "master" ]; then
+               echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
+               source "$WORKSPACE/kind-voltha/releases/${branch}"
+             else
+               echo "on master, using default settings for kind-voltha"
+             fi
+             export EXTRA_HELM_FLAGS+="--set log_agent.enabled=False ${extraHelmFlags} "
 
-           # Workflow-specific flags
-           export WITH_RADIUS=no
-           export WITH_EAPOL=no
-           export WITH_DHCP=yes
-           export WITH_IGMP=yes
-           export CONFIG_SADIS="external"
-           export BBSIM_CFG="configs/bbsim-sadis-tt.yaml"
+             # Workflow-specific flags
+             export WITH_RADIUS=no
+             export WITH_EAPOL=no
+             export WITH_DHCP=yes
+             export WITH_IGMP=yes
+             export CONFIG_SADIS="external"
+             export BBSIM_CFG="configs/bbsim-sadis-tt.yaml"
 
-           # start logging
-           mkdir -p $WORKSPACE/reconcilett
-           _TAG=kail-reconcile-tt kail -n voltha -n default > $WORKSPACE/reconcilett/onos-voltha-combined.log &
+             # start logging
+             mkdir -p $WORKSPACE/reconcilett
+             _TAG=kail-reconcile-tt kail -n voltha -n default > $WORKSPACE/reconcilett/onos-voltha-combined.log &
 
-           DEPLOY_K8S=n ./voltha up
+             DEPLOY_K8S=n ./voltha up
 
-           mkdir -p $ROBOT_LOGS_DIR
-           export ROBOT_MISC_ARGS="-d $ROBOT_LOGS_DIR -e PowerSwitch"
+             mkdir -p $ROBOT_LOGS_DIR
+             export ROBOT_MISC_ARGS="-d $ROBOT_LOGS_DIR -e PowerSwitch"
 
-           make -C $WORKSPACE/voltha-system-tests ${makeReconcileTtTestTarget} || true
+             make -C $WORKSPACE/voltha-system-tests ${makeReconcileTtTestTarget} || true
 
-           # stop logging
-           P_IDS="$(ps e -ww -A | grep "_TAG=kail-reconcile-tt" | grep -v grep | awk '{print $1}')"
-           if [ -n "$P_IDS" ]; then
-             echo $P_IDS
-             for P_ID in $P_IDS; do
-               kill -9 $P_ID
-             done
-           fi
+             # stop logging
+             P_IDS="$(ps e -ww -A | grep "_TAG=kail-reconcile-tt" | grep -v grep | awk '{print $1}')"
+             if [ -n "$P_IDS" ]; then
+               echo $P_IDS
+               for P_ID in $P_IDS; do
+                 kill -9 $P_ID
+               done
+             fi
 
-           # get pods information
-           kubectl get pods -o wide --all-namespaces > $WORKSPACE/reconcilett/pods.txt || true
-           '''
+             # get pods information
+             kubectl get pods -o wide --all-namespaces > $WORKSPACE/reconcilett/pods.txt || true
+             '''
+           }
       }
     }
   }
diff --git a/jjb/pipeline/voltha/voltha-2.7/voltha-physical-build-and-tests.groovy b/jjb/pipeline/voltha/voltha-2.7/voltha-physical-build-and-tests.groovy
index e719a08..f0b6787 100644
--- a/jjb/pipeline/voltha/voltha-2.7/voltha-physical-build-and-tests.groovy
+++ b/jjb/pipeline/voltha/voltha-2.7/voltha-physical-build-and-tests.groovy
@@ -63,6 +63,12 @@
     stage ('Initialize') {
       steps {
         sh returnStdout: false, script: """
+        if [ "${branch}" != "master" ]; then
+          echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
+          source "$WORKSPACE/kind-voltha/releases/${branch}"
+        else
+          echo "on master, using default settings for kind-voltha"
+        fi
         test -e $WORKSPACE/kind-voltha/voltha && cd $WORKSPACE/kind-voltha && ./voltha down
         cd $WORKSPACE
         rm -rf $WORKSPACE/*
diff --git a/jjb/pipeline/voltha/voltha-2.7/voltha-system-test-bbsim.groovy b/jjb/pipeline/voltha/voltha-2.7/voltha-system-test-bbsim.groovy
index 991ecee..aa08ecb 100644
--- a/jjb/pipeline/voltha/voltha-2.7/voltha-system-test-bbsim.groovy
+++ b/jjb/pipeline/voltha/voltha-2.7/voltha-system-test-bbsim.groovy
@@ -47,12 +47,20 @@
   stages {
     stage('Create Kubernetes Cluster') {
       steps {
-        sh """
-           git clone https://gerrit.opencord.org/kind-voltha
-           pushd kind-voltha/
-           JUST_K8S=y ./voltha up
-           popd
-           """
+        timeout(time: 10, unit: 'MINUTES') {
+          sh """
+             git clone https://gerrit.opencord.org/kind-voltha
+             pushd kind-voltha/
+             if [ "${branch}" != "master" ]; then
+               echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
+               source "$WORKSPACE/kind-voltha/releases/${branch}"
+             else
+               echo "on master, using default settings for kind-voltha"
+             fi
+             JUST_K8S=y ./voltha up
+             popd
+             """
+         }
       }
     }
 
@@ -67,48 +75,56 @@
 
     stage('Deploy Voltha') {
       steps {
-        sh """
-           export EXTRA_HELM_FLAGS=""
-           if [ "${branch}" != "master" ]; then
-             echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
-             source "$WORKSPACE/kind-voltha/releases/${branch}"
-           else
-             echo "on master, using default settings for kind-voltha"
-           fi
+        timeout(time: 10, unit: 'MINUTES') {
+          sh """
+             export EXTRA_HELM_FLAGS=""
+             if [ "${branch}" != "master" ]; then
+               echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
+               source "$WORKSPACE/kind-voltha/releases/${branch}"
+             else
+               echo "on master, using default settings for kind-voltha"
+             fi
 
-           EXTRA_HELM_FLAGS+="${params.extraHelmFlags} "
-           echo \$EXTRA_HELM_FLAGS
+             EXTRA_HELM_FLAGS+="${params.extraHelmFlags} "
+             echo \$EXTRA_HELM_FLAGS
 
-           pushd kind-voltha/
-           ./voltha up
-           popd
-           """
+             pushd kind-voltha/
+             ./voltha up
+             popd
+             """
+           }
       }
     }
 
     stage('Run E2E Tests') {
       steps {
-        sh '''
-           rm -rf $WORKSPACE/RobotLogs; mkdir -p $WORKSPACE/RobotLogs
-           git clone -b ${branch} https://gerrit.opencord.org/voltha-system-tests
-           make ROBOT_DEBUG_LOG_OPT="-l sanity_log.html -r sanity_report.html -o sanity_output.xml" -C $WORKSPACE/voltha-system-tests ${makeTarget}
-           '''
+        timeout(time: 10, unit: 'MINUTES') {
+          sh '''
+             rm -rf $WORKSPACE/RobotLogs; mkdir -p $WORKSPACE/RobotLogs
+             git clone -b ${branch} https://gerrit.opencord.org/voltha-system-tests
+             make ROBOT_DEBUG_LOG_OPT="-l sanity_log.html -r sanity_report.html -o sanity_output.xml" -C $WORKSPACE/voltha-system-tests ${makeTarget}
+             '''
+         }
       }
     }
 
     stage('Kubernetes ETCD Scale Test') {
       steps {
-        sh '''
-           make ROBOT_DEBUG_LOG_OPT="-l functional_log.html -r functional_report.html -o functional_output.xml" -C $WORKSPACE/voltha-system-tests system-scale-test
-           '''
+        timeout(time: 10, unit: 'MINUTES') {
+          sh '''
+             make ROBOT_DEBUG_LOG_OPT="-l functional_log.html -r functional_report.html -o functional_output.xml" -C $WORKSPACE/voltha-system-tests system-scale-test
+             '''
+         }
       }
     }
 
     stage('Kubernetes ETCD Failure Test') {
       steps {
-        sh '''
-           make ROBOT_DEBUG_LOG_OPT="-l failure_log.html -r failure_report.html -o failure_output.xml"  -C $WORKSPACE/voltha-system-tests failure-test
-           '''
+        timeout(time: 10, unit: 'MINUTES') {
+          sh '''
+             make ROBOT_DEBUG_LOG_OPT="-l failure_log.html -r failure_report.html -o failure_output.xml"  -C $WORKSPACE/voltha-system-tests failure-test
+             '''
+         }
       }
     }
 
@@ -149,6 +165,12 @@
          extract_errors_python voltha-ofagent >> $WORKSPACE/error-report.log
 
          ## shut down kind-voltha
+         if [ "${branch}" != "master" ]; then
+           echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
+           source "$WORKSPACE/kind-voltha/releases/${branch}"
+         else
+           echo "on master, using default settings for kind-voltha"
+         fi
          cd $WORKSPACE/kind-voltha
 	       WAIT_ON_DOWN=y ./voltha down