diff --git a/jjb/cord-test/voltha.yaml b/jjb/cord-test/voltha.yaml
index 60ef3cf..4c56478 100644
--- a/jjb/cord-test/voltha.yaml
+++ b/jjb/cord-test/voltha.yaml
@@ -10,7 +10,7 @@
 
     with-kind: false
     power-switch: False
-    work-flow: ''
+    work-flow: 'ATT'
     in-band-management: false
     num-of-openonu: '1'
     num-of-onos: '1'
diff --git a/jjb/pipeline/voltha-tt-physical-functional-tests.groovy b/jjb/pipeline/voltha-tt-physical-functional-tests.groovy
index d286789..a7f97ca 100644
--- a/jjb/pipeline/voltha-tt-physical-functional-tests.groovy
+++ b/jjb/pipeline/voltha-tt-physical-functional-tests.groovy
@@ -66,6 +66,14 @@
             [$class: 'CloneOption', depth: 0, noTags: false, reference: '', shallow: false],
           ],
         ])
+        script {
+          sh(script:"""
+            if [ '${volthaSystemTestsChange}' != '' ] ; then
+              cd $WORKSPACE/voltha-system-tests;
+              git fetch https://gerrit.opencord.org/voltha-system-tests ${volthaSystemTestsChange} && git checkout FETCH_HEAD
+            fi
+            """)
+        }
       }
     }
    stage('Clone cord-tester') {
@@ -178,6 +186,26 @@
         """
       }
     }
+
+    stage('Failure/Recovery Tests') {
+      environment {
+        ROBOT_CONFIG_FILE="$WORKSPACE/${configBaseDir}/${configDeploymentDir}/${configFileName}-TT.yaml"
+        ROBOT_FILE="Voltha_TT_FailureScenarios.robot"
+        ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/tt-workflow/FailureScenarios"
+      }
+      steps {
+        sh """
+        mkdir -p $ROBOT_LOGS_DIR
+        if ( ${powerSwitch} ); then
+             export ROBOT_MISC_ARGS="--removekeywords wuks -L TRACE -i functionalTT -i PowerSwitch -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
+        else
+             export ROBOT_MISC_ARGS="--removekeywords wuks -L TRACE -i functionalTT -e PowerSwitch -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
+        fi
+        make -C $WORKSPACE/voltha-system-tests voltha-tt-test || true
+        """
+      }
+    }
+
   }
   post {
     always {
diff --git a/jjb/pipeline/voltha/master/software-upgrades.groovy b/jjb/pipeline/voltha/master/software-upgrades.groovy
new file mode 100644
index 0000000..8a7d0ee
--- /dev/null
+++ b/jjb/pipeline/voltha/master/software-upgrades.groovy
@@ -0,0 +1,143 @@
+// Copyright 2021-present Open Networking Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// voltha-2.x e2e tests
+// uses bbsim to simulate OLT/ONUs
+// NOTE we are importing the library even if it's global so that it's
+// easier to change the keywords during a replay
+library identifier: 'cord-jenkins-libraries@master',
+    retriever: modernSCM([
+      $class: 'GitSCMSource',
+      remote: 'https://gerrit.opencord.org/ci-management.git'
+])
+def test_software_upgrade(name) {
+  stage('Deploy Voltha - '+ name) {
+      def extraHelmFlags = "${extraHelmFlags} --set global.log_level=DEBUG,onu=1,pon=1 "
+      // TODO: ONOS custom image handling
+      // if [ '${onosImg.trim()}' != '' ] && [ '\$GERRIT_PROJECT' != 'voltha-onos' ]; then
+      //   IFS=: read -r onosRepo onosTag <<< '${onosImg.trim()}'
+      //   extraHelmFlags = extraHelmFlags + "--set onos-classic.images.onos.repository=\$onosRepo,onos-classic.images.onos.tag=\$onosTag "
+      // fi
+      def localCharts = false
+      // Currently only testing with ATT workflow
+      // TODO: Support for other workflows
+      volthaDeploy([workflow: "att", extraHelmFlags: extraHelmFlags, localCharts: localCharts])
+      // start logging
+      sh """
+      mkdir -p $WORKSPACE/${name}
+      _TAG=kail-${name} kail -n infra -n voltha > $WORKSPACE/${name}/onos-voltha-combined.log &
+      """
+      // forward ONOS and VOLTHA ports
+      sh """
+      _TAG=onos-port-forward kubectl port-forward --address 0.0.0.0 -n infra svc/voltha-infra-onos-classic-hs 8101:8101&
+      _TAG=onos-port-forward kubectl port-forward --address 0.0.0.0 -n infra svc/voltha-infra-onos-classic-hs 8181:8181&
+      _TAG=voltha-port-forward kubectl port-forward --address 0.0.0.0 -n voltha svc/voltha-voltha-api 55555:55555&
+      """
+  }
+  stage('Test - '+ name) {
+      sh """
+        ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/${name}"
+        mkdir -p \$ROBOT_LOGS_DIR
+        if [[ ${name} == 'onos-app-upgrade' ]]; then
+          export ONOS_APPS_UNDER_TEST+=''
+          if [[ ${aaaVer.trim()} != '' && ${aaaOarUrl.trim()} != '' ]]; then
+            ONOS_APPS_UNDER_TEST+="org.opencord.aaa,${aaaVer.trim()},${aaaOarUrl.trim()}"
+          fi
+          export ROBOT_MISC_ARGS="-d \$ROBOT_LOGS_DIR -v onos_apps_under_test:\$ONOS_APPS_UNDER_TEST -e PowerSwitch"
+          export TARGET=onos-app-upgrade-test
+        fi
+        if [[ ${name} == 'voltha-component-upgrade' ]]; then
+          export VOLTHA_COMPS_UNDER_TEST+=''
+          if [[ ${adapterOpenOltImage.trim()} != '' ]]; then
+            VOLTHA_COMPS_UNDER_TEST+="adapter-open-olt,adapter-open-olt,${adapterOpenOltImage.trim()}"
+          fi
+          export ROBOT_MISC_ARGS="-d \$ROBOT_LOGS_DIR -v voltha_comps_under_test:\$VOLTHA_COMPS_UNDER_TEST -e PowerSwitch"
+          export TARGET=voltha-comp-upgrade-test
+        fi
+        export VOLTCONFIG=$HOME/.volt/config-minimal
+        export KUBECONFIG=$HOME/.kube/kind-config-voltha-minimal
+        # Run the specified tests
+        make -C $WORKSPACE/voltha-system-tests \$TARGET || true
+      """
+      // stop logging
+      sh """
+        P_IDS="\$(ps e -ww -A | grep "_TAG=kail-${name}" | grep -v grep | awk '{print \$1}')"
+        if [ -n "\$P_IDS" ]; then
+          echo \$P_IDS
+          for P_ID in \$P_IDS; do
+            kill -9 \$P_ID
+          done
+        fi
+      """
+      // remove port-forwarding
+      sh """
+        # remove orphaned port-forward from different namespaces
+        ps aux | grep port-forw | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9
+      """
+      // collect pod details
+      sh """
+      kubectl get pods --all-namespaces -o wide > \$WORKSPACE/${name}/pods.txt || true
+      kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\n'}" | sort | uniq | tee \$WORKSPACE/${name}/pod-images.txt || true
+      kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.imageID}{'\\n'}" | sort | uniq | tee \$WORKSPACE/${name}/pod-imagesId.txt || true
+      """
+      helmTeardown(['infra', 'voltha'])
+  }
+}
+pipeline {
+  /* no label, executor is determined by JJB */
+  agent {
+    label "${params.buildNode}"
+  }
+  options {
+    timeout(time: 30, unit: 'MINUTES')
+  }
+  environment {
+    PATH="$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin"
+    KUBECONFIG="$HOME/.kube/kind-config-voltha-minimal"
+  }
+  stages{
+    stage('Download Code') {
+      steps {
+        getVolthaCode([
+          branch: "${branch}",
+          volthaSystemTestsChange: "${volthaSystemTestsChange}",
+          volthaHelmChartsChange: "${volthaHelmChartsChange}",
+        ])
+      }
+    }
+    stage('Run Test') {
+      steps {
+        test_software_upgrade("onos-app-upgrade")
+        test_software_upgrade("voltha-component-upgrade")
+      }
+    }
+  }
+  post {
+    always {
+      sh '''
+      gzip $WORKSPACE/onos-app-upgrade/onos-voltha-combined.log || true
+      gzip $WORKSPACE/voltha-component-upgrade/onos-voltha-combined.log || true
+      '''
+      step([$class: 'RobotPublisher',
+         disableArchiveOutput: false,
+         logFileName: 'RobotLogs/*/log*.html',
+         otherFiles: '',
+         outputFileName: 'RobotLogs/*/output*.xml',
+         outputPath: '.',
+         passThreshold: 100,
+         reportFileName: 'RobotLogs/*/report*.html',
+         unstableThreshold: 0]);
+      archiveArtifacts artifacts: '*.log,**/*.log,**/*.gz,*.gz'
+    }
+  }
+}
diff --git a/jjb/pipeline/voltha/master/voltha-scale-test.groovy b/jjb/pipeline/voltha/master/voltha-scale-test.groovy
index e011257..ffd3fcd 100644
--- a/jjb/pipeline/voltha/master/voltha-scale-test.groovy
+++ b/jjb/pipeline/voltha/master/voltha-scale-test.groovy
@@ -238,7 +238,7 @@
               # ONOS custom image handling
               if [ '${onosImg.trim()}' != '' ] && [ '\$GERRIT_PROJECT' != 'voltha-onos' ]; then
                 IFS=: read -r onosRepo onosTag <<< '${onosImg.trim()}'
-                EXTRA_HELM_FLAGS+="--set onos-classic.images.onos.repository=\$onosRepo,onos-classic.images.onos.tag=\$onosTag "
+                EXTRA_HELM_FLAGS+="--set onos-classic.image.repository=\$onosRepo,onos-classic.image.tag=\$onosTag "
               fi
 
               # set BBSim parameters
@@ -278,7 +278,7 @@
               fi
 
               if [ '\$GERRIT_PROJECT' == 'voltha-onos' ]; then
-                EXTRA_HELM_FLAGS+="--set onos-classic.images.onos.repository=${dockerRegistry}/voltha/voltha-onos,onos-classic.images.onos.tag=voltha-scale "
+                EXTRA_HELM_FLAGS+="--set onos-classic.image.repository=${dockerRegistry}/voltha/voltha-onos,onos-classic.image.tag=voltha-scale "
               fi
 
               if [ '\$GERRIT_PROJECT' == 'bbsim' ]; then
@@ -320,13 +320,18 @@
             sh """
               set +x
 
-              echo -ne "\nWaiting for VOLTHA to start..."
+              echo -ne "\nWaiting for VOLTHA and ONOS to start..."
               voltha=\$(kubectl get pods --all-namespaces -l app.kubernetes.io/part-of=voltha --no-headers | grep "0/" | wc -l)
-              while [[ \$voltha != 0 ]]; do
+              onos=\$(kubectl get pods --all-namespaces -l app=onos-classic --no-headers | grep "0/" | wc -l)
+              while [[ \$voltha != 0 || \$onos != 0 ]]; do
                 sleep 5
                 echo -ne "."
                 voltha=\$(kubectl get pods --all-namespaces -l app.kubernetes.io/part-of=voltha --no-headers | grep "0/" | wc -l)
+                onos=\$(kubectl get pods --all-namespaces -l app=onos-classic --no-headers | grep "0/" | wc -l)
               done
+              echo -ne "\nVOLTHA and ONOS pods ready\n"
+              kubectl get pods --all-namespaces -l app.kubernetes.io/part-of=voltha --no-headers | grep "0/" | wc -l
+              kubectl get pods --all-namespaces -l app=onos-classic --no-headers | grep "0/" | wc -l
             """
             start_port_forward(olts)
           }
@@ -360,8 +365,8 @@
 
           sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager accumulatorMaxIdleMillis 500
 
-          sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg log:set ${logLevel} org.onosproject
-          sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg log:set ${logLevel} org.opencord
+          sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 log:set ${logLevel} org.onosproject
+          sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 log:set ${logLevel} org.opencord
 
 
           kubectl exec \$(kubectl get pods | grep -E "bbsim[0-9]" | awk 'NR==1{print \$1}') -- bbsimctl log ${logLevel.toLowerCase()} false
diff --git a/jjb/software-upgrades.yaml b/jjb/software-upgrades.yaml
new file mode 100644
index 0000000..4c3a4e1
--- /dev/null
+++ b/jjb/software-upgrades.yaml
@@ -0,0 +1,106 @@
+---
+# voltha 2.X tests
+
+- project:
+    name: software-upgrades
+    project-name: '{name}'
+
+    jobs:
+      - 'software-upgrades-test':
+          name: 'periodic-software-upgrade-test-bbsim'
+          pipeline-script: 'voltha/master/software-upgrades.groovy'
+          build-node: 'qct-pod4-node2'
+          code-branch: 'master'
+          aaa-version: '2.3.0.SNAPSHOT'
+          aaa-oar-url: 'https://oss.sonatype.org/content/groups/public/org/opencord/aaa-app/2.3.0-SNAPSHOT/aaa-app-2.3.0-20201210.223737-1.oar'
+          adapter-open-olt-image: 'voltha/voltha-openolt-adapter:3.1.3'
+          time-trigger: "H H/23 * * *"
+
+- job-template:
+    id: 'software-upgrades-test'
+    name: '{name}'
+    sandbox: true
+    volthaSystemTestsChange: ''
+    volthaHelmChartsChange: ''
+    kindVolthaChange: ''
+
+    description: |
+      <!-- Managed by Jenkins Job Builder -->
+      Created by {id} job-template from ci-management/jjb/software-upgrades.yaml  <br /><br />
+      E2E Validation for Voltha 2.X
+    properties:
+      - cord-infra-properties:
+          build-days-to-keep: '{build-days-to-keep}'
+          artifact-num-to-keep: '{artifact-num-to-keep}'
+
+    wrappers:
+      - lf-infra-wrappers:
+          build-timeout: '{build-timeout}'
+          jenkins-ssh-credential: '{jenkins-ssh-credential}'
+
+    parameters:
+      - string:
+          name: buildNode
+          default: '{build-node}'
+          description: 'Name of the Jenkins node to run the job on'
+
+      - string:
+          name: extraHelmFlags
+          default: ''
+          description: 'Helm flags to pass to ./voltha up'
+
+      - string:
+          name: volthaSystemTestsChange
+          default: ''
+          description: 'Download a change for gerrit in the voltha-system-tests repo, example value: "refs/changes/79/18779/13"'
+
+      - string:
+          name: volthaHelmChartsChange
+          default: ''
+          description: 'Download a change for gerrit in the voltha-helm-charts repo, example value: "refs/changes/79/18779/13"'
+
+      - string:
+          name: branch
+          default: '{code-branch}'
+          description: 'Name of the branch to use'
+
+      # deprecated params (not used in master, remove after 2.6 support is dropped)
+      - string:
+          name: kindVolthaChange
+          default: ''
+          description: 'Download a change for gerrit in the kind-voltha repo, example value: "refs/changes/32/19132/1"'
+
+      - string:
+          name: onosImg
+          default: ''
+          description: 'ONOS Image to use'
+
+      - string:
+          name: aaaVer
+          default: '{aaa-version}'
+          description: 'ONOS AAA App Version to Test Upgrade'
+
+      - string:
+          name: aaaOarUrl
+          default: '{aaa-oar-url}'
+          description: 'ONOS AAA App OAR File Url'
+
+      # TODO: Add other ONOS Apps
+
+      - string:
+          name: adapterOpenOltImage
+          default: '{adapter-open-olt-image}'
+          description: 'Voltha Adapter Open OLT Component Image'
+
+      # TODO: Add other Voltha Components
+
+    project-type: pipeline
+    concurrent: true
+
+    dsl: !include-raw-escape: pipeline/{pipeline-script}
+
+    triggers:
+      - timed: |
+                 TZ=America/Los_Angeles
+                 {time-trigger}
+