Merge "Add jobs for the pppoeagent repo"
diff --git a/jjb/omec-ci.yaml b/jjb/omec-ci.yaml
index c3097da..8f468b4 100644
--- a/jjb/omec-ci.yaml
+++ b/jjb/omec-ci.yaml
@@ -988,7 +988,12 @@
       - string:
           name: logSince
           default: '{log-since}'
-          description: 'Only upload logs newer than a relative duration e.g. 1h'
+          description: 'Only upload logs newer than a relative duration e.g. 1h. Leave empty if logSinceTime is specified'
+
+      - string:
+          name: logSinceTime
+          default: ''
+          description: 'Only upload logs newer than a date (RFC3339). Leave empty if logSince is specified'
 
     concurrent: false
 
diff --git a/jjb/pipeline/bbsim-scale.groovy b/jjb/pipeline/bbsim-scale.groovy
index 5af7063..a24d455 100644
--- a/jjb/pipeline/bbsim-scale.groovy
+++ b/jjb/pipeline/bbsim-scale.groovy
@@ -13,7 +13,7 @@
 
   environment {
       VOLTHA_LOG_LEVEL="DEBUG"
-      TYPE="minimal"
+      NAME="minimal"
       WITH_RADIUS="y"
       WITH_BBSIM="y"
       INSTALL_ONOS_APPS="y"
diff --git a/jjb/pipeline/device-management-mock-tests.groovy b/jjb/pipeline/device-management-mock-tests.groovy
index 6884144..ac575a5 100644
--- a/jjb/pipeline/device-management-mock-tests.groovy
+++ b/jjb/pipeline/device-management-mock-tests.groovy
@@ -29,7 +29,7 @@
     KUBECONFIG="$HOME/.kube/kind-config-voltha-minimal"
     VOLTCONFIG="$HOME/.volt/config-minimal"
     PATH="$WORKSPACE/voltha/kind-voltha/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-    TYPE="minimal"
+    NAME="minimal"
     FANCY=0
     WITH_SIM_ADAPTERS="n"
     WITH_RADIUS="y"
@@ -111,7 +111,7 @@
       steps {
         sh '''
              docker images | grep citest
-             for image in \$(docker images -f "reference=*/*citest" --format "{{.Repository}}"); do echo "Pushing \$image to nodes"; kind load docker-image \$image:citest --name voltha-\$TYPE --nodes voltha-\$TYPE-worker,voltha-\$TYPE-worker2; done
+             for image in \$(docker images -f "reference=*/*citest" --format "{{.Repository}}"); do echo "Pushing \$image to nodes"; kind load docker-image \$image:citest --name voltha-\$NAME --nodes voltha-\$NAME-worker,voltha-\$NAME-worker2; done
            '''
       }
     }
diff --git a/jjb/pipeline/device-management-physical-build-and-tests.groovy b/jjb/pipeline/device-management-physical-build-and-tests.groovy
index 6c183ff..6579cc4 100644
--- a/jjb/pipeline/device-management-physical-build-and-tests.groovy
+++ b/jjb/pipeline/device-management-physical-build-and-tests.groovy
@@ -35,7 +35,7 @@
     KUBECONFIG="$HOME/.kube/kind-config-voltha-minimal"
     VOLTCONFIG="$HOME/.volt/config-minimal"
     PATH="$WORKSPACE/voltha/kind-voltha/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-    TYPE="minimal"
+    NAME="minimal"
     FANCY=0
     //VOL-2194 ONOS SSH and REST ports hardcoded to 30115/30120 in tests
     ONOS_SSH_PORT=30115
@@ -166,7 +166,7 @@
       steps {
         sh '''
              docker images | grep citest
-             for image in \$(docker images -f "reference=*/*citest" --format "{{.Repository}}"); do echo "Pushing \$image to nodes"; kind load docker-image \$image:citest --name voltha-\$TYPE --nodes voltha-\$TYPE-worker,voltha-\$TYPE-worker2; done
+             for image in \$(docker images -f "reference=*/*citest" --format "{{.Repository}}"); do echo "Pushing \$image to nodes"; kind load docker-image \$image:citest --name voltha-\$NAME --nodes voltha-\$NAME-worker,voltha-\$NAME-worker2; done
            '''
       }
     }
diff --git a/jjb/pipeline/voltha-dt-physical-build-and-tests.groovy b/jjb/pipeline/voltha-dt-physical-build-and-tests.groovy
index da7bb3f..ef38bc6 100644
--- a/jjb/pipeline/voltha-dt-physical-build-and-tests.groovy
+++ b/jjb/pipeline/voltha-dt-physical-build-and-tests.groovy
@@ -44,7 +44,7 @@
     KUBECONFIG="$HOME/.kube/kind-config-voltha-minimal"
     VOLTCONFIG="$HOME/.volt/config-minimal"
     PATH="$WORKSPACE/voltha/kind-voltha/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-    TYPE="minimal"
+    NAME="minimal"
     FANCY=0
     //VOL-2194 ONOS SSH and REST ports hardcoded to 30115/30120 in tests
     ONOS_SSH_PORT=30115
diff --git a/jjb/pipeline/voltha-dt-physical-functional-tests.groovy b/jjb/pipeline/voltha-dt-physical-functional-tests.groovy
index c1f16c0..566d245 100644
--- a/jjb/pipeline/voltha-dt-physical-functional-tests.groovy
+++ b/jjb/pipeline/voltha-dt-physical-functional-tests.groovy
@@ -227,6 +227,26 @@
       }
     }
 
+    stage('Multiple OLT Tests') {
+      environment {
+        ROBOT_CONFIG_FILE="$WORKSPACE/${configBaseDir}/${configDeploymentDir}/${configFileName}-DT.yaml"
+        ROBOT_FILE="Voltha_DT_MultiOLT_Tests.robot"
+        ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/dt-workflow/MultipleOLTScenarios"
+      }
+      steps {
+        sh """
+        mkdir -p $ROBOT_LOGS_DIR
+        if ( ${powerSwitch} ); then
+             export ROBOT_MISC_ARGS="--removekeywords wuks -L TRACE -i functionalDt -i PowerSwitch -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE"
+        else
+             export ROBOT_MISC_ARGS="--removekeywords wuks -L TRACE -i functionalDt -e PowerSwitch -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE"
+        fi
+        make -C $WORKSPACE/voltha-system-tests voltha-dt-test || true
+        """
+      }
+    }
+
+
     stage('Error Scenario Tests') {
       environment {
         ROBOT_CONFIG_FILE="$WORKSPACE/${configBaseDir}/${configDeploymentDir}/${configFileName}-DT.yaml"
diff --git a/jjb/pipeline/voltha-go-multi-tests.groovy b/jjb/pipeline/voltha-go-multi-tests.groovy
index 60d0ab9..984d4cb 100644
--- a/jjb/pipeline/voltha-go-multi-tests.groovy
+++ b/jjb/pipeline/voltha-go-multi-tests.groovy
@@ -29,7 +29,7 @@
     KUBECONFIG="$HOME/.kube/kind-config-voltha-minimal"
     VOLTCONFIG="$HOME/.volt/config-minimal"
     PATH="$WORKSPACE/kind-voltha/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-    TYPE="minimal"
+    NAME="minimal"
     FANCY=0
     WITH_SIM_ADAPTERS="n"
     WITH_RADIUS="y"
diff --git a/jjb/pipeline/voltha-go-tests.groovy b/jjb/pipeline/voltha-go-tests.groovy
index bafca75..ab68755 100644
--- a/jjb/pipeline/voltha-go-tests.groovy
+++ b/jjb/pipeline/voltha-go-tests.groovy
@@ -29,7 +29,7 @@
     KUBECONFIG="$HOME/.kube/kind-config-voltha-minimal"
     VOLTCONFIG="$HOME/.volt/config-minimal"
     PATH="$HOME/kind-voltha/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-    TYPE="minimal"
+    NAME="minimal"
     FANCY=0
     WITH_SIM_ADAPTERS="no"
     WITH_RADIUS="yes"
diff --git a/jjb/pipeline/voltha-nightly-tests-bbsim.groovy b/jjb/pipeline/voltha-nightly-tests-bbsim.groovy
index 48af5c5..24c015b 100644
--- a/jjb/pipeline/voltha-nightly-tests-bbsim.groovy
+++ b/jjb/pipeline/voltha-nightly-tests-bbsim.groovy
@@ -29,7 +29,7 @@
     KUBECONFIG="$HOME/.kube/kind-config-voltha-minimal"
     VOLTCONFIG="$HOME/.volt/config-minimal"
     PATH="$PATH:$WORKSPACE/kind-voltha/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-    TYPE="minimal"
+    NAME="minimal"
     FANCY=0
     WITH_SIM_ADAPTERS="no"
     WITH_RADIUS="yes"
@@ -159,6 +159,21 @@
       }
     }
 
+    stage('Multiple OLT Tests') {
+      environment {
+        ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/MultipleOLTTests"
+      }
+      steps {
+        sh '''
+           set +e
+           mkdir -p $WORKSPACE/RobotLogs
+
+           export ROBOT_MISC_ARGS="-d $ROBOT_LOGS_DIR"
+           make -C $WORKSPACE/voltha-system-tests ${makeMultiOltTarget} || true
+           '''
+      }
+    }
+
     stage('Error Tests') {
       environment {
         ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/ErrorTests"
diff --git a/jjb/pipeline/voltha-openonu-go-tests.groovy b/jjb/pipeline/voltha-openonu-go-tests.groovy
index 2c5bfbb..19e57ef 100755
--- a/jjb/pipeline/voltha-openonu-go-tests.groovy
+++ b/jjb/pipeline/voltha-openonu-go-tests.groovy
@@ -29,7 +29,7 @@
     KUBECONFIG="$HOME/.kube/kind-config-voltha-minimal"
     VOLTCONFIG="$HOME/.volt/config-minimal"
     PATH="$WORKSPACE/kind-voltha/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-    TYPE="minimal"
+    NAME="minimal"
     FANCY=0
     WITH_SIM_ADAPTERS="n"
     WITH_RADIUS="y"
@@ -155,7 +155,7 @@
       steps {
         sh '''
            docker images | grep citest
-           for image in \$(docker images -f "reference=*/*citest" --format "{{.Repository}}"); do echo "Pushing \$image to nodes"; kind load docker-image \$image:citest --name voltha-\$TYPE --nodes voltha-\$TYPE-worker,voltha-\$TYPE-worker2; done
+           for image in \$(docker images -f "reference=*/*citest" --format "{{.Repository}}"); do echo "Pushing \$image to nodes"; kind load docker-image \$image:citest --name voltha-\$NAME --nodes voltha-\$NAME-worker,voltha-\$NAME-worker2; done
            '''
       }
     }
@@ -332,6 +332,54 @@
            '''
       }
     }
+
+    stage('TT workflow') {
+      environment {
+        ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/TTWorkflow"
+      }
+      steps {
+        sh '''
+           cd $WORKSPACE/kind-voltha/
+           WAIT_ON_DOWN=y DEPLOY_K8S=n ./voltha down
+
+           export EXTRA_HELM_FLAGS+="--set use_openonu_adapter_go=true,log_agent.enabled=False ${extraHelmFlags} "
+
+           IMAGES="adapter_open_onu_go"
+
+           for I in \$IMAGES
+           do
+             EXTRA_HELM_FLAGS+="--set images.\$I.tag=citest,images.\$I.pullPolicy=Never "
+           done
+
+           # Workflow-specific flags
+           export WITH_RADIUS=no
+           export WITH_EAPOL=no
+           export WITH_DHCP=yes
+           export WITH_IGMP=yes
+           export CONFIG_SADIS="external"
+           export BBSIM_CFG="configs/bbsim-sadis-tt.yaml"
+
+           DEPLOY_K8S=n ./voltha up
+
+           mkdir -p $ROBOT_LOGS_DIR
+           export ROBOT_MISC_ARGS="-d $ROBOT_LOGS_DIR -e PowerSwitch"
+
+           # By default, all tests tagged 'sanityTt' are run.  This covers basic functionality
+           # like running through the TT workflow for a single subscriber.
+           export TARGET=sanity-kind-tt
+
+           # If the Gerrit comment contains a line with "functional tests" then run the full
+           # functional test suite.  This covers tests tagged either 'sanityTt' or 'functionalTt'.
+           # Note: Gerrit comment text will be prefixed by "Patch set n:" and a blank line
+           REGEX="functional tests"
+           if [[ "$GERRIT_EVENT_COMMENT_TEXT" =~ \$REGEX ]]; then
+             TARGET=functional-single-kind-tt
+           fi
+
+           make -C $WORKSPACE/voltha-system-tests \$TARGET || true
+           '''
+      }
+    }
   }
   post {
     always {
diff --git a/jjb/pipeline/voltha-physical-build-and-tests.groovy b/jjb/pipeline/voltha-physical-build-and-tests.groovy
index 5f1bfff..a064b70 100644
--- a/jjb/pipeline/voltha-physical-build-and-tests.groovy
+++ b/jjb/pipeline/voltha-physical-build-and-tests.groovy
@@ -44,7 +44,7 @@
     KUBECONFIG="$HOME/.kube/kind-config-voltha-minimal"
     VOLTCONFIG="$HOME/.volt/config-minimal"
     PATH="$WORKSPACE/voltha/kind-voltha/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-    TYPE="minimal"
+    NAME="minimal"
     FANCY=0
     //VOL-2194 ONOS SSH and REST ports hardcoded to 30115/30120 in tests
     ONOS_SSH_PORT=30115
diff --git a/jjb/pipeline/voltha-scale-multi-stack.groovy b/jjb/pipeline/voltha-scale-multi-stack.groovy
index 1e00fc6..a7d3f71 100644
--- a/jjb/pipeline/voltha-scale-multi-stack.groovy
+++ b/jjb/pipeline/voltha-scale-multi-stack.groovy
@@ -40,11 +40,6 @@
     WITH_KAFKA="yes"
     WITH_ETCD="yes"
     VOLTHA_ETCD_PORT=9999
-
-    // VOLTHA namespaces are defined at runtime depending on the stack we're installing
-    // VOLTHA_NS="default"
-    // ADAPTER_NS="default"
-    // BBSIM_NS="default"
     INFRA_NS="infra"
 
     // configurable options
@@ -86,9 +81,6 @@
             helm repo add bbsim-sadis https://ciena.github.io/bbsim-sadis-server/charts
             helm repo update
 
-            # NOTE this is temporary, for now the bbsim-sadis-server service will be overridden and ONOS will use the new server
-            kubectl delete -n infra -f $HOME/bbsim-sadis-server/deployments/bbsim-sadis-server.yaml
-
             # removing ETCD port forward
             P_ID="\$(ps e -ww -A | grep "_TAG=etcd-port-forward" | grep -v grep | awk '{print \$1}')"
             if [ -n "\$P_ID" ]; then
@@ -107,8 +99,6 @@
 
             test -e $WORKSPACE/kind-voltha/voltha && cd $WORKSPACE/kind-voltha && ./voltha down
 
-            ps aux | grep port-forw | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9
-
             cd $WORKSPACE
             rm -rf $WORKSPACE/*
           """
@@ -244,10 +234,6 @@
         script {
           sh returnStdout: false, script: """
 
-          # NOTE this is temporary, for now the bbsim-sadis-server service will be overridden and ONOS will use the new server
-          helm del -n infra bbsim-sadis-server
-          kubectl apply -n infra -f $HOME/bbsim-sadis-server/deployments/bbsim-sadis-server.yaml
-
           # TODO this needs to be repeated per stack
           # kubectl exec \$(kubectl get pods | grep -E "bbsim[0-9]" | awk 'NR==1{print \$1}') -- bbsimctl log ${logLevel.toLowerCase()} false
 
@@ -276,7 +262,6 @@
           fi
 
           if [ ${withPcap} = true ] && [ ${volthaStacks} -eq 1 ] ; then
-            # FIXME ofagent pcap has to be replicated per stack
             # Start the tcp-dump in ofagent
             export OF_AGENT=\$(kubectl -n \$INFRA_NS get pods -l app=ofagent -o name)
             kubectl exec \$OF_AGENT -- apk update
@@ -313,7 +298,7 @@
           make vst_venv
         '''
         sh '''
-          if [ ${withProfiling} = true ] ; then
+          if [ ${withProfiling} = true ] && [ ${volthaStacks} -eq 1 ]; then
             mkdir -p $LOG_FOLDER/pprof
             cat << EOF > $WORKSPACE/pprof.sh
 timestamp() {
@@ -345,11 +330,10 @@
 
             _TAG="pprof"
             _TAG=$_TAG bash $WORKSPACE/pprof.sh &
+          else
+            echo "Profiling not supported for multiple VOLTHA stacks"
           fi
         '''
-        // bbsim-sadis server takes a while to cache the subscriber entries
-        // wait for that before starting the tests
-        sleep(60)
       }
     }
     stage('Run Test') {
@@ -379,7 +363,7 @@
           fi
         done
 
-        if [ ${withPcap} = true ] ; then
+        if [ ${withPcap} = true ] && [ ${volthaStacks} -eq 1 ]; then
           # stop ofAgent tcpdump
           P_ID="\$(ps e -ww -A | grep "_TAG=ofagent-tcpdump" | grep -v grep | awk '{print \$1}')"
           if [ -n "\$P_ID" ]; then
@@ -417,7 +401,7 @@
         fi
       '''
       sh '''
-        if [ ${withProfiling} = true ] ; then
+        if [ ${withProfiling} = true ] && [ ${volthaStacks} -eq 1 ]; then
           _TAG="pprof"
           P_IDS="$(ps e -ww -A | grep "_TAG=$_TAG" | grep -v grep | awk '{print $1}')"
           if [ -n "$P_IDS" ]; then
@@ -442,7 +426,7 @@
           [file: 'plots/plot-onos-flows-after.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],
           [file: 'plots/plot-onos-dhcp.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],
         ],
-        group: 'Voltha-Scale-Numbers', numBuilds: '20', style: 'line', title: "Scale Test (OLTs: ${olts}, PONs: ${pons}, ONUs: ${onus})", yaxis: 'Time (s)', useDescr: true
+        group: 'Voltha-Scale-Numbers', numBuilds: '20', style: 'line', title: "Scale Test (Stacks: ${params.volthaStacks}, OLTs: ${olts}, PONs: ${pons}, ONUs: ${onus})", yaxis: 'Time (s)', useDescr: true
       ])
       step([$class: 'RobotPublisher',
         disableArchiveOutput: false,
@@ -480,8 +464,8 @@
 
           for bbsim in "\${IDS[@]}"
           do
-            kubectl -n ${stack_ns} exec -t \$bbsim -- bbsimctl onu list > $LOG_FOLDER/\$bbsim-device-list.txt || true
-            kubectl -n ${stack_ns} exec -t \$bbsim -- bbsimctl service list > $LOG_FOLDER/\$bbsim-service-list.txt || true
+            kubectl -n ${stack_ns} exec -t \$bbsim -- bbsimctl onu list > $LOG_FOLDER/${stack_ns}/\$bbsim-device-list.txt || true
+            kubectl -n ${stack_ns} exec -t \$bbsim -- bbsimctl service list > $LOG_FOLDER/${stack_ns}/\$bbsim-service-list.txt || true
           done
           """
         }
@@ -532,20 +516,19 @@
         for (int i = 1; i <= params.volthaStacks.toInteger(); i++) {
           stack_ns="voltha"+i
           voltcfg="~/.volt/config-voltha"+i
-          println stack_ns
           try {
             sh """
-            voltctl -c $HOME/.volt/config-${stack_ns} -m 8MB device list -o json > $LOG_FOLDER/device-list.json || true
-            python -m json.tool $LOG_FOLDER/device-list.json > $LOG_FOLDER/voltha-devices-list.json || true
-            rm $LOG_FOLDER/device-list.json || true
-            voltctl -c $HOME/.volt/config-${stack_ns} -m 8MB device list > $LOG_FOLDER/voltha-devices-list.txt || true
+            voltctl -c $HOME/.volt/config-${stack_ns} -m 8MB device list -o json > $LOG_FOLDER/${stack_ns}/device-list.json || true
+            python -m json.tool $LOG_FOLDER/${stack_ns}/device-list.json > $LOG_FOLDER/${stack_ns}/voltha-devices-list.json || true
+            rm $LOG_FOLDER/${stack_ns}/device-list.json || true
+            voltctl -c $HOME/.volt/config-${stack_ns} -m 8MB device list > $LOG_FOLDER/${stack_ns}/voltha-devices-list.txt || true
 
             DEVICE_LIST=
-            printf '%s\n' \$(voltctl -c $HOME/.volt/config-${stack_ns} -m 8MB device list | grep olt | awk '{print \$1}') | xargs --no-run-if-empty -I# bash -c "voltctl -c $HOME/.volt/config-${stack_ns}-m 8MB device flows # > $LOG_FOLDER/voltha-device-flows-#.txt" || true
-            printf '%s\n' \$(voltctl -c $HOME/.volt/config-${stack_ns} -m 8MB device list | grep olt | awk '{print \$1}') | xargs --no-run-if-empty -I# bash -c "voltctl -c $HOME/.volt/config-${stack_ns} -m 8MB device port list --format 'table{{.PortNo}}\t{{.Label}}\t{{.Type}}\t{{.AdminState}}\t{{.OperStatus}}' # > $LOG_FOLDER/voltha-device-ports-#.txt" || true
+            printf '%s\n' \$(voltctl -c $HOME/.volt/config-${stack_ns} -m 8MB device list | grep olt | awk '{print \$1}') | xargs --no-run-if-empty -I# bash -c "voltctl -c $HOME/.volt/config-${stack_ns}-m 8MB device flows # > $LOG_FOLDER/${stack_ns}/voltha-device-flows-#.txt" || true
+            printf '%s\n' \$(voltctl -c $HOME/.volt/config-${stack_ns} -m 8MB device list | grep olt | awk '{print \$1}') | xargs --no-run-if-empty -I# bash -c "voltctl -c $HOME/.volt/config-${stack_ns} -m 8MB device port list --format 'table{{.PortNo}}\t{{.Label}}\t{{.Type}}\t{{.AdminState}}\t{{.OperStatus}}' # > $LOG_FOLDER/${stack_ns}/voltha-device-ports-#.txt" || true
 
-            printf '%s\n' \$(voltctl -c $HOME/.volt/config-${stack_ns} -m 8MB logicaldevice list -q) | xargs --no-run-if-empty -I# bash -c "voltctl -c $HOME/.volt/config-${stack_ns} -m 8MB logicaldevice flows # > $LOG_FOLDER/voltha-logicaldevice-flows-#.txt" || true
-            printf '%s\n' \$(voltctl -c $HOME/.volt/config-${stack_ns} -m 8MB logicaldevice list -q) | xargs --no-run-if-empty -I# bash -c "voltctl -c $HOME/.volt/config-${stack_ns} -m 8MB logicaldevice port list # > $LOG_FOLDER/voltha-logicaldevice-ports-#.txt" || true
+            printf '%s\n' \$(voltctl -c $HOME/.volt/config-${stack_ns} -m 8MB logicaldevice list -q) | xargs --no-run-if-empty -I# bash -c "voltctl -c $HOME/.volt/config-${stack_ns} -m 8MB logicaldevice flows # > $LOG_FOLDER/${stack_ns}/voltha-logicaldevice-flows-#.txt" || true
+            printf '%s\n' \$(voltctl -c $HOME/.volt/config-${stack_ns} -m 8MB logicaldevice list -q) | xargs --no-run-if-empty -I# bash -c "voltctl -c $HOME/.volt/config-${stack_ns} -m 8MB logicaldevice port list # > $LOG_FOLDER/${stack_ns}/voltha-logicaldevice-ports-#.txt" || true
             """
           } catch(e) {
             sh '''
@@ -563,7 +546,7 @@
         python tests/scale/sizing.py -o $WORKSPACE/plots || true
       fi
       '''
-      archiveArtifacts artifacts: 'kind-voltha/install-*.log,execution-time.txt,logs/*,logs/pprof/*,RobotLogs/**/*,plots/*,etcd-metrics/*'
+      archiveArtifacts artifacts: 'kind-voltha/install-*.log,execution-time-*.txt,logs/**/*,RobotLogs/**/*,plots/*,etcd-metrics/*'
     }
   }
 }
diff --git a/jjb/pipeline/voltha-scale-test.groovy b/jjb/pipeline/voltha-scale-test.groovy
index 6ff2d6d..76fa65a 100644
--- a/jjb/pipeline/voltha-scale-test.groovy
+++ b/jjb/pipeline/voltha-scale-test.groovy
@@ -320,9 +320,6 @@
           _TAG=kail-$app kail -l app=$app --since 1h > $LOG_FOLDER/$app.log&
         done
         '''
-        // bbsim-sadis server takes a while to cache the subscriber entries
-        // wait for that before starting the tests
-        sleep(120)
       }
     }
     stage('Configuration') {
diff --git a/jjb/verify/bbsim-sadis-server.yaml b/jjb/verify/bbsim-sadis-server.yaml
index d466482..5cdbc5c 100644
--- a/jjb/verify/bbsim-sadis-server.yaml
+++ b/jjb/verify/bbsim-sadis-server.yaml
@@ -22,9 +22,8 @@
           dest-gopath: "github.com/opencord"
           unit-test-targets: 'lint sca test'
           unit-test-keep-going: 'true'
-      # NOTE enable once kind-voltha uses it
-      # - 'voltha-patch-test':
-      #     pipeline-script: 'voltha-bbsim-tests.groovy'
+      - 'voltha-patch-test':
+          pipeline-script: 'voltha-bbsim-tests.groovy'
 
 - job-group:
     name: 'publish-bbsim-sadis-server-jobs'
diff --git a/jjb/voltha-e2e.yaml b/jjb/voltha-e2e.yaml
index f534efd..3d1c8e8 100644
--- a/jjb/voltha-e2e.yaml
+++ b/jjb/voltha-e2e.yaml
@@ -11,6 +11,7 @@
     make-target-failtest: bbsim-failurescenarios
     make-target-errortest: bbsim-errorscenarios
     make-target-alarmtest: bbsim-alarms-kind
+    make-target-multipleolt: bbsim-multiolt-kind
 
     jobs:
       - 'voltha-periodic-test':
@@ -21,6 +22,7 @@
           make-target-failtest: bbsim-failurescenarios
           make-target-errortest: bbsim-errorscenarios
           make-target-alarmtest: bbsim-alarms-kind
+          make-target-multipleolt: bbsim-multiolt-kind
           withAlarms: true
           code-branch: 'master'
           olts: 1
@@ -36,6 +38,7 @@
           make-target-failtest: bbsim-multiolt-failurescenarios
           make-target-errortest: bbsim-multiolt-errorscenarios
           make-target-alarmtest: bbsim-alarms-kind
+          make-target-multipleolt: bbsim-multiolt-kind
           withAlarms: false
           code-branch: 'master'
           olts: 2
@@ -50,6 +53,7 @@
           make-target-failtest: bbsim-failurescenarios
           make-target-errortest: bbsim-errorscenarios
           make-target-alarmtest: bbsim-alarms-kind
+          make-target-multipleolt: bbsim-multiolt-kind
           withAlarms: true
           make-target: functional-single-kind
           code-branch: 'master'
@@ -229,6 +233,11 @@
           description: 'Makefile target to invoke during failure/based test'
 
       - string:
+          name: makeMultiOltTarget
+          default: '{make-target-multipleolt}'
+          description: 'Makefile target to invoke during multiple olt test'
+
+      - string:
           name: makeErrortestTarget
           default: '{make-target-errortest}'
           description: 'Makefile target to invoke during error test'
diff --git a/jjb/voltha-scale.yaml b/jjb/voltha-scale.yaml
index cffc377..70970e3 100644
--- a/jjb/voltha-scale.yaml
+++ b/jjb/voltha-scale.yaml
@@ -102,8 +102,6 @@
           withDhcp: false
           withIgmp: false
           extraHelmFlags: '--set use_openonu_adapter_go=true'
-          volthaSystemTestsChange: refs/changes/27/21627/4
-          kindVolthaChange: refs/changes/85/21485/13
           openonuAdapterReplicas: 1
 
 
@@ -208,14 +206,14 @@
 
       # multi-stack jobs
       - 'voltha-scale-measurements':
-          name: 'voltha-scale-measurements-master-5-stacks-2-16-32-att-subscribers'
-          build-node: 'onf-pod1-head-node'
+          name: 'voltha-scale-measurements-master-10-stacks-2-16-32-att-subscribers'
+          build-node: 'voltha-scale-2'
           pipeline-script: 'voltha-scale-multi-stack.groovy'
-          'disable-job': true
+          'disable-job': false
           time-trigger: "H H/4 * * *"
           withMonitoring: false
           logLevel: WARN
-          volthaStacks: 5
+          volthaStacks: 10
           olts: 2
           pons: 16
           onus: 32
@@ -225,19 +223,18 @@
           withEapol: true
           withDhcp: true
           withIgmp: false
-          extraHelmFlags: '--set authRetry=true,dhcpRetry=true'
-          volthaSystemTestsChange: refs/changes/27/21627/4
-          kindVolthaChange: refs/changes/85/21485/13
+          extraHelmFlags: '--set use_openonu_adapter_go=true'
+          openonuAdapterReplicas: 1
 
       - 'voltha-scale-measurements':
-          name: 'voltha-scale-measurements-master-5-stacks-2-16-32-dt-subscribers'
-          build-node: 'onf-pod1-head-node'
+          name: 'voltha-scale-measurements-master-10-stacks-2-16-32-dt-subscribers'
+          build-node: 'voltha-scale-2'
           pipeline-script: 'voltha-scale-multi-stack.groovy'
-          'disable-job': true
+          'disable-job': false
           time-trigger: "H H/4 * * *"
           withMonitoring: false
           logLevel: WARN
-          volthaStacks: 5
+          volthaStacks: 10
           olts: 2
           pons: 16
           onus: 32
@@ -247,19 +244,18 @@
           withEapol: false
           withDhcp: false
           withIgmp: false
-          extraHelmFlags: '--set authRetry=true,dhcpRetry=true'
-          volthaSystemTestsChange: refs/changes/27/21627/4
-          kindVolthaChange: refs/changes/85/21485/13
+          extraHelmFlags: '--set use_openonu_adapter_go=true'
+          openonuAdapterReplicas: 1
 
       - 'voltha-scale-measurements':
-          name: 'voltha-scale-measurements-master-5-stacks-2-16-32-tt-subscribers'
-          build-node: 'onf-pod1-head-node'
+          name: 'voltha-scale-measurements-master-10-stacks-2-16-32-tt-subscribers'
+          build-node: 'voltha-scale-2'
           pipeline-script: 'voltha-scale-multi-stack.groovy'
-          'disable-job': true
+          'disable-job': false
           time-trigger: "H H/4 * * *"
           withMonitoring: false
           logLevel: WARN
-          volthaStacks: 5
+          volthaStacks: 10
           olts: 2
           pons: 16
           onus: 32
@@ -269,9 +265,8 @@
           withEapol: false
           withDhcp: true
           withIgmp: true
-          extraHelmFlags: '--set authRetry=true,dhcpRetry=true'
-          volthaSystemTestsChange: refs/changes/27/21627/4
-          kindVolthaChange: refs/changes/85/21485/13
+          extraHelmFlags: '--set use_openonu_adapter_go=true'
+          openonuAdapterReplicas: 1
 
       # voltha-2.5 Jobs
       - 'voltha-scale-measurements':