diff --git a/jjb/cord-test/nightly-build-pipeline.yaml b/jjb/cord-test/nightly-build-pipeline.yaml
index 354f92f..76244a6 100644
--- a/jjb/cord-test/nightly-build-pipeline.yaml
+++ b/jjb/cord-test/nightly-build-pipeline.yaml
@@ -917,6 +917,10 @@
           default: '{onos-version}'
           description: 'ONOS version that needs to be configured'
 
+      - string:
+          name: workFlow
+          default: '{work-flow}'
+          description: 'Installs the specified work flow on the POD'
 
       - bool:
           name: released
diff --git a/jjb/cord-test/voltha.yaml b/jjb/cord-test/voltha.yaml
index 2526400..f3ed114 100644
--- a/jjb/cord-test/voltha.yaml
+++ b/jjb/cord-test/voltha.yaml
@@ -12,6 +12,7 @@
     use-ofagent-go: false
     onos-version: '2.2'
     power-switch: False
+    work-flow: ''
 
     jobs:
       # flex OCP pod with olt/onu - manual test job, voltha master build job
@@ -22,7 +23,7 @@
           branch: 'master'
           test-repo: 'voltha-system-tests'
           Jenkinsfile: 'Jenkinsfile-voltha-bal31-build'
-          oltDebVersion: 'openolt_asfvolt16_bal3.2.deb'
+          oltDebVersion: 'openolt-2.3.0.deb'
           profile: '1T4GEM-bal31'
 
       # flex pod1 test job - using voltha branch
@@ -42,7 +43,7 @@
           branch: 'master'
           test-repo: 'voltha-system-tests'
           Jenkinsfile: 'Jenkinsfile-voltha-bal31-build'
-          oltDebVersion: 'openolt_asfvolt16_bal3.2.deb'
+          oltDebVersion: 'openolt-2.3.0.deb'
           profile: '1T4GEM-bal31'
 
       # onlab pod1 test job - BAL3.1 tests using voltha branch
@@ -63,7 +64,7 @@
           released: false
           test-repo: 'voltha-system-tests'
           Jenkinsfile: 'Jenkinsfile-voltha-bal31-build'
-          oltDebVersion: 'openolt_asfvolt16_bal3.2.deb'
+          oltDebVersion: 'openolt-2.3.0.deb'
           configurePod: true
           profile: 'Default'
           time: '1'
@@ -88,7 +89,7 @@
           released: false
           test-repo: 'voltha-system-tests'
           Jenkinsfile: 'Jenkinsfile-voltha-bal31-build'
-          oltDebVersion: 'openolt_asfvolt16_bal3.2.deb'
+          oltDebVersion: 'openolt-2.3.0.deb'
           configurePod: true
           profile: '1T4GEM-bal31'
           time: '3'
@@ -128,6 +129,34 @@
           test-repo: 'voltha-system-tests'
           profile: '1T4GEM-bal31'
 
+    # flex OCP pod with olt/onu - OFAGENT GO job, uses 1TCONT/4GEMs tech profile on voltha - timer based job
+      - 'build_voltha_pod_release':
+          build-node: 'qa-testvm-pod'
+          config-pod: 'flex-ocp-cord'
+          release: 'master'
+          branch: 'master'
+          released: false
+          name-extension: '_GO'
+          use-ofagent-go: true
+          test-repo: 'voltha-system-tests'
+          Jenkinsfile: 'Jenkinsfile-voltha-bal31-build'
+          oltDebVersion: 'openolt-2.3.0.deb'
+          configurePod: true
+          profile: '1T4GEM-bal31'
+          time: '6'
+
+      # flex pod1 test job - OFAGENT GO job, test job uses 1TCONT/4GEMs tech profile - using voltha branch
+      - 'build_voltha_pod_test':
+          build-node: 'qa-testvm-pod'
+          config-pod: 'flex-ocp-cord'
+          release: 'master'
+          branch: 'master'
+          name-extension: '_GO'
+          released: false
+          power-switch: True
+          test-repo: 'voltha-system-tests'
+          profile: '1T4GEM-bal31'
+
       # Menlo pod with olt/onu - Default tech profile and timer based job
       - 'build_voltha_pod_release':
           build-node: 'menlo-demo-pod'
@@ -137,7 +166,7 @@
           released: false
           test-repo: 'voltha-system-tests'
           Jenkinsfile: 'Jenkinsfile-voltha-bal31-build'
-          oltDebVersion: 'openolt_asfvolt16_bal3.2.deb'
+          oltDebVersion: 'openolt-2.3.0.deb'
           configurePod: true
           profile: 'Default'
           time: '1'
@@ -185,7 +214,7 @@
           released: false
           test-repo: 'voltha-system-tests'
           Jenkinsfile: 'Jenkinsfile-voltha-bal31-build'
-          oltDebVersion: 'openolt_asfvolt16_bal3.2.deb'
+          oltDebVersion: 'openolt-2.3.0.deb'
           configurePod: true
           profile: '1T4GEM-bal31'
           time: '4'
@@ -211,7 +240,7 @@
           released: false
           test-repo: 'voltha-system-tests'
           Jenkinsfile: 'Jenkinsfile-voltha-bal31-build'
-          oltDebVersion: 'openolt_asfvolt16_bal3.2.deb'
+          oltDebVersion: 'openolt-2.3.0.deb'
           configurePod: true
           profile: 'Default'
           time: '6'
@@ -233,14 +262,14 @@
           config-pod: 'onf-demo-pod'
           release: 'master'
           branch: 'master'
-          name-extension: '_GO'
-          use-ofagent-go: true
+          name-extension: '_DT'
+          work-flow: 'DT'
           released: false
           test-repo: 'voltha-system-tests'
           Jenkinsfile: 'Jenkinsfile-voltha-bal31-build'
-          oltDebVersion: 'openolt_asfvolt16_bal3.2.deb'
+          oltDebVersion: 'openolt-2.3.0.deb'
           configurePod: true
-          profile: '1T4GEM-bal31'
+          profile: '1T8GEM'
           time: '8'
 
       # Menlo pod test job - uses tech profile on voltha branch
@@ -249,10 +278,11 @@
           config-pod: 'onf-demo-pod'
           release: 'master'
           branch: 'master'
-          name-extension: '_GO'
+          name-extension: '_DT'
           released: false
           test-repo: 'voltha-system-tests'
-          profile: '1T4GEM-bal31'
+          profile: '1T8GEM'
+          pipeline-script: 'voltha-dt-physical-functional-tests.groovy'
 
       # Menlo DEMO-POD - 1 1TCONT 4 4GEMs TechProfile - Manual build and test job
       - 'build_pod_manual':
@@ -262,7 +292,7 @@
           branch: 'master'
           test-repo: 'voltha-system-tests'
           Jenkinsfile: 'Jenkinsfile-voltha-bal31-build'
-          oltDebVersion: 'openolt_asfvolt16_bal3.2.deb'
+          oltDebVersion: 'openolt-2.3.0.deb'
           configurePod: true
           profile: '1T4GEM-bal31'
       - 'build_pod_test':
@@ -345,3 +375,4 @@
           released: false
           test-repo: 'voltha-system-tests'
           profile: 'Default'
+          power-switch: True
diff --git a/jjb/pipeline/voltha-bbsim-tests.groovy b/jjb/pipeline/voltha-bbsim-tests.groovy
index b1a7413..6363770 100644
--- a/jjb/pipeline/voltha-bbsim-tests.groovy
+++ b/jjb/pipeline/voltha-bbsim-tests.groovy
@@ -64,7 +64,12 @@
       steps {
         sh """
            pushd voltha
-           repo download "${gerritProject}" "${gerritChangeNumber}/${gerritPatchsetNumber}"
+           if [ "${gerritProject}" != "" -a "${gerritChangeNumber}" != "" -a "${gerritPatchsetNumber}" != "" ]
+           then
+             repo download "${gerritProject}" "${gerritChangeNumber}/${gerritPatchsetNumber}"
+           else
+             echo "No patchset to download!"
+           fi
            popd
            """
       }
diff --git a/jjb/pipeline/voltha-dt-physical-functional-tests.groovy b/jjb/pipeline/voltha-dt-physical-functional-tests.groovy
new file mode 100644
index 0000000..98f4edb
--- /dev/null
+++ b/jjb/pipeline/voltha-dt-physical-functional-tests.groovy
@@ -0,0 +1,165 @@
+// Copyright 2017-present Open Networking Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+node {
+  // Need this so that deployment_config has global scope when it's read later
+  deployment_config = null
+}
+
+pipeline {
+  /* no label, executor is determined by JJB */
+  agent {
+    label "${params.buildNode}"
+  }
+  options {
+    timeout(time: 90, unit: 'MINUTES')
+  }
+
+  environment {
+    KUBECONFIG="$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf"
+    VOLTCONFIG="$HOME/.volt/config-minimal"
+    PATH="$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+  }
+
+  stages {
+    stage ('Initialize') {
+      steps {
+        step([$class: 'WsCleanup'])
+        sh returnStdout: false, script: "git clone -b ${branch} ${cordRepoUrl}/${configBaseDir}"
+        script {
+           deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}-DT.yaml"
+        }
+        // This checkout is just so that we can show changes in Jenkins
+        checkout(changelog: true,
+          poll: false,
+          scm: [$class: 'RepoScm',
+            manifestRepositoryUrl: "${params.manifestUrl}",
+            manifestBranch: "${params.manifestBranch}",
+            currentBranch: true,
+            destinationDir: 'voltha',
+            forceSync: true,
+            resetFirst: true,
+            quiet: true,
+            jobs: 4,
+            showAllChanges: true]
+          )
+        sh returnStdout: false, script: """
+        cd voltha
+        git clone -b ${branch} ${cordRepoUrl}/cord-tester
+        mkdir -p $WORKSPACE/bin
+        bash <( curl -sfL https://raw.githubusercontent.com/boz/kail/master/godownloader.sh) -b "$WORKSPACE/bin"
+        cd $WORKSPACE
+        git clone https://github.com/ciena/kind-voltha.git
+
+        VC_VERSION=\$(curl -sSL https://api.github.com/repos/opencord/voltctl/releases/latest | jq -r .tag_name | sed -e 's/^v//g')
+        HOSTOS=\$(uname -s | tr "[:upper:]" "[:lower:"])
+        HOSTARCH=\$(uname -m | tr "[:upper:]" "[:lower:"])
+        if [ \$HOSTARCH == "x86_64" ]; then
+            HOSTARCH="amd64"
+        fi
+        curl -o $WORKSPACE/bin/voltctl -sSL https://github.com/opencord/voltctl/releases/download/v\${VC_VERSION}/voltctl-\${VC_VERSION}-\${HOSTOS}-\${HOSTARCH}
+        chmod 755 $WORKSPACE/bin/voltctl
+        voltctl version --clientonly
+        """
+      }
+    }
+
+    stage('Functional Tests') {
+      environment {
+        ROBOT_CONFIG_FILE="$WORKSPACE/${configBaseDir}/${configDeploymentDir}/${configFileName}-DT.yaml"
+        ROBOT_FILE="Voltha_DT_PODTests.robot"
+        ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/dt-workflow"
+      }
+      steps {
+        sh """
+        cd $WORKSPACE/kind-voltha/scripts
+        ./log-collector.sh > /dev/null &
+        ./log-combine.sh > /dev/null &
+
+        mkdir -p $ROBOT_LOGS_DIR
+        if  ( ${released} ); then
+            export ROBOT_MISC_ARGS="--removekeywords wuks -i released -i sanityDt -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE"
+        else
+            export ROBOT_MISC_ARGS="--removekeywords wuks -i sanityDt -i functionalDt -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE"
+        fi
+        make -C $WORKSPACE/voltha/voltha-system-tests voltha-dt-test || true
+        """
+      }
+    }
+  }
+  post {
+    always {
+      sh returnStdout: false, script: '''
+      set +e
+      kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\t'}{.imageID}{'\\n'}" | sort | uniq -c
+      kubectl get nodes -o wide
+      kubectl get pods -n voltha -o wide
+
+      sleep 60 # Wait for log-collector and log-combine to complete
+
+      # Clean up "announcer" pod used by the tests if present
+      kubectl delete pod announcer || true
+
+      ## Pull out errors from log files
+      extract_errors_go() {
+        echo
+        echo "Error summary for $1:"
+        grep '"level":"error"' $WORKSPACE/kind-voltha/scripts/logger/combined/$1*
+        echo
+      }
+
+      extract_errors_python() {
+        echo
+        echo "Error summary for $1:"
+        grep 'ERROR' $WORKSPACE/kind-voltha/scripts/logger/combined/$1*
+        echo
+      }
+
+      extract_errors_go voltha-rw-core > $WORKSPACE/error-report.log
+      extract_errors_go adapter-open-olt >> $WORKSPACE/error-report.log
+      extract_errors_python adapter-open-onu >> $WORKSPACE/error-report.log
+      extract_errors_python voltha-ofagent >> $WORKSPACE/error-report.log
+      extract_errors_python onos >> $WORKSPACE/error-report.log
+
+      cd $WORKSPACE/kind-voltha/scripts/logger/combined/
+      tar czf $WORKSPACE/container-logs.tgz *
+
+      cd $WORKSPACE
+      gzip *-combined.log || true
+      '''
+      script {
+        deployment_config.olts.each { olt ->
+          sh returnStdout: false, script: """
+          sshpass -p ${olt.pass} scp ${olt.user}@${olt.ip}:/var/log/openolt.log $WORKSPACE/openolt-${olt.ip}.log || true
+          sed -i 's/\\x1b\\[[0-9;]*[a-zA-Z]//g' $WORKSPACE/openolt-${olt.ip}.log  # Remove escape sequences
+          """
+        }
+      }
+      step([$class: 'RobotPublisher',
+        disableArchiveOutput: false,
+        logFileName: '**/log*.html',
+        otherFiles: '',
+        outputFileName: '**/output*.xml',
+        outputPath: 'RobotLogs',
+        passThreshold: 100,
+        reportFileName: '**/report*.html',
+        unstableThreshold: 0
+        ]);
+      archiveArtifacts artifacts: '*.log,*.gz,*.tgz'
+    }
+    unstable {
+      step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "${notificationEmail}", sendToIndividuals: false])
+    }
+  }
+}
diff --git a/jjb/pipeline/voltha-go-multi-tests.groovy b/jjb/pipeline/voltha-go-multi-tests.groovy
index c8886ff..d95f23b 100644
--- a/jjb/pipeline/voltha-go-multi-tests.groovy
+++ b/jjb/pipeline/voltha-go-multi-tests.groovy
@@ -28,7 +28,7 @@
   environment {
     KUBECONFIG="$HOME/.kube/kind-config-voltha-minimal"
     VOLTCONFIG="$HOME/.volt/config-minimal"
-    PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$WORKSPACE/kind-voltha/bin"
+    PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$HOME/kind-voltha/bin"
     TYPE="minimal"
     FANCY=0
     WITH_SIM_ADAPTERS="n"
@@ -64,7 +64,10 @@
     stage('Download kind-voltha') {
       steps {
         sh """
-           git clone https://github.com/ciena/kind-voltha.git
+           cd $HOME
+           [ -d kind-voltha ] || git clone https://github.com/ciena/kind-voltha.git
+           cd $HOME/kind-voltha
+           git pull
            """
       }
     }
@@ -72,9 +75,8 @@
     stage('Deploy Voltha') {
       steps {
         sh """
-           cd kind-voltha/
-           JUST_K8S=y ./voltha up
-           kail -n voltha -n default > $WORKSPACE/onos-voltha-combined.log &
+           cd $HOME/kind-voltha/
+           WAIT_ON_DOWN=y DEPLOY_K8S=n ./voltha down || ./voltha down
            ./voltha up
            """
       }
@@ -83,9 +85,14 @@
     stage('Run E2E Tests') {
       steps {
         sh '''
+           set +e
            mkdir -p $WORKSPACE/RobotLogs
            git clone https://gerrit.opencord.org/voltha-system-tests
-           cd kind-voltha
+
+           cd $HOME/kind-voltha/scripts
+           ./log-collector.sh > /dev/null &
+           ./log-combine.sh > /dev/null &
+
            for i in \$(seq 1 ${testRuns})
            do
              make -C $WORKSPACE/voltha-system-tests ${makeTarget}
@@ -101,40 +108,44 @@
     always {
       sh '''
          set +e
-         cp $WORKSPACE/kind-voltha/install-minimal.log $WORKSPACE/
+         cp $HOME/kind-voltha/install-minimal.log $WORKSPACE/
          kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\t'}{.imageID}{'\\n'}" | sort | uniq -c
          kubectl get nodes -o wide
          kubectl get pods -o wide
          kubectl get pods -n voltha -o wide
 
-         sync
-         pkill kail || true
+         sleep 60 # Wait for log-collector and log-combine to complete
 
          ## Pull out errors from log files
          extract_errors_go() {
            echo
            echo "Error summary for $1:"
-           grep $1 $WORKSPACE/onos-voltha-combined.log | grep '"level":"error"' | cut -d ' ' -f 2- | jq -r '.msg'
+           grep '"level":"error"' $HOME/kind-voltha/scripts/logger/combined/$1*
            echo
          }
 
          extract_errors_python() {
            echo
            echo "Error summary for $1:"
-           grep $1 $WORKSPACE/onos-voltha-combined.log | grep 'ERROR' | cut -d ' ' -f 2-
+           grep 'ERROR' $HOME/kind-voltha/scripts/logger/combined/$1*
            echo
          }
 
          extract_errors_go voltha-rw-core > $WORKSPACE/error-report.log
          extract_errors_go adapter-open-olt >> $WORKSPACE/error-report.log
          extract_errors_python adapter-open-onu >> $WORKSPACE/error-report.log
-         extract_errors_python voltha-ofagent >> $WORKSPACE/error-report.log
+         extract_errors_go voltha-ofagent >> $WORKSPACE/error-report.log
+         extract_errors_python onos >> $WORKSPACE/error-report.log
 
-         gzip $WORKSPACE/onos-voltha-combined.log
+         cd $HOME/kind-voltha/scripts/logger/combined/
+         tar czf $WORKSPACE/container-logs.tgz *
 
-         ## shut down voltha
-         cd $WORKSPACE/kind-voltha/
-         WAIT_ON_DOWN=y ./voltha down
+         cd $WORKSPACE
+         gzip *-combined.log || true
+
+         ## shut down voltha but leave kind-voltha cluster
+         cd $HOME/kind-voltha/
+         DEPLOY_K8S=n WAIT_ON_DOWN=y ./voltha down
          '''
          step([$class: 'RobotPublisher',
             disableArchiveOutput: false,
@@ -145,7 +156,7 @@
             passThreshold: 100,
             reportFileName: 'RobotLogs/report*.html',
             unstableThreshold: 0]);
-         archiveArtifacts artifacts: '*.log,*.gz'
+         archiveArtifacts artifacts: '*.log,*.gz,*.tgz'
 
     }
   }
diff --git a/jjb/pipeline/voltha-go-tests.groovy b/jjb/pipeline/voltha-go-tests.groovy
index 5e519c2..c70f0ab 100644
--- a/jjb/pipeline/voltha-go-tests.groovy
+++ b/jjb/pipeline/voltha-go-tests.groovy
@@ -28,7 +28,7 @@
   environment {
     KUBECONFIG="$HOME/.kube/kind-config-voltha-minimal"
     VOLTCONFIG="$HOME/.volt/config-minimal"
-    PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$WORKSPACE/kind-voltha/bin"
+    PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$HOME/kind-voltha/bin"
     TYPE="minimal"
     FANCY=0
     WITH_SIM_ADAPTERS="n"
@@ -64,7 +64,10 @@
     stage('Download kind-voltha') {
       steps {
         sh """
-           git clone https://github.com/ciena/kind-voltha.git
+           cd $HOME
+           [ -d kind-voltha ] || git clone https://github.com/ciena/kind-voltha.git
+           cd $HOME/kind-voltha
+           git pull
            """
       }
     }
@@ -72,8 +75,8 @@
     stage('Deploy Voltha') {
       steps {
         sh """
-           cd kind-voltha/
-           JUST_K8S=y ./voltha up
+           cd $HOME/kind-voltha/
+           WAIT_ON_DOWN=y DEPLOY_K8S=n ./voltha down || ./voltha down
            ./voltha up
            """
       }
@@ -86,7 +89,7 @@
            mkdir -p $WORKSPACE/RobotLogs
            git clone https://gerrit.opencord.org/voltha-system-tests
 
-           cd $WORKSPACE/kind-voltha/scripts
+           cd $HOME/kind-voltha/scripts
            ./log-collector.sh > /dev/null &
            ./log-combine.sh > /dev/null &
 
@@ -100,7 +103,7 @@
     always {
       sh '''
          set +e
-         cp $WORKSPACE/kind-voltha/install-minimal.log $WORKSPACE/
+         cp $HOME/kind-voltha/install-minimal.log $WORKSPACE/
          kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\t'}{.imageID}{'\\n'}" | sort | uniq -c
          kubectl get nodes -o wide
          kubectl get pods -o wide
@@ -112,14 +115,14 @@
          extract_errors_go() {
            echo
            echo "Error summary for $1:"
-           grep '"level":"error"' $WORKSPACE/kind-voltha/scripts/logger/combined/$1*
+           grep '"level":"error"' $HOME/kind-voltha/scripts/logger/combined/$1*
            echo
          }
 
          extract_errors_python() {
            echo
            echo "Error summary for $1:"
-           grep 'ERROR' $WORKSPACE/kind-voltha/scripts/logger/combined/$1*
+           grep 'ERROR' $HOME/kind-voltha/scripts/logger/combined/$1*
            echo
          }
 
@@ -129,15 +132,15 @@
          extract_errors_go voltha-ofagent >> $WORKSPACE/error-report.log
          extract_errors_python onos >> $WORKSPACE/error-report.log
 
-         cd $WORKSPACE/kind-voltha/scripts/logger/combined/
+         cd $HOME/kind-voltha/scripts/logger/combined/
          tar czf $WORKSPACE/container-logs.tgz *
 
          cd $WORKSPACE
          gzip *-combined.log || true
 
-         ## shut down voltha
-         cd $WORKSPACE/kind-voltha/
-         WAIT_ON_DOWN=y ./voltha down
+         ## shut down voltha but leave kind-voltha cluster
+         cd $HOME/kind-voltha/
+         DEPLOY_K8S=n WAIT_ON_DOWN=y ./voltha down
          '''
          step([$class: 'RobotPublisher',
             disableArchiveOutput: false,
diff --git a/jjb/pipeline/voltha-physical-functional-tests.groovy b/jjb/pipeline/voltha-physical-functional-tests.groovy
index 23b01c9..47ace1e 100644
--- a/jjb/pipeline/voltha-physical-functional-tests.groovy
+++ b/jjb/pipeline/voltha-physical-functional-tests.groovy
@@ -29,7 +29,7 @@
   environment {
     KUBECONFIG="$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf"
     VOLTCONFIG="$HOME/.volt/config-minimal"
-    PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$WORKSPACE/bin"
+    PATH="$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
   }
 
   stages {
@@ -61,9 +61,20 @@
         bash <( curl -sfL https://raw.githubusercontent.com/boz/kail/master/godownloader.sh) -b "$WORKSPACE/bin"
         cd $WORKSPACE
         git clone https://github.com/ciena/kind-voltha.git
+
+        VC_VERSION=\$(curl -sSL https://api.github.com/repos/opencord/voltctl/releases/latest | jq -r .tag_name | sed -e 's/^v//g')
+        HOSTOS=\$(uname -s | tr "[:upper:]" "[:lower:"])
+        HOSTARCH=\$(uname -m | tr "[:upper:]" "[:lower:"])
+        if [ \$HOSTARCH == "x86_64" ]; then
+            HOSTARCH="amd64"
+        fi
+        curl -o $WORKSPACE/bin/voltctl -sSL https://github.com/opencord/voltctl/releases/download/v\${VC_VERSION}/voltctl-\${VC_VERSION}-\${HOSTOS}-\${HOSTARCH}
+        chmod 755 $WORKSPACE/bin/voltctl
+        voltctl version --clientonly
         """
       }
     }
+
     stage('Functional Tests') {
       environment {
         ROBOT_CONFIG_FILE="$WORKSPACE/${configBaseDir}/${configDeploymentDir}/${configFileName}.yaml"
diff --git a/jjb/pipeline/voltha-scale-measurements.groovy b/jjb/pipeline/voltha-scale-measurements.groovy
index 87b8656..b3cdef7 100644
--- a/jjb/pipeline/voltha-scale-measurements.groovy
+++ b/jjb/pipeline/voltha-scale-measurements.groovy
@@ -19,7 +19,6 @@
     ROBOT_MISC_ARGS="-d $WORKSPACE/RobotLogs -v teardown_device:False"
     SSHPASS="karaf"
     DEPLOY_K8S="n"
-    EXTRA_HELM_FLAGS="--set onu=${onuPerPon},pon=${ponPorts},delay=${BBSIMdelay},auth=${bbsimAuth},dhcp=${bbsimDhcp}"
   }
   stages {
     stage('set-description') {
@@ -32,7 +31,7 @@
     stage('cleanup') {
       steps {
         sh '''
-          rm -rf voltha-devices.txt onos-ports.txt total-time.txt onu-activation.txt
+          rm -rf voltha-devices-count.txt voltha-devices-time.txt onos-ports-count.txt onos-ports-time.txt onos-ports-list.txt voltha-devices-list.json onos-ports-time-num.txt voltha-devices-time-num.txt
           for hchart in \$(helm list -q | grep -E -v 'docker-registry|cord-kafka|etcd-operator');
           do
               echo "Purging chart: \${hchart}"
@@ -52,38 +51,56 @@
       }
     }
     stage('deploy-voltha') {
+      options {
+        timeout(time:10)
+      }
       steps {
         sh '''
-          helm install -n onos onf/onos --set images.onos.repository=voltha/voltha-onos --set images.onos.tag=4.0.1
+          helm install -n nem-monitoring cord/nem-monitoring
+
+          IFS=: read -r onosRepo onosTag <<< ${onosImg}
+          helm install -n onos onf/onos --set images.onos.repository=${onosRepo} --set images.onos.tag=${onosTag} ${extraHelmFlags}
 
           IFS=: read -r volthaRepo volthaTag <<< ${volthaImg}
-          helm install -n voltha onf/voltha -f /home/cord/voltha-scale/voltha-values.yaml --set images.voltha.repository=${volthaRepo},images.voltha.tag=${volthaTag}
+          IFS=: read -r ofAgentRepo ofAgentTag <<< ${ofAgentImg}
+          helm install -n voltha ${volthaChart} -f /home/cord/voltha-scale/voltha-values.yaml --set images.voltha.repository=${volthaRepo},images.voltha.tag=${volthaTag},images.ofagent.repository=${ofAgentRepo},images.ofagent.tag=${ofAgentTag} ${extraHelmFlags}
 
           IFS=: read -r openoltAdapterRepo openoltAdapterTag <<< ${openoltAdapterImg}
-          helm install -n openolt onf/voltha-adapter-openolt -f /home/cord/voltha-scale/voltha-values.yaml --set images.adapter_open_olt.repository=${openoltAdapterRepo},images.adapter_open_olt.tag=${openoltAdapterTag}
+          helm install -n openolt ${openoltAdapterChart} -f /home/cord/voltha-scale/voltha-values.yaml --set images.adapter_open_olt.repository=${openoltAdapterRepo},images.adapter_open_olt.tag=${openoltAdapterTag} ${extraHelmFlags}
 
           IFS=: read -r openonuAdapterRepo openonuAdapterTag <<< ${openonuAdapterImg}
-          helm install -n openonu onf/voltha-adapter-openonu -f /home/cord/voltha-scale/voltha-values.yaml --set images.adapter_open_olt.repository=${openonuAdapterRepo},images.adapter_open_olt.tag=${openonuAdapterTag}
+          helm install -n openonu ${openonuAdapterChart} -f /home/cord/voltha-scale/voltha-values.yaml --set images.adapter_open_onu.repository=${openonuAdapterRepo},images.adapter_open_onu.tag=${openonuAdapterTag} ${extraHelmFlags}
 
           IFS=: read -r bbsimRepo bbsimTag <<< ${bbsimImg}
-          helm install -n bbsim onf/bbsim --set pon=${ponPorts},onu=${onuPerPon},auth=${bbsimAuth},dhcp=${bbsimDhcp},delay=${BBSIMdelay},images.bbsim.repository=${bbsimRepo},images.bbsim.tag=${bbsimTag}
-          helm install -n radius onf/freeradius
+          helm install -n bbsim ${bbsimChart} --set enablePerf=true,pon=${ponPorts},onu=${onuPerPon},auth=${bbsimAuth},dhcp=${bbsimDhcp},delay=${BBSIMdelay},images.bbsim.repository=${bbsimRepo},images.bbsim.tag=${bbsimTag} ${extraHelmFlags}
 
-          if [ ! -z ${bbsimImg} ];
-          then
-            IFS=: read -r bbsimRepo bbsimTag <<< ${bbsimImg}
-            EXTRA_HELM_FLAGS+=",images.bbsim.repository=${bbsimRepo},images.bbsim.tag=${bbsimTag}"
-          fi
-          if [ ! -z ${volthaImg} ];
-          then
-            IFS=: read -r volthaRepo volthaTag <<< ${volthaImg}
-            EXTRA_HELM_FLAGS+=",images.voltha.repository=${volthaRepo},images.voltha.tag=${volthaTag}"
-          fi
+          helm install -n radius onf/freeradius ${extraHelmFlags}
+
           bash /home/cord/voltha-scale/wait_for_pods.sh
           bash /home/cord/voltha-scale/start_port_forward.sh
           '''
       }
     }
+    stage('wait for adapters to be registered') {
+      options {
+        timeout(time:5)
+      }
+      steps{
+        waitUntil {
+          script {
+            openolt_res = sh returnStdout: true, script: """
+            voltctl adapter list | grep openolt | wc -l
+            """
+
+            openonu_res = sh returnStdout: true, script: """
+            voltctl adapter list | grep brcm_openomci_onu | wc -l
+            """
+
+            return openolt_res.toInteger() == 1 && openonu_res.toInteger() == 1
+          }
+        }
+      }
+    }
     stage('MIB-template') {
       steps {
         sh '''
@@ -149,7 +166,7 @@
                 i=$(voltctl device list | grep -v OLT | grep ACTIVE | wc -l)
               done
               echo "${expectedOnus} ONUs Activated in $SECONDS seconds (time: $SECONDS)"
-              echo $SECONDS > voltha-devices.txt
+              echo $SECONDS > voltha-devices-time-num.txt
             '''
           }
         }
@@ -165,11 +182,12 @@
                 z=$(sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@localhost ports -e | grep BBSM | wc -l)
               done
               echo "${expectedOnus} ports enabled in $SECONDS seconds (time: $SECONDS)"
-              echo $SECONDS > onos-ports.txt
-              echo "ONOS-Duration(s)" > total-time.txt
-              echo "VOLTHA-Duration(s)" > onu-activation.txt
-              cat voltha-devices.txt >> onu-activation.txt
-              paste voltha-devices.txt onos-ports.txt | awk '{print ($1 + $2)}' >> total-time.txt
+              echo $SECONDS > temp.txt
+              paste voltha-devices-time-num.txt temp.txt | awk '{print ($1 + $2)}' > onos-ports-time-num.txt
+              echo "ONOS-Duration(s)" > onos-ports-time.txt
+              echo "VOLTHA-Duration(s)" > voltha-devices-time.txt
+              cat voltha-devices-time-num.txt >> voltha-devices-time.txt
+              cat onos-ports-time-num.txt >> onos-ports-time.txt
             '''
           }
         }
@@ -180,38 +198,41 @@
     success {
       plot([
         csvFileName: 'plot-onu-activation.csv',
-        csvSeries: [[displayTableFlag: false, exclusionValues: '', file: 'onu-activation.txt', inclusionFlag: 'OFF', url: ''], [displayTableFlag: false, exclusionValues: '', file: 'total-time.txt', inclusionFlag: 'OFF', url: '']],
+        csvSeries: [[displayTableFlag: false, exclusionValues: '', file: 'voltha-devices-time.txt', inclusionFlag: 'OFF', url: ''], [displayTableFlag: false, exclusionValues: '', file: 'onos-ports-time.txt', inclusionFlag: 'OFF', url: '']],
         group: 'Voltha-Scale-Numbers', numBuilds: '100', style: 'line', title: "Time (${BBSIMdelay}s Delay)", yaxis: 'Time (s)', useDescr: true
       ])
     }
     always {
       sh '''
         echo $(voltctl device list | grep -v OLT | grep ACTIVE | wc -l) > onus.txt
-        echo "#-of-ONUs" > no_onus.txt
-        cat onus.txt >> no_onus.txt
+        echo "#-of-ONUs" > voltha-devices-count.txt
+        cat onus.txt >> voltha-devices-count.txt
 
         echo $(sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@localhost ports -e | grep BBSM | wc -l) > ports.txt
-        echo "#-of-ports" > no_ports.txt
-        cat ports.txt >> no_ports.txt
+        echo "#-of-ports" > onos-ports-count.txt
+        cat ports.txt >> onos-ports-count.txt
 
         kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\t'}{.imageID}{'\\n'}" | sort | uniq -c
         voltctl device list -o json > device-list.json
-        python -m json.tool device-list.json > volt-device-list.json
-        sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@localhost ports > onos-ports.txt
+        python -m json.tool device-list.json > voltha-devices-list.json
+        sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@localhost ports > onos-ports-list.txt
+        curl -s -X GET -G http://127.0.0.1:31301/api/v1/query --data-urlencode 'query=avg(rate(container_cpu_usage_seconds_total[10m])*100) by (pod_name)' | jq . > cpu-usage.json
+        kubectl log deployment/adapter-open-olt > open-olt-logs.txt
+        kubectl log deployment/adapter-open-onu > open-onu-logs.txt
+        kubectl log deployment/voltha-rw-core > voltha-rw-core-logs.txt
+        kubectl log deployment/voltha-ofagent > voltha-ofagent-logs.txt
+        kubectl log deployment/bbsim > bbsim-logs.txt
+
+        rm -rf BBSM-12345123451234512345-00000000000001-v1.json device-list.json onus.txt ports.txt temp.txt
         '''
       plot([
         csvFileName: 'plot-numbers.csv',
-        csvSeries: [[displayTableFlag: false, exclusionValues: '', file: 'no_onus.txt', inclusionFlag: 'OFF', url: ''], [displayTableFlag: false, exclusionValues: '', file: 'no_ports.txt', inclusionFlag: 'OFF', url: '']],
+        csvSeries: [[displayTableFlag: false, exclusionValues: '', file: 'voltha-devices-count.txt', inclusionFlag: 'OFF', url: ''], [displayTableFlag: false, exclusionValues: '', file: 'onos-ports-count.txt', inclusionFlag: 'OFF', url: '']],
         group: 'Voltha-Scale-Numbers', numBuilds: '100', style: 'line', title: "Activated ONUs and Recognized Ports", yaxis: 'Number of Ports/ONUs', useDescr: true
       ])
 
       archiveArtifacts artifacts: '*.log,*.json,*txt'
 
-      script {
-        sh '''
-          rm -rf onus.txt ports.txt voltha-devices.txt onos-ports.txt total-time.txt onu-activation.txt device-list.json
-        '''
-      }
     }
   }
-}
\ No newline at end of file
+}
diff --git a/jjb/verify/pyvoltha.yaml b/jjb/verify/pyvoltha.yaml
index 4100715..b8564cb 100644
--- a/jjb/verify/pyvoltha.yaml
+++ b/jjb/verify/pyvoltha.yaml
@@ -11,6 +11,17 @@
       - 'publish-pyvoltha-jobs':
           branch-regexp: '{all-branches-regexp}'
           pypi-index: 'pypi'
+      - 'voltha-periodic-test':
+          name: 'nightly-pyvoltha-compatibility-test'
+          pipeline-script: 'voltha-bbsim-tests.groovy'
+          build-node: 'ubuntu16.04-basebuild-4c-8g'
+          default-image-tag: 'master'
+          code-branch: 'master'
+          make-target: sanity-single-kind
+          gerrit-project: pyvoltha
+          onus: 1
+          pons: 1
+          time-trigger: "@daily"
 
 - job-group:
     name: 'verify-pyvoltha-jobs'
@@ -20,9 +31,7 @@
           dependency-jobs: 'verify_pyvoltha_licensed'
       - 'python-unit-test':
           dependency-jobs: 'verify_pyvoltha_tag-collision'
-      - 'voltha-patch-test':
-          pipeline-script: 'voltha-bbsim-tests.groovy'
-          skip-vote: true
+
 
 - job-group:
     name: 'publish-pyvoltha-jobs'
diff --git a/jjb/verify/voltha-lib-go.yaml b/jjb/verify/voltha-lib-go.yaml
index fee00c3..9f148ef 100644
--- a/jjb/verify/voltha-lib-go.yaml
+++ b/jjb/verify/voltha-lib-go.yaml
@@ -8,6 +8,17 @@
     jobs:
       - 'verify-voltha-lib-go-jobs':
           branch-regexp: '{all-branches-regexp}'
+      - 'voltha-periodic-test':
+          name: 'nightly-voltha-lib-go-compatibility-test'
+          pipeline-script: 'voltha-bbsim-tests.groovy'
+          build-node: 'ubuntu16.04-basebuild-4c-8g'
+          default-image-tag: 'master'
+          code-branch: 'master'
+          make-target: sanity-single-kind
+          gerrit-project: voltha-lib-go
+          onus: 1
+          pons: 1
+          time-trigger: "@daily"
 
 - job-group:
     name: 'verify-voltha-lib-go-jobs'
@@ -21,6 +32,3 @@
           dest-gopath: "github.com/opencord"
           unit-test-targets: 'lint sca test'
           unit-test-keep-going: 'true'
-      - 'voltha-patch-test':
-          pipeline-script: 'voltha-bbsim-tests.groovy'
-          skip-vote: true
diff --git a/jjb/verify/voltha-protos.yaml b/jjb/verify/voltha-protos.yaml
index ef7b6ab..98afdb0 100644
--- a/jjb/verify/voltha-protos.yaml
+++ b/jjb/verify/voltha-protos.yaml
@@ -12,6 +12,17 @@
           branch-regexp: '{all-branches-regexp}'
           pypi-index: 'pypi'
           pypi-prep-commands: 'make python-protos'
+      - 'voltha-periodic-test':
+          name: 'nightly-voltha-protos-compatibility-test'
+          pipeline-script: 'voltha-bbsim-tests.groovy'
+          build-node: 'ubuntu16.04-basebuild-4c-8g'
+          default-image-tag: 'master'
+          code-branch: 'master'
+          make-target: sanity-single-kind
+          gerrit-project: voltha-protos
+          onus: 1
+          pons: 1
+          time-trigger: "@daily"
 
 - job-group:
     name: 'verify-voltha-protos-jobs'
@@ -23,9 +34,6 @@
           build-timeout: 20
           unit-test-targets: 'test'
           unit-test-keep-going: 'true'
-      - 'voltha-patch-test':
-          pipeline-script: 'voltha-bbsim-tests.groovy'
-          skip-vote: true
 
 - job-group:
     name: 'post-merge-voltha-protos-jobs'
diff --git a/jjb/voltha-e2e.yaml b/jjb/voltha-e2e.yaml
index e01e54e..e1539db 100644
--- a/jjb/voltha-e2e.yaml
+++ b/jjb/voltha-e2e.yaml
@@ -107,6 +107,7 @@
     pipeline-script: 'voltha-go-tests.groovy'
     test-runs: 1
     robot-args: ''
+    gerrit-project: ''
 
     description: |
       <!-- Managed by Jenkins Job Builder -->
@@ -150,6 +151,21 @@
           description: 'Repo manifest branch for code checkout (so we can display changes in Jenkins)'
 
       - string:
+          name: gerritProject
+          default: '{gerrit-project}'
+          description: 'Name of the Gerrit project'
+
+      - string:
+          name: gerritChangeNumber
+          default: ''
+          description: 'Changeset number in Gerrit'
+
+      - string:
+          name: gerritPatchsetNumber
+          default: ''
+          description: 'PatchSet number in Gerrit'
+
+      - string:
           name: testRuns
           default: '{test-runs}'
           description: 'How many times to repeat the tests'
@@ -173,7 +189,7 @@
 - job-template:
     id: 'voltha-patch-test'
     name: 'verify_{project}_sanity-test{name-extension}'
-    extra-helm-flags: ''
+    extra-helm-flags: '--set defaults.image_tag=master,use_ofagent_go=True,images.ofagent.repository=voltha/ofagent-go,images.ofagent.tag=master'
     skip-vote: false
 
     description: |
diff --git a/jjb/voltha-scale.yaml b/jjb/voltha-scale.yaml
index 33c33c5..71e20b6 100644
--- a/jjb/voltha-scale.yaml
+++ b/jjb/voltha-scale.yaml
@@ -24,20 +24,20 @@
           expectedOnus: 256
           BBSIMdelay: 1000
       - 'voltha-scale-measurements-periodic':
-          name: 'voltha-scale-measurements-periodic-8-48-200ms'
+          name: 'voltha-scale-measurements-periodic-10-20-200ms'
           build-node: 'onf-pod1-head-node'
           time-trigger: "H H/4 * * *"
-          onuPerPon: 48
-          ponPorts: 8
-          expectedOnus: 384
+          onuPerPon: 20
+          ponPorts: 10
+          expectedOnus: 200
           BBSIMdelay: 200
       - 'voltha-scale-measurements-periodic':
-          name: 'voltha-scale-measurements-periodic-8-48-1000ms'
+          name: 'voltha-scale-measurements-periodic-10-20-1000ms'
           build-node: 'onf-pod1-head-node'
           time-trigger: "H H/4 * * *"
-          onuPerPon: 48
-          ponPorts: 8
-          expectedOnus: 384
+          onuPerPon: 20
+          ponPorts: 10
+          expectedOnus: 200
           BBSIMdelay: 1000
       - 'voltha-scale-measurements-periodic':
           name: 'voltha-scale-measurements-periodic-4-32-200ms'
@@ -153,7 +153,7 @@
           BBSIMdelay: 1000
       - 'voltha-scale-measurements':
           name: 'voltha-scale-measurements-dev'
-          build-node: 'voltha-scale-dev'
+          build-node: 'voltha-scale'
           onuPerPon: 16
           ponPorts: 1
           expectedOnus: 16
@@ -252,20 +252,55 @@
           description: 'Custom image selection for BBSIM (repo:tag)'
 
       - string:
+          name: bbsimChart
+          default: onf/bbsim
+          description: 'BBSim chart name (or location on file system)'
+
+      - string:
           name: volthaImg
           default: 'voltha/voltha-rw-core:master'
           description: 'Custom image selection for VOLTHA (repo:tag)'
 
       - string:
+          name: ofAgentImg
+          default: 'voltha/voltha-ofagent:master'
+          description: 'Custom image selection for OfAgent (repo:tag)'
+
+      - string:
+          name: volthaChart
+          default: onf/voltha
+          description: 'VOLTHA chart name (or location on file system)'
+
+      - string:
           name: openoltAdapterImg
           default: voltha/voltha-openolt-adapter:master
           description: 'Custom image selection for Openolt Adapter (repo:tag)'
 
       - string:
+          name: openoltAdapterChart
+          default: onf/voltha-adapter-openolt
+          description: 'OpenOLT chart name (or location on file system)'
+
+      - string:
           name: openonuAdapterImg
           default: voltha/voltha-openonu-adapter:master
           description: 'Custom image selection for Openonu Adapter (repo:tag)'
 
+      - string:
+          name: openonuAdapterChart
+          default: onf/voltha-adapter-openonu
+          description: 'OpenONU chart name (or location on file system)'
+
+      - string:
+          name: onosImg
+          default: voltha/voltha-onos:4.0.1
+          description: 'Custom image selection for Openonu Adapter (repo:tag)'
+
+      - string:
+          name: extraHelmFlags
+          default: ''
+          description: 'Any extra helm parameters you want (passed to every helm install command)'
+
     project-type: pipeline
     concurrent: false
 
@@ -369,21 +404,56 @@
           description: 'Custom image selection for BBSIM (repo:tag)'
 
       - string:
+          name: bbsimChart
+          default: onf/bbsim
+          description: 'BBSim chart name (or location on file system)'
+
+      - string:
           name: volthaImg
           default: voltha/voltha-rw-core:master
           description: 'Custom image selection for VOLTHA (repo:tag)'
 
       - string:
+          name: ofAgentImg
+          default: 'voltha/voltha-ofagent:master'
+          description: 'Custom image selection for OfAgent (repo:tag)'
+
+      - string:
+          name: volthaChart
+          default: onf/voltha
+          description: 'VOLTHA chart name (or location on file system)'
+
+      - string:
           name: openoltAdapterImg
           default: voltha/voltha-openolt-adapter:master
           description: 'Custom image selection for Openolt Adapter (repo:tag)'
 
       - string:
+          name: openoltAdapterChart
+          default: onf/voltha-adapter-openolt
+          description: 'OpenOLT chart name (or location on file system)'
+
+      - string:
           name: openonuAdapterImg
           default: voltha/voltha-openonu-adapter:master
           description: 'Custom image selection for Openonu Adapter (repo:tag)'
 
+      - string:
+          name: openonuAdapterChart
+          default: onf/voltha-adapter-openonu
+          description: 'OpenONU chart name (or location on file system)'
+
+      - string:
+          name: onosImg
+          default: voltha/voltha-onos:4.0.1
+          description: 'Custom image selection for Openonu Adapter (repo:tag)'
+
+      - string:
+          name: extraHelmFlags
+          default: ''
+          description: 'Any extra helm parameters you want (passed to every helm install command)'
+
     project-type: pipeline
     concurrent: false
 
-    dsl: !include-raw-escape: pipeline/{pipeline-script}
\ No newline at end of file
+    dsl: !include-raw-escape: pipeline/{pipeline-script}