Merge "Using local chart for voltha-2.9 installations"
diff --git a/jjb/pipeline/voltha/master/bbsim-tests.groovy b/jjb/pipeline/voltha/master/bbsim-tests.groovy
index 527bf65..edfcb6c 100755
--- a/jjb/pipeline/voltha/master/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/master/bbsim-tests.groovy
@@ -42,6 +42,17 @@
       }
     }
   }
+  stage('Deploy common infrastructure') {
+    sh '''
+    helm repo add onf https://charts.opencord.org
+    helm repo update
+    if [ ${withMonitoring} = true ] ; then
+      helm install nem-monitoring onf/nem-monitoring \
+      --set prometheus.alertmanager.enabled=false,prometheus.pushgateway.enabled=false \
+      --set kpi_exporter.enabled=false,dashboards.xos=false,dashboards.onos=false,dashboards.aaa=false,dashboards.voltha=false
+    fi
+    '''
+  }
   stage('Deploy Voltha') {
     if (teardown) {
       timeout(10) {
@@ -104,6 +115,9 @@
         JENKINS_NODE_COOKIE="dontKillMe" _TAG="bbsim\${i}" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/bbsim\${i} \${bbsimDmiPortFwd}:50075; done"&
         ((bbsimDmiPortFwd++))
       done
+      if [ ${withMonitoring} = true ] ; then
+        JENKINS_NODE_COOKIE="dontKillMe" _TAG="nem-monitoring-prometheus-server" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n default svc/nem-monitoring-prometheus-server 31301:80; done"&
+      fi
       ps aux | grep port-forward
       """
       // setting ONOS log level
@@ -125,6 +139,16 @@
   }
   stage('Run test ' + testTarget + ' on ' + workflow + ' workFlow') {
     sh """
+    if [ ${withMonitoring} = true ] ; then
+      mkdir -p $WORKSPACE/voltha-pods-mem-consumption-${workflow}
+      cd $WORKSPACE/voltha-system-tests
+      make vst_venv
+      source ./vst_venv/bin/activate || true
+      # Collect initial memory consumption
+      python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace} || true
+    fi
+    """
+    sh """
     mkdir -p ${logsDir}
     export ROBOT_MISC_ARGS="-d ${logsDir} ${params.extraRobotArgs} "
     ROBOT_MISC_ARGS+="-v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace} -v container_log_dir:${logsDir} -v logging:${testLogging}"
@@ -140,6 +164,14 @@
       gzip *-combined.log || true
       rm *-combined.log || true
     """
+    sh """
+    if [ ${withMonitoring} = true ] ; then
+      cd $WORKSPACE/voltha-system-tests
+      source ./vst_venv/bin/activate || true
+      # Collect memory consumption of voltha pods once all the tests are complete
+      python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace} || true
+    fi
+    """
   }
 }
 
@@ -148,7 +180,7 @@
   sh """
   kubectl logs -n voltha -l app.kubernetes.io/part-of=voltha > $WORKSPACE/${exitStatus}/voltha.log || true
   """
-  archiveArtifacts artifacts: '**/*.log,**/*.gz,**/*.txt,**/*.html'
+  archiveArtifacts artifacts: '**/*.log,**/*.gz,**/*.txt,**/*.html,**/voltha-pods-mem-consumption-att/*,**/voltha-pods-mem-consumption-dt/*,**/voltha-pods-mem-consumption-tt/*'
   sh '''
     sync
     pkill kail || true
diff --git a/jjb/pipeline/voltha/master/software-upgrades.groovy b/jjb/pipeline/voltha/master/software-upgrades.groovy
index 0c259b9..f3e7a4c 100755
--- a/jjb/pipeline/voltha/master/software-upgrades.groovy
+++ b/jjb/pipeline/voltha/master/software-upgrades.groovy
@@ -25,7 +25,7 @@
 // returns the deployment version which is one less than the latest available tag of the repo, first voltha stack gets deployed using this;
 // returns the test version which is the latest tag of the repo, the component upgrade gets tested on this.
 // Note: if there is a major version change between deployment and test tags, then deployment tag will be same as test tag, i.e. both as latest.
-def get_voltha_comp_versions(component) {
+def get_voltha_comp_versions(component, base_deploy_tag) {
     def comp_test_tag = sh (
       script: "git ls-remote --refs --tags https://github.com/opencord/${component} | cut --delimiter='/' --fields=3 | tr '-' '~' | sort --version-sort | tail --lines=1 | sed 's/v//'",
       returnStdout: true
@@ -39,6 +39,10 @@
     if ( "${comp_deploy_major.trim()}" != "${comp_test_major.trim()}") {
       comp_deploy_tag = comp_test_tag
     }
+    if ( "${comp_test_tag.trim()}" == "${base_deploy_tag.trim()}") {
+      comp_deploy_tag = comp_test_tag
+      comp_test_tag = "master"
+    }
     println "${component}: deploy_tag: ${comp_deploy_tag}, test_tag: ${comp_test_tag}"
     return [comp_deploy_tag, comp_test_tag]
 }
@@ -91,13 +95,13 @@
       }
       if ("${name}" == "voltha-component-upgrade" || "${name}" == "voltha-component-rolling-upgrade") {
         // fetch voltha components versions/tags
-        (openolt_adapter_deploy_tag, openolt_adapter_test_tag) = get_voltha_comp_versions("voltha-openolt-adapter")
+        (openolt_adapter_deploy_tag, openolt_adapter_test_tag) = get_voltha_comp_versions("voltha-openolt-adapter", openoltAdapterDeployBaseTag.trim())
         extraHelmFlags = extraHelmFlags + " --set voltha-adapter-openolt.images.adapter_open_olt.tag=${openolt_adapter_deploy_tag} "
-        (openonu_adapter_deploy_tag, openonu_adapter_test_tag) = get_voltha_comp_versions("voltha-openonu-adapter-go")
+        (openonu_adapter_deploy_tag, openonu_adapter_test_tag) = get_voltha_comp_versions("voltha-openonu-adapter-go", openonuAdapterDeployBaseTag.trim())
         extraHelmFlags = extraHelmFlags + " --set voltha-adapter-openonu.images.adapter_open_onu_go.tag=${openonu_adapter_deploy_tag} "
-        (rw_core_deploy_tag, rw_core_test_tag) = get_voltha_comp_versions("voltha-go")
+        (rw_core_deploy_tag, rw_core_test_tag) = get_voltha_comp_versions("voltha-go", rwCoreDeployBaseTag.trim())
         extraHelmFlags = extraHelmFlags + " --set voltha.images.rw_core.tag=${rw_core_deploy_tag} "
-        (ofagent_deploy_tag, ofagent_test_tag) = get_voltha_comp_versions("ofagent-go")
+        (ofagent_deploy_tag, ofagent_test_tag) = get_voltha_comp_versions("ofagent-go", ofagentDeployBaseTag.trim())
         extraHelmFlags = extraHelmFlags + " --set voltha.images.ofagent.tag=${ofagent_deploy_tag} "
       }
       def localCharts = false
diff --git a/jjb/pipeline/voltha/master/voltha-scale-lwc-test.groovy b/jjb/pipeline/voltha/master/voltha-scale-lwc-test.groovy
index 3f66376..43e8e23 100644
--- a/jjb/pipeline/voltha/master/voltha-scale-lwc-test.groovy
+++ b/jjb/pipeline/voltha/master/voltha-scale-lwc-test.groovy
@@ -20,6 +20,12 @@
       remote: 'https://gerrit.opencord.org/ci-management.git'
 ])
 
+def lwc_helm_chart_path="/home/jenkins/Radisys_LWC_helm_charts"
+def value_file="/home/jenkins/lwc-values.yaml"
+def pon_count=16
+def onu_count=32
+def workflow="dt"
+
 pipeline {
 
   /* no label, executor is determined by JJB */
@@ -59,6 +65,7 @@
               // if we don't ONOS might get stuck because of all the events when BBSim goes down
               sh returnStdout: false, script: '''
               set +x
+              helm del -n infra voltha-infra || true
               helm del voltha-infra || true
               echo -ne "\nWaiting for ONOS to be removed..."
               onos=$(kubectl get pod -n default -l app=onos-classic --no-headers | wc -l)
@@ -98,9 +105,18 @@
         }
       }
     }
+    stage('Download Code') {
+      steps {
+        getVolthaCode([
+          branch: "${release}",
+          volthaSystemTestsChange: "${volthaSystemTestsChange}",
+          volthaHelmChartsChange: "${volthaHelmChartsChange}",
+        ])
+      }
+    }
     stage('Deploy Voltha') {
       steps {
-        timeout(time: 10, unit: 'MINUTES') {
+        timeout(time: 5, unit: 'MINUTES') {
           installVoltctl("${release}")
           script {
             startComponentsLogs([
@@ -116,7 +132,95 @@
             ])
           }
         }
+        timeout(time: 10, unit: 'MINUTES') {
+          sh """
+          cd /home/jenkins/Radisys_LWC_helm_charts
+
+          helm dep update ${lwc_helm_chart_path}/voltha-infra
+          helm upgrade --install --create-namespace -n infra voltha-infra ${lwc_helm_chart_path}/voltha-infra -f examples/${workflow}-values.yaml \
+            -f ${value_file} --wait
+
+          # helm dep update ${lwc_helm_chart_path}/voltha-stack
+          helm upgrade --install --create-namespace -n voltha1 voltha1 onf/voltha-stack \
+          --set voltha.ingress.enabled=true --set voltha.ingress.enableVirtualHosts=true --set voltha.fullHostnameOverride=voltha.scale1.dev \
+          -f ${value_file} --wait
+
+          helm upgrade --install -n voltha1 bbsim0 onf/bbsim --set olt_id=10 -f examples/${workflow}-values.yaml --set pon=${pon_count},onu=${onu_count} --version 4.6.0 --set oltRebootDelay=5 --wait
+          """
+        }
       }
     }
+    stage('Load MIB Template') {
+      when {
+        expression {
+          return params.withMibTemplate
+        }
+      }
+      steps {
+        sh """
+        # load MIB template
+        wget ${mibTemplateUrl} -O mibTemplate.json
+        cat mibTemplate.json | kubectl exec -it -n infra \$(kubectl get pods -n infra |grep etcd-0 | awk 'NR==1{print \$1}') -- etcdctl put service/voltha/omci_mibs/go_templates/BBSM/12345123451234512345/BBSM_IMG_00001
+        """
+      }
+    }
+    stage('Run Test') {
+      steps {
+        sh """
+          mkdir -p $WORKSPACE/RobotLogs
+          cd $WORKSPACE/voltha-system-tests
+          make vst_venv
+
+          daemonize -E JENKINS_NODE_COOKIE="dontKillMe" /usr/local/bin/kubectl port-forward -n infra svc/lwc 8182:8181 --address 0.0.0.0
+
+          source ./vst_venv/bin/activate
+          robot -d $WORKSPACE/RobotLogs \
+          --exitonfailure \
+          -v pon:${pon_count} -v onu:${onu_count} \
+          tests/scale/Voltha_Scale_Tests_lwc.robot
+
+          python tests/scale/collect-result.py -r $WORKSPACE/RobotLogs/output.xml -p $WORKSPACE/plots > $WORKSPACE/execution-time.txt || true
+          cat $WORKSPACE/execution-time.txt
+        """
+      }
+    }
+  }
+  post {
+    always {
+      stopComponentsLogs()
+      script {
+        try {
+          step([$class: 'RobotPublisher',
+            disableArchiveOutput: false,
+            logFileName: '**/log*.html',
+            otherFiles: '',
+            outputFileName: '**/output*.xml',
+            outputPath: 'RobotLogs',
+            passThreshold: 100,
+            reportFileName: '**/report*.html',
+            onlyCritical: true,
+            unstableThreshold: 0]);
+        } catch (Exception e) {
+            println "Cannot archive Robot Logs: ${e.toString()}"
+        }
+      }
+      plot([
+        csvFileName: 'scale-test.csv',
+        csvSeries: [
+          [file: 'plots/plot-lwc-olts.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],
+          [file: 'plots/plot-voltha-onus.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],
+          [file: 'plots/plot-lwc-ports.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],
+          [file: 'plots/plot-voltha-flows-before.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],
+          [file: 'plots/plot-voltha-openolt-flows-before.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],
+          [file: 'plots/plot-lwc-flows-before.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],
+          [file: 'plots/plot-voltha-flows-after.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],
+          [file: 'plots/plot-voltha-openolt-flows-after.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],
+          [file: 'plots/plot-lwc-flows-after.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],
+        ],
+        group: 'Voltha-Scale-Numbers', numBuilds: '20', style: 'line', title: "Scale Test (OLTs: ${olts}, PONs: ${pons}, ONUs: ${onus}, UNIs: ${unis})", yaxis: 'Time (s)', useDescr: true
+      ])
+      getPodsInfo("$LOG_FOLDER")
+      archiveArtifacts artifacts: 'execution-time.txt,logs/*,logs/pprof/*,RobotLogs/**/*,plots/*,etcd-metrics/*'
+    }
   }
 }
diff --git a/jjb/pipeline/voltha/master/voltha-scale-test.groovy b/jjb/pipeline/voltha/master/voltha-scale-test.groovy
index 2ac6094..43b5248 100644
--- a/jjb/pipeline/voltha/master/voltha-scale-test.groovy
+++ b/jjb/pipeline/voltha/master/voltha-scale-test.groovy
@@ -87,7 +87,7 @@
           }
           timeout(time: 10, unit: 'MINUTES') {
             script {
-              helmTeardown(["default"])
+              helmTeardown(["default", "voltha1", "infra"])
             }
             sh returnStdout: false, script: '''
               helm repo add onf https://charts.opencord.org
diff --git a/jjb/software-upgrades.yaml b/jjb/software-upgrades.yaml
index 8e1365c..ac458ac 100644
--- a/jjb/software-upgrades.yaml
+++ b/jjb/software-upgrades.yaml
@@ -12,19 +12,23 @@
           build-node: 'ubuntu18.04-basebuild-8c-15g'
           code-branch: 'master'
           aaa-version: '2.6.0.SNAPSHOT'
-          aaa-oar-url: 'https://oss.sonatype.org/content/groups/public/org/opencord/aaa-app/2.6.0-SNAPSHOT/aaa-app-2.6.0-20211227.162925-1.oar'
+          aaa-oar-url: 'https://oss.sonatype.org/content/groups/public/org/opencord/aaa-app/2.6.0-SNAPSHOT/aaa-app-2.6.0-20220318.145620-3.oar'
           olt-version: '5.1.0.SNAPSHOT'
-          olt-oar-url: 'https://oss.sonatype.org/content/groups/public/org/opencord/olt-app/5.1.0-SNAPSHOT/olt-app-5.1.0-20220217.163818-8.oar'
+          olt-oar-url: 'https://oss.sonatype.org/content/groups/public/org/opencord/olt-app/5.1.0-SNAPSHOT/olt-app-5.1.0-20220323.183932-13.oar'
           dhcpl2relay-version: '2.7.0.SNAPSHOT'
-          dhcpl2relay-oar-url: 'https://oss.sonatype.org/content/groups/public/org/opencord/dhcpl2relay-app/2.7.0-SNAPSHOT/dhcpl2relay-app-2.7.0-20211227.173623-1.oar'
+          dhcpl2relay-oar-url: 'https://oss.sonatype.org/content/groups/public/org/opencord/dhcpl2relay-app/2.7.0-SNAPSHOT/dhcpl2relay-app-2.7.0-20220318.145606-4.oar'
           igmpproxy-version: '2.5.0.SNAPSHOT'
-          igmpproxy-oar-url: 'https://oss.sonatype.org/content/groups/public/org/opencord/onos-app-igmpproxy-app/2.5.0-SNAPSHOT/onos-app-igmpproxy-app-2.5.0-20211227.182108-1.oar'
+          igmpproxy-oar-url: 'https://oss.sonatype.org/content/groups/public/org/opencord/onos-app-igmpproxy-app/2.5.0-SNAPSHOT/onos-app-igmpproxy-app-2.5.0-20220318.145612-3.oar'
           sadis-version: '5.6.0.SNAPSHOT'
-          sadis-oar-url: 'https://oss.sonatype.org/content/groups/public/org/opencord/sadis-app/5.6.0-SNAPSHOT/sadis-app-5.6.0-20220217.161332-5.oar'
+          sadis-oar-url: 'https://oss.sonatype.org/content/groups/public/org/opencord/sadis-app/5.6.0-SNAPSHOT/sadis-app-5.6.0-20220318.103602-7.oar'
           mcast-version: '2.6.0.SNAPSHOT'
-          mcast-oar-url: 'https://oss.sonatype.org/content/groups/public/org/opencord/mcast-app/2.6.0-SNAPSHOT/mcast-app-2.6.0-20211227.170629-1.oar'
+          mcast-oar-url: 'https://oss.sonatype.org/content/groups/public/org/opencord/mcast-app/2.6.0-SNAPSHOT/mcast-app-2.6.0-20220318.145711-3.oar'
           kafka-version: '2.9.0.SNAPSHOT'
-          kafka-oar-url: 'https://oss.sonatype.org/content/groups/public/org/opencord/kafka/2.9.0-SNAPSHOT/kafka-2.9.0-20211228.090937-1.oar'
+          kafka-oar-url: 'https://oss.sonatype.org/content/groups/public/org/opencord/kafka/2.9.0-SNAPSHOT/kafka-2.9.0-20220324.143019-4.oar'
+          openolt-adapter-deploy-base-tag: '4.2.2'
+          openonu-adapter-deploy-base-tag: '2.2.2'
+          rw-core-deploy-base-tag: '3.1.0'
+          ofagent-deploy-base-tag: '2.1.1'
           onu-image-version: 'BBSM_IMG_00002'
           onu-image-url: 'http://bbsim0:50074/images/software-image.img'
           onu-image-vendor: 'BBSM'
@@ -53,7 +57,7 @@
           kafka-version: '2.7.0'
           kafka-oar-url: 'https://oss.sonatype.org/content/groups/public/org/opencord/kafka/2.7.0/kafka-2.7.0.oar'
           adapter-open-olt-image: 'voltha/voltha-openolt-adapter:3.5.12'
-          adapter-open-onu-image: 'voltha/voltha-openonu-adapter-go:1.3.26'
+          adapter-open-onu-image: 'voltha/voltha-openonu-adapter-go:1.3.27'
           rw-core-image: 'voltha/voltha-rw-core:2.9.6'
           ofagent-image: 'voltha/voltha-ofagent-go:1.6.5'
           onu-image-version: 'BBSM_IMG_00002'
@@ -191,6 +195,26 @@
           description: 'ONOS Kafka App OAR File Url'
 
       - string:
+          name: openoltAdapterDeployBaseTag
+          default: '{openolt-adapter-deploy-base-tag}'
+          description: 'Minimum Version of OpenOLT Adapter to be used for Deployment'
+
+      - string:
+          name: openonuAdapterDeployBaseTag
+          default: '{openonu-adapter-deploy-base-tag}'
+          description: 'Minimum Version of OpenONU Adapter to be used for Deployment'
+
+      - string:
+          name: rwCoreDeployBaseTag
+          default: '{rw-core-deploy-base-tag}'
+          description: 'Minimum Version of RW Core to be used for Deployment'
+
+      - string:
+          name: ofagentDeployBaseTag
+          default: '{ofagent-deploy-base-tag}'
+          description: 'Minimum Version of Ofagent to be used for Deployment'
+
+      - string:
           name: onuImageVersion
           default: '{onu-image-version}'
           description: 'Version of ONU Image to Upgrade'
diff --git a/jjb/voltha-e2e.yaml b/jjb/voltha-e2e.yaml
index 9cbca48..3cb11f9 100755
--- a/jjb/voltha-e2e.yaml
+++ b/jjb/voltha-e2e.yaml
@@ -256,7 +256,7 @@
               logging: true
             - target: mib-upload-templating-openonu-go-adapter-test
               workflow: att
-              flags: "--set pon=2,onu=2,controlledActivation=only-onu"
+              flags: "--set pon=2,onu=2,controlledActivation=only-onu,injectOmciUnknownAttributes=true,injectOmciUnknownMe=true"
               teardown: true
               logging: true
             - target: reconcile-openonu-go-adapter-test-att
@@ -425,7 +425,7 @@
               logging: true
             - target: mib-upload-templating-openonu-go-adapter-test
               workflow: att
-              flags: "--set pon=2,onu=2,controlledActivation=only-onu"
+              flags: "--set pon=2,onu=2,controlledActivation=only-onu,injectOmciUnknownAttributes=true,injectOmciUnknownMe=true"
               teardown: true
               logging: true
             - target: reconcile-openonu-go-adapter-test-att
@@ -1333,6 +1333,45 @@
               teardown: true
               logging: true
 
+      - 'voltha-periodic-test':
+          name: 'periodic-voltha-memory-leak-test-bbsim'
+          build-node: 'ubuntu18.04-basebuild-4c-8g'
+          code-branch: 'master'
+          time-trigger: "H H/23 * * *"
+          extraHelmFlags: '--set global.image_tag=master --set onos-classic.image.tag=master'
+          withMonitoring: true
+          logLevel: 'DEBUG'
+          testTargets: |
+            - target: memory-leak-test-single-kind-att
+              workflow: att
+              flags: ""
+              teardown: true
+              logging: true
+            - target: memory-leak-test-single-kind-dt
+              workflow: dt
+              flags: ""
+              teardown: true
+              logging: true
+            - target: memory-leak-test-single-kind-tt
+              workflow: tt
+              flags: ""
+              teardown: true
+              logging: true
+          timeout: 360
+
+      - 'voltha-periodic-test':
+          name: 'periodic-voltha-dt-fttb-test-bbsim'
+          build-node: 'ubuntu18.04-basebuild-4c-8g'
+          code-branch: 'master'
+          time-trigger: "H H/23 * * *"
+          extraHelmFlags: '--set global.image_tag=master --set onos-classic.image.tag=master --set voltha-adapter-openonu.adapter_open_onu.uni_port_mask=0x00FF --set onu=2'
+          testTargets: |
+            - target: sanity-kind-dt-fttb
+              workflow: dt-fttb
+              flags: ""
+              teardown: true
+              logging: true
+
       # ATT Per-patchset Pod builds on Tucson pod (master)
       - 'verify_physical_voltha_patchset_auto':
           name: 'verify_physical_voltha_patchset_auto'
@@ -1402,6 +1441,7 @@
     timeout: 130
     logLevel: 'INFO'
     enableMacLearning: false
+    withMonitoring: false
 
     trigger-comment: vv7CBoQQYYonvaN8xcru
     time-trigger: 0 0 29 2 *
@@ -1498,6 +1538,11 @@
           default: '{logLevel}'
           description: 'Log level for all the components'
 
+      - bool:
+          name: withMonitoring
+          default: '{withMonitoring}'
+          description: 'Option to install Prometheus'
+
       - string:
           name: timeout
           default: '{timeout}'
@@ -1546,6 +1591,7 @@
     volthaHelmChartsChange: ''
     extraHelmFlags: ''
     enableMacLearning: false
+    withMonitoring: false
     robot-args: ''
     branch-regexp: '{all-branches-regexp}'
     testTargets: |
@@ -1657,6 +1703,11 @@
           default: '{registry}'
           description: 'Which registry to use (amazon vs menlo)'
 
+      - bool:
+          name: withMonitoring
+          default: '{withMonitoring}'
+          description: 'Option to install Prometheus'
+
       - string:
           name: logLevel
           default: '{logLevel}'
diff --git a/jjb/voltha-scale.yaml b/jjb/voltha-scale.yaml
index eb40623..968f4e3 100644
--- a/jjb/voltha-scale.yaml
+++ b/jjb/voltha-scale.yaml
@@ -521,6 +521,7 @@
           withIgmp: false
           extraHelmFlags: '--set authRetry=false,dhcpRetry=false'
           workflow: dt
+          volthaSystemTestsChange: refs/changes/10/31810/6
 
 
 # list of parameters for the VOLTHA Jobs,
diff --git a/jjb/voltha-test/voltha.yaml b/jjb/voltha-test/voltha.yaml
index 6350845..5790d0b 100644
--- a/jjb/voltha-test/voltha.yaml
+++ b/jjb/voltha-test/voltha.yaml
@@ -514,8 +514,8 @@
 
       # Berlin pod with olt/onu - voltha-2.8 versions timer based job , two OLTs
       - 'build_voltha_pod_release_timer':
-          build-node: 'dt-berlin-community-pod'
-          config-pod: 'dt-berlin-pod-multi-olt'
+          build-node: 'berlin-community-pod-1'
+          config-pod: 'berlin-community-pod-1-multi-olt'
           name-extension: '_DT'
           work-flow: 'DT'
           release: '2.8'
@@ -528,8 +528,8 @@
 
       # Berlin POD test job - voltha-2.8 versions: uses tech profile on voltha branch
       - 'build_voltha_pod_test':
-          build-node: 'dt-berlin-community-pod'
-          config-pod: 'dt-berlin-pod-multi-olt'
+          build-node: 'berlin-community-pod-1'
+          config-pod: 'berlin-community-pod-1-multi-olt'
           release: '2.8'
           branch: 'voltha-2.8'
           name-extension: '_DT'