Merge "Capturing ALLOC_ID and GEM_PORT info from BBSim in master scale test"
diff --git a/jjb/cord-test/voltha.yaml b/jjb/cord-test/voltha.yaml
index 4c56478..2fb494f 100644
--- a/jjb/cord-test/voltha.yaml
+++ b/jjb/cord-test/voltha.yaml
@@ -422,6 +422,7 @@
       - 'build_voltha_pod_soak_test':
           build-node: 'menlo-soak-pod'
           config-pod: 'onf-soak-pod'
+          'disable-job': true
           profile: '1T8GEM'
           branch: 'master'
           release: 'master'
@@ -437,6 +438,7 @@
       - 'build_voltha_pod_soak_test':
           build-node: 'menlo-soak-pod'
           config-pod: 'onf-soak-pod'
+          'disable-job': true
           profile: '1T8GEM'
           branch: 'master'
           release: 'master'
diff --git a/jjb/pipeline/voltha-dt-physical-functional-tests.groovy b/jjb/pipeline/voltha-dt-physical-functional-tests.groovy
index f07a112..868ff56 100644
--- a/jjb/pipeline/voltha-dt-physical-functional-tests.groovy
+++ b/jjb/pipeline/voltha-dt-physical-functional-tests.groovy
@@ -317,6 +317,14 @@
       gzip *-combined.log || true
       rm *-combined.log || true
 
+      # store information on running charts
+      helm ls > $WORKSPACE/helm-list.txt || true
+
+      # store information on the running pods
+      kubectl get pods -o wide > $WORKSPACE/pods.txt || true
+      kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\n'}" | sort | uniq | tee $WORKSPACE/pod-images.txt || true
+      kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.imageID}{'\\n'}" | sort | uniq | tee $WORKSPACE/pod-imagesId.txt || true
+
       # collect ETCD cluster logs
       mkdir -p $WORKSPACE/etcd
       printf '%s\n' $(kubectl get pods -l app=etcd -o=jsonpath="{.items[*]['metadata.name']}") | xargs -I% bash -c "kubectl logs % > $WORKSPACE/etcd/%.log"
@@ -347,7 +355,7 @@
         reportFileName: '**/report*.html',
         unstableThreshold: 0
         ]);
-      archiveArtifacts artifacts: '*.log,*.gz,*.tgz,etcd/*.log'
+      archiveArtifacts artifacts: '*.log,*.gz,*.tgz,etcd/*.log,*.txt'
     }
     unstable {
       step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "${notificationEmail}", sendToIndividuals: false])
diff --git a/jjb/pipeline/voltha-openonu-go-test-bbsim.groovy b/jjb/pipeline/voltha-openonu-go-test-bbsim.groovy
index e3129b8..0b73029 100755
--- a/jjb/pipeline/voltha-openonu-go-test-bbsim.groovy
+++ b/jjb/pipeline/voltha-openonu-go-test-bbsim.groovy
@@ -256,7 +256,6 @@
     }
 
     stage('Reconcile DT workflow') {
-      when { beforeAgent true; expression { return "${olts}" == "1" } }
       environment {
         ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/ReconcileDT"
       }
@@ -305,7 +304,6 @@
     }
 
     stage('Reconcile ATT workflow') {
-      when { beforeAgent true; expression { return "${olts}" == "1" } }
       environment {
         ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/ReconcileATT"
       }
@@ -360,7 +358,6 @@
     }
 
     stage('Reconcile TT workflow') {
-      when { beforeAgent true; expression { return "${olts}" == "1" } }
       environment {
         ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/ReconcileTT"
       }
diff --git a/jjb/pipeline/voltha-physical-functional-tests.groovy b/jjb/pipeline/voltha-physical-functional-tests.groovy
index 09e8c45..1c561be 100644
--- a/jjb/pipeline/voltha-physical-functional-tests.groovy
+++ b/jjb/pipeline/voltha-physical-functional-tests.groovy
@@ -283,6 +283,15 @@
       gzip *-combined.log || true
       rm *-combined.log || true
 
+      # store information on running charts
+      helm ls > $WORKSPACE/helm-list.txt || true
+
+      # store information on the running pods
+      kubectl get pods -o wide > $WORKSPACE/pods.txt || true
+      kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\n'}" | sort | uniq | tee $WORKSPACE/pod-images.txt || true
+      kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.imageID}{'\\n'}" | sort | uniq | tee $WORKSPACE/pod-imagesId.txt || true
+
+
       # collect ETCD cluster logs
       mkdir -p $WORKSPACE/etcd
       printf '%s\n' $(kubectl get pods -l app=etcd -o=jsonpath="{.items[*]['metadata.name']}") | xargs -I% bash -c "kubectl logs % > $WORKSPACE/etcd/%.log"
diff --git a/jjb/pipeline/voltha-tt-physical-functional-tests.groovy b/jjb/pipeline/voltha-tt-physical-functional-tests.groovy
index a7f97ca..2b8fb56 100644
--- a/jjb/pipeline/voltha-tt-physical-functional-tests.groovy
+++ b/jjb/pipeline/voltha-tt-physical-functional-tests.groovy
@@ -253,6 +253,15 @@
       cd $WORKSPACE
       gzip *-combined.log || true
       rm *-combined.log || true
+
+      # store information on running charts
+      helm ls > $WORKSPACE/helm-list.txt || true
+
+      # store information on the running pods
+      kubectl get pods -o wide > $WORKSPACE/pods.txt || true
+      kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\n'}" | sort | uniq | tee $WORKSPACE/pod-images.txt || true
+      kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.imageID}{'\\n'}" | sort | uniq | tee $WORKSPACE/pod-imagesId.txt || true
+
       '''
       script {
         deployment_config.olts.each { olt ->
diff --git a/jjb/pipeline/voltha/master/software-upgrades.groovy b/jjb/pipeline/voltha/master/software-upgrades.groovy
index 8a7d0ee..387cfe3 100644
--- a/jjb/pipeline/voltha/master/software-upgrades.groovy
+++ b/jjb/pipeline/voltha/master/software-upgrades.groovy
@@ -50,16 +50,43 @@
         mkdir -p \$ROBOT_LOGS_DIR
         if [[ ${name} == 'onos-app-upgrade' ]]; then
           export ONOS_APPS_UNDER_TEST+=''
-          if [[ ${aaaVer.trim()} != '' && ${aaaOarUrl.trim()} != '' ]]; then
-            ONOS_APPS_UNDER_TEST+="org.opencord.aaa,${aaaVer.trim()},${aaaOarUrl.trim()}"
+          if [ ${aaaVer.trim()} != '' ] && [ ${aaaOarUrl.trim()} != '' ]; then
+            ONOS_APPS_UNDER_TEST+="org.opencord.aaa,${aaaVer.trim()},${aaaOarUrl.trim()}*"
+          fi
+          if [ ${oltVer.trim()} != '' ] && [ ${oltOarUrl.trim()} != '' ]; then
+            ONOS_APPS_UNDER_TEST+="org.opencord.olt,${oltVer.trim()},${oltOarUrl.trim()}*"
+          fi
+          if [ ${dhcpl2relayVer.trim()} != '' ] && [ ${dhcpl2relayOarUrl.trim()} != '' ]; then
+            ONOS_APPS_UNDER_TEST+="org.opencord.dhcpl2relay,${dhcpl2relayVer.trim()},${dhcpl2relayOarUrl.trim()}*"
+          fi
+          if [ ${igmpproxyVer.trim()} != '' ] && [ ${igmpproxyOarUrl.trim()} != '' ]; then
+            ONOS_APPS_UNDER_TEST+="org.opencord.igmpproxy,${igmpproxyVer.trim()},${igmpproxyOarUrl.trim()}*"
+          fi
+          if [ ${sadisVer.trim()} != '' ] && [ ${sadisOarUrl.trim()} != '' ]; then
+            ONOS_APPS_UNDER_TEST+="org.opencord.sadis,${sadisVer.trim()},${sadisOarUrl.trim()}*"
+          fi
+          if [ ${mcastVer.trim()} != '' ] && [ ${mcastOarUrl.trim()} != '' ]; then
+            ONOS_APPS_UNDER_TEST+="org.opencord.mcast,${mcastVer.trim()},${mcastOarUrl.trim()}*"
+          fi
+          if [ ${kafkaVer.trim()} != '' ] && [ ${kafkaOarUrl.trim()} != '' ]; then
+            ONOS_APPS_UNDER_TEST+="org.opencord.kafka,${kafkaVer.trim()},${kafkaOarUrl.trim()}*"
           fi
           export ROBOT_MISC_ARGS="-d \$ROBOT_LOGS_DIR -v onos_apps_under_test:\$ONOS_APPS_UNDER_TEST -e PowerSwitch"
           export TARGET=onos-app-upgrade-test
         fi
         if [[ ${name} == 'voltha-component-upgrade' ]]; then
           export VOLTHA_COMPS_UNDER_TEST+=''
-          if [[ ${adapterOpenOltImage.trim()} != '' ]]; then
-            VOLTHA_COMPS_UNDER_TEST+="adapter-open-olt,adapter-open-olt,${adapterOpenOltImage.trim()}"
+          if [ ${adapterOpenOltImage.trim()} != '' ]; then
+            VOLTHA_COMPS_UNDER_TEST+="adapter-open-olt,adapter-open-olt,${adapterOpenOltImage.trim()}*"
+          fi
+          if [ ${adapterOpenOnuImage.trim()} != '' ]; then
+            VOLTHA_COMPS_UNDER_TEST+="adapter-open-onu,adapter-open-onu,${adapterOpenOnuImage.trim()}*"
+          fi
+          if [ ${rwCoreImage.trim()} != '' ]; then
+            VOLTHA_COMPS_UNDER_TEST+="rw-core,voltha,${rwCoreImage.trim()}*"
+          fi
+          if [ ${ofAgentImage.trim()} != '' ]; then
+            VOLTHA_COMPS_UNDER_TEST+="ofagent,ofagent,${ofAgentImage.trim()}*"
           fi
           export ROBOT_MISC_ARGS="-d \$ROBOT_LOGS_DIR -v voltha_comps_under_test:\$VOLTHA_COMPS_UNDER_TEST -e PowerSwitch"
           export TARGET=voltha-comp-upgrade-test
diff --git a/jjb/pipeline/voltha/master/voltha-scale-test.groovy b/jjb/pipeline/voltha/master/voltha-scale-test.groovy
index 68f5b21..a29c474 100644
--- a/jjb/pipeline/voltha/master/voltha-scale-test.groovy
+++ b/jjb/pipeline/voltha/master/voltha-scale-test.groovy
@@ -368,6 +368,10 @@
           sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 log:set ${logLevel} org.onosproject
           sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 log:set ${logLevel} org.opencord
 
+          sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 log:set DEBUG org.opencord.cordmcast
+          sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 log:set DEBUG org.onosproject.mcast
+          sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 log:set DEBUG org.opencord.igmpproxy
+          sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 log:set DEBUG org.opencord.olt
 
           kubectl exec \$(kubectl get pods | grep -E "bbsim[0-9]" | awk 'NR==1{print \$1}') -- bbsimctl log ${logLevel.toLowerCase()} false
 
@@ -500,7 +504,7 @@
       steps {
         sh '''
           set +e
-          mkdir -p $WORKSPACE/RobotLogs
+          mkdir -p $ROBOT_LOGS_DIR
           cd $WORKSPACE/voltha-system-tests
           make vst_venv
         '''
diff --git a/jjb/software-upgrades.yaml b/jjb/software-upgrades.yaml
index 4c3a4e1..ddf8923 100644
--- a/jjb/software-upgrades.yaml
+++ b/jjb/software-upgrades.yaml
@@ -13,7 +13,22 @@
           code-branch: 'master'
           aaa-version: '2.3.0.SNAPSHOT'
           aaa-oar-url: 'https://oss.sonatype.org/content/groups/public/org/opencord/aaa-app/2.3.0-SNAPSHOT/aaa-app-2.3.0-20201210.223737-1.oar'
+          olt-version: ''
+          olt-oar-url: ''
+          dhcpl2relay-version: ''
+          dhcpl2relay-oar-url: ''
+          igmpproxy-version: ''
+          igmpproxy-oar-url: ''
+          sadis-version: ''
+          sadis-oar-url: ''
+          mcast-version: ''
+          mcast-oar-url: ''
+          kafka-version: ''
+          kafka-oar-url: ''
           adapter-open-olt-image: 'voltha/voltha-openolt-adapter:3.1.3'
+          adapter-open-onu-image: 'voltha/voltha-openonu-adapter-go:1.2.4'
+          rw-core-image: 'voltha/voltha-rw-core:2.6.3'
+          ofagent-image: 'voltha/voltha-ofagent-go:1.5.1'
           time-trigger: "H H/23 * * *"
 
 - job-template:
@@ -85,14 +100,85 @@
           default: '{aaa-oar-url}'
           description: 'ONOS AAA App OAR File Url'
 
-      # TODO: Add other ONOS Apps
+      - string:
+          name: oltVer
+          default: '{olt-version}'
+          description: 'ONOS OLT App Version to Test Upgrade'
+
+      - string:
+          name: oltOarUrl
+          default: '{olt-oar-url}'
+          description: 'ONOS OLT App OAR File Url'
+
+      - string:
+          name: dhcpl2relayVer
+          default: '{dhcpl2relay-version}'
+          description: 'ONOS DHCP L2 Relay App Version to Test Upgrade'
+
+      - string:
+          name: dhcpl2relayOarUrl
+          default: '{dhcpl2relay-oar-url}'
+          description: 'ONOS DHCP L2 Relay App OAR File Url'
+
+      - string:
+          name: igmpproxyVer
+          default: '{igmpproxy-version}'
+          description: 'ONOS Igmp Proxy App Version to Test Upgrade'
+
+      - string:
+          name: igmpproxyOarUrl
+          default: '{igmpproxy-oar-url}'
+          description: 'ONOS Igmp Proxy App OAR File Url'
+
+      - string:
+          name: sadisVer
+          default: '{sadis-version}'
+          description: 'ONOS Sadis App Version to Test Upgrade'
+
+      - string:
+          name: sadisOarUrl
+          default: '{sadis-oar-url}'
+          description: 'ONOS Sadis App OAR File Url'
+
+      - string:
+          name: mcastVer
+          default: '{mcast-version}'
+          description: 'ONOS MCast App Version to Test Upgrade'
+
+      - string:
+          name: mcastOarUrl
+          default: '{mcast-oar-url}'
+          description: 'ONOS MCast App OAR File Url'
+
+      - string:
+          name: kafkaVer
+          default: '{kafka-version}'
+          description: 'ONOS Kafka App Version to Test Upgrade'
+
+      - string:
+          name: kafkaOarUrl
+          default: '{kafka-oar-url}'
+          description: 'ONOS Kafka App OAR File Url'
 
       - string:
           name: adapterOpenOltImage
           default: '{adapter-open-olt-image}'
           description: 'Voltha Adapter Open OLT Component Image'
 
-      # TODO: Add other Voltha Components
+      - string:
+          name: adapterOpenOnuImage
+          default: '{adapter-open-onu-image}'
+          description: 'Voltha Adapter Open ONU Component Image'
+
+      - string:
+          name: rwCoreImage
+          default: '{rw-core-image}'
+          description: 'Voltha RW Core Component Image'
+
+      - string:
+          name: ofAgentImage
+          default: '{ofagent-image}'
+          description: 'Voltha Ofagent Component Image'
 
     project-type: pipeline
     concurrent: true