Merge "[VOL-3993] Adding att (becuase of high packet handling) job with maclearner app"
diff --git a/jjb/pipeline/voltha/master/software-upgrades.groovy b/jjb/pipeline/voltha/master/software-upgrades.groovy
index a6a56b4..7d1b139 100755
--- a/jjb/pipeline/voltha/master/software-upgrades.groovy
+++ b/jjb/pipeline/voltha/master/software-upgrades.groovy
@@ -87,7 +87,7 @@
     }
   }
   stage('Test - '+ name) {
-    timeout(60) {
+    timeout(75) {
       sh """
         ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/${name}"
         mkdir -p \$ROBOT_LOGS_DIR
@@ -197,7 +197,7 @@
     label "${params.buildNode}"
   }
   options {
-    timeout(time: 120, unit: 'MINUTES')
+    timeout(time: 150, unit: 'MINUTES')
   }
   environment {
     PATH="$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin"
diff --git a/jjb/pipeline/voltha/master/voltha-scale-multi-stack.groovy b/jjb/pipeline/voltha/master/voltha-scale-multi-stack.groovy
index c58da51..21331f8 100644
--- a/jjb/pipeline/voltha/master/voltha-scale-multi-stack.groovy
+++ b/jjb/pipeline/voltha/master/voltha-scale-multi-stack.groovy
@@ -429,6 +429,7 @@
             -v withIgmp:${withIgmp} \
             --noncritical non-critical \
             -e igmp \
+            -e onu-upgrade \
             -e teardown "
 
           if [ ${withEapol} = false ] ; then
diff --git a/jjb/pipeline/voltha/master/voltha-scale-test.groovy b/jjb/pipeline/voltha/master/voltha-scale-test.groovy
index 39c1f05..0ef497e 100644
--- a/jjb/pipeline/voltha/master/voltha-scale-test.groovy
+++ b/jjb/pipeline/voltha/master/voltha-scale-test.groovy
@@ -466,7 +466,7 @@
               -v withDhcp:${withDhcp} \
               -v withIgmp:${withIgmp} \
               --noncritical non-critical \
-              -e igmp -e teardown "
+              -e onu-upgrade -e igmp -e teardown "
 
             if [ ${withEapol} = false ] ; then
               ROBOT_PARAMS+="-e authentication "
@@ -485,6 +485,10 @@
               ROBOT_PARAMS+="-i setup -i activation "
             fi
 
+            if [ ${withOnuUpgrade} = true ] ; then
+              ROBOT_PARAMS+="-e flow-before "
+            fi
+
             cd $WORKSPACE/voltha-system-tests
             source ./vst_venv/bin/activate
             robot -d $WORKSPACE/RobotLogs \
@@ -496,6 +500,71 @@
         }
       }
     }
+    stage('Run ONU Upgrade Tests') {
+      environment {
+        ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/OnuUpgradeTests"
+      }
+      when {
+        expression {
+          return params.withOnuUpgrade
+        }
+      }
+      options {
+          timeout(time: 11, unit: 'MINUTES')
+      }
+      steps {
+        sh '''
+          set +e
+          mkdir -p $ROBOT_LOGS_DIR
+          cd $WORKSPACE/voltha-system-tests
+          make vst_venv
+        '''
+        script {
+          Exception caughtException = null
+
+          catchError(buildResult: 'SUCCESS', stageResult: 'ABORTED') {
+            try {
+              sh '''
+                ROBOT_PARAMS="--exitonfailure \
+                  -v olt:${olts} \
+                  -v pon:${pons} \
+                  -v onu:${onus} \
+                  -v workflow:${workflow} \
+                  -v withEapol:${withEapol} \
+                  -v withDhcp:${withDhcp} \
+                  -v withIgmp:${withIgmp} \
+                  -v image_version:BBSM_IMG_00002 \
+                  -v image_url:http://bbsim0:50074/images/software-image.img \
+                  -v image_vendor:BBSM \
+                  -v image_activate_on_success:false \
+                  -v image_commit_on_success:false \
+                  -v image_crc:0 \
+                  -v ONOS_SSH_PORT:30115 \
+                  -v ONOS_REST_PORT:30120 \
+                  --noncritical non-critical \
+                  -i onu-upgrade \
+                  -e setup -e activation -e flow-before \
+                  -e authentication -e provision -e flow-after \
+                  -e dhcp -e igmp -e teardown "
+                cd $WORKSPACE/voltha-system-tests
+                source ./vst_venv/bin/activate
+                robot -d $ROBOT_LOGS_DIR \
+                $ROBOT_PARAMS tests/scale/Voltha_Scale_Tests.robot
+              '''
+            } catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e) {
+              // if the error is a timeout don't mark the build as failed
+              println "ONU Upgrade test timed out"
+            } catch (Throwable e) {
+              caughtException = e
+            }
+          }
+
+          if (caughtException) {
+            error caughtException.message
+          }
+        }
+      }
+    }
     stage('Run Igmp Tests') {
       environment {
         ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/IgmpTests"
@@ -538,7 +607,7 @@
                   -i igmp \
                   -e setup -e activation -e flow-before \
                   -e authentication -e provision -e flow-after \
-                  -e dhcp -e teardown "
+                  -e dhcp -e onu-upgrade -e teardown "
                 cd $WORKSPACE/voltha-system-tests
                 source ./vst_venv/bin/activate
                 robot -d $ROBOT_LOGS_DIR \
@@ -560,7 +629,7 @@
     }
     stage("Device removal") {
       options {
-          timeout(time: 5, unit: 'MINUTES')
+          timeout(time: 15, unit: 'MINUTES')
       }
       steps {
         sh '''
diff --git a/jjb/pipeline/voltha/voltha-2.8/voltha-scale-multi-stack.groovy b/jjb/pipeline/voltha/voltha-2.8/voltha-scale-multi-stack.groovy
index f7e82de..8161b70 100644
--- a/jjb/pipeline/voltha/voltha-2.8/voltha-scale-multi-stack.groovy
+++ b/jjb/pipeline/voltha/voltha-2.8/voltha-scale-multi-stack.groovy
@@ -426,6 +426,7 @@
             -v withIgmp:${withIgmp} \
             --noncritical non-critical \
             -e igmp \
+            -e onu-upgrade \
             -e teardown "
 
           if [ ${withEapol} = false ] ; then
diff --git a/jjb/verify/up4.yaml b/jjb/verify/up4.yaml
index 5baf8e3..f70bc05 100644
--- a/jjb/verify/up4.yaml
+++ b/jjb/verify/up4.yaml
@@ -8,6 +8,12 @@
     github_pr_auth_id: "64fe2b1a-b33a-4f13-8442-ad8360434003"
     github_pr_org_list:
       - "omec-project"
+
+    # Matrix
+    up4Ctrl:
+        - pfcp
+        - p4rt
+
     jobs:
       - "up4-jobs":
           stream: "master"
@@ -18,13 +24,13 @@
 - job-group:
     name: "up4-jobs"
     jobs:
-      - "up4-pr-verify"
-      - "up4-postmerge"
-      - "up4-devel-nightly"
+      - "up4-pr-verify-{up4Ctrl}"
+      - "up4-postmerge-{up4Ctrl}"
+      - "up4-devel-nightly-{up4Ctrl}"
 
 - job-template:
-    id: "up4-pr-verify"
-    name: "up4-pr-verify"
+    id: "up4-pr-verify-{up4Ctrl}"
+    name: "up4-pr-verify-{up4Ctrl}"
     description: |
       Created by {id} job-template from ci-management/up4.yaml<br/>
 
@@ -49,6 +55,10 @@
           name: sha1
           default: "origin/{stream}"
           description: "SHA string of the pull request commit to build. Populated by GitHUb Pull Request Builder."
+      - string:
+          name: up4Ctrl
+          default: "{up4Ctrl}"
+          description: "Run p4rt or pfpc -based STC scenarios: [p4rt, pfcp]"
 
     # Run pipeline that is checked-in in the same PR we want to verify.
     pipeline-scm:
@@ -68,7 +78,7 @@
       - cord-infra-github-pr-trigger:
           github_pr_org_list: "{obj:github_pr_org_list}"
           github_pr_auth_id: "{github_pr_auth_id}"
-          status_context: "up4-pr-verify"
+          status_context: "up4-pr-verify-{up4Ctrl}"
 
     wrappers:
       - lf-infra-wrappers:
@@ -80,8 +90,8 @@
               variable: CODECOV_TOKEN
 
 - job-template:
-    id: "up4-postmerge"
-    name: "up4-postmerge"
+    id: "up4-postmerge-{up4Ctrl}"
+    name: "up4-postmerge-{up4Ctrl}"
     description: |
       Created by {id} job-template from ci-management/up4.yaml<br/>
 
@@ -106,6 +116,10 @@
           name: commitHash
           default: 'origin/{stream}'
           description: 'SHA string of the merged commit to build. Populated by Generic Webhook Trigger.'
+      - string:
+          name: up4Ctrl
+          default: "{up4Ctrl}"
+          description: "Run p4rt or pfpc -based STC scenarios: [p4rt, pfcp]"
 
     pipeline-scm:
       script-path: ".jenkins/Jenkinsfile"
@@ -135,8 +149,8 @@
               variable: CODECOV_TOKEN
 
 - job-template:
-    id: "up4-devel-nightly"
-    name: "up4-devel-nightly"
+    id: "up4-devel-nightly-{up4Ctrl}"
+    name: "up4-devel-nightly-{up4Ctrl}"
     description: |
       Created by {id} job-template from ci-management/up4.yaml<br/>
 
@@ -165,6 +179,10 @@
           name: up4Env
           default: 'devel'
           description: 'UP4 STC scenarios environment (stable or devel)'
+      - string:
+          name: up4Ctrl
+          default: "{up4Ctrl}"
+          description: "Run p4rt or pfpc -based STC scenarios: [p4rt, pfcp]"
 
     pipeline-scm:
       script-path: ".jenkins/Jenkinsfile"
diff --git a/jjb/voltha-e2e.yaml b/jjb/voltha-e2e.yaml
index 3086a3e..98ce37c 100755
--- a/jjb/voltha-e2e.yaml
+++ b/jjb/voltha-e2e.yaml
@@ -181,7 +181,7 @@
               flags: ""
               teardown: false
               logging: true
-          timeout: 180
+          timeout: 360
 
       - 'voltha-periodic-test':
           name: 'periodic-voltha-multi-uni-test-bbsim-2.8'
@@ -269,9 +269,9 @@
               flags: ""
               teardown: true
               logging: true
-            - target: reconcile-openonu-go-adapter-test-tt
+            - target: reconcile-openonu-go-adapter-multi-uni-test-tt
               workflow: tt
-              flags: ""
+              flags: " --set voltha-adapter-openonu.adapter_open_onu.uni_port_mask=0x00FF "
               teardown: true
               logging: true
             - target: openonu-go-adapter-omci-hardening-passed-test
@@ -314,6 +314,7 @@
               flags: ""
               teardown: true
               logging: true
+          timeout: 180
 
       - 'voltha-periodic-test':
           name: 'periodic-voltha-openonu-go-test-bbsim-2.8'
@@ -422,9 +423,9 @@
               flags: ""
               teardown: true
               logging: true
-            - target: reconcile-openonu-go-adapter-test-tt
+            - target: reconcile-openonu-go-adapter-multi-uni-test-tt
               workflow: tt
-              flags: ""
+              flags: " --set voltha-adapter-openonu.adapter_open_onu.uni_port_mask=0x00FF "
               teardown: true
               logging: true
             - target: openonu-go-adapter-omci-hardening-passed-test
@@ -467,6 +468,7 @@
               flags: ""
               teardown: true
               logging: true
+          timeout: 180
 
       - 'voltha-periodic-test':
           name: 'patchset-voltha-2.8-openonu-go-test-bbsim'
@@ -570,9 +572,9 @@
               flags: ""
               teardown: true
               logging: true
-            - target: reconcile-openonu-go-adapter-multi-olt-test-tt
+            - target: reconcile-openonu-go-adapter-multi-olt-multi-uni-test-tt
               workflow: tt
-              flags: ""
+              flags: " --set voltha-adapter-openonu.adapter_open_onu.uni_port_mask=0x00FF "
               teardown: true
               logging: true
             - target: voltha-onu-omci-get-multiolt-kind-att
@@ -606,7 +608,7 @@
               teardown: true
               logging: true
           time-trigger: "H H/12 * * *"
-          timeout: 160
+          timeout: 235
 
       - 'voltha-periodic-test':
           name: 'periodic-voltha-multiple-olts-openonu-go-test-bbsim-2.8'
@@ -697,9 +699,9 @@
               flags: ""
               teardown: true
               logging: true
-            - target: reconcile-openonu-go-adapter-multi-olt-test-tt
+            - target: reconcile-openonu-go-adapter-multi-olt-multi-uni-test-tt
               workflow: tt
-              flags: ""
+              flags: " --set voltha-adapter-openonu.adapter_open_onu.uni_port_mask=0x00FF "
               teardown: true
               logging: true
             - target: voltha-onu-omci-get-multiolt-kind-att
@@ -733,7 +735,7 @@
               teardown: true
               logging: true
           olts: 2
-          timeout: 160
+          timeout: 235
 
       - 'voltha-periodic-test':
           name: 'patchset-voltha-2.8-multiple-olts-openonu-go-test-bbsim'
diff --git a/jjb/voltha-scale.yaml b/jjb/voltha-scale.yaml
index 93ceb4e..4439528 100644
--- a/jjb/voltha-scale.yaml
+++ b/jjb/voltha-scale.yaml
@@ -66,6 +66,22 @@
           extraHelmFlags: '--set authRetry=false,dhcpRetry=false -f /home/jenkins/voltha-scale/voltha-values.yaml --set etcd.persistence.enabled=true,etcd.persistence.storageClass=longhorn'
 
       - 'voltha-scale-measurements':
+          name: 'voltha-scale-measurements-master-onu-upgrade-2-16-32-att-onus'
+          'disable-job': false
+          build-node: 'voltha-scale-1'
+          time-trigger: "H H/12 * * *"
+          olts: 2
+          pons: 16
+          onus: 32
+          withOnuUpgrade: true
+          withFlows: true
+          provisionSubscribers: false
+          withEapol: false
+          withDhcp: false
+          withIgmp: false
+          extraHelmFlags: '--set authRetry=false,dhcpRetry=false -f /home/jenkins/voltha-scale/voltha-values.yaml --set etcd.persistence.enabled=true,etcd.persistence.storageClass=longhorn --set voltha-adapter-openonu.adapter_open_onu.omci_timeout=3s'
+
+      - 'voltha-scale-measurements':
           name: 'voltha-scale-measurements-master-2-16-32-dt-subscribers'
           'disable-job': false
           build-node: 'voltha-scale-1'
@@ -515,34 +531,39 @@
           description: 'Which workflow are we testing (att, dt, tt)'
 
       - bool:
+          name: withOnuUpgrade
+          default: '{withOnuUpgrade}'
+          description: 'Whether to run the ONU Upgrade test'
+
+      - bool:
           name: withFlows
           default: '{withFlows}'
-          description: 'Wheter to push flows from ONOS'
+          description: 'Whether to push flows from ONOS'
 
       - bool:
           name: provisionSubscribers
           default: '{provisionSubscribers}'
-          description: 'Wheter to provision subscribers durint the tests'
+          description: 'Whether to provision subscribers during the tests'
 
       - bool:
           name: withEapol
           default: '{withEapol}'
-          description: 'Wheter EAPOL is enabled for the test'
+          description: 'Whether EAPOL is enabled for the test'
 
       - bool:
           name: withDhcp
           default: '{withDhcp}'
-          description: 'Wheter DHCP is enabled for the test'
+          description: 'Whether DHCP is enabled for the test'
 
       - bool:
           name: withIgmp
           default: '{withIgmp}'
-          description: 'Wheter IGMP is enabled for the test'
+          description: 'Whether IGMP is enabled for the test'
 
       - bool:
           name: withLLDP
           default: '{withLLDP}'
-          description: 'Wheter Link Discovery is enabled for the test'
+          description: 'Whether Link Discovery is enabled for the test'
 
       - bool:
           name: withMonitoring
@@ -718,6 +739,7 @@
     pons: 2
     onus: 2
     workflow: att
+    withOnuUpgrade: false
     withFlows: false
     provisionSubscribers: false
     withEapol: true
@@ -823,6 +845,7 @@
     pons: 2
     onus: 2
     workflow: att
+    withOnuUpgrade: false
     withFlows: true
     provisionSubscribers: true
     withEapol: true