Merge "Supporting DT workflow in tucson pipeline"
diff --git a/jjb/omec-ci.yaml b/jjb/omec-ci.yaml
index fa09154..6e312ac 100644
--- a/jjb/omec-ci.yaml
+++ b/jjb/omec-ci.yaml
@@ -141,7 +141,7 @@
           build-node: 'ubuntu18.04-basebuild-8c-15g'
       - 'omec-container'
       - 'omec-container-remote':
-          pod: 'dev'
+          pod: 'ci-4g'
           remote-jenkins-name: 'Aether Jenkins'
           pipeline-file: 'omec-container-remote.groovy'
 
@@ -213,7 +213,7 @@
           build-node: 'ubuntu18.04-basebuild-1c-2g'
       - 'omec-container'
       - 'omec-container-remote':
-          pod: 'dev'
+          pod: 'ci-4g'
           remote-jenkins-name: 'Aether Jenkins'
           pipeline-file: 'omec-container-remote.groovy'
 
@@ -285,7 +285,7 @@
           extraEnvironmentVars: CPU=haswell
       - 'omec-container'
       - 'omec-container-remote':
-          pod: 'dev'
+          pod: 'ci-4g'
           remote-jenkins-name: 'Aether Jenkins'
           pipeline-file: 'omec-container-remote.groovy'
 
diff --git a/jjb/pipeline/voltha-scale-multi-stack.groovy b/jjb/pipeline/voltha-scale-multi-stack.groovy
index b10224d..8264387 100644
--- a/jjb/pipeline/voltha-scale-multi-stack.groovy
+++ b/jjb/pipeline/voltha-scale-multi-stack.groovy
@@ -103,7 +103,7 @@
             rm -rf $WORKSPACE/*
 
             # remove orphaned port-forward from different namespaces
-            ps aux | grep port-forw | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9
+            ps aux | grep port-forw | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9 || true
           """
         }
       }
diff --git a/jjb/pipeline/voltha/master/bbsim-tests.groovy b/jjb/pipeline/voltha/master/bbsim-tests.groovy
index 7a6fd92..a67d32a 100644
--- a/jjb/pipeline/voltha/master/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/master/bbsim-tests.groovy
@@ -98,7 +98,7 @@
       // remove port-forwarding
       sh """
         # remove orphaned port-forward from different namespaces
-        ps aux | grep port-forw | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9
+        ps aux | grep port-forw | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9 || true
       """
       // collect pod details
       getPodsInfo("$WORKSPACE/${name}")
diff --git a/jjb/pipeline/voltha/master/periodic-bbsim-tests.groovy b/jjb/pipeline/voltha/master/periodic-bbsim-tests.groovy
old mode 100644
new mode 100755
index f7ea4d7..d05a406
--- a/jjb/pipeline/voltha/master/periodic-bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/master/periodic-bbsim-tests.groovy
@@ -36,7 +36,7 @@
         timeout(1) {
           sh returnStdout: false, script: '''
           # remove orphaned port-forward from different namespaces
-          ps aux | grep port-forw | grep -v grep | awk '{print $2}' | xargs --no-run-if-empty kill -9
+          ps aux | grep port-forw | grep -v grep | awk '{print $2}' | xargs --no-run-if-empty kill -9 || true
           '''
         }
       }
@@ -162,7 +162,7 @@
     label "${params.buildNode}"
   }
   options {
-    timeout(time: 130, unit: 'MINUTES')
+    timeout(time: "${timeout}", unit: 'MINUTES')
   }
   environment {
     KUBECONFIG="$HOME/.kube/kind-${clusterName}"
diff --git a/jjb/pipeline/voltha/master/physical-build.groovy b/jjb/pipeline/voltha/master/physical-build.groovy
index 87cd9d4..05a059e 100644
--- a/jjb/pipeline/voltha/master/physical-build.groovy
+++ b/jjb/pipeline/voltha/master/physical-build.groovy
@@ -76,7 +76,7 @@
           timeout(1) {
             sh returnStdout: false, script: '''
             # remove orphaned port-forward from different namespaces
-            ps aux | grep port-forw | grep -v grep | awk '{print $2}' | xargs --no-run-if-empty kill -9
+            ps aux | grep port-forw | grep -v grep | awk '{print $2}' | xargs --no-run-if-empty kill -9 || true
             '''
           }
         }
diff --git a/jjb/pipeline/voltha/master/software-upgrades.groovy b/jjb/pipeline/voltha/master/software-upgrades.groovy
index 86443a3..aca2d7f 100644
--- a/jjb/pipeline/voltha/master/software-upgrades.groovy
+++ b/jjb/pipeline/voltha/master/software-upgrades.groovy
@@ -134,7 +134,7 @@
       // remove port-forwarding
       sh """
         # remove orphaned port-forward from different namespaces
-        ps aux | grep port-forw | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9
+        ps aux | grep port-forw | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9 || true
       """
       // collect pod details
       get_pods_info("$WORKSPACE/${name}")
@@ -185,7 +185,7 @@
         // remove port-forwarding
         sh """
           # remove orphaned port-forward from different namespaces
-          ps aux | grep port-forw | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9
+          ps aux | grep port-forw | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9 || true
         """
         helmTeardown(['infra', 'voltha'])
       }
diff --git a/jjb/pipeline/voltha/master/tucson-build-and-test.groovy b/jjb/pipeline/voltha/master/tucson-build-and-test.groovy
index 18a66c8..3c3b942 100644
--- a/jjb/pipeline/voltha/master/tucson-build-and-test.groovy
+++ b/jjb/pipeline/voltha/master/tucson-build-and-test.groovy
@@ -79,7 +79,7 @@
           timeout(1) {
             sh returnStdout: false, script: '''
             # remove orphaned port-forward from different namespaces
-            ps aux | grep port-forw | grep -v grep | awk '{print $2}' | xargs --no-run-if-empty kill -9
+            ps aux | grep port-forw | grep -v grep | awk '{print $2}' | xargs --no-run-if-empty kill -9 || true
             '''
           }
         }
diff --git a/jjb/pipeline/voltha/master/voltha-scale-multi-stack.groovy b/jjb/pipeline/voltha/master/voltha-scale-multi-stack.groovy
index 4536656..fb5e75c 100644
--- a/jjb/pipeline/voltha/master/voltha-scale-multi-stack.groovy
+++ b/jjb/pipeline/voltha/master/voltha-scale-multi-stack.groovy
@@ -55,7 +55,7 @@
         timeout(time: 11, unit: 'MINUTES') {
           sh """
           # remove orphaned port-forward from different namespaces
-            ps aux | grep port-forw | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9
+            ps aux | grep port-forw | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9 || true
           """
           script {
             def namespaces = ["infra"]
@@ -71,7 +71,7 @@
             helm repo update
 
             # remove all port-forward from different namespaces
-            ps aux | grep port-forw | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9
+            ps aux | grep port-forw | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9 || true
           """
         }
       }
@@ -347,7 +347,7 @@
             printf '%s\n' \$(voltctl -m 8MB logicaldevice list -q) | xargs --no-run-if-empty -I# bash -c "voltctl -m 8MB logicaldevice port list # > $LOG_FOLDER/${stack_ns}/voltha-logicaldevice-ports-#.txt" || true
 
             # remove VOLTHA port-forward
-            ps aux | grep port-forw | grep voltha-api | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9
+            ps aux | grep port-forw | grep voltha-api | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9 || true
             """
           } catch(e) {
             sh '''
@@ -455,7 +455,7 @@
         """
         sh """
           # remove VOLTHA port-forward
-          ps aux | grep port-forw | grep voltha-api | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9 2>&1 > /dev/null
+          ps aux | grep port-forw | grep voltha-api | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9 2>&1 > /dev/null || true
         """
       }
     }
diff --git a/jjb/pipeline/voltha/master/voltha-scale-test.groovy b/jjb/pipeline/voltha/master/voltha-scale-test.groovy
index e0b65d3..35604f8 100644
--- a/jjb/pipeline/voltha/master/voltha-scale-test.groovy
+++ b/jjb/pipeline/voltha/master/voltha-scale-test.groovy
@@ -69,7 +69,7 @@
             helm repo update
 
             # remove orphaned port-forward from different namespaces
-            ps aux | grep port-forw | grep -v grep | awk '{print $2}' | xargs --no-run-if-empty kill -9
+            ps aux | grep port-forw | grep -v grep | awk '{print $2}' | xargs --no-run-if-empty kill -9 || true
 
             cd $WORKSPACE
             rm -rf $WORKSPACE/*
diff --git a/jjb/pipeline/voltha/voltha-2.7/bbsim-tests.groovy b/jjb/pipeline/voltha/voltha-2.7/bbsim-tests.groovy
index 41720cd..c72c0f9 100644
--- a/jjb/pipeline/voltha/voltha-2.7/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/voltha-2.7/bbsim-tests.groovy
@@ -129,7 +129,7 @@
       // remove port-forwarding
       sh """
         # remove orphaned port-forward from different namespaces
-        ps aux | grep port-forw | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9
+        ps aux | grep port-forw | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9 || true
       """
       // collect pod details
       get_pods_info("$WORKSPACE/${name}")
diff --git a/jjb/pipeline/voltha/voltha-2.7/software-upgrades.groovy b/jjb/pipeline/voltha/voltha-2.7/software-upgrades.groovy
index 124e05b..7bdf278 100644
--- a/jjb/pipeline/voltha/voltha-2.7/software-upgrades.groovy
+++ b/jjb/pipeline/voltha/voltha-2.7/software-upgrades.groovy
@@ -130,7 +130,7 @@
       // remove port-forwarding
       sh """
         # remove orphaned port-forward from different namespaces
-        ps aux | grep port-forw | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9
+        ps aux | grep port-forw | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9 || true
       """
       // collect pod details
       get_pods_info("$WORKSPACE/${name}")
@@ -181,7 +181,7 @@
         // remove port-forwarding
         sh """
           # remove orphaned port-forward from different namespaces
-          ps aux | grep port-forw | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9
+          ps aux | grep port-forw | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9 || true
         """
         helmTeardown(['infra', 'voltha'])
       }
diff --git a/jjb/pipeline/voltha/voltha-2.7/voltha-scale-test.groovy b/jjb/pipeline/voltha/voltha-2.7/voltha-scale-test.groovy
index d88b180..11bb1c4 100644
--- a/jjb/pipeline/voltha/voltha-2.7/voltha-scale-test.groovy
+++ b/jjb/pipeline/voltha/voltha-2.7/voltha-scale-test.groovy
@@ -79,7 +79,7 @@
             done
 
             # remove orphaned port-forward from different namespaces
-            ps aux | grep port-forw | grep -v grep | awk '{print $2}' | xargs --no-run-if-empty kill -9
+            ps aux | grep port-forw | grep -v grep | awk '{print $2}' | xargs --no-run-if-empty kill -9 || true
 
             cd $WORKSPACE
             rm -rf $WORKSPACE/*
diff --git a/jjb/voltha-e2e.yaml b/jjb/voltha-e2e.yaml
index c7001de..958cd22 100755
--- a/jjb/voltha-e2e.yaml
+++ b/jjb/voltha-e2e.yaml
@@ -120,6 +120,14 @@
               workflow: tt
               flags: ""
               teardown: true
+            - target: openonu-go-adapter-omci-hardening-passed-test
+              workflow: att
+              flags: "--set omci_response_rate=9 --set omci_timeout=1s"
+              teardown: true
+            - target: openonu-go-adapter-omci-hardening-failed-test
+              workflow: att
+              flags: "--set omci_response_rate=7"
+              teardown: true
 
       - 'voltha-periodic-test':
           name: 'patchset-voltha-openonu-go-test-bbsim'
@@ -158,6 +166,14 @@
               workflow: tt
               flags: ""
               teardown: true
+            - target: openonu-go-adapter-omci-hardening-passed-test
+              workflow: att
+              flags: "--set omci_response_rate=9 --set omci_timeout=1s"
+              teardown: true
+            - target: openonu-go-adapter-omci-hardening-failed-test
+              workflow: att
+              flags: "--set omci_response_rate=7"
+              teardown: true
 
       - 'voltha-periodic-test-kind-voltha-based':
           name: 'periodic-voltha-openonu-go-test-bbsim-2.7'
@@ -241,6 +257,97 @@
               teardown: true
           olts: 2
 
+      - 'voltha-periodic-test':
+          name: 'periodic-voltha-pm-data-test-bbsim'
+          code-branch: 'master'
+          extraHelmFlags: '--set global.image_tag=master --set onos-classic.image.tag=master --set externalAccess.enabled=true,externalAccess.service.type=NodePort,externalAccess.service.nodePorts[0]=30201,externalAccess.service.domain=127.0.0.1'
+          time-trigger: "H H/23 * * *"
+          logLevel: 'DEBUG'
+          testTargets: |
+            - target: voltha-pm-data-single-kind-att
+              workflow: att
+              flags: ""
+              teardown: true
+            - target: voltha-pm-data-single-kind-dt
+              workflow: dt
+              flags: ""
+              teardown: true
+            - target: voltha-pm-data-single-kind-tt
+              workflow: tt
+              flags: ""
+              teardown: true
+          timeout: 140
+
+      - 'voltha-periodic-test':
+          name: 'patchset-voltha-pm-data-test-bbsim'
+          trigger-comment: "voltha test pm data singleolt"
+          code-branch: '$GERRIT_BRANCH'
+          extraHelmFlags: '--set global.image_tag=master --set onos-classic.image.tag=master --set externalAccess.enabled=true,externalAccess.service.type=NodePort,externalAccess.service.nodePorts[0]=30201,externalAccess.service.domain=127.0.0.1'
+          code-branch: '$GERRIT_BRANCH'
+          gerrit-project: '$GERRIT_PROJECT'
+          gerritRefspec: '$GERRIT_REFSPEC'
+          logLevel: 'DEBUG'
+          testTargets: |
+            - target: voltha-pm-data-single-kind-att
+              workflow: att
+              flags: ""
+              teardown: true
+            - target: voltha-pm-data-single-kind-dt
+              workflow: dt
+              flags: ""
+              teardown: true
+            - target: voltha-pm-data-single-kind-tt
+              workflow: tt
+              flags: ""
+              teardown: true
+          timeout: 140
+
+      - 'voltha-periodic-test':
+          name: 'periodic-voltha-multiple-olts-pm-data-test-bbsim'
+          code-branch: 'master'
+          extraHelmFlags: '--set global.image_tag=master --set onos-classic.image.tag=master --set onu=2,pon=2 --set externalAccess.enabled=true,externalAccess.service.type=NodePort,externalAccess.service.nodePorts[0]=30201,externalAccess.service.domain=127.0.0.1'
+          olts: 2
+          timeout: 180
+          logLevel: 'DEBUG'
+          testTargets: |
+            - target: voltha-pm-data-multiolt-kind-att
+              workflow: att
+              flags: ""
+              teardown: true
+            - target: voltha-pm-data-multiolt-kind-dt
+              workflow: dt
+              flags: ""
+              teardown: true
+            - target: voltha-pm-data-multiolt-kind-tt
+              workflow: tt
+              flags: ""
+              teardown: true
+          time-trigger: "H H/23 * * *"
+
+      - 'voltha-periodic-test':
+          name: 'patchset-voltha-multiple-olts-pm-data-test-bbsim'
+          trigger-comment: "voltha test pm data multiolt"
+          code-branch: '$GERRIT_BRANCH'
+          extraHelmFlags: '--set global.image_tag=master --set onos-classic.image.tag=master --set onu=2,pon=2 --set externalAccess.enabled=true,externalAccess.service.type=NodePort,externalAccess.service.nodePorts[0]=30201,externalAccess.service.domain=127.0.0.1'
+          gerrit-project: '$GERRIT_PROJECT'
+          gerritRefspec: '$GERRIT_REFSPEC'
+          logLevel: 'DEBUG'
+          testTargets: |
+            - target: voltha-pm-data-multiolt-kind-att
+              workflow: att
+              flags: ""
+              teardown: true
+            - target: voltha-pm-data-multiolt-kind-dt
+              workflow: dt
+              flags: ""
+              teardown: true
+            - target: voltha-pm-data-multiolt-kind-tt
+              workflow: tt
+              flags: ""
+              teardown: true
+          olts: 2
+          timeout: 180
+
       - 'voltha-periodic-test-kind-voltha-based':
           name: 'periodic-voltha-multiple-olts-openonu-go-test-bbsim-2.7'
           pipeline-script: 'voltha/voltha-2.7/voltha-openonu-go-test-bbsim.groovy'
@@ -429,6 +536,7 @@
     registry:  mirror.registry.opennetworking.org
     sandbox: true
     olts: 1
+    timeout: 130
     logLevel: 'INFO'
 
     trigger-comment: vv7CBoQQYYonvaN8xcru
@@ -519,6 +627,11 @@
           default: '{logLevel}'
           description: 'Log level for all the components'
 
+      - string:
+          name: timeout
+          default: '{timeout}'
+          description: 'Timeout of pipeline job [minutes]'
+
     project-type: pipeline
     concurrent: true
 
diff --git a/jjb/voltha-test/voltha-nightly-jobs.yaml b/jjb/voltha-test/voltha-nightly-jobs.yaml
index 608d1d7..e5ac6bc 100644
--- a/jjb/voltha-test/voltha-nightly-jobs.yaml
+++ b/jjb/voltha-test/voltha-nightly-jobs.yaml
@@ -19,6 +19,11 @@
           description: 'DEPRECATED - use buildNode instead'
 
       - string:
+          name: logLevel
+          default: '{logLevel}'
+          description: 'Log level for all the components'
+
+      - string:
           name: cordRepoUrl
           default: '{gerrit-server-url}'
           description: 'The URL of the CORD Project repository'
@@ -225,6 +230,7 @@
     release: '2.7'
     branch: 'voltha-2.7'
     volthaHelmChartsChange: '' # this is not supported in the VOLTHA-2.7 build, but the parameters are shared, ideally we should split them
+    logLevel: 'DEBUG'
 
     <<: *voltha-build-job-parameters
 
diff --git a/vars/createKubernetesCluster.groovy b/vars/createKubernetesCluster.groovy
index 36bd510..a1a2bde 100644
--- a/vars/createKubernetesCluster.groovy
+++ b/vars/createKubernetesCluster.groovy
@@ -76,10 +76,6 @@
       mkdir -p $HOME/.kube
       kind get kubeconfig --name ${cfg.name} > $HOME/.kube/config
 
-      # add helm repositories
-      helm repo add onf https://charts.opencord.org
-      helm repo update
-
       # download kail
       bash <( curl -sfL https://raw.githubusercontent.com/boz/kail/master/godownloader.sh) -b "$WORKSPACE/bin"
   """
diff --git a/vars/volthaDeploy.groovy b/vars/volthaDeploy.groovy
index 07146ee..d8d679f 100644
--- a/vars/volthaDeploy.groovy
+++ b/vars/volthaDeploy.groovy
@@ -35,10 +35,18 @@
       registryFlags += " --set onos-classic.atomix.image.repository=${cfg.dockerRegistry}/atomix/atomix "
       registryFlags += " --set freeradius.images.radius.registry=${cfg.dockerRegistry}/ "
 
-      // we want to alway leave the user provided flags at the end, to override changes
+      // we want to always leave the user provided flags at the end, to override changes
       cfg.extraHelmFlags = registryFlags + " " + cfg.extraHelmFlags
     }
 
+    // Add helm repositories
+    println "Updating helm repos"
+
+    sh """
+      helm repo add onf https://charts.opencord.org
+      helm repo update
+    """
+
     println "Deploying VOLTHA with the following parameters: ${cfg}."
 
     volthaInfraDeploy(cfg)