[VOL-4530] Added voltha components per patchset verification jobs and openonu-go periodic jobs for voltha-2.9 branch

Change-Id: Id450f17f747a9ce0bebb5312caa8e33d2ac75db4
diff --git a/jjb/pipeline/voltha/voltha-2.9/bbsim-tests.groovy b/jjb/pipeline/voltha/voltha-2.9/bbsim-tests.groovy
new file mode 100755
index 0000000..f039841
--- /dev/null
+++ b/jjb/pipeline/voltha/voltha-2.9/bbsim-tests.groovy
@@ -0,0 +1,279 @@
+// Copyright 2021-present Open Networking Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// voltha-2.x e2e tests for openonu-go
+// uses bbsim to simulate OLT/ONUs
+
+library identifier: 'cord-jenkins-libraries@master',
+    retriever: modernSCM([
+      $class: 'GitSCMSource',
+      remote: 'https://gerrit.opencord.org/ci-management.git'
+])
+
+def clusterName = "kind-ci"
+
+def execute_test(testTarget, workflow, testLogging, teardown, testSpecificHelmFlags = "") {
+  def infraNamespace = "default"
+  def volthaNamespace = "voltha"
+  def logsDir = "$WORKSPACE/${testTarget}"
+  stage('Cleanup') {
+    if (teardown) {
+      timeout(15) {
+        script {
+          helmTeardown(["default", infraNamespace, volthaNamespace])
+        }
+        timeout(1) {
+          sh returnStdout: false, script: '''
+          # remove orphaned port-forward from different namespaces
+          ps aux | grep port-forw | grep -v grep | awk '{print $2}' | xargs --no-run-if-empty kill -9 || true
+          '''
+        }
+      }
+    }
+  }
+  stage('Deploy Voltha') {
+    if (teardown) {
+      timeout(10) {
+        script {
+
+          sh """
+          mkdir -p ${logsDir}
+          _TAG=kail-startup kail -n ${infraNamespace} -n ${volthaNamespace} > ${logsDir}/onos-voltha-startup-combined.log &
+          """
+
+          // if we're downloading a voltha-helm-charts patch, then install from a local copy of the charts
+          def localCharts = false
+          if (volthaHelmChartsChange != "" || gerritProject == "voltha-helm-charts") {
+            localCharts = true
+          }
+
+          // NOTE temporary workaround expose ONOS node ports
+          def localHelmFlags = extraHelmFlags.trim() + " --set global.log_level=${logLevel.toUpperCase()} " +
+          " --set onos-classic.onosSshPort=30115 " +
+          " --set onos-classic.onosApiPort=30120 " +
+          " --set onos-classic.onosOfPort=31653 " +
+          " --set onos-classic.individualOpenFlowNodePorts=true " + testSpecificHelmFlags
+
+          if (gerritProject != "") {
+            localHelmFlags = "${localHelmFlags} " + getVolthaImageFlags("${gerritProject}")
+          }
+
+          volthaDeploy([
+            infraNamespace: infraNamespace,
+            volthaNamespace: volthaNamespace,
+            workflow: workflow.toLowerCase(),
+            extraHelmFlags: localHelmFlags,
+            localCharts: localCharts,
+            bbsimReplica: olts.toInteger(),
+            dockerRegistry: registry,
+            ])
+        }
+
+        // stop logging
+        sh """
+          P_IDS="\$(ps e -ww -A | grep "_TAG=kail-startup" | grep -v grep | awk '{print \$1}')"
+          if [ -n "\$P_IDS" ]; then
+            echo \$P_IDS
+            for P_ID in \$P_IDS; do
+              kill -9 \$P_ID
+            done
+          fi
+          cd ${logsDir}
+          gzip -k onos-voltha-startup-combined.log
+          rm onos-voltha-startup-combined.log
+        """
+      }
+      sh """
+      JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-voltha-api" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/voltha-voltha-api 55555:55555; done"&
+      JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-etcd" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-etcd 2379:2379; done"&
+      JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-kafka" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-kafka 9092:9092; done"&
+      bbsimDmiPortFwd=50075
+      for i in {0..${olts.toInteger() - 1}}; do
+        JENKINS_NODE_COOKIE="dontKillMe" _TAG="bbsim\${i}" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/bbsim\${i} \${bbsimDmiPortFwd}:50075; done"&
+        ((bbsimDmiPortFwd++))
+      done
+      ps aux | grep port-forward
+      """
+      // setting ONOS log level
+      script {
+        setOnosLogLevels([
+          onosNamespace: infraNamespace,
+          apps: [
+            'org.opencord.dhcpl2relay',
+            'org.opencord.olt',
+            'org.opencord.aaa',
+            'org.onosproject.net.flowobjective.impl.FlowObjectiveManager',
+            'org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager'
+          ],
+          logLevel: logLevel
+        ])
+      }
+    }
+  }
+  stage('Run test ' + testTarget + ' on ' + workflow + ' workFlow') {
+    sh """
+    mkdir -p ${logsDir}
+    export ROBOT_MISC_ARGS="-d ${logsDir} ${params.extraRobotArgs} "
+    ROBOT_MISC_ARGS+="-v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace} -v container_log_dir:${logsDir} -v logging:${testLogging}"
+    export KVSTOREPREFIX=voltha/voltha_voltha
+
+    make -C $WORKSPACE/voltha-system-tests ${testTarget} || true
+    """
+    getPodsInfo("${logsDir}")
+    sh """
+      set +e
+      # collect logs collected in the Robot Framework StartLogging keyword
+      cd ${logsDir}
+      gzip *-combined.log || true
+      rm *-combined.log || true
+    """
+  }
+}
+
+def collectArtifacts(exitStatus) {
+  getPodsInfo("$WORKSPACE/${exitStatus}")
+  sh """
+  kubectl logs -n voltha -l app.kubernetes.io/part-of=voltha > $WORKSPACE/${exitStatus}/voltha.log || true
+  """
+  archiveArtifacts artifacts: '**/*.log,**/*.gz,**/*.txt,**/*.html'
+  sh '''
+    sync
+    pkill kail || true
+    which voltctl
+    md5sum $(which voltctl)
+  '''
+  step([$class: 'RobotPublisher',
+    disableArchiveOutput: false,
+    logFileName: "**/*/log*.html",
+    otherFiles: '',
+    outputFileName: "**/*/output*.xml",
+    outputPath: '.',
+    passThreshold: 100,
+    reportFileName: "**/*/report*.html",
+    unstableThreshold: 0,
+    onlyCritical: true]);
+}
+
+pipeline {
+
+  /* no label, executor is determined by JJB */
+  agent {
+    label "${params.buildNode}"
+  }
+  options {
+    timeout(time: "${timeout}", unit: 'MINUTES')
+  }
+  environment {
+    KUBECONFIG="$HOME/.kube/kind-${clusterName}"
+    VOLTCONFIG="$HOME/.volt/config"
+    PATH="$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+    DIAGS_PROFILE="VOLTHA_PROFILE"
+    SSHPASS="karaf"
+  }
+  stages {
+    stage('Download Code') {
+      steps {
+        getVolthaCode([
+          branch: "${branch}",
+          gerritProject: "${gerritProject}",
+          gerritRefspec: "${gerritRefspec}",
+          volthaSystemTestsChange: "${volthaSystemTestsChange}",
+          volthaHelmChartsChange: "${volthaHelmChartsChange}",
+        ])
+      }
+    }
+    stage('Build patch') {
+      // build the patch only if gerritProject is specified
+      when {
+        expression {
+          return !gerritProject.isEmpty()
+        }
+      }
+      steps {
+        // NOTE that the correct patch has already been checked out
+        // during the getVolthaCode step
+        buildVolthaComponent("${gerritProject}")
+      }
+    }
+    stage('Create K8s Cluster') {
+      steps {
+        script {
+          def clusterExists = sh returnStdout: true, script: """
+          kind get clusters | grep ${clusterName} | wc -l
+          """
+          if (clusterExists.trim() == "0") {
+            createKubernetesCluster([nodes: 3, name: clusterName])
+          }
+        }
+      }
+    }
+    stage('Replace voltctl') {
+      // if the project is voltctl override the downloaded one with the built one
+      when {
+        expression {
+          return gerritProject == "voltctl"
+        }
+      }
+      steps{
+        sh """
+        mv `ls $WORKSPACE/voltctl/release/voltctl-*-linux-amd*` $WORKSPACE/bin/voltctl
+        chmod +x $WORKSPACE/bin/voltctl
+        """
+      }
+    }
+    stage('Load image in kind nodes') {
+      when {
+        expression {
+          return !gerritProject.isEmpty()
+        }
+      }
+      steps {
+        loadToKind()
+      }
+    }
+    stage('Parse and execute tests') {
+        steps {
+          script {
+            def tests = readYaml text: testTargets
+
+            for(int i = 0;i<tests.size();i++) {
+              def test = tests[i]
+              def target = test["target"]
+              def workflow = test["workflow"]
+              def flags = test["flags"]
+              def teardown = test["teardown"].toBoolean()
+              def logging = test["logging"].toBoolean()
+              def testLogging = 'False'
+              if (logging) {
+                  testLogging = 'True'
+              }
+              println "Executing test ${target} on workflow ${workflow} with logging ${testLogging} and extra flags ${flags}"
+              execute_test(target, workflow, testLogging, teardown, flags)
+            }
+          }
+        }
+    }
+  }
+  post {
+    aborted {
+      collectArtifacts("aborted")
+    }
+    failure {
+      collectArtifacts("failed")
+    }
+    always {
+      collectArtifacts("always")
+    }
+  }
+}
diff --git a/jjb/verify/bbsim-sadis-server.yaml b/jjb/verify/bbsim-sadis-server.yaml
index fea3a4b..51ebc94 100644
--- a/jjb/verify/bbsim-sadis-server.yaml
+++ b/jjb/verify/bbsim-sadis-server.yaml
@@ -11,6 +11,9 @@
       - 'verify-bbsim-sadis-server-jobs-voltha-2.8':
           name-extension: '-voltha-2.8'
           branch-regexp: '^voltha-2.8$'
+      - 'verify-bbsim-sadis-server-jobs-voltha-2.9':
+          name-extension: '-voltha-2.9'
+          branch-regexp: '^voltha-2.9$'
       - 'verify-bbsim-sadis-server-jobs-master':
           branch-regexp: '^master$'
       - 'publish-bbsim-sadis-server-jobs':
@@ -35,6 +38,12 @@
           pipeline-script: 'voltha/voltha-2.8/bbsim-tests.groovy'
 
 - job-group:
+    name: 'verify-bbsim-sadis-server-jobs-voltha-2.9'
+    jobs:
+      - 'voltha-patch-test':
+          pipeline-script: 'voltha/voltha-2.9/bbsim-tests.groovy'
+
+- job-group:
     name: 'verify-bbsim-sadis-server-jobs-master'
     jobs:
       - 'voltha-patch-test':
diff --git a/jjb/verify/bbsim.yaml b/jjb/verify/bbsim.yaml
index 22b44f7..d399b98 100644
--- a/jjb/verify/bbsim.yaml
+++ b/jjb/verify/bbsim.yaml
@@ -12,6 +12,10 @@
           name-extension: '-voltha-2.8'
           override-branch: 'voltha-2.8'
           branch-regexp: '^voltha-2.8$'
+      - 'verify-bbsim-jobs-voltha-2.9':
+          name-extension: '-voltha-2.9'
+          override-branch: 'voltha-2.9'
+          branch-regexp: '^voltha-2.9$'
       - 'verify-bbsim-jobs-master':
           branch-regexp: '^master$'
       - 'publish-bbsim-jobs':
@@ -52,6 +56,28 @@
               logging: true
 
 - job-group:
+    name: 'verify-bbsim-jobs-voltha-2.9'
+    jobs:
+      - 'voltha-patch-test':
+          pipeline-script: 'voltha/voltha-2.9/bbsim-tests.groovy'
+          testTargets: |
+            - target: sanity-bbsim-att
+              workflow: att
+              flags: ""
+              teardown: true
+              logging: true
+            - target: sanity-bbsim-dt
+              workflow: dt
+              flags: ""
+              teardown: true
+              logging: true
+            - target: sanity-bbsim-tt
+              workflow: tt
+              flags: ""
+              teardown: true
+              logging: true
+
+- job-group:
     name: 'verify-bbsim-jobs-master'
     jobs:
       - 'voltha-patch-test':
diff --git a/jjb/verify/ofagent-go.yaml b/jjb/verify/ofagent-go.yaml
index c629c6b..b929897 100644
--- a/jjb/verify/ofagent-go.yaml
+++ b/jjb/verify/ofagent-go.yaml
@@ -12,6 +12,10 @@
           name-extension: '-voltha-2.8'
           override-branch: 'voltha-2.8'
           branch-regexp: '{kind-voltha-regexp}'
+      - 'verify-ofagent-jobs-voltha-2.9':
+          name-extension: '-voltha-2.9'
+          override-branch: 'voltha-2.9'
+          branch-regexp: '{kind-voltha-regexp}'
       - 'verify-ofagent-jobs-master':
           branch-regexp: '^master$'
       - 'publish-ofagent-go-jobs':
@@ -37,6 +41,12 @@
           pipeline-script: 'voltha/voltha-2.8/bbsim-tests.groovy'
 
 - job-group:
+    name: 'verify-ofagent-jobs-voltha-2.9'
+    jobs:
+      - 'voltha-patch-test':
+          pipeline-script: 'voltha/voltha-2.9/bbsim-tests.groovy'
+
+- job-group:
     name: 'verify-ofagent-jobs-master'
     jobs:
       - 'voltha-patch-test':
diff --git a/jjb/verify/voltha-go.yaml b/jjb/verify/voltha-go.yaml
index 05125b7..fa82d1a 100644
--- a/jjb/verify/voltha-go.yaml
+++ b/jjb/verify/voltha-go.yaml
@@ -12,6 +12,10 @@
           name-extension: '-voltha-2.8'
           override-branch: 'voltha-2.8'
           branch-regexp: '^voltha-2.8$'
+      - 'verify-voltha-go-jobs-voltha-2.9':
+          name-extension: '-voltha-2.9'
+          override-branch: 'voltha-2.9'
+          branch-regexp: '^voltha-2.9$'
       - 'verify-voltha-go-jobs-master':
           branch-regexp: '^master$'
       - 'publish-voltha-go-jobs':
@@ -45,6 +49,12 @@
           pipeline-script: 'voltha/voltha-2.8/bbsim-tests.groovy'
 
 - job-group:
+    name: 'verify-voltha-go-jobs-voltha-2.9'
+    jobs:
+      - 'voltha-patch-test':
+          pipeline-script: 'voltha/voltha-2.9/bbsim-tests.groovy'
+
+- job-group:
     name: 'verify-voltha-go-jobs-master'
     jobs:
       - 'voltha-patch-test':
diff --git a/jjb/verify/voltha-helm-charts.yaml b/jjb/verify/voltha-helm-charts.yaml
index 0f3a58c..4ba63b8 100644
--- a/jjb/verify/voltha-helm-charts.yaml
+++ b/jjb/verify/voltha-helm-charts.yaml
@@ -12,6 +12,10 @@
           name-extension: '-voltha-2.8'
           override-branch: 'voltha-2.8'
           branch-regexp: '^voltha-2.8$'
+      - 'verify-voltha-helm-charts-jobs-voltha-2.9':
+          name-extension: '-voltha-2.9'
+          override-branch: 'voltha-2.9'
+          branch-regexp: '^voltha-2.9$'
       - 'verify-voltha-helm-charts-jobs-master':
           branch-regexp: '^master$'
 
@@ -31,6 +35,12 @@
           pipeline-script: 'voltha/voltha-2.8/bbsim-tests.groovy'
 
 - job-group:
+    name: 'verify-voltha-helm-charts-jobs-voltha-2.9'
+    jobs:
+      - 'voltha-patch-test':
+          pipeline-script: 'voltha/voltha-2.9/bbsim-tests.groovy'
+
+- job-group:
     name: 'verify-voltha-helm-charts-jobs-master'
     jobs:
       - 'voltha-patch-test'
diff --git a/jjb/verify/voltha-onos.yaml b/jjb/verify/voltha-onos.yaml
index aef2900..79f9dea 100644
--- a/jjb/verify/voltha-onos.yaml
+++ b/jjb/verify/voltha-onos.yaml
@@ -12,6 +12,10 @@
           name-extension: '-voltha-2.8'
           override-branch: 'voltha-2.8'
           branch-regexp: '^voltha-2.8$'
+      - 'verify-voltha-onos-jobs-voltha-2.9':
+          name-extension: '-voltha-2.9'
+          override-branch: 'voltha-2.9'
+          branch-regexp: '^voltha-2.9$'
       - 'verify-voltha-onos-jobs-master':
           branch-regexp: '^master$'
       - 'publish-voltha-onos-jobs':
@@ -31,6 +35,12 @@
           pipeline-script: 'voltha/voltha-2.8/bbsim-tests.groovy'
 
 - job-group:
+    name: 'verify-voltha-onos-jobs-voltha-2.9'
+    jobs:
+      - 'voltha-patch-test':
+          pipeline-script: 'voltha/voltha-2.9/bbsim-tests.groovy'
+
+- job-group:
     name: 'verify-voltha-onos-jobs-master'
     jobs:
       - 'voltha-patch-test':
diff --git a/jjb/verify/voltha-openolt-adapter.yaml b/jjb/verify/voltha-openolt-adapter.yaml
index 0f98077..6dc6b54 100644
--- a/jjb/verify/voltha-openolt-adapter.yaml
+++ b/jjb/verify/voltha-openolt-adapter.yaml
@@ -12,6 +12,10 @@
           name-extension: '-voltha-2.8'
           override-branch: 'voltha-2.8'
           branch-regexp: '^voltha-2.8$'
+      - 'verify-voltha-openolt-adapter-jobs-voltha-2.9':
+          name-extension: '-voltha-2.9'
+          override-branch: 'voltha-2.9'
+          branch-regexp: '^voltha-2.9$'
       - 'verify-voltha-openolt-adapter-jobs-master':
           branch-regexp: '^master$'
       - 'publish-voltha-openolt-adapter-jobs':
@@ -45,6 +49,12 @@
           pipeline-script: 'voltha/voltha-2.8/bbsim-tests.groovy'
 
 - job-group:
+    name: 'verify-voltha-openolt-adapter-jobs-voltha-2.9'
+    jobs:
+      - 'voltha-patch-test':
+          pipeline-script: 'voltha/voltha-2.9/bbsim-tests.groovy'
+
+- job-group:
     name: 'verify-voltha-openolt-adapter-jobs-master'
     jobs:
       - 'voltha-patch-test':
diff --git a/jjb/verify/voltha-openonu-adapter-go.yaml b/jjb/verify/voltha-openonu-adapter-go.yaml
index 8a1353a..6412711 100644
--- a/jjb/verify/voltha-openonu-adapter-go.yaml
+++ b/jjb/verify/voltha-openonu-adapter-go.yaml
@@ -12,6 +12,10 @@
           name-extension: '-voltha-2.8'
           override-branch: 'voltha-2.8'
           branch-regexp: '^voltha-2.8$'
+      - 'verify-voltha-openonu-adapter-go-jobs-voltha-2.9':
+          name-extension: '-voltha-2.9'
+          override-branch: 'voltha-2.9'
+          branch-regexp: '^voltha-2.9$'
       - 'verify-voltha-openonu-adapter-go-jobs-master':
           branch-regexp: '^master$'
       - 'publish-voltha-openonu-adapter-go-jobs':
@@ -44,6 +48,12 @@
           pipeline-script: 'voltha/voltha-2.8/bbsim-tests.groovy'
 
 - job-group:
+    name: 'verify-voltha-openonu-adapter-go-jobs-voltha-2.9'
+    jobs:
+      - 'voltha-patch-test':
+          pipeline-script: 'voltha/voltha-2.9/bbsim-tests.groovy'
+
+- job-group:
     name: 'verify-voltha-openonu-adapter-go-jobs-master'
     jobs:
       - 'voltha-patch-test':
diff --git a/jjb/verify/voltha-system-tests.yaml b/jjb/verify/voltha-system-tests.yaml
index 003d28e..1b14801 100644
--- a/jjb/verify/voltha-system-tests.yaml
+++ b/jjb/verify/voltha-system-tests.yaml
@@ -14,6 +14,11 @@
           name-extension: '-voltha-2.8'
           override-branch: 'voltha-2.8'
           branch-regexp: '^voltha-2.8$'
+      - 'verify-voltha-system-tests-jobs-voltha-2.9':
+          build-node:  'ubuntu18.04-basebuild-4c-8g'
+          name-extension: '-voltha-2.9'
+          override-branch: 'voltha-2.9'
+          branch-regexp: '^voltha-2.9$'
       - 'verify-voltha-system-tests-jobs-master':
           build-node:  'ubuntu18.04-basebuild-4c-8g'
           branch-regexp: '^master$'
@@ -36,6 +41,12 @@
           pipeline-script: 'voltha/voltha-2.8/bbsim-tests.groovy'
 
 - job-group:
+    name: 'verify-voltha-system-tests-jobs-voltha-2.9'
+    jobs:
+      - 'voltha-patch-test':
+          pipeline-script: 'voltha/voltha-2.9/bbsim-tests.groovy'
+
+- job-group:
     name: 'verify-voltha-system-tests-jobs-master'
     jobs:
       - 'voltha-patch-test':
diff --git a/jjb/voltha-e2e.yaml b/jjb/voltha-e2e.yaml
index 98ce37c..ea77e47 100755
--- a/jjb/voltha-e2e.yaml
+++ b/jjb/voltha-e2e.yaml
@@ -471,6 +471,77 @@
           timeout: 180
 
       - 'voltha-periodic-test':
+          name: 'patchset-voltha-2.9-openonu-go-test-bbsim'
+          trigger-comment: "voltha 2.9 test openonu singleolt"
+          pipeline-script: 'voltha/voltha-2.9/bbsim-tests.groovy'
+          code-branch: '$GERRIT_BRANCH'
+          gerritProject: '$GERRIT_PROJECT'
+          gerritRefspec: '$GERRIT_REFSPEC'
+          logLevel: 'DEBUG'
+          testTargets: |
+            - target: 1t1gem-openonu-go-adapter-test
+              workflow: att
+              flags: ""
+              teardown: true
+              logging: true
+            - target: 1t4gem-openonu-go-adapter-test
+              workflow: att
+              flags: ""
+              teardown: true
+              logging: true
+            - target: 1t8gem-openonu-go-adapter-test
+              workflow: att
+              flags: ""
+              teardown: true
+              logging: true
+            - target: mib-upload-templating-openonu-go-adapter-test
+              workflow: att
+              flags: "--set pon=2,onu=2,controlledActivation=only-onu"
+              teardown: true
+              logging: true
+            - target: reconcile-openonu-go-adapter-test-att
+              workflow: att
+              flags: ""
+              teardown: true
+              logging: true
+            - target: reconcile-openonu-go-adapter-test-dt
+              workflow: dt
+              flags: ""
+              teardown: true
+              logging: true
+            - target: reconcile-openonu-go-adapter-test-tt
+              workflow: tt
+              flags: ""
+              teardown: true
+              logging: true
+            - target: openonu-go-adapter-omci-hardening-passed-test
+              workflow: att
+              flags: "--set omci_response_rate=9 --set voltha-adapter-openonu.adapter_open_onu.omci_timeout=1s"
+              teardown: true
+              logging: true
+            - target: openonu-go-adapter-omci-hardening-failed-test
+              workflow: att
+              flags: "--set omci_response_rate=7"
+              teardown: true
+              logging: true
+            - target: voltha-onu-omci-get-single-kind-att
+              workflow: att
+              flags: ""
+              teardown: true
+              logging: true
+            - target: voltha-onu-omci-get-single-kind-dt
+              workflow: dt
+              flags: ""
+              teardown: true
+              logging: true
+            - target: voltha-onu-omci-get-single-kind-tt
+              workflow: tt
+              flags: ""
+              teardown: true
+              logging: true
+          timeout: 180
+
+      - 'voltha-periodic-test':
           name: 'patchset-voltha-2.8-openonu-go-test-bbsim'
           trigger-comment: "voltha 2.8 test openonu singleolt"
           pipeline-script: 'voltha/voltha-2.8/bbsim-tests.groovy'
@@ -611,6 +682,62 @@
           timeout: 235
 
       - 'voltha-periodic-test':
+          name: 'periodic-voltha-multiple-olts-openonu-go-test-bbsim-2.9'
+          pipeline-script: 'voltha/voltha-2.9/bbsim-tests.groovy'
+          code-branch: 'voltha-2.9'
+          extraHelmFlags: '--set onu=2,pon=2'
+          olts: 2
+          logLevel: 'DEBUG'
+          testTargets: |
+            - target: 1t1gem-openonu-go-adapter-multi-olt-test
+              workflow: att
+              flags: ""
+              teardown: true
+              logging: true
+            - target: 1t4gem-openonu-go-adapter-multi-olt-test
+              workflow: att
+              flags: ""
+              teardown: true
+              logging: true
+            - target: 1t8gem-openonu-go-adapter-multi-olt-test
+              workflow: att
+              flags: ""
+              teardown: true
+              logging: true
+            - target: reconcile-openonu-go-adapter-multi-olt-test-att
+              workflow: att
+              flags: ""
+              teardown: true
+              logging: true
+            - target: reconcile-openonu-go-adapter-multi-olt-test-dt
+              workflow: dt
+              flags: ""
+              teardown: true
+              logging: true
+            - target: reconcile-openonu-go-adapter-multi-olt-test-tt
+              workflow: tt
+              flags: ""
+              teardown: true
+              logging: true
+            - target: voltha-onu-omci-get-multiolt-kind-att
+              workflow: att
+              flags: ""
+              teardown: true
+              logging: true
+            - target: voltha-onu-omci-get-multiolt-kind-dt
+              workflow: dt
+              flags: ""
+              teardown: true
+              logging: true
+            - target: voltha-onu-omci-get-multiolt-kind-tt
+              workflow: tt
+              flags: ""
+              teardown: true
+              logging: true
+          time-trigger: "H H/23 * * *"
+          timeout: 235
+
+      - 'voltha-periodic-test':
           name: 'periodic-voltha-multiple-olts-openonu-go-test-bbsim-2.8'
           pipeline-script: 'voltha/voltha-2.8/bbsim-tests.groovy'
           code-branch: 'voltha-2.8'
@@ -738,6 +865,64 @@
           timeout: 235
 
       - 'voltha-periodic-test':
+          name: 'patchset-voltha-2.9-multiple-olts-openonu-go-test-bbsim'
+          trigger-comment: "voltha 2.9 test openonu multiolt"
+          pipeline-script: 'voltha/voltha-2.9/bbsim-tests.groovy'
+          code-branch: '$GERRIT_BRANCH'
+          gerritProject: '$GERRIT_PROJECT'
+          gerritRefspec: '$GERRIT_REFSPEC'
+          extraHelmFlags: '--set onu=2,pon=2'
+          olts: 2
+          logLevel: 'DEBUG'
+          testTargets: |
+            - target: 1t1gem-openonu-go-adapter-multi-olt-test
+              workflow: att
+              flags: ""
+              teardown: true
+              logging: true
+            - target: 1t4gem-openonu-go-adapter-multi-olt-test
+              workflow: att
+              flags: ""
+              teardown: true
+              logging: true
+            - target: 1t8gem-openonu-go-adapter-multi-olt-test
+              workflow: att
+              flags: ""
+              teardown: true
+              logging: true
+            - target: reconcile-openonu-go-adapter-multi-olt-test-att
+              workflow: att
+              flags: ""
+              teardown: true
+              logging: true
+            - target: reconcile-openonu-go-adapter-multi-olt-test-dt
+              workflow: dt
+              flags: ""
+              teardown: true
+              logging: true
+            - target: reconcile-openonu-go-adapter-multi-olt-test-tt
+              workflow: tt
+              flags: ""
+              teardown: true
+              logging: true
+            - target: voltha-onu-omci-get-multiolt-kind-att
+              workflow: att
+              flags: ""
+              teardown: true
+              logging: true
+            - target: voltha-onu-omci-get-multiolt-kind-dt
+              workflow: dt
+              flags: ""
+              teardown: true
+              logging: true
+            - target: voltha-onu-omci-get-multiolt-kind-tt
+              workflow: tt
+              flags: ""
+              teardown: true
+              logging: true
+          timeout: 235
+
+      - 'voltha-periodic-test':
           name: 'patchset-voltha-2.8-multiple-olts-openonu-go-test-bbsim'
           trigger-comment: "voltha 2.8 test openonu multiolt"
           pipeline-script: 'voltha/voltha-2.8/bbsim-tests.groovy'