Merge "Getting OfAgent to connect to all ONOS instance when running onos-ha test with the new olt app"
diff --git a/jjb/pipeline/voltha/master/bbsim-tests.groovy b/jjb/pipeline/voltha/master/bbsim-tests.groovy
index b2f0220..4c88e46 100755
--- a/jjb/pipeline/voltha/master/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/master/bbsim-tests.groovy
@@ -23,7 +23,7 @@
 
 def clusterName = "kind-ci"
 
-def execute_test(testTarget, workflow, teardown, testSpecificHelmFlags = "") {
+def execute_test(testTarget, workflow, testLogging, teardown, testSpecificHelmFlags = "") {
   def infraNamespace = "default"
   def volthaNamespace = "voltha"
   def robotLogsDir = "RobotLogs"
@@ -108,33 +108,14 @@
     }
   }
   stage('Run test ' + testTarget + ' on ' + workflow + ' workFlow') {
-    // start logging
-    sh """
-    mkdir -p $WORKSPACE/${testTarget}-components
-    _TAG=kail-${workflow} kail -n ${infraNamespace} -n ${volthaNamespace} > $WORKSPACE/${testTarget}-components/onos-voltha-combined.log &
-    """
     sh """
     mkdir -p $WORKSPACE/${robotLogsDir}/${testTarget}-robot
     export ROBOT_MISC_ARGS="-d $WORKSPACE/${robotLogsDir}/${testTarget}-robot "
-    ROBOT_MISC_ARGS+="-v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
+    ROBOT_MISC_ARGS+="-v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace} -v container_log_dir:$WORKSPACE/${robotLogsDir}/${testTarget}-robot -v logging:${testLogging}"
     export KVSTOREPREFIX=voltha/voltha_voltha
 
     make -C $WORKSPACE/voltha-system-tests ${testTarget} || true
     """
-    // stop logging
-    sh """
-      P_IDS="\$(ps e -ww -A | grep "_TAG=kail-${workflow}" | grep -v grep | awk '{print \$1}')"
-      if [ -n "\$P_IDS" ]; then
-        echo \$P_IDS
-        for P_ID in \$P_IDS; do
-          kill -9 \$P_ID
-        done
-      fi
-      cd $WORKSPACE/${testTarget}-components/
-      rm onos-voltha-combined.log.gz || true
-      gzip -k onos-voltha-combined.log
-      rm onos-voltha-combined.log
-    """
     getPodsInfo("$WORKSPACE/${testTarget}-components")
   }
 }
@@ -251,8 +232,13 @@
               def workflow = test["workflow"]
               def flags = test["flags"]
               def teardown = test["teardown"].toBoolean()
-              println "Executing test ${target} on workflow ${workflow} with extra flags ${flags}"
-              execute_test(target, workflow, teardown, flags)
+              def logging = test["logging"].toBoolean()
+              def testLogging = 'False'
+              if (logging) {
+                  testLogging = 'True'
+              }
+              println "Executing test ${target} on workflow ${workflow} with logging ${testLogging} and extra flags ${flags}"
+              execute_test(target, workflow, testLogging, teardown, flags)
             }
           }
         }
diff --git a/jjb/pipeline/voltha/master/software-upgrades.groovy b/jjb/pipeline/voltha/master/software-upgrades.groovy
index db0916b..2724510 100644
--- a/jjb/pipeline/voltha/master/software-upgrades.groovy
+++ b/jjb/pipeline/voltha/master/software-upgrades.groovy
@@ -25,8 +25,14 @@
   def volthaNamespace = "voltha"
   stage('Deploy Voltha - '+ name) {
     timeout(10) {
+      // start logging
+      sh """
+      rm -rf $WORKSPACE/${name} || true
+      mkdir -p $WORKSPACE/${name}
+      _TAG=kail-${name} kail -n ${infraNamespace} -n ${volthaNamespace} > $WORKSPACE/${name}/onos-voltha-startup-combined.log &
+      """
       def extraHelmFlags = extraHelmFlags.trim()
-      extraHelmFlags = extraHelmFlags + " --set global.log_level=DEBUG,onu=1,pon=1 --set onos-classic.replicas=3,onos-classic.atomix.replicas=3 "
+      extraHelmFlags = extraHelmFlags + " --set global.log_level=${logLevel.toUpperCase()},onu=1,pon=1 --set onos-classic.replicas=3,onos-classic.atomix.replicas=3 "
       if ("${name}" == "onos-app-upgrade" || "${name}" == "onu-software-upgrade") {
           extraHelmFlags = extraHelmFlags + "--set global.image_tag=master --set onos-classic.image.tag=master "
       }
@@ -46,11 +52,18 @@
       // Currently only testing with ATT workflow
       // TODO: Support for other workflows
       volthaDeploy([workflow: "att", extraHelmFlags: extraHelmFlags, localCharts: localCharts])
-      // start logging
+      // stop logging
       sh """
-      rm -rf $WORKSPACE/${name} || true
-      mkdir -p $WORKSPACE/${name}
-      _TAG=kail-${name} kail -n infra -n voltha > $WORKSPACE/${name}/onos-voltha-combined.log &
+        P_IDS="\$(ps e -ww -A | grep "_TAG=kail-${name}" | grep -v grep | awk '{print \$1}')"
+        if [ -n "\$P_IDS" ]; then
+          echo \$P_IDS
+          for P_ID in \$P_IDS; do
+            kill -9 \$P_ID
+          done
+        fi
+        cd $WORKSPACE/${name}/
+        gzip -k onos-voltha-startup-combined.log
+        rm onos-voltha-startup-combined.log
       """
       // forward ONOS and VOLTHA ports
       sh """
@@ -115,22 +128,16 @@
           export ROBOT_MISC_ARGS="-d \$ROBOT_LOGS_DIR -v image_version:${onuImageVersion.trim()} -v image_url:${onuImageUrl.trim()} -v image_vendor:${onuImageVendor.trim()} -v image_activate_on_success:${onuImageActivateOnSuccess.trim()} -v image_commit_on_success:${onuImageCommitOnSuccess.trim()} -v image_crc:${onuImageCrc.trim()} -e PowerSwitch"
           export TARGET=onu-upgrade-test
         fi
+        testLogging='False'
+        if [ ${logging} = true ]; then
+          testLogging='True'
+        fi
         export VOLTCONFIG=$HOME/.volt/config-minimal
         export KUBECONFIG=$HOME/.kube/kind-config-voltha-minimal
-        ROBOT_MISC_ARGS+=" -v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
+        ROBOT_MISC_ARGS+=" -v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace} -v container_log_dir:$WORKSPACE/RobotLogs/${name} -v logging:\$testLogging"
         # Run the specified tests
         make -C $WORKSPACE/voltha-system-tests \$TARGET || true
       """
-      // stop logging
-      sh """
-        P_IDS="\$(ps e -ww -A | grep "_TAG=kail-${name}" | grep -v grep | awk '{print \$1}')"
-        if [ -n "\$P_IDS" ]; then
-          echo \$P_IDS
-          for P_ID in \$P_IDS; do
-            kill -9 \$P_ID
-          done
-        fi
-      """
       // remove port-forwarding
       sh """
         # remove orphaned port-forward from different namespaces
@@ -212,11 +219,6 @@
       get_pods_info("$WORKSPACE/failed")
     }
     always {
-      sh '''
-      gzip $WORKSPACE/onos-app-upgrade/onos-voltha-combined.log || true
-      gzip $WORKSPACE/voltha-component-upgrade/onos-voltha-combined.log || true
-      gzip $WORKSPACE/onu-software-upgrade/onos-voltha-combined.log || true
-      '''
       step([$class: 'RobotPublisher',
          disableArchiveOutput: false,
          logFileName: 'RobotLogs/*/log*.html',
diff --git a/jjb/pipeline/voltha/voltha-2.8/bbsim-tests.groovy b/jjb/pipeline/voltha/voltha-2.8/bbsim-tests.groovy
index ef6964b..fd07107 100755
--- a/jjb/pipeline/voltha/voltha-2.8/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/voltha-2.8/bbsim-tests.groovy
@@ -23,7 +23,7 @@
 
 def clusterName = "kind-ci"
 
-def execute_test(testTarget, workflow, teardown, testSpecificHelmFlags = "") {
+def execute_test(testTarget, workflow, testLogging, teardown, testSpecificHelmFlags = "") {
   def infraNamespace = "default"
   def volthaNamespace = "voltha"
   def robotLogsDir = "RobotLogs"
@@ -108,33 +108,14 @@
     }
   }
   stage('Run test ' + testTarget + ' on ' + workflow + ' workFlow') {
-    // start logging
-    sh """
-    mkdir -p $WORKSPACE/${testTarget}-components
-    _TAG=kail-${workflow} kail -n ${infraNamespace} -n ${volthaNamespace} > $WORKSPACE/${testTarget}-components/onos-voltha-combined.log &
-    """
     sh """
     mkdir -p $WORKSPACE/${robotLogsDir}/${testTarget}-robot
     export ROBOT_MISC_ARGS="-d $WORKSPACE/${robotLogsDir}/${testTarget}-robot "
-    ROBOT_MISC_ARGS+="-v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v INFRA_NAMESPACE:${infraNamespace}"
+    ROBOT_MISC_ARGS+="-v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v INFRA_NAMESPACE:${infraNamespace} -v container_log_dir:$WORKSPACE/${robotLogsDir}/${testTarget}-robot -v logging:${testLogging}"
     export KVSTOREPREFIX=voltha/voltha_voltha
 
     make -C $WORKSPACE/voltha-system-tests ${testTarget} || true
     """
-    // stop logging
-    sh """
-      P_IDS="\$(ps e -ww -A | grep "_TAG=kail-${workflow}" | grep -v grep | awk '{print \$1}')"
-      if [ -n "\$P_IDS" ]; then
-        echo \$P_IDS
-        for P_ID in \$P_IDS; do
-          kill -9 \$P_ID
-        done
-      fi
-      cd $WORKSPACE/${testTarget}-components/
-      rm onos-voltha-combined.log.gz || true
-      gzip -k onos-voltha-combined.log
-      rm onos-voltha-combined.log
-    """
     getPodsInfo("$WORKSPACE/${testTarget}-components")
   }
 }
@@ -251,8 +232,13 @@
               def workflow = test["workflow"]
               def flags = test["flags"]
               def teardown = test["teardown"].toBoolean()
-              println "Executing test ${target} on workflow ${workflow} with extra flags ${flags}"
-              execute_test(target, workflow, teardown, flags)
+              def logging = test["logging"].toBoolean()
+              def testLogging = 'False'
+              if (logging) {
+                  testLogging = 'True'
+              }
+              println "Executing test ${target} on workflow ${workflow} with logging ${testLogging} and extra flags ${flags}"
+              execute_test(target, workflow, testLogging, teardown, flags)
             }
           }
         }
diff --git a/jjb/pipeline/voltha/voltha-2.8/software-upgrades.groovy b/jjb/pipeline/voltha/voltha-2.8/software-upgrades.groovy
index d5aa0ac..88fb129 100644
--- a/jjb/pipeline/voltha/voltha-2.8/software-upgrades.groovy
+++ b/jjb/pipeline/voltha/voltha-2.8/software-upgrades.groovy
@@ -21,10 +21,18 @@
       remote: 'https://gerrit.opencord.org/ci-management.git'
 ])
 def test_software_upgrade(name) {
+  def infraNamespace = "infra"
+  def volthaNamespace = "voltha"
   stage('Deploy Voltha - '+ name) {
     timeout(10) {
+      // start logging
+      sh """
+      rm -rf $WORKSPACE/${name} || true
+      mkdir -p $WORKSPACE/${name}
+      _TAG=kail-${name} kail -n ${infraNamespace} -n ${volthaNamespace} > $WORKSPACE/${name}/onos-voltha-startup-combined.log &
+      """
       def extraHelmFlags = extraHelmFlags.trim()
-      extraHelmFlags = extraHelmFlags + " --set global.log_level=DEBUG,onu=1,pon=1 --set onos-classic.replicas=3,onos-classic.atomix.replicas=3 "
+      extraHelmFlags = extraHelmFlags + " --set global.log_level=${logLevel.toUpperCase()},onu=1,pon=1 --set onos-classic.replicas=3,onos-classic.atomix.replicas=3 "
 
       extraHelmFlags = extraHelmFlags + " --set onos-classic.onosSshPort=30115 --set onos-classic.onosApiPort=30120 "
       extraHelmFlags = extraHelmFlags + " --set voltha.onos_classic.replicas=3"
@@ -42,11 +50,18 @@
       // Currently only testing with ATT workflow
       // TODO: Support for other workflows
       volthaDeploy([workflow: "att", extraHelmFlags: extraHelmFlags, localCharts: localCharts])
-      // start logging
+      // stop logging
       sh """
-      rm -rf $WORKSPACE/${name} || true
-      mkdir -p $WORKSPACE/${name}
-      _TAG=kail-${name} kail -n infra -n voltha > $WORKSPACE/${name}/onos-voltha-combined.log &
+        P_IDS="\$(ps e -ww -A | grep "_TAG=kail-${name}" | grep -v grep | awk '{print \$1}')"
+        if [ -n "\$P_IDS" ]; then
+          echo \$P_IDS
+          for P_ID in \$P_IDS; do
+            kill -9 \$P_ID
+          done
+        fi
+        cd $WORKSPACE/${name}/
+        gzip -k onos-voltha-startup-combined.log
+        rm onos-voltha-startup-combined.log
       """
       // forward ONOS and VOLTHA ports
       sh """
@@ -111,22 +126,16 @@
           export ROBOT_MISC_ARGS="-d \$ROBOT_LOGS_DIR -v image_version:${onuImageVersion.trim()} -v image_url:${onuImageUrl.trim()} -v image_vendor:${onuImageVendor.trim()} -v image_activate_on_success:${onuImageActivateOnSuccess.trim()} -v image_commit_on_success:${onuImageCommitOnSuccess.trim()} -v image_crc:${onuImageCrc.trim()} -e PowerSwitch"
           export TARGET=onu-upgrade-test
         fi
+        testLogging='False'
+        if [ ${logging} = true ]; then
+          testLogging='True'
+        fi
         export VOLTCONFIG=$HOME/.volt/config-minimal
         export KUBECONFIG=$HOME/.kube/kind-config-voltha-minimal
-        ROBOT_MISC_ARGS+=" -v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120"
+        ROBOT_MISC_ARGS+=" -v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace} -v container_log_dir:$WORKSPACE/RobotLogs/${name} -v logging:\$testLogging"
         # Run the specified tests
         make -C $WORKSPACE/voltha-system-tests \$TARGET || true
       """
-      // stop logging
-      sh """
-        P_IDS="\$(ps e -ww -A | grep "_TAG=kail-${name}" | grep -v grep | awk '{print \$1}')"
-        if [ -n "\$P_IDS" ]; then
-          echo \$P_IDS
-          for P_ID in \$P_IDS; do
-            kill -9 \$P_ID
-          done
-        fi
-      """
       // remove port-forwarding
       sh """
         # remove orphaned port-forward from different namespaces
@@ -208,11 +217,6 @@
       get_pods_info("$WORKSPACE/failed")
     }
     always {
-      sh '''
-      gzip $WORKSPACE/onos-app-upgrade/onos-voltha-combined.log || true
-      gzip $WORKSPACE/voltha-component-upgrade/onos-voltha-combined.log || true
-      gzip $WORKSPACE/onu-software-upgrade/onos-voltha-combined.log || true
-      '''
       step([$class: 'RobotPublisher',
          disableArchiveOutput: false,
          logFileName: 'RobotLogs/*/log*.html',
diff --git a/jjb/software-upgrades.yaml b/jjb/software-upgrades.yaml
index 3591992..15a1474 100644
--- a/jjb/software-upgrades.yaml
+++ b/jjb/software-upgrades.yaml
@@ -75,6 +75,8 @@
     volthaSystemTestsChange: ''
     volthaHelmChartsChange: ''
     kindVolthaChange: ''
+    logLevel: 'DEBUG'
+    logging: true
 
     description: |
       <!-- Managed by Jenkins Job Builder -->
@@ -247,6 +249,16 @@
           default: '{onu-image-crc}'
           description: 'CRC of ONU Image to Upgrade'
 
+      - string:
+          name: logLevel
+          default: '{logLevel}'
+          description: 'Log level for all the components'
+
+      - bool:
+          name: logging
+          default: '{logging}'
+          description: 'Option to collect voltha components logs per test case'
+
     project-type: pipeline
     concurrent: true
 
diff --git a/jjb/verify/bbsim.yaml b/jjb/verify/bbsim.yaml
index 3b64b88..22b44f7 100644
--- a/jjb/verify/bbsim.yaml
+++ b/jjb/verify/bbsim.yaml
@@ -39,14 +39,17 @@
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: sanity-bbsim-dt
               workflow: dt
               flags: ""
               teardown: true
+              logging: true
             - target: sanity-bbsim-tt
               workflow: tt
               flags: ""
               teardown: true
+              logging: true
 
 - job-group:
     name: 'verify-bbsim-jobs-master'
@@ -58,14 +61,17 @@
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: sanity-bbsim-dt
               workflow: dt
               flags: ""
               teardown: true
+              logging: true
             - target: sanity-bbsim-tt
               workflow: tt
               flags: ""
               teardown: true
+              logging: true
 
 - job-group:
     name: 'publish-bbsim-jobs'
diff --git a/jjb/verify/voltha-openonu-adapter-go.yaml b/jjb/verify/voltha-openonu-adapter-go.yaml
index efcef9f..8a1353a 100644
--- a/jjb/verify/voltha-openonu-adapter-go.yaml
+++ b/jjb/verify/voltha-openonu-adapter-go.yaml
@@ -53,18 +53,22 @@
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: sanity-kind-dt
               workflow: dt
               flags: ""
               teardown: true
+              logging: true
             - target: sanity-kind-tt
               workflow: tt
               flags: ""
               teardown: true
+              logging: true
             - target: 1t4gem-openonu-go-adapter-test
               workflow: att
               flags: ""
               teardown: true
+              logging: true
 
 - job-group:
     name: 'publish-voltha-openonu-adapter-go-jobs'
diff --git a/jjb/voltha-e2e.yaml b/jjb/voltha-e2e.yaml
index 0c6d2d5..9eae7b1 100755
--- a/jjb/voltha-e2e.yaml
+++ b/jjb/voltha-e2e.yaml
@@ -31,22 +31,27 @@
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: bbsim-alarms-kind
               workflow: att
               flags: ""
               teardown: false
+              logging: true
             - target: bbsim-failurescenarios
               workflow: att
               flags: ""
               teardown: false
+              logging: true
             - target: bbsim-errorscenarios
               workflow: att
               flags: ""
               teardown: false
+              logging: true
             - target: onos-ha-test
               workflow: att
               flags: ""
               teardown: false
+              logging: true
 
       - 'voltha-periodic-test':
           name: 'periodic-voltha-test-bbsim-new-olt-app'
@@ -58,22 +63,27 @@
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: bbsim-alarms-kind
               workflow: att
               flags: ""
               teardown: false
+              logging: true
             - target: bbsim-failurescenarios
               workflow: att
               flags: ""
               teardown: false
+              logging: true
             - target: bbsim-errorscenarios
               workflow: att
               flags: ""
               teardown: false
+              logging: true
             - target: onos-ha-test
               workflow: att
               flags: ""
               teardown: false
+              logging: true
 
       - 'voltha-periodic-test':
           name: 'periodic-voltha-multiple-olts-test-bbsim'
@@ -86,18 +96,22 @@
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: bbsim-multiolt-failurescenarios
               workflow: att
               flags: ""
               teardown: false
+              logging: true
             - target: bbsim-multiolt-errorscenarios
               workflow: att
               flags: ""
               teardown: false
+              logging: true
             - target: bbsim-multiolt-kind
               workflow: att
               flags: ""
               teardown: false
+              logging: true
 
       - 'voltha-periodic-test':
           name: 'periodic-voltha-test-bbsim-2.8'
@@ -109,18 +123,22 @@
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: bbsim-alarms-kind
               workflow: att
               flags: ""
               teardown: false
+              logging: true
             - target: bbsim-failurescenarios
               workflow: att
               flags: ""
               teardown: false
+              logging: true
             - target: bbsim-errorscenarios
               workflow: att
               flags: ""
               teardown: false
+              logging: true
 
       - 'voltha-periodic-test':
           name: 'periodic-voltha-multiple-olts-test-bbsim-2.8'
@@ -134,18 +152,22 @@
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: bbsim-multiolt-failurescenarios
               workflow: att
               flags: ""
               teardown: false
+              logging: true
             - target: bbsim-multiolt-errorscenarios
               workflow: att
               flags: ""
               teardown: false
+              logging: true
             - target: bbsim-multiolt-kind
               workflow: att
               flags: ""
               teardown: false
+              logging: true
 
       - 'voltha-periodic-test':
           name: 'periodic-voltha-multi-uni-test-bbsim'
@@ -157,14 +179,17 @@
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: bbsim-multiuni-failurescenarios-att
               workflow: att
               flags: ""
               teardown: false
+              logging: true
             - target: bbsim-multiuni-errorscenarios-att
               workflow: att
               flags: ""
               teardown: false
+              logging: true
 
       - 'voltha-periodic-test':
           name: 'periodic-voltha-multi-uni-multiple-olts-test-bbsim'
@@ -177,14 +202,17 @@
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: bbsim-multiuni-multiolt-failurescenarios-att
               workflow: att
               flags: ""
               teardown: false
+              logging: true
             - target: bbsim-multiuni-multiolt-errorscenarios-att
               workflow: att
               flags: ""
               teardown: false
+              logging: true
           timeout: 180
 
       - 'voltha-periodic-test':
@@ -198,14 +226,17 @@
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: bbsim-multiuni-failurescenarios-att
               workflow: att
               flags: ""
               teardown: false
+              logging: true
             - target: bbsim-multiuni-errorscenarios-att
               workflow: att
               flags: ""
               teardown: false
+              logging: true
 
       - 'voltha-periodic-test':
           name: 'periodic-voltha-multi-uni-multiple-olts-test-bbsim-2.8'
@@ -219,14 +250,17 @@
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: bbsim-multiuni-multiolt-failurescenarios-att
               workflow: att
               flags: ""
               teardown: false
+              logging: true
             - target: bbsim-multiuni-multiolt-errorscenarios-att
               workflow: att
               flags: ""
               teardown: false
+              logging: true
           timeout: 180
 
       # openonu Go periodic tests
@@ -241,62 +275,77 @@
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: 1t4gem-openonu-go-adapter-test
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: 1t8gem-openonu-go-adapter-test
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: mib-upload-templating-openonu-go-adapter-test
               workflow: att
               flags: "--set pon=2,onu=2,controlledActivation=only-onu"
               teardown: true
+              logging: true
             - target: reconcile-openonu-go-adapter-test-att
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: reconcile-openonu-go-adapter-test-dt
               workflow: dt
               flags: ""
               teardown: true
+              logging: true
             - target: reconcile-openonu-go-adapter-test-tt
               workflow: tt
               flags: ""
               teardown: true
+              logging: true
             - target: openonu-go-adapter-omci-hardening-passed-test
               workflow: att
               flags: "--set omci_response_rate=9 --set voltha-adapter-openonu.adapter_open_onu.omci_timeout=1s"
               teardown: true
+              logging: true
             - target: openonu-go-adapter-omci-hardening-failed-test
               workflow: att
               flags: "--set omci_response_rate=7"
               teardown: true
+              logging: true
             - target: voltha-onu-omci-get-single-kind-att
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-onu-omci-get-single-kind-dt
               workflow: dt
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-onu-omci-get-single-kind-tt
               workflow: tt
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-onu-flows-check-single-kind-att
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-onu-flows-check-single-kind-dt
               workflow: dt
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-onu-flows-check-single-kind-tt
               workflow: tt
               flags: ""
               teardown: true
+              logging: true
 
       - 'voltha-periodic-test':
           name: 'periodic-voltha-openonu-go-test-bbsim-new-olt-app'
@@ -309,62 +358,77 @@
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: 1t4gem-openonu-go-adapter-test
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: 1t8gem-openonu-go-adapter-test
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: mib-upload-templating-openonu-go-adapter-test
               workflow: att
               flags: "--set pon=2,onu=2,controlledActivation=only-onu"
               teardown: true
+              logging: true
             - target: reconcile-openonu-go-adapter-test-att
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: reconcile-openonu-go-adapter-test-dt
               workflow: dt
               flags: ""
               teardown: true
+              logging: true
             - target: reconcile-openonu-go-adapter-test-tt
               workflow: tt
               flags: ""
               teardown: true
+              logging: true
             - target: openonu-go-adapter-omci-hardening-passed-test
               workflow: att
               flags: "--set omci_response_rate=9 --set voltha-adapter-openonu.adapter_open_onu.omci_timeout=1s"
               teardown: true
+              logging: true
             - target: openonu-go-adapter-omci-hardening-failed-test
               workflow: att
               flags: "--set omci_response_rate=7"
               teardown: true
+              logging: true
             - target: voltha-onu-omci-get-single-kind-att
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-onu-omci-get-single-kind-dt
               workflow: dt
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-onu-omci-get-single-kind-tt
               workflow: tt
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-onu-flows-check-single-kind-att
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-onu-flows-check-single-kind-dt
               workflow: dt
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-onu-flows-check-single-kind-tt
               workflow: tt
               flags: ""
               teardown: true
+              logging: true
 
       - 'voltha-periodic-test':
           name: 'periodic-voltha-openonu-go-test-bbsim-2.8'
@@ -377,50 +441,62 @@
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: 1t4gem-openonu-go-adapter-test
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: 1t8gem-openonu-go-adapter-test
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: mib-upload-templating-openonu-go-adapter-test
               workflow: att
               flags: "--set pon=2,onu=2,controlledActivation=only-onu"
               teardown: true
+              logging: true
             - target: reconcile-openonu-go-adapter-test-att
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: reconcile-openonu-go-adapter-test-dt
               workflow: dt
               flags: ""
               teardown: true
+              logging: true
             - target: reconcile-openonu-go-adapter-test-tt
               workflow: tt
               flags: ""
               teardown: true
+              logging: true
             - target: openonu-go-adapter-omci-hardening-passed-test
               workflow: att
               flags: "--set omci_response_rate=9 --set voltha-adapter-openonu.adapter_open_onu.omci_timeout=1s"
               teardown: true
+              logging: true
             - target: openonu-go-adapter-omci-hardening-failed-test
               workflow: att
               flags: "--set omci_response_rate=7"
               teardown: true
+              logging: true
             - target: voltha-onu-omci-get-single-kind-att
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-onu-omci-get-single-kind-dt
               workflow: dt
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-onu-omci-get-single-kind-tt
               workflow: tt
               flags: ""
               teardown: true
+              logging: true
 
       - 'voltha-periodic-test':
           name: 'patchset-voltha-openonu-go-test-bbsim'
@@ -435,62 +511,77 @@
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: 1t4gem-openonu-go-adapter-test
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: 1t8gem-openonu-go-adapter-test
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: mib-upload-templating-openonu-go-adapter-test
               workflow: att
               flags: "--set pon=2,onu=2,controlledActivation=only-onu"
               teardown: true
+              logging: true
             - target: reconcile-openonu-go-adapter-test-att
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: reconcile-openonu-go-adapter-test-dt
               workflow: dt
               flags: ""
               teardown: true
+              logging: true
             - target: reconcile-openonu-go-adapter-test-tt
               workflow: tt
               flags: ""
               teardown: true
+              logging: true
             - target: openonu-go-adapter-omci-hardening-passed-test
               workflow: att
               flags: "--set omci_response_rate=9 --set voltha-adapter-openonu.adapter_open_onu.omci_timeout=1s"
               teardown: true
+              logging: true
             - target: openonu-go-adapter-omci-hardening-failed-test
               workflow: att
               flags: "--set omci_response_rate=7"
               teardown: true
+              logging: true
             - target: voltha-onu-omci-get-single-kind-att
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-onu-omci-get-single-kind-dt
               workflow: dt
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-onu-omci-get-single-kind-tt
               workflow: tt
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-onu-flows-check-single-kind-att
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-onu-flows-check-single-kind-dt
               workflow: dt
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-onu-flows-check-single-kind-tt
               workflow: tt
               flags: ""
               teardown: true
+              logging: true
 
       - 'voltha-periodic-test':
           name: 'periodic-voltha-multiple-olts-openonu-go-test-bbsim'
@@ -503,50 +594,62 @@
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: 1t4gem-openonu-go-adapter-multi-olt-test
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: 1t8gem-openonu-go-adapter-multi-olt-test
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: reconcile-openonu-go-adapter-multi-olt-test-att
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: reconcile-openonu-go-adapter-multi-olt-test-dt
               workflow: dt
               flags: ""
               teardown: true
+              logging: true
             - target: reconcile-openonu-go-adapter-multi-olt-test-tt
               workflow: tt
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-onu-omci-get-multiolt-kind-att
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-onu-omci-get-multiolt-kind-dt
               workflow: dt
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-onu-omci-get-multiolt-kind-tt
               workflow: tt
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-onu-flows-check-multiolt-kind-att
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-onu-flows-check-multiolt-kind-dt
               workflow: dt
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-onu-flows-check-multiolt-kind-tt
               workflow: tt
               flags: ""
               teardown: true
+              logging: true
           time-trigger: "H H/12 * * *"
 
       - 'voltha-periodic-test':
@@ -561,38 +664,47 @@
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: 1t4gem-openonu-go-adapter-multi-olt-test
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: 1t8gem-openonu-go-adapter-multi-olt-test
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: reconcile-openonu-go-adapter-multi-olt-test-att
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: reconcile-openonu-go-adapter-multi-olt-test-dt
               workflow: dt
               flags: ""
               teardown: true
+              logging: true
             - target: reconcile-openonu-go-adapter-multi-olt-test-tt
               workflow: tt
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-onu-omci-get-multiolt-kind-att
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-onu-omci-get-multiolt-kind-dt
               workflow: dt
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-onu-omci-get-multiolt-kind-tt
               workflow: tt
               flags: ""
               teardown: true
+              logging: true
           time-trigger: "H H/23 * * *"
 
       - 'voltha-periodic-test':
@@ -608,50 +720,62 @@
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: 1t4gem-openonu-go-adapter-multi-olt-test
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: 1t8gem-openonu-go-adapter-multi-olt-test
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: reconcile-openonu-go-adapter-multi-olt-test-att
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: reconcile-openonu-go-adapter-multi-olt-test-dt
               workflow: dt
               flags: ""
               teardown: true
+              logging: true
             - target: reconcile-openonu-go-adapter-multi-olt-test-tt
               workflow: tt
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-onu-omci-get-multiolt-kind-att
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-onu-omci-get-multiolt-kind-dt
               workflow: dt
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-onu-omci-get-multiolt-kind-tt
               workflow: tt
               flags: ""
               teardown: true
-             - target: voltha-onu-flows-check-multiolt-kind-att
+              logging: true
+            - target: voltha-onu-flows-check-multiolt-kind-att
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-onu-flows-check-multiolt-kind-dt
               workflow: dt
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-onu-flows-check-multiolt-kind-tt
               workflow: tt
               flags: ""
               teardown: true
+              logging: true
           olts: 2
 
       - 'voltha-periodic-test':
@@ -665,14 +789,17 @@
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-pm-data-single-kind-dt
               workflow: dt
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-pm-data-single-kind-tt
               workflow: tt
               flags: ""
               teardown: true
+              logging: true
           timeout: 140
 
       - 'voltha-periodic-test':
@@ -688,14 +815,17 @@
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-pm-data-single-kind-dt
               workflow: dt
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-pm-data-single-kind-tt
               workflow: tt
               flags: ""
               teardown: true
+              logging: true
           timeout: 140
 
       - 'voltha-periodic-test':
@@ -710,14 +840,17 @@
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-pm-data-single-kind-dt
               workflow: dt
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-pm-data-single-kind-tt
               workflow: tt
               flags: ""
               teardown: true
+              logging: true
           timeout: 140
 
       - 'voltha-periodic-test':
@@ -732,14 +865,17 @@
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-pm-data-multiolt-kind-dt
               workflow: dt
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-pm-data-multiolt-kind-tt
               workflow: tt
               flags: ""
               teardown: true
+              logging: true
           time-trigger: "H H/23 * * *"
 
       - 'voltha-periodic-test':
@@ -755,14 +891,17 @@
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-pm-data-multiolt-kind-dt
               workflow: dt
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-pm-data-multiolt-kind-tt
               workflow: tt
               flags: ""
               teardown: true
+              logging: true
           olts: 2
           timeout: 180
 
@@ -779,14 +918,17 @@
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-pm-data-multiolt-kind-dt
               workflow: dt
               flags: ""
               teardown: true
+              logging: true
             - target: voltha-pm-data-multiolt-kind-tt
               workflow: tt
               flags: ""
               teardown: true
+              logging: true
           time-trigger: "H H/23 * * *"
 
       - 'voltha-periodic-test':
@@ -799,6 +941,7 @@
               workflow: att
               flags: ""
               teardown: true
+              logging: true
 
       - 'voltha-periodic-test':
           name: 'periodic-voltha-test-DMI-2.8'
@@ -810,6 +953,7 @@
               workflow: att
               flags: ""
               teardown: true
+              logging: true
 
       - 'voltha-periodic-test':
           name: 'periodic-voltha-etcd-test'
@@ -822,6 +966,7 @@
               workflow: att
               flags: ""
               teardown: true
+              logging: true
 
       - 'voltha-periodic-test':
           name: 'periodic-voltha-etcd-test-2.8'
@@ -835,6 +980,7 @@
               workflow: att
               flags: ""
               teardown: true
+              logging: true
 
       - 'voltha-periodic-test':
           name: 'periodic-voltha-sanity-test-multi-runs'
@@ -846,22 +992,27 @@
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: sanity-kind
               workflow: att
               flags: ""
               teardown: false
+              logging: true
             - target: sanity-kind
               workflow: att
               flags: ""
               teardown: false
+              logging: true
             - target: sanity-kind
               workflow: att
               flags: ""
               teardown: false
+              logging: true
             - target: sanity-kind
               workflow: att
               flags: ""
               teardown: false
+              logging: true
 
       - 'voltha-periodic-test':
           name: 'periodic-voltha-sanity-test-multi-runs-2.8'
@@ -873,22 +1024,27 @@
               workflow: att
               flags: ""
               teardown: true
+              logging: true
             - target: sanity-kind
               workflow: att
               flags: ""
               teardown: false
+              logging: true
             - target: sanity-kind
               workflow: att
               flags: ""
               teardown: false
+              logging: true
             - target: sanity-kind
               workflow: att
               flags: ""
               teardown: false
+              logging: true
             - target: sanity-kind
               workflow: att
               flags: ""
               teardown: false
+              logging: true
 
       - 'voltha-periodic-test':
           name: 'nightly-voltha-DTflow-sanity-test'
@@ -901,6 +1057,7 @@
               workflow: dt
               flags: ""
               teardown: true
+              logging: true
 
       # ATT Per-patchset Pod builds on Tucson pod (master)
       - 'verify_physical_voltha_patchset_auto':
@@ -1009,6 +1166,7 @@
       #     workflow: att
       #     flags: ""
       #     teardown: true
+      #     logging: true
       - text:
           name: testTargets
           default: '{testTargets}'
@@ -1112,14 +1270,17 @@
         workflow: att
         flags: ""
         teardown: true
+        logging: true
       - target: sanity-kind-dt
         workflow: dt
         flags: ""
         teardown: true
+        logging: true
       - target: sanity-kind-tt
         workflow: tt
         flags: ""
         teardown: true
+        logging: true
 
     description: |
       <!-- Managed by Jenkins Job Builder -->
@@ -1181,6 +1342,7 @@
       #     workflow: att
       #     flags: ""
       #     teardown: true
+      #     logging: true
       - text:
           name: testTargets
           default: '{testTargets}'
diff --git a/jjb/voltha-test/voltha-certification.yaml b/jjb/voltha-test/voltha-certification.yaml
index f048043..994c543 100644
--- a/jjb/voltha-test/voltha-certification.yaml
+++ b/jjb/voltha-test/voltha-certification.yaml
@@ -36,7 +36,7 @@
           work-flow: 'DT'
           profile: '1T8GEM'
           time: '21'
-          extraHelmFlags: "--set onos-classic.image.repository=andreacampanella/voltha-onos --set onos-classic.image.tag=meter-fixes-simple"
+          extraHelmFlags: "--set onos-classic.image.repository=andreacampanella/voltha-onos --set onos-classic.image.tag=meter-fixes-simple --set voltha.global.rw_core.rpc_timeout=30s --set voltha.global.rw_core.internal_timeout=30s"
 
       # Certification (Radisys 1600G) POD test job - master versions: uses 1T8GEM tech profile on voltha branch
       - 'build_voltha_pod_test':
@@ -62,7 +62,7 @@
           work-flow: 'DT'
           profile: '1T8GEM'
           time: '1'
-          extraHelmFlags: "--set onos-classic.image.repository=andreacampanella/voltha-onos --set onos-classic.image.tag=meter-fixes-simple"
+          extraHelmFlags: "--set onos-classic.image.repository=andreacampanella/voltha-onos --set onos-classic.image.tag=meter-fixes-simple --set voltha.global.rw_core.rpc_timeout=30s --set voltha.global.rw_core.internal_timeout=30s"
 
       # Certification (Radisys 1600X) POD test job - master versions: uses 1T8GEM tech profile on voltha branch
       - 'build_voltha_pod_test':
@@ -88,7 +88,7 @@
           work-flow: 'DT'
           profile: '1T8GEM'
           time: '5'
-          extraHelmFlags: "--set onos-classic.image.repository=andreacampanella/voltha-onos --set onos-classic.image.tag=meter-fixes-simple"
+          extraHelmFlags: "--set onos-classic.image.repository=andreacampanella/voltha-onos --set onos-classic.image.tag=meter-fixes-simple --set voltha.global.rw_core.rpc_timeout=30s --set voltha.global.rw_core.internal_timeout=30s"
 
       # Certification (Radisys) POD test job - master versions: uses 1T8GEM tech profile on voltha branch
       - 'build_voltha_pod_test':
diff --git a/jjb/voltha-test/voltha.yaml b/jjb/voltha-test/voltha.yaml
index 431cb92..aa1b526 100644
--- a/jjb/voltha-test/voltha.yaml
+++ b/jjb/voltha-test/voltha.yaml
@@ -308,7 +308,7 @@
       - 'build_voltha_pod_soak_test':
           build-node: 'menlo-soak-pod'
           config-pod: 'onf-soak-pod'
-          'disable-job': false
+          'disable-job': true
           profile: '1T8GEM'
           branch: 'master'
           release: 'master'
@@ -324,7 +324,7 @@
       - 'build_voltha_pod_soak_test':
           build-node: 'menlo-soak-pod'
           config-pod: 'onf-soak-pod'
-          'disable-job': false
+          'disable-job': true
           profile: '1T8GEM'
           branch: 'master'
           release: 'master'
@@ -372,7 +372,7 @@
       - 'build_voltha_pod_soak_test':
           build-node: 'menlo-soak-pod'
           config-pod: 'onf-soak-pod'
-          'disable-job': false
+          'disable-job': true
           profile: '1T8GEM'
           branch: 'voltha-2.8'
           release: '2.8'
@@ -388,7 +388,7 @@
       - 'build_voltha_pod_soak_test':
           build-node: 'menlo-soak-pod'
           config-pod: 'onf-soak-pod'
-          'disable-job': false
+          'disable-job': true
           profile: '1T8GEM'
           branch: 'voltha-2.8'
           release: '2.8'