Merge "Combining all logs in a single folder per test suite"
diff --git a/jjb/pipeline/voltha/master/bbsim-tests.groovy b/jjb/pipeline/voltha/master/bbsim-tests.groovy
index 5d05764..0650373 100755
--- a/jjb/pipeline/voltha/master/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/master/bbsim-tests.groovy
@@ -105,6 +105,12 @@
       done
       ps aux | grep port-forward
       """
+      // setting ONOS log level
+      sh """
+      sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 log:set ${logLevel.toUpperCase()} org.opencord.olt
+      sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 log:set ${logLevel.toUpperCase()} org.opencord.aaa
+      sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 log:set ${logLevel.toUpperCase()} org.opencord.dhcpl2relay
+      """
     }
   }
   stage('Run test ' + testTarget + ' on ' + workflow + ' workFlow') {
@@ -159,6 +165,7 @@
     PATH="$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
     ROBOT_MISC_ARGS="-e PowerSwitch ${params.extraRobotArgs}"
     DIAGS_PROFILE="VOLTHA_PROFILE"
+    SSHPASS="karaf"
   }
   stages {
     stage('Download Code') {
diff --git a/jjb/pipeline/voltha/master/physical-build.groovy b/jjb/pipeline/voltha/master/physical-build.groovy
index ce4b8ad..51dea2c 100644
--- a/jjb/pipeline/voltha/master/physical-build.groovy
+++ b/jjb/pipeline/voltha/master/physical-build.groovy
@@ -92,6 +92,7 @@
     stage('Install Voltha')  {
       steps {
         timeout(20) {
+          installVoltctl("${branch}")
           script {
             // if we're downloading a voltha-helm-charts patch, then install from a local copy of the charts
             def localCharts = false
diff --git a/jjb/pipeline/voltha/master/voltha-dt-physical-functional-tests.groovy b/jjb/pipeline/voltha/master/voltha-dt-physical-functional-tests.groovy
index 07d58ff..a3236b7 100644
--- a/jjb/pipeline/voltha/master/voltha-dt-physical-functional-tests.groovy
+++ b/jjb/pipeline/voltha/master/voltha-dt-physical-functional-tests.groovy
@@ -99,25 +99,11 @@
         script {
            deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}-DT.yaml"
         }
+        installVoltctl("${branch}")
         sh returnStdout: false, script: """
         mkdir -p $WORKSPACE/bin
+        # download kail
         bash <( curl -sfL https://raw.githubusercontent.com/boz/kail/master/godownloader.sh) -b "$WORKSPACE/bin"
-        cd $WORKSPACE
-        if [ "${params.branch}" == "voltha-2.8" ]; then
-           VOLTCTL_VERSION=1.6.11
-        else
-           VOLTCTL_VERSION=\$(curl -sSL https://api.github.com/repos/opencord/voltctl/releases/latest | jq -r .tag_name | sed -e 's/^v//g')
-        fi
-
-        HOSTOS=\$(uname -s | tr "[:upper:]" "[:lower:"])
-        HOSTARCH=\$(uname -m | tr "[:upper:]" "[:lower:"])
-        if [ \$HOSTARCH == "x86_64" ]; then
-            HOSTARCH="amd64"
-        fi
-        curl -o $WORKSPACE/bin/voltctl -sSL https://github.com/opencord/voltctl/releases/download/v\${VOLTCTL_VERSION}/voltctl-\${VOLTCTL_VERSION}-\${HOSTOS}-\${HOSTARCH}
-        chmod 755 $WORKSPACE/bin/voltctl
-        voltctl version --clientonly
-
 
         # Default kind-voltha config doesn't work on ONF demo pod for accessing kvstore.
         # The issue is that the mgmt node is also one of the k8s nodes and so port forwarding doesn't work.
diff --git a/jjb/pipeline/voltha/master/voltha-physical-functional-tests.groovy b/jjb/pipeline/voltha/master/voltha-physical-functional-tests.groovy
index fa49464..f0f6d9e 100644
--- a/jjb/pipeline/voltha/master/voltha-physical-functional-tests.groovy
+++ b/jjb/pipeline/voltha/master/voltha-physical-functional-tests.groovy
@@ -87,24 +87,11 @@
         script {
           deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}.yaml"
         }
+        installVoltctl("${branch}")
         sh returnStdout: false, script: """
         mkdir -p $WORKSPACE/bin
+        # download kail
         bash <( curl -sfL https://raw.githubusercontent.com/boz/kail/master/godownloader.sh) -b "$WORKSPACE/bin"
-        cd $WORKSPACE
-        if [ "${params.branch}" == "voltha-2.8" ]; then
-           VOLTCTL_VERSION=1.6.11
-        else
-           VOLTCTL_VERSION=\$(curl -sSL https://api.github.com/repos/opencord/voltctl/releases/latest | jq -r .tag_name | sed -e 's/^v//g')
-        fi
-
-        HOSTOS=\$(uname -s | tr "[:upper:]" "[:lower:"])
-        HOSTARCH=\$(uname -m | tr "[:upper:]" "[:lower:"])
-        if [ \$HOSTARCH == "x86_64" ]; then
-            HOSTARCH="amd64"
-        fi
-        curl -o $WORKSPACE/bin/voltctl -sSL https://github.com/opencord/voltctl/releases/download/v\${VOLTCTL_VERSION}/voltctl-\${VOLTCTL_VERSION}-\${HOSTOS}-\${HOSTARCH}
-        chmod 755 $WORKSPACE/bin/voltctl
-        voltctl version --clientonly
 
         if [ "${params.branch}" == "master" ]; then
         # Default kind-voltha config doesn't work on ONF demo pod for accessing kvstore.
diff --git a/jjb/pipeline/voltha/master/voltha-physical-soak-dt-tests.groovy b/jjb/pipeline/voltha/master/voltha-physical-soak-dt-tests.groovy
index 8a93fc4..680ac5d 100644
--- a/jjb/pipeline/voltha/master/voltha-physical-soak-dt-tests.groovy
+++ b/jjb/pipeline/voltha/master/voltha-physical-soak-dt-tests.groovy
@@ -100,24 +100,11 @@
         script {
            deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}-DT.yaml"
         }
+        installVoltctl("${branch}")
         sh returnStdout: false, script: """
         mkdir -p $WORKSPACE/bin
+        # download kail
         bash <( curl -sfL https://raw.githubusercontent.com/boz/kail/master/godownloader.sh) -b "$WORKSPACE/bin"
-        cd $WORKSPACE
-        if [ "${params.branch}" == "voltha-2.8" ]; then
-           VC_VERSION=1.6.11
-        else
-           VC_VERSION=\$(curl -sSL https://api.github.com/repos/opencord/voltctl/releases/latest | jq -r .tag_name | sed -e 's/^v//g')
-        fi
-
-        HOSTOS=\$(uname -s | tr "[:upper:]" "[:lower:"])
-        HOSTARCH=\$(uname -m | tr "[:upper:]" "[:lower:"])
-        if [ \$HOSTARCH == "x86_64" ]; then
-            HOSTARCH="amd64"
-        fi
-        curl -o $WORKSPACE/bin/voltctl -sSL https://github.com/opencord/voltctl/releases/download/v\${VC_VERSION}/voltctl-\${VC_VERSION}-\${HOSTOS}-\${HOSTARCH}
-        chmod 755 $WORKSPACE/bin/voltctl
-        voltctl version --clientonly
 
         if [ "${params.branch}" == "master" ]; then
         # Default kind-voltha config doesn't work on ONF demo pod for accessing kvstore.
diff --git a/jjb/pipeline/voltha/master/voltha-tt-physical-functional-tests.groovy b/jjb/pipeline/voltha/master/voltha-tt-physical-functional-tests.groovy
index 0244751..f401c89 100644
--- a/jjb/pipeline/voltha/master/voltha-tt-physical-functional-tests.groovy
+++ b/jjb/pipeline/voltha/master/voltha-tt-physical-functional-tests.groovy
@@ -99,25 +99,11 @@
         script {
            deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}-TT.yaml"
         }
+        installVoltctl("${branch}")
         sh returnStdout: false, script: """
         mkdir -p $WORKSPACE/bin
+        # download kail
         bash <( curl -sfL https://raw.githubusercontent.com/boz/kail/master/godownloader.sh) -b "$WORKSPACE/bin"
-        cd $WORKSPACE
-        if [ "${params.branch}" == "voltha-2.8" ]; then
-           VOLTCTL_VERSION=1.6.11
-        else
-           VOLTCTL_VERSION=\$(curl -sSL https://api.github.com/repos/opencord/voltctl/releases/latest | jq -r .tag_name | sed -e 's/^v//g')
-        fi
-
-        HOSTOS=\$(uname -s | tr "[:upper:]" "[:lower:"])
-        HOSTARCH=\$(uname -m | tr "[:upper:]" "[:lower:"])
-        if [ \$HOSTARCH == "x86_64" ]; then
-            HOSTARCH="amd64"
-        fi
-        curl -o $WORKSPACE/bin/voltctl -sSL https://github.com/opencord/voltctl/releases/download/v\${VOLTCTL_VERSION}/voltctl-\${VOLTCTL_VERSION}-\${HOSTOS}-\${HOSTARCH}
-        chmod 755 $WORKSPACE/bin/voltctl
-        voltctl version --clientonly
-
 
         # Default kind-voltha config doesn't work on ONF demo pod for accessing kvstore.
         # The issue is that the mgmt node is also one of the k8s nodes and so port forwarding doesn't work.
@@ -224,7 +210,9 @@
           if (olt.type == null || olt.type == "" || olt.type == "openolt") {
              sh returnStdout: false, script: """
              sshpass -p ${olt.pass} scp ${olt.user}@${olt.sship}:/var/log/openolt.log $WORKSPACE/openolt-${olt.sship}.log || true
+             sshpass -p ${olt.pass} scp ${olt.user}@${olt.sship}:/var/log/dev_mgmt_daemon.log $WORKSPACE/dev_mgmt_daemon-${olt.sship}.log || true
              sed -i 's/\\x1b\\[[0-9;]*[a-zA-Z]//g' $WORKSPACE/openolt-${olt.sship}.log  # Remove escape sequences
+             sed -i 's/\\x1b\\[[0-9;]*[a-zA-Z]//g' $WORKSPACE/dev_mgmt_daemon-${olt.sship}.log  # Remove escape sequences
              """
           }
         }
diff --git a/jjb/pipeline/voltha/voltha-2.8/physical-build.groovy b/jjb/pipeline/voltha/voltha-2.8/physical-build.groovy
index d01b7ab..c1029bf 100644
--- a/jjb/pipeline/voltha/voltha-2.8/physical-build.groovy
+++ b/jjb/pipeline/voltha/voltha-2.8/physical-build.groovy
@@ -92,6 +92,7 @@
     stage('Install Voltha')  {
       steps {
         timeout(20) {
+          installVoltctl("$branch")
           script {
             // if we're downloading a voltha-helm-charts patch, then install from a local copy of the charts
             def localCharts = false
diff --git a/jjb/pipeline/voltha/voltha-2.8/tucson-build-and-test.groovy b/jjb/pipeline/voltha/voltha-2.8/tucson-build-and-test.groovy
index 78a0f5e..7e7048f 100644
--- a/jjb/pipeline/voltha/voltha-2.8/tucson-build-and-test.groovy
+++ b/jjb/pipeline/voltha/voltha-2.8/tucson-build-and-test.groovy
@@ -312,6 +312,7 @@
           ROBOT_MISC_ARGS+="${robotDataplaneKeyword}"
         fi
 
+        ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
         make -C $WORKSPACE/voltha-system-tests ${makeTarget} || true
         """
       }
diff --git a/jjb/pipeline/voltha/voltha-2.8/voltha-dt-physical-functional-tests.groovy b/jjb/pipeline/voltha/voltha-2.8/voltha-dt-physical-functional-tests.groovy
index 05e0902..369808a 100644
--- a/jjb/pipeline/voltha/voltha-2.8/voltha-dt-physical-functional-tests.groovy
+++ b/jjb/pipeline/voltha/voltha-2.8/voltha-dt-physical-functional-tests.groovy
@@ -23,6 +23,9 @@
   deployment_config = null
 }
 
+def infraNamespace = "infra"
+def volthaNamespace = "voltha"
+
 pipeline {
   /* no label, executor is determined by JJB */
   agent {
@@ -71,25 +74,11 @@
         script {
            deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}-DT.yaml"
         }
+        installVoltctl("${branch}")
         sh returnStdout: false, script: """
         mkdir -p $WORKSPACE/bin
+        # download kail
         bash <( curl -sfL https://raw.githubusercontent.com/boz/kail/master/godownloader.sh) -b "$WORKSPACE/bin"
-        cd $WORKSPACE
-        if [ "${params.branch}" == "voltha-2.8" ]; then
-           VOLTCTL_VERSION=1.6.11
-        else
-           VOLTCTL_VERSION=\$(curl -sSL https://api.github.com/repos/opencord/voltctl/releases/latest | jq -r .tag_name | sed -e 's/^v//g')
-        fi
-
-        HOSTOS=\$(uname -s | tr "[:upper:]" "[:lower:"])
-        HOSTARCH=\$(uname -m | tr "[:upper:]" "[:lower:"])
-        if [ \$HOSTARCH == "x86_64" ]; then
-            HOSTARCH="amd64"
-        fi
-        curl -o $WORKSPACE/bin/voltctl -sSL https://github.com/opencord/voltctl/releases/download/v\${VOLTCTL_VERSION}/voltctl-\${VOLTCTL_VERSION}-\${HOSTOS}-\${HOSTARCH}
-        chmod 755 $WORKSPACE/bin/voltctl
-        voltctl version --clientonly
-
 
         # Default kind-voltha config doesn't work on ONF demo pod for accessing kvstore.
         # The issue is that the mgmt node is also one of the k8s nodes and so port forwarding doesn't work.
@@ -124,6 +113,7 @@
         else
              export ROBOT_MISC_ARGS="--removekeywords wuks -e PowerSwitch -i sanityDt -i functionalDt -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
         fi
+        ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
         make -C $WORKSPACE/voltha-system-tests voltha-dt-test || true
         """
       }
@@ -143,6 +133,7 @@
         else
              export ROBOT_MISC_ARGS="--removekeywords wuks -L TRACE -i functionalDt -e PowerSwitch -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
         fi
+        ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
         make -C $WORKSPACE/voltha-system-tests voltha-dt-test || true
         """
       }
@@ -158,6 +149,7 @@
         sh """
         mkdir -p $ROBOT_LOGS_DIR
         export ROBOT_MISC_ARGS="--removekeywords wuks -i dataplaneDt -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
+        ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
         make -C $WORKSPACE/voltha-system-tests voltha-dt-test || true
         """
       }
@@ -172,6 +164,7 @@
         sh """
         mkdir -p $ROBOT_LOGS_DIR
         export ROBOT_MISC_ARGS="--removekeywords wuks -L TRACE -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v workflow:${params.workFlow} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
+        ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
         make -C $WORKSPACE/voltha-system-tests voltha-test || true
         """
       }
@@ -191,6 +184,7 @@
         else
              export ROBOT_MISC_ARGS="--removekeywords wuks -L TRACE -i functionalDt -e PowerSwitch -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
         fi
+        ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
         make -C $WORKSPACE/voltha-system-tests voltha-dt-test || true
         """
       }
@@ -207,6 +201,7 @@
         sh """
         mkdir -p $ROBOT_LOGS_DIR
         export ROBOT_MISC_ARGS="--removekeywords wuks -L TRACE -i functional -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v workflow:${params.workFlow} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
+        ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
         make -C $WORKSPACE/voltha-system-tests voltha-test || true
         """
       }
diff --git a/jjb/pipeline/voltha/voltha-2.8/voltha-physical-functional-tests.groovy b/jjb/pipeline/voltha/voltha-2.8/voltha-physical-functional-tests.groovy
index aca802b..040bb0c 100644
--- a/jjb/pipeline/voltha/voltha-2.8/voltha-physical-functional-tests.groovy
+++ b/jjb/pipeline/voltha/voltha-2.8/voltha-physical-functional-tests.groovy
@@ -23,6 +23,9 @@
   deployment_config = null
 }
 
+def infraNamespace = "infra"
+def volthaNamespace = "voltha"
+
 pipeline {
   /* no label, executor is determined by JJB */
   agent {
@@ -84,24 +87,11 @@
         script {
           deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}.yaml"
         }
+        installVoltctl("${branch}")
         sh returnStdout: false, script: """
         mkdir -p $WORKSPACE/bin
+        # download kail
         bash <( curl -sfL https://raw.githubusercontent.com/boz/kail/master/godownloader.sh) -b "$WORKSPACE/bin"
-        cd $WORKSPACE
-        if [ "${params.branch}" == "voltha-2.8" ]; then
-           VOLTCTL_VERSION=1.6.11
-        else
-           VOLTCTL_VERSION=\$(curl -sSL https://api.github.com/repos/opencord/voltctl/releases/latest | jq -r .tag_name | sed -e 's/^v//g')
-        fi
-
-        HOSTOS=\$(uname -s | tr "[:upper:]" "[:lower:"])
-        HOSTARCH=\$(uname -m | tr "[:upper:]" "[:lower:"])
-        if [ \$HOSTARCH == "x86_64" ]; then
-            HOSTARCH="amd64"
-        fi
-        curl -o $WORKSPACE/bin/voltctl -sSL https://github.com/opencord/voltctl/releases/download/v\${VOLTCTL_VERSION}/voltctl-\${VOLTCTL_VERSION}-\${HOSTOS}-\${HOSTARCH}
-        chmod 755 $WORKSPACE/bin/voltctl
-        voltctl version --clientonly
 
         if [ "${params.branch}" == "master" ]; then
         # Default kind-voltha config doesn't work on ONF demo pod for accessing kvstore.
@@ -135,6 +125,7 @@
         else
              export ROBOT_MISC_ARGS="--removekeywords wuks -e PowerSwitch -i sanity -i functional -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
         fi
+        ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
         make -C $WORKSPACE/voltha-system-tests voltha-test || true
         """
       }
@@ -154,6 +145,7 @@
         else
              export ROBOT_MISC_ARGS="--removekeywords wuks -L TRACE -i functional -e PowerSwitch -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
         fi
+        ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
         make -C $WORKSPACE/voltha-system-tests voltha-test || true
         """
       }
@@ -169,6 +161,7 @@
         sh """
         mkdir -p $ROBOT_LOGS_DIR
         export ROBOT_MISC_ARGS="--removekeywords wuks -i dataplane -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
+        ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
         make -C $WORKSPACE/voltha-system-tests voltha-test || true
         """
       }
@@ -183,6 +176,7 @@
        sh """
        mkdir -p $ROBOT_LOGS_DIR
        export ROBOT_MISC_ARGS="--removekeywords wuks -L TRACE -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v workflow:${params.workFlow} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
+       ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
        make -C $WORKSPACE/voltha-system-tests voltha-test || true
        """
       }
@@ -198,6 +192,7 @@
         sh """
         mkdir -p $ROBOT_LOGS_DIR
         export ROBOT_MISC_ARGS="--removekeywords wuks -L TRACE -i functional -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
+        ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
         make -C $WORKSPACE/voltha-system-tests voltha-test || true
         """
       }
diff --git a/jjb/pipeline/voltha/voltha-2.8/voltha-physical-soak-dt-tests.groovy b/jjb/pipeline/voltha/voltha-2.8/voltha-physical-soak-dt-tests.groovy
index a9a2005..17cb701 100644
--- a/jjb/pipeline/voltha/voltha-2.8/voltha-physical-soak-dt-tests.groovy
+++ b/jjb/pipeline/voltha/voltha-2.8/voltha-physical-soak-dt-tests.groovy
@@ -24,6 +24,7 @@
 }
 
 def volthaNamespace = "voltha"
+def infraNamespace = "infra"
 
 pipeline {
   /* no label, executor is determined by JJB */
@@ -37,7 +38,7 @@
   environment {
     KUBECONFIG="$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf"
     VOLTCONFIG="$HOME/.volt/config-minimal"
-    PATH="$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+    PATH="$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
   }
 
 
@@ -99,24 +100,11 @@
         script {
            deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}-DT.yaml"
         }
+        installVoltctl("${branch}")
         sh returnStdout: false, script: """
         mkdir -p $WORKSPACE/bin
+        # download kail
         bash <( curl -sfL https://raw.githubusercontent.com/boz/kail/master/godownloader.sh) -b "$WORKSPACE/bin"
-        cd $WORKSPACE
-        if [ "${params.branch}" == "voltha-2.8" ]; then
-           VC_VERSION=1.6.11
-        else
-           VC_VERSION=\$(curl -sSL https://api.github.com/repos/opencord/voltctl/releases/latest | jq -r .tag_name | sed -e 's/^v//g')
-        fi
-
-        HOSTOS=\$(uname -s | tr "[:upper:]" "[:lower:"])
-        HOSTARCH=\$(uname -m | tr "[:upper:]" "[:lower:"])
-        if [ \$HOSTARCH == "x86_64" ]; then
-            HOSTARCH="amd64"
-        fi
-        curl -o $WORKSPACE/bin/voltctl -sSL https://github.com/opencord/voltctl/releases/download/v\${VC_VERSION}/voltctl-\${VC_VERSION}-\${HOSTOS}-\${HOSTARCH}
-        chmod 755 $WORKSPACE/bin/voltctl
-        voltctl version --clientonly
 
         if [ "${params.branch}" == "master" ]; then
         # Default kind-voltha config doesn't work on ONF demo pod for accessing kvstore.
@@ -153,6 +141,7 @@
             else
                  export ROBOT_MISC_ARGS="--removekeywords wuks -e PowerSwitch -i soak -e dataplaneDt -e bbsim -e notready -d $ROBOT_LOGS_DIR -v SOAK_TEST:True -v logging:False -v teardown_device:False -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE"
             fi
+            ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
             make -C $WORKSPACE/voltha-system-tests voltha-dt-test || true
         fi
         """
@@ -170,6 +159,7 @@
         mkdir -p $ROBOT_LOGS_DIR
         if [ "${params.testType}" == "Failure" ]; then
            export ROBOT_MISC_ARGS="--removekeywords wuks -L TRACE -i soak -e PowerSwitch -e bbsim -e notready -d $ROBOT_LOGS_DIR -v SOAK_TEST:True -v logging:False -v teardown_device:False -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE"
+           ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
            make -C $WORKSPACE/voltha-system-tests voltha-dt-test || true
         fi
         """
@@ -187,6 +177,7 @@
         mkdir -p $ROBOT_LOGS_DIR
         if [ "${params.testType}" == "Dataplane" ]; then
            export ROBOT_MISC_ARGS="--removekeywords wuks -i soakDataplane -e bbsim -e notready -d $ROBOT_LOGS_DIR -v SOAK_TEST:True -v logging:False -v teardown_device:False -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE"
+           ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
            make -C $WORKSPACE/voltha-system-tests voltha-dt-test || true
         fi
         """
diff --git a/jjb/pipeline/voltha/voltha-2.8/voltha-tt-physical-functional-tests.groovy b/jjb/pipeline/voltha/voltha-2.8/voltha-tt-physical-functional-tests.groovy
index c6cd0d2..a139b7a 100644
--- a/jjb/pipeline/voltha/voltha-2.8/voltha-tt-physical-functional-tests.groovy
+++ b/jjb/pipeline/voltha/voltha-2.8/voltha-tt-physical-functional-tests.groovy
@@ -23,6 +23,9 @@
   deployment_config = null
 }
 
+def volthaNamespace = "voltha"
+def infraNamespace = "infra"
+
 pipeline {
   /* no label, executor is determined by JJB */
   agent {
@@ -96,25 +99,11 @@
         script {
            deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}-TT.yaml"
         }
+        installVoltctl("${branch}")
         sh returnStdout: false, script: """
         mkdir -p $WORKSPACE/bin
+        # download kail
         bash <( curl -sfL https://raw.githubusercontent.com/boz/kail/master/godownloader.sh) -b "$WORKSPACE/bin"
-        cd $WORKSPACE
-        if [ "${params.branch}" == "voltha-2.8" ]; then
-           VOLTCTL_VERSION=1.6.11
-        else
-           VOLTCTL_VERSION=\$(curl -sSL https://api.github.com/repos/opencord/voltctl/releases/latest | jq -r .tag_name | sed -e 's/^v//g')
-        fi
-
-        HOSTOS=\$(uname -s | tr "[:upper:]" "[:lower:"])
-        HOSTARCH=\$(uname -m | tr "[:upper:]" "[:lower:"])
-        if [ \$HOSTARCH == "x86_64" ]; then
-            HOSTARCH="amd64"
-        fi
-        curl -o $WORKSPACE/bin/voltctl -sSL https://github.com/opencord/voltctl/releases/download/v\${VOLTCTL_VERSION}/voltctl-\${VOLTCTL_VERSION}-\${HOSTOS}-\${HOSTARCH}
-        chmod 755 $WORKSPACE/bin/voltctl
-        voltctl version --clientonly
-
 
         # Default kind-voltha config doesn't work on ONF demo pod for accessing kvstore.
         # The issue is that the mgmt node is also one of the k8s nodes and so port forwarding doesn't work.
@@ -149,6 +138,7 @@
         else
              export ROBOT_MISC_ARGS="--removekeywords wuks -i functionalTT -e PowerSwitch -i sanityTT -i sanityTT-MCAST -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
         fi
+        ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
         make -C $WORKSPACE/voltha-system-tests voltha-tt-test || true
         """
       }
@@ -169,6 +159,7 @@
           else
                export ROBOT_MISC_ARGS="--removekeywords wuks -L TRACE -i functionalTT -e PowerSwitch -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
           fi
+          ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
           make -C $WORKSPACE/voltha-system-tests voltha-tt-test || true
         fi
         """
@@ -191,6 +182,7 @@
           else
             export ROBOT_MISC_ARGS="--removekeywords wuks -L TRACE -i functionalTT -e PowerSwitch -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel} -V $ROBOT_TEST_INPUT_FILE"
           fi
+          ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
           make -C $WORKSPACE/voltha-system-tests voltha-tt-test || true
         fi
         """
@@ -215,6 +207,8 @@
              sh returnStdout: false, script: """
              sshpass -p ${olt.pass} scp ${olt.user}@${olt.sship}:/var/log/openolt.log $WORKSPACE/openolt-${olt.sship}.log || true
              sed -i 's/\\x1b\\[[0-9;]*[a-zA-Z]//g' $WORKSPACE/openolt-${olt.sship}.log  # Remove escape sequences
+             sshpass -p ${olt.pass} scp ${olt.user}@${olt.sship}:/var/log/dev_mgmt_daemon.log $WORKSPACE/dev_mgmt_daemon-${olt.sship}.log || true
+             sed -i 's/\\x1b\\[[0-9;]*[a-zA-Z]//g' $WORKSPACE/dev_mgmt_daemon-${olt.sship}.log  # Remove escape sequences
              """
           }
         }
diff --git a/jjb/voltha-test/voltha.yaml b/jjb/voltha-test/voltha.yaml
index aa1b526..d4d39a0 100644
--- a/jjb/voltha-test/voltha.yaml
+++ b/jjb/voltha-test/voltha.yaml
@@ -129,6 +129,7 @@
           pipeline-script: 'voltha/master/voltha-tt-physical-functional-tests.groovy'
           test-repo: 'voltha-system-tests'
           profile: 'TP'
+          timeout: 330
 
     # flex OCP pod with olt/onu - Released versions Default tech profile and timer based job
       - 'build_voltha_pod_release_timer':
@@ -170,7 +171,7 @@
           name-extension: '_TT'
           work-flow: 'TT'
           profile: 'TP'
-          time: '15'
+          time: '16'
           VolthaEtcdPort: 9999
           num-of-onos: '3'
           num-of-atomix: '3'
@@ -278,7 +279,7 @@
       - 'build_voltha_pod_manual':
           build-node: 'menlo-soak-pod'
           config-pod: 'onf-soak-pod'
-          disable-job: true
+          disable-job: false
           bbsimReplicas: 1
           profile: '1T8GEM'
           name-extension: '_DT'
@@ -293,7 +294,7 @@
       - 'build_voltha_pod_manual_test':
           build-node: 'menlo-soak-pod'
           config-pod: 'onf-soak-pod'
-          disable-job: true
+          disable-job: false
           profile: '1T8GEM'
           branch: 'master'
           release: 'master'
@@ -308,7 +309,7 @@
       - 'build_voltha_pod_soak_test':
           build-node: 'menlo-soak-pod'
           config-pod: 'onf-soak-pod'
-          'disable-job': true
+          'disable-job': false
           profile: '1T8GEM'
           branch: 'master'
           release: 'master'
@@ -324,7 +325,7 @@
       - 'build_voltha_pod_soak_test':
           build-node: 'menlo-soak-pod'
           config-pod: 'onf-soak-pod'
-          'disable-job': true
+          'disable-job': false
           profile: '1T8GEM'
           branch: 'master'
           release: 'master'
diff --git a/vars/createKubernetesCluster.groovy b/vars/createKubernetesCluster.groovy
index 24a052f..d329de2 100644
--- a/vars/createKubernetesCluster.groovy
+++ b/vars/createKubernetesCluster.groovy
@@ -52,21 +52,10 @@
       curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.11.0/kind-linux-amd64
       chmod +x ./kind
       mv ./kind $WORKSPACE/bin/kind
-
-      # install voltctl
-      HOSTOS="\$(uname -s | tr "[:upper:]" "[:lower:"])"
-      HOSTARCH="\$(uname -m | tr "[:upper:]" "[:lower:"])"
-      if [ "\$HOSTARCH" == "x86_64" ]; then
-          HOSTARCH="amd64"
-      fi
-      if [ "${cfg.branch}" == "voltha-2.8" ]; then
-          VC_VERSION="1.6.11"
-      else
-          VC_VERSION="\$(curl --fail -sSL https://api.github.com/repos/opencord/voltctl/releases/latest | jq -r .tag_name | sed -e 's/^v//g')"
-      fi
-      curl -Lo $WORKSPACE/bin/voltctl https://github.com/opencord/voltctl/releases/download/v\$VC_VERSION/voltctl-\$VC_VERSION-\$HOSTOS-\$HOSTARCH
-      chmod +x $WORKSPACE/bin/voltctl
-
+    """
+    // install voltctl
+    installVoltctl("${cfg.branch}")
+    sh """
       # start the kind cluster
       kind create cluster --name ${cfg.name} --config kind.cfg
 
diff --git a/vars/installVoltctl.groovy b/vars/installVoltctl.groovy
new file mode 100644
index 0000000..3608537
--- /dev/null
+++ b/vars/installVoltctl.groovy
@@ -0,0 +1,29 @@
+// This keyword will install the voltctl based on the branch (e.g.: voltha-2.8 or master)
+def call(String branch) {
+
+  def voltctlVersion = ""
+  if (branch == "voltha-2.8") {
+    voltctlVersion = "1.6.11"
+  } else {
+    voltctlVersion = sh (
+      script: "curl -sSL https://api.github.com/repos/opencord/voltctl/releases/latest | jq -r .tag_name | sed -e 's/^v//g'",
+      returnStdout: true
+    ).trim()
+  }
+
+  println "Installing voltctl version ${voltctlVersion} on branch ${branch}"
+
+  sh returnStdout: false, script: """
+
+    mkdir -p $WORKSPACE/bin
+    cd $WORKSPACE
+    HOSTOS=\$(uname -s | tr "[:upper:]" "[:lower:"])
+    HOSTARCH=\$(uname -m | tr "[:upper:]" "[:lower:"])
+    if [ \$HOSTARCH == "x86_64" ]; then
+       HOSTARCH="amd64"
+    fi
+    curl -o $WORKSPACE/bin/voltctl -sSL https://github.com/opencord/voltctl/releases/download/v${voltctlVersion}/voltctl-${voltctlVersion}-\${HOSTOS}-\${HOSTARCH}
+    chmod 755 $WORKSPACE/bin/voltctl
+    voltctl version --clientonly
+  """
+}