diff --git a/Makefile b/Makefile
index cd0ef9d..6a0bfd7 100644
--- a/Makefile
+++ b/Makefile
@@ -1,3 +1,5 @@
+# -*- makefile -*-
+
 # Makefile for testing JJB jobs in a virtualenv
 .PHONY: test clean
 
@@ -23,4 +25,6 @@
 	jenkins-jobs -l DEBUG test --recursive --config-xml -o $(JOBCONFIG_DIR) jjb/ ;
 
 clean:
-	rm -rf $(VENV_DIR) $(JOBCONFIG_DIR)
+	$(RM) -r $(VENV_DIR) $(JOBCONFIG_DIR)
+
+# [EOF]
diff --git a/jjb/pipeline/voltha/master/bbsim-tests.groovy b/jjb/pipeline/voltha/master/bbsim-tests.groovy
index edfcb6c..c24a586 100755
--- a/jjb/pipeline/voltha/master/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/master/bbsim-tests.groovy
@@ -1,4 +1,4 @@
-// Copyright 2021-present Open Networking Foundation
+// Copyright 2021-2022 Open Networking Foundation (ONF) and the ONF Contributors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -27,7 +27,33 @@
   def infraNamespace = "default"
   def volthaNamespace = "voltha"
   def logsDir = "$WORKSPACE/${testTarget}"
-  stage('Cleanup') {
+
+    stage('IAM')
+    {
+	script
+	{
+	    String iam = [
+		'ci-management',
+		'jjb',
+		'pipeline',
+		'voltha',
+		'master',
+		'bbsim-tests.groovy'
+	    ].join('/')
+            println("** ${iam}: ENTER")
+
+	    String cmd = "which pkill"
+	    def stream = sh(
+		returnStatus:false,
+		returnStdout: true,
+		script: cmd)
+	    println(" ** ${cmd}:\n${stream}")
+	    
+            println("** ${iam}: LEAVE")
+	}
+    }
+
+    stage('Cleanup') {
     if (teardown) {
       timeout(15) {
         script {
diff --git a/jjb/pipeline/voltha/master/device-management-mock-tests.groovy b/jjb/pipeline/voltha/master/device-management-mock-tests.groovy
index 8a15ac6..28aeec6 100644
--- a/jjb/pipeline/voltha/master/device-management-mock-tests.groovy
+++ b/jjb/pipeline/voltha/master/device-management-mock-tests.groovy
@@ -1,4 +1,4 @@
-// Copyright 2017-present Open Networking Foundation
+// Copyright 2017-2022 Open Networking Foundation (ONF) and the ONF Contributors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
diff --git a/jjb/pipeline/voltha/master/dmi-build-and-test.groovy b/jjb/pipeline/voltha/master/dmi-build-and-test.groovy
index afa034f..926da46 100755
--- a/jjb/pipeline/voltha/master/dmi-build-and-test.groovy
+++ b/jjb/pipeline/voltha/master/dmi-build-and-test.groovy
@@ -1,4 +1,4 @@
-// Copyright 2022-present Open Networking Foundation
+// Copyright 2022 Open Networking Foundation (ONF) and the ONF Contributors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -16,6 +16,7 @@
 
 // NOTE we are importing the library even if it's global so that it's
 // easier to change the keywords during a replay
+
 library identifier: 'cord-jenkins-libraries@master',
     retriever: modernSCM([
       $class: 'GitSCMSource',
diff --git a/jjb/pipeline/voltha/master/physical-build.groovy b/jjb/pipeline/voltha/master/physical-build.groovy
index ab8928d..2cb5b2e 100755
--- a/jjb/pipeline/voltha/master/physical-build.groovy
+++ b/jjb/pipeline/voltha/master/physical-build.groovy
@@ -1,4 +1,4 @@
-// Copyright 2017-present Open Networking Foundation
+// Copyright 2017-2022 Open Networking Foundation (ONF) and the ONF Contributors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
diff --git a/jjb/pipeline/voltha/master/software-upgrades.groovy b/jjb/pipeline/voltha/master/software-upgrades.groovy
index 92f772b..e21e165 100755
--- a/jjb/pipeline/voltha/master/software-upgrades.groovy
+++ b/jjb/pipeline/voltha/master/software-upgrades.groovy
@@ -1,4 +1,4 @@
-// Copyright 2021-present Open Networking Foundation
+// Copyright 2021-2022 Open Networking Foundation (ONF) and the ONF Contributors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
diff --git a/jjb/pipeline/voltha/master/tucson-build-and-test.groovy b/jjb/pipeline/voltha/master/tucson-build-and-test.groovy
index f9f9e04..66aeff8 100644
--- a/jjb/pipeline/voltha/master/tucson-build-and-test.groovy
+++ b/jjb/pipeline/voltha/master/tucson-build-and-test.groovy
@@ -1,5 +1,4 @@
-
-// Copyright 2017-present Open Networking Foundation
+// Copyright 2017-2022 Open Networking Foundation (ONF) and the ONF Contributors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -15,6 +14,7 @@
 // used to deploy VOLTHA and configure ONOS physical PODs
 // NOTE we are importing the library even if it's global so that it's
 // easier to change the keywords during a replay
+
 library identifier: 'cord-jenkins-libraries@master',
     retriever: modernSCM([
       $class: 'GitSCMSource',
diff --git a/jjb/pipeline/voltha/master/voltha-dt-physical-functional-tests.groovy b/jjb/pipeline/voltha/master/voltha-dt-physical-functional-tests.groovy
index c6e63d2..864ca3b 100644
--- a/jjb/pipeline/voltha/master/voltha-dt-physical-functional-tests.groovy
+++ b/jjb/pipeline/voltha/master/voltha-dt-physical-functional-tests.groovy
@@ -1,4 +1,4 @@
-// Copyright 2017-present Open Networking Foundation
+// Copyright 2017-2022 Open Networking Foundation (ONF) and the ONF Contributors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
diff --git a/jjb/pipeline/voltha/master/voltha-physical-functional-tests.groovy b/jjb/pipeline/voltha/master/voltha-physical-functional-tests.groovy
index afc0127..1031cf9 100644
--- a/jjb/pipeline/voltha/master/voltha-physical-functional-tests.groovy
+++ b/jjb/pipeline/voltha/master/voltha-physical-functional-tests.groovy
@@ -1,4 +1,4 @@
-// Copyright 2017-present Open Networking Foundation
+// Copyright 2017-2022 Open Networking Foundation (ONF) and the ONF Contributors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
diff --git a/jjb/pipeline/voltha/master/voltha-physical-soak-dt-tests.groovy b/jjb/pipeline/voltha/master/voltha-physical-soak-dt-tests.groovy
index 5039890..5e655ce 100644
--- a/jjb/pipeline/voltha/master/voltha-physical-soak-dt-tests.groovy
+++ b/jjb/pipeline/voltha/master/voltha-physical-soak-dt-tests.groovy
@@ -1,4 +1,4 @@
-// Copyright 2017-present Open Networking Foundation
+// Copyright 2017-2022 Open Networking Foundation (ONF) and the ONF Contributors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
diff --git a/jjb/pipeline/voltha/master/voltha-scale-lwc-test.groovy b/jjb/pipeline/voltha/master/voltha-scale-lwc-test.groovy
index 75b48e2..8431216 100644
--- a/jjb/pipeline/voltha/master/voltha-scale-lwc-test.groovy
+++ b/jjb/pipeline/voltha/master/voltha-scale-lwc-test.groovy
@@ -1,4 +1,4 @@
-// Copyright 2019-present Open Networking Foundation
+// Copyright 2019-2022 Open Networking Foundation (ONF) and the ONF Contributors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -20,6 +20,7 @@
       remote: 'https://gerrit.opencord.org/ci-management.git'
 ])
 
+// [TODO] fix path, achilles heel for testing.
 def lwc_helm_chart_path="/home/jenkins/Radisys_LWC_helm_charts"
 def value_file="/home/jenkins/lwc-values.yaml"
 def workflow="dt"
diff --git a/jjb/pipeline/voltha/master/voltha-scale-multi-stack.groovy b/jjb/pipeline/voltha/master/voltha-scale-multi-stack.groovy
index 56ddf59..29a9f62 100644
--- a/jjb/pipeline/voltha/master/voltha-scale-multi-stack.groovy
+++ b/jjb/pipeline/voltha/master/voltha-scale-multi-stack.groovy
@@ -1,5 +1,4 @@
-// Copyright 2019-present Open Networking Foundation
-//
+// Copyright 2019-2022 Open Networking Foundation (ONF) and the ONF Contributors//
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
diff --git a/jjb/pipeline/voltha/master/voltha-scale-test.groovy b/jjb/pipeline/voltha/master/voltha-scale-test.groovy
index dc738e7..2c757d9 100644
--- a/jjb/pipeline/voltha/master/voltha-scale-test.groovy
+++ b/jjb/pipeline/voltha/master/voltha-scale-test.groovy
@@ -1,4 +1,4 @@
-// Copyright 2019-present Open Networking Foundation
+// Copyright 2019-2022 Open Networking Foundation (ONF) and the ONF Contributors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
diff --git a/jjb/pipeline/voltha/master/voltha-tt-physical-functional-tests.groovy b/jjb/pipeline/voltha/master/voltha-tt-physical-functional-tests.groovy
index 90470f5..a7ec0b6 100644
--- a/jjb/pipeline/voltha/master/voltha-tt-physical-functional-tests.groovy
+++ b/jjb/pipeline/voltha/master/voltha-tt-physical-functional-tests.groovy
@@ -1,4 +1,4 @@
-// Copyright 2017-present Open Networking Foundation
+// Copyright 2017-2022 Open Networking Foundation (ONF) and the ONF Contributors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
diff --git a/jjb/pipeline/voltha/voltha-2.10/bbsim-tests.groovy b/jjb/pipeline/voltha/voltha-2.10/bbsim-tests.groovy
index edfcb6c..98b0b23 100755
--- a/jjb/pipeline/voltha/voltha-2.10/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/voltha-2.10/bbsim-tests.groovy
@@ -1,5 +1,4 @@
-// Copyright 2021-present Open Networking Foundation
-//
+// Copyright 2021-2022 Open Networking Foundation (ONF) and the ONF Contributors//
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
@@ -27,7 +26,25 @@
   def infraNamespace = "default"
   def volthaNamespace = "voltha"
   def logsDir = "$WORKSPACE/${testTarget}"
-  stage('Cleanup') {
+
+    stage('IAM')
+    {
+	script
+	{
+	    String iam = [
+		'ci-management',
+		'jjb',
+		'pipeline',
+		'voltha',
+		'voltha-2.10',
+		'bbsim-tests.groovy'
+	    ].join('/')
+            println("** ${iam}: ENTER")
+            println("** ${iam}: LEAVE")
+	}
+    }
+
+    stage('Cleanup') {
     if (teardown) {
       timeout(15) {
         script {
@@ -311,3 +328,5 @@
     }
   }
 }
+
+// EOF
diff --git a/jjb/pipeline/voltha/voltha-2.11/README.md b/jjb/pipeline/voltha/voltha-2.11/README.md
new file mode 100644
index 0000000..fd9d0a0
--- /dev/null
+++ b/jjb/pipeline/voltha/voltha-2.11/README.md
@@ -0,0 +1,6 @@
+# Voltha 2.11 release tests.
+
+This directory will remain empty until the release.
+Modify test sources in the master/ directory until then.
+
+Refresh all scripts in this directory at release time.
\ No newline at end of file
diff --git a/jjb/pipeline/voltha/voltha-2.11/bbsim-tests.groovy b/jjb/pipeline/voltha/voltha-2.11/bbsim-tests.groovy
new file mode 100755
index 0000000..b35a8b1
--- /dev/null
+++ b/jjb/pipeline/voltha/voltha-2.11/bbsim-tests.groovy
@@ -0,0 +1,332 @@
+// -*- groovy -*-
+// Copyright 2021-2022 Open Networking Foundation (ONF) and the ONF Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// voltha-2.x e2e tests for openonu-go
+// uses bbsim to simulate OLT/ONUs
+
+library identifier: 'cord-jenkins-libraries@master',
+    retriever: modernSCM([
+      $class: 'GitSCMSource',
+      remote: 'https://gerrit.opencord.org/ci-management.git'
+])
+
+def clusterName = "kind-ci"
+
+def execute_test(testTarget, workflow, testLogging, teardown, testSpecificHelmFlags = "") {
+    def infraNamespace = "default"
+    def volthaNamespace = "voltha"
+    def logsDir = "$WORKSPACE/${testTarget}"
+
+    stage('IAM')
+    {
+	script
+	{
+	    String iam = [
+		'ci-management',
+		'jjb',
+		'pipeline',
+		'voltha',
+		'voltha-11',
+		'bbsim-tests.groovy'
+	    ].join('/')
+            println("** ${iam}: ENTER")
+            println("** ${iam}: LEAVE")
+	}
+    }
+
+  stage('Cleanup') {
+    if (teardown) {
+      timeout(15) {
+        script {
+          helmTeardown(["default", infraNamespace, volthaNamespace])
+        }
+        timeout(1) {
+          sh returnStdout: false, script: '''
+          # remove orphaned port-forward from different namespaces
+          ps aux | grep port-forw | grep -v grep | awk '{print $2}' | xargs --no-run-if-empty kill -9 || true
+          '''
+        }
+      }
+    }
+  }
+  stage('Deploy common infrastructure') {
+    sh '''
+    helm repo add onf https://charts.opencord.org
+    helm repo update
+    if [ ${withMonitoring} = true ] ; then
+      helm install nem-monitoring onf/nem-monitoring \
+      --set prometheus.alertmanager.enabled=false,prometheus.pushgateway.enabled=false \
+      --set kpi_exporter.enabled=false,dashboards.xos=false,dashboards.onos=false,dashboards.aaa=false,dashboards.voltha=false
+    fi
+    '''
+  }
+  stage('Deploy Voltha') {
+    if (teardown) {
+      timeout(10) {
+        script {
+
+          sh """
+          mkdir -p ${logsDir}
+          _TAG=kail-startup kail -n ${infraNamespace} -n ${volthaNamespace} > ${logsDir}/onos-voltha-startup-combined.log &
+          """
+
+          // if we're downloading a voltha-helm-charts patch, then install from a local copy of the charts
+          def localCharts = false
+          if (volthaHelmChartsChange != "" || gerritProject == "voltha-helm-charts") {
+            localCharts = true
+          }
+
+          // NOTE temporary workaround expose ONOS node ports
+          def localHelmFlags = extraHelmFlags.trim() + " --set global.log_level=${logLevel.toUpperCase()} " +
+          " --set onos-classic.onosSshPort=30115 " +
+          " --set onos-classic.onosApiPort=30120 " +
+          " --set onos-classic.onosOfPort=31653 " +
+          " --set onos-classic.individualOpenFlowNodePorts=true " + testSpecificHelmFlags
+
+          if (gerritProject != "") {
+            localHelmFlags = "${localHelmFlags} " + getVolthaImageFlags("${gerritProject}")
+          }
+
+          volthaDeploy([
+            infraNamespace: infraNamespace,
+            volthaNamespace: volthaNamespace,
+            workflow: workflow.toLowerCase(),
+            withMacLearning: enableMacLearning.toBoolean(),
+            extraHelmFlags: localHelmFlags,
+            localCharts: localCharts,
+            bbsimReplica: olts.toInteger(),
+            dockerRegistry: registry,
+            ])
+        }
+
+        // stop logging
+        sh """
+          P_IDS="\$(ps e -ww -A | grep "_TAG=kail-startup" | grep -v grep | awk '{print \$1}')"
+          if [ -n "\$P_IDS" ]; then
+            echo \$P_IDS
+            for P_ID in \$P_IDS; do
+              kill -9 \$P_ID
+            done
+          fi
+          cd ${logsDir}
+          gzip -k onos-voltha-startup-combined.log
+          rm onos-voltha-startup-combined.log
+        """
+      }
+      sh """
+      JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-voltha-api" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/voltha-voltha-api 55555:55555; done"&
+      JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-etcd" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-etcd 2379:2379; done"&
+      JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-kafka" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-kafka 9092:9092; done"&
+      bbsimDmiPortFwd=50075
+      for i in {0..${olts.toInteger() - 1}}; do
+        JENKINS_NODE_COOKIE="dontKillMe" _TAG="bbsim\${i}" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/bbsim\${i} \${bbsimDmiPortFwd}:50075; done"&
+        ((bbsimDmiPortFwd++))
+      done
+      if [ ${withMonitoring} = true ] ; then
+        JENKINS_NODE_COOKIE="dontKillMe" _TAG="nem-monitoring-prometheus-server" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n default svc/nem-monitoring-prometheus-server 31301:80; done"&
+      fi
+      ps aux | grep port-forward
+      """
+      // setting ONOS log level
+      script {
+        setOnosLogLevels([
+          onosNamespace: infraNamespace,
+          apps: [
+            'org.opencord.dhcpl2relay',
+            'org.opencord.olt',
+            'org.opencord.aaa',
+            'org.opencord.maclearner',
+            'org.onosproject.net.flowobjective.impl.FlowObjectiveManager',
+            'org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager'
+          ],
+          logLevel: logLevel
+        ])
+      }
+    }
+  }
+  stage('Run test ' + testTarget + ' on ' + workflow + ' workFlow') {
+    sh """
+    if [ ${withMonitoring} = true ] ; then
+      mkdir -p $WORKSPACE/voltha-pods-mem-consumption-${workflow}
+      cd $WORKSPACE/voltha-system-tests
+      make vst_venv
+      source ./vst_venv/bin/activate || true
+      # Collect initial memory consumption
+      python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace} || true
+    fi
+    """
+    sh """
+    mkdir -p ${logsDir}
+    export ROBOT_MISC_ARGS="-d ${logsDir} ${params.extraRobotArgs} "
+    ROBOT_MISC_ARGS+="-v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace} -v container_log_dir:${logsDir} -v logging:${testLogging}"
+    export KVSTOREPREFIX=voltha/voltha_voltha
+
+    make -C $WORKSPACE/voltha-system-tests ${testTarget} || true
+    """
+    getPodsInfo("${logsDir}")
+    sh """
+      set +e
+      # collect logs collected in the Robot Framework StartLogging keyword
+      cd ${logsDir}
+      gzip *-combined.log || true
+      rm *-combined.log || true
+    """
+    sh """
+    if [ ${withMonitoring} = true ] ; then
+      cd $WORKSPACE/voltha-system-tests
+      source ./vst_venv/bin/activate || true
+      # Collect memory consumption of voltha pods once all the tests are complete
+      python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace} || true
+    fi
+    """
+  }
+}
+
+def collectArtifacts(exitStatus) {
+  getPodsInfo("$WORKSPACE/${exitStatus}")
+  sh """
+  kubectl logs -n voltha -l app.kubernetes.io/part-of=voltha > $WORKSPACE/${exitStatus}/voltha.log || true
+  """
+  archiveArtifacts artifacts: '**/*.log,**/*.gz,**/*.txt,**/*.html,**/voltha-pods-mem-consumption-att/*,**/voltha-pods-mem-consumption-dt/*,**/voltha-pods-mem-consumption-tt/*'
+  sh '''
+    sync
+    pkill kail || true
+    which voltctl
+    md5sum $(which voltctl)
+  '''
+  step([$class: 'RobotPublisher',
+    disableArchiveOutput: false,
+    logFileName: "**/*/log*.html",
+    otherFiles: '',
+    outputFileName: "**/*/output*.xml",
+    outputPath: '.',
+    passThreshold: 100,
+    reportFileName: "**/*/report*.html",
+    unstableThreshold: 0,
+    onlyCritical: true]);
+}
+
+pipeline {
+
+  /* no label, executor is determined by JJB */
+  agent {
+    label "${params.buildNode}"
+  }
+  options {
+    timeout(time: "${timeout}", unit: 'MINUTES')
+  }
+  environment {
+    KUBECONFIG="$HOME/.kube/kind-${clusterName}"
+    VOLTCONFIG="$HOME/.volt/config"
+    PATH="$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+    DIAGS_PROFILE="VOLTHA_PROFILE"
+    SSHPASS="karaf"
+  }
+  stages {
+    stage('Download Code') {
+      steps {
+        getVolthaCode([
+          branch: "${branch}",
+          gerritProject: "${gerritProject}",
+          gerritRefspec: "${gerritRefspec}",
+          volthaSystemTestsChange: "${volthaSystemTestsChange}",
+          volthaHelmChartsChange: "${volthaHelmChartsChange}",
+        ])
+      }
+    }
+    stage('Build patch') {
+      // build the patch only if gerritProject is specified
+      when {
+        expression {
+          return !gerritProject.isEmpty()
+        }
+      }
+      steps {
+        // NOTE that the correct patch has already been checked out
+        // during the getVolthaCode step
+        buildVolthaComponent("${gerritProject}")
+      }
+    }
+    stage('Create K8s Cluster') {
+      steps {
+        script {
+          def clusterExists = sh returnStdout: true, script: """
+          kind get clusters | grep ${clusterName} | wc -l
+          """
+          if (clusterExists.trim() == "0") {
+            createKubernetesCluster([nodes: 3, name: clusterName])
+          }
+        }
+      }
+    }
+    stage('Replace voltctl') {
+      // if the project is voltctl override the downloaded one with the built one
+      when {
+        expression {
+          return gerritProject == "voltctl"
+        }
+      }
+      steps{
+        sh """
+        mv `ls $WORKSPACE/voltctl/release/voltctl-*-linux-amd*` $WORKSPACE/bin/voltctl
+        chmod +x $WORKSPACE/bin/voltctl
+        """
+      }
+    }
+    stage('Load image in kind nodes') {
+      when {
+        expression {
+          return !gerritProject.isEmpty()
+        }
+      }
+      steps {
+        loadToKind()
+      }
+    }
+    stage('Parse and execute tests') {
+        steps {
+          script {
+            def tests = readYaml text: testTargets
+
+            for(int i = 0;i<tests.size();i++) {
+              def test = tests[i]
+              def target = test["target"]
+              def workflow = test["workflow"]
+              def flags = test["flags"]
+              def teardown = test["teardown"].toBoolean()
+              def logging = test["logging"].toBoolean()
+              def testLogging = 'False'
+              if (logging) {
+                  testLogging = 'True'
+              }
+              println "Executing test ${target} on workflow ${workflow} with logging ${testLogging} and extra flags ${flags}"
+              execute_test(target, workflow, testLogging, teardown, flags)
+            }
+          }
+        }
+    }
+  }
+  post {
+    aborted {
+      collectArtifacts("aborted")
+    }
+    failure {
+      collectArtifacts("failed")
+    }
+    always {
+      collectArtifacts("always")
+    }
+  }
+}
diff --git a/jjb/pipeline/voltha/voltha-2.8/bbsim-tests.groovy b/jjb/pipeline/voltha/voltha-2.8/bbsim-tests.groovy
index 85683db..fadd09b 100755
--- a/jjb/pipeline/voltha/voltha-2.8/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/voltha-2.8/bbsim-tests.groovy
@@ -24,10 +24,28 @@
 def clusterName = "kind-ci"
 
 def execute_test(testTarget, workflow, testLogging, teardown, testSpecificHelmFlags = "") {
-  def infraNamespace = "default"
-  def volthaNamespace = "voltha"
-  def logsDir = "$WORKSPACE/${testTarget}"
-  stage('Cleanup') {
+    def infraNamespace = "default"
+    def volthaNamespace = "voltha"
+    def logsDir = "$WORKSPACE/${testTarget}"
+
+    stage('IAM')
+    {
+	script
+	{
+	    String iam = [
+		'ci-management',
+		'jjb',
+		'pipeline',
+		'voltha',
+		'voltha-2.8',
+		'bbsim-tests.groovy'
+	    ].join('/')
+            println("** ${iam}: ENTER")
+            println("** ${iam}: LEAVE")
+	}
+    }
+
+    stage('Cleanup') {
     if (teardown) {
       timeout(15) {
         script {
diff --git a/jjb/pipeline/voltha/voltha-2.9/bbsim-tests.groovy b/jjb/pipeline/voltha/voltha-2.9/bbsim-tests.groovy
index 2fdfc38..4e1895f 100755
--- a/jjb/pipeline/voltha/voltha-2.9/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/voltha-2.9/bbsim-tests.groovy
@@ -24,10 +24,28 @@
 def clusterName = "kind-ci"
 
 def execute_test(testTarget, workflow, testLogging, teardown, testSpecificHelmFlags = "") {
-  def infraNamespace = "default"
-  def volthaNamespace = "voltha"
-  def logsDir = "$WORKSPACE/${testTarget}"
-  stage('Cleanup') {
+    def infraNamespace = "default"
+    def volthaNamespace = "voltha"
+    def logsDir = "$WORKSPACE/${testTarget}"
+
+    stage('IAM')
+    {
+	script
+	{
+	    String iam = [
+		'ci-management',
+		'jjb',
+		'pipeline',
+		'voltha',
+		'voltha-2.9',
+		'bbsim-tests.groovy'
+	    ].join('/')
+            println("** ${iam}: ENTER")
+            println("** ${iam}: LEAVE")
+	}
+    }
+
+    stage('Cleanup') {
     if (teardown) {
       timeout(15) {
         script {
diff --git a/jjb/verify/bbsim.yaml b/jjb/verify/bbsim.yaml
index c7b4ae0..2c6cfd1 100644
--- a/jjb/verify/bbsim.yaml
+++ b/jjb/verify/bbsim.yaml
@@ -16,6 +16,14 @@
           name-extension: '-voltha-2.9'
           override-branch: 'voltha-2.9'
           branch-regexp: '^voltha-2.9$'
+      - 'verify-bbsim-jobs-voltha-2.10':
+          name-extension: '-voltha-2.10'
+          override-branch: 'voltha-2.10'
+          branch-regexp: '^voltha-2.10$'
+      - 'verify-bbsim-jobs-voltha-2.11':
+          name-extension: '-voltha-2.11'
+          override-branch: 'voltha-2.11'
+          branch-regexp: '^voltha-2.11$'
       - 'verify-bbsim-jobs-master':
           branch-regexp: '^master$'
       - 'publish-bbsim-jobs':
@@ -78,6 +86,50 @@
               logging: true
 
 - job-group:
+    name: 'verify-bbsim-jobs-voltha-2.10'
+    jobs:
+      - 'voltha-patch-test':
+          pipeline-script: 'voltha/voltha-2.10/bbsim-tests.groovy'
+          testTargets: |
+            - target: sanity-bbsim-att
+              workflow: att
+              flags: ""
+              teardown: true
+              logging: true
+            - target: sanity-bbsim-dt
+              workflow: dt
+              flags: ""
+              teardown: true
+              logging: true
+            - target: sanity-bbsim-tt
+              workflow: tt
+              flags: ""
+              teardown: true
+              logging: true
+
+- job-group:
+    name: 'verify-bbsim-jobs-voltha-2.11'
+    jobs:
+      - 'voltha-patch-test':
+          pipeline-script: 'voltha/voltha-2.11/bbsim-tests.groovy'
+          testTargets: |
+            - target: sanity-bbsim-att
+              workflow: att
+              flags: ""
+              teardown: true
+              logging: true
+            - target: sanity-bbsim-dt
+              workflow: dt
+              flags: ""
+              teardown: true
+              logging: true
+            - target: sanity-bbsim-tt
+              workflow: tt
+              flags: ""
+              teardown: true
+              logging: true
+
+- job-group:
     name: 'verify-bbsim-jobs-master'
     jobs:
       - 'voltha-patch-test':
diff --git a/jjb/voltha-test/voltha-nightly-jobs.yaml b/jjb/voltha-test/voltha-nightly-jobs.yaml
index b2e46b3..e34538d 100644
--- a/jjb/voltha-test/voltha-nightly-jobs.yaml
+++ b/jjb/voltha-test/voltha-nightly-jobs.yaml
@@ -160,12 +160,12 @@
       - string:
           name: volthaSystemTestsChange
           default: '{volthaSystemTestsChange}'
-          description: 'Download a change for gerrit in the voltha-system-tests repo, example value: "refs/changes/79/18779/13"'
+          description: 'Download delta from gerrit:voltha-system-tests, example value: "refs/changes/79/18779/13"'
 
       - string:
           name: volthaHelmChartsChange
           default: '{volthaHelmChartsChange}'
-          description: 'Download a change for gerrit in the voltha-helm-charts repo, example value: "refs/changes/32/19132/1"'
+          description: 'Download delta from gerrit:voltha-helm-charts, example value: "refs/changes/32/19132/1"'
 
       # NOTE is this needed/used?
       - string:
@@ -190,7 +190,14 @@
           default: '{waitTimerForOltUp}'
           description: 'Wait timer for the OLT to come up after reboot'
 
-# defualt properties for the VOLTHA scale jobs
+      - bool:
+          name: DebugMode
+          default: false
+          description: "Enable script debugging (unused)"
+
+# -----------------------------------------------------------------------
+# default properties for the VOLTHA scale jobs
+# -----------------------------------------------------------------------
 - voltha-pipe-job-boiler-plate: &voltha-pipe-job-boiler-plate
     name: voltha-pipe-job-boiler-plate
 
@@ -204,7 +211,9 @@
           build-days-to-keep: '{big-build-days-to-keep}'
           artifact-num-to-keep: '{big-artifact-num-to-keep}'
 
+# -----------------------------------------------------------------------
 # VOLTHA Build Jobs
+# -----------------------------------------------------------------------
 - job-template:
     name: 'build_{config-pod}_{profile}{name-extension}_voltha_{release}_manual'
     id: build_voltha_pod_manual
@@ -212,8 +221,8 @@
     description: |
                   Manual Build on POD {config-pod}, using pipeline/{pipeline-script} in {gerrit-server-url}/ci-management' <br /><br />
                   Created from job-template {id} from ci-management/jjb/voltha-test/voltha-nightly-jobs.yaml <br />
-                  Created by QA (Suchitra Vemuri - suchitra@opennetworking.org ) <br />
-                  Copyright (c) 2018 Open Networking Foundation (ONF)
+                  Created by QA (Suchitra Vemuri) <br />
+                  Copyright 2018-2022 Open Networking Foundation (ONF) and the ONF Contributors
 
     <<: *voltha-pipe-job-boiler-plate
     # default values
@@ -246,7 +255,9 @@
     project-type: pipeline
     dsl: !include-raw-escape: ../pipeline/{pipeline-script}
 
+# -----------------------------------------------------------------------
 # VOLTHA Build Jobs
+# -----------------------------------------------------------------------
 - job-template:
     name: 'build_{config-pod}_{profile}{name-extension}_voltha_{release}'
     id: build_voltha_pod_release_timer
@@ -254,8 +265,8 @@
     description: |
                   Manual Build on POD {config-pod}, using pipeline/{pipeline-script} in {gerrit-server-url}/ci-management' <br /><br />
                   Created from job-template {id} from ci-management/jjb/voltha-test/voltha-nightly-jobs.yaml <br />
-                  Created by QA (Suchitra Vemuri - suchitra@opennetworking.org ) <br />
-                  Copyright (c) 2018 Open Networking Foundation (ONF)
+                  Created by QA (Suchitra Vemuri) <br />
+                  Copyright 2018-2022 Open Networking Foundation (ONF) and the ONF Contributors
 
     <<: *voltha-pipe-job-boiler-plate
     # default values
@@ -293,8 +304,10 @@
                  TZ=America/Los_Angeles
                  H {time} * * *
 
+# -----------------------------------------------------------------------
 # VOLTHA Test Job
 # This job is automatically triggered after a build job has successfully completed
+# -----------------------------------------------------------------------
 - job-template:
     name: 'build_{config-pod}_{profile}_voltha{name-extension}_{release}_test'
     id: build_voltha_pod_test
@@ -302,8 +315,8 @@
     description: |
                   Post Tests on {config-pod} triggered by build_{config-pod}_{branch}, using {pipeline-script}<br /><br />
                   Created from job-template {id} from ci-management/jjb/voltha-test/voltha-nightly-jobs.yaml <br />
-                  Created by Suchitra Vemuri, suchitra@opennetworking.org <br />
-                  Copyright (c) 2017 Open Networking Foundation (ONF)
+                  Created by (Suchitra Vemuri) <br />
+                  Copyright 2017-2022 Open Networking Foundation (ONF) and the ONF Contributors
 
     <<: *voltha-pipe-job-boiler-plate
     pipeline-script: 'voltha/master/voltha-physical-functional-tests.groovy'
@@ -436,9 +449,10 @@
           jobs: 'build_{config-pod}_{profile}{name-extension}_voltha_{release}'
           result: 'success'
 
-
+# -----------------------------------------------------------------------
 # VOLTHA Soak Test Job
 # This job is triggered by a timer defined in the job
+# -----------------------------------------------------------------------
 - job-template:
     name: 'build_{config-pod}_{profile}_voltha{name-extension}_{release}_test'
     id: build_voltha_pod_soak_test
@@ -446,8 +460,8 @@
     description: |
                   Post Tests on {config-pod} triggered by build_{config-pod}_{branch}, using {pipeline-script}<br /><br />
                   Created from job-template {id} from ci-management/jjb/voltha-test/voltha-nightly-jobs.yaml <br />
-                  Created by Suchitra Vemuri, suchitra@opennetworking.org <br />
-                  Copyright (c) 2017 Open Networking Foundation (ONF)
+                  Created by (Suchitra Vemuri) <br />
+                  Copyright 2017-2022 Open Networking Foundation (ONF) and the ONF Contributors
 
     <<: *voltha-pipe-job-boiler-plate
     pipeline-script: 'voltha/master/voltha-physical-functional-tests.groovy'
@@ -562,6 +576,7 @@
           name: cordTesterChange
           default: '{cordTesterChange}'
           description: 'Download a change for gerrit in the kind-voltha repo, example value: "refs/changes/32/19132/1"'
+
     concurrent: true
 
     project-type: pipeline
@@ -572,9 +587,10 @@
                  TZ=America/Los_Angeles
                  {time-trigger}
 
-
+# -----------------------------------------------------------------------
 # VOLTHA Manual Test Job
 # This job is triggered manually
+# -----------------------------------------------------------------------
 - job-template:
     name: 'build_{config-pod}_{profile}{name-extension}_voltha_{release}_manual_test'
     id: build_voltha_pod_manual_test
@@ -582,8 +598,9 @@
     description: |
                   Post Tests on {config-pod} triggered by build_{config-pod}_{branch}, using {pipeline-script}<br /><br />
                   Created from job-template {id} from ci-management/jjb/voltha-test/voltha-nightly-jobs.yaml <br />
-                  Created by Suchitra Vemuri, suchitra@opennetworking.org <br />
-                  Copyright (c) 2017 Open Networking Foundation (ONF)
+                  Created by (Suchitra Vemuri) <br />
+                  Copyright 2017-2022 Open Networking Foundation (ONF) and the ONF Contributors
+
     <<: *voltha-pipe-job-boiler-plate
     pipeline-script: 'voltha/master/voltha-physical-functional-tests.groovy'
     manifest-url: 'https://gerrit.opencord.org/voltha-test-manifest.git'
@@ -708,8 +725,10 @@
     project-type: pipeline
     dsl: !include-raw-escape: ../pipeline/{pipeline-script}
 
+# -----------------------------------------------------------------------
 # VOLTHA DMI Test Job
 # This job is triggered by a timer defined in the job
+# -----------------------------------------------------------------------
 - job-template:
     name: 'verify_{config-pod}_{profile}{name-extension}_voltha_{release}_dmi'
     id: verify_voltha_pod_dmi_test
@@ -717,8 +736,8 @@
     description: |
                   Manual Build on POD {config-pod}, using pipeline/{pipeline-script} in {gerrit-server-url}/ci-management' <br /><br />
                   Created from job-template {id} from ci-management/jjb/voltha-test/voltha-nightly-jobs.yaml <br />
-                  Created by QA (Suchitra Vemuri - suchitra@opennetworking.org ) <br />
-                  Copyright (c) 2018 Open Networking Foundation (ONF)
+                  Created by QA (Suchitra Vemuri) <br />
+                  Copyright 2018-2022 Open Networking Foundation (ONF) and the ONF Contributors
 
     <<: *voltha-pipe-job-boiler-plate
     # default values
@@ -952,3 +971,5 @@
       - timed: |
                  TZ=America/Los_Angeles
                  H {time} * * *
+
+# [EOF]
\ No newline at end of file
diff --git a/vars/installVoltctl.groovy b/vars/installVoltctl.groovy
index dad20ff..cacf6e8 100644
--- a/vars/installVoltctl.groovy
+++ b/vars/installVoltctl.groovy
@@ -1,33 +1,110 @@
-// This keyword will install the voltctl based on the branch (e.g.: voltha-2.8 or master)
+#!/usr/bin/env groovy
+
 def call(String branch) {
 
-  def voltctlVersion = ""
-  if (branch == "voltha-2.8") {
-    voltctlVersion = "1.6.11"
-  } else if (branch == "voltha-2.9") {
-    voltctlVersion = "1.7.4"
-  } else if (branch == "voltha-2.10") {
-    voltctlVersion = "1.7.6"
-  } else {
-    voltctlVersion = sh (
-      script: "curl -sSL https://api.github.com/repos/opencord/voltctl/releases/latest | jq -r .tag_name | sed -e 's/^v//g'",
-      returnStdout: true
-    ).trim()
-  }
+    String iam = 'vars/installVoltctl.groovy'
+    println("** ${iam}: ENTER")
 
-  println "Installing voltctl version ${voltctlVersion} on branch ${branch}"
+    // This logic seems odd given we branch & tag repositories
+    // for release so hilight non-frozen until we know for sure.
+    def released=[
+	'voltha-2.10' : '1.7.6',
+	'voltha-2.9'  : '1.7.4',
+	'voltha-2.8'  : '1.6.11',
+    ]
 
-  sh returnStdout: false, script: """
+    boolean have_released = released.containsKey(branch)
+    boolean is_release  = false
+    // TODO: Enable with parameter: RELEASE_VOLTHA=
+    boolean has_binding = binding.hasVariable('WORKSPACE')
 
-    mkdir -p $WORKSPACE/bin
-    cd $WORKSPACE
-    HOSTOS=\$(uname -s | tr "[:upper:]" "[:lower:"])
-    HOSTARCH=\$(uname -m | tr "[:upper:]" "[:lower:"])
-    if [ \$HOSTARCH == "x86_64" ]; then
+    // WIP: Probe to find out what is available
+    print("""
+** have_released: ${have_released}
+**   has_binding: ${has_binding}
+""")
+
+    // ---------------------------------------------
+    // Sanity check: released version must be frozen
+    // ---------------------------------------------
+    if (is_release && ! released.containsKey(branch)) {
+	// Fingers crossed: jenkins may rewrite the callstack.
+	def myname = this.class.getName()
+
+	String url = [
+	    'https://docs.voltha.org/master',
+	    'howto/release/installVoltctl.html',
+	].join('/')
+
+	String error = [
+	    myname, "ERROR:",
+	    "Detected release version=[$branch]",
+	    "but voltctl is not frozen.",
+	    '',
+	    "See Also:", url,
+	].join(' ')
+	throw new Exception(error)
+    }
+
+    // --------------------------------
+    // General answer: latest available
+    // --------------------------------
+    if (! have_released) {
+	url = 'https://api.github.com/repos/opencord/voltctl/releases/latest'
+	get_version = [
+	    "curl -sSL ${url}",
+	    "jq -r .tag_name",
+	    "sed -e 's/^v//g'",
+	].join(' | ')
+	
+	print(" ** RUNNING: ${get_version}")
+	released[branch] = sh (
+	    script: get_version,
+	    returnStdout: true
+	).trim()
+    }
+
+    voltctlVersion = released[branch]
+    println "Installing voltctl version ${voltctlVersion} on branch ${branch}"
+
+    // -----------------------------------------------------------------------
+    // groovy expansion: ${var}
+    // shell  expansion: \${var}
+    // -----------------------------------------------------------------------
+    sh returnStdout: false, script: """#!/bin/bash
+
+    set -eu -o pipefail
+
+    bin_voltctl="$WORKSPACE/bin/voltctl"
+
+    mkdir -p "$WORKSPACE/bin"
+    cd "$WORKSPACE" || { echo "ERROR: cd $WORKSPACE failed"; exit 1; }
+
+    HOSTOS="\$(uname -s   | tr '[:upper:]' '[:lower:]')"
+    HOSTARCH="\$(uname -m | tr '[:upper:]' '[:lower:]')"
+    if [ "\$HOSTARCH" == "x86_64" ]; then
        HOSTARCH="amd64"
     fi
-    curl -o $WORKSPACE/bin/voltctl -sSL https://github.com/opencord/voltctl/releases/download/v${voltctlVersion}/voltctl-${voltctlVersion}-\${HOSTOS}-\${HOSTARCH}
-    chmod 755 $WORKSPACE/bin/voltctl
+
+    # Retrieve versioned voltctl binary
+    download_url="https://github.com/opencord/voltctl/releases/download"
+    vol_ver="v${voltctlVersion}"
+    vol_name="voltctl-${voltctlVersion}-\${HOSTOS}-\${HOSTARCH}"
+    curl -o "\$bin_voltctl" -sSL "\${download_url}/\${vol_ver}/\${vol_name}"
+
+    chmod u=rwx,go=rx "\$bin_voltctl"
+    chmod 755 "\$bin_voltctl"
+    /bin/ls -l "\$bin_voltctl"
+
+    ## Verify these are the same binary
+    "\${bin_voltctl}" version --clientonly
     voltctl version --clientonly
+
+    # Should use diff or md5sum here
+    /bin/ls -l \$(which voltctl)
   """
+
+    println("** ${iam}: LEAVE")
 }
+
+// [EOF]
diff --git a/vars/setOnosLogLevels.groovy b/vars/setOnosLogLevels.groovy
index 5b1bd45..a24b029 100644
--- a/vars/setOnosLogLevels.groovy
+++ b/vars/setOnosLogLevels.groovy
@@ -1,5 +1,10 @@
+#!/usr/bin/env groovy
+
 def call(Map config) {
 
+    String iam = 'vars/setOnosLogLevels.groovy'
+    println("** ${iam}: ENTER")
+
   def defaultConfig = [
       onosNamespace: "infra",
       apps: ['org.opencord.dhcpl2relay', 'org.opencord.olt', 'org.opencord.aaa'],
@@ -36,4 +41,6 @@
         ps aux | grep port-forward
       """
   }
+
+    println("** ${iam}: LEAVE")
 }
diff --git a/vars/volthaDeploy.groovy b/vars/volthaDeploy.groovy
index 8cb2622..cf76133 100644
--- a/vars/volthaDeploy.groovy
+++ b/vars/volthaDeploy.groovy
@@ -1,8 +1,15 @@
+#!/usr/bin/env groovy
+
 // this keyword is dedicated to deploy a single VOLTHA stack with infra
 // If you need to deploy different configurations you can use the volthaInfraDeploy and volthaStackDeploy keywords
 
 def call(Map config) {
+
+    String iam = 'vars/volthaDeploy.groovy'
+    println("** ${iam}: ENTER")
+
     // note that I can't define this outside the function as there's no global scope in Groovy
+    // [joey] A class method or library call can be used in place of globals, fqdn needed.
     def defaultConfig = [
       onosReplica: 1,
       atomixReplica: 1,
@@ -60,4 +67,6 @@
     if (cfg.withVolthaStack) {
       volthaStackDeploy(cfg)
     }
+
+    println("** ${iam}: LEAVE")
 }
diff --git a/vars/volthaInfraDeploy.groovy b/vars/volthaInfraDeploy.groovy
index a6870d8..7073e87 100644
--- a/vars/volthaInfraDeploy.groovy
+++ b/vars/volthaInfraDeploy.groovy
@@ -1,3 +1,5 @@
+#!/usr/bin/env groovy
+// -----------------------------------------------------------------------
 // usage
 //
 // stage('test stage') {
@@ -7,9 +9,50 @@
 //     ])
 //   }
 // }
+// -----------------------------------------------------------------------
 
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+def getIam(String func)
+{
+    // Cannot rely on a stack trace due to jenkins manipulation
+    String src = 'vars/volthaInfraDeploy.groovy'
+    String iam = [src, func].join('::')
+    return iam
+}
 
-def call(Map config) {
+// -----------------------------------------------------------------------
+// Intent: Display and interact with kubernetes namespaces.
+// -----------------------------------------------------------------------
+def doKubeNamespaces()
+{
+    String iam = getIam('doKubeNamespaces')
+    println("** ${iam}: ENTER")
+
+    /*
+     [joey] - should pre-existing hint the env is tainted (?)
+     05:24:57  + kubectl create namespace infra
+     05:24:57  Error from server (AlreadyExists): namespaces "infra" already exists
+     05:24:57  error: failed to create configmap: configmaps "kube-config" already exists
+
+     [joey] Thinking we should:
+            o A special case exists  (create namespace)
+            o helm upgrade --install (inital update)
+     */
+
+    sh('kubectl get namespaces || true')
+
+    println("** ${iam}: LEAVE")    
+    return
+}
+
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+def process(Map config)
+{
+    String iam = getIam('process')
+    println("** ${iam}: ENTER")
+
     // NOTE use params or directule extraHelmFlags??
     def defaultConfig = [
       onosReplica: 1,
@@ -25,10 +68,6 @@
       kubeconfig: null, // location of the kubernetes config file, if null we assume it's stored in the $KUBECONFIG environment variable
     ]
 
-    if (!config) {
-        config = [:]
-    }
-
     def cfg = defaultConfig + config
 
     def volthaInfraChart = "onf/voltha-infra"
@@ -50,6 +89,8 @@
       kubeconfig = env.KUBECONFIG
     }
 
+    doKubeNamespaces() // WIP: joey
+
     sh """
     kubectl create namespace ${cfg.infraNamespace} || true
     kubectl create configmap -n ${cfg.infraNamespace} kube-config "--from-file=kube_config=${kubeconfig}"  || true
@@ -72,4 +113,36 @@
           --set etcd.replicaCount=${cfg.etcdReplica} \
           -f $WORKSPACE/voltha-helm-charts/examples/${serviceConfigFile}-values.yaml ${cfg.extraHelmFlags}
     """
+
+    return
 }
+
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+def call(Map config)
+{
+    String iam = getIam('main')
+    println("** ${iam}: ENTER")
+
+    if (!config) {
+        config = [:]
+    }
+
+    try
+    {
+	process(config)
+    }
+    catch (Exception err)
+    {
+	println("** ${iam}: EXCEPTION ${err}")
+	// Cannot re-throw ATM: too many potential shell errors.
+	// throw err
+    }
+    finally
+    {
+	println("** ${iam}: LEAVE")
+    }
+    return
+}
+
+// [EOF]
diff --git a/vars/volthaStackDeploy.groovy b/vars/volthaStackDeploy.groovy
index c63850b..45d53a0 100644
--- a/vars/volthaStackDeploy.groovy
+++ b/vars/volthaStackDeploy.groovy
@@ -1,5 +1,18 @@
+#!/usr/bin/env groovy
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+def getIam(String func)
+{
+    // Cannot rely on a stack trace due to jenkins manipulation
+    String src = 'vars/volthaStackDeploy.groovy'
+    String iam = [src, func].join('::')
+    return iam
+}
 
-def call(Map config) {
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+def process(Map config)
+{
     // note that I can't define this outside the function as there's no global scope in Groovy
     def defaultConfig = [
       bbsimReplica: 1,
@@ -16,10 +29,6 @@
       adaptersToWait: 2,
     ]
 
-    if (!config) {
-        config = [:]
-    }
-
     def cfg = defaultConfig + config
 
     def volthaStackChart = "onf/voltha-stack"
@@ -84,6 +93,10 @@
 
     sh """
         set +x
+
+        # [joey]: debug
+        kubectl get pods -n ${cfg.volthaNamespace} -l app.kubernetes.io/part-of=voltha --no-headers
+
         voltha=\$(kubectl get pods -n ${cfg.volthaNamespace} -l app.kubernetes.io/part-of=voltha --no-headers | grep "0/" | wc -l)
         while [[ \$voltha != 0 ]]; do
           sleep 5
@@ -116,4 +129,36 @@
           config=\$(kubectl get pods -l app=onos-config-loader -n ${cfg.infraNamespace} --no-headers --field-selector=status.phase=Running | grep "0/" | wc -l)
         done
     """
+
+    return
 }
+
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+def call(Map config)
+{
+    String iam = getIam('main')
+    println("** ${iam}: ENTER")
+
+    if (!config) {
+        config = [:]
+    }
+
+    try
+    {
+	process(config)
+    }
+    catch (Exception err)
+    {
+	println("** ${iam}: EXCEPTION ${err}")
+	// Cannot re-throw ATM: too many potential shell errors.
+	// throw err
+    }
+    finally
+    {
+	println("** ${iam}: LEAVE")
+    }
+    return
+}
+
+// [EOF]
diff --git a/vars/waitForAdapters.groovy b/vars/waitForAdapters.groovy
index 14b8cdb..3c441da 100644
--- a/vars/waitForAdapters.groovy
+++ b/vars/waitForAdapters.groovy
@@ -1,4 +1,147 @@
-def call(Map config) {
+#!/usr/bin/env groovy
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+def getIam(String func)
+{
+    // Cannot rely on a stack trace due to jenkins manipulation
+    String src = 'vars/waitForAdapters.groovy'
+    String iam = [src, func].join('::')
+    return iam
+}
+
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+def getAdapters()
+{
+    String iam = getIam('getAdapters')
+
+    def adapters = ""
+    try
+    {
+        adapters = sh (
+            script: 'voltctl adapter list --format "{{gosince .LastCommunication}}"',
+            returnStdout: true,
+        ).trim()
+    }
+    catch (err)
+    {
+        // in some older versions of voltctl the command results in
+        // ERROR: Unexpected error while attempting to format results
+        // as table : template: output:1: function "gosince" not defined
+        // if that's the case we won't be able to check the timestamp so
+        // it's useless to wait
+        println("voltctl can't parse LastCommunication, skip waiting")
+	adapters = 'SKIP' // why not just retry a few times (?)
+    }
+
+    print("** ${iam}: returned $adapters")
+    return adapters
+}
+
+// -----------------------------------------------------------------------
+// Intent: Interrogate adapter states to determine if we should continue
+// looping waiting for adapter to start.  Passed values must all be
+// Integral, Valid and within range to assert we are done waiting.
+// Function will return incomplete at the first sign of an invalid value.
+// -----------------------------------------------------------------------
+// NOTE: list.find{} was replaced with a for loop to simplify
+// early loop termination and return.
+// -----------------------------------------------------------------------
+// RETURN: scalar
+//   'DOUBLE-DIGIT' - value exceeds 0-5sec response window.
+//   'NON-NUMERIC'  - garbage in the stream (letter, symbol, ctrl)
+//   'NULL'         - empty or null string detected
+//   'VALID'        - succes, all adapters are functional
+//   'SIX-NINE'     - detected a single numeric digit between 6 and 9.
+// -----------------------------------------------------------------------
+def getAdaptersState(String adapters0)
+{
+    String iam = getIam('getAdaptersState')
+    Boolean debug = true // for now
+
+    def adapters = adapters0.split('\n')
+
+    String ans = null
+    def found = []
+    for(i=0; i<adapters.size(); i++)
+    {
+	String elapsed = adapters[i]
+	if (debug)
+	{
+	    println("** ${iam} Checking elapsed[$i]: $elapsed")
+	}
+
+	if (! elapsed) // empty string or null
+	{
+	    ans = 'NULL'
+	    break
+	}
+
+	Integer size = elapsed.length()
+	if (size > 2) // 463765h58m52(s)
+	{
+	    // higlander: there can be only one
+	    ans = 'DOUBLE-DIGIT'
+	    break
+	}
+
+	if (elapsed.endsWith('s'))
+	{
+	    // safer than replaceAll('s'): ssss1s => 1
+	    elapsed = elapsed.substring(0, size-1)
+	}
+
+	// Line noise guard: 'a', 'f', '#', '!'
+	if (! elapsed.isInteger())
+	{
+	    ans = 'NON-NUMERIC'
+	    break
+	}
+
+	// Is value in range:
+	//   discard negative integers as just plain wonky.
+	//   HMMM(?): zero/no latency response is kinda odd to include imho.
+	Integer val = elapsed.toInteger()
+	if (5 >= val && val >= 0)
+	{
+	    found.add(elapsed)
+	}
+	else
+	{
+	    ans = 'SIX-NINE'
+	    break
+	}
+    } // for()
+
+    // Declare success IFF all values are:
+    //    o integral
+    //    o valid
+    //    o within range
+    if (ans == null)
+    {
+	Integer got = found.size()
+	Integer exp = adapters.size()
+	ans = (! exp)      ? 'NO-ADAPTERS'
+            : (got == exp) ? 'VALID'
+            : 'CONTINUE'
+    }
+
+    if (debug)
+    {
+	println("** ${iam} return: [$ans]")
+    }
+    return ans
+} // getAdaptersState
+
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+def process(Map config)
+{
+    String iam = getIam('process')
+    println("** ${iam}: ENTER")
 
     def defaultConfig = [
         volthaNamespace: "voltha",
@@ -6,10 +149,6 @@
         adaptersToWait: 2,
     ]
 
-    if (!config) {
-        config = [:]
-    }
-
     def cfg = defaultConfig + config
 
     if (cfg.adaptersToWait == 0){
@@ -18,7 +157,7 @@
        return
     }
 
-    println "Wait for adapters to be registered"
+    println("** ${iam}: Wait for adapters to be registered")
 
     // guarantee that at least the specified number of adapters are registered with VOLTHA before proceeding
      sh """
@@ -38,42 +177,70 @@
     // NOTE that we need to wait for LastCommunication to be equal or shorter that 5s
     // as that means the core can talk to the adapters
     // if voltctl can't read LastCommunication we skip this check
-    def done = false;
 
-    while (!done) {
-      sleep 1
-      def adapters = ""
-      try {
-        adapters = sh (
-          script: 'voltctl adapter list --format "{{gosince .LastCommunication}}"',
-          returnStdout: true,
-        ).trim()
-      } catch (err) {
-        // in some older versions of voltctl the command results in
-        // ERROR: Unexpected error while attempting to format results as table : template: output:1: function "gosince" not defined
-        // if that's the case we won't be able to check the timestamp so it's useless to wait
-        println("voltctl can't parse LastCommunication, skip waiting")
-        done = true
-        break
-      }
+    println("** ${iam}: Wait for adapter LastCommunication")
+    Integer countdown = 60 * 10
+    while (true)
+    {
+	sleep 1
+	def adapters = getAdapters()
+	// Why are we not failing hard on this condition ?
+	// Adapters in an unknown state == testing: roll the dice
+	if (adapters == 'SKIP') { break }
 
-      def waitingOn = adapters.split( '\n' ).find{since ->
-        since = since.replaceAll('s','') //remove seconds from the string
+	def state = getAdaptersState(adapters)
+	if (state == 'VALID') { break }   // IFF
 
-        // it has to be a single digit
-        if (since.length() > 1) {
-            return true
-        }
-        if ((since as Integer) > 5) {
-            return true
-        }
-        return false
-      }
-      done = (waitingOn == null || waitingOn.length() == 0)
+	// ----------------------------------------------------------
+	// Excessive timeout but unsure where startup time boundry is
+	// [TODO] profile for a baseline
+	// [TODO] groovy.transform.TimedInterrupt
+	// ----------------------------------------------------------
+	countdown -= 1
+	if (1 > countdown)
+	{
+	    throw new Exception("ERROR: Timed out waiting on adapter startup")
+	}
     }
 
-    sh """
-      set +x
-      ps aux | grep port-forw | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9 || true
-    """
+    println("** ${iam}: Tearing down port forwarding")
+    sh("""
+      ps aux \
+          | grep port-forw \
+          | grep -v grep \
+          | awk '{print \$2}' \
+          | xargs --no-run-if-empty kill -9 || true
+    """)
+
+    println("** ${iam}: LEAVE")
+    return
 }
+
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+def call(Map config)
+{
+    String iam = getIam('main')
+    println("** ${iam}: ENTER")
+
+    if (!config) {
+        config = [:]
+    }
+
+    try
+    {
+	process(config)
+    }
+    catch (Exception err)
+    {
+	println("** ${iam}: EXCEPTION ${err}")
+	throw err
+    }
+    finally
+    {
+	println("** ${iam}: LEAVE")
+    }
+    return
+}
+
+// EOF