Merge "voltha-test: pull pipeline-scrips from specific pipeline-branches"
diff --git a/jjb/software-upgrades.yaml b/jjb/software-upgrades.yaml
index 9076026..f23b929 100644
--- a/jjb/software-upgrades.yaml
+++ b/jjb/software-upgrades.yaml
@@ -11,6 +11,7 @@
       - 'software-upgrades-test-master':
           name: 'periodic-software-upgrade-test-bbsim'
           pipeline-script: 'voltha/master/software-upgrades.groovy'
+          pipeline-branch: 'master'
           build-node: 'ubuntu18.04-basebuild-8c-15g'
           code-branch: 'master'
           aaa-version: '2.9.0.SNAPSHOT'
@@ -46,6 +47,7 @@
       - 'software-upgrades-test-voltha-2.12':
           name: 'periodic-software-upgrade-test-bbsim-2.12'
           pipeline-script: 'voltha/voltha-2.12/software-upgrades.groovy'
+          pipeline-branch: 'voltha-2.12'
           build-node: 'ubuntu18.04-basebuild-8c-15g'
           code-branch: 'voltha-2.12'
           aaa-version: '2.10.3'
@@ -80,6 +82,7 @@
       - 'software-upgrades-test-voltha-2.11':
           name: 'periodic-software-upgrade-test-bbsim-2.11'
           pipeline-script: 'voltha/voltha-2.11/software-upgrades.groovy'
+          pipeline-branch: 'voltha-2.11'
           build-node: 'ubuntu18.04-basebuild-8c-15g'
           code-branch: 'voltha-2.11'
           aaa-version: '2.9.0'
@@ -331,7 +334,13 @@
     project-type: pipeline
     concurrent: true
 
-    dsl: !include-raw-escape: pipeline/{pipeline-script}
+    pipeline-scm:
+      scm:
+        - git:
+            url: https://gerrit.opencord.org/ci-management
+            branches:
+              - origin/{pipeline-branch}
+      script-path: jjb/pipeline/{pipeline-script}
 
     triggers:
       - timed: |
@@ -529,7 +538,13 @@
     project-type: pipeline
     concurrent: true
 
-    dsl: !include-raw-escape: pipeline/{pipeline-script}
+    pipeline-scm:
+      scm:
+        - git:
+            url: https://gerrit.opencord.org/ci-management
+            branches:
+              - origin/{pipeline-branch}
+      script-path: jjb/pipeline/{pipeline-script}
 
     triggers:
       - timed: |
@@ -728,7 +743,13 @@
     project-type: pipeline
     concurrent: true
 
-    dsl: !include-raw-escape: pipeline/{pipeline-script}
+    pipeline-scm:
+      scm:
+        - git:
+            url: https://gerrit.opencord.org/ci-management
+            branches:
+              - origin/{pipeline-branch}
+      script-path: jjb/pipeline/{pipeline-script}
 
     triggers:
       - timed: |
diff --git a/jjb/voltha-e2e.yaml b/jjb/voltha-e2e.yaml
index 3dada3d..22add45 100755
--- a/jjb/voltha-e2e.yaml
+++ b/jjb/voltha-e2e.yaml
@@ -2093,6 +2093,7 @@
     id: 'voltha-periodic-test'
     name: '{name}'
     pipeline-script: 'voltha/master/bbsim-tests.groovy'
+    pipeline-branch: 'master'
     build-node: 'ubuntu18.04-basebuild-8c-15g'
     robot-args: ''
     gerritProject: ''
@@ -2217,7 +2218,13 @@
     project-type: pipeline
     concurrent: true
 
-    dsl: !include-raw-escape: pipeline/{pipeline-script}
+    pipeline-scm:
+      scm:
+        - git:
+            url: https://gerrit.opencord.org/ci-management
+            branches:
+              - origin/{pipeline-branch}
+      script-path: jjb/pipeline/{pipeline-script}
 
     triggers:
       # patchset jobs will set this to "0 0 29 2 *" (feb 29th, it's once every 4 years)
@@ -2246,6 +2253,7 @@
     name: 'verify_{project}_sanity-test{name-extension}'
     build-node: 'ubuntu18.04-basebuild-4c-8g'
     pipeline-script: 'voltha/master/bbsim-tests.groovy'
+    pipeline-branch: 'master'
     override-branch: '$GERRIT_BRANCH'
     sandbox: true
     build-timeout: 20
@@ -2383,7 +2391,13 @@
     project-type: pipeline
     concurrent: true
 
-    dsl: !include-raw-escape: pipeline/{pipeline-script}
+    pipeline-scm:
+      scm:
+        - git:
+            url: https://gerrit.opencord.org/ci-management
+            branches:
+              - origin/{pipeline-branch}
+      script-path: jjb/pipeline/{pipeline-script}
 
     triggers:
       - gerrit:
@@ -2569,6 +2583,7 @@
                   Copyright 2019-2023 Open Networking Foundation (ONF) and the ONF Contributors
     sandbox: true
     pipeline-script: 'voltha/master/tucson-build-and-test.groovy'
+    pipeline-branch: 'master'
     default-test-args: '-i sanityORDeleteOLT -i PowerSwitch -X'
     build-node: 'tucson-pod'
     config-pod: 'tucson-pod'
@@ -2590,7 +2605,14 @@
     project-type: pipeline
     concurrent: true
 
-    dsl: !include-raw-escape: pipeline/{pipeline-script}
+    pipeline-scm:
+      scm:
+        - git:
+            url: https://gerrit.opencord.org/ci-management
+            branches:
+              - origin/{pipeline-branch}
+      script-path: jjb/pipeline/{pipeline-script}
+
 
     triggers:
       - gerrit:
@@ -2626,6 +2648,7 @@
     oltDebVersionMaster: 'openolt_asfvolt16-3.7.4-3b190f027136e845c5850a5b1a97897ce2b74ebf-40G-NNI.deb'
     oltDebVersionVoltha23: 'openolt_asfvolt16-3.4.9-e2a9597f3d690fe3a0ea0df244571dfc9e8c2833-40G-NNI.deb'
     pipeline-script: 'voltha/master/tucson-build-and-test.groovy'
+    pipeline-branch: 'master'
     trigger-string: 'hardware test'
     default-test-args: '-i sanityORDeleteOLT -i PowerSwitch -X'
     volthaSystemTestsChange: ''
@@ -2645,7 +2668,13 @@
     project-type: pipeline
     concurrent: true
 
-    dsl: !include-raw-escape: pipeline/{pipeline-script}
+    pipeline-scm:
+      scm:
+        - git:
+            url: https://gerrit.opencord.org/ci-management
+            branches:
+              - origin/{pipeline-branch}
+      script-path: jjb/pipeline/{pipeline-script}
 
     triggers:
       - gerrit:
diff --git a/jjb/voltha-e2e/master.yaml b/jjb/voltha-e2e/master.yaml
index f3baff1..0710265 100644
--- a/jjb/voltha-e2e/master.yaml
+++ b/jjb/voltha-e2e/master.yaml
@@ -86,6 +86,7 @@
           # 20230828 - show in the jenkins UI but w/o updates. 
           # ---------------------------------------------------------
           pipeline-script: 'voltha/master/bbsim-tests.groovy'
+          pipeline-branch: 'master'
           time-trigger: "H H/23 * * *"         # Build daily at 11pm
 #          time-trigger: "H/30 * * * *"        # Build every 30 min
           testTargets: |
@@ -154,6 +155,7 @@
           name: 'periodic-voltha-dt-test-bbsim-master'
           build-node: 'ubuntu18.04-basebuild-4c-8g'
           pipeline-script: 'voltha/master/bbsim-tests.groovy'
+          pipeline-branch: 'master'
           code-branch: 'master'
           time-trigger: "@daily"
           testTargets: |
diff --git a/jjb/voltha-e2e/voltha-2.11.yaml b/jjb/voltha-e2e/voltha-2.11.yaml
index c43c417..e04b9e1 100644
--- a/jjb/voltha-e2e/voltha-2.11.yaml
+++ b/jjb/voltha-e2e/voltha-2.11.yaml
@@ -27,6 +27,7 @@
       - 'voltha-periodic-test':
           name: 'periodic-voltha-pm-data-test-bbsim-2.11'
           pipeline-script: 'voltha/voltha-2.11/bbsim-tests.groovy'
+          pipeline-branch: 'voltha-2.11'
           code-branch: 'voltha-2.11'
           extraHelmFlags: '--set kafka.externalAccess.enabled=true,kafka.externalAccess.service.type=NodePort,kafka.externalAccess.service.nodePorts[0]=30201,kafka.externalAccess.service.domain=127.0.0.1'
           time-trigger: "H H/23 * * *"
@@ -52,6 +53,7 @@
       - 'voltha-periodic-test':
           name: 'periodic-voltha-multiple-olts-pm-data-test-bbsim-2.11'
           pipeline-script: 'voltha/voltha-2.11/bbsim-tests.groovy'
+          pipeline-branch: 'voltha-2.11'
           code-branch: 'voltha-2.11'
           extraHelmFlags: '--set onu=2,pon=2 --set kafka.externalAccess.enabled=true,kafka.externalAccess.service.type=NodePort,kafka.externalAccess.service.nodePorts[0]=30201,kafka.externalAccess.service.domain=127.0.0.1'
           olts: 2
@@ -78,6 +80,7 @@
       - 'voltha-periodic-test':
           name: 'periodic-voltha-test-bbsim-2.11'
           pipeline-script: 'voltha/voltha-2.11/bbsim-tests.groovy'
+          pipeline-branch: 'voltha-2.11'
           code-branch: 'voltha-2.11'
           logLevel: 'DEBUG'                     # Force job to regenerate
           time-trigger: "H H/23 * * *"
@@ -106,6 +109,7 @@
       - 'voltha-periodic-test':
           name: 'periodic-voltha-sanity-test-multi-runs-2.11'
           pipeline-script: 'voltha/voltha-2.11/bbsim-tests.groovy'
+          pipeline-branch: 'voltha-2.11'
           code-branch: 'voltha-2.11'
           time-trigger: "H H/23 * * *"
           testTargets: |
@@ -143,6 +147,7 @@
           name: 'periodic-voltha-dt-test-bbsim-2.11'
           build-node: 'ubuntu18.04-basebuild-4c-8g'
           pipeline-script: 'voltha/voltha-2.11/bbsim-tests.groovy'
+          pipeline-branch: 'voltha-2.11'
           code-branch: 'master'
           time-trigger: "@daily"
           extraHelmFlags: '--set global.image_tag=master --set onos-classic.image.tag=master '
diff --git a/jjb/voltha-e2e/voltha-2.12.yaml b/jjb/voltha-e2e/voltha-2.12.yaml
index 8d10ae0..0e43b55 100644
--- a/jjb/voltha-e2e/voltha-2.12.yaml
+++ b/jjb/voltha-e2e/voltha-2.12.yaml
@@ -27,6 +27,7 @@
       - 'voltha-periodic-test':
           name: 'periodic-voltha-pm-data-test-bbsim-2.12'
           pipeline-script: 'voltha/voltha-2.12/bbsim-tests.groovy'
+          pipeline-branch: 'voltha-2.12'
           code-branch: 'voltha-2.12'
           extraHelmFlags: '--set kafka.externalAccess.enabled=true,kafka.externalAccess.service.type=NodePort,kafka.externalAccess.service.nodePorts[0]=30201,kafka.externalAccess.service.domain=127.0.0.1'
           time-trigger: "H H/23 * * *"
@@ -52,6 +53,7 @@
       - 'voltha-periodic-test':
           name: 'periodic-voltha-multiple-olts-pm-data-test-bbsim-2.12'
           pipeline-script: 'voltha/voltha-2.12/bbsim-tests.groovy'
+          pipeline-branch: 'voltha-2.12'
           code-branch: 'voltha-2.12'
           extraHelmFlags: '--set onu=2,pon=2 --set kafka.externalAccess.enabled=true,kafka.externalAccess.service.type=NodePort,kafka.externalAccess.service.nodePorts[0]=30201,kafka.externalAccess.service.domain=127.0.0.1'
           olts: 2
@@ -83,6 +85,7 @@
           code-branch: 'voltha-2.12'
           logLevel: 'DEBUG'                      # force job to regenerate
           pipeline-script: 'voltha/voltha-2.12/bbsim-tests.groovy'
+          pipeline-branch: 'voltha-2.12'
           time-trigger: "H H/23 * * *"
           testTargets: |
             - target: functional-single-kind
@@ -114,6 +117,7 @@
       - 'voltha-periodic-test':
           name: 'periodic-voltha-sanity-test-multi-runs-2.12'
           pipeline-script: 'voltha/voltha-2.12/bbsim-tests.groovy'
+          pipeline-branch: 'voltha-2.12'
           code-branch: 'voltha-2.12'
           time-trigger: "H H/23 * * *"
           testTargets: |
@@ -152,6 +156,7 @@
           name: 'periodic-voltha-dt-test-bbsim-2.12'
           build-node: 'ubuntu18.04-basebuild-4c-8g'
           pipeline-script: 'voltha/voltha-2.12/bbsim-tests.groovy'
+          pipeline-branch: 'voltha-2.12'
           code-branch: 'voltha-2.12'
           time-trigger : '@daily'
           # -----------------------------------------------------------------------
diff --git a/jjb/voltha-e2e/voltha-2.8.yaml b/jjb/voltha-e2e/voltha-2.8.yaml
index 912e530..f7d3227 100644
--- a/jjb/voltha-e2e/voltha-2.8.yaml
+++ b/jjb/voltha-e2e/voltha-2.8.yaml
@@ -27,6 +27,7 @@
           name: 'periodic-voltha-pm-data-test-bbsim-2.8-playground'
           disable-job: true
           pipeline-script: 'voltha/voltha-2.8/bbsim-tests.groovy'
+          pipeline-branch: 'voltha-2.8'
           code-branch: 'voltha-2.8'
           extraHelmFlags: '--set kafka.externalAccess.enabled=true,kafka.externalAccess.service.type=NodePort,kafka.externalAccess.service.nodePorts[0]=30201,kafka.externalAccess.service.domain=127.0.0.1'
           time-trigger: "H H/23 * * *"
@@ -53,6 +54,7 @@
           name: 'periodic-voltha-multiple-olts-pm-data-test-bbsim-2.8'
           disable-job: true
           pipeline-script: 'voltha/voltha-2.8/bbsim-tests.groovy'
+          pipeline-branch: 'voltha-2.8'
           code-branch: 'voltha-2.8'
           extraHelmFlags: '--set onu=2,pon=2 --set kafka.externalAccess.enabled=true,kafka.externalAccess.service.type=NodePort,kafka.externalAccess.service.nodePorts[0]=30201,kafka.externalAccess.service.domain=127.0.0.1'
           olts: 2
@@ -80,6 +82,7 @@
           name: 'periodic-voltha-test-bbsim-2.8'
           disable-job: true                 # Reason: deprecated legacy build
           pipeline-script: 'voltha/voltha-2.8/bbsim-tests.groovy'
+          pipeline-branch: 'voltha-2.8'
           code-branch: 'voltha-2.8'
           time-trigger: "H H/23 * * *"
           testTargets: |
@@ -108,6 +111,7 @@
           name: 'periodic-voltha-sanity-test-multi-runs-2.8'
           disable-job: true
           pipeline-script: 'voltha/voltha-2.8/bbsim-tests.groovy'
+          pipeline-branch: 'voltha-2.8'
           code-branch: 'voltha-2.8'
           time-trigger: "H H/23 * * *"
           testTargets: |
diff --git a/jjb/voltha-scale.yaml b/jjb/voltha-scale.yaml
index bf0340e..e970d1b 100644
--- a/jjb/voltha-scale.yaml
+++ b/jjb/voltha-scale.yaml
@@ -31,6 +31,7 @@
           name: 'voltha-scale-measurements-master-experimental-multi-stack'
           build-node: 'voltha-scale-2'
           pipeline-script: 'voltha/master/voltha-scale-multi-stack.groovy'
+          pipeline-branch: 'master'
           # trigger on Feb 29th (a.k.a only trigger it manually)
           time-trigger: "H 0 29 2 *"
           withMonitoring: true
@@ -223,6 +224,7 @@
           name: 'voltha-scale-measurements-master-10-stacks-2-16-32-att-subscribers'
           build-node: 'voltha-scale-2'
           pipeline-script: 'voltha/master/voltha-scale-multi-stack.groovy'
+          pipeline-branch: 'master'
           time-trigger: "H H/4 * * *"
           disable-job: true
           logLevel: WARN
@@ -246,6 +248,7 @@
           name: 'voltha-scale-measurements-master-10-stacks-2-16-32-dt-subscribers'
           build-node: 'voltha-scale-2'
           pipeline-script: 'voltha/master/voltha-scale-multi-stack.groovy'
+          pipeline-branch: 'master'
           time-trigger: "H H/4 * * *"
           disable-job: true
           logLevel: WARN
@@ -269,6 +272,7 @@
           name: 'voltha-scale-measurements-master-10-stacks-2-16-32-tt-subscribers'
           build-node: 'voltha-scale-2'
           pipeline-script: 'voltha/master/voltha-scale-multi-stack.groovy'
+          pipeline-branch: 'master'
           time-trigger: "H H/4 * * *"
           disable-job: true
           logLevel: WARN
@@ -294,6 +298,7 @@
       - 'voltha-scale-measurements':
           name: 'voltha-scale-measurements-voltha-2.12-2-16-32-att-subscribers'
           pipeline-script: 'voltha/voltha-2.12/voltha-scale-test.groovy'
+          pipeline-branch: 'voltha-2.12'
           build-node: 'voltha-scale-1'
           time-trigger: "H H/4 * * *"
           disable-job: true
@@ -320,6 +325,7 @@
       - 'voltha-scale-measurements':
           name: 'voltha-scale-measurements-voltha-2.11-2-16-32-att-subscribers'
           pipeline-script: 'voltha/voltha-2.11/voltha-scale-test.groovy'
+          pipeline-branch: 'voltha-2.11'
           build-node: 'voltha-scale-1'
           time-trigger: "H H/4 * * *"
           disable-job: true
@@ -347,6 +353,7 @@
       - 'voltha-scale-measurements':
           name: 'voltha-scale-measurements-voltha-2.12-2-16-32-dt-subscribers'
           pipeline-script: 'voltha/voltha-2.12/voltha-scale-test.groovy'
+          pipeline-branch: 'voltha-2.12'
           build-node: 'voltha-scale-1'
           time-trigger: "H H/4 * * *"
           disable-job: true
@@ -373,6 +380,7 @@
       - 'voltha-scale-measurements':
           name: 'voltha-scale-measurements-voltha-2.11-2-16-32-dt-subscribers'
           pipeline-script: 'voltha/voltha-2.11/voltha-scale-test.groovy'
+          pipeline-branch: 'voltha-2.11'
           build-node: 'voltha-scale-1'
           time-trigger: "H H/4 * * *"
           disable-job: true
@@ -401,6 +409,7 @@
       - 'voltha-scale-measurements':
           name: 'voltha-scale-measurements-voltha-2.12-2-16-32-tt-subscribers'
           pipeline-script: 'voltha/voltha-2.12/voltha-scale-test.groovy'
+          pipeline-branch: 'voltha-2.12'
           build-node: 'voltha-scale-1'
           time-trigger: "H H/4 * * *"
           disable-job: true
@@ -429,6 +438,7 @@
       - 'voltha-scale-measurements':
           name: 'voltha-scale-measurements-voltha-2.11-2-16-32-tt-subscribers'
           pipeline-script: 'voltha/voltha-2.11/voltha-scale-test.groovy'
+          pipeline-branch: 'voltha-2.11'
           build-node: 'voltha-scale-1'
           time-trigger: "H H/4 * * *"
           disable-job: true
@@ -459,6 +469,7 @@
           name: 'voltha-scale-measurements-2.12-10-stacks-2-16-32-att-subscribers'
           build-node: 'voltha-scale-2'
           pipeline-script: 'voltha/voltha-2.12/voltha-scale-multi-stack.groovy'
+          pipeline-branch: 'voltha-2.12'
           time-trigger: "H H/4 * * *"
           disable-job: true
           withMonitoring: false
@@ -495,6 +506,7 @@
           name: 'voltha-scale-measurements-2.11-10-stacks-2-16-32-att-subscribers'
           build-node: 'voltha-scale-2'
           pipeline-script: 'voltha/voltha-2.11/voltha-scale-multi-stack.groovy'
+          pipeline-branch: 'voltha-2.11'
           time-trigger: "H H/4 * * *"
           disable-job: true
           withMonitoring: false
@@ -530,6 +542,7 @@
           name: 'voltha-scale-measurements-2.12-10-stacks-2-16-32-dt-subscribers'
           build-node: 'voltha-scale-2'
           pipeline-script: 'voltha/voltha-2.12/voltha-scale-multi-stack.groovy'
+          pipeline-branch: 'voltha-2.12'
           time-trigger: "H H/4 * * *"
           disable-job: true
           withMonitoring: false
@@ -565,6 +578,7 @@
           name: 'voltha-scale-measurements-2.11-10-stacks-2-16-32-dt-subscribers'
           build-node: 'voltha-scale-2'
           pipeline-script: 'voltha/voltha-2.11/voltha-scale-multi-stack.groovy'
+          pipeline-branch: 'voltha-2.11'
           time-trigger: "H H/4 * * *"
           disable-job: true
           withMonitoring: false
@@ -600,6 +614,7 @@
           name: 'voltha-scale-measurements-2.12-10-stacks-2-16-32-tt-subscribers'
           build-node: 'voltha-scale-2'
           pipeline-script: 'voltha/voltha-2.12/voltha-scale-multi-stack.groovy'
+          pipeline-branch: 'voltha-2.12'
           time-trigger: "H H/4 * * *"
           disable-job: true
           withMonitoring: false
@@ -635,6 +650,7 @@
           name: 'voltha-scale-measurements-2.11-10-stacks-2-16-32-tt-subscribers'
           build-node: 'voltha-scale-2'
           pipeline-script: 'voltha/voltha-2.11/voltha-scale-multi-stack.groovy'
+          pipeline-branch: 'voltha-2.11'
           time-trigger: "H H/4 * * *"
           disable-job: true
           withMonitoring: false
@@ -895,6 +911,7 @@
       - 'voltha-scale-measurements':
           name: 'voltha-scale-measurements-lwc-dt-512'
           pipeline-script: 'voltha/master/voltha-scale-lwc-test.groovy'
+          pipeline-branch: 'master'
           build-node: 'berlin-community-pod-2'
           time-trigger: "H H * * *"
           disable-job: true
@@ -914,6 +931,7 @@
       - 'voltha-scale-measurements':
           name: 'voltha-scale-measurements-lwc-dt-256'
           pipeline-script: 'voltha/master/voltha-scale-lwc-test.groovy'
+          pipeline-branch: 'master'
           build-node: 'berlin-community-pod-2'
           time-trigger: "H H * * *"
           disable-job: true
@@ -1197,6 +1215,7 @@
 
     # default values
     pipeline-script: 'voltha/master/voltha-scale-test.groovy'
+    pipeline-branch: 'master'
     release: master
 
     bbsimImg: voltha/bbsim:master
@@ -1260,7 +1279,13 @@
     project-type: pipeline
     concurrent: false
 
-    dsl: !include-raw-escape: pipeline/{pipeline-script}
+    pipeline-scm:
+      scm:
+        - git:
+            url: https://gerrit.opencord.org/ci-management
+            branches:
+              - origin/{pipeline-branch}
+      script-path: jjb/pipeline/{pipeline-script}
 
     triggers:
       # periodic jobs will set this every 4 hours
diff --git a/vars/volthaStackDeploy.groovy b/vars/volthaStackDeploy.groovy
index d432e03..9b198e5 100644
--- a/vars/volthaStackDeploy.groovy
+++ b/vars/volthaStackDeploy.groovy
@@ -106,50 +106,102 @@
 // -----------------------------------------------------------------------
 // Intent: Wait until the pod completed, meaning ONOS fully deployed
 // -----------------------------------------------------------------------
+// Todo: Move logic like this into a standalone script.
+//   Use jenkins stash or native JJB logic to publish out to nodes.
+// -----------------------------------------------------------------------
 void launchVolthaStack(Map cfg) {
     enter('launchVolthaStack')
 
-    sh(label   : "Wait for VOLTHA Stack ${cfg.stackName}::${cfg.volthaNamespace} to start",
+    /* -----------------------------------------------------------------------
+     * % kubectl get pods
+     17:40:15  bbsim0-868479698c-z66mk                          0/1   ContainerCreating   0     6s
+     17:40:15  voltha-voltha-adapter-openolt-68c84bf786-z98rh   0/1   Running             0     8s
+
+     * % kubectl port-forward --address 0.0.0.0
+     17:40:15  error: unable to forward port because pod is not running. Current status=Pending
+     * -----------------------------------------------------------------------
+     */
+    sh(label  : "Wait for VOLTHA Stack (stack=${cfg.stackName}, namespace=${cfg.volthaNamespace}) to start",
        script : """
 
 cat <<EOM
 
 ** -----------------------------------------------------------------------
-** Wait for VOLTHA Stack ${cfg.stackName}::${cfg.volthaNamespace} to start
+** Wait for VOLTHA Stack (stack=${cfg.stackName}, namespace=${cfg.volthaNamespace}) to start
 ** -----------------------------------------------------------------------
 EOM
 
 # set -euo pipefail
-set +x #        # Noisy when commented (default: uncommented)
+set +x #                 # Logs are noisy when commented
 
 declare -i count=0
+declare -i debug=1       # uncomment to enable debugging
+# declare -i verbose=1   # uncomment to enable debugging
 vsd_log='volthaStackDeploy.tmp'
 touch \$vsd_log
+
+declare -i rc=0          # exit status
 while true; do
 
-    ## Exit when the server begins showing signs of life
-    if grep -q '0/' \$vsd_log; then
-        echo 'volthaStackDeploy.groovy: Detected kubectl pods =~ 0/'
-        grep '0/' \$vsd_log
+    # Gather
+    kubectl get pods -n ${cfg.volthaNamespace} \
+        -l app.kubernetes.io/part-of=voltha --no-headers \
+        > \$vsd_log
+
+    count=\$((\$count - 1))
+
+    # Display activity every iteration ?
+    [[ -v verbose ]] && { count=0; }
+
+    # Display activity every minute or so {sleep(5) x count=10}
+    if [[ \$count -lt 1 ]]; then
+        count=10
+        cat \$vsd_log
+    fi
+
+    ## -----------------------
+    ## Probe for cluster state 
+    ## -----------------------
+    if grep -q -e 'ContainerCreating' \$vsd_log; then
+        echo -e '\nvolthaStackDeploy.groovy: ContainerCrating active'
+        [[ -v debug ]] && grep -e 'ContainerCreating' \$vsd_log
+
+    elif grep -q -e '0/' \$vsd_log; then
+        echo -e '\nvolthaStackDeploy.groovy: Waiting for status=Running'
+        [[ -v debug ]] && grep -e '0/' \$vsd_log
+
+    elif ! grep -q '/' \$vsd_log; then
+        echo -e '\nvolthaStackDeploy.groovy: Waiting for initial pod activity'
+        [[ ! -v verbose ]] && { cat \$vsd_log; }
+
+    # -----------------------------------------------------------------------
+    # voltha-adapter-openolt-68c84bf786-8xsfc   0/1   CrashLoopBackOff 2 69s
+    # voltha-adapter-openolt-68c84bf786-8xsfc   0/1   Error            3 85s
+    # -----------------------------------------------------------------------
+    elif grep -q 'Error' \$vsd_log; then
+        echo -e '\nvolthaStackDeploy.groovy: Detected cluster state=Error'
+        cat \$vsd_log
+        rc=1 # fatal
+        break
+
+    # -----------------------------------------------------------------------
+    # An extra conditon needed here but shell coding is tricky:
+    #    "svc x/y Running 0 6s
+    #    Verify (x == y) && (x > 0)
+    # Wait until job failure/we have an actual need for it. 
+    # -----------------------------------------------------------------------
+    else
+        echo -e '\nvolthaStackDeploy.groovy: Voltha stack has launched'
+        [[ ! -v verbose ]] && { cat \$vsd_log; }
         break
     fi
 
+    ## Support argument --timeout (?)
     sleep 5
-    count=\$((\$count - 1))
-
-    if [[ \$count -lt 1 ]]; then # [DEBUG] Display activity every minute or so
-        count=10
-        kubectl get pods -n ${cfg.volthaNamespace} \
-            -l app.kubernetes.io/part-of=voltha --no-headers \
-            | tee \$vsd_log
-    else
-        kubectl get pods -n ${cfg.volthaNamespace} \
-            -l app.kubernetes.io/part-of=voltha --no-headers \
-            > \$vsd_log
-    fi
 
 done
 rm -f \$vsd_log
+exit \$rc
 """)
 
     leave('launchVolthaStack')