Misc debugging edits

jjb/pipeline/voltha/master/bbsim-tests.groovy
jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
--------------------------------------------------
  o [VOL-5124 DEBUG] Display permisisons in ~/.volt
  o Display a banner for log readabilty around pgrep/pkill output.

Change-Id: Ibc757db01c33a9cec05c953f376218c0f99c6144
diff --git a/jjb/pipeline/voltha/master/bbsim-tests.groovy b/jjb/pipeline/voltha/master/bbsim-tests.groovy
index 3ffe223..3927ad3 100644
--- a/jjb/pipeline/voltha/master/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/master/bbsim-tests.groovy
@@ -89,6 +89,7 @@
     }
 
     // -----------------------------------------------------------------------
+    // Intent: Cleanup stale port-forwarding
     // -----------------------------------------------------------------------
     stage('Cleanup')
     {
@@ -496,39 +497,56 @@
         } // steps
     } // stage('Create K8s Cluster')
 
-	// -----------------------------------------------------------------------
-	// -----------------------------------------------------------------------
-	stage('Replace voltctl')
-	{
-            // if the project is voltctl, override the downloaded one with the built one
-            when {
-		expression {
-                    return gerritProject == 'voltctl'
-		}
+    // -----------------------------------------------------------------------
+    // -----------------------------------------------------------------------
+    stage('Replace voltctl')
+    {
+        // if the project is voltctl, override the downloaded one with the built one
+        when {
+            expression {
+                return gerritProject == 'voltctl'
             }
+        }
 
-            // Hmmmm(?) where did the voltctl download happen ?
-            // Likely Makefile but would be helpful to document here.
-            steps
-            {
-		println("${iam} Running: installVoltctl($branch)")
-		installVoltctl("$branch")
-            } // steps
-	} // stage
+        // Hmmmm(?) where did the voltctl download happen ?
+        // Likely Makefile but would be helpful to document here.
+        steps
+        {
+            println("${iam} Running: installVoltctl($branch)")
+            installVoltctl("$branch")
+        } // steps
+    } // stage
 
-	// -----------------------------------------------------------------------
-	// -----------------------------------------------------------------------
-	stage('Load image in kind nodes')
-	{
-            when {
-		expression {
-                    return !gerritProject.isEmpty()
-		}
-            }
-            steps {
-		loadToKind()
-            } // steps
-	} // stage
+    // -----------------------------------------------------------------------
+    // -----------------------------------------------------------------------
+    stage('voltctl [DEBUG]')
+    {
+        steps
+        {
+	    println("${iam} Display umask")
+	    sh('umask')
+		
+            println("${iam} Checking voltctl config permissions")
+            sh('/bin/ls -ld ~/.volt || true')
+
+            println("${iam} Running find")
+            sh('/bin/ls -l ~/.volt')
+        } // steps
+    } // stage
+
+    // -----------------------------------------------------------------------
+    // -----------------------------------------------------------------------
+    stage('Load image in kind nodes')
+    {
+        when {
+            expression {
+                return !gerritProject.isEmpty()
+	    }
+        }
+        steps {
+            loadToKind()
+        } // steps
+    } // stage
 
 	// -----------------------------------------------------------------------
 	// -----------------------------------------------------------------------
@@ -548,7 +566,14 @@
                         Boolean logging     = test['logging'].toBoolean()
                         String  testLogging = (logging) ? 'True' : 'False'
 
-                        println "Executing test ${target} on workflow ${workflow} with logging ${testLogging} and extra flags ${flags}"
+			print("""
+
+** -----------------------------------------------------------------------
+** Executing test ${target} on workflow ${workflow} with logging ${testLogging} and extra flags ${flags}"
+** -----------------------------------------------------------------------
+
+""")
+
                         println "Executing test ${target}: ENTER"
                         execute_test(target, workflow, testLogging, teardown, flags)
                         println "Executing test ${target}: LEAVE"
diff --git a/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy b/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
index a8663af..45349d9 100644
--- a/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
@@ -99,8 +99,23 @@
                     helmTeardown(['default', infraNamespace, volthaNamespace])
                 }
 
+	// -----------------------------------------------------------------------
+        // Verify pgrep/pkill behavior before replacing ps | kill -9
+        // -----------------------------------------------------------------------
+	script {
+	    println('''
+
+** -----------------------------------------------------------------------
+** pgrep process list for port-forward (pre-pkill)
+** -----------------------------------------------------------------------
+''')
+            sh('''pgrep --list-full port-forward || true''')
+	}
+
             // Comment timeout() if we hang (fix it VS mask problem)
             // timeout(1) {
+	// -----------------------------------------------------------------------
+        // -----------------------------------------------------------------------
 sh(returnStdout:true, script: '''
     sync
     cat <<EOM
@@ -110,8 +125,20 @@
 ** -----------------------------------------------------------------------
 EOM
     [[ $(pgrep --count port-forward) -gt 0 ]] && pkill --echo 'port-forward'
-    pgrep --list-full port-forward || true
  ''')
+
+	// -----------------------------------------------------------------------
+        // -----------------------------------------------------------------------
+	script {
+	    println('''
+
+** -----------------------------------------------------------------------
+** pgrep process list for port-forward (post-pkill)
+** -----------------------------------------------------------------------
+''')
+            sh('''pgrep --list-full port-forward || true''')
+	}
+		
             } // timeout(15)
         } // teardown()
         // timeout(1)
@@ -266,7 +293,7 @@
 	    sh('''ps e -ww -A | grep "_TAG=kail-startup"''')
 	}
 
-	/ -----------------------------------------------------------------------
+	// -----------------------------------------------------------------------
         // stop logging
         // -----------------------------------------------------------------------
         sh """
@@ -558,6 +585,23 @@
 
     // -----------------------------------------------------------------------
     // -----------------------------------------------------------------------
+    stage('voltctl [DEBUG]')
+    {
+        steps
+        {
+	    println("${iam} Display umask")
+	    sh('umask')
+		
+            println("${iam} Checking voltctl config permissions")
+	    sh('/bin/ls -ld ~/.volt || true')
+		
+            println("${iam} Running find")
+            sh('/bin/ls -l ~/.volt')
+        } // steps
+    } // stage
+
+    // -----------------------------------------------------------------------
+    // -----------------------------------------------------------------------
     stage('Load image in kind nodes')
     {
         when {
@@ -597,12 +641,13 @@
                     Boolean logging     = test['logging'].toBoolean()
 		    String  testLogging = (logging) ? 'True' : 'False'
 
-		    println([
-			    "Executing test ${target}",
-			    "on workflow ${workflow}",
-			    "with logging ${testLogging}",
-			    "and extra flags ${flags}",
-			].join(' ')
+                    print("""
+
+** -----------------------------------------------------------------------
+** Executing test ${target} on workflow ${workflow} with logging ${testLogging} and extra flags ${flags}"
+** -----------------------------------------------------------------------
+
+)
 
 		    try {
 			println "Executing test ${target}: ENTER"
diff --git a/jjb/voltha-test/voltha-pod-jobs/voltha-2.11.yaml b/jjb/voltha-test/voltha-pod-jobs/voltha-2.11.yaml
new file mode 100644
index 0000000..d254237
--- /dev/null
+++ b/jjb/voltha-test/voltha-pod-jobs/voltha-2.11.yaml
@@ -0,0 +1,60 @@
+---
+# POD Build Pipeline Jobs for Voltha Releases
+# -----------------------------------------------------------------------
+# Disabled, offline -- 'qa-testvm-pod'
+# -----------------------------------------------------------------------
+- project:
+    name: voltha-pod-jobs-2.11
+
+    project-name: '{name}'
+
+    build-timeout: '300'
+
+    with-kind: false
+    power-switch: false
+    power-cycle-olt: false
+    work-flow: 'ATT'
+    in-band-management: false
+    num-of-openonu: '1'
+    num-of-onos: '1'
+    num-of-atomix: '0'
+    reinstall-olt: true
+    test-type: ''
+    volthaSystemTestsChange: ''
+    kindVolthaChange: ''
+    cordTesterChange: ''
+    oltAdapterAppLabel: 'adapter-open-olt'
+    num-of-onus: ''
+    num-of-ponports: ''
+
+    jobs:
+      # -----------------------------------------------------------------------
+      # [VOL-4939] - flex pod1 test job - released versions: uses tech profile on voltha branch
+      # -----------------------------------------------------------------------
+      - 'build_voltha_pod_test':
+          disable-job: true
+          build-node: 'qa-testvm-pod'
+          config-pod: 'flex-ocp-cord'
+          release: '2.11'
+          branch: 'voltha-2.11'
+          power-switch: true
+          test-repo: 'voltha-system-tests'
+          profile: '1T4GEM'
+
+      # -----------------------------------------------------------------------
+      # [VOL-4939] - flex pod1 test job - released versions: uses tech profile on voltha branch
+      # -----------------------------------------------------------------------
+      - 'build_voltha_pod_test':
+          disable-job: true
+          build-node: 'qa-testvm-pod'
+          config-pod: 'flex-ocp-cord'
+          release: '2.11'
+          branch: 'voltha-2.11'
+          name-extension: '_TT'
+          work-flow: 'TT'
+          power-switch: true
+          pipeline-script: 'voltha/voltha-2.11/voltha-tt-physical-functional-tests.groovy'
+          test-repo: 'voltha-system-tests'
+          profile: 'TP'
+   
+# [EOF]
diff --git a/jjb/voltha-test/voltha-pod-jobs/voltha-2.11.yaml~ b/jjb/voltha-test/voltha-pod-jobs/voltha-2.11.yaml~
new file mode 100644
index 0000000..7e42f44
--- /dev/null
+++ b/jjb/voltha-test/voltha-pod-jobs/voltha-2.11.yaml~
@@ -0,0 +1,60 @@
+---
+# POD Build Pipeline Jobs for Voltha Releases
+# -----------------------------------------------------------------------
+# Disabled, offline -- 'qa-testvm-pod'
+# -----------------------------------------------------------------------
+- project:
+    name: voltha-pod-jobs
+
+    project-name: '{name}'
+
+    build-timeout: '300'
+
+    with-kind: false
+    power-switch: false
+    power-cycle-olt: false
+    work-flow: 'ATT'
+    in-band-management: false
+    num-of-openonu: '1'
+    num-of-onos: '1'
+    num-of-atomix: '0'
+    reinstall-olt: true
+    test-type: ''
+    volthaSystemTestsChange: ''
+    kindVolthaChange: ''
+    cordTesterChange: ''
+    oltAdapterAppLabel: 'adapter-open-olt'
+    num-of-onus: ''
+    num-of-ponports: ''
+
+    jobs:
+      # -----------------------------------------------------------------------
+      # [VOL-4939] - flex pod1 test job - released versions: uses tech profile on voltha branch
+      # -----------------------------------------------------------------------
+      - 'build_voltha_pod_test':
+          disable-job: true
+          build-node: 'qa-testvm-pod'
+          config-pod: 'flex-ocp-cord'
+          release: '2.11'
+          branch: 'voltha-2.11'
+          power-switch: true
+          test-repo: 'voltha-system-tests'
+          profile: '1T4GEM'
+
+      # -----------------------------------------------------------------------
+      # [VOL-4939] - flex pod1 test job - released versions: uses tech profile on voltha branch
+      # -----------------------------------------------------------------------
+      - 'build_voltha_pod_test':
+          disable-job: true
+          build-node: 'qa-testvm-pod'
+          config-pod: 'flex-ocp-cord'
+          release: '2.11'
+          branch: 'voltha-2.11'
+          name-extension: '_TT'
+          work-flow: 'TT'
+          power-switch: true
+          pipeline-script: 'voltha/voltha-2.11/voltha-tt-physical-functional-tests.groovy'
+          test-repo: 'voltha-system-tests'
+          profile: 'TP'
+   
+# [EOF]
diff --git a/jjb/voltha-test/voltha-pod-jobs/voltha-2.12.yaml b/jjb/voltha-test/voltha-pod-jobs/voltha-2.12.yaml
new file mode 100644
index 0000000..c00af37
--- /dev/null
+++ b/jjb/voltha-test/voltha-pod-jobs/voltha-2.12.yaml
@@ -0,0 +1,44 @@
+---
+# POD Build Pipeline Jobs for Voltha Releases
+# -----------------------------------------------------------------------
+# Disabled, offline -- 'qa-testvm-pod'
+# -----------------------------------------------------------------------
+- project:
+    name: voltha-pod-jobs-2.12
+
+    project-name: '{name}'
+
+    build-timeout: '300'
+
+    with-kind: false
+    power-switch: false
+    power-cycle-olt: false
+    work-flow: 'ATT'
+    in-band-management: false
+    num-of-openonu: '1'
+    num-of-onos: '1'
+    num-of-atomix: '0'
+    reinstall-olt: true
+    test-type: ''
+    volthaSystemTestsChange: ''
+    kindVolthaChange: ''
+    cordTesterChange: ''
+    oltAdapterAppLabel: 'adapter-open-olt'
+    num-of-onus: ''
+    num-of-ponports: ''
+
+    jobs:
+      # -----------------------------------------------------------------------
+      # [VOL-4939] - flex pod1 test job - released versions: uses tech profile on voltha branch
+      # -----------------------------------------------------------------------
+      - 'build_voltha_pod_test':
+          disable-job: true              # Reason: qa-testvm-pod offline
+          build-node: 'qa-testvm-pod'
+          config-pod: 'flex-ocp-cord'
+          release: '2.12'
+          branch: 'voltha-2.12'
+          power-switch: true
+          test-repo: 'voltha-system-tests'
+          profile: '1T4GEM'
+   
+# [EOF]
diff --git a/jjb/voltha-test/voltha-pod-jobs/voltha-2.12.yaml~ b/jjb/voltha-test/voltha-pod-jobs/voltha-2.12.yaml~
new file mode 100644
index 0000000..22002ac
--- /dev/null
+++ b/jjb/voltha-test/voltha-pod-jobs/voltha-2.12.yaml~
@@ -0,0 +1,44 @@
+---
+# POD Build Pipeline Jobs for Voltha Releases
+# -----------------------------------------------------------------------
+# Disabled, offline -- 'qa-testvm-pod'
+# -----------------------------------------------------------------------
+- project:
+    name: voltha-pod-jobs
+
+    project-name: '{name}'
+
+    build-timeout: '300'
+
+    with-kind: false
+    power-switch: false
+    power-cycle-olt: false
+    work-flow: 'ATT'
+    in-band-management: false
+    num-of-openonu: '1'
+    num-of-onos: '1'
+    num-of-atomix: '0'
+    reinstall-olt: true
+    test-type: ''
+    volthaSystemTestsChange: ''
+    kindVolthaChange: ''
+    cordTesterChange: ''
+    oltAdapterAppLabel: 'adapter-open-olt'
+    num-of-onus: ''
+    num-of-ponports: ''
+
+    jobs:
+      # -----------------------------------------------------------------------
+      # [VOL-4939] - flex pod1 test job - released versions: uses tech profile on voltha branch
+      # -----------------------------------------------------------------------
+      - 'build_voltha_pod_test':
+          disable-job: true              # Reason: qa-testvm-pod offline
+          build-node: 'qa-testvm-pod'
+          config-pod: 'flex-ocp-cord'
+          release: '2.12'
+          branch: 'voltha-2.12'
+          power-switch: true
+          test-repo: 'voltha-system-tests'
+          profile: '1T4GEM'
+   
+# [EOF]
diff --git a/jjb/voltha-test/voltha.yaml b/jjb/voltha-test/voltha.yaml
index 44f252d..e5df08a 100644
--- a/jjb/voltha-test/voltha.yaml
+++ b/jjb/voltha-test/voltha.yaml
@@ -68,30 +68,6 @@
           test-repo: 'voltha-system-tests'
           profile: 'Default'
 
-      # -----------------------------------------------------------------------
-      # [VOL-4939] - flex pod1 test job - released versions: uses tech profile on voltha branch
-      # -----------------------------------------------------------------------
-      - 'build_voltha_pod_test':
-          disable-job: true
-          build-node: 'qa-testvm-pod'
-          config-pod: 'flex-ocp-cord'
-          release: '2.12'
-          branch: 'voltha-2.12'
-          power-switch: true
-          test-repo: 'voltha-system-tests'
-          profile: '1T4GEM'
-          
-      # [VOL-4939] - flex pod1 test job - released versions: uses tech profile on voltha branch
-      - 'build_voltha_pod_test':
-          disable-job: true
-          build-node: 'qa-testvm-pod'
-          config-pod: 'flex-ocp-cord'
-          release: '2.11'
-          branch: 'voltha-2.11'
-          power-switch: true
-          test-repo: 'voltha-system-tests'
-          profile: '1T4GEM'
-
       # flex pod1 test job - released versions: uses tech profile on voltha branch
       - 'build_voltha_pod_test':
           disable-job: false
@@ -129,20 +105,6 @@
           disable-job: true
           build-node: 'qa-testvm-pod'
           config-pod: 'flex-ocp-cord'
-          release: '2.11'
-          branch: 'voltha-2.11'
-          name-extension: '_TT'
-          work-flow: 'TT'
-          power-switch: true
-          pipeline-script: 'voltha/voltha-2.11/voltha-tt-physical-functional-tests.groovy'
-          test-repo: 'voltha-system-tests'
-          profile: 'TP'
-
-      # [VOL-4939] - flex pod1 test job - released versions: uses tech profile on voltha branch
-      - 'build_voltha_pod_test':
-          disable-job: true
-          build-node: 'qa-testvm-pod'
-          config-pod: 'flex-ocp-cord'
           release: '2.12'
           branch: 'voltha-2.12'
           name-extension: '_TT'