VOL-5155 - triage failure in periodic-voltha-test-bbsim-2.12

jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
--------------------------------------------------
  o Fixed a brace matching problem for timeout(10) block.
    Some long shell() blocks were split into distinct sections to fail
    independently and carnage ensued.    On branch dev-joey

jjb/voltha-test/voltha-certification.yaml
jjb/voltha-test/voltha-certification/master.yaml
jjb/voltha-test/voltha-certification/voltha-2.11.yaml
jjb/voltha-test/voltha-certification/voltha-2.12.yaml
-----------------------------------------------------
  o Include some cosmetic config refactoring to force
    regeneraiton of job pipelines.

Change-Id: I59f8f7a23b51aab03c49805f0eec6fb2a4276a59
diff --git a/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy b/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
index 0023c04..2fe0a16 100644
--- a/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
@@ -272,10 +272,11 @@
 ** -----------------------------------------------------------------------
 ''')
 	    sh('''ps e -ww -A | grep "_TAG=kail-startup"''')
-	}
+	} // script
 
         // -----------------------------------------------------------------------
-        // stop logging
+	// stop logging
+	// [TODO] Replace this block with pgrep/pkill one-liner(s)	
         // -----------------------------------------------------------------------
 	sh("""
           P_IDS="\$(ps e -ww -A | grep "_TAG=kail-startup" | grep -v grep | awk '{print \$1}')"
@@ -287,6 +288,9 @@
           fi
 """)
 
+        // -----------------------------------------------------------------------
+	// Bundle onos-voltha / kail logs
+        // -----------------------------------------------------------------------
 	sh("""
 cat <<EOM
 
@@ -299,7 +303,7 @@
           rm onos-voltha-startup-combined.log
           popd
         """)
-       }
+	} // timeout(10)
 
 
         // -----------------------------------------------------------------------
diff --git a/jjb/voltha-test/voltha-certification.yaml b/jjb/voltha-test/voltha-certification.yaml
index 3a8c7ea..19eb4b5 100644
--- a/jjb/voltha-test/voltha-certification.yaml
+++ b/jjb/voltha-test/voltha-certification.yaml
@@ -97,49 +97,6 @@
           time: '1'
           extraHelmFlags: "--set onos-classic.image.tag=master --set voltha.global.rw_core.rpc_timeout=30s --set voltha.global.rw_core.internal_timeout=30s"
 
-      # Certification (Radisys) POD test job - master versions: uses 1T8GEM tech profile on voltha branch (ZZ)
-      - 'build_voltha_pod_test':
-          timeout: 330
-          build-node: 'menlo-certification-pod-3'
-          config-pod: 'menlo-certification-pod-radisys-3200g'
-          release: 'master'
-          branch: 'master'
-          name-extension: '_DT'
-          work-flow: 'DT'
-          test-repo: 'voltha-system-tests'
-          profile: '1T8GEM'
-          power-switch: true
-          pipeline-script: 'voltha/master/voltha-dt-physical-functional-tests.groovy'
-
-      # -----------------------------------------------------------------------
-      # Certification (Radisys) POD test job - master versions: uses 1T8GEM tech profile on voltha branch
-      # -----------------------------------------------------------------------
-      - 'build_voltha_pod_test':
-          build-node: 'menlo-certification-pod-3'
-          config-pod: 'menlo-certification-pod-radisys-3200g'
-          disable-job: false
-          release: '2.12'
-          branch: 'voltha-2.12'
-          name-extension: '_DT'
-          work-flow: 'DT'
-          test-repo: 'voltha-system-tests'
-          profile: '1T8GEM'
-          power-switch: true
-          pipeline-script: 'voltha/voltha-2.12/voltha-dt-physical-functional-tests.groovy'
-
-      - 'build_voltha_pod_test':
-          build-node: 'menlo-certification-pod-3'
-          config-pod: 'menlo-certification-pod-radisys-3200g'
-          disable-job: false
-          release: '2.11'
-          branch: 'voltha-2.11'
-          name-extension: '_DT'
-          work-flow: 'DT'
-          test-repo: 'voltha-system-tests'
-          profile: '1T8GEM'
-          power-switch: true
-          pipeline-script: 'voltha/voltha-2.11/voltha-dt-physical-functional-tests.groovy'
-
       # -----------------------------------------------------------------------
       # Certification (Radisys) pod with olt/onu - 2.8 version timer based job
       # -----------------------------------------------------------------------
diff --git a/jjb/voltha-test/voltha-certification/master.yaml b/jjb/voltha-test/voltha-certification/master.yaml
index 2384394..817193b 100644
--- a/jjb/voltha-test/voltha-certification/master.yaml
+++ b/jjb/voltha-test/voltha-certification/master.yaml
@@ -1,59 +1,50 @@
 ---
 # POD Build Pipeline Jobs for Voltha Releases
 
-# - project:
-#     name: voltha-certification-jobs-master
-# 
-#     project-name: '{name}'
-# 
-#     build-timeout: '300'
-# 
-#     with-kind: false
-#     power-switch: false
-#     power-cycle-olt: false
-#     work-flow: 'ATT'
-#     in-band-management: false
-#     num-of-openonu: 1
-#     num-of-onos: 1
-#     num-of-atomix: 1
-#     num-of-kafka: 1
-#     num-of-etcd: 1
-#     reinstall-olt: true
-#     test-type: ''
-#     volthaSystemTestsChange: ''
-#     kindVolthaChange: ''
-#     cordTesterChange: ''
-#     oltAdapterAppLabel: 'adapter-open-olt'
-#     num-of-onus: ''
-#     num-of-ponports: ''
-# 
-#     jobs:
-# 
-#       # -----------------------------------------------------------------------
-#       # VOL-5059
-#       # -----------------------------------------------------------------------
-#       #   o v2.8 and v2.11 release jobs exist.
-#       #   o Job does not exist on branch=master ?!?
-#       #   o Create a certification master job just in case, far easier to
-#       #     than recreate from scratch depending on need later on.
-#       #
-#       #  Duplicate job detected somewhere, likely drived
-# INFO:jenkins_jobs.local_yaml:Lazy loading of file template 'ScalarNode(tag='!include-raw:', value='{docker-get-container-tag-script}')' enabled
-# ERROR:jenkins_jobs.parser:Duplicate definitions for job 'build_menlo-certification-pod-radisys-3200g_1T8GEM_DT_voltha_master' specified
-#       # -----------------------------------------------------------------------
-#       # Certification (Radisys) pod with olt/onu - branch=master timer based job
-#       - 'build_voltha_pod_release_timer':
-#           build-node: 'menlo-certification-pod-3'
-#           config-pod: 'menlo-certification-pod-radisys-3200g'
-#           disable-job: false
-#           reinstall-olt: true
-#           release: 'master'
-#           branch: 'master'
-#           name-extension: '_DT'
-#           work-flow: 'DT'
-#           profile: '1T8GEM'
-#           time: '9'
-#           pipeline-script: 'voltha/master/physical-build.groovy'
-#           extraHelmFlags: "--set onos-classic.image.tag=master"
- 
+- project:
+    name: voltha-certification-jobs-master
+
+    project-name: '{name}'
+
+    build-timeout: '300'
+
+    with-kind: false
+    power-switch: false
+    power-cycle-olt: false
+    work-flow: 'ATT'
+    in-band-management: false
+    num-of-openonu: 1
+    num-of-onos: 1
+    num-of-atomix: 1
+    num-of-kafka: 1
+    num-of-etcd: 1
+    reinstall-olt: true
+    test-type: ''
+    volthaSystemTestsChange: ''
+    kindVolthaChange: ''
+    cordTesterChange: ''
+    oltAdapterAppLabel: 'adapter-open-olt'
+    num-of-onus: ''
+    num-of-ponports: ''
+
+    jobs:
+
+      # -----------------------------------------------------------------------
+      # Certification (Radisys) POD test job
+      # md5sum(master): e8cb3eba849b788944cee1fe58e4091b
+      # -----------------------------------------------------------------------
+      - 'build_voltha_pod_test':
+          timeout: 330
+          disable-job: true      # menlo-certification-pod-3 is offline
+          build-node: 'menlo-certification-pod-3'
+          config-pod: 'menlo-certification-pod-radisys-3200g'
+          release: 'master'
+          branch: 'master'
+          name-extension: '_DT'
+          work-flow: 'DT'
+          test-repo: 'voltha-system-tests'
+          profile: '1T8GEM'
+          power-switch: true
+          pipeline-script: 'voltha/master/voltha-dt-physical-functional-tests.groovy'
+
 # [EOF]
diff --git a/jjb/voltha-test/voltha-certification/voltha-2.11.yaml b/jjb/voltha-test/voltha-certification/voltha-2.11.yaml
index bc9781c..fd7d342 100644
--- a/jjb/voltha-test/voltha-certification/voltha-2.11.yaml
+++ b/jjb/voltha-test/voltha-certification/voltha-2.11.yaml
@@ -49,7 +49,7 @@
       - 'build_voltha_pod_release_timer':
           build-node: 'menlo-certification-pod-3'
           config-pod: 'menlo-certification-pod-radisys-3200g'
-          disable-job: true       # menlo-certification-pod-1: offline
+          disable-job: true       # menlo-certification-pod-3: offline
           reinstall-olt: true
           release: '2.11'
           branch: 'voltha-2.11'
@@ -73,5 +73,23 @@
           time: '13'
           pipeline-script: 'voltha/voltha-2.11/physical-build.groovy'
           extraHelmFlags: "--set onos-classic.image.tag=master"
-          
+
+      # -----------------------------------------------------------------------
+      # Certification (Radisys) POD test job
+      # md5sum(master): e8cb3eba849b788944cee1fe58e4091b
+      # -----------------------------------------------------------------------
+      - 'build_voltha_pod_test':
+          timeout: 330
+          disable-job: true      # menlo-certification-pod-3 is offline
+          build-node: 'menlo-certification-pod-3'
+          config-pod: 'menlo-certification-pod-radisys-3200g'
+          release: '2.11'
+          branch: 'voltha-2.11'
+          name-extension: '_DT'
+          work-flow: 'DT'
+          test-repo: 'voltha-system-tests'
+          profile: '1T8GEM'
+          power-switch: true
+          pipeline-script: 'voltha/voltha-2.11/voltha-dt-physical-functional-tests.groovy'
+
 # [EOF]
diff --git a/jjb/voltha-test/voltha-certification/voltha-2.12.yaml b/jjb/voltha-test/voltha-certification/voltha-2.12.yaml
index 41ee4e9..59320d3 100644
--- a/jjb/voltha-test/voltha-certification/voltha-2.12.yaml
+++ b/jjb/voltha-test/voltha-certification/voltha-2.12.yaml
@@ -76,5 +76,23 @@
           time: '13'
           pipeline-script: 'voltha/voltha-2.12/physical-build.groovy'
           extraHelmFlags: "--set onos-classic.image.tag=master"
-          
+
+      # -----------------------------------------------------------------------
+      # Certification (Radisys) POD test job
+      # md5sum(master): e8cb3eba849b788944cee1fe58e4091b
+      # -----------------------------------------------------------------------
+      - 'build_voltha_pod_test':
+          timeout: 330
+          disable-job: true      # menlo-certification-pod-3 is offline
+          build-node: 'menlo-certification-pod-3'
+          config-pod: 'menlo-certification-pod-radisys-3200g'
+          release: '2.12'
+          branch: 'voltha-2.12'
+          name-extension: '_DT'
+          work-flow: 'DT'
+          test-repo: 'voltha-system-tests'
+          profile: '1T8GEM'
+          power-switch: true
+          pipeline-script: 'voltha/voltha-2.12/voltha-dt-physical-functional-tests.groovy'
+
 # [EOF]
diff --git a/jjb/voltha-test/voltha-pod-jobs/voltha-2.11.yaml~ b/jjb/voltha-test/voltha-pod-jobs/voltha-2.11.yaml~
deleted file mode 100644
index 7e42f44..0000000
--- a/jjb/voltha-test/voltha-pod-jobs/voltha-2.11.yaml~
+++ /dev/null
@@ -1,60 +0,0 @@
----
-# POD Build Pipeline Jobs for Voltha Releases
-# -----------------------------------------------------------------------
-# Disabled, offline -- 'qa-testvm-pod'
-# -----------------------------------------------------------------------
-- project:
-    name: voltha-pod-jobs
-
-    project-name: '{name}'
-
-    build-timeout: '300'
-
-    with-kind: false
-    power-switch: false
-    power-cycle-olt: false
-    work-flow: 'ATT'
-    in-band-management: false
-    num-of-openonu: '1'
-    num-of-onos: '1'
-    num-of-atomix: '0'
-    reinstall-olt: true
-    test-type: ''
-    volthaSystemTestsChange: ''
-    kindVolthaChange: ''
-    cordTesterChange: ''
-    oltAdapterAppLabel: 'adapter-open-olt'
-    num-of-onus: ''
-    num-of-ponports: ''
-
-    jobs:
-      # -----------------------------------------------------------------------
-      # [VOL-4939] - flex pod1 test job - released versions: uses tech profile on voltha branch
-      # -----------------------------------------------------------------------
-      - 'build_voltha_pod_test':
-          disable-job: true
-          build-node: 'qa-testvm-pod'
-          config-pod: 'flex-ocp-cord'
-          release: '2.11'
-          branch: 'voltha-2.11'
-          power-switch: true
-          test-repo: 'voltha-system-tests'
-          profile: '1T4GEM'
-
-      # -----------------------------------------------------------------------
-      # [VOL-4939] - flex pod1 test job - released versions: uses tech profile on voltha branch
-      # -----------------------------------------------------------------------
-      - 'build_voltha_pod_test':
-          disable-job: true
-          build-node: 'qa-testvm-pod'
-          config-pod: 'flex-ocp-cord'
-          release: '2.11'
-          branch: 'voltha-2.11'
-          name-extension: '_TT'
-          work-flow: 'TT'
-          power-switch: true
-          pipeline-script: 'voltha/voltha-2.11/voltha-tt-physical-functional-tests.groovy'
-          test-repo: 'voltha-system-tests'
-          profile: 'TP'
-   
-# [EOF]
diff --git a/jjb/voltha-test/voltha-pod-jobs/voltha-2.12.yaml~ b/jjb/voltha-test/voltha-pod-jobs/voltha-2.12.yaml~
deleted file mode 100644
index 22002ac..0000000
--- a/jjb/voltha-test/voltha-pod-jobs/voltha-2.12.yaml~
+++ /dev/null
@@ -1,44 +0,0 @@
----
-# POD Build Pipeline Jobs for Voltha Releases
-# -----------------------------------------------------------------------
-# Disabled, offline -- 'qa-testvm-pod'
-# -----------------------------------------------------------------------
-- project:
-    name: voltha-pod-jobs
-
-    project-name: '{name}'
-
-    build-timeout: '300'
-
-    with-kind: false
-    power-switch: false
-    power-cycle-olt: false
-    work-flow: 'ATT'
-    in-band-management: false
-    num-of-openonu: '1'
-    num-of-onos: '1'
-    num-of-atomix: '0'
-    reinstall-olt: true
-    test-type: ''
-    volthaSystemTestsChange: ''
-    kindVolthaChange: ''
-    cordTesterChange: ''
-    oltAdapterAppLabel: 'adapter-open-olt'
-    num-of-onus: ''
-    num-of-ponports: ''
-
-    jobs:
-      # -----------------------------------------------------------------------
-      # [VOL-4939] - flex pod1 test job - released versions: uses tech profile on voltha branch
-      # -----------------------------------------------------------------------
-      - 'build_voltha_pod_test':
-          disable-job: true              # Reason: qa-testvm-pod offline
-          build-node: 'qa-testvm-pod'
-          config-pod: 'flex-ocp-cord'
-          release: '2.12'
-          branch: 'voltha-2.12'
-          power-switch: true
-          test-repo: 'voltha-system-tests'
-          profile: '1T4GEM'
-   
-# [EOF]