VOL-5155 - triage failure in periodic-voltha-test-bbsim-2.12
jjb/pipeline/voltha/master/bbsim-tests.groovy
---------------------------------------------
o v2.12 dt-bbsim passed, copy scirpt into master and update release branch.
Change-Id: Idcc776a0add50532cd5f414e2931c5e049543f4d
diff --git a/jjb/pipeline/voltha/master/bbsim-tests.groovy b/jjb/pipeline/voltha/master/bbsim-tests.groovy
index 4850e20..46cb252 100644
--- a/jjb/pipeline/voltha/master/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/master/bbsim-tests.groovy
@@ -39,11 +39,24 @@
}
// -----------------------------------------------------------------------
+// Intent: Difficult at times to determine when pipeline jobs have
+// regenerated. Hardcode a version string that can be assigned
+// per-script to be sure latest repository changes are being used.
+// -----------------------------------------------------------------------
+String pipelineVer() {
+ String version = '757ba9ea12d8d815c10301b7f265f3fcd7c41d26'
+ return(version)
+}
+
+// -----------------------------------------------------------------------
// Intent: Due to lack of a reliable stack trace, construct a literal.
-// Jenkins will re-write the call stack for serialization.
+// Jenkins will re-write the call stack for serialization.S
+// -----------------------------------------------------------------------
+// Note: Hardcoded version string used to visualize changes in jenkins UI
// -----------------------------------------------------------------------
String getIam(String func) {
String branchName = branchName()
+ String version = pipelineVer()
String src = [
'ci-management',
'jjb',
@@ -53,174 +66,221 @@
'bbsim-tests.groovy'
].join('/')
- String name = [src, func].join('::')
+ String name = [src, version, func].join('::')
return(name)
}
// -----------------------------------------------------------------------
+// Intent: Log progress message
+// -----------------------------------------------------------------------
+void enter(String name) {
+ // Announce ourselves for log usability
+ String iam = getIam(name)
+ println("${iam}: ENTER")
+ return
+}
+
+// -----------------------------------------------------------------------
+// Intent: Log progress message
+// -----------------------------------------------------------------------
+void leave(String name) {
+ // Announce ourselves for log usability
+ String iam = getIam(name)
+ println("${iam}: LEAVE")
+ return
+}
+
+// -----------------------------------------------------------------------
// Intent: Determine if working on a release branch.
// Note: Conditional is legacy, should also check for *-dev or *-pre
// -----------------------------------------------------------------------
-Boolean isReleaseBranch(String name)
-{
+Boolean isReleaseBranch(String name) {
// List modifiers = ['-dev', '-pre', 'voltha-x.y.z-pre']
// if branchName in modifiers
return(name != 'master') // OR branchName.contains('-')
}
// -----------------------------------------------------------------------
-// Intent:
+// Intent: Iterate over a list of test suites and invoke.
// -----------------------------------------------------------------------
-void execute_test(testTarget, workflow, testLogging, teardown, testSpecificHelmFlags='')
-{
+void execute_test\
+(
+ String testTarget, // functional-single-kind-dt
+ String workflow, // dt
+ String testLogging, // 'True'
+ Boolean teardown, // true
+ String testSpecificHelmFlags=''
+) {
String infraNamespace = 'default'
String volthaNamespace = 'voltha'
String logsDir = "$WORKSPACE/${testTarget}"
- stage('IAM')
- {
- script
- {
- // Announce ourselves for log usability
- String iam = getIam('execute_test')
- println("${iam}: ENTER")
- println("${iam}: LEAVE")
- }
- }
-
// -----------------------------------------------------------------------
// Intent: Cleanup stale port-forwarding
// -----------------------------------------------------------------------
- stage('Cleanup')
- {
- if (teardown) {
+ stage('Cleanup') {
+ if (teardown) {
timeout(15) {
- script {
+ script {
helmTeardown(['default', infraNamespace, volthaNamespace])
}
- timeout(1) {
- sh returnStdout: false, script: '''
- # remove orphaned port-forward from different namespaces
- ps aux | grep port-forw | grep -v grep | awk '{print $2}' | xargs --no-run-if-empty kill -9
- '''
- }
- }
- }
- }
+ } // timeout
+
+ timeout(5) {
+ script {
+ enter('Cleanup')
+ // remove orphaned port-forward from different namespaces
+ String proc = 'port-forw'
+ pgrep_proc(proc)
+ pkill_proc(proc)
+ pgrep_proc(proc) // proc count == 0
+ enter('Cleanup')
+ } // script
+ } // timeout
+ } // teardown
+ } // stage('Cleanup')
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
- stage('Deploy common infrastructure')
- {
- sh '''
+ stage('Deploy common infrastructure') {
+ script {
+ local dashargs = [
+ 'kpi_exporter.enabled=false',
+ 'dashboards.xos=false',
+ 'dashboards.onos=false',
+ 'dashboards.aaa=false',
+ 'dashboards.voltha=false',
+ ].join(',')
+
+ local promargs = [
+ 'prometheus.alertmanager.enabled=false',
+ 'prometheus.pushgateway.enabled=false',
+ ].join(',')
+
+ sh("""
helm repo add onf https://charts.opencord.org
helm repo update
+
+ echo -e "\nwithMonitoring=[$withMonitoring]"
if [ ${withMonitoring} = true ] ; then
helm install nem-monitoring onf/nem-monitoring \
- --set prometheus.alertmanager.enabled=false,prometheus.pushgateway.enabled=false \
- --set kpi_exporter.enabled=false,dashboards.xos=false,dashboards.onos=false,dashboards.aaa=false,dashboards.voltha=false
+ --set ${promargs} \
+ --set ${dashargs}
fi
- '''
- }
+ """)
+ } // script
+ } // stage('Deploy Common Infra')
- stage('Deploy Voltha')
- {
- if (teardown)
- {
- timeout(10)
- {
- script
- {
- sh """
+ // -----------------------------------------------------------------------
+ // [TODO] Check onos_log output
+ // -----------------------------------------------------------------------
+ stage('Deploy Voltha') {
+ if (teardown) {
+ timeout(10) {
+ script {
+ String iam = getIam('Deploy Voltha')
+ String onosLog = "${logsDir}/onos-voltha-startup-combined.log"
+ sh("""
mkdir -p ${logsDir}
- _TAG=kail-startup kail -n ${infraNamespace} -n ${volthaNamespace} > ${logsDir}/onos-voltha-startup-combined.log &
- """
+ touch "$onosLog"
+ echo "** kail-startup ENTER: \$(date)" > "$onosLog"
- // if we're downloading a voltha-helm-charts patch, then install from a local copy of the charts
- Boolean localCharts = false
+ # Intermixed output (tee -a &) may get conflusing but let(s) see
+ # what messages are logged during startup.
+ # _TAG=ka7il-startup kail -n ${infraNamespace} -n ${volthaNamespace} >> "$onosLog" &
+ _TAG=kail-startup kail -n ${infraNamespace} -n ${volthaNamespace} | tee -a "$onosLog" &
+ """)
- if (volthaHelmChartsChange != ''
- || gerritProject == 'voltha-helm-charts'
- || isReleaseBranch(branch) // branch != 'master'
- ) {
- localCharts = true
- }
+ // if we're downloading a voltha-helm-charts patch, then install from a local copy of the charts
+ Boolean localCharts = false
- String branchName = branchName()
- Boolean is_release = isReleaseBranch(branch)
- println([
- " ** localCharts=${localCharts}",
- "branchName=${branchName}",
- "branch=${branch}",
- "branch=isReleaseBranch=${is_release}",
- ].join(', '))
+ if (volthaHelmChartsChange != ''
+ || gerritProject == 'voltha-helm-charts'
+ || isReleaseBranch(branch) // branch != 'master'
+ ) {
+ localCharts = true
+ }
- // -----------------------------------------------------------------------
- // Rewrite localHelmFlags using array join, moving code around and
- // refactoring into standalone functions
- // -----------------------------------------------------------------------
- // hudson.remoting.ProxyException: groovy.lang.MissingMethodException:
- // No signature of method: java.lang.String.positive() is applicable for argument types: () values: []
- // -----------------------------------------------------------------------
- // NOTE temporary workaround expose ONOS node ports
- // -----------------------------------------------------------------------
- String localHelmFlags = [
- extraHelmFlags.trim(),
- "--set global.log_level=${logLevel.toUpperCase()}",
- '--set onos-classic.onosSshPort=30115',
- '--set onos-classic.onosApiPort=30120',
- '--set onos-classic.onosOfPort=31653',
- '--set onos-classic.individualOpenFlowNodePorts=true',
- testSpecificHelmFlags
- ].join(' ')
+ String branchName = branchName()
+ Boolean isRelease = isReleaseBranch(branch)
+ println([
+ " ** localCharts=${localCharts}",
+ "branchName=${branchName}",
+ "branch=${branch}",
+ "branch=isReleaseBranch=${isRelease}",
+ ].join(', '))
- println("** localHelmFlags = ${localHelmFlags}")
+ // -----------------------------------------------------------------------
+ // Rewrite localHelmFlags using array join, moving code around and
+ // refactoring into standalone functions
+ // -----------------------------------------------------------------------
+ // NOTE temporary workaround expose ONOS node ports
+ // -----------------------------------------------------------------------
+ String localHelmFlags = [
+ extraHelmFlags.trim(),
+ "--set global.log_level=${logLevel.toUpperCase()}",
+ '--set onos-classic.onosSshPort=30115',
+ '--set onos-classic.onosApiPort=30120',
+ '--set onos-classic.onosOfPort=31653',
+ '--set onos-classic.individualOpenFlowNodePorts=true',
+ testSpecificHelmFlags
+ ].join(' ')
- if (gerritProject != '') {
- localHelmFlags = "${localHelmFlags} " + getVolthaImageFlags("${gerritProject}")
- }
+ println("** ${iam} localHelmFlags = ${localHelmFlags}")
- println('volthaDeploy: ENTER')
- volthaDeploy([
- infraNamespace: infraNamespace,
- volthaNamespace: volthaNamespace,
- workflow: workflow.toLowerCase(),
- withMacLearning: enableMacLearning.toBoolean(),
- extraHelmFlags: localHelmFlags,
- localCharts: localCharts,
- bbsimReplica: olts.toInteger(),
- dockerRegistry: registry,
- ])
- println('volthaDeploy: LEAVE')
- } // script
+ if (gerritProject != '') {
+ localHelmFlags = "${localHelmFlags} " + getVolthaImageFlags("${gerritProject}")
+ }
- // -----------------------------------------------------------------------
- // Intent: Replacing P_IDS with pgrep/pkill is a step forward.
- // Why not simply use a pid file, capture _TAG=kail-startup above
- // Grep runs the risk of terminating stray commands (??-good <=> bad-??)
- // -----------------------------------------------------------------------
- script {
- println('Try out pgrep/pkill commands')
- sh('''pgrep --list-full kail-startup || true''')
- }
+ println("** ${iam}: ENTER")
+ volthaDeploy([
+ infraNamespace: infraNamespace,
+ volthaNamespace: volthaNamespace,
+ workflow: workflow.toLowerCase(),
+ withMacLearning: enableMacLearning.toBoolean(),
+ extraHelmFlags: localHelmFlags,
+ localCharts: localCharts,
+ bbsimReplica: olts.toInteger(),
+ dockerRegistry: registry,
+ ])
+ println("** ${iam}: LEAVE")
+ } // script
- // -----------------------------------------------------------------------
- // stop logging
- // -----------------------------------------------------------------------
- sh """
- P_IDS="\$(ps e -ww -A | grep "_TAG=kail-startup" | grep -v grep | awk '{print \$1}')"
- if [ -n "\$P_IDS" ]; then
- echo \$P_IDS
- for P_ID in \$P_IDS; do
- kill -9 \$P_ID
- done
- fi
- cd ${logsDir}
+ // -----------------------------------------------------------------------
+ // Intent: Replacing P_IDS with pgrep/pkill is a step forward.
+ // Why not simply use a pid file, capture _TAG=kail-startup above
+ // Grep runs the risk of terminating stray commands (??-good <=> bad-??)
+ // -----------------------------------------------------------------------
+ script {
+ String proc = '_TAG=kail-startup'
+
+ println("${iam}: ENTER")
+ println("${iam}: Shutdown process $proc")
+ pgrep_proc(proc)
+ pkill_proc(proc)
+ pgrep_proc(proc)
+ println("${iam}: LEAVE")
+ }
+
+ // -----------------------------------------------------------------------
+ // Bundle onos-voltha / kail logs
+ // -----------------------------------------------------------------------
+ sh("""
+cat <<EOM
+
+** -----------------------------------------------------------------------
+** Combine an compress voltha startup log(s)
+** -----------------------------------------------------------------------
+EOM
+ pushd "${logsDir}" || { echo "ERROR: pushd $logsDir failed"; exit 1; }
gzip -k onos-voltha-startup-combined.log
rm onos-voltha-startup-combined.log
- """
- }
+ popd
+ """)
+ } // timeout(10)
+ // -----------------------------------------------------------------------
+ // -----------------------------------------------------------------------
sh """
JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-voltha-api" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/voltha-voltha-api 55555:55555; done"&
JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-etcd" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-etcd 2379:2379; done"&
@@ -235,12 +295,21 @@
fi
ps aux | grep port-forward
"""
+ // ---------------------------------
+ // Sanity check port-forward spawned
+ // ---------------------------------
+ script {
+ enter('port-forward check')
+ String proc = 'port-forward'
+ println("Display spawned ${proc}")
+ pgrep_proc(proc)
+ leave('port-forward check')
+ }
// setting ONOS log level
- script
- {
- println('** setOnosLogLevels: ENTER')
- setOnosLogLevels([
+ script {
+ enter('setOnosLogLevels')
+ setOnosLogLevels([
onosNamespace: infraNamespace,
apps: [
'org.opencord.dhcpl2relay',
@@ -252,7 +321,7 @@
],
logLevel: logLevel
])
- println('** setOnosLogLevels: LEAVE')
+ enter('setOnosLogLevels')
} // script
} // if (teardown)
} // stage('Deploy Voltha')
@@ -262,7 +331,7 @@
stage("Run test ${testTarget} on workflow ${workflow}")
{
sh """
- echo -e '** Monitor memory consumption: ENTER'
+ echo -e "\n** Monitor using mem_consumption.py ?"
if [ ${withMonitoring} = true ] ; then
cat <<EOM
@@ -338,7 +407,10 @@
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
def collectArtifacts(exitStatus) {
- String iam = getIam('execute_test')
+ script {
+ String iam = getIam('collectArtifacts')
+ println("${iam}: ENTER (exitStatus=${exitStatus})")
+ }
echo '''
@@ -346,7 +418,6 @@
** collectArtifacts
** -----------------------------------------------------------------------
'''
- println("${iam}: ENTER (exitStatus=${exitStatus})")
getPodsInfo("$WORKSPACE/${exitStatus}")
@@ -356,25 +427,24 @@
archiveArtifacts artifacts: '**/*.log,**/*.gz,**/*.txt,**/*.html,**/voltha-pods-mem-consumption-att/*,**/voltha-pods-mem-consumption-dt/*,**/voltha-pods-mem-consumption-tt/*'
- sh(returnStdout:true, script: '''
- sync
- echo '** Running: pgrep --list-full kail-startup (ENTER)'
- pgrep --list-full 'kail-startup' || true
- [[ $(pgrep --count kail) -gt 0 ]] && pkill --echo kail
- echo '** Running: pgrep --list-full kail-startup (LEAVE)'
- ''')
+ script {
+ println("${iam}: ENTER")
+ pgrep_proc('kail-startup')
+ pkill_proc('kail')
+ println("${iam}: LEAVE")
+ }
println("${iam}: ENTER RobotPublisher")
step([$class: 'RobotPublisher',
- disableArchiveOutput: false,
- logFileName: '**/*/log*.html',
- otherFiles: '',
- outputFileName: '**/*/output*.xml',
- outputPath: '.',
- passThreshold: 100,
- reportFileName: '**/*/report*.html',
- unstableThreshold: 0,
- onlyCritical: true]);
+ disableArchiveOutput: false,
+ logFileName: '**/*/log*.html',
+ otherFiles: '',
+ outputFileName: '**/*/output*.xml',
+ outputPath: '.',
+ passThreshold: 100,
+ reportFileName: '**/*/report*.html',
+ unstableThreshold: 0,
+ onlyCritical: true])
println("${iam}: LEAVE RobotPublisher")
println("${iam}: LEAVE (exitStatus=${exitStatus})")
@@ -385,179 +455,166 @@
// Intent: main
// -----------------------------------------------------------------------
pipeline {
-
- /* no label, executor is determined by JJB */
- agent {
- label "${params.buildNode}"
- }
- options {
- timeout(time: "${timeout}", unit: 'MINUTES')
- }
- environment {
- KUBECONFIG = "$HOME/.kube/kind-${clusterName}"
- VOLTCONFIG = "$HOME/.volt/config"
- PATH = "$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
- DIAGS_PROFILE = 'VOLTHA_PROFILE'
- SSHPASS = 'karaf'
- }
- stages {
- stage('Download Code') {
- steps {
- getVolthaCode([
- branch: "${branch}",
- gerritProject: "${gerritProject}",
- gerritRefspec: "${gerritRefspec}",
- volthaSystemTestsChange: "${volthaSystemTestsChange}",
- volthaHelmChartsChange: "${volthaHelmChartsChange}",
- ])
- }
+ /* no label, executor is determined by JJB */
+ agent {
+ label "${params.buildNode}"
}
- stage('Build patch v1.1')
- {
- // build the patch only if gerritProject is specified
- when
- {
- expression
- {
- return !gerritProject.isEmpty()
- }
- }
-
- steps
- {
- // NOTE that the correct patch has already been checked out
- // during the getVolthaCode step
- buildVolthaComponent("${gerritProject}")
- }
+ options {
+ timeout(time: "${timeout}", unit: 'MINUTES')
}
- // -----------------------------------------------------------------------
- // -----------------------------------------------------------------------
- stage('Install Kail')
- {
- steps
- {
- script
- {
- String cmd = [
- 'make',
- '--no-print-directory',
- '-C', "$WORKSPACE/voltha-system-tests",
- "KAIL_PATH=\"$WORKSPACE/bin\"",
- 'kail',
- ].join(' ')
+ environment {
+ KUBECONFIG = "$HOME/.kube/kind-${clusterName}"
+ VOLTCONFIG = "$HOME/.volt/config"
+ PATH = "$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+ DIAGS_PROFILE = 'VOLTHA_PROFILE'
+ SSHPASS = 'karaf'
+ }
- println(" ** Running: ${cmd}:\n")
- sh("${cmd}")
- } // script
- } // steps
- } // stage
-
- // -----------------------------------------------------------------------
- // -----------------------------------------------------------------------
- stage('Install Kind')
- {
- steps
- {
- script
- {
-
- String cmd = [
- 'make',
- '--no-print-directory',
- '-C', "$WORKSPACE/voltha-system-tests",
- "KIND_PATH=\"$WORKSPACE/bin\"",
- 'install-command-kind',
- ].join(' ')
-
- println(" ** Running: ${cmd}:\n")
- sh("${cmd}")
- } // script
- } // steps
- } // stage
-
- // -----------------------------------------------------------------------
- // -----------------------------------------------------------------------
- stage('Create K8s Cluster')
- {
- steps
- {
- script
- {
- def clusterExists = sh(
- returnStdout: true,
- script: """kind get clusters | grep "${clusterName}" | wc -l""")
-
- if (clusterExists.trim() == '0')
- {
- createKubernetesCluster([nodes: 3, name: clusterName])
- }
- } // script
- } // steps
- } // stage('Create K8s Cluster')
-
- // -----------------------------------------------------------------------
- // -----------------------------------------------------------------------
- stage('Replace voltctl')
- {
- // if the project is voltctl, override the downloaded one with the built one
- when {
- expression {
- return gerritProject == 'voltctl'
- }
- }
-
- // Hmmmm(?) where did the voltctl download happen ?
- // Likely Makefile but would be helpful to document here.
- steps
- {
- println("${iam} Running: installVoltctl($branch)")
- installVoltctl("$branch")
- } // steps
- } // stage
-
- // -----------------------------------------------------------------------
- // -----------------------------------------------------------------------
- stage('voltctl [DEBUG]') {
- steps {
- script {
- String iam = getIam('execute_test')
-
- println("${iam} Display umask")
- sh('umask')
-
- println("${iam} Checking voltctl config permissions")
- sh('/bin/ls -ld ~/.volt ~/.volt/* || true')
- } // script
- } // steps
- } // stage
-
- // -----------------------------------------------------------------------
- // -----------------------------------------------------------------------
- stage('Load image in kind nodes')
- {
- when {
- expression {
- return !gerritProject.isEmpty()
- }
- }
- steps {
- loadToKind()
- } // steps
- } // stage
-
- // -----------------------------------------------------------------------
- // -----------------------------------------------------------------------
- stage('Parse and execute tests')
- {
+ stages {
+ stage('Download Code') {
steps {
- script {
- def tests = readYaml text: testTargets
+ getVolthaCode([
+ branch: "${branch}",
+ gerritProject: "${gerritProject}",
+ gerritRefspec: "${gerritRefspec}",
+ volthaSystemTestsChange: "${volthaSystemTestsChange}",
+ volthaHelmChartsChange: "${volthaHelmChartsChange}",
+ ])
+ }
+ }
- tests.eachWithIndex { test, idx ->
- println "** readYaml test suite[$idx]) test=[${test}]"
- // def test = tests[i]
+ stage('Build patch v1.1') {
+ // build the patch only if gerritProject is specified
+ when {
+ expression { return !gerritProject.isEmpty() }
+ }
+
+ steps {
+ // NOTE that the correct patch has already been checked out
+ // during the getVolthaCode step
+ buildVolthaComponent("${gerritProject}")
+ }
+ }
+
+ // -----------------------------------------------------------------------
+ // -----------------------------------------------------------------------
+ stage('Install Kail')
+ {
+ steps
+ {
+ script
+ {
+ String cmd = [
+ 'make',
+ '--no-print-directory',
+ '-C', "$WORKSPACE/voltha-system-tests",
+ "KAIL_PATH=\"$WORKSPACE/bin\"",
+ 'kail',
+ ].join(' ')
+
+ println(" ** Running: ${cmd}")
+ sh("${cmd}")
+ } // script
+ } // steps
+ } // stage
+
+ // -----------------------------------------------------------------------
+ // -----------------------------------------------------------------------
+ stage('Install Tools') {
+ steps {
+ script {
+ String branchName = branchName()
+ String iam = getIam('Install Tools')
+
+ println("${iam}: ENTER (branch=$branch)")
+ installKind(branch) // needed early by stage(Cleanup)
+ println("${iam}: LEAVE (branch=$branch)")
+ } // script
+ } // steps
+ } // stage
+
+ // -----------------------------------------------------------------------
+ // -----------------------------------------------------------------------
+ stage('Create K8s Cluster') {
+ steps {
+ script {
+ def clusterExists = sh(
+ returnStdout: true,
+ script: """kind get clusters | grep "${clusterName}" | wc -l""")
+
+ if (clusterExists.trim() == '0') {
+ createKubernetesCluster([nodes: 3, name: clusterName])
+ }
+ } // script
+ } // steps
+ } // stage('Create K8s Cluster')
+
+ // -----------------------------------------------------------------------
+ // -----------------------------------------------------------------------
+ stage('Replace voltctl') {
+ // if the project is voltctl, override the downloaded one with the built one
+ when {
+ expression { return gerritProject == 'voltctl' }
+ }
+
+ // Hmmmm(?) where did the voltctl download happen ?
+ // Likely Makefile but would be helpful to document here.
+ steps {
+ script {
+ String iam = getIam('Replace voltctl')
+
+ println("${iam} Running: installVoltctl($branch)")
+ println("${iam}: ENTER")
+ installVoltctl("$branch")
+ println("${iam}: LEAVE")
+ } // script
+ } // step
+ } // stage
+
+ // -----------------------------------------------------------------------
+ // -----------------------------------------------------------------------
+ stage('Load image in kind nodes')
+ {
+ when {
+ expression { return !gerritProject.isEmpty() }
+ }
+ steps {
+ loadToKind()
+ } // steps
+ } // stage
+
+ // -----------------------------------------------------------------------
+ // [TODO] verify testing output
+ // -----------------------------------------------------------------------
+ stage('Parse and execute tests')
+ {
+ steps {
+ script {
+ // Announce ourselves for log usability
+ enter('Parse and execute tests')
+
+ def tests = readYaml text: testTargets
+ println("** [DEBUG]: tests=$tests")
+
+ // Display expected tests for times when output goes dark
+ tests.eachWithIndex { test, idx ->
+ String target = test['target']
+ println("** test[${idx}]: ${target}\n")
+ }
+
+ println('''
+** -----------------------------------------------------------------------
+** NOTE: For odd/silent job failures verify a few details
+** - All tests mentioned in the tests-to-run index were logged.
+** - Test suites display ENTER/LEAVE mesasge pairs.
+** - Processing terminated prematurely when LEAVE strings are missing.
+** -----------------------------------------------------------------------
+''')
+ tests.eachWithIndex { test, idx ->
+ println "** readYaml test suite[$idx]) test=[${test}]"
+
String target = test['target']
String workflow = test['workflow']
String flags = test['flags']
@@ -565,21 +622,25 @@
Boolean logging = test['logging'].toBoolean()
String testLogging = (logging) ? 'True' : 'False'
- print("""
-
+ print("""
** -----------------------------------------------------------------------
-** Executing test ${target} on workflow ${workflow} with logging ${testLogging} and extra flags ${flags}"
+** Executing test ${target} on workflow ${workflow} with logging ${testLogging} and extra flags ${flags}
** -----------------------------------------------------------------------
-
""")
- println "Executing test ${target}: ENTER"
- execute_test(target, workflow, testLogging, teardown, flags)
- println "Executing test ${target}: LEAVE"
+ try {
+ leave("execute_test (target=$target)")
+ execute_test(target, workflow, testLogging, teardown, flags)
+ }
+ catch (Exception err) {
+ println("** ${iam}: EXCEPTION ${err}")
+ }
+ finally {
+ leave("execute_test (target=$target)")
+ }
} // for
-
- String iam = getIam('Parse and execute tests')
- println("** ${iam} ran to completion")
+ // Premature exit if this message is not logged
+ leave('Parse and execute tests')
} // script
} // steps
} // stage
@@ -593,4 +654,4 @@
}
} // pipeline
-// EOF
+// [EOF] - 2