VOL-5155 - triage failure in periodic-voltha-test-bbsim-2.12
jjb/voltha-e2e/master.yaml
jjb/voltha-e2e/voltha-2.11.yaml
jjb/voltha-e2e/voltha-2.12.yaml
jjb/voltha-e2e/voltha-2.8.yaml
-------------------------------
o Set loglevel:DEBUG to force peroidic pipeline job to regenerate.
jjb/pipeline/voltha/master/bbsim-tests.groovy
jjb/pipeline/voltha/voltha-2.11/bbsim-tests.groovy
jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
--------------------------------------------------
o Update getversion() changeset to verify latest script version in use.
Change-Id: If40246ea2b3cac3025df900c371f3fd918fc64a4
diff --git a/jjb/pipeline/voltha/master/bbsim-tests.groovy b/jjb/pipeline/voltha/master/bbsim-tests.groovy
index 674b4b0..ae5c58a 100644
--- a/jjb/pipeline/voltha/master/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/master/bbsim-tests.groovy
@@ -44,7 +44,7 @@
// per-script to be sure latest repository changes are being used.
// -----------------------------------------------------------------------
String pipelineVer() {
- String version = 'ebc180201844dad3a9a3d6551f3e257797e08ab7'
+ String version = '5addce3fac89095d103ac5c6eedff2bb02e9ec63'
return(version)
}
@@ -136,7 +136,7 @@
// todo: fatal unless (proc count==0)
pgrep_proc(proc)
- enter('Cleanup')
+ leave('Cleanup')
} // script
} // timeout
} // teardown
@@ -159,7 +159,8 @@
'prometheus.pushgateway.enabled=false',
].join(',')
- sh("""
+ sh(label : 'Deploy common infrastructure',
+ script : """
helm repo add onf https://charts.opencord.org
helm repo update
@@ -324,7 +325,7 @@
],
logLevel: logLevel
])
- enter('setOnosLogLevels')
+ leave('setOnosLogLevels')
} // script
} // if (teardown)
} // stage('Deploy Voltha')
@@ -333,7 +334,9 @@
// -----------------------------------------------------------------------
stage("Run test ${testTarget} on workflow ${workflow}")
{
- sh """
+ sh(
+ label : 'Monitor using mem_consumption.py',
+ script : """
echo -e "\n** Monitor using mem_consumption.py ?"
if [ ${withMonitoring} = true ] ; then
@@ -355,7 +358,7 @@
fi
echo -e '** Monitor memory consumption: LEAVE\n'
- """
+ """)
sh """
echo -e "\n** make testTarget=[${testTarget}]"
@@ -369,7 +372,9 @@
getPodsInfo("${logsDir}")
- sh """
+ sh(
+ label : 'Gather robot Framework logs',
+ script : """
echo -e '\n** Gather robot Framework logs: ENTER'
# set +e
# collect logs collected in the Robot Framework StartLogging keyword
@@ -378,11 +383,13 @@
rm -f *-combined.log
echo -e '** Gather robot Framework logs: LEAVE\n'
- """
+ """)
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
- sh """
+ sh(
+ label : 'Monitor pod-mem-consumption',
+ script : """
echo -e '** Monitor pod-mem-consumption: ENTER'
if [ ${withMonitoring} = true ] ; then
cat <<EOM
@@ -401,7 +408,7 @@
python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
fi
echo -e '** Monitor pod-mem-consumption: LEAVE\n'
- """
+ """)
} // stage
return
@@ -598,7 +605,7 @@
// Announce ourselves for log usability
enter('Parse and execute tests')
- def tests = readYaml text: testTargets
+ def tests = readYaml text: testTargets // typeof == Map (?)
println("** [DEBUG]: tests=$tests")
// Display expected tests for times when output goes dark
@@ -632,7 +639,7 @@
""")
try {
- leave("execute_test (target=$target)")
+ enter("execute_test (target=$target)")
execute_test(target, workflow, testLogging, teardown, flags)
}
catch (Exception err) {
diff --git a/jjb/pipeline/voltha/voltha-2.11/bbsim-tests.groovy b/jjb/pipeline/voltha/voltha-2.11/bbsim-tests.groovy
index c9293fc..79f648a 100644
--- a/jjb/pipeline/voltha/voltha-2.11/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/voltha-2.11/bbsim-tests.groovy
@@ -23,6 +23,49 @@
def clusterName = "kind-ci"
+// -----------------------------------------------------------------------
+// Intent:
+// -----------------------------------------------------------------------
+String branchName() {
+ String name = 'voltha-2.11'
+
+ // [TODO] Sanity check the target branch
+ // if (name != jenkins.branch) { fatal }
+ return(name)
+}
+
+// -----------------------------------------------------------------------
+// Intent: Difficult at times to determine when pipeline jobs have
+// regenerated. Hardcode a version string that can be assigned
+// per-script to be sure latest repository changes are being used.
+// -----------------------------------------------------------------------
+String pipelineVer() {
+ String version = '5addce3fac89095d103ac5c6eedff2bb02e9ec63'
+ return(version)
+}
+
+// -----------------------------------------------------------------------
+// Intent: Due to lack of a reliable stack trace, construct a literal.
+// Jenkins will re-write the call stack for serialization.S
+// -----------------------------------------------------------------------
+// Note: Hardcoded version string used to visualize changes in jenkins UI
+// -----------------------------------------------------------------------
+String getIam(String func) {
+ String branchName = branchName()
+ String version = pipelineVer()
+ String src = [
+ 'ci-management',
+ 'jjb',
+ 'pipeline',
+ 'voltha',
+ branchName,
+ 'bbsim-tests.groovy'
+ ].join('/')
+
+ String name = [src, version, func].join('::')
+ return(name)
+}
+
def execute_test(testTarget, workflow, testLogging, teardown, testSpecificHelmFlags = "")
{
def infraNamespace = "default"
@@ -72,16 +115,27 @@
stage ('Initialize')
{
- // VOL-4926 - Is voltha-system-tests available ?
- String cmd = [
- 'make',
- '-C', "$WORKSPACE/voltha-system-tests",
- "KAIL_PATH=\"$WORKSPACE/bin\"",
- 'kail',
- ].join(' ')
- println(" ** Running: ${cmd}:\n")
- sh("${cmd}")
- }
+ steps
+ {
+ script
+ {
+ String iam = getIam('Initialize')
+ println("${iam}: ENTER")
+
+ // VOL-4926 - Is voltha-system-tests available ?
+ String cmd = [
+ 'make',
+ '-C', "$WORKSPACE/voltha-system-tests",
+ "KAIL_PATH=\"$WORKSPACE/bin\"",
+ 'kail',
+ ].join(' ')
+ println(" ** Running: ${cmd}:\n")
+ sh("${cmd}")
+
+ println("${iam}: LEAVE")
+ } // script
+ } // steps
+ } // stage
stage('Deploy common infrastructure') {
sh '''
diff --git a/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy b/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
index cc6e43d..522d7f7 100644
--- a/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
@@ -44,7 +44,7 @@
// per-script to be sure latest repository changes are being used.
// -----------------------------------------------------------------------
String pipelineVer() {
- String version = 'ebc180201844dad3a9a3d6551f3e257797e08ab7'
+ String version = '5addce3fac89095d103ac5c6eedff2bb02e9ec63'
return(version)
}
@@ -136,7 +136,7 @@
// todo: fatal unless (proc count==0)
pgrep_proc(proc)
- enter('Cleanup')
+ leave('Cleanup')
} // script
} // timeout
} // teardown
@@ -159,7 +159,8 @@
'prometheus.pushgateway.enabled=false',
].join(',')
- sh("""
+ sh(label : 'Deploy common infrastructure',
+ script : """
helm repo add onf https://charts.opencord.org
helm repo update
@@ -324,7 +325,7 @@
],
logLevel: logLevel
])
- enter('setOnosLogLevels')
+ leave('setOnosLogLevels')
} // script
} // if (teardown)
} // stage('Deploy Voltha')
@@ -638,7 +639,7 @@
""")
try {
- leave("execute_test (target=$target)")
+ enter("execute_test (target=$target)")
execute_test(target, workflow, testLogging, teardown, flags)
}
catch (Exception err) {
diff --git a/jjb/voltha-e2e/master.yaml b/jjb/voltha-e2e/master.yaml
index 4c0433b..1398f29 100644
--- a/jjb/voltha-e2e/master.yaml
+++ b/jjb/voltha-e2e/master.yaml
@@ -72,11 +72,21 @@
logging: true
time-trigger: "H H/23 * * *"
+ # -----------------------------------------------------------------------
+ # md5sum(master): c606493da6ee7cbb18ca8aee97ea2957
+ # -----------------------------------------------------------------------
- 'voltha-periodic-test':
name: 'periodic-voltha-test-bbsim'
- code-branch: 'master'
+ code-branch : 'master'
extraHelmFlags: '--set global.image_tag=master --set onos-classic.image.tag=master --set onos-classic.replicas=3,onos-classic.atomix.replicas=3 --set voltha.onos_classic.replicas=3 '
- time-trigger: "H H/23 * * *" # Build daily at 11pm
+ logLevel: 'DEBUG' # Force job to regenerate
+ # ---------------------------------------------------------
+ # 20230828 - strange, release branches have pipeline-script
+ # 20230828 - while master branch did not (?) Script does
+ # 20230828 - show in the jenkins UI but w/o updates.
+ # ---------------------------------------------------------
+ pipeline-script: 'voltha/master/bbsim-tests.groovy'
+ time-trigger: "H H/23 * * *" # Build daily at 11pm
# time-trigger: "H/30 * * * *" # Build every 30 min
testTargets: |
- target: functional-single-kind
diff --git a/jjb/voltha-e2e/voltha-2.11.yaml b/jjb/voltha-e2e/voltha-2.11.yaml
index 5da941a..c43c417 100644
--- a/jjb/voltha-e2e/voltha-2.11.yaml
+++ b/jjb/voltha-e2e/voltha-2.11.yaml
@@ -79,6 +79,7 @@
name: 'periodic-voltha-test-bbsim-2.11'
pipeline-script: 'voltha/voltha-2.11/bbsim-tests.groovy'
code-branch: 'voltha-2.11'
+ logLevel: 'DEBUG' # Force job to regenerate
time-trigger: "H H/23 * * *"
testTargets: |
- target: functional-single-kind
diff --git a/jjb/voltha-e2e/voltha-2.12.yaml b/jjb/voltha-e2e/voltha-2.12.yaml
index b9cbb79..743e171 100644
--- a/jjb/voltha-e2e/voltha-2.12.yaml
+++ b/jjb/voltha-e2e/voltha-2.12.yaml
@@ -75,10 +75,14 @@
logging: true
time-trigger: "H H/23 * * *"
+ # -----------------------------------------------------------------------
+ # md5sum(master): c606493da6ee7cbb18ca8aee97ea2957
+ # -----------------------------------------------------------------------
- 'voltha-periodic-test':
name: 'periodic-voltha-test-bbsim-2.12'
- pipeline-script: 'voltha/voltha-2.12/bbsim-tests.groovy'
code-branch: 'voltha-2.12'
+ logLevel: 'DEBUG' # force job to regenerate
+ pipeline-script: 'voltha/voltha-2.12/bbsim-tests.groovy'
time-trigger: "H H/23 * * *"
testTargets: |
- target: functional-single-kind
@@ -101,6 +105,11 @@
flags: ""
teardown: false
logging: true
+ - target: onos-ha-test
+ workflow: att
+ flags: ""
+ teardown: false
+ logging: true
- 'voltha-periodic-test':
name: 'periodic-voltha-sanity-test-multi-runs-2.12'
diff --git a/jjb/voltha-e2e/voltha-2.8.yaml b/jjb/voltha-e2e/voltha-2.8.yaml
index 0803c33..b5e8ac2 100644
--- a/jjb/voltha-e2e/voltha-2.8.yaml
+++ b/jjb/voltha-e2e/voltha-2.8.yaml
@@ -77,7 +77,7 @@
- 'voltha-periodic-test':
name: 'periodic-voltha-test-bbsim-2.8'
- disable-job: true
+ disable-job: true # Reason: deprecated legacy build
pipeline-script: 'voltha/voltha-2.8/bbsim-tests.groovy'
code-branch: 'voltha-2.8'
time-trigger: "H H/23 * * *"