[VOL-5100] Fix bbsim test failure
jjb/pipeline/voltha/master/bbsim-tests.groovy
jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
--------------------------------------------------
o Add more debugging statements.
o Last job run reached log digestion successfully but ultimately failed.
o Last logic block is a monitor/mem_check routine which should not be enabled.
o Printfs will show if jenkins exited the script early.
Change-Id: Ibfd113d555bdd4ce9a2035adba2da220d9cde980
diff --git a/jjb/pipeline/voltha/master/bbsim-tests.groovy b/jjb/pipeline/voltha/master/bbsim-tests.groovy
index 4df9a78..611c948 100644
--- a/jjb/pipeline/voltha/master/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/master/bbsim-tests.groovy
@@ -135,15 +135,15 @@
_TAG=kail-startup kail -n ${infraNamespace} -n ${volthaNamespace} > ${logsDir}/onos-voltha-startup-combined.log &
"""
- // if we're downloading a voltha-helm-charts patch, then install from a local copy of the charts
- Boolean localCharts = false
-
- if (volthaHelmChartsChange != ''
- || gerritProject == 'voltha-helm-charts'
- || isReleaseBranch(branch) // branch != 'master'
- ) {
- localCharts = true
- }
+ // if we're downloading a voltha-helm-charts patch, then install from a local copy of the charts
+ Boolean localCharts = false
+
+ if (volthaHelmChartsChange != ''
+ || gerritProject == 'voltha-helm-charts'
+ || isReleaseBranch(branch) // branch != 'master'
+ ) {
+ localCharts = true
+ }
String branchName = branchName()
Boolean is_release = isReleaseBranch(branch)
@@ -189,9 +189,8 @@
localCharts: localCharts,
bbsimReplica: olts.toInteger(),
dockerRegistry: registry,
- ])
+ ])
println('volthaDeploy: LEAVE')
-
} // script
// -----------------------------------------------------------------------
@@ -262,9 +261,10 @@
} // if (teardown)
} // stage('Deploy Voltha')
- stage('Run test ' + testTarget + ' on ' + workflow + ' workFlow')
+ stage("Run test ${testTarget} on workflow ${workFlow}")
{
sh """
+ echo -e "\n** Monitor using mem_consumption.py ?"
if [ ${withMonitoring} = true ] ; then
mkdir -p "$WORKSPACE/voltha-pods-mem-consumption-${workflow}"
cd "$WORKSPACE/voltha-system-tests"
@@ -276,6 +276,7 @@
"""
sh """
+ echo -e "\n** make testTarget=[${testTarget}]"
mkdir -p ${logsDir}
export ROBOT_MISC_ARGS="-d ${logsDir} ${params.extraRobotArgs} "
ROBOT_MISC_ARGS+="-v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace} -v container_log_dir:${logsDir} -v logging:${testLogging}"
@@ -287,14 +288,18 @@
getPodsInfo("${logsDir}")
sh """
+ echo -e '\n** Gather robot Framework logs: ENTER'
# set +e
# collect logs collected in the Robot Framework StartLogging keyword
cd ${logsDir}
gzip *-combined.log
rm -f *-combined.log
+
+ echo -e '** Gather robot Framework logs: LEAVE\n'
"""
sh """
+ echo -e '** Monitor pod-mem-consumption: ENTER'
if [ ${withMonitoring} = true ] ; then
cd "$WORKSPACE/voltha-system-tests"
make venv-activate-script
@@ -302,6 +307,7 @@
# Collect memory consumption of voltha pods once all the tests are complete
python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
fi
+ echo -e '** Monitor pod-mem-consumption: LEAVE\n'
"""
} // stage
@@ -329,8 +335,6 @@
sh '''
sync
[[ $(pgrep --count kail) -gt 0 ]] && pkill --echo kail
- which voltctl
- md5sum $(which voltctl)
'''
step([$class: 'RobotPublisher',
@@ -515,7 +519,9 @@
testLogging = 'True'
}
println "Executing test ${target} on workflow ${workflow} with logging ${testLogging} and extra flags ${flags}"
+ println "Executing test ${target}: ENTER"
execute_test(target, workflow, testLogging, teardown, flags)
+ println "Executing test ${target}: LEAVE"
} // for
} // script
} // steps
diff --git a/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy b/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
index 34bf472..900e1e1 100644
--- a/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
@@ -25,12 +25,12 @@
//------------------//
//---] GLOBAL [---//
//------------------//
-def clusterName = 'kind-ci'
+String clusterName = 'kind-ci' // was def
// -----------------------------------------------------------------------
// Intent:
// -----------------------------------------------------------------------
-String getBranchName() {
+String branchName() {
String name = 'voltha-2.12'
// [TODO] Sanity check the target branch
@@ -43,13 +43,13 @@
// Jenkins will re-write the call stack for serialization.
// -----------------------------------------------------------------------
String getIam(String func) {
- String branch_name = getBranchName()
+ String branchName = branchName()
String src = [
'ci-management',
'jjb',
'pipeline',
'voltha',
- branch_name,
+ branchName,
'bbsim-tests.groovy'
].join('/')
@@ -64,30 +64,30 @@
Boolean isReleaseBranch(String name)
{
// List modifiers = ['-dev', '-pre', 'voltha-x.y.z-pre']
- // if branch_name in modifiers
- return(name != 'master') // OR branch_name.contains('-')
+ // if branchName in modifiers
+ return(name != 'master') // OR branchName.contains('-')
}
// -----------------------------------------------------------------------
// Intent:
// -----------------------------------------------------------------------
-def execute_test(testTarget, workflow, testLogging, teardown, testSpecificHelmFlags = "")
+void execute_test(testTarget, workflow, testLogging, teardown, testSpecificHelmFlags='')
{
- def infraNamespace = "default"
- def volthaNamespace = "voltha"
- def logsDir = "$WORKSPACE/${testTarget}"
+ String infraNamespace = 'default'
+ String volthaNamespace = 'voltha'
+ String logsDir = "$WORKSPACE/${testTarget}"
stage('IAM')
{
script
{
// Announce ourselves for log usability
- String iam = getIam('execute_test')
+ String iam = getIam('execute_test')
println("${iam}: ENTER")
println("${iam}: LEAVE")
}
}
-
+
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
stage('Cleanup')
@@ -95,7 +95,7 @@
if (teardown) {
timeout(15) {
script {
- helmTeardown(["default", infraNamespace, volthaNamespace])
+ helmTeardown(['default', infraNamespace, volthaNamespace])
}
timeout(1) {
sh returnStdout: false, script: '''
@@ -135,61 +135,78 @@
_TAG=kail-startup kail -n ${infraNamespace} -n ${volthaNamespace} > ${logsDir}/onos-voltha-startup-combined.log &
"""
- // if we're downloading a voltha-helm-charts patch, then install from a local copy of the charts
- def localCharts = false
+ // if we're downloading a voltha-helm-charts patch, then install from a local copy of the charts
+ Boolean localCharts = false
+
+ if (volthaHelmChartsChange != ''
+ || gerritProject == 'voltha-helm-charts'
+ || isReleaseBranch(branch) // branch != 'master'
+ ) {
+ localCharts = true
+ }
+
+ String branchName = branchName()
+ Boolean is_release = isReleaseBranch(branch)
+ println([
+ " ** localCharts=${localCharts}",
+ "branchName=${branchName}",
+ "branch=${branch}",
+ "branch=isReleaseBranch=${is_release}",
+ ].join(', '))
- if (volthaHelmChartsChange != ""
- || gerritProject == "voltha-helm-charts"
- || isReleaseBranch(branch) // branch != 'master'
- ) {
- localCharts = true
- }
- String branch_name = getBranchName()
- Boolean is_release = isReleaseBranch(branch)
- println([
- " ** localCharts=${localCharts}",
- "branch_name=${branch_name}",
- "branch=${branch}",
- "branch=isReleaseBranch=${is_release}",
- ].join(', '))
+ // -----------------------------------------------------------------------
+ // Rewrite localHelmFlags using array join, moving code around and
+ // refactoring into standalone functions
+ // -----------------------------------------------------------------------
+ // hudson.remoting.ProxyException: groovy.lang.MissingMethodException:
+ // No signature of method: java.lang.String.positive() is applicable for argument types: () values: []
+ // -----------------------------------------------------------------------
+ // NOTE temporary workaround expose ONOS node ports
+ // -----------------------------------------------------------------------
+ String localHelmFlags = [
+ extraHelmFlags.trim(),
+ "--set global.log_level=${logLevel.toUpperCase()}",
+ '--set onos-classic.onosSshPort=30115',
+ '--set onos-classic.onosApiPort=30120',
+ '--set onos-classic.onosOfPort=31653',
+ '--set onos-classic.individualOpenFlowNodePorts=true',
+ testSpecificHelmFlags
+ ].join(' ')
- // NOTE temporary workaround expose ONOS node ports
- def localHelmFlags = extraHelmFlags.trim()
- + " --set global.log_level=${logLevel.toUpperCase()} "
- + " --set onos-classic.onosSshPort=30115 "
- + " --set onos-classic.onosApiPort=30120 "
- + " --set onos-classic.onosOfPort=31653 "
- + " --set onos-classic.individualOpenFlowNodePorts=true "
- + testSpecificHelmFlags
+ println("** localHelmFlags = ${localHelmFlags}")
- if (gerritProject != "") {
- localHelmFlags = "${localHelmFlags} " + getVolthaImageFlags("${gerritProject}")
- }
+ if (gerritProject != '') {
+ localHelmFlags = "${localHelmFlags} " + getVolthaImageFlags("${gerritProject}")
+ }
- volthaDeploy([
- infraNamespace: infraNamespace,
- volthaNamespace: volthaNamespace,
- workflow: workflow.toLowerCase(),
- withMacLearning: enableMacLearning.toBoolean(),
- extraHelmFlags: localHelmFlags,
- localCharts: localCharts,
- bbsimReplica: olts.toInteger(),
- dockerRegistry: registry,
- ])
- }
+ println('volthaDeploy: ENTER')
+ volthaDeploy([
+ infraNamespace: infraNamespace,
+ volthaNamespace: volthaNamespace,
+ workflow: workflow.toLowerCase(),
+ withMacLearning: enableMacLearning.toBoolean(),
+ extraHelmFlags: localHelmFlags,
+ localCharts: localCharts,
+ bbsimReplica: olts.toInteger(),
+ dockerRegistry: registry,
+ ])
+ println('volthaDeploy: LEAVE')
+ } // script
// -----------------------------------------------------------------------
// Intent: Replacing P_IDS with pgrep/pkill is a step forward.
// Why not simply use a pid file, capture _TAG=kail-startup above
// Grep runs the risk of terminating stray commands (??-good <=> bad-??)
- // -----------------------------------------------------------------------
- echo 'Try out pgrep/pkill commands'
- def stream = sh(
- returnStatus:false,
- returnStdout:true,
- script: '''pgrep --list-full kail-startup || true'''
- )
- println("** pgrep output: ${stream}")
+ // -----------------------------------------------------------------------
+ script {
+ println('Try out pgrep/pkill commands')
+ def stream = sh(
+ returnStatus:false,
+ returnStdout:true,
+ script: '''pgrep --list-full kail-startup || true'''
+ )
+ println("** pgrep output: ${stream}")
+ }
// -----------------------------------------------------------------------
// stop logging
@@ -225,8 +242,9 @@
// setting ONOS log level
script
- {
- setOnosLogLevels([
+ {
+ println('** setOnosLogLevels: ENTER')
+ setOnosLogLevels([
onosNamespace: infraNamespace,
apps: [
'org.opencord.dhcpl2relay',
@@ -238,13 +256,15 @@
],
logLevel: logLevel
])
+ println('** setOnosLogLevels: LEAVE')
} // script
} // if (teardown)
} // stage('Deploy Voltha')
- stage('Run test ' + testTarget + ' on ' + workflow + ' workFlow')
+ stage("Run test ${testTarget} on workflow ${workFlow}")
{
sh """
+ echo -e "\n** Monitor using mem_consumption.py ?"
if [ ${withMonitoring} = true ] ; then
mkdir -p "$WORKSPACE/voltha-pods-mem-consumption-${workflow}"
cd "$WORKSPACE/voltha-system-tests"
@@ -256,6 +276,7 @@
"""
sh """
+ echo -e "\n** make testTarget=[${testTarget}]"
mkdir -p ${logsDir}
export ROBOT_MISC_ARGS="-d ${logsDir} ${params.extraRobotArgs} "
ROBOT_MISC_ARGS+="-v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace} -v container_log_dir:${logsDir} -v logging:${testLogging}"
@@ -267,14 +288,18 @@
getPodsInfo("${logsDir}")
sh """
+ echo -e '\n** Gather robot Framework logs: ENTER'
# set +e
# collect logs collected in the Robot Framework StartLogging keyword
cd ${logsDir}
gzip *-combined.log
rm -f *-combined.log
+
+ echo -e '** Gather robot Framework logs: LEAVE\n'
"""
sh """
+ echo -e '** Monitor pod-mem-consumption: ENTER'
if [ ${withMonitoring} = true ] ; then
cd "$WORKSPACE/voltha-system-tests"
make venv-activate-script
@@ -282,14 +307,16 @@
# Collect memory consumption of voltha pods once all the tests are complete
python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
fi
+ echo -e '** Monitor pod-mem-consumption: LEAVE\n'
"""
} // stage
+
+ return
} // execute_test()
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
-def collectArtifacts(exitStatus)
-{
+def collectArtifacts(exitStatus) {
echo '''
** -----------------------------------------------------------------------
@@ -308,22 +335,23 @@
sh '''
sync
[[ $(pgrep --count kail) -gt 0 ]] && pkill --echo kail
- which voltctl
- md5sum $(which voltctl)
'''
step([$class: 'RobotPublisher',
disableArchiveOutput: false,
- logFileName: "**/*/log*.html",
+ logFileName: '**/*/log*.html',
otherFiles: '',
- outputFileName: "**/*/output*.xml",
+ outputFileName: '**/*/output*.xml',
outputPath: '.',
passThreshold: 100,
- reportFileName: "**/*/report*.html",
+ reportFileName: '**/*/report*.html',
unstableThreshold: 0,
onlyCritical: true]);
}
+// -----------------------------------------------------------------------
+// Intent: main
+// -----------------------------------------------------------------------
pipeline {
/* no label, executor is determined by JJB */
@@ -337,8 +365,8 @@
KUBECONFIG="$HOME/.kube/kind-${clusterName}"
VOLTCONFIG="$HOME/.volt/config"
PATH="$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
- DIAGS_PROFILE="VOLTHA_PROFILE"
- SSHPASS="karaf"
+ DIAGS_PROFILE='VOLTHA_PROFILE'
+ SSHPASS='karaf'
}
stages {
stage('Download Code') {
@@ -429,7 +457,7 @@
returnStdout: true,
script: """kind get clusters | grep "${clusterName}" | wc -l""")
- if (clusterExists.trim() == "0")
+ if (clusterExists.trim() == '0')
{
createKubernetesCluster([nodes: 3, name: clusterName])
}
@@ -444,7 +472,7 @@
// if the project is voltctl, override the downloaded one with the built one
when {
expression {
- return gerritProject == "voltctl"
+ return gerritProject == 'voltctl'
}
}
@@ -475,26 +503,28 @@
// -----------------------------------------------------------------------
stage('Parse and execute tests')
{
- steps {
- script {
- def tests = readYaml text: testTargets
+ steps {
+ script {
+ def tests = readYaml text: testTargets
- for(int i = 0;i<tests.size();i++) {
- def test = tests[i]
- def target = test["target"]
- def workflow = test["workflow"]
- def flags = test["flags"]
- def teardown = test["teardown"].toBoolean()
- def logging = test["logging"].toBoolean()
+ for(int i = 0;i<tests.size();i++) {
+ def test = tests[i]
+ def target = test['target']
+ def workflow = test['workflow']
+ def flags = test['flags']
+ def teardown = test['teardown'].toBoolean()
+ def logging = test['logging'].toBoolean()
def testLogging = 'False'
if (logging) {
testLogging = 'True'
}
println "Executing test ${target} on workflow ${workflow} with logging ${testLogging} and extra flags ${flags}"
+ println "Executing test ${target}: ENTER"
execute_test(target, workflow, testLogging, teardown, flags)
- }
- }
- }
+ println "Executing test ${target}: LEAVE"
+ } // for
+ } // script
+ } // steps
} // stage
} // stages