VOL-5155 - triage failure in periodic-voltha-test-bbsim-2.12
vars/pgrep_proc.groovy
vars/pkill_proc.groovy
----------------------
o Pass arg --full to expand command matching to arguments.
o Include --uid to limit process consideration.
o Accept optional (Map args) parameter, will need more later.
jjb/pipeline/voltha/master/bbsim-tests.groovy
jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
--------------------------------------------------
o pgrep/pkill cleanup -- kubectl is the command to target for cleanup.
o port-forward is a command parameter and was used to filter tasks.
o pattern to display/terminate is 'kubectl.*port-forward'
Change-Id: I34b473a4e1c7fcd488910be62f458319d1499254
diff --git a/jjb/pipeline/voltha/master/bbsim-tests.groovy b/jjb/pipeline/voltha/master/bbsim-tests.groovy
index 46cb252..af5c554 100644
--- a/jjb/pipeline/voltha/master/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/master/bbsim-tests.groovy
@@ -130,10 +130,12 @@
script {
enter('Cleanup')
// remove orphaned port-forward from different namespaces
- String proc = 'port-forw'
+ String proc = 'kubectl .*port-forward' // was 'port-forw'
pgrep_proc(proc)
pkill_proc(proc)
- pgrep_proc(proc) // proc count == 0
+
+ // todo: fatal unless (proc count==0)
+ pgrep_proc(proc)
enter('Cleanup')
} // script
} // timeout
@@ -300,6 +302,7 @@
// ---------------------------------
script {
enter('port-forward check')
+ // String proc = 'kubectl.*port-forward' // was 'port-forward'
String proc = 'port-forward'
println("Display spawned ${proc}")
pgrep_proc(proc)
diff --git a/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy b/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
index d97877d..a6ce2f2 100644
--- a/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
@@ -130,10 +130,12 @@
script {
enter('Cleanup')
// remove orphaned port-forward from different namespaces
- String proc = 'port-forw'
+ String proc = 'kubectl .*port-forward' // was 'port-forw'
pgrep_proc(proc)
pkill_proc(proc)
- pgrep_proc(proc) // proc count == 0
+
+ // todo: fatal unless (proc count==0)
+ pgrep_proc(proc)
enter('Cleanup')
} // script
} // timeout
@@ -300,6 +302,7 @@
// ---------------------------------
script {
enter('port-forward check')
+ // String proc = 'kubectl.*port-forward' // was 'port-forward'
String proc = 'port-forward'
println("Display spawned ${proc}")
pgrep_proc(proc)
@@ -330,7 +333,9 @@
// -----------------------------------------------------------------------
stage("Run test ${testTarget} on workflow ${workflow}")
{
- sh """
+ sh(
+ label : 'Monitor using mem_consumption.py',
+ script : """
echo -e "\n** Monitor using mem_consumption.py ?"
if [ ${withMonitoring} = true ] ; then
@@ -352,7 +357,7 @@
fi
echo -e '** Monitor memory consumption: LEAVE\n'
- """
+ """)
sh """
echo -e "\n** make testTarget=[${testTarget}]"
@@ -366,7 +371,9 @@
getPodsInfo("${logsDir}")
- sh """
+ sh(
+ label : 'Gather robot Framework logs',
+ script : """
echo -e '\n** Gather robot Framework logs: ENTER'
# set +e
# collect logs collected in the Robot Framework StartLogging keyword
@@ -375,11 +382,13 @@
rm -f *-combined.log
echo -e '** Gather robot Framework logs: LEAVE\n'
- """
+ """)
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
- sh """
+ sh(
+ label : 'Monitor pod-mem-consumption',
+ script : """
echo -e '** Monitor pod-mem-consumption: ENTER'
if [ ${withMonitoring} = true ] ; then
cat <<EOM
@@ -398,7 +407,7 @@
python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
fi
echo -e '** Monitor pod-mem-consumption: LEAVE\n'
- """
+ """)
} // stage
return
@@ -595,7 +604,7 @@
// Announce ourselves for log usability
enter('Parse and execute tests')
- def tests = readYaml text: testTargets
+ def tests = readYaml text: testTargets // typeof == Map (?)
println("** [DEBUG]: tests=$tests")
// Display expected tests for times when output goes dark
diff --git a/vars/pgrep_proc.groovy b/vars/pgrep_proc.groovy
index 1ad5dfc..9559ce6 100644
--- a/vars/pgrep_proc.groovy
+++ b/vars/pgrep_proc.groovy
@@ -28,14 +28,21 @@
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
-Boolean process(String proc) {
- String iam = getIam('process')
+Boolean process(String proc, Map args) {
Boolean ans = true
+ String iam = getIam('process')
- println("** ${iam}: ENTER")
+ if (args.containsKey('debug')) {
+ println("** $iam [DEBUG]: proc=[$proc], args=[$args]")
+ }
- String cmdFull = "pgrep --list-full '${proc}'"
- String cmd = """if [[ \$(pgrep --count "${proc}") -gt 0 ]]; then ${cmdFull}; fi"""
+ String cmd = [
+ 'pgrep',
+ '--uid', '$(id -u)', // no stray signals
+ '--list-full',
+ '--full', // hmmm: conditional use (?)
+ "'${proc}",
+ ]
print("""
** -----------------------------------------------------------------------
@@ -46,8 +53,6 @@
label : 'pgrep_proc', // jenkins usability: label log entry 'step'
script : "${cmd}",
)
-
- println("** ${iam}: LEAVE")
return(ans)
}
@@ -56,22 +61,38 @@
// o Paramter branch is passed but not yet used.
// o Installer should be release friendly and checkout a frozen version
// -----------------------------------------------------------------------
-def call(String proc) {
+// groovylint-disable-next-line None, UnusedMethodParameter
+Boolean call\
+(
+ String proc, // name of process or arguments to terminate
+ Map args=[:],
+ Boolean filler=True // Groovy, why special case list comma handling (?)
+) {
String iam = getIam('main')
+ Boolean ans = True
println("** ${iam}: ENTER")
try {
- process(proc)
+ process(proc, args)
}
- catch (Exception err) {
+ catch (Exception err) { // groovylint-disable-line CatchException
+ ans = False
println("** ${iam}: EXCEPTION ${err}")
throw err
}
finally {
println("** ${iam}: LEAVE")
}
- return
+
+ return(ans)
}
+// -----------------------------------------------------------------------
+// [TODO] - Combine pkill_proc and pgrep_proc
+// - Usage: do_proc(pkill=true, pgrep=true, args='proc-forward', cmd='kubectl'
+// o When kill == grep == true: display procs, terminate, recheck: fatal if procs detected
+// o cmd && args (or command containing args) (or list of patterns passed)
+// - pass arg --full to match entire command line.
+// -----------------------------------------------------------------------
// [EOF]
diff --git a/vars/pkill_proc.groovy b/vars/pkill_proc.groovy
index 753a1f1..0166df9 100644
--- a/vars/pkill_proc.groovy
+++ b/vars/pkill_proc.groovy
@@ -28,13 +28,28 @@
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
-Boolean process(String proc) {
- String iam = getIam('process')
+Boolean process(String proc, Map args) {
Boolean ans = true
+ String iam = getIam('process')
- println("** ${iam}: ENTER")
+ if (args.containsKey('debug')) {
+ println("** $iam [DEBUG]: proc=[$proc], args=[$args]")
+ }
- String cmdKill = "pkill --echo '${proc}'"
+ String cmdKill = [
+ 'pkill',
+ '--uid', '$(id -u)', // no stray signals
+ '--echo',
+ '--full', // hmmm: conditional use (?)
+ "'${proc}",
+ ].join(' ')
+
+ /*
+ String cmdKill = (args['command_only')
+ ? "pkill --echo '${proc}'"
+ : "pkill --echo --full '${proc}'"
+ */
+
String cmd = """if [[ \$(pgrep --count "${proc}") -gt 0 ]]; then ${cmdKill}; fi"""
print("""
@@ -48,7 +63,6 @@
script : "${cmd}",
)
- println("** ${iam}: LEAVE")
return(ans)
}
@@ -57,22 +71,44 @@
// o Paramter branch is passed but not yet used.
// o Installer should be release friendly and checkout a frozen version
// -----------------------------------------------------------------------
-def call(String proc) {
+// groovylint-disable-next-line None, UnusedMethodParameter
+void call\
+(
+ String proc, // name of process or arguments to terminate
+ Map args=[:],
+ // Groovy, why special case list comma handling (?)
+ Boolean filler=True // groovylint-disable-line UnusedMethodParameter
+) {
+
String iam = getIam('main')
+ Boolean ans = True
println("** ${iam}: ENTER")
try {
- process(proc)
+ // Limit process matching by default
+ if (! mymap.containsKey('command_only')) {
+ mymap['command_only'] = True
+ }
+ process(proc, args)
}
- catch (Exception err) {
+ catch (Exception err) { // groovylint-disable-line CatchException
+ ans = False
println("** ${iam}: EXCEPTION ${err}")
throw err
}
finally {
println("** ${iam}: LEAVE")
}
- return
+
+ return(ans)
}
+// -----------------------------------------------------------------------
+// [TODO] - Combine pkill_proc and pgrep_proc
+// - Usage: do_proc(pkill=true, pgrep=true, args='proc-forward', cmd='kubectl'
+// o When kill == grep == true: display procs, terminate, recheck: fatal if procs detected
+// o cmd && args (or command containing args) (or list of patterns passed)
+// - pass arg --full to match entire command line.
+// -----------------------------------------------------------------------
// [EOF]