[VOL-5170] - Test debugging openolt-adapter-sanity-test-voltha
jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
--------------------------------------------------
o More debugging added around Create K8s Cluster, port forwarding failure.
o sh(label:'foo') more shell commands.
o Replace inlined pgrep/pkill commands with vars/p{grep,kill} scripts calls.
o Refactor port-forward cleanup logic into a named function.
o job post() routines augmented to call port cleanup function.
vars/pkill_port_forward.groovy
------------------------------
o Common port-forward script logic to display and kill procs if running.
Change-Id: Icc9d9dcbafd376926b95265bf2574b0dfd53baa8
diff --git a/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy b/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
index b3105c3..bc082a0 100644
--- a/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
@@ -18,8 +18,8 @@
// [TODO] Update syntax below to the latest supported
library identifier: 'cord-jenkins-libraries@master',
retriever: modernSCM([
- $class: 'GitSCMSource',
- remote: 'https://gerrit.opencord.org/ci-management.git'
+ $class: 'GitSCMSource',
+ remote: 'https://gerrit.opencord.org/ci-management.git'
])
//------------------//
@@ -28,14 +28,22 @@
String clusterName = 'kind-ci'
// -----------------------------------------------------------------------
-// Intent:
+// Intent: Return branch name for the script. A hardcoded value is used
+// as a guarantee release jobs are running in an expected sandbox.
// -----------------------------------------------------------------------
String branchName() {
- String name = 'voltha-2.12'
+ String br = 'voltha-2.12'
- // [TODO] Sanity check the target branch
- // if (name != jenkins.branch) { fatal }
- return(name)
+ // "${branch}" is assigned by jenkins
+ if (br != branch) {
+ String err = [
+ 'ERROR: Detected invalid branch',
+ '(expected=[$br] != found=[$branch])'
+ ].join(' ')
+ throw new Exception(err) // groovylint-disable-line CatchException
+ }
+
+ return (br)
}
// -----------------------------------------------------------------------
@@ -43,8 +51,8 @@
// regenerated. Hardcode a version string that can be assigned
// per-script to be sure latest repository changes are being used.
// -----------------------------------------------------------------------
-String pipelineVer() {
- String version = '45c11f80698d3be1416a86d2872d6e25aa24baa8'
+String pipelineVer() {
+ String version = '2563719757ee52af49cf9da3fe76ed4bb6877588'
return(version)
}
@@ -101,6 +109,25 @@
}
// -----------------------------------------------------------------------
+// Intent: Terminate orphaned port-forward from different namespaces
+// -----------------------------------------------------------------------
+void cleanupPortForward() {
+ enter('cleanupPortForward')
+
+ Map pkpfArgs =\
+ [
+ 'banner' : true, // display banner for logging
+ 'show_procs' : true, // display procs under consideration
+ 'filler' : true // fix conditional trailing comma
+ ]
+
+ // 'kubectl.*port-forward'
+ pkill_port_forward('port-forward', pkpfArgs)
+ leave('cleanupPortForward')
+ return
+}
+
+// -----------------------------------------------------------------------
// Intent: Iterate over a list of test suites and invoke.
// -----------------------------------------------------------------------
void execute_test\
@@ -118,10 +145,10 @@
// -----------------------------------------------------------------------
// Intent: Cleanup stale port-forwarding
// -----------------------------------------------------------------------
- stage('Cleanup') {
- if (teardown) {
+ stage('Cleanup') {
+ if (teardown) {
timeout(15) {
- script {
+ script {
helmTeardown(['default', infraNamespace, volthaNamespace])
}
} // timeout
@@ -129,39 +156,12 @@
timeout(5) {
script {
enter('Cleanup')
-
- // remove orphaned port-forward from different namespaces
- // String proc = 'kubectl.*port-forward' // was 'port-forw'
-
- /* Pass Gstring args to sh() having problems
- implicit cast from GString to java.lang.String unsupported
- pgrep_proc(proc)
- pkill_proc(proc)
- pgrep_proc(proc) // todo: fatal unless (proc count==0)
- */
-
- sh(label : 'pgrep_proc - kill-pre',
- script : """
-pgrep --uid "\$(id -u)" --list-full --full 'kubectl.*port-forward' || true
-""")
-
- sh(label : 'pkill_proc - kubectl.*port-forward',
- script : """
-if [[ \$(pgrep --count 'kubectl.*port-forward') -gt 0 ]]; then
- pkill --uid "\$(id -u)" --echo --full 'kubectl.*port-forward'
-fi
-""")
-
- sh(label : 'pgrep_proc - kill-post',
- script : """
-pgrep --uid "\$(id -u)" --list-full --full 'kubectl.*port-forward' || true
-""")
-
+ cleanupPortForward()
leave('Cleanup')
} // script
} // timeout
} // teardown
- } // stage('Cleanup')
+ }// stage('Cleanup')
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
@@ -204,18 +204,17 @@
script {
String iam = getIam('Deploy Voltha')
String onosLog = "${logsDir}/onos-voltha-startup-combined.log"
- sh("""
- mkdir -p "$logsDir"
- touch "$onosLog"
- echo "** kail-startup ENTER: \$(date)" > "$onosLog"
- # Intermixed output (tee -a &) may get conflusing but let(s) see
- # what messages are logged during startup.
- # _TAG=ka7il-startup kail -n ${infraNamespace} -n ${volthaNamespace} >> "$onosLog" &
- _TAG=kail-startup kail -n ${infraNamespace} -n ${volthaNamespace} | tee -a "$onosLog" &
- """)
+ sh(label : 'Launch kail-startup',
+ script : """
+mkdir -p "$logsDir"
+touch "$onosLog"
- // if we're downloading a voltha-helm-charts patch, then install from a local copy of the charts
+_TAG=kail-startup kail -n ${infraNamespace} -n ${volthaNamespace} > "$onosLog" &
+""")
+
+ // if we're downloading a voltha-helm-charts patch,
+ // install from a local copy of the charts
Boolean localCharts = false
if (volthaHelmChartsChange != ''
@@ -253,10 +252,10 @@
println("** ${iam} localHelmFlags = ${localHelmFlags}")
if (gerritProject != '') {
- localHelmFlags = "${localHelmFlags} " + getVolthaImageFlags("${gerritProject}")
+ localHelmFlags += getVolthaImageFlags(gerritProject)
}
- println("** ${iam}: ENTER")
+ enter('volthaDeploy')
volthaDeploy([
infraNamespace: infraNamespace,
volthaNamespace: volthaNamespace,
@@ -267,65 +266,47 @@
bbsimReplica: olts.toInteger(),
dockerRegistry: registry,
])
- println("** ${iam}: LEAVE")
+ leave('volthaDeploy')
} // script
- // -----------------------------------------------------------------------
- // Intent: Replacing P_IDS with pgrep/pkill is a step forward.
- // Why not simply use a pid file, capture _TAG=kail-startup above
- // Grep runs the risk of terminating stray commands (??-good <=> bad-??)
- // -----------------------------------------------------------------------
- script {
- String proc = '_TAG=kail-startup'
+ script { pgrep_port_forward() }
- println("${iam}: ENTER")
- println("${iam}: Shutdown process $proc")
- /*
- pgrep_proc(proc)
- pkill_proc(proc)
- pgrep_proc(proc)
- */
-
- sh(label : 'pgrep_proc - kill-pre',
- script : """
-pgrep --uid "\$(id -u)" --list-full --full '_TAG=kail-startup' || true
-""")
-
- sh(label : 'pkill_proc - _TAG=kail-startup',
- script : """
+ sh(label : 'Terminate kail-startup',
+ script : """
if [[ \$(pgrep --count '_TAG=kail-startup') -gt 0 ]]; then
- pkill --uid "\$(id -u)" --echo --full '_TAG=kail-startup'
+ pkill --uid \$(uid -u) --echo --list-full --full '_TAG=kail-startup'
fi
""")
- sh(label : 'pgrep_proc - kill-post',
- script : """
-pgrep --uid "\$(id -u)" --list-full --full '_TAG=kail-startup' || true
+ sh(label : 'Lingering kail-startup check',
+ script : """
+pgrep --uid \$(uid -u) --list-full --full 'kail-startup' || true
""")
-
- println("${iam}: LEAVE")
- }
// -----------------------------------------------------------------------
// Bundle onos-voltha / kail logs
// -----------------------------------------------------------------------
- sh("""
+ sh(
+ label : 'Bundle logs: onos-voltha-startup-combined',
+ script : """
cat <<EOM
** -----------------------------------------------------------------------
** Combine and compress voltha startup log(s)
** -----------------------------------------------------------------------
EOM
- pushd "${logsDir}" || { echo "ERROR: pushd $logsDir failed"; exit 1; }
- gzip -k onos-voltha-startup-combined.log
- rm onos-voltha-startup-combined.log
- popd
+
+pushd "${logsDir}" || { echo "ERROR: pushd $logsDir failed"; exit 1; }
+gzip -k onos-voltha-startup-combined.log
+rm onos-voltha-startup-combined.log
+popd || { echo "ERROR: popd $logsDir failed"; exit 1; }
""")
- } // timeout(10)
+ } // timeout(10)
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
- sh """
+ sh(label : 'while-true-port-forward',
+ """
JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-voltha-api" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/voltha-voltha-api 55555:55555; done"&
JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-etcd" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-etcd 2379:2379; done"&
JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-kafka" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-kafka 9092:9092; done"&
@@ -338,21 +319,12 @@
JENKINS_NODE_COOKIE="dontKillMe" _TAG="nem-monitoring-prometheus-server" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n default svc/nem-monitoring-prometheus-server 31301:80; done"&
fi
# ps aux | grep port-forward
- """
+""")
// ---------------------------------
// Sanity check port-forward spawned
// ---------------------------------
- script {
- enter('Display port-forward procs')
- // String proc = 'kubectl.*port-forward' // was 'port-forward'
- String proc = 'port-forward'
- println("Display spawned ${proc}")
- sh(label : 'pgrep_proc - check',
- script : """
-pgrep --uid "\$(id -u)" --list-full --full "port-forward" || true
-""")
- leave('Display port-forward procs')
- }
+ // [TODO] - Wait until forwarding successful else fatal
+ script { pgrep_port_forward() }
// setting ONOS log level
script {
@@ -376,90 +348,93 @@
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
- stage("Run test ${testTarget} on workflow ${workflow}")
- {
+ stage("Run test ${testTarget} on workflow ${workflow}") {
sh(
label : 'Monitor using mem_consumption.py',
script : """
- echo -e "\n** Monitor using mem_consumption.py ?"
+echo -e "\n** Monitor using mem_consumption.py ?"
- if [ ${withMonitoring} = true ] ; then
- cat <<EOM
+if [ ${withMonitoring} = true ] ; then
+ cat <<EOM
** -----------------------------------------------------------------------
** Monitoring memory usage with mem_consumption.py
** -----------------------------------------------------------------------
EOM
- mkdir -p "$WORKSPACE/voltha-pods-mem-consumption-${workflow}"
- cd "$WORKSPACE/voltha-system-tests"
+ mkdir -p "$WORKSPACE/voltha-pods-mem-consumption-${workflow}"
+ cd "$WORKSPACE/voltha-system-tests"
- echo '** Installing python virtualenv'
- make venv-activate-patched
+ echo '** Installing python virtualenv'
+ make venv-activate-patched
- set +u && source .venv/bin/activate && set -u
- # Collect initial memory consumption
- python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
- fi
+ # Collect initial memory consumption
+ set +u && source .venv/bin/activate && set -u
+ python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
+fi
- echo -e '** Monitor memory consumption: LEAVE\n'
- """)
+echo -e '** Monitor memory consumption: LEAVE\n'
+""")
- sh """
- echo -e "\n** make testTarget=[${testTarget}]"
- mkdir -p ${logsDir}
- export ROBOT_MISC_ARGS="-d ${logsDir} ${params.extraRobotArgs} "
- ROBOT_MISC_ARGS+="-v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace} -v container_log_dir:${logsDir} -v logging:${testLogging}"
- export KVSTOREPREFIX=voltha/voltha_voltha
+ sh(
+ label : "make testTarget=[${testTarget}]",
+ script : """
+echo -e "\n** make testTarget=[${testTarget}]"
+mkdir -p ${logsDir}
+export ROBOT_MISC_ARGS="-d ${logsDir} ${params.extraRobotArgs} "
+ROBOT_MISC_ARGS+="-v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace} -v container_log_dir:${logsDir} -v logging:${testLogging}"
+export KVSTOREPREFIX=voltha/voltha_voltha
- make -C "$WORKSPACE/voltha-system-tests" ${testTarget}
- """
+make -C "$WORKSPACE/voltha-system-tests" ${testTarget}
+""")
getPodsInfo("${logsDir}")
+ // [TODO] make conditional, bundle when logs are available
sh(
label : 'Gather robot Framework logs',
script : """
- echo -e '\n** Gather robot Framework logs: ENTER'
+echo -e '\n** Gather robot Framework logs: ENTER'
- # set +e
- # collect logs collected in the Robot Framework StartLogging keyword
- cd "${logsDir}"
+# set +e
+# collect logs collected in the Robot Framework StartLogging keyword
+cd "${logsDir}"
- echo "** Available logs:"
- /bin/ls -l "$logsDir"
- echo
+echo "** Available logs:"
+/bin/ls -l "$logsDir"
+echo
- echo '** Bundle combined log'
- gzip *-combined.log || true
- rm -f *-combined.log || true
+echo '** Bundle combined log'
+gzip *-combined.log || true
+rm -f *-combined.log || true
- echo -e '** Gather robot Framework logs: LEAVE\n'
- """)
+echo -e '** Gather robot Framework logs: LEAVE\n'
+""")
- // -----------------------------------------------------------------------
- // -----------------------------------------------------------------------
+ // -----------------------------------------------------------------------
+ // -----------------------------------------------------------------------
sh(
label : 'Monitor pod-mem-consumption',
script : """
- echo -e '** Monitor pod-mem-consumption: ENTER'
- if [ ${withMonitoring} = true ] ; then
+echo -e '** Monitor pod-mem-consumption: ENTER'
+if [ ${withMonitoring} = true ] ; then
cat <<EOM
** -----------------------------------------------------------------------
** Monitoring pod-memory-consumption using mem_consumption.py
** -----------------------------------------------------------------------
EOM
- cd "$WORKSPACE/voltha-system-tests"
- echo '** Installing python virtualenv'
- make venv-activate-patched
+cd "$WORKSPACE/voltha-system-tests"
- set +u && source .venv/bin/activate && set -u
- # Collect memory consumption of voltha pods once all the tests are complete
- python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
- fi
- echo -e '** Monitor pod-mem-consumption: LEAVE\n'
- """)
+echo '** Installing python virtualenv'
+make venv-activate-patched
+
+# Collect memory consumption of voltha pods once all the tests are complete
+set +u && source .venv/bin/activate && set -u
+python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
+fi
+echo -e '** Monitor pod-mem-consumption: LEAVE\n'
+""")
} // stage
return
@@ -467,7 +442,7 @@
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
-def collectArtifacts(exitStatus) {
+void collectArtifacts(exitStatus) {
script {
String iam = getIam('collectArtifacts')
println("${iam}: ENTER (exitStatus=${exitStatus})")
@@ -491,8 +466,8 @@
script {
println("${iam}: ENTER")
/*
- pgrep_proc('kail-startup')
- pkill_proc('kail')
+ pgrep_proc('kail-startup')
+ pkill_proc('kail')
*/
sh(label : 'pgrep_proc - kill-pre',
script : """
@@ -614,6 +589,7 @@
steps {
script {
def clusterExists = sh(
+ label : 'Create K8s Cluster',
returnStdout: true,
script: """kind get clusters | grep "${clusterName}" | wc -l""")
@@ -723,9 +699,18 @@
post
{
- aborted { collectArtifacts('aborted') }
- failure { collectArtifacts('failed') }
- always { collectArtifacts('always') }
+ aborted {
+ collectArtifacts('aborted')
+ script{ cleanupPortForward() }
+ }
+ failure {
+ collectArtifacts('failed')
+ script{ cleanupPortForward() }
+ }
+ always {
+ collectArtifacts('always')
+ script{ cleanupPortForward() }
+ }
}
} // pipeline
diff --git a/vars/pkill_port_forward.groovy b/vars/pkill_port_forward.groovy
new file mode 100644
index 0000000..8b2be6f
--- /dev/null
+++ b/vars/pkill_port_forward.groovy
@@ -0,0 +1,149 @@
+#!/usr/bin/env groovy
+// -----------------------------------------------------------------------
+// Copyright 2023 Open Networking Foundation (ONF) and the ONF Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// -----------------------------------------------------------------------
+// Install the voltctl command by branch name "voltha-xx"
+// -----------------------------------------------------------------------
+
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+String getIam(String func) {
+ // Cannot rely on a stack trace due to jenkins manipulation
+ String src = 'vars/pkill_port_forward.groovy'
+ String iam = [src, func].join('::')
+ return iam
+}
+
+// -----------------------------------------------------------------------
+// Intent: Log progress message
+// -----------------------------------------------------------------------
+void enter(String name) {
+ // Announce ourselves for log usability
+ String iam = getIam(name)
+ println("${iam}: ENTER")
+ return
+}
+
+// -----------------------------------------------------------------------
+// Intent: Log progress message
+// -----------------------------------------------------------------------
+void leave(String name) {
+ // Announce ourselves for log usability
+ String iam = getIam(name)
+ println("${iam}: LEAVE")
+ return
+}
+
+// -----------------------------------------------------------------------
+// Intent: Terminate a process by name.
+// -----------------------------------------------------------------------
+// Note: Due to an exception casting GString to java.lang.string:
+// - Used for parameterized construction of a command line with args
+// - Passed to jenkins sh("${cmd}")
+// - Command line invoked is currently hardcoded.
+// -----------------------------------------------------------------------
+Boolean process(String proc, Map args) {
+ Boolean ans = true
+ String iam = getIam('process')
+
+ println("** ${iam}: args passed: ${args}")
+
+ String cmd = [
+ 'pkill',
+ '--uid', '$(id -u)', // no stray signals
+ '--list-full',
+ '--full', // hmmm: conditional use (?)
+ "'${proc}",
+ ]
+
+ if (args['banner']) {
+ print("""
+** -----------------------------------------------------------------------
+** Running: $cmd
+** -----------------------------------------------------------------------
+""")
+ }
+
+ if (args['show_procs']) {
+ sh(
+ label : 'Display port forwarding (pre-pgrep-pkill)',
+ script : """
+pgrep --uid \$(uid -u) --list-full --full 'port-forw'
+""")
+ }
+
+ sh(
+ label : 'Display port forwarding',
+ // script : ${cmd}.toString(), -> Exception
+ script : """
+echo -e "\n** ${iam} [DEBUG]: pgrep-pkill check"
+if [[ \$(pgrep --count 'port-forw') -gt 0 ]]; then
+ pkill --uid \$(uid -u) --echo --list-full --full 'port-forw'
+fi
+""")
+
+ return(ans)
+}
+
+// -----------------------------------------------------------------------
+// Install: Display a list of port-forwarding processes.
+// -----------------------------------------------------------------------
+// groovylint-disable-next-line None, UnusedMethodParameter
+Boolean call\
+(
+ String proc, // name of process or arguments to terminate
+ Map args=[:],
+ Boolean filler = true // Groovy, why special case list comma handling (?)
+) {
+ Boolean ans = true
+
+ try {
+ enter('main')
+
+ // Assign defaults
+ ['banner', 'show_procs'].each{ key->
+ if (!args.containsKey(key)) {
+ args[key] = true
+ }
+ }
+
+ process(proc, args)
+ }
+ catch (Exception err) { // groovylint-disable-line CatchException
+ ans = false
+ println("** ${iam}: EXCEPTION ${err}")
+ throw err
+ }
+ finally {
+ enter('main')
+ }
+
+ return(ans)
+}
+
+/* groovylint-disable */
+
+// [SEE ALSO]
+// -----------------------------------------------------------------------
+// o String cmd = [ ... ].join('') -- GString cannot cast to java.String
+// o https://stackoverflow.com/questions/60304068/artifactory-in-jenkins-pipeline-org-codehaus-groovy-runtime-gstringimpl-cannot
+// -----------------------------------------------------------------------
+// [TODO] - Combine pkill_proc and pkill_proc
+// - Usage: do_proc(pkill=true, pkill=true, args='proc-forward', cmd='kubectl'
+// o When kill == grep == true: display procs, terminate, recheck: fatal if procs detected
+// o cmd && args (or command containing args) (or list of patterns passed)
+// - pass arg --full to match entire command line.
+// -----------------------------------------------------------------------
+// [EOF]