[VOL-5177] - verify_voltha-openolt-adapter_sanity-test failure
More debugging added for voltha-adapter-sanity-test.
https://jenkins.opencord.org/job/verify_voltha-openolt-adapter_sanity-test-voltha-2.12/17/console
vars/volthaStackDeploy.groovy
-----------------------------
o Last job failure was quiet, due to a timeout waiting for ONOS loading.
o ERR: Cancelling nested steps due to timeout
o Debugging added: display "kubectl get pods" early for error messages
and to see what is in the stream. We may need to split the 'wc -l'
one liner into separate commands to detect problems in the pipeline.
o Change getIam() return type from def to String so type cast problems
will become visible.
vars/waitForAdapter.groovy
--------------------------
o Replace several raw iam=getIam() calls with enter(msg), leave(msg).
o Replace raw pgrep/pkill with central debug library pkill_port_forward.groovy.
o Library logic can be replaced if/when we have port forwarding handled
by system services.
Change-Id: I5b07000c20e0bf2b9a12e87d39f384dbcc542a37
diff --git a/jjb/voltha-e2e/voltha-2.8.yaml b/jjb/voltha-e2e/voltha-2.8.yaml
index 994a79d..912e530 100644
--- a/jjb/voltha-e2e/voltha-2.8.yaml
+++ b/jjb/voltha-e2e/voltha-2.8.yaml
@@ -30,7 +30,7 @@
code-branch: 'voltha-2.8'
extraHelmFlags: '--set kafka.externalAccess.enabled=true,kafka.externalAccess.service.type=NodePort,kafka.externalAccess.service.nodePorts[0]=30201,kafka.externalAccess.service.domain=127.0.0.1'
time-trigger: "H H/23 * * *"
- logLevel: 'DEBUG'
+ logLevel: 'INFO'
testTargets: |
- target: voltha-pm-data-single-kind-att
workflow: att
diff --git a/vars/volthaStackDeploy.groovy b/vars/volthaStackDeploy.groovy
index fc7bbcd..e8390bb 100644
--- a/vars/volthaStackDeploy.groovy
+++ b/vars/volthaStackDeploy.groovy
@@ -17,7 +17,7 @@
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
-def getIam(String func)
+String getIam(String func)
{
// Cannot rely on a stack trace due to jenkins manipulation
String src = 'vars/volthaStackDeploy.groovy'
@@ -66,25 +66,25 @@
onosReplica: 1,
adaptersToWait: 2,
]
-
+
def cfg = defaultConfig + config
-
+
def volthaStackChart = "onf/voltha-stack"
def bbsimChart = "onf/bbsim"
-
+
if (cfg.localCharts) {
volthaStackChart = "$WORKSPACE/voltha-helm-charts/voltha-stack"
bbsimChart = "$WORKSPACE/voltha-helm-charts/bbsim"
-
+
sh """
pushd $WORKSPACE/voltha-helm-charts/voltha-stack
helm dep update
popd
"""
}
-
+
println "Deploying VOLTHA Stack with the following parameters: ${cfg}."
-
+
sh(label : "Create VOLTHA Stack ${cfg.stackName}, (namespace=${cfg.volthaNamespace})",
script : """
helm upgrade --install --create-namespace -n ${cfg.volthaNamespace} ${cfg.stackName} ${volthaStackChart} \
@@ -101,7 +101,7 @@
sh(label : "Create config[$i]: bbsimCfg${cfg.stackId}${i}.yaml",
script : "rm -f $WORKSPACE/bbsimCfg${cfg.stackId}${i}.yaml",
)
-
+
if (cfg.workflow == "att" || cfg.workflow == "tt") {
def startingStag = 900
def serviceConfigFile = cfg.workflow
@@ -120,7 +120,7 @@
"""
}
}
-
+
sh(label : "HELM: Create namespace=${cfg.volthaNamespace} bbsim${i}",
script : """
helm upgrade --install --create-namespace -n ${cfg.volthaNamespace} bbsim${i} ${bbsimChart} \
@@ -129,11 +129,18 @@
${cfg.extraHelmFlags}
""")
}
-
+
sh(label : "Wait for VOLTHA Stack ${cfg.stackName} to start",
script : """
-# set +x
+ # set +x # Noisy when uncommented
+cat << EOM
+
+** -----------------------------------------------------------------------
+** Wait for VOLTHA Stack ${cfg.stackName} to start
+** Display kubectl get pods prior to looping
+** -----------------------------------------------------------------------
+EOM
# [joey]: debug
kubectl get pods -n ${cfg.volthaNamespace} -l app.kubernetes.io/part-of=voltha --no-headers
@@ -142,8 +149,8 @@
sleep 5
voltha=\$(kubectl get pods -n ${cfg.volthaNamespace} -l app.kubernetes.io/part-of=voltha --no-headers | grep "0/" | wc -l)
done
- """)
-
+""")
+
waitForAdapters(cfg)
// also make sure that the ONOS config is loaded
@@ -152,17 +159,54 @@
// Wait until the pod completed, meaning ONOS fully deployed
sh(label : '"Wait for ONOS Config loader to fully deploy',
- script : """
-# set +x
- config=\$(kubectl get pods -l app=onos-config-loader -n ${cfg.infraNamespace} --no-headers --field-selector=status.phase=Running | grep "0/" | wc -l)
- while [[ \$config != 0 ]]; do
- sleep 5
- config=\$(kubectl get pods -l app=onos-config-loader -n ${cfg.infraNamespace} --no-headers --field-selector=status.phase=Running | grep "0/" | wc -l)
- done
- """)
+ script : """#!/bin/bash
+
+cat <<EOM
+
+** -----------------------------------------------------------------------
+** Wait for ONOS Config loader to fully deploy
+** IAM: vars/volthaStackDeploy.groovy
+** DBG: Polling loop initial kubectl get pods call
+** -----------------------------------------------------------------------
+** 17:06:07 Cancelling nested steps due to timeout
+** 17:06:07 Sending interrupt signal to process
+** 17:06:09 /w/workspace/verify_voltha-openolt-adapter_sanity-test-voltha-2.12@tmp/durable-18af2649/script.sh: line 7: 29716 Terminated sleep 5
+** 17:06:09 script returned exit code 143
+** -----------------------------------------------------------------------
+EOM
+
+# set -euo pipefail
+set +x# # Noisy when commented (default: uncommented)
+
+declare -i count=0
+vsd_log='volthaStackDeploy.tmp'
+touch \$vsd_log
+while true; do
+
+ ## Exit when the server begins showing signs of life
+ if grep -q '0/' \$vsd_log; then
+ echo 'volthaStackDeploy.groovy: Detected kubectl pods =~ 0/'
+ grep '0/' \$vsd_log
+ break
+ fi
+
+ sleep 5
+ count=\$((\$count - 1))
+
+ if [[ \$count -lt 1 ]]; then # [DEBUG] Display activity every minute or so
+ count=10
+ kubectl get pods -l app=onos-config-loader -n ${cfg.infraNamespace} --no-headers --field-selector=status.phase=Running \
+ | tee \$vsd_log
+ else
+ kubectl get pods -l app=onos-config-loader -n ${cfg.infraNamespace} --no-headers --field-selector=status.phase=Running> \$vsd_log
+ fi
+
+done
+rm -f \$vsd_log
+""")
leave('process')
-
+
return
}
@@ -171,7 +215,7 @@
def call(Map config=[:])
{
try
- {
+ {
enter('main')
process(config)
}
@@ -183,7 +227,6 @@
finally
{
leave('main')
- println("** ${iam}: LEAVE")
}
return
}
diff --git a/vars/waitForAdapters.groovy b/vars/waitForAdapters.groovy
index 92edf0d..5c9c5ce 100644
--- a/vars/waitForAdapters.groovy
+++ b/vars/waitForAdapters.groovy
@@ -47,7 +47,6 @@
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
def getAdapters() {
- String iam = getIam('getAdapters')
String adapters = ''
try {
@@ -66,7 +65,7 @@
adapters = 'SKIP' // why not just retry a few times (?)
}
- print("** ${iam}: returned $adapters")
+ leave("returned $adapters")
return adapters
}
@@ -149,8 +148,9 @@
: 'CONTINUE'
}
+
if (debug) {
- println("** ${iam} return: [$ans]")
+ leave("return: [$ans]")
}
return ans
} // getAdaptersState
@@ -229,14 +229,15 @@
}
println("** ${iam}: Tearing down port forwarding")
- // pgrep_port_forward-212
+ Map pkpfArgs =\
+ [
+ 'banner' : true, // display banner for logging
+ 'show_procs' : true, // display procs under consideration
+ 'filler' : true // fix conditional trailing comma
+ ]
- sh(label : 'waitForAdapters: Tear down port forwarding',
- script : """
-if [[ \$(pgrep --count 'port-forw') -gt 0 ]]; then
- pkill --uid "\$(id -u)" --echo --full 'port-forw'
-fi
-""")
+ // [TODO] 'kubectl.*port-forward'
+ pkill_port_forward('port-forward', pkpfArgs)
leave('process')
return
@@ -246,13 +247,12 @@
// -----------------------------------------------------------------------
def call(Map config=[:])
{
- String iam = getIam('main')
-
try {
enter('main')
process(config)
}
catch (Exception err) { // groovylint-disable-line CatchException
+ String iam = getIam('process')
println("** ${iam}: EXCEPTION ${err}")
throw err
}