[VOL-5180] - installVolthaStack: enhance polling loop
vars/volthaStackDeploy.groovy
-----------------------------
o More status detection needed to avoid a race condition.
o Added comments to document state transitions.
o grep '0/' will only detect activity VS state.
o Detect states: active, not Created, (~all) Running.
o Display kubectl get pod output states are blocked on.
o Display get pod output at exit condition for good measure.
Change-Id: I4395d4957a2bf732a3ea32f941fdbedf35c97a1a
diff --git a/jjb/voltha-e2e/voltha-2.8.yaml b/jjb/voltha-e2e/voltha-2.8.yaml
index 912e530..994a79d 100644
--- a/jjb/voltha-e2e/voltha-2.8.yaml
+++ b/jjb/voltha-e2e/voltha-2.8.yaml
@@ -30,7 +30,7 @@
code-branch: 'voltha-2.8'
extraHelmFlags: '--set kafka.externalAccess.enabled=true,kafka.externalAccess.service.type=NodePort,kafka.externalAccess.service.nodePorts[0]=30201,kafka.externalAccess.service.domain=127.0.0.1'
time-trigger: "H H/23 * * *"
- logLevel: 'INFO'
+ logLevel: 'DEBUG'
testTargets: |
- target: voltha-pm-data-single-kind-att
workflow: att
diff --git a/vars/volthaStackDeploy.groovy b/vars/volthaStackDeploy.groovy
index d432e03..452fb7d 100644
--- a/vars/volthaStackDeploy.groovy
+++ b/vars/volthaStackDeploy.groovy
@@ -106,47 +106,84 @@
// -----------------------------------------------------------------------
// Intent: Wait until the pod completed, meaning ONOS fully deployed
// -----------------------------------------------------------------------
+// Todo: Move logic like this into a standalone script.
+// Use jenkins stash or native JJB logic to publish out to nodes.
+// -----------------------------------------------------------------------
void launchVolthaStack(Map cfg) {
enter('launchVolthaStack')
- sh(label : "Wait for VOLTHA Stack ${cfg.stackName}::${cfg.volthaNamespace} to start",
+ /* -----------------------------------------------------------------------
+ * % kubectl get pods
+ 17:40:15 bbsim0-868479698c-z66mk 0/1 ContainerCreating 0 6s
+ 17:40:15 voltha-voltha-adapter-openolt-68c84bf786-z98rh 0/1 Running 0 8s
+
+ * % kubectl port-forward --address 0.0.0.0
+ 17:40:15 error: unable to forward port because pod is not running. Current status=Pending
+ * -----------------------------------------------------------------------
+ */
+ sh(label : "Wait for VOLTHA Stack (stack=${cfg.stackName}, namespace=${cfg.volthaNamespace}) to start",
script : """
cat <<EOM
** -----------------------------------------------------------------------
-** Wait for VOLTHA Stack ${cfg.stackName}::${cfg.volthaNamespace} to start
+** Wait for VOLTHA Stack (stack=${cfg.stackName}, namespace=${cfg.volthaNamespace}) to start
** -----------------------------------------------------------------------
EOM
# set -euo pipefail
-set +x # # Noisy when commented (default: uncommented)
+set +x # # Logs are noisy when commented
declare -i count=0
+declare -i debug=1 # uncomment to enable debugging
+# declare -i verbose=1 # uncomment to enable debugging
vsd_log='volthaStackDeploy.tmp'
touch \$vsd_log
+
while true; do
- ## Exit when the server begins showing signs of life
- if grep -q '0/' \$vsd_log; then
- echo 'volthaStackDeploy.groovy: Detected kubectl pods =~ 0/'
- grep '0/' \$vsd_log
+ # Gather
+ kubectl get pods -n ${cfg.volthaNamespace} \
+ -l app.kubernetes.io/part-of=voltha --no-headers \
+ > \$vsd_log
+
+ count=\$((\$count - 1))
+
+ # Display activity every iteration ?
+ [[ -v verbose ]] && { count=0; }
+
+ # Display activity every minute or so {sleep(5) x count=10}
+ if [[ \$count -lt 1 ]]; then
+ count=10
+ cat \$vsd_log
+ fi
+
+ ## -----------------------
+ ## Probe for cluster state
+ ## -----------------------
+ if grep -q -e 'ContainerCreating' \$vsd_log; then
+ echo -e '\nvolthaStackDeploy.groovy: ContainerCrating active'
+ [[ -v debug ]] && grep -e 'ContainerCreating' \$vsd_log
+ elif grep -q -e '0/' \$vsd_log; then
+ echo -e '\nvolthaStackDeploy.groovy: Waiting for status=Running'
+ [[ -v debug ]] && grep -e '0/' \$vsd_log
+ elif ! grep -q '/' \$vsd_log; then
+ echo -e '\nvolthaStackDeploy.groovy: Waiting for initial pod activity'
+ [[ ! -v verbose ]] && { cat \$vsd_log; }
+ # -----------------------------------------------------------------------
+ # An extra conditon needed here but shell coding is tricky:
+ # "svc x/y Running 0 6s
+ # Verify (x == y) && (x > 0)
+ # Wait until job failure/we have an actual need for it.
+ # -----------------------------------------------------------------------
+ else
+ echo -e '\nvolthaStackDeploy.groovy: Voltha stack has launched'
+ [[ ! -v verbose ]] && { cat \$vsd_log; }
break
fi
+ ## Support argument --timeout (?)
sleep 5
- count=\$((\$count - 1))
-
- if [[ \$count -lt 1 ]]; then # [DEBUG] Display activity every minute or so
- count=10
- kubectl get pods -n ${cfg.volthaNamespace} \
- -l app.kubernetes.io/part-of=voltha --no-headers \
- | tee \$vsd_log
- else
- kubectl get pods -n ${cfg.volthaNamespace} \
- -l app.kubernetes.io/part-of=voltha --no-headers \
- > \$vsd_log
- fi
done
rm -f \$vsd_log