Merge "voltha-e2e voltha-scale software-upgrades: use scm and branched scripts"
diff --git a/vars/volthaStackDeploy.groovy b/vars/volthaStackDeploy.groovy
index d432e03..9b198e5 100644
--- a/vars/volthaStackDeploy.groovy
+++ b/vars/volthaStackDeploy.groovy
@@ -106,50 +106,102 @@
// -----------------------------------------------------------------------
// Intent: Wait until the pod completed, meaning ONOS fully deployed
// -----------------------------------------------------------------------
+// Todo: Move logic like this into a standalone script.
+// Use jenkins stash or native JJB logic to publish out to nodes.
+// -----------------------------------------------------------------------
void launchVolthaStack(Map cfg) {
enter('launchVolthaStack')
- sh(label : "Wait for VOLTHA Stack ${cfg.stackName}::${cfg.volthaNamespace} to start",
+ /* -----------------------------------------------------------------------
+ * % kubectl get pods
+ 17:40:15 bbsim0-868479698c-z66mk 0/1 ContainerCreating 0 6s
+ 17:40:15 voltha-voltha-adapter-openolt-68c84bf786-z98rh 0/1 Running 0 8s
+
+ * % kubectl port-forward --address 0.0.0.0
+ 17:40:15 error: unable to forward port because pod is not running. Current status=Pending
+ * -----------------------------------------------------------------------
+ */
+ sh(label : "Wait for VOLTHA Stack (stack=${cfg.stackName}, namespace=${cfg.volthaNamespace}) to start",
script : """
cat <<EOM
** -----------------------------------------------------------------------
-** Wait for VOLTHA Stack ${cfg.stackName}::${cfg.volthaNamespace} to start
+** Wait for VOLTHA Stack (stack=${cfg.stackName}, namespace=${cfg.volthaNamespace}) to start
** -----------------------------------------------------------------------
EOM
# set -euo pipefail
-set +x # # Noisy when commented (default: uncommented)
+set +x # # Logs are noisy when commented
declare -i count=0
+declare -i debug=1 # uncomment to enable debugging
+# declare -i verbose=1 # uncomment to enable debugging
vsd_log='volthaStackDeploy.tmp'
touch \$vsd_log
+
+declare -i rc=0 # exit status
while true; do
- ## Exit when the server begins showing signs of life
- if grep -q '0/' \$vsd_log; then
- echo 'volthaStackDeploy.groovy: Detected kubectl pods =~ 0/'
- grep '0/' \$vsd_log
+ # Gather
+ kubectl get pods -n ${cfg.volthaNamespace} \
+ -l app.kubernetes.io/part-of=voltha --no-headers \
+ > \$vsd_log
+
+ count=\$((\$count - 1))
+
+ # Display activity every iteration ?
+ [[ -v verbose ]] && { count=0; }
+
+ # Display activity every minute or so {sleep(5) x count=10}
+ if [[ \$count -lt 1 ]]; then
+ count=10
+ cat \$vsd_log
+ fi
+
+ ## -----------------------
+ ## Probe for cluster state
+ ## -----------------------
+ if grep -q -e 'ContainerCreating' \$vsd_log; then
+ echo -e '\nvolthaStackDeploy.groovy: ContainerCrating active'
+ [[ -v debug ]] && grep -e 'ContainerCreating' \$vsd_log
+
+ elif grep -q -e '0/' \$vsd_log; then
+ echo -e '\nvolthaStackDeploy.groovy: Waiting for status=Running'
+ [[ -v debug ]] && grep -e '0/' \$vsd_log
+
+ elif ! grep -q '/' \$vsd_log; then
+ echo -e '\nvolthaStackDeploy.groovy: Waiting for initial pod activity'
+ [[ ! -v verbose ]] && { cat \$vsd_log; }
+
+ # -----------------------------------------------------------------------
+ # voltha-adapter-openolt-68c84bf786-8xsfc 0/1 CrashLoopBackOff 2 69s
+ # voltha-adapter-openolt-68c84bf786-8xsfc 0/1 Error 3 85s
+ # -----------------------------------------------------------------------
+ elif grep -q 'Error' \$vsd_log; then
+ echo -e '\nvolthaStackDeploy.groovy: Detected cluster state=Error'
+ cat \$vsd_log
+ rc=1 # fatal
+ break
+
+ # -----------------------------------------------------------------------
+ # An extra conditon needed here but shell coding is tricky:
+ # "svc x/y Running 0 6s
+ # Verify (x == y) && (x > 0)
+ # Wait until job failure/we have an actual need for it.
+ # -----------------------------------------------------------------------
+ else
+ echo -e '\nvolthaStackDeploy.groovy: Voltha stack has launched'
+ [[ ! -v verbose ]] && { cat \$vsd_log; }
break
fi
+ ## Support argument --timeout (?)
sleep 5
- count=\$((\$count - 1))
-
- if [[ \$count -lt 1 ]]; then # [DEBUG] Display activity every minute or so
- count=10
- kubectl get pods -n ${cfg.volthaNamespace} \
- -l app.kubernetes.io/part-of=voltha --no-headers \
- | tee \$vsd_log
- else
- kubectl get pods -n ${cfg.volthaNamespace} \
- -l app.kubernetes.io/part-of=voltha --no-headers \
- > \$vsd_log
- fi
done
rm -f \$vsd_log
+exit \$rc
""")
leave('launchVolthaStack')