wait_for_jobs.sh

#!/usr/bin/env bash

# Copyright 2018-2024 Open Networking Foundation (ONF) and the ONF Contributors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# wait_for_jobs.sh
# waits for all kubernetes jobs to complete before exiting
# inspired by similar scripts in Kolla-Kubernetes and Openstack Helm

set -eu -o pipefail
fail_wfj=0

# Set these to configure maximum timeout, and interval for checks
JOBS_TIMEOUT=${JOBS_TIMEOUT:-600}
CHECK_INTERVAL=${CHECK_INTERVAL:-5}
KUBECTL_ARGS=${KUBECTL_ARGS:-}

# calculate timeout time
START_TIME=$(date +%s)
END_TIME=$((START_TIME + JOBS_TIMEOUT))

echo "wait_for_jobs.sh - Waiting up to ${JOBS_TIMEOUT} seconds for all Kubernetes jobs to complete"
echo "Number printed is number of currently active jobs"

prev_job_count=0

while true; do
    NOW=$(date +%s)

    # handle timeout without completion
    if [ "$NOW" -gt "$END_TIME" ]
    then
        echo "Jobs didn't complete before timeout of ${JOBS_TIMEOUT} seconds"
        fail_wfj=1
        break
    fi

    # get list of active jobs, and count of them
    # jsonpath is picky about string vs comparison quoting, so have to have:
    # shellcheck disable=SC2026,SC2086
    active_jobs=$(kubectl get jobs $KUBECTL_ARGS -o=jsonpath='{range .items[?(@.status.active=='1')]}{.metadata.name}{"\n"}{end}')

    # this always is 1 or more, as echo leaves a newline in the output which wc
    # counts as a line
    active_job_count=$(echo -n "${active_jobs}" | wc -l)

    # if no jobs active, print runtime and break
    if [ -z "$active_jobs" ]
    then
        runtime=$((NOW - START_TIME))
        echo ""
        echo "All jobs completed in $runtime seconds"
        break
    fi

    # deal with changes in number of jobs
    if [ "$active_job_count" -ne "$prev_job_count" ]
    then
        echo ""
        echo "Number of active jobs changed - current jobs:"
        echo "$active_jobs"
    fi
    prev_job_count=$active_job_count

    # print number of remaining jobs every $CHECK_INTERVAL
    echo -n "$active_job_count "
    sleep "$CHECK_INTERVAL"
done

echo ""
echo "Job Status - Name | Start Time | Completion Time"
# shellcheck disable=SC2086
kubectl get jobs $KUBECTL_ARGS -o=jsonpath='{range .items[*]}{.metadata.name}{"\t| "}{.status.startTime}{" | "}{.status.completionTime}{"\n"}{end}'

exit ${fail_wfj}