python-upgrade/upgrade-sandbox-python.sh

#!/bin/bash
## -----------------------------------------------------------------------
## -----------------------------------------------------------------------

# set -euo pipefail

## -----------------------------------------------------------------------
## Intent: Dislpay an error message then exit
## -----------------------------------------------------------------------
function error()
{
    cat <<ERR

** -----------------------------------------------------------------------
** IAM: ${BASH_SOURCE[0]}
** IAM: ${FUNCNAME[1]} (LINENO:${BASH_LINENO[0]})
** ERR: $@
** -----------------------------------------------------------------------
ERR

    exit 1
}

## -----------------------------------------------------------------------
## Intent: Dislpay an error message then exit
## -----------------------------------------------------------------------
function banner()
{
    cat <<MSG

** -----------------------------------------------------------------------
** $@
** -----------------------------------------------------------------------
MSG
    
    return
}

## -----------------------------------------------------------------------
## Intent: Debug function, display minimalist script debugging
## -----------------------------------------------------------------------
function signpost()
{
    echo
    echo "** SIGNPOST[${FUNCNAME[1]}] (LINENO:${BASH_LINENO[1]}): $@"
    return
}

## -----------------------------------------------------------------------
## Intent: Identify system type
## https://stackoverflow.com/questions/394230/how-to-detect-the-os-from-a-bash-script
## -----------------------------------------------------------------------
function detect_arch()
{
    local -n ref=$1; shift

    ref=''
    if [[ "$OSTYPE" == "linux-gnu"* ]]; then
        ref='linux'
    elif [[ "$OSTYPE" == "darwin"* ]]; then
        ref='darwin'
    elif [[ "$OSTYPE" == "freebsd"* ]]; then
        ref='freebsd'
    else
        error "Deteted unhandled OS_TYPE=[$OS_TYPE]"
    fi

    return
}

## -----------------------------------------------------------------------
## Intent: Add more apt sources to install versioned packages
## -----------------------------------------------------------------------
function configure_apt()
{
    
    ## Perform once
    [[ -v configure_apt_seen ]] && { return; }
    declare -g -i configure_apt_seen=1

    sandbox '[ENTER]'
    sudo 'add-apt-repository' 'universe'
    sudo apt update
    sandbox '[LEAVE]'

    return
}

## -----------------------------------------------------------------------
## Intent: Ubuntu: apt install packages for a versioned python interpreter
## -----------------------------------------------------------------------
function install_apt_packages()
{
    local ver="$1"; shift

    pkgs+=("libpython${ver}")         # Shared Python runtime library (version ${ver})
    pkgs+=("libpython${ver}-dev")     # Header files and a static library for Python (v${ver})
    pkgs+=("libpython${ver}-minimal") # Minimal subset of the Python language (version ${ver})
    pkgs+=("python${ver}")            # - Interactive high-level object-oriented language (version ${ver})
    pkgs+=("python${ver}-dev")        # - Header files and a static library for Python (v${ver})
    pkgs+=("python${ver}-full")       # - Python Interpreter with complete class library (version ${ver})
    pkgs+=("python${ver}-venv")       # - Interactive high-level object-oriented language (pyvenv binary, version ${ver})

    pkgs+=('virtualenv')              # Convenience wrapper for installation

    sudo apt-get install -y "${pkgs[@]}"
    return
}

## -----------------------------------------------------------------------
## Intent:
##    o Install multiple interpreters for evaluation
##    o Installer is non-destructive
## -----------------------------------------------------------------------
## [TODO]
##    o Configure update-alternatives to simplify version switching.
##    o Careful, modifying /usr/bin/python can break system dependencies
## -----------------------------------------------------------------------
function install_python()
{
    local ver="$1"; shift
    local path="/usr/bin/python${ver}"

    echo "INSTALL PYTHON"
    local arch=''
    detect_arch arch

    ## Configure package manager to include versioned python packages.
    case "$arch" in
        'linux') configure_apt ;;
        'darwin')
            ## Logic needed if installing with: brew, etc
            error "Installer not yet configured for OSX (arch=[$arch])"
            ;;
        *) error "Installer not yet configured for arch=[$arch]" ;;
    esac


    ## -------------------------------
    ## If evaluating multiple versions
    ## -------------------------------
    declare -a versions=()
#    versions+=('3.8')
    versions+=('3.10')
#    versions+=('3.12')
#    versions+=('3.13')

    local version
    for version in "${versions[@]}";
    do
        local path="/usr/bin/python${version}"
        signpost "Interpreter: $(declare -p path)"
        
        if [[ -e "$path" ]]; then
            echo "[${FUNCNAME[0]}:SKIP] interpreter already installed [$path]"
            continue
        fi

        case "$arch" in
            'linux') install_apt_packages "$version" ;;

            ## Commands needed for brew, etc
            'darwin')
                error "Installer not yet configured for OSX (arch=[$arch])"
                ;;
            *) error "Installer not yet configured for arch=[$arch]" ;;
        esac
    done

    return
}

## -----------------------------------------------------------------------
## Intent: Install a versioned venv into a cloned sandbox
## -----------------------------------------------------------------------
## [NOTE] Early creation of {sandbox}/.venv will short circuit a makefile
##        makefile target that normally installs venv.
## -----------------------------------------------------------------------
function install_sandbox_virtualenv()
{
    local repo="$1"; shift
    local ver="$1";  shift
    local interpreter="/usr/bin/python${ver}"

    signpost '[ENTER]'
   
    pushd "$repo" >/dev/null || { error "pushd $repo failed"; }

    readarray -t cmd < <(which virtualenv 2>/dev/null)
    if [[ "${#cmd[@]}" -gt 0 ]] && [[ -x "${cmd[0]}" ]]; then
        ## Prefer tool:virtualenv for installation when available
        banner "** Install python using tool virtualenv"

        declare -a args=()
        [[ ! -v ARGV_no_quiet ]] && { args+=('--quiet'); }
        virtualenv "${args[@]}" -p "$interpreter" '.venv'

    else
        # fallback to dependency-less venv install
        banner "** Install python using a versioned interpreter"
        "$interpreter" -m venv '.venv'
    fi


    ## -------------------------------------------------------
    ## 1) Configure default interpreter version
    ## 2) Upgrade core interpreter packages: pip, setuptools
    ## 3) Install per-repository packages  : requirements.txt
    ## ------------------------------------------------------
    ## See Also:
    ##   % make venv
    ##   makefiles/virtualenv/include.mk
    ## -------------------------------------------------------
    source .venv/bin/activate

    declare -a pip_install=()
    pip_install+=('pip' 'install' '--no-cache-dir')
    [[ ! -v ARGV_no_quiet ]] && { pip_install+=('--quiet'); }
    # Upgrade core interpreter packages: pip, setuptools
    # pip --upgrade has chicken-n-egg problems
	python -m "${pip_install[@]}" --upgrade pip
    
	"${pip_install[@]}" --upgrade setuptools

    ## Install per-repository packages  : requirements.txt
    python -m "${pip_install[@]}" -r ./requirements.txt

    popd          >/dev/null || { error "pushd $repo failed"; }

    signpost '[LEAVE]'
   
    return
}

## -----------------------------------------------------------------------
## Intent:
##   Determine if python interpreter is valid/current.
##   No need for installation if the OS is current with recent packages
## -----------------------------------------------------------------------
function verify_python_version()
{
    local want="$1"; shift

    readarray -t version < <(/usr/bin/python3 --version)

    case "${version[*]}" in
        *"$want"*) local -i is_valid=1 ;;
    esac

    [[ -v is_valid ]] && { /bin/true; } || { /bin/false; }
    return
}

## -----------------------------------------------------------------------
## Intent:
##    o Clone a repository and create a user branch
##    o Installer is non-destructive
## -----------------------------------------------------------------------
function clone_and_branch_repo()
{
    local repo="$1"; shift
    local path="$repo"

    sandbox '[ENTER]'

    if [[ -v ARGV_clean ]] && [[ -d "$path" ]]; then
        echo "** Removing old sandbox: $repo"
        rm -fr "$repo"
    fi
    
    ## NOP if sandbox already checked out
    if [[ -e "$path" ]]; then
        echo "[${FUNCNAME[0]}:SKIP] repository already checked out"
        return
    fi

    banner "Clone and branch repo: ${repo}"

    local url="ssh://gerrit.opencord.org:29418/${repo}.git"
    git clone "$url"
    pushd "$repo" >/dev/null || { error "pushd $repo failed"; }

    local branch="dev-${USER}"
    banner "git checkout -b \"$branch\""
    git checkout -b "$branch"

    banner "git remote -v show"
    git remote -v show

    banner "git branch -a"
    git branch -a

    popd >/dev/null          || { error "popd $repo failed"; }

    sandbox '[LEAVE]'

    return
}

## -----------------------------------------------------------------------
## Intent: Alter requirements.txt to use latest package versions
## -----------------------------------------------------------------------
function upgrade_packages()
{
    local repo="$1"; shift

    signpost '[ENTER]'
        
    ## NOP: Already modified
    if ! grep '[=<>]' "$repo/requirements.txt"; then
        return
    fi

    pushd "$repo" >/dev/null || { error "pushd $repo failed"; }
    
    sed -i'' '\@zdw/robotframework-importresource@d' 'requirements.txt'

    # -------------------------------------------------
    # Disgusting but portable: prune version qualifiers
    # Pip will install latest packages by name
    # -------------------------------------------------
    cut -d'=' -f1 requirements.txt \
        | cut -d'<' -f1 \
        | cut -d'>' -f1 \
        > requirements.tmp

    echo "** Use git checkout requirements.txt to revert changes"
    mv requirements.tmp requirements.txt

    popd          >/dev/null || { error "popd $repo failed"; }

    signpost '[LEAVE]'
    
    return
}

## -----------------------------------------------------------------------
## Intent: If version strings
## -----------------------------------------------------------------------
function freeze_packages()
{
    local repo="$1"; shift

    signpost '[ENTER]'
    
    pushd "$repo" >/dev/null || { error "pushd $repo failed"; }
    source .venv/bin/activate

    declare -a pip_freeze=()
    pip_freeze+=('pip' 'freeze')
    pip_freeze+=('--local')                           # local directory
    pip_freeze+=('--requirement' 'requirements.txt')  # existing order

    "${pip_freeze[@]}" > 'requirements.txt.tmp'
    mv 'requirements.txt.tmp' 'requirements.txt'

    popd          >/dev/null || { error "popd $repo failed"; }

    signpost '[LEAVE]'
    
    return
}

## -----------------------------------------------------------------------
## Intent: Display misc information about the installation, known problems
##   and makefile targets used for building and testing.
## -----------------------------------------------------------------------
function show_readme()
{
    local repo="$1"; shift

    /bin/pwd
    pushd "$repo" >/dev/null || { error "pushd $repo failed"; }

    cat <<HOWTO

** -----------------------------------------------------------------------
** IAM: ${FUNCNAME[0]}
** PWD: $(/bin/pwd)
** -----------------------------------------------------------------------

% cd ${repo}
% source .venv/bin/activate
$(source .venv/bin/activate && python --version)

% make help       # Display available targets
% make all
% make lint
% make pre-commit
% make build
% make test

HOWTO

    readarray -t zdw < <(grep '/zdw/' requirements.txt)
    cat <<PROBLEMS

** -----------------------------------------------------------------------
** Known Problems: early failures that require attention
** -----------------------------------------------------------------------
[requirements.txt]
   o Comment out package /zdw/robotframework-import in equirements.txt.
   o Try --upgrade-packages to install latest versions

[grpc]
   o repo:voltha-protos and repo:voltha-system-tests both have
     problems building packages.
   o ERROR: Failed to build installable wheels for some pyproject.toml based projects (grpcio)

PROBLEMS
    
    popd          >/dev/null || { error "pushd $repo failed"; }
    return
}

## -----------------------------------------------------------------------
## -----------------------------------------------------------------------
function usage()
{
    cat <<EOHELP
Usage: $0
  --repo [r]          Repository name to checkout and configure.
  --version [v]       Install python virtualenv at version [v]

  --gerrit            Browse a list of VOLTHA repositories in the gerrit UI
  --upgrade-pacakges  Remove frozen version information from requirements.txt

  --no-quiet          Run commands in verbose mode (pip, virtualenv)
  --verbose             alias for --no-quiet
  --clean             Delete cloned sandbox prior to install
  --help              This message
EOHELP

    cat <<EOHELP

[EXAMPLES]
  % $0 --version 3.12 --repo voltha-system-tests
  % $0 --version 3.12 --repo voltha-lib-go

EOHELP

    return
}

##----------------##
##---]  MAIN  [---##
##----------------##
banner '[NOTE] If the status message [FIN] is displayed script ran successfully'


while [[ $# -gt 0 ]]; do
    arg="$1"; shift

    case "$arg" in
        '--no-quiet') declare -g -i ARGV_no_quiet=1;     ;;
        '--verbose')  declare -g -i ARGV_no_quiet=1;     ;;

        '--clean') declare -g -i ARGV_clean=1            ;;
        '--upgr'*) declare -g ARGV_upgrade_packages=1    ;;
        '--repo'*) declare -g ARGV_repo="$1";    shift   ;;
        '--ver'*)  declare -g ARGV_version="$1"; shift   ;;
        '--help')  usage; exit 0                         ;;
        '--gerrit')
            "${BROWSER:-firefox}" 'https://gerrit.opencord.org/admin/repos/q/filter:voltha-'
            exit 1
            ;;
        '-'*) error "Detected unknown switch [$arg]" ;;
        *) echo "[SKIP] unknown argument [$arg]"     ;;
    esac
done

[[ ! -v ARGV_version ]] && { error '--version is required (default: 2.13)'; }
[[ ! -v ARGV_repo ]]    && { error '--repo is required'; }

## -----------------------------------------------------------------------
## NOP if system interpreter is the latest
## -----------------------------------------------------------------------
if verify_python_version "$ARGV_version"; then

    readarray -t version < <(pyhon --version 2>/dev/null)
    echo "** Python Version: ${version[*]}"
    case "${version[*]}" in

        # v3.13 is in pre-release.
        # may be pre-installed for new distribution installations.
        # If unavailabler sources are available for download but
        #   will require building the interpreter.
        
        *'3.12'*|*'3.13'*)
            banner \
                "[SKIP] Python interpreter already installed (ver=${ARGV_version})" \
                '[NOTE] This script is no longer needed, detected latest python version'
            ;;
        *)
            banner "[SKIP] Python interpreter already installed (ver=${ARGV_version})"
            ;;
    esac

# [ELSE] we need to install
else

    banner "Installing python version ${ARGV_version}"

    ## ------------------------------------------------------------
    # package manager: Configure and install versioned interpreters
    ## ------------------------------------------------------------
    install_python "${ARGV_version}"    # OS packages

fi

mkdir -p sandbox # git clone into sandbox/ for easy cleanup
pushd 'sandbox' >/dev/null || { error 'pushd sandbox/ failed'; }

## Checkout VOLTHA code base
clone_and_branch_repo "$ARGV_repo"
echo

[[ -v ARGV_upgrade_packages ]] && { upgrade_packages "$ARGV_repo"; }

## Install upgraded python interpreter in the sandbox
install_sandbox_virtualenv  "${ARGV_repo}" "${ARGV_version}"    # sandbox/virtualenv/

freeze_packages "$ARGV_repo"   # capture all upgraded package versions
show_readme "${ARGV_repo}"     # display notes and caveats

popd >/dev/null || { error 'popd sandbox/ failed'; }


echo '[FIN] - installer ran successfully'

# [EOF]