Added a convenience script for upgrading python locally.
python-upgrade/
python-upgrade/README.md
python-upgrade/upgrade-sandbox-python.sh
----------------------------------------
o The upgrade-sandbox-script will install OS packages.
o Clone a git sandbox.
o Upgrade/install a virtualenv at a specified version.
Signed-off-by: Joey Armstrong <jarmstrong@linuxfoundation.org>
Change-Id: Ic939ab53c329d689e25c139b3bec9c00b4b9c68a
diff --git a/python-upgrade/upgrade-sandbox-python.sh b/python-upgrade/upgrade-sandbox-python.sh
new file mode 100755
index 0000000..eacb69b
--- /dev/null
+++ b/python-upgrade/upgrade-sandbox-python.sh
@@ -0,0 +1,531 @@
+#!/bin/bash
+## -----------------------------------------------------------------------
+## -----------------------------------------------------------------------
+
+# set -euo pipefail
+
+## -----------------------------------------------------------------------
+## Intent: Dislpay an error message then exit
+## -----------------------------------------------------------------------
+function error()
+{
+ cat <<ERR
+
+** -----------------------------------------------------------------------
+** IAM: ${BASH_SOURCE[0]}
+** IAM: ${FUNCNAME[1]} (LINENO:${BASH_LINENO[0]})
+** ERR: $@
+** -----------------------------------------------------------------------
+ERR
+
+ exit 1
+}
+
+## -----------------------------------------------------------------------
+## Intent: Dislpay an error message then exit
+## -----------------------------------------------------------------------
+function banner()
+{
+ cat <<MSG
+
+** -----------------------------------------------------------------------
+** $@
+** -----------------------------------------------------------------------
+MSG
+
+ return
+}
+
+## -----------------------------------------------------------------------
+## Intent: Debug function, display minimalist script debugging
+## -----------------------------------------------------------------------
+function signpost()
+{
+ echo
+ echo "** SIGNPOST[${FUNCNAME[1]}] (LINENO:${BASH_LINENO[1]}): $@"
+ return
+}
+
+## -----------------------------------------------------------------------
+## Intent: Identify system type
+## https://stackoverflow.com/questions/394230/how-to-detect-the-os-from-a-bash-script
+## -----------------------------------------------------------------------
+function detect_arch()
+{
+ local -n ref=$1; shift
+
+ ref=''
+ if [[ "$OSTYPE" == "linux-gnu"* ]]; then
+ ref='linux'
+ elif [[ "$OSTYPE" == "darwin"* ]]; then
+ ref='darwin'
+ elif [[ "$OSTYPE" == "freebsd"* ]]; then
+ ref='freebsd'
+ else
+ error "Deteted unhandled OS_TYPE=[$OS_TYPE]"
+ fi
+
+ return
+}
+
+## -----------------------------------------------------------------------
+## Intent: Add more apt sources to install versioned packages
+## -----------------------------------------------------------------------
+function configure_apt()
+{
+
+ ## Perform once
+ [[ -v configure_apt_seen ]] && { return; }
+ declare -g -i configure_apt_seen=1
+
+ sandbox '[ENTER]'
+ sudo 'add-apt-repository' 'universe'
+ sudo apt update
+ sandbox '[LEAVE]'
+
+ return
+}
+
+## -----------------------------------------------------------------------
+## Intent: Ubuntu: apt install packages for a versioned python interpreter
+## -----------------------------------------------------------------------
+function install_apt_packages()
+{
+ local ver="$1"; shift
+
+ pkgs+=("libpython${ver}") # Shared Python runtime library (version ${ver})
+ pkgs+=("libpython${ver}-dev") # Header files and a static library for Python (v${ver})
+ pkgs+=("libpython${ver}-minimal") # Minimal subset of the Python language (version ${ver})
+ pkgs+=("python${ver}") # - Interactive high-level object-oriented language (version ${ver})
+ pkgs+=("python${ver}-dev") # - Header files and a static library for Python (v${ver})
+ pkgs+=("python${ver}-full") # - Python Interpreter with complete class library (version ${ver})
+ pkgs+=("python${ver}-venv") # - Interactive high-level object-oriented language (pyvenv binary, version ${ver})
+
+ pkgs+=('virtualenv') # Convenience wrapper for installation
+
+ sudo apt-get install -y "${pkgs[@]}"
+ return
+}
+
+## -----------------------------------------------------------------------
+## Intent:
+## o Install multiple interpreters for evaluation
+## o Installer is non-destructive
+## -----------------------------------------------------------------------
+## [TODO]
+## o Configure update-alternatives to simplify version switching.
+## o Careful, modifying /usr/bin/python can break system dependencies
+## -----------------------------------------------------------------------
+function install_python()
+{
+ local ver="$1"; shift
+ local path="/usr/bin/python${ver}"
+
+ echo "INSTALL PYTHON"
+ local arch=''
+ detect_arch arch
+
+ ## Configure package manager to include versioned python packages.
+ case "$arch" in
+ 'linux') configure_apt ;;
+ 'darwin')
+ ## Logic needed if installing with: brew, etc
+ error "Installer not yet configured for OSX (arch=[$arch])"
+ ;;
+ *) error "Installer not yet configured for arch=[$arch]" ;;
+ esac
+
+
+ ## -------------------------------
+ ## If evaluating multiple versions
+ ## -------------------------------
+ declare -a versions=()
+# versions+=('3.8')
+ versions+=('3.10')
+# versions+=('3.12')
+# versions+=('3.13')
+
+ local version
+ for version in "${versions[@]}";
+ do
+ local path="/usr/bin/python${version}"
+ signpost "Interpreter: $(declare -p path)"
+
+ if [[ -e "$path" ]]; then
+ echo "[${FUNCNAME[0]}:SKIP] interpreter already installed [$path]"
+ continue
+ fi
+
+ case "$arch" in
+ 'linux') install_apt_packages "$version" ;;
+
+ ## Commands needed for brew, etc
+ 'darwin')
+ error "Installer not yet configured for OSX (arch=[$arch])"
+ ;;
+ *) error "Installer not yet configured for arch=[$arch]" ;;
+ esac
+ done
+
+ return
+}
+
+## -----------------------------------------------------------------------
+## Intent: Install a versioned venv into a cloned sandbox
+## -----------------------------------------------------------------------
+## [NOTE] Early creation of {sandbox}/.venv will short circuit a makefile
+## makefile target that normally installs venv.
+## -----------------------------------------------------------------------
+function install_sandbox_virtualenv()
+{
+ local repo="$1"; shift
+ local ver="$1"; shift
+ local interpreter="/usr/bin/python${ver}"
+
+ signpost '[ENTER]'
+
+ pushd "$repo" >/dev/null || { error "pushd $repo failed"; }
+
+ readarray -t cmd < <(which virtualenv 2>/dev/null)
+ if [[ "${#cmd[@]}" -gt 0 ]] && [[ -x "${cmd[0]}" ]]; then
+ ## Prefer tool:virtualenv for installation when available
+ banner "** Install python using tool virtualenv"
+
+ declare -a args=()
+ [[ ! -v ARGV_no_quiet ]] && { args+=('--quiet'); }
+ virtualenv "${args[@]}" -p "$interpreter" '.venv'
+
+ else
+ # fallback to dependency-less venv install
+ banner "** Install python using a versioned interpreter"
+ "$interpreter" -m venv '.venv'
+ fi
+
+
+ ## -------------------------------------------------------
+ ## 1) Configure default interpreter version
+ ## 2) Upgrade core interpreter packages: pip, setuptools
+ ## 3) Install per-repository packages : requirements.txt
+ ## ------------------------------------------------------
+ ## See Also:
+ ## % make venv
+ ## makefiles/virtualenv/include.mk
+ ## -------------------------------------------------------
+ source .venv/bin/activate
+
+ declare -a pip_install=()
+ pip_install+=('pip' 'install' '--no-cache-dir')
+ [[ ! -v ARGV_no_quiet ]] && { pip_install+=('--quiet'); }
+ # Upgrade core interpreter packages: pip, setuptools
+ # pip --upgrade has chicken-n-egg problems
+ python -m "${pip_install[@]}" --upgrade pip
+
+ "${pip_install[@]}" --upgrade setuptools
+
+ ## Install per-repository packages : requirements.txt
+ python -m "${pip_install[@]}" -r ./requirements.txt
+
+ popd >/dev/null || { error "pushd $repo failed"; }
+
+ signpost '[LEAVE]'
+
+ return
+}
+
+## -----------------------------------------------------------------------
+## Intent:
+## Determine if python interpreter is valid/current.
+## No need for installation if the OS is current with recent packages
+## -----------------------------------------------------------------------
+function verify_python_version()
+{
+ local want="$1"; shift
+
+ readarray -t version < <(/usr/bin/python3 --version)
+
+ case "${version[*]}" in
+ *"$want"*) local -i is_valid=1 ;;
+ esac
+
+ [[ -v is_valid ]] && { /bin/true; } || { /bin/false; }
+ return
+}
+
+## -----------------------------------------------------------------------
+## Intent:
+## o Clone a repository and create a user branch
+## o Installer is non-destructive
+## -----------------------------------------------------------------------
+function clone_and_branch_repo()
+{
+ local repo="$1"; shift
+ local path="$repo"
+
+ sandbox '[ENTER]'
+
+ if [[ -v ARGV_clean ]] && [[ -d "$path" ]]; then
+ echo "** Removing old sandbox: $repo"
+ rm -fr "$repo"
+ fi
+
+ ## NOP if sandbox already checked out
+ if [[ -e "$path" ]]; then
+ echo "[${FUNCNAME[0]}:SKIP] repository already checked out"
+ return
+ fi
+
+ banner "Clone and branch repo: ${repo}"
+
+ local url="ssh://gerrit.opencord.org:29418/${repo}.git"
+ git clone "$url"
+ pushd "$repo" >/dev/null || { error "pushd $repo failed"; }
+
+ local branch="dev-${USER}"
+ banner "git checkout -b \"$branch\""
+ git checkout -b "$branch"
+
+ banner "git remote -v show"
+ git remote -v show
+
+ banner "git branch -a"
+ git branch -a
+
+ popd >/dev/null || { error "popd $repo failed"; }
+
+ sandbox '[LEAVE]'
+
+ return
+}
+
+## -----------------------------------------------------------------------
+## Intent: Alter requirements.txt to use latest package versions
+## -----------------------------------------------------------------------
+function upgrade_packages()
+{
+ local repo="$1"; shift
+
+ signpost '[ENTER]'
+
+ ## NOP: Already modified
+ if ! grep '[=<>]' "$repo/requirements.txt"; then
+ return
+ fi
+
+ pushd "$repo" >/dev/null || { error "pushd $repo failed"; }
+
+ sed -i'' '\@zdw/robotframework-importresource@d' 'requirements.txt'
+
+ # -------------------------------------------------
+ # Disgusting but portable: prune version qualifiers
+ # Pip will install latest packages by name
+ # -------------------------------------------------
+ cut -d'=' -f1 requirements.txt \
+ | cut -d'<' -f1 \
+ | cut -d'>' -f1 \
+ > requirements.tmp
+
+ echo "** Use git checkout requirements.txt to revert changes"
+ mv requirements.tmp requirements.txt
+
+ popd >/dev/null || { error "popd $repo failed"; }
+
+ signpost '[LEAVE]'
+
+ return
+}
+
+## -----------------------------------------------------------------------
+## Intent: If version strings
+## -----------------------------------------------------------------------
+function freeze_packages()
+{
+ local repo="$1"; shift
+
+ signpost '[ENTER]'
+
+ pushd "$repo" >/dev/null || { error "pushd $repo failed"; }
+ source .venv/bin/activate
+
+ declare -a pip_freeze=()
+ pip_freeze+=('pip' 'freeze')
+ pip_freeze+=('--local') # local directory
+ pip_freeze+=('--requirement' 'requirements.txt') # existing order
+
+ "${pip_freeze[@]}" > 'requirements.txt.tmp'
+ mv 'requirements.txt.tmp' 'requirements.txt'
+
+ popd >/dev/null || { error "popd $repo failed"; }
+
+ signpost '[LEAVE]'
+
+ return
+}
+
+## -----------------------------------------------------------------------
+## Intent: Display misc information about the installation, known problems
+## and makefile targets used for building and testing.
+## -----------------------------------------------------------------------
+function show_readme()
+{
+ local repo="$1"; shift
+
+ /bin/pwd
+ pushd "$repo" >/dev/null || { error "pushd $repo failed"; }
+
+ cat <<HOWTO
+
+** -----------------------------------------------------------------------
+** IAM: ${FUNCNAME[0]}
+** PWD: $(/bin/pwd)
+** -----------------------------------------------------------------------
+
+% cd ${repo}
+% source .venv/bin/activate
+$(source .venv/bin/activate && python --version)
+
+% make help # Display available targets
+% make all
+% make lint
+% make pre-commit
+% make build
+% make test
+
+HOWTO
+
+ readarray -t zdw < <(grep '/zdw/' requirements.txt)
+ cat <<PROBLEMS
+
+** -----------------------------------------------------------------------
+** Known Problems: early failures that require attention
+** -----------------------------------------------------------------------
+[requirements.txt]
+ o Comment out package /zdw/robotframework-import in equirements.txt.
+ o Try --upgrade-packages to install latest versions
+
+[grpc]
+ o repo:voltha-protos and repo:voltha-system-tests both have
+ problems building packages.
+ o ERROR: Failed to build installable wheels for some pyproject.toml based projects (grpcio)
+
+PROBLEMS
+
+ popd >/dev/null || { error "pushd $repo failed"; }
+ return
+}
+
+## -----------------------------------------------------------------------
+## -----------------------------------------------------------------------
+function usage()
+{
+ cat <<EOHELP
+Usage: $0
+ --repo [r] Repository name to checkout and configure.
+ --version [v] Install python virtualenv at version [v]
+
+ --gerrit Browse a list of VOLTHA repositories in the gerrit UI
+ --upgrade-pacakges Remove frozen version information from requirements.txt
+
+ --no-quiet Run commands in verbose mode (pip, virtualenv)
+ --verbose alias for --no-quiet
+ --clean Delete cloned sandbox prior to install
+ --help This message
+EOHELP
+
+ cat <<EOHELP
+
+[EXAMPLES]
+ % $0 --version 3.12 --repo voltha-system-tests
+ % $0 --version 3.12 --repo voltha-lib-go
+
+EOHELP
+
+ return
+}
+
+##----------------##
+##---] MAIN [---##
+##----------------##
+banner '[NOTE] If the status message [FIN] is displayed script ran successfully'
+
+
+while [[ $# -gt 0 ]]; do
+ arg="$1"; shift
+
+ case "$arg" in
+ '--no-quiet') declare -g -i ARGV_no_quiet=1; ;;
+ '--verbose') declare -g -i ARGV_no_quiet=1; ;;
+
+ '--clean') declare -g -i ARGV_clean=1 ;;
+ '--upgr'*) declare -g ARGV_upgrade_packages=1 ;;
+ '--repo'*) declare -g ARGV_repo="$1"; shift ;;
+ '--ver'*) declare -g ARGV_version="$1"; shift ;;
+ '--help') usage; exit 0 ;;
+ '--gerrit')
+ "${BROWSER:-firefox}" 'https://gerrit.opencord.org/admin/repos/q/filter:voltha-'
+ exit 1
+ ;;
+ '-'*) error "Detected unknown switch [$arg]" ;;
+ *) echo "[SKIP] unknown argument [$arg]" ;;
+ esac
+done
+
+[[ ! -v ARGV_version ]] && { error '--version is required (default: 2.13)'; }
+[[ ! -v ARGV_repo ]] && { error '--repo is required'; }
+
+## -----------------------------------------------------------------------
+## NOP if system interpreter is the latest
+## -----------------------------------------------------------------------
+if verify_python_version "$ARGV_version"; then
+
+ readarray -t version < <(pyhon --version 2>/dev/null)
+ echo "** Python Version: ${version[*]}"
+ case "${version[*]}" in
+
+ # v3.13 is in pre-release.
+ # may be pre-installed for new distribution installations.
+ # If unavailabler sources are available for download but
+ # will require building the interpreter.
+
+ *'3.12'*|*'3.13'*)
+ banner \
+ "[SKIP] Python interpreter already installed (ver=${ARGV_version})" \
+ '[NOTE] This script is no longer needed, detected latest python version'
+ ;;
+ *)
+ banner "[SKIP] Python interpreter already installed (ver=${ARGV_version})"
+ ;;
+ esac
+
+# [ELSE] we need to install
+else
+
+ banner "Installing python version ${ARGV_version}"
+
+ ## ------------------------------------------------------------
+ # package manager: Configure and install versioned interpreters
+ ## ------------------------------------------------------------
+ install_python "${ARGV_version}" # OS packages
+
+fi
+
+mkdir -p sandbox # git clone into sandbox/ for easy cleanup
+pushd 'sandbox' >/dev/null || { error 'pushd sandbox/ failed'; }
+
+## Checkout VOLTHA code base
+clone_and_branch_repo "$ARGV_repo"
+echo
+
+[[ -v ARGV_upgrade_packages ]] && { upgrade_packages "$ARGV_repo"; }
+
+## Install upgraded python interpreter in the sandbox
+install_sandbox_virtualenv "${ARGV_repo}" "${ARGV_version}" # sandbox/virtualenv/
+
+freeze_packages "$ARGV_repo" # capture all upgraded package versions
+show_readme "${ARGV_repo}" # display notes and caveats
+
+popd >/dev/null || { error 'popd sandbox/ failed'; }
+
+
+echo '[FIN] - installer ran successfully'
+
+# [EOF]