Import makefile edits from repo:voltha-docs
makefiles/include.mk
makefiles/virtualenv/include.mk
makefiles/virtualenv/patches/
makefiles/virtualenv/python_310_migration.sh
--------------------------------------------
o Add deps and targets to create a python virtualenv
install patched for local use of v3.10+.
Change-Id: I9a3bf8cdfc9a98f09ac84c97c0246e6194c50d01
diff --git a/makefiles/commands/include.mk b/makefiles/commands/include.mk
index 01fa145..614986e 100644
--- a/makefiles/commands/include.mk
+++ b/makefiles/commands/include.mk
@@ -25,7 +25,7 @@
$(if $(DEBUG),$(warning ENTER))
-include $(ONF_MAKEDIR)/commands/kail.mk
+include $(ONF_MAKEDIR)/commands/kail/include.mk
$(if $(DEBUG),$(warning LEAVE))
diff --git a/makefiles/include.mk b/makefiles/include.mk
index 97ea39a..b6b90a1 100644
--- a/makefiles/include.mk
+++ b/makefiles/include.mk
@@ -41,7 +41,7 @@
include $(ONF_MAKEDIR)/etc/include.mk # banner macros
include $(ONF_MAKEDIR)/commands/include.mk # Tools and local installers
-include $(ONF_MAKEDIR)/virtualenv.mk# # lint-{jjb,python} depends on venv
+include $(ONF_MAKEDIR)/virtualenv/include.mk# # python, lint, JJB dependency
# include $(ONF_MAKEDIR)/patches/include.mk# # Patch when python 3.10+ in use
include $(ONF_MAKEDIR)/lint/include.mk
diff --git a/makefiles/virtualenv/HOWTO b/makefiles/virtualenv/HOWTO
new file mode 100644
index 0000000..ffdc30f
--- /dev/null
+++ b/makefiles/virtualenv/HOWTO
@@ -0,0 +1,21 @@
+# Howto create a python 3.10+ patch
+
+1) Checkout voltha-docs
+2) cd voltha-docs
+3) Create a virtual environment:
+ - make venv (default python version)
+ - make venv-activate-patched (for python v3.10+)
+4) make patch-init
+5) modify the file to be patched beneath staging/${relative_path_to_patch}
+6) make patch-create PATCH_PATH=${relative_path_to_patch}
+ o This will create patches/${relative_path_to_patch}/patch
+ o make patch-create PATCH_PATH=lib/python3.10/site-packages/sphinx/util/typing.py
+ lib/python3.10/site-packages/sphinx/util/typing/patch
+7) Verify
+ o make sterile
+ o make venv
+8) Validate
+ o make lint
+ o make test
+
+# [EOF]
diff --git a/makefiles/virtualenv/README.md b/makefiles/virtualenv/README.md
new file mode 100644
index 0000000..be948dc
--- /dev/null
+++ b/makefiles/virtualenv/README.md
@@ -0,0 +1,64 @@
+# Migration support for python 3.10+
+
+## Intent
+
+Applications of patches will enable local use of newer python versions
+available with a recent OS install or package manager installation.
+Makefile targets will fail out of the box w/o modifying collection imports.
+
+Create new patches as needed to help testing along.
+Eventually this hierarchy can be dismantled once the latest interpreter
+is generally in use.
+
+## patches/
+
+This directory contains patch files that can be directly applied to sources
+in the python virtualenv directory. Patches are created to support use of
+three python version variants for the robot ramework:
+ python 2x (deprecated)
+ python >= 3.10 (collections.abc required)
+ python < 3.10 (collections.abc optional)
+
+
+# .venv/
+
+Makefile will first create a python virtualenv directory to selectively
+use packages. After setup patches are applied to venv (as a transition)
+to fully enable support for local interpreter use for newer OS installs.
+
+
+# staging/
+
+The staging directory is used for comparison with the .venv directory
+to generate patches. Populate the directory with a copy of a cleanly
+patched virtual interpreter then modify files benath/ staging to generate
+a patch from.
+
+% make sterile
+% make venv
+% mkdir staging
+% rsync -rv --checksum .venv/. staging/.
+[NOTE] Make python 3.10+ migration edits beneath staging as needed
+% make patch-gather
+% make sterile
+% make lint
+
+
+# makefile targets
+
+make patch-gather:
+ * gather a list of potential sources to edit.
+
+make patch-apply:
+ * Gather a list of patch files.
+ * Apply each in turn to sources in the virtualenv directory.
+
+make patch-create:
+ * make patch-create VENV_NAME="venv_docs" PATCH_PATH="lib/python3.10/site-packages/sphinx/util/typing.py"
+ * make sterile
+ * make help # reinstall virtualenv and verify help target
+
+# Verify installation
+ * make lint -or- make test
+
+# [EOF]
\ No newline at end of file
diff --git a/makefiles/virtualenv.mk b/makefiles/virtualenv/include.mk
similarity index 71%
rename from makefiles/virtualenv.mk
rename to makefiles/virtualenv/include.mk
index cf24024..c20333f 100644
--- a/makefiles/virtualenv.mk
+++ b/makefiles/virtualenv/include.mk
@@ -15,7 +15,14 @@
# limitations under the License.
# -----------------------------------------------------------------------
# https://gerrit.opencord.org/plugins/gitiles/onf-make
-# ONF.makefile.version = 1.2
+# ONF.makefile.virtualenv.version = 1.2
+# -----------------------------------------------------------------------
+# Usage:
+# include makefiles/virtualenv/include.m
+#
+# Target Dependencies:
+# tgt : $(venv-activate-patched) python 3.10+ local use
+# tgt : $(venv-activate-script) python < v3.8
# -----------------------------------------------------------------------
$(if $(DEBUG),$(warning ENTER))
@@ -23,14 +30,14 @@
##-------------------##
##---] GLOBALS [---##
##-------------------##
-.PHONY: venv
+.PHONY: venv venv-patched
##------------------##
##---] LOCALS [---##
##------------------##
venv-name ?= .venv# # default install directory
-venv-abs-path := $(PWD)/$(venv-name)
-venv-activate-bin := $(venv-name)/bin
+venv-abs-path := $(PWD)/$(venv-name)# #
+venv-activate-bin := $(venv-name)/bin# # no whitespace
venv-activate-script := $(venv-activate-bin)/activate# # dependency
# Intent: activate= is a macro for accessing the virtualenv activation script#
@@ -45,9 +52,9 @@
## -----------------------------------------------------------------------
$(venv-activate-script):
@echo
- @echo "============================="
- @echo "Installing python virtual env"
- @echo "============================="
+ @echo '============================='
+ @echo 'Installing python virtual env'
+ @echo '============================='
virtualenv -p python3 $(venv-name)
$(activate) && python -m pip install --upgrade pip
$(activate) && pip install --upgrade setuptools
@@ -61,7 +68,20 @@
## Intent: Explicit named installer target w/o dependencies.
## Makefile targets should depend on venv-activate-script.
## -----------------------------------------------------------------------
-venv: $(venv-activate-script)
+venv-activate-patched := $(venv-activate-script).patched
+venv-activate-patched : $(venv-activate-patched)
+$(venv-activate-patched) : $(venv-activate-script)
+ $(call banner-enter,Target $@)
+ $(ONF_MAKEDIR)/virtualenv/python_310_migration.sh
+ touch $@
+ $(call banner-leave,Target $@)
+
+## -----------------------------------------------------------------------
+## Intent: Explicit named installer target w/o dependencies.
+## Makefile targets should depend on venv-activate-script.
+## -----------------------------------------------------------------------
+venv : $(venv-activate-script)
+venv-patched : $(venv-activate-patched)
## -----------------------------------------------------------------------
## Intent: Revert installation to a clean checkout
diff --git a/makefiles/virtualenv/lib/python3.10/site-packages/robot/utils/normalizing.py/patch b/makefiles/virtualenv/lib/python3.10/site-packages/robot/utils/normalizing.py/patch
new file mode 100644
index 0000000..ae8619a
--- /dev/null
+++ b/makefiles/virtualenv/lib/python3.10/site-packages/robot/utils/normalizing.py/patch
@@ -0,0 +1,14 @@
+--- vault/lib/python3.10/site-packages/robot/utils/normalizing.py 2022-11-26 06:59:47.438751606 -0500
++++ .venv/lib/python3.10/site-packages/robot/utils/normalizing.py 2022-11-26 06:57:29.960476182 -0500
+@@ -13,10 +13,7 @@
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+
+-try: # python >= 3.10
+- from collections.abc import MutableMapping
+-except ImportError: # python 2x
+- from collections import MutableMapping
++from collections import MutableMapping
+
+ from .platform import IRONPYTHON, PY_VERSION, PY3
+ from .robottypes import is_dict_like, is_unicode
diff --git a/makefiles/virtualenv/lib/python3.10/site-packages/robot/utils/robottypes3.py/patch b/makefiles/virtualenv/lib/python3.10/site-packages/robot/utils/robottypes3.py/patch
new file mode 100644
index 0000000..d1aa540
--- /dev/null
+++ b/makefiles/virtualenv/lib/python3.10/site-packages/robot/utils/robottypes3.py/patch
@@ -0,0 +1,16 @@
+--- vault/lib/python3.10/site-packages/robot/utils/robottypes3.py 2022-11-26 07:00:17.126386733 -0500
++++ .venv/lib/python3.10/site-packages/robot/utils/robottypes3.py 2022-11-26 06:57:29.956476232 -0500
+@@ -13,12 +13,7 @@
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+
+-try: # python >= 3.10
+- from collections.abc import Mapping
+- from collections import UserString
+-except ImportError: # python 2x
+- from collections import Mapping, UserString
+-
++from collections import Mapping, UserString
+ from io import IOBase
+
+ from .platform import RERAISED_EXCEPTIONS
diff --git a/makefiles/virtualenv/lib/python3.10/site-packages/sphinx/util/typing.py/patch b/makefiles/virtualenv/lib/python3.10/site-packages/sphinx/util/typing.py/patch
new file mode 100644
index 0000000..da24cd5
--- /dev/null
+++ b/makefiles/virtualenv/lib/python3.10/site-packages/sphinx/util/typing.py/patch
@@ -0,0 +1,18 @@
+--- staging/lib/python3.10/site-packages/sphinx/util/typing.py 2023-09-14 16:25:24.179214865 -0400
++++ .venv/lib/python3.10/site-packages/sphinx/util/typing.py 2023-09-14 16:22:16.545588690 -0400
+@@ -34,14 +34,7 @@
+ return ref._eval_type(globalns, localns)
+
+ if sys.version_info > (3, 10):
+-
+- try: # ver >= 3.10.12
+- from typing import Union as types_Union
+- except ImportError:
+- try: # ver >= 3.10.6
+- from types import Union as types_Union
+- except ImportError:
+- types_Union = None
++ from types import Union as types_Union
+ else:
+ types_Union = None
+
diff --git a/makefiles/virtualenv/lib/python3.10/site-packages/sphinxcontrib/openapi/utils.py/patch b/makefiles/virtualenv/lib/python3.10/site-packages/sphinxcontrib/openapi/utils.py/patch
new file mode 100644
index 0000000..3fda183
--- /dev/null
+++ b/makefiles/virtualenv/lib/python3.10/site-packages/sphinxcontrib/openapi/utils.py/patch
@@ -0,0 +1,29 @@
+--- staging/lib/python3.10/site-packages/sphinxcontrib/openapi/utils.py 2022-12-02 18:47:40.050490678 -0500
++++ venv_docs/lib/python3.10/site-packages/sphinxcontrib/openapi/utils.py 2022-12-02 18:45:53.023832002 -0500
+@@ -10,12 +10,7 @@
+
+ from __future__ import unicode_literals
+
+-try:
+- import collections.abc
+- from collections.abc import Mapping
+-except ImportError: # fall through
+- import collections
+- from collections import Mapping
++import collections
+
+ import jsonschema
+ try:
+@@ -40,10 +35,10 @@
+ resolver = jsonschema.RefResolver(uri, spec)
+
+ def _do_resolve(node):
+- if isinstance(node, Mapping) and '$ref' in node:
++ if isinstance(node, collections.Mapping) and '$ref' in node:
+ with resolver.resolving(node['$ref']) as resolved:
+ return resolved
+- elif isinstance(node, Mapping):
++ elif isinstance(node, collections.Mapping):
+ for k, v in node.items():
+ node[k] = _do_resolve(v)
+ elif isinstance(node, (list, tuple)):
diff --git a/makefiles/virtualenv/python_310_migration.sh b/makefiles/virtualenv/python_310_migration.sh
new file mode 100755
index 0000000..8ce4d5f
--- /dev/null
+++ b/makefiles/virtualenv/python_310_migration.sh
@@ -0,0 +1,113 @@
+#!/bin/bash
+# -----------------------------------------------------------------------
+# Copyright 2022-2023 Open Networking Foundation (ONF) and the ONF Contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# -----------------------------------------------------------------------
+
+##-------------------##
+##---] GLOBALS [---##
+##-------------------##
+set -euo pipefail
+
+## -----------------------------------------------------------------------
+## Intent: Display script documentation.
+## -----------------------------------------------------------------------
+function show_help()
+{
+ cat <<EOH
+Usage: $0
+ apply Patch virtualenv python modules by version (3.10+).
+ backup Create a tarball for work-in-progress.
+ gather Display a list of potential source files to patch.
+
+ --venv Installed venv directory to patch (override default)
+ --help This message
+
+See Also
+ patches/README.md Howto create a patch file.
+
+EOH
+ exit 0
+}
+
+##----------------##
+##---] MAIN [---##
+##----------------##
+declare dst='.venv' # "vst_venv"
+declare src="staging"
+declare pat="patches"
+
+## -----------------------
+## Slurp available patches
+## -----------------------
+pushd "$pat" >/dev/null
+readarray -t fyls < <(find . -name 'patch' -print)
+popd >/dev/null
+
+if [ $# -eq 0 ]; then set -- apply; fi
+
+while [ $# -gt 0 ]; do
+ opt="$1"; shift
+ case "$opt" in
+
+ -*help) show_help ;;
+ -*venv) dst="$1"; shift ;;
+
+ apply)
+ pushd "$dst" >/dev/null || { echo "pushd $dst failed"; exit 1; }
+ for fyl in "${fyls[@]}";
+ do
+ path="${fyl%/*}"
+
+ # Conditional install, jenkins may not support interpreter yet.
+ if [ ! -e "$path" ]; then
+ echo "[SKIP] $path"
+ continue
+ fi
+
+ echo "[APPLY] $path"
+ patch -R -p1 < "../$pat/${path}/patch"
+ done
+ popd >/dev/null || { echo "popd $dst failed"; exit 1; }
+ ;;
+
+ backup)
+ mkdir ~/backups
+ pushd "$src" || { echo "pushd $dst failed"; exit 1; }
+ tar czvf ~/backups/vault."$(date '+%Y%m%d%H%M%S')" "${fyls[@]}"
+ popd || { echo "popd $dst failed"; exit 1; }
+ ;;
+
+ gather)
+ for fyl in "${fyls[@]}";
+ do
+ patchDir="$pat/$fyl"
+ mkdir -p "$patchDir"
+ diff -Naur "$src/$fyl" "$dst/$fyl" | tee "$pat/$fyl/patch"
+ done
+ find "$pat" -print
+ ;;
+
+ help) show_help ;;
+
+ *)
+ echo "ERROR: Unknown action [$opt]"
+ exit 1
+ ;;
+ esac
+
+ echo
+done
+
+# [EOF]