VOL-4874 - Fix local lint target use problems.
scripts/which_deployment_owns_device.sh
---------------------------------------
* Fix a few shellcheck complaints.
makefiles/lint/robot.mk
-----------------------
* Update target logic to use && -vs- ;
* semicolon use can mask errors.
makefiles/lint/python.mk
------------------------
* Update target logic to use && -vs- ;
* Removed pylint option --py3k, switch not suppored by newer interpreters.
makefiles/lint.mk
makefiles/lint/shell.mk
-----------------------
* Added a lint target to invoke shellcheck on sources.
Makefile
--------
* target:clean modified to remove generated sources.
* target:gendocs: unique $(LIBDIRS) and replace for loop with a simple mkdir -vp call.
* cosmetic indentation cleanup.
* modified vst_venv target to apply patches after virtualenv created.
* patches address failures caused by python 3.10+ requiring collections.abc
* Ignore exit status from 310_migration for now to avoid failing other jobs.
patches/lib/python3.10/site-packages/robot/utils/normalizing.py/patch
patches/lib/python3.10/site-packages/robot/utils/robottypes3.py/patch
---------------------------------------------------------------------
* Attempt to import the new collections.abc module for Mapping and
friends so python 3.10+ can be supported.
* Fall back to existing module import supported by python interpreters
v3.5 <=> v3.9.
patches/python_310_migration.sh
-------------------------------
* Self contained helper script used to generate and apply python 3.10+ patches.
Followup edits
--------------
* Added missing license blocks reported by jenkins.
* Updated lint checking to detect license errors so problems can be fixed prior to checkin.
* Add set -x debugging, not clear why pyenv patching failed.
Change-Id: I2b8515bb26874eb8b1564bec87353ea0c1485b61
diff --git a/makefiles/lint/json.mk b/makefiles/lint/json.mk
index 0a11c71..cc9ecfa 100644
--- a/makefiles/lint/json.mk
+++ b/makefiles/lint/json.mk
@@ -15,7 +15,7 @@
# limitations under the License.
# -----------------------------------------------------------------------
-JSON_FILES ?= $(error JSON_FILES= is rqeuired)
+JSON_FILES ?= $(error JSON_FILES= is required)
.PHONY: lint-json
diff --git a/makefiles/lint/license/include.mk b/makefiles/lint/license/include.mk
new file mode 100644
index 0000000..61b0cfb
--- /dev/null
+++ b/makefiles/lint/license/include.mk
@@ -0,0 +1,82 @@
+# -*- makefile -*-
+# -----------------------------------------------------------------------
+# Copyright 2022 Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# -----------------------------------------------------------------------
+
+JSON_FILES ?= $(error JSON_FILES= is required)
+
+.PHONY: lint-license
+
+## -----------------------------------------------------------------------
+## -----------------------------------------------------------------------
+lint : lint-license
+
+lint-license-gargs += --recursive
+
+# ignore: png, xlsx
+# will utf8 be excluded(?)
+lint-license-gargs += --binary-files=without-match
+lint-license-gargs += --files-without-match
+
+# [TODO] license checking accepts either Copy or Apache.
+# [TODO] At least Copyright should be required (both?)
+lint-license-gargs += --extended-regexp
+lint-license-gargs += -e 'Copyright[[:space:]]+[[:digit:]]{4}'
+lint-license-gargs += -e 'Apache License'
+
+# [TODO] --strict, --strict-dates
+
+# TODO: Normalize into .venv for consistent filtering across projects.
+lint-license-gargs += --exclude-dir='.git'
+lint-license-gargs += --exclude-dir='vst_venv'
+lint-license-gargs += --exclude-dir='flog'
+
+lint-license-gargs += --exclude='*.json'
+lint-license-gargs += --exclude='*.md'
+lint-license-gargs += --exclude='*.pyc'
+lint-license-gargs += --exclude='*.xml'
+
+# [FILE(s)]
+lint-license-gargs += --exclude='VERSION'
+
+# [GIT]
+# lint-license-gargs += --exclude='.gitignore'
+# lint-license-gargs += --exclude='.gitreview'
+lint-license-gargs += --exclude='\.*'
+
+# [PYTHON]
+lint-license-gargs += --exclude='requirements.txt'
+
+# [WIP]
+lint-license-gargs += --exclude='patch'
+
+## -----------------------------------------------------------------------
+## -----------------------------------------------------------------------
+lint-license-new:
+ grep $(lint-license-gargs) $(dot)
+
+## -----------------------------------------------------------------------
+## Jenkins job checking logic.
+## -----------------------------------------------------------------------
+lint-license:
+ $(MAKEDIR)/lint/license/license-check.sh
+
+## -----------------------------------------------------------------------
+## -----------------------------------------------------------------------
+help::
+ @echo " lint-license Verify sources contain a license block."
+ @echo " lint-license-new Grep driven replacement logic."
+
+# [EOF]
diff --git a/makefiles/lint/license/license-check.sh b/makefiles/lint/license/license-check.sh
new file mode 100755
index 0000000..10d3376
--- /dev/null
+++ b/makefiles/lint/license/license-check.sh
@@ -0,0 +1,159 @@
+#!/usr/bin/env bash
+
+# -----------------------------------------------------------------------
+# Copyright 2022 Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# -----------------------------------------------------------------------
+
+# licensecheck.sh
+# checks for copyright/license headers on files
+# excludes filename extensions where this check isn't pertinent
+
+# --strict
+# --strict-dates
+# see https://github...
+
+## ---------------------------------------------------------------------------
+## find: warning: ‘-name’ matches against basenames only, but the given pattern
+## contains a directory separator (‘/’), thus the expression will evaluate to
+## false all the time. Did you mean ‘-wholename’?
+##
+## [TODO] find and fix plugin script, change --name to --path:
+## find ! -name "*/docs/*"
+## ---------------------------------------------------------------------------
+## [TODO] see license/include.mk, grep -L will ignore binaries so no need to
+## explicitly filter images, spreadsheets, etc. Files containing utf-8 are
+## the only question mark.
+## ---------------------------------------------------------------------------
+
+set +e -u -o pipefail
+fail_licensecheck=0
+
+declare -a gargs=()
+gargs+=('--extended-regexp')
+
+# Evil regex -- scripts detecting pattern are excluded from checking.
+gargs+=('-e' 'Apache License')
+
+# Good regex -- no false positives.
+gargs+=('-e' 'Copyright[[:space:]]+[[:digit:]]{4}')
+
+while IFS= read -r -d '' path
+do
+ if ! grep -q "${gargs[@]}" "${path}";
+ then
+ echo "ERROR: $path does not contain License Header"
+ fail_licensecheck=1
+ fi
+done < <(find . -name ".git" -prune -o -type f \
+ ! -iname "*.png" \
+ ! -name "*.asc" \
+ ! -name "*.bat" \
+ ! -name "*.bin" \
+ ! -name "*.cert" \
+ ! -name "*.cfg" \
+ ! -name "*.cnf" \
+ ! -name "*.conf" \
+ ! -name "*.cql" \
+ ! -name "*.crt" \
+ ! -name "*.csar" \
+ ! -name "*.csr" \
+ ! -name "*.csv" \
+ ! -name "*.ctmpl" \
+ ! -name "*.curl" \
+ ! -name "*.db" \
+ ! -name "*.der" \
+ ! -name "*.desc" \
+ ! -name "*.diff" \
+ ! -name "*.dnsmasq" \
+ ! -name "*.do" \
+ ! -name "*.docx" \
+ ! -name "*.eot" \
+ ! -name "*.gif" \
+ ! -name "*.gpg" \
+ ! -name "*.graffle" \
+ ! -name "*.ico" \
+ ! -name "*.iml" \
+ ! -name "*.in" \
+ ! -name "*.inc" \
+ ! -name "*.install" \
+ ! -name "*.j2" \
+ ! -name "*.jar" \
+ ! -name "*.jks" \
+ ! -name "*.jpg" \
+ ! -name "*.json" \
+ ! -name "*.jsonld" \
+ ! -name "*.JSON" \
+ ! -name "*.key" \
+ ! -name "*.list" \
+ ! -name "*.local" \
+ ! -path "*.lock" \
+ ! -name "*.log" \
+ ! -name "*.mak" \
+ ! -name "*.md" \
+ ! -name "*.MF" \
+ ! -name "*.oar" \
+ ! -name "*.p12" \
+ ! -name "*.patch" \
+ ! -name "*.pb.go" \
+ ! -name "*.pb.gw.go" \
+ ! -name "*.pdf" \
+ ! -name "*.pcap" \
+ ! -name "*.pem" \
+ ! -name "*.properties" \
+ ! -name "*.proto" \
+ ! -name "*.protoset" \
+ ! -name "*.pyc" \
+ ! -name "*.repo" \
+ ! -name "*.robot" \
+ ! -name "*.rst" \
+ ! -name "*.rules" \
+ ! -name "*.service" \
+ ! -name "*.svg" \
+ ! -name "*.swp" \
+ ! -name "*.tar" \
+ ! -name "*.tar.gz" \
+ ! -name "*.toml" \
+ ! -name "*.ttf" \
+ ! -name "*.txt" \
+ ! -name "*.woff" \
+ ! -name "*.xproto" \
+ ! -name "*.xtarget" \
+ ! -name "*ignore" \
+ ! -name "*rc" \
+ ! -name "*_pb2.py" \
+ ! -name "*_pb2_grpc.py" \
+ ! -name "Dockerfile" \
+ ! -name "Dockerfile.*" \
+ ! -name "go.mod" \
+ ! -name "go.sum" \
+ ! -name "README" \
+ ! -path "*/vendor/*" \
+ ! -path "*conf*" \
+ ! -path "*git*" \
+ ! -path "*swagger*" \
+ ! -path "*.drawio" \
+ ! -name "*.pb.h" \
+ ! -name "*.pb.cc" \
+ ! -path "*/docs/*" \
+ ! -name 'output.xml' \
+ ! -path "*/vst_venv/*" \
+ ! -name '*#*' \
+ ! -path '*scripts/flog/*' \
+ ! -name '*~' \
+ ! -name 'VERSION' \
+ ! -name 'patch' \
+ -print0 )
+
+exit ${fail_licensecheck}
diff --git a/makefiles/lint/license/license-check.sh.safe b/makefiles/lint/license/license-check.sh.safe
new file mode 100755
index 0000000..cd82c11
--- /dev/null
+++ b/makefiles/lint/license/license-check.sh.safe
@@ -0,0 +1,154 @@
+#!/usr/bin/env bash
+
+# -----------------------------------------------------------------------
+# Copyright 2022 Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# -----------------------------------------------------------------------
+
+# licensecheck.sh
+# checks for copyright/license headers on files
+# excludes filename extensions where this check isn't pertinent
+
+# --strict
+# --strict-dates
+# see https://github...
+
+## ---------------------------------------------------------------------------
+## find: warning: ‘-name’ matches against basenames only, but the given pattern
+## contains a directory separator (‘/’), thus the expression will evaluate to
+## false all the time. Did you mean ‘-wholename’?
+##
+## [TODO] find and fix plugin script, change --name to --path:
+## find ! -name "*/docs/*"
+## ---------------------------------------------------------------------------
+
+set +e -u -o pipefail
+fail_licensecheck=0
+
+declare -a gargs=()
+gargs+=('--extended-regexp')
+gargs+=('-e' 'Copyright[[:space:]]+[[:digit:]]{4}')
+# gargs+=('-e' 'Apache License')
+
+while IFS= read -r -d '' f
+do
+ grep -q "${gargs[@]}" "${f}"
+ rc=$?
+ if [[ $rc != 0 ]]; then
+ echo "ERROR: $f does not contain License Header"
+ fail_licensecheck=1
+ fi
+done < <(find . -name ".git" -prune -o -type f \
+ -name "*.*" \
+ ! -name "*.PNG" \
+ ! -name "*.asc" \
+ ! -name "*.bat" \
+ ! -name "*.bin" \
+ ! -name "*.cert" \
+ ! -name "*.cfg" \
+ ! -name "*.cnf" \
+ ! -name "*.conf" \
+ ! -name "*.cql" \
+ ! -name "*.crt" \
+ ! -name "*.csar" \
+ ! -name "*.csr" \
+ ! -name "*.csv" \
+ ! -name "*.ctmpl" \
+ ! -name "*.curl" \
+ ! -name "*.db" \
+ ! -name "*.der" \
+ ! -name "*.desc" \
+ ! -name "*.diff" \
+ ! -name "*.dnsmasq" \
+ ! -name "*.do" \
+ ! -name "*.docx" \
+ ! -name "*.eot" \
+ ! -name "*.gif" \
+ ! -name "*.gpg" \
+ ! -name "*.graffle" \
+ ! -name "*.ico" \
+ ! -name "*.iml" \
+ ! -name "*.in" \
+ ! -name "*.inc" \
+ ! -name "*.install" \
+ ! -name "*.j2" \
+ ! -name "*.jar" \
+ ! -name "*.jks" \
+ ! -name "*.jpg" \
+ ! -name "*.json" \
+ ! -name "*.jsonld" \
+ ! -name "*.JSON" \
+ ! -name "*.key" \
+ ! -name "*.list" \
+ ! -name "*.local" \
+ ! -path "*.lock" \
+ ! -name "*.log" \
+ ! -name "*.mak" \
+ ! -name "*.md" \
+ ! -name "*.MF" \
+ ! -name "*.mk" \
+ ! -name "*.oar" \
+ ! -name "*.p12" \
+ ! -name "*.patch" \
+ ! -name "*.pb.go" \
+ ! -name "*.pb.gw.go" \
+ ! -name "*.pdf" \
+ ! -name "*.pcap" \
+ ! -name "*.pem" \
+ ! -name "*.png" \
+ ! -name "*.properties" \
+ ! -name "*.proto" \
+ ! -name "*.protoset" \
+ ! -name "*.pyc" \
+ ! -name "*.repo" \
+ ! -name "*.robot" \
+ ! -name "*.rst" \
+ ! -name "*.rules" \
+ ! -name "*.service" \
+ ! -name "*.svg" \
+ ! -name "*.swp" \
+ ! -name "*.tar" \
+ ! -name "*.tar.gz" \
+ ! -name "*.toml" \
+ ! -name "*.ttf" \
+ ! -name "*.txt" \
+ ! -name "*.woff" \
+ ! -name "*.xproto" \
+ ! -name "*.xtarget" \
+ ! -name "*ignore" \
+ ! -name "*rc" \
+ ! -name "*_pb2.py" \
+ ! -name "*_pb2_grpc.py" \
+ ! -name "Dockerfile" \
+ ! -name "Dockerfile.*" \
+ ! -name "go.mod" \
+ ! -name "go.sum" \
+ ! -name "Makefile" \
+ ! -name "Makefile.*" \
+ ! -name "README" \
+ ! -path "*/vendor/*" \
+ ! -path "*conf*" \
+ ! -path "*git*" \
+ ! -path "*swagger*" \
+ ! -path "*.drawio" \
+ ! -name "*.pb.h" \
+ ! -name "*.pb.cc" \
+ ! -path "*/docs/*" \
+ ! -name 'output.xml' \
+ ! -path "*/vst_venv/*" \
+ ! -name '*#*' \
+ ! -name '*~' \
+ -print0 )
+
+exit ${fail_licensecheck}
diff --git a/makefiles/lint/license/license-helper.sh b/makefiles/lint/license/license-helper.sh
new file mode 100755
index 0000000..499f854
--- /dev/null
+++ b/makefiles/lint/license/license-helper.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+# -----------------------------------------------------------------------
+# Copyright 2022 Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# -----------------------------------------------------------------------
+
+declare -i status=$#
+
+while [ $# -gt 0 ]; do
+ arg="$1"; shift
+ echo "ERROR: Detected missing license header: ${arg}"
+done
+
+if [ $status -ne 0 ]; then
+ exit 1
+fi
+
+/bin/true # set $?
+
+# [EOF]
diff --git a/makefiles/lint/python.mk b/makefiles/lint/python.mk
index 5aecd58..86503a7 100644
--- a/makefiles/lint/python.mk
+++ b/makefiles/lint/python.mk
@@ -21,12 +21,11 @@
lint : lint-python
-# check deps for format and python3 cleanliness
lint-python: vst_venv
- source ./$</bin/activate \
- ; set -u \
- ; pylint --py3k $(PYTHON_FILES) \
- ; flake8 --max-line-length=99 --count $(PYTHON_FILES)
+ -source ./$</bin/activate \
+ && set -u \
+ && pylint $(PYTHON_FILES) \
+ && flake8 --max-line-length=99 --count $(PYTHON_FILES)
help::
@echo " lint-python Syntax check using pylint and flake8"
diff --git a/makefiles/lint/robot.mk b/makefiles/lint/robot.mk
index ec5579e..9808602 100644
--- a/makefiles/lint/robot.mk
+++ b/makefiles/lint/robot.mk
@@ -32,8 +32,8 @@
lint-robot: vst_venv
source ./$</bin/activate \
- ; set -u \
- ; rflint $(LINT_ARGS) $(ROBOT_FILES)
+ && set -u \
+ && rflint $(LINT_ARGS) $(ROBOT_FILES)
help::
@echo " lint-robot Syntax check robot sources using rflint"
diff --git a/makefiles/lint/shell.mk b/makefiles/lint/shell.mk
new file mode 100644
index 0000000..da92921
--- /dev/null
+++ b/makefiles/lint/shell.mk
@@ -0,0 +1,33 @@
+# -*- makefile -*-
+# -----------------------------------------------------------------------
+# Copyright 2017-2022 Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# -----------------------------------------------------------------------
+
+JSON_FILES ?= $(error JSON_FILES= is required)
+
+.PHONY: lint-shell
+
+lint : lint-shell
+
+lint-shell:
+ shellcheck --version
+ find . \( -name 'staging' -o -name 'vst_venv' \) -prune \
+ -o -name '*.sh' ! -name 'activate.sh' -print0 \
+ | xargs -0 -n1 shellcheck
+
+help::
+ @echo " lint-shell Syntax check bash,bourne,etc sources"
+
+# [EOF]