VOL-4977 - Failure in voltha-protos verification job.
Makefile
makefiles/consts.mk
makefiles/include.mk
virtualenv.mk
requirements.txt
--------------------
o Added library makefiles from repo:onf-make to handle virtualenv setup/removal.
o go-clean - augment clean rule, current behavior leaves directory unbuildable.
makefiles/docker/include.mk
---------------------------
o Replace inlined copies of docker cli arguments with macros: is-stdin, docker-run, etc.
o Moved docker command wrappers: PROTOC, etc into docker/include.mk
makefiles/help/*.mk
-------------------
o Update and align makefile target help text.
Change-Id: Ifcf72e8081efc6e0d828ee4b1b96c66a219a39b9
diff --git a/Makefile b/Makefile
index cb7a8cd..09c2531 100755
--- a/Makefile
+++ b/Makefile
@@ -24,10 +24,14 @@
TOP ?= .
MAKEDIR ?= $(TOP)/makefiles
-export SHELL := bash -e -o pipefail
-
$(if $(VERBOSE),$(eval export VERBOSE=$(VERBOSE))) # visible to include(s)
+##--------------------------
+## Enable setup.py debugging
+##--------------------------
+# https://docs.python.org/3/distutils/setupscript.html#debugging-the-setup-script
+export DISTUTILS_DEBUG := 1
+
# Makefile for voltha-protos
default: test
@@ -40,13 +44,6 @@
##--------------------##
include $(MAKEDIR)/include.mk
-# tool containers
-VOLTHA_TOOLS_VERSION ?= 2.4.0
-
-PROTOC = docker run --rm --user $$(id -u):$$(id -g) -v ${CURDIR}:/app $(shell test -t 0 && echo "-it") voltha/voltha-ci-tools:${VOLTHA_TOOLS_VERSION}-protoc protoc
-PROTOC_SH = docker run --rm --user $$(id -u):$$(id -g) -v ${CURDIR}:/go/src/github.com/opencord/voltha-protos/v5 $(shell test -t 0 && echo "-it") --workdir=/go/src/github.com/opencord/voltha-protos/v5 voltha/voltha-ci-tools:${VOLTHA_TOOLS_VERSION}-protoc sh -c
-GO = docker run --rm --user $$(id -u):$$(id -g) -v ${CURDIR}:/app $(shell test -t 0 && echo "-it") -v gocache:/.cache -v gocache-${VOLTHA_TOOLS_VERSION}:/go/pkg voltha/voltha-ci-tools:${VOLTHA_TOOLS_VERSION}-golang go
-
# Function to extract the last path component from go_package line in .proto files
define go_package_path
$(shell grep go_package $(1) | sed -n 's/.*\/\(.*\)";/\1/p')
@@ -67,6 +64,7 @@
PROTO_GO_PB:= $(foreach f, $(PROTO_FILES), $(patsubst protos/voltha_protos/%.proto,$(PROTO_GO_DEST_DIR)/$(call go_package_path,$(f))/%.pb.go,$(f)))
PROTO_JAVA_DEST_DIR := java
PROTO_JAVA_PB := $(foreach f, $(PROTO_FILES), $(patsubst protos/voltha_protos/%.proto,$(PROTO_JAVA_DEST_DIR)/$(call java_package_path,$(f))/%.pb.java,$(f)))
+
# Force pb file to be regenrated every time. Otherwise the make process assumes generated version is still valid
.PHONY: voltha.pb
@@ -83,22 +81,21 @@
test: python-test go-test java-test
-clean: python-clean java-clean go-clean
+clean :: python-clean java-clean go-clean
-sterile: clean
- $(RM) -r venv_protos
+sterile :: clean
# Python targets
python-protos: $(PROTO_PYTHON_PB2)
-venv_protos:
- virtualenv -p python3 $@;\
- source ./$@/bin/activate ; set -u ;\
- pip install grpcio==1.39.0 protobuf==3.17.3 grpcio-tools==1.39.0 googleapis-common-protos==1.52.0
-
-$(PROTO_PYTHON_DEST_DIR)/%_pb2.py: protos/voltha_protos/%.proto Makefile venv_protos
- source ./venv_protos/bin/activate ; set -u ;\
- python -m grpc_tools.protoc \
+## -----------------------------------------------------------------------
+## -----------------------------------------------------------------------
+$(PROTO_PYTHON_DEST_DIR)/%_pb2.py: \
+ protos/voltha_protos/%.proto \
+ Makefile \
+ $(venv-activate-script)
+ $(activate) \
+ && python -m grpc_tools.protoc \
-I protos \
--python_out=python \
--grpc_python_out=python \
@@ -128,28 +125,43 @@
python/__pycache__ \
python/test/__pycache__ \
python/voltha_protos.egg-info \
- venv_protos \
+ "$(venv-abs-path)" \
$(PROTO_PYTHON_DEST_DIR)/*.desc \
$(PROTO_PYTHON_PB2) \
$(PROTO_PYTHON_PB2_GRPC)
-# Why are we removing files under revision control ?
+## -----------------------------------------------------------------------
+## Intent: Revert go to a clean state.
+## o TODO - go/ directory should not be placed under revision control.
+## o Build should retrieve versioned sources from a central repo.
+## -----------------------------------------------------------------------
go-clean:
$(RM) -r go/*
+ $(HIDE)$(MAKE) repair
+
+## -----------------------------------------------------------------------
+## Intent: Recover from a fatal failed build state:
+## o build removes go/ while regenerating prototypes.
+## o chicken-n-egg: make becomes fatal when go/ is removed and proten fails.
+## -----------------------------------------------------------------------
+repair:
+ /usr/bin/env git checkout go
+
# Go targets
go-protos: voltha.pb
@echo "Creating *.go.pb files"
- @${PROTOC_SH} " \
+ @echo "PROTO_FILES=$(PROTO_FILES)" | tr ' ' '\n'
+ ${PROTOC_SH} $(quote-double)\
set -e -o pipefail; \
for x in ${PROTO_FILES}; do \
echo \$$x; \
protoc --go_out=plugins=grpc:/go/src -I protos \$$x; \
- done"
+ done$(quote-double)
voltha.pb:
@echo "Creating $@"
- @${PROTOC} -I protos -I protos/google/api \
+ $(HIDE)${PROTOC} -I protos -I protos/google/api \
--include_imports --include_source_info \
--descriptor_set_out=$@ \
${PROTO_FILES}
@@ -162,12 +174,12 @@
java-protos: voltha.pb
@echo "Creating java files"
@mkdir -p java_temp/src/main/java
- @${PROTOC_SH} " \
+ @${PROTOC_SH} $(quote-double) \
set -e -o pipefail; \
for x in ${PROTO_FILES}; do \
echo \$$x; \
protoc --java_out=java_temp/src/main/java -I protos \$$x; \
- done"
+ done$(quote-double)
#TODO: generate directly to the final location
@mkdir -p java
cp -r java_temp/src/main/java/* java/
diff --git a/README.md b/README.md
index 9ab210c..e3d9da0 100755
--- a/README.md
+++ b/README.md
@@ -103,3 +103,6 @@
`make test` will run tests for all languages.
+## Miscelaneous
+
+[setup.py](https://gerrit.opencord.org/plugins/gitiles/voltha-protos/+/refs/heads/master/setup.py) and [requirements.txt](https://gerrit.opencord.org/plugins/gitiles/voltha-protos/+/refs/heads/master/requirements.txt) versions must be kept in sync.
diff --git a/makefiles/consts.mk b/makefiles/consts.mk
index ce71862..c1306ac 100644
--- a/makefiles/consts.mk
+++ b/makefiles/consts.mk
@@ -13,25 +13,42 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+#
+# SPDX-FileCopyrightText: 2022-2023 Open Networking Foundation (ONF) and the ONF Contributors
+# SPDX-License-Identifier: Apache-2.0
+# -----------------------------------------------------------------------
+# https://gerrit.opencord.org/plugins/gitiles/onf-make
+# ONF.makefile.version = 1.0
# -----------------------------------------------------------------------
$(if $(DEBUG),$(warning ENTER))
-##-------------------##
-##---] GLOBALS [---##
-##-------------------##
-null :=#
-space :=$(null) $(null)
-dot :=.
+# include makefiles/constants.mk
+export dot :=.
+export null :=#
+export space := $(null) $(null)
+export quote-single := $(null)'$(null)#'
+export quote-double := $(null)"$(null)#"
-HIDE ?=@
+# [DEBUG] make {target} HIDE=
+HIDE ?= @
-env-clean = /usr/bin/env --ignore-environment
-xargs-n1 := xargs -0 -t -n1 --no-run-if-empty
+env-clean ?= /usr/bin/env --ignore-environment
+xargs-n1 := xargs -0 -t -n1 --no-run-if-empty
+xargs-n1-clean := $(env-clean) $(xargs-n1)
+## -----------------------------------------------------------------------
+## Default shell:
+## o set -e enable error checking
+## o set -u report undefined errors
+## o set -o pipefail propogate shell pipeline failures.
+## -----------------------------------------------------------------------
+SHELL ?= /bin/bash
have-shell-bash := $(filter bash,$(subst /,$(space),$(SHELL)))
$(if $(have-shell-bash),$(null),\
- $(eval export SHELL := /bin/bash -euo pipefail))
+ $(eval export SHELL := bash -euo pipefail))
+
+export SHELL ?= bash -euo pipefail
$(if $(DEBUG),$(warning LEAVE))
diff --git a/makefiles/docker/include.mk b/makefiles/docker/include.mk
new file mode 100644
index 0000000..d78d008
--- /dev/null
+++ b/makefiles/docker/include.mk
@@ -0,0 +1,38 @@
+# -*- makefile -*-
+# -----------------------------------------------------------------------
+# Copyright 2017-2023 Open Networking Foundation (ONF) and the ONF Contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.d
+# -----------------------------------------------------------------------
+
+$(if $(DEBUG),$(warning ENTER))
+
+# tool containers
+VOLTHA_TOOLS_VERSION ?= 2.4.0
+
+docker-run = docker run --rm --user $$(id -u):$$(id -g)# # Docker command stem
+docker-run-app = $(docker-run) -v ${CURDIR}:/app# # w/filesystem mount
+is-stdin = $(shell test -t 0 && echo "-it")
+
+# Docker volume mounts: container:/app/release <=> localhost:{pwd}/release
+vee-golang = -v gocache-${VOLTHA_TOOLS_VERSION}:/go/pkg
+vee-citools = voltha/voltha-ci-tools:${VOLTHA_TOOLS_VERSION}
+
+# Tool Containers
+PROTOC = $(docker-run) -v ${CURDIR}:/app $(is-stdin) $(vee-citools)-protoc protoc
+PROTOC_SH = $(docker-run) -v ${CURDIR}:/go/src/github.com/opencord/voltha-protos/v5 $(is-stdin) --workdir=/go/src/github.com/opencord/voltha-protos/v5 $(vee-citools)-protoc sh -c
+GO = $(docker-run) -v ${CURDIR}:/app $(is-stdin) -v gocache:/.cache $(vee-golang) $(vee-citools)-golang go
+
+$(if $(DEBUG),$(warning LEAVE))
+
+# [EOF]
diff --git a/makefiles/help/include.mk b/makefiles/help/include.mk
index 7a93bd0..ffd390f 100644
--- a/makefiles/help/include.mk
+++ b/makefiles/help/include.mk
@@ -18,29 +18,28 @@
# Parent makefile should include this early so help
# message will be prefixed by a usage statement.
help ::
- @echo "Usage: $(MAKE) [options] [target] ..."
@echo
- @echo '[Virtual Env]'
- @echo ' venv Create a python virtual environment'
- @echo ' venv_protos Create a python virtual environment'
- @echo " $(VENV_NAME)"
+ @echo "Usage: $(MAKE) [options] [target] ..."
+
@echo
@echo '[ACTION] - golang, java and python'
@echo ' protos'
@echo ' build'
@echo ' test'
- @echo
- @echo '[CLEAN]'
- @echo ' clean Remove generated targets'
- @echo ' sterile clean + remove virtual env interpreter install'
+
@echo
@echo '[HELP]'
- @echo ' help Display program help'
+ @echo ' help Display program help'
# @echo ' help-verbose Display additional targets and help'
+
@echo
@echo '[PROTOS: generate]'
@echo ' go-protos'
@echo ' java-protos'
@echo ' python-protos'
+ @echo
+ @echo '[MISC]'
+ @echo ' repair Recover from a common fatal build condition'
+
# [EOF]
diff --git a/makefiles/include.mk b/makefiles/include.mk
index 919ddaf..3cdbf2c 100644
--- a/makefiles/include.mk
+++ b/makefiles/include.mk
@@ -32,12 +32,17 @@
##--------------------##
##---] INCLUDES [---##
##--------------------##
-include $(MAKEDIR)/consts.mk
include $(MAKEDIR)/help/include.mk
+
+include $(MAKEDIR)/consts.mk
+include $(MAKEDIR)/virtualenv.mk
+
include $(MAKEDIR)/help/variables.mk
-include $(MAKEDIR)/lint/include.mk
include $(MAKEDIR)/todo.mk
+include $(MAKEDIR)/docker/include.mk
+include $(MAKEDIR)/lint/include.mk
+
$(if $(DEBUG),$(warning LEAVE))
# [EOF]
diff --git a/makefiles/lint/json.mk b/makefiles/lint/json.mk
index c3d5447..96eb87a 100644
--- a/makefiles/lint/json.mk
+++ b/makefiles/lint/json.mk
@@ -32,6 +32,6 @@
done
help::
- @echo " lint-json Syntax check json sources"
+ @echo " lint-json Syntax check json sources"
# [EOF]
diff --git a/makefiles/lint/makefile.mk b/makefiles/lint/makefile.mk
index 61de008..3cf41d2 100644
--- a/makefiles/lint/makefile.mk
+++ b/makefiles/lint/makefile.mk
@@ -37,6 +37,6 @@
## -----------------------------------------------------------------------
## -----------------------------------------------------------------------
help ::
- @echo ' lint-make Syntax check [Mm]akefile and *.mk'
+ @echo ' lint-make Syntax check [Mm]akefile and *.mk'
# [EOF]
diff --git a/makefiles/lint/python.mk b/makefiles/lint/python.mk
index 9252657..592ae98 100644
--- a/makefiles/lint/python.mk
+++ b/makefiles/lint/python.mk
@@ -34,6 +34,6 @@
## -----------------------------------------------------------------------
## -----------------------------------------------------------------------
help ::
- @echo ' lint-python Syntax check python sources'
+ @echo ' lint-python Syntax check python sources'
# [EOF]
diff --git a/makefiles/lint/robot.mk b/makefiles/lint/robot.mk
index f33c4ba..dfbf034 100644
--- a/makefiles/lint/robot.mk
+++ b/makefiles/lint/robot.mk
@@ -38,6 +38,6 @@
; rflint $(LINT_ARGS) $(ROBOT_FILES)
help::
- @echo " lint-robot Syntax check robot sources using rflint"
+ @echo " lint-robot Syntax check robot sources using rflint"
# [EOF]
diff --git a/makefiles/lint/shell.mk b/makefiles/lint/shell.mk
index 0180e90..5ccc34f 100644
--- a/makefiles/lint/shell.mk
+++ b/makefiles/lint/shell.mk
@@ -34,6 +34,6 @@
## -----------------------------------------------------------------------
## -----------------------------------------------------------------------
help ::
- @echo ' lint-shell Syntax check shell sources'
+ @echo ' lint-shell Syntax check shell sources'
# [EOF]
diff --git a/makefiles/lint/yaml/yamllint.mk b/makefiles/lint/yaml/yamllint.mk
index 0cf75de..b3ce6a5 100644
--- a/makefiles/lint/yaml/yamllint.mk
+++ b/makefiles/lint/yaml/yamllint.mk
@@ -42,7 +42,7 @@
## -----------------------------------------------------------------------
## -----------------------------------------------------------------------
help::
- @echo " lint-yaml Syntax check yaml sources"
+ @echo " lint-yaml Syntax check yaml sources"
$(if $(DEBUG),$(warning ENTER))
diff --git a/makefiles/todo.mk b/makefiles/todo.mk
index e2e826f..e59423d 100644
--- a/makefiles/todo.mk
+++ b/makefiles/todo.mk
@@ -23,7 +23,7 @@
## -----------------------------------------------------------------------
## -----------------------------------------------------------------------
help::
- @echo " todo Display future enhancement list"
+ @echo " todo Display future enhancement list"
todo ::
@echo
diff --git a/makefiles/virtualenv.mk b/makefiles/virtualenv.mk
new file mode 100644
index 0000000..93eb29d
--- /dev/null
+++ b/makefiles/virtualenv.mk
@@ -0,0 +1,82 @@
+# -*- makefile -*-
+## -----------------------------------------------------------------------
+# Copyright 2017-2023 Open Networking Foundation (ONF) and the ONF Contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+## -----------------------------------------------------------------------
+# https://gerrit.opencord.org/plugins/gitiles/onf-make
+# ONF.makefile.version = 1.0
+# -----------------------------------------------------------------------
+
+$(if $(DEBUG),$(warning ENTER))
+
+##-------------------##
+##---] GLOBALS [---##
+##-------------------##
+.PHONY: venv
+
+##------------------##
+##---] LOCALS [---##
+##------------------##
+venv-name ?= .venv# # default install directory
+venv-abs-path := $(PWD)/$(venv-name)
+
+venv-activate-script := $(venv-name)/bin/activate# # dependency
+
+# Intent: activate= is a macro for accessing the virtualenv activation script#
+# Usage: $(activate) && python
+activate ?= set +u && source $(venv-activate-script) && set -u
+
+## -----------------------------------------------------------------------
+## Intent: Activate script path dependency
+## Usage:
+## o place on the right side of colon as a target dependency
+## o When the script does not exist install the virtual env and display.
+## -----------------------------------------------------------------------
+$(venv-activate-script):
+ @echo
+ @echo "============================="
+ @echo "Installing python virtual env"
+ @echo "============================="
+ virtualenv -p python3 $(venv-name)
+ $(activate) && python -m pip install --upgrade pip
+ $(activate) && pip install --upgrade setuptools
+ $(activate) && { [[ -r requirements.txt ]] && python -m pip install -r requirements.txt; }
+ $(activate) && python --version
+
+## -----------------------------------------------------------------------
+## Intent: Explicit named installer target w/o dependencies.
+## Makefile targets should depend on venv-activate-script.
+## -----------------------------------------------------------------------
+venv: $(venv-activate-script)
+
+## -----------------------------------------------------------------------
+## Intent: Revert installation to a clean checkout
+## -----------------------------------------------------------------------
+sterile :: clean
+ $(RM) -r "$(venv-abs-path)"
+
+## -----------------------------------------------------------------------
+## -----------------------------------------------------------------------
+help ::
+ @echo
+ @echo '[VIRTUAL ENV]'
+ @echo ' venv Create a python virtual environment'
+ @echo ' venv-name= Subdir name for virtualenv install'
+ @echo ' venv-activate-script make macro name'
+ @echo ' $$(target) dependency install python virtualenv'
+ @echo ' source $$(macro) && cmd configure env and run cmd'
+
+$(if $(DEBUG),$(warning LEAVE))
+
+# [EOF]
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..98f43ad
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,5 @@
+grpcio==1.39.0
+protobuf==3.17.3
+grpcio-tools==1.39.0
+googleapis-common-protos==1.52.0
+nose2[coverage_plugin]