[VOL-1460]
Build Go stubs using the Makefile, instead of in a separate script.
New method is independent of GOPATH (which makes testing easier), and is
'go_package' aware, so any new .proto files will have directories
created and regenerated under go/
Cleaned up README.md and Makefile
Lexically sort input to protoc as --descriptor_set_out differs depending
on the order of file arguments passed to it.
Released v0.1.2
Change-Id: If9d5aabc89b4d73f3e069b01b38bd0bf8d1b3c43
diff --git a/.gitignore b/.gitignore
index 9713c4f..5d6f023 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,6 +6,9 @@
.tox
venv_protos
+# go related
+go_temp
+
# generated files that shouldn't be committed
*_pb2.py
*_pb2_grpc.py
diff --git a/Makefile b/Makefile
index f83e7e2..1eea24f 100644
--- a/Makefile
+++ b/Makefile
@@ -15,11 +15,22 @@
# Makefile for voltha-protos
default: test
+# set default shell options
+SHELL = bash -e -o pipefail
+
+# Function to extract the last path component from go_package line in .proto files
+define go_package_path
+$(shell grep go_package $(1) | sed -n 's/.*\/\(.*\)";/\1/p')
+endef
+
# Variables
-PROTO_FILES := $(wildcard protos/voltha_protos/*.proto)
+PROTO_FILES := $(sort $(wildcard protos/voltha_protos/*.proto))
+
PROTO_PYTHON_DEST_DIR := python/voltha_protos
PROTO_PYTHON_PB2 := $(foreach f, $(PROTO_FILES), $(patsubst protos/voltha_protos/%.proto,$(PROTO_PYTHON_DEST_DIR)/%_pb2.py,$(f)))
PROTO_PYTHON_PB2_GRPC := $(foreach f, $(PROTO_FILES), $(patsubst protos/voltha_protos/%.proto,$(PROTO_PYTHON_DEST_DIR)/%_pb2_grpc.py,$(f)))
+PROTO_GO_DEST_DIR := go
+PROTO_GO_PB:= $(foreach f, $(PROTO_FILES), $(patsubst protos/voltha_protos/%.proto,$(PROTO_GO_DEST_DIR)/$(call go_package_path,$(f))/%.pb.go,$(f)))
PROTOC_PREFIX := /usr/local
PROTOC_VERSION := "3.7.0"
@@ -30,11 +41,9 @@
PROTOC_BUILD_TMP_DIR := "/tmp/protobuf-build-$(shell uname -s | tr '[:upper:]' '[:lower:]')"
print:
- echo "Proto files: $(PROTO_FILES)"
- echo "Python PB2 files: $(PROTO_PYTHON_PB2)"
-
-# set default shell
-SHELL = bash -e -o pipefail
+ @echo "Proto files: $(PROTO_FILES)"
+ @echo "Python PB2 files: $(PROTO_PYTHON_PB2)"
+ @echo "Go PB files: $(PROTO_GO_PB)"
# Generic targets
protos: python-protos go-protos
@@ -56,30 +65,40 @@
$(PROTO_PYTHON_DEST_DIR)/%_pb2.py: protos/voltha_protos/%.proto Makefile venv_protos
source ./venv_protos/bin/activate ; set -u ;\
python -m grpc_tools.protoc \
- -I protos \
- --python_out=python \
- --grpc_python_out=python \
- --descriptor_set_out=$(PROTO_PYTHON_DEST_DIR)/$(basename $(notdir $<)).desc \
- --include_imports \
- --include_source_info \
- $<
+ -I protos \
+ --python_out=python \
+ --grpc_python_out=python \
+ --descriptor_set_out=$(PROTO_PYTHON_DEST_DIR)/$(basename $(notdir $<)).desc \
+ --include_imports \
+ --include_source_info \
+ $<
-python-build: setup.py
+python-build: setup.py python-protos
python ./setup.py sdist
python-test: tox.ini setup.py python-protos
tox
python-clean:
- rm -rf venv_protos .coverage coverage.xml nose2-results.xml dist $(PROTO_PYTHON_PB2) $(PROTO_PYTHON_PB2_GRPC) $(PROTO_PYTHON_DEST_DIR)/*.desc
find python/ -name '*.pyc' | xargs rm -f
- rm -rf python/voltha_protos.egg-info
- rm -rf .tox
- rm -rf python/__pycache__/
- rm -rf python/test/__pycache__/
+ rm -rf \
+ .coverage \
+ .tox \
+ coverage.xml \
+ dist \
+ nose2-results.xml \
+ python/__pycache__ \
+ python/test/__pycache__ \
+ python/voltha_protos.egg-info \
+ venv_protos \
+ $(PROTO_PYTHON_DEST_DIR)/*.desc \
+ $(PROTO_PYTHON_PB2) \
+ $(PROTO_PYTHON_PB2_GRPC)
# Go targets
-go-protos:
+go-protos: protoc_check_version $(PROTO_GO_PB) go/voltha.pb
+
+protoc_check_version:
ifeq ("", "$(shell which protoc)")
@echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
@echo "It looks like you don't have a version of protocol buffer tools."
@@ -88,13 +107,30 @@
@echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
exit 1
endif
- ./build_go_protos.sh protos
+
+go_temp:
+ mkdir -p go_temp
+
+$(PROTO_GO_PB): $(PROTO_FILES) go_temp
+ @echo "Creating $@"
+ cd protos && protoc \
+ --go_out=MAPS=Mgoogle/protobuf/descriptor.proto=github.com/golang/protobuf/protoc-gen-go/descriptor,plugins=grpc,paths=source_relative:../go_temp \
+ -I . voltha_protos/$$(echo $@ | sed -n 's/.*\/\(.*\).pb.go/\1.proto/p' )
+ mkdir -p $(dir $@)
+ mv go_temp/voltha_protos/$(notdir $@) $@
+
+go/voltha.pb: ${PROTO_FILES}
+ @echo "Creating $@"
+ protoc -I protos -I protos/google/api \
+ --include_imports --include_source_info \
+ --descriptor_set_out=$@ \
+ ${PROTO_FILES}
go-test:
ifneq ("libprotoc 3.7.0", "$(shell protoc --version)")
@echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
@echo "It looks like you don't have protocol buffer tools ${PROTOC_VERSION} installed."
->---@echo "To install this version, you can run:"
+ @echo "To install this version, you can run:"
@echo " make install-protoc"
@echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
exit 1
@@ -102,7 +138,7 @@
test/test-go-proto-consistency.sh
go-clean:
- echo "FIXME: Add golang cleanup"
+ rm -rf go_temp
install-protoc:
@echo "Downloading and installing protocol buffer support."
@@ -111,11 +147,11 @@
mkdir -p $(PROTOC_BUILD_TMP_DIR)
@echo "We ask for sudo credentials now so we can install at the end"; \
sudo echo "Thanks"; \
- cd $(PROTOC_BUILD_TMP_DIR); \
- wget $(PROTOC_DOWNLOAD_URI); \
- tar xzvf $(PROTOC_TARBALL); \
- cd $(PROTOC_DIR); \
- ./configure --prefix=$(PROTOC_PREFIX); \
- make; \
- sudo make install; \
- sudo ldconfig
+ cd $(PROTOC_BUILD_TMP_DIR); \
+ wget $(PROTOC_DOWNLOAD_URI); \
+ tar xzvf $(PROTOC_TARBALL); \
+ cd $(PROTOC_DIR); \
+ ./configure --prefix=$(PROTOC_PREFIX); \
+ make; \
+ sudo make install; \
+ sudo ldconfig
diff --git a/README.md b/README.md
index 208c839..6bcf0f0 100644
--- a/README.md
+++ b/README.md
@@ -2,19 +2,27 @@
Protobuf files used by [VOLTHA](https://wiki.opencord.org/display/CORD/VOLTHA).
-Currently this is used to generate both Golang and Python protobufs and gRPC stubs.
+Currently this is used to generate both Golang and Python protobufs and gRPC
+stubs.
-Protobuf definition files are located in `protos/voltha_protos` directory. This directory hierarchy and import scheme is required to allow the python code generated by the gRPC compiler to [have the correct import paths](https://github.com/grpc/grpc/issues/9575#issuecomment-293934506).
+Protobuf definition files are located in `protos/voltha_protos` directory. This
+directory hierarchy and import scheme is required to allow the python code
+generated by the gRPC compiler to [have the correct import
+paths](https://github.com/grpc/grpc/issues/9575#issuecomment-293934506).
-> NOTE: The `protos/google/api` directory has files copied from the [Google APIs](https://github.com/googleapis/googleapis), and is only included for initial compilation of the VOLTHA protobuf files - these API's should be installed independently via either the python [googleapis-common-protos](https://pypi.org/project/googleapis-common-protos/)
-> package or the golang [go-genproto](https://github.com/google/go-genproto) repo.
-
-
+> NOTE: The `protos/google/api` directory has files copied from the [Google
+> APIs](https://github.com/googleapis/googleapis), and is only included for
+> initial compilation of the VOLTHA protobuf files - these API's should be
+> installed independently via either the python
+> [googleapis-common-protos](https://pypi.org/project/googleapis-common-protos/)
+> package or the golang [go-genproto](https://github.com/google/go-genproto)
+> repo.
## Go Environment
Get the voltha-protos repository:
-```
+
+```sh
mkdir -p ~/source
cd ~/source
git clone https://gerrit.opencord.org/voltha-protos
@@ -23,7 +31,9 @@
### Setting up the Go environment
-After installing Go on a Mac or Linux environment, the GOPATH environment variable needs be set. These instructions assume its ~/go.
+After installing Go on a Mac or Linux environment, the GOPATH environment
+variable needs be set. These instructions assume it's `~/go`.
+
Create a symbolic link in the $GOPATH/src tree to the voltha-go repository:
```sh
@@ -31,13 +41,12 @@
ln -s ~/source/voltha-protos $GOPATH/src/github.com/opencord/voltha-protos
```
-
-
## Go Dependencies
### Install Dependencies
-Checkout and go install correct version of protoc-gen-go. make install build and runs go install
+Checkout and go install correct version of protoc-gen-go:
+
```sh
cd $GOPATH/src/github.com/opencord/voltha-protos
go get -d github.com/golang/protobuf/
@@ -48,7 +57,9 @@
## Building locally
-Install libprotoc 3.7.0 manually or use the Makefile target install it. Then build the python and golang stubs
+Install libprotoc 3.7.0 manually or use the Makefile target install it. Then
+build the python and golang stubs:
+
```sh
cd $GOPATH/src/github.com/opencord/voltha-protos
make install-protoc
@@ -58,8 +69,6 @@
use dist/*.tar.gz for local python imports
use go/ for local go imports
-
-
## Using voltha-protos in your project
### Python
@@ -76,8 +85,6 @@
`from voltha_protos import voltha_pb2`
-
-
### Go
```sh
@@ -85,13 +92,11 @@
cd $GOPATH/github.com/opencord/voltha-protos
make build
````
-protos should be importable from github.com/opencord/voltha-protos/go/packagename
+Protos are importable from `github.com/opencord/voltha-protos/go/<packagename>`
To use the libraries, import protos with the root path github.com/opencord/voltha-protos/go/
-
-
## Testing
`make test` will run tests for all languages.
diff --git a/VERSION b/VERSION
index 9d8d2c1..d917d3e 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-0.1.2-dev0
+0.1.2
diff --git a/build_go_protos.sh b/build_go_protos.sh
deleted file mode 100755
index 3d6090a..0000000
--- a/build_go_protos.sh
+++ /dev/null
@@ -1,74 +0,0 @@
-#!/bin/sh
-# Copyright 2018 the original author or authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -e
-
-export PARENT_DIR="$1"
-export SRC_DIR="${PARENT_DIR}/voltha_protos"
-
-export LD_LIBRARY_PATH=/usr/local/lib
-
-echo $SRC_DIR
-
-export MAPS=Mgoogle/protobuf/descriptor.proto=github.com/golang/protobuf/protoc-gen-go/descriptor
-export INCS="\
- -I $PARENT_DIR \
- -I ${PARENT_DIR}/google/api"
-
-export VOLTHA_PB="\
- $SRC_DIR/adapter.proto \
- $SRC_DIR/device.proto \
- $SRC_DIR/events.proto \
- $SRC_DIR/health.proto \
- $SRC_DIR/logical_device.proto \
- $SRC_DIR/ponsim.proto \
- $SRC_DIR/voltha.proto"
-
-export COMMON_PB="\
- $SRC_DIR/common.proto \
- $SRC_DIR/meta.proto \
- $SRC_DIR/yang_options.proto"
-
-export INTER_CONTAINER_PB="$SRC_DIR/inter_container.proto"
-export SCHEMA_PB="$SRC_DIR/schema.proto"
-export IETF_PB="$SRC_DIR/ietf_interfaces.proto"
-export OF_PB="$SRC_DIR/openflow_13.proto"
-export OMCI_PB="$SRC_DIR/omci*.proto"
-export AFROUTER_PB="$SRC_DIR/afrouter.proto"
-export OPENOLT_PB="$SRC_DIR/openolt.proto"
-
-export PB_VARS="\
- VOLTHA_PB \
- COMMON_PB \
- INTER_CONTAINER_PB \
- SCHEMA_PB \
- IETF_PB \
- OF_PB \
- OMCI_PB \
- AFROUTER_PB \
- OMCI_PB \
- OPENOLT_PB"
-
-export OTHER_INCLUDES="${PARENT_DIR}/google/api"
-export PROTO_DESC_FILE="go/voltha.pb"
-
-for pb_var in $PB_VARS
-do
- pbs="$(eval echo \$$pb_var)"
- echo "Compiling $pbs"
- protoc --go_out=$MAPS,plugins=grpc:$GOPATH/src $INCS $pbs
-done
-
-protoc -I ${PARENT_DIR} -I ${OTHER_INCLUDES} --include_imports --include_source_info --descriptor_set_out=${PROTO_DESC_FILE} ${SRC_DIR}/*.proto
diff --git a/test/test-go-proto-consistency.sh b/test/test-go-proto-consistency.sh
index 64719cb..76ac7b1 100755
--- a/test/test-go-proto-consistency.sh
+++ b/test/test-go-proto-consistency.sh
@@ -1,8 +1,4 @@
-set -e
-
-#This test validates that go proto outputs are committed to git.
-#It does this by generating them and validating whether go thinks they are the same.
-#If they become out of sync, there can be problems for consumers of the protos.
+#!/usr/bin/env bash
# Copyright 2018 the original author or authors.
#
@@ -17,6 +13,15 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
+# test-go-proto-consistency.sh
+#
+# This test validates that go proto outputs are committed to git. It does this
+# by regenerating them and validating whether git thinks they are the same. If
+# they become out of sync, there can be problems for consumers of the protos.
+
+set -eu -o pipefail
+
git status > /dev/null
STAGED="$(git diff-index --quiet HEAD -- || echo 'staged')"
UNTRACKED="$(git ls-files --exclude-standard --others)"
@@ -26,11 +31,12 @@
exit 1
fi
+# delete and regenerate go protos
+rm -rf go/voltha.pb go/*/*.pb.go go_temp
make go-protos
# Running git status ensures correct git diff-index picks up changes
git status > /dev/null
-
STAGED_POST="$(git diff-index --quiet HEAD -- || echo "staged")"
UNTRACKED_POST="$(git ls-files --exclude-standard --others)"