VOL-1449 Modify build environment to use pyvoltha
Depending on if LOCAL_PYVOLTHA is set use a build of
the library from the nearby source tree, for both
virtualenv and docker image builds. By default
requirements.txt uses the pip hosted dev build, which
may not be new enough for iterative development
Also by setting PYVOLTHA_BASE_IMAGE use a docker image produced
by pyvoltha as the base image.
Change-Id: Ibf7218b92450f92d6a3f1c9d837be46fcc3a84db
diff --git a/python/Makefile b/python/Makefile
index ec74b70..c78bb7a 100644
--- a/python/Makefile
+++ b/python/Makefile
@@ -14,12 +14,6 @@
# limitations under the License.
#
-ifneq ($(VOLTHA_BUILD),docker)
-ifeq ($(VOLTHA_BASE)_set,_set)
-$(error To get started, please source the env.sh file)
-endif
-endif
-
ifeq ($(TAG),)
TAG := latest
endif
@@ -28,12 +22,6 @@
TARGET_TAG := latest
endif
-# If no DOCKER_HOST_IP is specified grab a v4 IP address associated with
-# the default gateway
-ifeq ($(DOCKER_HOST_IP),)
-DOCKER_HOST_IP := $(shell ifconfig $$(netstat -rn | grep -E '^(default|0.0.0.0)' | head -1 | awk '{print $$NF}') | grep inet | awk '{print $$2}' | sed -e 's/addr://g')
-endif
-
ifneq ($(http_proxy)$(https_proxy),)
# Include proxies from the environment
DOCKER_PROXY_ARGS = \
@@ -55,103 +43,52 @@
--rm --force-rm \
$(DOCKER_BUILD_EXTRA_ARGS)
+DOCKER_IMAGE_LIST = \
+ voltha-openonu-adapter-base \
+ voltha-openonu-adapter
+
VENVDIR := venv-$(shell uname -s | tr '[:upper:]' '[:lower:]')
-DOCKER_IMAGE_LIST = \
- base \
- adapter-openonu \
-
-# The following list was scavanged from the compose / stack files as well as
-# from the Dockerfiles. If nothing else it highlights that VOLTHA is not
-# using consistent versions for some of the containers.
-
-FETCH_BUILD_IMAGE_LIST = \
- alpine:3.6 \
- centos:7 \
- centurylink/ca-certs:latest \
- grpc/python:latest \
- ubuntu:xenial
-
-FETCH_COMPOSE_IMAGE_LIST = \
- wurstmeister/kafka:latest \
- wurstmeister/zookeeper:latest
-
-# find k8s -type f | xargs grep image: | awk '{print $NF}' | sed -e 's/\"//g' | sed '/:.*$/!s/$/:latest/g' | sort -u | sed -e 's/^/ /g' -e 's/$/ \\/g'
-# Manually remove some image from this list as they don't reflect the new
-# naming conventions for the VOLTHA build
-FETCH_K8S_IMAGE_LIST = \
- wurstmeister/kafka:1.0.0 \
- zookeeper:3.4.11
-
-FETCH_IMAGE_LIST = $(shell echo $(FETCH_BUILD_IMAGE_LIST) $(FETCH_COMPOSE_IMAGE_LIST) $(FETCH_K8S_IMAGE_LIST) | tr ' ' '\n' | sort -u)
-
-.PHONY: $(DIRS) $(DIRS_CLEAN) $(DIRS_FLAKE8) base protoc protos adapter_openonu
+.PHONY: $(DIRS) $(DIRS_CLEAN) base openonu_adapter
# This should to be the first and default target in this Makefile
help:
@echo "Usage: make [<target>]"
@echo "where available targets are:"
@echo
- @echo "build : Build the Adapters protos and docker images.\n\
- If this is the first time you are building, choose \"make build\" option."
- @echo "clean : Remove files created by the build and tests"
- @echo "distclean : Remove venv directory"
- @echo "fetch : Pre-fetch artifacts for subsequent local builds"
- @echo "help : Print this help"
- @echo "protoc : Build a container with protoc installed"
- @echo "protos : Compile all grpc/protobuf files"
- @echo "rebuild-venv : Rebuild local Python virtualenv from scratch"
- @echo "venv : Build local Python virtualenv if did not exist yet"
- @echo "containers : Build all the docker containers"
- @echo "base : Build the base docker container used by all other dockers"
- @echo "adapter_openonu : Build the openonu openomci adapter docker container"
- @echo "tag : Tag a set of images"
- @echo "push : Push the docker images to an external repository"
- @echo "pull : Pull the docker images from a repository"
+ @echo "build : Build the adapter."
+ @echo "clean : Remove files created by the build and tests"
+ @echo "distclean : Remove venv directory"
+ @echo "help : Print this help"
+ @echo "rebuild-venv : Rebuild local Python virtualenv from scratch"
+ @echo "venv : Build local Python virtualenv if did not exist yet"
+ @echo "containers : Build all the docker containers"
+ @echo "base : Build a base docker image with a modern version of pip and requirements.txt installed"
+ @echo "adapter_openonu : Build the openonu openomci adapter docker container"
+ @echo "tag : Tag a set of images"
+ @echo "push : Push the docker images to an external repository"
+ @echo "pull : Pull the docker images from a repository"
@echo
-## New directories can be added here
-#DIRS:=
+build: containers
-## If one directory depends on another directory that
-## dependency can be expressed here
-##
-## For example, if the Tibit directory depended on the eoam
-## directory being built first, then that can be expressed here.
-## driver/tibit: eoam
-
-# Parallel Build
-$(DIRS):
- @echo " MK $@"
- $(Q)$(MAKE) -C $@
-
-# Parallel Clean
-DIRS_CLEAN = $(addsuffix .clean,$(DIRS))
-$(DIRS_CLEAN):
- @echo " CLEAN $(basename $@)"
- $(Q)$(MAKE) -C $(basename $@) clean
-
-# Parallel Flake8
-DIRS_FLAKE8 = $(addsuffix .flake8,$(DIRS))
-$(DIRS_FLAKE8):
- @echo " FLAKE8 $(basename $@)"
- -$(Q)$(MAKE) -C $(basename $@) flake8
-
-build: protoc protos containers
-
-containers: base adapter_openonu
+containers: base openonu_adapter
base:
- docker build $(DOCKER_BUILD_ARGS) -t ${REGISTRY}${REPOSITORY}voltha-base:${TAG} -f docker/Dockerfile.base .
+ifdef LOCAL_PYVOLTHA
+ mkdir -p pyvoltha/dist
+ cp ../../pyvoltha/dist/*.tar.gz pyvoltha/dist/
+ docker build $(DOCKER_BUILD_ARGS) -t ${REGISTRY}${REPOSITORY}voltha-openonu-adapter-base:${TAG} -f docker/Dockerfile.base_local .
+else
+ docker build $(DOCKER_BUILD_ARGS) -t ${REGISTRY}${REPOSITORY}voltha-openonu-adapter-base:${TAG} -f docker/Dockerfile.base .
+endif
-adapter_openonu:
- docker build $(DOCKER_BUILD_ARGS) -t ${REGISTRY}${REPOSITORY}voltha-adapter-openonu:${TAG} -f docker/Dockerfile.adapter_openonu .
-
-protoc:
- docker build $(DOCKER_BUILD_ARGS) -t ${REGISTRY}${REPOSITORY}voltha-protoc:${TAG} -f docker/Dockerfile.protoc .
-
-protos:
- docker build $(DOCKER_BUILD_ARGS) -t ${REGISTRY}${REPOSITORY}voltha-protos:${TAG} -f docker/Dockerfile.protos .
+openonu_adapter: base
+ifdef PYVOLTHA_BASE_IMAGE
+ docker build $(DOCKER_BUILD_ARGS) -t ${REGISTRY}${REPOSITORY}voltha-openonu-adapter:${TAG} -f docker/Dockerfile.openonu_adapter_pyvoltha .
+else
+ docker build $(DOCKER_BUILD_ARGS) -t ${REGISTRY}${REPOSITORY}voltha-openonu-adapter:${TAG} -f docker/Dockerfile.openonu_adapter .
+endif
tag: $(patsubst %,%.tag,$(DOCKER_IMAGE_LIST))
@@ -169,27 +106,18 @@
docker pull ${REGISTRY}${REPOSITORY}voltha-$(subst .pull,,$@):${TAG}
clean:
+ rm -rf pyvoltha
find . -name '*.pyc' | xargs rm -f
distclean: clean
rm -rf ${VENVDIR}
-fetch:
- @bash -c ' \
- for i in $(FETCH_IMAGE_LIST); do \
- docker pull $$i; \
- done'
-
purge-venv:
rm -fr ${VENVDIR}
rebuild-venv: purge-venv venv
-ifneq ($(VOLTHA_BUILD),docker)
venv: ${VENVDIR}/.built
-else
-venv:
-endif
${VENVDIR}/.built:
@ virtualenv ${VENVDIR}
@@ -203,7 +131,11 @@
uname -s > ${VENVDIR}/.built; \
fi
-
-flake8: $(DIRS_FLAKE8)
+ifdef LOCAL_PYVOLTHA
+ mkdir -p pyvoltha/dist
+ cp ../../pyvoltha/dist/*.tar.gz pyvoltha/dist/
+ @ . ${VENVDIR}/bin/activate && \
+ pip install pyvoltha/dist/*.tar.gz
+endif
# end file
diff --git a/python/adapters/brcm_openomci_onu/heartbeat.py b/python/adapters/brcm_openomci_onu/heartbeat.py
index 4a7ab1f..5242caa 100644
--- a/python/adapters/brcm_openomci_onu/heartbeat.py
+++ b/python/adapters/brcm_openomci_onu/heartbeat.py
@@ -49,7 +49,7 @@
self._defer = reactor.callLater(delay, self.check_pulse)
def _stop(self):
- d, self._defeered = self._defeered, None
+ d, self._defer = self._defer, None
if d is not None and not d.called():
d.cancel()
diff --git a/python/compose/adapters-openonu.yml b/python/compose/adapters-openonu.yml
index d151b0c..0f1231d 100644
--- a/python/compose/adapters-openonu.yml
+++ b/python/compose/adapters-openonu.yml
@@ -22,14 +22,14 @@
services:
adapter_openonu:
- image: "${REGISTRY}${REPOSITORY}voltha-adapter-openonu${TAG}"
+ image: "${REGISTRY}${REPOSITORY}voltha-openonu-adapter${TAG}"
logging:
driver: "json-file"
options:
max-size: "10m"
max-file: "3"
command: [
- "/voltha/python/adapters/brcm_openomci_onu/main.py",
+ "/voltha/voltha/adapters/brcm_openomci_onu/main.py",
"-v",
"--name=openonu",
"--kafka_adapter=${DOCKER_HOST_IP}:9092",
diff --git a/python/compose/system-test.yml b/python/compose/system-test.yml
index 872d4e2..fe8a9c8 100644
--- a/python/compose/system-test.yml
+++ b/python/compose/system-test.yml
@@ -155,7 +155,7 @@
adapter_openonu:
- image: "${REGISTRY}${REPOSITORY}voltha-adapter-openonu${TAG}"
+ image: "${REGISTRY}${REPOSITORY}voltha-openonu-adapter${TAG}"
logging:
driver: "json-file"
options:
diff --git a/python/docker/Dockerfile.adapter_openonu b/python/docker/Dockerfile.adp-old
similarity index 100%
rename from python/docker/Dockerfile.adapter_openonu
rename to python/docker/Dockerfile.adp-old
diff --git a/python/docker/Dockerfile.base b/python/docker/Dockerfile.base
index 1b912e0..898f7fa 100644
--- a/python/docker/Dockerfile.base
+++ b/python/docker/Dockerfile.base
@@ -18,17 +18,13 @@
# Update to have latest images
RUN apt-get update && \
- apt-get install -y python python-pip openssl iproute2 libpcap-dev wget
+ apt-get install -y python openssl iproute2 libpcap-dev wget build-essential git binutils python-dev libffi-dev libssl-dev
+# Install current version of pip rather than outdated pip from apt
+RUN wget -O /tmp/get-pip.py https://bootstrap.pypa.io/get-pip.py
+RUN python /tmp/get-pip.py
+
+# Install adapter requirements.
COPY requirements.txt /tmp/requirements.txt
+RUN pip install -r /tmp/requirements.txt
-# pip install cython enum34 six && \
-# Install app dependencies
-RUN wget https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
- dpkg -i *.deb && \
- rm -f *.deb && \
- apt-get update && \
- apt-get install -y wget build-essential make gcc binutils python-dev libffi-dev libssl-dev git && \
- pip install -r /tmp/requirements.txt && \
- apt-get purge -y wget build-essential make gcc binutils python-dev libffi-dev libssl-dev git && \
- apt-get autoremove -y
diff --git a/python/docker/Dockerfile.base_local b/python/docker/Dockerfile.base_local
new file mode 100644
index 0000000..9b6ce9e
--- /dev/null
+++ b/python/docker/Dockerfile.base_local
@@ -0,0 +1,34 @@
+# Copyright 2016 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM ubuntu:xenial
+
+MAINTAINER Voltha Community <info@opennetworking.org>
+
+# Update to have latest images
+RUN apt-get update && \
+ apt-get install -y python openssl iproute2 libpcap-dev wget build-essential git binutils python-dev libffi-dev libssl-dev
+
+# Install current version of pip rather than outdated pip from apt
+RUN wget -O /tmp/get-pip.py https://bootstrap.pypa.io/get-pip.py
+RUN python /tmp/get-pip.py
+
+# Install adapter requirements.
+COPY requirements.txt /tmp/requirements.txt
+RUN pip install -r /tmp/requirements.txt
+
+# Install local pyvoltha
+COPY pyvoltha/dist /pyvoltha/dist
+RUN pip install /pyvoltha/dist/*.tar.gz
+
diff --git a/python/docker/Dockerfile.openonu_adapter b/python/docker/Dockerfile.openonu_adapter
new file mode 100644
index 0000000..50251f1
--- /dev/null
+++ b/python/docker/Dockerfile.openonu_adapter
@@ -0,0 +1,31 @@
+# Copyright 2016 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+ARG TAG=latest
+ARG REGISTRY=
+ARG REPOSITORY=
+
+FROM ${REGISTRY}${REPOSITORY}voltha-openonu-adapter-base:${TAG}
+
+MAINTAINER Voltha Community <info@opennetworking.org>
+
+# Bundle app source
+RUN mkdir /voltha && touch /voltha/__init__.py
+RUN mkdir /voltha/voltha && touch /voltha/voltha/__init__.py
+RUN mkdir /voltha/voltha/adapters && touch /voltha/voltha/adapters/__init__.py
+ENV PYTHONPATH=/voltha
+COPY adapters/brcm_openomci_onu /voltha/voltha/adapters/brcm_openomci_onu
+
+# Exposing process and default entry point
+CMD ["python", "/voltha/python/adapters/openonu/main.py"]
+
diff --git a/python/docker/Dockerfile.openonu_adapter_pyvoltha b/python/docker/Dockerfile.openonu_adapter_pyvoltha
new file mode 100644
index 0000000..9d5e985
--- /dev/null
+++ b/python/docker/Dockerfile.openonu_adapter_pyvoltha
@@ -0,0 +1,31 @@
+# Copyright 2016 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+ARG TAG=latest
+ARG REGISTRY=
+ARG REPOSITORY=
+
+FROM ${REGISTRY}${REPOSITORY}pyvoltha:${TAG}
+
+MAINTAINER Voltha Community <info@opennetworking.org>
+
+# Bundle app source
+RUN mkdir /voltha && touch /voltha/__init__.py
+RUN mkdir /voltha/voltha && touch /voltha/voltha/__init__.py
+RUN mkdir /voltha/voltha/adapters && touch /voltha/voltha/adapters/__init__.py
+ENV PYTHONPATH=/voltha
+COPY adapters/brcm_openomci_onu /voltha/voltha/adapters/brcm_openomci_onu
+
+# Exposing process and default entry point
+CMD ["python", "/voltha/python/adapters/openonu/main.py"]
+
diff --git a/python/protos/Makefile b/python/protos/Makefile
index 0fad970..dc7c973 100644
--- a/python/protos/Makefile
+++ b/python/protos/Makefile
@@ -16,15 +16,20 @@
# Makefile to build all protobuf and gRPC related artifacts
+ifeq ($(VOLTHA_BASE)_set,_set)
+ $(error To get started, please source the env.sh file from Voltha top level directory)
+endif
+
default: third_party build
-PROTO_FILES := $(wildcard ../../protos/*.proto)
+PROTO_FILES := $(wildcard *.proto)
PROTO_GOOGLE_API := $(wildcard third_party/google/api/*.proto)
PROTO_ALL_FILES := $(PROTO_FILES) $(PROTO_GOOGLE_API)
PROTO_PB2_FILES := $(foreach f,$(PROTO_FILES),$(subst .proto,_pb2.py,$(f)))
PROTO_PB2_GOOGLE_API := $(foreach f,$(PROTO_GOOGLE_API),$(subst .proto,_pb2.py,$(f)))
-PROTO_PB2_GRPC_GOOGLE_API := $(foreach f,$(PROTO_GOOGLE_API),$(subst .proto,_pb2_grpc.py,$(f)))
-PROTO_DESC_GOOGLE_API := $(foreach f,$(PROTO_GOOGLE_API),$(subst .proto,.desc,$(f)))
+PROTO_All_PB2_C_FILES := $(foreach f,$(PROTO_ALL_FILES),$(subst .proto,_pb2.pyc,$(f)))
+PROTO_ALL_PB2_GPRC_FILES := $(foreach f,$(PROTO_ALL_FILES),$(subst .proto,_pb2_grpc.py,$(f)))
+PROTO_ALL_DESC_FILES := $(foreach f,$(PROTO_ALL_FILES),$(subst .proto,.desc,$(f)))
PROTOC_PREFIX := /usr/local
PROTOC_LIBDIR := $(PROTOC_PREFIX)/lib
@@ -58,20 +63,22 @@
%_pb2.py: %.proto Makefile
@echo "Building protocol buffer artifacts from $<"
env LD_LIBRARY_PATH=$(PROTOC_LIBDIR) python -m grpc.tools.protoc \
- -I../../protos \
+ -I. \
-I./third_party \
--python_out=. \
--grpc_python_out=. \
- --descriptor_set_out=./$(basename $(notdir $<)).desc \
+ --descriptor_set_out=$(basename $<).desc \
--include_imports \
--include_source_info \
$<
clean:
- rm -f *.desc *_pb2* \
- $(PROTO_PB2_GOOGLE_API) \
- $(PROTO_PB2_GRPC_GOOGLE_API)\
- $(PROTO_DESC_GOOGLE_API)
+ find . -name '*.pyc' | xargs rm -f
+ rm -f $(PROTO_PB2_FILES) \
+ $(PROTO_ALL_DESC_FILES) \
+ $(PROTO_ALL_PB2_GPRC_FILES) \
+ $(PROTO_All_PB2_C_FILES) \
+ $(PROTO_PB2_GOOGLE_API)
$(PROTOC):
@echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
@@ -98,4 +105,3 @@
uninstall-protoc:
cd $(PROTOC_BUILD_TMP_DIR)/$(PROTOC_DIR); \
sudo make uninstall
-
diff --git a/python/requirements.txt b/python/requirements.txt
index 56cb356..015660e 100644
--- a/python/requirements.txt
+++ b/python/requirements.txt
@@ -60,10 +60,6 @@
etcd3==0.7.0
pyparsing==2.2.0
packaging==17.1
-
-# python-consul>=0.6.1 we need the pre-released version for now, because 0.6.1 does not
-# yet support Twisted. Once this is released, it will be the 0.6.2 version
-git+https://github.com/cablehead/python-consul.git
-
-# Twisted Python kafka client
-git+https://github.com/ciena/afkak.git
+python-consul==0.6.2
+afkak==3.0.0.dev20181106
+pyvoltha==0.1.7