[SEBA-497]
Delayer and alpine-base chameleon container
Also fix formatting and minor bugs
Change-Id: I465b6b351ae11dca81dbc6c88d5b6080fdbc1f5c
diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000..7d8ee90
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,2 @@
+.tox
+venv-chameleon
diff --git a/.gitignore b/.gitignore
index c69f216..013d7fb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,9 +2,12 @@
*.swp
# Generated files
-protos/schema_pb2.py
-protos/schema_pb2_grpc.py
-protos/third_party/google/api/annotations_pb2.py
-protos/third_party/google/api/annotations_pb2_grpc.py
-protos/third_party/google/api/http_pb2.py
-protos/third_party/google/api/http_pb2_grpc.py
+protos/*_pb2.py
+protos/*_pb2_grpc.py
+
+# Testing
+.tox
+.coverage
+coverage.xml
+nose2-results.xml
+venv-chameleon
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..a99b5d7
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,56 @@
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# xosproject/chameleon
+FROM alpine:3.9.2
+
+# Install base software
+RUN apk add --no-cache \
+ build-base \
+ libffi-dev \
+ libressl-dev \
+ py-setuptools \
+ py2-pip \
+ python2-dev \
+ && mkdir /chameleon \
+ && touch /chameleon/__init__.py \
+ && pip freeze > /chameleon/pip_freeze_apk_`date -u +%Y%m%dT%H%M%S`
+
+# Copy over code
+COPY . /chameleon/chameleon
+
+# Install modules and build the protos
+RUN pip install -r /chameleon/chameleon/requirements.txt \
+ && pip freeze > /chameleon/pip_freeze_chameleon_`date -u +%Y%m%dT%H%M%S` \
+ && make -C /chameleon/chameleon/protos
+
+# Label image
+ARG org_label_schema_version=unknown
+ARG org_label_schema_vcs_url=unknown
+ARG org_label_schema_vcs_ref=unknown
+ARG org_label_schema_build_date=unknown
+ARG org_opencord_vcs_commit_date=unknown
+
+LABEL org.label-schema.schema-version=1.0 \
+ org.label-schema.name=chameleon \
+ org.label-schema.version=$org_label_schema_version \
+ org.label-schema.vcs-url=$org_label_schema_vcs_url \
+ org.label-schema.vcs-ref=$org_label_schema_vcs_ref \
+ org.label-schema.build-date=$org_label_schema_build_date \
+ org.opencord.vcs-commit-date=$org_opencord_vcs_commit_date
+
+ENV PYTHONPATH=/chameleon
+
+# Exposing process and default entry point
+CMD ["python", "/chameleon/chameleon/main.py"]
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..d97b262
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,82 @@
+# Copyright 2019-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Configure shell
+SHELL = bash -e -o pipefail
+
+# Variables
+VERSION ?= $(shell cat ./VERSION)
+CONTAINER_NAME ?= $(notdir $(abspath .))
+
+## Docker related
+DOCKER_REGISTRY ?=
+DOCKER_REPOSITORY ?=
+DOCKER_BUILD_ARGS ?=
+DOCKER_TAG ?= ${VERSION}
+DOCKER_IMAGENAME := ${DOCKER_REGISTRY}${DOCKER_REPOSITORY}${CONTAINER_NAME}:${DOCKER_TAG}
+
+## Docker labels. Only set ref and commit date if committed
+DOCKER_LABEL_VCS_URL ?= $(shell git remote get-url $(shell git remote))
+DOCKER_LABEL_VCS_REF ?= $(shell git diff-index --quiet HEAD -- && git rev-parse HEAD || echo "unknown")
+DOCKER_LABEL_COMMIT_DATE ?= $(shell git diff-index --quiet HEAD -- && git show -s --format=%cd --date=iso-strict HEAD || echo "unknown" )
+DOCKER_LABEL_BUILD_DATE ?= $(shell date -u "+%Y-%m-%dT%H:%M:%SZ")
+
+## xosgenx related - paths are relative to this directory
+XOS_DIR ?= "../xos"
+
+all: test
+
+docker-build:
+ docker build $(DOCKER_BUILD_ARGS) \
+ -t ${DOCKER_IMAGENAME} \
+ --build-arg org_label_schema_version="${VERSION}" \
+ --build-arg org_label_schema_vcs_url="${DOCKER_LABEL_VCS_URL}" \
+ --build-arg org_label_schema_vcs_ref="${DOCKER_LABEL_VCS_REF}" \
+ --build-arg org_label_schema_build_date="${DOCKER_LABEL_BUILD_DATE}" \
+ --build-arg org_opencord_vcs_commit_date="${DOCKER_LABEL_COMMIT_DATE}" \
+ -f Dockerfile .
+
+docker-push:
+ docker push ${DOCKER_IMAGENAME}
+
+# Test starting the image, loading TOSCA, deleting TOSCA, and cleaning up after
+# Not sure if this has been functional recently
+test-docker: docker-start test-create test-delete docker-clean
+
+test: test-unit
+
+test-unit: generate-protos
+ tox
+
+venv-chameleon:
+ virtualenv $@;\
+ source ./$@/bin/activate ; set -u ;\
+ pip install -r requirements.txt
+
+generate-protos: venv-chameleon
+ source ./venv-chameleon/bin/activate ; set -u ;\
+ make -C protos
+
+clean:
+ find . -name '*.pyc' | xargs rm -f
+ rm -rf \
+ .tox \
+ .coverage \
+ coverage \
+ coverage.xml \
+ nose2-results.xml \
+ protos/*_pb2.py \
+ protos/*_pb2_grpc.py \
+ venv-chameleon
+
diff --git a/VERSION b/VERSION
new file mode 100644
index 0000000..15a2799
--- /dev/null
+++ b/VERSION
@@ -0,0 +1 @@
+3.3.0
diff --git a/grpc_client/grpc_client.py b/grpc_client/grpc_client.py
index 9e7987c..f0c5f11 100644
--- a/grpc_client/grpc_client.py
+++ b/grpc_client/grpc_client.py
@@ -139,14 +139,14 @@
return
- except _Rendezvous, e:
+ except _Rendezvous as e:
if e.code() == grpc.StatusCode.UNAVAILABLE:
log.info('grpc-endpoint-not-available')
else:
log.exception('rendezvous error', e=e)
yield self._backoff('not-available')
- except Exception, e:
+ except Exception as e:
if not self.shutting_down:
log.exception('cannot-connect', endpoint=_endpoint)
yield self._backoff('unknown-error')
@@ -298,7 +298,7 @@
response, rendezvous = method.with_call(request, metadata=metadata)
returnValue((response, rendezvous.trailing_metadata()))
- except grpc._channel._Rendezvous, e:
+ except grpc._channel._Rendezvous as e:
code = e.code()
if code == grpc.StatusCode.UNAVAILABLE:
e = ServiceUnavailable()
diff --git a/main.py b/main.py
index 8187d25..6e87324 100755
--- a/main.py
+++ b/main.py
@@ -45,9 +45,9 @@
rest_port=os.environ.get('REST_PORT', 8881),
work_dir=os.environ.get('WORK_DIR', '/tmp/chameleon'),
swagger_url=os.environ.get('SWAGGER_URL', ''),
- enable_tls=os.environ.get('ENABLE_TLS',"True"),
- key=os.environ.get('KEY','/chameleon/pki/voltha.key'),
- cert=os.environ.get('CERT','/chameleon/pki/voltha.crt'),
+ enable_tls=os.environ.get('ENABLE_TLS', "True"),
+ key=os.environ.get('KEY', '/chameleon/pki/voltha.key'),
+ cert=os.environ.get('CERT', '/chameleon/pki/voltha.crt'),
)
@@ -217,9 +217,10 @@
path = os.path.join(dir, path)
path = os.path.abspath(path)
with open(path) as fd:
- config = yaml.load(fd)
+ config = yaml.safe_load(fd)
return config
+
banner = r'''
____ _ _
/ ___| | |__ __ _ _ __ ___ ___ | | ___ ___ _ __
@@ -229,6 +230,7 @@
'''
+
def print_banner(log):
for line in banner.strip('\n').splitlines():
log.info(line)
@@ -271,8 +273,8 @@
if args.enable_tls == "False":
self.log.info('tls-disabled-through-configuration')
self.rest_server = yield \
- WebServer(args.rest_port, args.work_dir, args.swagger_url,\
- self.grpc_client).start()
+ WebServer(args.rest_port, args.work_dir, args.swagger_url,
+ self.grpc_client).start()
else:
# If TLS is enabled, but the server key or cert is not found,
# then automatically disable TLS
@@ -284,21 +286,21 @@
self.log.error('cert-not-found')
self.log.info('disabling-tls-due-to-missing-pki-files')
self.rest_server = yield \
- WebServer(args.rest_port, args.work_dir,\
- args.swagger_url,\
- self.grpc_client).start()
+ WebServer(args.rest_port, args.work_dir,
+ args.swagger_url,
+ self.grpc_client).start()
else:
self.log.info('tls-enabled')
self.rest_server = yield \
- WebServer(args.rest_port, args.work_dir,\
- args.swagger_url,\
- self.grpc_client, args.key,\
- args.cert).start()
+ WebServer(args.rest_port, args.work_dir,
+ args.swagger_url,
+ self.grpc_client, args.key,
+ args.cert).start()
self.grpc_client.set_reconnect_callback(
self.rest_server.reload_generated_routes).start()
self.log.info('started-internal-services')
- except Exception, e:
+ except Exception as e:
self.log.exception('startup-failed', e=e)
@inlineCallbacks
diff --git a/protoc_plugins/descriptor_parser.py b/protoc_plugins/descriptor_parser.py
index 0034379..13bdc5e 100644
--- a/protoc_plugins/descriptor_parser.py
+++ b/protoc_plugins/descriptor_parser.py
@@ -21,7 +21,8 @@
from google.protobuf.message import Message
-class InvalidDescriptorError(Exception): pass
+class InvalidDescriptorError(Exception):
+ pass
class DescriptorParser(object):
@@ -133,8 +134,8 @@
return d
def parse_file_descriptors(self, descriptors,
- type_tag_name=None,
- fold_comments=False):
+ type_tag_name=None,
+ fold_comments=False):
return [self.parse_file_descriptor(descriptor,
type_tag_name=type_tag_name,
fold_comments=fold_comments)
diff --git a/protoc_plugins/protobuf_introspect.py b/protoc_plugins/protobuf_introspect.py
index 6e43af5..71a698b 100755
--- a/protoc_plugins/protobuf_introspect.py
+++ b/protoc_plugins/protobuf_introspect.py
@@ -35,7 +35,8 @@
from google.protobuf import descriptor_pb2
-class InvalidDescriptorError(Exception): pass
+class InvalidDescriptorError(Exception):
+ pass
class DescriptorParser(object):
@@ -123,7 +124,7 @@
location.get('leading_comments', '').strip(' '),
location.get('trailing_comments', '').strip(' '),
''.join(block.strip(' ') for block
- in location.get('leading_detached_comments', ''))
+ in location.get('leading_detached_comments', ''))
]).strip()
# ignore locations with no comments
diff --git a/protoc_plugins/swagger_template.py b/protoc_plugins/swagger_template.py
index e3aff17..a444f71 100644
--- a/protoc_plugins/swagger_template.py
+++ b/protoc_plugins/swagger_template.py
@@ -23,9 +23,16 @@
re_segment = re.compile(r'/(?P<absolute>[^{}/]+)|(?P<symbolic>{[^}]+})')
-class DuplicateMethodAndPathError(Exception): pass
-class ProtobufCompilationFailedError(Exception): pass
-class InvalidPathArgumentError(Exception): pass
+class DuplicateMethodAndPathError(Exception):
+ pass
+
+
+class ProtobufCompilationFailedError(Exception):
+ pass
+
+
+class InvalidPathArgumentError(Exception):
+ pass
def native_descriptors_to_swagger(native_descriptors):
@@ -117,7 +124,7 @@
for method in service.get('method', []):
# skip methods that do not have http options
options = method['options']
- if options.has_key('http'):
+ if 'http' in options:
full_name = service_prefix + '.' + method['name']
yield full_name, service, method
@@ -165,7 +172,7 @@
definition, types_referenced = make_definition(type, types)
definitions[full_name] = definition
for type_referenced in types_referenced:
- if not definitions.has_key(type_referenced):
+ if type_referenced not in definitions:
wanted.add(type_referenced)
return definitions
@@ -220,7 +227,7 @@
if properties:
definition['properties'] = properties
- if type.has_key('_description'):
+ if '_description' in type:
definition['description'] = type['_description']
return definition, referenced
@@ -288,7 +295,7 @@
'items': property
}
- if field.has_key('_description'):
+ if '_description' in field:
property['description'] = field['_description']
return field['name'], property, referenced
@@ -325,7 +332,7 @@
def lookup_type(input_type, field_name):
local_field_name, _, rest = field_name.partition('.')
properties = input_type['properties']
- if not properties.has_key(local_field_name):
+ if local_field_name not in properties:
raise InvalidPathArgumentError(
'Input type has no field {}'.format(field_name))
field = properties[local_field_name]
@@ -382,13 +389,13 @@
entry = {
'operationId': method['name'],
- 'tags': [service['name'],],
+ 'tags': [service['name'], ],
'responses': {
'200': { # TODO: code is 201 and 209 in POST/DELETE?
'description': unicode(""), # TODO: ever filled by proto?
'schema': {
'$ref': '#/definitions/{}'.format(
- method['output_type'].strip('.'))
+ method['output_type'].strip('.'))
}
},
# TODO shall we prefill with standard error (verb specific),
@@ -413,7 +420,7 @@
if verb in path_dict:
raise DuplicateMethodAndPathError(
'There is already a {} method defined for path ({})'.format(
- verb, path))
+ verb, path))
path_dict[verb] = entry
return paths
@@ -447,22 +454,22 @@
TYPE_MAP = {
- FieldDescriptor.TYPE_BOOL: ('boolean', 'boolean'),
- FieldDescriptor.TYPE_BYTES: ('string', 'byte'),
- FieldDescriptor.TYPE_DOUBLE: ('number', 'double'),
- FieldDescriptor.TYPE_ENUM: ('string', 'string'),
- FieldDescriptor.TYPE_FIXED32: ('integer', 'int64'),
- FieldDescriptor.TYPE_FIXED64: ('string', 'uint64'),
- FieldDescriptor.TYPE_FLOAT: ('number', 'float'),
- FieldDescriptor.TYPE_INT32: ('integer', 'int32'),
- FieldDescriptor.TYPE_INT64: ('string', 'int64'),
- FieldDescriptor.TYPE_SFIXED32: ('integer', 'int32'),
- FieldDescriptor.TYPE_SFIXED64: ('string', 'int64'),
- FieldDescriptor.TYPE_STRING: ('string', 'string'),
- FieldDescriptor.TYPE_SINT32: ('integer', 'int32'),
- FieldDescriptor.TYPE_SINT64: ('string', 'int64'),
- FieldDescriptor.TYPE_UINT32: ('integer', 'int64'),
- FieldDescriptor.TYPE_UINT64: ('string', 'uint64'),
- # FieldDescriptor.TYPE_MESSAGE:
- # FieldDescriptor.TYPE_GROUP:
+ FieldDescriptor.TYPE_BOOL: ('boolean', 'boolean'),
+ FieldDescriptor.TYPE_BYTES: ('string', 'byte'),
+ FieldDescriptor.TYPE_DOUBLE: ('number', 'double'),
+ FieldDescriptor.TYPE_ENUM: ('string', 'string'),
+ FieldDescriptor.TYPE_FIXED32: ('integer', 'int64'),
+ FieldDescriptor.TYPE_FIXED64: ('string', 'uint64'),
+ FieldDescriptor.TYPE_FLOAT: ('number', 'float'),
+ FieldDescriptor.TYPE_INT32: ('integer', 'int32'),
+ FieldDescriptor.TYPE_INT64: ('string', 'int64'),
+ FieldDescriptor.TYPE_SFIXED32: ('integer', 'int32'),
+ FieldDescriptor.TYPE_SFIXED64: ('string', 'int64'),
+ FieldDescriptor.TYPE_STRING: ('string', 'string'),
+ FieldDescriptor.TYPE_SINT32: ('integer', 'int32'),
+ FieldDescriptor.TYPE_SINT64: ('string', 'int64'),
+ FieldDescriptor.TYPE_UINT32: ('integer', 'int64'),
+ FieldDescriptor.TYPE_UINT64: ('string', 'uint64'),
+ # FieldDescriptor.TYPE_MESSAGE:
+ # FieldDescriptor.TYPE_GROUP:
}
diff --git a/protos/Makefile b/protos/Makefile
index 52e36b0..08967bf 100644
--- a/protos/Makefile
+++ b/protos/Makefile
@@ -1,4 +1,3 @@
-#
# Copyright 2017 the original author or authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,28 +11,20 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-#
# Makefile to build all protobuf and gRPC related artifacts
-ifeq ($(VOLTHA_BASE)_set,_set)
- $(error To get started, please source the env.sh file from Voltha top level directory)
-endif
-
default: build
PROTO_FILES := $(wildcard *.proto)
PROTO_PB2_FILES := $(foreach f,$(PROTO_FILES),$(subst .proto,_pb2.py,$(f)))
PROTO_DESC_FILES := $(foreach f,$(PROTO_FILES),$(subst .proto,.desc,$(f)))
-PROTOC_PREFIX := /usr/local
-PROTOC_LIBDIR := $(PROTOC_PREFIX)/lib
-
build: $(PROTO_PB2_FILES)
%_pb2.py: %.proto Makefile
@echo "Building protocol buffer artifacts from $<"
- env LD_LIBRARY_PATH=$(PROTOC_LIBDIR) python -m grpc.tools.protoc \
+ env python -m grpc.tools.protoc \
-I. \
--python_out=. \
--grpc_python_out=. \
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..a09582f
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,13 @@
+Jinja2==2.10
+PyYAML==5.1
+docker==3.7.0
+fluent-logger==0.9.3
+grpcio-tools==1.12.0
+grpcio==1.12.0
+klein==17.10.0
+netifaces==0.10.9
+pyOpenSSL==19.0.0
+python-consul==1.1.0
+service_identity==18.1.0
+simplejson==3.16.0
+structlog~=19.1.0
diff --git a/tox.ini b/tox.ini
new file mode 100644
index 0000000..3781b76
--- /dev/null
+++ b/tox.ini
@@ -0,0 +1,50 @@
+; Copyright 2019-present Open Networking Foundation
+;
+; Licensed under the Apache License, Version 2.0 (the "License");
+; you may not use this file except in compliance with the License.
+; You may obtain a copy of the License at
+;
+; http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing, software
+; distributed under the License is distributed on an "AS IS" BASIS,
+; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+; See the License for the specific language governing permissions and
+; limitations under the License.
+
+[tox]
+envlist = py27
+#,py35,py36,py37
+skip_missing_interpreters = true
+skipsdist = True
+
+[testenv]
+deps =
+ -r requirements.txt
+ nose2
+# flake8
+
+commands =
+ nose2 -c tox.ini --verbose --junit-xml
+# flake8
+
+[flake8]
+max-line-length = 119
+
+[unittest]
+plugins = nose2.plugins.junitxml
+
+[junit-xml]
+path = nose2-results.xml
+
+[coverage]
+always-on = True
+coverage =
+ protoc_plugins
+ grpc_client
+ utils
+ web_server
+coverage-report =
+ term
+ xml
+
diff --git a/utils/asleep.py b/utils/asleep.py
index 295f675..f8870e9 100644
--- a/utils/asleep.py
+++ b/utils/asleep.py
@@ -25,4 +25,4 @@
"""
d = Deferred()
reactor.callLater(dt, lambda: d.callback(None))
- return d
\ No newline at end of file
+ return d
diff --git a/utils/dockerhelpers.py b/utils/dockerhelpers.py
index b924628..4a0c434 100644
--- a/utils/dockerhelpers.py
+++ b/utils/dockerhelpers.py
@@ -55,7 +55,7 @@
docker_cli = DockerClient(base_url=docker_socket)
info = docker_cli.inspect_container(my_container_id)
- except Exception, e:
+ except Exception as e:
log.exception('failed', my_container_id=my_container_id, e=e)
raise
diff --git a/utils/nethelpers.py b/utils/nethelpers.py
index 16a57c7..b79d2b0 100644
--- a/utils/nethelpers.py
+++ b/utils/nethelpers.py
@@ -43,4 +43,4 @@
if __name__ == '__main__':
- print get_my_primary_local_ipv4()
\ No newline at end of file
+ print get_my_primary_local_ipv4()
diff --git a/utils/structlog_setup.py b/utils/structlog_setup.py
index 6a149c0..0cdb400 100644
--- a/utils/structlog_setup.py
+++ b/utils/structlog_setup.py
@@ -43,6 +43,7 @@
"""Our special version of OrderedDict that renders into string as a dict,
to make the log stream output cleaner.
"""
+
def __repr__(self, _repr_running={}):
'od.__repr__() <==> repr(od)'
call_key = id(self), _get_ident()
diff --git a/web_server/web_server.py b/web_server/web_server.py
index f817045..3f1ef3a 100644
--- a/web_server/web_server.py
+++ b/web_server/web_server.py
@@ -66,14 +66,14 @@
try:
log.debug(request=request)
return File(self.swagger_ui_root_dir)
- except Exception, e:
+ except Exception as e:
log.exception('file-not-found', request=request)
@app.route(swagger_url + '/v1/swagger.json')
def swagger_json(self, request):
try:
return File(os.path.join(self.work_dir, 'swagger.json'))
- except Exception, e:
+ except Exception as e:
log.exception('file-not-found', request=request)
@inlineCallbacks
@@ -95,7 +95,7 @@
@inlineCallbacks
def _open_endpoint(self):
try:
- if self.key == None or self.cert == None:
+ if self.key is None or self.cert is None:
endpoint = endpoints.TCP4ServerEndpoint(reactor, self.port)
else:
# Enforce TLSv1_2_METHOD
@@ -106,7 +106,7 @@
self.tcp_port = yield endpoint.listen(self.site)
log.info('web-server-started', port=self.port)
self.endpoint = endpoint
- except Exception, e:
+ except Exception as e:
self.log.exception('web-server-failed-to-start', e=e)
def reload_generated_routes(self):
@@ -139,4 +139,3 @@
else:
request.setResponseCode(500)
return json.dumps({'error': 'Internal Server Error', 'specific_error': failure.value.details()})
-