Fix to docker import issues and swagger error
This is actually two fixes in one change:
1. Fixing the import issues in the docker containers, also cleaning
it up.
2. Avoiding the creation of multiple swagger.json files by specifying
which of the proto files represent the top-level service.
Change-Id: I9fec5cf48df127725673ba53f0e91d2ed2e275ad
diff --git a/Dockerfile.chameleon b/Dockerfile.chameleon
index 6a14b69..600e6d7 100644
--- a/Dockerfile.chameleon
+++ b/Dockerfile.chameleon
@@ -35,10 +35,10 @@
rm -f *.deb
# Bundle app source
-COPY chameleon /chameleon
-COPY common /common
+RUN mkdir /chameleon && touch /chameleon/__init__.py
+ENV PYTHONPATH=/chameleon
+COPY common /chameleon/common
+COPY chameleon /chameleon/chameleon
# Exposing process and default entry point
-# EXPOSE 8000
-
-CMD ["python", "chameleon/main.py"]
+CMD ["python", "chameleon/chameleon/main.py"]
diff --git a/Dockerfile.ofagent b/Dockerfile.ofagent
index 4f23ad1..a64626c 100644
--- a/Dockerfile.ofagent
+++ b/Dockerfile.ofagent
@@ -35,10 +35,10 @@
rm -f *.deb
# Bundle app source
-COPY ofagent /ofagent
-COPY common /common
+RUN mkdir /ofagent && touch /ofagent/__init__.py
+ENV PYTHONPATH=/ofagent
+COPY common /ofagent/common
+COPY ofagent /ofagent/ofagent
# Exposing process and default entry point
-# EXPOSE 8000
-
-CMD ["python", "ofagent/main.py"]
+CMD ["python", "ofagent/ofagent/main.py"]
diff --git a/Dockerfile.voltha b/Dockerfile.voltha
index 405f499..f47f9c0 100644
--- a/Dockerfile.voltha
+++ b/Dockerfile.voltha
@@ -22,9 +22,11 @@
MAINTAINER Nathan Knuth <nathan.knuth@tibitcom.com>
# Bundle app source
-COPY voltha /voltha
-COPY common /common
+RUN mkdir /voltha && touch /voltha/__init__.py
+ENV PYTHONPATH=/voltha
+COPY common /voltha/common
+COPY voltha /voltha/voltha
# Exposing process and default entry point
# EXPOSE 8000
-CMD ["python", "voltha/main.py"]
+CMD ["python", "voltha/voltha/main.py"]
diff --git a/chameleon/grpc_client/grpc_client.py b/chameleon/grpc_client/grpc_client.py
index 9a95330..36618be 100644
--- a/chameleon/grpc_client/grpc_client.py
+++ b/chameleon/grpc_client/grpc_client.py
@@ -34,6 +34,7 @@
from werkzeug.exceptions import ServiceUnavailable
from common.utils.asleep import asleep
+from chameleon.protos import third_party
from chameleon.protos.schema_pb2 import SchemaServiceStub
from google.protobuf.empty_pb2 import Empty
@@ -101,8 +102,8 @@
log.info('connecting', endpoint=_endpoint)
self.channel = grpc.insecure_channel(_endpoint)
- self._retrieve_schema()
- self._compile_proto_files()
+ swagger_from = self._retrieve_schema()
+ self._compile_proto_files(swagger_from)
self._clear_backoff()
self.connected = True
@@ -192,8 +193,9 @@
length=len(desc_content))
with open(os.path.join(self.work_dir, desc_fname), 'wb') as f:
f.write(desc_content)
+ return schemas.swagger_from
- def _compile_proto_files(self):
+ def _compile_proto_files(self, swagger_from):
"""
For each *.proto file in the work directory, compile the proto
file into the respective *_pb2.py file as well as generate the
@@ -211,7 +213,8 @@
for fname in [f for f in os.listdir(self.work_dir)
if f.endswith('.proto')]:
- log.debug('compiling', file=fname)
+ need_swagger = fname == swagger_from
+ log.debug('compiling', file=fname, need_swagger=need_swagger)
cmd = (
'cd %s && '
'env PATH=%s PYTHONPATH=%s '
@@ -223,7 +226,7 @@
'--plugin=protoc-gen-gw=%s/gw_gen.py '
'--gw_out=. '
'--plugin=protoc-gen-swagger=%s/swagger_gen.py '
- '--swagger_out=. '
+ '%s'
'%s' % (
self.work_dir,
':'.join([os.environ['PATH'], self.plugin_dir]),
@@ -231,6 +234,7 @@
google_api_dir,
self.plugin_dir,
self.plugin_dir,
+ '--swagger_out=. ' if need_swagger else '',
fname)
)
log.debug('executing', cmd=cmd, file=fname)
diff --git a/chameleon/protos/schema.proto b/chameleon/protos/schema.proto
index 0c06feb..764f53f 100644
--- a/chameleon/protos/schema.proto
+++ b/chameleon/protos/schema.proto
@@ -17,6 +17,9 @@
// Proto files
repeated ProtoFile protos = 1;
+ // Name of proto file to generae swagger.json from
+ string swagger_from = 2;
+
}
// Schema services
diff --git a/chameleon/protos/schema_pb2.py b/chameleon/protos/schema_pb2.py
index 537ba02..6f45cc6 100644
--- a/chameleon/protos/schema_pb2.py
+++ b/chameleon/protos/schema_pb2.py
@@ -20,7 +20,7 @@
name='schema.proto',
package='schema',
syntax='proto3',
- serialized_pb=_b('\n\x0cschema.proto\x12\x06schema\x1a\x1bgoogle/protobuf/empty.proto\"A\n\tProtoFile\x12\x11\n\tfile_name\x18\x01 \x01(\t\x12\r\n\x05proto\x18\x02 \x01(\t\x12\x12\n\ndescriptor\x18\x03 \x01(\x0c\",\n\x07Schemas\x12!\n\x06protos\x18\x01 \x03(\x0b\x32\x11.schema.ProtoFile2G\n\rSchemaService\x12\x36\n\tGetSchema\x12\x16.google.protobuf.Empty\x1a\x0f.schema.Schemas\"\x00\x62\x06proto3')
+ serialized_pb=_b('\n\x0cschema.proto\x12\x06schema\x1a\x1bgoogle/protobuf/empty.proto\"A\n\tProtoFile\x12\x11\n\tfile_name\x18\x01 \x01(\t\x12\r\n\x05proto\x18\x02 \x01(\t\x12\x12\n\ndescriptor\x18\x03 \x01(\x0c\"B\n\x07Schemas\x12!\n\x06protos\x18\x01 \x03(\x0b\x32\x11.schema.ProtoFile\x12\x14\n\x0cswagger_from\x18\x02 \x01(\t2G\n\rSchemaService\x12\x36\n\tGetSchema\x12\x16.google.protobuf.Empty\x1a\x0f.schema.Schemas\"\x00\x62\x06proto3')
,
dependencies=[google_dot_protobuf_dot_empty__pb2.DESCRIPTOR,])
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
@@ -87,6 +87,13 @@
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
+ _descriptor.FieldDescriptor(
+ name='swagger_from', full_name='schema.Schemas.swagger_from', index=1,
+ number=2, type=9, cpp_type=9, label=1,
+ has_default_value=False, default_value=_b("").decode('utf-8'),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
],
extensions=[
],
@@ -100,7 +107,7 @@
oneofs=[
],
serialized_start=120,
- serialized_end=164,
+ serialized_end=186,
)
_SCHEMAS.fields_by_name['protos'].message_type = _PROTOFILE
@@ -172,6 +179,11 @@
class BetaSchemaServiceServicer(object):
+ """The Beta API is deprecated for 0.15.0 and later.
+
+ It is recommended to use the GA API (classes and functions in this
+ file not marked beta) for all further purposes. This class was generated
+ only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0."""
"""Schema services
"""
def GetSchema(self, request, context):
@@ -181,6 +193,11 @@
class BetaSchemaServiceStub(object):
+ """The Beta API is deprecated for 0.15.0 and later.
+
+ It is recommended to use the GA API (classes and functions in this
+ file not marked beta) for all further purposes. This class was generated
+ only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0."""
"""Schema services
"""
def GetSchema(self, request, timeout, metadata=None, with_call=False, protocol_options=None):
@@ -191,6 +208,11 @@
def beta_create_SchemaService_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None):
+ """The Beta API is deprecated for 0.15.0 and later.
+
+ It is recommended to use the GA API (classes and functions in this
+ file not marked beta) for all further purposes. This function was
+ generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0"""
request_deserializers = {
('schema.SchemaService', 'GetSchema'): google_dot_protobuf_dot_empty__pb2.Empty.FromString,
}
@@ -205,6 +227,11 @@
def beta_create_SchemaService_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None):
+ """The Beta API is deprecated for 0.15.0 and later.
+
+ It is recommended to use the GA API (classes and functions in this
+ file not marked beta) for all further purposes. This function was
+ generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0"""
request_serializers = {
('schema.SchemaService', 'GetSchema'): google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString,
}
diff --git a/compose/docker-compose-system-test.yml b/compose/docker-compose-system-test.yml
index ad23313..c186b8f 100644
--- a/compose/docker-compose-system-test.yml
+++ b/compose/docker-compose-system-test.yml
@@ -73,7 +73,7 @@
voltha:
image: cord/voltha
command: [
- "/voltha/main.py",
+ "/voltha/voltha/main.py",
"-v",
"--consul=${DOCKER_HOST_IP}:8500",
"--fluentd=fluentd:24224",
@@ -106,7 +106,7 @@
chameleon:
image: cord/chameleon
command: [
- "/chameleon/main.py",
+ "/chameleon/chameleon/main.py",
"-v",
"--consul=consul:8500",
"--fluentd=fluentd:24224",
@@ -133,7 +133,7 @@
ofagent:
image: cord/ofagent
command: [
- "/ofagent/main.py",
+ "/ofagent/ofagent/main.py",
"-v",
"--consul=${DOCKER_HOST_IP}:8500",
"--fluentd=fluentd:24224",
diff --git a/ofagent/connection_mgr.py b/ofagent/connection_mgr.py
index b55d47a..17af7b2 100644
--- a/ofagent/connection_mgr.py
+++ b/ofagent/connection_mgr.py
@@ -23,7 +23,7 @@
from common.utils.consulhelpers import get_endpoint_from_consul
from structlog import get_logger
import grpc
-# from ofagent.protos import third_party
+from ofagent.protos import third_party
from protos import voltha_pb2
from grpc_client import GrpcClient
diff --git a/ofagent/protos/Makefile b/ofagent/protos/Makefile
index 157052d..2d9c069 100644
--- a/ofagent/protos/Makefile
+++ b/ofagent/protos/Makefile
@@ -33,5 +33,5 @@
build: copyfiles
copyfiles:
- rsync -av --include '*/' --include '*.py' --exclude='*' $(SOURCE_PROTO_DIR)/ $(TARGET_PROTO_DIR)
+ rsync -av --include '*/' --exclude='third_party/__init__.py' --include '*.py' --exclude='*' $(SOURCE_PROTO_DIR)/ $(TARGET_PROTO_DIR)
diff --git a/ofagent/protos/third_party/__init__.py b/ofagent/protos/third_party/__init__.py
new file mode 100644
index 0000000..6dab4e7
--- /dev/null
+++ b/ofagent/protos/third_party/__init__.py
@@ -0,0 +1,50 @@
+#
+# Copyright 2016 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+This helps loading http_pb2 and annotations_pb2.
+Without this, the Python importer will not be able to process the lines:
+from google.api import http_pb2 or
+from google.api import annotations_pb2
+(Without importing these, the protobuf loader will not recognize http options
+in the protobuf definitions.)
+"""
+
+from importlib import import_module
+import os
+import sys
+
+
+class GoogleApiImporter(object):
+
+ def find_module(self, full_name, path=None):
+ if full_name == 'google.api':
+ self.path = [os.path.dirname(__file__)]
+ return self
+
+ def load_module(self, name):
+ if name in sys.modules:
+ return sys.modules[name]
+ full_name = 'ofagent.protos.third_party.' + name
+ import_module(full_name)
+ module = sys.modules[full_name]
+ sys.modules[name] = module
+ return module
+
+
+sys.meta_path.append(GoogleApiImporter())
+from google.api import http_pb2, annotations_pb2
+_ = http_pb2, annotations_pb2
diff --git a/voltha/northbound/grpc/grpc_server.py b/voltha/northbound/grpc/grpc_server.py
index 04eda7b..a0b3c8a 100644
--- a/voltha/northbound/grpc/grpc_server.py
+++ b/voltha/northbound/grpc/grpc_server.py
@@ -44,7 +44,8 @@
def __init__(self, thread_pool):
self.thread_pool = thread_pool
protos = self._load_schema()
- self.schemas = schema_pb2.Schemas(protos=protos)
+ self.schemas = schema_pb2.Schemas(protos=protos,
+ swagger_from='voltha.proto')
def stop(self):
pass
diff --git a/voltha/protos/schema.desc b/voltha/protos/schema.desc
index 1f15f1f..31ccf8b 100644
--- a/voltha/protos/schema.desc
+++ b/voltha/protos/schema.desc
Binary files differ
diff --git a/voltha/protos/schema.proto b/voltha/protos/schema.proto
index 5dc0eb8..e75f696 100644
--- a/voltha/protos/schema.proto
+++ b/voltha/protos/schema.proto
@@ -18,6 +18,9 @@
// Proto files
repeated ProtoFile protos = 1;
+ // Proto file name from which swagger.json shall be generated
+ string swagger_from = 2;
+
}
// Schema services
diff --git a/voltha/protos/schema_pb2.py b/voltha/protos/schema_pb2.py
index 266b463..3ebbb23 100644
--- a/voltha/protos/schema_pb2.py
+++ b/voltha/protos/schema_pb2.py
@@ -21,7 +21,7 @@
name='schema.proto',
package='schema',
syntax='proto3',
- serialized_pb=_b('\n\x0cschema.proto\x12\x06schema\x1a\x1cgoogle/api/annotations.proto\x1a\x1bgoogle/protobuf/empty.proto\"A\n\tProtoFile\x12\x11\n\tfile_name\x18\x01 \x01(\t\x12\r\n\x05proto\x18\x02 \x01(\t\x12\x12\n\ndescriptor\x18\x03 \x01(\x0c\",\n\x07Schemas\x12!\n\x06protos\x18\x01 \x03(\x0b\x32\x11.schema.ProtoFile2V\n\rSchemaService\x12\x45\n\tGetSchema\x12\x16.google.protobuf.Empty\x1a\x0f.schema.Schemas\"\x0f\x82\xd3\xe4\x93\x02\t\x12\x07/schemab\x06proto3')
+ serialized_pb=_b('\n\x0cschema.proto\x12\x06schema\x1a\x1cgoogle/api/annotations.proto\x1a\x1bgoogle/protobuf/empty.proto\"A\n\tProtoFile\x12\x11\n\tfile_name\x18\x01 \x01(\t\x12\r\n\x05proto\x18\x02 \x01(\t\x12\x12\n\ndescriptor\x18\x03 \x01(\x0c\"B\n\x07Schemas\x12!\n\x06protos\x18\x01 \x03(\x0b\x32\x11.schema.ProtoFile\x12\x14\n\x0cswagger_from\x18\x02 \x01(\t2V\n\rSchemaService\x12\x45\n\tGetSchema\x12\x16.google.protobuf.Empty\x1a\x0f.schema.Schemas\"\x0f\x82\xd3\xe4\x93\x02\t\x12\x07/schemab\x06proto3')
,
dependencies=[google_dot_api_dot_annotations__pb2.DESCRIPTOR,google_dot_protobuf_dot_empty__pb2.DESCRIPTOR,])
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
@@ -88,6 +88,13 @@
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
+ _descriptor.FieldDescriptor(
+ name='swagger_from', full_name='schema.Schemas.swagger_from', index=1,
+ number=2, type=9, cpp_type=9, label=1,
+ has_default_value=False, default_value=_b("").decode('utf-8'),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
],
extensions=[
],
@@ -101,7 +108,7 @@
oneofs=[
],
serialized_start=150,
- serialized_end=194,
+ serialized_end=216,
)
_SCHEMAS.fields_by_name['protos'].message_type = _PROTOFILE
@@ -173,6 +180,11 @@
class BetaSchemaServiceServicer(object):
+ """The Beta API is deprecated for 0.15.0 and later.
+
+ It is recommended to use the GA API (classes and functions in this
+ file not marked beta) for all further purposes. This class was generated
+ only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0."""
"""Schema services
"""
def GetSchema(self, request, context):
@@ -182,6 +194,11 @@
class BetaSchemaServiceStub(object):
+ """The Beta API is deprecated for 0.15.0 and later.
+
+ It is recommended to use the GA API (classes and functions in this
+ file not marked beta) for all further purposes. This class was generated
+ only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0."""
"""Schema services
"""
def GetSchema(self, request, timeout, metadata=None, with_call=False, protocol_options=None):
@@ -192,6 +209,11 @@
def beta_create_SchemaService_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None):
+ """The Beta API is deprecated for 0.15.0 and later.
+
+ It is recommended to use the GA API (classes and functions in this
+ file not marked beta) for all further purposes. This function was
+ generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0"""
request_deserializers = {
('schema.SchemaService', 'GetSchema'): google_dot_protobuf_dot_empty__pb2.Empty.FromString,
}
@@ -206,6 +228,11 @@
def beta_create_SchemaService_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None):
+ """The Beta API is deprecated for 0.15.0 and later.
+
+ It is recommended to use the GA API (classes and functions in this
+ file not marked beta) for all further purposes. This function was
+ generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0"""
request_serializers = {
('schema.SchemaService', 'GetSchema'): google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString,
}