Nomralize component start()/stop()
Also fixed the /schema swagger/rest entry. It did not work
because the 3rdparty protobuf_to_dict library cannot handle
Map fields. Changed the two map fields to a single list
entry.
Change-Id: Ib25a528701b67d58d32451687724c8247da6efa5
diff --git a/grpc_client/grpc_client.py b/grpc_client/grpc_client.py
index 9129b2e..75c78eb 100644
--- a/grpc_client/grpc_client.py
+++ b/grpc_client/grpc_client.py
@@ -47,31 +47,39 @@
"""
RETRY_BACKOFF = [0.05, 0.1, 0.2, 0.5, 1, 2, 5]
- def __init__(self, consul_endpoint, work_dir, endpoint='localhost:50055'):
+ def __init__(self, consul_endpoint, work_dir, endpoint='localhost:50055',
+ reconnect_callback=None):
self.consul_endpoint = consul_endpoint
self.endpoint = endpoint
self.work_dir = work_dir
+ self.reconnect_callback = reconnect_callback
+
self.plugin_dir = os.path.abspath(os.path.join(
os.path.dirname(__file__), '../protoc_plugins'))
self.channel = None
self.schema = None
self.retries = 0
- self.on_reconnect = None
self.shutting_down = False
self.connected = False
- def run(self, on_reconnect=None):
- self.on_reconnect = on_reconnect
+ def start(self):
+ log.debug('starting')
if not self.connected:
reactor.callLater(0, self.connect)
+ log.info('started')
return self
- def shutdown(self):
+ def stop(self):
+ log.debug('stopping')
if self.shutting_down:
return
self.shutting_down = True
- pass
+ log.info('stopped')
+
+ def set_reconnect_callback(self, reconnect_callback):
+ self.reconnect_callback = reconnect_callback
+ return self
@inlineCallbacks
def connect(self):
@@ -96,8 +104,8 @@
self._clear_backoff()
self.connected = True
- if self.on_reconnect is not None:
- reactor.callLater(0, self.on_reconnect)
+ if self.reconnect_callback is not None:
+ reactor.callLater(0, self.reconnect_callback)
return
@@ -157,7 +165,7 @@
assert isinstance(self.channel, grpc.Channel)
stub = SchemaServiceStub(self.channel)
# try:
- schema = stub.GetSchema(NullMessage())
+ schemas = stub.GetSchema(NullMessage())
# except _Rendezvous, e:
# if e.code == grpc.StatusCode.UNAVAILABLE:
#
@@ -168,19 +176,20 @@
os.system('rm -fr /tmp/%s/*' %
self.work_dir.replace('/tmp/', '')) # safer
- for fname in schema.protos:
- content = schema.protos[fname]
- log.debug('saving-proto',
- fname=fname, dir=self.work_dir, length=len(content))
- with open(os.path.join(self.work_dir, fname), 'w') as f:
- f.write(content)
+ for proto_file in schemas.protos:
+ proto_fname = proto_file.file_name
+ proto_content = proto_file.proto
+ log.debug('saving-proto', fname=proto_fname, dir=self.work_dir,
+ length=len(proto_content))
+ with open(os.path.join(self.work_dir, proto_fname), 'w') as f:
+ f.write(proto_content)
- for fname in schema.descriptors:
- content = decompress(schema.descriptors[fname])
- log.debug('saving-descriptor',
- fname=fname, dir=self.work_dir, length=len(content))
- with open(os.path.join(self.work_dir, fname), 'wb') as f:
- f.write(content)
+ desc_content = decompress(proto_file.descriptor)
+ desc_fname = proto_fname.replace('.proto', '.desc')
+ log.debug('saving-descriptor', fname=desc_fname, dir=self.work_dir,
+ length=len(desc_content))
+ with open(os.path.join(self.work_dir, desc_fname), 'wb') as f:
+ f.write(desc_content)
def _compile_proto_files(self):
"""
diff --git a/main.py b/main.py
index 14c854b..0065b66 100755
--- a/main.py
+++ b/main.py
@@ -224,9 +224,9 @@
self.grpc_client = yield \
GrpcClient(args.consul, args.work_dir, args.grpc_endpoint)
self.web_server = yield \
- WebServer(args.rest_port, args.work_dir, self.grpc_client).run()
- self.grpc_client.run(
- on_reconnect=self.web_server.reload_generated_routes)
+ WebServer(args.rest_port, args.work_dir, self.grpc_client).start()
+ self.grpc_client.set_reconnect_callback(
+ self.web_server.reload_generated_routes).start()
self.log.info('started-internal-services')
@inlineCallbacks
@@ -234,9 +234,9 @@
"""Execute before the reactor is shut down"""
self.log.info('exiting-on-keyboard-interrupt')
if self.rest_server is not None:
- yield self.rest_server.shutdown()
+ yield self.rest_server.stop()
if self.grpc_client is not None:
- yield self.grpc_client.shutdown()
+ yield self.grpc_client.stop()
def start_reactor(self):
from twisted.internet import reactor
diff --git a/protoc_plugins/gw_gen.py b/protoc_plugins/gw_gen.py
index 5398c32..20249c2 100755
--- a/protoc_plugins/gw_gen.py
+++ b/protoc_plugins/gw_gen.py
@@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+
import sys
from google.protobuf.compiler import plugin_pb2 as plugin
@@ -56,11 +57,22 @@
{% else %}
riase NotImplementedError('cannot handle specific body field list')
{% endif %}
- req = dict_to_protobuf({{ method['input_type'] }}, data)
+ try:
+ req = dict_to_protobuf({{ method['input_type'] }}, data)
+ except Exception, e:
+ log.error('cannot-convert-to-protobuf', e=e, data=data)
+ raise
res = grpc_client.invoke(
{{ '.'.join([package, method['service']]) }}Stub,
'{{ method['method'] }}', req)
- out_data = protobuf_to_dict(res, use_enum_labels=True)
+ try:
+ out_data = protobuf_to_dict(res, use_enum_labels=True)
+ except AttributeError, e:
+ filename = '/tmp/chameleon_failed_to_convert_data.pbd'
+ with file(filename, 'w') as f:
+ f.write(res.SerializeToString())
+ log.error('cannot-convert-from-protobuf', outdata_saved=filename)
+ raise
request.setHeader('Content-Type', 'application/json')
log.debug('{{ method_name }}', **out_data)
return dumps(out_data)
diff --git a/protoc_plugins/schema2dict.py b/protoc_plugins/schema2dict.py
new file mode 100644
index 0000000..31c0417
--- /dev/null
+++ b/protoc_plugins/schema2dict.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+Convert a schema.Schema object given on the standard input as protobuf data
+file (from standard output) to a Python dictionary.
+"""
+import json
+import sys
+from chameleon.protos.third_party.google.api import annotations_pb2
+_ = annotations_pb2
+from chameleon.protos import schema_pb2
+from protobuf_to_dict import protobuf_to_dict
+
+
+if __name__ == '__main__':
+
+ data = sys.stdin.read()
+ schemas = schema_pb2.Schemas()
+ schemas.ParseFromString(data)
+
+ data = protobuf_to_dict(schemas, use_enum_labels=True)
+ json.dump(data, sys.stdout)
diff --git a/protos/schema.proto b/protos/schema.proto
index 530d74a..2a1ec08 100644
--- a/protos/schema.proto
+++ b/protos/schema.proto
@@ -2,14 +2,18 @@
package schema;
-// Proto file and compiled descriptor for this interface
-message Schema {
+// Contains the name and content of a *.proto file
+message ProtoFile {
+ string file_name = 1; // name of proto file
+ string proto = 2; // content of proto file
+ bytes descriptor = 3; // compiled descriptor for proto (zlib compressed)
+}
- // file name -> proto file content
- map<string, string> protos = 1;
+// Proto files and compiled descriptors for this interface
+message Schemas {
- // file name -> gzip compressed protobuf of descriptor
- map<string, bytes> descriptors = 2;
+ // Proto files
+ repeated ProtoFile protos = 1;
}
@@ -19,7 +23,7 @@
// Schema services
service SchemaService {
- // Return active grpc schemas
- rpc GetSchema(NullMessage) returns (Schema) {}
+ // Return active grpc schemas
+ rpc GetSchema(NullMessage) returns (Schemas) {}
}
diff --git a/protos/schema_pb2.py b/protos/schema_pb2.py
index eab43f5..f5f17d2 100644
--- a/protos/schema_pb2.py
+++ b/protos/schema_pb2.py
@@ -19,67 +19,37 @@
name='schema.proto',
package='schema',
syntax='proto3',
- serialized_pb=_b('\n\x0cschema.proto\x12\x06schema\"\xcd\x01\n\x06Schema\x12*\n\x06protos\x18\x01 \x03(\x0b\x32\x1a.schema.Schema.ProtosEntry\x12\x34\n\x0b\x64\x65scriptors\x18\x02 \x03(\x0b\x32\x1f.schema.Schema.DescriptorsEntry\x1a-\n\x0bProtosEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x32\n\x10\x44\x65scriptorsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x0c:\x02\x38\x01\"\r\n\x0bNullMessage2C\n\rSchemaService\x12\x32\n\tGetSchema\x12\x13.schema.NullMessage\x1a\x0e.schema.Schema\"\x00\x62\x06proto3')
+ serialized_pb=_b('\n\x0cschema.proto\x12\x06schema\"A\n\tProtoFile\x12\x11\n\tfile_name\x18\x01 \x01(\t\x12\r\n\x05proto\x18\x02 \x01(\t\x12\x12\n\ndescriptor\x18\x03 \x01(\x0c\",\n\x07Schemas\x12!\n\x06protos\x18\x01 \x03(\x0b\x32\x11.schema.ProtoFile\"\r\n\x0bNullMessage2D\n\rSchemaService\x12\x33\n\tGetSchema\x12\x13.schema.NullMessage\x1a\x0f.schema.Schemas\"\x00\x62\x06proto3')
)
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
-_SCHEMA_PROTOSENTRY = _descriptor.Descriptor(
- name='ProtosEntry',
- full_name='schema.Schema.ProtosEntry',
+_PROTOFILE = _descriptor.Descriptor(
+ name='ProtoFile',
+ full_name='schema.ProtoFile',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
- name='key', full_name='schema.Schema.ProtosEntry.key', index=0,
+ name='file_name', full_name='schema.ProtoFile.file_name', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='value', full_name='schema.Schema.ProtosEntry.value', index=1,
+ name='proto', full_name='schema.ProtoFile.proto', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=133,
- serialized_end=178,
-)
-
-_SCHEMA_DESCRIPTORSENTRY = _descriptor.Descriptor(
- name='DescriptorsEntry',
- full_name='schema.Schema.DescriptorsEntry',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
_descriptor.FieldDescriptor(
- name='key', full_name='schema.Schema.DescriptorsEntry.key', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='value', full_name='schema.Schema.DescriptorsEntry.value', index=1,
- number=2, type=12, cpp_type=9, label=1,
+ name='descriptor', full_name='schema.ProtoFile.descriptor', index=2,
+ number=3, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
@@ -90,41 +60,35 @@
nested_types=[],
enum_types=[
],
- options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
+ options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
- serialized_start=180,
- serialized_end=230,
+ serialized_start=24,
+ serialized_end=89,
)
-_SCHEMA = _descriptor.Descriptor(
- name='Schema',
- full_name='schema.Schema',
+
+_SCHEMAS = _descriptor.Descriptor(
+ name='Schemas',
+ full_name='schema.Schemas',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
- name='protos', full_name='schema.Schema.protos', index=0,
+ name='protos', full_name='schema.Schemas.protos', index=0,
number=1, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
- _descriptor.FieldDescriptor(
- name='descriptors', full_name='schema.Schema.descriptors', index=1,
- number=2, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
],
extensions=[
],
- nested_types=[_SCHEMA_PROTOSENTRY, _SCHEMA_DESCRIPTORSENTRY, ],
+ nested_types=[],
enum_types=[
],
options=None,
@@ -133,8 +97,8 @@
extension_ranges=[],
oneofs=[
],
- serialized_start=25,
- serialized_end=230,
+ serialized_start=91,
+ serialized_end=135,
)
@@ -157,39 +121,28 @@
extension_ranges=[],
oneofs=[
],
- serialized_start=232,
- serialized_end=245,
+ serialized_start=137,
+ serialized_end=150,
)
-_SCHEMA_PROTOSENTRY.containing_type = _SCHEMA
-_SCHEMA_DESCRIPTORSENTRY.containing_type = _SCHEMA
-_SCHEMA.fields_by_name['protos'].message_type = _SCHEMA_PROTOSENTRY
-_SCHEMA.fields_by_name['descriptors'].message_type = _SCHEMA_DESCRIPTORSENTRY
-DESCRIPTOR.message_types_by_name['Schema'] = _SCHEMA
+_SCHEMAS.fields_by_name['protos'].message_type = _PROTOFILE
+DESCRIPTOR.message_types_by_name['ProtoFile'] = _PROTOFILE
+DESCRIPTOR.message_types_by_name['Schemas'] = _SCHEMAS
DESCRIPTOR.message_types_by_name['NullMessage'] = _NULLMESSAGE
-Schema = _reflection.GeneratedProtocolMessageType('Schema', (_message.Message,), dict(
-
- ProtosEntry = _reflection.GeneratedProtocolMessageType('ProtosEntry', (_message.Message,), dict(
- DESCRIPTOR = _SCHEMA_PROTOSENTRY,
- __module__ = 'schema_pb2'
- # @@protoc_insertion_point(class_scope:schema.Schema.ProtosEntry)
- ))
- ,
-
- DescriptorsEntry = _reflection.GeneratedProtocolMessageType('DescriptorsEntry', (_message.Message,), dict(
- DESCRIPTOR = _SCHEMA_DESCRIPTORSENTRY,
- __module__ = 'schema_pb2'
- # @@protoc_insertion_point(class_scope:schema.Schema.DescriptorsEntry)
- ))
- ,
- DESCRIPTOR = _SCHEMA,
+ProtoFile = _reflection.GeneratedProtocolMessageType('ProtoFile', (_message.Message,), dict(
+ DESCRIPTOR = _PROTOFILE,
__module__ = 'schema_pb2'
- # @@protoc_insertion_point(class_scope:schema.Schema)
+ # @@protoc_insertion_point(class_scope:schema.ProtoFile)
))
-_sym_db.RegisterMessage(Schema)
-_sym_db.RegisterMessage(Schema.ProtosEntry)
-_sym_db.RegisterMessage(Schema.DescriptorsEntry)
+_sym_db.RegisterMessage(ProtoFile)
+
+Schemas = _reflection.GeneratedProtocolMessageType('Schemas', (_message.Message,), dict(
+ DESCRIPTOR = _SCHEMAS,
+ __module__ = 'schema_pb2'
+ # @@protoc_insertion_point(class_scope:schema.Schemas)
+ ))
+_sym_db.RegisterMessage(Schemas)
NullMessage = _reflection.GeneratedProtocolMessageType('NullMessage', (_message.Message,), dict(
DESCRIPTOR = _NULLMESSAGE,
@@ -199,10 +152,6 @@
_sym_db.RegisterMessage(NullMessage)
-_SCHEMA_PROTOSENTRY.has_options = True
-_SCHEMA_PROTOSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
-_SCHEMA_DESCRIPTORSENTRY.has_options = True
-_SCHEMA_DESCRIPTORSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
import grpc
from grpc.beta import implementations as beta_implementations
from grpc.beta import interfaces as beta_interfaces
@@ -223,7 +172,7 @@
self.GetSchema = channel.unary_unary(
'/schema.SchemaService/GetSchema',
request_serializer=NullMessage.SerializeToString,
- response_deserializer=Schema.FromString,
+ response_deserializer=Schemas.FromString,
)
@@ -244,7 +193,7 @@
'GetSchema': grpc.unary_unary_rpc_method_handler(
servicer.GetSchema,
request_deserializer=NullMessage.FromString,
- response_serializer=Schema.SerializeToString,
+ response_serializer=Schemas.SerializeToString,
),
}
generic_handler = grpc.method_handlers_generic_handler(
@@ -276,7 +225,7 @@
('schema.SchemaService', 'GetSchema'): NullMessage.FromString,
}
response_serializers = {
- ('schema.SchemaService', 'GetSchema'): Schema.SerializeToString,
+ ('schema.SchemaService', 'GetSchema'): Schemas.SerializeToString,
}
method_implementations = {
('schema.SchemaService', 'GetSchema'): face_utilities.unary_unary_inline(servicer.GetSchema),
@@ -290,7 +239,7 @@
('schema.SchemaService', 'GetSchema'): NullMessage.SerializeToString,
}
response_deserializers = {
- ('schema.SchemaService', 'GetSchema'): Schema.FromString,
+ ('schema.SchemaService', 'GetSchema'): Schemas.FromString,
}
cardinalities = {
'GetSchema': cardinality.Cardinality.UNARY_UNARY,
diff --git a/web_server/web_server.py b/web_server/web_server.py
index c46ad88..c96be0f 100644
--- a/web_server/web_server.py
+++ b/web_server/web_server.py
@@ -47,11 +47,22 @@
self.shutting_down = False
@inlineCallbacks
- def run(self):
+ def start(self):
+ log.debug('starting')
yield self._open_endpoint()
+ log.info('started')
returnValue(self)
@inlineCallbacks
+ def stop(self):
+ log.debug('stopping')
+ self.shutting_down = True
+ if self.tcp_port is not None:
+ assert isinstance(self.tcp_port, Port)
+ yield self.tcp_port.socket.close()
+ log.info('stopped')
+
+ @inlineCallbacks
def _open_endpoint(self):
endpoint = endpoints.TCP4ServerEndpoint(reactor, self.port)
self.site = Site(self.app.resource())
@@ -59,13 +70,6 @@
log.info('web-server-started', port=self.port)
self.endpoint = endpoint
- @inlineCallbacks
- def shutdown(self):
- self.shutting_down = True
- if self.tcp_port is not None:
- assert isinstance(self.tcp_port, Port)
- yield self.tcp_port.socket.close()
-
def reload_generated_routes(self):
for fname in os.listdir(self.work_dir):
if fname.endswith('_gw.py'):