Nomralize component start()/stop()

Also fixed the /schema swagger/rest entry. It did not work
because the 3rdparty protobuf_to_dict library cannot handle
Map fields. Changed the two map fields to a single list
entry.

Change-Id: Ib25a528701b67d58d32451687724c8247da6efa5
diff --git a/grpc_client/grpc_client.py b/grpc_client/grpc_client.py
index 9129b2e..75c78eb 100644
--- a/grpc_client/grpc_client.py
+++ b/grpc_client/grpc_client.py
@@ -47,31 +47,39 @@
     """
     RETRY_BACKOFF = [0.05, 0.1, 0.2, 0.5, 1, 2, 5]
 
-    def __init__(self, consul_endpoint, work_dir, endpoint='localhost:50055'):
+    def __init__(self, consul_endpoint, work_dir, endpoint='localhost:50055',
+                 reconnect_callback=None):
         self.consul_endpoint = consul_endpoint
         self.endpoint = endpoint
         self.work_dir = work_dir
+        self.reconnect_callback = reconnect_callback
+
         self.plugin_dir = os.path.abspath(os.path.join(
             os.path.dirname(__file__), '../protoc_plugins'))
 
         self.channel = None
         self.schema = None
         self.retries = 0
-        self.on_reconnect = None
         self.shutting_down = False
         self.connected = False
 
-    def run(self, on_reconnect=None):
-        self.on_reconnect = on_reconnect
+    def start(self):
+        log.debug('starting')
         if not self.connected:
             reactor.callLater(0, self.connect)
+        log.info('started')
         return self
 
-    def shutdown(self):
+    def stop(self):
+        log.debug('stopping')
         if self.shutting_down:
             return
         self.shutting_down = True
-        pass
+        log.info('stopped')
+
+    def set_reconnect_callback(self, reconnect_callback):
+        self.reconnect_callback = reconnect_callback
+        return self
 
     @inlineCallbacks
     def connect(self):
@@ -96,8 +104,8 @@
             self._clear_backoff()
 
             self.connected = True
-            if self.on_reconnect is not None:
-                reactor.callLater(0, self.on_reconnect)
+            if self.reconnect_callback is not None:
+                reactor.callLater(0, self.reconnect_callback)
 
             return
 
@@ -157,7 +165,7 @@
         assert isinstance(self.channel, grpc.Channel)
         stub = SchemaServiceStub(self.channel)
         # try:
-        schema = stub.GetSchema(NullMessage())
+        schemas = stub.GetSchema(NullMessage())
         # except _Rendezvous, e:
         #     if e.code == grpc.StatusCode.UNAVAILABLE:
         #
@@ -168,19 +176,20 @@
         os.system('rm -fr /tmp/%s/*' %
                   self.work_dir.replace('/tmp/', ''))  # safer
 
-        for fname in schema.protos:
-            content = schema.protos[fname]
-            log.debug('saving-proto',
-                      fname=fname, dir=self.work_dir, length=len(content))
-            with open(os.path.join(self.work_dir, fname), 'w') as f:
-                f.write(content)
+        for proto_file in schemas.protos:
+            proto_fname = proto_file.file_name
+            proto_content = proto_file.proto
+            log.debug('saving-proto', fname=proto_fname, dir=self.work_dir,
+                      length=len(proto_content))
+            with open(os.path.join(self.work_dir, proto_fname), 'w') as f:
+                f.write(proto_content)
 
-        for fname in schema.descriptors:
-            content = decompress(schema.descriptors[fname])
-            log.debug('saving-descriptor',
-                      fname=fname, dir=self.work_dir, length=len(content))
-            with open(os.path.join(self.work_dir, fname), 'wb') as f:
-                f.write(content)
+            desc_content = decompress(proto_file.descriptor)
+            desc_fname = proto_fname.replace('.proto', '.desc')
+            log.debug('saving-descriptor', fname=desc_fname, dir=self.work_dir,
+                      length=len(desc_content))
+            with open(os.path.join(self.work_dir, desc_fname), 'wb') as f:
+                f.write(desc_content)
 
     def _compile_proto_files(self):
         """
diff --git a/main.py b/main.py
index 14c854b..0065b66 100755
--- a/main.py
+++ b/main.py
@@ -224,9 +224,9 @@
         self.grpc_client = yield \
             GrpcClient(args.consul, args.work_dir, args.grpc_endpoint)
         self.web_server = yield \
-            WebServer(args.rest_port, args.work_dir, self.grpc_client).run()
-        self.grpc_client.run(
-            on_reconnect=self.web_server.reload_generated_routes)
+            WebServer(args.rest_port, args.work_dir, self.grpc_client).start()
+        self.grpc_client.set_reconnect_callback(
+            self.web_server.reload_generated_routes).start()
         self.log.info('started-internal-services')
 
     @inlineCallbacks
@@ -234,9 +234,9 @@
         """Execute before the reactor is shut down"""
         self.log.info('exiting-on-keyboard-interrupt')
         if self.rest_server is not None:
-            yield self.rest_server.shutdown()
+            yield self.rest_server.stop()
         if self.grpc_client is not None:
-            yield self.grpc_client.shutdown()
+            yield self.grpc_client.stop()
 
     def start_reactor(self):
         from twisted.internet import reactor
diff --git a/protoc_plugins/gw_gen.py b/protoc_plugins/gw_gen.py
index 5398c32..20249c2 100755
--- a/protoc_plugins/gw_gen.py
+++ b/protoc_plugins/gw_gen.py
@@ -14,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+
 import sys
 
 from google.protobuf.compiler import plugin_pb2 as plugin
@@ -56,11 +57,22 @@
         {% else %}
         riase NotImplementedError('cannot handle specific body field list')
         {% endif %}
-        req = dict_to_protobuf({{ method['input_type'] }}, data)
+        try:
+            req = dict_to_protobuf({{ method['input_type'] }}, data)
+        except Exception, e:
+            log.error('cannot-convert-to-protobuf', e=e, data=data)
+            raise
         res = grpc_client.invoke(
             {{ '.'.join([package, method['service']]) }}Stub,
             '{{ method['method'] }}', req)
-        out_data = protobuf_to_dict(res, use_enum_labels=True)
+        try:
+            out_data = protobuf_to_dict(res, use_enum_labels=True)
+        except AttributeError, e:
+            filename = '/tmp/chameleon_failed_to_convert_data.pbd'
+            with file(filename, 'w') as f:
+                f.write(res.SerializeToString())
+            log.error('cannot-convert-from-protobuf', outdata_saved=filename)
+            raise
         request.setHeader('Content-Type', 'application/json')
         log.debug('{{ method_name }}', **out_data)
         return dumps(out_data)
diff --git a/protoc_plugins/schema2dict.py b/protoc_plugins/schema2dict.py
new file mode 100644
index 0000000..31c0417
--- /dev/null
+++ b/protoc_plugins/schema2dict.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+Convert a schema.Schema object given on the standard input as protobuf data
+file (from standard output) to a Python dictionary.
+"""
+import json
+import sys
+from chameleon.protos.third_party.google.api import annotations_pb2
+_ = annotations_pb2
+from chameleon.protos import schema_pb2
+from protobuf_to_dict import protobuf_to_dict
+
+
+if __name__ == '__main__':
+
+    data = sys.stdin.read()
+    schemas = schema_pb2.Schemas()
+    schemas.ParseFromString(data)
+
+    data = protobuf_to_dict(schemas, use_enum_labels=True)
+    json.dump(data, sys.stdout)
diff --git a/protos/schema.proto b/protos/schema.proto
index 530d74a..2a1ec08 100644
--- a/protos/schema.proto
+++ b/protos/schema.proto
@@ -2,14 +2,18 @@
 
 package schema;
 
-// Proto file and compiled descriptor for this interface
-message Schema {
+// Contains the name and content of a *.proto file
+message ProtoFile {
+    string file_name = 1;  // name of proto file
+    string proto = 2;  // content of proto file
+    bytes descriptor = 3;  // compiled descriptor for proto (zlib compressed)
+}
 
-  // file name -> proto file content
-  map<string, string> protos = 1;
+// Proto files and compiled descriptors for this interface
+message Schemas {
 
-  // file name -> gzip compressed protobuf of descriptor
-  map<string, bytes> descriptors = 2;
+    // Proto files
+    repeated ProtoFile protos = 1;
 
 }
 
@@ -19,7 +23,7 @@
 // Schema services
 service SchemaService {
 
-  // Return active grpc schemas
-  rpc GetSchema(NullMessage) returns (Schema) {}
+    // Return active grpc schemas
+    rpc GetSchema(NullMessage) returns (Schemas) {}
 
 }
diff --git a/protos/schema_pb2.py b/protos/schema_pb2.py
index eab43f5..f5f17d2 100644
--- a/protos/schema_pb2.py
+++ b/protos/schema_pb2.py
@@ -19,67 +19,37 @@
   name='schema.proto',
   package='schema',
   syntax='proto3',
-  serialized_pb=_b('\n\x0cschema.proto\x12\x06schema\"\xcd\x01\n\x06Schema\x12*\n\x06protos\x18\x01 \x03(\x0b\x32\x1a.schema.Schema.ProtosEntry\x12\x34\n\x0b\x64\x65scriptors\x18\x02 \x03(\x0b\x32\x1f.schema.Schema.DescriptorsEntry\x1a-\n\x0bProtosEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x32\n\x10\x44\x65scriptorsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x0c:\x02\x38\x01\"\r\n\x0bNullMessage2C\n\rSchemaService\x12\x32\n\tGetSchema\x12\x13.schema.NullMessage\x1a\x0e.schema.Schema\"\x00\x62\x06proto3')
+  serialized_pb=_b('\n\x0cschema.proto\x12\x06schema\"A\n\tProtoFile\x12\x11\n\tfile_name\x18\x01 \x01(\t\x12\r\n\x05proto\x18\x02 \x01(\t\x12\x12\n\ndescriptor\x18\x03 \x01(\x0c\",\n\x07Schemas\x12!\n\x06protos\x18\x01 \x03(\x0b\x32\x11.schema.ProtoFile\"\r\n\x0bNullMessage2D\n\rSchemaService\x12\x33\n\tGetSchema\x12\x13.schema.NullMessage\x1a\x0f.schema.Schemas\"\x00\x62\x06proto3')
 )
 _sym_db.RegisterFileDescriptor(DESCRIPTOR)
 
 
 
 
-_SCHEMA_PROTOSENTRY = _descriptor.Descriptor(
-  name='ProtosEntry',
-  full_name='schema.Schema.ProtosEntry',
+_PROTOFILE = _descriptor.Descriptor(
+  name='ProtoFile',
+  full_name='schema.ProtoFile',
   filename=None,
   file=DESCRIPTOR,
   containing_type=None,
   fields=[
     _descriptor.FieldDescriptor(
-      name='key', full_name='schema.Schema.ProtosEntry.key', index=0,
+      name='file_name', full_name='schema.ProtoFile.file_name', index=0,
       number=1, type=9, cpp_type=9, label=1,
       has_default_value=False, default_value=_b("").decode('utf-8'),
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       options=None),
     _descriptor.FieldDescriptor(
-      name='value', full_name='schema.Schema.ProtosEntry.value', index=1,
+      name='proto', full_name='schema.ProtoFile.proto', index=1,
       number=2, type=9, cpp_type=9, label=1,
       has_default_value=False, default_value=_b("").decode('utf-8'),
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=133,
-  serialized_end=178,
-)
-
-_SCHEMA_DESCRIPTORSENTRY = _descriptor.Descriptor(
-  name='DescriptorsEntry',
-  full_name='schema.Schema.DescriptorsEntry',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
     _descriptor.FieldDescriptor(
-      name='key', full_name='schema.Schema.DescriptorsEntry.key', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='value', full_name='schema.Schema.DescriptorsEntry.value', index=1,
-      number=2, type=12, cpp_type=9, label=1,
+      name='descriptor', full_name='schema.ProtoFile.descriptor', index=2,
+      number=3, type=12, cpp_type=9, label=1,
       has_default_value=False, default_value=_b(""),
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
@@ -90,41 +60,35 @@
   nested_types=[],
   enum_types=[
   ],
-  options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
+  options=None,
   is_extendable=False,
   syntax='proto3',
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=180,
-  serialized_end=230,
+  serialized_start=24,
+  serialized_end=89,
 )
 
-_SCHEMA = _descriptor.Descriptor(
-  name='Schema',
-  full_name='schema.Schema',
+
+_SCHEMAS = _descriptor.Descriptor(
+  name='Schemas',
+  full_name='schema.Schemas',
   filename=None,
   file=DESCRIPTOR,
   containing_type=None,
   fields=[
     _descriptor.FieldDescriptor(
-      name='protos', full_name='schema.Schema.protos', index=0,
+      name='protos', full_name='schema.Schemas.protos', index=0,
       number=1, type=11, cpp_type=10, label=3,
       has_default_value=False, default_value=[],
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       options=None),
-    _descriptor.FieldDescriptor(
-      name='descriptors', full_name='schema.Schema.descriptors', index=1,
-      number=2, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
   ],
   extensions=[
   ],
-  nested_types=[_SCHEMA_PROTOSENTRY, _SCHEMA_DESCRIPTORSENTRY, ],
+  nested_types=[],
   enum_types=[
   ],
   options=None,
@@ -133,8 +97,8 @@
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=25,
-  serialized_end=230,
+  serialized_start=91,
+  serialized_end=135,
 )
 
 
@@ -157,39 +121,28 @@
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=232,
-  serialized_end=245,
+  serialized_start=137,
+  serialized_end=150,
 )
 
-_SCHEMA_PROTOSENTRY.containing_type = _SCHEMA
-_SCHEMA_DESCRIPTORSENTRY.containing_type = _SCHEMA
-_SCHEMA.fields_by_name['protos'].message_type = _SCHEMA_PROTOSENTRY
-_SCHEMA.fields_by_name['descriptors'].message_type = _SCHEMA_DESCRIPTORSENTRY
-DESCRIPTOR.message_types_by_name['Schema'] = _SCHEMA
+_SCHEMAS.fields_by_name['protos'].message_type = _PROTOFILE
+DESCRIPTOR.message_types_by_name['ProtoFile'] = _PROTOFILE
+DESCRIPTOR.message_types_by_name['Schemas'] = _SCHEMAS
 DESCRIPTOR.message_types_by_name['NullMessage'] = _NULLMESSAGE
 
-Schema = _reflection.GeneratedProtocolMessageType('Schema', (_message.Message,), dict(
-
-  ProtosEntry = _reflection.GeneratedProtocolMessageType('ProtosEntry', (_message.Message,), dict(
-    DESCRIPTOR = _SCHEMA_PROTOSENTRY,
-    __module__ = 'schema_pb2'
-    # @@protoc_insertion_point(class_scope:schema.Schema.ProtosEntry)
-    ))
-  ,
-
-  DescriptorsEntry = _reflection.GeneratedProtocolMessageType('DescriptorsEntry', (_message.Message,), dict(
-    DESCRIPTOR = _SCHEMA_DESCRIPTORSENTRY,
-    __module__ = 'schema_pb2'
-    # @@protoc_insertion_point(class_scope:schema.Schema.DescriptorsEntry)
-    ))
-  ,
-  DESCRIPTOR = _SCHEMA,
+ProtoFile = _reflection.GeneratedProtocolMessageType('ProtoFile', (_message.Message,), dict(
+  DESCRIPTOR = _PROTOFILE,
   __module__ = 'schema_pb2'
-  # @@protoc_insertion_point(class_scope:schema.Schema)
+  # @@protoc_insertion_point(class_scope:schema.ProtoFile)
   ))
-_sym_db.RegisterMessage(Schema)
-_sym_db.RegisterMessage(Schema.ProtosEntry)
-_sym_db.RegisterMessage(Schema.DescriptorsEntry)
+_sym_db.RegisterMessage(ProtoFile)
+
+Schemas = _reflection.GeneratedProtocolMessageType('Schemas', (_message.Message,), dict(
+  DESCRIPTOR = _SCHEMAS,
+  __module__ = 'schema_pb2'
+  # @@protoc_insertion_point(class_scope:schema.Schemas)
+  ))
+_sym_db.RegisterMessage(Schemas)
 
 NullMessage = _reflection.GeneratedProtocolMessageType('NullMessage', (_message.Message,), dict(
   DESCRIPTOR = _NULLMESSAGE,
@@ -199,10 +152,6 @@
 _sym_db.RegisterMessage(NullMessage)
 
 
-_SCHEMA_PROTOSENTRY.has_options = True
-_SCHEMA_PROTOSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
-_SCHEMA_DESCRIPTORSENTRY.has_options = True
-_SCHEMA_DESCRIPTORSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
 import grpc
 from grpc.beta import implementations as beta_implementations
 from grpc.beta import interfaces as beta_interfaces
@@ -223,7 +172,7 @@
     self.GetSchema = channel.unary_unary(
         '/schema.SchemaService/GetSchema',
         request_serializer=NullMessage.SerializeToString,
-        response_deserializer=Schema.FromString,
+        response_deserializer=Schemas.FromString,
         )
 
 
@@ -244,7 +193,7 @@
       'GetSchema': grpc.unary_unary_rpc_method_handler(
           servicer.GetSchema,
           request_deserializer=NullMessage.FromString,
-          response_serializer=Schema.SerializeToString,
+          response_serializer=Schemas.SerializeToString,
       ),
   }
   generic_handler = grpc.method_handlers_generic_handler(
@@ -276,7 +225,7 @@
     ('schema.SchemaService', 'GetSchema'): NullMessage.FromString,
   }
   response_serializers = {
-    ('schema.SchemaService', 'GetSchema'): Schema.SerializeToString,
+    ('schema.SchemaService', 'GetSchema'): Schemas.SerializeToString,
   }
   method_implementations = {
     ('schema.SchemaService', 'GetSchema'): face_utilities.unary_unary_inline(servicer.GetSchema),
@@ -290,7 +239,7 @@
     ('schema.SchemaService', 'GetSchema'): NullMessage.SerializeToString,
   }
   response_deserializers = {
-    ('schema.SchemaService', 'GetSchema'): Schema.FromString,
+    ('schema.SchemaService', 'GetSchema'): Schemas.FromString,
   }
   cardinalities = {
     'GetSchema': cardinality.Cardinality.UNARY_UNARY,
diff --git a/web_server/web_server.py b/web_server/web_server.py
index c46ad88..c96be0f 100644
--- a/web_server/web_server.py
+++ b/web_server/web_server.py
@@ -47,11 +47,22 @@
         self.shutting_down = False
 
     @inlineCallbacks
-    def run(self):
+    def start(self):
+        log.debug('starting')
         yield self._open_endpoint()
+        log.info('started')
         returnValue(self)
 
     @inlineCallbacks
+    def stop(self):
+        log.debug('stopping')
+        self.shutting_down = True
+        if self.tcp_port is not None:
+            assert isinstance(self.tcp_port, Port)
+            yield self.tcp_port.socket.close()
+        log.info('stopped')
+
+    @inlineCallbacks
     def _open_endpoint(self):
         endpoint = endpoints.TCP4ServerEndpoint(reactor, self.port)
         self.site = Site(self.app.resource())
@@ -59,13 +70,6 @@
         log.info('web-server-started', port=self.port)
         self.endpoint = endpoint
 
-    @inlineCallbacks
-    def shutdown(self):
-        self.shutting_down = True
-        if self.tcp_port is not None:
-            assert isinstance(self.tcp_port, Port)
-            yield self.tcp_port.socket.close()
-
     def reload_generated_routes(self):
         for fname in os.listdir(self.work_dir):
             if fname.endswith('_gw.py'):