blob: c6841b687e6dbc47bef9157b02d5e29c4b6083fe [file] [log] [blame]
Zsolt Haraszti162de172016-09-25 14:49:30 -07001#!/usr/bin/env python
2#
Zsolt Haraszti3eb27a52017-01-03 21:56:48 -08003# Copyright 2017 the original author or authors.
Zsolt Haraszti162de172016-09-25 14:49:30 -07004#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17
Zsolt Harasztibae12752016-10-10 09:55:30 -070018"""
19Load a protobuf description file or protoc CodeGeneratorRequest an make
20sense of it
21"""
Zsolt Haraszti162de172016-09-25 14:49:30 -070022
Zsolt Haraszti162de172016-09-25 14:49:30 -070023import os
24import inspect
25from collections import OrderedDict
26
Zsolt Haraszti034db372016-10-03 22:26:41 -070027import sys
28
29from google.protobuf.compiler.plugin_pb2 import CodeGeneratorRequest
Zsolt Harasztid9f400f2016-09-25 17:19:49 -070030from google.protobuf.descriptor import FieldDescriptor, Descriptor
Zsolt Haraszti034db372016-10-03 22:26:41 -070031from google.protobuf.descriptor_pb2 import FileDescriptorProto, MethodOptions
32from google.protobuf.message import Message, DecodeError
Zsolt Haraszti162de172016-09-25 14:49:30 -070033from simplejson import dumps
34
35from google.protobuf import descriptor_pb2
36
Zsolt Haraszti162de172016-09-25 14:49:30 -070037
Zsolt Harasztid9f400f2016-09-25 17:19:49 -070038class InvalidDescriptorError(Exception): pass
Zsolt Haraszti162de172016-09-25 14:49:30 -070039
40
41class DescriptorParser(object):
42
43 def __init__(self, ignore_empty_source_code_info=True):
44 self.ignore_empty_source_code_info = ignore_empty_source_code_info
45 self.catalog = {}
Zsolt Harasztid9f400f2016-09-25 17:19:49 -070046 self.meta, blob = self.load_root_descriptor()
47 self.load_descriptor(blob)
48
49 def load_root_descriptor(self):
50 """Load descriptor.desc to make things more data driven"""
51 with open('descriptor.desc', 'r') as f:
52 blob = f.read()
53 proto = descriptor_pb2.FileDescriptorSet()
54 proto.ParseFromString(blob)
55 assert len(proto.file) == 1
56 fdp = proto.file[0]
57
58 # for i, (fd, v) in enumerate(fdp.ListFields()):
59 # assert isinstance(fd, FieldDescriptor)
60 # print fd.name, fd.full_name, fd.number, fd.type, fd.label, fd.message_type, type(v)
61
62 return fdp, blob
Zsolt Haraszti162de172016-09-25 14:49:30 -070063
64 def get_catalog(self):
65 return self.catalog
66
Zsolt Haraszti0650d1a2016-09-26 17:29:25 -070067 def load_descriptor(self, descriptor_blob,
68 fold_comments=True,
69 type_tag_name='_type'):
Zsolt Haraszti162de172016-09-25 14:49:30 -070070
Zsolt Haraszti034db372016-10-03 22:26:41 -070071 # decode file descriptor set or if that is not possible,
72 # try plugin request
73 try:
74 message = descriptor_pb2.FileDescriptorSet()
75 message.ParseFromString(descriptor_blob)
76 except DecodeError:
77 message = CodeGeneratorRequest()
78 message.ParseFromString(descriptor_blob)
Zsolt Haraszti162de172016-09-25 14:49:30 -070079
Zsolt Haraszti034db372016-10-03 22:26:41 -070080 d = self.parse(message, type_tag_name=type_tag_name)
81 print d.keys()
82 for _file in d.get('file', None) or d['proto_file']:
Zsolt Harasztid9f400f2016-09-25 17:19:49 -070083 if fold_comments:
84 self.fold_comments_in(_file)
85 self.catalog[_file['package']] = _file
Zsolt Haraszti162de172016-09-25 14:49:30 -070086
Zsolt Haraszti0650d1a2016-09-26 17:29:25 -070087 def parse_message(self, m, type_tag_name=None):
Zsolt Harasztid9f400f2016-09-25 17:19:49 -070088 assert isinstance(m, Message)
Zsolt Haraszti162de172016-09-25 14:49:30 -070089 d = OrderedDict()
Zsolt Harasztid9f400f2016-09-25 17:19:49 -070090 for fd, v in m.ListFields():
91 assert isinstance(fd, FieldDescriptor)
92 if fd.label in (1, 2):
Zsolt Haraszti0650d1a2016-09-26 17:29:25 -070093 d[fd.name] = self.parse(v, type_tag_name)
Zsolt Harasztid9f400f2016-09-25 17:19:49 -070094 elif fd.label == 3:
Zsolt Haraszti0650d1a2016-09-26 17:29:25 -070095 d[fd.name] = [self.parse(x, type_tag_name) for x in v]
Zsolt Harasztid9f400f2016-09-25 17:19:49 -070096 else:
97 raise InvalidDescriptorError()
98
Zsolt Haraszti0650d1a2016-09-26 17:29:25 -070099 if type_tag_name is not None:
100 d[type_tag_name] = m.DESCRIPTOR.full_name
101
Zsolt Haraszti162de172016-09-25 14:49:30 -0700102 return d
103
Zsolt Harasztid9f400f2016-09-25 17:19:49 -0700104 parser_table = {
105 unicode: lambda x: x,
106 int: lambda x: x,
107 bool: lambda x: x,
108 }
Zsolt Haraszti162de172016-09-25 14:49:30 -0700109
Zsolt Haraszti0650d1a2016-09-26 17:29:25 -0700110 def parse(self, o, type_tag_name=None):
Zsolt Harasztid9f400f2016-09-25 17:19:49 -0700111 if isinstance(o, Message):
Zsolt Haraszti0650d1a2016-09-26 17:29:25 -0700112 return self.parse_message(o, type_tag_name)
Zsolt Haraszti162de172016-09-25 14:49:30 -0700113 else:
Zsolt Harasztid9f400f2016-09-25 17:19:49 -0700114 return self.parser_table[type(o)](o)
Zsolt Haraszti162de172016-09-25 14:49:30 -0700115
116 def fold_comments_in(self, descriptor):
117 assert isinstance(descriptor, dict)
118
119 locations = descriptor.get('source_code_info', {}).get('location', [])
120 for location in locations:
Zsolt Harasztid9f400f2016-09-25 17:19:49 -0700121 path = location.get('path', [])
Zsolt Haraszti162de172016-09-25 14:49:30 -0700122 comments = ''.join([
Zsolt Harasztid9f400f2016-09-25 17:19:49 -0700123 location.get('leading_comments', '').strip(' '),
124 location.get('trailing_comments', '').strip(' '),
Zsolt Haraszti162de172016-09-25 14:49:30 -0700125 ''.join(block.strip(' ') for block
Zsolt Harasztid9f400f2016-09-25 17:19:49 -0700126 in location.get('leading_detached_comments', ''))
Zsolt Haraszti162de172016-09-25 14:49:30 -0700127 ]).strip()
Zsolt Haraszti162de172016-09-25 14:49:30 -0700128
Zsolt Harasztid9f400f2016-09-25 17:19:49 -0700129 # ignore locations with no comments
130 if not comments:
131 continue
Zsolt Haraszti162de172016-09-25 14:49:30 -0700132
Zsolt Harasztid9f400f2016-09-25 17:19:49 -0700133 # we ignore path with odd number of entries, since these do
134 # not address our schema nodes, but rather the meta schema
135 if (len(path) % 2 == 0):
136 node = self.find_node_by_path(
137 path, self.meta.DESCRIPTOR, descriptor)
138 assert isinstance(node, dict)
Zsolt Haraszti162de172016-09-25 14:49:30 -0700139 node['_description'] = comments
140
141 # remove source_code_info
142 del descriptor['source_code_info']
143
Zsolt Harasztid9f400f2016-09-25 17:19:49 -0700144 def find_node_by_path(self, path, meta, o):
Zsolt Haraszti162de172016-09-25 14:49:30 -0700145
Zsolt Harasztid9f400f2016-09-25 17:19:49 -0700146 # stop recursion when path is empty
Zsolt Haraszti162de172016-09-25 14:49:30 -0700147 if not path:
Zsolt Harasztid9f400f2016-09-25 17:19:49 -0700148 return o
Zsolt Haraszti162de172016-09-25 14:49:30 -0700149
Zsolt Harasztid9f400f2016-09-25 17:19:49 -0700150 # sanity check
151 assert len(path) >= 2
152 assert isinstance(meta, Descriptor)
153 assert isinstance(o, dict)
Zsolt Haraszti162de172016-09-25 14:49:30 -0700154
Zsolt Harasztid9f400f2016-09-25 17:19:49 -0700155 # find field name, then actual field
156 field_number = path.pop(0)
157 field_def = meta.fields_by_number[field_number]
158 field = o[field_def.name]
Zsolt Haraszti162de172016-09-25 14:49:30 -0700159
Zsolt Harasztid9f400f2016-09-25 17:19:49 -0700160 # field must be a list, extract entry with given index
161 assert isinstance(field, list) # expected to be a list field
Zsolt Haraszti162de172016-09-25 14:49:30 -0700162 index = path.pop(0)
Zsolt Harasztid9f400f2016-09-25 17:19:49 -0700163 child_o = field[index]
Zsolt Haraszti162de172016-09-25 14:49:30 -0700164
Zsolt Harasztid9f400f2016-09-25 17:19:49 -0700165 child_meta = field_def.message_type
166 return self.find_node_by_path(path, child_meta, child_o)
Zsolt Haraszti162de172016-09-25 14:49:30 -0700167
168
169if __name__ == '__main__':
Zsolt Haraszti162de172016-09-25 14:49:30 -0700170
Zsolt Haraszti80259ec2016-09-25 22:38:03 -0700171 # try loading voltha descriptor and turn it into JSON data as a preparation
172 # for generating JSON Schema / swagger file (to be done later)
Zsolt Haraszti034db372016-10-03 22:26:41 -0700173 if len(sys.argv) >= 2:
174 desc_file = sys.argv[1]
175 else:
176 desc_dir = os.path.dirname(inspect.getfile(voltha_pb2))
177 desc_file = os.path.join(desc_dir, 'voltha.desc')
178
Zsolt Haraszti5cd64702016-09-27 13:48:35 -0700179 from voltha.protos import voltha_pb2
Zsolt Haraszti162de172016-09-25 14:49:30 -0700180 with open(desc_file, 'rb') as f:
181 descriptor_blob = f.read()
Zsolt Haraszti034db372016-10-03 22:26:41 -0700182
Zsolt Haraszti162de172016-09-25 14:49:30 -0700183 parser = DescriptorParser()
Zsolt Haraszti034db372016-10-03 22:26:41 -0700184 parser.save_file_desc = '/tmp/grpc_introspection.out'
185
Zsolt Haraszti162de172016-09-25 14:49:30 -0700186 parser.load_descriptor(descriptor_blob)
Zsolt Haraszti162de172016-09-25 14:49:30 -0700187 print dumps(parser.get_catalog(), indent=4)
Zsolt Haraszti034db372016-10-03 22:26:41 -0700188 sys.exit(0)
Zsolt Haraszti162de172016-09-25 14:49:30 -0700189
Zsolt Haraszti80259ec2016-09-25 22:38:03 -0700190 # try to see if we can decode binary data into JSON automatically
Zsolt Haraszti0650d1a2016-09-26 17:29:25 -0700191 from random import seed, randint
192 seed(0)
193
194 def make_mc(name, n_children=0):
195 mc = voltha_pb2.MoreComplex(
196 name=name,
197 foo_counter=randint(0, 10000),
198 health=voltha_pb2.HealthStatus(
199 state=voltha_pb2.HealthStatus.OVERLOADED
200 ),
Zsolt Haraszti034db372016-10-03 22:26:41 -0700201 address=voltha_pb2.Address(
Zsolt Haraszti0650d1a2016-09-26 17:29:25 -0700202 street='1383 N McDowell Blvd',
203 city='Petaluma',
204 zip=94954,
205 state='CA'
206 ),
207 children=[make_mc('child%d' % (i + 1)) for i in xrange(n_children)]
208 )
Zsolt Haraszti80259ec2016-09-25 22:38:03 -0700209 return mc
Zsolt Haraszti162de172016-09-25 14:49:30 -0700210
Zsolt Haraszti0650d1a2016-09-26 17:29:25 -0700211 mc = make_mc('root', 3)
Zsolt Haraszti80259ec2016-09-25 22:38:03 -0700212 blob = mc.SerializeToString()
213 print len(blob), 'bytes'
214 mc2 = voltha_pb2.MoreComplex()
215 mc2.ParseFromString(blob)
216 assert mc == mc2
217
Zsolt Haraszti0650d1a2016-09-26 17:29:25 -0700218 print dumps(parser.parse(mc, type_tag_name='_type'), indent=4)