Khen Nursimulu | d768809 | 2016-11-17 00:08:57 -0500 | [diff] [blame^] | 1 | #!/usr/bin/env python |
| 2 | # |
| 3 | # Copyright 2016 the original author or authors. |
| 4 | # |
| 5 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | # you may not use this file except in compliance with the License. |
| 7 | # You may obtain a copy of the License at |
| 8 | # |
| 9 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | # |
| 11 | # Unless required by applicable law or agreed to in writing, software |
| 12 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | # See the License for the specific language governing permissions and |
| 15 | # limitations under the License. |
| 16 | # |
| 17 | |
| 18 | """protoc plugin to convert a protobuf schema to a yang schema |
| 19 | |
| 20 | - basic support for message, fields. enumeration, service, method |
| 21 | |
| 22 | - yang semantic rules needs to be implemented |
| 23 | |
| 24 | - to run this plugin : |
| 25 | |
| 26 | $ python -m grpc.tools.protoc -I. |
| 27 | --plugin=protoc-gen-custom=./proto2yang.py --custom_out=. <proto file>.proto |
| 28 | |
| 29 | - the above will produce a <proto file>.yang file formatted for yang |
| 30 | |
| 31 | - two examples of proto that can be used in the same directory are |
| 32 | yang.proto and addressbook.proto |
| 33 | |
| 34 | """ |
| 35 | |
| 36 | import sys |
| 37 | |
| 38 | from jinja2 import Template |
| 39 | from google.protobuf.compiler import plugin_pb2 as plugin |
| 40 | from descriptor_parser import DescriptorParser |
| 41 | |
| 42 | from google.protobuf.descriptor import FieldDescriptor |
| 43 | |
| 44 | template_yang = Template(""" |
| 45 | module {{ module.name }} { |
| 46 | |
| 47 | namespace "https://gerrit.opencord.org/voltha/{{ module.package }}"; |
| 48 | yang-version 1.1; |
| 49 | |
| 50 | prefix "voltha"; |
| 51 | |
| 52 | revision 2016-11-15 {{ module.revision }} { |
| 53 | {% if module.description %} |
| 54 | /* {{ message.description }} */ |
| 55 | {% else %} |
| 56 | description "Initial revision."; |
| 57 | {% endif %} |
| 58 | } |
| 59 | {% for message in module.messages recursive %} |
| 60 | grouping {{ message.name }} { |
| 61 | {% if message.description %} |
| 62 | /* {{ message.description }} */ |
| 63 | {% endif %} |
| 64 | {% if message.key %} |
| 65 | key {{ message.key_name }} ; |
| 66 | {% endif %} |
| 67 | {% for field in message.fields %} |
| 68 | {% if field.type_ref %} |
| 69 | {% if field.description %} |
| 70 | /* {{ field.description }} */ |
| 71 | {% endif %} |
| 72 | uses {{ field.type }} ; |
| 73 | {% else %} |
| 74 | leaf {{ field.name }} { |
| 75 | {% if field.type == "decimal64" %} |
| 76 | type {{ field.type }} { |
| 77 | fraction-digits 5; |
| 78 | } |
| 79 | {% else %} |
| 80 | type {{ field.type }} ; |
| 81 | {% endif %} |
| 82 | {% if field.description %} |
| 83 | description |
| 84 | "{{ field.description }}" ; |
| 85 | {% endif %} |
| 86 | } |
| 87 | {% endif %} |
| 88 | {% endfor %} |
| 89 | {% for enum_type in message.enums %} |
| 90 | {% if enum_type.description %} |
| 91 | /* {{ enum_type.description }} */ ; |
| 92 | {% endif %} |
| 93 | typedef {{ enum_type.name }} { |
| 94 | type enumeration { |
| 95 | {% for v in enum_type.value %} |
| 96 | {% if v.description %} |
| 97 | enum {{ v.name }} { |
| 98 | description "{{ v.description }}"; |
| 99 | } |
| 100 | {% else %} |
| 101 | enum {{ v.name }} ; |
| 102 | {% endif %} |
| 103 | {% endfor %} |
| 104 | } |
| 105 | } |
| 106 | {% endfor %} |
| 107 | {% for oneof in message.oneof %} |
| 108 | choice {{ oneof.name }} { |
| 109 | } |
| 110 | {% endfor %} |
| 111 | {% if message.messages %} |
| 112 | {{ loop (message.messages)|indent(4, false) }} |
| 113 | {% endif %} |
| 114 | } |
| 115 | {% endfor %} |
| 116 | |
| 117 | {% for service in module.services %} |
| 118 | {% if service.description %} |
| 119 | /* {{ service.description }}" */ |
| 120 | {% endif %} |
| 121 | |
| 122 | {% for method in service.methods %} |
| 123 | {% if method.description %} |
| 124 | /* {{ method.description }} */ |
| 125 | {% endif %} |
| 126 | rpc {{ service.service }}-{{ method.method }} { |
| 127 | {% if method.input %} |
| 128 | input { |
| 129 | {% if method.input_ref %} |
| 130 | uses {{ method.input }} ; |
| 131 | {% else %} |
| 132 | leaf {{ method.input }} { |
| 133 | type {{ method.input }} ; |
| 134 | } |
| 135 | {% endif %} |
| 136 | } |
| 137 | {% endif %} |
| 138 | {% if method.output %} |
| 139 | output { |
| 140 | {% if method.output_ref %} |
| 141 | uses {{ method.output }} ; |
| 142 | {% else %} |
| 143 | leaf {{ method.output }} { |
| 144 | type {{ method.output }} ; |
| 145 | } |
| 146 | {% endif %} |
| 147 | } |
| 148 | {% endif %} |
| 149 | } |
| 150 | {% endfor %} |
| 151 | {% endfor %} |
| 152 | } |
| 153 | """, trim_blocks=True, lstrip_blocks=True) |
| 154 | |
| 155 | |
| 156 | def _traverse_messages(message_types): |
| 157 | messages = [] |
| 158 | for message_type in message_types: |
| 159 | assert message_type['_type'] == 'google.protobuf.DescriptorProto' |
| 160 | # parse the fields |
| 161 | fields = _traverse_fields(message_type.get('field', [])) |
| 162 | |
| 163 | # parse the enums |
| 164 | enums = _traverse_enums(message_type.get('enum_type', [])) |
| 165 | |
| 166 | # parse nested messages |
| 167 | nested = message_type.get('nested_type', []) |
| 168 | nested_messages = _traverse_messages(nested) |
| 169 | messages.append( |
| 170 | { |
| 171 | 'name': message_type.get('name', ''), |
| 172 | 'fields': fields, |
| 173 | 'enums': enums, |
| 174 | # 'extensions': extensions, |
| 175 | 'messages': nested_messages, |
| 176 | 'description': message_type.get('_description', ''), |
| 177 | # 'extension_ranges': extension_ranges, |
| 178 | # 'oneof': oneof |
| 179 | } |
| 180 | ) |
| 181 | return messages |
| 182 | |
| 183 | |
| 184 | def _traverse_fields(fields_desc): |
| 185 | fields = [] |
| 186 | for field in fields_desc: |
| 187 | assert field['_type'] == 'google.protobuf.FieldDescriptorProto' |
| 188 | fields.append( |
| 189 | { |
| 190 | 'name': field.get('name', ''), |
| 191 | 'label': field.get('label', ''), |
| 192 | 'number': field.get('number', ''), |
| 193 | 'options': field.get('options', ''), |
| 194 | 'type_name': field.get('type_name', ''), |
| 195 | 'type': get_yang_type(field), |
| 196 | 'type_ref': not is_base_type(field['type']), |
| 197 | 'description': remove_unsupported_characters(field.get( |
| 198 | '_description', '')) |
| 199 | } |
| 200 | ) |
| 201 | return fields |
| 202 | |
| 203 | |
| 204 | def _traverse_enums(enums_desc): |
| 205 | enums = [] |
| 206 | for enum in enums_desc: |
| 207 | assert enum['_type'] == 'google.protobuf.EnumDescriptorProto' |
| 208 | enums.append( |
| 209 | { |
| 210 | 'name': enum.get('name', ''), |
| 211 | 'value': enum.get('value', ''), |
| 212 | 'description': enum.get('_description', '') |
| 213 | } |
| 214 | ) |
| 215 | return enums |
| 216 | |
| 217 | |
| 218 | def _traverse_services(service_desc): |
| 219 | services = [] |
| 220 | for service in service_desc: |
| 221 | methods = [] |
| 222 | for method in service.get('method', []): |
| 223 | assert method['_type'] == 'google.protobuf.MethodDescriptorProto' |
| 224 | input_name = method.get('input_type') |
| 225 | input_ref = False |
| 226 | if not is_base_type(input_name): |
| 227 | input_name = input_name.split('.')[-1] |
| 228 | input_ref = True |
| 229 | output_name = method.get('output_type') |
| 230 | output_ref = False |
| 231 | if not is_base_type(output_name): |
| 232 | output_name = output_name.split('.')[-1] |
| 233 | output_ref = True |
| 234 | methods.append( |
| 235 | { |
| 236 | 'method': method.get('name', ''), |
| 237 | 'input': input_name, |
| 238 | 'input_ref': input_ref, |
| 239 | 'output': output_name, |
| 240 | 'output_ref': output_ref, |
| 241 | 'description': method.get('_description', ''), |
| 242 | 'server_streaming': method.get('server_streaming', |
| 243 | False) == True |
| 244 | } |
| 245 | ) |
| 246 | services.append( |
| 247 | { |
| 248 | 'service': service.get('name', ''), |
| 249 | 'methods': methods, |
| 250 | 'description': service.get('_description', ''), |
| 251 | } |
| 252 | ) |
| 253 | return services |
| 254 | |
| 255 | |
| 256 | def _rchop(thestring, ending): |
| 257 | if thestring.endswith(ending): |
| 258 | return thestring[:-len(ending)] |
| 259 | return thestring |
| 260 | |
| 261 | |
| 262 | def _traverse_desc(descriptor): |
| 263 | name = _rchop(descriptor.get('name', ''), '.proto') |
| 264 | package = descriptor.get('package', '') |
| 265 | description = descriptor.get('_description', '') |
| 266 | messages = _traverse_messages(descriptor.get('message_type', [])) |
| 267 | enums = _traverse_enums(descriptor.get('enum_type', [])) |
| 268 | services = _traverse_services(descriptor.get('service', [])) |
| 269 | # extensions = _traverse_extensions(descriptors) |
| 270 | # options = _traverse_options(descriptors) |
| 271 | |
| 272 | data = { |
| 273 | 'name': name, |
| 274 | 'package': package, |
| 275 | 'description' : description, |
| 276 | 'messages': messages, |
| 277 | 'enums': enums, |
| 278 | 'services': services, |
| 279 | # 'extensions': extensions, |
| 280 | # 'options': options |
| 281 | } |
| 282 | |
| 283 | return data |
| 284 | |
| 285 | |
| 286 | def generate_code(request, response): |
| 287 | assert isinstance(request, plugin.CodeGeneratorRequest) |
| 288 | |
| 289 | parser = DescriptorParser() |
| 290 | |
| 291 | # idx = 1 |
| 292 | for proto_file in request.proto_file: |
| 293 | native_data = parser.parse_file_descriptor(proto_file, |
| 294 | type_tag_name='_type', |
| 295 | fold_comments=True) |
| 296 | |
| 297 | # print native_data |
| 298 | yang_data = _traverse_desc(native_data) |
| 299 | |
| 300 | f = response.file.add() |
| 301 | #TODO: We should have a separate file for each output. There is an |
| 302 | # issue reusing the same filename with an incremental suffix. Using |
| 303 | # a different file name works but not the actual proto file name |
| 304 | f.name = proto_file.name.replace('.proto', '.yang') |
| 305 | # f.name = '{}_{}{}'.format(_rchop(proto_file.name, '.proto'), idx, |
| 306 | # '.yang') |
| 307 | # idx += 1 |
| 308 | f.content = template_yang.render(module=yang_data) |
| 309 | |
| 310 | def get_yang_type(field): |
| 311 | type = field['type'] |
| 312 | if type in YANG_TYPE_MAP.keys(): |
| 313 | _type, _ = YANG_TYPE_MAP[type] |
| 314 | if _type in ['enumeration', 'message', 'group']: |
| 315 | return field['type_name'].split('.')[-1] |
| 316 | else: |
| 317 | return _type |
| 318 | else: |
| 319 | return type |
| 320 | |
| 321 | def is_base_type(type): |
| 322 | # check numeric value of the type first |
| 323 | if type in YANG_TYPE_MAP.keys(): |
| 324 | _type, _ = YANG_TYPE_MAP[type] |
| 325 | return _type not in ['message', 'group'] |
| 326 | else: |
| 327 | # proto name of the type |
| 328 | result = [ _format for ( _ , _format) in YANG_TYPE_MAP.values() if |
| 329 | _format == type and _format not in ['message', 'group']] |
| 330 | return len(result) > 0 |
| 331 | |
| 332 | def remove_unsupported_characters(text): |
| 333 | unsupported_characters = ["{", "}", "[", "]", "\"", "/", "\\"] |
| 334 | return ''.join([i if i not in unsupported_characters else ' ' for i in text]) |
| 335 | |
| 336 | YANG_TYPE_MAP = { |
| 337 | FieldDescriptor.TYPE_BOOL: ('boolean', 'boolean'), |
| 338 | FieldDescriptor.TYPE_BYTES: ('binary', 'byte'), |
| 339 | FieldDescriptor.TYPE_DOUBLE: ('decimal64', 'double'), |
| 340 | FieldDescriptor.TYPE_ENUM: ('enumeration', 'enum'), |
| 341 | FieldDescriptor.TYPE_FIXED32: ('int32', 'int64'), |
| 342 | FieldDescriptor.TYPE_FIXED64: ('int64', 'uint64'), |
| 343 | FieldDescriptor.TYPE_FLOAT: ('decimal64', 'float'), |
| 344 | FieldDescriptor.TYPE_INT32: ('int32', 'int32'), |
| 345 | FieldDescriptor.TYPE_INT64: ('int64', 'int64'), |
| 346 | FieldDescriptor.TYPE_SFIXED32: ('int32', 'int32'), |
| 347 | FieldDescriptor.TYPE_SFIXED64: ('int64', 'int64'), |
| 348 | FieldDescriptor.TYPE_STRING: ('string', 'string'), |
| 349 | FieldDescriptor.TYPE_SINT32: ('int32', 'int32'), |
| 350 | FieldDescriptor.TYPE_SINT64: ('int64', 'int64'), |
| 351 | FieldDescriptor.TYPE_UINT32: ('uint32', 'int64'), |
| 352 | FieldDescriptor.TYPE_UINT64: ('uint64', 'uint64'), |
| 353 | FieldDescriptor.TYPE_MESSAGE: ('message', 'message'), |
| 354 | FieldDescriptor.TYPE_GROUP: ('group', 'group') |
| 355 | } |
| 356 | |
| 357 | if __name__ == '__main__': |
| 358 | # Read request message from stdin |
| 359 | data = sys.stdin.read() |
| 360 | |
| 361 | # Parse request |
| 362 | request = plugin.CodeGeneratorRequest() |
| 363 | request.ParseFromString(data) |
| 364 | |
| 365 | # Create response |
| 366 | response = plugin.CodeGeneratorResponse() |
| 367 | |
| 368 | # Generate code |
| 369 | generate_code(request, response) |
| 370 | |
| 371 | # Serialise response message |
| 372 | output = response.SerializeToString() |
| 373 | |
| 374 | # Write to stdout |
| 375 | sys.stdout.write(output) |
| 376 | # print is_base_type(9) |