Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 1 | __author__ = "Dusan (Ph4r05) Klinec" |
Sapan Bhatia | b122587 | 2017-03-29 20:47:47 +0200 | [diff] [blame] | 2 | |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 3 | __copyright__ = "Copyright (C) 2014 Dusan (ph4r05) Klinec" |
| 4 | __license__ = "Apache License, Version 2.0" |
| 5 | __version__ = "1.0" |
| 6 | |
| 7 | import ply.lex as lex |
| 8 | import ply.yacc as yacc |
| 9 | from .model import * |
| 10 | |
Sapan Bhatia | b122587 | 2017-03-29 20:47:47 +0200 | [diff] [blame] | 11 | import pdb |
Sapan Bhatia | 64c7251 | 2017-06-23 02:32:45 -0700 | [diff] [blame] | 12 | from helpers import LexHelper, LU |
| 13 | from logicparser import FOLParser, FOLLexer |
Sapan Bhatia | b122587 | 2017-03-29 20:47:47 +0200 | [diff] [blame] | 14 | |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 15 | class ProtobufLexer(object): |
| 16 | keywords = ('double', 'float', 'int32', 'int64', 'uint32', 'uint64', 'sint32', 'sint64', |
| 17 | 'fixed32', 'fixed64', 'sfixed32', 'sfixed64', 'bool', 'string', 'bytes', |
Sapan Bhatia | 78fee77 | 2017-04-21 19:00:48 +0200 | [diff] [blame] | 18 | 'message', 'required', 'optional', 'repeated', 'enum', 'extensions', 'max', 'extend', |
Sapan Bhatia | 64c7251 | 2017-06-23 02:32:45 -0700 | [diff] [blame] | 19 | 'to', 'package', '_service', 'rpc', 'returns', 'true', 'false', 'option', 'import', 'manytoone', 'manytomany', 'onetoone', 'policy') |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 20 | |
| 21 | tokens = [ |
Sapan Bhatia | 64c7251 | 2017-06-23 02:32:45 -0700 | [diff] [blame] | 22 | 'POLICYBODY', |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 23 | 'NAME', |
| 24 | 'NUM', |
| 25 | 'STRING_LITERAL', |
Sapan Bhatia | 78fee77 | 2017-04-21 19:00:48 +0200 | [diff] [blame] | 26 | #'LINE_COMMENT', 'BLOCK_COMMENT', |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 27 | 'LBRACE', 'RBRACE', 'LBRACK', 'RBRACK', |
Dusan Klinec | c9b031a | 2014-11-10 13:21:08 +0100 | [diff] [blame] | 28 | 'LPAR', 'RPAR', 'EQ', 'SEMI', 'DOT', |
Sapan Bhatia | 78fee77 | 2017-04-21 19:00:48 +0200 | [diff] [blame] | 29 | 'ARROW', 'COLON', 'COMMA', 'SLASH', |
Sapan Bhatia | 64c7251 | 2017-06-23 02:32:45 -0700 | [diff] [blame] | 30 | 'DOUBLECOLON', |
Dusan Klinec | aa9ff47 | 2014-11-10 18:02:03 +0100 | [diff] [blame] | 31 | 'STARTTOKEN' |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 32 | ] + [k.upper() for k in keywords] |
Sapan Bhatia | 64c7251 | 2017-06-23 02:32:45 -0700 | [diff] [blame] | 33 | |
| 34 | |
| 35 | t_POLICYBODY = r'< (.|\n)*? >' |
| 36 | |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 37 | literals = '()+-*/=?:,.^|&~!=[]{};<>@%' |
| 38 | |
Sapan Bhatia | 87792a1 | 2017-04-10 19:35:05 -0700 | [diff] [blame] | 39 | t_NUM = r'[+-]?\d+(\.\d+)?' |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 40 | t_STRING_LITERAL = r'\"([^\\\n]|(\\.))*?\"' |
| 41 | |
| 42 | t_ignore_LINE_COMMENT = '//.*' |
| 43 | def t_BLOCK_COMMENT(self, t): |
| 44 | r'/\*(.|\n)*?\*/' |
| 45 | t.lexer.lineno += t.value.count('\n') |
| 46 | |
| 47 | t_LBRACE = '{' |
| 48 | t_RBRACE = '}' |
| 49 | t_LBRACK = '\\[' |
| 50 | t_RBRACK = '\\]' |
Sapan Bhatia | 64c7251 | 2017-06-23 02:32:45 -0700 | [diff] [blame] | 51 | |
| 52 | |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 53 | t_LPAR = '\\(' |
| 54 | t_RPAR = '\\)' |
| 55 | t_EQ = '=' |
| 56 | t_SEMI = ';' |
Sapan Bhatia | b122587 | 2017-03-29 20:47:47 +0200 | [diff] [blame] | 57 | t_ARROW = '\\-\\>' |
| 58 | t_COLON = '\\:' |
Sapan Bhatia | 78fee77 | 2017-04-21 19:00:48 +0200 | [diff] [blame] | 59 | t_SLASH = '\\/' |
Sapan Bhatia | b122587 | 2017-03-29 20:47:47 +0200 | [diff] [blame] | 60 | t_COMMA = '\\,' |
Dusan Klinec | a4fae11 | 2014-11-10 08:50:27 +0100 | [diff] [blame] | 61 | t_DOT = '\\.' |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 62 | t_ignore = ' \t\f' |
Dusan Klinec | aa9ff47 | 2014-11-10 18:02:03 +0100 | [diff] [blame] | 63 | t_STARTTOKEN = '\\+' |
Sapan Bhatia | 64c7251 | 2017-06-23 02:32:45 -0700 | [diff] [blame] | 64 | t_DOUBLECOLON = '\\:\\:' |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 65 | |
| 66 | def t_NAME(self, t): |
Sapan Bhatia | 78fee77 | 2017-04-21 19:00:48 +0200 | [diff] [blame] | 67 | '[A-Za-z_$][A-Za-z0-9_+$]*' |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 68 | if t.value in ProtobufLexer.keywords: |
Dusan Klinec | a4fae11 | 2014-11-10 08:50:27 +0100 | [diff] [blame] | 69 | #print "type: %s val %s t %s" % (t.type, t.value, t) |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 70 | t.type = t.value.upper() |
| 71 | return t |
| 72 | |
| 73 | def t_newline(self, t): |
| 74 | r'\n+' |
| 75 | t.lexer.lineno += len(t.value) |
| 76 | |
| 77 | def t_newline2(self, t): |
| 78 | r'(\r\n)+' |
| 79 | t.lexer.lineno += len(t.value) / 2 |
| 80 | |
| 81 | def t_error(self, t): |
| 82 | print("Illegal character '{}' ({}) in line {}".format(t.value[0], hex(ord(t.value[0])), t.lexer.lineno)) |
| 83 | t.lexer.skip(1) |
| 84 | |
Dusan Klinec | c9b031a | 2014-11-10 13:21:08 +0100 | [diff] [blame] | 85 | |
Sapan Bhatia | b122587 | 2017-03-29 20:47:47 +0200 | [diff] [blame] | 86 | def srcPort(x): |
| 87 | if (x): |
| 88 | return [FieldDirective(Name('port'),x)] |
| 89 | else: |
| 90 | return [] |
| 91 | |
| 92 | |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 93 | class ProtobufParser(object): |
| 94 | tokens = ProtobufLexer.tokens |
Dusan Klinec | aa9ff47 | 2014-11-10 18:02:03 +0100 | [diff] [blame] | 95 | offset = 0 |
| 96 | lh = LexHelper() |
Sapan Bhatia | 64c7251 | 2017-06-23 02:32:45 -0700 | [diff] [blame] | 97 | fol_lexer = lex.lex(module=FOLLexer())#, optimize=1) |
| 98 | fol_parser = yacc.yacc(module=FOLParser(), start='goal') |
Dusan Klinec | aa9ff47 | 2014-11-10 18:02:03 +0100 | [diff] [blame] | 99 | |
| 100 | def setOffset(self, of): |
| 101 | self.offset = of |
| 102 | self.lh.offset = of |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 103 | |
| 104 | def p_empty(self, p): |
| 105 | '''empty :''' |
| 106 | pass |
| 107 | |
| 108 | def p_field_modifier(self,p): |
| 109 | '''field_modifier : REQUIRED |
| 110 | | OPTIONAL |
| 111 | | REPEATED''' |
Dusan Klinec | aa9ff47 | 2014-11-10 18:02:03 +0100 | [diff] [blame] | 112 | p[0] = LU.i(p,1) |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 113 | |
| 114 | def p_primitive_type(self, p): |
| 115 | '''primitive_type : DOUBLE |
| 116 | | FLOAT |
| 117 | | INT32 |
| 118 | | INT64 |
| 119 | | UINT32 |
| 120 | | UINT64 |
| 121 | | SINT32 |
| 122 | | SINT64 |
| 123 | | FIXED32 |
| 124 | | FIXED64 |
| 125 | | SFIXED32 |
| 126 | | SFIXED64 |
| 127 | | BOOL |
| 128 | | STRING |
| 129 | | BYTES''' |
Dusan Klinec | aa9ff47 | 2014-11-10 18:02:03 +0100 | [diff] [blame] | 130 | p[0] = LU.i(p,1) |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 131 | |
Sapan Bhatia | b122587 | 2017-03-29 20:47:47 +0200 | [diff] [blame] | 132 | def p_link_type(self, p): |
| 133 | '''link_type : ONETOONE |
| 134 | | MANYTOONE |
| 135 | | MANYTOMANY''' |
| 136 | p[0] = LU.i(p,1) |
| 137 | |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 138 | def p_field_id(self, p): |
| 139 | '''field_id : NUM''' |
Dusan Klinec | aa9ff47 | 2014-11-10 18:02:03 +0100 | [diff] [blame] | 140 | p[0] = LU.i(p,1) |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 141 | |
| 142 | def p_rvalue(self, p): |
| 143 | '''rvalue : NUM |
| 144 | | TRUE |
| 145 | | FALSE''' |
Dusan Klinec | aa9ff47 | 2014-11-10 18:02:03 +0100 | [diff] [blame] | 146 | p[0] = LU.i(p,1) |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 147 | |
Sapan Bhatia | b122587 | 2017-03-29 20:47:47 +0200 | [diff] [blame] | 148 | def p_rvalue3(self, p): |
| 149 | '''rvalue : STRING_LITERAL''' |
| 150 | p[0] = Name(LU.i(p, 1)) |
| 151 | self.lh.set_parse_object(p[0], p) |
| 152 | p[0].deriveLex() |
| 153 | |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 154 | def p_rvalue2(self, p): |
| 155 | '''rvalue : NAME''' |
Dusan Klinec | aa9ff47 | 2014-11-10 18:02:03 +0100 | [diff] [blame] | 156 | p[0] = Name(LU.i(p, 1)) |
| 157 | self.lh.set_parse_object(p[0], p) |
| 158 | p[0].deriveLex() |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 159 | |
Sapan Bhatia | b122587 | 2017-03-29 20:47:47 +0200 | [diff] [blame] | 160 | def p_field_directives2(self, p): |
| 161 | '''field_directives : empty''' |
| 162 | p[0] = [] |
| 163 | |
| 164 | def p_field_directives(self, p): |
| 165 | '''field_directives : LBRACK field_directive_times RBRACK''' |
| 166 | p[0] = p[2] |
| 167 | #self.lh.set_parse_object(p[0], p) |
| 168 | |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 169 | def p_field_directive(self, p): |
Sapan Bhatia | b122587 | 2017-03-29 20:47:47 +0200 | [diff] [blame] | 170 | '''field_directive : NAME EQ rvalue''' |
| 171 | p[0] = FieldDirective(Name(LU.i(p, 1)), LU.i(p, 3)) |
Dusan Klinec | aa9ff47 | 2014-11-10 18:02:03 +0100 | [diff] [blame] | 172 | self.lh.set_parse_object(p[0], p) |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 173 | |
Sapan Bhatia | 87792a1 | 2017-04-10 19:35:05 -0700 | [diff] [blame] | 174 | |
| 175 | def p_csv_expr(self, p): |
| 176 | '''csv_expr : LPAR csv RPAR''' |
| 177 | p[0] = p[2] |
| 178 | |
| 179 | def p_csv_expr2(self, p): |
| 180 | '''csv_expr : empty''' |
| 181 | p[0] = [] |
| 182 | |
| 183 | def p_csv2(self, p): |
| 184 | '''csv : empty''' |
| 185 | |
| 186 | def p_csv(self, p): |
Sapan Bhatia | 2ddf83a | 2017-06-10 04:31:40 -0700 | [diff] [blame] | 187 | '''csv : dotname |
| 188 | | csv COMMA dotname''' |
Sapan Bhatia | 87792a1 | 2017-04-10 19:35:05 -0700 | [diff] [blame] | 189 | |
| 190 | if len(p) == 2: |
| 191 | p[0] = [LU(p,1)] |
| 192 | else: |
| 193 | p[0] = p[1] + [LU(p,3)] |
| 194 | |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 195 | def p_field_directive_times(self, p): |
| 196 | '''field_directive_times : field_directive_plus''' |
| 197 | p[0] = p[1] |
| 198 | |
| 199 | def p_field_directive_times2(self, p): |
| 200 | '''field_directive_times : empty''' |
| 201 | p[0] = [] |
| 202 | |
| 203 | def p_field_directive_plus(self, p): |
| 204 | '''field_directive_plus : field_directive |
Sapan Bhatia | b122587 | 2017-03-29 20:47:47 +0200 | [diff] [blame] | 205 | | field_directive_plus COMMA field_directive''' |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 206 | if len(p) == 2: |
Dusan Klinec | a9f6d36 | 2014-11-10 21:07:08 +0100 | [diff] [blame] | 207 | p[0] = [LU(p,1)] |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 208 | else: |
Sapan Bhatia | b122587 | 2017-03-29 20:47:47 +0200 | [diff] [blame] | 209 | p[0] = p[1] + [LU(p,3)] |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 210 | |
Dusan Klinec | a4fae11 | 2014-11-10 08:50:27 +0100 | [diff] [blame] | 211 | def p_dotname(self, p): |
| 212 | '''dotname : NAME |
| 213 | | dotname DOT NAME''' |
Dusan Klinec | aa9ff47 | 2014-11-10 18:02:03 +0100 | [diff] [blame] | 214 | if len(p) == 2: |
| 215 | p[0] = [LU(p,1)] |
| 216 | else: |
| 217 | p[0] = p[1] + [LU(p,3)] |
Dusan Klinec | a4fae11 | 2014-11-10 08:50:27 +0100 | [diff] [blame] | 218 | |
| 219 | # Hack for cases when there is a field named 'message' or 'max' |
| 220 | def p_fieldName(self, p): |
Sapan Bhatia | 78fee77 | 2017-04-21 19:00:48 +0200 | [diff] [blame] | 221 | '''field_name : STARTTOKEN |
| 222 | | NAME |
Dusan Klinec | a4fae11 | 2014-11-10 08:50:27 +0100 | [diff] [blame] | 223 | | MESSAGE |
| 224 | | MAX''' |
Dusan Klinec | aa9ff47 | 2014-11-10 18:02:03 +0100 | [diff] [blame] | 225 | p[0] = Name(LU.i(p,1)) |
| 226 | self.lh.set_parse_object(p[0], p) |
| 227 | p[0].deriveLex() |
Dusan Klinec | a4fae11 | 2014-11-10 08:50:27 +0100 | [diff] [blame] | 228 | |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 229 | def p_field_type(self, p): |
| 230 | '''field_type : primitive_type''' |
Dusan Klinec | aa9ff47 | 2014-11-10 18:02:03 +0100 | [diff] [blame] | 231 | p[0] = FieldType(LU.i(p,1)) |
| 232 | self.lh.set_parse_object(p[0], p) |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 233 | |
| 234 | def p_field_type2(self, p): |
Dusan Klinec | a4fae11 | 2014-11-10 08:50:27 +0100 | [diff] [blame] | 235 | '''field_type : dotname''' |
Dusan Klinec | aa9ff47 | 2014-11-10 18:02:03 +0100 | [diff] [blame] | 236 | p[0] = DotName(LU.i(p, 1)) |
| 237 | self.lh.set_parse_object(p[0], p) |
| 238 | p[0].deriveLex() |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 239 | |
Sapan Bhatia | 78fee77 | 2017-04-21 19:00:48 +0200 | [diff] [blame] | 240 | def p_slash_name(self, p): |
Sapan Bhatia | 2ddf83a | 2017-06-10 04:31:40 -0700 | [diff] [blame] | 241 | '''slash_name : SLASH dotname''' |
Sapan Bhatia | 78fee77 | 2017-04-21 19:00:48 +0200 | [diff] [blame] | 242 | p[0] = p[2] |
| 243 | #self.lh.set_parse_object(p[0], p) |
| 244 | |
| 245 | def p_slash_name2(self, p): |
| 246 | '''slash_name : empty''' |
| 247 | p[0] = None |
| 248 | |
Sapan Bhatia | b122587 | 2017-03-29 20:47:47 +0200 | [diff] [blame] | 249 | def p_colon_fieldname(self, p): |
| 250 | '''colon_fieldname : COLON field_name''' |
| 251 | p[0] = p[2] |
| 252 | self.lh.set_parse_object(p[0], p) |
| 253 | |
| 254 | def p_colon_fieldname2(self, p): |
| 255 | '''colon_fieldname : empty''' |
| 256 | p[0] = None |
| 257 | |
| 258 | # TODO: Add directives to link definition |
| 259 | def p_link_definition(self, p): |
Sapan Bhatia | 2ddf83a | 2017-06-10 04:31:40 -0700 | [diff] [blame] | 260 | '''link_definition : field_modifier link_type field_name ARROW dotname slash_name colon_fieldname EQ field_id field_directives SEMI''' |
Sapan Bhatia | b122587 | 2017-03-29 20:47:47 +0200 | [diff] [blame] | 261 | p[0] = LinkSpec( |
Sapan Bhatia | 78fee77 | 2017-04-21 19:00:48 +0200 | [diff] [blame] | 262 | FieldDefinition(LU.i(p,1), Name('int32'), LU.i(p, 3), LU.i(p, 9), [FieldDirective(Name('type'), Name('link')), FieldDirective(Name('model'),LU.i(p, 5))] + srcPort(LU.i(p,7)) + LU.i(p,10)), |
| 263 | LinkDefinition(LU.i(p,2), LU.i(p,3), LU.i(p,5), LU.i(p,6), LU.i(p,7))) |
Sapan Bhatia | b122587 | 2017-03-29 20:47:47 +0200 | [diff] [blame] | 264 | |
| 265 | self.lh.set_parse_object(p[0], p) |
| 266 | |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 267 | # Root of the field declaration. |
| 268 | def p_field_definition(self, p): |
Sapan Bhatia | b122587 | 2017-03-29 20:47:47 +0200 | [diff] [blame] | 269 | '''field_definition : field_modifier field_type field_name EQ field_id field_directives SEMI''' |
Dusan Klinec | aa9ff47 | 2014-11-10 18:02:03 +0100 | [diff] [blame] | 270 | p[0] = FieldDefinition(LU.i(p,1), LU.i(p,2), LU.i(p, 3), LU.i(p,5), LU.i(p,6)) |
| 271 | self.lh.set_parse_object(p[0], p) |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 272 | |
| 273 | # Root of the enum field declaration. |
| 274 | def p_enum_field(self, p): |
Dusan Klinec | a4fae11 | 2014-11-10 08:50:27 +0100 | [diff] [blame] | 275 | '''enum_field : field_name EQ NUM SEMI''' |
Dusan Klinec | aa9ff47 | 2014-11-10 18:02:03 +0100 | [diff] [blame] | 276 | p[0] = EnumFieldDefinition(LU.i(p, 1), LU.i(p,3)) |
| 277 | self.lh.set_parse_object(p[0], p) |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 278 | |
| 279 | def p_enum_body_part(self, p): |
| 280 | '''enum_body_part : enum_field |
| 281 | | option_directive''' |
| 282 | p[0] = p[1] |
| 283 | |
| 284 | def p_enum_body(self, p): |
| 285 | '''enum_body : enum_body_part |
| 286 | | enum_body enum_body_part''' |
| 287 | if len(p) == 2: |
| 288 | p[0] = [p[1]] |
| 289 | else: |
| 290 | p[0] = p[1] + [p[2]] |
| 291 | |
| 292 | def p_enum_body_opt(self, p): |
| 293 | '''enum_body_opt : empty''' |
| 294 | p[0] = [] |
| 295 | |
| 296 | def p_enum_body_opt2(self, p): |
| 297 | '''enum_body_opt : enum_body''' |
| 298 | p[0] = p[1] |
| 299 | |
Sapan Bhatia | 64c7251 | 2017-06-23 02:32:45 -0700 | [diff] [blame] | 300 | def p_policy_definition(self, p): |
| 301 | '''policy_definition : POLICY NAME POLICYBODY''' |
| 302 | fol = self.fol_parser.parse(p[3], lexer = self.fol_lexer) |
| 303 | p[0] = PolicyDefinition(Name(LU.i(p, 2)), fol) |
| 304 | self.lh.set_parse_object(p[0], p) |
| 305 | |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 306 | # Root of the enum declaration. |
| 307 | # enum_definition ::= 'enum' ident '{' { ident '=' integer ';' }* '}' |
| 308 | def p_enum_definition(self, p): |
| 309 | '''enum_definition : ENUM NAME LBRACE enum_body_opt RBRACE''' |
Dusan Klinec | aa9ff47 | 2014-11-10 18:02:03 +0100 | [diff] [blame] | 310 | p[0] = EnumDefinition(Name(LU.i(p, 2)), LU.i(p,4)) |
| 311 | self.lh.set_parse_object(p[0], p) |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 312 | |
| 313 | def p_extensions_to(self, p): |
| 314 | '''extensions_to : MAX''' |
| 315 | p[0] = ExtensionsMax() |
Dusan Klinec | aa9ff47 | 2014-11-10 18:02:03 +0100 | [diff] [blame] | 316 | self.lh.set_parse_object(p[0], p) |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 317 | |
| 318 | def p_extensions_to2(self, p): |
| 319 | '''extensions_to : NUM''' |
Dusan Klinec | aa9ff47 | 2014-11-10 18:02:03 +0100 | [diff] [blame] | 320 | p[0] = LU.i(p, 1) |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 321 | |
| 322 | # extensions_definition ::= 'extensions' integer 'to' integer ';' |
| 323 | def p_extensions_definition(self, p): |
| 324 | '''extensions_definition : EXTENSIONS NUM TO extensions_to SEMI''' |
Dusan Klinec | aa9ff47 | 2014-11-10 18:02:03 +0100 | [diff] [blame] | 325 | p[0] = ExtensionsDirective(LU.i(p,2), LU.i(p,4)) |
| 326 | self.lh.set_parse_object(p[0], p) |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 327 | |
| 328 | # message_extension ::= 'extend' ident '{' message_body '}' |
| 329 | def p_message_extension(self, p): |
| 330 | '''message_extension : EXTEND NAME LBRACE message_body RBRACE''' |
Dusan Klinec | aa9ff47 | 2014-11-10 18:02:03 +0100 | [diff] [blame] | 331 | p[0] = MessageExtension(Name(LU.i(p, 2)), LU.i(p,4)) |
| 332 | self.lh.set_parse_object(p[0], p) |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 333 | |
| 334 | def p_message_body_part(self, p): |
| 335 | '''message_body_part : field_definition |
Sapan Bhatia | b122587 | 2017-03-29 20:47:47 +0200 | [diff] [blame] | 336 | | link_definition |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 337 | | enum_definition |
Sapan Bhatia | 4a159ac | 2017-04-29 20:10:05 +0200 | [diff] [blame] | 338 | | option_directive |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 339 | | message_definition |
| 340 | | extensions_definition |
| 341 | | message_extension''' |
| 342 | p[0] = p[1] |
| 343 | |
| 344 | # message_body ::= { field_definition | enum_definition | message_definition | extensions_definition | message_extension }* |
| 345 | def p_message_body(self, p): |
| 346 | '''message_body : empty''' |
| 347 | p[0] = [] |
| 348 | |
| 349 | # message_body ::= { field_definition | enum_definition | message_definition | extensions_definition | message_extension }* |
| 350 | def p_message_body2(self, p): |
| 351 | '''message_body : message_body_part |
| 352 | | message_body message_body_part''' |
| 353 | if len(p) == 2: |
| 354 | p[0] = [p[1]] |
| 355 | else: |
| 356 | p[0] = p[1] + [p[2]] |
| 357 | |
| 358 | # Root of the message declaration. |
| 359 | # message_definition = MESSAGE_ - ident("messageId") + LBRACE + message_body("body") + RBRACE |
| 360 | def p_message_definition(self, p): |
Sapan Bhatia | 87792a1 | 2017-04-10 19:35:05 -0700 | [diff] [blame] | 361 | '''message_definition : MESSAGE NAME csv_expr LBRACE message_body RBRACE''' |
Sapan Bhatia | b122587 | 2017-03-29 20:47:47 +0200 | [diff] [blame] | 362 | p[0] = MessageDefinition(Name(LU.i(p, 2)), LU.i(p, 3), LU.i(p,5)) |
Dusan Klinec | aa9ff47 | 2014-11-10 18:02:03 +0100 | [diff] [blame] | 363 | self.lh.set_parse_object(p[0], p) |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 364 | |
| 365 | # method_definition ::= 'rpc' ident '(' [ ident ] ')' 'returns' '(' [ ident ] ')' ';' |
| 366 | def p_method_definition(self, p): |
| 367 | '''method_definition : RPC NAME LPAR NAME RPAR RETURNS LPAR NAME RPAR''' |
Dusan Klinec | aa9ff47 | 2014-11-10 18:02:03 +0100 | [diff] [blame] | 368 | p[0] = MethodDefinition(Name(LU.i(p, 2)), Name(LU.i(p, 4)), Name(LU.i(p, 8))) |
| 369 | self.lh.set_parse_object(p[0], p) |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 370 | |
| 371 | def p_method_definition_opt(self, p): |
| 372 | '''method_definition_opt : empty''' |
| 373 | p[0] = [] |
| 374 | |
| 375 | def p_method_definition_opt2(self, p): |
| 376 | '''method_definition_opt : method_definition |
| 377 | | method_definition_opt method_definition''' |
| 378 | if len(p) == 2: |
| 379 | p[0] = [p[1]] |
| 380 | else: |
| 381 | p[0] = p[1] + [p[2]] |
| 382 | |
| 383 | # service_definition ::= 'service' ident '{' method_definition* '}' |
| 384 | # service_definition = SERVICE_ - ident("serviceName") + LBRACE + ZeroOrMore(Group(method_definition)) + RBRACE |
| 385 | def p_service_definition(self, p): |
Sapan Bhatia | b122587 | 2017-03-29 20:47:47 +0200 | [diff] [blame] | 386 | '''service_definition : _SERVICE NAME LBRACE method_definition_opt RBRACE''' |
Dusan Klinec | aa9ff47 | 2014-11-10 18:02:03 +0100 | [diff] [blame] | 387 | p[0] = ServiceDefinition(Name(LU.i(p, 2)), LU.i(p,4)) |
| 388 | self.lh.set_parse_object(p[0], p) |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 389 | |
| 390 | # package_directive ::= 'package' ident [ '.' ident]* ';' |
| 391 | def p_package_directive(self,p): |
Dusan Klinec | a4fae11 | 2014-11-10 08:50:27 +0100 | [diff] [blame] | 392 | '''package_directive : PACKAGE dotname SEMI''' |
Dusan Klinec | aa9ff47 | 2014-11-10 18:02:03 +0100 | [diff] [blame] | 393 | p[0] = PackageStatement(Name(LU.i(p, 2))) |
| 394 | self.lh.set_parse_object(p[0], p) |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 395 | |
| 396 | # import_directive = IMPORT_ - quotedString("importFileSpec") + SEMI |
| 397 | def p_import_directive(self, p): |
| 398 | '''import_directive : IMPORT STRING_LITERAL SEMI''' |
Dusan Klinec | aa9ff47 | 2014-11-10 18:02:03 +0100 | [diff] [blame] | 399 | p[0] = ImportStatement(Literal(LU.i(p,2))) |
| 400 | self.lh.set_parse_object(p[0], p) |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 401 | |
| 402 | def p_option_rvalue(self, p): |
| 403 | '''option_rvalue : NUM |
| 404 | | TRUE |
| 405 | | FALSE''' |
Dusan Klinec | aa9ff47 | 2014-11-10 18:02:03 +0100 | [diff] [blame] | 406 | p[0] = LU(p, 1) |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 407 | |
| 408 | def p_option_rvalue2(self, p): |
| 409 | '''option_rvalue : STRING_LITERAL''' |
Dusan Klinec | aa9ff47 | 2014-11-10 18:02:03 +0100 | [diff] [blame] | 410 | p[0] = Literal(LU(p,1)) |
| 411 | |
| 412 | def p_option_rvalue3(self, p): |
| 413 | '''option_rvalue : NAME''' |
| 414 | p[0] = Name(LU.i(p,1)) |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 415 | |
| 416 | # option_directive = OPTION_ - ident("optionName") + EQ + quotedString("optionValue") + SEMI |
| 417 | def p_option_directive(self, p): |
| 418 | '''option_directive : OPTION NAME EQ option_rvalue SEMI''' |
Dusan Klinec | aa9ff47 | 2014-11-10 18:02:03 +0100 | [diff] [blame] | 419 | p[0] = OptionStatement(Name(LU.i(p, 2)), LU.i(p,4)) |
| 420 | self.lh.set_parse_object(p[0], p) |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 421 | |
Sapan Bhatia | ff86b01 | 2017-06-11 14:44:15 -0700 | [diff] [blame] | 422 | # topLevelStatement = Group(message_definition | message_extension | enum_definition | service_definition | import_directive | option_directive | package_definition) |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 423 | def p_topLevel(self,p): |
| 424 | '''topLevel : message_definition |
| 425 | | message_extension |
| 426 | | enum_definition |
Sapan Bhatia | 64c7251 | 2017-06-23 02:32:45 -0700 | [diff] [blame] | 427 | | policy_definition |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 428 | | service_definition |
| 429 | | import_directive |
Sapan Bhatia | ff86b01 | 2017-06-11 14:44:15 -0700 | [diff] [blame] | 430 | | package_directive |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 431 | | option_directive''' |
| 432 | p[0] = p[1] |
| 433 | |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 434 | def p_statements2(self, p): |
| 435 | '''statements : topLevel |
| 436 | | statements topLevel''' |
| 437 | if len(p) == 2: |
| 438 | p[0] = [p[1]] |
| 439 | else: |
| 440 | p[0] = p[1] + [p[2]] |
| 441 | |
| 442 | def p_statements(self, p): |
| 443 | '''statements : empty''' |
| 444 | p[0] = [] |
| 445 | |
| 446 | # parser = Optional(package_directive) + ZeroOrMore(topLevelStatement) |
Dusan Klinec | c9b031a | 2014-11-10 13:21:08 +0100 | [diff] [blame] | 447 | def p_protofile(self, p): |
Sapan Bhatia | ff86b01 | 2017-06-11 14:44:15 -0700 | [diff] [blame] | 448 | '''protofile : statements''' |
| 449 | p[0] = ProtoFile(LU.i(p,1)) |
Dusan Klinec | aa9ff47 | 2014-11-10 18:02:03 +0100 | [diff] [blame] | 450 | self.lh.set_parse_object(p[0], p) |
Dusan Klinec | c9b031a | 2014-11-10 13:21:08 +0100 | [diff] [blame] | 451 | |
| 452 | # Parsing starting point |
| 453 | def p_goal(self, p): |
Dusan Klinec | aa9ff47 | 2014-11-10 18:02:03 +0100 | [diff] [blame] | 454 | '''goal : STARTTOKEN protofile''' |
Dusan Klinec | c9b031a | 2014-11-10 13:21:08 +0100 | [diff] [blame] | 455 | p[0] = p[2] |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 456 | |
| 457 | def p_error(self, p): |
| 458 | print('error: {}'.format(p)) |
| 459 | |
| 460 | class ProtobufAnalyzer(object): |
| 461 | |
| 462 | def __init__(self): |
Sapan Bhatia | b122587 | 2017-03-29 20:47:47 +0200 | [diff] [blame] | 463 | self.lexer = lex.lex(module=ProtobufLexer())#, optimize=1) |
Sapan Bhatia | 4460911 | 2017-05-15 00:00:25 +0200 | [diff] [blame] | 464 | self.parser = yacc.yacc(module=ProtobufParser(), start='goal', debug=0, outputdir='/tmp')#optimize=1) |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 465 | |
| 466 | def tokenize_string(self, code): |
| 467 | self.lexer.input(code) |
| 468 | for token in self.lexer: |
| 469 | print(token) |
| 470 | |
| 471 | def tokenize_file(self, _file): |
| 472 | if type(_file) == str: |
| 473 | _file = file(_file) |
| 474 | content = '' |
| 475 | for line in _file: |
| 476 | content += line |
| 477 | return self.tokenize_string(content) |
| 478 | |
Dusan Klinec | aa9ff47 | 2014-11-10 18:02:03 +0100 | [diff] [blame] | 479 | def parse_string(self, code, debug=0, lineno=1, prefix='+'): |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 480 | self.lexer.lineno = lineno |
Dusan Klinec | aa9ff47 | 2014-11-10 18:02:03 +0100 | [diff] [blame] | 481 | self.parser.offset = len(prefix) |
Dusan Klinec | ccaa0d9 | 2014-11-09 03:21:31 +0100 | [diff] [blame] | 482 | return self.parser.parse(prefix + code, lexer=self.lexer, debug=debug) |
| 483 | |
| 484 | def parse_file(self, _file, debug=0): |
| 485 | if type(_file) == str: |
| 486 | _file = file(_file) |
| 487 | content = '' |
| 488 | for line in _file: |
| 489 | content += line |
| 490 | return self.parse_string(content, debug=debug) |