CORD-889: Re-adding xproto generation tool, now with base library on
github
Change-Id: Ibb31ce50e798436c5780cde3cf533a366d2539c4
diff --git a/containers/xos/pip_requirements.txt b/containers/xos/pip_requirements.txt
index 9d74a34..80a882d 100644
--- a/containers/xos/pip_requirements.txt
+++ b/containers/xos/pip_requirements.txt
@@ -156,3 +156,4 @@
wsgiref==0.1.2
zope.interface==4.3.3
redis==2.10.5
+git+https://github.com/sb98052/plyprotobuf
diff --git a/xos/genx/targets/django-split.xtarget b/xos/genx/targets/django-split.xtarget
new file mode 100644
index 0000000..07df559
--- /dev/null
+++ b/xos/genx/targets/django-split.xtarget
@@ -0,0 +1,28 @@
+{% for m in proto.messages %}
+
+from header import *
+
+{% if file_exists(m.name|lower+'_header.py') -%}from {{m.name|lower}}_header import *{% endif %}
+
+{% if file_exists(m.name|lower+'_top.py') -%}{{ include_file(m.name|lower+'_top.py') }}{% endif %}
+
+{%- for l in m.links %}
+from core.models.{{ l.peer | lower }} import {{ l.peer }}
+{%- endfor %}
+
+class {{ m.name }}{{ xproto_base_def(m.bases) }}:
+ # Primitive Fields (Not Relations)
+ {%- for f in m.fields %}
+ {{ f.name }} = {{ xproto_django_type(f.type, f.options) }}( {{ xproto_django_options_str(f) }} )
+ {%- endfor %}
+
+ # Relations
+ {%- for l in m.links %}
+ {{ l.src_port }} = {{ xproto_django_link_type(l) }}( {{ l.peer }}, {{ xproto_django_options_str(l, l.dst_port ) }} )
+ {%- endfor %}
+
+ {%- if file_exists(m.name|lower + '_model.py') -%}{{ include_file(m.name|lower + '_model.py') | indent(width=2)}}{%- endif %}
+
+{% if file_exists(m.name|lower+'_bottom.py') -%}{{ include_file(m.name|lower+'_bottom.py') }}{% endif %}
++++ models/{{m.name|lower}}.py
+{% endfor %}
diff --git a/xos/genx/targets/django.xtarget b/xos/genx/targets/django.xtarget
index b35d0be..5dff8d5 100644
--- a/xos/genx/targets/django.xtarget
+++ b/xos/genx/targets/django.xtarget
@@ -1,21 +1,27 @@
{% for m in proto.messages %}
+from header import *
+
+{% if file_exists(m.name|lower+'_header.py') -%}from {{m.name|lower}}_header import *{% endif %}
+
+{% if file_exists(m.name|lower+'_top.py') -%}{{ include_file(m.name|lower+'_top.py') }}{% endif %}
+
{%- for l in m.links %}
from core.models.{{ l.peer | lower }} import {{ l.peer }}
{%- endfor %}
-{{ include_file('header.py') }}
-{{ include_file(m+'_header.py') }}
-
-class {{ m.name }}({{ m.bclass }}):
+class {{ m.name }}{{ xproto_base_def(m.bases) }}:
# Primitive Fields (Not Relations)
{%- for f in m.fields %}
- {{ f.name }} = {{ xproto_django_type(f.type, f.options) }}( {{ xproto_django_options_str(f) }} );
+ {{ f.name }} = {{ xproto_django_type(f.type, f.options) }}( {{ xproto_django_options_str(f) }} )
{%- endfor %}
# Relations
{%- for l in m.links %}
- {{ l.src_port }} = {{ l.dst_port }} {{ xproto_django_link_type(l) }}( {{ l.peer }}, {{ xproto_django_options_str(l) }} );
+ {{ l.src_port }} = {{ xproto_django_link_type(l) }}( {{ l.peer }}, {{ xproto_django_options_str(l, l.dst_port ) }} )
{%- endfor %}
-}
+
+ {%- if file_exists(m.name|lower + '_model.py') -%}{{ include_file(m.name|lower + '_model.py') | indent(width=2)}}{%- endif %}
+
+{% if file_exists(m.name|lower+'_bottom.py') -%}{{ include_file(m.name|lower+'_bottom.py') }}{% endif %}
{% endfor %}
diff --git a/xos/genx/tool/lib.py b/xos/genx/tool/lib.py
new file mode 100644
index 0000000..fd7631e
--- /dev/null
+++ b/xos/genx/tool/lib.py
@@ -0,0 +1,104 @@
+import pdb
+
+def django_content_type_string(xptags):
+ # Check possibility of KeyError in caller
+ content_type = xptags['content_type']
+
+ if (content_type=='url'):
+ return 'URLField'
+ elif (content_type=='ip'):
+ return 'GenericIPAddressField'
+ elif (content_type=='stripped' or content_type=='"stripped"'):
+ return 'StrippedCharField'
+
+def django_string_type(xptags):
+ if ('content_type' in xptags):
+ return django_content_type_string(xptags)
+ elif ('indexed' not in xptags):
+ return 'TextField'
+ else:
+ return 'CharField'
+
+def xproto_base_def(base):
+ if (not base):
+ return ''
+ else:
+ return '(' + ','.join(base) + ')'
+
+def xproto_django_type(xptype, xptags):
+ if (xptype=='string'):
+ return django_string_type(xptags)
+ elif (xptype=='float'):
+ return 'FloatField'
+ elif (xptype=='bool'):
+ return 'BooleanField'
+ elif (xptype=='uint32'):
+ return 'IntegerField'
+ elif (xptype=='int32'):
+ return 'IntegerField'
+ elif (xptype=='int64'):
+ return 'BigIntegerField'
+ else:
+ raise Exception('Unknown Type: %s'%xptype)
+
+
+
+def xproto_django_link_type(f):
+ if (f['link_type']=='manytoone'):
+ return 'ForeignKey'
+ elif (f['link_type']=='manytomany'):
+ if (f['dst_port']):
+ return 'ManyToManyRelation'
+ else:
+ return 'GenericRelation'
+
+def format_options_string(d):
+ if (not d):
+ return ''
+ else:
+ lst = []
+ for k,v in d.items():
+ if (type(v)==str and v.startswith('"')):
+ try:
+ tup = eval(v[1:-1])
+ if (type(tup)==tuple):
+ lst.append('%s = %r'%(k,tup))
+ else:
+ lst.append('%s = %s'%(k,v))
+ except:
+ lst.append('%s = %s'%(k,v))
+ elif (type(v)==bool):
+ lst.append('%s = %r'%(k,bool(v)))
+ else:
+ try:
+ lst.append('%s = %r'%(k,int(v)))
+ except ValueError:
+ lst.append('%s = %s'%(k,v))
+
+ return ', '.join(lst)
+
+def map_xproto_to_django(f):
+ allowed_keys=['help_text','default','max_length','modifier','blank','choices','db_index','null']
+
+ m = {'modifier':{'optional':True, 'required':False, '_target':'null'}}
+ out = {}
+
+ for k,v in f['options'].items():
+ if (k in allowed_keys):
+ try:
+ kv2 = m[k]
+ out[kv2['_target']] = kv2[v]
+ except:
+ out[k] = v
+ return out
+
+
+def xproto_django_options_str(field, dport=None):
+ output_dict = map_xproto_to_django(field)
+
+ if (dport=='_'):
+ dport = '+'
+
+ if (dport):
+ output_dict['related_name'] = '%r'%dport
+ return format_options_string(output_dict)
diff --git a/xos/genx/tool/proto2xproto.py b/xos/genx/tool/proto2xproto.py
new file mode 100644
index 0000000..413ff0a
--- /dev/null
+++ b/xos/genx/tool/proto2xproto.py
@@ -0,0 +1,151 @@
+import plyproto.model as m
+import pdb
+import argparse
+import plyproto.parser as plyproto
+import traceback
+import sys
+import jinja2
+import os
+
+class Stack(list):
+ def push(self,x):
+ self.append(x)
+
+''' Proto2XProto overrides the underlying visitor pattern to transform the tree
+ in addition to traversing it '''
+class Proto2XProto(m.Visitor):
+ stack = Stack()
+ count_stack = Stack()
+ content=""
+ offset=0
+ statementsChanged=0
+
+ def map_field(self, obj, s):
+ if 'model' in s:
+ link = m.LinkDefinition('onetoone','src','name','dst', obj.linespan, obj.lexspan, obj.p)
+ lspec = m.LinkSpec(link, obj)
+ else:
+ lspec = obj
+ return lspec
+
+
+ def get_stack(self):
+ return stack
+
+ def __init__(self):
+ super(Proto2XProto, self).__init__()
+
+ self.verbose = 0
+ self.first_field = True
+ self.first_method = True
+
+ def visit_PackageStatement(self, obj):
+ '''Ignore'''
+ return True
+
+ def visit_ImportStatement(self, obj):
+ '''Ignore'''
+ return True
+
+ def visit_OptionStatement(self, obj):
+ '''Ignore'''
+ return True
+
+ def visit_LU(self, obj):
+ return True
+
+ def visit_default(self, obj):
+ return True
+
+ def visit_FieldDirective(self, obj):
+ return True
+
+ def visit_FieldDirective_post(self, obj):
+ try:
+ name = obj.name.value.pval
+ except AttributeError:
+ name = obj.name.value
+
+ try:
+ value = obj.value.value.pval
+ except AttributeError:
+ try:
+ value = obj.value.value
+ except AttributeError:
+ value = obj.value.pval
+
+ self.stack.push([name,value])
+ return True
+
+ def visit_FieldType(self, obj):
+ return True
+
+ def visit_LinkDefinition(self, obj):
+ return True
+
+ def visit_FieldDefinition(self, obj):
+ self.count_stack.push(len(obj.fieldDirective))
+ return True
+
+ def visit_FieldDefinition_post(self, obj):
+ opts = {}
+ n = self.count_stack.pop()
+ for i in range(0, n):
+ k,v = self.stack.pop()
+ opts[k] = v
+
+ f = self.map_field(obj, opts)
+ self.stack.push(f)
+ #self.stack::declare.push(f)
+ return True
+
+ def visit_EnumFieldDefinition(self, obj):
+ return True
+
+ def visit_EnumDefinition(self, obj):
+ return True
+
+ def visit_MessageDefinition(self, obj):
+ self.count_stack.push(len(obj.body))
+ return True
+
+ def visit_MessageDefinition_post(self, obj):
+ stack_num = self.count_stack.pop()
+ lst = []
+
+ for n in range(0,stack_num):
+ lst.append(self.stack.pop())
+
+ obj.body = lst
+ return True
+
+ def visit_MessageExtension(self, obj):
+ return True
+
+ def visit_MethodDefinition(self, obj):
+ return True
+
+ def visit_ServiceDefinition(self, obj):
+ return True
+
+ def visit_ExtensionsDirective(self, obj):
+ return True
+
+ def visit_Literal(self, obj):
+ return True
+
+ def visit_Name(self, obj):
+ return True
+
+ def visit_DotName(self, obj):
+ return True
+
+ def visit_Proto(self, obj):
+ self.count_stack.push(len(obj.body))
+ return True
+
+ def visit_Proto_post(self, obj):
+ return True
+
+ def visit_LinkSpec(self, obj):
+ return False
diff --git a/xos/genx/tool/xos2jinja.py b/xos/genx/tool/xos2jinja.py
new file mode 100644
index 0000000..09af05c
--- /dev/null
+++ b/xos/genx/tool/xos2jinja.py
@@ -0,0 +1,194 @@
+import plyproto.model as m
+import pdb
+import argparse
+import plyproto.parser as plyproto
+import traceback
+import sys
+import jinja2
+import os
+
+class Stack(list):
+ def push(self,x):
+ self.append(x)
+
+''' XOS2Jinja overrides the underlying visitor pattern to transform the tree
+ in addition to traversing it '''
+class XOS2Jinja(m.Visitor):
+ stack = Stack()
+ count_stack = Stack()
+ content=""
+ offset=0
+ doNameSanitization=False
+ statementsChanged=0
+ prefix=""
+
+ def get_stack(self):
+ return stack
+
+ def __init__(self):
+ super(XOS2Jinja, self).__init__()
+
+ self.verbose = 0
+ self.first_field = True
+ self.first_method = True
+
+ def visit_PackageStatement(self, obj):
+ '''Ignore'''
+ return True
+
+ def visit_ImportStatement(self, obj):
+ '''Ignore'''
+ return True
+
+ def visit_OptionStatement(self, obj):
+ '''Ignore'''
+ return True
+
+ def visit_LU(self, obj):
+ return True
+
+ def visit_default(self, obj):
+ return True
+
+ def visit_FieldDirective(self, obj):
+ return True
+
+ def visit_FieldDirective_post(self, obj):
+
+ try:
+ name = obj.name.value.pval
+ except AttributeError:
+ name = obj.name.value
+
+ try:
+ value = obj.value.value.pval
+ except AttributeError:
+ try:
+ value = obj.value.value
+ except AttributeError:
+ value = obj.value.pval
+
+ self.stack.push([name,value])
+ return True
+
+ def visit_FieldType(self, obj):
+ '''Field type, if type is name, then it may need refactoring consistent with refactoring rules according to the table'''
+ return True
+
+ def visit_LinkDefinition(self, obj):
+ s={}
+ s['link_type'] = obj.link_type.pval
+ s['src_port'] = obj.src_port.value.pval
+ s['name'] = obj.src_port.value.pval
+ try:
+ s['dst_port'] = obj.dst_port.value.pval
+ except AttributeError:
+ s['dst_port'] = ''
+ s['peer'] = obj.name
+ s['_type'] = 'link'
+ s['options'] = {'modifier':'optional'}
+
+ self.stack.push(s)
+ return True
+
+ def visit_FieldDefinition(self, obj):
+ self.count_stack.push(len(obj.fieldDirective))
+ return True
+
+ def visit_FieldDefinition_post(self, obj):
+ s={'_type':'field'}
+ if isinstance(obj.ftype, m.Name):
+ s['type'] = obj.ftype.value
+ else:
+ s['type'] = obj.ftype.name.pval
+ s['name'] = obj.name.value.pval
+ s['modifier'] = obj.field_modifier.pval
+ s['id'] = obj.fieldId.pval
+
+ opts = {'modifier':s['modifier']}
+ n = self.count_stack.pop()
+ for i in range(0, n):
+ k,v = self.stack.pop()
+ opts[k] = v
+
+ s['options'] = opts
+ self.stack.push(s)
+ return True
+
+ def visit_EnumFieldDefinition(self, obj):
+ if self.verbose > 4:
+ print "\tEnumField: name=%s, %s" % (obj.name, obj)
+
+ return True
+
+ def visit_EnumDefinition(self, obj):
+ '''New enum definition, refactor name'''
+ if self.verbose > 3:
+ print "Enum, [%s] body=%s\n\n" % (obj.name, obj.body)
+
+ self.prefixize(obj.name, obj.name.value)
+ return True
+
+ def visit_MessageDefinition(self, obj):
+ self.count_stack.push(len(obj.body))
+ return True
+
+ def visit_MessageDefinition_post(self, obj):
+ stack_num = self.count_stack.pop()
+ fields = []
+ links = []
+ last_field = None
+ obj.bases = map(lambda x:x.pval, obj.bases)
+
+ for i in range(0,stack_num):
+ f = self.stack.pop()
+ if (f['_type']=='link'):
+ f['options']={i:d[i] for d in [f['options'],last_field['options']] for i in d}
+
+ links.insert(0,f)
+ else:
+ fields.insert(0,f)
+ last_field = f
+
+ self.stack.push({'name':obj.name.value.pval,'fields':fields,'links':links, 'bases':obj.bases})
+ return True
+
+ def visit_MessageExtension(self, obj):
+ return True
+
+ def visit_MethodDefinition(self, obj):
+ return True
+
+ def visit_ServiceDefinition(self, obj):
+ return True
+
+ def visit_ExtensionsDirective(self, obj):
+ return True
+
+ def visit_Literal(self, obj):
+ return True
+
+ def visit_Name(self, obj):
+ return True
+
+ def visit_DotName(self, obj):
+ return True
+
+ def visit_Proto(self, obj):
+ self.count_stack.push(len(obj.body))
+ return True
+
+ def visit_Proto_post(self, obj):
+ count = self.count_stack.pop()
+ messages = []
+ for i in range(0,count):
+ m = self.stack.pop()
+ messages.insert(0,m)
+
+ self.messages = messages
+ return True
+
+ def visit_LinkSpec(self, obj):
+ count = self.count_stack.pop()
+ self.count_stack.push(count+1)
+ return True
diff --git a/xos/genx/tool/xosgen b/xos/genx/tool/xosgen
new file mode 100755
index 0000000..1585e9d
--- /dev/null
+++ b/xos/genx/tool/xosgen
@@ -0,0 +1,104 @@
+#!/usr/bin/python
+
+import plyproto.model as m
+import pdb
+import argparse
+import plyproto.parser as plyproto
+import traceback
+import sys
+import jinja2
+import os
+from xos2jinja import XOS2Jinja
+from proto2xproto import Proto2XProto
+
+import lib
+
+parse = argparse.ArgumentParser(description='XOS code generator')
+parse.add_argument('--rev', dest='rev', action='store_true',default=False, help='Convert proto to xproto')
+parse.add_argument('--input', dest='input', action='store',default=None, help='Filename in Protobufs')
+parse.add_argument('--output', dest='output', action='store',default=None, help='Output format, corresponding to <output>.yaml file')
+
+args = parse.parse_args()
+
+def file_exists(name):
+ return (os.path.exists(name))
+
+def include_file(name):
+ return open(name).read()
+ # FIXME: Support templates in the future
+ #return jinja2.Markup(loader.get_source(env, name)[0])
+
+loader = jinja2.PackageLoader(__name__, 'templates')
+env = jinja2.Environment(loader=loader)
+
+def main():
+ try:
+ if (not args.rev):
+ v = XOS2Jinja()
+ else:
+ v = Proto2XProto()
+
+ parser = plyproto.ProtobufAnalyzer()
+ input = open(args.input).read()
+
+ ast = parser.parse_string(input,debug=0)
+ ast.accept(v)
+
+ template_name = os.path.abspath(args.output)
+
+ os_template_loader = jinja2.FileSystemLoader( searchpath=[os.path.split(template_name)[0]])
+ os_template_env = jinja2.Environment(loader=os_template_loader)
+ os_template_env.globals['include_file'] = include_file
+ os_template_env.globals['file_exists'] = file_exists
+
+ for f in dir(lib):
+ if f.startswith('xproto'):
+ os_template_env.globals[f] = getattr(lib, f)
+
+ template = os_template_env.get_template(os.path.split(template_name)[1])
+ rendered = template.render({"proto": {'messages':v.messages}})
+
+ lines = rendered.splitlines()
+ current_buffer = []
+ for l in lines:
+ if (l.startswith('+++')):
+ path = l[4:]
+
+ direc,filename = path.rsplit('/',1)
+ os.system('mkdir -p %s'%direc)
+
+ fil = open(path,'w')
+ buf = '\n'.join(current_buffer)
+
+ obuf = buf
+
+ """
+ for d in options.dict:
+ df = open(d).read()
+ d = json.loads(df)
+
+ pattern = re.compile(r'\b(' + '|'.join(d.keys()) + r')\b')
+ obuf = pattern.sub(lambda x: d[x.group()], buf)
+ """
+
+ fil.write(obuf)
+ fil.close()
+
+ print 'Written to file %s'%path
+ current_buffer = []
+ else:
+ current_buffer.append(l)
+
+ if (current_buffer):
+ print '\n'.join(current_buffer)
+
+
+ except Exception as e:
+ print " Error occurred! file[%s]" % (args.input), e
+ print '-'*60
+ traceback.print_exc(file=sys.stdout)
+ print '-'*60
+ exit(1)
+
+if __name__=='__main__':
+ main()