[CORD-1440] Moving the generative toolchain in a library
Change-Id: Ifa8e8f930ac34e1f8952099b7e34842a52f4664d
diff --git a/lib/xos-genx/xosgenx/__init__.py b/lib/xos-genx/xosgenx/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/lib/xos-genx/xosgenx/__init__.py
diff --git a/lib/xos-genx/xosgenx/generator.py b/lib/xos-genx/xosgenx/generator.py
new file mode 100755
index 0000000..239a42a
--- /dev/null
+++ b/lib/xos-genx/xosgenx/generator.py
@@ -0,0 +1,223 @@
+import plyxproto.parser as plyxproto
+import jinja2
+import os
+from xos2jinja import XOS2Jinja
+from proto2xproto import Proto2XProto
+import jinja2_extensions
+import yaml
+
+loader = jinja2.PackageLoader(__name__, 'templates')
+env = jinja2.Environment(loader=loader)
+
+class XOSGenerator:
+
+ @staticmethod
+ def _read_input_from_files(files):
+ input = ''
+ for fname in files:
+ with open(fname) as infile:
+ input += infile.read()
+ return input
+
+ @staticmethod
+ def _attach_parser(ast, args):
+ if hasattr(args, 'rev') and args.rev:
+ v = Proto2XProto()
+ ast.accept(v)
+ else:
+ v = XOS2Jinja()
+ ast.accept(v)
+ return v
+
+ @staticmethod
+ def _get_template(target):
+ if not os.path.isabs(target):
+ return os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + '/targets/' + target)
+ return target
+
+ @staticmethod
+ def _file_exists(attic):
+ # NOTE this method can be used in the jinja template
+ def file_exists2(name):
+ if attic is not None:
+ path = attic + '/' + name
+ else:
+ path = name
+ return (os.path.exists(path))
+
+ return file_exists2
+
+ @staticmethod
+ def _include_file(attic):
+ # NOTE this method can be used in the jinja template
+ def include_file2(name):
+ if attic is not None:
+ path = attic + '/' + name
+ else:
+ path = name
+ return open(path).read()
+ return include_file2
+
+ @staticmethod
+ def _load_jinja2_extensions(os_template_env, attic):
+
+ os_template_env.globals['include_file'] = XOSGenerator._include_file(attic) # Generates a function
+ os_template_env.globals['file_exists'] = XOSGenerator._file_exists(attic) # Generates a function
+
+ os_template_env.filters['yaml'] = yaml.dump
+ for f in dir(jinja2_extensions):
+ if f.startswith('xproto'):
+ os_template_env.globals[f] = getattr(jinja2_extensions, f)
+ return os_template_env
+
+ @staticmethod
+ def _add_context(args):
+ if not hasattr(args, 'kv') or not args.kv:
+ return
+ try:
+ context = {}
+ for s in args.kv.split(','):
+ k, val = s.split(':')
+ context[k] = val
+ return context
+ except Exception, e:
+ print e.message
+
+ @staticmethod
+ def _write_single_file(rendered, dir, dest_file, quiet):
+
+ file_name = "%s/%s" % (dir, dest_file)
+ file = open(file_name, 'w')
+ file.write(rendered)
+ file.close()
+ if quiet == False:
+ print "Saved: %s" % file_name
+
+ @staticmethod
+ def _write_file_per_model(rendered, dir, extension, quiet):
+ for m in rendered:
+
+ file_name = "%s/%s.%s" % (dir, m.lower(), extension)
+ if not rendered[m]:
+ if quiet == False:
+ print "Not saving %s as it is empty" % file_name
+ else:
+ file = open(file_name, 'w')
+ file.write(rendered[m])
+ file.close()
+ if quiet == False:
+ print "Saved: %s" % file_name
+
+ @staticmethod
+ def _write_split_target(rendered, dir, quiet):
+
+ lines = rendered.splitlines()
+ current_buffer = []
+ for l in lines:
+ if (l.startswith('+++')):
+
+ if dir:
+ path = dir + '/' + l[4:].lower()
+
+ fil = open(path, 'w')
+ buf = '\n'.join(current_buffer)
+
+ obuf = buf
+
+ fil.write(obuf)
+ fil.close()
+
+ if quiet == False:
+ print "Save file to: %s" % path
+
+ current_buffer = []
+ else:
+ current_buffer.append(l)
+
+ @staticmethod
+ def _find_message_by_model_name(messages, model):
+ return next((x for x in messages if x['name'] == model), None)
+
+ @staticmethod
+ def generate(args):
+
+ # Setting defaults
+ if not hasattr(args, 'attic'):
+ args.attic = None
+ if not hasattr(args, 'write_to_file'):
+ args.write_to_file = None
+ if not hasattr(args, 'dest_file'):
+ args.dest_file = None
+ if not hasattr(args, 'dest_extension'):
+ args.dest_extension = None
+ if not hasattr(args, 'output'):
+ args.output = None
+ if not hasattr(args, 'quiet'):
+ args.quiet = True
+
+ # Validating
+ if args.write_to_file == 'single' and args.dest_file is None:
+ raise Exception("[XosGenX] write_to_file option is specified as 'single' but no dest_file is provided")
+ if args.write_to_file == 'model' and args.dest_extension is None:
+ raise Exception("[XosGenX] write_to_file option is specified as 'model' but no dest_extension is provided")
+
+ if args.output is not None and not os.path.isabs(args.output):
+ raise Exception("[XosGenX] The output dir must be an absolute path!")
+ if args.output is not None and not os.path.isdir(args.output):
+ raise Exception("[XosGenX] The output dir must be a directory!")
+
+ if hasattr(args, 'files'):
+ inputs = XOSGenerator._read_input_from_files(args.files)
+ elif hasattr(args, 'inputs'):
+ inputs = args.inputs
+ else:
+ raise Exception("[XosGenX] No inputs provided!")
+
+ template_path = XOSGenerator._get_template(args.target)
+ [template_folder, template_name] = os.path.split(template_path)
+ os_template_loader = jinja2.FileSystemLoader(searchpath=[template_folder])
+ os_template_env = jinja2.Environment(loader=os_template_loader)
+ os_template_env = XOSGenerator._load_jinja2_extensions(os_template_env, args.attic)
+ template = os_template_env.get_template(template_name)
+ context = XOSGenerator._add_context(args)
+
+ parser = plyxproto.ProtobufAnalyzer()
+ ast = parser.parse_string(inputs, debug=0)
+ v = XOSGenerator._attach_parser(ast, args)
+
+ if args.output is not None and args.write_to_file == "model":
+ rendered = {}
+ for i, model in enumerate(v.models):
+ models = {}
+ models[model] = v.models[model]
+ messages = [XOSGenerator._find_message_by_model_name(v.messages, model)]
+ rendered[model] = template.render(
+ {"proto":
+ {
+ 'message_table': models,
+ 'messages': messages,
+ 'message_names': [m['name'] for m in messages]
+ },
+ "context": context,
+ "options": v.options
+ }
+ )
+ XOSGenerator._write_file_per_model(rendered, args.output, args.dest_extension, args.quiet)
+ else:
+ rendered = template.render(
+ {"proto":
+ {
+ 'message_table': v.models,
+ 'messages': v.messages,
+ 'message_names': [m['name'] for m in v.messages]
+ },
+ "context": context,
+ "options": v.options
+ }
+ )
+ if args.output is not None and args.write_to_file == "target":
+ XOSGenerator._write_split_target(rendered, args.output, args.quiet)
+ elif args.output is not None and args.write_to_file == "single":
+ XOSGenerator._write_single_file(rendered, args.output, args.dest_file, args.quiet)
+
+ return rendered
\ No newline at end of file
diff --git a/lib/xos-genx/xosgenx/jinja2_extensions.py b/lib/xos-genx/xosgenx/jinja2_extensions.py
new file mode 100644
index 0000000..8377d5a
--- /dev/null
+++ b/lib/xos-genx/xosgenx/jinja2_extensions.py
@@ -0,0 +1,436 @@
+import pdb
+import re
+from pattern import en
+
+class FieldNotFound(Exception):
+ def __init__(self, message):
+ super(FieldNotFound, self).__init__(message)
+
+def xproto_unquote(s):
+ return unquote(s)
+
+def unquote(s):
+ if (s.startswith('"') and s.endswith('"')):
+ return s[1:-1]
+
+def xproto_singularize(field):
+ try:
+ # The user has set a singular, as an exception that cannot be handled automatically
+ singular = field['options']['singular']
+ singular = unquote(singular)
+ except KeyError:
+ singular = en.singularize(field['name'])
+
+ return singular
+
+def xproto_singularize_pluralize(field):
+ try:
+ # The user has set a plural, as an exception that cannot be handled automatically
+ plural = field['options']['plural']
+ plural = unquote(plural)
+ except KeyError:
+ plural = en.pluralize(en.singularize(field['name']))
+
+ return plural
+
+def xproto_pluralize(field):
+ try:
+ # The user has set a plural, as an exception that cannot be handled automatically
+ plural = field['options']['plural']
+ plural = unquote(plural)
+ except KeyError:
+ plural = en.pluralize(field['name'])
+
+ return plural
+
+def xproto_unquote(s):
+ if (s.startswith('"') and s.endswith('"')):
+ s = s[1:-1]
+ return s
+
+def xproto_links_to_modeldef_relations(llst):
+ outlist = []
+ seen = []
+ for l in llst:
+ try:
+ t = l['link_type']
+ except KeyError, e:
+ raise e
+
+ if l['peer']['fqn'] not in seen and t!='manytomany':
+ outlist.append('- {model: %s, type: %s}\n'%(l['peer']['name'], l['link_type']))
+ seen.append(l['peer'])
+
+ return outlist
+
+def django_content_type_string(xptags):
+ # Check possibility of KeyError in caller
+ content_type = xptags['content_type']
+
+ try:
+ content_type = eval(content_type)
+ except:
+ pass
+
+ if (content_type=='url'):
+ return 'URLField'
+ if (content_type=='date'):
+ return 'DateTimeField'
+ elif (content_type=='ip'):
+ return 'GenericIPAddressField'
+ elif (content_type=='stripped' or content_type=='"stripped"'):
+ return 'StrippedCharField'
+ else:
+ raise Exception('Unknown Type: %s'%content_type)
+
+def django_string_type(xptags):
+ try:
+ max_length = eval(xptags['max_length'])
+ except:
+ max_length = 1024 * 1024
+
+ if ('content_type' in xptags):
+ return django_content_type_string(xptags)
+ elif (max_length<1024*1024):
+ return 'CharField'
+ else:
+ return 'TextField'
+
+def xproto_base_def(model_name, base):
+ if (model_name=='XOSBase'):
+ return '(models.Model, PlModelMixIn)'
+ elif (not base):
+ return ''
+ else:
+ base = map(lambda s:s['name'], base)
+ return '(' + ','.join(base) + ')'
+
+def xproto_first_non_empty(lst):
+ for l in lst:
+ if l: return l
+
+def xproto_api_type(field):
+ try:
+ if (unquote(field['options']['content_type'])=='date'):
+ return 'float'
+ except KeyError:
+ pass
+
+ return field['type']
+
+def xproto_django_type(xptype, xptags):
+ if (xptype=='string'):
+ return django_string_type(xptags)
+ elif (xptype=='float'):
+ return 'FloatField'
+ elif (xptype=='bool'):
+ return 'BooleanField'
+ elif (xptype=='uint32'):
+ return 'IntegerField'
+ elif (xptype=='int32'):
+ return 'IntegerField'
+ elif (xptype=='int64'):
+ return 'BigIntegerField'
+ else:
+ raise Exception('Unknown Type: %s'%xptype)
+
+
+
+def xproto_django_link_type(f):
+ if (f['link_type']=='manytoone'):
+ return 'ForeignKey'
+ elif (f['link_type']=='manytomany'):
+ if (f['dst_port']):
+ return 'ManyToManyField'
+ else:
+ return 'GenericRelation'
+
+def format_options_string(d):
+ if (not d):
+ return ''
+ else:
+
+ lst = []
+ for k,v in d.items():
+ if (type(v)==str and k=='default' and v.endswith('()"')):
+ lst.append('%s = %s'%(k,v[1:-3]))
+ elif (type(v)==str and v.startswith('"')):
+ try:
+ tup = eval(v[1:-1])
+ if (type(tup)==tuple):
+ lst.append('%s = %r'%(k,tup))
+ else:
+ lst.append('%s = %s'%(k,v))
+ except:
+ lst.append('%s = %s'%(k,v))
+ elif (type(v)==bool):
+ lst.append('%s = %r'%(k,bool(v)))
+ else:
+ try:
+ lst.append('%s = %r'%(k,int(v)))
+ except ValueError:
+ lst.append('%s = %s'%(k,v))
+
+ return ', '.join(lst)
+
+def map_xproto_to_django(f):
+ allowed_keys=['help_text','default','max_length','modifier','blank','choices','db_index','null','editable','on_delete','verbose_name', 'auto_now_add']
+
+ m = {'modifier':{'optional':True, 'required':False, '_target':'null'}}
+ out = {}
+
+ for k,v in f['options'].items():
+ if (k in allowed_keys):
+ try:
+ kv2 = m[k]
+ out[kv2['_target']] = kv2[v]
+ except:
+ out[k] = v
+ return out
+
+
+def xproto_django_link_options_str(field, dport=None):
+ output_dict = map_xproto_to_django(field)
+
+ if (dport and (dport=='+' or '+' not in dport)):
+ output_dict['related_name'] = '%r'%dport
+
+ try:
+ if field['through']:
+ d = {}
+ if isinstance(field['through'], str):
+ split = field['through'].rsplit('.',1)
+ d['name'] = split[-1]
+ if len(split)==2:
+ d['package'] = split[0]
+ d['fqn'] = 'package' + '.' + d['name']
+ else:
+ d['fqn'] = d['name']
+ d['package'] = ''
+ else:
+ d = field['through']
+
+ if not d['name'].endswith('_'+field['name']):
+ output_dict['through'] = '%r'%d['fqn']
+ except KeyError:
+ pass
+
+ return format_options_string(output_dict)
+
+def xproto_django_options_str(field, dport=None):
+ output_dict = map_xproto_to_django(field)
+
+ if (dport=='_'):
+ dport = '+'
+
+ if (dport and (dport=='+' or '+' not in dport)):
+ output_dict['related_name'] = '%r'%dport
+
+ return format_options_string(output_dict)
+
+def xproto_base_name(n):
+ # Hack - Refactor NetworkParameter* to make this go away
+ if (n.startswith('NetworkParameter')):
+ return '_'
+
+ expr = r'^[A-Z]+[a-z]*'
+
+ try:
+ match = re.findall(expr, n)[0]
+ except:
+ return '_'
+
+ return match
+
+def xproto_base_fields(m, table):
+ fields = []
+
+ for b in m['bases']:
+ option1 = b['fqn']
+ try:
+ option2 = m['package'] + '.' + b['name']
+ except TypeError:
+ option2 = option1
+
+ accessor = None
+ if option1 in table: accessor = option1
+ elif option2 in table: accessor = option2
+
+ if accessor:
+ base_fields = xproto_base_fields(table[accessor], table)
+
+ model_fields = table[accessor]['fields']
+ fields.extend(base_fields)
+ fields.extend(model_fields)
+
+ return fields
+
+def xproto_base_rlinks(m, table):
+ links = []
+
+ for base in m['bases']:
+ b = base['name']
+ if b in table:
+ base_rlinks = xproto_base_rlinks(table[b], table)
+
+ model_rlinks = table[b]['rlinks']
+ links.extend(base_rlinks)
+ links.extend(model_rlinks)
+
+ return links
+
+def xproto_base_links(m, table):
+ links = []
+
+ for base in m['bases']:
+ b = base['name']
+ if b in table:
+ base_links = xproto_base_links(table[b], table)
+
+ model_links = table[b]['links']
+ links.extend(base_links)
+ links.extend(model_links)
+ return links
+
+
+def xproto_validators(f):
+ # To be cleaned up when we formalize validation in xproto
+ validators = []
+
+ # bound-based validators
+ bound_validators = [('max_length','maxlength'), ('min', 'min'), ('max', 'max')]
+
+ for v0, v1 in bound_validators:
+ try:
+ validators.append({'name':v1, 'int_value':int(f['options'][v0])})
+ except KeyError:
+ pass
+
+ # validators based on content_type
+ content_type_validators = ['ip', 'url', 'email']
+
+ for v in content_type_validators:
+ #if f['name']=='ip': pdb.set_trace()
+ try:
+ val = unquote(f['options']['content_type'])==v
+ if not val:
+ raise KeyError
+
+ validators.append({'name':v, 'bool_value': True})
+ except KeyError:
+ pass
+
+ # required validator
+ try:
+ required = f['options']['blank']=='False' and f['options']['null']=='False'
+ if required:
+ validators.append({'name':'required', 'bool_value':required})
+ except KeyError:
+ pass
+
+ return validators
+
+def xproto_string_type(xptags):
+ try:
+ max_length = eval(xptags['max_length'])
+ except:
+ max_length = 1024
+
+ if ('varchar' not in xptags):
+ return 'string'
+ else:
+ return 'text'
+
+def xproto_type_to_ui_type(f):
+ try:
+ content_type = f['options']['content_type']
+ content_type = eval(content_type)
+ except:
+ content_type = None
+ pass
+
+ if content_type == 'date':
+ return 'date'
+ elif f['type'] == 'bool':
+ return 'boolean'
+ elif f['type'] == 'string':
+ return xproto_string_type(f['options'])
+ elif f['type'] in ['int','uint32','int32'] or 'link' in f:
+ return 'number'
+ elif f['type'] in ['double','float']:
+ return 'string'
+
+def xproto_tuplify(nested_list_or_set):
+ if not isinstance(nested_list_or_set, list) and not isinstance(nested_list_or_set, set):
+ return nested_list_or_set
+ else:
+ return tuple([xproto_tuplify(i) for i in nested_list_or_set])
+
+def xproto_field_graph_components(fields, tag='unique_with'):
+ def find_components(graph):
+ pending = set(graph.keys())
+ components = []
+
+ while pending:
+ front = { pending.pop() }
+ component = set()
+
+ while front:
+ node = front.pop()
+ neighbours = graph[node]
+ neighbours-=component # These we have already visited
+ front |= neighbours
+
+ pending-=neighbours
+ component |= neighbours
+
+ components.append(component)
+
+ return components
+
+ field_graph = {}
+ field_names = {f['name'] for f in fields}
+
+ for f in fields:
+ try:
+ tagged_str = unquote(f['options'][tag])
+ tagged_fields = tagged_str.split(',')
+
+ for uf in tagged_fields:
+ if uf not in field_names:
+ raise FieldNotFound('Field %s not found'%uf)
+
+ field_graph.setdefault(f['name'],set()).add(uf)
+ field_graph.setdefault(uf,set()).add(f['name'])
+ except KeyError:
+ pass
+
+ components = find_components(field_graph)
+ return components
+
+def xproto_api_opts(field):
+ options = []
+ if 'max_length' in field['options'] and field['type']=='string':
+ options.append('(val).maxLength = %s'%field['options']['max_length'])
+
+ try:
+ if field['options']['null'] == 'False':
+ options.append('(val).nonNull = true')
+ except KeyError:
+ pass
+
+ if 'link' in field and 'model' in field['options']:
+ options.append('(foreignKey).modelName = "%s"'%field['options']['model'])
+
+ if options:
+ options_str = '[' + ', '.join(options) + ']'
+ else:
+ options_str = ''
+
+ return options_str
+
+def xproto_tosca_required(blank):
+ if blank == "False":
+ return "true"
+ return "false"
+
diff --git a/lib/xos-genx/xosgenx/proto2xproto.py b/lib/xos-genx/xosgenx/proto2xproto.py
new file mode 100644
index 0000000..f05c2bc
--- /dev/null
+++ b/lib/xos-genx/xosgenx/proto2xproto.py
@@ -0,0 +1,196 @@
+import plyxproto.model as m
+import pdb
+import argparse
+import plyxproto.parser as plyxproto
+import traceback
+import sys
+import jinja2
+import os
+
+class Stack(list):
+ def push(self,x):
+ self.append(x)
+
+def str_to_dict(s):
+ lst = s.rsplit('.',1)
+ name = lst[-1]
+
+ if len(lst)==2:
+ package = lst[0]
+ else:
+ package = ''
+
+ return {'name': name, 'package': package, 'fqn': s}
+
+def replace_link(obj):
+ try:
+ link = obj.link
+ try:
+ through = link['through']
+ except KeyError:
+ through = None
+
+ try:
+ through_str = through[1:-1]
+ except TypeError:
+ through_str = None
+
+ if through_str:
+ through_dict = str_to_dict(through_str)
+ else:
+ through_dict = {}
+
+ model_dict = str_to_dict(link['model'][1:-1])
+
+ ls = m.LinkSpec(obj, m.LinkDefinition(link['link'][1:-1],obj.name,model_dict,link['port'][1:-1],through_dict))
+ return ls
+ except:
+ return obj
+
+class Proto2XProto(m.Visitor):
+ def __init__(self):
+ super(Proto2XProto, self).__init__()
+
+ self.stack = Stack()
+ self.count_stack = Stack()
+ self.content=""
+ self.offset=0
+ self.statementsChanged=0
+ self.message_options = {}
+ self.options = {}
+ self.current_message_name = None
+
+ self.xproto_message_options = ['bases']
+ self.xproto_field_options = ['model']
+ self.verbose = 0
+ self.first_field = True
+ self.first_method = True
+
+ def proto_to_xproto_field(self, obj):
+ try:
+ opts = {}
+ for fd in obj.fieldDirective:
+ k = fd.pval.name.value.pval
+ v = fd.pval.value.value.pval
+ opts[k]=v
+
+ if ('model' in opts and 'link' in opts and 'port' in opts):
+ obj.link = opts
+ pass
+ except KeyError:
+ raise
+
+ def proto_to_xproto_message(self, obj):
+ try:
+ bases = self.message_options['bases'].split(',')
+ bases = map(lambda x:str_to_dict(x[1:-1]), bases)
+ obj.bases = bases
+ except KeyError:
+ raise
+
+ def map_field(self, obj, s):
+ if 'model' in s:
+ link = m.LinkDefinition('onetoone','src','name','dst', obj.linespan, obj.lexspan, obj.p)
+ lspec = m.LinkSpec(link, obj)
+ else:
+ lspec = obj
+ return lspec
+
+
+ def get_stack(self):
+ return self.stack
+
+ def visit_PackageStatement(self, obj):
+ '''Ignore'''
+ return True
+
+ def visit_ImportStatement(self, obj):
+ '''Ignore'''
+ return True
+
+ def visit_OptionStatement(self, obj):
+ if (self.current_message_name):
+ k = obj.name.value.pval
+ self.message_options[k] = obj.value.value.pval
+ if (k in self.xproto_message_options):
+ obj.mark_for_deletion = True
+ else:
+ self.options[obj.name.value.pval] = obj.value.value.pval
+
+ return True
+
+ def visit_LU(self, obj):
+ return True
+
+ def visit_default(self, obj):
+ return True
+
+ def visit_FieldDirective(self, obj):
+ return True
+
+ def visit_FieldDirective_post(self, obj):
+ return True
+
+ def visit_FieldType(self, obj):
+ return True
+
+ def visit_LinkDefinition(self, obj):
+ return True
+
+ def visit_FieldDefinition(self, obj):
+ return True
+
+ def visit_FieldDefinition_post(self, obj):
+ self.proto_to_xproto_field(obj)
+ return True
+
+ def visit_EnumFieldDefinition(self, obj):
+ return True
+
+ def visit_EnumDefinition(self, obj):
+ return True
+
+ def visit_MessageDefinition(self, obj):
+ self.current_message_name = obj.name.value.pval
+ self.message_options = {}
+
+ return True
+
+ def visit_MessageDefinition_post(self, obj):
+ self.proto_to_xproto_message(obj)
+ obj.body = filter(lambda x:not hasattr(x, 'mark_for_deletion'), obj.body)
+ obj.body = map(replace_link, obj.body)
+
+ self.current_message_name = None
+ return True
+
+ def visit_MessageExtension(self, obj):
+ return True
+
+ def visit_MethodDefinition(self, obj):
+ return True
+
+ def visit_ServiceDefinition(self, obj):
+ return True
+
+ def visit_ExtensionsDirective(self, obj):
+ return True
+
+ def visit_Literal(self, obj):
+ return True
+
+ def visit_Name(self, obj):
+ return True
+
+ def visit_DotName(self, obj):
+ return True
+
+ def visit_Proto(self, obj):
+ self.count_stack.push(len(obj.body))
+ return True
+
+ def visit_Proto_post(self, obj):
+ return True
+
+ def visit_LinkSpec(self, obj):
+ return False
diff --git a/lib/xos-genx/xosgenx/targets/chameleon_list_test.xtarget b/lib/xos-genx/xosgenx/targets/chameleon_list_test.xtarget
new file mode 100644
index 0000000..fcb02fb
--- /dev/null
+++ b/lib/xos-genx/xosgenx/targets/chameleon_list_test.xtarget
@@ -0,0 +1,19 @@
+source /opt/xos/coreapi/tests/testconfig-chameleon.sh
+
+# test modeldefs
+curl -f --silent http://$HOSTNAME:8080/xosapi/v1/modeldefs > /dev/null
+if [[ $? -ne 0 ]]; then
+ echo fail modeldefs
+fi
+
+{% for object in proto.messages %}
+{%- if object.name!='XOSBase' -%}
+curl -f --silent http://$HOSTNAME:8080/xosapi/v1/{{ xproto_unquote(options.app_label) }}/{{ xproto_pluralize(object) | lower }} > /dev/null
+if [[ $? -ne 0 ]]; then
+ echo fail {{ object.name }}
+fi
+{%endif-%}
+{%- endfor %}
+
+echo "okay"
+
diff --git a/lib/xos-genx/xosgenx/targets/django-split.xtarget b/lib/xos-genx/xosgenx/targets/django-split.xtarget
new file mode 100644
index 0000000..13212ae
--- /dev/null
+++ b/lib/xos-genx/xosgenx/targets/django-split.xtarget
@@ -0,0 +1,42 @@
+{% for m in proto.messages %}{% if not m.options.skip_django -%}
+{% if file_exists(xproto_base_name(m.name)|lower+'_header.py') -%}from {{xproto_base_name(m.name)|lower }}_header import *{%- else -%}from header import *{% endif %}
+{% if file_exists(xproto_base_name(m.name)|lower+'_top.py') -%}{{ include_file(xproto_base_name(m.name)|lower+'_top.py') }} {% endif %}
+{%- for l in m.links %}
+
+{% if l.peer.name != m.name %}
+from core.models.{{ l.peer.name | lower }} import {{ l.peer.name }}
+{% endif %}
+
+{%- endfor %}
+{% for b in m.bases %}
+{% if b.name!='XOSBase' and 'Mixin' not in b.name %}
+from core.models.{{b.name | lower}} import {{ b.name }}
+{% endif %}
+{% endfor %}
+
+
+class {{ m.name }}{{ xproto_base_def(m.name, m.bases) }}:
+ # Primitive Fields (Not Relations)
+ {% for f in m.fields %}
+ {%- if not f.link -%}
+ {{ f.name }} = {{ xproto_django_type(f.type, f.options) }}( {{ xproto_django_options_str(f) }} )
+ {% endif %}
+ {%- endfor %}
+
+ # Relations
+ {% for l in m.links %}{% set peer_name=l.peer.name %}
+ {{ l.src_port }} = {{ xproto_django_link_type(l) }}( {%- if peer_name==m.name -%}'self'{%- else -%}{{ peer_name }} {%- endif -%}, {{ xproto_django_link_options_str(l, l.dst_port ) }} )
+ {%- endfor %}
+
+ # Meta
+ {%- set uniques = xproto_field_graph_components(m.fields) %}
+ {%- if uniques %}
+ class Meta:
+ unique_together = {{ xproto_tuplify(uniques) }}
+ {%- endif %}
+ {% if file_exists(m.name|lower + '_model.py') -%}{{ include_file(m.name|lower + '_model.py') | indent(width=2)}}{%- endif %}
+ pass
+
+{% if file_exists(xproto_base_name(m.name)|lower+'_bottom.py') -%}{{ include_file(xproto_base_name(m.name)|lower+'_bottom.py') }}{% endif %}
++++ {{m.name|lower}}.py
+{% endif %}{% endfor %}
\ No newline at end of file
diff --git a/lib/xos-genx/xosgenx/targets/django.xtarget b/lib/xos-genx/xosgenx/targets/django.xtarget
new file mode 100644
index 0000000..20b4e51
--- /dev/null
+++ b/lib/xos-genx/xosgenx/targets/django.xtarget
@@ -0,0 +1,42 @@
+{% for m in proto.messages %}{% if not m.options.skip_django -%}
+{% if file_exists(xproto_base_name(m.name)|lower+'_header.py') -%}from {{xproto_base_name(m.name)|lower }}_header import *{%- else -%}from header import *{% endif %}
+{% if file_exists(xproto_base_name(m.name)|lower+'_top.py') -%}{{ include_file(xproto_base_name(m.name)|lower+'_top.py') }} {% endif %}
+
+{%- for l in m.links %}
+
+{% if l.peer.name != m.name %}
+from core.models.{{ l.peer.name | lower }} import {{ l.peer.name }}
+{% endif %}
+
+{%- endfor %}
+{% for b in m.bases %}
+{% if b.name!='XOSBase' and 'Mixin' not in b.name %}
+from core.models.{{b.name | lower}} import {{ b.name }}
+{% endif %}
+{% endfor %}
+
+
+class {{ m.name }}{{ xproto_base_def(m.name, m.bases) }}:
+ # Primitive Fields (Not Relations)
+ {% for f in m.fields %}
+ {%- if not f.link -%}
+ {{ f.name }} = {{ xproto_django_type(f.type, f.options) }}( {{ xproto_django_options_str(f) }} )
+ {% endif %}
+ {%- endfor %}
+
+ # Relations
+ {% for l in m.links %}{% set peer_name=l.peer.name %}
+ {{ l.src_port }} = {{ xproto_django_link_type(l) }}( {%- if peer_name==m.name -%}'self'{%- else -%}{{ peer_name }} {%- endif -%}, {{ xproto_django_link_options_str(l, l.dst_port ) }} )
+ {%- endfor %}
+
+ # Meta
+ {%- set uniques = xproto_field_graph_components(m.fields) %}
+ {%- if uniques %}
+ class Meta:
+ unique_together = {{ xproto_tuplify(uniques) }}
+ {%- endif %}
+ {% if file_exists(m.name|lower + '_model.py') -%}{{ include_file(m.name|lower + '_model.py') | indent(width=2)}}{%- endif %}
+ pass
+
+{% if file_exists(xproto_base_name(m.name)|lower+'_bottom.py') -%}{{ include_file(xproto_base_name(m.name)|lower+'_bottom.py') }}{% endif %}
+{% endif %}{% endfor %}
\ No newline at end of file
diff --git a/lib/xos-genx/xosgenx/targets/dot.xtarget b/lib/xos-genx/xosgenx/targets/dot.xtarget
new file mode 100644
index 0000000..96aceec
--- /dev/null
+++ b/lib/xos-genx/xosgenx/targets/dot.xtarget
@@ -0,0 +1,7 @@
+digraph {
+{% for m in proto.messages %}
+ {%- for l in m.links %}
+ {{ m.fqn }} -> {{ l.peer.name }};
+ {%- endfor %}
+{% endfor %}
+}
diff --git a/lib/xos-genx/xosgenx/targets/grpc_api.xtarget b/lib/xos-genx/xosgenx/targets/grpc_api.xtarget
new file mode 100644
index 0000000..a0373a3
--- /dev/null
+++ b/lib/xos-genx/xosgenx/targets/grpc_api.xtarget
@@ -0,0 +1,58 @@
+import base64
+import time
+from protos import xos_pb2
+from google.protobuf.empty_pb2 import Empty
+
+from django.contrib.auth import authenticate as django_authenticate
+from xos.exceptions import *
+from apihelper import XOSAPIHelperMixin, translate_exceptions
+
+class XosService(xos_pb2.xosServicer, XOSAPIHelperMixin):
+ def __init__(self, thread_pool):
+ self.thread_pool = thread_pool
+ XOSAPIHelperMixin.__init__(self)
+
+ def stop(self):
+ pass
+
+{% for object in proto.messages | sort(attribute='name') %}
+{%- if object.name!='XOSBase' %}
+ @translate_exceptions
+ def List{{ object.name }}(self, request, context):
+ user=self.authenticate(context)
+ model=self.get_model("{{ object.name }}")
+ return self.querysetToProto(model, model.objects.all())
+
+ @translate_exceptions
+ def Filter{{ object.name }}(self, request, context):
+ user=self.authenticate(context)
+ model=self.get_model("{{ object.name }}")
+ return self.filter(model, request)
+
+ @translate_exceptions
+ def Get{{ object.name }}(self, request, context):
+ user=self.authenticate(context)
+ model=self.get_model("{{ object.name }}")
+ return self.get(model, request.id)
+
+ @translate_exceptions
+ def Create{{ object.name }}(self, request, context):
+ user=self.authenticate(context)
+ model=self.get_model("{{ object.name }}")
+ return self.create(model, user, request)
+
+ @translate_exceptions
+ def Delete{{ object.name }}(self, request, context):
+ user=self.authenticate(context)
+ model=self.get_model("{{ object.name }}")
+ return self.delete(model, user, request.id)
+
+ @translate_exceptions
+ def Update{{ object.name }}(self, request, context):
+ user=self.authenticate(context)
+ model=self.get_model("{{ object.name }}")
+ return self.update(model, user, request.id, request, context)
+{%- endif %}
+{% endfor %}
+
+
diff --git a/lib/xos-genx/xosgenx/targets/grpc_list_test.xtarget b/lib/xos-genx/xosgenx/targets/grpc_list_test.xtarget
new file mode 100644
index 0000000..e968959
--- /dev/null
+++ b/lib/xos-genx/xosgenx/targets/grpc_list_test.xtarget
@@ -0,0 +1,49 @@
+import grpc_client
+from grpc_client import Empty
+from testconfig import *
+
+c=grpc_client.InsecureClient("xos-core.cord.lab")
+
+{% for object in proto.messages %}
+{%- if object.name!='XOSBase' %}
+print "testing insecure List{{ object.name }}...",
+c.stub.List{{ object.name }}(Empty())
+print "Okay"
+{%- endif %}
+{%- endfor %}
+
+c=grpc_client.SecureClient("xos-core.cord.lab", username=USERNAME, password=PASSWORD)
+
+{% for object in proto.messages %}
+{%- if object.name!='XOSBase' %}
+print "testing basic secure List{{ object.name }}...",
+c.stub.List{{ object.name }}(Empty())
+print "Okay"
+{%- endif %}
+{%- endfor %}
+
+# now try to login
+c=grpc_client.InsecureClient("xos-core.cord.lab")
+lr=grpc_client.LoginRequest()
+lr.username=USERNAME
+lr.password=PASSWORD
+session=c.utility.Login(lr)
+
+c=grpc_client.SecureClient("xos-core.cord.lab", sessionid=session.sessionid)
+{% for object in proto.messages %}
+{%- if object.name!='XOSBase' %}
+print "testing session secure List{{ object.name }}...",
+c.stub.List{{ object.name }}(Empty())
+print "Okay"
+{%- endif %}
+{%- endfor %}
+
+c=grpc_client.SecureClient("xos-core.cord.lab", sessionid=session.sessionid)
+{% for object in proto.messages %}
+{%- if object.name!='XOSBase' %}
+print "testing session secure xos_orm.{{ object.name }}.objects.all() ...",
+c.xos_orm.{{ object.name }}.objects.all()
+print "Okay"
+{%- endif %}
+{%- endfor %}
+
diff --git a/lib/xos-genx/xosgenx/targets/init.xtarget b/lib/xos-genx/xosgenx/targets/init.xtarget
new file mode 100644
index 0000000..235a657
--- /dev/null
+++ b/lib/xos-genx/xosgenx/targets/init.xtarget
@@ -0,0 +1,15 @@
+# The hardcoded entries cannot be inferred from the models. To get rid of in the long run
+#
+
+from .xosbase import XOSBase,XOSBaseManager,XOSBaseDeletionManager,PlModelMixIn,ModelLink
+from .contenttype import ContentType
+from .site import Site
+from .dashboardview import DashboardView
+from .user import User
+from .user import UserDashboardView
+
+{% for m in proto.messages -%}
+{% if not m.options.skip_init -%}
+from .{{ m.name | lower }} import {{ m.name }}
+{% endif -%}
+{% endfor -%}
diff --git a/lib/xos-genx/xosgenx/targets/link-graph.xtarget b/lib/xos-genx/xosgenx/targets/link-graph.xtarget
new file mode 100644
index 0000000..04c9dba
--- /dev/null
+++ b/lib/xos-genx/xosgenx/targets/link-graph.xtarget
@@ -0,0 +1,14 @@
+{
+{%- for model in proto.messages %}
+{%- if model.links %}
+ "{{ model.name }}": [
+ {% for l in model.links -%}
+ ["{{ l.peer }}", "{{ l.src_port }}", "{{ l.dst_port }}"]{% if not loop.last %},{% endif %}
+ {%- endfor %}
+ {%- if model.rlinks %},{% endif %}
+ {% for l in model.rlinks -%}
+ ["{{ l.peer }}", "{{ l.src_port }}", "{{ l.dst_port }}"]{% if not loop.last %},{% endif %}
+ {%- endfor %}
+ ],{% endif -%}
+{% endfor %}
+}
diff --git a/lib/xos-genx/xosgenx/targets/model-deps-graphviz.xtarget b/lib/xos-genx/xosgenx/targets/model-deps-graphviz.xtarget
new file mode 100644
index 0000000..b855a4a
--- /dev/null
+++ b/lib/xos-genx/xosgenx/targets/model-deps-graphviz.xtarget
@@ -0,0 +1,7 @@
+digraph {{ context.app_label }} {
+{%- for model in proto.messages %}
+ {%- for l in model.links %}
+ "{{ model.name }}" -> "{{ l.peer }}" [label="{{ l.src_port }}"];
+ {%- endfor %}
+{%- endfor %}
+}
diff --git a/lib/xos-genx/xosgenx/targets/model-deps.xtarget b/lib/xos-genx/xosgenx/targets/model-deps.xtarget
new file mode 100644
index 0000000..65a3742
--- /dev/null
+++ b/lib/xos-genx/xosgenx/targets/model-deps.xtarget
@@ -0,0 +1,8 @@
+{
+{%- for model in proto.messages %}
+{%- if model.links %}
+ "{{ model.name }}": [
+ {{ model.links | map(attribute='peer') | format_list("\"%s\"") |join(',\n\t\t') }}
+ ],{% endif -%}
+{% endfor %}
+}
diff --git a/lib/xos-genx/xosgenx/targets/modeldefs.xtarget b/lib/xos-genx/xosgenx/targets/modeldefs.xtarget
new file mode 100644
index 0000000..615a8cc
--- /dev/null
+++ b/lib/xos-genx/xosgenx/targets/modeldefs.xtarget
@@ -0,0 +1,37 @@
+items:
+{%- for m in proto.messages | sort(attribute='name') %}
+{%- if m.name != 'XOSBase' %}
+- app: {{ xproto_unquote(xproto_first_non_empty([m.options.app_label, context.app_label, options.app_label])) }}
+ fields:
+ {%- set id_field = {'type':'int32', 'name':'id', 'options':{}} %}
+ {% for f in (xproto_base_fields(m, proto.message_table) + m.fields + [id_field]) | sort(attribute='name') -%}
+ {% if not f.link or f.options.link_type != 'manytomany' -%}
+ - hint: {% if f.options.help_text %}{{ xproto_unquote(f.options.help_text) }}{% else %}''{% endif %}
+ {% if not f.link -%}
+ name: {{ f.name }}
+ {%- else -%}
+ name: {{ f.name }}_id
+ relation: {model: {{ f.options.model }}, type: {{ f.options.link_type }}}
+ {% endif %}
+ type: {{ xproto_type_to_ui_type(f) }}
+ {% set validators = xproto_validators(f) -%}
+ {% if validators -%}
+ validators:
+ {% for v in validators | sort(attribute='name',reverse=True) -%}
+ - {{ v | yaml }}
+ {% endfor %}
+ {% else -%}
+ validators: []
+ {% endif %}
+ {% endif -%}
+ {% endfor %}
+ name: {{ m.name }}
+ {%- set goodlinks = xproto_links_to_modeldef_relations( xproto_base_links(m, proto.message_table) + m.links ) %}
+ {% if goodlinks %}
+ relations:
+ {{ goodlinks | join('\n') | indent(width=2)}}
+ {%- else %}
+ relations: []
+ {%- endif %}
+{%- endif %}
+{% endfor -%}
diff --git a/lib/xos-genx/xosgenx/targets/plurals.xtarget b/lib/xos-genx/xosgenx/targets/plurals.xtarget
new file mode 100644
index 0000000..9cdd6c6
--- /dev/null
+++ b/lib/xos-genx/xosgenx/targets/plurals.xtarget
@@ -0,0 +1,6 @@
+{% for m in proto.messages %}
+Model {{ m.name }}:
+{% for f in m.fields -%}
+The plural of {{ f.name}} is {{ xproto_pluralize(f) }}.
+{% endfor %}
+{%- endfor %}
diff --git a/lib/xos-genx/xosgenx/targets/proto.xtarget b/lib/xos-genx/xosgenx/targets/proto.xtarget
new file mode 100644
index 0000000..27b9490
--- /dev/null
+++ b/lib/xos-genx/xosgenx/targets/proto.xtarget
@@ -0,0 +1,8 @@
+{% for m in proto.messages %}
+message {{ m.name }} {
+ option bases = "{{ m.bases | map(attribute='name') | join(",") }}";
+ {%- for f in m.fields %}
+ {{ f.modifier }} {{f.type}} {{f.name}} = {{ f.id }}{% if f.options %} [{% for k,v in f.options.iteritems() %} {{ k }} = "{{ v}}"{% if not loop.last %},{% endif %} {% endfor %}]{% endif %};
+ {%- endfor %}
+}
+{% endfor %}
diff --git a/lib/xos-genx/xosgenx/targets/protoapi.xtarget b/lib/xos-genx/xosgenx/targets/protoapi.xtarget
new file mode 100644
index 0000000..f375dae
--- /dev/null
+++ b/lib/xos-genx/xosgenx/targets/protoapi.xtarget
@@ -0,0 +1,88 @@
+syntax = "proto3";
+
+package xos;
+
+import "google/protobuf/empty.proto";
+import "google/api/annotations.proto";
+import "common.proto";
+import "xosoptions.proto";
+
+// Note: all fields are wrapped in a "oneof". This causes proto3 to always send
+// fields that are set by the caller, regardless if they are set to a default
+// value. XOS uses this to know when to apply a default value.
+
+{% for object in proto.messages|sort(attribute='name') %}
+{% if object.name != 'XOSBase' -%}
+message {{ object.name }} {
+ {%- if object.name=='CordSubscriberRoot' %}
+ option (contentTypeId) = "rcord.{{ object.name | lower }}";
+ {%- else %}
+ option (contentTypeId) = "{{ xproto_unquote(xproto_first_non_empty([object.options.name, object.options.app_label, options.name, context.app_label])) }}.{{ object.name | lower }}";
+ {%- endif %}
+ {%- set id_field = {'type':'int32', 'name':'id', 'options':{}} -%}
+ {%- for field in (xproto_base_fields(object, proto.message_table) + object.fields + [id_field]) | sort(attribute='name')%}
+ oneof {{ field.name }}_present {
+ {{ xproto_api_type(field) }} {{ field.name }}{% if field.link -%}_id{% endif %} = {{ loop.index }}{{ xproto_api_opts(field) }};
+ }
+ {%- endfor -%}
+
+ {%- for ref in xproto_base_rlinks(object, proto.message_table) + object.rlinks | sort(attribute='src_port') %}
+ {%- if '+' not in ref.src_port and '+' not in ref.dst_port %}
+ repeated int32 {{ ref.src_port }}_ids = {{ loop.index + 100 }} [(reverseForeignKey).modelName = "{{ ref.peer.name }}"];
+ {%- endif -%}
+ {%- endfor %}
+ string class_names = 201;
+ string self_content_type_id = 202;
+}
+
+message {{ xproto_pluralize(object) }} {
+ repeated {{ object.name }} items = 1;
+}
+
+{%- endif %}
+{% endfor %}
+
+service xos {
+{% for object in proto.messages | sort(attribute='name')%}
+{% if object.name != 'XOSBase' -%}
+ rpc List{{ object.name }}(google.protobuf.Empty) returns ({{ xproto_pluralize(object) }}) {
+ option (google.api.http) = {
+ {%- if object.name=='CordSubscriberRoot' %}
+ get: "/xosapi/v1/rcord/{{ xproto_pluralize(object) | lower }}"
+ {%- else %}
+ get: "/xosapi/v1/{{ xproto_unquote(xproto_first_non_empty([object.options.name, object.options.app_label, options.name, context.app_label])) }}/{{ xproto_pluralize(object) | lower }}"
+ {%- endif %}
+ };
+ }
+ rpc Filter{{ object.name }}(Query) returns ({{ xproto_pluralize(object) }}) {
+ }
+ rpc Get{{ object.name }}(ID) returns ({{ object.name }}) {
+ option (google.api.http) = {
+ {%- if object.name=='CordSubscriberRoot' %}
+ get: "/xosapi/v1/rcord/{{ xproto_pluralize(object) | lower }}/{id}"
+ {%- else %}
+ get: "/xosapi/v1/{{ xproto_unquote(xproto_first_non_empty([object.options.name, object.options.app_label, options.name, context.app_label])) }}/{{ xproto_pluralize(object) | lower }}/{id}"
+ {%- endif %}
+ };
+ }
+ rpc Create{{ object.name }}({{ object.name }}) returns ({{ object.name }}) {
+ option (google.api.http) = {
+ post: "/xosapi/v1/{{ xproto_unquote(xproto_first_non_empty([object.options.name, object.options.app_label, options.name, context.app_label])) }}/{{ xproto_pluralize(object) | lower }}"
+ body: "*"
+ };
+ }
+ rpc Update{{ object.name }}({{ object.name }}) returns ({{ object.name }}) {
+ option (google.api.http) = {
+ put: "/xosapi/v1/{{ xproto_unquote(xproto_first_non_empty([object.options.name, object.options.app_label, options.name, context.app_label])) }}/{{ xproto_pluralize(object) | lower }}/{id}"
+ body: "*"
+ };
+ }
+ rpc Delete{{ object.name }}(ID) returns (google.protobuf.Empty) {
+ option (google.api.http) = {
+ delete: "/xosapi/v1/{{ xproto_unquote(xproto_first_non_empty([object.options.name, object.options.app_label, options.name, context.app_label])) }}/{{ xproto_pluralize(object) | lower }}/{id}"
+ };
+ }
+{%- endif %}
+{% endfor %}
+}
+
diff --git a/lib/xos-genx/xosgenx/targets/service.xtarget b/lib/xos-genx/xosgenx/targets/service.xtarget
new file mode 100644
index 0000000..1ba3cff
--- /dev/null
+++ b/lib/xos-genx/xosgenx/targets/service.xtarget
@@ -0,0 +1,56 @@
+{% if options.legacy =='"True"' -%}
+{% set legacy_tag = '_decl' %}
+{% set legacy = True %}
+from core.models.xosbase import *
+{% else %}
+{% set legacy = False %}
+{% set legacy_tag = '' %}
+from header import *
+{% endif %}
+
+{% for m in proto.messages %}
+{% if file_exists(m.name|lower+'_header.py') -%}from {{m.name|lower }}_header import *{% endif %}
+{% if file_exists(m.name|lower+'_top.py') -%}{{ include_file(m.name|lower+'_top.py') }} {% endif %}
+
+{%- for l in m.links -%}{% set peer_name=l.peer.name %}
+{% if peer_name not in proto.message_names -%}
+from core.models import {{ peer_name }}
+{%- endif -%}
+{%- endfor -%}
+{%- for b in m.bases -%}
+{%- if b.name!='XOSBase' and 'Mixin' not in b.name %}
+from core.models import {{ b.name }}
+{%- endif -%}
+{% endfor %}
+
+{% endfor %}
+
+{% for m in proto.messages %}
+class {{ m.name }}{{ legacy_tag }}{{ xproto_base_def(m.name, m.bases) }}:
+
+ KIND = {{ xproto_first_non_empty([m.options.kind, options.kind, options.name, "Set a kind in your xproto!"]) }}
+
+ class Meta:
+ app_label = {{ xproto_first_non_empty([m.options.app_label, options.app_label, options.name, "Set an app label in your xproto!"]) | lower}}
+ # name = {{ xproto_first_non_empty([m.options.name, options.name, "Set a name in your xproto!"]) }}
+ verbose_name = {{ xproto_first_non_empty([m.options.verbose_name, options.verbose_name, "Set a verbose name in your xproto!"]) }}
+
+ # Primitive Fields (Not Relations)
+ {% for f in m.fields %}
+ {%- if not f.link -%}
+ {{ f.name }} = {{ xproto_django_type(f.type, f.options) }}( {{ xproto_django_options_str(f) }} )
+ {% endif %}
+ {%- endfor %}
+
+ # Relations
+ {% for l in m.links %}{% set peer_name=l.peer.name %}
+ {% if legacy and peer_name in proto.message_names %}{% set peer_tag = legacy_tag %}{% else %}{% set peer_tag = '' %}{% endif -%}
+ {{ l.src_port }} = {{ xproto_django_link_type(l) }}( {%- if peer_name==m.name -%}'self'{%- else -%}{{ peer_name }}{{ peer_tag }} {%- endif -%}, {{ xproto_django_options_str(l, l.dst_port ) }} )
+ {%- endfor %}
+
+ {% if file_exists(m.name|lower + '_model.py') -%}{{ include_file(m.name|lower + '_model.py') | indent(width=2)}}{%- endif %}
+ pass
+
+{% if file_exists(m.name|lower+'_bottom.py') -%}{{ include_file(m.name|lower+'_bottom.py') }}{% endif %}
+{% endfor %}
++++ models{{ legacy_tag }}.py
\ No newline at end of file
diff --git a/lib/xos-genx/xosgenx/targets/service_extender.xtarget b/lib/xos-genx/xosgenx/targets/service_extender.xtarget
new file mode 100644
index 0000000..5b4bc03
--- /dev/null
+++ b/lib/xos-genx/xosgenx/targets/service_extender.xtarget
@@ -0,0 +1,20 @@
+from core.models.xosbase import *
+{% for m in proto.messages %}
+from models_decl import {{ m.name }}_decl
+{%- endfor %}
+
+{% for m in proto.messages %}
+{% for l in m.links %}{% set peer_name=l.peer.name -%}
+{% if peer_name not in proto.message_names -%}
+from core.models import {{ peer_name }}
+{% endif -%}
+{% endfor -%}
+{% endfor -%}
+
+{% for m in proto.messages %}
+class {{ m.name }}({{ m.name }}_decl):
+ class Meta:
+ proxy = True
+
+{% endfor %}
++++ models.py
diff --git a/lib/xos-genx/xosgenx/targets/tosca.xtarget b/lib/xos-genx/xosgenx/targets/tosca.xtarget
new file mode 100644
index 0000000..b5e51ae
--- /dev/null
+++ b/lib/xos-genx/xosgenx/targets/tosca.xtarget
@@ -0,0 +1,31 @@
+tosca_definitions_version: tosca_simple_yaml_1_0
+
+node_types:
+{% for m in proto.messages %}
+ tosca.nodes.{{ m.name }}:
+ derived_from: tosca.nodes.Root
+ description: {{ m.name }}
+ properties:
+ no-delete:
+ type: boolean
+ default: false
+ description: Do not allow Tosca to delete this object
+ no-create:
+ type: boolean
+ default: false
+ description: Do not allow Tosca to create this object
+ no-update:
+ type: boolean
+ default: false
+ description: Do not allow Tosca to update this object
+ replaces:
+ type: string
+ required: false
+ descrption: Replaces/renames this object
+ {%- for f in m.fields %}
+ {{ f.name }}:
+ type: {{ f.type }}
+ required: {{ xproto_tosca_required(f.options.blank) }}
+ description: {{ f.options.help_text }}
+ {%- endfor %}
+{%- endfor %}
\ No newline at end of file
diff --git a/lib/xos-genx/xosgenx/targets/xproto.xtarget b/lib/xos-genx/xosgenx/targets/xproto.xtarget
new file mode 100644
index 0000000..513d7a7
--- /dev/null
+++ b/lib/xos-genx/xosgenx/targets/xproto.xtarget
@@ -0,0 +1,18 @@
+{% for k,v in options.iteritems() %}
+option {{ k }} = {{ v }};
+{% endfor %}
+
+{% for m in proto.messages %}
+message {{ m.name }} ({{ m.bases | join(",") }}){
+ {%- for f in m.fields %}
+ {% if not f.link %}
+ {{ f.modifier }} {{f.type}} {{f.name}} = {{ f.id }}{% if f.options %} [{% for k,v in f.options.iteritems() %} {{ k }} = {{ v }}{% if not loop.last %},{% endif %} {% endfor %}]{% endif %};
+ {% endif %}
+ {%- endfor %}
+
+ {%- for l in m.links %}{% set peer_name=l.peer.name %}
+ {{ l }}
+ {{ l.modifier }} {{ l.link_type }} {{ l.name }} -> {{ l.peer.name }}:{{ l.dst_port }} = 1 {% if l.options %} [{% for k,v in l.options.iteritems() %} {{ k }} = {{ v }}{% if not loop.last %},{% endif %} {% endfor %}]{% endif %};
+ {%- endfor -%}
+}
+{% endfor %}
diff --git a/lib/xos-genx/xosgenx/xos2jinja.py b/lib/xos-genx/xosgenx/xos2jinja.py
new file mode 100644
index 0000000..929526d
--- /dev/null
+++ b/lib/xos-genx/xosgenx/xos2jinja.py
@@ -0,0 +1,355 @@
+import plyxproto.model as m
+import pdb
+import argparse
+import plyxproto.parser as plyxproto
+import traceback
+import sys
+import jinja2
+import os
+import copy
+
+
+def dotname_to_fqn(dotname):
+ b_names = [part.pval for part in dotname]
+ package = '.'.join(b_names[:-1])
+ name = b_names[-1]
+ if package:
+ fqn = package + '.' + name
+ else:
+ fqn = name
+ return {'name': name, 'fqn': fqn, 'package': package}
+
+
+def dotname_to_name(dotname):
+ b_names = [part.pval for part in dotname]
+ return '.'.join(b_names)
+
+
+def count_messages(body):
+ count = 0
+ for e in body:
+ if (type(e) == m.MessageDefinition):
+ count += 1
+ return count
+
+
+def count_fields(body):
+ count = 0
+ for e in body:
+ if (type(e) in [m.LinkDefinition, m.FieldDefinition, m.LinkSpec]):
+ count += 1
+ return count
+
+def compute_rlinks(messages, message_dict):
+ rev_links = {}
+
+ link_opposite = {
+ 'manytomany': 'manytomany',
+ 'manytoone': 'onetomany',
+ 'onetoone': 'onetoone',
+ 'onetomany': 'manytoone'
+ }
+
+ for m in messages:
+ for l in m['links']:
+ rlink = copy.deepcopy(l)
+
+ rlink['_type'] = 'rlink' # An implicit link, not declared in the model
+ rlink['src_port'] = l['dst_port']
+ rlink['dst_port'] = l['src_port']
+ rlink['peer'] = {'name': m['name'], 'package': m['package'], 'fqn': m['fqn']}
+ rlink['link_type'] = link_opposite[l['link_type']]
+
+ try:
+ try:
+ rev_links[l['peer']['fqn']].append(rlink)
+ except TypeError:
+ pdb.set_trace()
+ pass
+ except KeyError:
+ rev_links[l['peer']['fqn']] = [rlink]
+
+ for m in messages:
+ try:
+ m['rlinks'] = rev_links[m['name']]
+ message_dict[m['name']]['rlinks'] = m['rlinks']
+ except KeyError:
+ pass
+
+
+def name_to_value(obj):
+ try:
+ value = obj.value.value.pval
+ except AttributeError:
+ try:
+ value = obj.value.value
+ except AttributeError:
+ value = obj.value.pval
+
+ return value
+
+
+class Stack(list):
+ def push(self, x):
+ self.append(x)
+
+
+''' XOS2Jinja overrides the underlying visitor pattern to transform the tree
+ in addition to traversing it '''
+
+
+class XOS2Jinja(m.Visitor):
+
+ def get_stack(self):
+ return self.stack
+
+ def __init__(self):
+ super(XOS2Jinja, self).__init__()
+
+ self.stack = Stack()
+ self.models = {}
+ self.options = {}
+ self.package = None
+ self.message_options = {}
+ self.count_stack = Stack()
+ self.content = ""
+ self.offset = 0
+ self.current_message_name = None
+ self.verbose = 0
+ self.first_field = True
+ self.first_method = True
+
+ def visit_PackageStatement(self, obj):
+ dotlist = obj.name.value
+ dotlist2 = [f.pval for f in dotlist]
+ dotstr = '.'.join(dotlist2)
+ self.package = dotstr
+ return True
+
+ def visit_ImportStatement(self, obj):
+ '''Ignore'''
+ return True
+
+ def visit_OptionStatement(self, obj):
+ if not hasattr(obj, 'mark_for_deletion'):
+ if (self.current_message_name):
+ self.message_options[obj.name.value.pval] = obj.value.value.pval
+ else:
+ self.options[obj.name.value.pval] = obj.value.value.pval
+
+ return True
+
+ def visit_LU(self, obj):
+ return True
+
+ def visit_default(self, obj):
+ return True
+
+ def visit_FieldDirective(self, obj):
+ return True
+
+ def visit_FieldDirective_post(self, obj):
+
+ try:
+ name = obj.name.value.pval
+ except AttributeError:
+ name = obj.name.value
+
+ if type(obj.value) == list:
+ value = dotname_to_name(obj.value)
+ else:
+ value = name_to_value(obj)
+
+ self.stack.push([name, value])
+ return True
+
+ def visit_FieldType(self, obj):
+ '''Field type, if type is name, then it may need refactoring consistent with refactoring rules according to the table'''
+ return True
+
+ def visit_LinkDefinition(self, obj):
+ s = {}
+
+ try:
+ s['link_type'] = obj.link_type.pval
+ except AttributeError:
+ s['link_type'] = obj.link_type
+
+ s['src_port'] = obj.src_port.value.pval
+ s['name'] = obj.src_port.value.pval
+
+ try:
+ s['dst_port'] = obj.dst_port.value.pval
+ except AttributeError:
+ s['dst_port'] = obj.dst_port
+
+ if type(obj.through) == list:
+ s['through'] = dotname_to_fqn(obj.through)
+ else:
+ try:
+ s['through'] = obj.through.pval
+ except AttributeError:
+ s['through'] = obj.through
+
+ if type(obj.name) == list:
+ s['peer'] = dotname_to_fqn(obj.name)
+ else:
+ try:
+ s['peer'] = obj.name.pval
+ except AttributeError:
+ s['peer'] = obj.name
+
+ s['_type'] = 'link'
+ s['options'] = {'modifier': 'optional'}
+
+ self.stack.push(s)
+ return True
+
+ def visit_FieldDefinition(self, obj):
+ self.count_stack.push(len(obj.fieldDirective))
+ return True
+
+ def visit_FieldDefinition_post(self, obj):
+ s = {}
+
+ if isinstance(obj.ftype, m.Name):
+ s['type'] = obj.ftype.value
+ else:
+ s['type'] = obj.ftype.name.pval
+
+ s['name'] = obj.name.value.pval
+ s['modifier'] = obj.field_modifier.pval
+ s['id'] = obj.fieldId.pval
+
+ opts = {'modifier': s['modifier']}
+ n = self.count_stack.pop()
+ for i in range(0, n):
+ k, v = self.stack.pop()
+
+ # The two lines below may be added to eliminate "" around an option.
+ # Right now, this is handled in targets. FIXME
+ #
+ # if (v.startswith('"') and v.endswith('"')):
+ # v = v[1:-1]
+
+ opts[k] = v
+
+ s['options'] = opts
+ try:
+ last_link = self.stack[-1]['_type']
+ if (last_link == 'link'):
+ s['link'] = True
+ except:
+ pass
+ s['_type'] = 'field'
+
+ self.stack.push(s)
+ return True
+
+ def visit_EnumFieldDefinition(self, obj):
+ if self.verbose > 4:
+ print "\tEnumField: name=%s, %s" % (obj.name, obj)
+
+ return True
+
+ def visit_EnumDefinition(self, obj):
+ '''New enum definition, refactor name'''
+ if self.verbose > 3:
+ print "Enum, [%s] body=%s\n\n" % (obj.name, obj.body)
+
+ return True
+
+ def visit_MessageDefinition(self, obj):
+ self.current_message_name = obj.name.value.pval
+ self.message_options = {}
+ self.count_stack.push(count_fields(obj.body))
+ return True
+
+ def visit_MessageDefinition_post(self, obj):
+ stack_num = self.count_stack.pop()
+ fields = []
+ links = []
+ last_field = None
+ try:
+ obj.bases = map(dotname_to_fqn, obj.bases)
+ except AttributeError:
+ pass
+
+ last_field = {}
+ for i in range(0, stack_num):
+ f = self.stack.pop()
+ if (f['_type'] == 'link'):
+ f['options'] = {i: d[i] for d in [f['options'], last_field['options']] for i in d}
+ assert (last_field == fields[0])
+ fields[0].setdefault('options', {})['link_type'] = f['link_type']
+ links.insert(0, f)
+ else:
+ fields.insert(0, f)
+ last_field = f
+
+ if self.package:
+ model_name = '.'.join([self.package, obj.name.value.pval])
+ else:
+ model_name = obj.name.value.pval
+
+ model_def = {'name':obj.name.value.pval,'fields':fields,'links':links, 'bases':obj.bases, 'options':self.message_options, 'package':self.package, 'fqn': model_name, 'rlinks': []}
+ self.stack.push(model_def)
+
+ self.models[model_name] = model_def
+
+ # Set message options
+ for k, v in self.options.iteritems():
+ try:
+ if k not in self.message_options:
+ self.message_options[k] = v
+ except KeyError:
+ pass
+
+ self.current_message_name = None
+ return True
+
+ def visit_MessageExtension(self, obj):
+ return True
+
+ def visit_MethodDefinition(self, obj):
+ return True
+
+ def visit_ServiceDefinition(self, obj):
+ return True
+
+ def visit_ExtensionsDirective(self, obj):
+ return True
+
+ def visit_Literal(self, obj):
+ return True
+
+ def visit_Name(self, obj):
+ return True
+
+ def visit_DotName(self, obj):
+ return True
+
+ def visit_Proto(self, obj):
+ self.count_stack.push(count_messages(obj.body))
+ return True
+
+ def visit_Proto_post(self, obj):
+ count = self.count_stack.pop()
+ messages = []
+ for i in range(0, count):
+ try:
+ m = self.stack.pop()
+ except IndexError:
+ pass
+
+ messages.insert(0, m)
+
+ compute_rlinks(messages, self.models)
+
+ self.messages = messages
+ return True
+
+ def visit_LinkSpec(self, obj):
+ count = self.count_stack.pop()
+ self.count_stack.push(count + 1)
+ return True
\ No newline at end of file
diff --git a/lib/xos-genx/xosgenx/xosgen.py b/lib/xos-genx/xosgenx/xosgen.py
new file mode 100755
index 0000000..349ca57
--- /dev/null
+++ b/lib/xos-genx/xosgenx/xosgen.py
@@ -0,0 +1,66 @@
+#!/usr/bin/python
+
+import argparse
+from generator import *
+
+parse = argparse.ArgumentParser(description='XOS Generative Toolchain')
+parse.add_argument('--rev', dest='rev', action='store_true',default=False, help='Convert proto to xproto')
+parse.add_argument('--target', dest='target', action='store',default=None, help='Output format, corresponding to <output>.yaml file', required=True)
+parse.add_argument('--output', dest='output', action='store',default=None, help='Destination dir')
+parse.add_argument('--attic', dest='attic', action='store',default=None, help='The location at which static files are stored')
+parse.add_argument('--kvpairs', dest='kv', action='store',default=None, help='Key value pairs to make available to the target')
+parse.add_argument('--write-to-file', dest='write_to_file', choices = ['single', 'model', 'target'], action='store',default=None, help='Single output file (single) or output file per model (model) or let target decide (target)')
+
+group = parse.add_mutually_exclusive_group()
+group.add_argument('--dest-file', dest='dest_file', action='store',default=None, help='Output file name (if write-to-file is set to single)')
+group.add_argument('--dest-extension', dest='dest_extension', action='store',default=None, help='Output file extension (if write-to-file is set to single)')
+
+parse.add_argument('files', metavar='<input file>', nargs='+', action='store', help='xproto files to compile')
+
+class XosGen:
+
+ @staticmethod
+ def init(args=None):
+
+ if not args:
+ args = parse.parse_args()
+
+ args.quiet = False
+
+ # convert output to absolute path
+ if args.output is not None and not os.path.isabs(args.output):
+ args.output = os.path.abspath(os.getcwd() + '/' + args.output)
+ if not '/' in args.target:
+ # if the target is not a path, it refer to a library included one
+ args.target = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/targets/" + args.target)
+ if not os.path.isabs(args.target):
+ args.target = os.path.abspath(os.getcwd() + '/' + args.target)
+
+ # check if there's a line that starts with +++ in the target
+ # if so, then the output file names are left to the target to decide
+ # also, if dest-file or dest-extension are supplied, then an error is generated.
+ plusplusplus = reduce(lambda acc, line: True if line.startswith('+++') else acc, open(args.target).read().splitlines(), False)
+
+ if plusplusplus and args.write_to_file != 'target':
+ parse.error('%s chooses the names of the files that it generates, you must set --write-to-file to "target"' % args.target)
+
+
+ if args.write_to_file != 'single' and (args.dest_file):
+ parse.error('--dest-file requires --write-to-file to be set to "single"')
+
+ if args.write_to_file != 'model' and (args.dest_extension):
+ parse.error('--dest-extension requires --write-to-file to be set to "model"')
+
+ inputs = []
+
+ for fname in args.files:
+ if not os.path.isabs(fname):
+ inputs.append(os.path.abspath(os.getcwd() + '/' + fname))
+ else:
+ inputs.append(fname)
+ args.files = inputs
+
+ generated = XOSGenerator.generate(args)
+
+ if not args.output and not args.write_to_file:
+ print generated
\ No newline at end of file