CORD-1026: Tool changes needed to generate protobufs that cover all of the
information in Django modeling
Change-Id: I19d5e70ed1693fd6e3cb7db71b06ed89b03008e7
diff --git a/xos/tools/apigen/lib.py b/xos/tools/apigen/lib.py
new file mode 100644
index 0000000..70d0ac4
--- /dev/null
+++ b/xos/tools/apigen/lib.py
@@ -0,0 +1,123 @@
+import pdb
+
+def format_options_string(d):
+ if (not d): return ''
+ lst = []
+ try:
+ for k,v in d.items():
+ if (type(v)==int or type(v)==bool or type(v)==float):
+ lst.append('%s = %r'%(k,v))
+ else:
+ lst.append('%s = "%s"'%(k,v))
+ except:
+ pass
+ return '['+', '.join(lst)+']'
+
+
+def map_django_to_xproto(f):
+ allowed_keys=['help_text','default','max_length','choices','blank','null','related_name','db_index']
+
+ m = {'help_text':{'_target':'help_text','':None}}
+ out = {}
+
+ for k in allowed_keys:
+ try:
+ v = getattr(f,k)
+ except:
+ continue
+
+ if (k=='choices' and v==[]):
+ continue
+
+ if (k=='default' and type(v)==float):
+ pass
+
+ if (k=='default' and 'function' in type(v).__name__):
+ v = v.__name__+"()"
+
+ try:
+ n = v.__name__
+ except AttributeError:
+ n = None
+ if (v is None or n=='NOT_PROVIDED'):
+ continue
+
+ if (k in allowed_keys):
+ try:
+ kv2 = m[k]
+ if (kv2[v] is None):
+ continue
+
+ out[kv2['_target']] = kv2[v]
+ except:
+ out[k] = v
+ return out
+
+
+def xp_options(field):
+ output_dict = map_django_to_xproto(field)
+ t0 = field.type
+
+ if (t0=='StrippedCharField'):
+ ctype = 'stripped'
+ elif (t0=='URLField'):
+ ctype = 'url'
+ elif (t0=='DateTimeField'):
+ ctype = 'date'
+ elif (t0=='GenericIPAddressField'):
+ ctype = 'ip'
+ else:
+ ctype = None
+
+ if (ctype):
+ output_dict['content_type'] = ctype
+
+
+ return format_options_string(output_dict)
+
+def xp_to_xproto(field, idx):
+ t = field.get_internal_type()
+ link = False
+
+ if (t=='CharField' or t=='TextField' or t=='SlugField'):
+ xptype = 'string'
+ elif (t=='BooleanField'):
+ xptype = 'bool'
+ elif (t=='ForeignKey'):
+ xptype = 'manytoone'
+ peer = field.related.model.__name__
+ if (field.related.name):
+ dst_port = ':' + field.related.name
+ else:
+ dst_port = ''
+ link = True
+ elif (t=='DateTimeField'):
+ xptype = 'string'
+ elif (t=='AutoField'):
+ xptype = 'int32'
+ elif (t=='BigIntegerField'):
+ xptype = 'int32'
+ elif (t=='IntegerField'):
+ xptype = 'int32'
+ elif (t=='PositiveIntegerField'):
+ xptype = 'uint32'
+ elif (t=='FloatField'):
+ xptype = 'float'
+ elif (t=='GenericIPAddressField'):
+ xptype = 'string'
+ elif (t=='OneToOneField'):
+ link = True
+
+
+ if (field.null==False):
+ modifier = 'required'
+ else:
+ modifier = 'optional'
+
+ if (link):
+ str = '%s %s %s->%s%s = %d'%(modifier, xptype, field.name, peer, dst_port, idx)
+ else:
+ str = '%s %s %s = %d'%(modifier, xptype, field.name, idx)
+
+ return str
+
diff --git a/xos/tools/apigen/modelgen2 b/xos/tools/apigen/modelgen2
new file mode 100755
index 0000000..25f4146
--- /dev/null
+++ b/xos/tools/apigen/modelgen2
@@ -0,0 +1,377 @@
+#!/usr/bin/python
+
+import inspect
+import os
+import pdb
+import copy
+import sys
+import json
+import re
+import jinja2
+from optparse import OptionParser
+import lib
+
+# Django set up
+
+import django
+sys.path.append('.')
+sys.path.append('/opt/xos')
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "xos.settings")
+from django.db.models.fields.related import ForeignKey, ManyToManyField
+from django.conf import settings
+from django.contrib.contenttypes.models import ContentType
+
+django.setup()
+
+from core.models import PlCoreBase
+
+options = None
+
+
+def is_model_class(model):
+ """ Return True if 'model' is something that we're interested in """
+ if not inspect.isclass(model):
+ return False
+ if model.__name__ in ["PlModelMixIn"]:
+ return False
+ bases = inspect.getmro(model)
+ bases = [x.__name__ for x in bases]
+ if ("PlCoreBase" in bases) or ("PlModelMixIn" in bases):
+ return True
+
+ return False
+
+def module_has_models(module):
+ """ return True if 'module' contains any models we're interested in """
+ for k in dir(module):
+ v=getattr(module,k)
+ if is_model_class(v):
+ return True
+
+ return False
+
+def app_get_models_module(app):
+ """ check whether 'app' includes XOS models """
+
+ app = app + ".models"
+ try:
+ models_module = __import__(app)
+ except ImportError:
+ return False
+
+ for part in app.split(".")[1:]:
+ if module_has_models(models_module):
+ return models_module
+ models_module = getattr(models_module,part)
+
+ if module_has_models(models_module):
+ return models_module
+
+ return None
+
+
+def singular(foo, keys):
+ for k in keys:
+ if (foo==k+'es'):
+ return k
+ elif (foo==k+'s'):
+ return k
+ raise Exception('Plural to singular error for %s'%foo)
+
+g = globals()
+
+def enum_classes(apps):
+ global app_map
+ global class_map
+ app_map = {}
+ class_map = {}
+ model_classes = []
+ for app in apps:
+ orig_app=app
+ models_module = app_get_models_module(app)
+
+ for classname in dir(models_module):
+ c = getattr(models_module, classname, None)
+
+ # For services, prevent loading of core models as it causes
+ # duplication.
+ if hasattr(c,"_meta") and hasattr(c._meta, "app_label"):
+ if (c._meta.app_label == "core") and (orig_app!="core"):
+ continue
+
+ if is_model_class(c) and c.__name__ not in options.blacklist:
+ model_classes.append(c)
+ app_map[c.__name__]=orig_app
+ c.class_name = c.__name__
+ file_name = c.__module__.rsplit('.',1)[1]
+ try:
+ if (file_name not in class_map[orig_app]):
+ class_map[orig_app].append({file_name:[c]})
+ else:
+ class_map[orig_app][file_name].append(c)
+
+ except KeyError:
+ class_map[orig_app] = [{file_name:[c]}]
+
+
+ return model_classes
+
+class GenObj(object):
+ def __str__(self):
+ return str(self.model.__name__.lower())
+
+ def __init__(self, m):
+ self.model = m
+ self.props = []
+ self.fields = []
+ self.all_fields = []
+ self.field_dict = []
+ self.refs = []
+ self.reverse_refs = []
+ self.plural_name = None
+ self.content_type_id = ContentType.objects.get_for_model(m).id
+
+ def plural(self):
+ if (self.plural_name):
+ return self.plural_name
+ else:
+ name = str(self)
+ if (name.endswith('s')):
+ return name+'es'
+ else:
+ return name+'s'
+
+ def singular(self):
+ return str(self)
+
+ def rest_name(self):
+ # These are things that either for historic reasons or due to incorrect naming,
+ # got called something different than the autogen thinks they should be
+ # called.
+ REST_FIXUP = {'controllernetworkses': 'controllernetworks',
+ 'controllerimageses': 'controllerimages',
+ 'controllersliceses': 'controllerslices',
+ 'controlleruserses': 'controllerusers',
+ 'sitedeploymentses': 'sitedeployments',
+ 'siteroles': 'site_roles',
+ 'sliceprivileges': 'slice_privileges',
+ 'sliceroles': 'slice_roles',
+ }
+ return REST_FIXUP.get(self.plural(), self.plural())
+
+ def camel(self):
+ name = str(self.model.__name__)
+ return name
+
+class Generator(dict):
+ def __init__(self):
+ self.apps = {}
+
+ def all(self):
+ return self.values()
+
+ def rest_models(self):
+ norest = [x.lower() for x in options.norest]
+ return [v for v in self.values() if not (str(v) in norest)]
+
+ def regex(self, r):
+ filtered = filter(lambda o:re.match(r,str(o)), self.values())
+ return filtered
+
+ def add_object(self, o):
+ global app_map
+ obj = GenObj(o)
+ fields = o._meta.fields
+ try:
+ obj.app = app_map[o.__name__] # full name
+ if hasattr(o, "_meta") and hasattr(o._meta, "app_label"):
+ obj.app_name = o._meta.app_label
+ else:
+ obj.app_name = app_map[o.__name__].split(".")[-1] # only the last part
+ except KeyError:
+ print "KeyError: %r"%o.__name__
+
+ obj.class_name = o.class_name
+
+ file_name = o.__module__.rsplit('.',1)[1]
+
+ try:
+ if (file_name not in self.apps[obj.app]):
+ self.apps[obj.app][file_name]=[obj]
+ else:
+ self.apps[obj.app][file_name].append(obj)
+
+ except KeyError:
+ self.apps[obj.app] = {file_name:[obj]}
+
+ self[str(obj).lower()]=obj
+
+ def compute_links(self):
+ base_props = [f.name for f in PlCoreBase._meta.fields] + ['id']
+
+ for obj in self.values():
+ #if (str(obj)=='network'):
+ # pdb.set_trace()
+ fields = obj.model._meta.fields
+ for f in fields:
+ if (f and f.rel):
+ to_name = str(f.rel.to)
+ else:
+ to_name = None
+
+ if type(f)==ForeignKey and to_name and to_name in self.keys():
+ refobj = self[f.to_name]
+
+ if (str(obj)=='slice' and f.to_name=='networks'):
+ obj.refs.append(refobj)
+ related_name = f.related_query_name()
+ if (related_name!='+' and related_name.lower()!=str(obj).lower()):
+ cobj = copy.deepcopy(obj)
+ cobj.multi = True
+ cobj.plural_name = related_name
+ refobj.refs.append(cobj)
+ elif f.name.endswith("_ptr"):
+ # django inherited model, for example HPCService
+ # cause swagger and REST to break
+ pass
+ else:
+ f.type = f.__class__.__name__
+ if (type(f)==ForeignKey):
+ f.related.model.class_name = f.related.model.__name__
+ if (f.name not in base_props):
+ obj.fields.append(f)
+ obj.all_fields.append(f)
+ obj.props.append(f.name)
+
+ m2m = obj.model._meta.many_to_many
+ for f in m2m:
+ try:
+ related_model_name = f.m2m_reverse_field_name()
+ except:
+ related_model_name = f.m2m_db_table().rsplit('_',1)[-1]
+
+ related_name = f.related_query_name()
+ if related_model_name in self.keys():
+ #print "XXX1", obj, f, related_name, related_model_name
+ refobj = self[related_model_name]
+ cobj = copy.deepcopy(obj)
+ cobj.multi=True
+ refobj.refs.append(cobj)
+
+ # deal with upgradeFrom_rel_+
+ if (related_name.endswith("+")):
+ continue
+
+ if (related_name!='+') and related_model_name in self: # and related_name.lower()!=str(obj).lower()):
+ refobj = self[related_model_name]
+ #print "XXX2", obj, f, related_name, related_model_name, refobj.plural_name
+ cobj = copy.deepcopy(refobj)
+ cobj.multi = True
+
+ obj.refs.append(cobj)
+
+ for obj in self.values():
+ # generate foreign key reverse references
+ for f in obj.model._meta.related_objects:
+ related_model = getattr(f, "related_model", None)
+ if not f.related_name:
+ continue
+ if "+" in f.related_name:
+ continue
+ if related_model and (related_model.__name__.lower() in self.keys()):
+ cobj = copy.deepcopy(self[related_model.__name__.lower()])
+ cobj.related_name = f.related_name
+ obj.reverse_refs.append(cobj)
+
+def main():
+ global options
+ parser = OptionParser(usage="modelgen [options] <template_fn>", )
+
+ parser.add_option("-d", "--dict", dest="dict",
+ help="dictionary to replace text in output", metavar="DICT", default=[], action="append")
+
+ parser.add_option("-a", "--app", dest="apps",
+ help="list of applications to parse", metavar="APP", default=[], action="append")
+ parser.add_option("-b", "--blacklist", dest="blacklist",
+ help="add model name to blacklist", metavar="MODEL", default=["SingletonModel", "PlCoreBase"], action="append")
+ parser.add_option("-n", "--no-rest", dest="norest",
+ help="do not generate rest api for model", metavar="MODEL", default=["SingletonModel", "PlCoreBase"], action="append")
+ parser.add_option("-l", "--local", dest="local",
+ help="skip base models", metavar="MODEL", default=False, action="store_true")
+ (options, args) = parser.parse_args(sys.argv[1:])
+
+ template_name = os.path.abspath(args[0])
+
+ # try to make sure we're running from the right place
+ if (not os.path.exists("core")):
+ if (os.path.exists("../core")):
+ os.chdir("..")
+ elif (os.path.exists("../../core")):
+ os.chdir("../..")
+ else:
+ print >> sys.stderr, "Are you sure you're running modelgen from the root of an XOS installation"
+ sys.exit(-1)
+
+ if not options.apps:
+ options.apps = ["core"]
+
+ if options.apps == ["*"]:
+ options.apps = [x for x in settings.INSTALLED_APPS if app_get_models_module(x)]
+
+ if len(args)!=1:
+ print 'Usage: modelgen [options] <template_fn>'
+ exit(1)
+
+ generator = Generator()
+
+ models = enum_classes(options.apps)
+
+ for m in models:
+ generator.add_object(m)
+
+ generator.compute_links()
+
+ os_template_loader = jinja2.FileSystemLoader( searchpath=[os.path.split(template_name)[0]])
+ os_template_env = jinja2.Environment(loader=os_template_loader)
+
+ for x in dir(lib):
+ if x.startswith('xp'):
+ os_template_env.globals[x] = getattr(lib, x)
+
+ template = os_template_env.get_template(os.path.split(template_name)[1])
+ rendered = template.render({"generator": generator})
+
+ lines = rendered.splitlines()
+ current_buffer = []
+ for l in lines:
+ if (l.startswith('+++')):
+ path = l[4:]
+
+ direc,filename = path.rsplit('/',1)
+ os.system('mkdir -p %s'%direc)
+
+ pdb.set_trace()
+ fil = open(path,'w')
+ buf = '\n'.join(current_buffer)
+
+ obuf = buf
+ for d in options.dict:
+ df = open(d).read()
+ d = json.loads(df)
+
+ pattern = re.compile(r'\b(' + '|'.join(d.keys()) + r')\b')
+ obuf = pattern.sub(lambda x: d[x.group()], buf)
+
+ fil.write(obuf)
+ fil.close()
+
+ print 'Written to file %s'%path
+ current_buffer = []
+ else:
+ current_buffer.append(l)
+ if (current_buffer):
+ print '\n'.join(current_buffer)
+
+
+if (__name__=='__main__'):
+ main()
diff --git a/xos/tools/apigen/xproto.template.txt b/xos/tools/apigen/xproto.template.txt
new file mode 100644
index 0000000..61734bc
--- /dev/null
+++ b/xos/tools/apigen/xproto.template.txt
@@ -0,0 +1,10 @@
+{% for object in generator.all() %}
+
+message {{ object.camel() }} {
+ {%- for field in object.fields %}
+ {{ xp_to_xproto(field, loop.index) }} {{xp_options(field)}};
+ {%- endfor %}
+}
+
++++ protos/{{ object }}.xproto
+{% endfor %}