CORD-1166: Unit test suite for xproto generation. Includes 12 tests covering
parsing, jinja2 target functionalities, xproto to protobuf conversion, django generation, pure protobuf generation. Also added README.md file.
Change-Id: I66a6982fafdc972966ec33bbd24d0333de040c61
diff --git a/xos/genx/tool/Makefile b/xos/genx/tool/Makefile
index a4dfcae..0d94772 100644
--- a/xos/genx/tool/Makefile
+++ b/xos/genx/tool/Makefile
@@ -1,25 +1,38 @@
# Replace the line below with the location of xosgen
-PREFIX=.
+PREFIX=../
XOSGEN=$(PREFIX)/tool/xosgen
DJANGO_TARGET=$(PREFIX)/targets/django-split.xtarget
INIT_TARGET=$(PREFIX)/targets/init.xtarget
XPROTOS_TMP := $(shell mktemp)
+# Rule to compile Python files
%.py: %.xproto
$(XOSGEN) --attic attic --input $< --target $(DJANGO_TARGET) --output $@
+# Rule to produce test results
+%.ok: %.py
+ PYTHONPATH=$(PYTHONPATH):$(PREFIX)/tool python -m $(subst /,.,$(subst .py,,$<))
+
+# List of xprotos
xprotos = $(wildcard *.xproto)
pys = $(xprotos:.xproto=.py)
+# List of tests
+tests = $(wildcard tests/*test.py)
+oks = $(tests:.py=.ok)
+
all: $(pys) __init__.py
__init__.py: $(xprotos)
cat $(xprotos) > $(XPROTOS_TMP)
$(XOSGEN) --attic attic --input $(XPROTOS_TMP) --target $(INIT_TARGET) --output $@ | awk '!seen[$$0]++' > __init__.py
-.PHONY: clean all init
+tests: $(oks)
+
+.PHONY: clean all init tests
clean:
rm -f $(pys)
+ rm -f $(oks)
rm -f __init__.py
diff --git a/xos/genx/tool/README.md b/xos/genx/tool/README.md
new file mode 100644
index 0000000..fd99542
--- /dev/null
+++ b/xos/genx/tool/README.md
@@ -0,0 +1,11 @@
+## Synopsis
+
+`xproto` is the layer that encodes XOS data models. The toolset here lets you generate various components from their `xproto` representations.
+
+## How to use
+
+Drop the Makefile in a directory with `xproto` files, set `PREFIX` to the location of `xproto` tools, and run `make` to generate XOS data models in Django.
+
+## Tests
+
+Run `make tests`
diff --git a/xos/genx/tool/__init__.py b/xos/genx/tool/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/xos/genx/tool/__init__.py
diff --git a/xos/genx/tool/tests/__init__.py b/xos/genx/tool/tests/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/xos/genx/tool/tests/__init__.py
diff --git a/xos/genx/tool/tests/counts b/xos/genx/tool/tests/counts
new file mode 100755
index 0000000..c08e39b
--- /dev/null
+++ b/xos/genx/tool/tests/counts
@@ -0,0 +1,34 @@
+#!/usr/bin/python
+
+import core.models
+import inspect
+from core.models import PlCoreBase, PlModelMixIn
+import pdb
+
+def count(lst):
+ c = 0
+ for l in lst[0]:
+ ll = l.lstrip()
+ if (ll and not ll.startswith('#') and ll.rstrip()!='pass' and 'ModelLink' not in ll and 'CHOICES' not in ll):
+ c+=1
+ return c
+
+def is_model_class(model):
+ """ Return True if 'model' is something that we're interested in """
+ if not inspect.isclass(model):
+ return False
+ if model.__name__ in ["PlModelMixIn"]:
+ return False
+ bases = inspect.getmro(model)
+ bases = [x.__name__ for x in bases]
+ if ("PlCoreBase" in bases) or ("PlModelMixIn" in bases):
+ return True
+
+ return False
+
+for a in dir(core.models):
+ x = getattr(core.models,a)
+ if (is_model_class(x)):
+ lines = inspect.getsourcelines(x)
+ print x.__name__,":",count(lines)
+
diff --git a/xos/genx/tool/tests/django_generator_test.py b/xos/genx/tool/tests/django_generator_test.py
new file mode 100644
index 0000000..4ea339a
--- /dev/null
+++ b/xos/genx/tool/tests/django_generator_test.py
@@ -0,0 +1,66 @@
+from xproto_test_base import *
+
+# Generate Protobuf from Xproto and then parse the resulting Protobuf
+class XProtoProtobufGeneratorTest(XProtoTest):
+ def test_proto_generator(self):
+ xproto = \
+"""
+message VRouterPort (PlCoreBase){
+ optional string name = 1 [help_text = "port friendly name", max_length = 20, null = True, db_index = False, blank = True];
+ required string openflow_id = 2 [help_text = "port identifier in ONOS", max_length = 21, null = False, db_index = False, blank = False];
+ required manytoone vrouter_device->VRouterDevice:ports = 3 [db_index = True, null = False, blank = False];
+ required manytoone vrouter_service->VRouterService:device_ports = 4 [db_index = True, null = False, blank = False];
+}
+"""
+ target = \
+"""
+from header import *
+{% for m in proto.messages %}
+{% if file_exists(xproto_base_name(m.name)|lower+'_header.py') -%}from {{xproto_base_name(m.name)|lower }}_header import *{% endif %}
+{% if file_exists(xproto_base_name(m.name)|lower+'_top.py') -%}{{ include_file(xproto_base_name(m.name)|lower+'_top.py') }} {% endif %}
+
+{%- for l in m.links %}
+
+{% if l.peer != m.name %}
+from core.models.{{ l.peer | lower }} import {{ l.peer }}
+{% endif %}
+
+{%- endfor %}
+{% for b in m.bases %}
+{% if b!='PlCoreBase' and 'Mixin' not in b%}
+from core.models.{{b | lower}} import {{ b }}
+{% endif %}
+{% endfor %}
+
+
+class {{ m.name }}{{ xproto_base_def(m.bases) }}:
+ # Primitive Fields (Not Relations)
+ {% for f in m.fields %}
+ {%- if not f.link -%}
+ {{ f.name }} = {{ xproto_django_type(f.type, f.options) }}( {{ xproto_django_options_str(f) }} )
+ {% endif %}
+ {%- endfor %}
+
+ # Relations
+ {% for l in m.links %}
+ {{ l.src_port }} = {{ xproto_django_link_type(l) }}( {%- if l.peer==m.name -%}'self'{%- else -%}{{ l.peer }} {%- endif -%}, {{ xproto_django_link_options_str(l, l.dst_port ) }} )
+ {%- endfor %}
+
+ {% if file_exists(m.name|lower + '_model.py') -%}{{ include_file(m.name|lower + '_model.py') | indent(width=2)}}{%- endif %}
+ pass
+
+{% if file_exists(xproto_base_name(m.name)|lower+'_bottom.py') -%}{{ include_file(xproto_base_name(m.name)|lower+'_bottom.py') }}{% endif %}
+{% endfor %}
+"""
+
+ self.generate(xproto = xproto, target = target)
+
+ fields = filter(lambda s:'Field(' in s, self.get_output().splitlines())
+ self.assertEqual(len(fields), 2)
+ links = filter(lambda s:'Key(' in s, self.get_output().splitlines())
+ self.assertEqual(len(links), 2)
+
+if __name__ == '__main__':
+ unittest.main()
+
+
diff --git a/xos/genx/tool/tests/parse_test.py b/xos/genx/tool/tests/parse_test.py
new file mode 100644
index 0000000..cfba4bd
--- /dev/null
+++ b/xos/genx/tool/tests/parse_test.py
@@ -0,0 +1,115 @@
+from xproto_test_base import *
+
+class XProtoParseTests(XProtoTest):
+ def test_global_options(self):
+ xproto = \
+"""
+ option kind = "vsg";
+ option verbose_name = "vSG Service";
+"""
+ self.generate(xproto = xproto, target = "{{ options }}")
+ self.assertIn("vsg", self.get_output())
+ self.assertIn("vSG Service", self.get_output())
+
+ def test_basic_proto(self):
+ # Picked up standard protobuf file from https://github.com/google/protobuf/blob/master/examples/addressbook.proto
+ xproto = \
+"""
+// See README.txt for information and build instructions.
+//
+// Note: START and END tags are used in comments to define sections used in
+// tutorials. They are not part of the syntax for Protocol Buffers.
+//
+// To get an in-depth walkthrough of this file and the related examples, see:
+// https://developers.google.com/protocol-buffers/docs/tutorials
+
+// [START declaration]
+package tutorial;
+// [END declaration]
+
+// [START java_declaration]
+option java_package = "com.example.tutorial";
+option java_outer_classname = "AddressBookProtos";
+// [END java_declaration]
+
+// [START csharp_declaration]
+option csharp_namespace = "Google.Protobuf.Examples.AddressBook";
+// [END csharp_declaration]
+
+// [START messages]
+message Person {
+ required string name = 1;
+ required int32 id = 2; // Unique ID number for this person.
+ optional string email = 3 [symphony = "da da da dum"];
+
+ enum PhoneType {
+ MOBILE = 0;
+ HOME = 1;
+ WORK = 2;
+ }
+
+ required string number = 1;
+ optional PhoneType type = 2;
+
+ repeated PhoneNumber phones = 4;
+}
+
+// Our address book file is just one of these.
+message AddressBook {
+ repeated Person people = 1;
+}
+// [END messages]
+"""
+ self.generate(xproto = xproto, target = "{{ proto }}")
+ self.assertIn("PhoneNumber", self.get_output())
+
+ def test_link_extensions(self):
+ xproto = \
+"""
+message links {
+ required manytoone vrouter_service->VRouterService:device_ports = 4 [db_index = True, null = False, blank = False];
+}
+"""
+ self.generate(xproto = xproto, target = "{{ proto.messages.0.links }}")
+ self.assertIn("VRouterService", self.get_output())
+
+ pass
+
+ def test_through_extensions(self):
+ xproto = \
+"""
+message links {
+ required manytomany vrouter_service->VRouterService/ServiceProxy:device_ports = 4 [db_index = True, null = False, blank = False];
+}
+"""
+ self.generate(xproto = xproto, target = "{{ proto.messages.0.links.0.through }}")
+ self.assertIn("ServiceProxy", self.get_output())
+
+ pass
+
+ def test_message_options(self):
+ xproto = \
+"""
+message link {
+ option type = "e1000";
+}
+"""
+ self.generate(xproto = xproto, target = "{{ proto.messages.0.options.type }}")
+ self.assertIn("e1000", self.get_output())
+
+ pass
+
+ def test_message_base(self):
+ xproto = \
+"""
+message base(Base) {
+}
+"""
+ self.generate(xproto = xproto, target = "{{ proto.messages.0.bases }}")
+ self.assertIn("Base", self.get_output())
+ pass
+
+if __name__ == '__main__':
+ unittest.main()
+
+
diff --git a/xos/genx/tool/tests/proto_generator_test.py b/xos/genx/tool/tests/proto_generator_test.py
new file mode 100644
index 0000000..b36f109
--- /dev/null
+++ b/xos/genx/tool/tests/proto_generator_test.py
@@ -0,0 +1,38 @@
+from xproto_test_base import *
+
+# Generate Protobuf from Xproto and then parse the resulting Protobuf
+class XProtoProtobufGeneratorTest(XProtoTest):
+ # This test is disabled because of a bug in Protobuf generation from xproto
+ # Namely, options appear with repeated double quotes: foo=""bar""
+ # TODO: Fix this bug, and re-enable this test
+
+ def __disabled_test_proto_generator(self):
+ xproto = \
+"""
+message VRouterPort (PlCoreBase){
+ optional string name = 1 [help_text = "port friendly name", max_length = 20, null = True, db_index = False, blank = True];
+ required string openflow_id = 2 [help_text = "port identifier in ONOS", max_length = 21, null = False, db_index = False, blank = False];
+ required manytoone vrouter_device->VRouterDevice:ports = 3 [db_index = True, null = False, blank = False];
+ required manytoone vrouter_service->VRouterService:device_ports = 4 [db_index = True, null = False, blank = False];
+}
+"""
+ target = \
+"""
+{% for m in proto.messages %}
+message {{ m.name }} {
+ option bases = "{{ m.bases | join(",") }}";
+ {%- for f in m.fields %}
+ {{ f.modifier }} {{f.type}} {{f.name}} = {{ f.id }}{% if f.options %} [{% for k,v in f.options.iteritems() %} {{ k }} = "{{ v}}"{% if not loop.last %},{% endif %} {% endfor %}]{% endif %};
+ {%- endfor %}
+}
+{% endfor %}
+"""
+
+ self.generate(xproto = xproto, target = target)
+ self.generate(xproto = self.get_output(), target = "{{ proto }}")
+ output = self.get_output()
+
+if __name__ == '__main__':
+ unittest.main()
+
+
diff --git a/xos/genx/tool/tests/pure_proto_test.py b/xos/genx/tool/tests/pure_proto_test.py
new file mode 100644
index 0000000..8402c93
--- /dev/null
+++ b/xos/genx/tool/tests/pure_proto_test.py
@@ -0,0 +1,80 @@
+from xproto_test_base import *
+
+# Generate from xproto, then generate from equivalent proto
+class XPureProtobufGenerator(XProtoTest):
+ def test_pure_proto(self):
+ xproto = \
+"""
+message VRouterPort (PlCoreBase){
+ optional string name = 1 [help_text = "port friendly name", max_length = 20, null = True, db_index = False, blank = True];
+ required string openflow_id = 2 [help_text = "port identifier in ONOS", max_length = 21, null = False, db_index = False, blank = False];
+ required manytoone vrouter_device->VRouterDevice:ports = 3 [db_index = True, null = False, blank = False];
+ required manytoone vrouter_service->VRouterService:device_ports = 4 [db_index = True, null = False, blank = False];
+}
+"""
+
+ proto = \
+"""
+message VRouterPort {
+ option bases = "PlCoreBase";
+ optional string name = 1 [ null = "True", max_length = "20", blank = "True", help_text = "port friendly name", modifier = "optional", db_index = "False" ];
+ required string openflow_id = 2 [ null = "False", max_length = "21", blank = "False", help_text = "port identifier in ONOS", modifier = "required", db_index = "False" ];
+ required int32 vrouter_device = 3 [ null = "False", blank = "False", model = "VRouterDevice", modifier = "required", type = "link", port = "ports", db_index = "True", link = "manytoone"];
+ required int32 vrouter_service = 4 [ null = "False", blank = "False", model = "VRouterService", modifier = "required", type = "link", port = "device_ports", db_index = "True", link = "manytoone"];
+}
+"""
+ target = \
+"""
+from header import *
+{% for m in proto.messages %}
+{% if file_exists(xproto_base_name(m.name)|lower+'_header.py') -%}from {{xproto_base_name(m.name)|lower }}_header import *{% endif %}
+{% if file_exists(xproto_base_name(m.name)|lower+'_top.py') -%}{{ include_file(xproto_base_name(m.name)|lower+'_top.py') }} {% endif %}
+
+{%- for l in m.links %}
+
+{% if l.peer != m.name %}
+from core.models.{{ l.peer | lower }} import {{ l.peer }}
+{% endif %}
+
+{%- endfor %}
+{% for b in m.bases %}
+{% if b!='PlCoreBase' and 'Mixin' not in b%}
+from core.models.{{b | lower}} import {{ b }}
+{% endif %}
+{% endfor %}
+
+
+class {{ m.name }}{{ xproto_base_def(m.bases) }}:
+ # Primitive Fields (Not Relations)
+ {% for f in m.fields %}
+ {%- if not f.link -%}
+ {{ f.name }} = {{ xproto_django_type(f.type, f.options) }}( {{ xproto_django_options_str(f) }} )
+ {% endif %}
+ {%- endfor %}
+
+ # Relations
+ {% for l in m.links %}
+ {{ l.src_port }} = {{ xproto_django_link_type(l) }}( {%- if l.peer==m.name -%}'self'{%- else -%}{{ l.peer }} {%- endif -%}, {{ xproto_django_link_options_str(l, l.dst_port ) }} )
+ {%- endfor %}
+
+ {% if file_exists(m.name|lower + '_model.py') -%}{{ include_file(m.name|lower + '_model.py') | indent(width=2)}}{%- endif %}
+ pass
+
+{% if file_exists(xproto_base_name(m.name)|lower+'_bottom.py') -%}{{ include_file(xproto_base_name(m.name)|lower+'_bottom.py') }}{% endif %}
+{% endfor %}
+"""
+
+ self.generate(xproto = xproto, target = target)
+ xproto_gen = self.get_output()
+ count1 = len(xproto_gen.split('\n'))
+
+ self.generate(xproto = proto, target = target, rev = True)
+ proto_gen = self.get_output()
+ count2 = len(proto_gen.split('\n'))
+
+ self.assertEqual(count1, count2)
+
+if __name__ == '__main__':
+ unittest.main()
+
+
diff --git a/xos/genx/tool/tests/target_test.py b/xos/genx/tool/tests/target_test.py
new file mode 100644
index 0000000..b73bf12
--- /dev/null
+++ b/xos/genx/tool/tests/target_test.py
@@ -0,0 +1,34 @@
+from xproto_test_base import *
+
+class XProtoTargetTests(XProtoTest):
+ def test_file_methods(self):
+ target = \
+"""
+ {%% if file_exists("%s") %%}
+ {{ include_file("%s") }}
+ {%% endif %%}
+"""%(TEST_FILE, TEST_FILE)
+
+ self.generate(target=target)
+ self.assertIn(TEST_OUTPUT, self.get_output())
+
+ def test_xproto_lib(self):
+ target = \
+"""
+ {{ xproto_first_non_empty([None, None, None, None, None, None, "Eureka"]) }}
+"""
+ self.generate(target=target)
+ self.assertIn("Eureka", self.get_output())
+
+ def test_context(self):
+ target = \
+"""
+ {{ context.what }}
+"""
+ self.generate(target=target, kv='what:what is what')
+ self.assertIn("what is what", self.get_output())
+
+if __name__ == '__main__':
+ unittest.main()
+
+
diff --git a/xos/genx/tool/xproto_test_base.py b/xos/genx/tool/xproto_test_base.py
new file mode 100644
index 0000000..2315eaf
--- /dev/null
+++ b/xos/genx/tool/xproto_test_base.py
@@ -0,0 +1,64 @@
+import unittest
+import shutil
+import os
+from generator import *
+
+TEST_FILE = "test_file"
+TEST_OUTPUT = "Do re mi fa so la ti do"
+
+XPROTO_FILE = 'test.xproto'
+OUTPUT_FILE = 'test.output'
+TARGET_FILE = 'test.xtarget'
+XPROTO_DIR = "/tmp/xproto-tests"
+
+TEST_PATH = '/'.join([XPROTO_DIR, TEST_FILE])
+XPROTO_PATH = '/'.join([XPROTO_DIR, XPROTO_FILE])
+TARGET_PATH = '/'.join([XPROTO_DIR, TARGET_FILE])
+OUTPUT_PATH = '/'.join([XPROTO_DIR, OUTPUT_FILE])
+
+class FakeArgs:
+ pass
+
+class XProtoTest(unittest.TestCase):
+ def setUp(self):
+ if not os.path.exists(XPROTO_DIR):
+ os.mkdir(XPROTO_DIR)
+ open('/'.join([XPROTO_DIR, TEST_FILE]),'w').write(TEST_OUTPUT)
+ #print "Test %s Started" % (self.id())
+
+
+ def tearDown(self):
+ #if os.path.exists(XPROTO_DIR):
+ # shutil.rmtree(XPROTO_DIR)
+ pass
+
+ def generate(self, xproto = None, target = None, kv = '', rev = False):
+ if (not xproto):
+ xproto = \
+"""
+ message X(Y) {}
+"""
+ pass
+
+ if (not target):
+ target = '{{ proto }}'
+
+ target+='\n+++ %s'%OUTPUT_PATH
+
+ open(TARGET_PATH, 'w').write(target)
+ open(XPROTO_PATH, 'w').write(xproto)
+
+ args = FakeArgs()
+ args.template_dir = XPROTO_DIR
+ args.quiet = True
+ args.rev = rev
+ args.kv = kv
+ args.attic = XPROTO_DIR
+ args.input = XPROTO_PATH
+ args.output = OUTPUT_PATH
+ args.target = TARGET_FILE
+ g = XOSGenerator(args)
+ g.generate()
+
+ def get_output(self):
+ return open(OUTPUT_PATH).read()