[SEBA-450] (part 1)

Refactoring, python3 compat, and tox tests on:

- xosconfig
- xosgenx
- xosutil

Eliminate use of yaml.load() which is unsafe, switch to yaml.safe_load()

More diagnostics during database migration

Change-Id: I0fae5782fca401603a7c4e4ec2b9269ad24bda97
diff --git a/lib/xos-genx/xosgenx/generator.py b/lib/xos-genx/xosgenx/generator.py
index af99a23..9a6a249 100644
--- a/lib/xos-genx/xosgenx/generator.py
+++ b/lib/xos-genx/xosgenx/generator.py
@@ -12,17 +12,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import absolute_import, print_function
 
-from __future__ import print_function
-import plyxproto.parser as plyxproto
-import jinja2
 import os
-from xos2jinja import XOS2Jinja
-from proto2xproto import Proto2XProto
-import jinja2_extensions
+import jinja2
+import plyxproto.parser as plyxproto
 import yaml
 from colorama import Fore
 
+from . import jinja2_extensions
+from .proto2xproto import Proto2XProto
+from .xos2jinja import XOS2Jinja
+
 loader = jinja2.PackageLoader(__name__, "templates")
 env = jinja2.Environment(loader=loader)
 
@@ -147,7 +148,7 @@
                 context[k] = val
             return context
         except Exception as e:
-            print(e.message)
+            print(e)
 
     @staticmethod
     def _write_single_file(rendered, dir, dest_file, quiet):
@@ -308,7 +309,11 @@
                 if message["name"] in args.include_models:
                     message["is_included"] = True
                 else:
-                    app_label = message.get("options", {}).get("app_label").strip('"')
+                    app_label = (
+                        message.get("options", {})
+                        .get("app_label")
+                        .strip('"')
+                    )
                     if app_label in args.include_apps:
                         message["is_included"] = True
         else:
diff --git a/lib/xos-genx/xosgenx/jinja2_extensions/__init__.py b/lib/xos-genx/xosgenx/jinja2_extensions/__init__.py
index bf7a812..a50bf3a 100644
--- a/lib/xos-genx/xosgenx/jinja2_extensions/__init__.py
+++ b/lib/xos-genx/xosgenx/jinja2_extensions/__init__.py
@@ -12,6 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Ignore this file as the * imports for Jinja2 functions are too numerous to list
+# flake8: noqa
+
+from __future__ import absolute_import
 from .django import *
 from .base import *
 from .fol2 import *
diff --git a/lib/xos-genx/xosgenx/jinja2_extensions/base.py b/lib/xos-genx/xosgenx/jinja2_extensions/base.py
index 96e8dc2..f4929a9 100644
--- a/lib/xos-genx/xosgenx/jinja2_extensions/base.py
+++ b/lib/xos-genx/xosgenx/jinja2_extensions/base.py
@@ -12,8 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
-from __future__ import print_function
+from __future__ import absolute_import, print_function
 import pdb
 import re
 from inflect import engine as inflect_engine_class
@@ -319,10 +318,11 @@
 
 
 def xproto_string_type(xptags):
-    try:
-        max_length = eval(xptags["max_length"])
-    except BaseException:
-        max_length = 1024
+    # FIXME: this try/except block assigns but never uses max_length?
+    #   try:
+    #       max_length = eval(xptags["max_length"])
+    #   except BaseException:
+    #       max_length = 1024
 
     if "varchar" not in xptags:
         return "string"
@@ -340,26 +340,59 @@
 
 
 def xproto_field_graph_components(fields, model, tag="unique_with"):
+    """
+    NOTE: Don't use set theory operators if you want repeatable tests - many
+    of them have non-deterministic behavior
+    """
+
     def find_components(graph):
+
+        # 'graph' dict structure:
+        #   - keys are strings
+        #   - values are sets containing strings that are names of other keys in 'graph'
+
+        # take keys from 'graph' dict and put in 'pending' set
         pending = set(graph.keys())
+
+        # create an empty list named 'components'
         components = []
 
+        # loop while 'pending' is true - while there are still items in the 'pending' set
         while pending:
+
+            # remove a random item from pending set, and put in 'front'
+            # this is the primary source of nondeterminism
             front = {pending.pop()}
+
+            # create an empty set named 'component'
             component = set()
 
+            # loop while 'front' is true. Front is modified below
             while front:
+
+                # take the (only?) item out of the 'front' dict, and put in 'node'
                 node = front.pop()
+
+                # from 'graph' dict take set with key of 'node' and put into 'neighbors'
                 neighbours = graph[node]
+
+                # remove the set of items in components from neighbors
                 neighbours -= component  # These we have already visited
+
+                # add all remaining neighbors to front
                 front |= neighbours
 
+                # remove neighbors from pending
                 pending -= neighbours
+
+                # add neighbors to component
                 component |= neighbours
 
-            components.append(component)
+            # append component set to components list, sorted
+            components.append(sorted(component))
 
-        return components
+        # return 'components', which is a list of sets
+        return sorted(components)
 
     field_graph = {}
     field_names = {f["name"] for f in fields}
@@ -378,6 +411,7 @@
 
                 field_graph.setdefault(f["name"], set()).add(uf)
                 field_graph.setdefault(uf, set()).add(f["name"])
+
         except KeyError:
             pass
 
@@ -434,12 +468,12 @@
 
 def xproto_field_to_swagger_enum(f):
     if "choices" in f["options"]:
-        list = []
+        c_list = []
 
         for c in eval(xproto_unquote(f["options"]["choices"])):
-            list.append(c[0])
+            c_list.append(c[0])
 
-        return list
+        return sorted(c_list)
     else:
         return False
 
diff --git a/lib/xos-genx/xosgenx/jinja2_extensions/checklib.py b/lib/xos-genx/xosgenx/jinja2_extensions/checklib.py
index db61f01..c361b64 100644
--- a/lib/xos-genx/xosgenx/jinja2_extensions/checklib.py
+++ b/lib/xos-genx/xosgenx/jinja2_extensions/checklib.py
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import absolute_import
 import ast
 
 
@@ -27,13 +28,13 @@
     except SyntaxError:
         return "511 Could not parse sync step %s" % sync_step_path
 
-    classes = filter(lambda x: isinstance(x, ast.ClassDef), sync_step_ast.body)
+    classes = [x for x in sync_step_ast.body if isinstance(x, ast.ClassDef)]
     found_sync_step_class = False
 
     for c in classes:
         base_names = [v.id for v in c.bases]
         if "SyncStep" in base_names or "SyncInstanceUsingAnsible" in base_names:
-            attributes = filter(lambda x: isinstance(x, ast.Assign), c.body)
+            attributes = [x for x in c.body if isinstance(x, ast.Assign)]
             for a in attributes:
                 target_names = [t.id for t in a.targets]
                 values = a.value.elts if isinstance(a.value, ast.List) else [a.value]
@@ -66,7 +67,7 @@
     except SyntaxError:
         return "511 Could not parse sync step %s" % model_policy_path
 
-    classes = filter(lambda x: isinstance(x, ast.ClassDef), model_policy_ast.body)
+    classes = [x for x in model_policy_ast.body if isinstance(x, ast.ClassDef)]
     found_model_policy_class = False
     for c in classes:
         base_names = [v.id for v in c.bases]
diff --git a/lib/xos-genx/xosgenx/jinja2_extensions/django.py b/lib/xos-genx/xosgenx/jinja2_extensions/django.py
index 3909c00..0c34d3e 100644
--- a/lib/xos-genx/xosgenx/jinja2_extensions/django.py
+++ b/lib/xos-genx/xosgenx/jinja2_extensions/django.py
@@ -12,11 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import print_function
-from base import *
-import pdb
+from __future__ import absolute_import, print_function
+from .base import unquote
 import re
 import sys
+from six.moves import map
 
 
 def django_content_type_string(xptags):
@@ -48,7 +48,7 @@
 
     if "content_type" in xptags:
         return django_content_type_string(xptags)
-    elif max_length < 1024 * 1024:
+    elif int(max_length) < 1024 * 1024:
         return "CharField"
     else:
         return "TextField"
@@ -208,7 +208,7 @@
     else:
 
         lst = []
-        for k, v in d.items():
+        for k, v in sorted(d.items(), key=lambda t: t[0]):  # sorted by key
             if k in known_validators:
                 validator_lst.append(use_native_django_validators(k, v))
             elif isinstance(v, str) and k == "default" and v.endswith('()"'):
@@ -261,7 +261,7 @@
 def xproto_validations(options):
     try:
         return [
-            map(str.strip, validation.split(":"))
+            list(map(str.strip, validation.split(":")))
             for validation in unquote(options["validators"]).split(",")
         ]
     except KeyError:
@@ -280,7 +280,7 @@
     optioned_fields = []
     for f in fields:
         option_names = []
-        for k, v in f["options"].items():
+        for k, v in sorted(f["options"].items(), key=lambda t: t[0]):  # sorted by key
             option_names.append(k)
 
         if option in option_names and f["options"][option] == val:
diff --git a/lib/xos-genx/xosgenx/jinja2_extensions/fol2.py b/lib/xos-genx/xosgenx/jinja2_extensions/fol2.py
index 6d66117..0997e00 100644
--- a/lib/xos-genx/xosgenx/jinja2_extensions/fol2.py
+++ b/lib/xos-genx/xosgenx/jinja2_extensions/fol2.py
@@ -12,14 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
-from __future__ import print_function
+from __future__ import absolute_import, print_function
 import astunparse
 import ast
 import random
 import string
 import jinja2
-from plyxproto.parser import *
+from plyxproto.parser import lex, yacc
+from plyxproto.logicparser import FOLParser, FOLLexer
+from six.moves import range
+from six.moves import input
 
 BINOPS = ["|", "&", "->"]
 QUANTS = ["exists", "forall"]
@@ -45,11 +47,13 @@
         self.idx = 0
         return self
 
-    def next(self):
+    def __next__(self):
         var = "i%d" % self.idx
         self.idx += 1
         return var
 
+    next = __next__  # 2to3
+
 
 def gen_random_string():
     return "".join(
@@ -63,18 +67,18 @@
         self.loopvar = iter(AutoVariable("i"))
         self.verdictvar = iter(AutoVariable("result"))
 
-        self.loop_variable = self.loopvar.next()
-        self.verdict_variable = self.verdictvar.next()
+        self.loop_variable = next(self.loopvar)
+        self.verdict_variable = next(self.verdictvar)
         self.context_map = context_map
 
         if not self.context_map:
             self.context_map = {"user": "self", "obj": "obj"}
 
     def loop_next(self):
-        self.loop_variable = self.loopvar.next()
+        self.loop_variable = next(self.loopvar)
 
     def verdict_next(self):
-        self.verdict_variable = self.verdictvar.next()
+        self.verdict_variable = next(self.verdictvar)
 
     def gen_enumerate(self, fol):
         pass
@@ -398,7 +402,8 @@
                     if result['result'] == 'True':
                         return {'hoist': ['const'], 'result': result['hoist'][1]}
                     elif result['hoist'][0] in BINOPS:
-                        return {'hoist': ['const'], 'result': {result['hoist'][0]: [result['hoist'][1], {k: [var2, result['result']]}]}}
+                        return {'hoist': ['const'], 'result': {result['hoist'][0]:
+                                [result['hoist'][1], {k: [var2, result['result']]}]}}
                     else:
                         return {'hoist': ['const'], 'result': {k: [var2, result['result']]}}
                 else:
@@ -731,9 +736,10 @@
 
     if fol_reduced in ["True", "False"] and fol != fol_reduced:
         raise TrivialPolicy(
-            "Policy %(name)s trivially reduces to %(reduced)s. If this is what you want, replace its contents with %(reduced)s" % {
-                "name": policy,
-                "reduced": fol_reduced})
+            "Policy %(name)s trivially reduces to %(reduced)s."
+            "If this is what you want, replace its contents with %(reduced)s"
+            % {"name": policy, "reduced": fol_reduced}
+        )
 
     a = f2p.gen_test_function(fol_reduced, policy, tag="security_check")
 
@@ -749,9 +755,10 @@
 
     if fol_reduced in ["True", "False"] and fol != fol_reduced:
         raise TrivialPolicy(
-            "Policy %(name)s trivially reduces to %(reduced)s. If this is what you want, replace its contents with %(reduced)s" % {
-                "name": policy,
-                "reduced": fol_reduced})
+            "Policy %(name)s trivially reduces to %(reduced)s."
+            "If this is what you want, replace its contents with %(reduced)s"
+            % {"name": policy, "reduced": fol_reduced}
+        )
 
     a = f2p.gen_validation_function(fol_reduced, policy, message, tag="validator")
 
@@ -762,7 +769,7 @@
     while True:
         inp = ""
         while True:
-            inp_line = raw_input()
+            inp_line = input()
             if inp_line == "EOF":
                 break
             else:
diff --git a/lib/xos-genx/xosgenx/jinja2_extensions/gui.py b/lib/xos-genx/xosgenx/jinja2_extensions/gui.py
index 4cb644a..07dbc35 100644
--- a/lib/xos-genx/xosgenx/jinja2_extensions/gui.py
+++ b/lib/xos-genx/xosgenx/jinja2_extensions/gui.py
@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
-from base import xproto_string_type, unquote
+from __future__ import absolute_import
+from .base import xproto_string_type, unquote
 
 
 def xproto_type_to_ui_type(f):
@@ -49,6 +49,20 @@
         return None
 
 
+def xproto_dict_to_sorted_string(d):
+    """
+    sorts the dict by key and returns a string representation, which makes
+    for better consistency when testing
+    """
+    ft = []  # formatted tuples
+    for k, v in sorted(d.items(), key=lambda t: t[0]):  # sorted by key
+        if v is not None:
+            ft.append("'%s': '%s'" % (k, v))
+        else:
+            ft.append("'%s': None" % k)
+    return "{%s}" % ", ".join(ft)
+
+
 def xproto_validators(f):
     # To be cleaned up when we formalize validation in xproto
     validators = []
diff --git a/lib/xos-genx/xosgenx/jinja2_extensions/tosca.py b/lib/xos-genx/xosgenx/jinja2_extensions/tosca.py
index d8eada7..4424824 100644
--- a/lib/xos-genx/xosgenx/jinja2_extensions/tosca.py
+++ b/lib/xos-genx/xosgenx/jinja2_extensions/tosca.py
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import absolute_import
 from xosgenx.jinja2_extensions import xproto_field_graph_components
 
 
@@ -58,18 +59,19 @@
         ):
             keys.append("%s_id" % f["name"])
     # if not keys are specified and there is a name field, use that as key.
-    if len(keys) == 0 and "name" in map(lambda f: f["name"], fields):
+    if len(keys) == 0 and "name" in [f["name"] for f in fields]:
         keys.append("name")
 
-    for of in one_of:
+    for of in sorted(one_of):
         # check if the field is a link, and in case add _id
         for index, f in enumerate(of):
             try:
-                field = [
+                # FIXME: the 'field' var appears to be dead code, but exists to cause the IndexError?
+                field = [  # noqa: F841
                     x for x in fields if x["name"] == f and ("link" in x and x["link"])
                 ][0]
                 of[index] = "%s_id" % f
-            except IndexError as e:
+            except IndexError:
                 # the field is not a link
                 pass
 
diff --git a/lib/xos-genx/xosgenx/proto2xproto.py b/lib/xos-genx/xosgenx/proto2xproto.py
index f1dc959..6157aee 100644
--- a/lib/xos-genx/xosgenx/proto2xproto.py
+++ b/lib/xos-genx/xosgenx/proto2xproto.py
@@ -12,19 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import absolute_import
 
-import plyxproto.model as m
-from plyxproto.helpers import Visitor
-import pdb
-import argparse
-import plyxproto.parser as plyxproto
-from plyxproto.logicparser import FOLParser, FOLLexer
-import traceback
-import sys
-import jinja2
-import os
 import ply.lex as lex
 import ply.yacc as yacc
+import plyxproto.model as m
+from plyxproto.helpers import Visitor
+from plyxproto.logicparser import FOLLexer, FOLParser
+from six.moves import map
 
 
 class Stack(list):
@@ -136,7 +131,7 @@
             except KeyError:
                 bases = []
 
-            bases = map(lambda x: str_to_dict(x[1:-1]), bases)
+            bases = [str_to_dict(x[1:-1]) for x in bases]
             obj.bases = bases
         except KeyError:
             raise
@@ -212,8 +207,8 @@
 
     def visit_MessageDefinition_post(self, obj):
         self.proto_to_xproto_message(obj)
-        obj.body = filter(lambda x: not hasattr(x, "mark_for_deletion"), obj.body)
-        obj.body = map(replace_link, obj.body)
+        obj.body = [x for x in obj.body if not hasattr(x, "mark_for_deletion")]
+        obj.body = list(map(replace_link, obj.body))
 
         self.current_message_name = None
         return True
diff --git a/lib/xos-genx/xosgenx/targets/link-graph.xtarget b/lib/xos-genx/xosgenx/targets/link-graph.xtarget
index 04c9dba..e455129 100644
--- a/lib/xos-genx/xosgenx/targets/link-graph.xtarget
+++ b/lib/xos-genx/xosgenx/targets/link-graph.xtarget
@@ -5,7 +5,7 @@
         {% for l in model.links -%}
             ["{{ l.peer }}", "{{ l.src_port }}", "{{ l.dst_port }}"]{% if not loop.last %},{% endif %}
         {%- endfor %}
-        {%- if model.rlinks %},{% endif %}        
+        {%- if model.rlinks %},{% endif %}
         {% for l in model.rlinks -%}
             ["{{ l.peer }}", "{{ l.src_port }}", "{{ l.dst_port }}"]{% if not loop.last %},{% endif %}
         {%- endfor %}
diff --git a/lib/xos-genx/xosgenx/targets/modeldefs.xtarget b/lib/xos-genx/xosgenx/targets/modeldefs.xtarget
index 15c43d7..ec7e5f2 100644
--- a/lib/xos-genx/xosgenx/targets/modeldefs.xtarget
+++ b/lib/xos-genx/xosgenx/targets/modeldefs.xtarget
@@ -8,7 +8,7 @@
   {%- if m.options.verbose_name %}
   verbose_name: "{{ xproto_unquote(m.options.verbose_name) }}"
   {%- endif %}
-  fields: 
+  fields:
   {%- set id_field = {'type':'int32', 'name':'id', 'options':{}} %}
   {% for f in (xproto_base_fields(m, proto.message_table) + m.fields + [id_field]) | sort(attribute='name') -%}
   {% if xproto_unquote(xproto_first_non_empty([f.options.gui_hidden, 'False'])) != 'True' and (not f.link or f.options.link_type != 'manytomany') -%}
@@ -25,7 +25,7 @@
   {%- if f.options.choices %}
     options:
     {% for o in xproto_options_choices_to_dict(xproto_unquote(f.options.choices)) %}
-    - {{ o }}
+    - {{ xproto_dict_to_sorted_string(o) }}
     {% endfor %}
   {%- endif %}
     type: {{ xproto_type_to_ui_type(f) }}
@@ -50,4 +50,4 @@
   relations: []
   {%- endif %}
 {%- endif %}
-{% endfor -%} 
+{% endfor -%}
diff --git a/lib/xos-genx/xosgenx/version.py b/lib/xos-genx/xosgenx/version.py
index 79c5702..98215ea 100644
--- a/lib/xos-genx/xosgenx/version.py
+++ b/lib/xos-genx/xosgenx/version.py
@@ -14,5 +14,6 @@
 
 import os
 
-# This file will be replaced by setup.py
-__version__ = "unknown"
+# read the version in from VERSION file, which is installed next to this file.
+with open(os.path.join(os.path.dirname(__file__), "VERSION")) as vf:
+    __version__ = vf.read()
diff --git a/lib/xos-genx/xosgenx/xos2jinja.py b/lib/xos-genx/xosgenx/xos2jinja.py
index 471be93..e4ff7db 100644
--- a/lib/xos-genx/xosgenx/xos2jinja.py
+++ b/lib/xos-genx/xosgenx/xos2jinja.py
@@ -12,18 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import absolute_import, print_function
 
-from __future__ import print_function
+import copy
+import sys
+
 import plyxproto.model as m
 from plyxproto.helpers import Visitor
-import argparse
-import plyxproto.parser as plyxproto
-import traceback
-import sys
-import jinja2
-import os
-import copy
-import pdb
+from six.moves import map, range
 
 
 class MissingPolicyException(Exception):
@@ -175,7 +171,10 @@
         return True
 
     def visit_FieldType(self, obj):
-        """Field type, if type is name, then it may need refactoring consistent with refactoring rules according to the table"""
+        """
+        Field type, if type is name, then it may need refactoring consistent
+        with refactoring rules according to the table
+        """
         return True
 
     def visit_LinkDefinition(self, obj):
@@ -295,13 +294,12 @@
         stack_num = self.count_stack.pop()
         fields = []
         links = []
-        last_field = None
+        last_field = {}
         try:
-            obj.bases = map(dotname_to_fqn, obj.bases)
+            obj.bases = list(map(dotname_to_fqn, obj.bases))
         except AttributeError:
             pass
 
-        last_field = {}
         for i in range(0, stack_num):
             f = self.stack.pop()
             if f["_type"] == "link":
@@ -340,7 +338,7 @@
         self.models[model_name] = model_def
 
         # Set message options
-        for k, v in self.options.iteritems():
+        for k, v in sorted(self.options.items(), key=lambda t: t[0]):  # sorted by key
             try:
                 if k not in self.message_options:
                     self.message_options[k] = v
@@ -406,47 +404,47 @@
             "onetomany": "manytoone",
         }
 
-        for m in messages:
-            for l in m["links"]:
-                rlink = copy.deepcopy(l)
+        for msg in messages:
+            for lnk in msg["links"]:
+                rlink = copy.deepcopy(lnk)
 
                 rlink["_type"] = "rlink"  # An implicit link, not declared in the model
-                rlink["src_port"] = l["dst_port"]
-                rlink["dst_port"] = l["src_port"]
+                rlink["src_port"] = lnk["dst_port"]
+                rlink["dst_port"] = lnk["src_port"]
                 rlink["peer"] = {
-                    "name": m["name"],
-                    "package": m["package"],
-                    "fqn": m["fqn"],
+                    "name": msg["name"],
+                    "package": msg["package"],
+                    "fqn": msg["fqn"],
                 }
-                rlink["link_type"] = link_opposite[l["link_type"]]
-                rlink["reverse_id"] = l["reverse_id"]
+                rlink["link_type"] = link_opposite[lnk["link_type"]]
+                rlink["reverse_id"] = lnk["reverse_id"]
 
-                if (not l["reverse_id"]) and (self.args.verbosity >= 1):
+                if (not lnk["reverse_id"]) and (self.args.verbosity >= 1):
                     print(
                         "WARNING: Field %s in model %s has no reverse_id"
-                        % (l["src_port"], m["name"]),
+                        % (lnk["src_port"], msg["name"]),
                         file=sys.stderr,
                     )
 
-                if l["reverse_id"] and (
-                    (int(l["reverse_id"]) < 1000) or (int(l["reverse_id"]) >= 1900)
+                if lnk["reverse_id"] and (
+                    (int(lnk["reverse_id"]) < 1000) or (int(lnk["reverse_id"]) >= 1900)
                 ):
                     raise Exception(
                         "reverse id for field %s in model %s should be between 1000 and 1899"
-                        % (l["src_port"], m["name"])
+                        % (lnk["src_port"], msg["name"])
                     )
 
                 try:
                     try:
-                        rev_links[l["peer"]["fqn"]].append(rlink)
+                        rev_links[lnk["peer"]["fqn"]].append(rlink)
                     except TypeError:
                         pass
                 except KeyError:
-                    rev_links[l["peer"]["fqn"]] = [rlink]
+                    rev_links[lnk["peer"]["fqn"]] = [rlink]
 
-        for m in messages:
+        for msg in messages:
             try:
-                m["rlinks"] = rev_links[m["name"]]
-                message_dict[m["name"]]["rlinks"] = m["rlinks"]
+                msg["rlinks"] = rev_links[msg["name"]]
+                message_dict[msg["name"]]["rlinks"] = msg["rlinks"]
             except KeyError:
                 pass
diff --git a/lib/xos-genx/xosgenx/xosgen.py b/lib/xos-genx/xosgenx/xosgen.py
index 1318242..2da8962 100755
--- a/lib/xos-genx/xosgenx/xosgen.py
+++ b/lib/xos-genx/xosgenx/xosgen.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
 # Copyright 2017-present Open Networking Foundation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,13 +14,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import absolute_import, print_function
 
-#!/usr/bin/python
-
-from __future__ import print_function
 import argparse
-from generator import *
-from version import __version__
+import os
+from functools import reduce
+
+from six.moves import range
+
+from .generator import XOSProcessor, XOSProcessorArgs
+from .version import __version__
 
 parse = argparse.ArgumentParser(description="XOS Generative Toolchain")
 parse.add_argument(
@@ -147,7 +152,7 @@
             else [args.target]
         )
 
-        for i in xrange(len(operators)):
+        for i in range(len(operators)):
             if "/" not in operators[i]:
                 # if the target is not a path, it refer to a library included one
                 operators[i] = os.path.abspath(