[SEBA-450] (part 1)
Refactoring, python3 compat, and tox tests on:
- xosconfig
- xosgenx
- xosutil
Eliminate use of yaml.load() which is unsafe, switch to yaml.safe_load()
More diagnostics during database migration
Change-Id: I0fae5782fca401603a7c4e4ec2b9269ad24bda97
diff --git a/lib/xos-genx/xos-genx-tests/helpers.py b/lib/xos-genx/xos-genx-tests/helpers.py
index 4687bb3..e6d87ab 100644
--- a/lib/xos-genx/xos-genx-tests/helpers.py
+++ b/lib/xos-genx/xos-genx-tests/helpers.py
@@ -13,6 +13,7 @@
# limitations under the License.
+from __future__ import absolute_import
import os
# Constants
diff --git a/lib/xos-genx/xos-genx-tests/test_cli.py b/lib/xos-genx/xos-genx-tests/test_cli.py
index 3f94865..087c904 100644
--- a/lib/xos-genx/xos-genx-tests/test_cli.py
+++ b/lib/xos-genx/xos-genx-tests/test_cli.py
@@ -13,6 +13,7 @@
# limitations under the License.
+from __future__ import absolute_import
import unittest
import os
from mock import patch
diff --git a/lib/xos-genx/xos-genx-tests/test_django_generator.py b/lib/xos-genx/xos-genx-tests/test_django_generator.py
index a81f80c..022a640 100644
--- a/lib/xos-genx/xos-genx-tests/test_django_generator.py
+++ b/lib/xos-genx/xos-genx-tests/test_django_generator.py
@@ -13,6 +13,7 @@
# limitations under the License.
+from __future__ import absolute_import
import unittest
import os
from xosgenx.generator import XOSProcessor, XOSProcessorArgs
@@ -32,9 +33,9 @@
args = XOSProcessorArgs(files=[VROUTER_XPROTO], target="django.xtarget")
output = XOSProcessor.process(args)
- fields = filter(lambda s: "Field(" in s, output.splitlines())
+ fields = [s for s in output.splitlines() if "Field(" in s]
self.assertEqual(len(fields), 2)
- links = filter(lambda s: "Key(" in s, output.splitlines())
+ links = [s for s in output.splitlines() if "Key(" in s]
self.assertEqual(len(links), 2)
def test_optional_relations(self):
@@ -59,11 +60,11 @@
args = XOSProcessorArgs(inputs=xproto, target="django.xtarget")
output = XOSProcessor.process(args)
- null_true = filter(lambda s: "null = True" in s, output.splitlines())
- null_false = filter(lambda s: "null = False" in s, output.splitlines())
+ null_true = [s for s in output.splitlines() if "null = True" in s]
+ null_false = [s for s in output.splitlines() if "null = False" in s]
- blank_true = filter(lambda s: "blank = True" in s, output.splitlines())
- blank_false = filter(lambda s: "blank = False" in s, output.splitlines())
+ blank_true = [s for s in output.splitlines() if "blank = True" in s]
+ blank_false = [s for s in output.splitlines() if "blank = False" in s]
self.assertEqual(len(null_true), 1)
self.assertEqual(len(null_false), 1)
diff --git a/lib/xos-genx/xos-genx-tests/test_field_graph.py b/lib/xos-genx/xos-genx-tests/test_field_graph.py
index cfb4c8b..c3c08a7 100644
--- a/lib/xos-genx/xos-genx-tests/test_field_graph.py
+++ b/lib/xos-genx/xos-genx-tests/test_field_graph.py
@@ -13,10 +13,11 @@
# limitations under the License.
+from __future__ import absolute_import
import unittest
from xosgenx.jinja2_extensions import FieldNotFound
-from helpers import XProtoTestHelpers
from xosgenx.generator import XOSProcessor, XOSProcessorArgs
+from helpers import XProtoTestHelpers
from functools import reduce
@@ -75,7 +76,7 @@
generate()
self.assertEqual(
- e.exception.message,
+ str(e.exception),
'Field "B" not found in model "Foo", referenced from field "A" by option "unique_with"',
)
diff --git a/lib/xos-genx/xos-genx-tests/test_general_security.py b/lib/xos-genx/xos-genx-tests/test_general_security.py
index c675b16..26a4736 100644
--- a/lib/xos-genx/xos-genx-tests/test_general_security.py
+++ b/lib/xos-genx/xos-genx-tests/test_general_security.py
@@ -12,28 +12,32 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
+from __future__ import absolute_import
+from __future__ import print_function
import unittest
from xosgenx.generator import XOSProcessor, XOSProcessorArgs
from helpers import XProtoTestHelpers, FakeObject
-"""The function below is for eliminating warnings arising due to the missing output_security_check,
-which is generated and loaded dynamically.
-"""
-
def output_security_check(x, y):
+ """
+ This function eliminates warnings arising due to the missing
+ output_security_check, which is generated and loaded dynamically. This is
+ defined in the global namespace, and in python3 the globals() namespace has
+ to be passed to calls to "exec()" for the xproto-generated version to
+ redefine this function.
+ """
+
raise Exception("Security enforcer not generated. Test failed.")
return False
-"""
-The tests below use the Python code target to generate
-Python security policies, set up an appropriate environment and execute the Python.
-"""
-
-
class XProtoSecurityTest(unittest.TestCase):
+ """
+ Use the Python code target to generate Python security policies, set up an
+ appropriate environment and execute the Python.
+ """
+
def setUp(self):
self.target = XProtoTestHelpers.write_tmp_target(
"""
@@ -48,10 +52,8 @@
policy output < True >
"""
args = XOSProcessorArgs(inputs=xproto, target=self.target)
-
output = XOSProcessor.process(args)
-
- exec(output) # This loads the generated function, which should look like this:
+ exec(output, globals()) # This loads the generated function, which should look like this:
"""
def output_security_check(obj, ctx):
@@ -68,10 +70,8 @@
"""
args = XOSProcessorArgs(inputs=xproto, target=self.target)
-
output = XOSProcessor.process(args)
-
- exec(output) # This loads the generated function, which should look like this:
+ exec(output, globals()) # This loads the generated function, which should look like this:
"""
def output_security_check(obj, ctx):
@@ -93,12 +93,8 @@
"""
args = XOSProcessorArgs(inputs=xproto, target=self.target)
-
output = XOSProcessor.process(args)
-
- exec(
- output, globals()
- ) # This loads the generated function, which should look like this:
+ exec(output, globals()) # This loads the generated function, which should look like this:
"""
def sub_policy_security_check(obj, ctx):
@@ -130,12 +126,8 @@
"""
args = XOSProcessorArgs(inputs=xproto, target=self.target)
-
output = XOSProcessor.process(args)
-
- exec(
- output, globals()
- ) # This loads the generated function, which should look like this:
+ exec(output, globals()) # This loads the generated function, which should look like this:
"""
def sub_policy_security_check(obj, ctx):
@@ -165,9 +157,8 @@
"""
args = XOSProcessorArgs(inputs=xproto, target=self.target)
-
output = XOSProcessor.process(args)
- exec(output) # This loads the generated function, which should look like this:
+ exec(output, globals()) # This loads the generated function, which should look like this:
"""
def output_security_check(obj, ctx):
@@ -192,9 +183,8 @@
policy output < exists Privilege: Privilege.object_id = obj.id >
"""
args = XOSProcessorArgs(inputs=xproto, target=self.target)
-
output = XOSProcessor.process(args)
- exec(output) # This loads the generated function, which should look like this:
+ exec(output, globals()) # This loads the generated function, which should look like this:
"""
def output_security_check(obj, ctx):
@@ -210,7 +200,7 @@
"""
args = XOSProcessorArgs(inputs=xproto, target=self.target)
output = XOSProcessor.process(args)
- exec(output) # This loads the generated function, which should look like this:
+ exec(output, globals()) # This loads the generated function, which should look like this:
"""
def output_security_check(obj, ctx):
@@ -228,15 +218,15 @@
"""
args = XOSProcessorArgs(inputs=xproto, target=self.target)
-
output = XOSProcessor.process(args)
+ exec(output, globals())
+
"""
def output_security_check(obj, ctx):
i2 = Credential.objects.filter((~ Q(obj_id=obj_id)))[0]
i1 = (not i2)
return i1
"""
- exec(output)
if __name__ == "__main__":
diff --git a/lib/xos-genx/xos-genx-tests/test_general_validation.py b/lib/xos-genx/xos-genx-tests/test_general_validation.py
index 0e2a785..a2b5924 100644
--- a/lib/xos-genx/xos-genx-tests/test_general_validation.py
+++ b/lib/xos-genx/xos-genx-tests/test_general_validation.py
@@ -13,6 +13,7 @@
# limitations under the License.
+from __future__ import absolute_import
import unittest
from xosgenx.generator import XOSProcessor, XOSProcessorArgs
from helpers import XProtoTestHelpers, FakeObject
diff --git a/lib/xos-genx/xos-genx-tests/test_generator.py b/lib/xos-genx/xos-genx-tests/test_generator.py
index 1de6bd8..8212d51 100644
--- a/lib/xos-genx/xos-genx-tests/test_generator.py
+++ b/lib/xos-genx/xos-genx-tests/test_generator.py
@@ -13,6 +13,7 @@
# limitations under the License.
+from __future__ import absolute_import
import unittest
import os
from helpers import OUTPUT_DIR
diff --git a/lib/xos-genx/xos-genx-tests/test_graph.py b/lib/xos-genx/xos-genx-tests/test_graph.py
index c6cfea7..1e73955 100644
--- a/lib/xos-genx/xos-genx-tests/test_graph.py
+++ b/lib/xos-genx/xos-genx-tests/test_graph.py
@@ -13,6 +13,7 @@
# limitations under the License.
+from __future__ import absolute_import
import unittest
from xosgenx.generator import XOSProcessor, XOSProcessorArgs
from helpers import XProtoTestHelpers
diff --git a/lib/xos-genx/xos-genx-tests/test_jinja2_base.py b/lib/xos-genx/xos-genx-tests/test_jinja2_base.py
index 859d640..3b988e5 100644
--- a/lib/xos-genx/xos-genx-tests/test_jinja2_base.py
+++ b/lib/xos-genx/xos-genx-tests/test_jinja2_base.py
@@ -13,6 +13,7 @@
# limitations under the License.
+from __future__ import absolute_import
import unittest
from xosgenx.jinja2_extensions.base import *
diff --git a/lib/xos-genx/xos-genx-tests/test_jinja2_django.py b/lib/xos-genx/xos-genx-tests/test_jinja2_django.py
index d5da2d3..9b993df 100644
--- a/lib/xos-genx/xos-genx-tests/test_jinja2_django.py
+++ b/lib/xos-genx/xos-genx-tests/test_jinja2_django.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
import unittest
from xosgenx.jinja2_extensions.django import *
diff --git a/lib/xos-genx/xos-genx-tests/test_optimize.py b/lib/xos-genx/xos-genx-tests/test_optimize.py
index c86b736..5791542 100644
--- a/lib/xos-genx/xos-genx-tests/test_optimize.py
+++ b/lib/xos-genx/xos-genx-tests/test_optimize.py
@@ -13,6 +13,7 @@
# limitations under the License.
+from __future__ import absolute_import
import unittest
from xosgenx.jinja2_extensions.fol2 import FOL2Python
diff --git a/lib/xos-genx/xos-genx-tests/test_package.py b/lib/xos-genx/xos-genx-tests/test_package.py
index 03911bd..d364431 100644
--- a/lib/xos-genx/xos-genx-tests/test_package.py
+++ b/lib/xos-genx/xos-genx-tests/test_package.py
@@ -13,6 +13,7 @@
# limitations under the License.
+from __future__ import absolute_import
import unittest
import os
from xosgenx.generator import XOSProcessor, XOSProcessorArgs
diff --git a/lib/xos-genx/xos-genx-tests/test_parse.py b/lib/xos-genx/xos-genx-tests/test_parse.py
index 8d1ccf5..ade625a 100644
--- a/lib/xos-genx/xos-genx-tests/test_parse.py
+++ b/lib/xos-genx/xos-genx-tests/test_parse.py
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
+from __future__ import absolute_import
import unittest
from xosgenx.generator import XOSProcessor, XOSProcessorArgs
from helpers import XProtoTestHelpers
diff --git a/lib/xos-genx/xos-genx-tests/test_policy.py b/lib/xos-genx/xos-genx-tests/test_policy.py
index e8b5a76..9cac78a 100644
--- a/lib/xos-genx/xos-genx-tests/test_policy.py
+++ b/lib/xos-genx/xos-genx-tests/test_policy.py
@@ -13,10 +13,10 @@
# limitations under the License.
+from __future__ import absolute_import
import unittest
from xosgenx.generator import XOSProcessor, XOSProcessorArgs
from helpers import FakeObject, XProtoTestHelpers
-import pdb
"""
The tests below convert the policy logic expression
diff --git a/lib/xos-genx/xos-genx-tests/test_pure_proto.py b/lib/xos-genx/xos-genx-tests/test_pure_proto.py
index c4f680d..fdf1fe7 100644
--- a/lib/xos-genx/xos-genx-tests/test_pure_proto.py
+++ b/lib/xos-genx/xos-genx-tests/test_pure_proto.py
@@ -13,6 +13,7 @@
# limitations under the License.
+from __future__ import absolute_import
import unittest
from xosgenx.generator import XOSProcessor, XOSProcessorArgs
from helpers import XProtoTestHelpers
diff --git a/lib/xos-genx/xos-genx-tests/test_rlinks.py b/lib/xos-genx/xos-genx-tests/test_rlinks.py
index c0ad406..67e31dd 100644
--- a/lib/xos-genx/xos-genx-tests/test_rlinks.py
+++ b/lib/xos-genx/xos-genx-tests/test_rlinks.py
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
+from __future__ import absolute_import
import unittest
from xosgenx.generator import XOSProcessor, XOSProcessorArgs
from helpers import XProtoTestHelpers
diff --git a/lib/xos-genx/xos-genx-tests/test_swagger.py b/lib/xos-genx/xos-genx-tests/test_swagger.py
index 3af997e..1886a76 100644
--- a/lib/xos-genx/xos-genx-tests/test_swagger.py
+++ b/lib/xos-genx/xos-genx-tests/test_swagger.py
@@ -12,10 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
+from __future__ import absolute_import
import unittest
-
-import yaml
from xosgenx.generator import XOSProcessor, XOSProcessorArgs
from helpers import OUTPUT_DIR
diff --git a/lib/xos-genx/xos-genx-tests/test_target.py b/lib/xos-genx/xos-genx-tests/test_target.py
index c468018..4e70de7 100644
--- a/lib/xos-genx/xos-genx-tests/test_target.py
+++ b/lib/xos-genx/xos-genx-tests/test_target.py
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
+from __future__ import absolute_import
import unittest
import os
from xosgenx.generator import XOSProcessor, XOSProcessorArgs
diff --git a/lib/xos-genx/xos-genx-tests/test_tosca.py b/lib/xos-genx/xos-genx-tests/test_tosca.py
index c5a0f17..6e585f5 100644
--- a/lib/xos-genx/xos-genx-tests/test_tosca.py
+++ b/lib/xos-genx/xos-genx-tests/test_tosca.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
import unittest
from xosgenx.generator import XOSProcessor, XOSProcessorArgs
from helpers import XProtoTestHelpers
@@ -145,7 +146,7 @@
args.inputs = xproto
args.target = self.target_tosca_keys
output = XOSProcessor.process(args)
- self.assertIn("['name', ['key_4', 'key_3'], ['key_1', 'key_2']]", output)
+ self.assertIn("['name', ['key_1', 'key_2'], ['key_3', 'key_4']]", output)
xproto = """
option app_label = "test";
@@ -163,4 +164,4 @@
args.inputs = xproto
output = XOSProcessor.process(args)
- self.assertIn("['name', ['key_1_id', 'key_3_id', 'key_2_id']]", output)
+ self.assertIn("['name', ['key_1_id', 'key_2_id', 'key_3_id']]", output)
diff --git a/lib/xos-genx/xos-genx-tests/test_translator.py b/lib/xos-genx/xos-genx-tests/test_translator.py
index 320021b..f98894b 100644
--- a/lib/xos-genx/xos-genx-tests/test_translator.py
+++ b/lib/xos-genx/xos-genx-tests/test_translator.py
@@ -13,6 +13,7 @@
# limitations under the License.
+from __future__ import absolute_import
import unittest
import os
from xosgenx.generator import XOSProcessor, XOSProcessorArgs
@@ -335,7 +336,7 @@
args.target = "modeldefs.xtarget"
output = XOSProcessor.process(args)
- read_only = filter(lambda s: "read_only: True" in s, output.splitlines())
+ read_only = [s for s in output.splitlines() if "read_only: True" in s]
self.assertEqual(len(read_only), 3) # readonly is 1 for ParentFoo and 2 for Foo
diff --git a/lib/xos-genx/xos-genx-tests/test_xos_security.py b/lib/xos-genx/xos-genx-tests/test_xos_security.py
index 3bd4653..b75f3c7 100644
--- a/lib/xos-genx/xos-genx-tests/test_xos_security.py
+++ b/lib/xos-genx/xos-genx-tests/test_xos_security.py
@@ -12,29 +12,29 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
+from __future__ import absolute_import
import unittest
from xosgenx.generator import XOSProcessor, XOSProcessorArgs
from helpers import XProtoTestHelpers
-"""The function below is for eliminating warnings arising due to the missing policy_output_enforcer,
-which is generated and loaded dynamically.
-"""
-
def policy_output_enforcer(x, y):
+ """
+ eliminating warnings arising due to the missing policy_output_enforcer,
+ which is generated and loaded dynamically.
+ """
raise Exception("Security enforcer not generated. Test failed.")
return False
-"""
-The tests below use the Python code target to generate
-Python security policies, set up an appropriate environment and execute the Python.
-The security policies here deliberately made complex in order to stress the processor.
-"""
-
class XProtoXOSSecurityTest(unittest.TestCase):
+ """
+ Use the Python code target to generate Python security policies, set up an
+ appropriate environment and execute the Python. The security policies here
+ deliberately made complex in order to stress the processor.
+ """
+
def setUp(self):
self.target = XProtoTestHelpers.write_tmp_target(
"{{ xproto_fol_to_python_test('output',proto.policies.test_policy, None, '0') }}"
diff --git a/lib/xos-genx/xos-genx-tests/test_xos_validation.py b/lib/xos-genx/xos-genx-tests/test_xos_validation.py
index 257eb4d..65555a3 100644
--- a/lib/xos-genx/xos-genx-tests/test_xos_validation.py
+++ b/lib/xos-genx/xos-genx-tests/test_xos_validation.py
@@ -12,28 +12,26 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
+from __future__ import absolute_import
import unittest
from xosgenx.generator import XOSProcessor, XOSProcessorArgs
from helpers import FakeObject, XProtoTestHelpers
-"""The function below is for eliminating warnings arising due to the missing policy_output_validator,
-which is generated and loaded dynamically.
-"""
-
def policy_output_validator(x, y):
+ """
+ For eliminating warnings arising due to the missing
+ policy_output_validator, which is generated and loaded dynamically.
+ """
raise Exception("Validator not generated. Test failed.")
return False
-
-"""
-The tests below use the Python code target to generate
-Python validation policies, set up an appropriate environment and execute the Python.
-"""
-
-
class XProtoXOSModelValidationTest(unittest.TestCase):
+ """
+ Use the Python code target to generate Python validation policies, set up an
+ appropriate environment and execute the Python.
+ """
+
def setUp(self):
self.target = XProtoTestHelpers.write_tmp_target(
"{{ xproto_fol_to_python_validator('output', proto.policies.test_policy, None, 'Necessary Failure') }}"