[SEBA-450] (part 1)
Refactoring, python3 compat, and tox tests on:
- xosconfig
- xosgenx
- xosutil
Eliminate use of yaml.load() which is unsafe, switch to yaml.safe_load()
More diagnostics during database migration
Change-Id: I0fae5782fca401603a7c4e4ec2b9269ad24bda97
diff --git a/.gitignore b/.gitignore
index 23ed2a9..088aa5e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -38,13 +38,13 @@
# testing/coverage files
nose2-junit.xml
+nose2-results.xml
.coverage
-coverage.xml
+*coverage.xml
+cover
+.tox
-lib/xos-config/cover/
-lib/xos-genx/cover/
-.vscode
-
-lib/xos-kafka/build/
-lib/xos-kafka/dist/
-lib/xos-kafka/xoskafka.egg-info/
\ No newline at end of file
+# python module build files
+*.egg-info
+dist
+build
diff --git a/VERSION b/VERSION
index 21bb5e1..bda8fbe 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.2.5
+2.2.6
diff --git a/containers/chameleon/Dockerfile.chameleon b/containers/chameleon/Dockerfile.chameleon
index 7cc050d..ee2ddbe 100644
--- a/containers/chameleon/Dockerfile.chameleon
+++ b/containers/chameleon/Dockerfile.chameleon
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/chameleon
-FROM xosproject/xos-base:2.2.5
+FROM xosproject/xos-base:2.2.6
# xos-base already has protoc and dependencies installed
diff --git a/containers/xos/Dockerfile.client b/containers/xos/Dockerfile.client
index e995605..f7fcd50 100644
--- a/containers/xos/Dockerfile.client
+++ b/containers/xos/Dockerfile.client
@@ -13,13 +13,13 @@
# limitations under the License.
# xosproject/xos-client
-FROM xosproject/xos-libraries:2.2.5
+FROM xosproject/xos-libraries:2.2.6
# Install XOS client
COPY lib/xos-api /tmp/xos-api
# Install the VERSION file
-COPY VERSION tmp/xos-api
+COPY VERSION /tmp/xos-api/VERSION
# install the client library and xossh
RUN chdir /tmp/xos-api && python ./setup.py install
diff --git a/containers/xos/Dockerfile.libraries b/containers/xos/Dockerfile.libraries
index 3b963d5..a7ecf03 100644
--- a/containers/xos/Dockerfile.libraries
+++ b/containers/xos/Dockerfile.libraries
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/xos-libraries
-FROM xosproject/xos-base:2.2.5
+FROM xosproject/xos-base:2.2.6
# Add libraries
COPY lib /opt/xos/lib
diff --git a/containers/xos/Dockerfile.synchronizer-base b/containers/xos/Dockerfile.synchronizer-base
index 58c5a07..64dd9cb 100644
--- a/containers/xos/Dockerfile.synchronizer-base
+++ b/containers/xos/Dockerfile.synchronizer-base
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/xos-synchronizer-base
-FROM xosproject/xos-client:2.2.5
+FROM xosproject/xos-client:2.2.6
COPY xos/synchronizers/new_base /opt/xos/synchronizers/new_base
COPY xos/xos/logger.py /opt/xos/xos/logger.py
@@ -25,7 +25,7 @@
# For library-based synchronizers
COPY lib/xos-synchronizer /tmp/xos-synchronizer
-COPY VERSION tmp/xos-synchronizer
+COPY VERSION /tmp/xos-synchronizer/VERSION
RUN cd /tmp/xos-synchronizer && python setup.py install
# Label image
diff --git a/containers/xos/Dockerfile.xos-core b/containers/xos/Dockerfile.xos-core
index f8a356d..c1aa1d0 100644
--- a/containers/xos/Dockerfile.xos-core
+++ b/containers/xos/Dockerfile.xos-core
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/xos-core
-FROM xosproject/xos-libraries:2.2.5
+FROM xosproject/xos-libraries:2.2.6
# Install XOS
ADD xos /opt/xos
diff --git a/lib/xos-config/.gitignore b/lib/xos-config/.gitignore
index 10432e2..a04aaf4 100644
--- a/lib/xos-config/.gitignore
+++ b/lib/xos-config/.gitignore
@@ -1,7 +1,2 @@
-.noseids
-build
-XosConfig.egg-info
-dist
-.coverage
-coverage.xml
-cover
\ No newline at end of file
+# setup.py copies this, don't commit it
+xosconfig/VERSION
diff --git a/lib/xos-config/MANIFEST.in b/lib/xos-config/MANIFEST.in
index 2887117..0bb6996 100644
--- a/lib/xos-config/MANIFEST.in
+++ b/lib/xos-config/MANIFEST.in
@@ -1,2 +1,4 @@
+include requirements.txt
+include xosconfig/VERSION
+include xosconfig/synchronizer-config-schema.yaml
include xosconfig/xos-config-schema.yaml
-include xosconfig/synchronizer-config-schema.yaml
\ No newline at end of file
diff --git a/lib/xos-config/requirements.txt b/lib/xos-config/requirements.txt
new file mode 100644
index 0000000..5528415
--- /dev/null
+++ b/lib/xos-config/requirements.txt
@@ -0,0 +1,2 @@
+PyYAML~=3.12
+pykwalify~=1.6.0
diff --git a/lib/xos-config/setup.py b/lib/xos-config/setup.py
index 1e683f1..57f2939 100644
--- a/lib/xos-config/setup.py
+++ b/lib/xos-config/setup.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,23 +12,40 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-try:
- from xosutil.autoversion_setup import setup_with_auto_version as setup
-except ImportError:
- # xosutil is not installed. Expect this to happen when we build an egg, in which case xosgenx.version will
- # automatically have the right version.
- from setuptools import setup
+from __future__ import absolute_import
-from xosconfig.version import __version__
+import os
+from shutil import copyfile
+
+from setuptools import setup
+
+
+def version():
+ # Copy VERSION file of parent to module directory if not found
+ if not os.path.exists("xosconfig/VERSION"):
+ copyfile("../../VERSION", "xosconfig/VERSION")
+ with open("xosconfig/VERSION") as f:
+ return f.read().strip()
+
+
+def parse_requirements(filename):
+ # parse a requirements.txt file, allowing for blank lines and comments
+ requirements = []
+ for line in open(filename):
+ if line and line.startswith("#"):
+ requirements.append(line)
+ return requirements
+
setup(
- name="XosConfig",
- version=__version__,
+ name="xosconfig",
+ version=version(),
description="XOS Config Library",
author="Matteo Scandolo",
- author_email="teo@onlab.us",
+ author_email="teo@opennetworking.org",
+ classifiers=["License :: OSI Approved :: Apache Software License"],
+ license="Apache v2",
packages=["xosconfig"],
+ install_requires=parse_requirements("requirements.txt"),
include_package_data=True,
- # TODO add all deps to the install_requires section
- install_requires=["pykwalify>=1.6.0"],
)
diff --git a/lib/xos-config/tests/test_config.py b/lib/xos-config/tests/test_config.py
index 5eb86af..edec0b5 100644
--- a/lib/xos-config/tests/test_config.py
+++ b/lib/xos-config/tests/test_config.py
@@ -12,9 +12,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
-import unittest
import os
+import unittest
+
from xosconfig import Config
from xosconfig import Config as Config2
@@ -79,7 +81,7 @@
with self.assertRaises(Exception) as e:
Config.init(sample_conf)
Config2.init(sample_conf)
- self.assertEqual(e.exception.message, "[XOS-Config] Module already initialized")
+ self.assertEqual(str(e.exception), "[XOS-Config] Module already initialized")
def test_config_not_initialized(self):
"""
@@ -88,7 +90,7 @@
with self.assertRaises(Exception) as e:
Config.get("database")
self.assertEqual(
- e.exception.message, "[XOS-Config] Module has not been initialized"
+ str(e.exception), "[XOS-Config] Module has not been initialized"
)
def test_missing_file_exception(self):
@@ -98,7 +100,7 @@
with self.assertRaises(Exception) as e:
Config.init("missing_conf")
self.assertEqual(
- e.exception.message, "[XOS-Config] Config file not found at: missing_conf"
+ str(e.exception), "[XOS-Config] Config file not found at: missing_conf"
)
def test_yaml_not_valid(self):
@@ -108,7 +110,7 @@
with self.assertRaises(Exception) as e:
Config.init(yaml_not_valid)
self.assertTrue(
- e.exception.message.startswith("[XOS-Config] The config format is wrong:")
+ str(e.exception).startswith("[XOS-Config] The config format is wrong:")
)
def test_invalid_format(self):
@@ -118,7 +120,7 @@
with self.assertRaises(Exception) as e:
Config.init(invalid_format)
self.assertEqual(
- e.exception.message,
+ str(e.exception),
(
"[XOS-Config] The config format is wrong: Schema validation failed:\n"
" - Value '['I am', 'a yaml', 'but the', 'format is not', 'correct']' is not a dict. Value path: ''."
@@ -133,7 +135,7 @@
with self.assertRaises(Exception) as e:
Config.init("missing_conf")
self.assertEqual(
- e.exception.message, "[XOS-Config] Config file not found at: env.yaml"
+ str(e.exception), "[XOS-Config] Config file not found at: env.yaml"
)
del os.environ["XOS_CONFIG_FILE"]
@@ -145,10 +147,10 @@
with self.assertRaises(Exception) as e:
Config.init(basic_conf)
self.assertRegexpMatches(
- e.exception.message,
+ str(e.exception),
r"\[XOS\-Config\] Config schema not found at: (.+)env-schema\.yaml",
)
- # self.assertEqual(e.exception.message, "[XOS-Config] Config schema not found at: env-schema.yaml")
+ # self.assertEqual(str(e.exception), "[XOS-Config] Config schema not found at: env-schema.yaml")
del os.environ["XOS_CONFIG_SCHEMA"]
def test_schema_override_usage(self):
@@ -159,7 +161,7 @@
with self.assertRaises(Exception) as e:
Config.init(basic_conf)
self.assertEqual(
- e.exception.message,
+ str(e.exception),
(
"[XOS-Config] The config format is wrong: Schema validation failed:\n"
" - Key 'database' was not defined. Path: ''."
diff --git a/lib/xos-config/tox.ini b/lib/xos-config/tox.ini
new file mode 100644
index 0000000..55981ac
--- /dev/null
+++ b/lib/xos-config/tox.ini
@@ -0,0 +1,43 @@
+; Copyright 2019-present Open Networking Foundation
+;
+; Licensed under the Apache License, Version 2.0 (the "License");
+; you may not use this file except in compliance with the License.
+; You may obtain a copy of the License at
+;
+; http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing, software
+; distributed under the License is distributed on an "AS IS" BASIS,
+; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+; See the License for the specific language governing permissions and
+; limitations under the License.
+
+[tox]
+envlist = py27,py35,py36,py37
+skip_missing_interpreters = true
+
+[testenv]
+deps =
+ -r requirements.txt
+ nose2
+ flake8
+
+commands =
+ nose2 -c tox.ini --verbose --junit-xml
+ flake8
+
+[flake8]
+max-line-length = 119
+
+[unittest]
+plugins=nose2.plugins.junitxml
+
+[junit-xml]
+path=nose2-results.xml
+
+[coverage]
+always-on = True
+coverage = xosconfig
+coverage-report =
+ xml
+ term
diff --git a/lib/xos-config/xosconfig/__init__.py b/lib/xos-config/xosconfig/__init__.py
index 9a0b30c..0ae9199 100644
--- a/lib/xos-config/xosconfig/__init__.py
+++ b/lib/xos-config/xosconfig/__init__.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
from .config import Config
__all__ = ["Config"]
diff --git a/lib/xos-config/xosconfig/config.py b/lib/xos-config/xosconfig/config.py
index aac6ffb..1fe6707 100644
--- a/lib/xos-config/xosconfig/config.py
+++ b/lib/xos-config/xosconfig/config.py
@@ -13,12 +13,16 @@
# limitations under the License.
+from __future__ import absolute_import
+
import os
import sys
-import yaml
-import default
-from pykwalify.core import Core as PyKwalify
+
import pykwalify
+import yaml
+from pykwalify.core import Core as PyKwalify
+
+from . import default
pykwalify.init_logging(1)
diff --git a/lib/xos-genx/.gitignore b/lib/xos-genx/.gitignore
index b1547d1..9c6e673 100644
--- a/lib/xos-genx/.gitignore
+++ b/lib/xos-genx/.gitignore
@@ -1,6 +1,2 @@
-build
-dist
-XosGenX.egg-info
-.coverage
-coverage.xml
-cover
\ No newline at end of file
+# setup.py copies this, don't commit it
+xosgenx/VERSION
diff --git a/lib/xos-genx/MANIFEST.in b/lib/xos-genx/MANIFEST.in
index a058bb4..d530ad3 100644
--- a/lib/xos-genx/MANIFEST.in
+++ b/lib/xos-genx/MANIFEST.in
@@ -1,3 +1,5 @@
-include xosgenx/targets/*
+include requirements.txt
+include xosgenx/VERSION
include xosgenx/checkers/*
include xosgenx/jinja2_extensions/*
+include xosgenx/targets/*
diff --git a/lib/xos-genx/requirements.txt b/lib/xos-genx/requirements.txt
new file mode 100644
index 0000000..b210781
--- /dev/null
+++ b/lib/xos-genx/requirements.txt
@@ -0,0 +1,8 @@
+Jinja2~=2.10
+PyYAML~=3.12
+astunparse~=1.5.0
+colorama~=0.4.1
+inflect~=1.0.1
+plyxproto~=4.0.0
+ply~=3.11
+six~=1.12.0
diff --git a/lib/xos-genx/setup.py b/lib/xos-genx/setup.py
index c294ca4..f1bc5a5 100644
--- a/lib/xos-genx/setup.py
+++ b/lib/xos-genx/setup.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,24 +12,41 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-try:
- from xosutil.autoversion_setup import setup_with_auto_version as setup
-except ImportError:
- # xosutil is not installed. Expect this to happen when we build an egg, in which case xosgenx.version will
- # automatically have the right version.
- from setuptools import setup
+from __future__ import absolute_import
-from xosgenx.version import __version__
+import os
+from shutil import copyfile
+
+from setuptools import setup
+
+
+def version():
+ # Copy VERSION file of parent to module directory if not found
+ if not os.path.exists("xosgenx/VERSION"):
+ copyfile("../../VERSION", "xosgenx/VERSION")
+ with open("xosgenx/VERSION") as f:
+ return f.read().strip()
+
+
+def parse_requirements(filename):
+ # parse a requirements.txt file, allowing for blank lines and comments
+ requirements = []
+ for line in open(filename):
+ if line and line.startswith("#"):
+ requirements.append(line)
+ return requirements
+
setup(
- name="XosGenX",
- version=__version__,
+ name="xosgenx",
+ version=version(),
description="XOS Generative Toolchain",
author="Sapan Bhatia, Matteo Scandolo",
author_email="sapan@opennetworking.org, teo@opennetworking.org",
+ classifiers=["License :: OSI Approved :: Apache Software License"],
+ license="Apache v2",
packages=["xosgenx"],
scripts=["bin/xosgenx"],
+ install_requires=parse_requirements("requirements.txt"),
include_package_data=True,
- # TODO add all deps to the install_requires section
- install_requires=["inflect>=1.0.1", "astunparse>=1.5.0"],
)
diff --git a/lib/xos-genx/tox.ini b/lib/xos-genx/tox.ini
index 56c85a5..7b85e32 100644
--- a/lib/xos-genx/tox.ini
+++ b/lib/xos-genx/tox.ini
@@ -12,10 +12,34 @@
; See the License for the specific language governing permissions and
; limitations under the License.
-; using tox.ini to store config information for other tools
+[tox]
+envlist = py27,py35,py36,py37
+skip_missing_interpreters = true
+
+[testenv]
+deps =
+ -r requirements.txt
+ nose2
+ flake8
+ mock
+
+commands =
+ nose2 -c tox.ini --verbose --junit-xml
+; only check the code in xosgenx, tests aren't flake8 clean yet
+ flake8 xosgenx
[flake8]
-; F821, ignoring undefined names would be valuable, but our testing dynamically loads them
-; W503, allow breaks before binary operators (see: https://github.com/PyCQA/pycodestyle/issues/498)
-ignore = F821, W503
-max-line-length = 200
+max-line-length = 119
+
+[unittest]
+plugins=nose2.plugins.junitxml
+
+[junit-xml]
+path=nose2-results.xml
+
+[coverage]
+always-on = True
+coverage = xosgenx
+coverage-report =
+ xml
+ term
diff --git a/lib/xos-genx/xos-genx-tests/helpers.py b/lib/xos-genx/xos-genx-tests/helpers.py
index 4687bb3..e6d87ab 100644
--- a/lib/xos-genx/xos-genx-tests/helpers.py
+++ b/lib/xos-genx/xos-genx-tests/helpers.py
@@ -13,6 +13,7 @@
# limitations under the License.
+from __future__ import absolute_import
import os
# Constants
diff --git a/lib/xos-genx/xos-genx-tests/test_cli.py b/lib/xos-genx/xos-genx-tests/test_cli.py
index 3f94865..087c904 100644
--- a/lib/xos-genx/xos-genx-tests/test_cli.py
+++ b/lib/xos-genx/xos-genx-tests/test_cli.py
@@ -13,6 +13,7 @@
# limitations under the License.
+from __future__ import absolute_import
import unittest
import os
from mock import patch
diff --git a/lib/xos-genx/xos-genx-tests/test_django_generator.py b/lib/xos-genx/xos-genx-tests/test_django_generator.py
index a81f80c..022a640 100644
--- a/lib/xos-genx/xos-genx-tests/test_django_generator.py
+++ b/lib/xos-genx/xos-genx-tests/test_django_generator.py
@@ -13,6 +13,7 @@
# limitations under the License.
+from __future__ import absolute_import
import unittest
import os
from xosgenx.generator import XOSProcessor, XOSProcessorArgs
@@ -32,9 +33,9 @@
args = XOSProcessorArgs(files=[VROUTER_XPROTO], target="django.xtarget")
output = XOSProcessor.process(args)
- fields = filter(lambda s: "Field(" in s, output.splitlines())
+ fields = [s for s in output.splitlines() if "Field(" in s]
self.assertEqual(len(fields), 2)
- links = filter(lambda s: "Key(" in s, output.splitlines())
+ links = [s for s in output.splitlines() if "Key(" in s]
self.assertEqual(len(links), 2)
def test_optional_relations(self):
@@ -59,11 +60,11 @@
args = XOSProcessorArgs(inputs=xproto, target="django.xtarget")
output = XOSProcessor.process(args)
- null_true = filter(lambda s: "null = True" in s, output.splitlines())
- null_false = filter(lambda s: "null = False" in s, output.splitlines())
+ null_true = [s for s in output.splitlines() if "null = True" in s]
+ null_false = [s for s in output.splitlines() if "null = False" in s]
- blank_true = filter(lambda s: "blank = True" in s, output.splitlines())
- blank_false = filter(lambda s: "blank = False" in s, output.splitlines())
+ blank_true = [s for s in output.splitlines() if "blank = True" in s]
+ blank_false = [s for s in output.splitlines() if "blank = False" in s]
self.assertEqual(len(null_true), 1)
self.assertEqual(len(null_false), 1)
diff --git a/lib/xos-genx/xos-genx-tests/test_field_graph.py b/lib/xos-genx/xos-genx-tests/test_field_graph.py
index cfb4c8b..c3c08a7 100644
--- a/lib/xos-genx/xos-genx-tests/test_field_graph.py
+++ b/lib/xos-genx/xos-genx-tests/test_field_graph.py
@@ -13,10 +13,11 @@
# limitations under the License.
+from __future__ import absolute_import
import unittest
from xosgenx.jinja2_extensions import FieldNotFound
-from helpers import XProtoTestHelpers
from xosgenx.generator import XOSProcessor, XOSProcessorArgs
+from helpers import XProtoTestHelpers
from functools import reduce
@@ -75,7 +76,7 @@
generate()
self.assertEqual(
- e.exception.message,
+ str(e.exception),
'Field "B" not found in model "Foo", referenced from field "A" by option "unique_with"',
)
diff --git a/lib/xos-genx/xos-genx-tests/test_general_security.py b/lib/xos-genx/xos-genx-tests/test_general_security.py
index c675b16..26a4736 100644
--- a/lib/xos-genx/xos-genx-tests/test_general_security.py
+++ b/lib/xos-genx/xos-genx-tests/test_general_security.py
@@ -12,28 +12,32 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
+from __future__ import absolute_import
+from __future__ import print_function
import unittest
from xosgenx.generator import XOSProcessor, XOSProcessorArgs
from helpers import XProtoTestHelpers, FakeObject
-"""The function below is for eliminating warnings arising due to the missing output_security_check,
-which is generated and loaded dynamically.
-"""
-
def output_security_check(x, y):
+ """
+ This function eliminates warnings arising due to the missing
+ output_security_check, which is generated and loaded dynamically. This is
+ defined in the global namespace, and in python3 the globals() namespace has
+ to be passed to calls to "exec()" for the xproto-generated version to
+ redefine this function.
+ """
+
raise Exception("Security enforcer not generated. Test failed.")
return False
-"""
-The tests below use the Python code target to generate
-Python security policies, set up an appropriate environment and execute the Python.
-"""
-
-
class XProtoSecurityTest(unittest.TestCase):
+ """
+ Use the Python code target to generate Python security policies, set up an
+ appropriate environment and execute the Python.
+ """
+
def setUp(self):
self.target = XProtoTestHelpers.write_tmp_target(
"""
@@ -48,10 +52,8 @@
policy output < True >
"""
args = XOSProcessorArgs(inputs=xproto, target=self.target)
-
output = XOSProcessor.process(args)
-
- exec(output) # This loads the generated function, which should look like this:
+ exec(output, globals()) # This loads the generated function, which should look like this:
"""
def output_security_check(obj, ctx):
@@ -68,10 +70,8 @@
"""
args = XOSProcessorArgs(inputs=xproto, target=self.target)
-
output = XOSProcessor.process(args)
-
- exec(output) # This loads the generated function, which should look like this:
+ exec(output, globals()) # This loads the generated function, which should look like this:
"""
def output_security_check(obj, ctx):
@@ -93,12 +93,8 @@
"""
args = XOSProcessorArgs(inputs=xproto, target=self.target)
-
output = XOSProcessor.process(args)
-
- exec(
- output, globals()
- ) # This loads the generated function, which should look like this:
+ exec(output, globals()) # This loads the generated function, which should look like this:
"""
def sub_policy_security_check(obj, ctx):
@@ -130,12 +126,8 @@
"""
args = XOSProcessorArgs(inputs=xproto, target=self.target)
-
output = XOSProcessor.process(args)
-
- exec(
- output, globals()
- ) # This loads the generated function, which should look like this:
+ exec(output, globals()) # This loads the generated function, which should look like this:
"""
def sub_policy_security_check(obj, ctx):
@@ -165,9 +157,8 @@
"""
args = XOSProcessorArgs(inputs=xproto, target=self.target)
-
output = XOSProcessor.process(args)
- exec(output) # This loads the generated function, which should look like this:
+ exec(output, globals()) # This loads the generated function, which should look like this:
"""
def output_security_check(obj, ctx):
@@ -192,9 +183,8 @@
policy output < exists Privilege: Privilege.object_id = obj.id >
"""
args = XOSProcessorArgs(inputs=xproto, target=self.target)
-
output = XOSProcessor.process(args)
- exec(output) # This loads the generated function, which should look like this:
+ exec(output, globals()) # This loads the generated function, which should look like this:
"""
def output_security_check(obj, ctx):
@@ -210,7 +200,7 @@
"""
args = XOSProcessorArgs(inputs=xproto, target=self.target)
output = XOSProcessor.process(args)
- exec(output) # This loads the generated function, which should look like this:
+ exec(output, globals()) # This loads the generated function, which should look like this:
"""
def output_security_check(obj, ctx):
@@ -228,15 +218,15 @@
"""
args = XOSProcessorArgs(inputs=xproto, target=self.target)
-
output = XOSProcessor.process(args)
+ exec(output, globals())
+
"""
def output_security_check(obj, ctx):
i2 = Credential.objects.filter((~ Q(obj_id=obj_id)))[0]
i1 = (not i2)
return i1
"""
- exec(output)
if __name__ == "__main__":
diff --git a/lib/xos-genx/xos-genx-tests/test_general_validation.py b/lib/xos-genx/xos-genx-tests/test_general_validation.py
index 0e2a785..a2b5924 100644
--- a/lib/xos-genx/xos-genx-tests/test_general_validation.py
+++ b/lib/xos-genx/xos-genx-tests/test_general_validation.py
@@ -13,6 +13,7 @@
# limitations under the License.
+from __future__ import absolute_import
import unittest
from xosgenx.generator import XOSProcessor, XOSProcessorArgs
from helpers import XProtoTestHelpers, FakeObject
diff --git a/lib/xos-genx/xos-genx-tests/test_generator.py b/lib/xos-genx/xos-genx-tests/test_generator.py
index 1de6bd8..8212d51 100644
--- a/lib/xos-genx/xos-genx-tests/test_generator.py
+++ b/lib/xos-genx/xos-genx-tests/test_generator.py
@@ -13,6 +13,7 @@
# limitations under the License.
+from __future__ import absolute_import
import unittest
import os
from helpers import OUTPUT_DIR
diff --git a/lib/xos-genx/xos-genx-tests/test_graph.py b/lib/xos-genx/xos-genx-tests/test_graph.py
index c6cfea7..1e73955 100644
--- a/lib/xos-genx/xos-genx-tests/test_graph.py
+++ b/lib/xos-genx/xos-genx-tests/test_graph.py
@@ -13,6 +13,7 @@
# limitations under the License.
+from __future__ import absolute_import
import unittest
from xosgenx.generator import XOSProcessor, XOSProcessorArgs
from helpers import XProtoTestHelpers
diff --git a/lib/xos-genx/xos-genx-tests/test_jinja2_base.py b/lib/xos-genx/xos-genx-tests/test_jinja2_base.py
index 859d640..3b988e5 100644
--- a/lib/xos-genx/xos-genx-tests/test_jinja2_base.py
+++ b/lib/xos-genx/xos-genx-tests/test_jinja2_base.py
@@ -13,6 +13,7 @@
# limitations under the License.
+from __future__ import absolute_import
import unittest
from xosgenx.jinja2_extensions.base import *
diff --git a/lib/xos-genx/xos-genx-tests/test_jinja2_django.py b/lib/xos-genx/xos-genx-tests/test_jinja2_django.py
index d5da2d3..9b993df 100644
--- a/lib/xos-genx/xos-genx-tests/test_jinja2_django.py
+++ b/lib/xos-genx/xos-genx-tests/test_jinja2_django.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
import unittest
from xosgenx.jinja2_extensions.django import *
diff --git a/lib/xos-genx/xos-genx-tests/test_optimize.py b/lib/xos-genx/xos-genx-tests/test_optimize.py
index c86b736..5791542 100644
--- a/lib/xos-genx/xos-genx-tests/test_optimize.py
+++ b/lib/xos-genx/xos-genx-tests/test_optimize.py
@@ -13,6 +13,7 @@
# limitations under the License.
+from __future__ import absolute_import
import unittest
from xosgenx.jinja2_extensions.fol2 import FOL2Python
diff --git a/lib/xos-genx/xos-genx-tests/test_package.py b/lib/xos-genx/xos-genx-tests/test_package.py
index 03911bd..d364431 100644
--- a/lib/xos-genx/xos-genx-tests/test_package.py
+++ b/lib/xos-genx/xos-genx-tests/test_package.py
@@ -13,6 +13,7 @@
# limitations under the License.
+from __future__ import absolute_import
import unittest
import os
from xosgenx.generator import XOSProcessor, XOSProcessorArgs
diff --git a/lib/xos-genx/xos-genx-tests/test_parse.py b/lib/xos-genx/xos-genx-tests/test_parse.py
index 8d1ccf5..ade625a 100644
--- a/lib/xos-genx/xos-genx-tests/test_parse.py
+++ b/lib/xos-genx/xos-genx-tests/test_parse.py
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
+from __future__ import absolute_import
import unittest
from xosgenx.generator import XOSProcessor, XOSProcessorArgs
from helpers import XProtoTestHelpers
diff --git a/lib/xos-genx/xos-genx-tests/test_policy.py b/lib/xos-genx/xos-genx-tests/test_policy.py
index e8b5a76..9cac78a 100644
--- a/lib/xos-genx/xos-genx-tests/test_policy.py
+++ b/lib/xos-genx/xos-genx-tests/test_policy.py
@@ -13,10 +13,10 @@
# limitations under the License.
+from __future__ import absolute_import
import unittest
from xosgenx.generator import XOSProcessor, XOSProcessorArgs
from helpers import FakeObject, XProtoTestHelpers
-import pdb
"""
The tests below convert the policy logic expression
diff --git a/lib/xos-genx/xos-genx-tests/test_pure_proto.py b/lib/xos-genx/xos-genx-tests/test_pure_proto.py
index c4f680d..fdf1fe7 100644
--- a/lib/xos-genx/xos-genx-tests/test_pure_proto.py
+++ b/lib/xos-genx/xos-genx-tests/test_pure_proto.py
@@ -13,6 +13,7 @@
# limitations under the License.
+from __future__ import absolute_import
import unittest
from xosgenx.generator import XOSProcessor, XOSProcessorArgs
from helpers import XProtoTestHelpers
diff --git a/lib/xos-genx/xos-genx-tests/test_rlinks.py b/lib/xos-genx/xos-genx-tests/test_rlinks.py
index c0ad406..67e31dd 100644
--- a/lib/xos-genx/xos-genx-tests/test_rlinks.py
+++ b/lib/xos-genx/xos-genx-tests/test_rlinks.py
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
+from __future__ import absolute_import
import unittest
from xosgenx.generator import XOSProcessor, XOSProcessorArgs
from helpers import XProtoTestHelpers
diff --git a/lib/xos-genx/xos-genx-tests/test_swagger.py b/lib/xos-genx/xos-genx-tests/test_swagger.py
index 3af997e..1886a76 100644
--- a/lib/xos-genx/xos-genx-tests/test_swagger.py
+++ b/lib/xos-genx/xos-genx-tests/test_swagger.py
@@ -12,10 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
+from __future__ import absolute_import
import unittest
-
-import yaml
from xosgenx.generator import XOSProcessor, XOSProcessorArgs
from helpers import OUTPUT_DIR
diff --git a/lib/xos-genx/xos-genx-tests/test_target.py b/lib/xos-genx/xos-genx-tests/test_target.py
index c468018..4e70de7 100644
--- a/lib/xos-genx/xos-genx-tests/test_target.py
+++ b/lib/xos-genx/xos-genx-tests/test_target.py
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
+from __future__ import absolute_import
import unittest
import os
from xosgenx.generator import XOSProcessor, XOSProcessorArgs
diff --git a/lib/xos-genx/xos-genx-tests/test_tosca.py b/lib/xos-genx/xos-genx-tests/test_tosca.py
index c5a0f17..6e585f5 100644
--- a/lib/xos-genx/xos-genx-tests/test_tosca.py
+++ b/lib/xos-genx/xos-genx-tests/test_tosca.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
import unittest
from xosgenx.generator import XOSProcessor, XOSProcessorArgs
from helpers import XProtoTestHelpers
@@ -145,7 +146,7 @@
args.inputs = xproto
args.target = self.target_tosca_keys
output = XOSProcessor.process(args)
- self.assertIn("['name', ['key_4', 'key_3'], ['key_1', 'key_2']]", output)
+ self.assertIn("['name', ['key_1', 'key_2'], ['key_3', 'key_4']]", output)
xproto = """
option app_label = "test";
@@ -163,4 +164,4 @@
args.inputs = xproto
output = XOSProcessor.process(args)
- self.assertIn("['name', ['key_1_id', 'key_3_id', 'key_2_id']]", output)
+ self.assertIn("['name', ['key_1_id', 'key_2_id', 'key_3_id']]", output)
diff --git a/lib/xos-genx/xos-genx-tests/test_translator.py b/lib/xos-genx/xos-genx-tests/test_translator.py
index 320021b..f98894b 100644
--- a/lib/xos-genx/xos-genx-tests/test_translator.py
+++ b/lib/xos-genx/xos-genx-tests/test_translator.py
@@ -13,6 +13,7 @@
# limitations under the License.
+from __future__ import absolute_import
import unittest
import os
from xosgenx.generator import XOSProcessor, XOSProcessorArgs
@@ -335,7 +336,7 @@
args.target = "modeldefs.xtarget"
output = XOSProcessor.process(args)
- read_only = filter(lambda s: "read_only: True" in s, output.splitlines())
+ read_only = [s for s in output.splitlines() if "read_only: True" in s]
self.assertEqual(len(read_only), 3) # readonly is 1 for ParentFoo and 2 for Foo
diff --git a/lib/xos-genx/xos-genx-tests/test_xos_security.py b/lib/xos-genx/xos-genx-tests/test_xos_security.py
index 3bd4653..b75f3c7 100644
--- a/lib/xos-genx/xos-genx-tests/test_xos_security.py
+++ b/lib/xos-genx/xos-genx-tests/test_xos_security.py
@@ -12,29 +12,29 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
+from __future__ import absolute_import
import unittest
from xosgenx.generator import XOSProcessor, XOSProcessorArgs
from helpers import XProtoTestHelpers
-"""The function below is for eliminating warnings arising due to the missing policy_output_enforcer,
-which is generated and loaded dynamically.
-"""
-
def policy_output_enforcer(x, y):
+ """
+ eliminating warnings arising due to the missing policy_output_enforcer,
+ which is generated and loaded dynamically.
+ """
raise Exception("Security enforcer not generated. Test failed.")
return False
-"""
-The tests below use the Python code target to generate
-Python security policies, set up an appropriate environment and execute the Python.
-The security policies here deliberately made complex in order to stress the processor.
-"""
-
class XProtoXOSSecurityTest(unittest.TestCase):
+ """
+ Use the Python code target to generate Python security policies, set up an
+ appropriate environment and execute the Python. The security policies here
+ deliberately made complex in order to stress the processor.
+ """
+
def setUp(self):
self.target = XProtoTestHelpers.write_tmp_target(
"{{ xproto_fol_to_python_test('output',proto.policies.test_policy, None, '0') }}"
diff --git a/lib/xos-genx/xos-genx-tests/test_xos_validation.py b/lib/xos-genx/xos-genx-tests/test_xos_validation.py
index 257eb4d..65555a3 100644
--- a/lib/xos-genx/xos-genx-tests/test_xos_validation.py
+++ b/lib/xos-genx/xos-genx-tests/test_xos_validation.py
@@ -12,28 +12,26 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
+from __future__ import absolute_import
import unittest
from xosgenx.generator import XOSProcessor, XOSProcessorArgs
from helpers import FakeObject, XProtoTestHelpers
-"""The function below is for eliminating warnings arising due to the missing policy_output_validator,
-which is generated and loaded dynamically.
-"""
-
def policy_output_validator(x, y):
+ """
+ For eliminating warnings arising due to the missing
+ policy_output_validator, which is generated and loaded dynamically.
+ """
raise Exception("Validator not generated. Test failed.")
return False
-
-"""
-The tests below use the Python code target to generate
-Python validation policies, set up an appropriate environment and execute the Python.
-"""
-
-
class XProtoXOSModelValidationTest(unittest.TestCase):
+ """
+ Use the Python code target to generate Python validation policies, set up an
+ appropriate environment and execute the Python.
+ """
+
def setUp(self):
self.target = XProtoTestHelpers.write_tmp_target(
"{{ xproto_fol_to_python_validator('output', proto.policies.test_policy, None, 'Necessary Failure') }}"
diff --git a/lib/xos-genx/xosgenx/generator.py b/lib/xos-genx/xosgenx/generator.py
index af99a23..9a6a249 100644
--- a/lib/xos-genx/xosgenx/generator.py
+++ b/lib/xos-genx/xosgenx/generator.py
@@ -12,17 +12,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import, print_function
-from __future__ import print_function
-import plyxproto.parser as plyxproto
-import jinja2
import os
-from xos2jinja import XOS2Jinja
-from proto2xproto import Proto2XProto
-import jinja2_extensions
+import jinja2
+import plyxproto.parser as plyxproto
import yaml
from colorama import Fore
+from . import jinja2_extensions
+from .proto2xproto import Proto2XProto
+from .xos2jinja import XOS2Jinja
+
loader = jinja2.PackageLoader(__name__, "templates")
env = jinja2.Environment(loader=loader)
@@ -147,7 +148,7 @@
context[k] = val
return context
except Exception as e:
- print(e.message)
+ print(e)
@staticmethod
def _write_single_file(rendered, dir, dest_file, quiet):
@@ -308,7 +309,11 @@
if message["name"] in args.include_models:
message["is_included"] = True
else:
- app_label = message.get("options", {}).get("app_label").strip('"')
+ app_label = (
+ message.get("options", {})
+ .get("app_label")
+ .strip('"')
+ )
if app_label in args.include_apps:
message["is_included"] = True
else:
diff --git a/lib/xos-genx/xosgenx/jinja2_extensions/__init__.py b/lib/xos-genx/xosgenx/jinja2_extensions/__init__.py
index bf7a812..a50bf3a 100644
--- a/lib/xos-genx/xosgenx/jinja2_extensions/__init__.py
+++ b/lib/xos-genx/xosgenx/jinja2_extensions/__init__.py
@@ -12,6 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+# Ignore this file as the * imports for Jinja2 functions are too numerous to list
+# flake8: noqa
+
+from __future__ import absolute_import
from .django import *
from .base import *
from .fol2 import *
diff --git a/lib/xos-genx/xosgenx/jinja2_extensions/base.py b/lib/xos-genx/xosgenx/jinja2_extensions/base.py
index 96e8dc2..f4929a9 100644
--- a/lib/xos-genx/xosgenx/jinja2_extensions/base.py
+++ b/lib/xos-genx/xosgenx/jinja2_extensions/base.py
@@ -12,8 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
-from __future__ import print_function
+from __future__ import absolute_import, print_function
import pdb
import re
from inflect import engine as inflect_engine_class
@@ -319,10 +318,11 @@
def xproto_string_type(xptags):
- try:
- max_length = eval(xptags["max_length"])
- except BaseException:
- max_length = 1024
+ # FIXME: this try/except block assigns but never uses max_length?
+ # try:
+ # max_length = eval(xptags["max_length"])
+ # except BaseException:
+ # max_length = 1024
if "varchar" not in xptags:
return "string"
@@ -340,26 +340,59 @@
def xproto_field_graph_components(fields, model, tag="unique_with"):
+ """
+ NOTE: Don't use set theory operators if you want repeatable tests - many
+ of them have non-deterministic behavior
+ """
+
def find_components(graph):
+
+ # 'graph' dict structure:
+ # - keys are strings
+ # - values are sets containing strings that are names of other keys in 'graph'
+
+ # take keys from 'graph' dict and put in 'pending' set
pending = set(graph.keys())
+
+ # create an empty list named 'components'
components = []
+ # loop while 'pending' is true - while there are still items in the 'pending' set
while pending:
+
+ # remove a random item from pending set, and put in 'front'
+ # this is the primary source of nondeterminism
front = {pending.pop()}
+
+ # create an empty set named 'component'
component = set()
+ # loop while 'front' is true. Front is modified below
while front:
+
+ # take the (only?) item out of the 'front' dict, and put in 'node'
node = front.pop()
+
+ # from 'graph' dict take set with key of 'node' and put into 'neighbors'
neighbours = graph[node]
+
+ # remove the set of items in components from neighbors
neighbours -= component # These we have already visited
+
+ # add all remaining neighbors to front
front |= neighbours
+ # remove neighbors from pending
pending -= neighbours
+
+ # add neighbors to component
component |= neighbours
- components.append(component)
+ # append component set to components list, sorted
+ components.append(sorted(component))
- return components
+ # return 'components', which is a list of sets
+ return sorted(components)
field_graph = {}
field_names = {f["name"] for f in fields}
@@ -378,6 +411,7 @@
field_graph.setdefault(f["name"], set()).add(uf)
field_graph.setdefault(uf, set()).add(f["name"])
+
except KeyError:
pass
@@ -434,12 +468,12 @@
def xproto_field_to_swagger_enum(f):
if "choices" in f["options"]:
- list = []
+ c_list = []
for c in eval(xproto_unquote(f["options"]["choices"])):
- list.append(c[0])
+ c_list.append(c[0])
- return list
+ return sorted(c_list)
else:
return False
diff --git a/lib/xos-genx/xosgenx/jinja2_extensions/checklib.py b/lib/xos-genx/xosgenx/jinja2_extensions/checklib.py
index db61f01..c361b64 100644
--- a/lib/xos-genx/xosgenx/jinja2_extensions/checklib.py
+++ b/lib/xos-genx/xosgenx/jinja2_extensions/checklib.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
import ast
@@ -27,13 +28,13 @@
except SyntaxError:
return "511 Could not parse sync step %s" % sync_step_path
- classes = filter(lambda x: isinstance(x, ast.ClassDef), sync_step_ast.body)
+ classes = [x for x in sync_step_ast.body if isinstance(x, ast.ClassDef)]
found_sync_step_class = False
for c in classes:
base_names = [v.id for v in c.bases]
if "SyncStep" in base_names or "SyncInstanceUsingAnsible" in base_names:
- attributes = filter(lambda x: isinstance(x, ast.Assign), c.body)
+ attributes = [x for x in c.body if isinstance(x, ast.Assign)]
for a in attributes:
target_names = [t.id for t in a.targets]
values = a.value.elts if isinstance(a.value, ast.List) else [a.value]
@@ -66,7 +67,7 @@
except SyntaxError:
return "511 Could not parse sync step %s" % model_policy_path
- classes = filter(lambda x: isinstance(x, ast.ClassDef), model_policy_ast.body)
+ classes = [x for x in model_policy_ast.body if isinstance(x, ast.ClassDef)]
found_model_policy_class = False
for c in classes:
base_names = [v.id for v in c.bases]
diff --git a/lib/xos-genx/xosgenx/jinja2_extensions/django.py b/lib/xos-genx/xosgenx/jinja2_extensions/django.py
index 3909c00..0c34d3e 100644
--- a/lib/xos-genx/xosgenx/jinja2_extensions/django.py
+++ b/lib/xos-genx/xosgenx/jinja2_extensions/django.py
@@ -12,11 +12,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import print_function
-from base import *
-import pdb
+from __future__ import absolute_import, print_function
+from .base import unquote
import re
import sys
+from six.moves import map
def django_content_type_string(xptags):
@@ -48,7 +48,7 @@
if "content_type" in xptags:
return django_content_type_string(xptags)
- elif max_length < 1024 * 1024:
+ elif int(max_length) < 1024 * 1024:
return "CharField"
else:
return "TextField"
@@ -208,7 +208,7 @@
else:
lst = []
- for k, v in d.items():
+ for k, v in sorted(d.items(), key=lambda t: t[0]): # sorted by key
if k in known_validators:
validator_lst.append(use_native_django_validators(k, v))
elif isinstance(v, str) and k == "default" and v.endswith('()"'):
@@ -261,7 +261,7 @@
def xproto_validations(options):
try:
return [
- map(str.strip, validation.split(":"))
+ list(map(str.strip, validation.split(":")))
for validation in unquote(options["validators"]).split(",")
]
except KeyError:
@@ -280,7 +280,7 @@
optioned_fields = []
for f in fields:
option_names = []
- for k, v in f["options"].items():
+ for k, v in sorted(f["options"].items(), key=lambda t: t[0]): # sorted by key
option_names.append(k)
if option in option_names and f["options"][option] == val:
diff --git a/lib/xos-genx/xosgenx/jinja2_extensions/fol2.py b/lib/xos-genx/xosgenx/jinja2_extensions/fol2.py
index 6d66117..0997e00 100644
--- a/lib/xos-genx/xosgenx/jinja2_extensions/fol2.py
+++ b/lib/xos-genx/xosgenx/jinja2_extensions/fol2.py
@@ -12,14 +12,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
-from __future__ import print_function
+from __future__ import absolute_import, print_function
import astunparse
import ast
import random
import string
import jinja2
-from plyxproto.parser import *
+from plyxproto.parser import lex, yacc
+from plyxproto.logicparser import FOLParser, FOLLexer
+from six.moves import range
+from six.moves import input
BINOPS = ["|", "&", "->"]
QUANTS = ["exists", "forall"]
@@ -45,11 +47,13 @@
self.idx = 0
return self
- def next(self):
+ def __next__(self):
var = "i%d" % self.idx
self.idx += 1
return var
+ next = __next__ # 2to3
+
def gen_random_string():
return "".join(
@@ -63,18 +67,18 @@
self.loopvar = iter(AutoVariable("i"))
self.verdictvar = iter(AutoVariable("result"))
- self.loop_variable = self.loopvar.next()
- self.verdict_variable = self.verdictvar.next()
+ self.loop_variable = next(self.loopvar)
+ self.verdict_variable = next(self.verdictvar)
self.context_map = context_map
if not self.context_map:
self.context_map = {"user": "self", "obj": "obj"}
def loop_next(self):
- self.loop_variable = self.loopvar.next()
+ self.loop_variable = next(self.loopvar)
def verdict_next(self):
- self.verdict_variable = self.verdictvar.next()
+ self.verdict_variable = next(self.verdictvar)
def gen_enumerate(self, fol):
pass
@@ -398,7 +402,8 @@
if result['result'] == 'True':
return {'hoist': ['const'], 'result': result['hoist'][1]}
elif result['hoist'][0] in BINOPS:
- return {'hoist': ['const'], 'result': {result['hoist'][0]: [result['hoist'][1], {k: [var2, result['result']]}]}}
+ return {'hoist': ['const'], 'result': {result['hoist'][0]:
+ [result['hoist'][1], {k: [var2, result['result']]}]}}
else:
return {'hoist': ['const'], 'result': {k: [var2, result['result']]}}
else:
@@ -731,9 +736,10 @@
if fol_reduced in ["True", "False"] and fol != fol_reduced:
raise TrivialPolicy(
- "Policy %(name)s trivially reduces to %(reduced)s. If this is what you want, replace its contents with %(reduced)s" % {
- "name": policy,
- "reduced": fol_reduced})
+ "Policy %(name)s trivially reduces to %(reduced)s."
+ "If this is what you want, replace its contents with %(reduced)s"
+ % {"name": policy, "reduced": fol_reduced}
+ )
a = f2p.gen_test_function(fol_reduced, policy, tag="security_check")
@@ -749,9 +755,10 @@
if fol_reduced in ["True", "False"] and fol != fol_reduced:
raise TrivialPolicy(
- "Policy %(name)s trivially reduces to %(reduced)s. If this is what you want, replace its contents with %(reduced)s" % {
- "name": policy,
- "reduced": fol_reduced})
+ "Policy %(name)s trivially reduces to %(reduced)s."
+ "If this is what you want, replace its contents with %(reduced)s"
+ % {"name": policy, "reduced": fol_reduced}
+ )
a = f2p.gen_validation_function(fol_reduced, policy, message, tag="validator")
@@ -762,7 +769,7 @@
while True:
inp = ""
while True:
- inp_line = raw_input()
+ inp_line = input()
if inp_line == "EOF":
break
else:
diff --git a/lib/xos-genx/xosgenx/jinja2_extensions/gui.py b/lib/xos-genx/xosgenx/jinja2_extensions/gui.py
index 4cb644a..07dbc35 100644
--- a/lib/xos-genx/xosgenx/jinja2_extensions/gui.py
+++ b/lib/xos-genx/xosgenx/jinja2_extensions/gui.py
@@ -12,8 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
-from base import xproto_string_type, unquote
+from __future__ import absolute_import
+from .base import xproto_string_type, unquote
def xproto_type_to_ui_type(f):
@@ -49,6 +49,20 @@
return None
+def xproto_dict_to_sorted_string(d):
+ """
+ sorts the dict by key and returns a string representation, which makes
+ for better consistency when testing
+ """
+ ft = [] # formatted tuples
+ for k, v in sorted(d.items(), key=lambda t: t[0]): # sorted by key
+ if v is not None:
+ ft.append("'%s': '%s'" % (k, v))
+ else:
+ ft.append("'%s': None" % k)
+ return "{%s}" % ", ".join(ft)
+
+
def xproto_validators(f):
# To be cleaned up when we formalize validation in xproto
validators = []
diff --git a/lib/xos-genx/xosgenx/jinja2_extensions/tosca.py b/lib/xos-genx/xosgenx/jinja2_extensions/tosca.py
index d8eada7..4424824 100644
--- a/lib/xos-genx/xosgenx/jinja2_extensions/tosca.py
+++ b/lib/xos-genx/xosgenx/jinja2_extensions/tosca.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
from xosgenx.jinja2_extensions import xproto_field_graph_components
@@ -58,18 +59,19 @@
):
keys.append("%s_id" % f["name"])
# if not keys are specified and there is a name field, use that as key.
- if len(keys) == 0 and "name" in map(lambda f: f["name"], fields):
+ if len(keys) == 0 and "name" in [f["name"] for f in fields]:
keys.append("name")
- for of in one_of:
+ for of in sorted(one_of):
# check if the field is a link, and in case add _id
for index, f in enumerate(of):
try:
- field = [
+ # FIXME: the 'field' var appears to be dead code, but exists to cause the IndexError?
+ field = [ # noqa: F841
x for x in fields if x["name"] == f and ("link" in x and x["link"])
][0]
of[index] = "%s_id" % f
- except IndexError as e:
+ except IndexError:
# the field is not a link
pass
diff --git a/lib/xos-genx/xosgenx/proto2xproto.py b/lib/xos-genx/xosgenx/proto2xproto.py
index f1dc959..6157aee 100644
--- a/lib/xos-genx/xosgenx/proto2xproto.py
+++ b/lib/xos-genx/xosgenx/proto2xproto.py
@@ -12,19 +12,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
-import plyxproto.model as m
-from plyxproto.helpers import Visitor
-import pdb
-import argparse
-import plyxproto.parser as plyxproto
-from plyxproto.logicparser import FOLParser, FOLLexer
-import traceback
-import sys
-import jinja2
-import os
import ply.lex as lex
import ply.yacc as yacc
+import plyxproto.model as m
+from plyxproto.helpers import Visitor
+from plyxproto.logicparser import FOLLexer, FOLParser
+from six.moves import map
class Stack(list):
@@ -136,7 +131,7 @@
except KeyError:
bases = []
- bases = map(lambda x: str_to_dict(x[1:-1]), bases)
+ bases = [str_to_dict(x[1:-1]) for x in bases]
obj.bases = bases
except KeyError:
raise
@@ -212,8 +207,8 @@
def visit_MessageDefinition_post(self, obj):
self.proto_to_xproto_message(obj)
- obj.body = filter(lambda x: not hasattr(x, "mark_for_deletion"), obj.body)
- obj.body = map(replace_link, obj.body)
+ obj.body = [x for x in obj.body if not hasattr(x, "mark_for_deletion")]
+ obj.body = list(map(replace_link, obj.body))
self.current_message_name = None
return True
diff --git a/lib/xos-genx/xosgenx/targets/link-graph.xtarget b/lib/xos-genx/xosgenx/targets/link-graph.xtarget
index 04c9dba..e455129 100644
--- a/lib/xos-genx/xosgenx/targets/link-graph.xtarget
+++ b/lib/xos-genx/xosgenx/targets/link-graph.xtarget
@@ -5,7 +5,7 @@
{% for l in model.links -%}
["{{ l.peer }}", "{{ l.src_port }}", "{{ l.dst_port }}"]{% if not loop.last %},{% endif %}
{%- endfor %}
- {%- if model.rlinks %},{% endif %}
+ {%- if model.rlinks %},{% endif %}
{% for l in model.rlinks -%}
["{{ l.peer }}", "{{ l.src_port }}", "{{ l.dst_port }}"]{% if not loop.last %},{% endif %}
{%- endfor %}
diff --git a/lib/xos-genx/xosgenx/targets/modeldefs.xtarget b/lib/xos-genx/xosgenx/targets/modeldefs.xtarget
index 15c43d7..ec7e5f2 100644
--- a/lib/xos-genx/xosgenx/targets/modeldefs.xtarget
+++ b/lib/xos-genx/xosgenx/targets/modeldefs.xtarget
@@ -8,7 +8,7 @@
{%- if m.options.verbose_name %}
verbose_name: "{{ xproto_unquote(m.options.verbose_name) }}"
{%- endif %}
- fields:
+ fields:
{%- set id_field = {'type':'int32', 'name':'id', 'options':{}} %}
{% for f in (xproto_base_fields(m, proto.message_table) + m.fields + [id_field]) | sort(attribute='name') -%}
{% if xproto_unquote(xproto_first_non_empty([f.options.gui_hidden, 'False'])) != 'True' and (not f.link or f.options.link_type != 'manytomany') -%}
@@ -25,7 +25,7 @@
{%- if f.options.choices %}
options:
{% for o in xproto_options_choices_to_dict(xproto_unquote(f.options.choices)) %}
- - {{ o }}
+ - {{ xproto_dict_to_sorted_string(o) }}
{% endfor %}
{%- endif %}
type: {{ xproto_type_to_ui_type(f) }}
@@ -50,4 +50,4 @@
relations: []
{%- endif %}
{%- endif %}
-{% endfor -%}
+{% endfor -%}
diff --git a/lib/xos-genx/xosgenx/version.py b/lib/xos-genx/xosgenx/version.py
index 79c5702..98215ea 100644
--- a/lib/xos-genx/xosgenx/version.py
+++ b/lib/xos-genx/xosgenx/version.py
@@ -14,5 +14,6 @@
import os
-# This file will be replaced by setup.py
-__version__ = "unknown"
+# read the version in from VERSION file, which is installed next to this file.
+with open(os.path.join(os.path.dirname(__file__), "VERSION")) as vf:
+ __version__ = vf.read()
diff --git a/lib/xos-genx/xosgenx/xos2jinja.py b/lib/xos-genx/xosgenx/xos2jinja.py
index 471be93..e4ff7db 100644
--- a/lib/xos-genx/xosgenx/xos2jinja.py
+++ b/lib/xos-genx/xosgenx/xos2jinja.py
@@ -12,18 +12,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import, print_function
-from __future__ import print_function
+import copy
+import sys
+
import plyxproto.model as m
from plyxproto.helpers import Visitor
-import argparse
-import plyxproto.parser as plyxproto
-import traceback
-import sys
-import jinja2
-import os
-import copy
-import pdb
+from six.moves import map, range
class MissingPolicyException(Exception):
@@ -175,7 +171,10 @@
return True
def visit_FieldType(self, obj):
- """Field type, if type is name, then it may need refactoring consistent with refactoring rules according to the table"""
+ """
+ Field type, if type is name, then it may need refactoring consistent
+ with refactoring rules according to the table
+ """
return True
def visit_LinkDefinition(self, obj):
@@ -295,13 +294,12 @@
stack_num = self.count_stack.pop()
fields = []
links = []
- last_field = None
+ last_field = {}
try:
- obj.bases = map(dotname_to_fqn, obj.bases)
+ obj.bases = list(map(dotname_to_fqn, obj.bases))
except AttributeError:
pass
- last_field = {}
for i in range(0, stack_num):
f = self.stack.pop()
if f["_type"] == "link":
@@ -340,7 +338,7 @@
self.models[model_name] = model_def
# Set message options
- for k, v in self.options.iteritems():
+ for k, v in sorted(self.options.items(), key=lambda t: t[0]): # sorted by key
try:
if k not in self.message_options:
self.message_options[k] = v
@@ -406,47 +404,47 @@
"onetomany": "manytoone",
}
- for m in messages:
- for l in m["links"]:
- rlink = copy.deepcopy(l)
+ for msg in messages:
+ for lnk in msg["links"]:
+ rlink = copy.deepcopy(lnk)
rlink["_type"] = "rlink" # An implicit link, not declared in the model
- rlink["src_port"] = l["dst_port"]
- rlink["dst_port"] = l["src_port"]
+ rlink["src_port"] = lnk["dst_port"]
+ rlink["dst_port"] = lnk["src_port"]
rlink["peer"] = {
- "name": m["name"],
- "package": m["package"],
- "fqn": m["fqn"],
+ "name": msg["name"],
+ "package": msg["package"],
+ "fqn": msg["fqn"],
}
- rlink["link_type"] = link_opposite[l["link_type"]]
- rlink["reverse_id"] = l["reverse_id"]
+ rlink["link_type"] = link_opposite[lnk["link_type"]]
+ rlink["reverse_id"] = lnk["reverse_id"]
- if (not l["reverse_id"]) and (self.args.verbosity >= 1):
+ if (not lnk["reverse_id"]) and (self.args.verbosity >= 1):
print(
"WARNING: Field %s in model %s has no reverse_id"
- % (l["src_port"], m["name"]),
+ % (lnk["src_port"], msg["name"]),
file=sys.stderr,
)
- if l["reverse_id"] and (
- (int(l["reverse_id"]) < 1000) or (int(l["reverse_id"]) >= 1900)
+ if lnk["reverse_id"] and (
+ (int(lnk["reverse_id"]) < 1000) or (int(lnk["reverse_id"]) >= 1900)
):
raise Exception(
"reverse id for field %s in model %s should be between 1000 and 1899"
- % (l["src_port"], m["name"])
+ % (lnk["src_port"], msg["name"])
)
try:
try:
- rev_links[l["peer"]["fqn"]].append(rlink)
+ rev_links[lnk["peer"]["fqn"]].append(rlink)
except TypeError:
pass
except KeyError:
- rev_links[l["peer"]["fqn"]] = [rlink]
+ rev_links[lnk["peer"]["fqn"]] = [rlink]
- for m in messages:
+ for msg in messages:
try:
- m["rlinks"] = rev_links[m["name"]]
- message_dict[m["name"]]["rlinks"] = m["rlinks"]
+ msg["rlinks"] = rev_links[msg["name"]]
+ message_dict[msg["name"]]["rlinks"] = msg["rlinks"]
except KeyError:
pass
diff --git a/lib/xos-genx/xosgenx/xosgen.py b/lib/xos-genx/xosgenx/xosgen.py
index 1318242..2da8962 100755
--- a/lib/xos-genx/xosgenx/xosgen.py
+++ b/lib/xos-genx/xosgenx/xosgen.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,13 +14,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import, print_function
-#!/usr/bin/python
-
-from __future__ import print_function
import argparse
-from generator import *
-from version import __version__
+import os
+from functools import reduce
+
+from six.moves import range
+
+from .generator import XOSProcessor, XOSProcessorArgs
+from .version import __version__
parse = argparse.ArgumentParser(description="XOS Generative Toolchain")
parse.add_argument(
@@ -147,7 +152,7 @@
else [args.target]
)
- for i in xrange(len(operators)):
+ for i in range(len(operators)):
if "/" not in operators[i]:
# if the target is not a path, it refer to a library included one
operators[i] = os.path.abspath(
diff --git a/lib/xos-migrate/xosmigrate/main.py b/lib/xos-migrate/xosmigrate/main.py
index 3fcbe89..338474d 100644
--- a/lib/xos-migrate/xosmigrate/main.py
+++ b/lib/xos-migrate/xosmigrate/main.py
@@ -146,7 +146,7 @@
raise Exception("Config file not found at: %s" % config)
cfg_file = open(config)
- cfg = yaml.load(cfg_file)
+ cfg = yaml.safe_load(cfg_file)
return cfg["name"]
diff --git a/lib/xos-util/.gitignore b/lib/xos-util/.gitignore
index 7fa3589..02a3a55 100644
--- a/lib/xos-util/.gitignore
+++ b/lib/xos-util/.gitignore
@@ -1,6 +1 @@
-build
-dist
-XosUtil.egg-info
-.coverage
-coverage.xml
-cover
\ No newline at end of file
+tests/test_version.py
diff --git a/lib/xos-util/setup.py b/lib/xos-util/setup.py
index a3707ef..a542cb9 100644
--- a/lib/xos-util/setup.py
+++ b/lib/xos-util/setup.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
from xosutil.autoversion_setup import setup_with_auto_version
from xosutil.version import __version__
diff --git a/lib/xos-util/tests/test_util.py b/lib/xos-util/tests/test_util.py
index 2b5be07..f4c0d3a 100644
--- a/lib/xos-util/tests/test_util.py
+++ b/lib/xos-util/tests/test_util.py
@@ -12,6 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
+
import os
import unittest
diff --git a/lib/xos-util/tox.ini b/lib/xos-util/tox.ini
new file mode 100644
index 0000000..2e1275a
--- /dev/null
+++ b/lib/xos-util/tox.ini
@@ -0,0 +1,42 @@
+; Copyright 2019-present Open Networking Foundation
+;
+; Licensed under the Apache License, Version 2.0 (the "License");
+; you may not use this file except in compliance with the License.
+; You may obtain a copy of the License at
+;
+; http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing, software
+; distributed under the License is distributed on an "AS IS" BASIS,
+; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+; See the License for the specific language governing permissions and
+; limitations under the License.
+
+[tox]
+envlist = py27,py35,py36,py37
+skip_missing_interpreters = true
+
+[testenv]
+deps =
+ nose2
+ flake8
+
+commands =
+ nose2 -c tox.ini --verbose --junit-xml
+ flake8
+
+[flake8]
+max-line-length = 119
+
+[unittest]
+plugins=nose2.plugins.junitxml
+
+[junit-xml]
+path=nose2-results.xml
+
+[coverage]
+always-on = True
+coverage = xosutil
+coverage-report =
+ xml
+ term
diff --git a/lib/xos-util/xosutil/autodiscover_version.py b/lib/xos-util/xosutil/autodiscover_version.py
index 8f4ea47..2a74273 100644
--- a/lib/xos-util/xosutil/autodiscover_version.py
+++ b/lib/xos-util/xosutil/autodiscover_version.py
@@ -20,6 +20,8 @@
of the caller.
"""
+from __future__ import absolute_import
+
import inspect
import os
diff --git a/lib/xos-util/xosutil/autoversion_setup.py b/lib/xos-util/xosutil/autoversion_setup.py
index e027d5c..a0f30f0 100644
--- a/lib/xos-util/xosutil/autoversion_setup.py
+++ b/lib/xos-util/xosutil/autoversion_setup.py
@@ -23,14 +23,16 @@
automatically load the version number from the VERSION file, if one is detected.
"""
-import os
-from setuptools import setup
-
-from setuptools.command.sdist import sdist
-from setuptools.command.build_py import build_py
+from __future__ import absolute_import
import inspect
-from autodiscover_version import autodiscover_version
+import os
+
+from setuptools import setup
+from setuptools.command.build_py import build_py
+from setuptools.command.sdist import sdist
+
+from .autodiscover_version import autodiscover_version
class SdistCommand(sdist):
diff --git a/xos/core/migrations/0002_initial_data.py b/xos/core/migrations/0002_initial_data.py
index 22227f8..78db513 100644
--- a/xos/core/migrations/0002_initial_data.py
+++ b/xos/core/migrations/0002_initial_data.py
@@ -26,7 +26,7 @@
def load_data_from_yaml():
file = open(FIXTURES, "r").read()
try:
- data = yaml.load(file)
+ data = yaml.safe_load(file)
return data
except Exception as e:
raise Exception("Cannot load inital data file: %s" % e.message)
diff --git a/xos/core/migrations/0006_auto_20190304_2346.py b/xos/core/migrations/0006_auto_20190304_2346.py
new file mode 100644
index 0000000..ce9558b
--- /dev/null
+++ b/xos/core/migrations/0006_auto_20190304_2346.py
@@ -0,0 +1,45 @@
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.11 on 2019-03-05 04:46
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('core', '0005_delete_models'),
+ ]
+
+ operations = [
+ migrations.AlterUniqueTogether(
+ name='image_decl',
+ unique_together=set([('name', 'tag')]),
+ ),
+ migrations.AlterUniqueTogether(
+ name='networkslice_decl',
+ unique_together=set([('network', 'slice')]),
+ ),
+ migrations.AlterUniqueTogether(
+ name='port_decl',
+ unique_together=set([('network', 'service_instance')]),
+ ),
+ migrations.AlterUniqueTogether(
+ name='serviceinstanceattribute_decl',
+ unique_together=set([('name', 'service_instance')]),
+ ),
+ ]
diff --git a/xos/coreapi/xos_modeldefs_api.py b/xos/coreapi/xos_modeldefs_api.py
index edbfac9..4830677 100644
--- a/xos/coreapi/xos_modeldefs_api.py
+++ b/xos/coreapi/xos_modeldefs_api.py
@@ -64,7 +64,7 @@
@track_request_time("Modeldefs", "ListModelDefs")
def ListModelDefs(self, request, context):
ystr = open("protos/modeldefs.yaml").read()
- yaml_repr = yaml.load(ystr)
+ yaml_repr = yaml.safe_load(ystr)
modeldefs = modeldefs_pb2.ModelDefs()
diff --git a/xos/manage.py b/xos/manage.py
index c81ffb9..9e02242 100644
--- a/xos/manage.py
+++ b/xos/manage.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,8 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
-#!/usr/bin/env python
import os
import sys
@@ -23,6 +23,7 @@
from django.core.management import execute_from_command_line
+ # this if seems circular - the xos-manage script calls this script!
if "--makemigrations" in sys.argv:
os.system("/opt/xos/tools/xos-manage makemigrations")
sys.argv.remove("--makemigrations")
diff --git a/xos/tools/xos-manage b/xos/tools/xos-manage
index 819c887..4f4f4e8 100755
--- a/xos/tools/xos-manage
+++ b/xos/tools/xos-manage
@@ -134,11 +134,13 @@
}
function makemigrations {
+ echo "Making migrations for core"
python ./manage.py makemigrations core
if [[ -e /opt/xos/xos/xosbuilder_migration_list ]]; then
cat /opt/xos/xos/xosbuilder_migration_list | while read line; do
if [[ ! -z "$line" ]]; then
+ echo "Making migrations for $line"
python ./manage.py makemigrations $line
fi
done