[SEBA-450] (part 2)
Add tox testing support on additional XOS library modules:
- xos-api
- xos-kafka (has no tests)
- xos-migrate (has no tests)
- xos-synchronizer
Change-Id: I98195bc9747971d3515882d517affe058dd86ac5
diff --git a/VERSION b/VERSION
index a6333e4..0d3ad67 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.2.9
+2.2.10
diff --git a/containers/chameleon/Dockerfile.chameleon b/containers/chameleon/Dockerfile.chameleon
index 51fe650..79debb6 100644
--- a/containers/chameleon/Dockerfile.chameleon
+++ b/containers/chameleon/Dockerfile.chameleon
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/chameleon
-FROM xosproject/xos-base:2.2.9
+FROM xosproject/xos-base:2.2.10
# xos-base already has protoc and dependencies installed
diff --git a/containers/xos/Dockerfile.client b/containers/xos/Dockerfile.client
index 5f0e13f..0501e60 100644
--- a/containers/xos/Dockerfile.client
+++ b/containers/xos/Dockerfile.client
@@ -13,13 +13,13 @@
# limitations under the License.
# xosproject/xos-client
-FROM xosproject/xos-libraries:2.2.9
+FROM xosproject/xos-libraries:2.2.10
# Install XOS client
COPY lib/xos-api /tmp/xos-api
# Install the VERSION file
-COPY VERSION /tmp/xos-api/VERSION
+COPY VERSION /tmp/xos-api/xosapi/VERSION
# install the client library and xossh
RUN chdir /tmp/xos-api && python ./setup.py install
@@ -47,5 +47,4 @@
org.opencord.component.chameleon.vcs-url=$org_opencord_component_chameleon_vcs_url \
org.opencord.component.chameleon.vcs-ref=$org_opencord_component_chameleon_vcs_ref
-
CMD ["xossh"]
diff --git a/containers/xos/Dockerfile.libraries b/containers/xos/Dockerfile.libraries
index 57aa334..8d9652c 100644
--- a/containers/xos/Dockerfile.libraries
+++ b/containers/xos/Dockerfile.libraries
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/xos-libraries
-FROM xosproject/xos-base:2.2.9
+FROM xosproject/xos-base:2.2.10
# Add libraries
COPY lib /opt/xos/lib
diff --git a/containers/xos/Dockerfile.synchronizer-base b/containers/xos/Dockerfile.synchronizer-base
index 3e5793a..619fef8 100644
--- a/containers/xos/Dockerfile.synchronizer-base
+++ b/containers/xos/Dockerfile.synchronizer-base
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/xos-synchronizer-base
-FROM xosproject/xos-client:2.2.9
+FROM xosproject/xos-client:2.2.10
COPY xos/synchronizers/new_base /opt/xos/synchronizers/new_base
COPY xos/xos/logger.py /opt/xos/xos/logger.py
@@ -25,7 +25,7 @@
# For library-based synchronizers
COPY lib/xos-synchronizer /tmp/xos-synchronizer
-COPY VERSION /tmp/xos-synchronizer/VERSION
+COPY VERSION /tmp/xos-synchronizer/xossynchronizer/VERSION
RUN cd /tmp/xos-synchronizer && python setup.py install
# Label image
diff --git a/containers/xos/Dockerfile.xos-core b/containers/xos/Dockerfile.xos-core
index 1c3d962..15a7222 100644
--- a/containers/xos/Dockerfile.xos-core
+++ b/containers/xos/Dockerfile.xos-core
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/xos-core
-FROM xosproject/xos-libraries:2.2.9
+FROM xosproject/xos-libraries:2.2.10
# Install XOS
ADD xos /opt/xos
diff --git a/lib/xos-api/.gitignore b/lib/xos-api/.gitignore
index f82b325..b8cf076 100644
--- a/lib/xos-api/.gitignore
+++ b/lib/xos-api/.gitignore
@@ -1,3 +1,2 @@
-build
-xosapi/chameleon
-xosapi.egg-info
+# setup.py copies this, don't commit it
+xosapi/VERSION
diff --git a/lib/xos-api/MANIFEST.in b/lib/xos-api/MANIFEST.in
new file mode 100644
index 0000000..223efb9
--- /dev/null
+++ b/lib/xos-api/MANIFEST.in
@@ -0,0 +1,2 @@
+include requirements.txt
+include xosapi/VERSION
diff --git a/lib/xos-api/requirements.txt b/lib/xos-api/requirements.txt
new file mode 100644
index 0000000..976b014
--- /dev/null
+++ b/lib/xos-api/requirements.txt
@@ -0,0 +1,9 @@
+Twisted~=16.6.0
+multistructlog~=2.1.0
+xosconfig~=2.2.6
+xosgenx~=2.2.6
+googleapis-common-protos~=1.5.6
+pykwalify~=1.6.0
+PyYAML~=3.12
+python-consul~=1.1.0
+grpcio~=1.12.0
diff --git a/lib/xos-api/setup.py b/lib/xos-api/setup.py
index 32c566f..92c7898 100644
--- a/lib/xos-api/setup.py
+++ b/lib/xos-api/setup.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,17 +12,30 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
+
import os
+from shutil import copyfile
+
+from setuptools import setup
from setuptools.command.install import install
-try:
- from xosutil.autoversion_setup import setup_with_auto_version as setup
-except ImportError:
- # xosutil is not installed. Expect this to happen when we build an egg, in which case xosgenx.version will
- # automatically have the right version.
- from setuptools import setup
-from xosapi.version import __version__
+def version():
+ # Copy VERSION file of parent to module directory if not found
+ if not os.path.exists("xosapi/VERSION"):
+ copyfile("../../VERSION", "xosapi/VERSION")
+ with open("xosapi/VERSION") as f:
+ return f.read().strip()
+
+
+def parse_requirements(filename):
+ # parse a requirements.txt file, allowing for blank lines and comments
+ requirements = []
+ for line in open(filename):
+ if line and not line.startswith("#"):
+ requirements.append(line)
+ return requirements
class InstallFixChameleonPermissions(install):
@@ -41,9 +52,11 @@
setup_result = setup(
name="xosapi",
- version=__version__,
+ version=version(),
+ classifiers=["License :: OSI Approved :: Apache Software License"],
+ license="Apache v2",
cmdclass={"install": InstallFixChameleonPermissions},
- description="XOS api client",
+ description="XOS API client",
packages=[
"xosapi.chameleon_client",
"xosapi.chameleon_client.protos",
@@ -51,5 +64,11 @@
"xosapi",
"xosapi.convenience",
],
+ install_requires=parse_requirements("requirements.txt"),
+ include_package_data=True,
+ package_data={
+ "xosapi.chameleon_client.protos": ["*.proto"],
+ "xosapi.chameleon_client.protoc_plugins": ["*.desc"],
+ },
scripts=["xossh"],
)
diff --git a/lib/xos-api/xosapi/fake_stub.py b/lib/xos-api/tests/fake_stub.py
similarity index 98%
rename from lib/xos-api/xosapi/fake_stub.py
rename to lib/xos-api/tests/fake_stub.py
index abfce00..81c3988 100644
--- a/lib/xos-api/xosapi/fake_stub.py
+++ b/lib/xos-api/tests/fake_stub.py
@@ -17,6 +17,8 @@
Implements a simple fake grpc stub to use for unit testing.
"""
+from __future__ import absolute_import
+
import functools
ContentTypeMap = {}
@@ -492,9 +494,9 @@
items = []
if query.kind == FakeQuery.SYNCHRONIZER_DELETED_OBJECTS:
- objs = self.deleted_objs.items()
+ objs = list(self.deleted_objs.items())
else:
- objs = self.objs.items()
+ objs = list(self.objs.items())
for (k, v) in objs:
(this_classname, id) = k.split(":")
diff --git a/lib/xos-api/tests/test_chameleon_client.py b/lib/xos-api/tests/test_chameleon_client.py
new file mode 100644
index 0000000..24b19ca
--- /dev/null
+++ b/lib/xos-api/tests/test_chameleon_client.py
@@ -0,0 +1,26 @@
+# Copyright 2019-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import xosapi.chameleon_client.grpc_client as chameleon_client
+
+
+class TestChameleonClient(unittest.TestCase):
+
+ def test_chameleon_init(self):
+ """ test that chameleon_client can be loaded """
+
+ cclient = chameleon_client.GrpcClient("consul_endpoint", "work_dir")
+ self.assertEqual(type(cclient), chameleon_client.GrpcClient)
diff --git a/lib/xos-api/xosapi/test_config.yaml b/lib/xos-api/tests/test_config.yaml
similarity index 100%
rename from lib/xos-api/xosapi/test_config.yaml
rename to lib/xos-api/tests/test_config.yaml
diff --git a/lib/xos-api/xosapi/test_orm.py b/lib/xos-api/tests/test_orm.py
similarity index 97%
rename from lib/xos-api/xosapi/test_orm.py
rename to lib/xos-api/tests/test_orm.py
index 88bf6d4..86be969 100644
--- a/lib/xos-api/xosapi/test_orm.py
+++ b/lib/xos-api/tests/test_orm.py
@@ -13,14 +13,20 @@
# limitations under the License.
from __future__ import print_function
-import exceptions
+
import os
import random
import string
import sys
import unittest
-from mock import patch
-from StringIO import StringIO
+
+try: # python 3
+ from io import StringIO
+ from unittest.mock import patch
+except ImportError: # python 2
+ from StringIO import StringIO
+ from mock import patch
+
# by default, use fake stub rather than real core
USE_FAKE_STUB = True
@@ -498,7 +504,7 @@
with self.assertRaises(Exception) as e:
self.assertEqual(
- e.exception.message,
+ str(e.exception),
"Content_type does_not_exist not found in self.content_type_map",
)
@@ -520,7 +526,7 @@
self.assertTrue(tag.id > 0)
with self.assertRaises(Exception) as e:
self.assertEqual(
- e.exception.message, "Object 4567 of model Site was not found"
+ str(e.exception), "Object 4567 of model Site was not found"
)
def test_generic_foreign_key_set(self):
@@ -600,7 +606,7 @@
with self.assertRaises(Exception) as e:
tm.intfile = None
self.assertEqual(
- e.exception.message,
+ str(e.exception),
"Setting a non-foreignkey field to None is not supported",
)
@@ -1013,11 +1019,11 @@
with self.assertRaises(Exception) as e:
lobjs.add(123)
- self.assertEqual(e.exception.message, "Only ManyToMany lists are writeable")
+ self.assertEqual(str(e.exception), "Only ManyToMany lists are writeable")
with self.assertRaises(Exception) as e:
lobjs.remove(123)
- self.assertEqual(e.exception.message, "Only ManyToMany lists are writeable")
+ self.assertEqual(str(e.exception), "Only ManyToMany lists are writeable")
def test_ORMLocalObjectManager_add(self):
orm = self.make_coreapi()
@@ -1067,9 +1073,11 @@
try:
sys.argv = sys.argv[
:1
- ] # unittest does not like xos_grpc_client's command line arguments (TODO: find a cooperative approach)
+ ]
+ # unittest does not like xos_grpc_client's command line
+ # arguments (TODO: find a cooperative approach)
unittest.main()
- except exceptions.SystemExit as e:
+ except SystemExit as e:
global exitStatus
exitStatus = e.code
diff --git a/lib/xos-api/xosapi/test_wrapper.py b/lib/xos-api/tests/test_wrapper.py
similarity index 97%
rename from lib/xos-api/xosapi/test_wrapper.py
rename to lib/xos-api/tests/test_wrapper.py
index fe4f9c1..8cb5255 100644
--- a/lib/xos-api/xosapi/test_wrapper.py
+++ b/lib/xos-api/tests/test_wrapper.py
@@ -13,7 +13,7 @@
# limitations under the License.
from __future__ import print_function
-import exceptions
+
import os
import random
import string
@@ -213,9 +213,11 @@
try:
sys.argv = sys.argv[
:1
- ] # unittest does not like xos_grpc_client's command line arguments (TODO: find a cooperative approach)
+ ]
+ # unittest does not like xos_grpc_client's command line arguments
+ # (TODO: find a cooperative approach)
unittest.main()
- except exceptions.SystemExit as e:
+ except SystemExit as e:
global exitStatus
exitStatus = e.code
diff --git a/lib/xos-api/tox.ini b/lib/xos-api/tox.ini
new file mode 100644
index 0000000..c2a889c
--- /dev/null
+++ b/lib/xos-api/tox.ini
@@ -0,0 +1,49 @@
+; Copyright 2017-present Open Networking Foundation
+;
+; Licensed under the Apache License, Version 2.0 (the "License");
+; you may not use this file except in compliance with the License.
+; You may obtain a copy of the License at
+;
+; http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing, software
+; distributed under the License is distributed on an "AS IS" BASIS,
+; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+; See the License for the specific language governing permissions and
+; limitations under the License.
+
+[tox]
+envlist = py27,py35,py36,py37
+skip_missing_interpreters = true
+
+[testenv]
+deps =
+ -r requirements.txt
+ nose2
+ flake8
+
+commands =
+ nose2 -c tox.ini --verbose --junit-xml
+ flake8
+
+[flake8]
+exclude =
+ .tox
+ build
+ xosapi/chameleon_client/protos
+max-line-length = 119
+
+[unittest]
+plugins=nose2.plugins.junitxml
+
+[junit-xml]
+path=nose2-results.xml
+
+[coverage]
+always-on = True
+coverage =
+ xosapi
+ tests
+coverage-report =
+ xml
+ term
diff --git a/lib/xos-api/xosapi/.gitignore b/lib/xos-api/xosapi/.gitignore
new file mode 100644
index 0000000..b8cf076
--- /dev/null
+++ b/lib/xos-api/xosapi/.gitignore
@@ -0,0 +1,2 @@
+# setup.py copies this, don't commit it
+xosapi/VERSION
diff --git a/lib/xos-api/xosapi/chameleon_client/asleep.py b/lib/xos-api/xosapi/chameleon_client/asleep.py
index 295f675..cdba793 100644
--- a/lib/xos-api/xosapi/chameleon_client/asleep.py
+++ b/lib/xos-api/xosapi/chameleon_client/asleep.py
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+from __future__ import absolute_import
from twisted.internet import reactor
from twisted.internet.defer import Deferred
@@ -25,4 +26,4 @@
"""
d = Deferred()
reactor.callLater(dt, lambda: d.callback(None))
- return d
\ No newline at end of file
+ return d
diff --git a/lib/xos-api/xosapi/chameleon_client/grpc_client.py b/lib/xos-api/xosapi/chameleon_client/grpc_client.py
index a8ad69d..7d3839d 100644
--- a/lib/xos-api/xosapi/chameleon_client/grpc_client.py
+++ b/lib/xos-api/xosapi/chameleon_client/grpc_client.py
@@ -20,6 +20,7 @@
semantics are derived from the recovered schema.
"""
+from __future__ import absolute_import
import os
import sys
import time
@@ -33,12 +34,12 @@
from structlog import get_logger
from twisted.internet import reactor
from twisted.internet.defer import inlineCallbacks, returnValue
-from werkzeug.exceptions import ServiceUnavailable
+from twisted.internet.error import ConnectError
-from protos.schema_pb2_grpc import SchemaServiceStub
+from .protos.schema_pb2_grpc import SchemaServiceStub
from google.protobuf.empty_pb2 import Empty
-from asleep import asleep
+from .asleep import asleep
log = get_logger()
@@ -143,14 +144,14 @@
return
- except _Rendezvous, e:
+ except _Rendezvous as e:
if e.code() == grpc.StatusCode.UNAVAILABLE:
log.info('grpc-endpoint-not-available')
else:
log.exception('rendezvous error', e=e)
yield self._backoff('not-available')
- except Exception, e:
+ except Exception:
if not self.shutting_down:
log.exception('cannot-connect', endpoint=_endpoint)
yield self._backoff('unknown-error')
@@ -277,7 +278,7 @@
if f.endswith('_pb2.py')]:
modname = fname[:-len('.py')]
log.debug('test-import', modname=modname)
- _ = __import__(modname)
+ _ = __import__(modname) # noqa: F841
@inlineCallbacks
def invoke(self, stub, method_name, request, metadata, retry=1):
@@ -291,17 +292,17 @@
"""
if not self.connected:
- raise ServiceUnavailable()
+ raise ConnectError()
try:
method = getattr(stub(self.channel), method_name)
response, rendezvous = method.with_call(request, metadata=metadata)
returnValue((response, rendezvous.trailing_metadata()))
- except grpc._channel._Rendezvous, e:
+ except grpc._channel._Rendezvous as e:
code = e.code()
if code == grpc.StatusCode.UNAVAILABLE:
- e = ServiceUnavailable()
+ e = ConnectError()
if self.connected:
self.connected = False
diff --git a/lib/xos-api/xosapi/chameleon_client/protoc_plugins/gw_gen.py b/lib/xos-api/xosapi/chameleon_client/protoc_plugins/gw_gen.py
index 9cd2f6f..cafee67 100755
--- a/lib/xos-api/xosapi/chameleon_client/protoc_plugins/gw_gen.py
+++ b/lib/xos-api/xosapi/chameleon_client/protoc_plugins/gw_gen.py
@@ -15,6 +15,7 @@
# limitations under the License.
#
+from __future__ import absolute_import
import sys
from google.protobuf.compiler import plugin_pb2 as plugin
@@ -23,7 +24,7 @@
from jinja2 import Template
from simplejson import dumps
-from xosapi.chameleon_client.protos import annotations_pb2, http_pb2
+from xosapi.chameleon_client.protos import http_pb2
template = Template("""
# Generated file; please do not edit
diff --git a/lib/xos-api/xosapi/chameleon_client/protos/annotations_pb2.py b/lib/xos-api/xosapi/chameleon_client/protos/annotations_pb2.py
index 5abc663..aa451e5 100644
--- a/lib/xos-api/xosapi/chameleon_client/protos/annotations_pb2.py
+++ b/lib/xos-api/xosapi/chameleon_client/protos/annotations_pb2.py
@@ -18,6 +18,7 @@
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: annotations.proto
+from __future__ import absolute_import
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
from google.protobuf import descriptor as _descriptor
@@ -30,7 +31,7 @@
_sym_db = _symbol_database.Default()
-import http_pb2 as http__pb2
+from . import http_pb2 as http__pb2
from google.protobuf import descriptor_pb2 as google_dot_protobuf_dot_descriptor__pb2
diff --git a/lib/xos-api/xosapi/chameleon_client/protos/annotations_pb2_grpc.py b/lib/xos-api/xosapi/chameleon_client/protos/annotations_pb2_grpc.py
index 972edf5..1ec9f40 100644
--- a/lib/xos-api/xosapi/chameleon_client/protos/annotations_pb2_grpc.py
+++ b/lib/xos-api/xosapi/chameleon_client/protos/annotations_pb2_grpc.py
@@ -16,4 +16,5 @@
#
# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+from __future__ import absolute_import
import grpc
diff --git a/lib/xos-api/xosapi/chameleon_client/protos/http_pb2.py b/lib/xos-api/xosapi/chameleon_client/protos/http_pb2.py
index f420e61..8e359f5 100644
--- a/lib/xos-api/xosapi/chameleon_client/protos/http_pb2.py
+++ b/lib/xos-api/xosapi/chameleon_client/protos/http_pb2.py
@@ -18,6 +18,7 @@
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: http.proto
+from __future__ import absolute_import
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
from google.protobuf import descriptor as _descriptor
diff --git a/lib/xos-api/xosapi/chameleon_client/protos/http_pb2_grpc.py b/lib/xos-api/xosapi/chameleon_client/protos/http_pb2_grpc.py
index 972edf5..1ec9f40 100644
--- a/lib/xos-api/xosapi/chameleon_client/protos/http_pb2_grpc.py
+++ b/lib/xos-api/xosapi/chameleon_client/protos/http_pb2_grpc.py
@@ -16,4 +16,5 @@
#
# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+from __future__ import absolute_import
import grpc
diff --git a/lib/xos-api/xosapi/chameleon_client/protos/schema_pb2.py b/lib/xos-api/xosapi/chameleon_client/protos/schema_pb2.py
index 06909b4..0e0159a 100644
--- a/lib/xos-api/xosapi/chameleon_client/protos/schema_pb2.py
+++ b/lib/xos-api/xosapi/chameleon_client/protos/schema_pb2.py
@@ -18,6 +18,7 @@
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: schema.proto
+from __future__ import absolute_import
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
from google.protobuf import descriptor as _descriptor
diff --git a/lib/xos-api/xosapi/chameleon_client/protos/schema_pb2_grpc.py b/lib/xos-api/xosapi/chameleon_client/protos/schema_pb2_grpc.py
index 65c9afa..06c1456 100644
--- a/lib/xos-api/xosapi/chameleon_client/protos/schema_pb2_grpc.py
+++ b/lib/xos-api/xosapi/chameleon_client/protos/schema_pb2_grpc.py
@@ -16,10 +16,11 @@
#
# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+from __future__ import absolute_import
import grpc
from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2
-import schema_pb2 as schema__pb2
+from . import schema_pb2 as schema__pb2
class SchemaServiceStub(object):
diff --git a/lib/xos-api/xosapi/convenience/addresspool.py b/lib/xos-api/xosapi/convenience/addresspool.py
index 13006c5..e83c818 100644
--- a/lib/xos-api/xosapi/convenience/addresspool.py
+++ b/lib/xos-api/xosapi/convenience/addresspool.py
@@ -13,6 +13,8 @@
# limitations under the License.
+from __future__ import absolute_import
+
from xosapi.orm import ORMWrapper, register_convenience_wrapper
diff --git a/lib/xos-api/xosapi/convenience/controller.py b/lib/xos-api/xosapi/convenience/controller.py
index 437b54a..4e13dac 100644
--- a/lib/xos-api/xosapi/convenience/controller.py
+++ b/lib/xos-api/xosapi/convenience/controller.py
@@ -12,6 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
+
from xosapi.orm import ORMWrapper, register_convenience_wrapper
diff --git a/lib/xos-api/xosapi/convenience/instance.py b/lib/xos-api/xosapi/convenience/instance.py
index 3941643..f373f12 100644
--- a/lib/xos-api/xosapi/convenience/instance.py
+++ b/lib/xos-api/xosapi/convenience/instance.py
@@ -13,6 +13,8 @@
# limitations under the License.
+from __future__ import absolute_import
+
from xosapi.orm import ORMWrapper, register_convenience_wrapper
diff --git a/lib/xos-api/xosapi/convenience/network.py b/lib/xos-api/xosapi/convenience/network.py
index ffa89f7..95cb972 100644
--- a/lib/xos-api/xosapi/convenience/network.py
+++ b/lib/xos-api/xosapi/convenience/network.py
@@ -12,7 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from xosapi.orm import ORMWrapper, ORMLocalObjectManager, register_convenience_wrapper
+from __future__ import absolute_import
+
+from xosapi.orm import (ORMLocalObjectManager, ORMWrapper,
+ register_convenience_wrapper)
class ORMWrapperNetwork(ORMWrapper):
diff --git a/lib/xos-api/xosapi/convenience/onosapp.py b/lib/xos-api/xosapi/convenience/onosapp.py
index fd791a8..2970581 100644
--- a/lib/xos-api/xosapi/convenience/onosapp.py
+++ b/lib/xos-api/xosapi/convenience/onosapp.py
@@ -12,8 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from xosapi.orm import register_convenience_wrapper
+from __future__ import absolute_import
+
from xosapi.convenience.serviceinstance import ORMWrapperServiceInstance
+from xosapi.orm import register_convenience_wrapper
class ORMWrapperONOSApp(ORMWrapperServiceInstance):
diff --git a/lib/xos-api/xosapi/convenience/port.py b/lib/xos-api/xosapi/convenience/port.py
index 61e7377..2c9b9a6 100644
--- a/lib/xos-api/xosapi/convenience/port.py
+++ b/lib/xos-api/xosapi/convenience/port.py
@@ -12,6 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
+
from xosapi.orm import ORMWrapper, register_convenience_wrapper
diff --git a/lib/xos-api/xosapi/convenience/privilege.py b/lib/xos-api/xosapi/convenience/privilege.py
index 9bdb635..0071842 100644
--- a/lib/xos-api/xosapi/convenience/privilege.py
+++ b/lib/xos-api/xosapi/convenience/privilege.py
@@ -13,6 +13,8 @@
# limitations under the License.
+from __future__ import absolute_import
+
from xosapi.orm import ORMWrapper, register_convenience_wrapper
diff --git a/lib/xos-api/xosapi/convenience/service.py b/lib/xos-api/xosapi/convenience/service.py
index ab82908..8c9985a 100644
--- a/lib/xos-api/xosapi/convenience/service.py
+++ b/lib/xos-api/xosapi/convenience/service.py
@@ -12,6 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
+
from xosapi.orm import ORMWrapper, register_convenience_wrapper
diff --git a/lib/xos-api/xosapi/convenience/serviceinstance.py b/lib/xos-api/xosapi/convenience/serviceinstance.py
index 33ab553..10adb16 100644
--- a/lib/xos-api/xosapi/convenience/serviceinstance.py
+++ b/lib/xos-api/xosapi/convenience/serviceinstance.py
@@ -11,10 +11,11 @@
# limitations under the License.
-from xosapi.orm import ORMWrapper, register_convenience_wrapper
+from __future__ import absolute_import
-from xosconfig import Config
from multistructlog import create_logger
+from xosapi.orm import ORMWrapper, register_convenience_wrapper
+from xosconfig import Config
log = create_logger(Config().get("logging"))
@@ -28,11 +29,6 @@
return attrs
@property
- def tenantattribute_dict(self):
- log.warn("tenantattribute_dict method is deprecated")
- return self.serviceinstanceattribute_dict
-
- @property
def westbound_service_instances(self):
links = self.provided_links.all()
instances = []
@@ -50,32 +46,6 @@
return instances
- def create_eastbound_instance(self):
-
- # Already has a chain
- if len(self.eastbound_service_instances) > 0 and not self.is_new:
- log.debug("MODEL_POLICY: Subscriber %s is already part of a chain" % si.id)
- return
-
- # if it does not have a chain,
- # Find links to the next element in the service chain
- # and create one
-
- links = self.owner.subscribed_dependencies.all()
-
- for link in links:
- si_class = link.provider_service.get_service_instance_class_name()
- log.info(" %s creating %s" % (self.model_name, si_class))
-
- eastbound_si_class = model_accessor.get_model_class(si_class)
- eastbound_si = eastbound_si_class()
- eastbound_si.owner_id = link.provider_service_id
- eastbound_si.save()
- link = ServiceInstanceLink(
- provider_service_instance=eastbound_si, subscriber_service_instance=si
- )
- link.save()
-
def get_westbound_service_instance_properties(self, prop_name, include_self=False):
if include_self and hasattr(self, prop_name):
return getattr(self, prop_name)
diff --git a/lib/xos-api/xosapi/convenience/slice.py b/lib/xos-api/xosapi/convenience/slice.py
index abdd485..0a10311 100644
--- a/lib/xos-api/xosapi/convenience/slice.py
+++ b/lib/xos-api/xosapi/convenience/slice.py
@@ -12,7 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from xosapi.orm import ORMWrapper, ORMLocalObjectManager, register_convenience_wrapper
+from __future__ import absolute_import
+
+from xosapi.orm import (ORMLocalObjectManager, ORMWrapper,
+ register_convenience_wrapper)
class ORMWrapperSlice(ORMWrapper):
diff --git a/lib/xos-api/xosapi/convenience/tag.py b/lib/xos-api/xosapi/convenience/tag.py
index aaeaf65..eb6490f 100644
--- a/lib/xos-api/xosapi/convenience/tag.py
+++ b/lib/xos-api/xosapi/convenience/tag.py
@@ -12,6 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
+
from xosapi.orm import ORMWrapper, register_convenience_wrapper
diff --git a/lib/xos-api/xosapi/convenience/user.py b/lib/xos-api/xosapi/convenience/user.py
index 3205680..7dda414 100644
--- a/lib/xos-api/xosapi/convenience/user.py
+++ b/lib/xos-api/xosapi/convenience/user.py
@@ -12,7 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
+
import hashlib
+
from xosapi.orm import ORMWrapper, register_convenience_wrapper
diff --git a/lib/xos-api/xosapi/orm.py b/lib/xos-api/xosapi/orm.py
index 41d775f..08b73e3 100644
--- a/lib/xos-api/xosapi/orm.py
+++ b/lib/xos-api/xosapi/orm.py
@@ -12,15 +12,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import print_function
+from __future__ import absolute_import, print_function
+
+import imp
import os
import sys
import threading
import time
-import imp
import traceback
-from xosconfig import Config
+
from multistructlog import create_logger
+from xosconfig import Config
"""
Django-like ORM layer for gRPC
@@ -43,23 +45,26 @@
u=c.xos_orm.User.objects.get(id=1)
"""
-
log = create_logger(Config().get("logging"))
convenience_wrappers = {}
-# Find the topmost synchronizer-specific function in the call stack
+
def get_synchronizer_function():
+ """
+ Find the topmost synchronizer-specific function in the call stack
+ """
result = None
- for file,line,func,stmt in traceback.extract_stack():
+ for file, line, func, stmt in traceback.extract_stack():
if file.startswith("/opt/xos/synchronizers"):
if not result:
- result = "%s:%s()" % (file,func)
+ result = "%s:%s()" % (file, func)
if not file.startswith("/opt/xos/synchronizers/new_base"):
- result = "%s:%s()" % (file,func)
+ result = "%s:%s()" % (file, func)
break
return result
+
class ORMGenericContentNotFoundException(Exception):
pass
@@ -106,7 +111,7 @@
def diff(self):
d1 = self._initial
d2 = self._dict
- all_field_names = self._wrapped_class.DESCRIPTOR.fields_by_name.keys()
+ all_field_names = list(self._wrapped_class.DESCRIPTOR.fields_by_name.keys())
diffs = []
for k in all_field_names:
if d1.get(k, None) != d2.get(k, None):
@@ -127,11 +132,11 @@
that are set to default values.
"""
if self.is_new:
- return self._dict.keys()
- return self.diff.keys()
+ return list(self._dict.keys())
+ return list(self.diff.keys())
def has_field_changed(self, field_name):
- return field_name in self.diff.keys()
+ return field_name in list(self.diff.keys())
def get_field_diff(self, field_name):
return self.diff.get(field_name, None)
@@ -166,7 +171,7 @@
def gen_fkmap(self):
fkmap = {}
- all_field_names = self._wrapped_class.DESCRIPTOR.fields_by_name.keys()
+ all_field_names = list(self._wrapped_class.DESCRIPTOR.fields_by_name.keys())
for (name, field) in self._wrapped_class.DESCRIPTOR.fields_by_name.items():
if name.endswith("_id"):
@@ -349,10 +354,10 @@
if name == "pk":
name = "id"
- if name in self._fkmap.keys():
+ if name in list(self._fkmap.keys()):
return self.fk_resolve(name)
- if name in self._reverse_fkmap.keys():
+ if name in list(self._reverse_fkmap.keys()):
return self.reverse_fk_resolve(name)
try:
@@ -368,7 +373,7 @@
return getattr(self._wrapped_class, name, *args, **kwargs)
def __setattr__(self, name, value):
- if name in self._fkmap.keys():
+ if name in list(self._fkmap.keys()):
self.fk_set(name, value)
elif name in self.__dict__:
super(ORMWrapper, self).__setattr__(name, value)
@@ -423,17 +428,28 @@
):
classname = self._wrapped_class.__class__.__name__
if self.is_new:
- log.debug("save(): is new", classname=classname, syncstep=get_synchronizer_function())
- new_class = self.stub.invoke(
- "Create%s" % classname, self._wrapped_class
+ log.debug(
+ "save(): is new",
+ classname=classname,
+ syncstep=get_synchronizer_function(),
)
+ new_class = self.stub.invoke("Create%s" % classname, self._wrapped_class)
self._wrapped_class = new_class
self.is_new = False
else:
if self.has_changed:
- log.debug("save(): updated", classname=classname, changed_fields=self.changed_fields, syncstep=get_synchronizer_function())
+ log.debug(
+ "save(): updated",
+ classname=classname,
+ changed_fields=self.changed_fields,
+ syncstep=get_synchronizer_function(),
+ )
else:
- log.debug("save(): no changes", classname=classname, syncstep=get_synchronizer_function())
+ log.debug(
+ "save(): no changes",
+ classname=classname,
+ syncstep=get_synchronizer_function(),
+ )
metadata = []
if update_fields:
metadata.append(("update_fields", ",".join(update_fields)))
@@ -444,9 +460,7 @@
if is_sync_save:
metadata.append(("is_sync_save", "1"))
self.stub.invoke(
- "Update%s" % classname,
- self._wrapped_class,
- metadata=metadata,
+ "Update%s" % classname, self._wrapped_class, metadata=metadata
)
self.do_post_save_fixups()
@@ -648,7 +662,7 @@
return self.wrap_list(self._stub.invoke("Filter%s" % self._modelName, q))
def get(self, **kwargs):
- if kwargs.keys() == ["id"]:
+ if list(kwargs.keys()) == ["id"]:
# the fast and easy case, look it up by id
return self.wrap_single(
self._stub.invoke(
diff --git a/lib/xos-api/xosapi/version.py b/lib/xos-api/xosapi/version.py
index 2c84950..524d081 100644
--- a/lib/xos-api/xosapi/version.py
+++ b/lib/xos-api/xosapi/version.py
@@ -12,5 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# This file will be replaced by setup.py
-__version__ = "unknown"
+from __future__ import absolute_import
+
+import os
+
+# read the version in from VERSION file, which is installed next to this file.
+with open(os.path.join(os.path.dirname(__file__), "VERSION")) as vf:
+ __version__ = vf.read()
diff --git a/lib/xos-api/xosapi/xos_grpc_client.py b/lib/xos-api/xosapi/xos_grpc_client.py
index a80843f..fba9ce4 100644
--- a/lib/xos-api/xosapi/xos_grpc_client.py
+++ b/lib/xos-api/xosapi/xos_grpc_client.py
@@ -12,35 +12,33 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import print_function
+from __future__ import absolute_import, print_function
+
import argparse
import base64
import functools
-import grpc
-import orm
+import inspect
import os
import sys
-from twisted.internet import reactor
from google.protobuf.empty_pb2 import Empty
-from grpc import (
- metadata_call_credentials,
- composite_channel_credentials,
- ssl_channel_credentials,
-)
-# fix up sys.path for chameleon
-import inspect
+import grpc
+from grpc import (composite_channel_credentials, metadata_call_credentials,
+ ssl_channel_credentials)
-currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
-sys.path = [currentdir] + sys.path
-
+from twisted.internet import reactor
from xosconfig import Config
-import chameleon_client.grpc_client as chameleon_client
+
+from xosapi import orm
+import xosapi.chameleon_client.grpc_client as chameleon_client
from multistructlog import create_logger
log = create_logger(Config().get("logging"))
+currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
+sys.path = [currentdir] + sys.path
+
SERVER_CA = "/usr/local/share/ca-certificates/local_certs.crt"
@@ -97,7 +95,7 @@
for cm in response.convenience_methods:
log.debug("Saving convenience method", method=cm.filename)
save_path = os.path.join(convenience_methods_dir, cm.filename)
- file(save_path, "w").write(cm.contents)
+ open(save_path, "w").write(cm.contents)
else:
log.exception(
"Cannot load convenience methods, restarting the synchronzier"
diff --git a/lib/xos-genx/xosgenx/generator.py b/lib/xos-genx/xosgenx/generator.py
index ec21d34..707f87b 100644
--- a/lib/xos-genx/xosgenx/generator.py
+++ b/lib/xos-genx/xosgenx/generator.py
@@ -86,7 +86,7 @@
input = ""
for fname in files:
with open(fname) as infile:
- line_map.append( (len(input.split("\n")), fname) )
+ line_map.append((len(input.split("\n")), fname))
input += infile.read()
return (input, line_map)
@@ -335,7 +335,7 @@
validator = XProtoValidator(v.models, line_map)
validator.validate()
if validator.errors:
- if args.strict_validation or (args.verbosity>=0):
+ if args.strict_validation or (args.verbosity >= 0):
validator.print_errors()
if args.strict_validation:
sys.exit(-1)
diff --git a/lib/xos-genx/xosgenx/validator.py b/lib/xos-genx/xosgenx/validator.py
index 384689b..3007a1a 100644
--- a/lib/xos-genx/xosgenx/validator.py
+++ b/lib/xos-genx/xosgenx/validator.py
@@ -22,18 +22,19 @@
with other design aspects of XOS such as the XOS gRPC API implementation.
"""
-
from __future__ import print_function
import sys
import os
# Options that are always allowed
-COMMON_OPTIONS = ["help_text", "gui_hidden", "tosca_key", "tosca_key_one_of", "feedback_state", "unique", "unique_with"]
+COMMON_OPTIONS = ["help_text", "gui_hidden", "tosca_key", "tosca_key_one_of",
+ "feedback_state", "unique", "unique_with"]
# Options that must be either "True" or "False"
BOOLEAN_OPTIONS = ["blank", "db_index", "feedback_state", "gui_hidden", "null", "tosca_key", "unique", "varchar"]
+
class XProtoValidator(object):
def __init__(self, models, line_map):
"""
@@ -55,7 +56,7 @@
error_filename = "unknown"
error_line_offset = 0
for (start_line, fn) in self.line_map:
- if start_line>error_first_line_number:
+ if start_line > error_first_line_number:
break
error_filename = fn
error_line_offset = start_line
@@ -70,7 +71,7 @@
def print_errors(self):
# Sort by line number
- for error in sorted(self.errors, key=lambda error:error["absolute_line_number"]):
+ for error in sorted(self.errors, key=lambda error: error["absolute_line_number"]):
model = error["model"]
field = error["field"]
message = error["message"]
@@ -83,11 +84,11 @@
linestr = "%d" % first_line_number
print("[ERROR] %s:%s %s.%s (Type %s): %s" % (os.path.basename(error["filename"]),
- linestr,
- model.get("name"),
- field.get("name"),
- field.get("type"),
- message), file=sys.stderr)
+ linestr,
+ model.get("name"),
+ field.get("name"),
+ field.get("type"),
+ message), file=sys.stderr)
def is_option_true(self, field, name):
options = field.get("options")
@@ -109,14 +110,14 @@
for option in options:
if "=" in option:
(optname, optval) = option.split("=")
- if optname==k and optval==v:
+ if optname == k and optval == v:
allowed = True
else:
- if option==k:
+ if option == k:
allowed = True
if not allowed:
- self.error(model, field, "Option %s=%s is not allowed" % (k,v))
+ self.error(model, field, "Option %s=%s is not allowed" % (k, v))
if k in BOOLEAN_OPTIONS and (v not in ["True", "False"]):
self.error(model, field, "Option `%s` must be either True or False, but is '%s'" % (k, v))
@@ -130,7 +131,7 @@
def check_modifier_consistent(self, model, field):
""" Validates that "modifier" is consistent with options.
-
+
Required/optional imply some settings for blank= and null=. These settings are dependent on the type
of field. See also jinja2_extensions/django.py which has to implement some of the same logic.
"""
@@ -159,11 +160,14 @@
# print an error if there's a field conflict
for kmo in mod_out.keys():
if (kmo in options) and (options[kmo] != mod_out[kmo]):
- self.error(model, field, "Option `%s`=`%s` is inconsistent with modifier `%s`" % (kmo, options[kmo], modifier))
+ self.error(model, field, "Option `%s`=`%s` is inconsistent with modifier `%s`" %
+ (kmo, options[kmo], modifier))
def validate_field_date(self, model, field):
self.check_modifier_consistent(model, field)
- self.allow_options(model, field, ["auto_now_add", "blank", "db_index", "default", "max_length", "modifier", "null", "content_type"])
+ self.allow_options(model, field,
+ ["auto_now_add", "blank", "db_index", "default",
+ "max_length", "modifier", "null", "content_type"])
def validate_field_string(self, model, field):
# A string with a `content_type="date"` is actually a date
@@ -179,8 +183,8 @@
self.check_modifier_consistent(model, field)
self.allow_options(model, field,
- ["blank", "choices", "content_type", "db_index", "default", "max_length", "modifier", "null",
- "varchar"])
+ ["blank", "choices", "content_type", "db_index", "default",
+ "max_length", "modifier", "null", "varchar"])
def validate_field_bool(self, model, field):
self.check_modifier_consistent(model, field)
@@ -204,7 +208,7 @@
"modifier", "null", "port", "type=link"])
def validate_field_link(self, model, field):
- link_type = field.get("options",{}).get("link_type")
+ link_type = field.get("options", {}).get("link_type")
if link_type == "manytoone":
self.validate_field_link_onetomany(model, field)
elif link_type == "manytomany":
@@ -219,7 +223,8 @@
return
self.check_modifier_consistent(model, field)
- self.allow_options(model, field, ["blank", "db_index", "default", "max_value", "min_value", "modifier", "null"])
+ self.allow_options(model, field,
+ ["blank", "db_index", "default", "max_value", "min_value", "modifier", "null"])
if self.is_option_true(field, "blank") and not self.is_option_true(field, "null"):
self.error(model, field, "If blank is true then null must also be true")
@@ -243,4 +248,4 @@
def validate(self):
""" Validate all models. This is the main entrypoint for validating xproto. """
for (name, model) in self.models.items():
- self.validate_model(model)
\ No newline at end of file
+ self.validate_model(model)
diff --git a/lib/xos-kafka/.gitignore b/lib/xos-kafka/.gitignore
new file mode 100644
index 0000000..ca52aa7
--- /dev/null
+++ b/lib/xos-kafka/.gitignore
@@ -0,0 +1,2 @@
+# setup.py copies this, don't commit it
+xoskafka/VERSION
diff --git a/lib/xos-kafka/MANIFEST.in b/lib/xos-kafka/MANIFEST.in
index 9561fb1..b9708d6 100644
--- a/lib/xos-kafka/MANIFEST.in
+++ b/lib/xos-kafka/MANIFEST.in
@@ -1 +1,3 @@
include README.rst
+include requirements.txt
+include xoskafka/VERSION
diff --git a/lib/xos-kafka/requirements.txt b/lib/xos-kafka/requirements.txt
new file mode 100644
index 0000000..010540a
--- /dev/null
+++ b/lib/xos-kafka/requirements.txt
@@ -0,0 +1,3 @@
+confluent-kafka~=0.11.5
+multistructlog~=2.1.0
+xosconfig~=2.2.6
diff --git a/lib/xos-kafka/setup.py b/lib/xos-kafka/setup.py
index 3d05854..5662afd 100644
--- a/lib/xos-kafka/setup.py
+++ b/lib/xos-kafka/setup.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
# Copyright 2018-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,8 +12,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from xosutil.autoversion_setup import setup_with_auto_version
-from xosutil.version import __version__
+from __future__ import absolute_import
+
+import os
+from shutil import copyfile
+
+from setuptools import setup
def readme():
@@ -23,21 +25,33 @@
return f.read()
-setup_with_auto_version(
+def version():
+ # Copy VERSION file of parent to module directory if not found
+ if not os.path.exists("xoskafka/VERSION"):
+ copyfile("../../VERSION", "xoskafka/VERSION")
+ with open("xoskafka/VERSION") as f:
+ return f.read().strip()
+
+
+def parse_requirements(filename):
+ # parse a requirements.txt file, allowing for blank lines and comments
+ requirements = []
+ for line in open(filename):
+ if line and not line.startswith("#"):
+ requirements.append(line)
+ return requirements
+
+
+setup(
name="xoskafka",
- version=__version__,
+ version=version(),
description="Wrapper around kafka for XOS",
long_description=readme(),
- classifiers=["License :: OSI Approved :: Apache Software License"],
author="Zack Williams",
author_email="zdw@opennetworking.org",
- packages=["xoskafka"],
+ classifiers=["License :: OSI Approved :: Apache Software License"],
license="Apache v2",
- install_requires=[
- "confluent-kafka>=0.11.5",
- "xosconfig>=2.1.0",
- "multistructlog>=1.5",
- ],
+ packages=["xoskafka"],
+ install_requires=parse_requirements("requirements.txt"),
include_package_data=True,
- zip_safe=False,
)
diff --git a/lib/xos-kafka/tox.ini b/lib/xos-kafka/tox.ini
new file mode 100644
index 0000000..cec6c72
--- /dev/null
+++ b/lib/xos-kafka/tox.ini
@@ -0,0 +1,43 @@
+; Copyright 2019-present Open Networking Foundation
+;
+; Licensed under the Apache License, Version 2.0 (the "License");
+; you may not use this file except in compliance with the License.
+; You may obtain a copy of the License at
+;
+; http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing, software
+; distributed under the License is distributed on an "AS IS" BASIS,
+; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+; See the License for the specific language governing permissions and
+; limitations under the License.
+
+[tox]
+envlist = py27,py35,py36,py37
+skip_missing_interpreters = true
+
+[testenv]
+deps =
+ -r requirements.txt
+ nose2
+ flake8
+
+commands =
+ nose2 -c tox.ini --verbose --junit-xml
+ flake8
+
+[flake8]
+max-line-length = 119
+
+[unittest]
+plugins=nose2.plugins.junitxml
+
+[junit-xml]
+path=nose2-results.xml
+
+[coverage]
+always-on = True
+coverage = xoskafka
+coverage-report =
+ xml
+ term
diff --git a/lib/xos-migrate/.gitignore b/lib/xos-migrate/.gitignore
index ed8b98a..81c7233 100644
--- a/lib/xos-migrate/.gitignore
+++ b/lib/xos-migrate/.gitignore
@@ -1,6 +1,2 @@
-build
-dist
-XosMigrate.egg-info
-.coverage
-coverage.xml
-cover
+# setup.py copies this, don't commit it
+xosmigrate/VERSION
diff --git a/lib/xos-migrate/MANIFEST.in b/lib/xos-migrate/MANIFEST.in
index a38c6ba..7b800b6 100644
--- a/lib/xos-migrate/MANIFEST.in
+++ b/lib/xos-migrate/MANIFEST.in
@@ -1,2 +1,4 @@
-include xosmigrate/migration_cfg_schema.yaml
+include requirements.txt
+include xosmigrate/VERSION
include xosmigrate/migration_cfg.yaml
+include xosmigrate/migration_cfg_schema.yaml
diff --git a/lib/xos-migrate/bin/xos-migrate b/lib/xos-migrate/bin/xos-migrate
old mode 100644
new mode 100755
index 574bc40..243e915
--- a/lib/xos-migrate/bin/xos-migrate
+++ b/lib/xos-migrate/bin/xos-migrate
@@ -1,5 +1,18 @@
#!/usr/bin/env python
-# from xosmigrate import run
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
from xosmigrate.main import run
run()
diff --git a/lib/xos-migrate/requirements.txt b/lib/xos-migrate/requirements.txt
new file mode 100644
index 0000000..803fecb
--- /dev/null
+++ b/lib/xos-migrate/requirements.txt
@@ -0,0 +1,4 @@
+PyYAML~=3.12
+multistructlog~=2.1.0
+xosconfig~=2.2.6
+xosgenx~=2.2.6
diff --git a/lib/xos-migrate/setup.py b/lib/xos-migrate/setup.py
index fe795de..97dc454 100644
--- a/lib/xos-migrate/setup.py
+++ b/lib/xos-migrate/setup.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,24 +12,41 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-try:
- from xosutil.autoversion_setup import setup_with_auto_version as setup
-except ImportError:
- # xosutil is not installed. Expect this to happen when we build an egg, in which case xosgenx.version will
- # automatically have the right version.
- from setuptools import setup
+from __future__ import absolute_import
-from xosgenx.version import __version__
+import os
+from shutil import copyfile
+
+from setuptools import setup
+
+
+def version():
+ # Copy VERSION file of parent to module directory if not found
+ if not os.path.exists("xosmigrate/VERSION"):
+ copyfile("../../VERSION", "xosmigrate/VERSION")
+ with open("xosmigrate/VERSION") as f:
+ return f.read().strip()
+
+
+def parse_requirements(filename):
+ # parse a requirements.txt file, allowing for blank lines and comments
+ requirements = []
+ for line in open(filename):
+ if line and not line.startswith("#"):
+ requirements.append(line)
+ return requirements
+
setup(
- name="XosMigrate",
- version=__version__,
+ name="xosmigrate",
+ version=version(),
description="XOS Migrations Toolkit",
author="Matteo Scandolo",
author_email="teo@opennetworking.org",
+ classifiers=["License :: OSI Approved :: Apache Software License"],
+ license="Apache v2",
packages=["xosmigrate"],
scripts=["bin/xos-migrate"],
+ install_requires=parse_requirements("requirements.txt"),
include_package_data=True,
- # TODO add all deps to the install_requires section
- install_requires=[],
)
diff --git a/lib/xos-migrate/tox.ini b/lib/xos-migrate/tox.ini
new file mode 100644
index 0000000..92ae12f
--- /dev/null
+++ b/lib/xos-migrate/tox.ini
@@ -0,0 +1,44 @@
+; Copyright 2017-present Open Networking Foundation
+;
+; Licensed under the Apache License, Version 2.0 (the "License");
+; you may not use this file except in compliance with the License.
+; You may obtain a copy of the License at
+;
+; http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing, software
+; distributed under the License is distributed on an "AS IS" BASIS,
+; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+; See the License for the specific language governing permissions and
+; limitations under the License.
+
+[tox]
+envlist = py27,py35,py36,py37
+skip_missing_interpreters = true
+
+[testenv]
+deps =
+ -r requirements.txt
+ nose2
+ flake8
+
+commands =
+ nose2 -c tox.ini --verbose --junit-xml
+ flake8
+
+[flake8]
+max-line-length = 119
+
+[unittest]
+plugins=nose2.plugins.junitxml
+
+[junit-xml]
+path=nose2-results.xml
+
+[coverage]
+always-on = True
+coverage = xosmigrate
+coverage-report =
+ xml
+ term
+
diff --git a/lib/xos-migrate/xosmigrate/__init__.py b/lib/xos-migrate/xosmigrate/__init__.py
index 0612f4f..b0fb0b2 100644
--- a/lib/xos-migrate/xosmigrate/__init__.py
+++ b/lib/xos-migrate/xosmigrate/__init__.py
@@ -11,4 +11,3 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-import main
diff --git a/lib/xos-migrate/xosmigrate/main.py b/lib/xos-migrate/xosmigrate/main.py
index 338474d..f8f5d6b 100644
--- a/lib/xos-migrate/xosmigrate/main.py
+++ b/lib/xos-migrate/xosmigrate/main.py
@@ -32,6 +32,7 @@
from xosconfig import Config
from multistructlog import create_logger
+
def get_abs_path(dir_):
""" Convert a path specified by the user, which might be relative or based on
home directory location, into an absolute path.
@@ -330,7 +331,6 @@
# find absolute path to the code
xos_path = get_abs_path(os.path.join(args.repo_root, "orchestration/xos/xos/"))
- django_path = get_abs_path(os.path.join(xos_path, "manage.py"))
core_dir = get_abs_path(os.path.join(xos_path, "core/models/"))
service_base_dir = get_abs_path(os.path.join(xos_path, "../../xos_services/"))
service_dest_dir = get_abs_path(os.path.join(xos_path, "services/"))
diff --git a/lib/xos-synchronizer/.gitignore b/lib/xos-synchronizer/.gitignore
index 5be50bc..b0b91ee 100644
--- a/lib/xos-synchronizer/.gitignore
+++ b/lib/xos-synchronizer/.gitignore
@@ -4,4 +4,7 @@
dist
.coverage
coverage.xml
-cover
\ No newline at end of file
+cover
+
+# setup.py copies this, don't commit it
+VERSION
diff --git a/lib/xos-synchronizer/MANIFEST.in b/lib/xos-synchronizer/MANIFEST.in
new file mode 100644
index 0000000..101c2b1
--- /dev/null
+++ b/lib/xos-synchronizer/MANIFEST.in
@@ -0,0 +1,2 @@
+include requirements.txt
+include xossynchronizer/VERSION
diff --git a/lib/xos-synchronizer/requirements.txt b/lib/xos-synchronizer/requirements.txt
new file mode 100644
index 0000000..dc7e5b9
--- /dev/null
+++ b/lib/xos-synchronizer/requirements.txt
@@ -0,0 +1,14 @@
+Jinja2~=2.10
+confluent-kafka==0.11.5
+multistructlog~=2.1.0
+networkx~=1.11
+inflect~=1.0.1
+astunparse~=1.5.0
+xosconfig~=2.2.6
+xosgenx~=2.2.6
+xosutil~=2.2.6
+
+# remove once xosconfig/xosgenx are updated with correct requirements.txt
+plyxproto~=4.0.0
+pykwalify~=1.6.0
+PyYAML~=3.12
diff --git a/lib/xos-synchronizer/setup.py b/lib/xos-synchronizer/setup.py
index 305e7f6..51d9d78 100644
--- a/lib/xos-synchronizer/setup.py
+++ b/lib/xos-synchronizer/setup.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,24 +12,46 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
-from xosutil.autoversion_setup import setup_with_auto_version
-from xosutil.version import __version__
+import os
+from shutil import copyfile
-setup_with_auto_version(
- name="XosSynchronizer",
- version=__version__,
+from setuptools import setup
+
+
+def version():
+ # Copy VERSION file of parent to module directory if not found
+ if not os.path.exists("xossynchronizer/VERSION"):
+ copyfile("../../VERSION", "xossynchronizer/VERSION")
+ with open("xossynchronizer/VERSION") as f:
+ return f.read().strip()
+
+
+def parse_requirements(filename):
+ # parse a requirements.txt file, allowing for blank lines and comments
+ requirements = []
+ for line in open(filename):
+ if line and not line.startswith("#"):
+ requirements.append(line)
+ return requirements
+
+
+setup(
+ name="xossynchronizer",
+ version=version(),
description="XOS Synchronizer Framework",
author="Scott Baker",
author_email="scottb@opennetworking.org",
+ classifiers=["License :: OSI Approved :: Apache Software License"],
+ license="Apache v2",
packages=[
"xossynchronizer",
"xossynchronizer.steps",
"xossynchronizer.event_steps",
"xossynchronizer.pull_steps",
- "xossynchronizer.model_policies"],
+ "xossynchronizer.model_policies"
+ ],
+ install_requires=parse_requirements("requirements.txt"),
include_package_data=True,
- test_suite='nose2.collector.collector',
- tests_require=['nose2'],
- install_requires=["xosconfig>=2.1.35",],
)
diff --git a/lib/xos-synchronizer/tox.ini b/lib/xos-synchronizer/tox.ini
new file mode 100644
index 0000000..2e09d5e
--- /dev/null
+++ b/lib/xos-synchronizer/tox.ini
@@ -0,0 +1,45 @@
+; Copyright 2019-present Open Networking Foundation
+;
+; Licensed under the Apache License, Version 2.0 (the "License");
+; you may not use this file except in compliance with the License.
+; You may obtain a copy of the License at
+;
+; http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing, software
+; distributed under the License is distributed on an "AS IS" BASIS,
+; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+; See the License for the specific language governing permissions and
+; limitations under the License.
+
+[tox]
+envlist = py27,py35,py36,py37
+skip_missing_interpreters = true
+
+[testenv]
+deps =
+ -r requirements.txt
+ nose2
+ flake8
+ mock
+
+commands =
+ nose2 -c tox.ini --verbose --junit-xml
+; should widen flake8 to also cover xos-synchronizer-tests
+ flake8 xossynchronizer
+
+[flake8]
+; F821, ignoring undefined names would be valuable, but synchronizer dynamically loads them
+; W503, allow breaks before binary operators (see: https://github.com/PyCQA/pycodestyle/issues/498)
+;ignore = F821,W503
+max-line-length = 119
+
+[unittest]
+plugins=nose2.plugins.junitxml
+
+[coverage]
+always-on = True
+coverage = xossynchronizer
+coverage-report =
+ xml
+ term
diff --git a/lib/xos-synchronizer/xos-synchronizer-tests/test_diffs.py b/lib/xos-synchronizer/xos-synchronizer-tests/test_diffs.py
index 9e09c0f..9e0b9d9 100644
--- a/lib/xos-synchronizer/xos-synchronizer-tests/test_diffs.py
+++ b/lib/xos-synchronizer/xos-synchronizer-tests/test_diffs.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
import unittest
from mock import patch, call, Mock, PropertyMock
import json
@@ -24,6 +25,7 @@
xos_dir = os.path.join(test_path, "..", "..", "..", "xos")
services_dir = os.path.join(xos_dir, "../../xos_services")
+
class TestDiffs(unittest.TestCase):
""" These tests are for the mock modelaccessor, to make sure it behaves like the real one """
@@ -39,9 +41,7 @@
Config.init(config, "synchronizer-config-schema.yaml")
# END Setting up the config module
- from xossynchronizer.mock_modelaccessor_build import (
- build_mock_modelaccessor,
- )
+ from xossynchronizer.mock_modelaccessor_build import build_mock_modelaccessor
# FIXME this is to get jenkins to pass the tests, somehow it is running tests in a different order
# and apparently it is not overriding the generated model accessor
@@ -52,9 +52,7 @@
for (
k,
v,
- ) in (
- xossynchronizer.modelaccessor.model_accessor.all_model_classes.items()
- ):
+ ) in xossynchronizer.modelaccessor.model_accessor.all_model_classes.items():
globals()[k] = v
self.log = Mock()
@@ -68,7 +66,7 @@
self.assertEqual(site.is_new, True)
self.assertEqual(site._dict, {"name": "mysite"})
self.assertEqual(site.diff, {})
- self.assertEqual(site.changed_fields, ["name"])
+ self.assertEqual(list(site.changed_fields), ["name"])
self.assertEqual(site.has_field_changed("name"), False)
self.assertEqual(site.has_field_changed("login_base"), False)
@@ -93,7 +91,7 @@
self.assertEqual(site.is_new, False)
self.assertEqual(site._dict, {"id": 1, "name": "mysite", "login_base": "foo"})
self.assertEqual(site.diff, {})
- self.assertEqual(site.changed_fields, [])
+ self.assertEqual(list(site.changed_fields), [])
self.assertEqual(site.has_field_changed("name"), False)
self.assertEqual(site.has_field_changed("login_base"), False)
diff --git a/lib/xos-synchronizer/xos-synchronizer-tests/test_event_engine.py b/lib/xos-synchronizer/xos-synchronizer-tests/test_event_engine.py
index bc1cb97..352c7e3 100644
--- a/lib/xos-synchronizer/xos-synchronizer-tests/test_event_engine.py
+++ b/lib/xos-synchronizer/xos-synchronizer-tests/test_event_engine.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import, print_function
import confluent_kafka
import functools
import unittest
@@ -28,7 +29,7 @@
sync_lib_dir = os.path.join(test_path, "..", "xossynchronizer")
xos_dir = os.path.join(test_path, "..", "..", "..", "xos")
-print os.getcwd()
+print(os.getcwd())
def config_get_mock(orig, overrides, key):
if key in overrides:
diff --git a/lib/xos-synchronizer/xos-synchronizer-tests/test_event_steps/event_step.py b/lib/xos-synchronizer/xos-synchronizer-tests/test_event_steps/event_step.py
index 372d6d3..77402ba 100644
--- a/lib/xos-synchronizer/xos-synchronizer-tests/test_event_steps/event_step.py
+++ b/lib/xos-synchronizer/xos-synchronizer-tests/test_event_steps/event_step.py
@@ -13,6 +13,7 @@
# limitations under the License.
from __future__ import print_function
+from __future__ import absolute_import
from xossynchronizer.event_steps.eventstep import EventStep
from mock_modelaccessor import *
diff --git a/lib/xos-synchronizer/xos-synchronizer-tests/test_load.py b/lib/xos-synchronizer/xos-synchronizer-tests/test_load.py
index 774bc97..73ea213 100644
--- a/lib/xos-synchronizer/xos-synchronizer-tests/test_load.py
+++ b/lib/xos-synchronizer/xos-synchronizer-tests/test_load.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
import unittest
from mock import patch
import mock
@@ -21,6 +22,12 @@
import os
import sys
+try:
+ # Python 2: "reload" is built-in
+ reload # pylint: disable=reload-builtin
+except NameError:
+ from importlib import reload
+
test_path = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
sync_lib_dir = os.path.join(test_path, "..", "xossynchronizer")
xos_dir = os.path.join(test_path, "..", "..", "..", "xos")
@@ -77,10 +84,8 @@
self.synchronizer.load_sync_steps()
model_to_step = self.synchronizer.model_to_step
step_lookup = self.synchronizer.step_lookup
- self.assertIn(
- ("Port", ["SyncPort"]), model_to_step.items()
- )
- self.assertIn(("Image", ["SyncImages"]), model_to_step.items())
+ self.assertIn(("Port", ["SyncPort"]), list(model_to_step.items()))
+ self.assertIn(("Image", ["SyncImages"]), list(model_to_step.items()))
for k, v in model_to_step.items():
val = v[0]
diff --git a/lib/xos-synchronizer/xos-synchronizer-tests/test_loadmodels.py b/lib/xos-synchronizer/xos-synchronizer-tests/test_loadmodels.py
index 6353104..ddb88f3 100644
--- a/lib/xos-synchronizer/xos-synchronizer-tests/test_loadmodels.py
+++ b/lib/xos-synchronizer/xos-synchronizer-tests/test_loadmodels.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
import unittest
from mock import patch, Mock, MagicMock
@@ -20,6 +21,15 @@
import os
import sys
+# Python 3 renamed __builtin__ -> builtins
+# py_builtins is used to help with mocking 'open'
+try:
+ import builtins
+ py_builtins = "builtins"
+except ImportError:
+ import __builtin__ as builtins
+ py_builtins = "__builtin__"
+
test_path = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
@@ -30,7 +40,7 @@
def mock_exists(file_map, fn):
""" mock os.path.exists() """
- return (fn in file_map)
+ return fn in file_map
def mock_open(orig_open, file_map, fn, *args, **kwargs):
@@ -79,6 +89,7 @@
from xossynchronizer import loadmodels
from xossynchronizer.loadmodels import ModelLoadClient
+
self.loadmodels = loadmodels
self.api = MagicMock()
@@ -90,22 +101,31 @@
os.chdir(self.cwd_save)
def test_upload_models(self):
- dir_map = {"models_dir": ["models.xproto", "models.py"],
- "models_dir/convenience": ["convenience1.py"],
- "models_dir/../migrations": ["migration1.py", "migration2.py"]}
+ dir_map = {
+ "models_dir": ["models.xproto", "models.py"],
+ "models_dir/convenience": ["convenience1.py"],
+ "models_dir/../migrations": ["migration1.py", "migration2.py"],
+ }
- file_map = {"models_dir/models.xproto": u"some xproto",
- "models_dir/models.py": u"print `python models file`",
- "models_dir/convenience": u"directory",
- "models_dir/convenience/convenience1.py": u"print `python convenience file`",
- "models_dir/../migrations": u"directory",
- "models_dir/../migrations/migration1.py": u"print `first migration`",
- "models_dir/../migrations/migration2.py": u"print `second migration`"}
+ file_map = {
+ "models_dir/models.xproto": u"some xproto",
+ "models_dir/models.py": u"print `python models file`",
+ "models_dir/convenience": u"directory",
+ "models_dir/convenience/convenience1.py": u"print `python convenience file`",
+ "models_dir/../migrations": u"directory",
+ "models_dir/../migrations/migration1.py": u"print `first migration`",
+ "models_dir/../migrations/migration2.py": u"print `second migration`",
+ }
- orig_open = open
- with patch("os.listdir", side_effect=functools.partial(mock_listdir, dir_map)), \
- patch("os.path.exists", side_effect=functools.partial(mock_exists, file_map)), \
- patch("__builtin__.open", side_effect=functools.partial(mock_open, orig_open, file_map)):
+ orig_open = builtins.open
+ with patch(
+ "os.listdir", side_effect=functools.partial(mock_listdir, dir_map)
+ ), patch(
+ "os.path.exists", side_effect=functools.partial(mock_exists, file_map)
+ ), patch(
+ py_builtins + ".open",
+ side_effect=functools.partial(mock_open, orig_open, file_map),
+ ):
self.loader.upload_models("myservice", "models_dir", "1.2")
request = self.api.dynamicload.LoadModels.call_args[0][0]
@@ -118,17 +138,28 @@
self.assertEqual(len(request.decls.items), 1)
self.assertEqual(request.decls.items[0].filename, "models.py")
- self.assertEqual(request.decls.items[0].contents, u"print `python models file`")
+ self.assertEqual(
+ request.decls.items[0].contents, u"print `python models file`"
+ )
self.assertEqual(len(request.convenience_methods.items), 1)
- self.assertEqual(request.convenience_methods.items[0].filename, "convenience1.py")
- self.assertEqual(request.convenience_methods.items[0].contents, u"print `python convenience file`")
+ self.assertEqual(
+ request.convenience_methods.items[0].filename, "convenience1.py"
+ )
+ self.assertEqual(
+ request.convenience_methods.items[0].contents,
+ u"print `python convenience file`",
+ )
self.assertEqual(len(request.migrations.items), 2)
self.assertEqual(request.migrations.items[0].filename, "migration1.py")
- self.assertEqual(request.migrations.items[0].contents, u"print `first migration`")
+ self.assertEqual(
+ request.migrations.items[0].contents, u"print `first migration`"
+ )
self.assertEqual(request.migrations.items[1].filename, "migration2.py")
- self.assertEqual(request.migrations.items[1].contents, u"print `second migration`")
+ self.assertEqual(
+ request.migrations.items[1].contents, u"print `second migration`"
+ )
if __name__ == "__main__":
diff --git a/lib/xos-synchronizer/xos-synchronizer-tests/test_run.py b/lib/xos-synchronizer/xos-synchronizer-tests/test_run.py
index a9c6c19..f21428e 100644
--- a/lib/xos-synchronizer/xos-synchronizer-tests/test_run.py
+++ b/lib/xos-synchronizer/xos-synchronizer-tests/test_run.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
import json
import unittest
import mock
@@ -19,6 +20,12 @@
import os
import sys
+try:
+ # Python 2: "reload" is built-in
+ reload # pylint: disable=reload-builtin
+except NameError:
+ from importlib import reload
+
test_path = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
sync_lib_dir = os.path.join(test_path, "..", "xossynchronizer")
xos_dir = os.path.join(test_path, "..", "..", "..", "xos")
@@ -47,13 +54,15 @@
Config.clear()
Config.init(config, "synchronizer-config-schema.yaml")
- from xossynchronizer.mock_modelaccessor_build import (
- build_mock_modelaccessor,
+ from xossynchronizer.mock_modelaccessor_build import build_mock_modelaccessor
+
+ build_mock_modelaccessor(
+ sync_lib_dir, xos_dir, services_dir=None, service_xprotos=[]
)
- build_mock_modelaccessor(sync_lib_dir, xos_dir, services_dir=None, service_xprotos=[])
-
- os.chdir(os.path.join(test_path, "..")) # config references xos-synchronizer-tests/model-deps
+ os.chdir(
+ os.path.join(test_path, "..")
+ ) # config references xos-synchronizer-tests/model-deps
import xossynchronizer.event_loop
@@ -72,7 +81,9 @@
b = xossynchronizer.backend.Backend(model_accessor=model_accessor)
steps_dir = Config.get("steps_dir")
self.steps = b.load_sync_step_modules(steps_dir)
- self.synchronizer = xossynchronizer.event_loop.XOSObserver(self.steps, model_accessor)
+ self.synchronizer = xossynchronizer.event_loop.XOSObserver(
+ self.steps, model_accessor
+ )
try:
os.remove("/tmp/sync_ports")
except OSError:
diff --git a/lib/xos-synchronizer/xos-synchronizer-tests/test_scheduler.py b/lib/xos-synchronizer/xos-synchronizer-tests/test_scheduler.py
index 0aa5334..1dc8fdf 100644
--- a/lib/xos-synchronizer/xos-synchronizer-tests/test_scheduler.py
+++ b/lib/xos-synchronizer/xos-synchronizer-tests/test_scheduler.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
import unittest
from mock import patch
import mock
@@ -21,12 +22,18 @@
import os
import sys
+try:
+ # Python 2: "reload" is built-in
+ reload # pylint: disable=reload-builtin
+except NameError:
+ from importlib import reload
+
test_path = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
sync_lib_dir = os.path.join(test_path, "..", "xossynchronizer")
xos_dir = os.path.join(test_path, "..", "..", "..", "xos")
-class TestScheduling(unittest.TestCase):
+class TestScheduling(unittest.TestCase):
def setUp(self):
global mock_enumerator, event_loop
@@ -39,15 +46,18 @@
Config.clear()
Config.init(config, "synchronizer-config-schema.yaml")
- from xossynchronizer.mock_modelaccessor_build import (
- build_mock_modelaccessor,
+ from xossynchronizer.mock_modelaccessor_build import build_mock_modelaccessor
+
+ build_mock_modelaccessor(
+ sync_lib_dir, xos_dir, services_dir=None, service_xprotos=[]
)
- build_mock_modelaccessor(sync_lib_dir, xos_dir, services_dir=None, service_xprotos=[])
-
- os.chdir(os.path.join(test_path, "..")) # config references xos-synchronizer-tests/model-deps
+ os.chdir(
+ os.path.join(test_path, "..")
+ ) # config references xos-synchronizer-tests/model-deps
import xossynchronizer.event_loop
+
event_loop = xossynchronizer.event_loop
reload(xossynchronizer.event_loop)
@@ -64,7 +74,9 @@
b = xossynchronizer.backend.Backend(model_accessor=model_accessor)
steps_dir = Config.get("steps_dir")
self.steps = b.load_sync_step_modules(steps_dir)
- self.synchronizer = xossynchronizer.event_loop.XOSObserver(self.steps, model_accessor)
+ self.synchronizer = xossynchronizer.event_loop.XOSObserver(
+ self.steps, model_accessor
+ )
def tearDown(self):
sys.path = self.sys_path_save
@@ -106,6 +118,7 @@
self.assertTrue(verdict)
self.assertEqual(edge_type, event_loop.DIRECT_EDGE)
+
def test_concrete_object_path_distant(self):
p = ComputeServiceInstance()
s = Slice()
diff --git a/lib/xos-synchronizer/xos-synchronizer-tests/test_services.py b/lib/xos-synchronizer/xos-synchronizer-tests/test_services.py
index 3ff1c43..96fd534 100644
--- a/lib/xos-synchronizer/xos-synchronizer-tests/test_services.py
+++ b/lib/xos-synchronizer/xos-synchronizer-tests/test_services.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
import unittest
from mock import patch
import mock
@@ -21,6 +22,12 @@
import os
import sys
+try:
+ # Python 2: "reload" is built-in
+ reload # pylint: disable=reload-builtin
+except NameError:
+ from importlib import reload
+
test_path = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
sync_lib_dir = os.path.join(test_path, "..", "xossynchronizer")
xos_dir = os.path.join(test_path, "..", "..", "..", "xos")
@@ -37,13 +44,15 @@
Config.clear()
Config.init(config, "synchronizer-config-schema.yaml")
- from xossynchronizer.mock_modelaccessor_build import (
- build_mock_modelaccessor,
+ from xossynchronizer.mock_modelaccessor_build import build_mock_modelaccessor
+
+ build_mock_modelaccessor(
+ sync_lib_dir, xos_dir, services_dir=None, service_xprotos=[]
)
- build_mock_modelaccessor(sync_lib_dir, xos_dir, services_dir=None, service_xprotos=[])
-
- os.chdir(os.path.join(test_path, "..")) # config references xos-synchronizer-tests/model-deps
+ os.chdir(
+ os.path.join(test_path, "..")
+ ) # config references xos-synchronizer-tests/model-deps
import xossynchronizer.event_loop
@@ -60,7 +69,9 @@
b = xossynchronizer.backend.Backend(model_accessor=model_accessor)
steps_dir = Config.get("steps_dir")
self.steps = b.load_sync_step_modules(steps_dir)
- self.synchronizer = xossynchronizer.event_loop.XOSObserver(self.steps, model_accessor)
+ self.synchronizer = xossynchronizer.event_loop.XOSObserver(
+ self.steps, model_accessor
+ )
def tearDown(self):
sys.path = self.sys_path_save
diff --git a/lib/xos-synchronizer/xos-synchronizer-tests/test_steps/sync_images.py b/lib/xos-synchronizer/xos-synchronizer-tests/test_steps/sync_images.py
index b3ed9bd..2ed1e40 100644
--- a/lib/xos-synchronizer/xos-synchronizer-tests/test_steps/sync_images.py
+++ b/lib/xos-synchronizer/xos-synchronizer-tests/test_steps/sync_images.py
@@ -13,8 +13,10 @@
# limitations under the License.
+from __future__ import absolute_import
from xossynchronizer.steps.syncstep import SyncStep
+
class SyncImages(SyncStep):
requested_interval = 0
observes = ["Image"]
diff --git a/lib/xos-synchronizer/xos-synchronizer-tests/test_steps/sync_ports.py b/lib/xos-synchronizer/xos-synchronizer-tests/test_steps/sync_ports.py
index a7eb7d1..1fbbbf5 100644
--- a/lib/xos-synchronizer/xos-synchronizer-tests/test_steps/sync_ports.py
+++ b/lib/xos-synchronizer/xos-synchronizer-tests/test_steps/sync_ports.py
@@ -13,6 +13,7 @@
# limitations under the License.
+from __future__ import absolute_import
import os
import base64
from xossynchronizer.steps.syncstep import SyncStep
diff --git a/lib/xos-synchronizer/xossynchronizer/__init__.py b/lib/xos-synchronizer/xossynchronizer/__init__.py
index 18ab956..ea109c8 100644
--- a/lib/xos-synchronizer/xossynchronizer/__init__.py
+++ b/lib/xos-synchronizer/xossynchronizer/__init__.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
from .synchronizer import Synchronizer
__all__ = ["Synchronizer"]
diff --git a/lib/xos-synchronizer/xossynchronizer/ansible_helper.py b/lib/xos-synchronizer/xossynchronizer/ansible_helper.py
index c607607..846aeb9 100644
--- a/lib/xos-synchronizer/xossynchronizer/ansible_helper.py
+++ b/lib/xos-synchronizer/xossynchronizer/ansible_helper.py
@@ -14,24 +14,20 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import print_function
-import jinja2
-import tempfile
-import os
-import json
-import pickle
-import pdb
-import string
-import random
-import re
-import traceback
-import subprocess
-import threading
+from __future__ import absolute_import, print_function
-from multiprocessing import Process, Queue
-from xosconfig import Config
+import json
+import os
+import pickle
+import random
+import string
+import tempfile
+
+import jinja2
from multistructlog import create_logger
+from xosconfig import Config
+from six.moves import range
log = create_logger(Config().get("logging"))
@@ -108,7 +104,7 @@
stats = result.get("stats", None)
aresults = result.get("aresults", None)
- except Exception as e:
+ except BaseException:
log.exception("Exception running ansible_main")
stats = None
aresults = None
@@ -220,7 +216,7 @@
if t["failures"] > 0:
raise ValueError("Ansible playbook reported failures for host %s" % h)
- except ValueError as e:
+ except ValueError:
if error_msg:
try:
error = " // ".join(error_msg)
@@ -230,7 +226,7 @@
else:
raise
- processed_results = map(lambda x: x._result, ok_results)
+ processed_results = [x._result for x in ok_results]
return processed_results[1:] # 0 is setup
diff --git a/lib/xos-synchronizer/xossynchronizer/ansible_main.py b/lib/xos-synchronizer/xossynchronizer/ansible_main.py
index 08283a4..6a8c711 100644
--- a/lib/xos-synchronizer/xossynchronizer/ansible_main.py
+++ b/lib/xos-synchronizer/xossynchronizer/ansible_main.py
@@ -12,15 +12,21 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
import os
import pickle
import sys
-
-# import json
import traceback
+
from xosconfig import Config
+try:
+ # Python 2: "reload" is built-in
+ reload # pylint: disable=reload-builtin
+except NameError:
+ from importlib import reload
+
sys.path.append("/opt/xos")
@@ -42,7 +48,7 @@
except KeyError:
pass
- import ansible_runner
+ from . import ansible_runner
reload(ansible_runner)
@@ -52,7 +58,7 @@
)
stats, aresults = runner.run()
- except Exception as e:
+ except Exception:
return {"stats": None, "aresults": None, "exception": traceback.format_exc()}
return {"stats": stats, "aresults": aresults}
diff --git a/lib/xos-synchronizer/xossynchronizer/ansible_runner.py b/lib/xos-synchronizer/xossynchronizer/ansible_runner.py
index d20feb5..2615346 100644
--- a/lib/xos-synchronizer/xossynchronizer/ansible_runner.py
+++ b/lib/xos-synchronizer/xossynchronizer/ansible_runner.py
@@ -14,22 +14,30 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from multistructlog import create_logger
-from xosconfig import Config
-from ansible.plugins.callback import CallbackBase
-from ansible.utils.display import Display
-from ansible.executor import playbook_executor
-from ansible.parsing.dataloader import DataLoader
-from ansible.vars.manager import VariableManager
-from ansible.inventory.manager import InventoryManager
-from tempfile import NamedTemporaryFile
-import os
-import sys
-import pdb
+from __future__ import absolute_import
+
import json
+import os
import uuid
from ansible import constants
+from ansible.executor import playbook_executor
+from ansible.inventory.manager import InventoryManager
+from ansible.parsing.dataloader import DataLoader
+from ansible.plugins.callback import CallbackBase
+from ansible.utils.display import Display
+from ansible.vars.manager import VariableManager
+
+from multistructlog import create_logger
+from xosconfig import Config
+
+try:
+ # Python 2: "reload" is built-in
+ # pylint: disable=W1626
+ reload
+except NameError:
+ # Python 3: "reload" is part of importlib
+ from importlib import reload
constants = reload(constants)
diff --git a/lib/xos-synchronizer/xossynchronizer/apiaccessor.py b/lib/xos-synchronizer/xossynchronizer/apiaccessor.py
index a56381b..b4e3b13 100644
--- a/lib/xos-synchronizer/xossynchronizer/apiaccessor.py
+++ b/lib/xos-synchronizer/xossynchronizer/apiaccessor.py
@@ -13,9 +13,11 @@
# limitations under the License.
-from modelaccessor import ModelAccessor
+from __future__ import absolute_import
+
import datetime
-import time
+
+from .modelaccessor import ModelAccessor
class CoreApiModelAccessor(ModelAccessor):
diff --git a/lib/xos-synchronizer/xossynchronizer/backend.py b/lib/xos-synchronizer/xossynchronizer/backend.py
index 55977b2..96e412f 100644
--- a/lib/xos-synchronizer/xossynchronizer/backend.py
+++ b/lib/xos-synchronizer/xossynchronizer/backend.py
@@ -12,22 +12,22 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import print_function
-import os
-import inspect
+from __future__ import absolute_import, print_function
+
import imp
+import inspect
+import os
import sys
import threading
import time
-from xossynchronizer.steps.syncstep import SyncStep
+
+from multistructlog import create_logger
+from xosconfig import Config
+from xossynchronizer.event_engine import XOSEventEngine
from xossynchronizer.event_loop import XOSObserver
from xossynchronizer.model_policy_loop import XOSPolicyEngine
-from xossynchronizer.event_engine import XOSEventEngine
from xossynchronizer.pull_step_engine import XOSPullStepEngine
-from xosconfig import Config
-from multistructlog import create_logger
-
log = create_logger(Config().get("logging"))
@@ -100,7 +100,11 @@
# if we have at least one sync_step
if len(sync_steps) > 0:
# start the observer
- self.log.info("Starting XOSObserver", sync_steps=sync_steps, model_accessor=self.model_accessor)
+ self.log.info(
+ "Starting XOSObserver",
+ sync_steps=sync_steps,
+ model_accessor=self.model_accessor,
+ )
observer = XOSObserver(sync_steps, self.model_accessor, self.log)
observer_thread = threading.Thread(
target=observer.run, name="synchronizer"
@@ -131,14 +135,20 @@
self.log.info("Skipping event engine due to synchronizer unloading.")
else:
self.log.info("Starting XOSEventEngine", event_steps_dir=event_steps_dir)
- event_engine = XOSEventEngine(model_accessor=self.model_accessor, log=self.log)
+ event_engine = XOSEventEngine(
+ model_accessor=self.model_accessor, log=self.log
+ )
event_engine.load_event_step_modules(event_steps_dir)
event_engine.start()
# start model policies thread
policies_dir = Config.get("model_policies_dir")
if policies_dir:
- policy_engine = XOSPolicyEngine(policies_dir=policies_dir, model_accessor=self.model_accessor, log=self.log)
+ policy_engine = XOSPolicyEngine(
+ policies_dir=policies_dir,
+ model_accessor=self.model_accessor,
+ log=self.log,
+ )
model_policy_thread = threading.Thread(
target=policy_engine.run, name="policy_engine"
)
diff --git a/lib/xos-synchronizer/xossynchronizer/backend_modelpolicy.py b/lib/xos-synchronizer/xossynchronizer/backend_modelpolicy.py
index a8e826b..82eb7b8 100644
--- a/lib/xos-synchronizer/xossynchronizer/backend_modelpolicy.py
+++ b/lib/xos-synchronizer/xossynchronizer/backend_modelpolicy.py
@@ -12,18 +12,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import print_function
-import os
-import inspect
-import imp
+from __future__ import absolute_import, print_function
+
import sys
import threading
import time
-from syncstep import SyncStep
-from synchronizers.new_base.event_loop import XOSObserver
-from xosconfig import Config
from multistructlog import create_logger
+from xosconfig import Config
log = create_logger(Config().get("logging"))
diff --git a/lib/xos-synchronizer/xossynchronizer/dependency_walker_new.py b/lib/xos-synchronizer/xossynchronizer/dependency_walker_new.py
index 138c26d..8c9c8f9 100644
--- a/lib/xos-synchronizer/xossynchronizer/dependency_walker_new.py
+++ b/lib/xos-synchronizer/xossynchronizer/dependency_walker_new.py
@@ -12,26 +12,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
-#!/usr/bin/env python
-
# TODO: Moved this into the synchronizer, as it appeared to require model
# access. Verify whether or not that's true and reconcile with
# generate/dependency_walker.py
-from __future__ import print_function
-import os
-import imp
-import inspect
-import time
-import traceback
-import commands
-import threading
-from xosconfig import Config
+from __future__ import absolute_import, print_function
+
import json
-from xosconfig import Config
from multistructlog import create_logger
+from xosconfig import Config
log = create_logger(Config().get("logging"))
diff --git a/lib/xos-synchronizer/xossynchronizer/event_engine.py b/lib/xos-synchronizer/xossynchronizer/event_engine.py
index 694a1a8..0455f0d 100644
--- a/lib/xos-synchronizer/xossynchronizer/event_engine.py
+++ b/lib/xos-synchronizer/xossynchronizer/event_engine.py
@@ -12,12 +12,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import confluent_kafka
+from __future__ import absolute_import
+
import imp
import inspect
import os
import threading
import time
+
+import confluent_kafka
+
from xosconfig import Config
@@ -130,7 +134,9 @@
)
try:
- self.step(model_accessor=self.model_accessor, log=self.log).process_event(event_msg)
+ self.step(
+ model_accessor=self.model_accessor, log=self.log
+ ).process_event(event_msg)
except BaseException:
self.log.exception(
@@ -207,7 +213,9 @@
for step in self.event_steps:
if step.technology == "kafka":
- thread = XOSKafkaThread(step, [eventbus_endpoint], self.model_accessor, self.log)
+ thread = XOSKafkaThread(
+ step, [eventbus_endpoint], self.model_accessor, self.log
+ )
thread.start()
self.threads.append(thread)
else:
diff --git a/lib/xos-synchronizer/xossynchronizer/event_loop.py b/lib/xos-synchronizer/xossynchronizer/event_loop.py
index 5cbfda9..15f005c 100644
--- a/lib/xos-synchronizer/xossynchronizer/event_loop.py
+++ b/lib/xos-synchronizer/xossynchronizer/event_loop.py
@@ -16,23 +16,28 @@
# Add unit tests:
# - 2 sets of Instance, ControllerSlice, ControllerNetworks - delete and create case
-import time
-import threading
-import json
+from __future__ import absolute_import
+import json
+import threading
+import time
from collections import defaultdict
+
+from multistructlog import create_logger
from networkx import (
DiGraph,
- weakly_connected_component_subgraphs,
- all_shortest_paths,
NetworkXNoPath,
+ all_shortest_paths,
+ weakly_connected_component_subgraphs,
)
from networkx.algorithms.dag import topological_sort
-
-from xossynchronizer.steps.syncstep import InnocuousException, DeferredException, SyncStep
-
from xosconfig import Config
-from multistructlog import create_logger
+from xossynchronizer.steps.syncstep import (
+ DeferredException,
+ InnocuousException,
+ SyncStep,
+)
+from six.moves import range
log = create_logger(Config().get("logging"))
@@ -113,10 +118,12 @@
# src_port is the field that accesses Model2 from Model1
# dst_port is the field that accesses Model1 from Model2
static_dependencies = json.loads(dep_graph_str)
- dynamic_dependencies = [] # Dropped Service and ServiceInstance dynamic dependencies
+ dynamic_dependencies = (
+ []
+ ) # Dropped Service and ServiceInstance dynamic dependencies
joint_dependencies = dict(
- static_dependencies.items() + dynamic_dependencies
+ list(static_dependencies.items()) + dynamic_dependencies
)
model_dependency_graph = DiGraph()
@@ -267,7 +274,6 @@
def handle_sync_exception(self, o, e):
self.log.exception("sync step failed!", e=e, **o.tologdict())
- current_code = o.backend_code
if hasattr(e, "message"):
status = str(e.message)
@@ -359,7 +365,6 @@
sc_log = self.log.new(thread_id=threading.current_thread().ident)
try:
- start_time = time.time()
sc_log.debug("Starting to work on cohort", cohort=cohort, deletion=deletion)
cohort_emptied = False
@@ -417,14 +422,13 @@
self.run_once()
def fetch_pending(self, deletion=False):
- unique_model_list = list(set(self.model_to_step.keys()))
pending_objects = []
pending_steps = []
- step_list = self.step_lookup.values()
+ step_list = list(self.step_lookup.values())
for e in self.external_dependencies:
s = SyncStep
- if isinstance(e,str):
+ if isinstance(e, str):
# external dependency is a string that names a model class
s.observes = self.model_accessor.get_model_class(e)
else:
@@ -439,7 +443,9 @@
if not hasattr(step, "call"):
pending = step.fetch_pending(deletion)
for obj in pending:
- step = step_class(driver=self.driver, model_accessor=self.model_accessor)
+ step = step_class(
+ driver=self.driver, model_accessor=self.model_accessor
+ )
step.log = self.log.new(step=step)
obj.synchronizer_step = step
@@ -460,7 +466,7 @@
if o is None:
return [], None
try:
- o_lst = [o for o in o.all()]
+ o_lst = [oa for oa in o.all()]
edge_type = PROXY_EDGE
except (AttributeError, TypeError):
o_lst = [o]
@@ -606,10 +612,8 @@
"""
def compute_dependent_cohorts(self, objects, deletion):
- model_map = defaultdict(list)
n = len(objects)
- r = range(n)
- indexed_objects = zip(r, objects)
+ r = list(range(n))
oG = DiGraph()
@@ -652,8 +656,6 @@
# Why are we checking the DB connection here?
self.model_accessor.check_db_connection_okay()
- loop_start = time.time()
-
# Two passes. One for sync, the other for deletion.
for deletion in (False, True):
objects_to_process = []
@@ -693,8 +695,6 @@
except Exception as e:
self.log.exception("Legacy step failed", step=step, e=e)
- loop_end = time.time()
-
except Exception as e:
self.log.exception(
"Core error. This seems like a misconfiguration or bug. This error will not be relayed to the user!",
diff --git a/lib/xos-synchronizer/xossynchronizer/loadmodels.py b/lib/xos-synchronizer/xossynchronizer/loadmodels.py
index 1c1f8bd..43b429e 100644
--- a/lib/xos-synchronizer/xossynchronizer/loadmodels.py
+++ b/lib/xos-synchronizer/xossynchronizer/loadmodels.py
@@ -12,9 +12,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
+
import os
-from xosconfig import Config
+
from multistructlog import create_logger
+from xosconfig import Config
log = create_logger(Config().get("logging"))
@@ -89,4 +92,3 @@
result = self.api.dynamicload.UnloadModels(request)
return result
-
diff --git a/lib/xos-synchronizer/xossynchronizer/mock_modelaccessor_build.py b/lib/xos-synchronizer/xossynchronizer/mock_modelaccessor_build.py
index 99b2d46..1db6f26 100644
--- a/lib/xos-synchronizer/xossynchronizer/mock_modelaccessor_build.py
+++ b/lib/xos-synchronizer/xossynchronizer/mock_modelaccessor_build.py
@@ -12,10 +12,19 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
+
import os
-import cPickle
import subprocess
+try:
+ # Python 2 has separate pickle and cPickle
+ # pylint: disable=W1648
+ import cPickle
+except ImportError:
+ # Python 3 will use cPickle by dfault
+ import pickle as cPickle
+
"""
Support for autogenerating mock_modelaccessor.
@@ -31,7 +40,7 @@
# TODO: deprecate the dest_dir argument
# force modelaccessor to be found in /tmp
- dest_dir="/tmp/mock_modelaccessor"
+ dest_dir = "/tmp/mock_modelaccessor"
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
dest_fn = os.path.join(dest_dir, "mock_modelaccessor.py")
@@ -44,13 +53,13 @@
# Check to see if we've already run xosgenx. If so, don't run it again.
context_fn = dest_fn + ".context"
this_context = (xos_dir, services_dir, service_xprotos, target)
- need_xosgenx = True
+
if os.path.exists(context_fn):
try:
- context = cPickle.loads(open(context_fn).read())
+ context = cPickle.loads(open(context_fn, 'rb').read())
if context == this_context:
return
- except (cPickle.UnpicklingError, EOFError):
+ except (cPickle.UnpicklingError, EOFError, ValueError):
# Something went wrong with the file read or depickling
pass
@@ -74,7 +83,8 @@
)
# Save the context of this invocation of xosgenx
- open(context_fn, "w").write(cPickle.dumps(this_context))
+ open(context_fn, "wb").write(cPickle.dumps(this_context))
+
# generate model from xproto
def get_models_fn(services_dir, service_name, xproto_name):
@@ -86,8 +96,11 @@
if os.path.exists(os.path.join(services_dir, name)):
return name
raise Exception("Unable to find service=%s xproto=%s" % (service_name, xproto_name))
+
+
# END generate model from xproto
+
def mock_modelaccessor_config(test_dir, services):
""" Automatically configure the mock modelaccessor.
@@ -99,13 +112,13 @@
while not orchestration_dir.endswith("orchestration"):
# back up a level
orchestration_dir = os.path.dirname(orchestration_dir)
- if len(orchestration_dir)<10:
+ if len(orchestration_dir) < 10:
raise Exception("Failed to autodiscovery repository tree")
xos_dir = os.path.join(orchestration_dir, "xos", "xos")
services_dir = os.path.join(orchestration_dir, "xos_services")
- service_xprotos=[]
+ service_xprotos = []
for (service_name, xproto_name) in services:
service_xprotos.append(get_models_fn(services_dir, service_name, xproto_name))
diff --git a/lib/xos-synchronizer/xossynchronizer/model_policies/policy.py b/lib/xos-synchronizer/xossynchronizer/model_policies/policy.py
index 5877279..b33a634 100644
--- a/lib/xos-synchronizer/xossynchronizer/model_policies/policy.py
+++ b/lib/xos-synchronizer/xossynchronizer/model_policies/policy.py
@@ -18,8 +18,10 @@
Base Classes for Model Policies
"""
-from xosconfig import Config
+from __future__ import absolute_import
+
from multistructlog import create_logger
+from xosconfig import Config
log = create_logger(Config().get("logging"))
diff --git a/lib/xos-synchronizer/xossynchronizer/model_policy_loop.py b/lib/xos-synchronizer/xossynchronizer/model_policy_loop.py
index 20144a5..5259778 100644
--- a/lib/xos-synchronizer/xossynchronizer/model_policy_loop.py
+++ b/lib/xos-synchronizer/xossynchronizer/model_policy_loop.py
@@ -12,11 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import, print_function
-from __future__ import print_function
-from xossynchronizer.dependency_walker_new import *
-from xossynchronizer.model_policies.policy import Policy
-
+import os
import imp
import inspect
import time
@@ -25,11 +23,13 @@
class XOSPolicyEngine(object):
def __init__(self, policies_dir, model_accessor, log):
+
+ self.log = log # has to come before self.load_model_policies(), which logs
+
self.model_accessor = model_accessor
self.model_policies = self.load_model_policies(policies_dir)
self.policies_by_name = {}
self.policies_by_class = {}
- self.log = log
for policy in self.model_policies:
if policy.model_name not in self.policies_by_name:
@@ -53,7 +53,7 @@
def update_dep(self, d, o):
try:
- print("Trying to update %s" % d)
+ self.log.info("Trying to update %s", d)
save_fields = []
if d.updated < o.updated:
save_fields = ["updated"]
@@ -61,15 +61,15 @@
if save_fields:
d.save(update_fields=save_fields)
except AttributeError as e:
- log.exception("AttributeError in update_dep", e=e)
+ self.log.exception("AttributeError in update_dep", e=e)
raise e
except Exception as e:
- log.exception("Exception in update_dep", e=e)
+ self.log.exception("Exception in update_dep", e=e)
def delete_if_inactive(self, d, o):
try:
d.delete()
- print("Deleted %s (%s)" % (d, d.__class__.__name__))
+ self.log.info("Deleted %s (%s)" % (d, d.__class__.__name__))
except BaseException:
pass
return
@@ -103,13 +103,13 @@
and (c not in policies)
):
if not c.model_name:
- log.info(
+ self.log.info(
"load_model_policies: skipping model policy",
classname=classname,
)
continue
if not self.model_accessor.has_model_class(c.model_name):
- log.error(
+ self.log.error(
"load_model_policies: unable to find model policy",
classname=classname,
model=c.model_name,
@@ -117,12 +117,12 @@
c.model = self.model_accessor.get_model_class(c.model_name)
policies.append(c)
- log.info("Loaded model policies", policies=policies)
+ self.log.info("Loaded model policies", policies=policies)
return policies
def execute_model_policy(self, instance, action):
# These are the models whose children get deleted when they are
- delete_policy_models = ["Slice", "Instance", "Network"]
+ # delete_policy_models = ["Slice", "Instance", "Network"]
sender_name = getattr(instance, "model_name", instance.__class__.__name__)
# if (action != "deleted"):
@@ -136,15 +136,17 @@
method_name = "handle_%s" % action
if hasattr(policy, method_name):
try:
- log.debug(
+ self.log.debug(
"MODEL POLICY: calling handler",
sender_name=sender_name,
instance=instance,
policy=policy.__name__,
method=method_name,
)
- getattr(policy(model_accessor=self.model_accessor), method_name)(instance)
- log.debug(
+ getattr(policy(model_accessor=self.model_accessor), method_name)(
+ instance
+ )
+ self.log.debug(
"MODEL POLICY: completed handler",
sender_name=sender_name,
instance=instance,
@@ -152,7 +154,7 @@
method=method_name,
)
except Exception as e:
- log.exception("MODEL POLICY: Exception when running handler", e=e)
+ self.log.exception("MODEL POLICY: Exception when running handler", e=e)
policies_failed = True
try:
@@ -160,7 +162,7 @@
instance.policy_code = 2
instance.save(update_fields=["policy_status", "policy_code"])
except Exception as e:
- log.exception(
+ self.log.exception(
"MODEL_POLICY: Exception when storing policy_status", e=e
)
@@ -173,11 +175,13 @@
instance.save(update_fields=["policed", "policy_status", "policy_code"])
if hasattr(policy, "after_policy_save"):
- policy(model_accessor=self.model_accessor).after_policy_save(instance)
+ policy(model_accessor=self.model_accessor).after_policy_save(
+ instance
+ )
- log.info("MODEL_POLICY: Saved", o=instance)
+ self.log.info("MODEL_POLICY: Saved", o=instance)
except BaseException:
- log.exception(
+ self.log.exception(
"MODEL POLICY: Object failed to update policed timestamp",
instance=instance,
)
@@ -191,7 +195,7 @@
try:
self.run_policy_once()
except Exception as e:
- log.exception("MODEL_POLICY: Exception in run()", e=e)
+ self.log.exception("MODEL_POLICY: Exception in run()", e=e)
if time.time() - start < 5:
time.sleep(5)
@@ -199,7 +203,7 @@
# ways to combine them.
def run_policy_once(self):
- models = self.policies_by_class.keys()
+ models = list(self.policies_by_class.keys())
self.model_accessor.check_db_connection_okay()
@@ -222,4 +226,4 @@
self.model_accessor.reset_queries()
except Exception as e:
# this shouldn't happen, but in case it does, catch it...
- log.exception("MODEL POLICY: exception in reset_queries", e)
+ self.log.exception("MODEL POLICY: exception in reset_queries", e)
diff --git a/lib/xos-synchronizer/xossynchronizer/modelaccessor.py b/lib/xos-synchronizer/xossynchronizer/modelaccessor.py
index 72fc3c5..b4b26f3 100644
--- a/lib/xos-synchronizer/xossynchronizer/modelaccessor.py
+++ b/lib/xos-synchronizer/xossynchronizer/modelaccessor.py
@@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
""" ModelAccessor
A class for abstracting access to models. Used to get any djangoisms out
@@ -23,24 +22,28 @@
models into the calling module's scope.
"""
+from __future__ import absolute_import
+
import functools
import importlib
import os
import signal
import sys
from threading import Timer
-from loadmodels import ModelLoadClient
from xosconfig import Config
-from multistructlog import create_logger
from xosutil.autodiscover_version import autodiscover_version_of_main
+from .loadmodels import ModelLoadClient
+
+from multistructlog import create_logger
log = create_logger(Config().get("logging"))
after_reactor_exit_code = None
orig_sigint = None
model_accessor = None
+
class ModelAccessor(object):
def __init__(self):
self.all_model_classes = self.get_all_model_classes()
@@ -161,6 +164,7 @@
reactor.callLater(1, functools.partial(keep_trying, client, reactor))
+
def unload_models(client, reactor, version):
# This function is called by a timer until it succeeds.
log.info("unload_models initiated by timer")
@@ -180,13 +184,14 @@
if result.status == result.TRYAGAIN:
log.info("TRYAGAIN received. Expect to try again in 30 seconds.")
- except Exception as e:
+ except Exception:
# If the synchronizer is operational, then assume the ORM's restart_on_disconnect will deal with the
# connection being lost.
log.exception("Error while unloading. Expect to try again in 30 seconds.")
Timer(30, functools.partial(unload_models, client, reactor, version)).start()
+
def exit_while_inside_reactor(reactor, code):
""" Calling sys.exit() while inside reactor ends up trapped by reactor.
@@ -261,7 +266,6 @@
exit_while_inside_reactor(reactor, 1)
return
-
log.exception("failed to onboard models")
# If it's some other error, then we don't need to force a reconnect. Just try the LoadModels() again.
reactor.callLater(10, functools.partial(grpcapi_reconnect, client, reactor))
@@ -279,7 +283,7 @@
client.xos_orm.restart_on_disconnect = True
- from apiaccessor import CoreApiModelAccessor
+ from .apiaccessor import CoreApiModelAccessor
model_accessor = CoreApiModelAccessor(orm=client.xos_orm)
@@ -370,7 +374,7 @@
global model_accessor
# the mock model accessor always gets built to a temporary location
- if not "/tmp/mock_modelaccessor" in sys.path:
+ if "/tmp/mock_modelaccessor" not in sys.path:
sys.path.append("/tmp/mock_modelaccessor")
from mock_modelaccessor import model_accessor as mock_model_accessor
diff --git a/lib/xos-synchronizer/xossynchronizer/pull_step_engine.py b/lib/xos-synchronizer/xossynchronizer/pull_step_engine.py
index a58bfa2..c3a2557 100644
--- a/lib/xos-synchronizer/xossynchronizer/pull_step_engine.py
+++ b/lib/xos-synchronizer/xossynchronizer/pull_step_engine.py
@@ -12,13 +12,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
+
import imp
import inspect
import os
import threading
import time
-from xosconfig import Config
+
from multistructlog import create_logger
+from xosconfig import Config
log = create_logger(Config().get("logging"))
@@ -44,7 +47,10 @@
threads = []
for step in self.steps:
- thread = threading.Thread(target=step(model_accessor=self.model_accessor).pull_records, name="pull_step")
+ thread = threading.Thread(
+ target=step(model_accessor=self.model_accessor).pull_records,
+ name="pull_step",
+ )
threads.append(thread)
for t in threads:
@@ -100,5 +106,7 @@
log.info("Starting pull steps engine", steps=self.pull_steps)
for step in self.pull_steps:
- sched = XOSPullStepScheduler(steps=self.pull_steps, model_accessor=self.model_accessor)
+ sched = XOSPullStepScheduler(
+ steps=self.pull_steps, model_accessor=self.model_accessor
+ )
sched.run()
diff --git a/lib/xos-synchronizer/xossynchronizer/steps/ansiblesyncstep.py b/lib/xos-synchronizer/xossynchronizer/steps/ansiblesyncstep.py
index 116f8c2..3781e1f 100644
--- a/lib/xos-synchronizer/xossynchronizer/steps/ansiblesyncstep.py
+++ b/lib/xos-synchronizer/xossynchronizer/steps/ansiblesyncstep.py
@@ -12,8 +12,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
+
from xossynchronizer.ansible_helper import run_template
-from syncstep import SyncStep
+
+from .syncstep import SyncStep
+
class AnsibleSyncStep(SyncStep):
def sync_record(self, o):
@@ -57,4 +61,4 @@
if res[0].get("rc", 0) != 0:
raise Exception("Nonzero rc from Ansible during delete_record")
- self.log.debug("Finished default delete record", **o.tologdict())
\ No newline at end of file
+ self.log.debug("Finished default delete record", **o.tologdict())
diff --git a/lib/xos-synchronizer/xossynchronizer/steps/syncstep.py b/lib/xos-synchronizer/xossynchronizer/steps/syncstep.py
index 7644822..08637ed 100644
--- a/lib/xos-synchronizer/xossynchronizer/steps/syncstep.py
+++ b/lib/xos-synchronizer/xossynchronizer/steps/syncstep.py
@@ -13,9 +13,12 @@
# limitations under the License.
-from xosconfig import Config
+from __future__ import absolute_import
+
from functools import reduce
+from xosconfig import Config
+
def f7(seq):
seen = set()
@@ -113,7 +116,6 @@
result.append(class_or_name)
return result
-
def fetch_pending(self, deletion=False):
# This is the most common implementation of fetch_pending
# Steps should override it if they have their own logic
@@ -121,12 +123,10 @@
return self.model_accessor.fetch_pending(self.observes_classes, deletion)
-
def sync_record(self, o):
self.log.debug("In abstract sync record", **o.tologdict())
# This method should be overridden by the service
-
def delete_record(self, o):
self.log.debug("In abstract delete record", **o.tologdict())
# This method should be overridden by the service
diff --git a/lib/xos-synchronizer/xossynchronizer/synchronizer.py b/lib/xos-synchronizer/xossynchronizer/synchronizer.py
index f204785..42344e9 100644
--- a/lib/xos-synchronizer/xossynchronizer/synchronizer.py
+++ b/lib/xos-synchronizer/xossynchronizer/synchronizer.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,17 +12,20 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
+
import time
-from xosconfig import Config
from multistructlog import create_logger
+from xosconfig import Config
+
class Synchronizer(object):
def __init__(self):
self.log = create_logger(Config().get("logging"))
def create_model_accessor(self):
- from modelaccessor import model_accessor
+ from .modelaccessor import model_accessor
self.model_accessor = model_accessor
@@ -33,7 +34,8 @@
wait = False
while not models_active:
try:
- _i = self.model_accessor.Site.objects.first()
+ # variable is unused
+ _i = self.model_accessor.Site.objects.first() # noqa: F841
models_active = True
except Exception as e:
self.log.info("Exception", e=e)
@@ -54,12 +56,8 @@
# use `from xossynchronizer.modelaccessor import ...` and require the model accessor to be initialized before
# their code can be imported.
- from backend import Backend
+ from .backend import Backend
log_closure = self.log.bind(synchronizer_name=Config().get("name"))
backend = Backend(log=log_closure, model_accessor=self.model_accessor)
backend.run()
-
-
-if __name__ == "__main__":
- main()
diff --git a/scripts/xos_dev_reqs.txt b/scripts/xos_dev_reqs.txt
index ff3c3ac..ad15a6c 100644
--- a/scripts/xos_dev_reqs.txt
+++ b/scripts/xos_dev_reqs.txt
@@ -21,3 +21,4 @@
django-extensions==2.1.5
djangorestframework==3.9.1
flake8==3.7.5
+python-consul~=1.1.0
diff --git a/tox.ini b/tox.ini
index ea9ee50..2b634a9 100644
--- a/tox.ini
+++ b/tox.ini
@@ -15,7 +15,6 @@
; using tox.ini only to store config information for other tools
[flake8]
-; F821, ignoring undefined names would be valuable, but our testing dynamically loads them
; W503, allow breaks before binary operators (see: https://github.com/PyCQA/pycodestyle/issues/498)
ignore = W503
-max-line-length = 120
+max-line-length = 119