[SEBA-449]
Makefile for running all XOS tests, used both by developers and
Jenkins jobs.
Includes llp's "pass over migration" #13204 patchset:
Add ability for core migration to be run/checked from outside of the repo
tree (useful in tests)
Change-Id: I47d027a5492a51273a92296e9da5310b8bc49a8c
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..817a93c
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,60 @@
+# Copyright 2019-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Makefile for testing and developing XOS
+
+# set default shell
+SHELL = bash -e -o pipefail
+
+# Variables
+XOS_LIBRARIES := $(wildcard lib/*)
+XOS_DIR := "."
+
+venv-xos:
+ ./scripts/setup_venv.sh
+
+# tests
+test: lib-test xos-test migration-test
+
+lib-test:
+ for lib in $(XOS_LIBRARIES); do pushd $$lib; tox; popd; done
+
+xos-test: venv-xos
+ source ./venv-xos/bin/activate ; set -u ;\
+ nose2 -c tox.ini --verbose --junit-xml
+ # FIXME: should run `flake8 xos` as a part of this target
+
+migration-test: venv-xos
+ source ./venv-xos/bin/activate ; set -u ;\
+ xos-migrate -x xos -s core --check
+
+clean:
+ find . -name '*.pyc' | xargs rm -f
+ find . -name '__pycache__' | xargs rm -rf
+ rm -rf \
+ .coverage \
+ coverage.xml \
+ nose2-results.xml \
+ venv-xos \
+ lib/*/.tox \
+ lib/*/build \
+ lib/*/dist \
+ lib/*/*.egg-info \
+ lib/*/.coverage \
+ lib/*/coverage.xml \
+ lib/*/*results.xml \
+ lib/*/*/VERSION \
+ lib/xos-genx/xos-genx-tests/out/* \
+ lib/xos-util/tests/test_version.py
+
diff --git a/README.md b/README.md
index 0a15459..f348c22 100644
--- a/README.md
+++ b/README.md
@@ -1,15 +1,23 @@
# XOS
-XOS is a framework for operationalizing a collection of
+XOS is a framework for operationalizing a collection of
disaggregated services. It is packaged as a project in
the Open CORD initiative, with source code managed through
-`https://gerrit.opencord.org`. It is also mirrored at
-`https://github.com/opencord`.
+<https://gerrit.opencord.org>. It is also mirrored at
+<https://github.com/opencord>.
You can download and use XOS either as part of CORD (see the
-[CORD Guide](https://guide.opencord.org/) for details)
+[CORD Guide](https://guide.opencord.org/) for details)
or as a standalone component (see the
[XOS Guide](https://guide.xosproject.org/) for details).
+
Also visit the
[XOS Wiki Page](https://wiki.opencord.org/display/CORD/XOS+and+the+CORD+Controller)
for additional information.
+
+## Testing the XOS Core
+
+Run `make test` to run tests on the XOS codebase.
+
+XOS's component libraries are stored in `lib` and are tested by running
+[tox](https://tox.readthedocs.io) in the individual library directories.
diff --git a/VERSION b/VERSION
index 98c938e..f2b42fc 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.2.12
+2.2.13
diff --git a/containers/chameleon/Dockerfile.chameleon b/containers/chameleon/Dockerfile.chameleon
index 7cad4ff..e2a5dc4 100644
--- a/containers/chameleon/Dockerfile.chameleon
+++ b/containers/chameleon/Dockerfile.chameleon
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/chameleon
-FROM xosproject/xos-base:2.2.12
+FROM xosproject/xos-base:2.2.13
# xos-base already has protoc and dependencies installed
diff --git a/containers/xos/Dockerfile.client b/containers/xos/Dockerfile.client
index e2a698e..491acfc 100644
--- a/containers/xos/Dockerfile.client
+++ b/containers/xos/Dockerfile.client
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/xos-client
-FROM xosproject/xos-libraries:2.2.12
+FROM xosproject/xos-libraries:2.2.13
# Install XOS client
COPY lib/xos-api /tmp/xos-api
diff --git a/containers/xos/Dockerfile.libraries b/containers/xos/Dockerfile.libraries
index 9edd9d1..4c3a7c6 100644
--- a/containers/xos/Dockerfile.libraries
+++ b/containers/xos/Dockerfile.libraries
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/xos-libraries
-FROM xosproject/xos-base:2.2.12
+FROM xosproject/xos-base:2.2.13
# Add libraries
COPY lib /opt/xos/lib
diff --git a/containers/xos/Dockerfile.synchronizer-base b/containers/xos/Dockerfile.synchronizer-base
index faaf185..8f1dc73 100644
--- a/containers/xos/Dockerfile.synchronizer-base
+++ b/containers/xos/Dockerfile.synchronizer-base
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/xos-synchronizer-base
-FROM xosproject/xos-client:2.2.12
+FROM xosproject/xos-client:2.2.13
COPY xos/synchronizers/new_base /opt/xos/synchronizers/new_base
COPY xos/xos/logger.py /opt/xos/xos/logger.py
diff --git a/containers/xos/Dockerfile.xos-core b/containers/xos/Dockerfile.xos-core
index 6dae309..d82a9c4 100644
--- a/containers/xos/Dockerfile.xos-core
+++ b/containers/xos/Dockerfile.xos-core
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/xos-core
-FROM xosproject/xos-libraries:2.2.12
+FROM xosproject/xos-libraries:2.2.13
# Install XOS
ADD xos /opt/xos
diff --git a/docs/README.md b/docs/README.md
index 424dd76..553d36c 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -17,26 +17,20 @@
monitoring information, allocate resources and isolate performance,
and distribute/migrate functionality.
-XOS is currently being used in three projects:
-
-* **CORD Controller:** XOS is a central part of CORD (Central Office
- Re-architected as a Datacenter), providing a coherent service
- control plane that runs on on top of a mix of disaggregated
- access technologies, legacy VNFs running in OpenStack VMs, horizontally
- scalable micro-services running in Kubernetes, and SDN control
- applications running on ONOS.
+XOS currently has two use cases:
* **Network Edge Mediator (NEM):** XOS is being used to provide a
- mediation layer for SEBA (Software-Enabled Broadband Access),
- addressing the challenge of how to integrate an edge sites with different
- (and potentially multiple) global orchestrators and legacy OSS/BSS.
+ mediation layer for CORD-based edge solutions, including SEBA
+ (SDN-Enabled Broadband Access) and COMAC (Converged Multi-Access
+ and Core). NEM replaces the "CORD Controller" component of earlier
+ CORD solutions (e.g., R-CORD, M-CORD, and E-CORD).
* **End-to-End Service Chains in a Multi-Cloud:** XOS is being used
to manage end-to-end service chains that span customer premises,
operator edge sites, Internet exchanges, and commodity clouds.
For additional white papers describing XOS, see the project
-[wiki page](https://wiki.opencord.org/display/CORD/XOS+and+the+CORD+Controller).
+[wiki page](https://wiki.opencord.org/display/CORD/XOS+and+NEM).
diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md
index adb2f98..84860ec 100644
--- a/docs/SUMMARY.md
+++ b/docs/SUMMARY.md
@@ -14,6 +14,7 @@
* [Synchronizer Implementation](dev/sync_impl.md)
* [Synchronizer Reference](dev/sync_reference.md)
* [Core Models](core_models.md)
+ * [Migrating XOS Models](dev/xosmigrate.md)
* [Security Policies](security_policies.md)
* [Tutorial](tutorials/basic_synchronizer.md)
* [XOS Shell (xossh)](dev/xossh.md)
diff --git a/docs/dev/xosmigrate.md b/docs/dev/xosmigrate.md
index 57490f2..324089b 100644
--- a/docs/dev/xosmigrate.md
+++ b/docs/dev/xosmigrate.md
@@ -1,27 +1,28 @@
-# Generating migrations for XOS and Services
+# Migrating XOS Models
-This document describes how to use the XOS toolchain to generate basic migrations
-for service models. The autogenerated migrations can later be extended with custom
-logic to support more complex scenarios.
+This document describes an XOS toolchain that can be used to generate
+basic migrations for service models. The autogenerated migrations can
+later be extended with custom logic to support more complex scenarios.
-> NOTE from now on we assume you have obtained the code as described in
-> [this guide](https://guide.opencord.org/developer/getting_the_code.html) and the
-> entire tree is now available in `~/cord`
+> NOTE: The following assumes you have downloaded the entire source
+> code tree in `~/cord`.
-## Install the XOS Toolchain
+## Installing the Toolchain
-The XOS toolchain consists of a set of python libraries. There is an helper script
-to install the toolchain in the XOS repo, so just execute:
+The XOS migration toolchain consists of a set of python
+libraries. There is a helper script to install the toolchain.
+Just run:
```bash
cd ~/cord/orchestration/xos
bash scripts/setup_venv.sh
```
-## Generate the migrations
+## Generating Migrations
-Once the toolchain is available, you'll be able to generate migrations for the services (or the core).
-The `xos-migrate` command is now available and you can see the options by running:
+Once the toolchain is available, you will be able to generate
+migrations for the services (or the core) using the `xos-migrate`
+command. Execute the command to see the available options:
```bash
usage: xos-migrate [-h] -s SERVICE_NAMES [-r REPO_ROOT] [--check] [-v]
@@ -40,37 +41,39 @@
required arguments:
-s SERVICE_NAMES, --service SERVICE_NAMES
The name of the folder containing the service in
- cord/orchestration/xos_services
+ cord/orchestration/xos-services
```
-Supposing that your code is living under `~/Sites/cord` and you want to generate migrations
-for `core` and `fabric`, this is the command you'll run:
+For example, if the code you want to migrate is in `~/Sites` and you
+want to generate migrations for `core` and `fabric`, then run the
+following command:
```bash
-xos-migrate -r ~/Sites/cord -s core -s fabric
+xos-migrate -r ~/Sites -s core -s fabric
```
-If no migrations were present for your service you'll see a new folder `migrations`
-that is a sibling of `models` containing a file `0001-initial.py`.
-If the service already had migrations you'll see a new file in that folder,
-for example: `0002-fabricservice_fullname.py`
+If no migrations were present for your service, you will see a new
+folder `migrations` (a sibling of `models`) that contains the file
+`0001-initial.py`. If the service already had migrations you will see
+a new file in that folder, for example: `0002-fabricservice_fullname.py`
-> NOTE that all the migration files needs to be committed together with the code
-> as they'll be loaded into the core together with the models.
+> NOTE: All the migration files need to be committed together with the
+> code as they will be loaded into the core together with the models.
-## Customize the migrations
+## Customizing Migrations
-The autogenerated migrations only operates on the database altering tables, they
-won't make any modifications to the data. You can write custom code to do that,
-following this example.
+The autogenerated migrations operate on only the database tables. They
+do not make any modifications to the data in those tables. You can write
+custom code to make such changes, as illustrated in the following
+example.
-We assume you already have a model called `FabricService` that has two properties:
-`first_name` and `last_name`.
+Assume you already have a model called `FabricService` that has two
+properties: `first_name` and `last_name`, and as part of your changes
+you want to add a third property called `full_name`, defined as
+`first_name` + `last_name`.
-As part of your changes you want to add a third property called `full_name` that
-is defined as `first_name` + `last_name`.
-
-This is the migration code that the tool will generate:
+The following is the migration code that the tool will automatically
+generate:
```python
@@ -102,7 +105,7 @@
and adding it to the `operations` list.
-Here is a complete example of the customized migration:
+The following is a complete example of the customized migration:
```python
def forwards(apps, schema_editor):
@@ -113,7 +116,6 @@
class Migration(migrations.Migration):
-
dependencies = [
('fabric', '0001_initial'),
]
@@ -128,21 +130,25 @@
]
```
-For more informations about migrations you can refer to the official
+For more information about migrations you can refer to the official
[Django guide](https://docs.djangoproject.com/en/2.1/howto/writing-migrations/).
-Here are some good reads about migrations:
+Additional information about migration can be found in the following:
- <https://realpython.com/django-migrations-a-primer/#creating-migrations>
- <https://realpython.com/data-migrations/>
-## Development workflow
+## Development Workflow
-During development multiple changes to the models are often necessary. In order to continuously upgrade the service to
-proceed with development we suggest to generate a new migration every time the models are changed. This is required to
-upgrade the service multiple times during the development loop (as the core expects new migrations).
+It is sometimes necessary to make multiple changes to the models
+during development. In order to continuously upgrade the service to
+proceed with development we suggest you generate a new migration every
+time the models are changed. This is required to upgrade the service
+multiple times during the development loop (as the core expects new
+migrations).
-This will probably lead to multiple migration files by the time your feature is complete, for example:
+This will probably lead to multiple migration files by the time your
+feature is complete, for example:
```yaml
- 0003-modelA-fieldA.py
@@ -151,15 +157,18 @@
- 0007-modelB-fieldX.py
```
-However, in order to maintain clarity, we suggest to submit a single migration as part of a patch.
-To do that you can simply remove all the migrations you have generated as part of your development and run the
-`xos-migrate` tool again. This will generate a single migration file for all your changes.
+To maintain clarity, however, we suggest you submit a single
+migration as part of a patch. To do that, simply remove all the
+migrations you have generated as part of your development and run the
+`xos-migrate` tool again. This will generate a single migration file
+for all your changes.
-## Validate migration status
+## Validating Migration Status
-Once you are done with development you want to make sure that all the necessary migrations are generated and checked in.
-To do that you can run the `xos-migrate` tool using the `--check` flag. Here is an example:
+Once you are done with development, you should make sure that all the
+necessary migrations are generated and checked in. To do that, run the
+`xos-migrate` tool using the `--check` flag. Here is an example:
```shell
xos-migrate -s fabric --check
-```
\ No newline at end of file
+```
diff --git a/lib/__init__.py b/lib/__init__.py
deleted file mode 100644
index b0fb0b2..0000000
--- a/lib/__init__.py
+++ /dev/null
@@ -1,13 +0,0 @@
-# Copyright 2017-present Open Networking Foundation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
diff --git a/lib/xos-migrate/xosmigrate/main.py b/lib/xos-migrate/xosmigrate/main.py
index f8f5d6b..8d76197 100644
--- a/lib/xos-migrate/xosmigrate/main.py
+++ b/lib/xos-migrate/xosmigrate/main.py
@@ -293,12 +293,22 @@
help="The name of the folder containing the service in cord/orchestration/xos_services"
)
-parser.add_argument(
+pathgroup = parser.add_mutually_exclusive_group()
+
+pathgroup.add_argument(
"-r",
"--repo",
- default=get_abs_path("~/cord"),
+ default=get_abs_path("../.."),
dest="repo_root",
- help="The location of the folder containing the CORD repo root (default to ~/cord)"
+ help="Path to the CORD repo root (defaults to '../..'). Mutually exclusive with '--xos'."
+)
+
+pathgroup.add_argument(
+ "-x",
+ "--xos",
+ default=None,
+ dest="xos_root",
+ help="Path to directory of the XOS repo. Incompatible with '--repo' and only works for core migrations."
)
parser.add_argument(
@@ -330,10 +340,17 @@
print_banner(args.repo_root)
# find absolute path to the code
- xos_path = get_abs_path(os.path.join(args.repo_root, "orchestration/xos/xos/"))
+ if args.xos_root: # if args.xos_root is set, testing only the core
+ xos_path = get_abs_path(args.xos_root)
+ if args.service_names != ["core"]:
+ log.error("When using --xos, can only check the core models")
+ sys.exit(1)
+ else:
+ xos_path = get_abs_path(os.path.join(args.repo_root, "orchestration/xos/xos/"))
+ service_base_dir = get_abs_path(os.path.join(xos_path, "../../xos_services/"))
+ service_dest_dir = get_abs_path(os.path.join(xos_path, "services/"))
+
core_dir = get_abs_path(os.path.join(xos_path, "core/models/"))
- service_base_dir = get_abs_path(os.path.join(xos_path, "../../xos_services/"))
- service_dest_dir = get_abs_path(os.path.join(xos_path, "services/"))
# we need to append the xos folder to sys.path
original_sys_path = sys.path
diff --git a/lib/xos-synchronizer/tox.ini b/lib/xos-synchronizer/tox.ini
index 2e09d5e..b98744a 100644
--- a/lib/xos-synchronizer/tox.ini
+++ b/lib/xos-synchronizer/tox.ini
@@ -25,18 +25,18 @@
commands =
nose2 -c tox.ini --verbose --junit-xml
-; should widen flake8 to also cover xos-synchronizer-tests
+; FIXME: should widen flake8 to also cover xos-synchronizer-tests
flake8 xossynchronizer
[flake8]
-; F821, ignoring undefined names would be valuable, but synchronizer dynamically loads them
-; W503, allow breaks before binary operators (see: https://github.com/PyCQA/pycodestyle/issues/498)
-;ignore = F821,W503
max-line-length = 119
[unittest]
plugins=nose2.plugins.junitxml
+[junit-xml]
+path=nose2-results.xml
+
[coverage]
always-on = True
coverage = xossynchronizer
diff --git a/tox.ini b/tox.ini
index 2b634a9..d78bf8b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -18,3 +18,21 @@
; W503, allow breaks before binary operators (see: https://github.com/PyCQA/pycodestyle/issues/498)
ignore = W503
max-line-length = 119
+
+[unittest]
+plugins = nose2.plugins.junitxml
+code-directories =
+ xos-genx
+ xos-api
+ xos-config
+ xos-synchronizer
+ coreapi
+
+[junit-xml]
+path = nose2-results.xml
+
+[coverage]
+always-on = True
+coverage = xos
+coverage-report = term
+coverage-report = xml
diff --git a/unittest.cfg b/unittest.cfg
deleted file mode 100644
index 259c7b6..0000000
--- a/unittest.cfg
+++ /dev/null
@@ -1,15 +0,0 @@
-[unittest]
-plugins=nose2.plugins.junitxml
-
-code-directories=xos-genx
- xos-api
- xos-config
- xos-synchronizer
- coreapi
-
-[coverage]
-always-on = True
-coverage = xos
- lib
-coverage-report = term
-coverage-report = xml