[SEBA-496]
Update pip packages used in XOS containers, flatten containers
Fixed invalid YAML generated by xosgenx modeldefs.xtarget
Change-Id: Ic051170f3f9ef5a5d0dbba61aa6843458a435d49
diff --git a/VERSION b/VERSION
index cb2b00e..b502146 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-3.0.1
+3.0.2
diff --git a/containers/chameleon/Dockerfile.chameleon b/containers/chameleon/Dockerfile.chameleon
index 1f36eb8..63084e1 100644
--- a/containers/chameleon/Dockerfile.chameleon
+++ b/containers/chameleon/Dockerfile.chameleon
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/chameleon
-FROM xosproject/xos-base:3.0.1
+FROM xosproject/xos-base:3.0.2
# xos-base already has protoc and dependencies installed
@@ -23,7 +23,19 @@
COPY tmp.chameleon /chameleon/chameleon
# Build the protos
-RUN pip install docker==3.1.4 klein==17.10.0 fluent-logger==0.9.2 \
+RUN pip install \
+ Jinja2==2.10 \
+ PyYAML==5.1 \
+ docker==3.7.0 \
+ fluent-logger==0.9.3 \
+ grpcio-tools==1.12.0 \
+ grpcio==1.12.0 \
+ klein==17.10.0 \
+ netifaces==0.10.9 \
+ pyOpenSSL==19.0.0 \
+ python-consul==1.1.0 \
+ service_identity==18.1.0 \
+ simplejson==3.16.0 \
&& chdir chameleon/chameleon/protos \
&& VOLTHA_BASE=anything make
diff --git a/containers/xos/Dockerfile.base b/containers/xos/Dockerfile.base
index 7af9954..811901b 100644
--- a/containers/xos/Dockerfile.base
+++ b/containers/xos/Dockerfile.base
@@ -32,15 +32,10 @@
&& pip freeze > /var/xos/pip_freeze_apt_`date -u +%Y%m%dT%H%M%S`
# Install python packages with pip
-# Upgrade methodology - pip_requested.txt is the manually generated list of pip
-# packages desired, without any added dependencies. It is used to generate
-# pip_requirements.txt by running: pip install -r pip_requested.txt then use
-# `pipdeptree` to make sure dependencies aren't broken, then
-# `pip freeze > pip_requirements.txt` and manually clean up.
-COPY pip_requirements.txt /tmp/pip_requirements.txt
-RUN pip install --no-deps -U -I pip==9.0.3 setuptools==39.0.1 requests==2.18.4 \
- && pip install --no-deps -r /tmp/pip_requirements.txt \
- && pip freeze > /var/xos/pip_freeze_pip_up_`date -u +%Y%m%dT%H%M%S`
+COPY requirements.txt /tmp/requirements.txt
+RUN pip install -U -I pip==9.0.3 setuptools==39.0.1 \
+ && pip install -r /tmp/requirements.txt \
+ && pip freeze > /var/xos/pip_freeze_base_`date -u +%Y%m%dT%H%M%S`
# Label image
ARG org_label_schema_schema_version=1.0
diff --git a/containers/xos/Dockerfile.client b/containers/xos/Dockerfile.client
index c6bae83..5ba8a8e 100644
--- a/containers/xos/Dockerfile.client
+++ b/containers/xos/Dockerfile.client
@@ -13,16 +13,7 @@
# limitations under the License.
# xosproject/xos-client
-FROM xosproject/xos-libraries:3.0.1
-
-# Install XOS client
-COPY lib/xos-api /tmp/xos-api
-
-# Install the VERSION file
-COPY VERSION /tmp/xos-api/xosapi/VERSION
-
-# install the client library and xossh
-RUN chdir /tmp/xos-api && python ./setup.py install
+FROM xosproject/xos-libraries:3.0.2
# Label image
ARG org_label_schema_schema_version=1.0
diff --git a/containers/xos/Dockerfile.libraries b/containers/xos/Dockerfile.libraries
index 420ba33..acc7ccb 100644
--- a/containers/xos/Dockerfile.libraries
+++ b/containers/xos/Dockerfile.libraries
@@ -13,17 +13,20 @@
# limitations under the License.
# xosproject/xos-libraries
-FROM xosproject/xos-base:3.0.1
+FROM xosproject/xos-base:3.0.2
# Add libraries
COPY lib /opt/xos/lib
COPY VERSION /opt/xos
-# Install python modules included with XOS
-RUN cd /opt/xos/lib/xos-util && python setup.py install \
- && cd /opt/xos/lib/xos-config && python setup.py install \
- && cd /opt/xos/lib/xos-genx && python setup.py install \
- && cd /opt/xos/lib/xos-kafka && python setup.py install
+# Install python using pip so their dependencies are installed
+RUN pip install -e /opt/xos/lib/xos-util \
+ && pip install -e /opt/xos/lib/xos-config \
+ && pip install -e /opt/xos/lib/xos-genx \
+ && pip install -e /opt/xos/lib/xos-kafka \
+ && pip install -e /opt/xos/lib/xos-synchronizer \
+ && pip install -e /opt/xos/lib/xos-api \
+ && pip freeze > /var/xos/pip_freeze_libraries_`date -u +%Y%m%dT%H%M%S`
# Label image
ARG org_label_schema_schema_version=1.0
diff --git a/containers/xos/Dockerfile.synchronizer-base b/containers/xos/Dockerfile.synchronizer-base
index df1e203..c8c7e5b 100644
--- a/containers/xos/Dockerfile.synchronizer-base
+++ b/containers/xos/Dockerfile.synchronizer-base
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/xos-synchronizer-base
-FROM xosproject/xos-client:3.0.1
+FROM xosproject/xos-client:3.0.2
COPY xos/xos/logger.py /opt/xos/xos/logger.py
COPY xos/xos/__init__.py /opt/xos/xos/__init__.py
@@ -22,11 +22,6 @@
# Copy over ansible hosts
COPY containers/xos/ansible-hosts /etc/ansible/hosts
-# For library-based synchronizers
-COPY lib/xos-synchronizer /tmp/xos-synchronizer
-COPY VERSION /tmp/xos-synchronizer/xossynchronizer/VERSION
-RUN cd /tmp/xos-synchronizer && python setup.py install
-
# Label image
ARG org_label_schema_schema_version=1.0
ARG org_label_schema_name=xos-synchronizer-base
diff --git a/containers/xos/Dockerfile.xos-core b/containers/xos/Dockerfile.xos-core
index bddc63b..7fff670 100644
--- a/containers/xos/Dockerfile.xos-core
+++ b/containers/xos/Dockerfile.xos-core
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/xos-core
-FROM xosproject/xos-libraries:3.0.1
+FROM xosproject/xos-libraries:3.0.2
# Install XOS
ADD xos /opt/xos
diff --git a/containers/xos/nodesource.gpg.key b/containers/xos/nodesource.gpg.key
deleted file mode 100644
index 1dc1d10..0000000
--- a/containers/xos/nodesource.gpg.key
+++ /dev/null
@@ -1,52 +0,0 @@
------BEGIN PGP PUBLIC KEY BLOCK-----
-Version: GnuPG v1
-Comment: GPGTools - https://gpgtools.org
-
-mQINBFObJLYBEADkFW8HMjsoYRJQ4nCYC/6Eh0yLWHWfCh+/9ZSIj4w/pOe2V6V+
-W6DHY3kK3a+2bxrax9EqKe7uxkSKf95gfns+I9+R+RJfRpb1qvljURr54y35IZgs
-fMG22Np+TmM2RLgdFCZa18h0+RbH9i0b+ZrB9XPZmLb/h9ou7SowGqQ3wwOtT3Vy
-qmif0A2GCcjFTqWW6TXaY8eZJ9BCEqW3k/0Cjw7K/mSy/utxYiUIvZNKgaG/P8U7
-89QyvxeRxAf93YFAVzMXhoKxu12IuH4VnSwAfb8gQyxKRyiGOUwk0YoBPpqRnMmD
-Dl7SdmY3oQHEJzBelTMjTM8AjbB9mWoPBX5G8t4u47/FZ6PgdfmRg9hsKXhkLJc7
-C1btblOHNgDx19fzASWX+xOjZiKpP6MkEEzq1bilUFul6RDtxkTWsTa5TGixgCB/
-G2fK8I9JL/yQhDc6OGY9mjPOxMb5PgUlT8ox3v8wt25erWj9z30QoEBwfSg4tzLc
-Jq6N/iepQemNfo6Is+TG+JzI6vhXjlsBm/Xmz0ZiFPPObAH/vGCY5I6886vXQ7ft
-qWHYHT8jz/R4tigMGC+tvZ/kcmYBsLCCI5uSEP6JJRQQhHrCvOX0UaytItfsQfLm
-EYRd2F72o1yGh3yvWWfDIBXRmaBuIGXGpajC0JyBGSOWb9UxMNZY/2LJEwARAQAB
-tB9Ob2RlU291cmNlIDxncGdAbm9kZXNvdXJjZS5jb20+iQI4BBMBAgAiBQJTmyS2
-AhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRAWVaCraFdigHTmD/9OKhUy
-jJ+h8gMRg6ri5EQxOExccSRU0i7UHktecSs0DVC4lZG9AOzBe+Q36cym5Z1di6JQ
-kHl69q3zBdV3KTW+H1pdmnZlebYGz8paG9iQ/wS9gpnSeEyx0Enyi167Bzm0O4A1
-GK0prkLnz/yROHHEfHjsTgMvFwAnf9uaxwWgE1d1RitIWgJpAnp1DZ5O0uVlsPPm
-XAhuBJ32mU8S5BezPTuJJICwBlLYECGb1Y65Cil4OALU7T7sbUqfLCuaRKxuPtcU
-VnJ6/qiyPygvKZWhV6Od0Yxlyed1kftMJyYoL8kPHfeHJ+vIyt0s7cropfiwXoka
-1iJB5nKyt/eqMnPQ9aRpqkm9ABS/r7AauMA/9RALudQRHBdWIzfIg0Mlqb52yyTI
-IgQJHNGNX1T3z1XgZhI+Vi8SLFFSh8x9FeUZC6YJu0VXXj5iz+eZmk/nYjUt4Mtc
-pVsVYIB7oIDIbImODm8ggsgrIzqxOzQVP1zsCGek5U6QFc9GYrQ+Wv3/fG8hfkDn
-xXLww0OGaEQxfodm8cLFZ5b8JaG3+Yxfe7JkNclwvRimvlAjqIiW5OK0vvfHco+Y
-gANhQrlMnTx//IdZssaxvYytSHpPZTYw+qPEjbBJOLpoLrz8ZafN1uekpAqQjffI
-AOqW9SdIzq/kSHgl0bzWbPJPw86XzzftewjKNbkCDQRTmyS2ARAAxSSdQi+WpPQZ
-fOflkx9sYJa0cWzLl2w++FQnZ1Pn5F09D/kPMNh4qOsyvXWlekaV/SseDZtVziHJ
-Km6V8TBG3flmFlC3DWQfNNFwn5+pWSB8WHG4bTA5RyYEEYfpbekMtdoWW/Ro8Kmh
-41nuxZDSuBJhDeFIp0ccnN2Lp1o6XfIeDYPegyEPSSZqrudfqLrSZhStDlJgXjea
-JjW6UP6txPtYaaila9/Hn6vF87AQ5bR2dEWB/xRJzgNwRiax7KSU0xca6xAuf+TD
-xCjZ5pp2JwdCjquXLTmUnbIZ9LGV54UZ/MeiG8yVu6pxbiGnXo4Ekbk6xgi1ewLi
-vGmz4QRfVklV0dba3Zj0fRozfZ22qUHxCfDM7ad0eBXMFmHiN8hg3IUHTO+UdlX/
-aH3gADFAvSVDv0v8t6dGc6XE9Dr7mGEFnQMHO4zhM1HaS2Nh0TiL2tFLttLbfG5o
-QlxCfXX9/nasj3K9qnlEg9G3+4T7lpdPmZRRe1O8cHCI5imVg6cLIiBLPO16e0fK
-yHIgYswLdrJFfaHNYM/SWJxHpX795zn+iCwyvZSlLfH9mlegOeVmj9cyhN/VOmS3
-QRhlYXoA2z7WZTNoC6iAIlyIpMTcZr+ntaGVtFOLS6fwdBqDXjmSQu66mDKwU5Ek
-fNlbyrpzZMyFCDWEYo4AIR/18aGZBYUAEQEAAYkCHwQYAQIACQUCU5sktgIbDAAK
-CRAWVaCraFdigIPQEACcYh8rR19wMZZ/hgYv5so6Y1HcJNARuzmffQKozS/rxqec
-0xM3wceL1AIMuGhlXFeGd0wRv/RVzeZjnTGwhN1DnCDy1I66hUTgehONsfVanuP1
-PZKoL38EAxsMzdYgkYH6T9a4wJH/IPt+uuFTFFy3o8TKMvKaJk98+Jsp2X/QuNxh
-qpcIGaVbtQ1bn7m+k5Qe/fz+bFuUeXPivafLLlGc6KbdgMvSW9EVMO7yBy/2JE15
-ZJgl7lXKLQ31VQPAHT3an5IV2C/ie12eEqZWlnCiHV/wT+zhOkSpWdrheWfBT+ac
-hR4jDH80AS3F8jo3byQATJb3RoCYUCVc3u1ouhNZa5yLgYZ/iZkpk5gKjxHPudFb
-DdWjbGflN9k17VCf4Z9yAb9QMqHzHwIGXrb7ryFcuROMCLLVUp07PrTrRxnO9A/4
-xxECi0l/BzNxeU1gK88hEaNjIfviPR/h6Gq6KOcNKZ8rVFdwFpjbvwHMQBWhrqfu
-G3KaePvbnObKHXpfIKoAM7X2qfO+IFnLGTPyhFTcrl6vZBTMZTfZiC1XDQLuGUnd
-sckuXINIU3DFWzZGr0QrqkuE/jyr7FXeUJj9B7cLo+s/TXo+RaVfi3kOc9BoxIvy
-/qiNGs/TKy2/Ujqp/affmIMoMXSozKmga81JSwkADO1JMgUy6dApXz9kP4EE3g==
-=CLGF
------END PGP PUBLIC KEY BLOCK-----
diff --git a/containers/xos/pip_requested.txt b/containers/xos/pip_requested.txt
deleted file mode 100644
index 889110e..0000000
--- a/containers/xos/pip_requested.txt
+++ /dev/null
@@ -1,52 +0,0 @@
-Django==1.11.11
-Jinja2==2.10
-inflect==1.0.1
-PyYAML==3.12
-Twisted==16.6.0
-Werkzeug==0.14.1
-ansible==2.5.0
-astunparse==1.5.0
-confluent-kafka==0.11.5
-django-extensions==2.0.6
-django-timezones==0.2
-djangorestframework==3.7.7
-dnspython==1.15.0
-functools32==3.2.3-2
-futures==3.2.0
-google-api-python-client==1.6.5
-grpcio-tools==1.12.0
-grpcio==1.12.0
-kafkaloghandler==0.9.0
-keystoneauth1==3.4.0
-mock==2.0.0
-multistructlog==2.0.0
-netaddr==0.7.19
-networkx==1.11
-nose==1.3.7
-oslo.log==3.37.0
-oslo.messaging==5.36.0
-oslo.serialization==2.25.0
-oslo.utils==3.36.0
-ply==3.11
-plyxproto==4.0.0
-protobuf==3.5.2
-prometheus_client==0.4.0
-# Avoids a warning, see http://initd.org/psycopg/docs/faq.html#faq-compile
-psycopg2==2.7.4 --no-binary psycopg2
-pyOpenSSL==17.5.0
-pykwalify==1.6.1
-python-consul==0.7.2
-python-glanceclient==2.9.1
-python-keystoneclient==3.15.0
-python-logstash==0.4.6
-python-neutronclient==5.1.0
-python-novaclient==10.1.0
-python-openstackclient==3.14.0
-pytz==2018.3
-requests==2.18.4
-service-identity==17.0.0
-shade==1.27.1
-six==1.11.0
-structlog==18.1.0
-tosca-parser==0.9.0
-semver==2.8.1
diff --git a/containers/xos/pip_requirements.txt b/containers/xos/pip_requirements.txt
deleted file mode 100644
index cb62170..0000000
--- a/containers/xos/pip_requirements.txt
+++ /dev/null
@@ -1,143 +0,0 @@
-amqp==2.3.2
-ansible==2.5.0
-appdirs==1.4.3
-asn1crypto==0.24.0
-astunparse==1.5.0
-attrs==18.2.0
-Babel==2.6.0
-bcrypt==3.1.4
-cachetools==2.1.0
-certifi==2018.8.24
-cffi==1.11.5
-chardet==3.0.4
-cliff==2.13.0
-cmd2==0.8.9
-colorama==0.3.9
-confluent-kafka==0.11.5
-constantly==15.1.0
-contextlib2==0.5.5
-cryptography==2.3.1
-debtcollector==1.20.0
-decorator==4.3.0
-deprecation==2.0.6
-Django==1.11.11
-django-extensions==2.0.6
-django-timezones==0.2
-djangorestframework==3.7.7
-dnspython==1.15.0
-docopt==0.6.2
-dogpile.cache==0.6.7
-enum34==1.1.6
-eventlet==0.24.1
-fasteners==0.14.1
-funcsigs==1.0.2
-functools32==3.2.3.post2
-futures==3.2.0
-futurist==1.7.0
-google-api-python-client==1.6.5
-greenlet==0.4.15
-grpcio==1.12.0
-grpcio-tools==1.12.0
-httplib2==0.11.3
-idna==2.6
-incremental==17.5.0
-inflect==1.0.1
-ipaddress==1.0.22
-iso8601==0.1.12
-Jinja2==2.10
-jmespath==0.9.3
-jsonpatch==1.23
-jsonpointer==2.0
-jsonschema==2.6.0
-kafkaloghandler==0.9.0
-keystoneauth1==3.4.0
-kombu==4.2.1
-MarkupSafe==1.0
-mock==2.0.0
-monotonic==1.5
-msgpack==0.5.6
-multistructlog==2.1.0
-munch==2.3.2
-netaddr==0.7.19
-netifaces==0.10.7
-networkx==1.11
-nose==1.3.7
-oauth2client==4.1.3
-openstacksdk==0.17.2
-os-client-config==1.31.2
-os-service-types==1.3.0
-osc-lib==1.11.1
-oslo.concurrency==3.28.1
-oslo.config==6.5.1
-oslo.context==2.21.0
-oslo.i18n==3.22.1
-oslo.log==3.37.0
-oslo.messaging==5.36.0
-oslo.middleware==3.36.0
-oslo.serialization==2.25.0
-oslo.service==1.32.0
-oslo.utils==3.36.0
-packaging==18.0
-paramiko==2.4.2
-Paste==2.0.3
-PasteDeploy==1.5.2
-pbr==4.2.0
-pika==0.12.0
-pika-pool==0.1.3
-pipdeptree==0.13.0
-ply==3.11
-plyxproto==4.0.0
-prettytable==0.7.2
-prometheus-client==0.4.0
-protobuf==3.5.2
-# Avoids a warning, see http://initd.org/psycopg/docs/faq.html#faq-compile
-psycopg2==2.7.4 --no-binary psycopg2
-pyasn1==0.4.4
-pyasn1-modules==0.2.2
-pycparser==2.19
-pyinotify==0.9.6
-pykwalify==1.6.1
-PyNaCl==1.3.0
-pyOpenSSL==17.5.0
-pyparsing==2.2.2
-pyperclip==1.7.0
-python-cinderclient==4.0.1
-python-consul==0.7.2
-python-dateutil==2.7.3
-python-glanceclient==2.9.1
-python-keystoneclient==3.15.0
-python-logstash==0.4.6
-python-neutronclient==5.1.0
-python-novaclient==10.1.0
-python-openstackclient==3.14.0
-pytz==2018.3
-PyYAML==3.12
-repoze.lru==0.7
-requests==2.18.4
-requestsexceptions==1.4.0
-rfc3986==1.1.0
-Routes==2.4.1
-rsa==4.0
-service-identity==17.0.0
-shade==1.27.1
-simplejson==3.16.0
-six==1.11.0
-statsd==3.3.0
-stevedore==1.29.0
-structlog==18.1.0
-subprocess32==3.5.2
-tenacity==5.0.2
-tosca-parser==0.9.0
-Twisted==16.6.0
-typing==3.6.6
-unicodecsv==0.14.1
-uritemplate==3.0.0
-urllib3==1.22
-vine==1.1.4
-warlock==1.3.0
-wcwidth==0.1.7
-WebOb==1.8.2
-Werkzeug==0.14.1
-wrapt==1.10.11
-zope.interface==4.5.0
-semver==2.8.1
diff --git a/containers/xos/requirements.txt b/containers/xos/requirements.txt
new file mode 100644
index 0000000..64fec10
--- /dev/null
+++ b/containers/xos/requirements.txt
@@ -0,0 +1,14 @@
+Django~=1.11.20
+confluent-kafka==0.11.5
+django-extensions~=2.1.6
+django-timezones~=0.2
+djangorestframework~=3.9.1
+functools32~=3.2.3-2
+futures~=3.2.0
+kafkaloghandler~=0.9.0
+multistructlog~=2.1.0
+prometheus_client~=0.6.0
+psycopg2-binary~=2.7.7
+pytz~=2018.9
+semver~=2.8.1
+tosca-parser~=1.4.0
diff --git a/lib/xos-api/requirements.txt b/lib/xos-api/requirements.txt
index f990247..1fb843d 100644
--- a/lib/xos-api/requirements.txt
+++ b/lib/xos-api/requirements.txt
@@ -1,11 +1,9 @@
-PyYAML~=3.12
-Twisted~=16.6.0
-googleapis-common-protos~=1.5.6
-grpcio-tools~=1.12.0
-grpcio~=1.12.0
+Twisted~=18.9.0
+googleapis-common-protos~=1.5.8
+grpcio-tools~=1.19.0
+grpcio~=1.19.0
multistructlog~=2.1.0
-pykwalify~=1.6.0
python-consul~=1.1.0
simplejson~=3.16.0
-xosconfig~=3.0.0
-xosgenx~=3.0.0
+xosconfig~=3.0.1
+xosgenx~=3.0.1
diff --git a/lib/xos-config/Makefile b/lib/xos-config/Makefile
deleted file mode 100644
index 650eb5f..0000000
--- a/lib/xos-config/Makefile
+++ /dev/null
@@ -1,2 +0,0 @@
-test:
- nosetests -s -v --with-id --with-coverage --cover-html --cover-erase --cover-xml --cover-package="xosconfig"
\ No newline at end of file
diff --git a/lib/xos-config/requirements.txt b/lib/xos-config/requirements.txt
index 5528415..1ed683f 100644
--- a/lib/xos-config/requirements.txt
+++ b/lib/xos-config/requirements.txt
@@ -1,2 +1,2 @@
-PyYAML~=3.12
-pykwalify~=1.6.0
+PyYAML~=5.1
+pykwalify~=1.7.0
diff --git a/lib/xos-genx/Makefile b/lib/xos-genx/Makefile
deleted file mode 100644
index feee40e..0000000
--- a/lib/xos-genx/Makefile
+++ /dev/null
@@ -1,3 +0,0 @@
-test:
- nosetests -s -v --with-id --with-coverage --cover-html --cover-erase --cover-xml --cover-package="xosgenx"
-
diff --git a/lib/xos-genx/requirements.txt b/lib/xos-genx/requirements.txt
index b210781..924fbb6 100644
--- a/lib/xos-genx/requirements.txt
+++ b/lib/xos-genx/requirements.txt
@@ -1,8 +1,8 @@
Jinja2~=2.10
-PyYAML~=3.12
-astunparse~=1.5.0
+PyYAML~=5.1
+astunparse~=1.6.2
colorama~=0.4.1
-inflect~=1.0.1
+inflect~=2.1.0
plyxproto~=4.0.0
ply~=3.11
six~=1.12.0
diff --git a/lib/xos-genx/xos-genx-tests/test_translator.py b/lib/xos-genx/xos-genx-tests/test_translator.py
index f98894b..ae4289a 100644
--- a/lib/xos-genx/xos-genx-tests/test_translator.py
+++ b/lib/xos-genx/xos-genx-tests/test_translator.py
@@ -125,8 +125,7 @@
args.inputs = xproto
args.target = "modeldefs.xtarget"
output = XOSProcessor.process(args)
-
- yaml_ir = yaml.load(output)
+ yaml_ir = yaml.safe_load(output)
self.assertEqual(len(yaml_ir["items"]), 4)
def test_gui_hidden_models(self):
@@ -147,7 +146,7 @@
args.inputs = xproto
args.target = "modeldefs.xtarget"
output = XOSProcessor.process(args)
- yaml_ir = yaml.load(output)
+ yaml_ir = yaml.safe_load(output)
self.assertEqual(len(yaml_ir["items"]), 1)
self.assertIn("Bar", output)
self.assertNotIn("Foo", output)
@@ -165,7 +164,7 @@
args.inputs = xproto
args.target = "modeldefs.xtarget"
output = XOSProcessor.process(args)
- yaml_ir = yaml.load(output)
+ yaml_ir = yaml.safe_load(output)
self.assertEqual(len(yaml_ir["items"]), 1)
self.assertIn("name", output)
self.assertNotIn("secret", output)
@@ -336,7 +335,7 @@
args.target = "modeldefs.xtarget"
output = XOSProcessor.process(args)
- read_only = [s for s in output.splitlines() if "read_only: True" in s]
+ read_only = [s for s in output.splitlines() if "read_only: true" in s]
self.assertEqual(len(read_only), 3) # readonly is 1 for ParentFoo and 2 for Foo
diff --git a/lib/xos-genx/xosgenx/jinja2_extensions/gui.py b/lib/xos-genx/xosgenx/jinja2_extensions/gui.py
index 07dbc35..a48d95a 100644
--- a/lib/xos-genx/xosgenx/jinja2_extensions/gui.py
+++ b/lib/xos-genx/xosgenx/jinja2_extensions/gui.py
@@ -146,7 +146,7 @@
elif l["link_type"] == "onetomany":
on_field = l["dst_port"]
outlist.append(
- "- {model: %s, type: %s, on_field: %s}\n"
+ "{model: %s, type: %s, on_field: %s}"
% (l["peer"]["name"], l["link_type"], on_field)
)
seen.append(l["peer"])
diff --git a/lib/xos-genx/xosgenx/targets/modeldefs.xtarget b/lib/xos-genx/xosgenx/targets/modeldefs.xtarget
index ec7e5f2..94b4136 100644
--- a/lib/xos-genx/xosgenx/targets/modeldefs.xtarget
+++ b/lib/xos-genx/xosgenx/targets/modeldefs.xtarget
@@ -1,53 +1,57 @@
+---
+# Created by modeldefs.xtarget
items:
{%- for m in proto.messages | sort(attribute='name') %}
-{%- if m.name != 'XOSBase' and xproto_unquote(xproto_first_non_empty([m.options.gui_hidden, 'False'])) != 'True' %}
-- app: {{ xproto_unquote(xproto_first_non_empty([m.options.name, m.options.app_label, options.name, context.app_label])) }}
- {%- if m.options.description %}
- description: "{{ xproto_unquote(m.options.description) }}"
- {%- endif %}
- {%- if m.options.verbose_name %}
- verbose_name: "{{ xproto_unquote(m.options.verbose_name) }}"
- {%- endif %}
- fields:
- {%- set id_field = {'type':'int32', 'name':'id', 'options':{}} %}
- {% for f in (xproto_base_fields(m, proto.message_table) + m.fields + [id_field]) | sort(attribute='name') -%}
- {% if xproto_unquote(xproto_first_non_empty([f.options.gui_hidden, 'False'])) != 'True' and (not f.link or f.options.link_type != 'manytomany') -%}
- - hint: {% if f.options.help_text %}{{ xproto_unquote(f.options.help_text) }}{% else %}''{% endif %}
- {% if not f.link -%}
- name: {{ f.name }}
- {%- else -%}
- name: {{ f.name }}_id
- relation: {model: {{ f.options.model }}, type: {{ f.options.link_type }}}
- {% endif %}
- {%- if f.options.default %}
- default: "{{ xproto_unquote(xproto_default_to_gui(f.options.default)) }}"
- {%- endif %}
- {%- if f.options.choices %}
- options:
- {% for o in xproto_options_choices_to_dict(xproto_unquote(f.options.choices)) %}
- - {{ xproto_dict_to_sorted_string(o) }}
- {% endfor %}
- {%- endif %}
- type: {{ xproto_type_to_ui_type(f) }}
- read_only: {{ xproto_is_true(f.options.feedback_state) }}
- {% set validators = xproto_validators(f) -%}
- {% if validators -%}
- validators:
- {% for v in validators | sort(attribute='name',reverse=True) -%}
- - {{ v | yaml }}
- {% endfor %}
- {% else -%}
- validators: []
- {% endif %}
- {% endif -%}
- {% endfor %}
- name: {{ m.name }}
- {%- set goodlinks = xproto_links_to_modeldef_relations( xproto_base_links(m, proto.message_table) + m.links ) + xproto_links_to_modeldef_relations( xproto_base_rlinks(m, proto.message_table) + m.rlinks )%}
- {% if goodlinks %}
- relations:
- {{ goodlinks | join('\n') | indent(width=2)}}
- {%- else %}
- relations: []
- {%- endif %}
+{% if m.name != 'XOSBase' and xproto_unquote(xproto_first_non_empty([m.options.gui_hidden, 'False'])) != 'True' %}
+ - name: {{ m.name }}
+ app: {{ xproto_unquote(xproto_first_non_empty([m.options.name, m.options.app_label, options.name, context.app_label])) }}
+ {%- if m.options.verbose_name %}
+ verbose_name: "{{ xproto_unquote(m.options.verbose_name) }}"
+ {%- endif %}
+ {%- if m.options.description %}
+ description: "{{ xproto_unquote(m.options.description) }}"
+ {%- endif %}
+ {%- set goodlinks = xproto_links_to_modeldef_relations( xproto_base_links(m, proto.message_table) + m.links ) + xproto_links_to_modeldef_relations( xproto_base_rlinks(m, proto.message_table) + m.rlinks )%}
+ {%- if goodlinks %}
+ relations:
+ {%- for gl in goodlinks %}
+ - {{ gl }}
+ {%- endfor %}
+ {%- else %}
+ relations: []
+ {%- endif %}
+ fields:
+ {%- set id_field = {'type':'int32', 'name':'id', 'options':{}} %}
+ {%- for f in (xproto_base_fields(m, proto.message_table) + m.fields + [id_field]) | sort(attribute='name') %}
+ {%- if xproto_unquote(xproto_first_non_empty([f.options.gui_hidden, 'False'])) != 'True' and (not f.link or f.options.link_type != 'manytomany') %}
+ {%- if not f.link %}
+ - name: {{ f.name }}
+ {%- else %}
+ - name: {{ f.name }}_id
+ relation: {model: {{ f.options.model }}, type: {{ f.options.link_type }}}
+ {%- endif %}
+ hint: {% if f.options.help_text %}"{{ xproto_unquote(f.options.help_text) }}"{% else %}""{% endif %}
+ {%- if f.options.default %}
+ default: "{{ xproto_unquote(xproto_default_to_gui(f.options.default)) }}"
+ {%- endif %}
+ {%- if f.options.choices %}
+ options:
+ {%- for o in xproto_options_choices_to_dict(xproto_unquote(f.options.choices)) %}
+ - {{ xproto_dict_to_sorted_string(o) }}
+ {%- endfor %}
+ {%- endif %}
+ type: {{ xproto_type_to_ui_type(f) }}
+ read_only: {{ xproto_is_true(f.options.feedback_state) | lower }}
+ {%- set validators = xproto_validators(f) %}
+ {%- if validators %}
+ validators:
+ {%- for v in validators | sort(attribute='name', reverse=True) %}
+ - {{ v | yaml | indent(width=12)}}
+ {%- endfor %}
+ {%- else %}
+ validators: []
+ {%- endif %}
+ {%- endif %}
+ {%- endfor %}
{%- endif %}
-{% endfor -%}
+{%- endfor %}
diff --git a/lib/xos-kafka/requirements.txt b/lib/xos-kafka/requirements.txt
index 1888a03..dee0cea 100644
--- a/lib/xos-kafka/requirements.txt
+++ b/lib/xos-kafka/requirements.txt
@@ -1,3 +1,3 @@
-confluent-kafka~=0.11.5
+confluent-kafka==0.11.5
multistructlog~=2.1.0
-xosconfig~=3.0.0
+xosconfig~=3.0.1
diff --git a/lib/xos-migrate/requirements.txt b/lib/xos-migrate/requirements.txt
index 0ea31a5..4643655 100644
--- a/lib/xos-migrate/requirements.txt
+++ b/lib/xos-migrate/requirements.txt
@@ -1,7 +1,6 @@
-PyYAML~=3.12
+Django~=1.11.20
django-extensions~=2.1.6
djangorestframework~=3.9.1
-django~=1.11.20
multistructlog~=2.1.0
-xosconfig~=3.0.0
-xosgenx~=3.0.0
+xosconfig~=3.0.1
+xosgenx~=3.0.1
diff --git a/lib/xos-synchronizer/requirements.txt b/lib/xos-synchronizer/requirements.txt
index 616f5fb..c539816 100644
--- a/lib/xos-synchronizer/requirements.txt
+++ b/lib/xos-synchronizer/requirements.txt
@@ -1,8 +1,10 @@
Jinja2~=2.10
-astunparse~=1.5.0
+ansible~=2.7.9
+astunparse~=1.6.2
confluent-kafka==0.11.5
-inflect~=1.0.1
+inflect~=2.1.0
multistructlog~=2.1.0
-networkx~=1.11
-xosconfig~=3.0.0
-xosgenx~=3.0.0
+networkx~=2.2
+six~=1.12.0
+xosconfig~=3.0.1
+xosgenx~=3.0.1
diff --git a/lib/xos-synchronizer/xossynchronizer/event_loop.py b/lib/xos-synchronizer/xossynchronizer/event_loop.py
index 15f005c..0c69694 100644
--- a/lib/xos-synchronizer/xossynchronizer/event_loop.py
+++ b/lib/xos-synchronizer/xossynchronizer/event_loop.py
@@ -136,7 +136,7 @@
"dst_accessor": dst_accessor,
}
model_dependency_graph.add_edge(
- src_model, dst_model, edge_label
+ src_model, dst_model, **edge_label
)
model_dependency_graph_rev = model_dependency_graph.reverse(copy=True)
@@ -635,14 +635,14 @@
edge_type = oG[i1][i0]["type"]
if edge_type == PROXY_EDGE:
oG.remove_edge(i1, i0)
- oG.add_edge(i0, i1, {"type": edge_type})
+ oG.add_edge(i0, i1, type=edge_type)
except KeyError:
- oG.add_edge(i0, i1, {"type": edge_type})
+ oG.add_edge(i0, i1, type=edge_type)
except KeyError:
pass
components = weakly_connected_component_subgraphs(oG)
- cohort_indexes = [reversed(topological_sort(g)) for g in components]
+ cohort_indexes = [reversed(list(topological_sort(g))) for g in components]
cohorts = [
[objects[i] for i in cohort_index] for cohort_index in cohort_indexes
]
diff --git a/lib/xos-util/Makefile b/lib/xos-util/Makefile
deleted file mode 100644
index 4143dba..0000000
--- a/lib/xos-util/Makefile
+++ /dev/null
@@ -1,2 +0,0 @@
-test:
- nosetests -s -v --with-id --with-coverage --cover-html --cover-erase --cover-xml --cover-package="xosutil"
diff --git a/scripts/setup_venv.sh b/scripts/setup_venv.sh
index 85955a1..dba8b15 100755
--- a/scripts/setup_venv.sh
+++ b/scripts/setup_venv.sh
@@ -21,7 +21,7 @@
WORKSPACE=${WORKSPACE:-.}
XOS_DIR=${XOS_DIR:-.}
-PIP_REQS=${PIP_REQS:-${XOS_DIR}/scripts/xos_dev_reqs.txt}
+PIP_REQS=${PIP_REQS:-${XOS_DIR}/containers/xos/requirements.txt}
VENVDIR="${WORKSPACE}/venv-xos"
# create venv if it's not yet there
@@ -34,7 +34,7 @@
echo "Installing python requirements in virtualenv with pip"
source "${VENVDIR}/bin/activate"
pip install --upgrade pip
-pip install -r "$PIP_REQS"
+pip install -r "$PIP_REQS" nose2 mock
pushd "$XOS_DIR/lib/xos-util"
python setup.py install
diff --git a/scripts/xos_dev_reqs.txt b/scripts/xos_dev_reqs.txt
deleted file mode 100644
index c881531..0000000
--- a/scripts/xos_dev_reqs.txt
+++ /dev/null
@@ -1,24 +0,0 @@
-Django==1.11.11
-Jinja2==2.10
-PyYAML==3.12
-Twisted==16.6.0
-astunparse==1.5.0
-colorama==0.3.9
-confluent-kafka==0.11.5
-django-extensions==2.1.5
-djangorestframework==3.9.1
-flake8==3.7.5
-grpcio-tools==1.12.0
-grpcio==1.12.0
-inflect==1.0.1
-ipaddress==1.0.19
-multistructlog==2.1.0
-netaddr==0.7.19
-networkx==1.11
-nose2==0.7.4
-ply==3.11
-plyxproto==4.0.0
-pykwalify==1.6.1
-python-consul~=1.1.0
-requests-mock==1.5.0
-tosca-parser==0.9.0