[SEBA-496]
Update pip packages used in XOS containers, flatten containers
Fixed invalid YAML generated by xosgenx modeldefs.xtarget
Change-Id: Ic051170f3f9ef5a5d0dbba61aa6843458a435d49
diff --git a/containers/chameleon/Dockerfile.chameleon b/containers/chameleon/Dockerfile.chameleon
index 1f36eb8..63084e1 100644
--- a/containers/chameleon/Dockerfile.chameleon
+++ b/containers/chameleon/Dockerfile.chameleon
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/chameleon
-FROM xosproject/xos-base:3.0.1
+FROM xosproject/xos-base:3.0.2
# xos-base already has protoc and dependencies installed
@@ -23,7 +23,19 @@
COPY tmp.chameleon /chameleon/chameleon
# Build the protos
-RUN pip install docker==3.1.4 klein==17.10.0 fluent-logger==0.9.2 \
+RUN pip install \
+ Jinja2==2.10 \
+ PyYAML==5.1 \
+ docker==3.7.0 \
+ fluent-logger==0.9.3 \
+ grpcio-tools==1.12.0 \
+ grpcio==1.12.0 \
+ klein==17.10.0 \
+ netifaces==0.10.9 \
+ pyOpenSSL==19.0.0 \
+ python-consul==1.1.0 \
+ service_identity==18.1.0 \
+ simplejson==3.16.0 \
&& chdir chameleon/chameleon/protos \
&& VOLTHA_BASE=anything make
diff --git a/containers/xos/Dockerfile.base b/containers/xos/Dockerfile.base
index 7af9954..811901b 100644
--- a/containers/xos/Dockerfile.base
+++ b/containers/xos/Dockerfile.base
@@ -32,15 +32,10 @@
&& pip freeze > /var/xos/pip_freeze_apt_`date -u +%Y%m%dT%H%M%S`
# Install python packages with pip
-# Upgrade methodology - pip_requested.txt is the manually generated list of pip
-# packages desired, without any added dependencies. It is used to generate
-# pip_requirements.txt by running: pip install -r pip_requested.txt then use
-# `pipdeptree` to make sure dependencies aren't broken, then
-# `pip freeze > pip_requirements.txt` and manually clean up.
-COPY pip_requirements.txt /tmp/pip_requirements.txt
-RUN pip install --no-deps -U -I pip==9.0.3 setuptools==39.0.1 requests==2.18.4 \
- && pip install --no-deps -r /tmp/pip_requirements.txt \
- && pip freeze > /var/xos/pip_freeze_pip_up_`date -u +%Y%m%dT%H%M%S`
+COPY requirements.txt /tmp/requirements.txt
+RUN pip install -U -I pip==9.0.3 setuptools==39.0.1 \
+ && pip install -r /tmp/requirements.txt \
+ && pip freeze > /var/xos/pip_freeze_base_`date -u +%Y%m%dT%H%M%S`
# Label image
ARG org_label_schema_schema_version=1.0
diff --git a/containers/xos/Dockerfile.client b/containers/xos/Dockerfile.client
index c6bae83..5ba8a8e 100644
--- a/containers/xos/Dockerfile.client
+++ b/containers/xos/Dockerfile.client
@@ -13,16 +13,7 @@
# limitations under the License.
# xosproject/xos-client
-FROM xosproject/xos-libraries:3.0.1
-
-# Install XOS client
-COPY lib/xos-api /tmp/xos-api
-
-# Install the VERSION file
-COPY VERSION /tmp/xos-api/xosapi/VERSION
-
-# install the client library and xossh
-RUN chdir /tmp/xos-api && python ./setup.py install
+FROM xosproject/xos-libraries:3.0.2
# Label image
ARG org_label_schema_schema_version=1.0
diff --git a/containers/xos/Dockerfile.libraries b/containers/xos/Dockerfile.libraries
index 420ba33..acc7ccb 100644
--- a/containers/xos/Dockerfile.libraries
+++ b/containers/xos/Dockerfile.libraries
@@ -13,17 +13,20 @@
# limitations under the License.
# xosproject/xos-libraries
-FROM xosproject/xos-base:3.0.1
+FROM xosproject/xos-base:3.0.2
# Add libraries
COPY lib /opt/xos/lib
COPY VERSION /opt/xos
-# Install python modules included with XOS
-RUN cd /opt/xos/lib/xos-util && python setup.py install \
- && cd /opt/xos/lib/xos-config && python setup.py install \
- && cd /opt/xos/lib/xos-genx && python setup.py install \
- && cd /opt/xos/lib/xos-kafka && python setup.py install
+# Install python using pip so their dependencies are installed
+RUN pip install -e /opt/xos/lib/xos-util \
+ && pip install -e /opt/xos/lib/xos-config \
+ && pip install -e /opt/xos/lib/xos-genx \
+ && pip install -e /opt/xos/lib/xos-kafka \
+ && pip install -e /opt/xos/lib/xos-synchronizer \
+ && pip install -e /opt/xos/lib/xos-api \
+ && pip freeze > /var/xos/pip_freeze_libraries_`date -u +%Y%m%dT%H%M%S`
# Label image
ARG org_label_schema_schema_version=1.0
diff --git a/containers/xos/Dockerfile.synchronizer-base b/containers/xos/Dockerfile.synchronizer-base
index df1e203..c8c7e5b 100644
--- a/containers/xos/Dockerfile.synchronizer-base
+++ b/containers/xos/Dockerfile.synchronizer-base
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/xos-synchronizer-base
-FROM xosproject/xos-client:3.0.1
+FROM xosproject/xos-client:3.0.2
COPY xos/xos/logger.py /opt/xos/xos/logger.py
COPY xos/xos/__init__.py /opt/xos/xos/__init__.py
@@ -22,11 +22,6 @@
# Copy over ansible hosts
COPY containers/xos/ansible-hosts /etc/ansible/hosts
-# For library-based synchronizers
-COPY lib/xos-synchronizer /tmp/xos-synchronizer
-COPY VERSION /tmp/xos-synchronizer/xossynchronizer/VERSION
-RUN cd /tmp/xos-synchronizer && python setup.py install
-
# Label image
ARG org_label_schema_schema_version=1.0
ARG org_label_schema_name=xos-synchronizer-base
diff --git a/containers/xos/Dockerfile.xos-core b/containers/xos/Dockerfile.xos-core
index bddc63b..7fff670 100644
--- a/containers/xos/Dockerfile.xos-core
+++ b/containers/xos/Dockerfile.xos-core
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/xos-core
-FROM xosproject/xos-libraries:3.0.1
+FROM xosproject/xos-libraries:3.0.2
# Install XOS
ADD xos /opt/xos
diff --git a/containers/xos/nodesource.gpg.key b/containers/xos/nodesource.gpg.key
deleted file mode 100644
index 1dc1d10..0000000
--- a/containers/xos/nodesource.gpg.key
+++ /dev/null
@@ -1,52 +0,0 @@
------BEGIN PGP PUBLIC KEY BLOCK-----
-Version: GnuPG v1
-Comment: GPGTools - https://gpgtools.org
-
-mQINBFObJLYBEADkFW8HMjsoYRJQ4nCYC/6Eh0yLWHWfCh+/9ZSIj4w/pOe2V6V+
-W6DHY3kK3a+2bxrax9EqKe7uxkSKf95gfns+I9+R+RJfRpb1qvljURr54y35IZgs
-fMG22Np+TmM2RLgdFCZa18h0+RbH9i0b+ZrB9XPZmLb/h9ou7SowGqQ3wwOtT3Vy
-qmif0A2GCcjFTqWW6TXaY8eZJ9BCEqW3k/0Cjw7K/mSy/utxYiUIvZNKgaG/P8U7
-89QyvxeRxAf93YFAVzMXhoKxu12IuH4VnSwAfb8gQyxKRyiGOUwk0YoBPpqRnMmD
-Dl7SdmY3oQHEJzBelTMjTM8AjbB9mWoPBX5G8t4u47/FZ6PgdfmRg9hsKXhkLJc7
-C1btblOHNgDx19fzASWX+xOjZiKpP6MkEEzq1bilUFul6RDtxkTWsTa5TGixgCB/
-G2fK8I9JL/yQhDc6OGY9mjPOxMb5PgUlT8ox3v8wt25erWj9z30QoEBwfSg4tzLc
-Jq6N/iepQemNfo6Is+TG+JzI6vhXjlsBm/Xmz0ZiFPPObAH/vGCY5I6886vXQ7ft
-qWHYHT8jz/R4tigMGC+tvZ/kcmYBsLCCI5uSEP6JJRQQhHrCvOX0UaytItfsQfLm
-EYRd2F72o1yGh3yvWWfDIBXRmaBuIGXGpajC0JyBGSOWb9UxMNZY/2LJEwARAQAB
-tB9Ob2RlU291cmNlIDxncGdAbm9kZXNvdXJjZS5jb20+iQI4BBMBAgAiBQJTmyS2
-AhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRAWVaCraFdigHTmD/9OKhUy
-jJ+h8gMRg6ri5EQxOExccSRU0i7UHktecSs0DVC4lZG9AOzBe+Q36cym5Z1di6JQ
-kHl69q3zBdV3KTW+H1pdmnZlebYGz8paG9iQ/wS9gpnSeEyx0Enyi167Bzm0O4A1
-GK0prkLnz/yROHHEfHjsTgMvFwAnf9uaxwWgE1d1RitIWgJpAnp1DZ5O0uVlsPPm
-XAhuBJ32mU8S5BezPTuJJICwBlLYECGb1Y65Cil4OALU7T7sbUqfLCuaRKxuPtcU
-VnJ6/qiyPygvKZWhV6Od0Yxlyed1kftMJyYoL8kPHfeHJ+vIyt0s7cropfiwXoka
-1iJB5nKyt/eqMnPQ9aRpqkm9ABS/r7AauMA/9RALudQRHBdWIzfIg0Mlqb52yyTI
-IgQJHNGNX1T3z1XgZhI+Vi8SLFFSh8x9FeUZC6YJu0VXXj5iz+eZmk/nYjUt4Mtc
-pVsVYIB7oIDIbImODm8ggsgrIzqxOzQVP1zsCGek5U6QFc9GYrQ+Wv3/fG8hfkDn
-xXLww0OGaEQxfodm8cLFZ5b8JaG3+Yxfe7JkNclwvRimvlAjqIiW5OK0vvfHco+Y
-gANhQrlMnTx//IdZssaxvYytSHpPZTYw+qPEjbBJOLpoLrz8ZafN1uekpAqQjffI
-AOqW9SdIzq/kSHgl0bzWbPJPw86XzzftewjKNbkCDQRTmyS2ARAAxSSdQi+WpPQZ
-fOflkx9sYJa0cWzLl2w++FQnZ1Pn5F09D/kPMNh4qOsyvXWlekaV/SseDZtVziHJ
-Km6V8TBG3flmFlC3DWQfNNFwn5+pWSB8WHG4bTA5RyYEEYfpbekMtdoWW/Ro8Kmh
-41nuxZDSuBJhDeFIp0ccnN2Lp1o6XfIeDYPegyEPSSZqrudfqLrSZhStDlJgXjea
-JjW6UP6txPtYaaila9/Hn6vF87AQ5bR2dEWB/xRJzgNwRiax7KSU0xca6xAuf+TD
-xCjZ5pp2JwdCjquXLTmUnbIZ9LGV54UZ/MeiG8yVu6pxbiGnXo4Ekbk6xgi1ewLi
-vGmz4QRfVklV0dba3Zj0fRozfZ22qUHxCfDM7ad0eBXMFmHiN8hg3IUHTO+UdlX/
-aH3gADFAvSVDv0v8t6dGc6XE9Dr7mGEFnQMHO4zhM1HaS2Nh0TiL2tFLttLbfG5o
-QlxCfXX9/nasj3K9qnlEg9G3+4T7lpdPmZRRe1O8cHCI5imVg6cLIiBLPO16e0fK
-yHIgYswLdrJFfaHNYM/SWJxHpX795zn+iCwyvZSlLfH9mlegOeVmj9cyhN/VOmS3
-QRhlYXoA2z7WZTNoC6iAIlyIpMTcZr+ntaGVtFOLS6fwdBqDXjmSQu66mDKwU5Ek
-fNlbyrpzZMyFCDWEYo4AIR/18aGZBYUAEQEAAYkCHwQYAQIACQUCU5sktgIbDAAK
-CRAWVaCraFdigIPQEACcYh8rR19wMZZ/hgYv5so6Y1HcJNARuzmffQKozS/rxqec
-0xM3wceL1AIMuGhlXFeGd0wRv/RVzeZjnTGwhN1DnCDy1I66hUTgehONsfVanuP1
-PZKoL38EAxsMzdYgkYH6T9a4wJH/IPt+uuFTFFy3o8TKMvKaJk98+Jsp2X/QuNxh
-qpcIGaVbtQ1bn7m+k5Qe/fz+bFuUeXPivafLLlGc6KbdgMvSW9EVMO7yBy/2JE15
-ZJgl7lXKLQ31VQPAHT3an5IV2C/ie12eEqZWlnCiHV/wT+zhOkSpWdrheWfBT+ac
-hR4jDH80AS3F8jo3byQATJb3RoCYUCVc3u1ouhNZa5yLgYZ/iZkpk5gKjxHPudFb
-DdWjbGflN9k17VCf4Z9yAb9QMqHzHwIGXrb7ryFcuROMCLLVUp07PrTrRxnO9A/4
-xxECi0l/BzNxeU1gK88hEaNjIfviPR/h6Gq6KOcNKZ8rVFdwFpjbvwHMQBWhrqfu
-G3KaePvbnObKHXpfIKoAM7X2qfO+IFnLGTPyhFTcrl6vZBTMZTfZiC1XDQLuGUnd
-sckuXINIU3DFWzZGr0QrqkuE/jyr7FXeUJj9B7cLo+s/TXo+RaVfi3kOc9BoxIvy
-/qiNGs/TKy2/Ujqp/affmIMoMXSozKmga81JSwkADO1JMgUy6dApXz9kP4EE3g==
-=CLGF
------END PGP PUBLIC KEY BLOCK-----
diff --git a/containers/xos/pip_requested.txt b/containers/xos/pip_requested.txt
deleted file mode 100644
index 889110e..0000000
--- a/containers/xos/pip_requested.txt
+++ /dev/null
@@ -1,52 +0,0 @@
-Django==1.11.11
-Jinja2==2.10
-inflect==1.0.1
-PyYAML==3.12
-Twisted==16.6.0
-Werkzeug==0.14.1
-ansible==2.5.0
-astunparse==1.5.0
-confluent-kafka==0.11.5
-django-extensions==2.0.6
-django-timezones==0.2
-djangorestframework==3.7.7
-dnspython==1.15.0
-functools32==3.2.3-2
-futures==3.2.0
-google-api-python-client==1.6.5
-grpcio-tools==1.12.0
-grpcio==1.12.0
-kafkaloghandler==0.9.0
-keystoneauth1==3.4.0
-mock==2.0.0
-multistructlog==2.0.0
-netaddr==0.7.19
-networkx==1.11
-nose==1.3.7
-oslo.log==3.37.0
-oslo.messaging==5.36.0
-oslo.serialization==2.25.0
-oslo.utils==3.36.0
-ply==3.11
-plyxproto==4.0.0
-protobuf==3.5.2
-prometheus_client==0.4.0
-# Avoids a warning, see http://initd.org/psycopg/docs/faq.html#faq-compile
-psycopg2==2.7.4 --no-binary psycopg2
-pyOpenSSL==17.5.0
-pykwalify==1.6.1
-python-consul==0.7.2
-python-glanceclient==2.9.1
-python-keystoneclient==3.15.0
-python-logstash==0.4.6
-python-neutronclient==5.1.0
-python-novaclient==10.1.0
-python-openstackclient==3.14.0
-pytz==2018.3
-requests==2.18.4
-service-identity==17.0.0
-shade==1.27.1
-six==1.11.0
-structlog==18.1.0
-tosca-parser==0.9.0
-semver==2.8.1
diff --git a/containers/xos/pip_requirements.txt b/containers/xos/pip_requirements.txt
deleted file mode 100644
index cb62170..0000000
--- a/containers/xos/pip_requirements.txt
+++ /dev/null
@@ -1,143 +0,0 @@
-amqp==2.3.2
-ansible==2.5.0
-appdirs==1.4.3
-asn1crypto==0.24.0
-astunparse==1.5.0
-attrs==18.2.0
-Babel==2.6.0
-bcrypt==3.1.4
-cachetools==2.1.0
-certifi==2018.8.24
-cffi==1.11.5
-chardet==3.0.4
-cliff==2.13.0
-cmd2==0.8.9
-colorama==0.3.9
-confluent-kafka==0.11.5
-constantly==15.1.0
-contextlib2==0.5.5
-cryptography==2.3.1
-debtcollector==1.20.0
-decorator==4.3.0
-deprecation==2.0.6
-Django==1.11.11
-django-extensions==2.0.6
-django-timezones==0.2
-djangorestframework==3.7.7
-dnspython==1.15.0
-docopt==0.6.2
-dogpile.cache==0.6.7
-enum34==1.1.6
-eventlet==0.24.1
-fasteners==0.14.1
-funcsigs==1.0.2
-functools32==3.2.3.post2
-futures==3.2.0
-futurist==1.7.0
-google-api-python-client==1.6.5
-greenlet==0.4.15
-grpcio==1.12.0
-grpcio-tools==1.12.0
-httplib2==0.11.3
-idna==2.6
-incremental==17.5.0
-inflect==1.0.1
-ipaddress==1.0.22
-iso8601==0.1.12
-Jinja2==2.10
-jmespath==0.9.3
-jsonpatch==1.23
-jsonpointer==2.0
-jsonschema==2.6.0
-kafkaloghandler==0.9.0
-keystoneauth1==3.4.0
-kombu==4.2.1
-MarkupSafe==1.0
-mock==2.0.0
-monotonic==1.5
-msgpack==0.5.6
-multistructlog==2.1.0
-munch==2.3.2
-netaddr==0.7.19
-netifaces==0.10.7
-networkx==1.11
-nose==1.3.7
-oauth2client==4.1.3
-openstacksdk==0.17.2
-os-client-config==1.31.2
-os-service-types==1.3.0
-osc-lib==1.11.1
-oslo.concurrency==3.28.1
-oslo.config==6.5.1
-oslo.context==2.21.0
-oslo.i18n==3.22.1
-oslo.log==3.37.0
-oslo.messaging==5.36.0
-oslo.middleware==3.36.0
-oslo.serialization==2.25.0
-oslo.service==1.32.0
-oslo.utils==3.36.0
-packaging==18.0
-paramiko==2.4.2
-Paste==2.0.3
-PasteDeploy==1.5.2
-pbr==4.2.0
-pika==0.12.0
-pika-pool==0.1.3
-pipdeptree==0.13.0
-ply==3.11
-plyxproto==4.0.0
-prettytable==0.7.2
-prometheus-client==0.4.0
-protobuf==3.5.2
-# Avoids a warning, see http://initd.org/psycopg/docs/faq.html#faq-compile
-psycopg2==2.7.4 --no-binary psycopg2
-pyasn1==0.4.4
-pyasn1-modules==0.2.2
-pycparser==2.19
-pyinotify==0.9.6
-pykwalify==1.6.1
-PyNaCl==1.3.0
-pyOpenSSL==17.5.0
-pyparsing==2.2.2
-pyperclip==1.7.0
-python-cinderclient==4.0.1
-python-consul==0.7.2
-python-dateutil==2.7.3
-python-glanceclient==2.9.1
-python-keystoneclient==3.15.0
-python-logstash==0.4.6
-python-neutronclient==5.1.0
-python-novaclient==10.1.0
-python-openstackclient==3.14.0
-pytz==2018.3
-PyYAML==3.12
-repoze.lru==0.7
-requests==2.18.4
-requestsexceptions==1.4.0
-rfc3986==1.1.0
-Routes==2.4.1
-rsa==4.0
-service-identity==17.0.0
-shade==1.27.1
-simplejson==3.16.0
-six==1.11.0
-statsd==3.3.0
-stevedore==1.29.0
-structlog==18.1.0
-subprocess32==3.5.2
-tenacity==5.0.2
-tosca-parser==0.9.0
-Twisted==16.6.0
-typing==3.6.6
-unicodecsv==0.14.1
-uritemplate==3.0.0
-urllib3==1.22
-vine==1.1.4
-warlock==1.3.0
-wcwidth==0.1.7
-WebOb==1.8.2
-Werkzeug==0.14.1
-wrapt==1.10.11
-zope.interface==4.5.0
-semver==2.8.1
diff --git a/containers/xos/requirements.txt b/containers/xos/requirements.txt
new file mode 100644
index 0000000..64fec10
--- /dev/null
+++ b/containers/xos/requirements.txt
@@ -0,0 +1,14 @@
+Django~=1.11.20
+confluent-kafka==0.11.5
+django-extensions~=2.1.6
+django-timezones~=0.2
+djangorestframework~=3.9.1
+functools32~=3.2.3-2
+futures~=3.2.0
+kafkaloghandler~=0.9.0
+multistructlog~=2.1.0
+prometheus_client~=0.6.0
+psycopg2-binary~=2.7.7
+pytz~=2018.9
+semver~=2.8.1
+tosca-parser~=1.4.0