don't run anything by default in xos container, create directories
get rid of database port/ip from env vars hack
default to 'xos_db' for the database host
don't require redis container
update to docker-compose v2 format, to use docker networking
more compose v2 fixes
add networks to ui/sync containers
update docker/docker-compose versions
update ansible, reduce errors during xos-manage execution
try localhost instead of 127.0.0.1
dependencies for docker-compose, legacy django db restore (unfortunately)
Change-Id: I7d30a1731d7a0906b2cf836693470936d2597ed0
diff --git a/containers/xos/Dockerfile.base b/containers/xos/Dockerfile.base
index aa3ea15..a89b09c 100644
--- a/containers/xos/Dockerfile.base
+++ b/containers/xos/Dockerfile.base
@@ -4,7 +4,7 @@
FROM ubuntu:14.04.5
MAINTAINER Zack Williams <zdw@cs.arizona.edu>
-# Install apt packages
+# Install apt packages, including docker
RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y \
apt-transport-https \
curl \
@@ -30,6 +30,12 @@
supervisor \
unzip \
wget \
+ && apt-key adv --keyserver hkp://ha.pool.sks-keyservers.net:80 \
+ --recv-keys 58118E89F3A912897C070ADBF76221572C52609D \
+ && echo "deb https://apt.dockerproject.org/repo ubuntu-trusty main" | \
+ sudo tee /etc/apt/sources.list.d/docker.list \
+ && apt-get update \
+ && apt-get install -y docker-engine \
&& rm -rf /var/lib/apt/lists/* \
&& mkdir /var/xos \
&& pip freeze > /var/xos/pip_freeze_apt_`date -u +%Y%m%dT%H%M%S`
@@ -70,25 +76,3 @@
mv jquery-ui-themes-*/themes/smoothness/jquery-ui.min.css /usr/local/lib/python2.7/dist-packages/suit/static/suit/css/ && \
rm -rf jquery-ui-themes.zip jquery-ui-themes-*
-# -----------------------------------------------------------------------------
-# actions formerly in onboarding synchronizer
-
-ENV DOCKER_URL https://get.docker.com/builds/Linux/x86_64/docker-1.10.3
-ENV DOCKER_SHA256 d0df512afa109006a450f41873634951e19ddabf8c7bd419caeb5a526032d86d
-
-RUN curl -fLsS $DOCKER_URL -o docker && \
- echo "${DOCKER_SHA256} docker" | sha256sum -c - && \
- mv docker /usr/local/bin/docker && \
- chmod +x /usr/local/bin/docker
-
-# Install docker-compose
-ENV DOCKER_COMPOSE_URL https://github.com/docker/compose/releases/download/1.5.2/docker-compose-Linux-x86_64
-ENV DOCKER_COMPOSE_SHA256 b6b975badc5389647ef1c16fe8a33bdc5935c61f6afd5a15a28ff765427d01e3
-
-RUN curl -fLsS $DOCKER_COMPOSE_URL -o docker-compose && \
- echo "${DOCKER_COMPOSE_SHA256} docker-compose" | sha256sum -c - && \
- mv docker-compose /usr/local/bin/docker-compose && \
- chmod +x /usr/local/bin/docker-compose
-
-# end actions formerly in onboarding synchronizer
-# ----------------------------------------------------------------------------
diff --git a/containers/xos/Dockerfile.devel b/containers/xos/Dockerfile.devel
index 5a554be..c666792 100644
--- a/containers/xos/Dockerfile.devel
+++ b/containers/xos/Dockerfile.devel
@@ -14,17 +14,13 @@
ADD xos /opt/xos
RUN chmod +x /opt/xos/tools/xos-manage \
+ && mkdir -p /opt/cord_profile /opt/xos_services /opt/xos_libraries \
&& sync \
&& /opt/xos/tools/xos-manage genkeys
-EXPOSE 8000
-
# Set environment variables
ENV HOME /root
# Define working directory
WORKDIR /opt/xos
-# Define default command
-CMD python /opt/xos/manage.py runserver 0.0.0.0:8000 --insecure --makemigrations
-
diff --git a/containers/xos/ansible-hosts b/containers/xos/ansible-hosts
index 0dd74f1..75304a0 100644
--- a/containers/xos/ansible-hosts
+++ b/containers/xos/ansible-hosts
@@ -1,2 +1,2 @@
[localhost]
-127.0.0.1
+localhost ansible_connection=local
diff --git a/containers/xos/pip_requirements.txt b/containers/xos/pip_requirements.txt
index 8e2ae4d..769273c 100644
--- a/containers/xos/pip_requirements.txt
+++ b/containers/xos/pip_requirements.txt
@@ -10,12 +10,14 @@
Routes==2.3.1
WebOb==1.6.1
amqp==1.4.9
-ansible==2.1.1.0
+ansible==2.2.0.0
anyjson==0.3.3
appdirs==1.4.0
argparse==1.2.1
+backports.ssl-match-hostname==3.5.0.1
ca-certs-locater==1.0
cachetools==1.1.6
+cached-property==1.3.0
cffi==1.7.0
chardet==2.0.1
cliff==2.2.0
@@ -37,6 +39,11 @@
django-timezones==0.2
djangorestframework==3.3.3
dnslib==0.9.6
+docker-compose==1.9.0
+docker-py==1.10.6
+docker_pycreds==0.2.1
+dockerpty==0.4.1
+docopt==0.6.2
dogpile.cache==0.6.2
enum34==1.1.6
eventlet==0.19.0
@@ -126,11 +133,13 @@
six==1.10.0
stevedore==1.17.1
supervisor==3.0b2
+texttable==0.8.7
tosca-parser==0.6.0
unicodecsv==0.14.1
uritemplate==0.6
urllib3==1.7.1
warlock==1.2.0
+websocket_client==0.39.0
wheel==0.24.0
wrapt==1.10.8
wsgiref==0.1.2
diff --git a/xos/core/models/xosmodel.py b/xos/core/models/xosmodel.py
index 8ca65f4..e1ff93b 100644
--- a/xos/core/models/xosmodel.py
+++ b/xos/core/models/xosmodel.py
@@ -10,12 +10,10 @@
class XOS(PlCoreBase):
name = StrippedCharField(max_length=200, unique=True, help_text="Name of XOS", default="XOS")
ui_port = models.IntegerField(help_text="Port for XOS UI", default=80)
- bootstrap_ui_port = models.IntegerField(help_text="Port for XOS UI", default=81)
+ bootstrap_ui_port = models.IntegerField(help_text="Port for XOS Bootstrap UI", default=81)
db_container_name = StrippedCharField(max_length=200, help_text="name of XOS db container", default="xos_db")
- redis_container_name = StrippedCharField(max_length=200, help_text="name of XOS redis container", default="xos_redis")
+ redis_container_name = StrippedCharField(max_length=200, help_text="name of XOS redis container", blank=True, default="")
docker_project_name = StrippedCharField(max_length=200, help_text="docker project name")
- #FIXME: duplicate, delete?
- db_container_name = StrippedCharField(max_length=200, help_text="database container name")
enable_build = models.BooleanField(help_text="True if Onboarding Synchronizer should build XOS as necessary", default=True)
frontend_only = models.BooleanField(help_text="If True, XOS will not start synchronizer containers", default=False)
source_ui_image = StrippedCharField(max_length=200, default="xosproject/xos")
diff --git a/xos/synchronizers/onboarding/onboarding_synchronizer_config b/xos/synchronizers/onboarding/onboarding_synchronizer_config
index e2b92fd..6c08c57 100644
--- a/xos/synchronizers/onboarding/onboarding_synchronizer_config
+++ b/xos/synchronizers/onboarding/onboarding_synchronizer_config
@@ -1,17 +1,16 @@
-
[plc]
name=plc
-deployment=VICCI
+deployment=plc
[db]
name=xos
user=postgres
password=password
-host=localhost
+host=xos_db
port=5432
[api]
-host=128.112.171.237
+host=localhost
port=8000
ssl_key=None
ssl_cert=None
@@ -33,6 +32,3 @@
pretend=False
save_ansible_output=True
-[feefie]
-client_id='vicci_dev_central'
-user_id='pl'
diff --git a/xos/synchronizers/onboarding/steps/sync_servicecontroller.yaml b/xos/synchronizers/onboarding/steps/sync_servicecontroller.yaml
index 9431427..b81cde1 100644
--- a/xos/synchronizers/onboarding/steps/sync_servicecontroller.yaml
+++ b/xos/synchronizers/onboarding/steps/sync_servicecontroller.yaml
@@ -1,6 +1,5 @@
---
-- hosts: 127.0.0.1
- connection: local
+- hosts: localhost
vars:
dockerfiles:
diff --git a/xos/synchronizers/onboarding/steps/sync_xos.yaml b/xos/synchronizers/onboarding/steps/sync_xos.yaml
index 8a98873..49d86fe 100644
--- a/xos/synchronizers/onboarding/steps/sync_xos.yaml
+++ b/xos/synchronizers/onboarding/steps/sync_xos.yaml
@@ -1,15 +1,14 @@
---
-- hosts: 127.0.0.1
- connection: local
+- hosts: localhost
vars:
dockerfiles:
{% for dockerfile in dockerfiles %}
- docker_image_name: {{ dockerfile.docker_image_name }}
dockerfile_fn: {{ dockerfile.dockerfile_fn }}
- {% endfor %}
+ {% endfor %}
- tasks:
+ tasks:
{% for dockerfile in dockerfiles %}
- name: build_docker_{{ dockerfile.docker_image_name }}
shell: chdir={{ build_dir }} docker build -f {{ dockerfile.dockerfile_fn }} --rm -t {{ dockerfile.docker_image_name }} .
diff --git a/xos/synchronizers/onboarding/templates/docker-compose.yml.j2 b/xos/synchronizers/onboarding/templates/docker-compose.yml.j2
index 0479160..72824f8 100644
--- a/xos/synchronizers/onboarding/templates/docker-compose.yml.j2
+++ b/xos/synchronizers/onboarding/templates/docker-compose.yml.j2
@@ -1,54 +1,72 @@
-{% for container_name, container in containers.iteritems() %}
+version: '2'
-{{ container_name}}:
+{% if networks %}
+networks:
+{% for network in networks %}
+ {{ network }}:
+ external: true
+{% endfor %}
+{% endif %}
+
+services:
+{% for container_name, container in containers.iteritems() %}
+ {{ container_name }}:
# container_name: {{ container.container_base_name }}_{{ container_name }}_1
- image: {{ container.image }}
+ image: {{ container.image }}
+{%- if container.networks %}
+ networks:
+{%- for network in container.networks %}
+ - {{ network }}
+{%- endfor %}
+{%- endif %}
{%- if container.command %}
- command: {{ container.command }}
+ command: {{ container.command }}
{%- endif %}
{%- if container.ports %}
- ports:
+ ports:
{%- for src,dest in container.ports.iteritems() %}
- - "{{ src }}:{{ dest }}"
+ - "{{ src }}:{{ dest }}"
{%- endfor %}
{%- endif %}
{%- if container.links %}
- links:
+ links:
{%- for link in container.links %}
- - {{ link }}
+ - {{ link }}
{%- endfor %}
{%- endif %}
{%- if container.external_links %}
- external_links:
+ external_links:
{%- for link in container.external_links %}
- - {{ link }}
+ - {{ link }}
{%- endfor %}
{%- endif %}
{%- if container.volumes %}
- volumes:
+ volumes:
{%- for volume in container.volumes %}
{%- if volume.read_only %}
- - {{ volume.host_path }}:{{ volume.container_path }}:ro
+ - {{ volume.host_path }}:{{ volume.container_path }}:ro
{%- else %}
- - {{ volume.host_path }}:{{ volume.container_path }}
+ - {{ volume.host_path }}:{{ volume.container_path }}
{%- endif %}
{%- endfor %}
{%- endif %}
{%- if container.expose %}
- expose:
+ expose:
{%- for expose in container.expose %}
- - "{{ expose }}"
+ - "{{ expose }}"
{%- endfor %}
{%- endif %}
{%- if container.extra_hosts %}
- extra_hosts:
+ extra_hosts:
{%- for host in container.extra_hosts %}
- - "{{ host }}"
+ - "{{ host }}"
{%- endfor %}
{%- endif %}
- log_driver: "json-file"
- log_opt:
- max-size: "100k"
- max-file: "5"
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "1000k"
+ max-file: "5"
{%- endfor %}
+
diff --git a/xos/synchronizers/onboarding/xosbuilder.py b/xos/synchronizers/onboarding/xosbuilder.py
index 30a8c90..d352865 100644
--- a/xos/synchronizers/onboarding/xosbuilder.py
+++ b/xos/synchronizers/onboarding/xosbuilder.py
@@ -319,6 +319,8 @@
else:
extra_hosts = []
+ networks = [ "xos" ] # docker networks used by XOS, parameterize in the future
+
containers = {}
# containers["xos_db"] = \
@@ -334,6 +336,7 @@
containers["xos_ui"] = {
"image": "xosproject/xos-ui",
"command": "python /opt/xos/manage.py runserver 0.0.0.0:%d --insecure --makemigrations" % xos.ui_port,
+ "networks": networks,
"ports": {"%d" % xos.ui_port: "%d" % xos.ui_port},
# "links": ["xos_db"],
# "external_links": ["%s:%s" % (xos.db_container_name, "xos_db")],
@@ -374,6 +377,7 @@
containers[c.name] = {
"image": c.image,
"command": c.command,
+ "networks": networks,
"ports": {
port[0]: port[1]
},
@@ -397,6 +401,7 @@
containers["xos_synchronizer_%s" % c.name] = {
"image": "xosproject/xos-synchronizer-%s" % c.name,
"command": command,
+ "networks": networks,
"external_links": external_links,
"extra_hosts": extra_hosts,
"volumes": volume_list
@@ -405,7 +410,7 @@
if c.no_start:
containers["xos_synchronizer_%s" % c.name]["command"] = "sleep 864000"
- vars = {"containers": containers}
+ vars = {"containers": containers, "networks": networks }
template_loader = jinja2.FileSystemLoader("/opt/xos/synchronizers/onboarding/templates/")
template_env = jinja2.Environment(loader=template_loader)
diff --git a/xos/tools/xos-manage b/xos/tools/xos-manage
index 2045f19..363b00c 100755
--- a/xos/tools/xos-manage
+++ b/xos/tools/xos-manage
@@ -80,6 +80,7 @@
function syncdb {
echo "Syncing XOS services..."
python $XOS_DIR/manage.py syncdb --noinput
+ python $XOS_DIR/manage.py migrate
if [[ $DJANGO_17 ]]; then
echo "Loading initial data from fixture..."
python $XOS_DIR/manage.py --noobserver loaddata $XOS_DIR/core/fixtures/core_initial_data.json
@@ -140,7 +141,6 @@
rm -rf /opt/xos/*/migrations /opt/xos/services/*/migrations
python ./manage.py makemigrations core
python ./manage.py makemigrations syndicate_storage
- python ./manage.py makemigrations mcord
if [[ -e /opt/xos/xos/xosbuilder_migration_list ]]; then
cat /opt/xos/xos/xosbuilder_migration_list | while read line; do
diff --git a/xos/xos/settings.py b/xos/xos/settings.py
index 1b68832..bddeb3b 100644
--- a/xos/xos/settings.py
+++ b/xos/xos/settings.py
@@ -7,28 +7,8 @@
# Django settings for XOS.
from config import Config
-from config import set_override
config = Config()
-
-# Override the config from the environment. This is used leverage the LINK
-# capability of docker. It would be far better to use DNS and that can be
-# done in environments like kubernetes. Look for environment variables that
-# match the link pattern and set the appropriate overeides. It is expected
-# that the set of overrides will be expanded with need
-def overrideDbSettings(v):
- parsed = urlparse(v)
- config.db_host = parsed.hostname
- config.db_port = parsed.port
-
-env_to_config_dict = {
- "XOS_DB_PORT": overrideDbSettings
-}
-
-for key, ofunc in env_to_config_dict.items():
- if key in os.environ:
- ofunc(os.environ[key])
-
GEOIP_PATH = "/usr/share/GeoIP"
XOS_DIR = "/opt/xos"
@@ -61,7 +41,6 @@
AUTH_USER_MODEL = 'core.User'
-
# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
ALLOWED_HOSTS = ["*"]
diff --git a/xos/xos_configuration/xos_common_config b/xos/xos_configuration/xos_common_config
index 2855816..beede12 100755
--- a/xos/xos_configuration/xos_common_config
+++ b/xos/xos_configuration/xos_common_config
@@ -6,7 +6,7 @@
name=xos
user=postgres
password=password
-host=localhost
+host=xos_db
port=5432
[api]
@@ -43,3 +43,4 @@
branding_name=Open Cloud
branding_icon=/static/logo.png
branding_favicon=/static/favicon.png
+