Merge pull request #136 from open-cloud/feature/syndicate-ms
add syndicate devel configuration and syndicate-ms docker container
diff --git a/containers/syndicate-ms/Dockerfile b/containers/syndicate-ms/Dockerfile
new file mode 100644
index 0000000..e74db92
--- /dev/null
+++ b/containers/syndicate-ms/Dockerfile
@@ -0,0 +1,51 @@
+# Syndicate Metadata Server
+# See also https://github.com/syndicate-storage/syndicate-docker
+
+FROM ubuntu:14.04.4
+MAINTAINER Zack Williams <zdw@cs.arizona.edu>
+
+# vars
+ENV APT_KEY butler_opencloud_cs_arizona_edu_pub.gpg
+ENV MS_PORT 8080
+ENV GAE_SDK google_appengine_1.9.35.zip
+
+# Prep apt to be able to download over https
+RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y --force-yes\
+ apt-transport-https
+
+# copy over and trust https cert
+COPY butler.crt /usr/local/share/ca-certificates
+RUN update-ca-certificates
+
+# Install Syndicate MS
+COPY $APT_KEY /tmp/
+RUN apt-key add /tmp/$APT_KEY
+
+RUN echo "deb https://butler.opencloud.cs.arizona.edu/repos/release/syndicate syndicate main" > /etc/apt/sources.list.d/butler.list
+
+RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y --force-yes\
+ syndicate-core \
+ syndicate-ms \
+ wget \
+ unzip
+
+# setup syndicate user
+RUN groupadd -r syndicate && useradd -m -r -g syndicate syndicate
+USER syndicate
+ENV HOME /home/syndicate
+WORKDIR $HOME
+
+# setup GAE
+RUN wget -nv https://storage.googleapis.com/appengine-sdks/featured/$GAE_SDK
+RUN unzip -q $GAE_SDK
+
+# Expose the MS port
+EXPOSE $MS_PORT
+
+# Create a storage location
+RUN mkdir $HOME/datastore
+
+# run the MS under GAE
+CMD $HOME/google_appengine/dev_appserver.py --admin_host=0.0.0.0 --host=0.0.0.0 --storage_path=$HOME/datastore --skip_sdk_update_check=true /usr/src/syndicate/ms
+
+
diff --git a/containers/syndicate-ms/Makefile b/containers/syndicate-ms/Makefile
new file mode 100644
index 0000000..2c24afc
--- /dev/null
+++ b/containers/syndicate-ms/Makefile
@@ -0,0 +1,19 @@
+IMAGE_NAME:=xosproject/syndicate-ms
+CONTAINER_NAME:=xos-syndicate-ms
+NO_DOCKER_CACHE?=false
+
+.PHONY: build
+build: ; docker build --no-cache=${NO_DOCKER_CACHE} --rm -t ${IMAGE_NAME} .
+
+.PHONY: run
+run: ; docker run -d -p 8080:8080 --name ${CONTAINER_NAME} ${IMAGE_NAME}
+
+.PHONY: stop
+stop: ; docker stop ${CONTAINER_NAME}
+
+.PHONY: rm
+rm: ; docker rm ${CONTAINER_NAME}
+
+.PHONY: rmi
+rmi: ; docker rmi ${IMAGE_NAME}
+
diff --git a/containers/syndicate-ms/butler.crt b/containers/syndicate-ms/butler.crt
new file mode 100644
index 0000000..be60161
--- /dev/null
+++ b/containers/syndicate-ms/butler.crt
@@ -0,0 +1,37 @@
+-----BEGIN CERTIFICATE-----
+MIIGgjCCBWqgAwIBAgIRAJ26ZC+oEixlqDU7+7cazpIwDQYJKoZIhvcNAQELBQAw
+djELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAk1JMRIwEAYDVQQHEwlBbm4gQXJib3Ix
+EjAQBgNVBAoTCUludGVybmV0MjERMA8GA1UECxMISW5Db21tb24xHzAdBgNVBAMT
+FkluQ29tbW9uIFJTQSBTZXJ2ZXIgQ0EwHhcNMTYwMzIyMDAwMDAwWhcNMTkwMzIy
+MjM1OTU5WjCBqzELMAkGA1UEBhMCVVMxDjAMBgNVBBETBTg1NzIxMQswCQYDVQQI
+EwJBWjEPMA0GA1UEBxMGVHVjc29uMSIwIAYDVQQKExlUaGUgVW5pdmVyc2l0eSBv
+ZiBBcml6b25hMSAwHgYDVQQLExdDb21wdXRlciBTY2llbmNlICgwNDEyKTEoMCYG
+A1UEAxMfYnV0bGVyLm9wZW5jbG91ZC5jcy5hcml6b25hLmVkdTCCAiIwDQYJKoZI
+hvcNAQEBBQADggIPADCCAgoCggIBAKHUqBxVP6fvTm015n8hXfe53B2IHbMbkwCj
+6eqc2mak8PEVIoD1Ds2TlrvS6xWtFJfNdKlMTNQMh3dVjUC8xcB+OUdr1Q3qv9to
+qiUJC+kTnJNDtOqYqJzX9koH+tHD0zr5/cqyT4vLkJZJXiZ5NGKyHUeh9INTj/ZG
+yHHVrDiF5gUyNl7HrN53AMPpJAxO0rurN5tI3ozK8TE60sslVdxE5zWwnSGazS+0
+hcz7uIyDTpyuo8H6iA/F5L5/USLqAYHLTk10Hg/7vnbRMbaz6sdXPFm+gtZPm5mG
+L2P9I4GM6L/TBXL7+etUtPAgVMoYrdDGZ3wmWOrWukD6ax3BVaX+dJxFNUTju2MZ
+1By6nJIzBBezHE7j4dhjRDaGwsxmdvEjn8weoeWS8ngT3fnm6btFgzO0O2CC3QN9
+M6pk5kJGm8kyhcc8nX4gv/Tkl1gHAd9VNgEJPY3YFXWigtjK7fSYGe9GDQsploUG
+OubK5S8eelSej1u9XW/NgqdxwgQWmxeppWxSwWb4wVyunVX03UHFmk6XnSdtF54E
+iy8VIuItRYyZGni8gAyCx8z6ke2zd8+wWgzsjxQ3dHjbLFxV1O57ZyNyb8TuZ5hk
+0QoJqdR0X6EXc+z4+tV+yYQGQZ5L3vgz7REp3TnlgG8acp3JfZpH8gng05cX6sBi
+I+NbZEmPAgMBAAGjggHTMIIBzzAfBgNVHSMEGDAWgBQeBaN3j2yW4luHS6a0hqxx
+AAznODAdBgNVHQ4EFgQUDfCqsiaVDm70iLaq32jUEmKr9pIwDgYDVR0PAQH/BAQD
+AgWgMAwGA1UdEwEB/wQCMAAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMC
+MGcGA1UdIARgMF4wUgYMKwYBBAGuIwEEAwEBMEIwQAYIKwYBBQUHAgEWNGh0dHBz
+Oi8vd3d3LmluY29tbW9uLm9yZy9jZXJ0L3JlcG9zaXRvcnkvY3BzX3NzbC5wZGYw
+CAYGZ4EMAQICMEQGA1UdHwQ9MDswOaA3oDWGM2h0dHA6Ly9jcmwuaW5jb21tb24t
+cnNhLm9yZy9JbkNvbW1vblJTQVNlcnZlckNBLmNybDB1BggrBgEFBQcBAQRpMGcw
+PgYIKwYBBQUHMAKGMmh0dHA6Ly9jcnQudXNlcnRydXN0LmNvbS9JbkNvbW1vblJT
+QVNlcnZlckNBXzIuY3J0MCUGCCsGAQUFBzABhhlodHRwOi8vb2NzcC51c2VydHJ1
+c3QuY29tMCoGA1UdEQQjMCGCH2J1dGxlci5vcGVuY2xvdWQuY3MuYXJpem9uYS5l
+ZHUwDQYJKoZIhvcNAQELBQADggEBACUaI/yYc0pxAuwIWi0985f06MdKEMJo+qEO
+YLXENApQrJhTPdV9OaChlzI4x2ExmffPZEmhyD0q7z57mT9QkBYQwEJqwbRqfY2v
+0iQ4nLLkyXh7SJSS7J4WSG+cFEN1nFZ8/YGg/TD8spEIPeUGsUvRoJmJm9z90uqd
++ETDc+79TZHxserOY3AJtlvzPScJa1HAqgDJGzgwGdUn82+bKZF5WGsGbfwUS6uS
+Ua2SsOxVZOn5ukF2g9vYs3dcO8u5ITAWrR1s6ACg/wGxvfvXwazpeiFx/RxilpcV
+6W7mTwbE76ZbkafrXbnZ6ihhIPARsVJhJsnClnf5OM7IqrX5g80=
+-----END CERTIFICATE-----
diff --git a/containers/syndicate-ms/butler_opencloud_cs_arizona_edu_pub.gpg b/containers/syndicate-ms/butler_opencloud_cs_arizona_edu_pub.gpg
new file mode 100644
index 0000000..92a2ae4
--- /dev/null
+++ b/containers/syndicate-ms/butler_opencloud_cs_arizona_edu_pub.gpg
@@ -0,0 +1,29 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1
+
+mQINBFb+uuwBEADgmbb2CPnQ2LofLdx5rJN4O75TAjYjJAPyyyIZL2bKmhhuRYwK
+a/gZAlOy5Y/4o5pRgG5s1BFkrSvWRIP+Y3D+PHz7wppjlo31NGm4+34stzlzGu4K
+EEUZpCiUiD1tCxX/H9jZTo5Dm2YvdLxnkWSkbf1ZkIzwNjM3bnYily2a/1NwMmqt
+18Hsy+3ivvUEZO0FmO2reP1l7Eb0hLR2QPxSA4/PxQ81+EJ3CObRYaUZ9KjgIRah
+eyP+PsXaFnxkoikGHod9ll2iWPzpkOUh+xXAu73YK4ikCrIUZ5Oe98Euja8h856H
+xiRRLGVL3iqzgAQJxG/0cXbiobN7bNYGlvLLyp+qRNbmgSYonsJxON4aVG+wjiLi
+gYCOQ/FQT0tYGeDprPBWRj6iGiic6K7W9BDXkxPqlYIYomMrjrqW5kX0YGMp+V7c
+2QG3yfh4+3pfpM+ZYfrAtCdgklYmCYBhoaieMrjIYw31PWqMuzxeb3xBS6++6ksH
+d9TlJKLgJ1UPiKLgDOEyIbYVWhPs2sQoRRstuKfPF9Gdv0UUAnqlyA8siVrvZfB2
+7D05PM4mv83GshoZ8ZAkV7uS6PFJIg6JM11dUM50LTfvHe7ig93CBvbFzm+RqxjQ
+JYf1XWd19912TW7NcNz6lg5jxEYLh8WYJin2xC2aLLb+hpy5NHE/Ien2aQARAQAB
+tCVPcGVuQ2xvdWQgQnV0bGVyIDxzaXRlc0BvcGVuY2xvdWQudXM+iQI9BBMBCgAn
+BQJW/rrsAhsDBQkDwmcABQsJCAcDBRUKCQgLBRYDAgEAAh4BAheAAAoJELvMx3QD
+/Cyyc2UQAIw2A8qrNMQt4skrR/87uKQjfJ/OXC7MEBDTLSL0Ed0VIuRrA/E1s1D/
+YJpdsFfKJyDbZ2Id25L+1QclvEjnsEDCIiURGcRmXLLsqjHCw4N2C16P2JasQVWo
+i1lkqUHC8kCzvR75u+agzpn16Qhb8FqLQxBSxd8vhMEw2LnrjRsjHGwErKhpYfOg
+LFXyurKKBb4KYOLortICgcE3Wz6eqgbNInrTMrSOSf5P7nsPINCFTyemzUyT53IU
+07RmJwTOrcgqJR5klghHQnFXJBkB55EMvFLjUrL4dpnAmlbkKhyFX8aRsBD5Frt2
+93LkHWDa35SELEzfIQznIsfok1rHgDR8kAh7m+tEbmn/Qk3llJ7c/r4JqG0RVGfe
+OfYZDT4I12H6ZWIoLjktnAP4QlDf+olILEYAD0PvKEQU7sQpMmex5QBMt6vvGAj6
+RfPn1iFhUZdOPB7GyWtUn8hmBCEfLAoAAntgoW9NC+PI/chFrm6Nugjz60TbMMOd
+i4s5J998AuJeF2RJogIi61a4VYcprSMTkF5b8kxBhV4N4J5jJQEQxo3ztdw7USvj
+ce8/3/69mBT7rIXgk39FvqnSIz9SmyQ+wgLb94Gcpy1id64yab2P1LNm3pORafSN
+F59uVqgEv5W2g/frt5QMSBO06dvzNjStIV7/uxlOHuSNooIClr//
+=JFDF
+-----END PGP PUBLIC KEY BLOCK-----
diff --git a/xos/configurations/syndicate/MS.mk b/xos/configurations/syndicate/MS.mk
new file mode 100644
index 0000000..b51d7ec
--- /dev/null
+++ b/xos/configurations/syndicate/MS.mk
@@ -0,0 +1,38 @@
+# MS build parameters
+
+MS_APP_ADMIN_EMAIL ?= sites@opencloud.us
+MS_APP_ADMIN_PUBLIC_KEY ?= ms/admin.pub
+MS_APP_ADMIN_PRIVATE_KEY ?= ms/admin.pem
+
+MS_APP_NAME ?= syndicate-ms
+MS_APP_PUBLIC_KEY ?= ms/syndicate.pub
+MS_APP_PRIVATE_KEY ?= ms/syndicate.pem
+
+MS_DEVEL ?= true
+
+$(MS_APP_ADMIN_PRIVATE_KEY):
+ openssl genrsa 4096 > "$@"
+
+$(MS_APP_ADMIN_PUBLIC_KEY): $(MS_APP_ADMIN_PRIVATE_KEY)
+ openssl rsa -in "$<" -pubout > "$@"
+
+$(MS_APP_PRIVATE_KEY):
+ openssl genrsa 4096 > "$@"
+
+$(MS_APP_PUBLIC_KEY): $(MS_APP_PRIVATE_KEY)
+ openssl rsa -in "$<" -pubout > "$@"
+
+ms/admin_info.py: ms/admin_info.pyin $(MS_APP_ADMIN_PUBLIC_KEY) $(MS_APP_PUBLIC_KEY) $(MS_APP_PRIVATE_KEY)
+ mkdir -p "$(@D)"
+ cat "$<" | \
+ sed -e 's~@MS_APP_NAME@~$(MS_APP_NAME)~g;' | \
+ sed -e 's~@MS_APP_ADMIN_EMAIL@~$(MS_APP_ADMIN_EMAIL)~g;' | \
+ sed -e 's~@MS_DEVEL@~$(MS_DEVEL)~g;' | \
+ sed -e 's~@MS_APP_ADMIN_PUBLIC_KEY@~$(shell cat $(MS_APP_ADMIN_PUBLIC_KEY) | tr "\n" "@" | sed 's/@/\\n/g')~g;' | \
+ sed -e 's~@MS_APP_PRIVATE_KEY@~$(shell cat $(MS_APP_PRIVATE_KEY) | tr "\n" "@" | sed 's/@/\\n/g')~g;' | \
+ sed -e 's~@MS_APP_PUBLIC_KEY@~$(shell cat $(MS_APP_PUBLIC_KEY) | tr "\n" "@" | sed 's/@/\\n/g')~g;' > "$@"
+
+ms/app.yaml: ms/app.yamlin
+ mkdir -p "$(@D)"
+ cat "$<" | \
+ sed -e 's~@MS_APP_NAME@~$(MS_APP_NAME)~g;' > "$@"
diff --git a/xos/configurations/syndicate/Makefile b/xos/configurations/syndicate/Makefile
new file mode 100644
index 0000000..f057182
--- /dev/null
+++ b/xos/configurations/syndicate/Makefile
@@ -0,0 +1,61 @@
+MYIP:=$(shell hostname -i)
+
+
+cloudlab: common_cloudlab xos
+
+devstack: upgrade_pkgs common_devstack xos
+
+xos: syndicate_config
+ sudo MYIP=$(MYIP) docker-compose up -d
+ bash ../common/wait_for_xos.sh
+ sudo MYIP=$(MYIP) docker-compose run xos python /opt/xos/tosca/run.py padmin@vicci.org /opt/xos/configurations/common/base.yaml
+ sudo MYIP=$(MYIP) docker-compose run xos python /opt/xos/tosca/run.py padmin@vicci.org /root/setup/nodes.yaml
+
+containers:
+ cd ../../../containers/xos; make devel
+ cd ../../../containers/synchronizer; make
+ cd ../../../containers/syndicate-ms; make
+
+include MS.mk
+# see also
+syndicate_config: ms/admin_info.py ms/app.yaml
+
+common_cloudlab:
+ make -C ../common -f Makefile.cloudlab
+
+common_devstack:
+ make -C ../common -f Makefile.devstack
+
+stop:
+ sudo MYIP=$(MYIP) docker-compose stop
+
+showlogs:
+ sudo MYIP=$(MYIP) docker-compose logs
+
+rm: stop
+ sudo MYIP=$(MYIP) docker-compose rm --force
+
+ps:
+ sudo MYIP=$(MYIP) docker-compose ps
+
+enter-xos:
+ sudo docker exec -it syndicate_xos_1 bash
+
+enter-synchronizer:
+ sudo docker exec -it syndicate_xos_synchronizer_openstack_1 bash
+
+enter-ms:
+ sudo docker exec -it syndicate_xos_syndicate_ms_1 bash
+
+upgrade_pkgs:
+ sudo pip install httpie --upgrade
+
+rebuild_xos:
+ make -C ../../../containers/xos devel
+
+rebuild_synchronizer:
+ make -C ../../../containers/synchronizer
+
+rebuild_syndicate_ms:
+ make -C ../../../containers/syndicate-ms
+
diff --git a/xos/configurations/syndicate/README.md b/xos/configurations/syndicate/README.md
new file mode 100644
index 0000000..f78afd9
--- /dev/null
+++ b/xos/configurations/syndicate/README.md
@@ -0,0 +1,6 @@
+# XOS w/Syndicate environment
+
+This is a test environment derived from the [devel](../devel) environment.
+
+It implements the Metadata Service (MS) within an additional Docker container
+
diff --git a/xos/configurations/syndicate/docker-compose.yml b/xos/configurations/syndicate/docker-compose.yml
new file mode 100644
index 0000000..d2d3dbb
--- /dev/null
+++ b/xos/configurations/syndicate/docker-compose.yml
@@ -0,0 +1,51 @@
+xos_db:
+ image: xosproject/xos-postgres
+ expose:
+ - "5432"
+
+xos_syndicate_ms:
+ build: ../../../containers/syndicate-ms/
+ expose:
+ - "8080"
+ volumes:
+ - ./ms/app.yaml:/usr/src/syndicate/ms/app.yaml
+ - ./ms/admin_info.py:/usr/src/syndicate/ms/common/admin_info.py
+
+xos_synchronizer_openstack:
+ image: xosproject/xos-synchronizer-openstack
+ command: bash -c "sleep 120; python /opt/xos/synchronizers/openstack/xos-synchronizer.py"
+ labels:
+ org.xosproject.kind: synchronizer
+ org.xosproject.target: openstack
+ links:
+ - xos_db
+ extra_hosts:
+ - ctl:${MYIP}
+ volumes:
+ - ../common/xos_common_config:/opt/xos/xos_configuration/xos_common_config:ro
+ - ./images:/opt/xos/images:ro
+
+xos_synchronizer_exampleservice:
+ image: xosproject/xos-synchronizer-openstack
+ command: bash -c "sleep 120; python /opt/xos/synchronizers/exampleservice/exampleservice-synchronizer.py -C /opt/xos/synchronizers/exampleservice/exampleservice_config"
+ labels:
+ org.xosproject.kind: synchronizer
+ org.xosproject.target: exampleservice
+ links:
+ - xos_db
+ extra_hosts:
+ - ctl:${MYIP}
+ volumes:
+ - ../common/xos_common_config:/opt/xos/xos_configuration/xos_common_config:ro
+ - ../setup/id_rsa:/opt/xos/synchronizers/exampleservice/exampleservice_private_key:ro
+
+xos:
+ image: xosproject/xos
+ command: python /opt/xos/manage.py runserver 0.0.0.0:8000 --insecure --makemigrations
+ ports:
+ - "9999:8000"
+ links:
+ - xos_db
+ volumes:
+ - ../setup:/root/setup:ro
+ - ../common/xos_common_config:/opt/xos/xos_configuration/xos_common_config:ro
diff --git a/xos/configurations/syndicate/ms/admin_info.pyin b/xos/configurations/syndicate/ms/admin_info.pyin
new file mode 100644
index 0000000..9dee928
--- /dev/null
+++ b/xos/configurations/syndicate/ms/admin_info.pyin
@@ -0,0 +1,12 @@
+#!/usr/bin/python
+
+# AUTO-GENERATED FILE
+
+ADMIN_PUBLIC_KEY = """@MS_APP_ADMIN_PUBLIC_KEY@""".strip()
+ADMIN_EMAIL = "@MS_APP_ADMIN_EMAIL@".strip()
+ADMIN_ID = 0
+
+SYNDICATE_NAME = "@MS_APP_NAME@".strip()
+SYNDICATE_PUBKEY = """@MS_APP_PUBLIC_KEY@""".strip()
+SYNDICATE_PRIVKEY = """@MS_APP_PRIVATE_KEY@""".strip()
+
diff --git a/xos/configurations/syndicate/ms/app.yamlin b/xos/configurations/syndicate/ms/app.yamlin
new file mode 100644
index 0000000..c480565
--- /dev/null
+++ b/xos/configurations/syndicate/ms/app.yamlin
@@ -0,0 +1,45 @@
+application: @MS_APP_NAME@
+version: 1
+runtime: python27
+api_version: 1
+threadsafe: yes
+
+inbound_services:
+- warmup
+
+builtins:
+- appstats: on
+- admin_redirect: on
+- deferred: on
+
+handlers:
+- url: /cron.*
+ script: cron.app
+ login: admin
+
+- url: /CERT.*
+ script: msapp.app
+ secure: always
+
+- url: /USER.*
+ script: msapp.app
+ secure: always
+
+- url: /VOLUME.*
+ script: msapp.app
+ secure: always
+
+- url: .*
+ script: msapp.app
+ secure: never
+
+libraries:
+- name: webapp2
+ version: "2.5.2"
+- name: lxml
+ version: latest
+- name: pycrypto
+ version: latest
+- name: django
+ version: 1.4
+