Scott Baker | 7581c25 | 2016-05-27 13:12:47 -0700 | [diff] [blame] | 1 | import os |
| 2 | import base64 |
Scott Baker | aab8a29 | 2016-06-03 16:32:45 -0700 | [diff] [blame] | 3 | import jinja2 |
Scott Baker | 7581c25 | 2016-05-27 13:12:47 -0700 | [diff] [blame] | 4 | import string |
| 5 | import sys |
Scott Baker | b0eb23e | 2016-06-01 16:08:04 -0700 | [diff] [blame] | 6 | import urllib2 |
| 7 | import urlparse |
Scott Baker | 7581c25 | 2016-05-27 13:12:47 -0700 | [diff] [blame] | 8 | import xmlrpclib |
| 9 | |
Scott Baker | 7581c25 | 2016-05-27 13:12:47 -0700 | [diff] [blame] | 10 | from xos.config import Config |
Scott Baker | aab8a29 | 2016-06-03 16:32:45 -0700 | [diff] [blame] | 11 | from core.models import Service, ServiceController, ServiceControllerResource, XOS |
Scott Baker | 7581c25 | 2016-05-27 13:12:47 -0700 | [diff] [blame] | 12 | from xos.logger import Logger, logging |
| 13 | |
| 14 | logger = Logger(level=logging.INFO) |
| 15 | |
| 16 | class XOSBuilder(object): |
Scott Baker | 7806f62 | 2016-06-07 17:45:04 -0700 | [diff] [blame] | 17 | UI_KINDS=["models", "admin", "django_library", "rest_service", "rest_tenant", "tosca_custom_types", "tosca_resource","public_key"] |
Scott Baker | 18c8917 | 2016-06-02 16:40:25 -0700 | [diff] [blame] | 18 | SYNC_CONTROLLER_KINDS=["synchronizer", "private_key", "public_key"] |
Scott Baker | 513ea45 | 2016-06-02 16:03:26 -0700 | [diff] [blame] | 19 | SYNC_ALLCONTROLLER_KINDS=["models", "django_library"] |
Scott Baker | b0eb23e | 2016-06-01 16:08:04 -0700 | [diff] [blame] | 20 | |
Scott Baker | 7581c25 | 2016-05-27 13:12:47 -0700 | [diff] [blame] | 21 | def __init__(self): |
Scott Baker | b0eb23e | 2016-06-01 16:08:04 -0700 | [diff] [blame] | 22 | self.source_ui_image = "xosproject/xos" |
| 23 | self.source_sync_image = "xosproject/xos-synchronizer-openstack" |
| 24 | self.build_dir = "/opt/xos/BUILD/" |
| 25 | |
| 26 | # stuff that has to do with downloading |
| 27 | |
| 28 | def get_dest_dir(self, scr): |
| 29 | xos_base = "opt/xos" |
| 30 | service_name = scr.service_controller.name |
| 31 | base_dirs = {"models": "%s/services/%s/" % (xos_base, service_name), |
| 32 | "admin": "%s/services/%s/" % (xos_base, service_name), |
| 33 | "django_library": "%s/services/%s/" % (xos_base, service_name), |
| 34 | "synchronizer": "%s/synchronizers/%s/" % (xos_base, service_name), |
| 35 | "tosca_custom_types": "%s/tosca/custom_types/" % (xos_base), |
Scott Baker | 18c8917 | 2016-06-02 16:40:25 -0700 | [diff] [blame] | 36 | "tosca_resource": "%s/tosca/resources/" % (xos_base), |
Scott Baker | 16cfb9c | 2016-06-07 15:37:03 -0700 | [diff] [blame] | 37 | "rest_service": "%s/api/service/" % (xos_base), |
Scott Baker | 7806f62 | 2016-06-07 17:45:04 -0700 | [diff] [blame] | 38 | "rest_tenant": "%s/api/tenant/" % (xos_base), |
Scott Baker | 18c8917 | 2016-06-02 16:40:25 -0700 | [diff] [blame] | 39 | "private_key": "%s/services/%s/keys" % (xos_base, service_name), |
| 40 | "public_key": "%s/services/%s/keys/" % (xos_base, service_name)} |
Scott Baker | 260a21c | 2016-06-13 10:42:49 -0700 | [diff] [blame^] | 41 | dest_dir = base_dirs[scr.kind] |
| 42 | |
| 43 | if scr.subdirectory: |
| 44 | dest_dir = os.path.join(dest_dir, scr.subdirectory) |
| 45 | |
| 46 | return dest_dir |
Scott Baker | b0eb23e | 2016-06-01 16:08:04 -0700 | [diff] [blame] | 47 | |
| 48 | def get_build_fn(self, scr): |
| 49 | dest_dir = self.get_dest_dir(scr) |
| 50 | dest_fn = os.path.split(urlparse.urlsplit(scr.full_url).path)[-1] |
| 51 | return os.path.join(dest_dir, dest_fn) |
| 52 | |
| 53 | def get_download_fn(self, scr): |
| 54 | dest_fn = self.get_build_fn(scr) |
| 55 | return os.path.join(self.build_dir, dest_fn) |
| 56 | |
| 57 | def read_manifest(self, scr, fn): |
| 58 | manifest = [] |
| 59 | manifest_lines = file(fn).readlines() |
| 60 | manifest_lines = [x.strip() for x in manifest_lines] |
Scott Baker | c2a2fe9 | 2016-06-02 13:19:10 -0700 | [diff] [blame] | 61 | manifest_lines = [x for x in manifest_lines if x] |
Scott Baker | b0eb23e | 2016-06-01 16:08:04 -0700 | [diff] [blame] | 62 | for line in manifest_lines: |
Scott Baker | c2a2fe9 | 2016-06-02 13:19:10 -0700 | [diff] [blame] | 63 | url_parts = urlparse.urlsplit(scr.full_url) |
| 64 | new_path = os.path.join(os.path.join(*os.path.split(url_parts.path)[:-1]),line) |
| 65 | url = urlparse.urlunsplit( (url_parts.scheme, url_parts.netloc, new_path, url_parts.query, url_parts.fragment) ) |
Scott Baker | b0eb23e | 2016-06-01 16:08:04 -0700 | [diff] [blame] | 66 | |
| 67 | build_fn = os.path.join(self.get_dest_dir(scr), line) |
Scott Baker | c2a2fe9 | 2016-06-02 13:19:10 -0700 | [diff] [blame] | 68 | download_fn = os.path.join(self.build_dir, build_fn) |
Scott Baker | b0eb23e | 2016-06-01 16:08:04 -0700 | [diff] [blame] | 69 | |
| 70 | manifest.append( (url, download_fn, build_fn) ) |
| 71 | return manifest |
| 72 | |
| 73 | def download_file(self, url, dest_fn): |
| 74 | logger.info("Download %s to %s" % (url, dest_fn)) |
| 75 | if not os.path.exists(os.path.dirname(dest_fn)): |
| 76 | os.makedirs(os.path.dirname(dest_fn)) |
| 77 | obj = urllib2.urlopen(url) |
| 78 | file(dest_fn,"w").write(obj.read()) |
| 79 | |
Scott Baker | c2a2fe9 | 2016-06-02 13:19:10 -0700 | [diff] [blame] | 80 | # make python files executable |
| 81 | if dest_fn.endswith(".py"): # and contents.startswith("#!"): |
| 82 | os.chmod(dest_fn, 0755) |
| 83 | |
Scott Baker | b0eb23e | 2016-06-01 16:08:04 -0700 | [diff] [blame] | 84 | def download_resource(self, scr): |
| 85 | if scr.format == "manifest": |
| 86 | manifest_fn = self.get_download_fn(scr) |
| 87 | self.download_file(scr.full_url, manifest_fn) |
Scott Baker | c2a2fe9 | 2016-06-02 13:19:10 -0700 | [diff] [blame] | 88 | manifest = self.read_manifest(scr, manifest_fn) |
Scott Baker | b0eb23e | 2016-06-01 16:08:04 -0700 | [diff] [blame] | 89 | for (url, download_fn, build_fn) in manifest: |
| 90 | self.download_file(url, download_fn) |
| 91 | else: |
| 92 | self.download_file(scr.full_url, self.get_download_fn(scr)) |
| 93 | |
| 94 | def get_docker_lines(self, scr): |
| 95 | if scr.format == "manifest": |
| 96 | manifest_fn = self.get_download_fn(scr) |
| 97 | manifest = self.read_manifest(scr, manifest_fn) |
| 98 | lines = [] |
| 99 | for (url, download_fn, build_fn) in manifest: |
| 100 | lines.append("ADD %s /%s" % (build_fn, build_fn)) |
| 101 | return lines |
| 102 | else: |
| 103 | build_fn = self.get_build_fn(scr) |
| 104 | return ["ADD %s /%s" % (build_fn, build_fn)] |
| 105 | |
| 106 | def get_controller_docker_lines(self, controller, kinds): |
Scott Baker | 7806f62 | 2016-06-07 17:45:04 -0700 | [diff] [blame] | 107 | need_service_init_py = False |
Scott Baker | b0eb23e | 2016-06-01 16:08:04 -0700 | [diff] [blame] | 108 | dockerfile=[] |
| 109 | for scr in controller.service_controller_resources.all(): |
| 110 | if scr.kind in kinds: |
| 111 | lines = self.get_docker_lines(scr) |
| 112 | dockerfile = dockerfile + lines |
Scott Baker | 7806f62 | 2016-06-07 17:45:04 -0700 | [diff] [blame] | 113 | if scr.kind in ["admin", "models"]: |
| 114 | need_service_init_py = True |
| 115 | |
| 116 | if need_service_init_py: |
| 117 | file(os.path.join(self.build_dir, "opt/xos/empty__init__.py"),"w").write("") |
| 118 | dockerfile.append("ADD opt/xos/empty__init__.py /opt/xos/services/%s/__init__.py" % controller.name) |
| 119 | |
Scott Baker | b0eb23e | 2016-06-01 16:08:04 -0700 | [diff] [blame] | 120 | return dockerfile |
| 121 | |
Scott Baker | 7806f62 | 2016-06-07 17:45:04 -0700 | [diff] [blame] | 122 | def check_controller_unready(self, controller): |
| 123 | unready_resources=[] |
| 124 | for scr in controller.service_controller_resources.all(): |
| 125 | if (not scr.backend_status) or (not scr.backend_status.startswith("1")): |
| 126 | unready_resources.append(scr) |
| 127 | |
| 128 | return unready_resources |
| 129 | |
Scott Baker | b0eb23e | 2016-06-01 16:08:04 -0700 | [diff] [blame] | 130 | # stuff that has to do with building |
| 131 | |
Scott Baker | 513ea45 | 2016-06-02 16:03:26 -0700 | [diff] [blame] | 132 | def create_xos_app_data(self, name, dockerfile, app_list, migration_list): |
Scott Baker | ae196c3 | 2016-06-01 23:29:22 -0700 | [diff] [blame] | 133 | if not os.path.exists(os.path.join(self.build_dir,"opt/xos/xos")): |
| 134 | os.makedirs(os.path.join(self.build_dir,"opt/xos/xos")) |
| 135 | |
| 136 | if app_list: |
Scott Baker | 513ea45 | 2016-06-02 16:03:26 -0700 | [diff] [blame] | 137 | dockerfile.append("COPY opt/xos/xos/%s_xosbuilder_app_list /opt/xos/xos/xosbuilder_app_list" % name) |
| 138 | file(os.path.join(self.build_dir, "opt/xos/xos/%s_xosbuilder_app_list") % name, "w").write("\n".join(app_list)+"\n") |
Scott Baker | ae196c3 | 2016-06-01 23:29:22 -0700 | [diff] [blame] | 139 | |
| 140 | if migration_list: |
Scott Baker | 513ea45 | 2016-06-02 16:03:26 -0700 | [diff] [blame] | 141 | dockerfile.append("COPY opt/xos/xos/%s_xosbuilder_migration_list /opt/xos/xos/xosbuilder_migration_list" % name) |
| 142 | file(os.path.join(self.build_dir, "opt/xos/xos/%s_xosbuilder_migration_list") % name, "w").write("\n".join(migration_list)+"\n") |
Scott Baker | ae196c3 | 2016-06-01 23:29:22 -0700 | [diff] [blame] | 143 | |
Scott Baker | b0eb23e | 2016-06-01 16:08:04 -0700 | [diff] [blame] | 144 | def create_ui_dockerfile(self): |
| 145 | dockerfile_fn = "Dockerfile.UI" |
| 146 | |
Scott Baker | ae196c3 | 2016-06-01 23:29:22 -0700 | [diff] [blame] | 147 | app_list = [] |
| 148 | migration_list = [] |
| 149 | |
Scott Baker | b0eb23e | 2016-06-01 16:08:04 -0700 | [diff] [blame] | 150 | dockerfile = ["FROM %s" % self.source_ui_image] |
| 151 | for controller in ServiceController.objects.all(): |
Scott Baker | 7806f62 | 2016-06-07 17:45:04 -0700 | [diff] [blame] | 152 | if self.check_controller_unready(controller): |
| 153 | logger.warning("Controller %s has unready resources" % str(controller)) |
| 154 | continue |
| 155 | |
Scott Baker | b0eb23e | 2016-06-01 16:08:04 -0700 | [diff] [blame] | 156 | dockerfile = dockerfile + self.get_controller_docker_lines(controller, self.UI_KINDS) |
Scott Baker | ae196c3 | 2016-06-01 23:29:22 -0700 | [diff] [blame] | 157 | if controller.service_controller_resources.filter(kind="models").exists(): |
| 158 | app_list.append("services." + controller.name) |
| 159 | migration_list.append(controller.name) |
| 160 | |
Scott Baker | 513ea45 | 2016-06-02 16:03:26 -0700 | [diff] [blame] | 161 | self.create_xos_app_data("ui", dockerfile, app_list, migration_list) |
Scott Baker | b0eb23e | 2016-06-01 16:08:04 -0700 | [diff] [blame] | 162 | |
| 163 | file(os.path.join(self.build_dir, dockerfile_fn), "w").write("\n".join(dockerfile)+"\n") |
| 164 | |
Scott Baker | 8d252b6 | 2016-06-01 23:08:04 -0700 | [diff] [blame] | 165 | return {"dockerfile_fn": dockerfile_fn, |
| 166 | "docker_image_name": "xosproject/xos-ui"} |
Scott Baker | b0eb23e | 2016-06-01 16:08:04 -0700 | [diff] [blame] | 167 | |
| 168 | def create_synchronizer_dockerfile(self, controller): |
Scott Baker | 513ea45 | 2016-06-02 16:03:26 -0700 | [diff] [blame] | 169 | # bake in the synchronizer from this controller |
| 170 | sync_lines = self.get_controller_docker_lines(controller, self.SYNC_CONTROLLER_KINDS) |
| 171 | if not sync_lines: |
Scott Baker | b0eb23e | 2016-06-01 16:08:04 -0700 | [diff] [blame] | 172 | return [] |
| 173 | |
| 174 | dockerfile_fn = "Dockerfile.%s" % controller.name |
| 175 | dockerfile = ["FROM %s" % self.source_sync_image] |
Scott Baker | 513ea45 | 2016-06-02 16:03:26 -0700 | [diff] [blame] | 176 | |
| 177 | # Now bake in models from this controller as well as the others |
| 178 | # It's important to bake all services in, because some services' |
| 179 | # synchronizers may depend on models from another service. |
| 180 | app_list = [] |
| 181 | for c in ServiceController.objects.all(): |
| 182 | dockerfile = dockerfile + self.get_controller_docker_lines(c, self.SYNC_ALLCONTROLLER_KINDS) |
| 183 | if controller.service_controller_resources.filter(kind="models").exists(): |
| 184 | app_list.append("services." + controller.name) |
| 185 | |
| 186 | self.create_xos_app_data(controller.name, dockerfile, app_list, None) |
| 187 | |
| 188 | dockerfile = dockerfile + sync_lines |
Scott Baker | c2a2fe9 | 2016-06-02 13:19:10 -0700 | [diff] [blame] | 189 | file(os.path.join(self.build_dir, dockerfile_fn), "w").write("\n".join(dockerfile)+"\n") |
Scott Baker | b0eb23e | 2016-06-01 16:08:04 -0700 | [diff] [blame] | 190 | |
Scott Baker | 8d252b6 | 2016-06-01 23:08:04 -0700 | [diff] [blame] | 191 | return {"dockerfile_fn": dockerfile_fn, |
| 192 | "docker_image_name": "xosproject/xos-synchronizer-%s" % controller.name} |
Scott Baker | b0eb23e | 2016-06-01 16:08:04 -0700 | [diff] [blame] | 193 | |
Scott Baker | aab8a29 | 2016-06-03 16:32:45 -0700 | [diff] [blame] | 194 | def create_docker_compose(self): |
| 195 | xos = XOS.objects.all()[0] |
| 196 | |
Scott Baker | 29b677b | 2016-06-07 10:20:00 -0700 | [diff] [blame] | 197 | volume_list = [] |
| 198 | for volume in xos.volumes.all(): |
| 199 | volume_list.append({"host_path": volume.host_path, |
| 200 | "container_path": volume.container_path, |
| 201 | "read_only": volume.read_only}) |
Scott Baker | aab8a29 | 2016-06-03 16:32:45 -0700 | [diff] [blame] | 202 | |
| 203 | containers = {} |
| 204 | |
| 205 | containers["xos_db"] = \ |
| 206 | {"image": "xosproject/xos-postgres", |
| 207 | "expose": [5432]} |
| 208 | |
Scott Baker | 584cd89 | 2016-06-09 16:10:31 -0700 | [diff] [blame] | 209 | db_container_name = xos.docker_project_name + "_xos_db_1" |
| 210 | |
Scott Baker | aab8a29 | 2016-06-03 16:32:45 -0700 | [diff] [blame] | 211 | containers["xos_ui"] = \ |
| 212 | {"image": "xosproject/xos-ui", |
| 213 | "command": "python /opt/xos/manage.py runserver 0.0.0.0:%d --insecure --makemigrations" % xos.ui_port, |
| 214 | "ports": {"%d"%xos.ui_port : "%d"%xos.ui_port}, |
| 215 | "links": ["xos_db"], |
Scott Baker | 584cd89 | 2016-06-09 16:10:31 -0700 | [diff] [blame] | 216 | #"external_links": [db_container_name], |
Scott Baker | aab8a29 | 2016-06-03 16:32:45 -0700 | [diff] [blame] | 217 | "volumes": volume_list} |
| 218 | |
Scott Baker | 584cd89 | 2016-06-09 16:10:31 -0700 | [diff] [blame] | 219 | # containers["xos_bootstrap_ui"] = {"image": "xosproject/xos", |
| 220 | # "command": "python /opt/xos/manage.py runserver 0.0.0.0:%d --insecure --makemigrations" % xos.bootstrap_ui_port, |
| 221 | # "ports": {"%d"%xos.bootstrap_ui_port : "%d"%xos.bootstrap_ui_port}, |
| 222 | # #"external_links": [db_container_name], |
| 223 | # "links": ["xos_db"], |
| 224 | # "volumes": volume_list} |
Scott Baker | be41a12 | 2016-06-06 10:40:40 -0700 | [diff] [blame] | 225 | |
Scott Baker | aab8a29 | 2016-06-03 16:32:45 -0700 | [diff] [blame] | 226 | for c in ServiceController.objects.all(): |
Scott Baker | 7806f62 | 2016-06-07 17:45:04 -0700 | [diff] [blame] | 227 | if self.check_controller_unready(c): |
| 228 | logger.warning("Controller %s has unready resources" % str(c)) |
| 229 | continue |
| 230 | |
Scott Baker | aab8a29 | 2016-06-03 16:32:45 -0700 | [diff] [blame] | 231 | containers["xos_synchronizer_%s" % c.name] = \ |
Scott Baker | 7806f62 | 2016-06-07 17:45:04 -0700 | [diff] [blame] | 232 | {"image": "xosproject/xos-synchronizer-%s" % c.name, |
Scott Baker | ff903a9 | 2016-06-08 15:23:34 -0700 | [diff] [blame] | 233 | "command": 'bash -c "sleep 120; cd /opt/xos/synchronizers/%s; bash ./run.sh"' % c.name, |
Scott Baker | 584cd89 | 2016-06-09 16:10:31 -0700 | [diff] [blame] | 234 | #"external_links": [db_container_name], |
Scott Baker | aab8a29 | 2016-06-03 16:32:45 -0700 | [diff] [blame] | 235 | "links": ["xos_db"], |
| 236 | "volumes": volume_list} |
| 237 | |
| 238 | vars = { "containers": containers } |
| 239 | |
| 240 | template_loader = jinja2.FileSystemLoader( "/opt/xos/synchronizers/onboarding/templates/" ) |
| 241 | template_env = jinja2.Environment(loader=template_loader) |
| 242 | template = template_env.get_template("docker-compose.yml.j2") |
| 243 | buffer = template.render(vars) |
| 244 | |
| 245 | if not os.path.exists("/opt/xos/synchronizers/onboarding/docker-compose"): |
| 246 | os.makedirs("/opt/xos/synchronizers/onboarding/docker-compose") |
| 247 | file("/opt/xos/synchronizers/onboarding/docker-compose/docker-compose.yml", "w").write(buffer) |
| 248 | |
Scott Baker | 8d252b6 | 2016-06-01 23:08:04 -0700 | [diff] [blame] | 249 | # def build_xos(self): |
| 250 | # dockerfiles=[] |
| 251 | # dockerfiles.append(self.create_ui_dockerfile()) |
| 252 | # |
| 253 | # for controller in ServiceController.objects.all(): |
| 254 | # dockerfiles.append(self.create_synchronizer_dockerfile(controller)) |
Scott Baker | b0eb23e | 2016-06-01 16:08:04 -0700 | [diff] [blame] | 255 | |
| 256 | |
| 257 | |