docker cannot efficiently build many COPY lines
diff --git a/xos/synchronizers/onboarding/xosbuilder.py b/xos/synchronizers/onboarding/xosbuilder.py
index 80a41c8..6d33110 100644
--- a/xos/synchronizers/onboarding/xosbuilder.py
+++ b/xos/synchronizers/onboarding/xosbuilder.py
@@ -92,33 +92,69 @@
else:
self.download_file(scr.full_url, self.get_download_fn(scr))
- def get_docker_lines(self, scr):
+# XXX docker creates a new container and commits it for every single COPY
+# line in the dockerfile. This causes services with many files (for example,
+# vsg) to take ~ 10-15 minutes to build the docker file. So instead we'll copy
+# the whole build directory, and then run a script that copies the files
+# we want.
+
+# def get_docker_lines(self, scr):
+# if scr.format == "manifest":
+# manifest_fn = self.get_download_fn(scr)
+# manifest = self.read_manifest(scr, manifest_fn)
+# lines = []
+# for (url, download_fn, build_fn) in manifest:
+# script.append("mkdir -p
+# #lines.append("COPY %s /%s" % (build_fn, build_fn))
+# return lines
+# else:
+# build_fn = self.get_build_fn(scr)
+# #return ["COPY %s /%s" % (build_fn, build_fn)]
+
+# def get_controller_docker_lines(self, controller, kinds):
+# need_service_init_py = False
+# dockerfile=[]
+# for scr in controller.service_controller_resources.all():
+# if scr.kind in kinds:
+# lines = self.get_docker_lines(scr)
+# dockerfile = dockerfile + lines
+# if scr.kind in ["admin", "models"]:
+# need_service_init_py = True
+#
+# if need_service_init_py:
+# file(os.path.join(self.build_dir, "opt/xos/empty__init__.py"),"w").write("")
+# dockerfile.append("COPY opt/xos/empty__init__.py /opt/xos/services/%s/__init__.py" % controller.name)
+#
+# return dockerfile
+
+ def get_script_lines(self, scr):
if scr.format == "manifest":
manifest_fn = self.get_download_fn(scr)
manifest = self.read_manifest(scr, manifest_fn)
lines = []
for (url, download_fn, build_fn) in manifest:
- lines.append("ADD %s /%s" % (build_fn, build_fn))
+ lines.append("mkdir -p /%s" % os.path.dirname(build_fn))
+ lines.append("cp /build/%s /%s" % (build_fn, build_fn))
return lines
else:
build_fn = self.get_build_fn(scr)
- return ["ADD %s /%s" % (build_fn, build_fn)]
+ return ["mkdir -p /%s" % os.path.dirname(build_fn),
+ "cp /build/%s /%s" % (build_fn, build_fn)]
- def get_controller_docker_lines(self, controller, kinds):
+ def get_controller_script_lines(self, controller, kinds):
need_service_init_py = False
- dockerfile=[]
+ script=[]
for scr in controller.service_controller_resources.all():
if scr.kind in kinds:
- lines = self.get_docker_lines(scr)
- dockerfile = dockerfile + lines
+ lines = self.get_script_lines(scr)
+ script = script + lines
if scr.kind in ["admin", "models"]:
need_service_init_py = True
if need_service_init_py:
- file(os.path.join(self.build_dir, "opt/xos/empty__init__.py"),"w").write("")
- dockerfile.append("ADD opt/xos/empty__init__.py /opt/xos/services/%s/__init__.py" % controller.name)
+ script.append("echo > /opt/xos/services/%s/__init__.py" % controller.name)
- return dockerfile
+ return script
def check_controller_unready(self, controller):
unready_resources=[]
@@ -130,16 +166,20 @@
# stuff that has to do with building
- def create_xos_app_data(self, name, dockerfile, app_list, migration_list):
+ def create_xos_app_data(self, name, script, app_list, migration_list):
if not os.path.exists(os.path.join(self.build_dir,"opt/xos/xos")):
os.makedirs(os.path.join(self.build_dir,"opt/xos/xos"))
if app_list:
- dockerfile.append("COPY opt/xos/xos/%s_xosbuilder_app_list /opt/xos/xos/xosbuilder_app_list" % name)
+ script.append("mkdir -p /opt/xos/xos")
+ script.append("cp /build/opt/xos/xos/%s_xosbuilder_app_list /opt/xos/xos/xosbuilder_app_list" % name)
+ #dockerfile.append("COPY opt/xos/xos/%s_xosbuilder_app_list /opt/xos/xos/xosbuilder_app_list" % name)
file(os.path.join(self.build_dir, "opt/xos/xos/%s_xosbuilder_app_list") % name, "w").write("\n".join(app_list)+"\n")
if migration_list:
- dockerfile.append("COPY opt/xos/xos/%s_xosbuilder_migration_list /opt/xos/xos/xosbuilder_migration_list" % name)
+ script.append("mkdir -p /opt/xos/xos")
+ script.append("cp /build/opt/xos/xos/%s_xosbuilder_migration_list /opt/xos/xos/xosbuilder_migration_list" % name)
+ #dockerfile.append("COPY opt/xos/xos/%s_xosbuilder_migration_list /opt/xos/xos/xosbuilder_migration_list" % name)
file(os.path.join(self.build_dir, "opt/xos/xos/%s_xosbuilder_migration_list") % name, "w").write("\n".join(migration_list)+"\n")
def create_ui_dockerfile(self):
@@ -149,17 +189,23 @@
migration_list = []
dockerfile = ["FROM %s" % self.source_ui_image]
+ script = []
for controller in ServiceController.objects.all():
if self.check_controller_unready(controller):
logger.warning("Controller %s has unready resources" % str(controller))
continue
- dockerfile = dockerfile + self.get_controller_docker_lines(controller, self.UI_KINDS)
+ #dockerfile = dockerfile + self.get_controller_docker_lines(controller, self.UI_KINDS)
+ script = script + self.get_controller_script_lines(controller, self.UI_KINDS)
if controller.service_controller_resources.filter(kind="models").exists():
app_list.append("services." + controller.name)
migration_list.append(controller.name)
- self.create_xos_app_data("ui", dockerfile, app_list, migration_list)
+ self.create_xos_app_data("ui", script, app_list, migration_list)
+
+ file(os.path.join(self.build_dir, "install-xos.sh"), "w").write("\n".join(script)+"\n")
+ dockerfile.append("COPY . /build/")
+ dockerfile.append("RUN bash /build/install-xos.sh")
file(os.path.join(self.build_dir, dockerfile_fn), "w").write("\n".join(dockerfile)+"\n")
@@ -168,25 +214,32 @@
def create_synchronizer_dockerfile(self, controller):
# bake in the synchronizer from this controller
- sync_lines = self.get_controller_docker_lines(controller, self.SYNC_CONTROLLER_KINDS)
+ sync_lines = self.get_controller_script_lines(controller, self.SYNC_CONTROLLER_KINDS)
if not sync_lines:
return []
dockerfile_fn = "Dockerfile.%s" % controller.name
dockerfile = ["FROM %s" % self.source_sync_image]
+ script = []
# Now bake in models from this controller as well as the others
# It's important to bake all services in, because some services'
# synchronizers may depend on models from another service.
app_list = []
for c in ServiceController.objects.all():
- dockerfile = dockerfile + self.get_controller_docker_lines(c, self.SYNC_ALLCONTROLLER_KINDS)
+ #dockerfile = dockerfile + self.get_controller_docker_lines(c, self.SYNC_ALLCONTROLLER_KINDS)
+ script = script + self.get_controller_script_lines(c, self.SYNC_ALLCONTROLLER_KINDS)
if controller.service_controller_resources.filter(kind="models").exists():
- app_list.append("services." + controller.name)
+ app_list.append("services." + c.name)
- self.create_xos_app_data(controller.name, dockerfile, app_list, None)
+ self.create_xos_app_data(controller.name, script, app_list, None)
- dockerfile = dockerfile + sync_lines
+ script = script + sync_lines
+
+ file(os.path.join(self.build_dir, "install-%s.sh" % controller.name), "w").write("\n".join(script)+"\n")
+ dockerfile.append("COPY . /build/")
+ dockerfile.append("RUN bash /build/install-%s.sh" % controller.name)
+
file(os.path.join(self.build_dir, dockerfile_fn), "w").write("\n".join(dockerfile)+"\n")
return {"dockerfile_fn": dockerfile_fn,
@@ -232,7 +285,7 @@
if c.service_controller_resources.filter(kind="synchronizer").exists():
if c.synchronizer_run and c.synchronizer_config:
- command = 'bash -c "sleep 120; cd /opt/xos/synchronizers/%s; python ./%s -C %s" % (c.name, c.synchronizer_run, c.synchronizer_config)
+ command = 'bash -c "sleep 120; cd /opt/xos/synchronizers/%s; python ./%s -C %s"' % (c.name, c.synchronizer_run, c.synchronizer_config)
else:
command = 'bash -c "sleep 120; cd /opt/xos/synchronizers/%s; bash ./run.sh"' % c.name