SEBA-463 Submit migration scripts from sync to core, use instead of autogenned
Change-Id: I9e46b068caf97082c27d6f910f0961ccd5d10c2f
diff --git a/VERSION b/VERSION
index 440e866..c95a9d4 100644
--- a/VERSION
+++ b/VERSION
@@ -1,2 +1,2 @@
-2.1.43
+2.1.44
diff --git a/containers/chameleon/Dockerfile.chameleon b/containers/chameleon/Dockerfile.chameleon
index 2e7c9cb..05df0d9 100644
--- a/containers/chameleon/Dockerfile.chameleon
+++ b/containers/chameleon/Dockerfile.chameleon
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/chameleon
-FROM xosproject/xos-base:2.1.43
+FROM xosproject/xos-base:2.1.44
# xos-base already has protoc and dependencies installed
diff --git a/containers/xos/Dockerfile.client b/containers/xos/Dockerfile.client
index 99ebe8c..bfc7649 100644
--- a/containers/xos/Dockerfile.client
+++ b/containers/xos/Dockerfile.client
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/xos-client
-FROM xosproject/xos-libraries:2.1.43
+FROM xosproject/xos-libraries:2.1.44
# Install XOS client
COPY lib/xos-api /tmp/xos-api
diff --git a/containers/xos/Dockerfile.libraries b/containers/xos/Dockerfile.libraries
index 29d1bb9..ae8000c 100644
--- a/containers/xos/Dockerfile.libraries
+++ b/containers/xos/Dockerfile.libraries
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/xos-libraries
-FROM xosproject/xos-base:2.1.43
+FROM xosproject/xos-base:2.1.44
# Add libraries
COPY lib /opt/xos/lib
diff --git a/containers/xos/Dockerfile.synchronizer-base b/containers/xos/Dockerfile.synchronizer-base
index 4d52363..b0f2da1 100644
--- a/containers/xos/Dockerfile.synchronizer-base
+++ b/containers/xos/Dockerfile.synchronizer-base
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/xos-synchronizer-base
-FROM xosproject/xos-client:2.1.43
+FROM xosproject/xos-client:2.1.44
COPY xos/synchronizers/new_base /opt/xos/synchronizers/new_base
COPY xos/xos/logger.py /opt/xos/xos/logger.py
diff --git a/containers/xos/Dockerfile.xos-core b/containers/xos/Dockerfile.xos-core
index ee2d2d8..632347f 100644
--- a/containers/xos/Dockerfile.xos-core
+++ b/containers/xos/Dockerfile.xos-core
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/xos-core
-FROM xosproject/xos-libraries:2.1.43
+FROM xosproject/xos-libraries:2.1.44
# Install XOS
ADD xos /opt/xos
diff --git a/lib/xos-synchronizer/xossynchronizer/loadmodels.py b/lib/xos-synchronizer/xossynchronizer/loadmodels.py
index 7e82ac9..78fa1a6 100644
--- a/lib/xos-synchronizer/xossynchronizer/loadmodels.py
+++ b/lib/xos-synchronizer/xossynchronizer/loadmodels.py
@@ -57,4 +57,13 @@
item.filename = fn
item.contents = open(os.path.join(api_convenience_dir, fn)).read()
+ # migrations directory is a sibling to the models directory
+ migrations_dir = os.path.join(dir, "..", "migrations")
+ if os.path.exists(migrations_dir):
+ for fn in os.listdir(migrations_dir):
+ if fn.endswith(".py") and "test" not in fn:
+ item = request.migrations.add()
+ item.filename = fn
+ item.contents = open(os.path.join(migrations_dir, fn)).read()
+
result = self.api.dynamicload.LoadModels(request)
diff --git a/xos/coreapi/app_list_builder.py b/xos/coreapi/app_list_builder.py
index 4aad92e..512562c 100644
--- a/xos/coreapi/app_list_builder.py
+++ b/xos/coreapi/app_list_builder.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import json
import os
@@ -24,21 +25,45 @@
def __init__(self):
self.app_metadata_dir = "/opt/xos/xos"
self.services_dest_dir = "/opt/xos/services"
+ self.service_manifests_dir = "/opt/xos/dynamic_services/manifests"
+
+ def load_manifests(self):
+ """ Load the manifests that were saved from LoadModels() calls """
+
+ if not os.path.exists(self.service_manifests_dir):
+ # No manifests dir means no services have been dynamically loaded yet
+ return {}
+
+ manifests = {}
+ for fn in os.listdir(self.service_manifests_dir):
+ manifest_fn = os.path.join(self.service_manifests_dir, fn)
+ manifest = json.loads(open(manifest_fn).read())
+ if not "name" in manifest:
+ # sanity check
+ continue
+ manifests[manifest["name"]] = manifest
+
+ return manifests
def generate_app_lists(self):
- # TODO: Once static onboarding is no more, we will get these from the manifests rather than using listdir
app_names = []
- for fn in os.listdir(self.services_dest_dir):
- service_dir = os.path.join(self.services_dest_dir, fn)
- if (not fn.startswith(".")) and os.path.isdir(service_dir):
- models_fn = os.path.join(service_dir, "models.py")
- if os.path.exists(models_fn):
- app_names.append(fn)
+ automigrate_app_names = []
- # Generate the migration list
+ manifests = self.load_manifests()
+ for manifest in manifests.values():
+ # We're only interested in apps that contain models
+ if manifest.get("xprotos"):
+ app_names.append(manifest["name"])
+
+ # Only apps that do not already have migration scripts will get automigrated
+ # TODO(smbaker): Eventually all apps will have migration scripts. Drop this when that happens.
+ if not manifest.get("migrations"):
+ automigrate_app_names.append(manifest["name"])
+
+ # Generate the auto-migration list
mig_list_fn = os.path.join(self.app_metadata_dir, "xosbuilder_migration_list")
makedirs_if_noexist(os.path.dirname(mig_list_fn))
- file(mig_list_fn, "w").write("\n".join(app_names) + "\n")
+ file(mig_list_fn, "w").write("\n".join(automigrate_app_names) + "\n")
# Generate the app list
app_list_fn = os.path.join(self.app_metadata_dir, "xosbuilder_app_list")
diff --git a/xos/coreapi/dynamicbuild.py b/xos/coreapi/dynamicbuild.py
index 1de6b9e..582fd8b 100644
--- a/xos/coreapi/dynamicbuild.py
+++ b/xos/coreapi/dynamicbuild.py
@@ -72,6 +72,10 @@
self.pre_validate_file(item)
self.pre_validate_python(item)
+ for item in request.migrations:
+ self.pre_validate_file(item)
+ self.pre_validate_python(item)
+
def get_manifests(self):
if not os.path.exists(self.manifest_dir):
return []
@@ -234,6 +238,24 @@
{"filename": item.filename, "path": save_path}
)
+ if request.migrations:
+ # These can be saved directly to the service destination directory, since they are not processed by
+ # xosgenx.
+ migrations_dir = os.path.join(service_manifest["dest_dir"], "migrations")
+ service_manifest["migrations_dir"] = migrations_dir
+ service_manifest["migrations"] = []
+ if not os.path.exists(migrations_dir):
+ os.makedirs(migrations_dir)
+ for item in request.migrations:
+ file(os.path.join(migrations_dir, item.filename), "w").write(
+ item.contents
+ )
+ service_manifest["migrations"].append({"filename": item.filename})
+
+ migrations_init_py_filename = os.path.join(migrations_dir, "__init__.py")
+ if not os.path.exists(migrations_init_py_filename):
+ open(migrations_init_py_filename, "w").write("# created by dynamicbuild")
+
return service_manifest
def run_xosgenx_service(self, manifest):
diff --git a/xos/coreapi/protos/dynamicload.proto b/xos/coreapi/protos/dynamicload.proto
index 0279f71..38b55fc 100644
--- a/xos/coreapi/protos/dynamicload.proto
+++ b/xos/coreapi/protos/dynamicload.proto
@@ -25,6 +25,11 @@
string contents = 2;
};
+message MigrationFile {
+ string filename = 1;
+ string contents = 2;
+};
+
message LoadModelsRequest {
string name = 1;
string version = 2;
@@ -32,6 +37,7 @@
repeated DeclFile decls = 4;
repeated AtticFile attics = 5;
repeated APIConvenienceFile convenience_methods = 6;
+ repeated MigrationFile migrations = 7;
};
message ListConvenienceMethodsReply {
diff --git a/xos/coreapi/test_dynamicbuild.py b/xos/coreapi/test_dynamicbuild.py
index eda5b4c..8931355 100644
--- a/xos/coreapi/test_dynamicbuild.py
+++ b/xos/coreapi/test_dynamicbuild.py
@@ -33,6 +33,7 @@
self.xprotos = []
self.decls = []
self.attics = []
+ self.migrations = []
for (k, v) in kwargs.items():
setattr(self, k, v)
@@ -168,6 +169,64 @@
)
self.assertEqual(manifest.get("state"), "load")
+ def test_handle_loadmodels_request_with_migrations(self):
+ with patch.object(
+ dynamicbuild.DynamicBuilder, "save_models", wraps=self.builder.save_models
+ ) as save_models, patch.object(
+ dynamicbuild.DynamicBuilder,
+ "run_xosgenx_service",
+ wraps=self.builder.run_xosgenx_service,
+ ) as run_xosgenx_service, patch.object(
+ dynamicbuild.DynamicBuilder,
+ "remove_service",
+ wraps=self.builder.remove_service,
+ ) as remove_service:
+ migration1_contents = "print 'one'"
+ migration1 = DynamicLoadItem(
+ filename="migration1.py", contents=migration1_contents
+ )
+ migration2_contents = "print 'one'"
+ migration2 = DynamicLoadItem(
+ filename="migration2.py", contents=migration2_contents
+ )
+
+ self.example_request.migrations = [migration1, migration2]
+
+ result = self.builder.handle_loadmodels_request(self.example_request)
+
+ save_models.assert_called()
+ run_xosgenx_service.assert_called()
+ remove_service.assert_not_called()
+
+ self.assertEqual(result, self.builder.SOMETHING_CHANGED)
+
+ self.assertTrue(os.path.exists(self.builder.manifest_dir))
+ self.assertTrue(
+ os.path.exists(
+ os.path.join(self.builder.manifest_dir, "exampleservice.json")
+ )
+ )
+
+ service_dir = os.path.join(self.base_dir, "services", "exampleservice")
+
+ self.assertTrue(os.path.exists(service_dir))
+ self.assertTrue(os.path.exists(os.path.join(service_dir, "__init__.py")))
+ self.assertTrue(os.path.exists(os.path.join(service_dir, "models.py")))
+ self.assertTrue(os.path.exists(os.path.join(service_dir, "security.py")))
+
+ self.assertTrue(os.path.exists(os.path.join(service_dir, "migrations", "migration1.py")))
+ self.assertTrue(os.path.exists(os.path.join(service_dir, "migrations", "migration2.py")))
+ self.assertTrue(os.path.exists(os.path.join(service_dir, "migrations", "__init__.py")))
+
+ manifest = json.loads(
+ open(
+ os.path.join(self.builder.manifest_dir, "exampleservice.json"), "r"
+ ).read()
+ )
+ self.assertEqual(manifest.get("state"), "load")
+
+ self.assertEqual(manifest.get("migrations"), [{u'filename': u'migration1.py'}, {u'filename': u'migration2.py'}])
+
def test_handle_unloadmodels_request(self):
with patch.object(
dynamicbuild.DynamicBuilder, "save_models", wraps=self.builder.save_models