SEBA-622 Implement filetransfer API;
Support gRPC Reflection;
Mark bookkeeping fields in xproto
Change-Id: Ia8e925a520b7821e72f7c3e9c018ce9cceb8a3ab
diff --git a/Dockerfile.client b/Dockerfile.client
index fb649a2..27582dc 100644
--- a/Dockerfile.client
+++ b/Dockerfile.client
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/xos-client
-FROM xosproject/alpine-grpc-base:0.9.0
+FROM xosproject/alpine-grpc-base:0.9.1
# Add libraries
RUN mkdir -p /opt/xos
diff --git a/Dockerfile.core b/Dockerfile.core
index 2e5dcf5..443452e 100644
--- a/Dockerfile.core
+++ b/Dockerfile.core
@@ -13,7 +13,7 @@
# limitations under the License.
# xosproject/xos-core
-FROM xosproject/alpine-grpc-base:0.9.0
+FROM xosproject/alpine-grpc-base:0.9.1
# Install libraries and python requirements
COPY requirements.txt /tmp/requirements.txt
diff --git a/VERSION b/VERSION
index fb93c04..5ae69bd 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-3.2.5-dev
+3.2.5
diff --git a/lib/xos-genx/xosgenx/jinja2_extensions/base.py b/lib/xos-genx/xosgenx/jinja2_extensions/base.py
index c8f5bfe..26564bb 100644
--- a/lib/xos-genx/xosgenx/jinja2_extensions/base.py
+++ b/lib/xos-genx/xosgenx/jinja2_extensions/base.py
@@ -425,6 +425,15 @@
if "max_length" in field["options"] and field["type"] == "string":
options.append("(val).maxLength = %s" % field["options"]["max_length"])
+ if field["options"].get("feedback_state"):
+ options.append("(feedbackState) = true")
+
+ if field["options"].get("gui_hidden"):
+ options.append("(guiHidden) = true")
+
+ if field["options"].get("bookkeeping_state"):
+ options.append("(bookkeepingState) = true")
+
try:
if field["options"]["null"] == "False":
options.append("(val).nonNull = true")
diff --git a/lib/xos-genx/xosgenx/targets/protoapi.xtarget b/lib/xos-genx/xosgenx/targets/protoapi.xtarget
index e87e7d8..0359c46 100644
--- a/lib/xos-genx/xosgenx/targets/protoapi.xtarget
+++ b/lib/xos-genx/xosgenx/targets/protoapi.xtarget
@@ -35,8 +35,8 @@
repeated int32 {{ ref.src_port }}_ids = {{ ref["id"] }} [(reverseForeignKey).modelName = "{{ ref.peer.name }}"];
{%- endif -%}
{%- endfor %}
- string class_names = 2046;
- string self_content_type_id = 2047;
+ string class_names = 2046 [(bookkeepingState) = true];
+ string self_content_type_id = 2047 [(bookkeepingState) = true];
}
message {{ xproto_pluralize(object) }} {
diff --git a/lib/xos-genx/xosgenx/validator.py b/lib/xos-genx/xosgenx/validator.py
index d95c5d7..48f0fb5 100644
--- a/lib/xos-genx/xosgenx/validator.py
+++ b/lib/xos-genx/xosgenx/validator.py
@@ -29,10 +29,11 @@
# Options that are always allowed
COMMON_OPTIONS = ["help_text", "gui_hidden", "tosca_key", "tosca_key_one_of",
- "feedback_state", "unique", "unique_with"]
+ "bookkeeping_state", "feedback_state", "unique", "unique_with"]
# Options that must be either "True" or "False"
-BOOLEAN_OPTIONS = ["blank", "db_index", "feedback_state", "gui_hidden", "null", "tosca_key", "unique", "text"]
+BOOLEAN_OPTIONS = ["blank", "db_index", "bookkeeping_state", "feedback_state", "gui_hidden", "null",
+ "tosca_key", "unique", "text"]
class XProtoValidator(object):
diff --git a/xos/core/models/core.xproto b/xos/core/models/core.xproto
index f7e5fc5..079f830 100644
--- a/xos/core/models/core.xproto
+++ b/xos/core/models/core.xproto
@@ -10,28 +10,28 @@
option custom_python=True;
// field 1 is reserved for "id"
- required string created = 2 [content_type = "date", auto_now_add = True, help_text = "Time this model was created"];
- required string updated = 3 [default = "now()", content_type = "date", help_text = "Time this model was changed by a non-synchronizer"];
- optional string enacted = 4 [content_type = "date", blank = True, default = None, help_text = "When synced, set to the timestamp of the data that was synced"];
- optional string policed = 5 [content_type = "date", blank = True, default = None, help_text = "When policed, set to the timestamp of the data that was policed"];
+ required string created = 2 [content_type = "date", auto_now_add = True, bookkeeping_state = True, help_text = "Time this model was created"];
+ required string updated = 3 [default = "now()", content_type = "date", bookkeeping_state = True, help_text = "Time this model was changed by a non-synchronizer"];
+ optional string enacted = 4 [content_type = "date", blank = True, default = None, bookkeeping_state = True, help_text = "When synced, set to the timestamp of the data that was synced"];
+ optional string policed = 5 [content_type = "date", blank = True, default = None, bookkeeping_state = True, help_text = "When policed, set to the timestamp of the data that was policed"];
optional string backend_register = 6 [default = "{}", max_length = 1024, feedback_state = True];
- required bool backend_need_delete = 7 [default = False];
- required bool backend_need_reap = 8 [default = False];
+ required bool backend_need_delete = 7 [default = False, bookkeeping_state = True];
+ required bool backend_need_reap = 8 [default = False, bookkeeping_state = True];
required string backend_status = 9 [default = "Provisioning in progress", max_length = 1024, feedback_state = True];
required int32 backend_code = 10 [default = 0, feedback_state = True];
- required bool deleted = 11 [default = False];
- required bool write_protect = 12 [default = False];
- required bool lazy_blocked = 13 [default = False];
- required bool no_sync = 14 [default = False];
- required bool no_policy = 15 [default = False];
+ required bool deleted = 11 [default = False, bookkeeping_state = True];
+ required bool write_protect = 12 [default = False, bookkeeping_state = True];
+ required bool lazy_blocked = 13 [default = False, bookkeeping_state = True];
+ required bool no_sync = 14 [default = False, bookkeeping_state = True];
+ required bool no_policy = 15 [default = False, bookkeeping_state = True];
optional string policy_status = 16 [default = "Policy in process", max_length = 1024, feedback_state = True];
optional int32 policy_code = 17 [default = 0, feedback_state = True];
- required string leaf_model_name = 18 [max_length = 1024, help_text = "The most specialized model in this chain of inheritance, often defined by a service developer"];
- required bool backend_need_delete_policy = 19 [default = False, help_text = "True if delete model_policy must be run before object can be reaped"];
+ required string leaf_model_name = 18 [max_length = 1024, bookkeeping_state = True, help_text = "The most specialized model in this chain of inheritance, often defined by a service developer"];
+ required bool backend_need_delete_policy = 19 [default = False, bookkeeping_state = True, help_text = "True if delete model_policy must be run before object can be reaped"];
required bool xos_managed = 20 [default = True, help_text = "True if xos is responsible for creating/deleting this object", gui_hidden = True];
optional string backend_handle = 21 [max_length = 1024, feedback_state = True, blank=True, help_text = "Handle used by the backend to track this object", gui_hidden = True];
- optional string changed_by_step = 22 [content_type = "date", blank = True, default = None, gui_hidden = True, help_text = "Time this model was changed by a sync step"];
- optional string changed_by_policy = 23 [content_type = "date", blank = True, default = None, gui_hidden = True, help_text = "Time this model was changed by a model policy"];
+ optional string changed_by_step = 22 [content_type = "date", blank = True, default = None, bookkeeping_state = True, gui_hidden = True, help_text = "Time this model was changed by a sync step"];
+ optional string changed_by_policy = 23 [content_type = "date", blank = True, default = None, bookkeeping_state = True, gui_hidden = True, help_text = "Time this model was changed by a model policy"];
}
// The calling user represents the user being accessed, or is a site admin.
diff --git a/xos/coreapi/grpc_server.py b/xos/coreapi/grpc_server.py
index 915e2c3..6374a49 100644
--- a/xos/coreapi/grpc_server.py
+++ b/xos/coreapi/grpc_server.py
@@ -19,8 +19,10 @@
from collections import OrderedDict
from os.path import abspath, basename, dirname, join
import grpc
+from grpc_reflection.v1alpha import reflection
from concurrent import futures
import zlib
+from protos import schema_pb2, schema_pb2_grpc
# initialize config and logger
from xosconfig import Config
@@ -32,8 +34,6 @@
Config.init()
log = create_logger(Config().get("logging"))
-from protos import schema_pb2, schema_pb2_grpc
-
SERVER_KEY = "/opt/cord_profile/core_api_key.pem"
SERVER_CERT = "/opt/cord_profile/core_api_cert.pem"
SERVER_CA = "/usr/local/share/ca-certificates/local_certs.crt"
@@ -95,7 +95,7 @@
self.model_status = model_status
self.model_output = model_output
log.info("Initializing GRPC Server", port=port)
- self.thread_pool = futures.ThreadPoolExecutor(max_workers=1)
+ self.thread_pool = futures.ThreadPoolExecutor(max_workers=10)
self.server = grpc.server(self.thread_pool)
self.django_initialized = False
self.django_apps = []
@@ -128,7 +128,7 @@
def register_dynamicload(self):
from xos_dynamicload_api import DynamicLoadService
- from protos import dynamicload_pb2_grpc
+ from protos import dynamicload_pb2_grpc, dynamicload_pb2
dynamic_load_service = DynamicLoadService(self.thread_pool, self)
self.register(
@@ -138,17 +138,21 @@
)
dynamic_load_service.set_django_apps(self.django_apps)
+ self.service_names.append(dynamicload_pb2.DESCRIPTOR.services_by_name['dynamicload'].full_name)
+
def register_core(self):
from xos_grpc_api import XosService
- from protos import xos_pb2_grpc
+ from protos import xos_pb2_grpc, xos_pb2
self.register(
"xos", xos_pb2_grpc.add_xosServicer_to_server, XosService(self.thread_pool)
)
+ self.service_names.append(xos_pb2.DESCRIPTOR.services_by_name['xos'].full_name)
+
def register_utility(self):
from xos_utility_api import UtilityService
- from protos import utility_pb2_grpc
+ from protos import utility_pb2_grpc, utility_pb2
self.register(
"utility",
@@ -156,9 +160,11 @@
UtilityService(self.thread_pool),
)
+ self.service_names.append(utility_pb2.DESCRIPTOR.services_by_name['utility'].full_name)
+
def register_modeldefs(self):
from xos_modeldefs_api import ModelDefsService
- from protos import modeldefs_pb2_grpc
+ from protos import modeldefs_pb2_grpc, modeldefs_pb2
self.register(
"modeldefs",
@@ -166,9 +172,25 @@
ModelDefsService(self.thread_pool),
)
+ self.service_names.append(modeldefs_pb2.DESCRIPTOR.services_by_name['modeldefs'].full_name)
+
+ def register_filetransfer(self):
+ from xos_filetransfer_api import FileTransferService
+ from protos import filetransfer_pb2_grpc, filetransfer_pb2
+
+ self.register(
+ "filetransfer",
+ filetransfer_pb2_grpc.add_filetransferServicer_to_server,
+ FileTransferService(self.thread_pool),
+ )
+
+ self.service_names.append(filetransfer_pb2.DESCRIPTOR.services_by_name['filetransfer'].full_name)
+
def start(self):
log.info("Starting GRPC Server")
+ self.service_names = []
+
self.register(
"schema",
schema_pb2_grpc.add_SchemaServiceServicer_to_server,
@@ -189,6 +211,10 @@
self.register_core()
self.register_utility()
self.register_modeldefs()
+ self.register_filetransfer()
+
+ self.service_names.append(reflection.SERVICE_NAME)
+ reflection.enable_server_reflection(self.service_names, self.server)
# open port
self.server.add_insecure_port("[::]:%s" % self.port)
diff --git a/xos/coreapi/protos/filetransfer.proto b/xos/coreapi/protos/filetransfer.proto
new file mode 100644
index 0000000..cce1e38
--- /dev/null
+++ b/xos/coreapi/protos/filetransfer.proto
@@ -0,0 +1,36 @@
+syntax = "proto3";
+
+package xos;
+
+import "google/protobuf/empty.proto";
+import "common.proto";
+import "xosoptions.proto";
+
+message FileRequest {
+ string uri = 1;
+};
+
+message FileContents {
+ string chunk = 1;
+}
+
+message FileUploadChunk {
+ string uri = 1;
+ string chunk = 2;
+}
+
+message FileUploadStatus {
+ enum FileUploadStatusCode {
+ SUCCESS = 0;
+ FAILED = 1;
+ }
+ FileUploadStatusCode status = 1;
+ string checksum = 2;
+ int32 chunks_received = 3;
+ int32 bytes_received = 4;
+}
+
+service filetransfer {
+ rpc Download(FileRequest) returns (stream FileContents) {}
+ rpc Upload(stream FileUploadChunk) returns (FileUploadStatus) {}
+}
diff --git a/xos/coreapi/protos/xosoptions.proto b/xos/coreapi/protos/xosoptions.proto
index 0494ada..abae9cd 100644
--- a/xos/coreapi/protos/xosoptions.proto
+++ b/xos/coreapi/protos/xosoptions.proto
@@ -28,6 +28,9 @@
ForeignKeyRule foreignKey = 1002;
ReverseForeignKeyRule reverseForeignKey = 1003;
ManyToManyForeignKeyRule manyToManyForeignKey = 1004;
+ bool guiHidden = 1005;
+ bool feedbackState = 1006;
+ bool bookkeepingState = 1007;
}
extend google.protobuf.MessageOptions {
diff --git a/xos/coreapi/xos_filetransfer_api.py b/xos/coreapi/xos_filetransfer_api.py
new file mode 100644
index 0000000..bf632d3
--- /dev/null
+++ b/xos/coreapi/xos_filetransfer_api.py
@@ -0,0 +1,88 @@
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from apistats import track_request_time
+from authhelper import XOSAuthHelperMixin
+from decorators import translate_exceptions, require_authentication
+import os
+from protos import filetransfer_pb2, filetransfer_pb2_grpc
+
+import hashlib
+from importlib import import_module
+from django.conf import settings
+
+SessionStore = import_module(settings.SESSION_ENGINE).SessionStore
+
+
+class FileTransferService(filetransfer_pb2_grpc.filetransferServicer, XOSAuthHelperMixin):
+ def __init__(self, thread_pool):
+ self.thread_pool = thread_pool
+ XOSAuthHelperMixin.__init__(self)
+
+ def stop(self):
+ pass
+
+ @translate_exceptions("FileTransfer", "Download")
+ @track_request_time("FileTransfer", "Download")
+ @require_authentication
+ def Download(self, request, context):
+ if not request.uri.startswith("file://"):
+ raise Exception("Uri must start with file://")
+
+ backend_filename = request.uri[7:]
+ if not os.path.exists(backend_filename):
+ raise Exception("File %s does not exist" % backend_filename)
+
+ with open(backend_filename) as backend_f:
+ while True:
+ chunk = backend_f.read(65536)
+ if not chunk:
+ return
+ yield filetransfer_pb2.FileContents(chunk=chunk)
+
+ @translate_exceptions("FileTransfer", "Upload")
+ @track_request_time("FileTransfer", "Upload")
+ @require_authentication
+ def Upload(self, request_iterator, context):
+ backend_file = None
+ try:
+ hasher = hashlib.sha1()
+ chunks_received = 0
+ bytes_received = 0
+ for chunk in request_iterator:
+ # The first chunk had better have the URI so we can open the file
+ if not backend_file:
+ if not chunk.uri.startswith("file://"):
+ raise Exception("Uri must start with file://")
+
+ backend_filename = chunk.uri[7:]
+
+ backend_file = open(backend_filename, "w")
+
+ backend_file.write(chunk.chunk)
+ chunks_received += 1
+ bytes_received += len(chunk.chunk)
+ hasher.update(chunk.chunk)
+
+ response = filetransfer_pb2.FileUploadStatus()
+ response.status = response.SUCCESS
+ response.chunks_received = chunks_received
+ response.bytes_received = bytes_received
+ response.checksum = "sha1:" + hasher.hexdigest()
+
+ return response
+ finally:
+ # Files do not always close automatically when going out of scope
+ if backend_file:
+ backend_file.close()