SEBA-580 Add uuid to backupoperation;
Add version endpoint
Change-Id: I7e2877332e7005fc74b3fda7cfb16a8cbf9c4b64
diff --git a/Dockerfile.core b/Dockerfile.core
index 443452e..9ec33ab 100644
--- a/Dockerfile.core
+++ b/Dockerfile.core
@@ -28,8 +28,7 @@
COPY lib /opt/xos/lib
# Install XOS libraries
-RUN pip install -e /opt/xos/lib/xos-util \
- && pip install -e /opt/xos/lib/xos-config \
+RUN pip install -e /opt/xos/lib/xos-config \
&& pip install -e /opt/xos/lib/xos-genx \
&& pip install -e /opt/xos/lib/xos-kafka \
&& pip freeze > /var/xos/pip_freeze_xos-core_libs_`date -u +%Y%m%dT%H%M%S` \
@@ -54,6 +53,9 @@
ARG org_label_schema_build_date=unknown
ARG org_opencord_vcs_commit_date=unknown
+# Record git build information
+RUN echo $org_label_schema_vcs_ref > /opt/xos/COMMIT
+
LABEL org.label-schema.schema-version=1.0 \
org.label-schema.name=xos-core \
org.label-schema.version=$org_label_schema_version \
diff --git a/VERSION b/VERSION
index 5ae69bd..db5486c 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-3.2.5
+3.2.6-dev
diff --git a/xos/core/migrations/0012_backupoperation_decl_uuid.py b/xos/core/migrations/0012_backupoperation_decl_uuid.py
new file mode 100644
index 0000000..e8c0d2f
--- /dev/null
+++ b/xos/core/migrations/0012_backupoperation_decl_uuid.py
@@ -0,0 +1,34 @@
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-05-10 23:14
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('core', '0011_auto_20190430_1254'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='backupoperation_decl',
+ name='uuid',
+ field=models.CharField(blank=True, help_text=b'unique identifer of this request', max_length=80, null=True),
+ ),
+ ]
diff --git a/xos/core/models/backupoperation.py b/xos/core/models/backupoperation.py
new file mode 100644
index 0000000..204a981
--- /dev/null
+++ b/xos/core/models/backupoperation.py
@@ -0,0 +1,30 @@
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import socket
+import struct
+from uuid import uuid4
+
+from xos.exceptions import *
+from backupoperation_decl import *
+
+
+class BackupOperation(BackupOperation_decl):
+ class Meta:
+ proxy = True
+
+ def save(self, *args, **kwargs):
+ if not self.uuid:
+ self.uuid = uuid4()
+ super(BackupOperation, self).save(*args, **kwargs)
diff --git a/xos/core/models/core.xproto b/xos/core/models/core.xproto
index 079f830..8a77234 100644
--- a/xos/core/models/core.xproto
+++ b/xos/core/models/core.xproto
@@ -146,6 +146,8 @@
}
message BackupOperation (XOSBase) {
+ option custom_python=True;
+
// `file` is required for restores.
// `file` is optional for backups. If file is unspecified then XOS will create a backup file using
// a default mechanism.
@@ -178,6 +180,10 @@
help_text = "the time and date the operation was completed",
content_type = "date",
feedback_state = True];
+ optional string uuid = 7 [
+ help_text = "unique identifer of this request",
+ bookkeeping_state = True,
+ max_length = 80];
}
message ComputeServiceInstance (ServiceInstance) {
diff --git a/xos/coreapi/backupprocessor.py b/xos/coreapi/backupprocessor.py
index e02bfa0..b270275 100644
--- a/xos/coreapi/backupprocessor.py
+++ b/xos/coreapi/backupprocessor.py
@@ -84,8 +84,9 @@
else:
self.log.error(error_msg)
- self.log.info("Finalizing response", status=status, id=request["id"])
+ self.log.info("Finalizing response", status=status, id=request["id"], uuid=request["uuid"])
response["id"] = request["id"]
+ response["uuid"] = request["uuid"]
response["status"] = status
response["operation"] = request["operation"]
response["file_details"] = request["file_details"]
@@ -208,6 +209,7 @@
try:
id = request["id"]
+ uuid = request["uuid"]
operation = request["operation"]
backend_filename = request["file_details"]["backend_filename"]
except Exception:
@@ -218,6 +220,7 @@
self.log.info(
"Processing request",
id=id,
+ uuid=uuid,
operation=operation,
backend_filename=backend_filename)
diff --git a/xos/coreapi/backupsetwatcher.py b/xos/coreapi/backupsetwatcher.py
index 0778e48..6d2fe1c 100644
--- a/xos/coreapi/backupsetwatcher.py
+++ b/xos/coreapi/backupsetwatcher.py
@@ -101,10 +101,10 @@
continue
os.remove(fn)
- def process_response_create(self, id, operation, status, response):
+ def process_response_create(self, uuid, operation, status, response):
file_details = response["file_details"]
- backupops = BackupOperation.objects.filter(id=id)
+ backupops = BackupOperation.objects.filter(uuid=uuid)
if not backupops:
log.exception("Backup response refers to a backupop that does not exist", id=id)
raise BackupDoesNotExist()
@@ -125,7 +125,7 @@
update_fields=["backend_code", "backend_status", "effective_date", "enacted", "file", "status",
"error_msg"])
- def process_response_restore(self, id, operation, status, response):
+ def process_response_restore(self, uuid, operation, status, response):
file_details = response["file_details"]
# If the restore was successful, then look for any inprogress backups and mark them orphaned.
@@ -141,11 +141,12 @@
# It's likely the Restore operation doesn't exist, because it went away during the restore
# process. Check for the existing operation first, and if it doesn't exist, then create
# one to stand in its place.
- backupops = BackupOperation.objects.filter(id=id)
+ backupops = BackupOperation.objects.filter(uuid=uuid)
if backupops:
backupop = backupops[0]
log.info("Resolved existing backupop model", backupop=backupop)
else:
+ # TODO: Should this use a UUID also?
backupfiles = BackupFile.objects.filter(id=file_details["id"])
if backupfiles:
backupfile = backupfiles[0]
@@ -159,7 +160,8 @@
log.info("Created backupfile model", backupfile=backupfile)
backupop = BackupOperation(operation=operation,
- file=backupfile)
+ file=backupfile,
+ uuid=uuid)
backupop.save(allow_modify_feedback=True)
log.info("Created backupop model", backupop=backupop)
@@ -195,7 +197,7 @@
raise BackupUnreadable()
try:
- id = contents["id"]
+ uuid = contents["uuid"]
operation = contents["operation"]
status = contents["status"]
_ = contents["file_details"]["backend_filename"]
@@ -205,9 +207,9 @@
raise BackupUnreadable()
if operation == "create":
- self.process_response_create(id, operation, status, contents)
+ self.process_response_create(uuid, operation, status, contents)
elif operation == "restore":
- self.process_response_restore(id, operation, status, contents)
+ self.process_response_restore(uuid, operation, status, contents)
# We've successfully concluded. Delete the response file
os.remove(fn)
@@ -218,6 +220,7 @@
def save_request(self, backupop):
request = {"id": backupop.id,
+ "uuid": backupop.uuid,
"operation": backupop.operation}
request["file_details"] = {
diff --git a/xos/coreapi/protos/utility.proto b/xos/coreapi/protos/utility.proto
index 554e0a9..2f81055 100644
--- a/xos/coreapi/protos/utility.proto
+++ b/xos/coreapi/protos/utility.proto
@@ -34,6 +34,15 @@
string xproto = 1;
};
+message VersionInfo {
+ string version = 1;
+ string pythonVersion = 2;
+ string gitCommit = 3;
+ string buildTime = 4;
+ string os = 5;
+ string arch = 6;
+};
+
message PopulatedServiceInstance {
option (contentTypeId) = "core.serviceinstance";
oneof id_present {
@@ -108,4 +117,12 @@
get: "/xosapi/v1/core/populatedserviceinstance/{id}"
};
}
+
+ rpc GetVersion(google.protobuf.Empty) returns (VersionInfo) {
+ option (googleapi.http) = {
+ get: "/xosapi/v1/version"
+ };
+ }
+
+
};
diff --git a/xos/coreapi/test_backupprocessor.py b/xos/coreapi/test_backupprocessor.py
index c2bbd24..70a9244 100644
--- a/xos/coreapi/test_backupprocessor.py
+++ b/xos/coreapi/test_backupprocessor.py
@@ -14,14 +14,12 @@
import json
import os
-import pdb
import unittest
from mock import MagicMock, patch, ANY, call
from pyfakefs import fake_filesystem_unittest
from io import open
-# pyfakefs breaks these
-from __builtin__ import dir as builtin_dir, True as builtin_True, False as builtin_False
+from __builtin__ import True as builtin_True, False as builtin_False
from xosconfig import Config
@@ -81,6 +79,7 @@
"uri": "file://" + backend_filename,
"backend_filename": backend_filename}
req = {"id": 3,
+ "uuid": "three",
"operation": operation,
"file_details": file_details,
"request_fn": request_fn,
@@ -108,6 +107,7 @@
u'effective_date': ANY,
u'operation': u'create',
u'id': 3,
+ u'uuid': u'three',
u'file_details': {u'backend_filename':
u'/var/run/xos/backup/local/mybackup',
u'checksum': u'1234',
@@ -137,6 +137,7 @@
u'effective_date': ANY,
u'operation': u'create',
u'id': 3,
+ u'uuid': u'three',
u'file_details': {u'backend_filename': u'/var/run/xos/backup/local/mybackup',
u'checksum': u'sha1:5eee38381388b6f30efdd5c5c6f067dbf32c0bb3',
u'uri': u'file:///var/run/xos/backup/local/mybackup',
@@ -171,6 +172,7 @@
u'effective_date': ANY,
u'operation': u'restore',
u'id': 3,
+ u'uuid': u'three',
u'file_details': {u'backend_filename': u'/var/run/xos/backup/local/mybackup',
u'uri': u'file:///var/run/xos/backup/local/mybackup',
u'name': u'mybackup',
@@ -210,6 +212,7 @@
u'effective_date': ANY,
u'operation': u'restore',
u'id': 3,
+ u'uuid': u'three',
u'file_details': {u'backend_filename': u'/var/run/xos/backup/local/mybackup',
u'uri': u'file:///var/run/xos/backup/local/mybackup',
u'name': u'mybackup',
diff --git a/xos/coreapi/test_backupsetwatcher.py b/xos/coreapi/test_backupsetwatcher.py
index d95a2ee..3a17441 100644
--- a/xos/coreapi/test_backupsetwatcher.py
+++ b/xos/coreapi/test_backupsetwatcher.py
@@ -15,16 +15,12 @@
import functools
import json
import os
-import pdb
import sys
import unittest
from mock import MagicMock, Mock, patch
from pyfakefs import fake_filesystem_unittest
from io import open
-# pyfakefs breaks these
-from __builtin__ import dir as builtin_dir
-
from xosconfig import Config
@@ -96,7 +92,7 @@
self.backupsetwatcher.BackupOperation.objects.filter.return_value = []
with self.assertRaises(self.backupsetwatcher.BackupDoesNotExist):
- self.watcher.process_response_create(id=1, operation="create", status="created", response=response)
+ self.watcher.process_response_create(uuid="one", operation="create", status="created", response=response)
def test_process_response_create(self):
file_details = {"checksum": "1234"}
@@ -108,7 +104,7 @@
self.backupsetwatcher.BackupOperation.objects.filter.return_value = [op]
- self.watcher.process_response_create(id=1, operation="create", status="created", response=response)
+ self.watcher.process_response_create(uuid="one", operation="create", status="created", response=response)
self.assertEqual(op.file.checksum, "1234")
op.file.save.assert_called()
@@ -135,7 +131,7 @@
self.backupsetwatcher.BackupOperation.objects.filter.return_value = []
self.backupsetwatcher.BackupOperation.side_effect = functools.partial(make_model, mockvars, "newop")
- self.watcher.process_response_restore(id=1, operation="restore", status="restored", response=response)
+ self.watcher.process_response_restore(uuid="one", operation="restore", status="restored", response=response)
newfile = mockvars["newfile"]
self.assertEqual(newfile.name, "mybackup")
@@ -161,7 +157,7 @@
self.backupsetwatcher.BackupOperation.objects.filter.return_value = [op]
- self.watcher.process_response_restore(id=1, operation="restore", status="restored", response=response)
+ self.watcher.process_response_restore(uuid="one", operation="restore", status="restored", response=response)
self.assertEqual(op.status, "restored")
self.assertEqual(op.error_msg, "")
@@ -175,7 +171,7 @@
self.assertTrue(os.path.exists(self.watcher.backup_response_dir))
file_details = {"backend_filename": "/mybackup"}
- resp = {"id": 7, "operation": "create", "status": "created", "file_details": file_details}
+ resp = {"uuid": "seven", "operation": "create", "status": "created", "file_details": file_details}
resp_fn = os.path.join(self.watcher.backup_response_dir, "response")
with open(resp_fn, "w") as resp_f:
@@ -195,7 +191,7 @@
self.assertTrue(os.path.exists(self.watcher.backup_response_dir))
file_details = {"backend_filename": "/mybackup"}
- resp = {"id": 7, "operation": "restore", "status": "restored", "file_details": file_details}
+ resp = {"uuid": "seven", "operation": "restore", "status": "restored", "file_details": file_details}
resp_fn = os.path.join(self.watcher.backup_response_dir, "response")
with open(resp_fn, "w") as resp_f:
@@ -216,6 +212,7 @@
file.name = "mybackup",
request = Mock(id=3,
+ uuid="three",
file=file,
operation="create")
@@ -228,6 +225,7 @@
expected_data = {u'operation': u'create',
u'id': 3,
+ u'uuid': "three",
u'file_details': {u'backend_filename': u'/mybackup',
u'checksum': u'1234',
u'uri': u'file:///mybackup',
@@ -243,6 +241,7 @@
file.name = "mybackup",
request = Mock(id=3,
+ uuid="three",
file=file,
component="xos",
operation="create",
@@ -271,6 +270,7 @@
file.name = "mybackup",
request = Mock(id=3,
+ uuid="three",
file=file,
component="xos",
operation="restore",
@@ -299,6 +299,7 @@
file.name = "mybackup",
request = Mock(id=3,
+ uuid="three",
file=file,
component="somethingelse",
operation="create",
diff --git a/xos/coreapi/xos_dynamicload_api.py b/xos/coreapi/xos_dynamicload_api.py
index 284c114..1ee3bcb 100644
--- a/xos/coreapi/xos_dynamicload_api.py
+++ b/xos/coreapi/xos_dynamicload_api.py
@@ -15,19 +15,18 @@
from protos import dynamicload_pb2
from protos import dynamicload_pb2_grpc
-from xosutil.autodiscover_version import autodiscover_version_of_main
from dynamicbuild import DynamicBuilder
from apistats import REQUEST_COUNT, track_request_time
from authhelper import XOSAuthHelperMixin
from decorators import translate_exceptions, require_authentication
import grpc
import semver
-import re
from xosconfig import Config
from multistructlog import create_logger
log = create_logger(Config().get("logging"))
+
class DynamicLoadService(dynamicload_pb2_grpc.dynamicloadServicer, XOSAuthHelperMixin):
def __init__(self, thread_pool, server):
self.thread_pool = thread_pool
@@ -62,19 +61,26 @@
django_models[k] = v
self.django_app_models[app.name] = django_models
+ def get_core_version(self):
+ try:
+ core_version = open("/opt/xos/VERSION").readline().strip()
+ except Exception:
+ core_version = "unknown"
+
+ return core_version
+
@track_request_time("DynamicLoad", "LoadModels")
@translate_exceptions("DynamicLoad", "LoadModels")
@require_authentication
def LoadModels(self, request, context):
try:
-
- core_version = autodiscover_version_of_main()
+ core_version = self.get_core_version()
requested_core_version = request.core_version
log.info("Loading service models",
service=request.name,
service_version=request.version,
requested_core_version=requested_core_version
- )
+ )
if not requested_core_version:
requested_core_version = ">=2.2.1"
@@ -91,7 +97,7 @@
core_version=core_version,
requested_min_core_version=min_requested,
requested_max_core_version=max_requested
- )
+ )
context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
msg = "Service %s is requesting core version between %s and %s but actual version is %s" % (
request.name,
@@ -109,7 +115,7 @@
core_version=core_version, requested_core_version=requested_core_version)
context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
msg = "Service %s is requesting core version %s but actual version is %s" % (
- request.name, requested_core_version, core_version)
+ request.name, requested_core_version, core_version)
context.set_details(msg)
raise Exception(msg)
@@ -223,7 +229,7 @@
# the core is always onboarded, so doesn't have an explicit manifest
item = response.services.add()
item.name = "core"
- item.version = autodiscover_version_of_main()
+ item.version = self.get_core_version()
if "core" in self.django_apps_by_name:
item.state = "present"
else:
diff --git a/xos/coreapi/xos_utility_api.py b/xos/coreapi/xos_utility_api.py
index 1d2ec96..9ab9220 100644
--- a/xos/coreapi/xos_utility_api.py
+++ b/xos/coreapi/xos_utility_api.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import datetime
import inspect
from apistats import REQUEST_COUNT, track_request_time
import grpc
@@ -25,12 +26,14 @@
import fnmatch
import os
import sys
-import traceback
from protos import utility_pb2, utility_pb2_grpc
from google.protobuf.empty_pb2 import Empty
-
from importlib import import_module
from django.conf import settings
+from xosconfig import Config
+from multistructlog import create_logger
+
+log = create_logger(Config().get("logging"))
SessionStore = import_module(settings.SESSION_ENGINE).SessionStore
@@ -269,3 +272,35 @@
"xos-core", "Utilities", "GetPopulatedServiceInstances", grpc.StatusCode.OK
).inc()
return response
+
+ @translate_exceptions("Utilities", "GetXproto")
+ @track_request_time("Utilities", "GetXproto")
+ def GetVersion(self, request, context):
+ res = utility_pb2.VersionInfo()
+
+ try:
+ res.version = open("/opt/xos/VERSION").readline().strip()
+ except Exception:
+ log.exception("Exception while determining build version")
+ res.version = "unknown"
+
+ try:
+ res.gitCommit = open("/opt/xos/COMMIT").readline().strip()
+ res.buildTime = datetime.datetime.utcfromtimestamp(
+ os.stat("/opt/xos/COMMIT").st_ctime).strftime("%Y-%m-%dT%H:%M:%SZ")
+ except Exception:
+ log.exception("Exception while determining build information")
+ res.buildDate = "unknown"
+ res.gitCommit = "unknown"
+
+ res.pythonVersion = sys.version.split("\n")[0].strip()
+ res.os = os.uname()[0].lower()
+ res.arch = os.uname()[4].lower()
+
+ # TODO(smbaker): res.builTime
+ # TODO(smbaker): res.gitCommit
+
+ REQUEST_COUNT.labels(
+ "xos-core", "Utilities", "GetVersion", grpc.StatusCode.OK
+ ).inc()
+ return res