[SEBA-547] TP NEM Integration
Change-Id: I6c0dab752f4728d5f6a777ea297321ebb75de769
diff --git a/requirements.txt b/requirements.txt
index 669c29b..bad8c6b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,4 @@
xossynchronizer~=3.2.1
xosapi~=3.2.1
xoskafka~=3.2.1
+etcd3==0.7.0
diff --git a/xos/synchronizer/migrations/0006_technologyprofile_technologyprofile_decl.py b/xos/synchronizer/migrations/0006_technologyprofile_technologyprofile_decl.py
new file mode 100644
index 0000000..5a6e8b9
--- /dev/null
+++ b/xos/synchronizer/migrations/0006_technologyprofile_technologyprofile_decl.py
@@ -0,0 +1,77 @@
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-05-13 11:46
+from __future__ import unicode_literals
+
+import core.models.xosbase_header
+import django.core.validators
+from django.db import migrations, models
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('volt', '0005_auto_20190410_0026'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='TechnologyProfile_decl',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('created', models.DateTimeField(auto_now_add=True, help_text=b'Time this model was created')),
+ ('updated', models.DateTimeField(default=django.utils.timezone.now, help_text=b'Time this model was changed by a non-synchronizer')),
+ ('enacted', models.DateTimeField(blank=True, default=None, help_text=b'When synced, set to the timestamp of the data that was synced', null=True)),
+ ('policed', models.DateTimeField(blank=True, default=None, help_text=b'When policed, set to the timestamp of the data that was policed', null=True)),
+ ('backend_register', models.CharField(blank=True, default=b'{}', max_length=1024, null=True)),
+ ('backend_need_delete', models.BooleanField(default=False)),
+ ('backend_need_reap', models.BooleanField(default=False)),
+ ('backend_status', models.CharField(default=b'Provisioning in progress', max_length=1024)),
+ ('backend_code', models.IntegerField(default=0)),
+ ('deleted', models.BooleanField(default=False)),
+ ('write_protect', models.BooleanField(default=False)),
+ ('lazy_blocked', models.BooleanField(default=False)),
+ ('no_sync', models.BooleanField(default=False)),
+ ('no_policy', models.BooleanField(default=False)),
+ ('policy_status', models.CharField(blank=True, default=b'Policy in process', max_length=1024, null=True)),
+ ('policy_code', models.IntegerField(blank=True, default=0, null=True)),
+ ('leaf_model_name', models.CharField(help_text=b'The most specialized model in this chain of inheritance, often defined by a service developer', max_length=1024)),
+ ('backend_need_delete_policy', models.BooleanField(default=False, help_text=b'True if delete model_policy must be run before object can be reaped')),
+ ('xos_managed', models.BooleanField(default=True, help_text=b'True if xos is responsible for creating/deleting this object')),
+ ('backend_handle', models.CharField(blank=True, help_text=b'Handle used by the backend to track this object', max_length=1024, null=True)),
+ ('changed_by_step', models.DateTimeField(blank=True, default=None, help_text=b'Time this model was changed by a sync step', null=True)),
+ ('changed_by_policy', models.DateTimeField(blank=True, default=None, help_text=b'Time this model was changed by a model policy', null=True)),
+ ('technology', models.CharField(db_index=True, help_text=b'The technology being utilized by the adaptor', max_length=16)),
+ ('profile_id', models.IntegerField(db_index=True, help_text=b'The numeric id of the profile', validators=[django.core.validators.MaxValueValidator(255), django.core.validators.MinValueValidator(64)])),
+ ('profile_value', models.CharField(help_text=b'The technology profile value in JSON format', max_length=4096)),
+ ],
+ options={
+ 'verbose_name': 'Technology Profile',
+ },
+ bases=(models.Model, core.models.xosbase_header.PlModelMixIn),
+ ),
+ migrations.CreateModel(
+ name='TechnologyProfile',
+ fields=[
+ ],
+ options={
+ 'proxy': True,
+ 'indexes': [],
+ },
+ bases=('volt.technologyprofile_decl',),
+ ),
+ ]
diff --git a/xos/synchronizer/models/models.py b/xos/synchronizer/models/models.py
index 4d5b8b8..59b6e66 100644
--- a/xos/synchronizer/models/models.py
+++ b/xos/synchronizer/models/models.py
@@ -25,6 +25,9 @@
from models_decl import ONUDevice_decl
from models_decl import PONONUPort_decl
from models_decl import UNIPort_decl
+from models_decl import TechnologyProfile_decl
+
+import json
class VOLTService(VOLTService_decl):
class Meta:
@@ -116,3 +119,28 @@
class Meta:
proxy = True
+class TechnologyProfile(TechnologyProfile_decl):
+ class Meta:
+ proxy = True
+
+ def save(self, *args, **kwargs):
+
+ caller_kind = None
+ if "caller_kind" in kwargs:
+ caller_kind = kwargs.get("caller_kind")
+
+ # only synchronizer is allowed to update the model
+ if not self.is_new and caller_kind != "synchronizer":
+ if not self.deleted:
+ existing = TechnologyProfile.objects.filter(id=self.id)
+ raise XOSValidationError('Modification operation is not allowed on Technology Profile [/%s/%s]. Delete it and add again' % (existing[0].technology, existing[0].profile_id))
+
+ # validate if technology profile value is valid JSON format string
+ if self.profile_value != None:
+ try:
+ tp_json_val = json.loads(self.profile_value)
+ except ValueError as e:
+ raise XOSValidationError('Technology Profile value not in valid JSON format')
+
+ super(TechnologyProfile, self).save(*args, **kwargs)
+
diff --git a/xos/synchronizer/models/test_models.py b/xos/synchronizer/models/test_models.py
index 3ede98c..3048071 100644
--- a/xos/synchronizer/models/test_models.py
+++ b/xos/synchronizer/models/test_models.py
@@ -113,6 +113,9 @@
self.onu_device.is_new = True
self.onu_device.serial_number = 1234
+ def tearDown(self):
+ self.module_patcher.stop()
+
def test_delete(self):
self.onu_device.delete()
self.models_decl.ONUDevice_decl.delete.assert_called()
@@ -129,5 +132,67 @@
'ONU "1234" can\'t be deleted as it has subscribers associated with it')
self.models_decl.OLTDevice_decl.delete.assert_not_called()
+class TestTechnologyProfileModel(unittest.TestCase):
+
+ def setUp(self):
+ self.xos = XOS
+
+ self.models_decl = Mock()
+ self.models_decl.TechnologyProfile_decl = MagicMock
+ self.models_decl.TechnologyProfile_decl.save = Mock()
+ self.models_decl.TechnologyProfile_decl.objects = Mock()
+ self.models_decl.TechnologyProfile_decl.objects.filter.return_value = []
+
+ modules = {
+ 'xos': MagicMock(),
+ 'xos.exceptions': self.xos.exceptions,
+ 'models_decl': self.models_decl
+ }
+
+ self.module_patcher = patch.dict('sys.modules', modules)
+ self.module_patcher.start()
+
+ from models import TechnologyProfile
+
+ self.technology_profile = TechnologyProfile()
+ self.technology_profile.deleted = False
+ self.technology_profile.id = None # this is a new model
+ self.technology_profile.is_new = True
+ self.technology_profile.technology = 'xgspon'
+ self.technology_profile.profile_id = 64
+ self.technology_profile.profile_value = '{ "name": "4QueueHybridProfileMap1", "profile_type": "XPON", "version": 1, "num_gem_ports": 4, "instance_control": { "onu": "multi-instance", "uni": "single-instance", "max_gem_payload_size": "auto" }, "us_scheduler": { "additional_bw": "auto", "direction": "UPSTREAM", "priority": 0, "weight": 0, "q_sched_policy": "hybrid" }, "ds_scheduler": { "additional_bw": "auto", "direction": "DOWNSTREAM", "priority": 0, "weight": 0, "q_sched_policy": "hybrid" }, "upstream_gem_port_attribute_list": [ { "pbit_map": "0b00000101", "aes_encryption": "True", "scheduling_policy": "WRR", "priority_q": 4, "weight": 25, "discard_policy": "TailDrop", "max_q_size": "auto", "discard_config": { "max_threshold": 0, "min_threshold": 0, "max_probability": 0 } }, { "pbit_map": "0b00011010", "aes_encryption": "True", "scheduling_policy": "WRR", "priority_q": 3, "weight": 75, "discard_policy": "TailDrop", "max_q_size": "auto", "discard_config": { "min_threshold": 0, "max_threshold": 0, "max_probability": 0 } }, { "pbit_map": "0b00100000", "aes_encryption": "True", "scheduling_policy": "StrictPriority", "priority_q": 2, "weight": 0, "discard_policy": "TailDrop", "max_q_size": "auto", "discard_config": { "min_threshold": 0, "max_threshold": 0, "max_probability": 0 } }, { "pbit_map": "0b11000000", "aes_encryption": "True", "scheduling_policy": "StrictPriority", "priority_q": 1, "weight": 25, "discard_policy": "TailDrop", "max_q_size": "auto", "discard_config": { "min_threshold": 0, "max_threshold": 0, "max_probability": 0 } } ], "downstream_gem_port_attribute_list": [ { "pbit_map": "0b00000101", "aes_encryption": "True", "scheduling_policy": "WRR", "priority_q": 4, "weight": 10, "discard_policy": "TailDrop", "max_q_size": "auto", "discard_config": { "min_threshold": 0, "max_threshold": 0, "max_probability": 0 } }, { "pbit_map": "0b00011010", "aes_encryption": "True", "scheduling_policy": "WRR", "priority_q": 3, "weight": 90, "discard_policy": "TailDrop", "max_q_size": "auto", "discard_config": { "min_threshold": 0, "max_threshold": 0, "max_probability": 0 } }, { "pbit_map": "0b00100000", "aes_encryption": "True", "scheduling_policy": "StrictPriority", "priority_q": 2, "weight": 0, "discard_policy": "TailDrop", "max_q_size": "auto", "discard_config": { "min_threshold": 0, "max_threshold": 0, "max_probability": 0 } }, { "pbit_map": "0b11000000", "aes_encryption": "True", "scheduling_policy": "StrictPriority", "priority_q": 1, "weight": 25, "discard_policy": "TailDrop", "max_q_size": "auto", "discard_config": { "min_threshold": 0, "max_threshold": 0, "max_probability": 0 } } ]}'
+
+ def tearDown(self):
+ self.module_patcher.stop()
+
+ def test_save(self):
+ self.technology_profile.save()
+ self.models_decl.TechnologyProfile_decl.save.assert_called()
+
+ def test_prevent_modify(self):
+ self.technology_profile.is_new = False
+ self.technology_profile.id = 1
+ self.technology_profile.profile_value = '{"name": "someValue", "profile_type": "someValue"}'
+
+ self.models_decl.TechnologyProfile_decl.objects.filter.return_value = [self.technology_profile]
+
+ with self.assertRaises(Exception) as e:
+ self.technology_profile.save()
+
+ self.assertEqual(e.exception.message,
+ 'Modification operation is not allowed on Technology Profile [/xgspon/64]. Delete it and add again')
+ self.models_decl.TechnologyProfile_decl.save.assert_not_called()
+
+ def test_invalid_tech_profile_value_format(self):
+ self.technology_profile.profile_value = 'someTechProfileValue'
+
+ with self.assertRaises(Exception) as e:
+ self.technology_profile.save()
+
+ self.assertEqual(e.exception.message,
+ 'Technology Profile value not in valid JSON format')
+ self.models_decl.TechnologyProfile_decl.save.assert_not_called()
+
+
if __name__ == '__main__':
unittest.main()
diff --git a/xos/synchronizer/models/volt.xproto b/xos/synchronizer/models/volt.xproto
index 5ae85ea..8856c0f 100644
--- a/xos/synchronizer/models/volt.xproto
+++ b/xos/synchronizer/models/volt.xproto
@@ -230,3 +230,21 @@
help_text = "ONUDevice that belongs to this Subscriber chain",
db_index = True];
}
+
+message TechnologyProfile (XOSBase) {
+ option verbose_name = "Technology Profile";
+ option description = "The Technology Profile that is utilized by VOLTHA";
+
+ required string technology = 1 [
+ help_text = "The technology being utilized by the adaptor",
+ db_index = True,
+ max_length = 16];
+ required int32 profile_id = 2 [
+ help_text = "The numeric id of the profile",
+ db_index = True,
+ min_value = 64,
+ max_value = 255];
+ required string profile_value = 3 [
+ help_text = "The technology profile value in JSON format",
+ max_length = 4096];
+}
diff --git a/xos/synchronizer/steps/sync_tech_profile.py b/xos/synchronizer/steps/sync_tech_profile.py
new file mode 100644
index 0000000..01e7da3
--- /dev/null
+++ b/xos/synchronizer/steps/sync_tech_profile.py
@@ -0,0 +1,70 @@
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os, sys
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from helpers import Helpers
+
+import requests
+from multistructlog import create_logger
+from requests.auth import HTTPBasicAuth
+from xossynchronizer.modelaccessor import TechnologyProfile, model_accessor
+from xossynchronizer.steps.syncstep import SyncStep
+from xosconfig import Config
+
+import etcd3
+
+ETCD_HOST_URL = 'etcd-cluster.default.svc.cluster.local'
+ETCD_PORT = 2379
+
+log = create_logger(Config().get("logging"))
+
+class SyncTechnologyProfile(SyncStep):
+ provides = [TechnologyProfile]
+
+ observes = TechnologyProfile
+
+ def update_etcd(self, operation, key, value):
+ log.info('Update Etcd store: ', operation=operation, key=key, value=value)
+
+ etcd = etcd3.client(host=ETCD_HOST_URL, port=ETCD_PORT)
+ if operation == 'PUT':
+ etcd.put(key, value)
+ log.info('Technology Profile [%s] saved successfully to Etcd store' % key)
+ elif operation == 'DELETE':
+ if False == etcd.delete(key):
+ log.error('Error while deleting Technology Profile [%s] from Etcd store' % key)
+ raise Exception('Failed to delete Technology Profile')
+ else:
+ log.info('Technology Profile [%s] deleted successfully from Etcd store' % key)
+ else:
+ log.warning('Invalid or unsupported Etcd operation: %s' % operation)
+
+ def sync_record(self, model):
+ log.info('Synching TechnologyProfile', object=str(model), **model.tologdict())
+
+ log.info('TechnologyProfile: %s : %s' % (model.technology, model.profile_id))
+
+ tp_key = u'/' + model.technology + u'/' + str(model.profile_id)
+ self.update_etcd('PUT', tp_key, model.profile_value)
+
+ def delete_record(self, model):
+ log.info('Deleting TechnologyProfile', object=str(model), **model.tologdict())
+
+ log.info('TechnologyProfile: %s : %s' % (model.technology, model.profile_id))
+
+ tp_key = u'/' + model.technology + u'/' + str(model.profile_id)
+ self.update_etcd('DELETE', tp_key, None)
+