[CORD-3078] Pulling ONUs
Change-Id: I211f3669bc59ee217d5c084a69383be535a9c4c6
diff --git a/xos/synchronizer/steps/helpers.py b/xos/synchronizer/helpers.py
similarity index 100%
rename from xos/synchronizer/steps/helpers.py
rename to xos/synchronizer/helpers.py
diff --git a/xos/synchronizer/models/volt.xproto b/xos/synchronizer/models/volt.xproto
index 874e523..a73b8ed 100644
--- a/xos/synchronizer/models/volt.xproto
+++ b/xos/synchronizer/models/volt.xproto
@@ -52,8 +52,15 @@
option verbose_name = "ONU Device";
option description = "Represents a physical ONU device";
- required manytoone volt_device->OLTDevice:onu_devices = 1 [db_index = True, null = False, blank = False];
- required string serial_number = 1 [max_length = 254, null = False, db_index = False, blank = False];
+ required manytoone olt_device->OLTDevice:onu_devices = 1 [db_index = True, null = False, blank = False];
+ required string serial_number = 2 [max_length = 254, null = False, db_index = False, blank = False, tosca_key=True];
+ required string vendor = 3 [max_length = 254, null = False, db_index = False, blank = False];
+ required string device_id = 4 [max_length = 254, null = False, db_index = False, blank = False];
+
+ required string device_type = 5 [help_text = "Device Type", default = "asfvolt16_olt", max_length = 254, null = False, db_index = False, blank = False];
+ optional string admin_state = 6 [help_text = "admin_state", null = True, db_index = False, blank = False, feedback_state = True];
+ optional string oper_status = 7 [help_text = "oper_status", null = True, db_index = False, blank = False, feedback_state = True];
+ optional string connect_status = 8 [help_text = "connect_status", null = True, db_index = False, blank = False, feedback_state = True];
}
message PONPort (XOSBase){
diff --git a/xos/synchronizer/pull_steps/pull_olts.py b/xos/synchronizer/pull_steps/pull_olts.py
index 0fc5e2d..36cac45 100644
--- a/xos/synchronizer/pull_steps/pull_olts.py
+++ b/xos/synchronizer/pull_steps/pull_olts.py
@@ -22,39 +22,21 @@
from requests import ConnectionError
from requests.models import InvalidURL
+import os, sys
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from helpers import Helpers
+
log = create_logger(Config().get('logging'))
class OLTDevicePullStep(PullStep):
def __init__(self):
super(OLTDevicePullStep, self).__init__(observed_model=OLTDevice)
- # NOTE move helpers where they can be loaded by multiple modules?
- @staticmethod
- def format_url(url):
- if 'http' in url:
- return url
- else:
- return 'http://%s' % url
-
- @staticmethod
- def get_voltha_info(olt_service):
- return {
- 'url': OLTDevicePullStep.format_url(olt_service.voltha_url),
- 'port': olt_service.voltha_port,
- 'user': olt_service.voltha_user,
- 'pass': olt_service.voltha_pass
- }
-
- @staticmethod
- def datapath_id_to_hex(id):
- if isinstance(id, basestring):
- id = int(id)
- return "{0:0{1}x}".format(id, 16)
-
@staticmethod
def get_ids_from_logical_device(o):
- voltha_url = OLTDevicePullStep.get_voltha_info(o.volt_service)['url']
- voltha_port = OLTDevicePullStep.get_voltha_info(o.volt_service)['port']
+ voltha_url = Helpers.get_voltha_info(o.volt_service)['url']
+ voltha_port = Helpers.get_voltha_info(o.volt_service)['port']
r = requests.get("%s:%s/api/v1/logical_devices" % (voltha_url, voltha_port))
@@ -66,11 +48,10 @@
for ld in res["items"]:
if ld["root_device_id"] == o.device_id:
o.of_id = ld["id"]
- o.dp_id = "of:" + OLTDevicePullStep.datapath_id_to_hex(ld["datapath_id"]) # convert to hex
+ o.dp_id = "of:" + Helpers.datapath_id_to_hex(ld["datapath_id"]) # convert to hex
return o
raise Exception("Can't find a logical device for device id: %s" % o.device_id)
- # end note
def pull_records(self):
log.info("pulling OLT devices from VOLTHA")
@@ -81,8 +62,8 @@
log.warn('VOLTService not found')
return
- voltha_url = OLTDevicePullStep.get_voltha_info(self.volt_service)['url']
- voltha_port = OLTDevicePullStep.get_voltha_info(self.volt_service)['port']
+ voltha_url = Helpers.get_voltha_info(self.volt_service)['url']
+ voltha_port = Helpers.get_voltha_info(self.volt_service)['port']
try:
r = requests.get("%s:%s/api/v1/devices" % (voltha_url, voltha_port))
diff --git a/xos/synchronizer/pull_steps/pull_onus.py b/xos/synchronizer/pull_steps/pull_onus.py
new file mode 100644
index 0000000..b48ab1b
--- /dev/null
+++ b/xos/synchronizer/pull_steps/pull_onus.py
@@ -0,0 +1,118 @@
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from synchronizers.new_base.pullstep import PullStep
+from synchronizers.new_base.modelaccessor import model_accessor, ONUDevice, VOLTService, OLTDevice
+
+from xosconfig import Config
+from multistructlog import create_logger
+
+import requests
+from requests import ConnectionError
+from requests.models import InvalidURL
+
+import os, sys
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from helpers import Helpers
+
+log = create_logger(Config().get('logging'))
+
+class ONUDevicePullStep(PullStep):
+ def __init__(self):
+ super(ONUDevicePullStep, self).__init__(observed_model=ONUDevice)
+
+ def pull_records(self):
+ log.info("pulling ONU devices from VOLTHA")
+
+ try:
+ self.volt_service = VOLTService.objects.all()[0]
+ except IndexError:
+ log.warn('VOLTService not found')
+ return
+
+ voltha_url = Helpers.get_voltha_info(self.volt_service)['url']
+ voltha_port = Helpers.get_voltha_info(self.volt_service)['port']
+
+ try:
+ r = requests.get("%s:%s/api/v1/devices" % (voltha_url, voltha_port))
+
+ if r.status_code != 200:
+ log.info("It was not possible to fetch devices from VOLTHA")
+
+ # keeping only ONUs
+ devices = [d for d in r.json()["items"] if "onu" in d["type"]]
+
+ log.debug("received devices", onus=devices)
+
+ # TODO
+ # [ ] delete ONUS as ONUDevice.objects.all() - updated OLTs
+
+ if r.status_code != 200:
+ log.info("It was not possible to fetch devices from VOLTHA")
+
+ onus_in_voltha = self.create_or_update_onus(devices)
+
+ except ConnectionError, e:
+ log.warn("It was not possible to connect to VOLTHA", reason=e)
+ return
+ except InvalidURL, e:
+ log.warn("VOLTHA url is invalid, is it configured in the VOLTService?", reason=e)
+ return
+
+ def create_or_update_onus(self, onus):
+
+ updated_onus = []
+
+ for onu in onus:
+ try:
+
+ model = ONUDevice.objects.filter(serial_number=onu["serial_number"])[0]
+ log.debug("ONUDevice already exists, updating it", serial_number=onu["serial_number"])
+
+ if model.enacted < model.updated:
+ log.info("Skipping pull on ONUDevice %s as enacted < updated" % model.name, name=model.name, id=model.id, enacted=model.enacted, updated=model.updated)
+ return
+
+ except IndexError:
+ model = ONUDevice()
+ model.serial_number = onu["serial_number"]
+
+ log.debug("ONUDevice is new, creating it", serial_number=onu["serial_number"])
+
+ # Adding feedback state to the device
+ model.vendor = onu["vendor"]
+ model.device_type = onu["type"]
+ model.device_id = onu["id"]
+
+ model.admin_state = onu["admin_state"]
+ model.oper_status = onu["oper_status"]
+ model.connect_status = onu["connect_status"]
+
+ olt = OLTDevice.objects.get(device_id=onu["proxy_address"]["device_id"])
+
+ model.olt_device = olt
+ model.olt_device_id = olt.id
+
+ model.save()
+
+ updated_onus.append(model)
+
+ return updated_onus
+
+
+
+
+
+
diff --git a/xos/synchronizer/pull_steps/test_pull_onus.py b/xos/synchronizer/pull_steps/test_pull_onus.py
new file mode 100644
index 0000000..ae95db6
--- /dev/null
+++ b/xos/synchronizer/pull_steps/test_pull_onus.py
@@ -0,0 +1,186 @@
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+from mock import patch, call, Mock, PropertyMock
+import requests_mock
+
+import os, sys
+
+# Hack to load synchronizer framework
+test_path=os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
+xos_dir=os.path.join(test_path, "../../..")
+if not os.path.exists(os.path.join(test_path, "new_base")):
+ xos_dir=os.path.join(test_path, "../../../../../../orchestration/xos/xos")
+ services_dir = os.path.join(xos_dir, "../../xos_services")
+sys.path.append(xos_dir)
+sys.path.append(os.path.join(xos_dir, 'synchronizers', 'new_base'))
+# END Hack to load synchronizer framework
+
+# generate model from xproto
+def get_models_fn(service_name, xproto_name):
+ name = os.path.join(service_name, "xos", xproto_name)
+ if os.path.exists(os.path.join(services_dir, name)):
+ return name
+ else:
+ name = os.path.join(service_name, "xos", "synchronizer", "models", xproto_name)
+ if os.path.exists(os.path.join(services_dir, name)):
+ return name
+ raise Exception("Unable to find service=%s xproto=%s" % (service_name, xproto_name))
+# END generate model from xproto
+
+class TestPullONUDevice(unittest.TestCase):
+
+ def setUp(self):
+ global DeferredException
+
+ self.sys_path_save = sys.path
+ sys.path.append(xos_dir)
+ sys.path.append(os.path.join(xos_dir, 'synchronizers', 'new_base'))
+
+ # Setting up the config module
+ from xosconfig import Config
+ config = os.path.join(test_path, "../model_policies/test_config.yaml")
+ Config.clear()
+ Config.init(config, "synchronizer-config-schema.yaml")
+ # END Setting up the config module
+
+ from synchronizers.new_base.mock_modelaccessor_build import build_mock_modelaccessor
+ # build_mock_modelaccessor(xos_dir, services_dir, [get_models_fn("olt-service", "volt.xproto")])
+
+ # FIXME this is to get jenkins to pass the tests, somehow it is running tests in a different order
+ # and apparently it is not overriding the generated model accessor
+ build_mock_modelaccessor(xos_dir, services_dir, [get_models_fn("olt-service", "volt.xproto"),
+ get_models_fn("vsg", "vsg.xproto"),
+ get_models_fn("../profiles/rcord", "rcord.xproto")])
+ import synchronizers.new_base.modelaccessor
+ from pull_onus import ONUDevicePullStep, model_accessor
+
+ # import all class names to globals
+ for (k, v) in model_accessor.all_model_classes.items():
+ globals()[k] = v
+
+ self.sync_step = ONUDevicePullStep
+
+ # mock volt service
+ self.volt_service = Mock()
+ self.volt_service.id = "volt_service_id"
+ self.volt_service.voltha_url = "voltha_url"
+ self.volt_service.voltha_user = "voltha_user"
+ self.volt_service.voltha_pass = "voltha_pass"
+ self.volt_service.voltha_port = 1234
+
+ # mock OLTDevice
+ self.olt = Mock()
+ self.olt.id = 1
+
+ # mock voltha responses
+ self.devices = {
+ "items": [
+ {
+ "id": "0001130158f01b2d",
+ "type": "broadcom_onu",
+ "vendor": "Broadcom",
+ "serial_number": "BRCM22222222",
+ "vendor_id": "BRCM",
+ "adapter": "broadcom_onu",
+ "vlan": 0,
+ "admin_state": "ENABLED",
+ "oper_status": "ACTIVE",
+ "connect_status": "REACHABLE",
+ "proxy_address": {
+ "device_id": "00010fc93996afea"
+ }
+ }
+ ]
+ }
+
+ def tearDown(self):
+ sys.path = self.sys_path_save
+
+ @requests_mock.Mocker()
+ def test_missing_volt_service(self, m):
+ self.assertFalse(m.called)
+
+ @requests_mock.Mocker()
+ def test_pull(self, m):
+
+ with patch.object(VOLTService.objects, "all") as olt_service_mock, \
+ patch.object(OLTDevice.objects, "get") as mock_olt_device, \
+ patch.object(ONUDevice, "save") as mock_save:
+ olt_service_mock.return_value = [self.volt_service]
+ mock_olt_device.return_value = self.olt
+
+ m.get("http://voltha_url:1234/api/v1/devices", status_code=200, json=self.devices)
+
+ self.sync_step().pull_records()
+
+ # TODO how to asster this?
+ # self.assertEqual(existing_olt.admin_state, "ENABLED")
+ # self.assertEqual(existing_olt.oper_status, "ACTIVE")
+ # self.assertEqual(existing_olt.volt_service_id, "volt_service_id")
+ # self.assertEqual(existing_olt.device_id, "test_id")
+ # self.assertEqual(existing_olt.of_id, "of_id")
+ # self.assertEqual(existing_olt.dp_id, "of:0000000ce2314000")
+
+ mock_save.assert_called()
+
+ @requests_mock.Mocker()
+ def _test_pull_existing(self, m):
+
+ existing_olt = Mock()
+ existing_olt.enacted = 2
+ existing_olt.updated = 1
+
+ with patch.object(VOLTService.objects, "all") as olt_service_mock, \
+ patch.object(OLTDevice.objects, "filter") as mock_get, \
+ patch.object(existing_olt, "save") as mock_save:
+ olt_service_mock.return_value = [self.volt_service]
+ mock_get.return_value = [existing_olt]
+
+ m.get("http://voltha_url:1234/api/v1/devices", status_code=200, json=self.devices)
+ m.get("http://voltha_url:1234/api/v1/logical_devices", status_code=200, json=self.logical_devices)
+
+ self.sync_step().pull_records()
+
+ self.assertEqual(existing_olt.admin_state, "ENABLED")
+ self.assertEqual(existing_olt.oper_status, "ACTIVE")
+ self.assertEqual(existing_olt.volt_service_id, "volt_service_id")
+ self.assertEqual(existing_olt.device_id, "test_id")
+ self.assertEqual(existing_olt.of_id, "of_id")
+ self.assertEqual(existing_olt.dp_id, "of:0000000ce2314000")
+
+ mock_save.assert_called()
+
+ @requests_mock.Mocker()
+ def _test_pull_existing_do_not_sync(self, m):
+ existing_olt = Mock()
+ existing_olt.enacted = 1
+ existing_olt.updated = 2
+
+ with patch.object(VOLTService.objects, "all") as olt_service_mock, \
+ patch.object(OLTDevice.objects, "get") as mock_get, \
+ patch.object(existing_olt, "save") as mock_save:
+ olt_service_mock.return_value = [self.volt_service]
+ mock_get.return_value = existing_olt
+
+ m.get("http://voltha_url:1234/api/v1/devices", status_code=200, json=self.devices)
+ m.get("http://voltha_url:1234/api/v1/logical_devices", status_code=200, json=self.logical_devices)
+
+ self.sync_step().pull_records()
+
+ mock_save.assert_not_called()
+
+if __name__ == "__main__":
+ unittest.main()
\ No newline at end of file
diff --git a/xos/synchronizer/steps/sync_olt_device.py b/xos/synchronizer/steps/sync_olt_device.py
index 4aaebdf..5997de5 100644
--- a/xos/synchronizer/steps/sync_olt_device.py
+++ b/xos/synchronizer/steps/sync_olt_device.py
@@ -12,15 +12,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import json
-from synchronizers.new_base.SyncInstanceUsingAnsible import SyncStep
-from synchronizers.new_base.modelaccessor import OLTDevice
-
-from xosconfig import Config
-from multistructlog import create_logger
from time import sleep
+
import requests
+from multistructlog import create_logger
from requests.auth import HTTPBasicAuth
+from synchronizers.new_base.SyncInstanceUsingAnsible import SyncStep
+from synchronizers.new_base.modelaccessor import OLTDevice, model_accessor
+from xosconfig import Config
+
+import os, sys
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
from helpers import Helpers
log = create_logger(Config().get('logging'))
diff --git a/xos/synchronizer/steps/sync_volt_service_instance.py b/xos/synchronizer/steps/sync_volt_service_instance.py
index eef6f7e..6703d82 100644
--- a/xos/synchronizer/steps/sync_volt_service_instance.py
+++ b/xos/synchronizer/steps/sync_volt_service_instance.py
@@ -12,16 +12,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from synchronizers.new_base.syncstep import SyncStep, DeferredException
-from synchronizers.new_base.modelaccessor import model_accessor
-from synchronizers.new_base.modelaccessor import VOLTService, VOLTServiceInstance, ServiceInstance, OLTDevice
+import os, sys
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-from xosconfig import Config
-from multistructlog import create_logger
-import requests
-from requests.auth import HTTPBasicAuth
from helpers import Helpers
+import requests
+from multistructlog import create_logger
+from requests.auth import HTTPBasicAuth
+from synchronizers.new_base.modelaccessor import VOLTService, VOLTServiceInstance, ServiceInstance, OLTDevice, model_accessor
+from synchronizers.new_base.syncstep import SyncStep, DeferredException
+from xosconfig import Config
+
log = create_logger(Config().get("logging"))
class SyncVOLTServiceInstance(SyncStep):
diff --git a/xos/synchronizer/steps/test_sync_volt_service_instance.py b/xos/synchronizer/steps/test_sync_volt_service_instance.py
index 60d2d7d..cacf765 100644
--- a/xos/synchronizer/steps/test_sync_volt_service_instance.py
+++ b/xos/synchronizer/steps/test_sync_volt_service_instance.py
@@ -43,7 +43,7 @@
def mock_get_westbound_service_instance_properties(prop):
return prop
-class TestSyncOLTDevice(unittest.TestCase):
+class TestSyncVOLTServiceInstance(unittest.TestCase):
def setUp(self):
global DeferredException
diff --git a/xos/synchronizer/steps/test_helpers.py b/xos/synchronizer/test_helpers.py
similarity index 97%
rename from xos/synchronizer/steps/test_helpers.py
rename to xos/synchronizer/test_helpers.py
index f97625e..1ec5df7 100644
--- a/xos/synchronizer/steps/test_helpers.py
+++ b/xos/synchronizer/test_helpers.py
@@ -13,9 +13,12 @@
# limitations under the License.
import unittest
-from mock import patch, call, Mock, PropertyMock
+
+from mock import Mock
+
from helpers import Helpers
+
class TestHelpers(unittest.TestCase):
def setUp(self):