VOL-948: Implementation of ResourceManager module and corresponding changes in OpenOlt adapter
Change-Id: I7df8226ea1a13198f1be1e15838c69cd54890239
diff --git a/common/pon_resource_manager/__init__.py b/common/pon_resource_manager/__init__.py
new file mode 100644
index 0000000..2d104e0
--- /dev/null
+++ b/common/pon_resource_manager/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/common/pon_resource_manager/resource_kv_store.py b/common/pon_resource_manager/resource_kv_store.py
new file mode 100644
index 0000000..0ccbed8
--- /dev/null
+++ b/common/pon_resource_manager/resource_kv_store.py
@@ -0,0 +1,110 @@
+#
+# Copyright 2018 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Resource KV store - interface between Resource Manager and backend store."""
+import structlog
+
+from voltha.core.config.config_backend import ConsulStore
+from voltha.core.config.config_backend import EtcdStore
+
+# KV store uses this prefix to store resource info
+PATH_PREFIX = 'resource_manager/{}/{}'
+
+
+class ResourceKvStore(object):
+ """Implements apis to store/get/remove resource in backend store."""
+
+ def __init__(self, technology, device_id, backend, host, port):
+ """
+ Create ResourceKvStore object.
+
+ Based on backend ('consul' and 'etcd' use the host and port
+ to create the respective object.
+
+ :param technology: PON technology
+ :param device_id: OLT device id
+ :param backend: Type of backend storage (etcd or consul)
+ :param host: host ip info for backend storage
+ :param port: port for the backend storage
+ :raises exception when invalid backend store passed as an argument
+ """
+ # logger
+ self._log = structlog.get_logger()
+
+ path = PATH_PREFIX.format(technology, device_id)
+ try:
+ if backend == 'consul':
+ self._kv_store = ConsulStore(host, port, path)
+ self._recurse = '?recurse'
+ elif backend == 'etcd':
+ self._kv_store = EtcdStore(host, port, path)
+ self._recurse = ''
+ else:
+ self._log.error('Invalid-backend')
+ raise Exception("Invalid-backend-for-kv-store")
+ except Exception as e:
+ self._log.exception("exception-in-init")
+ raise Exception(e)
+
+ def update_to_kv_store(self, path, resource):
+ """
+ Update resource.
+
+ :param path: path to update the resource
+ :param resource: updated resource
+ """
+ try:
+ self._kv_store[path] = str(resource)
+ self._log.debug("Resource-updated-in-kv-store", path=path)
+ return True
+ except BaseException:
+ self._log.exception("Resource-update-in-kv-store-failed",
+ path=path, resource=resource)
+ return False
+
+ def get_from_kv_store(self, path):
+ """
+ Get resource.
+
+ :param path: path to get the resource
+ """
+ resource = None
+ try:
+ resource = self._kv_store[path]
+ self._log.debug("Got-resource-from-kv-store", path=path)
+ except KeyError:
+ self._log.info("Resource-not-found-updating-resource",
+ path=path)
+ except BaseException:
+ self._log.exception("Getting-resource-from-kv-store-failed",
+ path=path)
+ return resource
+
+ def remove_from_kv_store(self, path):
+ """
+ Remove resource.
+
+ :param path: path to remove the resource
+ """
+ path = path + self._recurse
+ try:
+ del self._kv_store[path]
+ self._log.debug("Resource-deleted-in-kv-store", path=path)
+ return True
+ except BaseException:
+ self._log.exception("Resource-delete-in-kv-store-failed",
+ path=path)
+ return False
diff --git a/common/pon_resource_manager/resource_manager.py b/common/pon_resource_manager/resource_manager.py
new file mode 100644
index 0000000..4abead3
--- /dev/null
+++ b/common/pon_resource_manager/resource_manager.py
@@ -0,0 +1,460 @@
+#
+# Copyright 2018 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+Resource Manager will be unique for each OLT device.
+
+It exposes APIs to create/free alloc_ids/onu_ids/gemport_ids. Resource Manager
+uses a KV store in backend to ensure resiliency of the data.
+"""
+import json
+import structlog
+from bitstring import BitArray
+from twisted.internet.defer import returnValue, inlineCallbacks
+
+from common.kvstore.kvstore import create_kv_client
+from common.utils.asleep import asleep
+
+
+class PONResourceManager(object):
+ """Implements APIs to initialize/allocate/release alloc/gemport/onu IDs."""
+
+ # Constants to identify resource pool
+ ONU_ID = 'ONU_ID'
+ ALLOC_ID = 'ALLOC_ID'
+ GEMPORT_ID = 'GEMPORT_ID'
+
+ # The resource ranges for a given device vendor_type should be placed
+ # at 'resource_manager/<technology>/resource_ranges/<olt_vendor_type>'
+ # path on the KV store.
+ # If Resource Range parameters are to be read from the external KV store,
+ # they are expected to be stored in the following format.
+ # Note: All parameters are MANDATORY for now.
+ '''
+ {
+ "onu_start_idx": 1,
+ "onu_end_idx": 127,
+ "alloc_id_start_idx": 1024,
+ "alloc_id_end_idx": 65534,
+ "gem_port_id_start_idx": 1024,
+ "gem_port_id_end_idx": 16383,
+ "num_of_pon_port": 16
+ }
+ '''
+ # constants used as keys to reference the resource range parameters from
+ # and external KV store.
+ ONU_START_IDX = "onu_start_idx"
+ ONU_END_IDX = "onu_end_idx"
+ ALLOC_ID_START_IDX = "alloc_id_start_idx"
+ ALLOC_ID_END_IDX = "alloc_id_end_idx"
+ GEM_PORT_ID_START_IDX = "gem_port_id_start_idx"
+ GEM_PORT_ID_END_IDX = "gem_port_id_end_idx"
+ NUM_OF_PON_PORT = "num_of_pon_port"
+
+ # PON Resource range configuration on the KV store.
+ # Format: 'resource_manager/<technology>/resource_ranges/<olt_vendor_type>'
+ PON_RESOURCE_RANGE_CONFIG_PATH = 'resource_manager/{}/resource_ranges/{}'
+
+ # resource path in kv store
+ ALLOC_ID_POOL_PATH = 'resource_manager/{}/{}/alloc_id_pool/{}'
+ GEMPORT_ID_POOL_PATH = 'resource_manager/{}/{}/gemport_id_pool/{}'
+ ONU_ID_POOL_PATH = 'resource_manager/{}/{}/onu_id_pool/{}'
+
+ # Constants for internal usage.
+ PON_INTF_ID = 'pon_intf_id'
+ START_IDX = 'start_idx'
+ END_IDX = 'end_idx'
+ POOL = 'pool'
+
+ def __init__(self, technology, olt_vendor_type, device_id,
+ backend, host, port):
+ """
+ Create PONResourceManager object.
+
+ :param technology: PON technology
+ :param: olt_vendor_type: This string defines the OLT vendor type
+ and is used as a Key to load the resource range configuration from
+ KV store location.
+ :param device_id: OLT device id
+ :param backend: backend store
+ :param host: ip of backend store
+ :param port: port on which backend store listens
+ :raises exception when invalid backend store passed as an argument
+ """
+ # logger
+ self._log = structlog.get_logger()
+
+ try:
+ self._kv_store = create_kv_client(backend, host, port)
+ self.technology = technology
+ self.olt_vendor_type = olt_vendor_type
+ self.device_id = device_id
+ # Below attribute, pon_resource_ranges, should be initialized
+ # by reading from KV store.
+ self.pon_resource_ranges = dict()
+ except Exception as e:
+ self._log.exception("exception-in-init")
+ raise Exception(e)
+
+ @inlineCallbacks
+ def init_pon_resource_ranges(self):
+ # Try to initialize the PON Resource Ranges from KV store if available
+ status = yield self.init_resource_ranges_from_kv_store()
+ # If reading from KV store fails, initialize to default values.
+ if not status:
+ self._log.error("failed-to-read-resource-ranges-from-kv-store")
+ self.init_default_pon_resource_ranges()
+
+ @inlineCallbacks
+ def init_resource_ranges_from_kv_store(self):
+ path = self.PON_RESOURCE_RANGE_CONFIG_PATH.format(
+ self.technology, self.olt_vendor_type)
+ # get resource from kv store
+ result = yield self._kv_store.get(path)
+ resource_range_config = result[0]
+
+ if resource_range_config is not None:
+ self.pon_resource_ranges = eval(resource_range_config.value)
+ self._log.debug("Init-resource-ranges-from-kvstore-success",
+ pon_resource_ranges=self.pon_resource_ranges,
+ path=path)
+ returnValue(True)
+
+ returnValue(False)
+
+ def init_default_pon_resource_ranges(self, onu_start_idx=1,
+ onu_end_idx=127,
+ alloc_id_start_idx=1024,
+ alloc_id_end_idx=65534,
+ gem_port_id_start_idx=1024,
+ gem_port_id_end_idx=16383,
+ num_of_pon_ports=16):
+ self._log.info("initialize-default-resource-range-values")
+ self.pon_resource_ranges[PONResourceManager.ONU_START_IDX] = onu_start_idx
+ self.pon_resource_ranges[PONResourceManager.ONU_END_IDX] = onu_end_idx
+ self.pon_resource_ranges[PONResourceManager.ALLOC_ID_START_IDX] = alloc_id_start_idx
+ self.pon_resource_ranges[PONResourceManager.ALLOC_ID_END_IDX] = alloc_id_end_idx
+ self.pon_resource_ranges[
+ PONResourceManager.GEM_PORT_ID_START_IDX] = gem_port_id_start_idx
+ self.pon_resource_ranges[
+ PONResourceManager.GEM_PORT_ID_END_IDX] = gem_port_id_end_idx
+ self.pon_resource_ranges[PONResourceManager.NUM_OF_PON_PORT] = num_of_pon_ports
+
+ def init_device_resource_pool(self):
+ i = 0
+ while i < self.pon_resource_ranges[PONResourceManager.NUM_OF_PON_PORT]:
+ self.init_resource_id_pool(
+ pon_intf_id=i,
+ resource_type=PONResourceManager.ONU_ID,
+ start_idx=self.pon_resource_ranges[
+ PONResourceManager.ONU_START_IDX],
+ end_idx=self.pon_resource_ranges[
+ PONResourceManager.ONU_END_IDX])
+
+ self.init_resource_id_pool(
+ pon_intf_id=i,
+ resource_type=PONResourceManager.ALLOC_ID,
+ start_idx=self.pon_resource_ranges[
+ PONResourceManager.ALLOC_ID_START_IDX],
+ end_idx=self.pon_resource_ranges[
+ PONResourceManager.ALLOC_ID_END_IDX])
+
+ self.init_resource_id_pool(
+ pon_intf_id=i,
+ resource_type=PONResourceManager.GEMPORT_ID,
+ start_idx=self.pon_resource_ranges[
+ PONResourceManager.GEM_PORT_ID_START_IDX],
+ end_idx=self.pon_resource_ranges[
+ PONResourceManager.GEM_PORT_ID_END_IDX])
+ i += 1
+
+ def clear_device_resource_pool(self):
+ i = 0
+ while i < self.pon_resource_ranges[PONResourceManager.NUM_OF_PON_PORT]:
+ self.clear_resource_id_pool(
+ pon_intf_id=i,
+ resource_type=PONResourceManager.ONU_ID,
+ )
+
+ self.clear_resource_id_pool(
+ pon_intf_id=i,
+ resource_type=PONResourceManager.ALLOC_ID,
+ )
+
+ self.clear_resource_id_pool(
+ pon_intf_id=i,
+ resource_type=PONResourceManager.GEMPORT_ID,
+ )
+ i += 1
+
+ @inlineCallbacks
+ def init_resource_id_pool(self, pon_intf_id, resource_type, start_idx,
+ end_idx):
+ """
+ Initialize Resource ID pool for a given Resource Type on a given PON Port
+
+ :param pon_intf_id: OLT PON interface id
+ :param resource_type: String to identify type of resource
+ :param start_idx: start index for onu id pool
+ :param end_idx: end index for onu id pool
+ :return boolean: True if resource id pool initialized else false
+ """
+ status = False
+ path = self._get_path(pon_intf_id, resource_type)
+ if path is None:
+ returnValue(status)
+
+ # In case of adapter reboot and reconciliation resource in kv store
+ # checked for its presence if not kv store update happens
+ resource = yield self._get_resource(path)
+
+ if resource is not None:
+ self._log.info("Resource-already-present-in-store", path=path)
+ status = True
+ else:
+ resource = self._format_resource(pon_intf_id, start_idx, end_idx)
+ self._log.info("Resource-initialized", path=path)
+
+ # Add resource as json in kv store.
+ result = yield self._kv_store.put(path, resource)
+ if result is None:
+ status = True
+ returnValue(status)
+
+ @inlineCallbacks
+ def get_resource_id(self, pon_intf_id, resource_type, num_of_id=1):
+ """
+ Create alloc/gemport/onu id for given OLT PON interface.
+
+ :param pon_intf_id: OLT PON interface id
+ :param resource_type: String to identify type of resource
+ :param num_of_id: required number of ids
+ :return list/int/None: list, int or None if resource type is
+ alloc_id/gemport_id, onu_id or invalid type
+ respectively
+ """
+ result = None
+ path = self._get_path(pon_intf_id, resource_type)
+ if path is None:
+ returnValue(result)
+
+ resource = yield self._get_resource(path)
+ try:
+ if resource is not None and resource_type == \
+ PONResourceManager.ONU_ID:
+ result = self._generate_next_id(resource)
+ elif resource is not None and (
+ resource_type == PONResourceManager.GEMPORT_ID or
+ resource_type == PONResourceManager.ALLOC_ID):
+ result = list()
+ while num_of_id > 0:
+ result.append(self._generate_next_id(resource))
+ num_of_id -= 1
+
+ # Update resource in kv store
+ self._update_resource(path, resource)
+
+ except BaseException:
+ self._log.exception("Get-" + resource_type + "-id-failed",
+ path=path)
+ self._log.debug("Get-" + resource_type + "-success", result=result,
+ path=path)
+ returnValue(result)
+
+ @inlineCallbacks
+ def free_resource_id(self, pon_intf_id, resource_type, release_content):
+ """
+ Release alloc/gemport/onu id for given OLT PON interface.
+
+ :param pon_intf_id: OLT PON interface id
+ :param resource_type: String to identify type of resource
+ :param release_content: required number of ids
+ :return boolean: True if all IDs in given release_content released
+ else False
+ """
+ status = False
+ path = self._get_path(pon_intf_id, resource_type)
+ if path is None:
+ returnValue(status)
+
+ resource = yield self._get_resource(path)
+ try:
+ if resource is not None and resource_type == \
+ PONResourceManager.ONU_ID:
+ self._release_id(resource, release_content)
+ elif resource is not None and (
+ resource_type == PONResourceManager.ALLOC_ID or
+ resource_type == PONResourceManager.GEMPORT_ID):
+ for content in release_content:
+ self._release_id(resource, content)
+ self._log.debug("Free-" + resource_type + "-success", path=path)
+
+ # Update resource in kv store
+ status = yield self._update_resource(path, resource)
+
+ except BaseException:
+ self._log.exception("Free-" + resource_type + "-failed", path=path)
+ returnValue(status)
+
+ @inlineCallbacks
+ def clear_resource_id_pool(self, pon_intf_id, resource_type):
+ """
+ Clear Resource Pool for a given Resource Type on a given PON Port.
+
+ :return boolean: True if removed else False
+ """
+ path = self._get_path(pon_intf_id, resource_type)
+ if path is None:
+ returnValue(False)
+
+ result = yield self._kv_store.delete(path)
+ if result is None:
+ self._log.debug("Resource-pool-cleared", device_id=self.device_id,
+ path=path)
+ returnValue(True)
+ self._log.error("Clear-resource-pool-failed", device_id=self.device_id,
+ path=path)
+ returnValue(False)
+
+ def _generate_next_id(self, resource):
+ """
+ Generate unique id having OFFSET as start index.
+
+ :param resource: resource used to generate ID
+ :return int: generated id
+ """
+ pos = resource[PONResourceManager.POOL].find('0b0')
+ resource[PONResourceManager.POOL].set(1, pos)
+ return pos[0] + resource[PONResourceManager.START_IDX]
+
+ def _release_id(self, resource, unique_id):
+ """
+ Release unique id having OFFSET as start index.
+
+ :param resource: resource used to release ID
+ :param unique_id: id need to be released
+ """
+ pos = ((int(unique_id)) - resource[PONResourceManager.START_IDX])
+ resource[PONResourceManager.POOL].set(0, pos)
+
+ def _get_path(self, pon_intf_id, resource_type):
+ """
+ Get path for given resource type.
+
+ :param pon_intf_id: OLT PON interface id
+ :param resource_type: String to identify type of resource
+ :return: path for given resource type
+ """
+ path = None
+ if resource_type == PONResourceManager.ONU_ID:
+ path = self._get_onu_id_resource_path(pon_intf_id)
+ elif resource_type == PONResourceManager.ALLOC_ID:
+ path = self._get_alloc_id_resource_path(pon_intf_id)
+ elif resource_type == PONResourceManager.GEMPORT_ID:
+ path = self._get_gemport_id_resource_path(pon_intf_id)
+ else:
+ self._log.error("invalid-resource-pool-identifier")
+ return path
+
+ def _get_alloc_id_resource_path(self, pon_intf_id):
+ """
+ Get alloc id resource path.
+
+ :param pon_intf_id: OLT PON interface id
+ :return: alloc id resource path
+ """
+ return PONResourceManager.ALLOC_ID_POOL_PATH.format(
+ self.technology, self.device_id, pon_intf_id)
+
+ def _get_gemport_id_resource_path(self, pon_intf_id):
+ """
+ Get gemport id resource path.
+
+ :param pon_intf_id: OLT PON interface id
+ :return: gemport id resource path
+ """
+ return PONResourceManager.GEMPORT_ID_POOL_PATH.format(
+ self.technology, self.device_id, pon_intf_id)
+
+ def _get_onu_id_resource_path(self, pon_intf_id):
+ """
+ Get onu id resource path.
+
+ :param pon_intf_id: OLT PON interface id
+ :return: onu id resource path
+ """
+ return PONResourceManager.ONU_ID_POOL_PATH.format(
+ self.technology, self.device_id, pon_intf_id)
+
+ @inlineCallbacks
+ def _update_resource(self, path, resource):
+ """
+ Update resource in resource kv store.
+
+ :param path: path to update resource
+ :param resource: resource need to be updated
+ :return boolean: True if resource updated in kv store else False
+ """
+ resource[PONResourceManager.POOL] = \
+ resource[PONResourceManager.POOL].bin
+ result = yield self._kv_store.put(path, json.dumps(resource))
+ if result is None:
+ returnValue(True)
+ returnValue(False)
+
+ @inlineCallbacks
+ def _get_resource(self, path):
+ """
+ Get resource from kv store.
+
+ :param path: path to get resource
+ :return: resource if resource present in kv store else None
+ """
+ # get resource from kv store
+ result = yield self._kv_store.get(path)
+ resource = result[0]
+
+ if resource is not None:
+ # decode resource fetched from backend store to dictionary
+ resource = eval(resource.value)
+
+ # resource pool in backend store stored as binary string whereas to
+ # access the pool to generate/release IDs it need to be converted
+ # as BitArray
+ resource[PONResourceManager.POOL] = \
+ BitArray('0b' + resource[PONResourceManager.POOL])
+
+ returnValue(resource)
+
+ def _format_resource(self, pon_intf_id, start_idx, end_idx):
+ """
+ Format resource as json.
+
+ :param pon_intf_id: OLT PON interface id
+ :param start_idx: start index for id pool
+ :param end_idx: end index for id pool
+ :return dictionary: resource formatted as dictionary
+ """
+ # Format resource as json to be stored in backend store
+ resource = dict()
+ resource[PONResourceManager.PON_INTF_ID] = pon_intf_id
+ resource[PONResourceManager.START_IDX] = start_idx
+ resource[PONResourceManager.END_IDX] = end_idx
+
+ # resource pool stored in backend store as binary string
+ resource[PONResourceManager.POOL] = BitArray(end_idx).bin
+
+ return json.dumps(resource)