| # SPDX-FileCopyrightText: 2020 The Magma Authors. |
| # SPDX-FileCopyrightText: 2022 Open Networking Foundation <support@opennetworking.org> |
| # |
| # SPDX-License-Identifier: BSD-3-Clause |
| |
| import asyncio |
| from xml.etree import ElementTree |
| |
| from aiohttp import web |
| from common.misc_utils import get_ip_from_if |
| from configuration.service_configs import load_service_config |
| from enodeb_status import get_enb_status, update_status_metrics |
| from logger import EnodebdLogger as logger |
| from state_machines.enb_acs import EnodebAcsStateMachine |
| from state_machines.enb_acs_manager import StateMachineManager |
| |
| import metrics |
| |
| |
| class StatsManager: |
| """ HTTP server to receive performance management uploads from eNodeB and |
| translate to metrics """ |
| # Dict to map performance counter names (from eNodeB) to metrics |
| # For eNodeB sub-counters, the counter name is shown as |
| # '<counter>:<sub-counter>' |
| PM_FILE_TO_METRIC_MAP = { |
| 'RRC.AttConnEstab': metrics.STAT_RRC_ESTAB_ATT, |
| 'RRC.SuccConnEstab': metrics.STAT_RRC_ESTAB_SUCC, |
| 'RRC.AttConnReestab': metrics.STAT_RRC_REESTAB_ATT, |
| 'RRC.AttConnReestab._Cause:RRC.AttConnReestab.RECONF_FAIL': |
| metrics.STAT_RRC_REESTAB_ATT_RECONF_FAIL, |
| 'RRC.AttConnReestab._Cause:RRC.AttConnReestab.HO_FAIL': |
| metrics.STAT_RRC_REESTAB_ATT_HO_FAIL, |
| 'RRC.AttConnReestab._Cause:RRC.AttConnReestab.OTHER': |
| metrics.STAT_RRC_REESTAB_ATT_OTHER, |
| 'RRC.SuccConnReestab': metrics.STAT_RRC_REESTAB_SUCC, |
| 'ERAB.NbrAttEstab': metrics.STAT_ERAB_ESTAB_ATT, |
| 'ERAB.NbrSuccEstab': metrics.STAT_ERAB_ESTAB_SUCC, |
| 'ERAB.NbrFailEstab': metrics.STAT_ERAB_ESTAB_FAIL, |
| 'ERAB.NbrReqRelEnb': metrics.STAT_ERAB_REL_REQ, |
| 'ERAB.NbrReqRelEnb.CauseUserInactivity': |
| metrics.STAT_ERAB_REL_REQ_USER_INAC, |
| 'ERAB.NbrReqRelEnb.Normal': metrics.STAT_ERAB_REL_REQ_NORMAL, |
| 'ERAB.NbrReqRelEnb._Cause:ERAB.NbrReqRelEnb.CauseRADIORESOURCESNOTAVAILABLE': |
| metrics.STAT_ERAB_REL_REQ_RES_NOT_AVAIL, |
| 'ERAB.NbrReqRelEnb._Cause:ERAB.NbrReqRelEnb.CauseREDUCELOADINSERVINGCELL': |
| metrics.STAT_ERAB_REL_REQ_REDUCE_LOAD, |
| 'ERAB.NbrReqRelEnb._Cause:ERAB.NbrReqRelEnb.CauseFAILUREINTHERADIOINTERFACEPROCEDURE': |
| metrics.STAT_ERAB_REL_REQ_FAIL_IN_RADIO, |
| 'ERAB.NbrReqRelEnb._Cause:ERAB.NbrReqRelEnb.CauseRELEASEDUETOEUTRANGENERATEDREASONS': |
| metrics.STAT_ERAB_REL_REQ_EUTRAN_REAS, |
| 'ERAB.NbrReqRelEnb._Cause:ERAB.NbrReqRelEnb.CauseRADIOCONNECTIONWITHUELOST': |
| metrics.STAT_ERAB_REL_REQ_RADIO_CONN_LOST, |
| 'ERAB.NbrReqRelEnb._Cause:ERAB.NbrReqRelEnb.CauseOAMINTERVENTION': |
| metrics.STAT_ERAB_REL_REQ_OAM_INTV, |
| 'PDCP.UpOctUl': metrics.STAT_PDCP_USER_PLANE_BYTES_UL, |
| 'PDCP.UpOctDl': metrics.STAT_PDCP_USER_PLANE_BYTES_DL, |
| } |
| |
| # Check if radio transmit is turned on every 10 seconds. |
| CHECK_RF_TX_PERIOD = 10 |
| |
| def __init__(self, enb_acs_manager: StateMachineManager): |
| self.enb_manager = enb_acs_manager |
| self.loop = asyncio.get_event_loop() |
| self._prev_rf_tx = False |
| self.mme_timeout_handler = None |
| |
| def run(self) -> None: |
| """ Create and start HTTP server """ |
| svc_config = load_service_config("enodebd") |
| |
| app = web.Application() |
| app.router.add_route( |
| 'POST', "/{something}", |
| self._post_and_put_handler, |
| ) |
| app.router.add_route( |
| 'PUT', "/{something}", |
| self._post_and_put_handler, |
| ) |
| |
| handler = app.make_handler() |
| create_server_func = self.loop.create_server( |
| handler, |
| host=get_ip_from_if(svc_config['tr069']['interface']), |
| port=svc_config['tr069']['perf_mgmt_port'], |
| ) |
| |
| self._periodic_check_rf_tx() |
| self.loop.run_until_complete(create_server_func) |
| |
| def _periodic_check_rf_tx(self) -> None: |
| self._check_rf_tx() |
| self.mme_timeout_handler = self.loop.call_later( |
| self.CHECK_RF_TX_PERIOD, |
| self._periodic_check_rf_tx, |
| ) |
| |
| def _check_rf_tx(self) -> None: |
| """ |
| Check if eNodeB should be connected to MME but isn't, and maybe reboot. |
| |
| If the eNB doesn't report connection to MME within a timeout period, |
| get it to reboot in the hope that it will fix things. |
| |
| Usually, enodebd polls the eNodeB for whether it is connected to MME. |
| This method checks the last polled MME connection status, and if |
| eNodeB should be connected to MME but it isn't. |
| """ |
| # Clear stats when eNodeB stops radiating. This is |
| # because eNodeB stops sending performance metrics at this point. |
| serial_list = self.enb_manager.get_connected_serial_id_list() |
| for enb_serial in serial_list: |
| handler = self.enb_manager.get_handler_by_serial(enb_serial) |
| if handler: |
| self._check_rf_tx_for_handler(handler) |
| |
| def _check_rf_tx_for_handler(self, handler: EnodebAcsStateMachine) -> None: |
| status = get_enb_status(handler) |
| if self._prev_rf_tx and not status.rf_tx_on: |
| self._clear_stats() |
| self._prev_rf_tx = status.rf_tx_on |
| |
| # Update status metrics |
| update_status_metrics(status) |
| |
| def _get_enb_label_from_request(self, request) -> str: |
| label = 'default' |
| ip = request.headers.get('X-Forwarded-For') |
| |
| if ip is None: |
| peername = request.transport.get_extra_info('peername') |
| if peername is not None: |
| ip, _ = peername |
| |
| if ip is None: |
| return label |
| |
| label = self.enb_manager.get_serial_of_ip(ip) |
| if label: |
| logger.debug('Found serial %s for ip %s', label, ip) |
| else: |
| logger.error("Couldn't find serial for ip", ip) |
| return label |
| |
| @asyncio.coroutine |
| def _post_and_put_handler(self, request) -> web.Response: |
| """ HTTP POST handler """ |
| # Read request body and convert to XML tree |
| body = yield from request.read() |
| |
| root = ElementTree.fromstring(body) |
| label = self._get_enb_label_from_request(request) |
| if label: |
| self._parse_pm_xml(label, root) |
| |
| # Return success response |
| return web.Response() |
| |
| def _parse_pm_xml(self, enb_label, xml_root) -> None: |
| """ |
| Parse performance management XML from eNodeB and populate metrics. |
| The schema for this XML document, along with an example, is shown in |
| tests/stats_manager_tests.py. |
| """ |
| for measurement in xml_root.findall('Measurements'): |
| object_type = measurement.findtext('ObjectType') |
| names = measurement.find('PmName') |
| data = measurement.find('PmData') |
| if object_type == 'EutranCellTdd': |
| self._parse_tdd_counters(enb_label, names, data) |
| elif object_type == 'ManagedElement': |
| # Currently no counters to parse |
| pass |
| elif object_type == 'SctpAssoc': |
| # Currently no counters to parse |
| pass |
| |
| def _parse_tdd_counters(self, enb_label, names, data): |
| """ |
| Parse eNodeB performance management counters from TDD structure. |
| Most of the logic is just to extract the correct counter based on the |
| name of the statistic. Each counter is either of type 'V', which is a |
| single integer value, or 'CV', which contains multiple integer |
| sub-elements, named 'SV', which we add together. E.g: |
| <V i="9">0</V> |
| <CV i="10"> |
| <SN>RRC.AttConnReestab.RECONF_FAIL</SN> |
| <SV>0</SV> |
| <SN>RRC.AttConnReestab.HO_FAIL</SN> |
| <SV>0</SV> |
| <SN>RRC.AttConnReestab.OTHER</SN> |
| <SV>0</SV> |
| </CV> |
| See tests/stats_manager_tests.py for a more complete example. |
| """ |
| index_data_map = self._build_index_to_data_map(data) |
| name_index_map = self._build_name_to_index_map(names) |
| |
| # For each performance metric, extract value from XML document and set |
| # internal metric to that value. |
| for pm_name, metric in self.PM_FILE_TO_METRIC_MAP.items(): |
| |
| elements = pm_name.split(':') |
| counter = elements.pop(0) |
| if len(elements) == 0: |
| subcounter = None |
| else: |
| subcounter = elements.pop(0) |
| |
| index = name_index_map.get(counter) |
| if index is None: |
| logger.warning('PM counter %s not found in PmNames', counter) |
| continue |
| |
| data_el = index_data_map.get(index) |
| if data_el is None: |
| logger.warning('PM counter %s not found in PmData', counter) |
| continue |
| |
| if data_el.tag == 'V': |
| if subcounter is not None: |
| logger.warning('No subcounter in PM counter %s', counter) |
| continue |
| |
| # Data is singular value |
| try: |
| value = int(data_el.text) |
| except ValueError: |
| logger.info( |
| 'PM value (%s) of counter %s not integer', |
| data_el.text, counter, |
| ) |
| continue |
| elif data_el.tag == 'CV': |
| # Check whether we want just one subcounter, or sum them all |
| subcounter_index = None |
| if subcounter is not None: |
| index = 0 |
| for sub_name_el in data_el.findall('SN'): |
| if sub_name_el.text == subcounter: |
| subcounter_index = index |
| index = index + 1 |
| |
| if subcounter is not None and subcounter_index is None: |
| logger.warning('PM subcounter (%s) not found', subcounter) |
| continue |
| |
| # Data is multiple sub-elements. Sum them, or select the one |
| # of interest |
| value = 0 |
| try: |
| index = 0 |
| for sub_data_el in data_el.findall('SV'): |
| if subcounter_index is None or \ |
| subcounter_index == index: |
| value = value + int(sub_data_el.text) |
| index = index + 1 |
| except ValueError: |
| logger.error( |
| 'PM value (%s) of counter %s not integer', |
| sub_data_el.text, pm_name, |
| ) |
| continue |
| else: |
| logger.warning( |
| 'Unknown PM data type (%s) of counter %s', |
| data_el.tag, pm_name, |
| ) |
| continue |
| |
| # Apply new value to metric |
| if pm_name == 'PDCP.UpOctUl' or pm_name == 'PDCP.UpOctDl': |
| metric.labels(enb_label).set(value) |
| else: |
| metric.set(value) |
| |
| def _build_index_to_data_map(self, data_etree): |
| """ |
| Parse XML ElementTree and build a dict mapping index to data XML |
| element. The relevant part of XML schema being parsed is: |
| <xs:element name="PmData"> |
| <xs:complexType> |
| <xs:sequence minOccurs="0" maxOccurs="unbounded"> |
| <xs:element name="Pm"> |
| <xs:complexType> |
| <xs:choice minOccurs="0" maxOccurs="unbounded"> |
| <xs:element name="V"> |
| <xs:complexType> |
| <xs:simpleContent> |
| <xs:extension base="xs:string"> |
| <xs:attribute name="i" type="xs:integer" use="required"/> |
| </xs:extension> |
| </xs:simpleContent> |
| </xs:complexType> |
| </xs:element> |
| <xs:element name="CV"> |
| <xs:complexType> |
| <xs:sequence minOccurs="0" maxOccurs="unbounded"> |
| <xs:element name="SN" type="xs:string"/> |
| <xs:element name="SV" type="xs:string"/> |
| </xs:sequence> |
| <xs:attribute name="i" type="xs:integer" use="required"/> |
| </xs:complexType> |
| </xs:element> |
| </xs:choice> |
| <xs:attribute name="Dn" type="xs:string" use="required"/> |
| <xs:attribute name="UserLabel" type="xs:string" use="required"/> |
| </xs:complexType> |
| </xs:element> |
| </xs:sequence> |
| </xs:complexType> |
| </xs:element> |
| |
| Inputs: |
| - XML elementree element corresponding to 'PmData' in above schema |
| Outputs: |
| - Dict mapping index ('i' in above schema) to data elementree |
| elements ('V' and 'CV' in above schema) |
| """ |
| # Construct map of index to pm_data XML element |
| index_data_map = {} |
| for pm_el in data_etree.findall('Pm'): |
| for data_el in pm_el.findall('V'): |
| index = data_el.get('i') |
| if index is not None: |
| index_data_map[index] = data_el |
| for data_el in pm_el.findall('CV'): |
| index = data_el.get('i') |
| if index is not None: |
| index_data_map[index] = data_el |
| |
| return index_data_map |
| |
| def _build_name_to_index_map(self, name_etree): |
| """ |
| Parse XML ElementTree and build a dict mapping name to index. The |
| relevant part of XML schema being parsed is: |
| <xs:element name="PmName"> |
| <xs:complexType> |
| <xs:sequence minOccurs="0" maxOccurs="unbounded"> |
| <xs:element name="N"> |
| <xs:complexType> |
| <xs:simpleContent> |
| <xs:extension base="xs:string"> |
| <xs:attribute name="i" type="xs:integer" use="required"/> |
| </xs:extension> |
| </xs:simpleContent> |
| </xs:complexType> |
| </xs:element> |
| </xs:sequence> |
| </xs:complexType> |
| </xs:element> |
| |
| Inputs: |
| - XML elementree element corresponding to 'PmName' in above schema |
| Outputs: |
| - Dict mapping name ('N' in above schema) to index ('i' in above |
| schema) |
| """ |
| # Construct map of pm_name to index |
| name_index_map = {} |
| for name_el in name_etree.findall('N'): |
| name_index_map[name_el.text] = name_el.get('i') |
| |
| return name_index_map |
| |
| def _clear_stats(self) -> None: |
| """ |
| Clear statistics. Called when eNodeB management plane disconnects |
| """ |
| logger.info('Clearing performance counter statistics') |
| # Set all metrics to 0 if eNodeB not connected |
| for pm_name, metric in self.PM_FILE_TO_METRIC_MAP.items(): |
| # eNB data usage metrics will not be cleared |
| if pm_name not in ('PDCP.UpOctUl', 'PDCP.UpOctDl'): |
| metric.set(0) |